Example structured

Path to notebook: https://www.github.com/data-privacy-stack/presidio/blob/main/docs/samples/python/example_structured.ipynb

from presidio_structured import StructuredEngine, JsonAnalysisBuilder, PandasAnalysisBuilder, StructuredAnalysis, CsvReader, JsonReader, JsonDataProcessor, PandasDataProcessor

This sample showcases presidio-structured on structured and semi-structured data containing sensitive data like names, emails, and addresses. It differs from the sample for the batch analyzer/anonymizer engines example, which includes narrative phrases that might contain sensitive data. The presence of personal data embedded in these phrases requires to analyze and to anonymize the text inside the cells, which is not the case for our structured sample, where the sensitive data is already separated into columns.

Loading in data

sample_df = CsvReader().read("./csv_sample_data/test_structured.csv")
sample_df

	id	name	email	street	city	state	non_pii
0	1	John Doe	[email protected]	123 Main St	Anytown	CA	reallynotpii
1	2	Jane Smith	[email protected]	456 Elm St	Somewhere	TX	reallynotapii
2	3	Alice Johnson	[email protected]	789 Pine St	Elsewhere	NY	reallynotapiiatall

sample_json = JsonReader().read("./sample_data/test_structured.json")
sample_json

{'id': 1,
 'name': 'John Doe',
 'email': '[email protected]',
 'address': {'street': '123 Main St',
  'city': 'Anytown',
  'state': 'CA',
  'non_pii': 'reallynotapiiatall'}}

# contains nested objects in lists
sample_complex_json = JsonReader().read("./sample_data/test_structured_complex.json")
sample_complex_json

{'users': [{'id': 1,
   'name': 'John Doe',
   'email': '[email protected]',
   'address': {'street': '123 Main St',
    'city': 'Anytown',
    'state': 'CA',
    'non_pii': 'reallynotpii'}},
  {'id': 2,
   'name': 'Jane Smith',
   'email': '[email protected]',
   'address': {'street': '456 Elm St',
    'city': 'Somewhere',
    'state': 'TX',
    'non_pii': 'reallynotapii'}},
  {'id': 3,
   'name': 'Alice Johnson',
   'email': '[email protected]',
   'address': {'street': '789 Pine St',
    'city': 'Elsewhere',
    'state': 'NY',
    'non_pii': 'reallynotapiiatall'}}]}

Tabular (csv) data: defining & generating tabular analysis, anonymization.

# Automatically detect the entity for the columns
tabular_analysis = PandasAnalysisBuilder().generate_analysis(sample_df)
tabular_analysis

StructuredAnalysis(entity_mapping={'name': 'PERSON', 'email': 'URL', 'city': 'LOCATION', 'state': 'LOCATION'})

# anonymized data defaults to be replaced with None, unless operators is specified

pandas_engine = StructuredEngine(data_processor=PandasDataProcessor())
df_to_be_anonymized = sample_df.copy() # in-place anonymization
anonymized_df = pandas_engine.anonymize(df_to_be_anonymized, tabular_analysis, operators=None) # explicit None for clarity
anonymized_df

	id	name	email	street	city	state	non_pii
0	1	<None>	<None>	123 Main St	<None>	<None>	reallynotpii
1	2	<None>	<None>	456 Elm St	<None>	<None>	reallynotapii
2	3	<None>	<None>	789 Pine St	<None>	<None>	reallynotapiiatall

We can also define operators using OperatorConfig similar as to the AnonymizerEngine:

from presidio_anonymizer.entities.engine import OperatorConfig
from faker import Faker
fake = Faker()

operators = {
    "PERSON": OperatorConfig("replace", {"new_value": "person..."}),
    "EMAIL_ADDRESS": OperatorConfig("custom", {"lambda": lambda x: fake.safe_email()})
    # etc...
    }
anonymized_df = pandas_engine.anonymize(sample_df, tabular_analysis, operators=operators)
anonymized_df

	id	name	email	street	city	state	non_pii
0	1	person...	<None>	123 Main St	<None>	<None>	reallynotpii
1	2	person...	<None>	456 Elm St	<None>	<None>	reallynotapii
2	3	person...	<None>	789 Pine St	<None>	<None>	reallynotapiiatall

Semi-structured (JSON) data: simple and complex analysis, anonymization

json_analysis = JsonAnalysisBuilder().generate_analysis(sample_json)
json_analysis

StructuredAnalysis(entity_mapping={'name': 'PERSON', 'email': 'EMAIL_ADDRESS', 'address.city': 'LOCATION'})

# Currently does not support nested objects in lists
try:
    json_complex_analysis = JsonAnalysisBuilder().generate_analysis(sample_complex_json)
except ValueError as e:
    print(e)

# however, we can define it manually:
json_complex_analysis = StructuredAnalysis(entity_mapping={
    "users.name":"PERSON",
    "users.address.street":"LOCATION",
    "users.address.city":"LOCATION",
    "users.address.state":"LOCATION",
    "users.email": "EMAIL_ADDRESS",
})

Analyzer.analyze_iterator only works on primitive types (int, float, bool, str). Lists of objects are not yet supported.

# anonymizing simple data
json_engine = StructuredEngine(data_processor=JsonDataProcessor())
anonymized_json = json_engine.anonymize(sample_json, json_analysis, operators=operators)
anonymized_json

{'id': 1,
 'name': 'person...',
 'email': '[email protected]',
 'address': {'street': '123 Main St',
  'city': '<None>',
  'state': 'CA',
  'non_pii': 'reallynotapiiatall'}}

anonymized_complex_json = json_engine.anonymize(sample_complex_json, json_complex_analysis, operators=operators)
anonymized_complex_json

{'users': [{'id': 1,
   'name': 'person...',
   'email': '[email protected]',
   'address': {'street': '<None>',
    'city': '<None>',
    'state': '<None>',
    'non_pii': 'reallynotpii'}},
  {'id': 2,
   'name': 'person...',
   'email': '[email protected]',
   'address': {'street': '<None>',
    'city': '<None>',
    'state': '<None>',
    'non_pii': 'reallynotapii'}},
  {'id': 3,
   'name': 'person...',
   'email': '[email protected]',
   'address': {'street': '<None>',
    'city': '<None>',
    'state': '<None>',
    'non_pii': 'reallynotapiiatall'}}]}