Anonymize with sha256

pii-anonymizer.json

@@ -9,7 +9,7 @@
     "level": "medium"
   },
   "anonymize": {
-    "mode": "redact",
+    "mode": "hash",
     "output_file_path": "./output"
   }
 }

pii_anonymizer/common/config_validator.py

@@ -1,6 +1,6 @@
 from pii_anonymizer.common.constants import ANONYMIZE
 
-anonymize_mode = ["redact", "drop"]
+anonymize_mode = ["redact", "drop", "hash"]
 anonymize_mode_err_msg = f"{ANONYMIZE}'s mode must be {' or '.join(anonymize_mode)}"
 
 

pii_anonymizer/spark/analyze/detectors/pii_detector.py

@@ -95,6 +95,10 @@ def get_redacted_text(self, input_data_frame: DataFrame, report: DataFrame):
                 result = input_data_frame.rdd.map(
                     lambda row: Anonymizer.redact(row, pii_list)
                 ).toDF(column)
+            case "hash":
+                result = input_data_frame.rdd.map(
+                    lambda row: Anonymizer.hash(row, pii_list)
+                ).toDF(column)
             case _:
                 result = input_data_frame.rdd.map(
                     lambda row: Anonymizer.drop(row, pii_list)

pii_anonymizer/spark/anonymize/anonymizer.py

@@ -1,3 +1,6 @@
+from hashlib import sha256
+
+
 class Anonymizer:
     @staticmethod
     def drop(row, pii_list):
@@ -18,3 +21,13 @@ def redact(row, pii_list):
                     cell = cell.replace(word, "[Redacted]")
             new_row.append(cell)
         return new_row
+
+    @staticmethod
+    def hash(row, pii_list):
+        new_row = []
+        for cell in row:
+            for word in pii_list:
+                if word in cell:
+                    cell = cell.replace(word, sha256(word.encode("utf-8")).hexdigest())
+            new_row.append(cell)
+        return new_row

pii_anonymizer/spark/anonymize/tests/test_hash_anonymizer.py

@@ -0,0 +1,42 @@
+from unittest import TestCase
+from pyspark.sql import SparkSession
+from pii_anonymizer.spark.anonymize.anonymizer import Anonymizer
+from hashlib import sha256
+
+
+class TestHashAnonymizer(TestCase):
+    def setUp(self) -> None:
+        self.SPARK = (
+            SparkSession.builder.master("local")
+            .appName("Test PIIDetector")
+            .getOrCreate()
+        )
+
+    def test_hash_for_single_analyzer_result(self):
+        test_data_frame = self.SPARK.createDataFrame(
+            [("text containing pii", "something else")]
+        )
+        hashed = sha256("pii".encode("utf-8")).hexdigest()
+        analyzer_results = ["pii"]
+        result = test_data_frame.rdd.map(
+            lambda row: Anonymizer.hash(row, analyzer_results)
+        ).toDF()
+
+        actual = result.collect()[0][0]
+
+        self.assertEqual(actual, f"text containing {hashed}")
+
+    def test_hash_for_multiple_analyzer_results(self):
+        test_data_frame = self.SPARK.createDataFrame(
+            [("text containing pii1 and pii2", "something else")]
+        )
+        analyzer_results = ["pii1", "pii2"]
+        hashed1 = sha256("pii1".encode("utf-8")).hexdigest()
+        hashed2 = sha256("pii2".encode("utf-8")).hexdigest()
+        result = test_data_frame.rdd.map(
+            lambda row: Anonymizer.hash(row, analyzer_results)
+        ).toDF()
+
+        actual = result.collect()[0][0]
+
+        self.assertEqual(actual, f"text containing {hashed1} and {hashed2}")

pii_anonymizer/standalone/analyze/detectors/pii_detector.py

@@ -54,6 +54,8 @@ def analyze_and_anonymize(self, text: str):
                 redacted_text = Anonymizer.drop(text, analyzer_results)
             case "redact":
                 redacted_text = Anonymizer.redact(text, analyzer_results)
+            case "hash":
+                redacted_text = Anonymizer.hash(text, analyzer_results)
             case _:
                 redacted_text = Anonymizer.drop(text, analyzer_results)
 

pii_anonymizer/standalone/anonymize/anonymizer.py

@@ -1,5 +1,7 @@
 from pii_anonymizer.standalone.analyze.utils.analyzer_result import AnalyzerResult
 
+from hashlib import sha256
+
 
 class Anonymizer:
     @staticmethod
@@ -13,3 +15,11 @@ def redact(text: str, analyzer_results: [AnalyzerResult]):
         for result in analyzer_results:
             text = text.replace(result.text, "[Redacted]")
         return text
+
+    @staticmethod
+    def hash(text: str, analyzer_results: [AnalyzerResult]):
+        for result in analyzer_results:
+            text = text.replace(
+                result.text, sha256(result.text.encode("utf-8")).hexdigest()
+            )
+        return text

pii_anonymizer/standalone/anonymize/tests/test_hash_anonymizer.py

@@ -0,0 +1,24 @@
+from unittest import TestCase
+from pii_anonymizer.standalone.anonymize.anonymizer import Anonymizer
+from pii_anonymizer.standalone.analyze.utils.analyzer_result import AnalyzerResult
+from hashlib import sha256
+
+
+class TestRedactAnonymizer(TestCase):
+    def test_hash_for_single_analyzer_result(self):
+        text = "text containing pii"
+        hashed = sha256("pii".encode("utf-8")).hexdigest()
+        analyzer_results = [AnalyzerResult("pii", "PII_DETECTOR", 16, 18)]
+        result = Anonymizer.hash(text, analyzer_results)
+        self.assertEqual(result, f"text containing {hashed}")
+
+    def test_hash_for_multiple_analyzer_results(self):
+        text = "text containing pii1 and pii2"
+        hashed1 = sha256("pii1".encode("utf-8")).hexdigest()
+        hashed2 = sha256("pii2".encode("utf-8")).hexdigest()
+        analyzer_results = [
+            AnalyzerResult("pii1", "PII_DETECTOR", 16, 19),
+            AnalyzerResult("pii2", "PII_DETECTOR", 25, 28),
+        ]
+        result = Anonymizer.hash(text, analyzer_results)
+        self.assertEqual(result, f"text containing {hashed1} and {hashed2}")