add PII column tag (#136)

* add PII tag * add orders_augmented * add powerbi PII tag as well * comment out hooli_bi from workspace
dagster-io · Dec 5, 2024 · 7a79320 · 7a79320
1 parent 8c5909e
commit 7a79320
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 3 deletions.
diff --git a/dbt_project/models/ANALYTICS/analytics.yml b/dbt_project/models/ANALYTICS/analytics.yml
@@ -5,4 +5,11 @@ models:
         freshness_check:
           deadline_cron: "0 9 * * MON"
           lower_bound_delta_seconds: 1980
-          severity: "WARN"
+          severity: "WARN"
+  - name: orders_augmented
+    columns:
+      - name: user_id
+        description: "Platform id of the user that placed this order."
+        tests:
+          - not_null
+        tags: ["PII"]
diff --git a/dbt_project/models/CLEANED/schema.yml b/dbt_project/models/CLEANED/schema.yml
@@ -16,6 +16,7 @@ models:
         data_type: "int"
         tests:
           - not_null
+        tags: ["PII"]
       - name: quantity
         description: "The quantity of items purchased in this order."
       - name: purchase_price
@@ -42,10 +43,12 @@ models:
     # group: core-data-eng
     description: "Raw users data with test accounts removed"
     config:
-      tags: "core_kpis"
+      tags:
+        - "core_kpis"
     columns:
       - name: "user_id"
         description: "Platform id of the user."
+        tags: ["PII"]
         data_type: "int"
       - name: "company"
         description: "The name of the company that this user belongs to."
@@ -63,6 +66,7 @@ models:
     description: "Locations data with standardized column names"
     columns:
       - name: "user_id"
+        tags: ["PII"]
         description: "The unique identifier for the user"
         data_type: "int"
       - name: "street_address"

diff --git a/hooli-bi/hooli_bi/powerbi_assets.py b/hooli-bi/hooli_bi/powerbi_assets.py
@@ -30,7 +30,8 @@ def get_semantic_model_spec(self, data: PowerBIContentData) -> AssetSpec:
                                 metadata={"dagster/column_schema": TableSchema(
             columns=[TableColumn(
                 name=col["name"], 
-                type=col["dataType"]) 
+                type=col["dataType"],
+                tags={"PII":""} if col["name"] == "USER_ID" else None) 
                 for col in data.properties["tables"][0]["columns"]])},
                 tags={"core_kpis":""},
         )

diff --git a/workspace.yaml b/workspace.yaml
@@ -9,3 +9,4 @@ load_from:
   # - python_module:
   #     module_name: hooli_bi
   #     working_directory: hooli-bi/
+
Original file line number	Diff line number	Diff line change
Expand Up		@@ -9,3 +9,4 @@ load_from:
		# - python_module:
		# module_name: hooli_bi
		# working_directory: hooli-bi/