Merge pull request #17 from cleardataeng/dataform_support

add support for dataform repositories and workspaces

requirements.txt

@@ -1,5 +1,5 @@
 google-api-python-client-helpers>=1.2.6
-google-api-python-client==2.0.2
+google-api-python-client~=2.126.0
 jmespath
 tenacity
 python-dateutil

rpe/engines/python.py

@@ -23,11 +23,9 @@
 
 
 class PythonPolicyEngine:
-
     counter = 0
 
     def __init__(self, package_path):
-
         self._policies = {}
         self.package_path = package_path
         PythonPolicyEngine.counter += 1
@@ -85,7 +83,6 @@ def evaluate(self, resource):
 
         for policy_name, policy_cls in matched_policies.items():
             try:
-
                 if hasattr(policy_cls, "evaluate"):
                     eval_result = policy_cls.evaluate(resource)
                     if not isinstance(eval_result, EvaluationResult):

rpe/extractors/gcp_auditlogs.py

@@ -65,7 +65,6 @@ def extract(cls, log_message):
 
     @classmethod
     def is_audit_log(cls, message_data):
-
         log_type = jmespath.search('protoPayload."@type"', message_data)
         log_name = message_data.get("logName", "")
 
@@ -83,7 +82,6 @@ def is_audit_log(cls, message_data):
 
     @classmethod
     def get_metadata(cls, message_data):
-
         method_name = jmespath.search("protoPayload.methodName", message_data)
         insert_id = message_data.get("insertId")
 
@@ -106,7 +104,6 @@ def get_metadata(cls, message_data):
 
     @classmethod
     def get_operation_type(cls, method_name):
-
         last = method_name.split(".")[-1].lower()
         # For batch methods, look for the verb after the word 'batch'
         if last.startswith("batch"):
@@ -148,7 +145,6 @@ def get_operation_type(cls, method_name):
 
     @classmethod
     def get_resources(cls, message):
-
         resources = []
 
         res_type = jmespath.search("resource.type", message)
@@ -170,7 +166,6 @@ def add_resource():
         if res_type == "cloudsql_database" and method_name.startswith(
             "cloudsql.instances"
         ):
-
             resource_data = {
                 "resource_type": "sqladmin.googleapis.com/Instance",
                 # CloudSQL logs are inconsistent. See https://issuetracker.google.com/issues/137629452
@@ -233,7 +228,6 @@ def add_resource():
             or "DisableService" in method_name
             or "ctivateService" in method_name
         ):
-
             resource_data = {
                 "resource_type": "serviceusage.googleapis.com/Service",
                 "project_id": prop("resource.labels.project_id"),
@@ -298,7 +292,6 @@ def add_resource():
             add_resource()
 
         elif res_type == "gce_instance":
-
             instance_name = prop("protoPayload.resourceName").split("/")[-1]
 
             resource_data = {
@@ -321,7 +314,6 @@ def add_resource():
             disks = prop("protoPayload.request.disks") or []
 
             for disk in disks:
-
                 # The name of the disk is complicated. If the diskName is set in initParams use that
                 # If not AND its the boot disk, use the instance name
                 # Otherwise use the device name
@@ -433,4 +425,22 @@ def add_resource():
             }
             add_resource()
 
+        elif (
+            res_type == "audited_resource"
+            and prop("resource.labels.service") == "dataform.googleapis.com"
+        ):
+            name_bits = prop("protoPayload.resourceName").split("/")
+            resource_data = {
+                "name": name_bits[len(name_bits) - 1],
+                "project_id": name_bits[1],
+                "location": name_bits[3],
+            }
+            if len(name_bits) == 6 and name_bits[4] == "repositories":
+                resource_data["resource_type"] = "dataform.googleapis.com/Repository"
+                add_resource()
+            elif len(name_bits) == 8 and name_bits[6] == "workspaces":
+                resource_data["resource_type"] = "dataform.googleapis.com/Workspace"
+                resource_data["repository"] = name_bits[4]
+                add_resource()
+
         return resources

rpe/extractors/micromanager.py

@@ -33,7 +33,6 @@ class MicromanagerMetadata(PubsubMessageMetadata, ExtractedMetadata):
 class MicromanagerEvaluationRequest(Extractor):
     @classmethod
     def extract(cls, message):
-
         message_data = json.loads(message.data)
 
         name = message_data.get("name")

rpe/policy.py

@@ -27,7 +27,6 @@ class _EvaluationTrigger:
 # that has the results of an eval without details about what triggered it
 @dataclass
 class EvaluationResult:
-
     compliant: bool
     remediable: bool
 

rpe/resources/base.py

@@ -17,7 +17,6 @@
 
 
 class Resource(ABC):
-
     # Returns a dictionary representing the resource. Must contain a 'type' key
     # indicating what type of resource it is
     @abstractmethod