diff --git a/requirements.txt b/requirements.txt index f94d62a..767793d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ google-api-python-client-helpers>=1.2.6 -google-api-python-client==2.0.2 +google-api-python-client~=2.126.0 jmespath tenacity python-dateutil diff --git a/rpe/engines/python.py b/rpe/engines/python.py index 1640a3a..869928d 100644 --- a/rpe/engines/python.py +++ b/rpe/engines/python.py @@ -23,11 +23,9 @@ class PythonPolicyEngine: - counter = 0 def __init__(self, package_path): - self._policies = {} self.package_path = package_path PythonPolicyEngine.counter += 1 @@ -85,7 +83,6 @@ def evaluate(self, resource): for policy_name, policy_cls in matched_policies.items(): try: - if hasattr(policy_cls, "evaluate"): eval_result = policy_cls.evaluate(resource) if not isinstance(eval_result, EvaluationResult): diff --git a/rpe/extractors/gcp_auditlogs.py b/rpe/extractors/gcp_auditlogs.py index 0c32195..b886d9b 100644 --- a/rpe/extractors/gcp_auditlogs.py +++ b/rpe/extractors/gcp_auditlogs.py @@ -65,7 +65,6 @@ def extract(cls, log_message): @classmethod def is_audit_log(cls, message_data): - log_type = jmespath.search('protoPayload."@type"', message_data) log_name = message_data.get("logName", "") @@ -83,7 +82,6 @@ def is_audit_log(cls, message_data): @classmethod def get_metadata(cls, message_data): - method_name = jmespath.search("protoPayload.methodName", message_data) insert_id = message_data.get("insertId") @@ -106,7 +104,6 @@ def get_metadata(cls, message_data): @classmethod def get_operation_type(cls, method_name): - last = method_name.split(".")[-1].lower() # For batch methods, look for the verb after the word 'batch' if last.startswith("batch"): @@ -148,7 +145,6 @@ def get_operation_type(cls, method_name): @classmethod def get_resources(cls, message): - resources = [] res_type = jmespath.search("resource.type", message) @@ -170,7 +166,6 @@ def add_resource(): if res_type == "cloudsql_database" and method_name.startswith( "cloudsql.instances" ): - resource_data = { "resource_type": "sqladmin.googleapis.com/Instance", # CloudSQL logs are inconsistent. See https://issuetracker.google.com/issues/137629452 @@ -233,7 +228,6 @@ def add_resource(): or "DisableService" in method_name or "ctivateService" in method_name ): - resource_data = { "resource_type": "serviceusage.googleapis.com/Service", "project_id": prop("resource.labels.project_id"), @@ -298,7 +292,6 @@ def add_resource(): add_resource() elif res_type == "gce_instance": - instance_name = prop("protoPayload.resourceName").split("/")[-1] resource_data = { @@ -321,7 +314,6 @@ def add_resource(): disks = prop("protoPayload.request.disks") or [] for disk in disks: - # The name of the disk is complicated. If the diskName is set in initParams use that # If not AND its the boot disk, use the instance name # Otherwise use the device name @@ -433,4 +425,22 @@ def add_resource(): } add_resource() + elif ( + res_type == "audited_resource" + and prop("resource.labels.service") == "dataform.googleapis.com" + ): + name_bits = prop("protoPayload.resourceName").split("/") + resource_data = { + "name": name_bits[len(name_bits) - 1], + "project_id": name_bits[1], + "location": name_bits[3], + } + if len(name_bits) == 6 and name_bits[4] == "repositories": + resource_data["resource_type"] = "dataform.googleapis.com/Repository" + add_resource() + elif len(name_bits) == 8 and name_bits[6] == "workspaces": + resource_data["resource_type"] = "dataform.googleapis.com/Workspace" + resource_data["repository"] = name_bits[4] + add_resource() + return resources diff --git a/rpe/extractors/micromanager.py b/rpe/extractors/micromanager.py index 7c8eaa7..e10de77 100644 --- a/rpe/extractors/micromanager.py +++ b/rpe/extractors/micromanager.py @@ -33,7 +33,6 @@ class MicromanagerMetadata(PubsubMessageMetadata, ExtractedMetadata): class MicromanagerEvaluationRequest(Extractor): @classmethod def extract(cls, message): - message_data = json.loads(message.data) name = message_data.get("name") diff --git a/rpe/policy.py b/rpe/policy.py index dafa674..3bf7fc8 100644 --- a/rpe/policy.py +++ b/rpe/policy.py @@ -27,7 +27,6 @@ class _EvaluationTrigger: # that has the results of an eval without details about what triggered it @dataclass class EvaluationResult: - compliant: bool remediable: bool diff --git a/rpe/resources/base.py b/rpe/resources/base.py index bd3377b..76cee17 100644 --- a/rpe/resources/base.py +++ b/rpe/resources/base.py @@ -17,7 +17,6 @@ class Resource(ABC): - # Returns a dictionary representing the resource. Must contain a 'type' key # indicating what type of resource it is @abstractmethod diff --git a/rpe/resources/gcp.py b/rpe/resources/gcp.py index f3f8997..d0c8cb1 100644 --- a/rpe/resources/gcp.py +++ b/rpe/resources/gcp.py @@ -32,7 +32,6 @@ class GoogleAPIResource(Resource): - # Names of the get method of the root resource get_method = "get" required_resource_data = ["name"] @@ -56,7 +55,6 @@ class GoogleAPIResource(Resource): inferred_data_map = None def __init__(self, client_kwargs=None, http=None, **resource_data): - if client_kwargs is None: client_kwargs = {} @@ -78,7 +76,6 @@ def __init__(self, client_kwargs=None, http=None, **resource_data): def _validate_resource_data(self): """Verify we have all the required data for this resource""" if not all(arg in self._resource_data for arg in self.required_resource_data): - raise ResourceException( "Missing data required for resource creation. Expected data: {}; Got: {}".format( ",".join(self.required_resource_data), @@ -104,6 +101,8 @@ def _extract_cai_name_data(name): "cluster": r"/clusters/([^\/]+)/", # ServiceAccounts "service_account": r"serviceAccounts/([^\/]+)/", + # Dataform repository, used to query dataform workspaces + "repository": r"/repositories/([^\/]+)/", } resource_data = {} @@ -118,6 +117,10 @@ def _extract_cai_name_data(name): @classmethod def subclass_by_type(cls, resource_type): + # maps resource_type to the actual resource for cai events and audit log events + # cai events use the from_cai_data method to pass in the asset_type, which is used to map to the resource + # audit log events use from_resource_data from_resource_data to pass in the resource_type, + # which is used to map to the resource mapper = {res_cls.resource_type: res_cls for res_cls in cls.__subclasses__()} try: @@ -156,7 +159,6 @@ def from_cai_data( return res_cls(client_kwargs=client_kwargs, http=http, **resource_data) def to_dict(self): - self._refresh_inferred_data() details = self._resource_data.copy() @@ -175,7 +177,6 @@ def to_dict(self): # Some useful resource data may not be available when instantiated def _refresh_inferred_data(self): - if not self.inferred_data_map: return @@ -210,7 +211,6 @@ def full_resource_name(self): # If we inject it into the resource, we can use it in policy evaluation to # simplify the structure of our policies def gen_full_resource_name(self): - method = getattr(self.service, self.get_method) uri = method(**self._get_request_args()).uri @@ -293,7 +293,6 @@ def _get_component(self, component): return component_metadata def get(self, refresh=True): - if not refresh and self._resource_metadata: return self._resource_metadata @@ -410,7 +409,6 @@ def client_kwargs(self): @client_kwargs.setter def client_kwargs(self, client_kwargs): - # Invalidate service/parent because client_kwargs changed self._service = None @@ -419,7 +417,6 @@ def client_kwargs(self, client_kwargs): @property def service(self): if self._service is None: - full_resource_path = "{}.{}".format(self.service_name, self.resource_path) self._service = build_subresource( full_resource_path, self.version, **self._client_kwargs, http=self._http @@ -462,7 +459,6 @@ def uniquifier(self): class GcpAppEngineInstance(GoogleAPIResource): - service_name = "appengine" resource_path = "apps.services.versions.instances" version = "v1" @@ -487,7 +483,6 @@ def _get_request_args(self): class GcpBigqueryDataset(GoogleAPIResource): - service_name = "bigquery" resource_path = "datasets" version = "v2" @@ -508,7 +503,6 @@ def _get_request_args(self): class GcpBigtableInstance(GoogleAPIResource): - service_name = "bigtableadmin" resource_path = "projects.instances" version = "v2" @@ -539,7 +533,6 @@ def _get_iam_request_args(self): class GcpCloudFunction(GoogleAPIResource): - service_name = "cloudfunctions" resource_path = "projects.locations.functions" version = "v1" @@ -580,7 +573,6 @@ def _get_iam_request_args(self): class GcpComputeInstance(GoogleAPIResource): - service_name = "compute" resource_path = "instances" version = "v1" @@ -602,7 +594,6 @@ def _get_request_args(self): class GcpComputeDisk(GoogleAPIResource): - service_name = "compute" resource_path = "disks" version = "v1" @@ -624,7 +615,6 @@ def _get_request_args(self): class GcpComputeRegionDisk(GoogleAPIResource): - service_name = "compute" resource_path = "regionDisks" version = "v1" @@ -646,7 +636,6 @@ def _get_request_args(self): class GcpComputeNetwork(GoogleAPIResource): - service_name = "compute" resource_path = "networks" version = "v1" @@ -667,7 +656,6 @@ def _get_request_args(self): class GcpComputeSubnetwork(GoogleAPIResource): - service_name = "compute" resource_path = "subnetworks" version = "v1" @@ -689,7 +677,6 @@ def _get_request_args(self): class GcpComputeFirewall(GoogleAPIResource): - service_name = "compute" resource_path = "firewalls" version = "v1" @@ -771,7 +758,6 @@ def _get_iam_request_args(self): class GcpGkeCluster(GoogleAPIResource): - service_name = "container" resource_path = "projects.locations.clusters" version = "v1" @@ -800,7 +786,6 @@ def _get_request_args(self): class GcpGkeClusterNodepool(GoogleAPIResource): - service_name = "container" resource_path = "projects.locations.clusters.nodePools" version = "v1" @@ -829,7 +814,6 @@ def _get_request_args(self): class GcpIamServiceAccount(GoogleAPIResource): - service_name = "iam" resource_path = "projects.serviceAccounts" version = "v1" @@ -851,7 +835,6 @@ def _get_request_args(self): class GcpIamServiceAccountKey(GoogleAPIResource): - service_name = "iam" resource_path = "projects.serviceAccounts.keys" version = "v1" @@ -875,7 +858,6 @@ def _get_request_args(self): class GcpPubsubSubscription(GoogleAPIResource): - service_name = "pubsub" resource_path = "projects.subscriptions" version = "v1" @@ -904,7 +886,6 @@ def _get_iam_request_args(self): class GcpPubsubTopic(GoogleAPIResource): - service_name = "pubsub" resource_path = "projects.topics" version = "v1" @@ -933,7 +914,6 @@ def _get_iam_request_args(self): class GcpStorageBucket(GoogleAPIResource): - service_name = "storage" resource_path = "buckets" version = "v1" @@ -958,7 +938,6 @@ def _get_request_args(self): class GcpSqlInstance(GoogleAPIResource): - service_name = "sqladmin" resource_path = "instances" version = "v1beta4" @@ -978,7 +957,6 @@ def _get_request_args(self): class GcpOrganization(GoogleAPIResource): - service_name = "cloudresourcemanager" resource_path = "organizations" version = "v1" @@ -997,7 +975,6 @@ def _get_iam_request_args(self): class GcpProject(GoogleAPIResource): - service_name = "cloudresourcemanager" resource_path = "projects" version = "v1" @@ -1020,7 +997,6 @@ def _get_iam_request_args(self): class GcpProjectService(GoogleAPIResource): - service_name = "serviceusage" resource_path = "services" version = "v1" @@ -1038,7 +1014,6 @@ def _get_request_args(self): class GcpDataflowJob(GoogleAPIResource): - service_name = "dataflow" resource_path = "projects.locations.jobs" version = "v1b3" @@ -1061,7 +1036,6 @@ def _get_request_args(self): class GcpRedisInstance(GoogleAPIResource): - service_name = "redis" resource_path = "projects.locations.instances" version = "v1" @@ -1089,7 +1063,6 @@ def _get_request_args(self): class GcpMemcacheInstance(GoogleAPIResource): - service_name = "memcache" resource_path = "projects.locations.instances" version = "v1" @@ -1114,3 +1087,62 @@ def _get_request_args(self): self._resource_data["name"], ), } + + +class GcpDataformRepository(GoogleAPIResource): + service_name = "dataform" + resource_path = "projects.locations.repositories" + version = "v1beta1" + + required_resource_data = ["name", "location", "project_id"] + + resource_components = { + "iam": "getIamPolicy", + } + + resource_type = "dataform.googleapis.com/Repository" + + inferred_data_map = { + "uniquifier": "createTime", + } + + def _get_resource_string(self): + return "projects/{}/locations/{}/repositories/{}".format( + self._resource_data["project_id"], + self._resource_data["location"], + self._resource_data["name"], + ) + + def _get_request_args(self): + return {"name": self._get_resource_string()} + + def _get_iam_request_args(self): + return {"resource": self._get_resource_string()} + + +class GcpDataformWorkspace(GoogleAPIResource): + service_name = "dataform" + resource_path = "projects.locations.repositories.workspaces" + version = "v1beta1" + + required_resource_data = ["name", "location", "project_id", "repository"] + + resource_components = { + "iam": "getIamPolicy", + } + + resource_type = "dataform.googleapis.com/Workspace" + + def _get_resource_string(self): + return "projects/{}/locations/{}/repositories/{}/workspaces/{}".format( + self._resource_data["project_id"], + self._resource_data["location"], + self._resource_data["repository"], + self._resource_data["name"], + ) + + def _get_request_args(self): + return {"name": self._get_resource_string()} + + def _get_iam_request_args(self): + return {"resource": self._get_resource_string()} diff --git a/tests/data/dataform-create-repository.json b/tests/data/dataform-create-repository.json new file mode 100644 index 0000000..2ac1ce9 --- /dev/null +++ b/tests/data/dataform-create-repository.json @@ -0,0 +1,57 @@ +{ + "protoPayload": { + "@type": "type.googleapis.com/google.cloud.audit.AuditLog", + "status": {}, + "authenticationInfo": { + "principalEmail": "yash.vaidya@cleardata.com" + }, + "requestMetadata": { + "callerIp": "::1", + "requestAttributes": { + "time": "2024-04-16T13:01:55.220243141Z", + "auth": {} + }, + "destinationAttributes": {} + }, + "serviceName": "dataform.googleapis.com", + "methodName": "google.cloud.dataform.v1beta1.Dataform.CreateRepository", + "authorizationInfo": [ + { + "resource": "projects/test-project/locations/us-east5/repositories/test-repository", + "permission": "dataform.repositories.create", + "granted": true, + "resourceAttributes": { + "service": "dataform.googleapis.com", + "name": "projects/test-project/locations/us-east5/repositories/test-repository", + "type": "dataform.googleapis.com/Repository" + }, + "permissionType": "ADMIN_WRITE" + } + ], + "resourceName": "projects/test-project/locations/us-east5/repositories/test-repository", + "request": { + "@type": "type.googleapis.com/google.cloud.dataform.v1beta1.CreateRepositoryRequest" + }, + "response": { + "@type": "type.googleapis.com/google.cloud.dataform.v1beta1.Repository" + }, + "resourceLocation": { + "currentLocations": [ + "us-east5" + ] + } + }, + "insertId": "a01kboditbv", + "resource": { + "type": "audited_resource", + "labels": { + "method": "google.cloud.dataform.v1beta1.Dataform.CreateRepository", + "project_id": "test-project", + "service": "dataform.googleapis.com" + } + }, + "timestamp": "2024-04-16T13:01:57.609876595Z", + "severity": "NOTICE", + "logName": "projects/test-project/logs/cloudaudit.googleapis.com%2Factivity", + "receiveTimestamp": "2024-04-16T13:01:58.328125429Z" +} \ No newline at end of file diff --git a/tests/data/dataform-create-workspace.json b/tests/data/dataform-create-workspace.json new file mode 100644 index 0000000..52350cf --- /dev/null +++ b/tests/data/dataform-create-workspace.json @@ -0,0 +1,57 @@ +{ + "protoPayload": { + "@type": "type.googleapis.com/google.cloud.audit.AuditLog", + "status": {}, + "authenticationInfo": { + "principalEmail": "yash.vaidya@cleardata.com" + }, + "requestMetadata": { + "callerIp": "::1", + "requestAttributes": { + "time": "2024-04-16T13:02:05.896006678Z", + "auth": {} + }, + "destinationAttributes": {} + }, + "serviceName": "dataform.googleapis.com", + "methodName": "google.cloud.dataform.v1beta1.Dataform.CreateWorkspace", + "authorizationInfo": [ + { + "resource": "projects/test-project/locations/us-east5/repositories/test-repository/workspaces/test-workspace", + "permission": "dataform.workspaces.create", + "granted": true, + "resourceAttributes": { + "service": "dataform.googleapis.com", + "name": "projects/test-project/locations/us-east5/repositories/test-repository/workspaces/test-workspace", + "type": "dataform.googleapis.com/Workspace" + }, + "permissionType": "ADMIN_WRITE" + } + ], + "resourceName": "projects/test-project/locations/us-east5/repositories/test-repository/workspaces/test-workspace", + "request": { + "@type": "type.googleapis.com/google.cloud.dataform.v1beta1.CreateWorkspaceRequest" + }, + "response": { + "@type": "type.googleapis.com/google.cloud.dataform.v1beta1.Workspace" + }, + "resourceLocation": { + "currentLocations": [ + "us-east5" + ] + } + }, + "insertId": "1cu2hlce2l3dd", + "resource": { + "type": "audited_resource", + "labels": { + "project_id": "test-project", + "method": "google.cloud.dataform.v1beta1.Dataform.CreateWorkspace", + "service": "dataform.googleapis.com" + } + }, + "timestamp": "2024-04-16T13:02:06.781641484Z", + "severity": "NOTICE", + "logName": "projects/test-project/logs/cloudaudit.googleapis.com%2Factivity", + "receiveTimestamp": "2024-04-16T13:02:07.768003559Z" +} \ No newline at end of file diff --git a/tests/data/dataform-update-repository.json b/tests/data/dataform-update-repository.json new file mode 100644 index 0000000..ea3fec7 --- /dev/null +++ b/tests/data/dataform-update-repository.json @@ -0,0 +1,57 @@ +{ + "protoPayload": { + "@type": "type.googleapis.com/google.cloud.audit.AuditLog", + "status": {}, + "authenticationInfo": { + "principalEmail": "yash.vaidya@cleardata.com" + }, + "requestMetadata": { + "callerIp": "::1", + "requestAttributes": { + "time": "2024-04-17T19:57:47.218933650Z", + "auth": {} + }, + "destinationAttributes": {} + }, + "serviceName": "dataform.googleapis.com", + "methodName": "google.cloud.dataform.v1beta1.Dataform.UpdateRepository", + "authorizationInfo": [ + { + "resource": "projects/test-project/locations/us-east5/repositories/test-repository", + "permission": "dataform.repositories.update", + "granted": true, + "resourceAttributes": { + "service": "dataform.googleapis.com", + "name": "projects/test-project/locations/us-east5/repositories/test-repository", + "type": "dataform.googleapis.com/Repository" + }, + "permissionType": "ADMIN_WRITE" + } + ], + "resourceName": "projects/test-project/locations/us-east5/repositories/test-repository", + "request": { + "@type": "type.googleapis.com/google.cloud.dataform.v1beta1.UpdateRepositoryRequest" + }, + "response": { + "@type": "type.googleapis.com/google.cloud.dataform.v1beta1.Repository" + }, + "resourceLocation": { + "currentLocations": [ + "us-east5" + ] + } + }, + "insertId": "1lfo2qse2tixp", + "resource": { + "type": "audited_resource", + "labels": { + "service": "dataform.googleapis.com", + "project_id": "test-project", + "method": "google.cloud.dataform.v1beta1.Dataform.UpdateRepository" + } + }, + "timestamp": "2024-04-17T19:57:47.314676541Z", + "severity": "NOTICE", + "logName": "projects/test-project/logs/cloudaudit.googleapis.com%2Factivity", + "receiveTimestamp": "2024-04-17T19:57:48.059129560Z" +} \ No newline at end of file diff --git a/tests/test_extractors.py b/tests/test_extractors.py index 7d5986e..716f031 100644 --- a/tests/test_extractors.py +++ b/tests/test_extractors.py @@ -208,6 +208,24 @@ def get_test_data(filename): "create", "test-instance", ), + ( + "dataform-create-repository.json", + "dataform.googleapis.com/Repository", + "create", + "test-repository", + ), + ( + "dataform-create-workspace.json", + "dataform.googleapis.com/Workspace", + "create", + "test-workspace", + ), + ( + "dataform-update-repository.json", + "dataform.googleapis.com/Repository", + "update", + "test-repository", + ), ] test_micromanager_log = [ diff --git a/tests/test_resources.py b/tests/test_resources.py index 946a991..ebf17e6 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -32,6 +32,8 @@ GcpDataflowJob, GcpDatafusionInstance, GcpDataprocCluster, + GcpDataformRepository, + GcpDataformWorkspace, GcpGkeCluster, GcpGkeClusterNodepool, GcpMemcacheInstance, @@ -436,6 +438,41 @@ ), uniquifier="createTime", ), + ResourceTestCase( + resource_data={ + "name": test_resource_name, + "location": "us-central1", + "project_id": test_project, + }, + cls=GcpDataformRepository, + resource_type="dataform.googleapis.com/Repository", + name="//dataform.googleapis.com/projects/my_project/locations/us-central1/repositories/my_resource", + http=HttpMockSequence( + [ + ({"status": 200}, '{"createTime":"createTime"}'), + ({"status": 200}, "{}"), + ] + ), + uniquifier="createTime", + ), + ResourceTestCase( + resource_data={ + "name": test_resource_name, + "location": "us-central1", + "project_id": test_project, + "repository": "test_repository", + }, + cls=GcpDataformWorkspace, + resource_type="dataform.googleapis.com/Workspace", + name="//dataform.googleapis.com/projects/my_project/locations/us-central1/repositories/test_repository/workspaces/test_resource_name", + http=HttpMockSequence( + [ + ({"status": 200}, '{"createTime":"createTime"}'), + ({"status": 200}, "{}"), + ] + ), + uniquifier="createTime", + ), ] @@ -492,7 +529,6 @@ def test_gcp_location(case): def test_missing_resource_data(): with pytest.raises(ResourceException) as excinfo: - GcpAppEngineInstance(name=test_resource_name) assert "Missing data required for resource creation" in str(excinfo.value) diff --git a/tests/test_resources_cai.py b/tests/test_resources_cai.py index cee36e5..5d95f06 100644 --- a/tests/test_resources_cai.py +++ b/tests/test_resources_cai.py @@ -32,6 +32,8 @@ GcpDataflowJob, GcpDatafusionInstance, GcpDataprocCluster, + GcpDataformRepository, + GcpDataformWorkspace, GcpGkeCluster, GcpGkeClusterNodepool, GcpIamServiceAccount, @@ -228,6 +230,20 @@ }, resource_cls=GcpMemcacheInstance, ), + CaiTestCase( + data={ + "name": "//dataform.googleapis.com/projects/test-project/locations/us-central1/repositories/test-resource", + "asset_type": "dataform.googleapis.com/Repository", + }, + resource_cls=GcpDataformRepository, + ), + CaiTestCase( + data={ + "name": "//dataform.googleapis.com/projects/test-project/locations/us-central1/repositories/test-repository/workspaces/test-resource", + "asset_type": "dataform.googleapis.com/Workspace", + }, + resource_cls=GcpDataformWorkspace, + ), ] @@ -245,7 +261,6 @@ def test_gcp_resource_from_cai_data(case): def test_bad_resource_type(): - with pytest.raises(ResourceException) as excinfo: GoogleAPIResource.from_cai_data( "//cloudfakeservice.googleapis.com/widgets/test-resource",