Merge pull request #10 from cleardataeng/add_inferred_data_support

Add inferred data support to GoogleAPIResources

rpe/resources/gcp.py

@@ -43,8 +43,6 @@ class GoogleAPIResource(Resource):
     # Other properties of a resource we might need to perform evaluations, such as iam policy
     resource_components = {}
 
-    uniquifier_path = None
-
     # If a resource is not in a ready state, we can't update it. If we retrieve
     # it, and the state changes, updates will be rejected because the ETAG will
     # have changed. If a resource defines readiness criteria, the get() call
@@ -55,6 +53,8 @@ class GoogleAPIResource(Resource):
     readiness_value = None
     readiness_terminal_values = []
 
+    inferred_data_map = None
+
     def __init__(self, client_kwargs=None, http=None, **resource_data):
 
         if client_kwargs is None:
@@ -156,6 +156,9 @@ def from_cai_data(
         return res_cls(client_kwargs=client_kwargs, http=http, **resource_data)
 
     def to_dict(self):
+
+        self._refresh_inferred_data()
+
         details = self._resource_data.copy()
         details.update(
             {
@@ -170,6 +173,26 @@ def to_dict(self):
 
         return details
 
+    # Some useful resource data may not be available when instantiated
+    def _refresh_inferred_data(self):
+
+        if not self.inferred_data_map:
+            return
+
+        try:
+            resource_metadata = self.get(refresh=False)["resource"]
+        except Exception:
+            return
+
+        for key, path in self.inferred_data_map.items():
+            # Don't replace existing data
+            if key in self._resource_data:
+                continue
+            value = jmespath.search(path, resource_metadata)
+
+            if value is not None:
+                self._resource_data[key] = value
+
     def type(self):
         return self.resource_type
 
@@ -433,11 +456,9 @@ def location(self):
     # creation timestamp of the resource. Not all resources have fields that make this  possible.
     @property
     def uniquifier(self):
-        if not self.uniquifier_path:
-            return None
+        self._refresh_inferred_data()
 
-        resource_metadata = self.get(refresh=False).get("resource")
-        return jmespath.search(self.uniquifier_path, resource_metadata)
+        return self._resource_data.get("uniquifier")
 
 
 class GcpAppEngineInstance(GoogleAPIResource):
@@ -452,7 +473,9 @@ class GcpAppEngineInstance(GoogleAPIResource):
 
     required_resource_data = ["name", "app", "service", "version"]
 
-    uniquifier_path = "startTime"
+    inferred_data_map = {
+        "uniquifier": "startTime",
+    }
 
     def _get_request_args(self):
         return {
@@ -473,7 +496,9 @@ class GcpBigqueryDataset(GoogleAPIResource):
 
     resource_type = "bigquery.googleapis.com/Dataset"
 
-    uniquifier_path = "id"
+    inferred_data_map = {
+        "uniquifier": "id",
+    }
 
     def _get_request_args(self):
         return {
@@ -564,7 +589,9 @@ class GcpComputeInstance(GoogleAPIResource):
 
     resource_type = "compute.googleapis.com/Instance"
 
-    uniquifier_path = "id"
+    inferred_data_map = {
+        "uniquifier": "id",
+    }
 
     def _get_request_args(self):
         return {
@@ -584,7 +611,9 @@ class GcpComputeDisk(GoogleAPIResource):
 
     resource_type = "compute.googleapis.com/Disk"
 
-    uniquifier_path = "id"
+    inferred_data_map = {
+        "uniquifier": "id",
+    }
 
     def _get_request_args(self):
         return {
@@ -604,7 +633,9 @@ class GcpComputeRegionDisk(GoogleAPIResource):
 
     resource_type = "compute.googleapis.com/RegionDisk"
 
-    uniquifier_path = "id"
+    inferred_data_map = {
+        "uniquifier": "id",
+    }
 
     def _get_request_args(self):
         return {
@@ -624,7 +655,9 @@ class GcpComputeNetwork(GoogleAPIResource):
 
     resource_type = "compute.googleapis.com/Network"
 
-    uniquifier_path = "id"
+    inferred_data_map = {
+        "uniquifier": "id",
+    }
 
     def _get_request_args(self):
         return {
@@ -643,7 +676,9 @@ class GcpComputeSubnetwork(GoogleAPIResource):
 
     resource_type = "compute.googleapis.com/Subnetwork"
 
-    uniquifier_path = "id"
+    inferred_data_map = {
+        "uniquifier": "id",
+    }
 
     def _get_request_args(self):
         return {
@@ -663,7 +698,9 @@ class GcpComputeFirewall(GoogleAPIResource):
 
     resource_type = "compute.googleapis.com/Firewall"
 
-    uniquifier_path = "id"
+    inferred_data_map = {
+        "uniquifier": "id",
+    }
 
     def _get_request_args(self):
         return {
@@ -681,7 +718,9 @@ class GcpDataprocCluster(GoogleAPIResource):
 
     resource_type = "dataproc.googleapis.com/Cluster"
 
-    uniquifier_path = "clusterUuid"
+    inferred_data_map = {
+        "uniquifier": "clusterUuid",
+    }
 
     def _get_request_args(self):
         return {
@@ -708,7 +747,9 @@ class GcpDatafusionInstance(GoogleAPIResource):
 
     resource_type = "datafusion.googleapis.com/Instance"
 
-    uniquifier_path = "createTime"
+    inferred_data_map = {
+        "uniquifier": "createTime",
+    }
 
     def _get_request_args(self):
         return {
@@ -744,7 +785,9 @@ class GcpGkeCluster(GoogleAPIResource):
 
     resource_type = "container.googleapis.com/Cluster"
 
-    uniquifier_path = "id"
+    inferred_data_map = {
+        "uniquifier": "id",
+    }
 
     def _get_request_args(self):
         return {
@@ -795,7 +838,9 @@ class GcpIamServiceAccount(GoogleAPIResource):
 
     resource_type = "iam.googleapis.com/ServiceAccount"
 
-    uniquifier_path = "uniqueId"
+    inferred_data_map = {
+        "uniquifier": "uniqueId",
+    }
 
     def _get_request_args(self):
         return {
@@ -815,7 +860,9 @@ class GcpIamServiceAccountKey(GoogleAPIResource):
 
     resource_type = "iam.googleapis.com/ServiceAccountKey"
 
-    uniquifier_path = "privateKeyData"
+    inferred_data_map = {
+        "uniquifier": "privateKeyData",
+    }
 
     def _get_request_args(self):
         return {
@@ -899,7 +946,10 @@ class GcpStorageBucket(GoogleAPIResource):
 
     resource_type = "storage.googleapis.com/Bucket"
 
-    uniquifier_path = "timeCreated"
+    inferred_data_map = {
+        "location": "location",
+        "uniquifier": "timeCreated",
+    }
 
     def _get_request_args(self):
         return {
@@ -958,7 +1008,9 @@ class GcpProject(GoogleAPIResource):
 
     resource_type = "cloudresourcemanager.googleapis.com/Project"  # beta
 
-    uniquifier_path = "projectNumber"
+    inferred_data_map = {
+        "uniquifier": "projectNumber",
+    }
 
     def _get_request_args(self):
         return {"projectId": self._resource_data["name"]}
@@ -995,7 +1047,9 @@ class GcpDataflowJob(GoogleAPIResource):
 
     resource_type = "dataflow.googleapis.com/Job"
 
-    uniquifier_path = "id"
+    inferred_data_map = {
+        "uniquifier": "id",
+    }
 
     def _get_request_args(self):
         return {
@@ -1020,7 +1074,9 @@ class GcpRedisInstance(GoogleAPIResource):
 
     resource_type = "redis.googleapis.com/Instance"
 
-    uniquifier_path = "createTime"
+    inferred_data_map = {
+        "uniquifier": "createTime",
+    }
 
     def _get_request_args(self):
         return {
@@ -1046,7 +1102,9 @@ class GcpMemcacheInstance(GoogleAPIResource):
 
     resource_type = "memcache.googleapis.com/Instance"
 
-    uniquifier_path = "createTime"
+    inferred_data_map = {
+        "uniquifier": "createTime",
+    }
 
     def _get_request_args(self):
         return {

tests/test_gcp_resource_inferred_data.py

@@ -0,0 +1,81 @@
+from rpe.resources.gcp import GoogleAPIResource
+from googleapiclient.http import HttpMockSequence
+import pytest
+
+
+@pytest.mark.parametrize(
+    "name,asset_type,http,inferred_key",
+    [
+        pytest.param(
+            "//storage.googleapis.com/buckets/test-bucket",
+            "storage.googleapis.com/Bucket",
+            HttpMockSequence(
+                [
+                    ({"status": 200}, '{"location": "US"}'),
+                    ({"status": 200}, '{"iam_policy":""}'),
+                ]
+            ),
+            "location",
+            id="bucket_location",
+        ),
+        pytest.param(
+            "//compute.googleapis.com/projects/test-project/zones/test-zone/instances/test-instance",
+            "compute.googleapis.com/Instance",
+            HttpMockSequence(
+                [
+                    ({"status": 200}, '{"id": "1234567890"}'),
+                    ({"status": 200}, '{"iam_policy":""}'),
+                ]
+            ),
+            "uniquifier",
+            id="instance_uniquifier",
+        ),
+    ],
+)
+def test_inferred_data_lookup_success(name, asset_type, http, inferred_key):
+    res = GoogleAPIResource.from_cai_data(name, asset_type, http=http)
+
+    assert inferred_key not in res._resource_data
+    res.to_dict()
+    assert inferred_key in res._resource_data
+
+
+@pytest.mark.parametrize(
+    "name,asset_type,http",
+    [
+        pytest.param(
+            "//storage.googleapis.com/buckets/test-bucket",
+            "storage.googleapis.com/Bucket",
+            HttpMockSequence(
+                [
+                    ({"status": 401}, '{"msg": "oops"}'),
+                ]
+            ),
+            id="resource_401",
+        ),
+        pytest.param(
+            "//compute.googleapis.com/projects/test-project/zones/test-zone/instances/test-instance",
+            "compute.googleapis.com/Instance",
+            HttpMockSequence(
+                [
+                    ({"status": 404}, '{"msg": "something went wrong"}'),
+                ]
+            ),
+            id="resource_404",
+        ),
+        pytest.param(
+            "//compute.googleapis.com/projects/test-project/zones/test-zone/instances/test-instance",
+            "compute.googleapis.com/Instance",
+            HttpMockSequence(
+                [
+                    ({"status": 200}, "HTTP ERROR: NOT JSON"),
+                ]
+            ),
+            id="malformed_json",
+        ),
+    ],
+)
+def test_todict_eats_exceptions(name, asset_type, http):
+    res = GoogleAPIResource.from_cai_data(name, asset_type, http=http)
+
+    res.to_dict()