From 85d618354d17b1c74f351fcfce52ddcda0d1fbd5 Mon Sep 17 00:00:00 2001 From: Kirill Date: Fri, 22 Nov 2024 17:30:42 +0000 Subject: [PATCH] feat: make unique queries if resources have same name --- plugins/gcp/fix_plugin_gcp/collector.py | 3 ++- plugins/gcp/fix_plugin_gcp/resources/compute.py | 14 +++++++------- plugins/gcp/fix_plugin_gcp/resources/monitoring.py | 4 +++- plugins/gcp/fix_plugin_gcp/resources/sqladmin.py | 6 +++--- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/plugins/gcp/fix_plugin_gcp/collector.py b/plugins/gcp/fix_plugin_gcp/collector.py index 4474efe7f..51205538e 100644 --- a/plugins/gcp/fix_plugin_gcp/collector.py +++ b/plugins/gcp/fix_plugin_gcp/collector.py @@ -172,7 +172,8 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> None: if region := cast(GcpRegion, resource.region()): resource_queries: List[monitoring.GcpMonitoringQuery] = resource.collect_usage_metrics(builder) if resource_queries: - lookup_map[resource.id] = resource + # set unique GcpMonitoringQuery.ref_id + lookup_map[f"{resource.kind}/{resource.id}/{region.id}"] = resource for query in resource_queries: query_region = query.region or region start = builder.metrics_delta diff --git a/plugins/gcp/fix_plugin_gcp/resources/compute.py b/plugins/gcp/fix_plugin_gcp/resources/compute.py index f0c05cc69..542d58438 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/compute.py +++ b/plugins/gcp/fix_plugin_gcp/resources/compute.py @@ -1214,7 +1214,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer GcpMonitoringQuery.create( query_name="compute.googleapis.com/instance/disk/average_io_queue_depth", period=delta, - ref_id=self.id, + ref_id=f"{self.kind}/{self.id}/{self.region().id}", resource_name=self.id, metric_name=MetricName.VolumeQueueLength, normalization=normalizer_factory.count, @@ -1230,7 +1230,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer GcpMonitoringQuery.create( query_name=name, period=delta, - ref_id=self.id, + ref_id=f"{self.kind}/{self.id}/{self.region().id}", resource_name=self.id, metric_name=metric_name, normalization=normalizer_factory.count, @@ -1250,7 +1250,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer GcpMonitoringQuery.create( query_name=name, period=delta, - ref_id=self.id, + ref_id=f"{self.kind}/{self.id}/{self.region().id}", resource_name=self.id, metric_name=metric_name, normalization=normalizer_factory.count, @@ -3627,7 +3627,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer GcpMonitoringQuery.create( query_name="compute.googleapis.com/instance/cpu/utilization", period=delta, - ref_id=self.id, + ref_id=f"{self.kind}/{self.id}/{self.region().id}", resource_name=self.id, metric_name=MetricName.CpuUtilization, normalization=normalizer_factory.percent, @@ -3642,7 +3642,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer GcpMonitoringQuery.create( query_name=name, period=delta, - ref_id=self.id, + ref_id=f"{self.kind}/{self.id}/{self.region().id}", resource_name=self.id, metric_name=metric_name, normalization=normalizer_factory.count, @@ -3662,7 +3662,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer GcpMonitoringQuery.create( query_name=name, period=delta, - ref_id=self.id, + ref_id=f"{self.kind}/{self.id}/{self.region().id}", resource_name=self.id, metric_name=metric_name, normalization=normalizer_factory.count, @@ -3682,7 +3682,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer GcpMonitoringQuery.create( query_name=name, period=delta, - ref_id=self.id, + ref_id=f"{self.kind}/{self.id}/{self.region().id}", resource_name=self.id, metric_name=metric_name, normalization=normalizer_factory.count, diff --git a/plugins/gcp/fix_plugin_gcp/resources/monitoring.py b/plugins/gcp/fix_plugin_gcp/resources/monitoring.py index ced12c845..bdff3777f 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/monitoring.py +++ b/plugins/gcp/fix_plugin_gcp/resources/monitoring.py @@ -60,7 +60,8 @@ class GcpMonitoringQuery: query_name: str # name of the metric (e.g., GCP metric type) resource_name: str # name of resource period: timedelta # period of the metric - ref_id: str # reference ID for the resource (e.g., instance ID) + ref_id: str # A unique identifier for the resource, formatted as `{resource_kind}/{resource_id}/{resource_region}`. + # Example: "gcp_instance/12345/us-central1". This is used to uniquely reference resources across kinds and regions. metric_id: str # unique metric identifier (metric_name + instance_id) stat: str # aggregation type, supports ALIGN_MEAN, ALIGN_MAX, ALIGN_MIN label_name: str @@ -219,6 +220,7 @@ def update_resource_metrics( resources_map: Dict[str, V], monitoring_metric_result: Dict[GcpMonitoringQuery, GcpMonitoringMetricData], ) -> None: + a = None for query, metric in monitoring_metric_result.items(): resource = resources_map.get(query.ref_id) if resource is None: diff --git a/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py b/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py index 1c9f9f21e..6a3772a73 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py +++ b/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py @@ -776,7 +776,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer GcpMonitoringQuery.create( query_name="cloudsql.googleapis.com/database/cpu/utilization", period=delta, - ref_id=self.id, + ref_id=f"{self.kind}/{self.id}/{self.region().id}", resource_name=f"{builder.project.id}:{self.id}", metric_name=MetricName.CpuUtilization, normalization=normalizer_factory.percent, @@ -792,7 +792,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer GcpMonitoringQuery.create( query_name=name, period=delta, - ref_id=self.id, + ref_id=f"{self.kind}/{self.id}/{self.region().id}", resource_name=f"{builder.project.id}:{self.id}", metric_name=metric_name, normalization=normalizer_factory.count, @@ -813,7 +813,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer GcpMonitoringQuery.create( query_name=name, period=delta, - ref_id=self.id, + ref_id=f"{self.kind}/{self.id}/{self.region().id}", resource_name=f"{builder.project.id}:{self.id}", metric_name=metric_name, normalization=normalizer_factory.iops,