From 088e9b9d3fa6519f31175a510ead503bb7ec6431 Mon Sep 17 00:00:00 2001 From: Kirill Date: Sat, 9 Nov 2024 13:24:03 +0000 Subject: [PATCH 1/6] feat: init filestore --- plugins/gcp/fix_plugin_gcp/collector.py | 3 +- .../gcp/fix_plugin_gcp/resources/filestore.py | 294 ++++++++++++++++++ plugins/gcp/tools/model_gen.py | 4 +- 3 files changed, 299 insertions(+), 2 deletions(-) create mode 100644 plugins/gcp/fix_plugin_gcp/resources/filestore.py diff --git a/plugins/gcp/fix_plugin_gcp/collector.py b/plugins/gcp/fix_plugin_gcp/collector.py index edd81cfec7..528ddce05b 100644 --- a/plugins/gcp/fix_plugin_gcp/collector.py +++ b/plugins/gcp/fix_plugin_gcp/collector.py @@ -4,7 +4,7 @@ from fix_plugin_gcp.config import GcpConfig from fix_plugin_gcp.gcp_client import GcpApiSpec -from fix_plugin_gcp.resources import compute, container, billing, sqladmin, storage, aiplatform, firestore +from fix_plugin_gcp.resources import compute, container, billing, sqladmin, storage, aiplatform, firestore, filestore from fix_plugin_gcp.resources.base import GcpResource, GcpProject, ExecutorQueue, GraphBuilder, GcpRegion, GcpZone from fix_plugin_gcp.utils import Credentials from fixlib.baseresources import Cloud @@ -20,6 +20,7 @@ + storage.resources + aiplatform.resources + firestore.resources + + filestore.resources ) diff --git a/plugins/gcp/fix_plugin_gcp/resources/filestore.py b/plugins/gcp/fix_plugin_gcp/resources/filestore.py new file mode 100644 index 0000000000..b90db71c82 --- /dev/null +++ b/plugins/gcp/fix_plugin_gcp/resources/filestore.py @@ -0,0 +1,294 @@ +from datetime import datetime +import logging +from typing import ClassVar, Dict, Optional, List, Type + +from attr import define, field + +from fix_plugin_gcp.gcp_client import GcpApiSpec +from fix_plugin_gcp.resources.base import GraphBuilder, GcpErrorHandler, GcpResource, GcpDeprecationStatus +from fixlib.baseresources import BaseNetworkShare +from fixlib.json_bender import Bender, S, Bend, ForallBend +from fixlib.types import Json + +log = logging.getLogger("fix.plugins.gcp") + + +service_name = "filestore" + + +@define(eq=False, slots=False) +class GcpFilestoreBackup(GcpResource): + kind: ClassVar[str] = "gcp_filestore_backup" + api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( + service="file", + version="v1", + accessors=["projects", "locations", "backups"], + action="list", + request_parameter={"parent": "projects/{project}/locations/-"}, + request_parameter_in={"project"}, + response_path="backups", + response_regional_sub_path=None, + ) + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("name").or_else(S("id")).or_else(S("selfLink")), + "name": S("name"), + "ctime": S("creationTimestamp"), + "description": S("description"), + "link": S("selfLink"), + "label_fingerprint": S("labelFingerprint"), + "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), + "capacity_gb": S("capacityGb"), + "create_time": S("createTime"), + "download_bytes": S("downloadBytes"), + "file_system_protocol": S("fileSystemProtocol"), + "kms_key": S("kmsKey"), + "satisfies_pzi": S("satisfiesPzi"), + "satisfies_pzs": S("satisfiesPzs"), + "source_file_share": S("sourceFileShare"), + "source_instance": S("sourceInstance"), + "source_instance_tier": S("sourceInstanceTier"), + "state": S("state"), + "storage_bytes": S("storageBytes"), + "tags": S("tags"), + } + capacity_gb: Optional[str] = field(default=None) + create_time: Optional[datetime] = field(default=None) + download_bytes: Optional[str] = field(default=None) + file_system_protocol: Optional[str] = field(default=None) + kms_key: Optional[str] = field(default=None) + satisfies_pzi: Optional[bool] = field(default=None) + satisfies_pzs: Optional[bool] = field(default=None) + source_file_share: Optional[str] = field(default=None) + source_instance: Optional[str] = field(default=None) + source_instance_tier: Optional[str] = field(default=None) + state: Optional[str] = field(default=None) + storage_bytes: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpNfsExportOptions: + kind: ClassVar[str] = "gcp_nfs_export_options" + mapping: ClassVar[Dict[str, Bender]] = { + "access_mode": S("accessMode"), + "anon_gid": S("anonGid"), + "anon_uid": S("anonUid"), + "ip_ranges": S("ipRanges", default=[]), + "squash_mode": S("squashMode"), + } + access_mode: Optional[str] = field(default=None) + anon_gid: Optional[str] = field(default=None) + anon_uid: Optional[str] = field(default=None) + ip_ranges: Optional[List[str]] = field(default=None) + squash_mode: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpFileShareConfig: + kind: ClassVar[str] = "gcp_file_share_config" + mapping: ClassVar[Dict[str, Bender]] = { + "capacity_gb": S("capacityGb"), + "name": S("name"), + "nfs_export_options": S("nfsExportOptions", default=[]) >> ForallBend(GcpNfsExportOptions.mapping), + "source_backup": S("sourceBackup"), + } + capacity_gb: Optional[str] = field(default=None) + name: Optional[str] = field(default=None) + nfs_export_options: Optional[List[GcpNfsExportOptions]] = field(default=None) + source_backup: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpNetworkConfig: + kind: ClassVar[str] = "gcp_network_config" + mapping: ClassVar[Dict[str, Bender]] = { + "connect_mode": S("connectMode"), + "ip_addresses": S("ipAddresses", default=[]), + "modes": S("modes", default=[]), + "network": S("network"), + "reserved_ip_range": S("reservedIpRange"), + } + connect_mode: Optional[str] = field(default=None) + ip_addresses: Optional[List[str]] = field(default=None) + modes: Optional[List[str]] = field(default=None) + network: Optional[str] = field(default=None) + reserved_ip_range: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpPerformanceConfig: + kind: ClassVar[str] = "gcp_performance_config" + mapping: ClassVar[Dict[str, Bender]] = { + "fixed_iops": S("fixedIops", "maxReadIops"), + "iops_per_tb": S("iopsPerTb", "maxReadIopsPerTb"), + } + fixed_iops: Optional[str] = field(default=None) + iops_per_tb: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpPerformanceLimits: + kind: ClassVar[str] = "gcp_performance_limits" + mapping: ClassVar[Dict[str, Bender]] = { + "max_read_iops": S("maxReadIops"), + "max_read_throughput_bps": S("maxReadThroughputBps"), + "max_write_iops": S("maxWriteIops"), + "max_write_throughput_bps": S("maxWriteThroughputBps"), + } + max_read_iops: Optional[str] = field(default=None) + max_read_throughput_bps: Optional[str] = field(default=None) + max_write_iops: Optional[str] = field(default=None) + max_write_throughput_bps: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpReplicaConfig: + kind: ClassVar[str] = "gcp_replica_config" + mapping: ClassVar[Dict[str, Bender]] = { + "last_active_sync_time": S("lastActiveSyncTime"), + "peer_instance": S("peerInstance"), + "state": S("state"), + "state_reasons": S("stateReasons", default=[]), + } + last_active_sync_time: Optional[datetime] = field(default=None) + peer_instance: Optional[str] = field(default=None) + state: Optional[str] = field(default=None) + state_reasons: Optional[List[str]] = field(default=None) + + +@define(eq=False, slots=False) +class GcpReplication: + kind: ClassVar[str] = "gcp_replication" + mapping: ClassVar[Dict[str, Bender]] = { + "replicas": S("replicas", default=[]) >> ForallBend(GcpReplicaConfig.mapping), + "role": S("role"), + } + replicas: Optional[List[GcpReplicaConfig]] = field(default=None) + role: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpFilestoreInstance(GcpResource, BaseNetworkShare): + kind: ClassVar[str] = "gcp_filestore_instance" + api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( + service="file", + version="v1", + accessors=["projects", "locations", "instances"], + action="list", + request_parameter={"parent": "projects/{project}/locations/-"}, + request_parameter_in={"project"}, + response_path="instances", + response_regional_sub_path=None, + ) + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("name").or_else(S("id")).or_else(S("selfLink")), + "name": S("name"), + "ctime": S("creationTimestamp"), + "description": S("description"), + "link": S("selfLink"), + "label_fingerprint": S("labelFingerprint"), + "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), + "configurable_performance_enabled": S("configurablePerformanceEnabled"), + "create_time": S("createTime"), + "deletion_protection_enabled": S("deletionProtectionEnabled"), + "deletion_protection_reason": S("deletionProtectionReason"), + "etag": S("etag"), + "file_shares": S("fileShares", default=[]) >> ForallBend(GcpFileShareConfig.mapping), + "kms_key_name": S("kmsKeyName"), + "networks": S("networks", default=[]) >> ForallBend(GcpNetworkConfig.mapping), + "performance_config": S("performanceConfig", default={}) >> Bend(GcpPerformanceConfig.mapping), + "performance_limits": S("performanceLimits", default={}) >> Bend(GcpPerformanceLimits.mapping), + "protocol": S("protocol"), + "replication": S("replication", default={}) >> Bend(GcpReplication.mapping), + "satisfies_pzi": S("satisfiesPzi"), + "satisfies_pzs": S("satisfiesPzs"), + "state": S("state"), + "status_message": S("statusMessage"), + "suspension_reasons": S("suspensionReasons", default=[]), + "tags": S("tags"), + "tier": S("tier"), + } + configurable_performance_enabled: Optional[bool] = field(default=None) + create_time: Optional[datetime] = field(default=None) + deletion_protection_enabled: Optional[bool] = field(default=None) + deletion_protection_reason: Optional[str] = field(default=None) + etag: Optional[str] = field(default=None) + file_shares: Optional[List[GcpFileShareConfig]] = field(default=None) + kms_key_name: Optional[str] = field(default=None) + networks: Optional[List[GcpNetworkConfig]] = field(default=None) + performance_config: Optional[GcpPerformanceConfig] = field(default=None) + performance_limits: Optional[GcpPerformanceLimits] = field(default=None) + protocol: Optional[str] = field(default=None) + replication: Optional[GcpReplication] = field(default=None) + satisfies_pzi: Optional[bool] = field(default=None) + satisfies_pzs: Optional[bool] = field(default=None) + state: Optional[str] = field(default=None) + status_message: Optional[str] = field(default=None) + suspension_reasons: Optional[List[str]] = field(default=None) + tier: Optional[str] = field(default=None) + + @classmethod + def called_collect_apis(cls) -> List[GcpApiSpec]: + return [ + cls.api_spec, + GcpApiSpec( + service="file", + version="v1", + accessors=["projects", "locations", "instances", "snapshots"], + action="list", + request_parameter={"parent": "projects/{project}/locations/{location}/instances/{instanceId}"}, + request_parameter_in={"project", "location", "instanceId"}, + response_path="snapshots", + response_regional_sub_path=None, + ), + ] + + def post_process(self, graph_builder: GraphBuilder, source: Json) -> None: + def collect_snapshots() -> None: + spec = GcpApiSpec( + service="file", + version="v1", + accessors=["projects", "locations", "instances", "snapshots"], + action="list", + request_parameter={"parent": f"{self.id}"}, + request_parameter_in=set(), + response_path="snapshots", + response_regional_sub_path=None, + ) + with GcpErrorHandler( + spec.action, + graph_builder.error_accumulator, + spec.service, + graph_builder.region.safe_name if graph_builder.region else None, + set(), + f" in {graph_builder.project.id} kind {GcpFilestoreSnapshot.kind}", + ): + items = graph_builder.client.list(spec) + GcpFilestoreSnapshot.collect(items, graph_builder) + log.info(f"[GCP:{graph_builder.project.id}] finished collecting: {GcpFilestoreSnapshot.kind}") + + graph_builder.submit_work(collect_snapshots) + + +@define(eq=False, slots=False) +class GcpFilestoreSnapshot(GcpResource): + # collected via GcpFilestoreInstance() + kind: ClassVar[str] = "gcp_filestore_snapshot" + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("name").or_else(S("id")).or_else(S("selfLink")), + "name": S("name"), + "ctime": S("creationTimestamp"), + "description": S("description"), + "link": S("selfLink"), + "label_fingerprint": S("labelFingerprint"), + "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), + "create_time": S("createTime"), + "filesystem_used_bytes": S("filesystemUsedBytes"), + "state": S("state"), + "tags": S("tags"), + } + create_time: Optional[datetime] = field(default=None) + filesystem_used_bytes: Optional[str] = field(default=None) + state: Optional[str] = field(default=None) + + +resources: List[Type[GcpResource]] = [GcpFilestoreBackup, GcpFilestoreInstance, GcpFilestoreSnapshot] diff --git a/plugins/gcp/tools/model_gen.py b/plugins/gcp/tools/model_gen.py index db5704cf53..64fc3cb372 100644 --- a/plugins/gcp/tools/model_gen.py +++ b/plugins/gcp/tools/model_gen.py @@ -511,6 +511,7 @@ def generate_test_classes() -> None: "parent": "projects/{project}/locations/{region}", }, "firestore": {"parent": "projects/{project_id}/databases/{database_id}/documents", "collectionId": "", "name": ""}, + "file": {"name": "", "parent": "projects/{projectId}/locations/-"}, } # See https://googleapis.github.io/google-api-python-client/docs/dyn/ for the list of available resources @@ -521,8 +522,9 @@ def generate_test_classes() -> None: # ("sqladmin", "v1", "Sql", ["Tier"]), # ("cloudbilling", "v1", "", []), # ("storage", "v1", "", []) - # ("aiplatform", "v1", "", []) + # # ("aiplatform", "v1", "", []) ("firestore", "v1", "", []), + ("file", "v1", "", []) ] From 9f34b6930944359c57551c828992c7fe9b8cc2cd Mon Sep 17 00:00:00 2001 From: Kirill Date: Sat, 9 Nov 2024 13:46:47 +0000 Subject: [PATCH 2/6] feat: added reference kinds --- plugins/gcp/fix_plugin_gcp/resources/filestore.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/plugins/gcp/fix_plugin_gcp/resources/filestore.py b/plugins/gcp/fix_plugin_gcp/resources/filestore.py index b90db71c82..cf00823142 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/filestore.py +++ b/plugins/gcp/fix_plugin_gcp/resources/filestore.py @@ -6,7 +6,7 @@ from fix_plugin_gcp.gcp_client import GcpApiSpec from fix_plugin_gcp.resources.base import GraphBuilder, GcpErrorHandler, GcpResource, GcpDeprecationStatus -from fixlib.baseresources import BaseNetworkShare +from fixlib.baseresources import BaseNetworkShare, ModelReference from fixlib.json_bender import Bender, S, Bend, ForallBend from fixlib.types import Json @@ -169,6 +169,13 @@ class GcpReplication: @define(eq=False, slots=False) class GcpFilestoreInstance(GcpResource, BaseNetworkShare): kind: ClassVar[str] = "gcp_filestore_instance" + _reference_kinds: ClassVar[ModelReference] = { + "successors": { + "default": [ + "gcp_filestore_snapshot", + ], + }, + } api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( service="file", version="v1", @@ -263,7 +270,9 @@ def collect_snapshots() -> None: f" in {graph_builder.project.id} kind {GcpFilestoreSnapshot.kind}", ): items = graph_builder.client.list(spec) - GcpFilestoreSnapshot.collect(items, graph_builder) + snapshots = GcpFilestoreSnapshot.collect(items, graph_builder) + for snapshot in snapshots: + graph_builder.add_edge(self, node=snapshot) log.info(f"[GCP:{graph_builder.project.id}] finished collecting: {GcpFilestoreSnapshot.kind}") graph_builder.submit_work(collect_snapshots) From ecb28ad9620bdba1e467c81f1d0249bf14757fbc Mon Sep 17 00:00:00 2001 From: Kirill Date: Sat, 9 Nov 2024 13:56:05 +0000 Subject: [PATCH 3/6] feat: added info fields --- .../gcp/fix_plugin_gcp/resources/filestore.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/plugins/gcp/fix_plugin_gcp/resources/filestore.py b/plugins/gcp/fix_plugin_gcp/resources/filestore.py index cf00823142..7216ba1819 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/filestore.py +++ b/plugins/gcp/fix_plugin_gcp/resources/filestore.py @@ -1,6 +1,6 @@ from datetime import datetime import logging -from typing import ClassVar, Dict, Optional, List, Type +from typing import ClassVar, Dict, Optional, List, Type, Any from attr import define, field @@ -19,6 +19,14 @@ @define(eq=False, slots=False) class GcpFilestoreBackup(GcpResource): kind: ClassVar[str] = "gcp_filestore_backup" + _kind_display: ClassVar[str] = "GCP Filestore Backup" + _kind_description: ClassVar[str] = ( + "GCP Filestore Backup is a service that allows you to create backups of your Filestore instances." + " It provides a way to protect your data and restore it in case of data loss." + ) + _docs_url: ClassVar[str] = "https://cloud.google.com/filestore/docs/backups" + _kind_service: ClassVar[Optional[str]] = "filestore" + _metadata: ClassVar[Dict[str, Any]] = {"icon": "backup", "group": "storage"} api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( service="file", version="v1", @@ -169,6 +177,14 @@ class GcpReplication: @define(eq=False, slots=False) class GcpFilestoreInstance(GcpResource, BaseNetworkShare): kind: ClassVar[str] = "gcp_filestore_instance" + _kind_display: ClassVar[str] = "GCP Filestore Instance" + _kind_description: ClassVar[str] = ( + "GCP Filestore Instance is a fully managed file storage service that provides scalable and high-performance" + " file systems for applications running on Google Cloud." + ) + _docs_url: ClassVar[str] = "https://cloud.google.com/filestore/docs/instances" + _kind_service: ClassVar[Optional[str]] = "filestore" + _metadata: ClassVar[Dict[str, Any]] = {"icon": "network_share", "group": "storage"} _reference_kinds: ClassVar[ModelReference] = { "successors": { "default": [ @@ -282,6 +298,14 @@ def collect_snapshots() -> None: class GcpFilestoreSnapshot(GcpResource): # collected via GcpFilestoreInstance() kind: ClassVar[str] = "gcp_filestore_snapshot" + _kind_display: ClassVar[str] = "GCP Filestore Snapshot" + _kind_description: ClassVar[str] = ( + "GCP Filestore Snapshot is a point-in-time copy of a Filestore instance, allowing you to restore" + " data to a previous state or create new instances from the snapshot." + ) + _docs_url: ClassVar[str] = "https://cloud.google.com/filestore/docs/snapshots" + _kind_service: ClassVar[Optional[str]] = "filestore" + _metadata: ClassVar[Dict[str, Any]] = {"icon": "snapshot", "group": "storage"} mapping: ClassVar[Dict[str, Bender]] = { "id": S("name").or_else(S("id")).or_else(S("selfLink")), "name": S("name"), From 560d8ab7cee5f97d2608a3cc46798fde58a725e8 Mon Sep 17 00:00:00 2001 From: Kirill Date: Mon, 11 Nov 2024 12:24:46 +0000 Subject: [PATCH 4/6] feat: added tests --- .../gcp/fix_plugin_gcp/resources/filestore.py | 20 ++--- plugins/gcp/test/files/filestore_backup.json | 31 +++++++ .../gcp/test/files/filestore_instance.json | 86 +++++++++++++++++++ .../files/filestore_instance_snapshot.json | 22 +++++ plugins/gcp/test/test_filestore.py | 29 +++++++ 5 files changed, 178 insertions(+), 10 deletions(-) create mode 100644 plugins/gcp/test/files/filestore_backup.json create mode 100644 plugins/gcp/test/files/filestore_instance.json create mode 100644 plugins/gcp/test/files/filestore_instance_snapshot.json create mode 100644 plugins/gcp/test/test_filestore.py diff --git a/plugins/gcp/fix_plugin_gcp/resources/filestore.py b/plugins/gcp/fix_plugin_gcp/resources/filestore.py index 7216ba1819..7f7714fa15 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/filestore.py +++ b/plugins/gcp/fix_plugin_gcp/resources/filestore.py @@ -57,7 +57,7 @@ class GcpFilestoreBackup(GcpResource): "source_instance_tier": S("sourceInstanceTier"), "state": S("state"), "storage_bytes": S("storageBytes"), - "tags": S("tags"), + "tags": S("tags", default={}), } capacity_gb: Optional[str] = field(default=None) create_time: Optional[datetime] = field(default=None) @@ -188,7 +188,7 @@ class GcpFilestoreInstance(GcpResource, BaseNetworkShare): _reference_kinds: ClassVar[ModelReference] = { "successors": { "default": [ - "gcp_filestore_snapshot", + "gcp_filestore_instance_snapshot", ], }, } @@ -227,7 +227,7 @@ class GcpFilestoreInstance(GcpResource, BaseNetworkShare): "state": S("state"), "status_message": S("statusMessage"), "suspension_reasons": S("suspensionReasons", default=[]), - "tags": S("tags"), + "tags": S("tags", default={}), "tier": S("tier"), } configurable_performance_enabled: Optional[bool] = field(default=None) @@ -283,21 +283,21 @@ def collect_snapshots() -> None: spec.service, graph_builder.region.safe_name if graph_builder.region else None, set(), - f" in {graph_builder.project.id} kind {GcpFilestoreSnapshot.kind}", + f" in {graph_builder.project.id} kind {GcpFilestoreInstanceSnapshot.kind}", ): items = graph_builder.client.list(spec) - snapshots = GcpFilestoreSnapshot.collect(items, graph_builder) + snapshots = GcpFilestoreInstanceSnapshot.collect(items, graph_builder) for snapshot in snapshots: graph_builder.add_edge(self, node=snapshot) - log.info(f"[GCP:{graph_builder.project.id}] finished collecting: {GcpFilestoreSnapshot.kind}") + log.info(f"[GCP:{graph_builder.project.id}] finished collecting: {GcpFilestoreInstanceSnapshot.kind}") graph_builder.submit_work(collect_snapshots) @define(eq=False, slots=False) -class GcpFilestoreSnapshot(GcpResource): +class GcpFilestoreInstanceSnapshot(GcpResource): # collected via GcpFilestoreInstance() - kind: ClassVar[str] = "gcp_filestore_snapshot" + kind: ClassVar[str] = "gcp_filestore_instance_snapshot" _kind_display: ClassVar[str] = "GCP Filestore Snapshot" _kind_description: ClassVar[str] = ( "GCP Filestore Snapshot is a point-in-time copy of a Filestore instance, allowing you to restore" @@ -317,11 +317,11 @@ class GcpFilestoreSnapshot(GcpResource): "create_time": S("createTime"), "filesystem_used_bytes": S("filesystemUsedBytes"), "state": S("state"), - "tags": S("tags"), + "tags": S("tags", default={}), } create_time: Optional[datetime] = field(default=None) filesystem_used_bytes: Optional[str] = field(default=None) state: Optional[str] = field(default=None) -resources: List[Type[GcpResource]] = [GcpFilestoreBackup, GcpFilestoreInstance, GcpFilestoreSnapshot] +resources: List[Type[GcpResource]] = [GcpFilestoreBackup, GcpFilestoreInstance, GcpFilestoreInstanceSnapshot] diff --git a/plugins/gcp/test/files/filestore_backup.json b/plugins/gcp/test/files/filestore_backup.json new file mode 100644 index 0000000000..2af88e45dd --- /dev/null +++ b/plugins/gcp/test/files/filestore_backup.json @@ -0,0 +1,31 @@ +{ + "backups": [ + { + "id": "projects/sample-project/locations/us-central1/backups/backup-1", + "name": "backup-1", + "ctime": "2024-11-11T08:30:00Z", + "description": "Daily backup for critical data", + "link": "https://www.googleapis.com/filestore/v1/projects/sample-project/locations/us-central1/backups/backup-1", + "label_fingerprint": "abc123", + "deprecation_status": { + "state": "ACTIVE", + "deleted": null, + "deprecated": null, + "obsolete": null, + "replacement": null + }, + "capacity_gb": "1024", + "create_time": "2024-11-11T08:30:00Z", + "download_bytes": "5368709120", + "file_system_protocol": "NFSv3", + "kms_key": "projects/sample-project/locations/us-central1/keyRings/sample-ring/cryptoKeys/sample-key", + "satisfies_pzi": true, + "satisfies_pzs": false, + "source_file_share": "share-1", + "source_instance": "projects/sample-project/locations/us-central1/instances/instance-1", + "source_instance_tier": "STANDARD", + "state": "READY", + "storage_bytes": "5368709120" + } + ] +} \ No newline at end of file diff --git a/plugins/gcp/test/files/filestore_instance.json b/plugins/gcp/test/files/filestore_instance.json new file mode 100644 index 0000000000..28ceeb251c --- /dev/null +++ b/plugins/gcp/test/files/filestore_instance.json @@ -0,0 +1,86 @@ +{ + "instances": [ + { + "id": "projects/sample-project/locations/us-central1/instances/instance-1", + "name": "instance-1", + "ctime": "2024-10-01T14:00:00Z", + "description": "Primary file storage instance", + "link": "https://www.googleapis.com/filestore/v1/projects/sample-project/locations/us-central1/instances/instance-1", + "label_fingerprint": "def456", + "deprecation_status": { + "state": "ACTIVE", + "deleted": null, + "deprecated": null, + "obsolete": null, + "replacement": null + }, + "configurable_performance_enabled": true, + "create_time": "2024-10-01T14:00:00Z", + "deletion_protection_enabled": false, + "deletion_protection_reason": null, + "etag": "etag-789", + "file_shares": [ + { + "capacity_gb": "2048", + "name": "file-share-1", + "nfs_export_options": [ + { + "access_mode": "READ_WRITE", + "anon_gid": "65534", + "anon_uid": "65534", + "ip_ranges": [ + "192.168.1.0/24" + ], + "squash_mode": "NO_ROOT_SQUASH" + } + ], + "source_backup": "projects/sample-project/locations/us-central1/backups/backup-1" + } + ], + "kms_key_name": "projects/sample-project/locations/us-central1/keyRings/sample-ring/cryptoKeys/sample-key", + "networks": [ + { + "connect_mode": "PRIVATE_SERVICE_ACCESS", + "ip_addresses": [ + "10.1.2.3" + ], + "modes": [ + "MODE_IPV4" + ], + "network": "projects/sample-project/global/networks/default", + "reserved_ip_range": "10.1.0.0/24" + } + ], + "performance_config": { + "fixed_iops": "1000", + "iops_per_tb": "300" + }, + "performance_limits": { + "max_read_iops": "3000", + "max_read_throughput_bps": "104857600", + "max_write_iops": "2000", + "max_write_throughput_bps": "52428800" + }, + "protocol": "NFSv3", + "replication": { + "replicas": [ + { + "last_active_sync_time": "2024-11-01T12:00:00Z", + "peer_instance": "projects/sample-project/locations/us-central1/instances/replica-1", + "state": "ACTIVE", + "state_reasons": [ + "Synced successfully" + ] + } + ], + "role": "PRIMARY" + }, + "satisfies_pzi": true, + "satisfies_pzs": true, + "state": "READY", + "status_message": "Instance is operating normally", + "suspension_reasons": [], + "tier": "STANDARD" + } + ] +} \ No newline at end of file diff --git a/plugins/gcp/test/files/filestore_instance_snapshot.json b/plugins/gcp/test/files/filestore_instance_snapshot.json new file mode 100644 index 0000000000..eb9cfdb9d1 --- /dev/null +++ b/plugins/gcp/test/files/filestore_instance_snapshot.json @@ -0,0 +1,22 @@ +{ + "snapshots": [ + { + "id": "projects/sample-project/locations/us-central1/snapshots/snapshot-1", + "name": "snapshot-1", + "ctime": "2024-10-15T09:00:00Z", + "description": "Snapshot before maintenance", + "link": "https://www.googleapis.com/filestore/v1/projects/sample-project/locations/us-central1/snapshots/snapshot-1", + "label_fingerprint": "ghi789", + "deprecation_status": { + "state": "ACTIVE", + "deleted": null, + "deprecated": null, + "obsolete": null, + "replacement": null + }, + "create_time": "2024-10-15T09:00:00Z", + "filesystem_used_bytes": "1073741824", + "state": "READY" + } + ] +} \ No newline at end of file diff --git a/plugins/gcp/test/test_filestore.py b/plugins/gcp/test/test_filestore.py new file mode 100644 index 0000000000..355b8c5c34 --- /dev/null +++ b/plugins/gcp/test/test_filestore.py @@ -0,0 +1,29 @@ +import json +import os + +from fix_plugin_gcp.resources.base import GraphBuilder +from fix_plugin_gcp.resources.filestore import GcpFilestoreBackup, GcpFilestoreInstance, GcpFilestoreInstanceSnapshot + + +def test_gcp_filestore_backup(random_builder: GraphBuilder) -> None: + with open(os.path.dirname(__file__) + "/files/filestore_backup.json") as f: + GcpFilestoreBackup.collect(raw=json.load(f)["backups"], builder=random_builder) + + functions = random_builder.nodes(clazz=GcpFilestoreBackup) + assert len(functions) == 1 + + +def test_gcp_filestore_instance(random_builder: GraphBuilder) -> None: + with open(os.path.dirname(__file__) + "/files/filestore_instance.json") as f: + GcpFilestoreInstance.collect(raw=json.load(f)["instances"], builder=random_builder) + + functions = random_builder.nodes(clazz=GcpFilestoreInstance) + assert len(functions) == 1 + + +def test_gcp_filestore_instance_snapshot(random_builder: GraphBuilder) -> None: + with open(os.path.dirname(__file__) + "/files/filestore_instance_snapshot.json") as f: + GcpFilestoreInstanceSnapshot.collect(raw=json.load(f)["snapshots"], builder=random_builder) + + functions = random_builder.nodes(clazz=GcpFilestoreInstanceSnapshot) + assert len(functions) == 1 From c58bbaedf1822e6a600663aa664de687f23a29f6 Mon Sep 17 00:00:00 2001 From: Kirill Date: Mon, 11 Nov 2024 12:42:32 +0000 Subject: [PATCH 5/6] fixed incorrect names --- plugins/gcp/test/test_filestore.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/gcp/test/test_filestore.py b/plugins/gcp/test/test_filestore.py index 355b8c5c34..394cab86f8 100644 --- a/plugins/gcp/test/test_filestore.py +++ b/plugins/gcp/test/test_filestore.py @@ -9,21 +9,21 @@ def test_gcp_filestore_backup(random_builder: GraphBuilder) -> None: with open(os.path.dirname(__file__) + "/files/filestore_backup.json") as f: GcpFilestoreBackup.collect(raw=json.load(f)["backups"], builder=random_builder) - functions = random_builder.nodes(clazz=GcpFilestoreBackup) - assert len(functions) == 1 + backups = random_builder.nodes(clazz=GcpFilestoreBackup) + assert len(backups) == 1 def test_gcp_filestore_instance(random_builder: GraphBuilder) -> None: with open(os.path.dirname(__file__) + "/files/filestore_instance.json") as f: GcpFilestoreInstance.collect(raw=json.load(f)["instances"], builder=random_builder) - functions = random_builder.nodes(clazz=GcpFilestoreInstance) - assert len(functions) == 1 + instances = random_builder.nodes(clazz=GcpFilestoreInstance) + assert len(instances) == 1 def test_gcp_filestore_instance_snapshot(random_builder: GraphBuilder) -> None: with open(os.path.dirname(__file__) + "/files/filestore_instance_snapshot.json") as f: GcpFilestoreInstanceSnapshot.collect(raw=json.load(f)["snapshots"], builder=random_builder) - functions = random_builder.nodes(clazz=GcpFilestoreInstanceSnapshot) - assert len(functions) == 1 + snapshots = random_builder.nodes(clazz=GcpFilestoreInstanceSnapshot) + assert len(snapshots) == 1 From 2ebe747aab2a0bc42a1a0067a481f6998043f3ca Mon Sep 17 00:00:00 2001 From: Kirill Date: Mon, 11 Nov 2024 14:37:56 +0000 Subject: [PATCH 6/6] rebase main --- plugins/gcp/tools/model_gen.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/gcp/tools/model_gen.py b/plugins/gcp/tools/model_gen.py index 64fc3cb372..06bd6af2ea 100644 --- a/plugins/gcp/tools/model_gen.py +++ b/plugins/gcp/tools/model_gen.py @@ -521,9 +521,9 @@ def generate_test_classes() -> None: # ("container", "v1", "Container", ["UsableSubnetwork"]), # ("sqladmin", "v1", "Sql", ["Tier"]), # ("cloudbilling", "v1", "", []), - # ("storage", "v1", "", []) - # # ("aiplatform", "v1", "", []) - ("firestore", "v1", "", []), + # ("storage", "v1", "", []), + # # ("aiplatform", "v1", "", []), + # ("firestore", "v1", "", []), ("file", "v1", "", []) ]