Skip to content

Commit

Permalink
[azure][feat]: Update configs collection for mysql and postgresql (#2157
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@1101-1
1101-1 authored Aug 5, 2024
1 parent 3410e81 commit f692309
Show file tree
Hide file tree
Showing 6 changed files with 244 additions and 174 deletions.
78 changes: 36 additions & 42 deletions plugins/azure/fix_plugin_azure/resource/mysql.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
AzureSystemData,
)
from fix_plugin_azure.resource.microsoft_graph import MicrosoftGraphServicePrincipal, MicrosoftGraphUser
from fix_plugin_azure.utils import from_str_to_typed
from fixlib.baseresources import (
BaseDatabase,
BaseDatabaseInstanceType,
Expand All @@ -21,7 +22,7 @@
ModelReference,
)
from fixlib.graph import BySearchCriteria
from fixlib.json_bender import K, AsBool, Bender, S, ForallBend, Bend, MapEnum, MapValue
from fixlib.json_bender import K, Bender, S, ForallBend, Bend, MapEnum, MapValue
from fixlib.types import Json

service_name = "azure_mysql"
Expand Down Expand Up @@ -318,39 +319,34 @@ def collect(
class AzureMysqlServerConfiguration(MicrosoftResource):
kind: ClassVar[str] = "azure_mysql_server_configuration"
# Collect via AzureMysqlServer()
mapping: ClassVar[Dict[str, Bender]] = {
"id": S("id"),
"tags": S("tags", default={}),
"name": S("name"),
"system_data": S("systemData") >> Bend(AzureSystemData.mapping),
"type": S("type"),
"ctime": S("systemData", "createdAt"),
"mtime": S("systemData", "lastModifiedAt"),
"allowed_values": S("properties", "allowedValues"),
"current_value": S("properties", "currentValue"),
"data_type": S("properties", "dataType"),
"default_value": S("properties", "defaultValue"),
"description": S("properties", "description"),
"documentation_link": S("properties", "documentationLink"),
"is_config_pending_restart": S("properties", "isConfigPendingRestart") >> AsBool(),
"is_dynamic_config": S("properties", "isDynamicConfig") >> AsBool(),
"is_read_only": S("properties", "isReadOnly") >> AsBool(),
"source": S("properties", "source"),
"value": S("properties", "value"),
}
allowed_values: Optional[str] = field(default=None, metadata={'description': 'Allowed values of the configuration.'}) # fmt: skip
current_value: Optional[str] = field(default=None, metadata={"description": "Current value of the configuration."})
data_type: Optional[str] = field(default=None, metadata={"description": "Data type of the configuration."})
default_value: Optional[str] = field(default=None, metadata={"description": "Default value of the configuration."})
description: Optional[str] = field(default=None, metadata={"description": "Description of the configuration."})
documentation_link: Optional[str] = field(default=None, metadata={'description': 'The link used to get the document from community or Azure site.'}) # fmt: skip
is_config_pending_restart: Optional[bool] = field(default=None, metadata={'description': 'If is the configuration pending restart or not.'}) # fmt: skip
is_dynamic_config: Optional[bool] = field(default=None, metadata={'description': 'If is the configuration dynamic.'}) # fmt: skip
is_read_only: Optional[bool] = field(default=None, metadata={"description": "If is the configuration read only."})
source: Optional[str] = field(default=None, metadata={"description": "Source of the configuration."})
value: Optional[str] = field(default=None, metadata={"description": "Value of the configuration."})
system_data: Optional[AzureSystemData] = field(default=None, metadata={'description': 'Metadata pertaining to creation and last modification of the resource.'}) # fmt: skip
type: Optional[str] = field(default=None, metadata={'description': 'The type of the resource. E.g. Microsoft.Compute/virtualMachines or Microsoft.Storage/storageAccounts '}) # fmt: skip
config: Json = field(factory=dict)

@classmethod
def collect_configs(
cls,
server_id: str,
raw: List[Json],
builder: GraphBuilder,
) -> List["AzureMysqlServerConfiguration"]:
if not raw:
return []
configuration_instance = AzureMysqlServerConfiguration(id=server_id)
for js in raw:
properties = js.get("properties")
if not properties:
continue
if (
(data_type := properties.get("dataType"))
and (val := properties.get("currentValue") or properties.get("value"))
and (config_name := js.get("name"))
):
value = from_str_to_typed(data_type, val)
if not value:
continue
configuration_instance.config[config_name] = value
if (added := builder.add_node(configuration_instance, configuration_instance.config)) is not None:
return [added]
return []


@define(eq=False, slots=False)
Expand Down Expand Up @@ -667,14 +663,12 @@ def _collect_items(
items = graph_builder.client.list(api_spec)
if not items:
return
collected = class_instance.collect(items, graph_builder)
for clazz in collected:
graph_builder.add_edge(
self,
edge_type=EdgeType.default,
id=clazz.id,
clazz=class_instance,
)
if issubclass(AzureMysqlServerConfiguration, class_instance): # type: ignore
collected = class_instance.collect_configs(self.id, items, graph_builder) # type: ignore
else:
collected = class_instance.collect(items, graph_builder)
for resource in collected:
graph_builder.add_edge(self, node=resource)

def post_process(self, graph_builder: GraphBuilder, source: Json) -> None:
if server_id := self.id:
Expand Down
65 changes: 33 additions & 32 deletions plugins/azure/fix_plugin_azure/resource/postgresql.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
AzureServerMaintenanceWindow,
AzureServerNetwork,
)
from fix_plugin_azure.utils import from_str_to_typed
from fixlib.baseresources import BaseDatabase, BaseDatabaseInstanceType, DatabaseInstanceStatus, ModelReference
from fixlib.graph import BySearchCriteria
from fixlib.json_bender import K, Bender, S, ForallBend, Bend, MapEnum, MapValue
Expand Down Expand Up @@ -313,37 +314,34 @@ def collect(cls, raw: List[Json], builder: GraphBuilder) -> List[AzurePostgresql
class AzurePostgresqlServerConfiguration(MicrosoftResource, AzureProxyResource):
kind: ClassVar[str] = "azure_postgresql_server_configuration"
# Collect via AzurePostgresqlServer()
mapping: ClassVar[Dict[str, Bender]] = AzureProxyResource.mapping | {
"id": S("id"),
"tags": S("tags", default={}),
"name": S("name"),
"ctime": S("systemData", "createdAt"),
"mtime": S("systemData", "lastModifiedAt"),
"allowed_values": S("properties", "allowedValues"),
"data_type": S("properties", "dataType"),
"default_value": S("properties", "defaultValue"),
"description": S("properties", "description"),
"documentation_link": S("properties", "documentationLink"),
"is_config_pending_restart": S("properties", "isConfigPendingRestart"),
"is_dynamic_config": S("properties", "isDynamicConfig"),
"is_read_only": S("properties", "isReadOnly"),
"configuration_source": S("properties", "source"),
"system_data": S("systemData") >> Bend(AzureSystemData.mapping),
"unit": S("properties", "unit"),
"value": S("properties", "value"),
}
allowed_values: Optional[str] = field(default=None, metadata={'description': 'Allowed values of the configuration.'}) # fmt: skip
data_type: Optional[str] = field(default=None, metadata={"description": "Data type of the configuration."})
default_value: Optional[str] = field(default=None, metadata={"description": "Default value of the configuration."})
description: Optional[str] = field(default=None, metadata={"description": "Description of the configuration."})
documentation_link: Optional[str] = field(default=None, metadata={'description': 'Configuration documentation link.'}) # fmt: skip
is_config_pending_restart: Optional[bool] = field(default=None, metadata={'description': 'Configuration is pending restart or not.'}) # fmt: skip
is_dynamic_config: Optional[bool] = field(default=None, metadata={'description': 'Configuration dynamic or static.'}) # fmt: skip
is_read_only: Optional[bool] = field(default=None, metadata={"description": "Configuration read-only or not."})
configuration_source: Optional[str] = field(default=None, metadata={"description": "Source of the configuration."})
system_data: Optional[AzureSystemData] = field(default=None, metadata={'description': 'Metadata pertaining to creation and last modification of the resource.'}) # fmt: skip
unit: Optional[str] = field(default=None, metadata={"description": "Configuration unit."})
value: Optional[str] = field(default=None, metadata={"description": "Value of the configuration."})
config: Json = field(factory=dict)

@classmethod
def collect_configs(
cls,
server_id: str,
raw: List[Json],
builder: GraphBuilder,
) -> List[AzurePostgresqlServerConfiguration]:
if not raw:
return []
configuration_instance = AzurePostgresqlServerConfiguration(id=server_id)
for js in raw:
properties = js.get("properties")
if not properties:
continue
if (
(data_type := properties.get("dataType"))
and (val := properties.get("value"))
and (config_name := js.get("name"))
):
value = from_str_to_typed(data_type, val)
if not value:
continue
configuration_instance.config[config_name] = value
if (added := builder.add_node(configuration_instance, configuration_instance.config)) is not None:
return [added]
return []


@define(eq=False, slots=False)
Expand Down Expand Up @@ -542,7 +540,10 @@ def _collect_items(
items = graph_builder.client.list(api_spec)
if not items:
return
collected = class_instance.collect(items, graph_builder)
if issubclass(AzurePostgresqlServerConfiguration, class_instance): # type: ignore
collected = class_instance.collect_configs(self.id, items, graph_builder) # type: ignore
else:
collected = class_instance.collect(items, graph_builder)
for clazz in collected:
graph_builder.add_edge(self, node=clazz)

Expand Down
23 changes: 23 additions & 0 deletions plugins/azure/fix_plugin_azure/utils.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import logging
from typing import Callable, Dict, TypeVar, Any
from attr import frozen
import functools
Expand All @@ -6,6 +7,7 @@


T = TypeVar("T")
log = logging.getLogger("fix.plugins.azure")


def rgetattr(obj: Any, attr: str, *args: Any) -> Any:
Expand Down Expand Up @@ -44,6 +46,27 @@ def case_insensitive_eq(left: T, right: T) -> bool:
return left == right


def from_str_to_typed(config_type: str, value: str) -> Any:
def set_bool(val: str) -> bool:
if val.lower() == "on":
return True
return False

type_mapping = {
"Enumeration": lambda x: set_bool(x) if x.lower() in ["on", "off"] else str(x),
"Integer": int,
"Numeric": float,
"Set": lambda x: x.split(","),
"String": str,
"Boolean": set_bool,
}
try:
return type_mapping[config_type](value) # type: ignore
except Exception as e:
log.warning(f"An error occured while typing value: {e}")
return None


@frozen(kw_only=True)
class MetricNormalization:
metric_name: MetricName
Expand Down
4 changes: 2 additions & 2 deletions plugins/azure/test/collector_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ def test_collect(
config, Cloud(id="azure"), azure_subscription, credentials, core_feedback
)
subscription_collector.collect()
assert len(subscription_collector.graph.nodes) == 428
assert len(subscription_collector.graph.edges) == 668
assert len(subscription_collector.graph.nodes) == 420
assert len(subscription_collector.graph.edges) == 652

graph_collector = MicrosoftGraphOrganizationCollector(
config, Cloud(id="azure"), MicrosoftGraphOrganization(id="test", name="test"), credentials, core_feedback
Expand Down
Loading

0 comments on commit f692309

Please sign in to comment.