Skip to content

Commit

Permalink
feat: updated config collection for postgres service
Browse files Browse the repository at this point in the history
  • Loading branch information
@1101-1
1101-1 committed Aug 2, 2024
1 parent 932a34b commit c15925a
Show file tree
Hide file tree
Showing 5 changed files with 152 additions and 63 deletions.
27 changes: 4 additions & 23 deletions plugins/azure/fix_plugin_azure/resource/mysql.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
MicrosoftResourceType,
)
from fix_plugin_azure.resource.microsoft_graph import MicrosoftGraphServicePrincipal, MicrosoftGraphUser
from fix_plugin_azure.utils import from_str_to_typed
from fixlib.baseresources import (
BaseDatabase,
BaseDatabaseInstanceType,
Expand Down Expand Up @@ -315,26 +316,6 @@ def collect(
return result


def from_str_to_typed(config_type: str, value: str) -> Any:
def set_bool(val: str) -> bool:
if val == "ON":
return True
return False

type_mapping = {
"Enumeration": lambda x: set_bool(x) if x in ["ON", "OFF"] else str(x),
"Integer": int,
"Numeric": float,
"Set": lambda x: x.split(","),
"String": str,
}
try:
return type_mapping[config_type](value) # type: ignore
except Exception as e:
log.warning(f"An error occured while defining type of configuration value: {e}")
return None


@define(eq=False, slots=False)
class AzureMysqlServerConfiguration(MicrosoftResource):
kind: ClassVar[str] = "azure_mysql_server_configuration"
Expand All @@ -343,10 +324,10 @@ class AzureMysqlServerConfiguration(MicrosoftResource):

@classmethod
def collect(
cls: Type[MicrosoftResourceType],
cls,
raw: List[Json],
builder: GraphBuilder,
) -> List[MicrosoftResourceType]:
) -> List["AzureMysqlServerConfiguration"]:
if not raw:
return []
server_id = raw[0].get("serverID")
Expand All @@ -360,7 +341,7 @@ def collect(
continue
if (
(data_type := properties.get("dataType"))
and (val := properties.get("currentValue"))
and (val := properties.get("currentValue") or properties.get("value"))
and (config_name := js.get("name"))
):
value = from_str_to_typed(data_type, val)
Expand Down
66 changes: 35 additions & 31 deletions plugins/azure/fix_plugin_azure/resource/postgresql.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
AzureServerMaintenanceWindow,
AzureServerNetwork,
)
from fix_plugin_azure.utils import from_str_to_typed
from fixlib.baseresources import BaseDatabase, BaseDatabaseInstanceType, DatabaseInstanceStatus, ModelReference
from fixlib.graph import BySearchCriteria
from fixlib.json_bender import K, Bender, S, ForallBend, Bend, MapEnum, MapValue
Expand Down Expand Up @@ -313,37 +314,37 @@ def collect(cls, raw: List[Json], builder: GraphBuilder) -> List[AzurePostgresql
class AzurePostgresqlServerConfiguration(MicrosoftResource, AzureProxyResource):
kind: ClassVar[str] = "azure_postgresql_server_configuration"
# Collect via AzurePostgresqlServer()
mapping: ClassVar[Dict[str, Bender]] = AzureProxyResource.mapping | {
"id": S("id"),
"tags": S("tags", default={}),
"name": S("name"),
"ctime": S("systemData", "createdAt"),
"mtime": S("systemData", "lastModifiedAt"),
"allowed_values": S("properties", "allowedValues"),
"data_type": S("properties", "dataType"),
"default_value": S("properties", "defaultValue"),
"description": S("properties", "description"),
"documentation_link": S("properties", "documentationLink"),
"is_config_pending_restart": S("properties", "isConfigPendingRestart"),
"is_dynamic_config": S("properties", "isDynamicConfig"),
"is_read_only": S("properties", "isReadOnly"),
"configuration_source": S("properties", "source"),
"system_data": S("systemData") >> Bend(AzureSystemData.mapping),
"unit": S("properties", "unit"),
"value": S("properties", "value"),
}
allowed_values: Optional[str] = field(default=None, metadata={'description': 'Allowed values of the configuration.'}) # fmt: skip
data_type: Optional[str] = field(default=None, metadata={"description": "Data type of the configuration."})
default_value: Optional[str] = field(default=None, metadata={"description": "Default value of the configuration."})
description: Optional[str] = field(default=None, metadata={"description": "Description of the configuration."})
documentation_link: Optional[str] = field(default=None, metadata={'description': 'Configuration documentation link.'}) # fmt: skip
is_config_pending_restart: Optional[bool] = field(default=None, metadata={'description': 'Configuration is pending restart or not.'}) # fmt: skip
is_dynamic_config: Optional[bool] = field(default=None, metadata={'description': 'Configuration dynamic or static.'}) # fmt: skip
is_read_only: Optional[bool] = field(default=None, metadata={"description": "Configuration read-only or not."})
configuration_source: Optional[str] = field(default=None, metadata={"description": "Source of the configuration."})
system_data: Optional[AzureSystemData] = field(default=None, metadata={'description': 'Metadata pertaining to creation and last modification of the resource.'}) # fmt: skip
unit: Optional[str] = field(default=None, metadata={"description": "Configuration unit."})
value: Optional[str] = field(default=None, metadata={"description": "Value of the configuration."})
config: Json = field(factory=dict)

@classmethod
def collect(
cls,
raw: List[Json],
builder: GraphBuilder,
) -> List[AzurePostgresqlServerConfiguration]:
if not raw:
return []
server_id = raw[0].get("serverID")
if not server_id:
return []
configuration_instance = cls(id=server_id)
if isinstance(configuration_instance, AzurePostgresqlServerConfiguration):
for js in raw:
properties = js.get("properties")
if not properties:
continue
if (
(data_type := properties.get("dataType"))
and (val := properties.get("value"))
and (config_name := js.get("name"))
):
value = from_str_to_typed(data_type, val)
if not value:
continue
configuration_instance.config[config_name] = value
if (added := builder.add_node(configuration_instance, configuration_instance.config)) is not None:
return [added] # type: ignore
return []


@define(eq=False, slots=False)
Expand Down Expand Up @@ -542,6 +543,9 @@ def _collect_items(
items = graph_builder.client.list(api_spec)
if not items:
return
if issubclass(class_instance, AzurePostgresqlServerConfiguration): # type: ignore
for item in items:
item["serverID"] = self.id
collected = class_instance.collect(items, graph_builder)
for clazz in collected:
graph_builder.add_edge(self, node=clazz)
Expand Down
20 changes: 20 additions & 0 deletions plugins/azure/fix_plugin_azure/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,26 @@ def case_insensitive_eq(left: T, right: T) -> bool:
return left == right


def from_str_to_typed(config_type: str, value: str) -> Any:
def set_bool(val: str) -> bool:
if val.lower() == "on":
return True
return False

type_mapping = {
"Enumeration": lambda x: set_bool(x) if x in ["ON", "OFF", "on", "off"] else str(x),
"Integer": int,
"Numeric": float,
"Set": lambda x: x.split(","),
"String": str,
"Boolean": set_bool,
}
try:
return type_mapping[config_type](value) # type: ignore
except Exception:
return None


@frozen(kw_only=True)
class MetricNormalization:
metric_name: MetricName
Expand Down
4 changes: 2 additions & 2 deletions plugins/azure/test/collector_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ def test_collect(
config, Cloud(id="azure"), azure_subscription, credentials, core_feedback
)
subscription_collector.collect()
assert len(subscription_collector.graph.nodes) == 428
assert len(subscription_collector.graph.edges) == 668
assert len(subscription_collector.graph.nodes) == 420
assert len(subscription_collector.graph.edges) == 652

graph_collector = MicrosoftGraphOrganizationCollector(
config, Cloud(id="azure"), MicrosoftGraphOrganization(id="test", name="test"), credentials, core_feedback
Expand Down
98 changes: 91 additions & 7 deletions plugins/azure/test/files/postgresql/configurations.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,106 @@
"value": [
{
"properties": {
"value": "",
"description": "Sets the application name to be reported in statistics and logs.",
"defaultValue": "",
"allowedValues": "ON,OFF",
"currentValue": "ON",
"dataType": "Enumeration",
"defaultValue": "ON",
"description": "Whether to enable automatic activation of all granted roles when users log in to the server.",
"documentationLink": "https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_activate_all_roles_on_login",
"isConfigPendingRestart": "False",
"isDynamicConfig": "True",
"isReadOnly": "False",
"source": "system-default",
"value": "ON"
},
"type": "Microsoft.DBforPostgreSQL/flexibleServers/configurations",
"resourceGroup": "foo",
"id": "/subscriptions/subid/resourceGroups/foo/providers/Microsoft.DBforPostgreSQL/flexibleServers/test/configurations/activate_all_roles_on_login",
"name": "activate_all_roles_on_login"
},
{
"properties": {
"allowedValues": "0-65535",
"currentValue": "33062",
"dataType": "Integer",
"defaultValue": "33062",
"description": "The TCP/IP port number to use for connections on the administrative network interface.",
"documentationLink": "https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_admin_port",
"isConfigPendingRestart": "False",
"isDynamicConfig": "False",
"isReadOnly": "True",
"source": "system-default",
"value": "33062"
},
"type": "Microsoft.DBforPostgreSQL/flexibleServers/configurations",
"id": "/subscriptions/subid/resourceGroups/foo/providers/Microsoft.DBforPostgreSQL/flexibleServers/test/configurations/admin_port",
"name": "admin_port",
"resourceGroup": "foo"
},
{
"properties": {
"allowedValues": "",
"currentValue": "log_filter_internal; log_sink_internal",
"dataType": "String",
"allowedValues": "[A-Za-z0-9._-]*",
"source": "system-default"
"defaultValue": "",
"description": "The admin_ssl_ca system variable is like ssl_ca, except that it applies to the administrative connection interface rather than the main connection interface.",
"documentationLink": "https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_admin_ssl_ca",
"isConfigPendingRestart": "False",
"isDynamicConfig": "True",
"isReadOnly": "True",
"source": "system-default",
"systemData": null,
"value": ""
},
"id": "/subscriptions/ffffffff-ffff-ffff-ffff-ffffffffffff/resourceGroups/testrg/providers/Microsoft.DBforPostgreSQL/flexibleServers/testserver/configurations/application_name",
"name": "application_name",
"id": "/subscriptions/subid/resourceGroups/foo/providers/Microsoft.DBforPostgreSQL/flexibleServers/test/configurations/admin_ssl_ca",
"name": "admin_ssl_ca",
"resourceGroup": "foo",
"type": "Microsoft.DBforPostgreSQL/flexibleServers/configurations"
},
{
"properties": {
"source": "system-default",
"value": "TLSv1.2",
"allowedValues": "TLSv1.2,TLSv1.3",
"currentValue": "TLSv1.2",
"dataType": "Set",
"defaultValue": "TLSv1.2",
"description": "The admin_tls_version system variable is like tls_version, except that it applies to the administrative connection interface rather than the main connection interface.",
"documentationLink": "https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_admin_tls_version",
"isConfigPendingRestart": "False",
"isDynamicConfig": "True",
"isReadOnly": "True"
},
"id": "/subscriptions/subid/resourceGroups/foo/providers/Microsoft.DBforPostgreSQL/flexibleServers/test/configurations/admin_tls_version",
"name": "admin_tls_version",
"type": "Microsoft.DBforPostgreSQL/flexibleServers/configurations",
"resourceGroup": "foo"
},
{
"properties": {
"source": "system-default",
"value": "90",
"isConfigPendingRestart": "False",
"isDynamicConfig": "True",
"isReadOnly": "False",
"allowedValues": "0-99.99",
"currentValue": "90",
"dataType": "Numeric",
"defaultValue": "90",
"description": "InnoDB tries to flush data from the buffer pool so that the percentage of dirty pages does not exceed this value.",
"documentationLink": "https://dev.mysql.com/doc/refman/8.0/en/innodb-parameters.html#sysvar_innodb_max_dirty_pages_pct"
},
"type": "Microsoft.DBforPostgreSQL/flexibleServers/configurations",
"resourceGroup": "foo",
"id": "/subscriptions/subid/resourceGroups/foo/providers/Microsoft.DBforPostgreSQL/flexibleServers/test/configurations/innodb_max_dirty_pages_pct",
"name": "innodb_max_dirty_pages_pct"
},
{
"properties": {
"value": "on",
"description": "Enables input of NULL elements in arrays.",
"defaultValue": "on",
"currentValue": "on",
"dataType": "Boolean",
"allowedValues": "on,off",
"source": "system-default"
Expand Down

0 comments on commit c15925a

Please sign in to comment.