From 8cb14d77bc588b85d7b200801f049bc48cf3cac3 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Tue, 19 Sep 2023 11:07:37 +0200 Subject: [PATCH] [feat][resotocore] Load static benchmarks and checks in multi-tenant mode --- resotocore/resotocore/__main__.py | 8 +- resotocore/resotocore/dependencies.py | 4 +- resotocore/resotocore/model/model_handler.py | 5 + .../resotocore/report/inspector_service.py | 157 +++++++++++++----- resotocore/resotocore/web/directives.py | 2 +- resotocore/tests/resotocore/conftest.py | 4 +- .../report/inspector_service_test.py | 22 ++- 7 files changed, 150 insertions(+), 52 deletions(-) diff --git a/resotocore/resotocore/__main__.py b/resotocore/resotocore/__main__.py index e1e0f4fe52..4165639cf7 100644 --- a/resotocore/resotocore/__main__.py +++ b/resotocore/resotocore/__main__.py @@ -52,10 +52,10 @@ from resotocore.infra_apps.package_manager import PackageManager from resotocore.message_bus import MessageBus from resotocore.model.db_updater import GraphMerger -from resotocore.model.model_handler import ModelHandlerDB +from resotocore.model.model_handler import ModelHandlerDB, ModelHandlerFromCodeAndDB from resotocore.model.typed_model import to_json, class_fqn from resotocore.query.template_expander_service import TemplateExpanderService -from resotocore.report.inspector_service import InspectorService +from resotocore.report.inspector_service import InspectorConfigService, InspectorFileService from resotocore.system_start import db_access, setup_process, parse_args, system_info, reconfigure_logging from resotocore.task.scheduler import APScheduler, NoScheduler from resotocore.task.subscribers import SubscriptionHandlerService @@ -197,7 +197,7 @@ async def direct_tenant(deps: TenantDependencies) -> None: default_env = {"graph": config.cli.default_graph, "section": config.cli.default_section} cli = deps.add(ServiceNames.cli, CLIService(deps, all_commands(deps), default_env, alias_names())) deps.add(ServiceNames.template_expander, TemplateExpanderService(db.template_entity_db, cli)) - inspector = deps.add(ServiceNames.inspector, InspectorService(cli)) + inspector = deps.add(ServiceNames.inspector, InspectorConfigService(cli)) subscriptions = deps.add( ServiceNames.subscription_handler, SubscriptionHandlerService(db.subscribers_db, message_bus) ) @@ -251,6 +251,8 @@ async def direct_tenant(deps: TenantDependencies) -> None: async def multi_tenancy(deps: Dependencies) -> None: deps.add(ServiceNames.message_bus, MessageBus()) deps.add(ServiceNames.forked_tasks, Queue()) + InspectorFileService.on_startup() + ModelHandlerFromCodeAndDB.on_startup() def with_config( diff --git a/resotocore/resotocore/dependencies.py b/resotocore/resotocore/dependencies.py index 78485fe1ac..178513c465 100644 --- a/resotocore/resotocore/dependencies.py +++ b/resotocore/resotocore/dependencies.py @@ -42,7 +42,7 @@ from resotocore.query.template_expander import TemplateExpander from resotocore.query.template_expander_service import TemplateExpanderService from resotocore.report import Inspector -from resotocore.report.inspector_service import InspectorService +from resotocore.report.inspector_service import InspectorFileService from resotocore.service import Service from resotocore.system_start import SystemInfo from resotocore.task.scheduler import NoScheduler @@ -419,7 +419,7 @@ def standard_database() -> StandardDatabase: default_env = {"graph": config.cli.default_graph, "section": config.cli.default_section} cli = deps.add(ServiceNames.cli, CLIService(deps, all_commands(deps), default_env, alias_names())) deps.add(ServiceNames.template_expander, TemplateExpanderService(db.template_entity_db, cli)) - inspector = deps.add(ServiceNames.inspector, InspectorService(cli)) + inspector = deps.add(ServiceNames.inspector, InspectorFileService(cli)) subscriptions = deps.add(ServiceNames.subscription_handler, NoSubscriptionHandler()) core_config_handler = deps.add( ServiceNames.core_config_handler, diff --git a/resotocore/resotocore/model/model_handler.py b/resotocore/resotocore/model/model_handler.py index 6f7d06e5d6..f10ced1346 100644 --- a/resotocore/resotocore/model/model_handler.py +++ b/resotocore/resotocore/model/model_handler.py @@ -257,3 +257,8 @@ async def load_model(self, graph_name: GraphName, *, force: bool = False) -> Mod return code_model() else: return await super().load_model(graph_name, force=force) + + @staticmethod + def on_startup() -> None: + # make sure the model is loaded on startup + code_model() diff --git a/resotocore/resotocore/report/inspector_service.py b/resotocore/resotocore/report/inspector_service.py index cb58701eb6..ea51cc9dba 100644 --- a/resotocore/resotocore/report/inspector_service.py +++ b/resotocore/resotocore/report/inspector_service.py @@ -1,6 +1,8 @@ import asyncio import logging +from abc import abstractmethod from collections import defaultdict +from functools import lru_cache from typing import Optional, List, Dict, Tuple, Callable, AsyncIterator, cast from aiostream import stream @@ -89,39 +91,31 @@ def includes_severity(self, severity: ReportSeverity) -> bool: class InspectorService(Inspector, Service): def __init__(self, cli: CLI) -> None: super().__init__() - self.config_handler: ConfigHandler = cli.dependencies.config_handler self.db_access = cli.dependencies.db_access self.cli = cli self.template_expander = cli.dependencies.template_expander self.model_handler = cli.dependencies.model_handler self.event_sender = cli.dependencies.event_sender - async def start(self) -> None: - if not self.cli.dependencies.config.multi_tenant_setup: - # TODO: we need a migration path for checks added in existing configs - config_ids = {i async for i in self.config_handler.list_config_ids()} - overwrite = False # Only here to simplify development. True until we reach a stable version. - for name, js in BenchmarkConfig.from_files().items(): - if overwrite or benchmark_id(name) not in config_ids: - cid = benchmark_id(name) - log.info(f"Creating benchmark config {cid}") - await self.config_handler.put_config(ConfigEntity(cid, {BenchmarkConfigRoot: js}), validate=False) - for name, js in ReportCheckCollectionConfig.from_files().items(): - if overwrite or check_id(name) not in config_ids: - cid = check_id(name) - log.info(f"Creating check collection config {cid}") - await self.config_handler.put_config(ConfigEntity(cid, {CheckConfigRoot: js}), validate=False) + @abstractmethod + async def _report_values(self) -> Json: + pass - async def list_benchmarks(self) -> List[Benchmark]: - return [ - await self.__benchmark(i) - async for i in self.config_handler.list_config_ids() - if i.startswith(BenchmarkConfigPrefix) - ] + @abstractmethod + async def _check_ids(self) -> List[ConfigId]: + pass + + @abstractmethod + async def _checks(self, cfg_id: ConfigId) -> List[ReportCheck]: + pass + + @abstractmethod + async def _benchmark(self, cfg_id: ConfigId) -> Benchmark: + pass async def benchmark(self, name: str) -> Optional[Benchmark]: try: - return await self.__benchmark(benchmark_id(name)) + return await self._benchmark(benchmark_id(name)) except ValueError: return None @@ -168,7 +162,7 @@ async def load_benchmarks( model = QueryModel(Query.by(term), await self.model_handler.load_model(graph)) # collect all checks - benchmarks = {name: await self.__benchmark(benchmark_id(name)) for name in benchmark_names} + benchmarks = {name: await self._benchmark(benchmark_id(name)) for name in benchmark_names} check_ids = {check for b in benchmarks.values() for check in b.nested_checks()} checks = await self.list_checks(check_ids=list(check_ids), context=context) check_lookup = {check.id: check for check in checks} @@ -199,7 +193,7 @@ async def perform_benchmarks( report_run_id: Optional[str] = None, ) -> Dict[str, BenchmarkResult]: context = CheckContext(accounts=accounts, severity=severity, only_failed=only_failing) - benchmarks = {name: await self.__benchmark(benchmark_id(name)) for name in benchmark_names} + benchmarks = {name: await self._benchmark(benchmark_id(name)) for name in benchmark_names} # collect all checks check_ids = {check for b in benchmarks.values() for check in b.nested_checks()} checks = await self.list_checks(check_ids=list(check_ids), context=context) @@ -261,22 +255,12 @@ async def perform_checks( await self.event_sender.core_event(CoreEvent.BenchmarkPerformed, {"benchmark": benchmark.id}) return self.__to_result(benchmark, check_by_id, results, context) - async def __benchmark(self, cfg_id: ConfigId) -> Benchmark: - cfg = await self.config_handler.get_config(cfg_id) - if cfg is None or BenchmarkConfigRoot not in cfg.config: - raise ValueError(f"Unknown benchmark: {cfg_id}") - return BenchmarkConfig.from_config(cfg) - async def filter_checks(self, report_filter: Optional[Callable[[ReportCheck], bool]] = None) -> List[ReportCheck]: - cfg_ids = [i async for i in self.config_handler.list_config_ids() if i.startswith(CheckConfigPrefix)] - loaded = await asyncio.gather(*[self.config_handler.get_config(cfg_id) for cfg_id in cfg_ids]) - # fmt: off + cfg_ids = await self._check_ids() + list_of_lists = await asyncio.gather(*[self._checks(cfg_id) for cfg_id in cfg_ids]) return [ - check - for entry in loaded if isinstance(entry, ConfigEntity) and CheckConfigRoot in entry.config - for check in ReportCheckCollectionConfig.from_config(entry) if report_filter is None or report_filter(check) + check for entries in list_of_lists for check in entries if report_filter is None or report_filter(check) ] - # fmt: on async def list_failing_resources( self, graph: GraphName, check_uid: str, account_ids: Optional[List[str]] = None @@ -290,8 +274,7 @@ async def list_failing_resources( model = await self.model_handler.load_model(graph) inspection = checks[0] # load configuration - cfg_entity = await self.config_handler.get_config(ResotoReportValues) - cfg = cfg_entity.config if cfg_entity else {} + cfg = await self._report_values() return await self.__list_failing_resources(graph, model, inspection, cfg, context) async def __list_failing_resources( @@ -374,8 +357,7 @@ async def __perform_checks( # load model model = await self.model_handler.load_model(graph) # load configuration - cfg_entity = await self.config_handler.get_config(ResotoReportValues) - cfg = cfg_entity.config if cfg_entity else {} + cfg = await self._report_values() async def perform_single(check: ReportCheck) -> Tuple[str, SingleCheckResult]: return check.id, await self.__perform_check(graph, model, check, cfg, context) @@ -476,3 +458,94 @@ async def iterate_nodes() -> AsyncIterator[Tuple[NodeId, Json]]: yield NodeId(node_id), dict(issues=issues) return iterate_nodes() + + +@lru_cache(maxsize=1) +def benchmarks_from_file() -> Dict[ConfigId, Benchmark]: + result = {} + for name, js in BenchmarkConfig.from_files().items(): + cid = benchmark_id(name) + benchmark = BenchmarkConfig.from_config(ConfigEntity(cid, {BenchmarkConfigRoot: js})) + result[cid] = benchmark + return result + + +@lru_cache(maxsize=1) +def checks_from_file() -> Dict[ConfigId, List[ReportCheck]]: + result = {} + for name, js in ReportCheckCollectionConfig.from_files().items(): + cid = check_id(name) + result[cid] = ReportCheckCollectionConfig.from_config(ConfigEntity(cid, {CheckConfigRoot: js})) + return result + + +class InspectorFileService(InspectorService): + async def _report_values(self) -> Json: + return {} # default values + + async def _check_ids(self) -> List[ConfigId]: + return list(checks_from_file().keys()) + + async def _checks(self, cfg_id: ConfigId) -> List[ReportCheck]: + return checks_from_file().get(cfg_id, []) + + async def _benchmark(self, cfg_id: ConfigId) -> Benchmark: + return benchmarks_from_file()[cfg_id] + + async def list_benchmarks(self) -> List[Benchmark]: + return list(benchmarks_from_file().values()) + + @staticmethod + def on_startup() -> None: + # make sure benchmarks and checks are loaded + benchmarks_from_file() + checks_from_file() + + +class InspectorConfigService(InspectorService): + def __init__(self, cli: CLI) -> None: + super().__init__(cli) + self.config_handler: ConfigHandler = cli.dependencies.config_handler + + async def start(self) -> None: + if not self.cli.dependencies.config.multi_tenant_setup: + # TODO: we need a migration path for checks added in existing configs + config_ids = {i async for i in self.config_handler.list_config_ids()} + overwrite = False # Only here to simplify development. True until we reach a stable version. + for name, js in BenchmarkConfig.from_files().items(): + if overwrite or benchmark_id(name) not in config_ids: + cid = benchmark_id(name) + log.info(f"Creating benchmark config {cid}") + await self.config_handler.put_config(ConfigEntity(cid, {BenchmarkConfigRoot: js}), validate=False) + for name, js in ReportCheckCollectionConfig.from_files().items(): + if overwrite or check_id(name) not in config_ids: + cid = check_id(name) + log.info(f"Creating check collection config {cid}") + await self.config_handler.put_config(ConfigEntity(cid, {CheckConfigRoot: js}), validate=False) + + async def _report_values(self) -> Json: + cfg_entity = await self.config_handler.get_config(ResotoReportValues) + return cfg_entity.config if cfg_entity else {} + + async def _check_ids(self) -> List[ConfigId]: + return [i async for i in self.config_handler.list_config_ids() if i.startswith(CheckConfigPrefix)] + + async def _checks(self, cfg_id: ConfigId) -> List[ReportCheck]: + config = await self.config_handler.get_config(cfg_id) + if config is not None and CheckConfigRoot in config.config: + return ReportCheckCollectionConfig.from_config(config) + else: + return [] + + async def _benchmark(self, cfg_id: ConfigId) -> Benchmark: + cfg = await self.config_handler.get_config(cfg_id) + if cfg is None or BenchmarkConfigRoot not in cfg.config: + raise ValueError(f"Unknown benchmark: {cfg_id}") + return BenchmarkConfig.from_config(cfg) + + async def list_benchmarks(self) -> List[Benchmark]: + return [ + await self._benchmark(i) + async for i in self.config_handler.list_config_ids() + if i.startswith(BenchmarkConfigPrefix) + ] diff --git a/resotocore/resotocore/web/directives.py b/resotocore/resotocore/web/directives.py index da6765d9fb..1a0b44f91c 100644 --- a/resotocore/resotocore/web/directives.py +++ b/resotocore/resotocore/web/directives.py @@ -79,7 +79,7 @@ async def metrics_handler(request: Request, handler: RequestHandler) -> StreamRe raise ex finally: resp_time = perf_now() - request["start_time"] - log.debug(f"Request {request} took {resp_time} ms") + log.debug(f"Request {request} took {resp_time} s") RequestLatency.labels(request.path).observe(resp_time) RequestInProgress.labels(request.path, request.method).dec() diff --git a/resotocore/tests/resotocore/conftest.py b/resotocore/tests/resotocore/conftest.py index 92501176ce..5039f43463 100644 --- a/resotocore/tests/resotocore/conftest.py +++ b/resotocore/tests/resotocore/conftest.py @@ -73,7 +73,7 @@ from resotocore.model.typed_model import to_js from resotocore.query.template_expander import TemplateExpander from resotocore.report import BenchmarkConfigPrefix, CheckConfigPrefix, Benchmark -from resotocore.report.inspector_service import InspectorService +from resotocore.report.inspector_service import InspectorService, InspectorConfigService from resotocore.report.report_config import BenchmarkConfig from resotocore.system_start import empty_config, parse_args from resotocore.task.model import Subscriber, Subscription @@ -553,7 +553,7 @@ def cli(dependencies: TenantDependencies) -> CLIService: @fixture async def inspector_service(cli: CLIService) -> InspectorService: - async with InspectorService(cli) as service: + async with InspectorConfigService(cli) as service: cli.dependencies.lookup["inspector"] = service return service diff --git a/resotocore/tests/resotocore/report/inspector_service_test.py b/resotocore/tests/resotocore/report/inspector_service_test.py index 0e68d811ca..02944da671 100644 --- a/resotocore/tests/resotocore/report/inspector_service_test.py +++ b/resotocore/tests/resotocore/report/inspector_service_test.py @@ -8,7 +8,13 @@ from resotocore.ids import ConfigId, GraphName from resotocore.query.model import P, Query from resotocore.report import BenchmarkConfigRoot, CheckConfigRoot, BenchmarkResult -from resotocore.report.inspector_service import InspectorService, check_id, benchmark_id +from resotocore.report.inspector_service import ( + InspectorService, + check_id, + benchmark_id, + InspectorConfigService, + InspectorFileService, +) from resotocore.report.report_config import ( config_model, ReportCheckCollectionConfig, @@ -22,12 +28,17 @@ async def inspector_service_with_test_benchmark( cli: CLIService, inspection_check_collection: Json, benchmark: Json ) -> InspectorService: - service = InspectorService(cli) + service = InspectorConfigService(cli) await service.config_handler.put_config(ConfigEntity(check_id("test"), inspection_check_collection)) await service.config_handler.put_config(ConfigEntity(benchmark_id("test"), benchmark)) return service +@fixture +async def inspector_file_service(cli: CLIService) -> InspectorService: + return InspectorFileService(cli) + + @fixture def inspection_check_collection() -> Json: return dict( @@ -184,3 +195,10 @@ async def test_list_failing(inspector_service_with_test_benchmark: InspectorServ assert len(search_res_account) == 0 cmd_res_account = [r async for r in await inspector.list_failing_resources(graph, "test_test_cmd", ["n/a"])] assert len(cmd_res_account) == 0 + + +async def test_file_inspector(inspector_file_service: InspectorService) -> None: + assert len(await inspector_file_service.list_benchmarks()) >= 1 + assert len(await inspector_file_service.filter_checks()) >= 100 + aws_cis = await inspector_file_service.benchmark("aws_cis_1_5") + assert aws_cis is not None