Skip to content

Commit

Permalink
[core][feat] account security score details (#2162)
Browse files Browse the repository at this point in the history
  • Loading branch information
aquamatthias authored Aug 7, 2024
1 parent dcd08d9 commit c06a775
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 11 deletions.
35 changes: 28 additions & 7 deletions fixcore/fixcore/report/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,23 @@ def to_json(self) -> Json:
}


@define
class BenchmarkScore:
score: int
failed_checks: Dict[ReportSeverity, int]
failed_resources: Dict[ReportSeverity, int]

def to_json(self) -> Json:
return {
"score": self.score,
"failed": {
severity.value: {"checks": count, "resources": fr}
for severity, count in self.failed_checks.items()
if (fr := self.failed_resources.get(severity, 0)) and count > 0
},
}


@define
class Remediation:
kind: ClassVar[str] = "fix_core_report_check_remediation"
Expand Down Expand Up @@ -368,23 +385,27 @@ def visit_check_collection(collection: CheckCollectionResult) -> None:
visit_check_collection(self)
return result

def score_for(self, account: str) -> int:
failing: Dict[ReportSeverity, int] = defaultdict(int)
# score, failed_checks, failed_resources
def score_for(self, account: str) -> BenchmarkScore:
failing_checks: Dict[ReportSeverity, int] = defaultdict(int)
failing_resources: Dict[ReportSeverity, int] = defaultdict(int)
available: Dict[ReportSeverity, int] = defaultdict(int)

def available_failing(check: CheckCollectionResult) -> None:
for result in check.checks:
fr = result.number_of_resources_failing_by_account.get(account, 0)
available[result.check.severity] += 1
failing[result.check.severity] += (
1 if result.number_of_resources_failing_by_account.get(account, 0) else 0
)
failing_checks[result.check.severity] += 1 if fr else 0
failing_resources[result.check.severity] += fr
for child in check.children:
available_failing(child)

available_failing(self) # walk the benchmark hierarchy
missing = sum(severity.score * count for severity, count in failing.items())
missing = sum(severity.score * count for severity, count in failing_checks.items())
total = sum(severity.score * count for severity, count in available.items())
return int((max(0, total - missing) * 100) // total) if total > 0 else 100
return BenchmarkScore(
int((max(0, total - missing) * 100) // total) if total > 0 else 100, failing_checks, failing_resources
)


class Inspector(ABC):
Expand Down
25 changes: 22 additions & 3 deletions fixcore/fixcore/report/inspector_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,21 @@ async def perform_benchmarks(
result = {
name: self.__to_result(benchmark, check_lookup, results, context) for name, benchmark in benchmarks.items()
}
all_checks = {cr.check.id: cr for res in result.values() for cr in res.check_results()}

def account_failing(account_id: str) -> Json:
failing_checks: Dict[ReportSeverity, int] = defaultdict(int)
failing_resources: Dict[ReportSeverity, int] = defaultdict(int)
for cr in all_checks.values():
if fr := cr.resources_failing_by_account.get(account_id, []):
failing_checks[cr.check.severity] += 1
failing_resources[cr.check.severity] += len(fr)
return {
sev.value: {"checks": count, "resources": failing_resources[sev]}
for sev, count in failing_checks.items()
if count > 0 and failing_resources[sev] > 0
}

if sync_security_section:
model = await self.model_handler.load_model(graph)
# In case no run_id is provided, we invent a report run id here.
Expand All @@ -278,9 +293,13 @@ async def perform_benchmarks(
if (node_id := value_in_path(acc, NodePath.node_id)) and (
acc_id := value_in_path(acc, NodePath.reported_id)
):
bench_score = {br.id: br.score_for(acc_id) for br in result.values()}
account_score = sum(bench_score.values()) / len(bench_score) if bench_score else 100
patch = {"score": account_score, "benchmark": bench_score}
scores = {br.id: br.score_for(acc_id) for br in result.values()}
account_score = sum(a.score for a in scores.values()) / len(scores) if scores else 100
patch = {
"score": account_score,
"failed": account_failing(acc_id),
"benchmark": {br: score.to_json() for br, score in scores.items()},
}
async for _ in db.update_nodes_metadata(model, patch, [node_id]):
pass
return result
Expand Down
4 changes: 3 additions & 1 deletion fixcore/tests/fixcore/report/inspector_service_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,10 @@ def assert_result(results: Dict[str, BenchmarkResult]) -> None:
db = inspector_service.db_access.get_graph_db(graph_name)
async with await db.search_list(QueryModel(Query.by("account"), foo_model)) as crsr:
async for elem in crsr:
expected = {"critical": {"checks": 1, "resources": 10}, "medium": {"checks": 1, "resources": 10}}
assert elem["metadata"]["score"] == 0
assert elem["metadata"]["benchmark"]["test"] == 0
assert elem["metadata"]["failed"] == expected
assert elem["metadata"]["benchmark"]["test"] == {"score": 0, "failed": expected}


async def test_perform_benchmark_ignored(
Expand Down

0 comments on commit c06a775

Please sign in to comment.