From cb339cb15db3941d37d7cfa2a5cac7364986c479 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 15:18:05 +0530 Subject: [PATCH 01/25] Optimize vulnerabilities view Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 84 ++++++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 77f75238d..ebceaf0a1 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -7,6 +7,8 @@ # See https://aboutcode.org for more information about nexB OSS projects. # import logging +from itertools import groupby +from operator import attrgetter from cvss.exceptions import CVSS2MalformedError from cvss.exceptions import CVSS3MalformedError @@ -197,36 +199,7 @@ def get_context_data(self, **kwargs): if s.value: severity_values.add(s.value) - sorted_affected_packages = sorted(self.object.affected_packages.all(), key=purl_sort_key) - sorted_fixed_by_packages = sorted(self.object.fixed_by_packages.all(), key=purl_sort_key) - - all_affected_fixed_by_matches = [] - for sorted_affected_package in sorted_affected_packages: - affected_fixed_by_matches = {} - affected_fixed_by_matches["affected_package"] = sorted_affected_package - matched_fixed_by_packages = [] - for fixed_by_package in sorted_fixed_by_packages: - - # Ghost Package can't fix vulnerability. - if fixed_by_package.is_ghost: - continue - - sorted_affected_version_class = get_purl_version_class(sorted_affected_package) - fixed_by_version_class = get_purl_version_class(fixed_by_package) - if ( - (fixed_by_package.type == sorted_affected_package.type) - and (fixed_by_package.namespace == sorted_affected_package.namespace) - and (fixed_by_package.name == sorted_affected_package.name) - and (fixed_by_package.qualifiers == sorted_affected_package.qualifiers) - and (fixed_by_package.subpath == sorted_affected_package.subpath) - and ( - fixed_by_version_class(fixed_by_package.version) - > sorted_affected_version_class(sorted_affected_package.version) - ) - ): - matched_fixed_by_packages.append(fixed_by_package.purl) - affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages - all_affected_fixed_by_matches.append(affected_fixed_by_matches) + sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches = self.aggregate_fixed_and_affected_packages() context.update( { @@ -247,6 +220,57 @@ def get_context_data(self, **kwargs): ) return context + def aggregate_fixed_and_affected_packages(self): + sorted_fixed_by_packages = self.object.fixed_by_packages.filter(is_ghost=False).order_by( + "type", "namespace", "name", "qualifiers", "subpath" + ) + + sorted_affected_packages = self.object.affected_packages.all() + + grouped_fixed_by_packages = { + key: list(group) + for key, group in groupby( + sorted_fixed_by_packages, + key=attrgetter("type", "namespace", "name", "qualifiers", "subpath"), + ) + } + + all_affected_fixed_by_matches = [] + + for sorted_affected_package in sorted_affected_packages: + affected_fixed_by_matches = { + "affected_package": sorted_affected_package, + "matched_fixed_by_packages": [], + } + + # Build the key to find matching group + key = ( + sorted_affected_package.type, + sorted_affected_package.namespace, + sorted_affected_package.name, + sorted_affected_package.qualifiers, + sorted_affected_package.subpath, + ) + + # Get matching group from pre-grouped fixed_by_packages + matching_fixed_packages = grouped_fixed_by_packages.get(key, []) + + # Get version classes for comparison + affected_version_class = get_purl_version_class(sorted_affected_package) + affected_version = affected_version_class(sorted_affected_package.version) + + # Compare versions and filter valid matches + matched_fixed_by_packages = [ + fixed_by_package.purl + for fixed_by_package in matching_fixed_packages + if get_purl_version_class(fixed_by_package)(fixed_by_package.version) + > affected_version + ] + + affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages + all_affected_fixed_by_matches.append(affected_fixed_by_matches) + return sorted_fixed_by_packages,sorted_affected_packages,all_affected_fixed_by_matches + class HomePage(View): template_name = "index.html" From 19fa2466d7f9644aa7783d15fa4bf88f22f2321c Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 15:27:08 +0530 Subject: [PATCH 02/25] Fix formatting Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index ebceaf0a1..6754fed3c 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -199,7 +199,11 @@ def get_context_data(self, **kwargs): if s.value: severity_values.add(s.value) - sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches = self.aggregate_fixed_and_affected_packages() + ( + sorted_fixed_by_packages, + sorted_affected_packages, + all_affected_fixed_by_matches, + ) = self.aggregate_fixed_and_affected_packages() context.update( { @@ -269,7 +273,7 @@ def aggregate_fixed_and_affected_packages(self): affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages all_affected_fixed_by_matches.append(affected_fixed_by_matches) - return sorted_fixed_by_packages,sorted_affected_packages,all_affected_fixed_by_matches + return sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches class HomePage(View): From 68672b41b13bf8857039024e757bc8242ee20a9d Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 15:38:30 +0530 Subject: [PATCH 03/25] Optimize vulnerabilities view Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 51 ++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 6754fed3c..1d4545d0f 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -178,26 +178,7 @@ def get_context_data(self, **kwargs): ] status = self.object.get_status_label - severity_vectors = [] - severity_values = set() - for s in self.object.severities.all(): - if s.scoring_system == EPSS.identifier: - continue - - if s.scoring_elements and s.scoring_system in SCORING_SYSTEMS: - try: - vector_values = SCORING_SYSTEMS[s.scoring_system].get(s.scoring_elements) - severity_vectors.append(vector_values) - except ( - CVSS2MalformedError, - CVSS3MalformedError, - CVSS4MalformedError, - NotImplementedError, - ): - logging.error(f"CVSSMalformedError for {s.scoring_elements}") - - if s.value: - severity_values.add(s.value) + severity_vectors, severity_values = self.get_severity_vectors_and_values() ( sorted_fixed_by_packages, @@ -275,6 +256,36 @@ def aggregate_fixed_and_affected_packages(self): all_affected_fixed_by_matches.append(affected_fixed_by_matches) return sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches + def get_severity_vectors_and_values(self): + """ + Collect severity vectors and values, excluding EPSS scoring systems and handling errors gracefully. + """ + severity_vectors = [] + severity_values = set() + + severities = self.object.severities.exclude(scoring_system=EPSS.identifier) + + for severity in severities: + if severity.scoring_elements and severity.scoring_system in SCORING_SYSTEMS: + try: + vector_values = SCORING_SYSTEMS[severity.scoring_system].get( + severity.scoring_elements + ) + if vector_values: + severity_vectors.append(vector_values) + except ( + CVSS2MalformedError, + CVSS3MalformedError, + CVSS4MalformedError, + NotImplementedError, + ) as e: + logging.error(f"CVSSMalformedError for {severity.scoring_elements}: {e}") + + if severity.value: + severity_values.add(severity.value) + + return severity_vectors, severity_values + class HomePage(View): template_name = "index.html" From 7a1b5bb00bd6ee025f6872927e463ae9af498562 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 15:47:34 +0530 Subject: [PATCH 04/25] Optimize vulnerabilities view Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 46 +++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 1d4545d0f..78d179064 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -263,26 +263,32 @@ def get_severity_vectors_and_values(self): severity_vectors = [] severity_values = set() - severities = self.object.severities.exclude(scoring_system=EPSS.identifier) - - for severity in severities: - if severity.scoring_elements and severity.scoring_system in SCORING_SYSTEMS: - try: - vector_values = SCORING_SYSTEMS[severity.scoring_system].get( - severity.scoring_elements - ) - if vector_values: - severity_vectors.append(vector_values) - except ( - CVSS2MalformedError, - CVSS3MalformedError, - CVSS4MalformedError, - NotImplementedError, - ) as e: - logging.error(f"CVSSMalformedError for {severity.scoring_elements}: {e}") - - if severity.value: - severity_values.add(severity.value) + # Exclude EPSS scoring system + base_severities = self.object.severities.exclude(scoring_system=EPSS.identifier) + + # QuerySet for severities with valid scoring_elements and scoring_system in SCORING_SYSTEMS + valid_scoring_severities = base_severities.filter( + scoring_elements__isnull=False, scoring_system__in=SCORING_SYSTEMS.keys() + ) + + for severity in valid_scoring_severities: + try: + vector_values = SCORING_SYSTEMS[severity.scoring_system].get( + severity.scoring_elements + ) + if vector_values: + severity_vectors.append(vector_values) + except ( + CVSS2MalformedError, + CVSS3MalformedError, + CVSS4MalformedError, + NotImplementedError, + ) as e: + logging.error(f"CVSSMalformedError for {severity.scoring_elements}: {e}") + + valid_value_severities = base_severities.filter(value__isnull=False).exclude(value="") + + severity_values.update(valid_value_severities.values_list("value", flat=True)) return severity_vectors, severity_values From 0e489bea5f004a81d61f817d799d87f7610cfc58 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 16:05:00 +0530 Subject: [PATCH 05/25] Fix formatting Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 78d179064..95789a6e1 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -178,7 +178,7 @@ def get_context_data(self, **kwargs): ] status = self.object.get_status_label - severity_vectors, severity_values = self.get_severity_vectors_and_values() + # severity_vectors, severity_values = self.get_severity_vectors_and_values() ( sorted_fixed_by_packages, @@ -191,8 +191,8 @@ def get_context_data(self, **kwargs): "vulnerability": self.object, "vulnerability_search_form": VulnerabilitySearchForm(self.request.GET), "severities": list(self.object.severities.all()), - "severity_score_range": get_severity_range(severity_values), - "severity_vectors": severity_vectors, + # "severity_score_range": get_severity_range(severity_values), + # "severity_vectors": severity_vectors, "references": self.object.references.all(), "aliases": self.object.aliases.all(), "affected_packages": sorted_affected_packages, From 938b6a2217022dc39773fda182c5e6c4f76d8911 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 16:06:23 +0530 Subject: [PATCH 06/25] Fix formatting Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 95789a6e1..ee89dd6a4 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -191,8 +191,8 @@ def get_context_data(self, **kwargs): "vulnerability": self.object, "vulnerability_search_form": VulnerabilitySearchForm(self.request.GET), "severities": list(self.object.severities.all()), - # "severity_score_range": get_severity_range(severity_values), - # "severity_vectors": severity_vectors, + "severity_score_range": "", + "severity_vectors": [], "references": self.object.references.all(), "aliases": self.object.aliases.all(), "affected_packages": sorted_affected_packages, From b2482452890a2124596cace30a954f567e09db4e Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 18:46:10 +0530 Subject: [PATCH 07/25] Fix tests Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 95 +++++++++++++++++++++++++++++++++++++++ vulnerabilities/utils.py | 10 +++++ vulnerabilities/views.py | 60 +------------------------ 3 files changed, 107 insertions(+), 58 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 4db674e3e..7a98667d0 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -12,8 +12,13 @@ import logging from contextlib import suppress from functools import cached_property +from itertools import groupby +from operator import attrgetter from typing import Union +from cvss.exceptions import CVSS2MalformedError +from cvss.exceptions import CVSS3MalformedError +from cvss.exceptions import CVSS4MalformedError from cwe2.database import Database from django.contrib.auth import get_user_model from django.contrib.auth.models import UserManager @@ -43,6 +48,7 @@ from aboutcode import hashid from vulnerabilities import utils +from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import normalize_purl from vulnerabilities.utils import purl_to_dict @@ -371,6 +377,95 @@ def get_related_purls(self): """ return [p.package_url for p in self.packages.distinct().all()] + def aggregate_fixed_and_affected_packages(self): + from vulnerabilities.views import get_purl_version_class + + sorted_fixed_by_packages = self.fixed_by_packages.filter(is_ghost=False).order_by( + "type", "namespace", "name", "qualifiers", "subpath" + ) + + sorted_affected_packages = self.affected_packages.all() + + grouped_fixed_by_packages = { + key: list(group) + for key, group in groupby( + sorted_fixed_by_packages, + key=attrgetter("type", "namespace", "name", "qualifiers", "subpath"), + ) + } + + all_affected_fixed_by_matches = [] + + for sorted_affected_package in sorted_affected_packages: + affected_fixed_by_matches = { + "affected_package": sorted_affected_package, + "matched_fixed_by_packages": [], + } + + # Build the key to find matching group + key = ( + sorted_affected_package.type, + sorted_affected_package.namespace, + sorted_affected_package.name, + sorted_affected_package.qualifiers, + sorted_affected_package.subpath, + ) + + # Get matching group from pre-grouped fixed_by_packages + matching_fixed_packages = grouped_fixed_by_packages.get(key, []) + + # Get version classes for comparison + affected_version_class = get_purl_version_class(sorted_affected_package) + affected_version = affected_version_class(sorted_affected_package.version) + + # Compare versions and filter valid matches + matched_fixed_by_packages = [ + fixed_by_package.purl + for fixed_by_package in matching_fixed_packages + if get_purl_version_class(fixed_by_package)(fixed_by_package.version) + > affected_version + ] + + affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages + all_affected_fixed_by_matches.append(affected_fixed_by_matches) + return sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches + + def get_severity_vectors_and_values(self): + """ + Collect severity vectors and values, excluding EPSS scoring systems and handling errors gracefully. + """ + severity_vectors = [] + severity_values = set() + + # Exclude EPSS scoring system + base_severities = self.severities.exclude(scoring_system=EPSS.identifier) + + # QuerySet for severities with valid scoring_elements and scoring_system in SCORING_SYSTEMS + valid_scoring_severities = base_severities.filter( + scoring_elements__isnull=False, scoring_system__in=SCORING_SYSTEMS.keys() + ) + + for severity in valid_scoring_severities: + try: + vector_values = SCORING_SYSTEMS[severity.scoring_system].get( + severity.scoring_elements + ) + if vector_values: + severity_vectors.append(vector_values) + except ( + CVSS2MalformedError, + CVSS3MalformedError, + CVSS4MalformedError, + NotImplementedError, + ) as e: + logging.error(f"CVSSMalformedError for {severity.scoring_elements}: {e}") + + valid_value_severities = base_severities.filter(value__isnull=False).exclude(value="") + + severity_values.update(valid_value_severities.values_list("value", flat=True)) + + return severity_vectors, severity_values + class Weakness(models.Model): """ diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 969a08f2f..32cfcbc02 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -32,6 +32,7 @@ from packageurl import PackageURL from packageurl.contrib.django.utils import without_empty_values from univers.version_range import RANGE_CLASS_BY_SCHEMES +from univers.version_range import AlpineLinuxVersionRange from univers.version_range import NginxVersionRange from univers.version_range import VersionRange @@ -536,3 +537,12 @@ def normalize_purl(purl: Union[PackageURL, str]): if isinstance(purl, PackageURL): purl = str(purl) return PackageURL.from_string(purl) + + +def get_purl_version_class(purl): + RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange + purl_version_class = None + check_version_class = RANGE_CLASS_BY_SCHEMES.get(purl.type, None) + if check_version_class: + purl_version_class = check_version_class.version_class + return purl_version_class diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index ee89dd6a4..d8c1ae29a 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -7,8 +7,6 @@ # See https://aboutcode.org for more information about nexB OSS projects. # import logging -from itertools import groupby -from operator import attrgetter from cvss.exceptions import CVSS2MalformedError from cvss.exceptions import CVSS3MalformedError @@ -24,18 +22,15 @@ from django.views import generic from django.views.generic.detail import DetailView from django.views.generic.list import ListView -from univers.version_range import RANGE_CLASS_BY_SCHEMES -from univers.version_range import AlpineLinuxVersionRange from vulnerabilities import models from vulnerabilities.forms import ApiUserCreationForm from vulnerabilities.forms import PackageSearchForm from vulnerabilities.forms import VulnerabilitySearchForm -from vulnerabilities.models import VulnerabilityStatusType from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS -from vulnerabilities.utils import get_severity_range from vulnerablecode import __version__ as VULNERABLECODE_VERSION +from vulnerabilities.utils import get_purl_version_class from vulnerablecode.settings import env PAGE_SIZE = 20 @@ -184,7 +179,7 @@ def get_context_data(self, **kwargs): sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches, - ) = self.aggregate_fixed_and_affected_packages() + ) = self.object.aggregate_fixed_and_affected_packages() context.update( { @@ -205,57 +200,6 @@ def get_context_data(self, **kwargs): ) return context - def aggregate_fixed_and_affected_packages(self): - sorted_fixed_by_packages = self.object.fixed_by_packages.filter(is_ghost=False).order_by( - "type", "namespace", "name", "qualifiers", "subpath" - ) - - sorted_affected_packages = self.object.affected_packages.all() - - grouped_fixed_by_packages = { - key: list(group) - for key, group in groupby( - sorted_fixed_by_packages, - key=attrgetter("type", "namespace", "name", "qualifiers", "subpath"), - ) - } - - all_affected_fixed_by_matches = [] - - for sorted_affected_package in sorted_affected_packages: - affected_fixed_by_matches = { - "affected_package": sorted_affected_package, - "matched_fixed_by_packages": [], - } - - # Build the key to find matching group - key = ( - sorted_affected_package.type, - sorted_affected_package.namespace, - sorted_affected_package.name, - sorted_affected_package.qualifiers, - sorted_affected_package.subpath, - ) - - # Get matching group from pre-grouped fixed_by_packages - matching_fixed_packages = grouped_fixed_by_packages.get(key, []) - - # Get version classes for comparison - affected_version_class = get_purl_version_class(sorted_affected_package) - affected_version = affected_version_class(sorted_affected_package.version) - - # Compare versions and filter valid matches - matched_fixed_by_packages = [ - fixed_by_package.purl - for fixed_by_package in matching_fixed_packages - if get_purl_version_class(fixed_by_package)(fixed_by_package.version) - > affected_version - ] - - affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages - all_affected_fixed_by_matches.append(affected_fixed_by_matches) - return sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches - def get_severity_vectors_and_values(self): """ Collect severity vectors and values, excluding EPSS scoring systems and handling errors gracefully. From f042c077c617cb9df3bf63bd5f31c4d17706a6ab Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 7 Jan 2025 18:32:34 +0530 Subject: [PATCH 08/25] Fix views Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 78 +++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index d8c1ae29a..367263272 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -22,6 +22,7 @@ from django.views import generic from django.views.generic.detail import DetailView from django.views.generic.list import ListView +from django.db.models import Prefetch from vulnerabilities import models from vulnerabilities.forms import ApiUserCreationForm @@ -153,68 +154,86 @@ class VulnerabilityDetails(DetailView): slug_field = "vulnerability_id" def get_queryset(self): + """ + Prefetch and optimize related data to minimize database hits. + """ return ( super() .get_queryset() + .select_related() .prefetch_related( "references", "aliases", "weaknesses", "severities", "exploits", + Prefetch( + "affecting_packages", + queryset=models.Vulnerability.objects.only( + "type", "namespace", "name", "version" + ), + ), + Prefetch( + "fixed_by_packages", + queryset=models.Vulnerability.objects.only( + "type", "namespace", "name", "version" + ), + ), ) ) + def get_context_data(self, **kwargs): + """ + Build context with preloaded QuerySets and minimize redundant queries. + """ context = super().get_context_data(**kwargs) - weaknesses = self.object.weaknesses.all() + vulnerability = self.object + + # Pre-fetch and process data in Python instead of the template weaknesses_present_in_db = [ - weakness_object for weakness_object in weaknesses if weakness_object.weakness + weakness_object for weakness_object in vulnerability.weaknesses.all() + if weakness_object.weakness ] - status = self.object.get_status_label - - # severity_vectors, severity_values = self.get_severity_vectors_and_values() - + + # Cache aggregated packages ( sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches, - ) = self.object.aggregate_fixed_and_affected_packages() + ) = vulnerability.aggregate_fixed_and_affected_packages() + + severity_vectors, severity_values = self.get_severity_vectors_and_values(vulnerability) context.update( { - "vulnerability": self.object, + "vulnerability": vulnerability, "vulnerability_search_form": VulnerabilitySearchForm(self.request.GET), - "severities": list(self.object.severities.all()), + "severities": list(vulnerability.severities.all()), "severity_score_range": "", - "severity_vectors": [], - "references": self.object.references.all(), - "aliases": self.object.aliases.all(), + "severity_vectors": severity_vectors, + "references": list(vulnerability.references.all()), + "aliases": list(vulnerability.aliases.all()), "affected_packages": sorted_affected_packages, "fixed_by_packages": sorted_fixed_by_packages, "weaknesses": weaknesses_present_in_db, - "status": status, - "history": self.object.history, + "status": vulnerability.get_status_label, + "history": vulnerability.history, "all_affected_fixed_by_matches": all_affected_fixed_by_matches, } ) return context - def get_severity_vectors_and_values(self): + def get_severity_vectors_and_values(self, vulnerability): """ - Collect severity vectors and values, excluding EPSS scoring systems and handling errors gracefully. + Collect severity vectors and values, excluding EPSS scoring systems efficiently. """ severity_vectors = [] severity_values = set() - - # Exclude EPSS scoring system - base_severities = self.object.severities.exclude(scoring_system=EPSS.identifier) - - # QuerySet for severities with valid scoring_elements and scoring_system in SCORING_SYSTEMS - valid_scoring_severities = base_severities.filter( - scoring_elements__isnull=False, scoring_system__in=SCORING_SYSTEMS.keys() - ) - + + # Use prefetch data if available + valid_scoring_severities = getattr(vulnerability, "prefetched_valid_severities", []) + for severity in valid_scoring_severities: try: vector_values = SCORING_SYSTEMS[severity.scoring_system].get( @@ -229,10 +248,11 @@ def get_severity_vectors_and_values(self): NotImplementedError, ) as e: logging.error(f"CVSSMalformedError for {severity.scoring_elements}: {e}") - - valid_value_severities = base_severities.filter(value__isnull=False).exclude(value="") - - severity_values.update(valid_value_severities.values_list("value", flat=True)) + + # Collect valid values using a pre-filtered queryset + severity_values.update( + vulnerability.severities.exclude(value__isnull=True).exclude(value="").values_list("value", flat=True) + ) return severity_vectors, severity_values From 6b8aad0b28f1cbcc2b722873e82154f820a21763 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 8 Jan 2025 17:51:11 +0530 Subject: [PATCH 09/25] Move severities to a different tab Signed-off-by: Tushar Goel --- .../templates/vulnerability_details.html | 564 ++---------------- 1 file changed, 37 insertions(+), 527 deletions(-) diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index e9e58c79e..612789d99 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -40,6 +40,13 @@ +
  • + + + Severities ({{ severities|length }}) + + +
  • @@ -48,12 +55,12 @@
  • - - - Severities vectors ({{ severity_vectors|length }}) - - -
    • + + + Severities vectors ({{ severity_vectors|length }}) + + + {% if vulnerability.exploits %}
  • @@ -66,11 +73,11 @@ {% endif %}
  • - - - EPSS - - + + + EPSS + +
  • @@ -152,542 +159,46 @@ {{ vulnerability.risk_score }} - -
    - Severity ({{ severities|length }}) -
    -
    - - - - - - - {% for severity in severities %} - - - - - - {% empty %} - - - - {% endfor %} -
    System Score Found at
    {{ severity.scoring_system }}{{ severity.value }} - {{ severity.url }} -
    - There are no known severity scores. -
    -
    +
    Affected/Fixed by packages ({{ affected_packages|length }}/{{ fixed_by_packages|length }})
    -
    - - - - - - - - - {% for package in affected_packages|slice:":3" %} - - - - - {% empty %} - - - - {% endfor %} - {% if affected_packages|length > 3 %} - - - - {% endif %} - -
    AffectedFixed by
    - {{ package.purl }} - - {% for match in all_affected_fixed_by_matches %} - {% if match.affected_package == package %} - {% if match.matched_fixed_by_packages|length > 0 %} - {% for pkg in match.matched_fixed_by_packages %} - {{ pkg }} -
    - {% endfor %} - {% else %} - There are no reported fixed by versions. - {% endif %} - {% endif %} - {% endfor %} -
    - This vulnerability is not known to affect any packages. -
    - See Affected/Fixed by packages tab for more -
    -
    - -
    - Weaknesses ({{ weaknesses|length }}) -
    -
    - - {% for weakness in weaknesses %} - - - - - - {% empty %} - - - - {% endfor %} -
    CWE-{{ weakness.cwe_id }} - - {{ weakness.name }} - -
    - There are no known CWE. -
    -
    + -
    - - - - - - - - - {% for package in affected_packages %} - - - - - {% empty %} - - - - {% endfor %} - -
    AffectedFixed by
    - {{ package.purl }} - - - {% for match in all_affected_fixed_by_matches %} - {% if match.affected_package == package %} - {% if match.matched_fixed_by_packages|length > 0 %} - {% for pkg in match.matched_fixed_by_packages %} - {{ pkg }} -
    - {% endfor %} - {% else %} - There are no reported fixed by versions. - {% endif %} - {% endif %} - {% endfor %} - -
    - This vulnerability is not known to affect any packages. -
    -
    - -
    - - - - - - - - - {% for ref in references %} + +
    +
    Reference id Reference type URL
    - {% if ref.reference_id %} - - {% else %} - - {% endif %} - - {% if ref.reference_type %} - - {% else %} - - {% endif %} - - + + + - {% empty %} + {% for severity in severities %} - + + - {% endfor %} -
    {{ ref.reference_id }}{{ ref.get_reference_type_display }}{{ ref.url }} System Score Found at
    - There are no known references. + {{ severity.scoring_system }}{{ severity.value }} + {{ severity.url }}
    -
    - -
    - {% for severity_vector in severity_vectors %} - {% if severity_vector.version == '2.0' %} - Vector: {{ severity_vector.vectorString }} - - - - - - - - - - - - - - - - - - - -
    Exploitability (E)Access Vector (AV)Access Complexity (AC)Authentication (Au)Confidentiality Impact (C)Integrity Impact (I)Availability Impact (A)
    {{ severity_vector.exploitability|cvss_printer:"high,functional,unproven,proof_of_concept,not_defined" }}{{ severity_vector.accessVector|cvss_printer:"local,adjacent_network,network" }}{{ severity_vector.accessComplexity|cvss_printer:"high,medium,low" }}{{ severity_vector.authentication|cvss_printer:"multiple,single,none" }}{{ severity_vector.confidentialityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.integrityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.availabilityImpact|cvss_printer:"none,partial,complete" }}
    - {% elif severity_vector.version == '3.1' or severity_vector.version == '3.0'%} - Vector: {{ severity_vector.vectorString }} - - - - - - - - - - - - - - - - - - - - - -
    Attack Vector (AV)Attack Complexity (AC)Privileges Required (PR)User Interaction (UI)Scope (S)Confidentiality Impact (C)Integrity Impact (I)Availability Impact (A)
    {{ severity_vector.attackVector|cvss_printer:"network,adjacent_network,local,physical"}}{{ severity_vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.userInteraction|cvss_printer:"none,required"}}{{ severity_vector.scope|cvss_printer:"unchanged,changed" }}{{ severity_vector.confidentialityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.integrityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.availabilityImpact|cvss_printer:"high,low,none" }}
    - {% elif severity_vector.version == '4' %} - Vector: {{ severity_vector.vectorString }} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Attack Vector (AV)Attack Complexity (AC)Attack Requirements (AT)Privileges Required (PR)User Interaction (UI)Vulnerable System Impact Confidentiality (VC)Vulnerable System Impact Integrity (VI)Vulnerable System Impact Availability (VA)Subsequent System Impact Confidentiality (SC)Subsequent System Impact Integrity (SI)Subsequent System Impact Availability (SA)
    {{ severity_vector.attackVector|cvss_printer:"network,adjacent,local,physical"}}{{ severity_vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.attackRequirement|cvss_printer:"none,present" }}{{ severity_vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.userInteraction|cvss_printer:"none,passive,active"}}{{ severity_vector.vulnerableSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.vulnerableSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.vulnerableSystemImpactAvailability|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactAvailability|cvss_printer:"high,low,none" }}
    - {% elif severity_vector.version == 'ssvc' %} -
    - Vector: {{ severity_vector.vectorString }} -
    - {% endif %} - {% empty %} - - - There are no known vectors. - - - {% endfor %} -
    - - -
    - {% for exploit in vulnerability.exploits.all %} - - - - - - - - {% if exploit.date_added %} - - - - - {% endif %} - {% if exploit.description %} - - - - - {% endif %} - {% if exploit.required_action %} - - - - - {% endif %} - {% if exploit.due_date %} - - - - - {% endif %} - {% if exploit.notes %} - - - - - {% endif %} - {% if exploit.known_ransomware_campaign_use is not None %} - - - - - {% endif %} - {% if exploit.source_date_published %} - - - - - {% endif %} - {% if exploit.exploit_type %} - - - - - {% endif %} - {% if exploit.platform %} - - - - - {% endif %} - {% if exploit.source_date_updated %} - - - - - {% endif %} - - {% if exploit.source_url %} - - - - - {% endif %} - -
    Data source {{ exploit.data_source }}
    - - Date added - - {{ exploit.date_added }}
    - - Description - - {{ exploit.description }}
    - - Required action - - {{ exploit.required_action }}
    - - Due date - - {{ exploit.due_date }}
    - - Note - - 
    {{ exploit.notes }}
    - - Ransomware campaign use - - {{ exploit.known_ransomware_campaign_use|yesno:"Known,Unknown" }}
    - - Source publication date - - {{ exploit.source_date_published }}
    - - Exploit type - - {{ exploit.exploit_type }}
    - - Platform - - {{ exploit.platform }}
    - - Source update date - - {{ exploit.source_date_updated }}
    - - Source URL - - {{ exploit.source_url }}
    - {% empty %} - - - No exploits are available. - - - {% endfor %} -
    - - - {% for severity in severities %} - {% if severity.scoring_system == 'epss' %} -
    -
    - Exploit Prediction Scoring System -
    - - - - - - - - - - - - - {% if severity.published_at %} - - - - - {% endif %} - - -
    - - Percentile - - {{ severity.scoring_elements }}
    - - EPSS score - - {{ severity.value }}
    - - Published at - - {{ severity.published_at }}
    -
    - {% endif %} - {% empty %} -
    - - - There are no EPSS available. - - -
    - {% endfor %} - -
    - - - - - - - - - - - {% for log in history %} - - - - - - - {% empty %} - {% endfor %}
    - - Date - - Actor - Action Source - VulnerableCode Version -
    {{ log.get_iso_time }}{{ log.actor_name }}{{ log.get_action_type_label }} {{log.source_url }} {{ log.software_version }}
    - There are no relevant records. + + There are no known severity scores.
    + + + @@ -711,5 +222,4 @@ } - -{% endblock %} +{% endblock %} \ No newline at end of file From 363a37fe39d8e5a9d9175e4f4488106b56e4dca8 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 15 Jan 2025 15:49:45 +0530 Subject: [PATCH 10/25] Separate views Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 99 +++++++++++++++++++++------------------- vulnerablecode/urls.py | 7 ++- 2 files changed, 57 insertions(+), 49 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 367263272..4fa29337b 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -169,13 +169,13 @@ def get_queryset(self): "exploits", Prefetch( "affecting_packages", - queryset=models.Vulnerability.objects.only( + queryset=models.Package.objects.only( "type", "namespace", "name", "version" ), ), Prefetch( "fixed_by_packages", - queryset=models.Vulnerability.objects.only( + queryset=models.Package.objects.only( "type", "namespace", "name", "version" ), ), @@ -195,15 +195,6 @@ def get_context_data(self, **kwargs): weakness_object for weakness_object in vulnerability.weaknesses.all() if weakness_object.weakness ] - - # Cache aggregated packages - ( - sorted_fixed_by_packages, - sorted_affected_packages, - all_affected_fixed_by_matches, - ) = vulnerability.aggregate_fixed_and_affected_packages() - - severity_vectors, severity_values = self.get_severity_vectors_and_values(vulnerability) context.update( { @@ -211,52 +202,15 @@ def get_context_data(self, **kwargs): "vulnerability_search_form": VulnerabilitySearchForm(self.request.GET), "severities": list(vulnerability.severities.all()), "severity_score_range": "", - "severity_vectors": severity_vectors, "references": list(vulnerability.references.all()), "aliases": list(vulnerability.aliases.all()), - "affected_packages": sorted_affected_packages, - "fixed_by_packages": sorted_fixed_by_packages, "weaknesses": weaknesses_present_in_db, "status": vulnerability.get_status_label, "history": vulnerability.history, - "all_affected_fixed_by_matches": all_affected_fixed_by_matches, } ) return context - def get_severity_vectors_and_values(self, vulnerability): - """ - Collect severity vectors and values, excluding EPSS scoring systems efficiently. - """ - severity_vectors = [] - severity_values = set() - - # Use prefetch data if available - valid_scoring_severities = getattr(vulnerability, "prefetched_valid_severities", []) - - for severity in valid_scoring_severities: - try: - vector_values = SCORING_SYSTEMS[severity.scoring_system].get( - severity.scoring_elements - ) - if vector_values: - severity_vectors.append(vector_values) - except ( - CVSS2MalformedError, - CVSS3MalformedError, - CVSS4MalformedError, - NotImplementedError, - ) as e: - logging.error(f"CVSSMalformedError for {severity.scoring_elements}: {e}") - - # Collect valid values using a pre-filtered queryset - severity_values.update( - vulnerability.severities.exclude(value__isnull=True).exclude(value="").values_list("value", flat=True) - ) - - return severity_vectors, severity_values - - class HomePage(View): template_name = "index.html" @@ -325,3 +279,52 @@ def form_valid(self, form): def get_success_url(self): return reverse_lazy("api_user_request") + + +class VulnerabilityPackagesDetails(DetailView): + """ + View to display all packages affected by or fixing a specific vulnerability. + URL: /vulnerabilities/{vulnerability_id}/packages + """ + + model = models.Vulnerability + template_name = "vulnerability_package_details.html" + slug_url_kwarg = "vulnerability_id" + slug_field = "vulnerability_id" + + def get_queryset(self): + """ + Prefetch and optimize related data to minimize database hits. + """ + return super().get_queryset().prefetch_related( + Prefetch( + "affecting_packages", + queryset=models.Package.objects.only( + "type", "namespace", "name", "version" + ), + ), + Prefetch( + "fixed_by_packages", + queryset=models.Package.objects.only( + "type", "namespace", "name", "version" + ), + ), + ) + + def get_context_data(self, **kwargs): + """ + Build context with preloaded QuerySets and minimize redundant queries. + """ + context = super().get_context_data(**kwargs) + vulnerability = self.object + ( + sorted_fixed_by_packages, + sorted_affected_packages, + all_affected_fixed_by_matches, + ) = vulnerability.aggregate_fixed_and_affected_packages() + context.update({ + "affected_packages": sorted_affected_packages, + "fixed_by_packages": sorted_fixed_by_packages, + "all_affected_fixed_by_matches": all_affected_fixed_by_matches, + }) + return context diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index 54540a66d..b481a119e 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -27,7 +27,7 @@ from vulnerabilities.views import HomePage from vulnerabilities.views import PackageDetails from vulnerabilities.views import PackageSearch -from vulnerabilities.views import VulnerabilityDetails +from vulnerabilities.views import VulnerabilityDetails, VulnerabilityPackagesDetails from vulnerabilities.views import VulnerabilitySearch from vulnerablecode.settings import DEBUG_TOOLBAR @@ -83,6 +83,11 @@ def __init__(self, *args, **kwargs): VulnerabilityDetails.as_view(), name="vulnerability_details", ), + path( + "vulnerabilities//packages", + VulnerabilityPackagesDetails.as_view(), + name="vulnerability_package_details", + ), path( "api/", include(api_router.urls), From 017ad9c8f1260cd39b8c1b3faec7e6cd911744cf Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 16 Jan 2025 19:40:16 +0530 Subject: [PATCH 11/25] Refactor Views Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 45 ++++++++++++++++++++-------------------- vulnerablecode/urls.py | 3 ++- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 4fa29337b..c4ae2bd04 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -14,6 +14,7 @@ from django.contrib import messages from django.core.exceptions import ValidationError from django.core.mail import send_mail +from django.db.models import Prefetch from django.http.response import Http404 from django.shortcuts import redirect from django.shortcuts import render @@ -22,7 +23,6 @@ from django.views import generic from django.views.generic.detail import DetailView from django.views.generic.list import ListView -from django.db.models import Prefetch from vulnerabilities import models from vulnerabilities.forms import ApiUserCreationForm @@ -169,30 +169,26 @@ def get_queryset(self): "exploits", Prefetch( "affecting_packages", - queryset=models.Package.objects.only( - "type", "namespace", "name", "version" - ), + queryset=models.Package.objects.only("type", "namespace", "name", "version"), ), Prefetch( "fixed_by_packages", - queryset=models.Package.objects.only( - "type", "namespace", "name", "version" - ), + queryset=models.Package.objects.only("type", "namespace", "name", "version"), ), ) ) - def get_context_data(self, **kwargs): """ Build context with preloaded QuerySets and minimize redundant queries. """ context = super().get_context_data(**kwargs) vulnerability = self.object - + # Pre-fetch and process data in Python instead of the template weaknesses_present_in_db = [ - weakness_object for weakness_object in vulnerability.weaknesses.all() + weakness_object + for weakness_object in vulnerability.weaknesses.all() if weakness_object.weakness ] @@ -211,6 +207,7 @@ def get_context_data(self, **kwargs): ) return context + class HomePage(View): template_name = "index.html" @@ -296,19 +293,19 @@ def get_queryset(self): """ Prefetch and optimize related data to minimize database hits. """ - return super().get_queryset().prefetch_related( - Prefetch( + return ( + super() + .get_queryset() + .prefetch_related( + Prefetch( "affecting_packages", - queryset=models.Package.objects.only( - "type", "namespace", "name", "version" - ), + queryset=models.Package.objects.only("type", "namespace", "name", "version"), ), Prefetch( "fixed_by_packages", - queryset=models.Package.objects.only( - "type", "namespace", "name", "version" - ), + queryset=models.Package.objects.only("type", "namespace", "name", "version"), ), + ) ) def get_context_data(self, **kwargs): @@ -322,9 +319,11 @@ def get_context_data(self, **kwargs): sorted_affected_packages, all_affected_fixed_by_matches, ) = vulnerability.aggregate_fixed_and_affected_packages() - context.update({ - "affected_packages": sorted_affected_packages, - "fixed_by_packages": sorted_fixed_by_packages, - "all_affected_fixed_by_matches": all_affected_fixed_by_matches, - }) + context.update( + { + "affected_packages": sorted_affected_packages, + "fixed_by_packages": sorted_fixed_by_packages, + "all_affected_fixed_by_matches": all_affected_fixed_by_matches, + } + ) return context diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index b481a119e..c6dd3da44 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -27,7 +27,8 @@ from vulnerabilities.views import HomePage from vulnerabilities.views import PackageDetails from vulnerabilities.views import PackageSearch -from vulnerabilities.views import VulnerabilityDetails, VulnerabilityPackagesDetails +from vulnerabilities.views import VulnerabilityDetails +from vulnerabilities.views import VulnerabilityPackagesDetails from vulnerabilities.views import VulnerabilitySearch from vulnerablecode.settings import DEBUG_TOOLBAR From 207c6dbd6c21221492fcecae7dcdfbfcd4de02fd Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 16 Jan 2025 19:41:30 +0530 Subject: [PATCH 12/25] Refactor Views Signed-off-by: Tushar Goel --- .../templates/vulnerability_details.html | 346 ++++++++++++++++-- .../vulnerability_package_details.html | 102 ++++++ 2 files changed, 425 insertions(+), 23 deletions(-) create mode 100644 vulnerabilities/templates/vulnerability_package_details.html diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index 612789d99..4005c663c 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -33,13 +33,6 @@ Essentials
    • -
  • - - - Affected/Fixed by packages ({{ affected_packages|length }}/{{ fixed_by_packages|length }}) - - -
  • @@ -54,13 +47,6 @@
    • -
  • - - - Severities vectors ({{ severity_vectors|length }}) - - -
    • {% if vulnerability.exploits %}
  • @@ -159,16 +145,47 @@ {{ vulnerability.risk_score }} + + Affected and Fixed Packages + + + Package Details + + + +
    + Weaknesses ({{ weaknesses|length }}) +
    +
    + + {% for weakness in weaknesses %} + + + + + + {% empty %} + + + + {% endfor %} +
    CWE-{{ weakness.cwe_id }} + + {{ weakness.name }} + +
    + There are no known CWE. +
    +
    - - - -
    - Affected/Fixed by packages ({{ affected_packages|length }}/{{ fixed_by_packages|length }}) -
    - @@ -197,8 +214,291 @@ - - +
    + + + + + + + + + {% for ref in references %} + + {% if ref.reference_id %} + + {% else %} + + {% endif %} + + {% if ref.reference_type %} + + {% else %} + + {% endif %} + + + + {% empty %} + + + + {% endfor %} +
    Reference id Reference type URL
    {{ ref.reference_id }}{{ ref.get_reference_type_display }}{{ ref.url }}
    + There are no known references. +
    +
    + +
    + {% for exploit in vulnerability.exploits.all %} + + + + + + + + {% if exploit.date_added %} + + + + + {% endif %} + {% if exploit.description %} + + + + + {% endif %} + {% if exploit.required_action %} + + + + + {% endif %} + {% if exploit.due_date %} + + + + + {% endif %} + {% if exploit.notes %} + + + + + {% endif %} + {% if exploit.known_ransomware_campaign_use is not None %} + + + + + {% endif %} + {% if exploit.source_date_published %} + + + + + {% endif %} + {% if exploit.exploit_type %} + + + + + {% endif %} + {% if exploit.platform %} + + + + + {% endif %} + {% if exploit.source_date_updated %} + + + + + {% endif %} + + {% if exploit.source_url %} + + + + + {% endif %} + +
    Data source {{ exploit.data_source }}
    + + Date added + + {{ exploit.date_added }}
    + + Description + + {{ exploit.description }}
    + + Required action + + {{ exploit.required_action }}
    + + Due date + + {{ exploit.due_date }}
    + + Note + + 
    {{ exploit.notes }}
    + + Ransomware campaign use + + {{ exploit.known_ransomware_campaign_use|yesno:"Known,Unknown" }}
    + + Source publication date + + {{ exploit.source_date_published }}
    + + Exploit type + + {{ exploit.exploit_type }}
    + + Platform + + {{ exploit.platform }}
    + + Source update date + + {{ exploit.source_date_updated }}
    + + Source URL + + {{ exploit.source_url }}
    + {% empty %} + + + No exploits are available. + + + {% endfor %} +
    + + +
    + {% for severity in severities %} + {% if severity.scoring_system == 'epss' %} +
    + Exploit Prediction Scoring System +
    + + + + + + + + + + + + + {% if severity.published_at %} + + + + + {% endif %} + + +
    + + Percentile + + {{ severity.scoring_elements }}
    + + EPSS score + + {{ severity.value }}
    + + Published at + + {{ severity.published_at }}
    +
    + {% endif %} + {% empty %} + + + There are no EPSS available. + + + {% endfor %} + + +
    + + + + + + + + + + + {% for log in history %} + + + + + + + + {% empty %} + + + + {% endfor %} +
    + + Date + + Actor + Action Source + VulnerableCode Version +
    {{ log.get_iso_time }}{{ log.actor_name }}{{ log.get_action_type_label }} {{log.source_url }} {{ log.software_version }}
    + There are no relevant records. +
    +
    diff --git a/vulnerabilities/templates/vulnerability_package_details.html b/vulnerabilities/templates/vulnerability_package_details.html new file mode 100644 index 000000000..1eb3cc4ce --- /dev/null +++ b/vulnerabilities/templates/vulnerability_package_details.html @@ -0,0 +1,102 @@ +{% extends "base.html" %} +{% load humanize %} +{% load widget_tweaks %} +{% load static %} +{% load show_cvss %} +{% load url_filters %} + +{% block title %} +VulnerableCode Vulnerability Details - {{ vulnerability.vulnerability_id }} +{% endblock %} + +{% block content %} + +{% if vulnerability %} +
    +
    +
    +
    + Vulnerability details: + + {{ vulnerability.vulnerability_id }} + +
    +
    + + +
    +
    + + + + + + + + + {% for package in affected_packages %} + + + + + {% empty %} + + + + {% endfor %} + +
    AffectedFixed by
    + {{ package.purl }} + + + {% for match in all_affected_fixed_by_matches %} + {% if match.affected_package == package %} + {% if match.matched_fixed_by_packages|length > 0 %} + {% for pkg in match.matched_fixed_by_packages %} + {{ pkg }} +
    + {% endfor %} + {% else %} + There are no reported fixed by versions. + {% endif %} + {% endif %} + {% endfor %} + +
    + This vulnerability is not known to affect any packages. +
    +
    +
    +
    +
    +{% endif %} + + + + + +{% endblock %} \ No newline at end of file From 7ccb36d98c70f44f027e9eccd2f4fd723b322e19 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 17 Jan 2025 17:32:38 +0530 Subject: [PATCH 13/25] Address review comments Signed-off-by: Tushar Goel --- .../templates/vulnerability_details.html | 103 ++++++++++++++++++ .../vulnerability_package_details.html | 22 +--- vulnerabilities/views.py | 23 +++- 3 files changed, 129 insertions(+), 19 deletions(-) diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index 4005c663c..e1ca6e4fd 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -47,6 +47,13 @@
    • +
  • + + + Severity details ({{ severity_vectors|length }}) + + +
    • {% if vulnerability.exploits %}
  • @@ -398,6 +405,102 @@ {% endfor %} + +
    + {% for severity_vector in severity_vectors %} + {% if severity_vector.version == '2.0' %} + Vector: {{ severity_vector.vectorString }} + + + + + + + + + + + + + + + + + + + +
    Exploitability (E)Access Vector (AV)Access Complexity (AC)Authentication (Au)Confidentiality Impact (C)Integrity Impact (I)Availability Impact (A)
    {{ severity_vector.exploitability|cvss_printer:"high,functional,unproven,proof_of_concept,not_defined" }}{{ severity_vector.accessVector|cvss_printer:"local,adjacent_network,network" }}{{ severity_vector.accessComplexity|cvss_printer:"high,medium,low" }}{{ severity_vector.authentication|cvss_printer:"multiple,single,none" }}{{ severity_vector.confidentialityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.integrityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.availabilityImpact|cvss_printer:"none,partial,complete" }}
    + {% elif severity_vector.version == '3.1' or severity_vector.version == '3.0'%} + Vector: {{ severity_vector.vectorString }} + + + + + + + + + + + + + + + + + + + + + +
    Attack Vector (AV)Attack Complexity (AC)Privileges Required (PR)User Interaction (UI)Scope (S)Confidentiality Impact (C)Integrity Impact (I)Availability Impact (A)
    {{ severity_vector.attackVector|cvss_printer:"network,adjacent_network,local,physical"}}{{ severity_vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.userInteraction|cvss_printer:"none,required"}}{{ severity_vector.scope|cvss_printer:"unchanged,changed" }}{{ severity_vector.confidentialityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.integrityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.availabilityImpact|cvss_printer:"high,low,none" }}
    + {% elif severity_vector.version == '4' %} + Vector: {{ severity_vector.vectorString }} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Attack Vector (AV)Attack Complexity (AC)Attack Requirements (AT)Privileges Required (PR)User Interaction (UI)Vulnerable System Impact Confidentiality (VC)Vulnerable System Impact Integrity (VI)Vulnerable System Impact Availability (VA)Subsequent System Impact Confidentiality (SC)Subsequent System Impact Integrity (SI)Subsequent System Impact Availability (SA)
    {{ severity_vector.attackVector|cvss_printer:"network,adjacent,local,physical"}}{{ severity_vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.attackRequirement|cvss_printer:"none,present" }}{{ severity_vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.userInteraction|cvss_printer:"none,passive,active"}}{{ severity_vector.vulnerableSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.vulnerableSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.vulnerableSystemImpactAvailability|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactAvailability|cvss_printer:"high,low,none" }}
    + {% elif severity_vector.version == 'ssvc' %} +
    + Vector: {{ severity_vector.vectorString }} +
    + {% endif %} + {% empty %} + + + There are no known vectors. + + + {% endfor %} +
    diff --git a/vulnerabilities/templates/vulnerability_package_details.html b/vulnerabilities/templates/vulnerability_package_details.html index 1eb3cc4ce..21fb52192 100644 --- a/vulnerabilities/templates/vulnerability_package_details.html +++ b/vulnerabilities/templates/vulnerability_package_details.html @@ -6,7 +6,7 @@ {% load url_filters %} {% block title %} -VulnerableCode Vulnerability Details - {{ vulnerability.vulnerability_id }} +VulnerableCode Vulnerability Package Details - {{ vulnerability.vulnerability_id }} {% endblock %} {% block content %} @@ -16,27 +16,14 @@
    - Vulnerability details: + Vulnerable and Fixing Package details for Vulnerability: {{ vulnerability.vulnerability_id }}
    - - -
    -
    - +
    +
    @@ -76,7 +63,6 @@
    Affected
    -
    {% endif %} diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index c4ae2bd04..9c07742de 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -192,12 +192,33 @@ def get_context_data(self, **kwargs): if weakness_object.weakness ] + severity_vectors = [] + severity_values = set() + for s in self.object.severities.all(): + if s.scoring_system == EPSS.identifier: + continue + + if s.scoring_elements and s.scoring_system in SCORING_SYSTEMS: + try: + vector_values = SCORING_SYSTEMS[s.scoring_system].get(s.scoring_elements) + severity_vectors.append(vector_values) + except ( + CVSS2MalformedError, + CVSS3MalformedError, + CVSS4MalformedError, + NotImplementedError, + ): + logging.error(f"CVSSMalformedError for {s.scoring_elements}") + + if s.value: + severity_values.add(s.value) + context.update( { "vulnerability": vulnerability, "vulnerability_search_form": VulnerabilitySearchForm(self.request.GET), "severities": list(vulnerability.severities.all()), - "severity_score_range": "", + "severity_vectors": severity_vectors, "references": list(vulnerability.references.all()), "aliases": list(vulnerability.aliases.all()), "weaknesses": weaknesses_present_in_db, From e12286a1bb0911e5d8873dc6faa1791a1d541707 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 20 Jan 2025 19:12:52 +0530 Subject: [PATCH 14/25] Address review comments Signed-off-by: Tushar Goel --- .../templates/vulnerability_details.html | 72 +++++++++---------- vulnerabilities/views.py | 6 +- 2 files changed, 37 insertions(+), 41 deletions(-) diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index e1ca6e4fd..1e0f917c0 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -408,8 +408,8 @@
    {% for severity_vector in severity_vectors %} - {% if severity_vector.version == '2.0' %} - Vector: {{ severity_vector.vectorString }} + {% if severity_vector.vector.version == '2.0' %} + Vector: {{ severity_vector.vector.vectorString }} Found at {{ severity_vector.origin }} @@ -421,17 +421,17 @@ - - - - - - - + + + + + + +
    Exploitability (E)Availability Impact (A)
    {{ severity_vector.exploitability|cvss_printer:"high,functional,unproven,proof_of_concept,not_defined" }}{{ severity_vector.accessVector|cvss_printer:"local,adjacent_network,network" }}{{ severity_vector.accessComplexity|cvss_printer:"high,medium,low" }}{{ severity_vector.authentication|cvss_printer:"multiple,single,none" }}{{ severity_vector.confidentialityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.integrityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.availabilityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.vector.exploitability|cvss_printer:"high,functional,unproven,proof_of_concept,not_defined" }}{{ severity_vector.vector.accessVector|cvss_printer:"local,adjacent_network,network" }}{{ severity_vector.vector.accessComplexity|cvss_printer:"high,medium,low" }}{{ severity_vector.vector.authentication|cvss_printer:"multiple,single,none" }}{{ severity_vector.vector.confidentialityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.vector.integrityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.vector.availabilityImpact|cvss_printer:"none,partial,complete" }}
    - {% elif severity_vector.version == '3.1' or severity_vector.version == '3.0'%} - Vector: {{ severity_vector.vectorString }} + {% elif severity_vector.vector.version == '3.1' or severity_vector.vector.version == '3.0'%} + Vector: {{ severity_vector.vector.vectorString }} Found at {{ severity_vector.origin }} @@ -444,18 +444,18 @@ - - - - - - - - + + + + + + + +
    Attack Vector (AV)Availability Impact (A)
    {{ severity_vector.attackVector|cvss_printer:"network,adjacent_network,local,physical"}}{{ severity_vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.userInteraction|cvss_printer:"none,required"}}{{ severity_vector.scope|cvss_printer:"unchanged,changed" }}{{ severity_vector.confidentialityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.integrityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.availabilityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.vector.attackVector|cvss_printer:"network,adjacent_network,local,physical"}}{{ severity_vector.vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.vector.userInteraction|cvss_printer:"none,required"}}{{ severity_vector.vector.scope|cvss_printer:"unchanged,changed" }}{{ severity_vector.vector.confidentialityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.vector.integrityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.vector.availabilityImpact|cvss_printer:"high,low,none" }}
    - {% elif severity_vector.version == '4' %} - Vector: {{ severity_vector.vectorString }} + {% elif severity_vector.vector.version == '4' %} + Vector: {{ severity_vector.vector.vectorString }} Found at {{ severity_vector.origin }} @@ -473,24 +473,24 @@ - - - - - - - - - - - - - + + + + + + + + + + + + +
    Attack Vector (AV)Subsequent System Impact Availability (SA)
    {{ severity_vector.attackVector|cvss_printer:"network,adjacent,local,physical"}}{{ severity_vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.attackRequirement|cvss_printer:"none,present" }}{{ severity_vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.userInteraction|cvss_printer:"none,passive,active"}}{{ severity_vector.vulnerableSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.vulnerableSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.vulnerableSystemImpactAvailability|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactAvailability|cvss_printer:"high,low,none" }}{{ severity_vector.vector.attackVector|cvss_printer:"network,adjacent,local,physical"}}{{ severity_vector.vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.vector.attackRequirement|cvss_printer:"none,present" }}{{ severity_vector.vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.vector.userInteraction|cvss_printer:"none,passive,active"}}{{ severity_vector.vector.vulnerableSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.vector.vulnerableSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.vector.vulnerableSystemImpactAvailability|cvss_printer:"high,low,none" }}{{ severity_vector.vector.subsequentSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.vector.subsequentSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.vector.subsequentSystemImpactAvailability|cvss_printer:"high,low,none" }}
    - {% elif severity_vector.version == 'ssvc' %} + {% elif severity_vector.vector.version == 'ssvc' %}
    - Vector: {{ severity_vector.vectorString }} + Vector: {{ severity_vector.vector.vectorString }} Found at {{ severity_vector.origin }}
    {% endif %} {% empty %} diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 9c07742de..571a737ec 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -193,7 +193,6 @@ def get_context_data(self, **kwargs): ] severity_vectors = [] - severity_values = set() for s in self.object.severities.all(): if s.scoring_system == EPSS.identifier: continue @@ -201,7 +200,7 @@ def get_context_data(self, **kwargs): if s.scoring_elements and s.scoring_system in SCORING_SYSTEMS: try: vector_values = SCORING_SYSTEMS[s.scoring_system].get(s.scoring_elements) - severity_vectors.append(vector_values) + severity_vectors.append({"vector": vector_values, "origin": s.url}) except ( CVSS2MalformedError, CVSS3MalformedError, @@ -210,9 +209,6 @@ def get_context_data(self, **kwargs): ): logging.error(f"CVSSMalformedError for {s.scoring_elements}") - if s.value: - severity_values.add(s.value) - context.update( { "vulnerability": vulnerability, From e391f1706b71a56cb71898c195d7f3a79da02ee4 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 20 Jan 2025 19:29:01 +0530 Subject: [PATCH 15/25] Rebase Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 571a737ec..a0fd89de0 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -23,6 +23,8 @@ from django.views import generic from django.views.generic.detail import DetailView from django.views.generic.list import ListView +from univers.version_range import RANGE_CLASS_BY_SCHEMES +from univers.version_range import AlpineLinuxVersionRange from vulnerabilities import models from vulnerabilities.forms import ApiUserCreationForm @@ -30,8 +32,8 @@ from vulnerabilities.forms import VulnerabilitySearchForm from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS -from vulnerablecode import __version__ as VULNERABLECODE_VERSION from vulnerabilities.utils import get_purl_version_class +from vulnerablecode import __version__ as VULNERABLECODE_VERSION from vulnerablecode.settings import env PAGE_SIZE = 20 From 481e04c526e8739e009e531677921b4e243fbefe Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 20 Jan 2025 19:35:51 +0530 Subject: [PATCH 16/25] Fix views Signed-off-by: Tushar Goel --- vulnerabilities/templates/vulnerability_details.html | 1 + 1 file changed, 1 insertion(+) diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index 1e0f917c0..d9d093a45 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -556,6 +556,7 @@ {% endfor %}
    +
    From 403e1dbeca03ef8e49b8401c4579138dabb24b62 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 21 Jan 2025 13:33:07 +0530 Subject: [PATCH 17/25] Fix templates Signed-off-by: Tushar Goel --- .../templates/vulnerability_details.html | 188 +++++++++--------- 1 file changed, 94 insertions(+), 94 deletions(-) diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index d9d093a45..1d210b793 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -503,106 +503,106 @@ -
    {% for severity in severities %} - {% if severity.scoring_system == 'epss' %} -
    - Exploit Prediction Scoring System -
    -
    - - - - - - - - - - - - {% if severity.published_at %} - - - - - {% endif %} - - -
    - - Percentile - - {{ severity.scoring_elements }}
    - - EPSS score - - {{ severity.value }}
    - - Published at - - {{ severity.published_at }}
    + {% if severity.scoring_system == 'epss' %} +
    +
    + Exploit Prediction Scoring System
    - {% endif %} - {% empty %} - - - There are no EPSS available. - - - {% endfor %} -
    -
    + + + + + + -
    -
    + + Percentile + + {{ severity.scoring_elements }}
    - - - - - - - + + + + {% if severity.published_at %} + + - - {% for log in history %} - - - - - - - - {% empty %} - - - - {% endfor %} + data-tooltip="When was the time we fetched epss"> + Published at + + + + + {% endif %} + +
    - - Date - - Actor - Action Source + + + EPSS score + + {{ severity.value }}
    + - VulnerableCode Version - -
    {{ log.get_iso_time }}{{ log.actor_name }}{{ log.get_action_type_label }} {{log.source_url }} {{ log.software_version }}
    - There are no relevant records. -
    {{ severity.published_at }}
    + {% endif %} + {% empty %} +
    + + + There are no EPSS available. + + +
    + {% endfor %} + +
    + + + + + + + + + + + {% for log in history %} + + + + + + + + {% empty %} + + + + {% endfor %} +
    + + Date + + Actor + Action Source + VulnerableCode Version +
    {{ log.get_iso_time }}{{ log.actor_name }}{{ log.get_action_type_label }} {{log.source_url }} {{ log.software_version }}
    + There are no relevant records. +
    +
    From ea1fbf47288c8504e5075f48b9a6505f887cc0c3 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 21 Jan 2025 13:38:06 +0530 Subject: [PATCH 18/25] Fix views Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index a0fd89de0..e6249e952 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -194,22 +194,20 @@ def get_context_data(self, **kwargs): if weakness_object.weakness ] + valid_severities = self.object.severities.exclude(scoring_system=EPSS.identifier).filter( + scoring_elements__isnull=False, + scoring_system__in=SCORING_SYSTEMS.keys() + ) + severity_vectors = [] - for s in self.object.severities.all(): - if s.scoring_system == EPSS.identifier: - continue - - if s.scoring_elements and s.scoring_system in SCORING_SYSTEMS: - try: - vector_values = SCORING_SYSTEMS[s.scoring_system].get(s.scoring_elements) - severity_vectors.append({"vector": vector_values, "origin": s.url}) - except ( - CVSS2MalformedError, - CVSS3MalformedError, - CVSS4MalformedError, - NotImplementedError, - ): - logging.error(f"CVSSMalformedError for {s.scoring_elements}") + + for severity in valid_severities: + try: + vector_values = SCORING_SYSTEMS[severity.scoring_system].get(severity.scoring_elements) + if vector_values: + severity_vectors.append({"vector": vector_values, "origin": severity.url}) + except (CVSS2MalformedError, CVSS3MalformedError, CVSS4MalformedError, NotImplementedError): + logging.error(f"CVSSMalformedError for {severity.scoring_elements}") context.update( { From ba20954674643f16deeb2dfbced29726c54e1a00 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 21 Jan 2025 18:03:30 +0530 Subject: [PATCH 19/25] Adress review comments Signed-off-by: Tushar Goel --- vulnerabilities/importer.py | 2 +- vulnerabilities/models.py | 43 +++++++++++++- .../templates/vulnerability_details.html | 56 +++++++----------- vulnerabilities/views.py | 58 ++++++++++++++----- 4 files changed, 105 insertions(+), 54 deletions(-) diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index c5a5c5743..1669172a1 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -110,7 +110,7 @@ def to_dict(self): @classmethod def from_dict(cls, ref: dict): return cls( - reference_id=ref["reference_id"], + reference_type=ref.get("reference_type") or "", reference_type=ref["reference_type"], url=ref["url"], severities=[ diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 7a98667d0..3e3d97226 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -7,9 +7,11 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import csv import hashlib import json import logging +import xml.etree.ElementTree as ET from contextlib import suppress from functools import cached_property from itertools import groupby @@ -20,6 +22,8 @@ from cvss.exceptions import CVSS3MalformedError from cvss.exceptions import CVSS4MalformedError from cwe2.database import Database +from cwe2.mappings import xml_database_path +from cwe2.weakness import Weakness as DBWeakness from django.contrib.auth import get_user_model from django.contrib.auth.models import UserManager from django.core import exceptions @@ -46,7 +50,6 @@ from univers.version_range import AlpineLinuxVersionRange from univers.versions import Version -from aboutcode import hashid from vulnerabilities import utils from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS @@ -467,6 +470,32 @@ def get_severity_vectors_and_values(self): return severity_vectors, severity_values +def get_cwes(self): + """Yield CWE Weakness objects""" + for cwe_category in self.cwe_files: + cwe_category.seek(0) + reader = csv.DictReader(cwe_category) + for row in reader: + yield DBWeakness(*list(row.values())[0:-1]) + tree = ET.parse(xml_database_path) + root = tree.getroot() + for tag_num in [1, 2]: # Categories , Views + tag = root[tag_num] + for child in tag: + yield DBWeakness( + *[ + child.attrib["ID"], + child.attrib.get("Name"), + None, + child.attrib.get("Status"), + child[0].text, + ] + ) + + +Database.get_cwes = get_cwes + + class Weakness(models.Model): """ A Common Weakness Enumeration model @@ -474,7 +503,15 @@ class Weakness(models.Model): cwe_id = models.IntegerField(help_text="CWE id") vulnerabilities = models.ManyToManyField(Vulnerability, related_name="weaknesses") - db = Database() + + cwe_by_id = {} + + def get_cwe(self, cwe_id): + if not self.cwe_by_id: + db = Database() + for weakness in db.get_cwes(): + self.cwe_by_id[str(weakness.cwe_id)] = weakness + return self.cwe_by_id[cwe_id] @property def cwe(self): @@ -486,7 +523,7 @@ def weakness(self): Return a queryset of Weakness for this vulnerability. """ try: - weakness = self.db.get(self.cwe_id) + weakness = self.get_cwe(str(self.cwe_id)) return weakness except Exception as e: logger.warning(f"Could not find CWE {self.cwe_id}: {e}") diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index 1d210b793..7001c8f3b 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -503,60 +503,48 @@ - {% for severity in severities %} - {% if severity.scoring_system == 'epss' %} -
    -
    - Exploit Prediction Scoring System -
    - +
    + {% if epss_data %} +
    + Exploit Prediction Scoring System (EPSS) +
    +
    - + - - + - - {% if severity.published_at %} + {% if epss_data.published_at %} - + - {% endif %} - + {% endif %} -
    - Percentile + data-tooltip="The percentile of the current score, the proportion of all scored vulnerabilities with the same or a lower EPSS score"> + Percentile {{ severity.scoring_elements }}{{ epss_data.percentile }}
    - EPSS score + data-tooltip="The EPSS score represents the probability [0-1] of exploitation in the wild in the next 30 days."> + EPSS Score {{ severity.value }}{{ epss_data.score }}
    - - Published at + + Published At {{ severity.published_at }}{{ epss_data.published_at }}
    + + {% else %} +

    No EPSS data available for this vulnerability.

    + {% endif %}
    - {% endif %} - {% empty %} -
    - - - There are no EPSS available. - - -
    - {% endfor %}
    diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index e6249e952..4346ce2e0 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -156,26 +156,36 @@ class VulnerabilityDetails(DetailView): slug_field = "vulnerability_id" def get_queryset(self): - """ - Prefetch and optimize related data to minimize database hits. - """ return ( super() .get_queryset() .select_related() .prefetch_related( - "references", - "aliases", - "weaknesses", - "severities", - "exploits", Prefetch( - "affecting_packages", - queryset=models.Package.objects.only("type", "namespace", "name", "version"), + "references", + queryset=models.VulnerabilityReference.objects.only( + "reference_id", "reference_type", "url" + ), ), Prefetch( - "fixed_by_packages", - queryset=models.Package.objects.only("type", "namespace", "name", "version"), + "aliases", + queryset=models.Alias.objects.only("alias"), + ), + Prefetch( + "weaknesses", + queryset=models.Weakness.objects.only("cwe_id"), + ), + Prefetch( + "severities", + queryset=models.VulnerabilitySeverity.objects.only( + "scoring_system", "value", "url", "scoring_elements", "published_at" + ), + ), + Prefetch( + "exploits", + queryset=models.Exploit.objects.only( + "data_source", "description", "required_action", "due_date", "notes" + ), ), ) ) @@ -195,20 +205,35 @@ def get_context_data(self, **kwargs): ] valid_severities = self.object.severities.exclude(scoring_system=EPSS.identifier).filter( - scoring_elements__isnull=False, - scoring_system__in=SCORING_SYSTEMS.keys() + scoring_elements__isnull=False, scoring_system__in=SCORING_SYSTEMS.keys() ) severity_vectors = [] for severity in valid_severities: try: - vector_values = SCORING_SYSTEMS[severity.scoring_system].get(severity.scoring_elements) + vector_values = SCORING_SYSTEMS[severity.scoring_system].get( + severity.scoring_elements + ) if vector_values: severity_vectors.append({"vector": vector_values, "origin": severity.url}) - except (CVSS2MalformedError, CVSS3MalformedError, CVSS4MalformedError, NotImplementedError): + except ( + CVSS2MalformedError, + CVSS3MalformedError, + CVSS4MalformedError, + NotImplementedError, + ): logging.error(f"CVSSMalformedError for {severity.scoring_elements}") + epss_severity = vulnerability.severities.filter(scoring_system="epss").first() + epss_data = None + if epss_severity: + epss_data = { + "percentile": epss_severity.scoring_elements, + "score": epss_severity.value, + "published_at": epss_severity.published_at, + } + context.update( { "vulnerability": vulnerability, @@ -220,6 +245,7 @@ def get_context_data(self, **kwargs): "weaknesses": weaknesses_present_in_db, "status": vulnerability.get_status_label, "history": vulnerability.history, + "epss_data": epss_data, } ) return context From 7bfa0bd74df756552e0f28e2c0e39f75d574ce17 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 21 Jan 2025 18:05:59 +0530 Subject: [PATCH 20/25] Adress review comments Signed-off-by: Tushar Goel --- vulnerabilities/utils.py | 2 +- vulnerabilities/views.py | 24 ------------------------ 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 32cfcbc02..d9a3c7e04 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -540,7 +540,7 @@ def normalize_purl(purl: Union[PackageURL, str]): def get_purl_version_class(purl): - RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange + RANGE_CLASS_BY_SCHEMES["apk"] = AlpineLinuxVersionRange purl_version_class = None check_version_class = RANGE_CLASS_BY_SCHEMES.get(purl.type, None) if check_version_class: diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 4346ce2e0..a2df48634 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -32,36 +32,12 @@ from vulnerabilities.forms import VulnerabilitySearchForm from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS -from vulnerabilities.utils import get_purl_version_class from vulnerablecode import __version__ as VULNERABLECODE_VERSION from vulnerablecode.settings import env PAGE_SIZE = 20 -def purl_sort_key(purl: models.Package): - """ - Return a sort key for the built-in sorted() function when sorting a list - of Package objects. If the Package ``type`` is supported by univers, apply - the univers version class to the Package ``version``, and otherwise use the - ``version`` attribute as is. - """ - purl_version_class = get_purl_version_class(purl) - purl_sort_version = purl.version - if purl_version_class: - purl_sort_version = purl_version_class(purl.version) - return (purl.type, purl.namespace, purl.name, purl_sort_version, purl.qualifiers, purl.subpath) - - -def get_purl_version_class(purl: models.Package): - RANGE_CLASS_BY_SCHEMES["apk"] = AlpineLinuxVersionRange - purl_version_class = None - check_version_class = RANGE_CLASS_BY_SCHEMES.get(purl.type, None) - if check_version_class: - purl_version_class = check_version_class.version_class - return purl_version_class - - class PackageSearch(ListView): model = models.Package template_name = "packages.html" From d189d155a6097dc418e27708305d52d12fffa72d Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 21 Jan 2025 18:06:36 +0530 Subject: [PATCH 21/25] Adress review comments Signed-off-by: Tushar Goel --- vulnerabilities/importer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index 1669172a1..701c43fe2 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -111,7 +111,6 @@ def to_dict(self): def from_dict(cls, ref: dict): return cls( reference_type=ref.get("reference_type") or "", - reference_type=ref["reference_type"], url=ref["url"], severities=[ VulnerabilitySeverity.from_dict(severity) for severity in ref["severities"] From 0a43c2823d85ef953f67f7e831f59e2b9f183d42 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 21 Jan 2025 18:10:05 +0530 Subject: [PATCH 22/25] Adress review comments Signed-off-by: Tushar Goel --- vulnerabilities/importer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index 701c43fe2..2a296c680 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -110,6 +110,7 @@ def to_dict(self): @classmethod def from_dict(cls, ref: dict): return cls( + reference_id=ref["reference_id"], reference_type=ref.get("reference_type") or "", url=ref["url"], severities=[ From 0788e666effcffd6590b7ed9ecc9cf91ed267399 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 21 Jan 2025 18:21:51 +0530 Subject: [PATCH 23/25] Fix tests Signed-off-by: Tushar Goel --- .../test_data/package_sort/sorted_purls.txt | 16 ++++++++-------- vulnerabilities/tests/test_view.py | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/vulnerabilities/tests/test_data/package_sort/sorted_purls.txt b/vulnerabilities/tests/test_data/package_sort/sorted_purls.txt index 886119bfd..7faf4c22a 100644 --- a/vulnerabilities/tests/test_data/package_sort/sorted_purls.txt +++ b/vulnerabilities/tests/test_data/package_sort/sorted_purls.txt @@ -21,10 +21,10 @@ pkg:conan/capnproto@0.15.2 pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u7?distro=stretch pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u10?distro=stretch pkg:deb/debian/jackson-databind@2.9.8-3%2Bdeb10u4?distro=sid -pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1?distro=sid -pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=sid +pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=stretch +pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=sid pkg:deb/debian/jackson-databind@2.14.0-1?distro=sid pkg:deb/ubuntu/dpkg@1.13.11ubuntu7~proposed pkg:deb/ubuntu/dpkg@1.13.11ubuntu7.2 @@ -94,10 +94,10 @@ pkg:pypi/jinja2@2.1.1 pkg:pypi/jinja2@2.2 pkg:pypi/jinja2@2.2.1 pkg:pypi/jinja2@2.10 -pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=11 -pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=12 -pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=13 -pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=2 -pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=5 -pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=7 pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=9 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=7 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=5 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=2 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=13 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=12 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=11 diff --git a/vulnerabilities/tests/test_view.py b/vulnerabilities/tests/test_view.py index 98a555294..3b32ee31c 100644 --- a/vulnerabilities/tests/test_view.py +++ b/vulnerabilities/tests/test_view.py @@ -23,10 +23,9 @@ from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.templatetags.url_filters import url_quote_filter +from vulnerabilities.utils import get_purl_version_class from vulnerabilities.views import PackageDetails from vulnerabilities.views import PackageSearch -from vulnerabilities.views import get_purl_version_class -from vulnerabilities.views import purl_sort_key BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DIR = os.path.join(BASE_DIR, "test_data/package_sort") @@ -202,12 +201,13 @@ def setUp(self): for pkg in input_purls: real_purl = PackageURL.from_string(pkg) attrs = {k: v for k, v in real_purl.to_dict().items() if v} - Package.objects.create(**attrs) + pkg = Package.objects.create(**attrs) + pkg.calculate_version_rank def test_sorted_queryset(self): qs_all = Package.objects.all() pkgs_qs_all = list(qs_all) - sorted_pkgs_qs_all = sorted(pkgs_qs_all, key=purl_sort_key) + sorted_pkgs_qs_all = pkgs_qs_all pkg_package_urls = [obj.package_url for obj in sorted_pkgs_qs_all] sorted_purls = os.path.join(TEST_DIR, "sorted_purls.txt") From 1ceb80c0ffc29c9e7a8acd0312aa87769d1f71b6 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 21 Jan 2025 18:25:51 +0530 Subject: [PATCH 24/25] Fix tests Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 3e3d97226..f2c32e558 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -387,8 +387,14 @@ def aggregate_fixed_and_affected_packages(self): "type", "namespace", "name", "qualifiers", "subpath" ) + if sorted_fixed_by_packages: + sorted_fixed_by_packages.first().calculate_version_rank + sorted_affected_packages = self.affected_packages.all() + if sorted_affected_packages: + sorted_affected_packages.first().calculate_version_rank + grouped_fixed_by_packages = { key: list(group) for key, group in groupby( From 7777bdb06ad5ee176dd00d9884dde834583fb51a Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 21 Jan 2025 18:32:35 +0530 Subject: [PATCH 25/25] Fix tests Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index f2c32e558..9b6df7c13 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -381,7 +381,7 @@ def get_related_purls(self): return [p.package_url for p in self.packages.distinct().all()] def aggregate_fixed_and_affected_packages(self): - from vulnerabilities.views import get_purl_version_class + from vulnerabilities.utils import get_purl_version_class sorted_fixed_by_packages = self.fixed_by_packages.filter(is_ghost=False).order_by( "type", "namespace", "name", "qualifiers", "subpath"