From 327969e2fb443f55700b4a1facb5ac042a45815a Mon Sep 17 00:00:00 2001 From: John Andersen Date: Mon, 17 Jun 2024 15:50:57 +0000 Subject: [PATCH] feat: documentation on out of tree parsers Signed-off-by: John --- cve_bin_tool/cvedb.py | 9 +++ cve_bin_tool/parsers/__init__.py | 1 + cve_bin_tool/parsers/env.py | 131 +++++++++++++++++++++++++++++++ test/test_parsers.py | 62 ++++++++++++++- 4 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 cve_bin_tool/parsers/env.py diff --git a/cve_bin_tool/cvedb.py b/cve_bin_tool/cvedb.py index 1451eaa996..99a29e4cca 100644 --- a/cve_bin_tool/cvedb.py +++ b/cve_bin_tool/cvedb.py @@ -8,6 +8,7 @@ import asyncio import datetime +import contextlib import json import logging import shutil @@ -1193,3 +1194,11 @@ def fetch_from_mirror(self, mirror, pubkey, ignore_signature, log_signature_erro else: self.clear_cached_data() return -1 + + @contextlib.contextmanager + def with_cursor(self): + cursor = self.db_open_and_get_cursor() + try: + yield cursor + finally: + self.db_close() diff --git a/cve_bin_tool/parsers/__init__.py b/cve_bin_tool/parsers/__init__.py index 1e39cc33f5..7657cdfb2d 100644 --- a/cve_bin_tool/parsers/__init__.py +++ b/cve_bin_tool/parsers/__init__.py @@ -25,6 +25,7 @@ "php", "perl", "dart", + "env", ] diff --git a/cve_bin_tool/parsers/env.py b/cve_bin_tool/parsers/env.py new file mode 100644 index 0000000000..d8c5675825 --- /dev/null +++ b/cve_bin_tool/parsers/env.py @@ -0,0 +1,131 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: GPL-3.0-or-later + +import json +import re +import pathlib +import subprocess +import contextlib +import dataclasses +from re import MULTILINE, compile, search + +from packaging.version import parse as parse_version + +from cve_bin_tool.parsers import Parser +from cve_bin_tool.strings import parse_strings +from cve_bin_tool.util import ProductInfo, ScanInfo + +import snoop + + +@dataclasses.dataclass +class EnvNamespaceConfig: + ad_hoc_cve_id: str + vendor: str + product: str + version: str + + +@dataclasses.dataclass +class EnvConfig: + namespaces: dict[str, EnvNamespaceConfig] + + +class EnvParser(Parser): + """ + Parser for Python requirements files. + This parser is designed to parse Python requirements files (usually named + requirements.txt) and generate PURLs (Package URLs) for the listed packages. + """ + + PARSER_MATCH_FILENAMES = [ + ".env", + ] + + def __init__(self, cve_db, logger): + """Initialize the python requirements file parser.""" + super().__init__(cve_db, logger) + self.purl_pkg_type = "ad-hoc" + + def generate_purl(self, product, vendor, qualifier={}, subpath=None): + """Generates PURL after normalizing all components.""" + product = re.sub(r"[^a-zA-Z0-9._-]", "", product).lower() + + if not product: + return None + + purl = super().generate_purl( + product, + vendor, + qualifier, + subpath, + ) + + return purl + + @staticmethod + @snoop + def parse_file_contents(contents): + lines = list( + [ + line + for line in contents.replace("\r\n", "\n").split("\n") + if line.strip() and line.startswith("CVE_BIN_TOOL_") + ] + ) + namespaces = {} + for i, line in enumerate(lines): + key, value = line.split("=", maxsplit=1) + namespace, key = key[len("CVE_BIN_TOOL_") :].split("_", maxsplit=1) + if value.startswith('"'): + value = value[1:] + if value.endswith('"'): + value = value[:-1] + namespaces.setdefault(namespace, {}) + namespaces[namespace][key.lower()] = value + for namespace, config in namespaces.items(): + namespaces[namespace] = EnvNamespaceConfig(**config) + return EnvConfig(namespaces=namespaces) + + @snoop + def run_checker(self, filename): + """ + Parse the .env file and yield ScanInfo objects for the listed packages. + Args: + filename (str): The path to the .env file. + Yields: + str: ScanInfo objects for the packages listed in the file. + """ + self.filename = filename + contents = pathlib.Path(self.filename).read_text() + + env_config = self.parse_file_contents(contents) + + snoop.pp(self.cve_db) + + # TODO Create SCITT_URN_FOR_MANIFEST_OF_EXECUTED_WORKFLOW_WITH_SARIF_OUTPUTS_DEREFERENCEABLE + # by making a request to the poligy engine and getting it's workflow + # manifest as output and deriving from that or extend it to return that. + data_source = "SCITT_URN_FOR_MANIFEST_OF_EXECUTED_WORKFLOW_WITH_SARIF_OUTPUTS_DEREFERENCEABLE" + affected_data = [ + { + "cve_id": cve.ad_hoc_cve_id, + "vendor": cve.vendor, + "product": cve.product, + "version": cve.version, + "versionStartIncluding": "", + # "versionStartIncluding": cve.version, + "versionStartExcluding": "", + "versionEndIncluding": "", + # "versionEndIncluding": cve.version, + "versionEndExcluding": "", + } + for _namespace, cve in env_config.namespaces.items() + ] + with self.cve_db.with_cursor() as cursor: + self.cve_db.populate_affected(affected_data, cursor, data_source) + + for _namespace, cve in env_config.namespaces.items(): + yield from self.find_vendor(cve.product, cve.version) + + # TODO VEX attached via linked data to ad-hoc CVE-ID diff --git a/test/test_parsers.py b/test/test_parsers.py index 09e6e88c98..44e668b4d2 100644 --- a/test/test_parsers.py +++ b/test/test_parsers.py @@ -1,6 +1,18 @@ -import pytest +import os +import sys +import shutil +import atexit +import pathlib +import textwrap +import tempfile import unittest +import contextlib + +import pytest +from cve_bin_tool.cvedb import CVEDB +from cve_bin_tool.util import ProductInfo, ScanInfo +from cve_bin_tool.log import LOGGER from cve_bin_tool.parsers.parse import valid_files as actual_valid_files from cve_bin_tool.parsers.dart import DartParser from cve_bin_tool.parsers.go import GoParser @@ -13,7 +25,18 @@ from cve_bin_tool.parsers.ruby import RubyParser from cve_bin_tool.parsers.rust import RustParser from cve_bin_tool.parsers.swift import SwiftParser +from cve_bin_tool.parsers.env import EnvParser + +import snoop + +cve_db = CVEDB() +logger = LOGGER.getChild(__name__) +stack = contextlib.ExitStack().__enter__() +tmpdir = stack.enter_context( + tempfile.TemporaryDirectory(prefix="cve-bin-tool-TEST_ENV") +) +atexit.register(lambda: stack.__exit__(None, None, None)) EXPECTED_VALID_FILES = { "pom.xml": [JavaParser], @@ -29,10 +52,22 @@ "composer.lock": [PhpParser], "cpanfile": [PerlParser], "pubspec.lock": [DartParser], + ".env": [EnvParser], } +PARSER_ENV_TEST_0001_ENV_CONTENTS = textwrap.dedent( + """ + CVE_BIN_TOOL_0_PRODUCT="myproduct" + CVE_BIN_TOOL_0_VENDOR="myvendor" + CVE_BIN_TOOL_0_VERSION="v0.0.0.dev-15abff2d529396937e18c657ecee1ed224842000" + CVE_BIN_TOOL_0_AD_HOC_CVE_ID="CVE-0001-15004435-aa84-43ff-9c26-f703a26069f8" + """ +) + class TestParsers: + maxDiff = None + @pytest.mark.asyncio async def test_parser_match_filenames_results_in_correct_valid_files(self): unittest.TestCase().assertDictEqual( @@ -40,3 +75,28 @@ async def test_parser_match_filenames_results_in_correct_valid_files(self): actual_valid_files, "Expected registered file types not the same as loaded file types, second dict is actual file types loaded, first is expected", ) + + @pytest.mark.asyncio + async def test_parser_env_test_0001(self): + with snoop(): + file_path = pathlib.Path(tmpdir, ".env") + file_path.write_text(PARSER_ENV_TEST_0001_ENV_CONTENTS) + env_parser = EnvParser(cve_db, logger) + results = list(env_parser.run_checker(file_path)) + unittest.TestCase().assertListEqual( + results, + [ + ScanInfo( + product_info=ProductInfo( + vendor="myvendor", + product="myproduct", + version="v0.0.0.dev-15abff2d529396937e18c657ecee1ed224842000", + # TODO location? + location="/usr/local/bin/product", + # TODO purl + purl=None, + ), + file_path=file_path, + ) + ], + )