feat: documentation on out of tree parsers

Signed-off-by: John Andersen <[email protected]>

cve_bin_tool/cvedb.py

@@ -7,6 +7,7 @@
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import datetime
 import json
 import logging
@@ -1193,3 +1194,11 @@ def fetch_from_mirror(self, mirror, pubkey, ignore_signature, log_signature_erro
         else:
             self.clear_cached_data()
             return -1
+
+    @contextlib.contextmanager
+    def with_cursor(self):
+        cursor = self.db_open_and_get_cursor()
+        try:
+            yield cursor
+        finally:
+            self.db_close()

cve_bin_tool/parsers/__init__.py

@@ -25,6 +25,7 @@
     "php",
     "perl",
     "dart",
+    "env",
 ]
 
 

cve_bin_tool/parsers/env.py

@@ -0,0 +1,134 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+
+import dataclasses
+import pathlib
+import re
+
+from packageurl import PackageURL
+
+from cve_bin_tool.parsers import Parser
+from cve_bin_tool.util import ProductInfo, ScanInfo
+
+
+@dataclasses.dataclass
+class EnvNamespaceConfig:
+    ad_hoc_cve_id: str
+    vendor: str
+    product: str
+    version: str
+    location: str = "/usr/local/bin/product"
+
+
+@dataclasses.dataclass
+class EnvConfig:
+    namespaces: dict[str, EnvNamespaceConfig]
+
+
+class EnvParser(Parser):
+    """
+    Parser for Python requirements files.
+    This parser is designed to parse Python requirements files (usually named
+    requirements.txt) and generate PURLs (Package URLs) for the listed packages.
+    """
+
+    PARSER_MATCH_FILENAMES = [
+        ".env",
+    ]
+
+    @staticmethod
+    def parse_file_contents(contents):
+        lines = list(
+            [
+                line
+                for line in contents.replace("\r\n", "\n").split("\n")
+                if line.strip() and line.startswith("CVE_BIN_TOOL_")
+            ]
+        )
+        namespaces = {}
+        for i, line in enumerate(lines):
+            key, value = line.split("=", maxsplit=1)
+            namespace, key = key[len("CVE_BIN_TOOL_") :].split("_", maxsplit=1)
+            if value.startswith('"'):
+                value = value[1:]
+            if value.endswith('"'):
+                value = value[:-1]
+            namespaces.setdefault(namespace, {})
+            namespaces[namespace][key.lower()] = value
+        for namespace, config in namespaces.items():
+            namespaces[namespace] = EnvNamespaceConfig(**config)
+        return EnvConfig(namespaces=namespaces)
+
+    def run_checker(self, filename):
+        """
+        Parse the .env file and yield ScanInfo objects for the listed packages.
+        Args:
+            filename (str): The path to the .env file.
+        Yields:
+            str: ScanInfo objects for the packages listed in the file.
+        """
+        self.filename = filename
+        contents = pathlib.Path(self.filename).read_text()
+
+        env_config = self.parse_file_contents(contents)
+
+        data_source = "environment"
+        affected_data = [
+            {
+                "cve_id": cve.ad_hoc_cve_id,
+                "vendor": cve.vendor,
+                "product": cve.product,
+                # TODO Version MUST be unique to this bug!
+                "version": cve.version,
+                "versionStartIncluding": "",
+                # "versionStartIncluding": cve.version,
+                "versionStartExcluding": "",
+                "versionEndIncluding": "",
+                # "versionEndIncluding": cve.version,
+                "versionEndExcluding": "",
+            }
+            for _namespace, cve in env_config.namespaces.items()
+        ]
+        severity_data = [
+            {
+                "ID": cve.ad_hoc_cve_id,
+                # TODO severity
+                "severity": "LOW",
+                # TODO description
+                "description": "TODO",
+                # TODO score
+                "score": 0,
+                # TODO CVSS_version
+                "CVSS_version": 3,
+                # TODO CVSS_vector
+                "CVSS_vector": "",
+                "last_modified": "",
+            }
+            for _namespace, cve in env_config.namespaces.items()
+        ]
+
+        with self.cve_db.with_cursor() as cursor:
+            self.cve_db.populate_cve_metrics(severity_data, cursor)
+            self.cve_db.populate_severity(severity_data, cursor, data_source)
+            self.cve_db.populate_affected(affected_data, cursor, data_source)
+
+        for _namespace, cve in env_config.namespaces.items():
+            yield ScanInfo(
+                ProductInfo(
+                    cve.vendor,
+                    cve.product,
+                    cve.version,
+                    cve.location,
+                    PackageURL(
+                        type="ad-hoc",
+                        namespace=cve.vendor,
+                        name=re.sub(r"[^a-zA-Z0-9._-]", "", cve.product).lower(),
+                        version=cve.version,
+                        qualifiers={},
+                        subpath=None,
+                    ),
+                ),
+                pathlib.Path(filename).resolve(),
+            )

doc/PARSERS.rst

@@ -0,0 +1,139 @@
+Adding a new parser to cve-bin-tool
+===================================
+
+Overview
+--------
+
+Parsers enhance ``cve-bin-tool`` by helping it discover vulnerabilities for
+different file types and manifest formats.
+
+Parsers
+-------
+
+The following parsers have been added to the project:
+
+- **DartParser**
+- **GoParser**
+- **JavaParser**
+- **JavascriptParser**
+- **PerlParser**
+- **PhpParser**
+- **PythonParser**
+- **PythonRequirementsParser**
+- **RParser**
+- **RubyParser**
+- **RustParser**
+- **SwiftParser**
+- **BanditParser**
+
+Usage
+-----
+
+To utilize these parsers, ensure that your project includes the following imports:
+
+.. code-block:: python
+
+    from cve_bin_tool.parsers.dart import DartParser
+    from cve_bin_tool.parsers.go import GoParser
+    from cve_bin_tool.parsers.java import JavaParser
+    from cve_bin_tool.parsers.javascript import JavascriptParser
+    from cve_bin_tool.parsers.perl import PerlParser
+    from cve_bin_tool.parsers.php import PhpParser
+    from cve_bin_tool.parsers.python import PythonParser, PythonRequirementsParser
+    from cve_bin_tool.parsers.r import RParser
+    from cve_bin_tool.parsers.ruby import RubyParser
+    from cve_bin_tool.parsers.rust import RustParser
+    from cve_bin_tool.parsers.swift import SwiftParser
+    from cve_bin_tool.parsers.bandit import BanditParser
+
+Setting Up a New Package and Entry Point
+----------------------------------------
+
+To implement a new parser plugin, such as a Bandit parser, follow these steps:
+
+1. Create the Parser Class
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+First, create the parser class. This class should be located in the appropriate directory within your project. For example, you might place it in ``cve_bin_tool_parser_env/env.py``.
+
+.. literalinclude:: /../cve_bin_tool/parsers/env.py
+
+2. Set Up ``setup.py``
+^^^^^^^^^^^^^^^^^^^^^^
+
+Next, configure the ``setup.py`` file boilerplate.
+
+.. literalinclude:: /../example/oot-parser/setup.py
+
+3. Set Up ``setup.cfg``
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Next, configure the ``setup.cfg`` file to include your new parser as an entry point. This allows the parser to be dynamically discovered and used by the project.
+
+.. literalinclude:: /../example/oot-parser/setup.cfg
+
+4. Create ``entry_points.txt``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You may also need to configure an ``entry_points.txt`` file if your project uses it to manage entry points.
+
+.. literalinclude:: /../example/oot-parser/entry_points.txt
+
+5. Install your plugin
+^^^^^^^^^^^^^^^^^^^^^^
+
+You need to activate your virtualenv before installing if you set one up.
+
+.. code-block:: console
+
+    $ python -m pip install -e .
+
+6. Populate the to-be-parsed file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In this example we implemented the ``EnvParser`` which is the format of
+``/etc/environment``.
+
+.. literalinclude:: /../test/parser_env_test_0001.env
+
+7. Run ``cve-bin-tool`` and see your plugin's findings
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Let's test that our defined CVE comes up by scanning a ``.env`` file.
+
+.. code-block:: console
+
+    $ cve-bin-tool --log debug setup.py
+
+Advanced Example: Ad-Hoc CVEs
+-----------------------------
+
+For more information see: https://github.com/ossf/wg-vulnerability-disclosures/issues/94
+
+1. Create the Parser Class
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+First, create the parser class. This class should be located in the appropriate directory within your project. For example, you might place it in ``cve_bin_tool_parser_static_analysis_bandit/bandit.py``.
+
+.. literalinclude:: /../example/oot-parser/cve_bin_tool_parser_static_analysis_bandit/static_analysis_bandit.py
+
+2. Run ``cve-bin-tool`` and see your plugin's findings
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In this example we implemented the ``BanditParser`` which is a static
+analysis tool for Python files. We'll test that it loads by scanning
+a ``.py`` file.
+
+.. code-block:: console
+
+    $ cve-bin-tool --log debug setup.py
+
+Test Implementation
+-------------------
+
+A new test class `TestParsers` has been introduced to verify that the expected file types are correctly mapped to their respective parsers. The test ensures that the actual valid files match the expected valid files.
+
+Test Method
+^^^^^^^^^^^
+
+- `test_parser_match_filenames_results_in_correct_valid_files`: This test compares the `EXPECTED_VALID_FILES` dictionary with the `actual_valid_files` dictionary imported from `cve_bin_tool.parsers.parse`. If there is any discrepancy between the two, the test will fail, indicating that the loaded file types do not match the expected registered file types.

doc/index.rst

@@ -19,6 +19,7 @@ The CVE Binary Tool helps you determine if your system includes known vulnerabil
    RELEASE.md
    CONTRIBUTING.md
    CHECKERS.md
+   PARSERS.rst
    sboms_for_humans/README.md
    new-contributor-tips.md
    pypi_downloads.md