feat: enable out of tree parsers

Asciinema: https://asciinema.org/a/664036
Signed-off-by: John Andersen <[email protected]>

cve_bin_tool/egg_updater.py

@@ -2,11 +2,12 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import ast
+import importlib.util
 import os
 import sys
 from io import StringIO
 
-from setuptools import Distribution, find_packages
+from setuptools import Distribution
 
 try:
     from cve_bin_tool.version import VERSION
@@ -59,44 +60,21 @@ def update_egg() -> None:
     with StringIO() as f:
         cwd = os.getcwd()
         os.chdir(os.path.join(os.path.dirname(__file__), ".."))
+        setup_spec = importlib.util.spec_from_file_location(
+            "setup", os.path.join(os.path.dirname(__file__), "..", "setup.py")
+        )
+        setup_module = importlib.util.module_from_spec(setup_spec)
+        setup_spec.loader.exec_module(setup_module)
+        setup_kwargs = setup_module.setup_kwargs
         sys.stdout = f
         sys.stderr = f
-        dist = Distribution(
+        setup_kwargs.update(
             dict(
                 script_name="setup.py",
                 script_args=["egg_info"],
-                name="cve-bin-tool",
-                version=VERSION,
-                packages=find_packages(
-                    exclude=["locales", "presentation"],
-                ),
-                entry_points={
-                    "console_scripts": [
-                        "cve-bin-tool = cve_bin_tool.cli:main",
-                        "csv2cve = cve_bin_tool.csv2cve:main",
-                    ],
-                    "cve_bin_tool.checker": [
-                        "{} = cve_bin_tool.checkers.{}:{}".format(
-                            filename.replace(".py", ""),
-                            filename.replace(".py", ""),
-                            "".join(
-                                (filename.replace(".py", "") + " checker")
-                                .replace("_", " ")
-                                .title()
-                                .split()
-                            ),
-                        )
-                        for filename in os.listdir(
-                            os.path.join(
-                                os.path.abspath(os.path.dirname(__file__)),
-                                "checkers",
-                            )
-                        )
-                        if filename.endswith(".py") and "__init__" not in filename
-                    ],
-                },
             )
         )
+        dist = Distribution(setup_kwargs)
         dist.parse_command_line()
         dist.run_commands()
         sys.stdout = sys.__stdout__

cve_bin_tool/parsers/__init__.py

@@ -12,6 +12,7 @@
 from cve_bin_tool.util import ProductInfo, ScanInfo
 
 __all__ = [
+    "parse",
     "Parser",
     "java",
     "javascript",

cve_bin_tool/parsers/dart.py

@@ -15,6 +15,10 @@ class DartParser(Parser):
     https://dart.dev/overview
     """
 
+    PARSER_MATCH_FILENAMES = [
+        "pubspec.lock",
+    ]
+
     def __init__(self, cve_db, logger):
         super().__init__(cve_db, logger)
         self.purl_pkg_type = "pub"

cve_bin_tool/parsers/go.py

@@ -25,6 +25,10 @@ class GoParser(Parser):
 
     """
 
+    PARSER_MATCH_FILENAMES = [
+        "go.mod",
+    ]
+
     def __init__(self, cve_db, logger):
         super().__init__(cve_db, logger)
         self.purl_pkg_type = "golang"

cve_bin_tool/parsers/java.py

@@ -13,6 +13,10 @@
 class JavaParser(Parser):
     """Class to handle parsing Java-based Packages."""
 
+    PARSER_MATCH_FILENAMES = [
+        "pom.xml",
+    ]
+
     def __init__(self, cve_db, logger, validate=True):
         super().__init__(cve_db, logger)
         self.validate = validate

cve_bin_tool/parsers/javascript.py

@@ -11,6 +11,10 @@
 class JavascriptParser(Parser):
     """Parser for javascript's package-lock.json files"""
 
+    PARSER_MATCH_FILENAMES = [
+        "package-lock.json",
+    ]
+
     def __init__(self, cve_db, logger):
         super().__init__(cve_db, logger)
         self.purl_pkg_type = "npm"

cve_bin_tool/parsers/parse.py

@@ -1,40 +1,44 @@
 # Copyright (C) 2022 Intel Corporation
 # SPDX-License-Identifier: GPL-3.0-or-later
+import sys
+from typing import Type
 
-from cve_bin_tool.parsers.dart import DartParser
-from cve_bin_tool.parsers.go import GoParser
-from cve_bin_tool.parsers.java import JavaParser
-from cve_bin_tool.parsers.javascript import JavascriptParser
-from cve_bin_tool.parsers.perl import PerlParser
-from cve_bin_tool.parsers.php import PhpParser
-from cve_bin_tool.parsers.python import PythonParser, PythonRequirementsParser
-from cve_bin_tool.parsers.r import RParser
-from cve_bin_tool.parsers.ruby import RubyParser
-from cve_bin_tool.parsers.rust import RustParser
-from cve_bin_tool.parsers.swift import SwiftParser
-
-valid_files = {
-    "pom.xml": JavaParser,
-    "package-lock.json": JavascriptParser,
-    "Cargo.lock": RustParser,
-    "renv.lock": RParser,
-    "requirements.txt": PythonRequirementsParser,
-    "go.mod": GoParser,
-    "PKG-INFO: ": PythonParser,
-    "METADATA: ": PythonParser,
-    "Gemfile.lock": RubyParser,
-    "Package.resolved": SwiftParser,
-    "composer.lock": PhpParser,
-    "cpanfile": PerlParser,
-    "pubspec.lock": DartParser,
-}
+if sys.version_info >= (3, 10):
+    from importlib import metadata as importlib_metadata
+else:
+    import importlib_metadata
+
+from cve_bin_tool.parsers import Parser
+
+PARSERS_ENTRYPOINT = "cve_bin_tool.parsers"
+
+
+def load_valid_files() -> dict[str, list[Type[Parser]]]:
+    """Loads file parsers"""
+    valid_files: dict[str, list[Type[Parser]]] = {}
+    for entrypoint in importlib_metadata.entry_points().select(
+        group=PARSERS_ENTRYPOINT
+    ):
+        parser_cls = entrypoint.load()
+        for match_filename in getattr(parser_cls, "PARSER_MATCH_FILENAMES", []):
+            valid_files.setdefault(match_filename, [])
+            valid_files[match_filename].append(parser_cls)
+    for match_filename in valid_files:
+        valid_files[match_filename] = list(set(valid_files[match_filename]))
+    return valid_files
+
+
+valid_files = load_valid_files()
 
 
 def parse(filename, output, cve_db, logger):
     """
     Parses the given filename using the appropriate parser.
     """
+    parsers = []
     for file in list(valid_files.keys()):
         if file in output:
-            parser = valid_files[file](cve_db, logger)
-    yield from parser.run_checker(filename)
+            for valid_file_parser in valid_files[file]:
+                parsers.append(valid_file_parser(cve_db, logger))
+    for parser in parsers:
+        yield from parser.run_checker(filename)

cve_bin_tool/parsers/perl.py

@@ -9,6 +9,10 @@
 class PerlParser(Parser):
     """Parser for perl's cpan files"""
 
+    PARSER_MATCH_FILENAMES = [
+        "cpanfile",
+    ]
+
     def __init__(self, cve_db, logger):
         super().__init__(cve_db, logger)
         self.purl_pkg_type = "cpan"

cve_bin_tool/parsers/php.py

@@ -14,6 +14,10 @@ class PhpParser(Parser):
     generate PURLs (Package URLs) for the listed packages.
     """
 
+    PARSER_MATCH_FILENAMES = [
+        "composer.lock",
+    ]
+
     def __init__(self, cve_db, logger):
         """Initialize the PhpParser."""
         super().__init__(cve_db, logger)

cve_bin_tool/parsers/python.py

@@ -20,6 +20,10 @@ class PythonRequirementsParser(Parser):
     requirements.txt) and generate PURLs (Package URLs) for the listed packages.
     """
 
+    PARSER_MATCH_FILENAMES = [
+        "requirements.txt",
+    ]
+
     def __init__(self, cve_db, logger):
         """Initialize the python requirements file parser."""
         super().__init__(cve_db, logger)
@@ -114,6 +118,11 @@ class PythonParser(Parser):
     PKG-INFO or METADATA) and generate PURLs (Package URLs) for the package.
     """
 
+    PARSER_MATCH_FILENAMES = [
+        "PKG-INFO: ",
+        "METADATA: ",
+    ]
+
     def __init__(self, cve_db, logger):
         """Initialize the python package metadata parser."""
         super().__init__(cve_db, logger)

cve_bin_tool/parsers/r.py

@@ -26,6 +26,10 @@ class RParser(Parser):
 
     """
 
+    PARSER_MATCH_FILENAMES = [
+        "renv.lock",
+    ]
+
     def __init__(self, cve_db, logger):
         super().__init__(cve_db, logger)
         self.purl_pkg_type = "cran"

cve_bin_tool/parsers/ruby.py

@@ -25,6 +25,10 @@ class RubyParser(Parser):
 
     """
 
+    PARSER_MATCH_FILENAMES = [
+        "Gemfile.lock",
+    ]
+
     def __init__(self, cve_db, logger):
         super().__init__(cve_db, logger)
         self.purl_pkg_type = "gem"

cve_bin_tool/parsers/rust.py

@@ -24,6 +24,10 @@ class RustParser(Parser):
             Parse the Rust dependency file and yield valid PURLs for the packages listed in the file.
     """
 
+    PARSER_MATCH_FILENAMES = [
+        "Cargo.lock",
+    ]
+
     def __init__(self, cve_db, logger):
         super().__init__(cve_db, logger)
         self.purl_pkg_type = "cargo"

cve_bin_tool/parsers/swift.py

@@ -28,6 +28,10 @@ class SwiftParser(Parser):
 
     """
 
+    PARSER_MATCH_FILENAMES = [
+        "Package.resolved",
+    ]
+
     def __init__(self, cve_db, logger):
         super().__init__(cve_db, logger)
         self.purl_pkg_type = "swift"

cve_bin_tool/version_scanner.py

@@ -2,10 +2,11 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 from __future__ import annotations
 
+import itertools
 import subprocess
 import sys
 from logging import Logger
-from pathlib import Path, PurePath
+from pathlib import Path
 from typing import Iterator
 
 from cve_bin_tool.checkers import Checker
@@ -30,10 +31,6 @@
     from importlib import metadata as importlib_metadata
 else:
     import importlib_metadata
-if sys.version_info >= (3, 9):
-    import importlib.resources as resources
-else:
-    import importlib_resources as resources
 
 
 class InvalidFileError(Exception):
@@ -129,16 +126,7 @@ def number_of_checkers(self) -> int:
     @classmethod
     def available_language_checkers(cls) -> list[str]:
         """Find Language checkers"""
-        language_directory = resources.files(cls.LANGUAGE_CHECKER_ENTRYPOINT)
-        parsers = language_directory.iterdir()
-        language_checkers = []
-        for parser in parsers:
-            if str(parser).endswith(".py"):
-                language = PurePath(parser).name.replace(".py", "").capitalize()
-                if language not in ["__init__", "Parse"]:
-                    language_checkers.append(language)
-
-        return sorted(language_checkers)
+        return list(sorted(map(str, set(itertools.chain(*valid_files.values())))))
 
     def print_language_checkers(self) -> None:
         """Logs the message that lists the names of the language checkers"""

setup.py

@@ -3,9 +3,13 @@
 
 import ast
 import os
+import pathlib
+import re
 
 from setuptools import find_packages, setup
 
+PACKAGE_ROOT_PATH = pathlib.Path(__file__).parent.resolve()
+
 with open("README.md", encoding="utf-8") as f:
     readme = f.read()
 
@@ -18,6 +22,30 @@
             VERSION = ast.literal_eval(line.strip().split("=")[-1].strip())
             break
 
+
+def enumerate_entry_points_parsers():
+    """Reads the files in cve_bin_tool/parsers/to auto determine list"""
+    parsers = {}
+    for path in PACKAGE_ROOT_PATH.joinpath(
+        "cve_bin_tool",
+        "parsers",
+    ).glob("*.py"):
+        if "__init__" == path.stem:
+            continue
+        contents = path.read_text()
+        for re_match in re.finditer(r"^class (\w+)", contents, re.MULTILINE):
+            parser_cls_name = re_match[1]
+            parsers[".".join([path.stem, parser_cls_name])] = ":".join(
+                [
+                    str(path.relative_to(PACKAGE_ROOT_PATH).with_suffix("")).replace(
+                        os.path.sep, "."
+                    ),
+                    parser_cls_name,
+                ],
+            )
+    return parsers
+
+
 setup_kwargs = dict(
     name="cve-bin-tool",
     version=VERSION,
@@ -89,7 +117,18 @@
             )
             if filename.endswith(".py") and "__init__" not in filename
         ],
+        "cve_bin_tool.parsers": [
+            "{} = {}".format(
+                parser_entry_point_name,
+                entry_point_path,
+            )
+            for (
+                parser_entry_point_name,
+                entry_point_path,
+            ) in enumerate_entry_points_parsers().items()
+        ],
     },
 )
 
-setup(**setup_kwargs)
+if __name__ == "__main__":
+    setup(**setup_kwargs)