Merge branch 'main' into feature/add-task-details-to-plugin-detail-hi…

…story-tables

.pre-commit-config.yaml

@@ -112,13 +112,15 @@ repos:
       ^rocky/.*/templates/.*$ |
       ^rocky/reports/report_types/.*/.*\.html
       )
+    exclude: '^rocky/rocky/templates/admin/.*\.html$'
 
   - id: djlint-django
     files: |
       (?x)(
       ^rocky/.*/templates/.*$ |
       ^rocky/reports/report_types/.*/.*\.html
       )
+    exclude: '^rocky/rocky/templates/admin/.*\.html$'
 
 - repo: https://github.com/thibaudcolas/pre-commit-stylelint
   rev: v15.10.1

boefjes/boefjes/api.py

@@ -14,14 +14,14 @@
 from boefjes.clients.scheduler_client import SchedulerAPIClient, TaskStatus
 from boefjes.config import settings
 from boefjes.job_handler import (
-    _collect_default_mime_types,
     _find_ooi_in_past,
     get_environment_settings,
     get_octopoes_api_connector,
     serialize_ooi,
 )
 from boefjes.job_models import BoefjeMeta
 from boefjes.katalogus.local_repository import LocalPluginRepository, get_local_repository
+from boefjes.plugins.models import _default_meta_mime_types
 from octopoes.models import Reference
 
 app = FastAPI(title="Boefje API")
@@ -128,7 +128,7 @@ async def boefje_output(
     bytes_client.save_boefje_meta(boefje_meta)
 
     if boefje_output.files:
-        mime_types = _collect_default_mime_types(task.p_item.data)
+        mime_types = _default_meta_mime_types(task.p_item.data)
         for file in boefje_output.files:
             raw = base64.b64decode(file.content)
             # when supported, also save file.name to Bytes

boefjes/boefjes/docker_boefjes_runner.py

@@ -31,9 +31,9 @@ def run(self) -> None:
             raise RuntimeError("Boefje does not have OCI image")
 
         # local import to prevent circular dependency
-        from boefjes import job_handler
+        import boefjes.plugins.models
 
-        stderr_mime_types = job_handler._collect_default_mime_types(self.boefje_meta)
+        stderr_mime_types = boefjes.plugins.models._default_meta_mime_types(self.boefje_meta)
 
         task_id = str(self.boefje_meta.id)
         self.scheduler_client.patch_task(task_id, TaskStatus.RUNNING)

boefjes/boefjes/job_handler.py

@@ -3,7 +3,7 @@
 import traceback
 from datetime import datetime, timedelta, timezone
 from enum import Enum
-from typing import Any, Dict, List, Set
+from typing import Any, Dict, List
 
 import requests
 from pydantic.tools import parse_obj_as
@@ -18,6 +18,7 @@
     NormalizerPlainOOI,
 )
 from boefjes.katalogus.local_repository import LocalPluginRepository
+from boefjes.plugins.models import _default_meta_mime_types
 from boefjes.runtime_interfaces import BoefjeJobRunner, Handler, NormalizerJobRunner
 from octopoes.api.models import Declaration, Observation
 from octopoes.connector.octopoes import OctopoesAPIConnector
@@ -99,28 +100,6 @@ def get_environment_settings(boefje_meta: BoefjeMeta, environment_keys: List[str
         logger.exception("Error getting environment settings")
         raise
 
-    return {}
-
-
-def _collect_default_mime_types(boefje_meta: BoefjeMeta) -> Set[str]:
-    boefje_id = boefje_meta.boefje.id
-
-    mime_types = {
-        boefje_id,
-        f"boefje/{boefje_id}",
-        f"boefje/{boefje_id}-{boefje_meta.parameterized_arguments_hash}",
-    }
-
-    if boefje_meta.boefje.version is not None:
-        mime_types = mime_types.union(
-            {
-                f"boefje/{boefje_id}-{boefje_meta.boefje.version}",
-                f"boefje/{boefje_id}-{boefje_meta.parameterized_arguments_hash}-{boefje_meta.boefje.version}",
-            }
-        )
-
-    return mime_types
-
 
 class BoefjeHandler(Handler):
     def __init__(self, job_runner, local_repository: LocalPluginRepository):
@@ -155,7 +134,7 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
         boefje_meta.runnable_hash = boefje_resource.runnable_hash
         boefje_meta.environment = get_environment_settings(boefje_meta, env_keys) if env_keys else {}
 
-        mime_types = _collect_default_mime_types(boefje_meta)
+        mime_types = _default_meta_mime_types(boefje_meta)
 
         logger.info("Starting boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
 

boefjes/boefjes/katalogus/local_repository.py

@@ -1,11 +1,10 @@
-import hashlib
 import json
 import logging
 import pkgutil
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple
 
-from boefjes.katalogus.models import RESERVED_LOCAL_ID, Boefje, Normalizer, PluginType
+from boefjes.katalogus.models import PluginType
 from boefjes.plugins.models import (
     BOEFJE_DEFINITION_FILE,
     BOEFJES_DIR,
@@ -27,8 +26,8 @@ def __init__(self, path: Path):
         self._cached_normalizers = None
 
     def get_all(self) -> List[PluginType]:
-        all_plugins = [self._boefje_to_plugin(boefje) for boefje in self.resolve_boefjes().values()]
-        normalizers = [self._normalizer_to_plugin(normalizer) for normalizer in self.resolve_normalizers().values()]
+        all_plugins = [boefje_resource.boefje for boefje_resource in self.resolve_boefjes().values()]
+        normalizers = [normalizer_resource.normalizer for normalizer_resource in self.resolve_normalizers().values()]
 
         all_plugins += normalizers
 
@@ -38,12 +37,12 @@ def by_id(self, plugin_id: str) -> PluginType:
         boefjes = self.resolve_boefjes()
 
         if plugin_id in boefjes:
-            return self._boefje_to_plugin(boefjes[plugin_id])
+            return boefjes[plugin_id].boefje
 
         normalizers = self.resolve_normalizers()
 
         if plugin_id in normalizers:
-            return self._normalizer_to_plugin(normalizers[plugin_id])
+            return normalizers[plugin_id].normalizer
 
         raise Exception(f"Can't find plugin {plugin_id}")
 
@@ -151,42 +150,6 @@ def create_relative_import_statement_from_cwd(package_dir: Path) -> str:
 
         return f"{relative_path[1:].replace('/', '.')}."  # Turns into "boefjes.plugins."
 
-    @staticmethod
-    def _boefje_to_plugin(boefje: BoefjeResource) -> Boefje:
-        def_file = boefje.path / "boefje.json"
-        def_obj = json.loads(def_file.read_text())
-        def_obj["repository_id"] = RESERVED_LOCAL_ID
-        def_obj["runnable_hash"] = get_runnable_hash(boefje.path)
-
-        return Boefje.parse_obj(def_obj)
-
-    @staticmethod
-    def _normalizer_to_plugin(normalizer: NormalizerResource) -> Normalizer:
-        def_file = normalizer.path / "normalizer.json"
-        def_obj = json.loads(def_file.read_text())
-        def_obj["repository_id"] = RESERVED_LOCAL_ID
-
-        normalizer: Normalizer = Normalizer.parse_obj(def_obj)
-        normalizer.consumes.append(f"normalizer/{normalizer.id}")
-
-        return normalizer
-
 
 def get_local_repository():
     return LocalPluginRepository(BOEFJES_DIR)
-
-
-def get_runnable_hash(path: Path) -> str:
-    """Returns sha256(file1 + file2 + ...) of all files in the given path."""
-
-    folder_hash = hashlib.sha256()
-
-    for file in sorted(path.glob("**/*")):
-        # Note that the hash does not include *.pyc files
-        # Thus there may be a desync between the source code and the cached, compiled bytecode
-        if file.is_file() and file.suffix != ".pyc":
-            with file.open("rb") as f:
-                while chunk := f.read(32768):
-                    folder_hash.update(chunk)
-
-    return folder_hash.hexdigest()

boefjes/boefjes/katalogus/models.py

@@ -39,6 +39,7 @@ class Boefje(Plugin):
     scan_level: int = 1
     consumes: Set[str] = Field(default_factory=set)
     produces: List[str] = Field(default_factory=list)
+    mime_types: Set[str] = Field(default_factory=set)
     options: Optional[List[str]]
     runnable_hash: Optional[str]
     oci_image: Optional[str]

boefjes/boefjes/katalogus/tests/test_plugins.py

@@ -107,6 +107,20 @@ def test_get_plugin(self):
         res = self.client.get("/v1/organisations/test-org/repositories/test-repo/plugins/test-boefje-1")
         self.assertEqual(200, res.status_code)
 
+        # Simpler endpoint works as well, but due to the mock the default mime_types are not dynamically added
+        res = self.client.get("/v1/organisations/test-org/plugins/test-boefje-1")
+        self.assertEqual(200, res.status_code)
+        assert "mime_types" in res.json()
+        assert not res.json()["mime_types"]
+
+        # For boefjes that are pulled from the local repository, we actually get the default mime_types
+        assert set(self.client.get("/v1/organisations/test-org/plugins/kat_test").json()["mime_types"]) == set(
+            [
+                "kat_test",
+                "boefje/kat_test",
+            ]
+        )
+
     def test_non_existing_plugin(self):
         res = self.client.get("/v1/organisations/test-org/repositories/test-repo/plugins/future-plugin")
         self.assertEqual(404, res.status_code)

boefjes/boefjes/plugins/kat_adr_finding_types/boefje.json

@@ -8,7 +8,6 @@
     "produces": [
         "ADRFindingType"
     ],
-    "environment_keys": [],
     "scan_level": 0,
     "enabled": true
 }

boefjes/boefjes/plugins/kat_adr_validator/main.py

@@ -28,9 +28,4 @@ def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
 
     output = run_adr_validator(url)
 
-    return [
-        (
-            set(),
-            output,
-        ),
-    ]
+    return [(set(), output)]

boefjes/boefjes/plugins/kat_binaryedge/schema.json

@@ -11,5 +11,8 @@
   },
   "required": [
     "BINARYEDGE_API"
+  ],
+  "secret": [
+    "BINARYEDGE_API"
   ]
 }

boefjes/boefjes/plugins/kat_censys/schema.json

@@ -18,5 +18,8 @@
     "required": [
         "CENSYS_API_ID",
         "CENSYS_API_SECRET"
+    ],
+    "secret": [
+        "CENSYS_API_SECRET"
     ]
 }

boefjes/boefjes/plugins/kat_crt_sh/boefje.json

@@ -9,6 +9,5 @@
         "Hostname",
         "X509Certificate"
     ],
-    "environment_keys": [],
     "scan_level": 1
 }

boefjes/boefjes/plugins/kat_cwe_finding_types/boefje.json

@@ -8,7 +8,6 @@
     "produces": [
         "CWEFindingType"
     ],
-    "environment_keys": [],
     "scan_level": 0,
     "enabled": true
 }

boefjes/boefjes/plugins/kat_dicom/boefje.json

@@ -13,6 +13,5 @@
         "Finding",
         "Software"
     ],
-    "environment_keys": [],
     "scan_level": 2
 }

boefjes/boefjes/plugins/kat_dns/boefje.json

@@ -20,6 +20,5 @@
         "IPAddressV4",
         "DNSZone"
     ],
-    "environment_keys": [],
     "scan_level": 1
 }

boefjes/boefjes/plugins/kat_dns_zone/boefje.json

@@ -10,6 +10,5 @@
         "DNSZone",
         "DNSSOARecord"
     ],
-    "environment_keys": [],
     "scan_level": 1
 }

boefjes/boefjes/plugins/kat_dnssec/boefje.json

@@ -9,6 +9,5 @@
         "KATFindingType",
         "Finding"
     ],
-    "environment_keys": [],
     "scan_level": 1
 }

boefjes/boefjes/plugins/kat_external_db/main.py

@@ -11,7 +11,7 @@ def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
     """Fetch external database response."""
     api_format = getenv(
         "DB_ENDPOINT_FORMAT",
-        "{DB_URL}/api/v1/participants/assets/{DB_ORGANIZATION_IDENTIFIER}?access_token={DB_ACCESS_TOKEN}",
+        "{DB_URL}/api/v1/organizations/assets/{DB_ORGANIZATION_IDENTIFIER}?access_token={DB_ACCESS_TOKEN}",
     )
     request_timeout = 100
 

boefjes/boefjes/plugins/kat_external_db/schema.json

@@ -29,5 +29,8 @@
     },
     "required": [
         "DB_URL"
+    ],
+    "secret": [
+      "DB_ACCESS_TOKEN"
     ]
 }

boefjes/boefjes/plugins/kat_fierce/boefje.json

@@ -12,6 +12,5 @@
         "DNSAAAARecord",
         "IPAddressV4"
     ],
-    "environment_keys": [],
     "scan_level": 3
 }

boefjes/boefjes/plugins/kat_green_hosting/boefje.json

@@ -9,6 +9,5 @@
         "KATFindingType",
         "Finding"
     ],
-    "environment_keys": [],
     "scan_level": 1
 }

boefjes/boefjes/plugins/kat_kat_finding_types/boefje.json

@@ -8,7 +8,6 @@
     "produces": [
         "KATFindingType"
     ],
-    "environment_keys": [],
     "scan_level": 0,
     "enabled": true
 }

boefjes/boefjes/plugins/kat_leakix/schema.json

@@ -11,5 +11,8 @@
   },
   "required": [
     "LEAKIX_API"
+  ],
+  "secret": [
+    "LEAKIX_API"
   ]
 }

boefjes/boefjes/plugins/kat_manyportsopen/boefje.json

@@ -10,6 +10,5 @@
             "KATFindingType",
             "Finding"
         ],
-        "environment_keys": [],
         "scan_level": 1
     }

boefjes/boefjes/plugins/kat_nmap/boefje.json

@@ -1,21 +1,21 @@
-    {
-        "id": "nmap",
-        "name": "Nmap",
-        "description": "Defaults to top 250 TCP ports. Includes service detection.",
-        "consumes": [
-            "IPAddressV4",
-            "IPAddressV6"
-        ],
-        "produces": [
-            "IPAddressV6",
-            "Service",
-            "IPPort",
-            "IPAddressV4",
-            "IPService"
-        ],
-        "environment_keys": [
-            "TOP_PORTS",
-            "PROTOCOL"
-        ],
-        "scan_level": 2
-    }
+{
+    "id": "nmap",
+    "name": "Nmap",
+    "description": "Defaults to top 250 TCP ports. Includes service detection.",
+    "consumes": [
+        "IPAddressV4",
+        "IPAddressV6"
+    ],
+    "produces": [
+        "IPAddressV6",
+        "Service",
+        "IPPort",
+        "IPAddressV4",
+        "IPService"
+    ],
+    "environment_keys": [
+        "TOP_PORTS",
+        "PROTOCOL"
+    ],
+    "scan_level": 2
+}