Merge branch 'main' into feat/security-legacy-urls

.env-defaults

@@ -13,7 +13,7 @@ SCHEDULER_API=http://scheduler:8000
 KEIKO_API=http://keiko:8000
 KATALOGUS_API=http://katalogus:8000
 XTDB_URI=http://crux:3000
-BOEFJE_API=http://boefje:8000
+BOEFJES_API=http://boefje:8000
 
 # Bytes uses JWT for authentication
 BYTES_API=http://bytes:8000

.pre-commit-config.yaml

@@ -84,12 +84,6 @@ repos:
       ^boefjes/tools |
       ^keiko/templates |
       ^mula/whitelist\.py$ |
-      ^octopoes/bits |
-      ^octopoes/octopoes/core |
-      ^octopoes/octopoes/events |
-      ^octopoes/octopoes/models |
-      ^octopoes/octopoes/repositories |
-      ^octopoes/octopoes/xtdb |
       ^octopoes/tools |
       ^rocky/whitelist\.py$ |
       /tests/ |

boefjes/boefjes/api.py

@@ -13,16 +13,12 @@
 from boefjes.clients.bytes_client import BytesAPIClient
 from boefjes.clients.scheduler_client import SchedulerAPIClient, TaskStatus
 from boefjes.config import settings
-from boefjes.job_handler import (
-    _find_ooi_in_past,
-    get_environment_settings,
-    get_octopoes_api_connector,
-    serialize_ooi,
-)
+from boefjes.job_handler import get_environment_settings, get_octopoes_api_connector, serialize_ooi
 from boefjes.job_models import BoefjeMeta
 from boefjes.katalogus.local_repository import LocalPluginRepository, get_local_repository
 from boefjes.plugins.models import _default_mime_types
 from octopoes.models import Reference
+from octopoes.models.exception import ObjectNotFoundException
 
 app = FastAPI(title="Boefje API")
 logger = logging.getLogger(__name__)
@@ -162,12 +158,13 @@ def create_boefje_meta(task, local_repository):
     input_ooi = task.p_item.data.input_ooi
     arguments = {}
     if input_ooi:
-        arguments["input"] = serialize_ooi(
-            _find_ooi_in_past(
-                Reference.from_str(input_ooi),
-                get_octopoes_api_connector(organization),
-            )
-        )
+        reference = Reference.from_str(input_ooi)
+        try:
+            ooi = get_octopoes_api_connector(organization).get(reference, valid_time=datetime.now(timezone.utc))
+        except ObjectNotFoundException as e:
+            raise ObjectNotFoundException(f"Object {reference} not found in Octopoes") from e
+
+        arguments["input"] = serialize_ooi(ooi)
 
     boefje_meta = BoefjeMeta(
         id=task.id,

boefjes/boefjes/config.py

@@ -68,6 +68,10 @@ class Settings(BaseSettings):
         validation_alias="KATALOGUS_DB_URI",
     )
 
+    db_connection_pool_size: int = Field(
+        16, description="Database connection pool size", validation_alias="KATALOGUS_DB_CONNECTION_POOL_SIZE"
+    )
+
     scheduler_api: AnyHttpUrl = Field(
         ..., examples=["http://localhost:8004"], description="Mula API URL", validation_alias="SCHEDULER_API"
     )

boefjes/boefjes/job_handler.py

@@ -2,7 +2,7 @@
 import os
 import traceback
 from collections.abc import Callable
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timezone
 from enum import Enum
 from typing import Any, cast
 
@@ -13,12 +13,7 @@
 from boefjes.clients.bytes_client import BytesAPIClient
 from boefjes.config import settings
 from boefjes.docker_boefjes_runner import DockerBoefjesRunner
-from boefjes.job_models import (
-    BoefjeMeta,
-    NormalizerMeta,
-    NormalizerPlainOOI,
-    NormalizerScanProfile,
-)
+from boefjes.job_models import BoefjeMeta, NormalizerMeta, NormalizerPlainOOI, NormalizerScanProfile
 from boefjes.katalogus.local_repository import LocalPluginRepository
 from boefjes.plugins.models import _default_mime_types
 from boefjes.runtime_interfaces import BoefjeJobRunner, Handler, NormalizerJobRunner
@@ -28,6 +23,8 @@
 from octopoes.models.exception import ObjectNotFoundException
 from octopoes.models.types import OOIType
 
+MIMETYPE_MIN_LENGTH = 5  # two chars before, and 2 chars after the slash ought to be reasonable
+
 logger = logging.getLogger(__name__)
 
 bytes_api_client = BytesAPIClient(
@@ -37,24 +34,6 @@
 )
 
 
-def _find_ooi_in_past(reference: Reference, connector: OctopoesAPIConnector, lookback_days: int = 4) -> OOI:
-    # Source OOIs may not live in XTDB since we currently have TTLs in place (to be removed soon).
-    valid_time = datetime.now(timezone.utc)
-
-    for days_in_past in range(lookback_days):
-        try:
-            return connector.get(reference, valid_time=valid_time)
-        except ObjectNotFoundException:
-            logger.debug(
-                "Object %s not found in Octopoes, looking into other valid times...",
-                reference,
-            )
-            date = datetime.now(timezone.utc) - timedelta(days=days_in_past)
-            valid_time = date.replace(hour=0, minute=0, second=0, microsecond=0)
-
-    raise ObjectNotFoundException(f"Object {reference} not found in Octopoes")
-
-
 def _serialize_value(value: Any, required: bool) -> Any:
     if isinstance(value, list):
         return [_serialize_value(item, required) for item in value]
@@ -118,12 +97,10 @@ def __init__(
         job_runner: BoefjeJobRunner,
         local_repository: LocalPluginRepository,
         bytes_client: BytesAPIClient,
-        octopoes_factory=get_octopoes_api_connector,
     ):
         self.job_runner = job_runner
         self.local_repository = local_repository
         self.bytes_client = bytes_client
-        self.octopoes_factory = octopoes_factory
 
     def handle(self, boefje_meta: BoefjeMeta) -> None:
         logger.info("Handling boefje %s[task_id=%s]", boefje_meta.boefje.id, str(boefje_meta.id))
@@ -141,12 +118,15 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
             return docker_runner.run()
 
         if boefje_meta.input_ooi:
-            boefje_meta.arguments["input"] = serialize_ooi(
-                _find_ooi_in_past(
-                    Reference.from_str(boefje_meta.input_ooi),
-                    self.octopoes_factory(boefje_meta.organization),
+            reference = Reference.from_str(boefje_meta.input_ooi)
+            try:
+                ooi = get_octopoes_api_connector(boefje_meta.organization).get(
+                    reference, valid_time=datetime.now(timezone.utc)
                 )
-            )
+            except ObjectNotFoundException as e:
+                raise ObjectNotFoundException(f"Object {reference} not found in Octopoes") from e
+
+            boefje_meta.arguments["input"] = serialize_ooi(ooi)
 
         env_keys = boefje_resource.environment_keys
 
@@ -176,9 +156,17 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
 
             if boefje_results:
                 for boefje_added_mime_types, output in boefje_results:
-                    raw_file_id = self.bytes_client.save_raw(
-                        boefje_meta.id, output, mime_types.union(boefje_added_mime_types)
-                    )
+                    valid_mimetypes = set()
+                    for mimetype in boefje_added_mime_types:
+                        if len(mimetype) < MIMETYPE_MIN_LENGTH or "/" not in mimetype:
+                            logger.warning(
+                                "Invalid mime-type encountered in output for boefje %s[%s]",
+                                boefje_meta.boefje.id,
+                                str(boefje_meta.id),
+                            )
+                        else:
+                            valid_mimetypes.add(mimetype)
+                    raw_file_id = self.bytes_client.save_raw(boefje_meta.id, output, mime_types.union(valid_mimetypes))
                     logger.debug(
                         "Saved raw file %s for boefje %s[%s]", raw_file_id, boefje_meta.boefje.id, boefje_meta.id
                     )

boefjes/boefjes/katalogus/dependencies/plugins.py

@@ -8,14 +8,8 @@
 from jsonschema.validators import validate
 from sqlalchemy.orm import Session
 
-from boefjes.katalogus.clients import (
-    PluginRepositoryClient,
-    PluginRepositoryClientInterface,
-)
-from boefjes.katalogus.local_repository import (
-    LocalPluginRepository,
-    get_local_repository,
-)
+from boefjes.katalogus.clients import PluginRepositoryClient, PluginRepositoryClientInterface
+from boefjes.katalogus.local_repository import LocalPluginRepository, get_local_repository
 from boefjes.katalogus.models import RESERVED_LOCAL_ID, PluginType, Repository
 from boefjes.katalogus.storage.interfaces import (
     NotFound,

boefjes/boefjes/katalogus/routers/organisations.py

@@ -2,10 +2,7 @@
 
 from boefjes.katalogus.dependencies.organisations import get_organisations_store
 from boefjes.katalogus.models import Organisation
-from boefjes.katalogus.storage.interfaces import (
-    OrganisationNotFound,
-    OrganisationStorage,
-)
+from boefjes.katalogus.storage.interfaces import OrganisationNotFound, OrganisationStorage
 from boefjes.sql.db import ObjectNotFoundException
 
 router = APIRouter(prefix="/organisations", tags=["organisations"])

boefjes/boefjes/katalogus/storage/diskcache.py

@@ -3,10 +3,7 @@
 from diskcache import Cache
 
 from boefjes.katalogus.models import Organisation, Repository
-from boefjes.katalogus.storage.interfaces import (
-    OrganisationStorage,
-    RepositoryStorage,
-)
+from boefjes.katalogus.storage.interfaces import OrganisationStorage, RepositoryStorage
 
 # todo: improve duplicate code
 

boefjes/boefjes/katalogus/tests/test_organisations.py

@@ -4,9 +4,7 @@
 from fastapi.testclient import TestClient
 
 from boefjes.katalogus.api import app
-from boefjes.katalogus.dependencies.organisations import (
-    get_organisations_store,
-)
+from boefjes.katalogus.dependencies.organisations import get_organisations_store
 from boefjes.katalogus.models import Organisation
 from boefjes.katalogus.storage.memory import OrganisationStorageMemory
 

boefjes/boefjes/katalogus/tests/test_plugin_service.py

@@ -6,11 +6,7 @@
 from boefjes.katalogus.local_repository import LocalPluginRepository
 from boefjes.katalogus.models import RESERVED_LOCAL_ID, Bit, Boefje, Normalizer, Repository
 from boefjes.katalogus.storage.interfaces import SettingsNotConformingToSchema
-from boefjes.katalogus.storage.memory import (
-    PluginStatesStorageMemory,
-    RepositoryStorageMemory,
-    SettingsStorageMemory,
-)
+from boefjes.katalogus.storage.memory import PluginStatesStorageMemory, RepositoryStorageMemory, SettingsStorageMemory
 
 
 def get_plugin_seed():

boefjes/boefjes/plugins/kat_binaryedge/containers/normalize.py

@@ -5,14 +5,7 @@
 from boefjes.job_models import NormalizerMeta
 from octopoes.models import OOI, Reference
 from octopoes.models.ooi.findings import Finding, KATFindingType
-from octopoes.models.ooi.network import (
-    IPAddressV4,
-    IPAddressV6,
-    IPPort,
-    Network,
-    PortState,
-    Protocol,
-)
+from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol
 from octopoes.models.ooi.software import Software, SoftwareInstance
 
 

boefjes/boefjes/plugins/kat_binaryedge/databases/normalize.py

@@ -5,14 +5,7 @@
 from boefjes.job_models import NormalizerMeta
 from octopoes.models import OOI, Reference
 from octopoes.models.ooi.findings import Finding, KATFindingType
-from octopoes.models.ooi.network import (
-    IPAddressV4,
-    IPAddressV6,
-    IPPort,
-    Network,
-    PortState,
-    Protocol,
-)
+from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol
 from octopoes.models.ooi.software import Software, SoftwareInstance
 
 

boefjes/boefjes/plugins/kat_binaryedge/http_web/normalize.py

@@ -5,14 +5,7 @@
 from boefjes.job_models import NormalizerMeta
 from boefjes.plugins.kat_binaryedge.services.normalize import get_name_from_cpe
 from octopoes.models import OOI, Reference
-from octopoes.models.ooi.network import (
-    IPAddressV4,
-    IPAddressV6,
-    IPPort,
-    Network,
-    PortState,
-    Protocol,
-)
+from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol
 from octopoes.models.ooi.software import Software, SoftwareInstance
 
 

boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalize.py

@@ -5,14 +5,7 @@
 from boefjes.job_models import NormalizerMeta
 from octopoes.models import OOI, Reference
 from octopoes.models.ooi.findings import Finding, KATFindingType
-from octopoes.models.ooi.network import (
-    IPAddressV4,
-    IPAddressV6,
-    IPPort,
-    Network,
-    PortState,
-    Protocol,
-)
+from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol
 from octopoes.models.ooi.software import Software
 
 

boefjes/boefjes/plugins/kat_binaryedge/protocols/normalize.py

@@ -5,14 +5,7 @@
 from boefjes.job_models import NormalizerMeta
 from octopoes.models import OOI, Reference
 from octopoes.models.ooi.findings import Finding, KATFindingType
-from octopoes.models.ooi.network import (
-    IPAddressV4,
-    IPAddressV6,
-    IPPort,
-    Network,
-    PortState,
-    Protocol,
-)
+from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol
 
 
 def run(normalizer_meta: NormalizerMeta, raw: bytes | str) -> Iterable[OOI]:

boefjes/boefjes/plugins/kat_binaryedge/remote_desktop/normalize.py

@@ -5,14 +5,7 @@
 from boefjes.job_models import NormalizerMeta
 from octopoes.models import OOI, Reference
 from octopoes.models.ooi.findings import Finding, KATFindingType
-from octopoes.models.ooi.network import (
-    IPAddressV4,
-    IPAddressV6,
-    IPPort,
-    Network,
-    PortState,
-    Protocol,
-)
+from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol
 from octopoes.models.ooi.service import IPService, Service
 
 

boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalize.py

@@ -6,14 +6,7 @@
 from boefjes.plugins.kat_binaryedge.services.normalize import get_name_from_cpe
 from octopoes.models import OOI, Reference
 from octopoes.models.ooi.findings import Finding, KATFindingType
-from octopoes.models.ooi.network import (
-    IPAddressV4,
-    IPAddressV6,
-    IPPort,
-    Network,
-    PortState,
-    Protocol,
-)
+from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol
 from octopoes.models.ooi.service import IPService, Service
 from octopoes.models.ooi.software import Software, SoftwareInstance
 

boefjes/boefjes/plugins/kat_binaryedge/services/normalize.py

@@ -4,14 +4,7 @@
 
 from boefjes.job_models import NormalizerMeta
 from octopoes.models import OOI, Reference
-from octopoes.models.ooi.network import (
-    IPAddressV4,
-    IPAddressV6,
-    IPPort,
-    Network,
-    PortState,
-    Protocol,
-)
+from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol
 from octopoes.models.ooi.software import Software, SoftwareInstance
 
 

boefjes/boefjes/plugins/kat_censys/normalize.py

@@ -6,20 +6,10 @@
 from octopoes.models import OOI, Reference
 from octopoes.models.ooi.certificate import X509Certificate
 from octopoes.models.ooi.dns.zone import Hostname
-from octopoes.models.ooi.network import (
-    IPPort,
-    Network,
-    PortState,
-    Protocol,
-)
+from octopoes.models.ooi.network import IPPort, Network, PortState, Protocol
 from octopoes.models.ooi.service import IPService, Service
 from octopoes.models.ooi.software import Software, SoftwareInstance
-from octopoes.models.ooi.web import (
-    HTTPHeader,
-    HTTPResource,
-    IPAddressHTTPURL,
-    Website,
-)
+from octopoes.models.ooi.web import HTTPHeader, HTTPResource, IPAddressHTTPURL, Website
 
 
 def run(normalizer_meta: NormalizerMeta, raw: bytes | str) -> Iterable[OOI]: