Containerize all non-docker boefjes

boefjes/Makefile

@@ -36,10 +36,9 @@ help: ## Show this help.
 build: images
 
 images:  # Build the images for the containerized boefjes
-  # Dns-records is disabled for now, see the discussion in https://github.com/minvws/nl-kat-coordination/pull/2709
-	# docker build -f images/base.Dockerfile -t ghcr.io/minvws/openkat/dns-records --build-arg BOEFJE_PATH=./boefjes/plugins/kat_dns .
 	docker build -f ./boefjes/plugins/kat_dnssec/boefje.Dockerfile -t ghcr.io/minvws/openkat/dns-sec:latest .
 	docker build -f ./boefjes/plugins/kat_nmap_tcp/boefje.Dockerfile -t ghcr.io/minvws/openkat/nmap:latest  .
+	docker build -f images/generic.Dockerfile -t ghcr.io/minvws/openkat/generic:latest .
 
 
 ##

boefjes/boefjes/dependencies/plugins.py

@@ -52,7 +52,8 @@ def _get_all_without_enabled(self) -> dict[str, PluginType]:
         all_plugins = {plugin.id: plugin for plugin in self.local_repo.get_all()}
 
         for plugin in self.plugin_storage.get_all():
-            all_plugins[plugin.id] = plugin
+            if plugin.id not in all_plugins:
+                all_plugins[plugin.id] = plugin
 
         return all_plugins
 

boefjes/boefjes/docker_boefjes_runner.py

@@ -15,6 +15,9 @@
 
 
 class DockerBoefjesRunner:
+    CACHE_VOLUME_NAME = "openkat_cache"
+    CACHE_VOLUME_TARGET = "/home/nonroot/openkat_cache"
+
     def __init__(self, boefje_resource: Boefje, boefje_meta: BoefjeMeta):
         self.boefje_resource = boefje_resource
         self.boefje_meta = boefje_meta
@@ -46,6 +49,7 @@ def run(self) -> None:
                 stderr=True,
                 remove=True,
                 network=settings.docker_network,
+                volumes=[f"{self.CACHE_VOLUME_NAME}:{self.CACHE_VOLUME_TARGET}"],
             )
 
             task = self.scheduler_client.get_task(task_id)

boefjes/boefjes/plugins/kat_adr_finding_types/boefje.json

@@ -6,5 +6,9 @@
     "ADRFindingType"
   ],
   "scan_level": 0,
-  "enabled": true
+  "enabled": true,
+  "oci_image": "ghcr.io/minvws/openkat/generic:latest",
+  "oci_arguments": [
+    "kat_adr_finding_types.main"
+  ]
 }

boefjes/boefjes/plugins/kat_adr_finding_types/main.py

@@ -1,11 +1,9 @@
 import json
 
-from boefjes.job_models import BoefjeMeta
-
 FINDING_TYPE_PATH = "boefjes/plugins/kat_adr_finding_types/adr_finding_types.json"
 
 
-def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]:
+def run(boefje_meta: dict) -> list[tuple[set, bytes | str]]:
     with open(FINDING_TYPE_PATH) as json_file:
         data = json.load(json_file)
         return [(set(), json.dumps(data))]

boefjes/boefjes/plugins/kat_binaryedge/boefje.json

@@ -6,5 +6,9 @@
     "IPAddressV4",
     "IPAddressV6"
   ],
-  "scan_level": 2
+  "scan_level": 2,
+  "oci_image": "ghcr.io/minvws/openkat/generic:latest",
+  "oci_arguments": [
+    "kat_binaryedge.main"
+  ]
 }

boefjes/boefjes/plugins/kat_binaryedge/main.py

@@ -4,14 +4,12 @@
 
 from pybinaryedge import BinaryEdge
 
-from boefjes.job_models import BoefjeMeta
 
-
-def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]:
+def run(boefje_meta: dict) -> list[tuple[set, bytes | str]]:
     be = BinaryEdge(getenv("BINARYEDGE_API"))
     results: dict[str, list] = {"results": []}
 
-    input_ = boefje_meta.arguments["input"]
+    input_ = boefje_meta["arguments"]["input"]
 
     if input_["object_type"] in ["IPAddressV4", "IPAddressV6"]:
         ip = input_["address"]

boefjes/boefjes/plugins/kat_binaryedge/requirements.txt

@@ -0,0 +1 @@
+pybinaryedge == 0.5

boefjes/boefjes/plugins/kat_censys/boefje.json

@@ -6,5 +6,9 @@
     "IPAddressV4",
     "IPAddressV6"
   ],
-  "scan_level": 1
+  "scan_level": 1,
+  "oci_image": "ghcr.io/minvws/openkat/generic:latest",
+  "oci_arguments": [
+    "kat_cencys.main"
+  ]
 }

boefjes/boefjes/plugins/kat_censys/main.py

@@ -2,12 +2,10 @@
 
 from censys.search import CensysHosts
 
-from boefjes.job_models import BoefjeMeta
 
-
-def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]:
+def run(boefje_meta: dict) -> list[tuple[set, bytes | str]]:
     h = CensysHosts()
-    input_ = boefje_meta.arguments["input"]
+    input_ = boefje_meta["arguments"]["input"]
     ip = input_["address"]
     host = h.view(ip)
 

boefjes/boefjes/plugins/kat_censys/requirements.txt

@@ -0,0 +1 @@
+censys == 2.1.8

boefjes/boefjes/plugins/kat_crt_sh/boefje.json

@@ -5,5 +5,9 @@
   "consumes": [
     "DNSZone"
   ],
-  "scan_level": 1
+  "scan_level": 1,
+  "oci_image": "ghcr.io/minvws/openkat/generic:latest",
+  "oci_arguments": [
+    "kat_crt_sh.main"
+  ]
 }

boefjes/boefjes/plugins/kat_crt_sh/main.py

@@ -2,8 +2,6 @@
 
 import requests
 
-from boefjes.job_models import BoefjeMeta
-
 CRT_SH_API = "https://crt.sh/"
 MATCHES = ("=", "ILIKE", "LIKE", "single", "any", "FTS")
 SEARCH_TYPES = (
@@ -55,8 +53,8 @@ def request_certs(search_string, search_type="Identity", match="=", deduplicate=
     return response.text
 
 
-def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]:
-    input_ = boefje_meta.arguments["input"]
+def run(boefje_meta: dict) -> list[tuple[set, bytes | str]]:
+    input_ = boefje_meta["arguments"]["input"]
     fqdn = input_["hostname"]["name"]
     results = request_certs(fqdn)
 

boefjes/boefjes/plugins/kat_crt_sh/requirements.txt

@@ -0,0 +1 @@
+requests == 2.32.3

boefjes/boefjes/plugins/kat_cve_2023_34039/boefje.json

@@ -5,5 +5,9 @@
   "consumes": [
     "IPService"
   ],
-  "scan_level": 4
+  "scan_level": 4,
+  "oci_image": "ghcr.io/minvws/openkat/generic:latest",
+  "oci_arguments": [
+    "kat_cve_2023_34039.main"
+  ]
 }

boefjes/boefjes/plugins/kat_cve_2023_34039/main.py

@@ -16,11 +16,9 @@
 
 import os
 
-from boefjes.job_models import BoefjeMeta
 
-
-def run(boefje_meta: BoefjeMeta) -> list[tuple[set, str | bytes]]:
-    input_ = boefje_meta.arguments["input"]  # input is IPService
+def run(boefje_meta: dict) -> list[tuple[set, str | bytes]]:
+    input_ = boefje_meta["arguments"]["input"]  # input is IPService
     ip_port = input_["ip_port"]
     if input_["service"]["name"] != "ssh":
         return [({"info/boefje"}, "Skipping because service is not an ssh service")]

boefjes/boefjes/plugins/kat_cve_2023_35078/boefje.json

@@ -5,5 +5,9 @@
   "consumes": [
     "Website"
   ],
-  "scan_level": 2
+  "scan_level": 2,
+  "oci_image": "ghcr.io/minvws/openkat/generic:latest",
+  "oci_arguments": [
+    "kat_cve_2023_35078.main"
+  ]
 }

boefjes/boefjes/plugins/kat_cve_2023_35078/main.py

@@ -2,13 +2,11 @@
 
 import requests
 
-from boefjes.job_models import BoefjeMeta
-
 ENDPOINT_PATH = "/mifs/c/windows/api/v2/device/registration"
 
 
-def run(boefje_meta: BoefjeMeta) -> list[tuple[set, str | bytes]]:
-    input_ = boefje_meta.arguments["input"]  # input is website
+def run(boefje_meta: dict) -> list[tuple[set, str | bytes]]:
+    input_ = boefje_meta["arguments"]["input"]  # input is website
     hostname = input_["hostname"]["name"]
     service = input_["ip_service"]["service"]["name"]
     website = f"{service}://{hostname}"

boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json

@@ -6,5 +6,9 @@
     "CVEFindingType"
   ],
   "scan_level": 0,
-  "enabled": true
+  "enabled": true,
+  "oci_image": "ghcr.io/minvws/openkat/generic:latest",
+  "oci_arguments": [
+    "kat_cve_finding_types.main"
+  ]
 }

boefjes/boefjes/plugins/kat_cve_finding_types/main.py

@@ -2,11 +2,9 @@
 
 import requests
 
-from boefjes.job_models import BoefjeMeta
 
-
-def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]:
-    cve_id = boefje_meta.arguments["input"]["id"]
+def run(boefje_meta: dict) -> list[tuple[set, bytes | str]]:
+    cve_id = boefje_meta["arguments"]["input"]["id"]
     cveapi_url = getenv("CVEAPI_URL", "https://cve.openkat.dev/v1")
     response = requests.get(f"{cveapi_url}/{cve_id}.json", timeout=30)
 

boefjes/boefjes/plugins/kat_cwe_finding_types/boefje.json

@@ -6,5 +6,9 @@
     "CWEFindingType"
   ],
   "scan_level": 0,
-  "enabled": true
+  "enabled": true,
+  "oci_image": "ghcr.io/minvws/openkat/generic:latest",
+  "oci_arguments": [
+    "kat_cwe_finding_types.main"
+  ]
 }

boefjes/boefjes/plugins/kat_cwe_finding_types/main.py

@@ -2,13 +2,11 @@
 
 import defusedxml.ElementTree as ET
 
-from boefjes.job_models import BoefjeMeta
-
 FINDING_TYPE_PATH = "boefjes/plugins/kat_cwe_finding_types/cwec_v4.11.xml"
 
 
-def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]:
-    cwe_id = boefje_meta.arguments["input"]["id"]
+def run(boefje_meta: dict) -> list[tuple[set, bytes | str]]:
+    cwe_id = boefje_meta["arguments"]["input"]["id"]
 
     root = ET.parse(FINDING_TYPE_PATH)
     root = root.getroot()

boefjes/boefjes/plugins/kat_dicom/boefje.json

@@ -6,5 +6,9 @@
     "IPAddressV4",
     "IPAddressV6"
   ],
-  "scan_level": 2
+  "scan_level": 2,
+  "oci_image": "ghcr.io/minvws/openkat/generic:latest",
+  "oci_arguments": [
+    "kat_dicom.main"
+  ]
 }

boefjes/boefjes/plugins/kat_dicom/main.py

@@ -3,11 +3,9 @@
 
 from pynetdicom import AE
 
-from boefjes.job_models import BoefjeMeta
 
-
-def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]:
-    input_ = boefje_meta.arguments["input"]
+def run(boefje_meta: dict) -> list[tuple[set, bytes | str]]:
+    input_ = boefje_meta["arguments"]["input"]
     ip = input_["address"]
 
     # it prints errors if the port is not open, ignore these errors as we expect them to happen a lot

boefjes/boefjes/plugins/kat_dicom/requirements.txt

@@ -0,0 +1,2 @@
+pynetdicom == 2.0.2
+pydicom == 2.4.4

boefjes/boefjes/plugins/kat_dns/boefje.json

@@ -5,5 +5,9 @@
   "consumes": [
     "Hostname"
   ],
-  "scan_level": 1
+  "scan_level": 1,
+  "oci_image": "ghcr.io/minvws/openkat/generic:latest",
+  "oci_arguments": [
+    "kat_dns.main"
+  ]
 }

boefjes/boefjes/plugins/kat_dns/main.py

@@ -10,8 +10,6 @@
 from dns.name import Name
 from dns.resolver import Answer
 
-from boefjes.job_models import BoefjeMeta
-
 logger = logging.getLogger(__name__)
 DEFAULT_RECORD_TYPES = {"A", "AAAA", "CAA", "CERT", "RP", "SRV", "TXT", "MX", "NS", "CNAME", "DNAME", "SOA"}
 
@@ -34,8 +32,8 @@ def get_record_types() -> set[str]:
     return set(parsed_requested_record_types).intersection(DEFAULT_RECORD_TYPES)
 
 
-def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]:
-    hostname = boefje_meta.arguments["input"]["name"]
+def run(boefje_meta: dict) -> list[tuple[set, bytes | str]]:
+    hostname = boefje_meta["arguments"]["input"]["name"]
 
     requested_dns_name = dns.name.from_text(hostname)
     resolver = dns.resolver.Resolver()

boefjes/boefjes/plugins/kat_dns/requirements.txt

@@ -0,0 +1 @@
+dnspython==2.6.1

boefjes/boefjes/plugins/kat_dns_version/boefje.json

@@ -5,5 +5,9 @@
   "consumes": [
     "IPService"
   ],
-  "scan_level": 2
+  "scan_level": 2,
+  "oci_image": "ghcr.io/minvws/openkat/generic:latest",
+  "oci_arguments": [
+    "kat_dns_version.main"
+  ]
 }

boefjes/boefjes/plugins/kat_dns_version/main.py

@@ -7,11 +7,9 @@
 import dns.message
 import dns.query
 
-from boefjes.job_models import BoefjeMeta
 
-
-def run(boefje_meta: BoefjeMeta) -> list[tuple[set, str | bytes]]:
-    input_ = boefje_meta.arguments["input"]  # input is IPService
+def run(boefje_meta: dict) -> list[tuple[set, str | bytes]]:
+    input_ = boefje_meta["arguments"]["input"]  # input is IPService
     ip_port = input_["ip_port"]
     if input_["service"]["name"] != "domain":
         return [({"error/boefje"}, "Not a DNS service")]

boefjes/boefjes/plugins/kat_dns_zone/boefje.json

@@ -5,5 +5,9 @@
   "consumes": [
     "DNSZone"
   ],
-  "scan_level": 1
+  "scan_level": 1,
+  "oci_image": "ghcr.io/minvws/openkat/generic:latest",
+  "oci_arguments": [
+    "kat_dns_zone.main"
+  ]
 }

boefjes/boefjes/plugins/kat_dns_zone/main.py

@@ -6,17 +6,15 @@
 from dns.name import Name
 from dns.resolver import Answer
 
-from boefjes.job_models import BoefjeMeta
-
 logger = logging.getLogger(__name__)
 
 
 class ZoneNotFoundException(Exception):
     pass
 
 
-def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]:
-    input_ = boefje_meta.arguments["input"]
+def run(boefje_meta: dict) -> list[tuple[set, bytes | str]]:
+    input_ = boefje_meta["arguments"]["input"]
     zone_ooi = input_["hostname"]["name"]
 
     zone_name = dns.name.from_text(zone_ooi)

boefjes/boefjes/plugins/kat_dns_zone/requirements.txt

@@ -0,0 +1 @@
+dnspython==2.6.1

boefjes/boefjes/plugins/kat_external_db/boefje.json

@@ -5,5 +5,9 @@
   "consumes": [
     "Network"
   ],
-  "scan_level": 0
+  "scan_level": 0,
+  "oci_image": "ghcr.io/minvws/openkat/generic:latest",
+  "oci_arguments": [
+    "kat_external_db.main"
+  ]
 }