Feat/security legacy urls

boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json

@@ -504,45 +504,52 @@
         "recommendation": "Ideally to minimize the attack surface as much as possible these panels should not be directly exposed to the internet"
     },
     "KAT-CRITICAL-BAD-CIPHER": {
-          "description": "Ciphers are used that are labeled as bad. These should not be used anymore",
-          "source": null,
-          "risk": "critical",
-          "impact": null,
-          "recommendation": "It is recommended to only use ciphers labelled as 'good': TLS_AES_256_GCM_SHA384, TLS_CHACHA20_POLY1305_SHA256, TLS_AES_128_GCM_SHA256"
+        "description": "Ciphers are used that are labeled as bad. These should not be used anymore",
+        "source": null,
+        "risk": "critical",
+        "impact": null,
+        "recommendation": "It is recommended to only use ciphers labelled as 'good': TLS_AES_256_GCM_SHA384, TLS_CHACHA20_POLY1305_SHA256, TLS_AES_128_GCM_SHA256"
     },
     "KAT-MEDIUM-BAD-CIPHER": {
-          "description": "Ciphers are used that are labeled as bad. These should not be used anymore",
-          "source": null,
-          "risk": "medium",
-          "impact": null,
-          "recommendation": "It is recommended to only use ciphers labelled as 'good': TLS_AES_256_GCM_SHA384, TLS_CHACHA20_POLY1305_SHA256, TLS_AES_128_GCM_SHA256"
+        "description": "Ciphers are used that are labeled as bad. These should not be used anymore",
+        "source": null,
+        "risk": "medium",
+        "impact": null,
+        "recommendation": "It is recommended to only use ciphers labelled as 'good': TLS_AES_256_GCM_SHA384, TLS_CHACHA20_POLY1305_SHA256, TLS_AES_128_GCM_SHA256"
     },
     "KAT-RECOMMENDATION-BAD-CIPHER": {
-          "description": "Ciphers are used that are labeled as bad. These should not be used anymore",
-          "source": null,
-          "risk": "recommendation",
-          "impact": null,
-          "recommendation": "It is recommended to only use ciphers labelled as 'good': TLS_AES_256_GCM_SHA384, TLS_CHACHA20_POLY1305_SHA256, TLS_AES_128_GCM_SHA256"
+        "description": "Ciphers are used that are labeled as bad. These should not be used anymore",
+        "source": null,
+        "risk": "recommendation",
+        "impact": null,
+        "recommendation": "It is recommended to only use ciphers labelled as 'good': TLS_AES_256_GCM_SHA384, TLS_CHACHA20_POLY1305_SHA256, TLS_AES_128_GCM_SHA256"
     },
     "KAT-NO-RPKI": {
-            "description": "The IP address does not have a route announcement that is matched by the published Route Policy and Authorization (RPKI)",
-            "source": null,
-            "risk": "low",
-            "impact": "Without RPKI validation, your servers might be more vulnerable to unintended or malicious routing configuration errors, potentially leading to inaccessibility of your servers or interception of internet traffic directed to them.",
-            "recommendation": "Work on implementing RPKI for your IP addresses. This may involve creating Route Origin Authorizations (ROAs) that specify which Autonomous Systems (AS) are authorized to announce your IP addresses."
+        "description": "The IP address does not have a route announcement that is matched by the published Route Policy and Authorization (RPKI)",
+        "source": null,
+        "risk": "low",
+        "impact": "Without RPKI validation, your servers might be more vulnerable to unintended or malicious routing configuration errors, potentially leading to inaccessibility of your servers or interception of internet traffic directed to them.",
+        "recommendation": "Work on implementing RPKI for your IP addresses. This may involve creating Route Origin Authorizations (ROAs) that specify which Autonomous Systems (AS) are authorized to announce your IP addresses."
     },
     "KAT-EXPIRED-RPKI": {
-            "description": "The route announcement that is matched by the published Route Policy and Authorization (RPKI) is expired",
-            "source": null,
-            "risk": "low",
-            "impact": "Without RPKI validation, your servers might be more vulnerable to unintended or malicious routing configuration errors, potentially leading to inaccessibility of your servers or interception of internet traffic directed to them.",
-            "recommendation": "Make sure that the Route Origin Authorizations (ROAs) that specify which Autonomous Systems (AS) are authorized to announce your IP addresses are valid and not expired."
+        "description": "The route announcement that is matched by the published Route Policy and Authorization (RPKI) is expired",
+        "source": null,
+        "risk": "low",
+        "impact": "Without RPKI validation, your servers might be more vulnerable to unintended or malicious routing configuration errors, potentially leading to inaccessibility of your servers or interception of internet traffic directed to them.",
+        "recommendation": "Make sure that the Route Origin Authorizations (ROAs) that specify which Autonomous Systems (AS) are authorized to announce your IP addresses are valid and not expired."
     },
     "KAT-NO-CAA": {
         "description": "This zone does not carry at least one CAA record.",
         "source": null,
         "risk": "low",
         "impact": null,
         "recommendation": "Set a CAA record to limit which CA's are allowed to issue certs."
+    },
+    "KAT-LEGACY-SECURITY-LOCATION": {
+        "description": "This website only has a legacy location security.txt file.",
+        "source": "https://www.rfc-editor.org/rfc/rfc9116#section-3-1",
+        "risk": "info",
+        "impact": null,
+        "recommendation": "Add a security.txt file location in the .well-known folder."
     }
 }

boefjes/boefjes/plugins/kat_security_txt_downloader/boefje.json

@@ -5,5 +5,8 @@
     "consumes": [
         "Website"
     ],
+    "environment_keys": [
+        "USERAGENT"
+    ],
     "scan_level": 2
 }

boefjes/boefjes/plugins/kat_security_txt_downloader/main.py

@@ -27,17 +27,17 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]:
             session.mount(uri, ForcedIPHTTPSAdapter(dest_ip=ip))
         else:
             addr = ipaddress.ip_address(ip)
-            netloc = f"[{ip}]" if addr.version == 6 else ip
-
-            uri = f"{scheme}://{netloc}/{path}"
+            iploc = f"[{ip}]" if addr.version == 6 else ip
+            uri = f"{scheme}://{iploc}/{path}"
 
         response = do_request(netloc, session, uri, useragent)
 
         # if the response is 200, return the content
         if response.status_code == 200:
             results[path] = {"content": response.content.decode(), "url": response.url, "ip": ip, "status": 200}
         # if the response is 301, we need to follow the location header to the correct security txt,
-        # we can not force the ip anymore
+        # we can not force the ip anymore because we dont know it yet.
+        # TODO return a redirected URL and have OpenKAT figure out if we want to follow this.
         elif response.status_code in [301, 302, 307, 308]:
             uri = response.headers["Location"]
             response = requests.get(uri, stream=True, timeout=30, verify=False)  # noqa: S501

boefjes/boefjes/plugins/kat_security_txt_downloader/normalize.py

@@ -9,6 +9,7 @@
 from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network
 from octopoes.models.ooi.service import IPService, Service
 from octopoes.models.ooi.web import URL, SecurityTXT, Website
+from octopoes.models.types import Finding, KATFindingType
 
 
 def run(normalizer_meta: NormalizerMeta, raw: bytes | str) -> Iterable[OOI]:
@@ -17,6 +18,7 @@ def run(normalizer_meta: NormalizerMeta, raw: bytes | str) -> Iterable[OOI]:
     website_original = Reference.from_str(boefje_meta.input_ooi)
     input_ = boefje_meta.arguments["input"]
 
+    location_rfc_compliant = False
     for path, details in results.items():
         if details["content"] is None:
             continue
@@ -27,6 +29,16 @@ def run(normalizer_meta: NormalizerMeta, raw: bytes | str) -> Iterable[OOI]:
         yield url_original
         url = URL(raw=details["url"], network=Network(name=input_["hostname"]["network"]["name"]).reference)
         yield url
+
+        # Check for legacy url https://www.rfc-editor.org/rfc/rfc9116#section-3-1
+        if path == ".well-known/security.txt":
+            location_rfc_compliant = True
+        elif path == "security.txt" and not location_rfc_compliant:
+            ft = KATFindingType(id="KAT-LEGACY-SECURITY-LOCATION")
+            yield ft
+            yield Finding(
+                description="Only legacy Security.txt location found.", finding_type=ft.reference, ooi=url.reference
+            )
         url_parts = urlparse(details["url"])
         # we need to check if the website of the response is the same as the input website
         if (