Merge branch 'main' into feature/misc-language-updates

.github/ISSUE_TEMPLATE/user_story.md

@@ -0,0 +1,15 @@
+---
+name: User story
+about: Create a task that is phrased as a user story
+title: ''
+labels: ''
+assignees: ''
+---
+
+## User Story
+As a [type of user],
+I want [some goal or desired outcome],
+so that [some reason or benefit].
+
+## Acceptance Criteria:
+- [List specific conditions or criteria that must be met for the story to be considered complete.]

.github/pull_request_template.md

@@ -6,8 +6,8 @@ _Please add a link to the issue after "Closes". If there is no issue for this PR
 
 Closes ...
 
-### Proof
-_Please add some proof of your working change here, unless this is not required (e.g. this PR is trivial)._
+### Demo
+_Please add some proof in the form of screenshots or screen recordings to show (off) new functionality, if there are interesting new features for end-users._
 
 ---
 

README.rst

@@ -2,14 +2,6 @@
 What is OpenKAT?
 ================
 
-.. image:: https://hosted.weblate.org/widget/openkat/287x66-white.png
-   :target: https://hosted.weblate.org/engage/openkat/
-   :alt: Translation status (summary)
-
-.. image:: https://hosted.weblate.org/widget/openkat/multi-auto.svg
-   :target: https://hosted.weblate.org/engage/openkat/
-   :alt: Translation status (bar chart)
-
 OpenKAT aims to monitor, record and analyze the status of information systems. The basic premise is that many of the major security incidents are caused by small errors and known vulnerabilities, and that if you can find them in time your systems and infrastructure become a lot more secure.
 
 OpenKAT scans, collects, analyzes and reports in an ongoing process:
@@ -42,6 +34,13 @@ The high level documentation on OpenKAT explains the purpose and operation of Op
 
 Translations
 ============
+.. image:: https://hosted.weblate.org/widget/openkat/287x66-white.png
+   :target: https://hosted.weblate.org/engage/openkat/
+   :alt: Translation status (summary)
+
+.. image:: https://hosted.weblate.org/widget/openkat/multi-auto.svg
+   :target: https://hosted.weblate.org/engage/openkat/
+   :alt: Translation status (bar chart)
 
 We gratefully use `Weblate <https://hosted.weblate.org/engage/openkat/>`_ to manage the translations.
 See `the docs <https://docs.openkat.nl/guidelines/contributions.html#contribute-translations>`_ for more information.

boefjes/boefjes/job_handler.py

@@ -27,6 +27,7 @@
 from octopoes.models.types import OOIType
 
 logger = logging.getLogger(__name__)
+
 bytes_api_client = BytesAPIClient(
     settings.bytes_api,
     username=settings.bytes_username,
@@ -153,7 +154,6 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
             boefje_meta.ended_at = datetime.now(timezone.utc)
             logger.info("Saving to Bytes for boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
 
-            bytes_api_client.login()
             bytes_api_client.save_boefje_meta(boefje_meta)
 
             if boefje_results:
@@ -175,7 +175,6 @@ def __init__(self, job_runner):
     def handle(self, normalizer_meta: NormalizerMeta) -> None:
         logger.info("Handling normalizer %s[%s]", normalizer_meta.normalizer.id, normalizer_meta.id)
 
-        bytes_api_client.login()
         raw = bytes_api_client.get_raw(normalizer_meta.raw_data.id)
 
         normalizer_meta.started_at = datetime.now(timezone.utc)

boefjes/boefjes/plugins/kat_binaryedge/schema.json

@@ -11,5 +11,8 @@
   },
   "required": [
     "BINARYEDGE_API"
+  ],
+  "secret": [
+    "BINARYEDGE_API"
   ]
 }

boefjes/boefjes/plugins/kat_censys/schema.json

@@ -18,5 +18,8 @@
     "required": [
         "CENSYS_API_ID",
         "CENSYS_API_SECRET"
+    ],
+    "secret": [
+        "CENSYS_API_SECRET"
     ]
 }

boefjes/boefjes/plugins/kat_external_db/schema.json

@@ -29,5 +29,8 @@
     },
     "required": [
         "DB_URL"
+    ],
+    "secret": [
+      "DB_ACCESS_TOKEN"
     ]
 }

boefjes/boefjes/plugins/kat_leakix/schema.json

@@ -11,5 +11,8 @@
   },
   "required": [
     "LEAKIX_API"
+  ],
+  "secret": [
+    "LEAKIX_API"
   ]
 }

boefjes/boefjes/plugins/kat_shodan/schema.json

@@ -11,5 +11,8 @@
   },
   "required": [
     "SHODAN_API"
+  ],
+  "secret": [
+    "SHODAN_API"
   ]
 }

boefjes/boefjes/plugins/kat_wpscan/schema.json

@@ -9,5 +9,7 @@
       "description": "An optional WordPress Vulnerability Database API token to retrieve WordPress vulnerability data in real time. See https://wpscan.com/api and https://github.com/wpscanteam/wpscan/wiki/WPScan-User-Documentation#optional-wordpress-vulnerability-database-api for more information."
     }
   },
-  "required": []
+  "secret": [
+    "WP_SCAN_API"
+  ]
 }

bytes/bytes/database/sql_meta_repository.py

@@ -96,14 +96,14 @@ def get_normalizer_meta_by_id(self, normalizer_meta_id: uuid.UUID) -> Normalizer
     def get_normalizer_meta(self, query_filter: NormalizerMetaFilter) -> List[NormalizerMeta]:
         logger.debug("Querying normalizer meta: %s", query_filter.json())
 
+        query = self.session.query(NormalizerMetaInDB)
+
         if query_filter.raw_id is not None:
-            query = self.session.query(NormalizerMetaInDB).filter(
-                NormalizerMetaInDB.raw_file_id == str(query_filter.raw_id)
-            )
-        else:
+            query = query.filter(NormalizerMetaInDB.raw_file_id == str(query_filter.raw_id))
+
+        if query_filter.organization is not None:
             query = (
-                self.session.query(NormalizerMetaInDB)
-                .join(RawFileInDB)
+                query.join(RawFileInDB)
                 .join(BoefjeMetaInDB)
                 .filter(RawFileInDB.boefje_meta_id == BoefjeMetaInDB.id)
                 .filter(BoefjeMetaInDB.organization == query_filter.organization)
@@ -143,17 +143,13 @@ def save_raw(self, raw: RawData) -> uuid.UUID:
 
     def get_raw(self, query_filter: RawDataFilter) -> List[RawDataMeta]:
         logger.debug("Querying raw data: %s", query_filter.json())
+        query = self.session.query(RawFileInDB)
 
         if query_filter.boefje_meta_id:
-            query = self.session.query(RawFileInDB).filter(
-                RawFileInDB.boefje_meta_id == str(query_filter.boefje_meta_id)
-            )
-        else:
-            query = (
-                self.session.query(RawFileInDB)
-                .join(BoefjeMetaInDB)
-                .filter(BoefjeMetaInDB.organization == query_filter.organization)
-            )
+            query = query.filter(RawFileInDB.boefje_meta_id == str(query_filter.boefje_meta_id))
+
+        if query_filter.organization:
+            query = query.join(BoefjeMetaInDB).filter(BoefjeMetaInDB.organization == query_filter.organization)
 
         if query_filter.normalized:
             query = query.join(NormalizerMetaInDB, isouter=False)

bytes/bytes/repositories/meta_repository.py

@@ -1,7 +1,7 @@
-from typing import Any, Dict, List, Optional, Type
+from typing import Dict, List, Optional, Type
 from uuid import UUID
 
-from pydantic import BaseModel, Field, root_validator
+from pydantic import BaseModel, Field
 
 from bytes.models import BoefjeMeta, MimeType, NormalizerMeta, RawData, RawDataMeta
 
@@ -33,15 +33,6 @@ class RawDataFilter(BaseModel):
     limit: int = 1
     offset: int = 0
 
-    @root_validator(pre=False)
-    def either_organization_or_boefje_meta_id(  # pylint: disable=no-self-argument
-        cls, values: Dict[str, Any]
-    ) -> Dict[str, Any]:
-        if values.get("organization") or values.get("boefje_meta_id"):
-            return values
-
-        raise ValueError("boefje_meta_id and organization cannot both be None.")
-
 
 class MetaDataRepository:
     def __enter__(self) -> None:

bytes/tests/conftest.py

@@ -84,11 +84,14 @@ def bytes_api_client(settings) -> Iterator[BytesAPIClient]:
     alembicArgs = ["--config", "/app/bytes/bytes/alembic.ini", "--raiseerr", "upgrade", "head"]
     alembic.config.main(argv=alembicArgs)
 
-    yield BytesAPIClient(
+    client = BytesAPIClient(
         "http://ci_bytes:8000",
         settings.username,
         settings.password,
     )
+    client.login()
+
+    yield client
 
     sessionmaker(bind=get_engine(settings.db_uri), autocommit=True)().execute(
         ";".join([f"TRUNCATE TABLE {t} CASCADE" for t in SQL_BASE.metadata.tables])

bytes/tests/integration/test_meta_repository.py

@@ -170,7 +170,35 @@ def test_filter_raw_on_organization(meta_repository: SQLMetaDataRepository) -> N
     assert len(meta_repository.get_raw(query_filter)) == 1
 
     query_filter.organization = "test2"
-    assert len(meta_repository.get_raw(query_filter)) == 1
+    assert len(meta_repository.get_raw(query_filter)) == 0
+
+
+def test_filter_raw_not_on_organization(meta_repository: SQLMetaDataRepository) -> None:
+    raw = get_raw_data()
+    raw2 = get_raw_data()
+    raw2.boefje_meta.id = "c5d7d1da-7d94-4ac4-b0f6-ac065eeb0c97"
+
+    with meta_repository:
+        meta_repository.save_boefje_meta(raw.boefje_meta)
+        meta_repository.save_raw(raw)
+        meta_repository.save_raw(raw)
+        meta_repository.save_raw(raw)
+        meta_repository.save_raw(raw)
+        meta_repository.save_raw(raw)
+        meta_repository.save_boefje_meta(raw2.boefje_meta)
+        meta_repository.save_raw(raw2)
+        meta_repository.save_raw(raw2)
+
+    # Test offset-limit
+    assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=2))) == 2
+    assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=6))) == 5
+    assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=2, offset=1))) == 2
+    assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=2, offset=4))) == 1
+
+    # Test without boefje_meta id
+    assert len(meta_repository.get_raw(RawDataFilter(limit=2, offset=4))) == 2
+    assert len(meta_repository.get_raw(RawDataFilter(limit=100))) == 7
+    assert len(meta_repository.get_raw(RawDataFilter(limit=100, normalized=False))) == 7
 
 
 def test_filter_normalizer_meta(meta_repository: SQLMetaDataRepository) -> None:

bytes/tests/unit/test_raw_repository.py

@@ -1,12 +1,10 @@
 import uuid
 
 import pytest
-from pydantic import ValidationError
 
 from bytes.config import get_settings
 from bytes.raw.file_raw_repository import FileRawRepository
 from bytes.raw.middleware import NaclBoxMiddleware
-from bytes.repositories.meta_repository import RawDataFilter
 from tests.loading import get_raw_data
 
 
@@ -34,11 +32,3 @@ def test_nacl_middleware(nacl_middleware: NaclBoxMiddleware) -> None:
 
     assert encrypted != msg
     assert decrypted == msg
-
-
-def test_filter_validator() -> None:
-    with pytest.raises(ValidationError):
-        RawDataFilter()
-
-    RawDataFilter(organization="Not None")
-    RawDataFilter(boefje_meta_id="3a9f1b33-f703-4c32-b0ae-44d0473c8aa5")

docs/source/developer_documentation/boefjes.md

@@ -3,7 +3,7 @@
 This module has several entry points discussed below, but let us first consider the prerequisites and scope.
 If you already have running setup and want to learn where each bit of functionality goes, read the following page:
 
-[Developing Openkat Plugins](README.md#your-first-boefje)
+[Developing Openkat Plugins]([https://docs.openkat.nl/introduction/makeyourown.html])
 
 ## Prerequisites
 

docs/source/guidelines/contributions.rst

@@ -82,6 +82,19 @@ Any translation updates in Weblate will be automatically submitted as a GitHub P
 If you contribute to the translation effort, you will receive a mention in the source code.
 
 Note that editing the English localization requires changing the source string in Django, which must be done through a GitHub PR manually.
+In addition, take care not to edit the ``.po`` files through GitHub directly, as this may cause merge conflicts when Weblate has pending translations.
+
+Adding a new language
+---------------------
+You can add a new language to OpenKAT by following these steps:
+
+1. Add a new language through Weblate's interface
+    Alternatively: create a new ``rocky/rocky/locale/$LANGUAGE_CODE/LC_MESSAGES/django.po`` file directly in the repository.
+    You can use ``rocky/rocky/locale/django.pot`` as a template.
+2. Add the language code to the ``LANGUAGES`` list in ``rocky/rocky/settings.py``
+    Note that languages not supported by Django require that you also add a custom dictionary entry in ``EXTRA_LANG_INFO``.
+
+The new language should be automatically picked up by both Weblate and Django.
 
 Contributor Social Contract
 ===========================

docs/source/guidelines/feature_flow.md

@@ -81,3 +81,19 @@ Ideally we would follow the following QA procedure on each of these environments
 
 #### Checking Performance
 - [ ] Verify that there is no significant performance regression
+
+---
+
+## Tips and tricks for pull request QA testing
+
+### Think outside the box
+- Feel free to deviate from the checklist: testing things that are not obviously related to the PR is a good way to find bugs.
+- Thoroughness is key: embrace the "hacker mindset" and try to break (new) functionality by providing unexpected input, and attempt to perform unauthorized actions.
+- Try to break the UI: try resizing the window, using zoom functionality, and test multiple browsers.
+- Always remember that you are taking on the role of a user that is probably not as familiar with the application as you are: everything you encounter should feel intuitive and easy to use. Lack of intuitiveness deserves a QA comment.
+
+### Be pragmatic but versatile
+- Features updating the data model should usually be backward compatible, so we should not run `make reset` upon every review. Switch tactics with respect to updating your local environment regularly.
+- Small documentation changes do not require rebuilding and restarting all services to performa a QA review.
+- Properly gauge the impact of a feature: API changes in the KATalogus, for example, can affect Rocky, Mula and Octopoes, but never Bytes (in the current setup).
+- Changes that hit the core of every service (package updates) require performing the extended QA checklist.

docs/source/modules/api.rst

@@ -0,0 +1,4 @@
+API
+The API is used to query the status of tasks within OpenKAT. By querying a task identifier you can see if the task has completed or failed. In the future the API can be used to create your own boefjes.
+
+Boefjes receive their task input and save their output using REST APIs.

mula/scheduler/app.py

@@ -175,22 +175,23 @@ def collect_metrics(self) -> None:
 
         This method that allows to collect metrics throughout the application.
         """
-        for s in self.schedulers.values():
-            self.ctx.metrics_qsize.labels(
-                scheduler_id=s.scheduler_id,
-            ).set(
-                s.queue.qsize(),
-            )
-
-            status_counts = self.ctx.datastores.task_store.get_status_counts(s.scheduler_id)
-            for status, count in status_counts.items():
-                self.ctx.metrics_task_status_counts.labels(
+        with self.lock:
+            for s in self.schedulers.values():
+                self.ctx.metrics_qsize.labels(
                     scheduler_id=s.scheduler_id,
-                    status=status,
                 ).set(
-                    count,
+                    s.queue.qsize(),
                 )
 
+                status_counts = self.ctx.datastores.task_store.get_status_counts(s.scheduler_id)
+                for status, count in status_counts.items():
+                    self.ctx.metrics_task_status_counts.labels(
+                        scheduler_id=s.scheduler_id,
+                        status=status,
+                    ).set(
+                        count,
+                    )
+
     def run(self) -> None:
         """Start the main scheduler application, and run in threads the
         following processes: