Merge branch 'main' into feature/mime-types-from-bytes

.github/ISSUE_TEMPLATE/user_story.md

@@ -0,0 +1,15 @@
+---
+name: User story
+about: Create a task that is phrased as a user story
+title: ''
+labels: ''
+assignees: ''
+---
+
+## User Story
+As a [type of user],
+I want [some goal or desired outcome],
+so that [some reason or benefit].
+
+## Acceptance Criteria:
+- [List specific conditions or criteria that must be met for the story to be considered complete.]

.github/pull_request_template.md

@@ -6,8 +6,8 @@ _Please add a link to the issue after "Closes". If there is no issue for this PR
 
 Closes ...
 
-### Proof
-_Please add some proof of your working change here, unless this is not required (e.g. this PR is trivial)._
+### Demo
+_Please add some proof in the form of screenshots or screen recordings to show (off) new functionality, if there are interesting new features for end-users._
 
 ---
 

README.rst

@@ -2,14 +2,6 @@
 What is OpenKAT?
 ================
 
-.. image:: https://hosted.weblate.org/widget/openkat/287x66-white.png
-   :target: https://hosted.weblate.org/engage/openkat/
-   :alt: Translation status (summary)
-
-.. image:: https://hosted.weblate.org/widget/openkat/multi-auto.svg
-   :target: https://hosted.weblate.org/engage/openkat/
-   :alt: Translation status (bar chart)
-
 OpenKAT aims to monitor, record and analyze the status of information systems. The basic premise is that many of the major security incidents are caused by small errors and known vulnerabilities, and that if you can find them in time your systems and infrastructure become a lot more secure.
 
 OpenKAT scans, collects, analyzes and reports in an ongoing process:
@@ -42,6 +34,13 @@ The high level documentation on OpenKAT explains the purpose and operation of Op
 
 Translations
 ============
+.. image:: https://hosted.weblate.org/widget/openkat/287x66-white.png
+   :target: https://hosted.weblate.org/engage/openkat/
+   :alt: Translation status (summary)
+
+.. image:: https://hosted.weblate.org/widget/openkat/multi-auto.svg
+   :target: https://hosted.weblate.org/engage/openkat/
+   :alt: Translation status (bar chart)
 
 We gratefully use `Weblate <https://hosted.weblate.org/engage/openkat/>`_ to manage the translations.
 See `the docs <https://docs.openkat.nl/guidelines/contributions.html#contribute-translations>`_ for more information.

bytes/bytes/database/sql_meta_repository.py

@@ -96,14 +96,14 @@ def get_normalizer_meta_by_id(self, normalizer_meta_id: uuid.UUID) -> Normalizer
     def get_normalizer_meta(self, query_filter: NormalizerMetaFilter) -> List[NormalizerMeta]:
         logger.debug("Querying normalizer meta: %s", query_filter.json())
 
+        query = self.session.query(NormalizerMetaInDB)
+
         if query_filter.raw_id is not None:
-            query = self.session.query(NormalizerMetaInDB).filter(
-                NormalizerMetaInDB.raw_file_id == str(query_filter.raw_id)
-            )
-        else:
+            query = query.filter(NormalizerMetaInDB.raw_file_id == str(query_filter.raw_id))
+
+        if query_filter.organization is not None:
             query = (
-                self.session.query(NormalizerMetaInDB)
-                .join(RawFileInDB)
+                query.join(RawFileInDB)
                 .join(BoefjeMetaInDB)
                 .filter(RawFileInDB.boefje_meta_id == BoefjeMetaInDB.id)
                 .filter(BoefjeMetaInDB.organization == query_filter.organization)
@@ -143,7 +143,24 @@ def save_raw(self, raw: RawData) -> uuid.UUID:
 
     def get_raw(self, query_filter: RawDataFilter) -> List[RawDataMeta]:
         logger.debug("Querying raw data: %s", query_filter.json())
-        query = query_filter.apply(self.session.query(RawFileInDB))
+        query = self.session.query(RawFileInDB)
+
+        if query_filter.boefje_meta_id:
+            query = query.filter(RawFileInDB.boefje_meta_id == str(query_filter.boefje_meta_id))
+
+        if query_filter.organization:
+            query = query.join(BoefjeMetaInDB).filter(BoefjeMetaInDB.organization == query_filter.organization)
+
+        if query_filter.normalized:
+            query = query.join(NormalizerMetaInDB, isouter=False)
+
+        if query_filter.normalized is False:  # it can also be None, in which case we do not want a filter
+            query = query.join(NormalizerMetaInDB, isouter=True).filter(NormalizerMetaInDB.id.is_(None))
+
+        if query_filter.mime_types:
+            query = query.filter(RawFileInDB.mime_types.contains([m.value for m in query_filter.mime_types]))
+
+        query = query.offset(query_filter.offset).limit(query_filter.limit)
 
         return [to_raw_meta(raw_file_in_db) for raw_file_in_db in query]
 

bytes/tests/conftest.py

@@ -84,11 +84,14 @@ def bytes_api_client(settings) -> Iterator[BytesAPIClient]:
     alembicArgs = ["--config", "/app/bytes/bytes/alembic.ini", "--raiseerr", "upgrade", "head"]
     alembic.config.main(argv=alembicArgs)
 
-    yield BytesAPIClient(
+    client = BytesAPIClient(
         "http://ci_bytes:8000",
         settings.username,
         settings.password,
     )
+    client.login()
+
+    yield client
 
     sessionmaker(bind=get_engine(settings.db_uri), autocommit=True)().execute(
         ";".join([f"TRUNCATE TABLE {t} CASCADE" for t in SQL_BASE.metadata.tables])

bytes/tests/integration/test_meta_repository.py

@@ -170,7 +170,35 @@ def test_filter_raw_on_organization(meta_repository: SQLMetaDataRepository) -> N
     assert len(meta_repository.get_raw(query_filter)) == 1
 
     query_filter.organization = "test2"
-    assert len(meta_repository.get_raw(query_filter)) == 1
+    assert len(meta_repository.get_raw(query_filter)) == 0
+
+
+def test_filter_raw_not_on_organization(meta_repository: SQLMetaDataRepository) -> None:
+    raw = get_raw_data()
+    raw2 = get_raw_data()
+    raw2.boefje_meta.id = "c5d7d1da-7d94-4ac4-b0f6-ac065eeb0c97"
+
+    with meta_repository:
+        meta_repository.save_boefje_meta(raw.boefje_meta)
+        meta_repository.save_raw(raw)
+        meta_repository.save_raw(raw)
+        meta_repository.save_raw(raw)
+        meta_repository.save_raw(raw)
+        meta_repository.save_raw(raw)
+        meta_repository.save_boefje_meta(raw2.boefje_meta)
+        meta_repository.save_raw(raw2)
+        meta_repository.save_raw(raw2)
+
+    # Test offset-limit
+    assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=2))) == 2
+    assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=6))) == 5
+    assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=2, offset=1))) == 2
+    assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=2, offset=4))) == 1
+
+    # Test without boefje_meta id
+    assert len(meta_repository.get_raw(RawDataFilter(limit=2, offset=4))) == 2
+    assert len(meta_repository.get_raw(RawDataFilter(limit=100))) == 7
+    assert len(meta_repository.get_raw(RawDataFilter(limit=100, normalized=False))) == 7
 
 
 def test_filter_normalizer_meta(meta_repository: SQLMetaDataRepository) -> None:

bytes/tests/unit/test_raw_repository.py

@@ -1,12 +1,10 @@
 import uuid
 
 import pytest
-from pydantic import ValidationError
 
 from bytes.config import get_settings
 from bytes.raw.file_raw_repository import FileRawRepository
 from bytes.raw.middleware import NaclBoxMiddleware
-from bytes.repositories.meta_repository import RawDataFilter
 from tests.loading import get_raw_data
 
 
@@ -34,11 +32,3 @@ def test_nacl_middleware(nacl_middleware: NaclBoxMiddleware) -> None:
 
     assert encrypted != msg
     assert decrypted == msg
-
-
-def test_filter_validator() -> None:
-    with pytest.raises(ValidationError):
-        RawDataFilter()
-
-    RawDataFilter(organization="Not None")
-    RawDataFilter(boefje_meta_id="3a9f1b33-f703-4c32-b0ae-44d0473c8aa5")

docs/source/guidelines/contributions.rst

@@ -82,6 +82,19 @@ Any translation updates in Weblate will be automatically submitted as a GitHub P
 If you contribute to the translation effort, you will receive a mention in the source code.
 
 Note that editing the English localization requires changing the source string in Django, which must be done through a GitHub PR manually.
+In addition, take care not to edit the ``.po`` files through GitHub directly, as this may cause merge conflicts when Weblate has pending translations.
+
+Adding a new language
+---------------------
+You can add a new language to OpenKAT by following these steps:
+
+1. Add a new language through Weblate's interface
+    Alternatively: create a new ``rocky/rocky/locale/$LANGUAGE_CODE/LC_MESSAGES/django.po`` file directly in the repository.
+    You can use ``rocky/rocky/locale/django.pot`` as a template.
+2. Add the language code to the ``LANGUAGES`` list in ``rocky/rocky/settings.py``
+    Note that languages not supported by Django require that you also add a custom dictionary entry in ``EXTRA_LANG_INFO``.
+
+The new language should be automatically picked up by both Weblate and Django.
 
 Contributor Social Contract
 ===========================

docs/source/guidelines/feature_flow.md

@@ -81,3 +81,19 @@ Ideally we would follow the following QA procedure on each of these environments
 
 #### Checking Performance
 - [ ] Verify that there is no significant performance regression
+
+---
+
+## Tips and tricks for pull request QA testing
+
+### Think outside the box
+- Feel free to deviate from the checklist: testing things that are not obviously related to the PR is a good way to find bugs.
+- Thoroughness is key: embrace the "hacker mindset" and try to break (new) functionality by providing unexpected input, and attempt to perform unauthorized actions.
+- Try to break the UI: try resizing the window, using zoom functionality, and test multiple browsers.
+- Always remember that you are taking on the role of a user that is probably not as familiar with the application as you are: everything you encounter should feel intuitive and easy to use. Lack of intuitiveness deserves a QA comment.
+
+### Be pragmatic but versatile
+- Features updating the data model should usually be backward compatible, so we should not run `make reset` upon every review. Switch tactics with respect to updating your local environment regularly.
+- Small documentation changes do not require rebuilding and restarting all services to performa a QA review.
+- Properly gauge the impact of a feature: API changes in the KATalogus, for example, can affect Rocky, Mula and Octopoes, but never Bytes (in the current setup).
+- Changes that hit the core of every service (package updates) require performing the extended QA checklist.

mula/scheduler/models/__init__.py

@@ -1,7 +1,6 @@
 from .base import Base
 from .boefje import Boefje, BoefjeMeta
 from .events import RawData, RawDataReceivedEvent
-from .filter import Filter
 from .health import ServiceHealth
 from .normalizer import Normalizer
 from .ooi import OOI, MutationOperationType, ScanProfile, ScanProfileMutation

mula/scheduler/queues/pq.py

@@ -3,7 +3,7 @@
 import abc
 import logging
 import threading
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Optional
 
 import pydantic
 
@@ -90,12 +90,12 @@ def __init__(
         self.pq_store: storage.PriorityQueueStore = pq_store
         self.lock: threading.Lock = threading.Lock()
 
-    def pop(self, filters: Optional[List[models.Filter]] = None) -> Optional[models.PrioritizedItem]:
+    def pop(self, filters: Optional[storage.filters.FilterRequest] = None) -> Optional[models.PrioritizedItem]:
         """Remove and return the highest priority item from the queue.
         Optionally apply filters to the queue.
 
         Args:
-            filters: A list of filters to be applied to the queue.
+            filters: A FilterRequest instance that defines the filters
 
         Returns:
             The highest priority item from the queue.

mula/scheduler/schedulers/boefje.py

@@ -20,6 +20,7 @@
     ScanProfileMutation,
     TaskStatus,
 )
+from scheduler.storage import filters
 
 from .scheduler import Scheduler
 
@@ -144,13 +145,16 @@ def push_tasks_for_scan_profile_mutations(self, mutation: ScanProfileMutation) -
             # remove them from the queue.
             items, _ = self.ctx.datastores.pq_store.get_items(
                 scheduler_id=self.scheduler_id,
-                filters=[
-                    models.Filter(
-                        field="input_ooi",
-                        operator="eq",
-                        value=ooi.primary_key,
-                    ),
-                ],
+                filters=filters.FilterRequest(
+                    filters=[
+                        filters.Filter(
+                            column="data",
+                            field="input_ooi",
+                            operator="eq",
+                            value=ooi.primary_key,
+                        ),
+                    ],
+                ),
             )
 
             # Delete all items for this ooi, update all tasks for this ooi

mula/scheduler/schedulers/scheduler.py

@@ -6,7 +6,7 @@
 from datetime import datetime, timezone
 from typing import Any, Callable, Dict, List, Optional
 
-from scheduler import connectors, context, models, queues, utils
+from scheduler import connectors, context, models, queues, storage, utils
 from scheduler.utils import thread
 
 
@@ -132,11 +132,13 @@ def post_pop(self, p_item: models.PrioritizedItem) -> None:
 
         return None
 
-    def pop_item_from_queue(self, filters: Optional[List[models.Filter]] = None) -> Optional[models.PrioritizedItem]:
+    def pop_item_from_queue(
+        self, filters: Optional[storage.filters.FilterRequest] = None
+    ) -> Optional[models.PrioritizedItem]:
         """Pop an item from the queue.
 
         Args:
-            filters: A list of filters to apply to get a filtered set of
+            filters: A FilterRequest instance to filter the
             prioritized items from the queue.
 
         Returns: