diff --git a/requirements/base.in b/requirements/base.in index 022dd9f805..87b6380fc4 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -49,6 +49,7 @@ django-cors-headers django-decorator-include django-digid-eherkenning django-hijack +django-log-outgoing-requests django-modeltranslation django-ordered-model django-privates diff --git a/requirements/base.txt b/requirements/base.txt index 74f93fa268..a1dea2d4c5 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: # # ./bin/compile_dependencies.sh # @@ -103,6 +103,7 @@ django==3.2.19 # django-filter # django-formtools # django-hijack + # django-log-outgoing-requests # django-modeltranslation # django-otp # django-phonenumber-field @@ -166,6 +167,8 @@ django-hijack==3.1.6 # via -r requirements/base.in django-ipware==3.0.1 # via django-axes +django-log-outgoing-requests==0.1.0 + # via -r requirements/base.in django-modeltranslation==0.18.5 # via -r requirements/base.in django-ordered-model==3.6 @@ -376,6 +379,7 @@ redis==4.5.4 requests==2.31.0 # via # django-camunda + # django-log-outgoing-requests # gemma-zds-client # maykin-python3-saml # mozilla-django-oidc diff --git a/requirements/ci.txt b/requirements/ci.txt index b7e77a7c3b..cc65695680 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: # # ./bin/compile_dependencies.sh # @@ -178,6 +178,7 @@ django==3.2.19 # django-formtools # django-hijack # django-jenkins + # django-log-outgoing-requests # django-modeltranslation # django-otp # django-phonenumber-field @@ -280,6 +281,10 @@ django-ipware==3.0.1 # django-axes django-jenkins==0.110.0 # via -r requirements/test-tools.in +django-log-outgoing-requests==0.1.0 + # via + # -c requirements/base.txt + # -r requirements/base.txt django-modeltranslation==0.18.5 # via # -c requirements/base.txt @@ -725,6 +730,7 @@ requests==2.31.0 # -c requirements/base.txt # -r requirements/base.txt # django-camunda + # django-log-outgoing-requests # gemma-zds-client # maykin-python3-saml # mozilla-django-oidc diff --git a/requirements/dev.txt b/requirements/dev.txt index 2146653d0b..be39d69a59 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: # # ./bin/compile_dependencies.sh # @@ -205,6 +205,7 @@ django==3.2.19 # django-formtools # django-hijack # django-jenkins + # django-log-outgoing-requests # django-modeltranslation # django-otp # django-phonenumber-field @@ -315,6 +316,10 @@ django-jenkins==0.110.0 # via # -c requirements/ci.txt # -r requirements/ci.txt +django-log-outgoing-requests==0.1.0 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt django-modeltranslation==0.18.5 # via # -c requirements/ci.txt @@ -858,6 +863,7 @@ requests==2.31.0 # -r requirements/ci.txt # ddt-api-calls # django-camunda + # django-log-outgoing-requests # django-rosetta # django-silk # gemma-zds-client diff --git a/requirements/extensions.txt b/requirements/extensions.txt index baa57599de..395e31f022 100644 --- a/requirements/extensions.txt +++ b/requirements/extensions.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: # # ./bin/compile_dependencies.sh # @@ -142,6 +142,7 @@ django==3.2.19 # django-filter # django-formtools # django-hijack + # django-log-outgoing-requests # django-modeltranslation # django-otp # django-phonenumber-field @@ -240,6 +241,10 @@ django-ipware==3.0.1 # via # -r requirements/base.txt # django-axes +django-log-outgoing-requests==0.1.0 + # via + # -c requirements/base.in + # -r requirements/base.txt django-modeltranslation==0.18.5 # via # -c requirements/base.in @@ -579,6 +584,7 @@ requests==2.31.0 # via # -r requirements/base.txt # django-camunda + # django-log-outgoing-requests # gemma-zds-client # maykin-python3-saml # mozilla-django-oidc diff --git a/src/openforms/conf/base.py b/src/openforms/conf/base.py index f9a2081398..3bf23a9ca5 100644 --- a/src/openforms/conf/base.py +++ b/src/openforms/conf/base.py @@ -8,6 +8,7 @@ import sentry_sdk from celery.schedules import crontab from corsheaders.defaults import default_headers as default_cors_headers +from log_outgoing_requests.formatters import HttpFormatter from csp_post_processor.constants import NONCE_HTTP_HEADER @@ -185,6 +186,7 @@ "cspreports", "csp_post_processor", "django_camunda", + "log_outgoing_requests", # Project applications. "openforms.accounts", "openforms.analytics_tools", @@ -376,6 +378,7 @@ "performance": { "format": "%(asctime)s %(process)d | %(thread)d | %(message)s", }, + "outgoing_requests": {"()": HttpFormatter}, }, "filters": { "require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}, @@ -419,6 +422,15 @@ "maxBytes": 1024 * 1024 * 10, # 10 MB "backupCount": 10, }, + "log_outgoing_requests": { + "level": "DEBUG", + "formatter": "outgoing_requests", + "class": "logging.StreamHandler", + }, + "save_outgoing_requests": { + "level": "DEBUG", + "class": "log_outgoing_requests.handlers.DatabaseOutgoingRequestsHandler", + }, }, "loggers": { "openforms": { @@ -445,9 +457,27 @@ "handlers": ["project"] if not LOG_STDOUT else ["console"], "level": "DEBUG", }, + "requests": { + "handlers": ["log_outgoing_requests", "save_outgoing_requests"], + "level": "DEBUG", + "propagate": True, + }, }, } +LOG_OUTGOING_REQUESTS_DB_SAVE = True +LOG_OUTGOING_REQUESTS_DB_SAVE_BODY = True +LOG_OUTGOING_REQUESTS_CONTENT_TYPES = [ + "text/*", + "application/json", + "application/xml", + "application/soap+xml", +] +LOG_OUTGOING_REQUESTS_EMIT_BODY = True +LOG_OUTGOING_REQUESTS_MAX_AGE = config("LOG_OUTGOING_REQUESTS_MAX_AGE", default=7 * 24) +LOG_OUTGOING_REQUESTS_MAX_CONTENT_LENGTH = 524_288 # 0.5MB +LOG_OUTGOING_REQUESTS_LOG_BODY_TO_STDOUT = True + # # AUTH settings - user accounts, passwords, backends... # @@ -668,6 +698,11 @@ "task": "openforms.forms.admin.tasks.clear_forms_export", "schedule": crontab(hour=0, minute=0, day_of_week="sunday"), }, + "cleanup-outgoing-request-logs": { + "task": "openforms.logging.tasks.cleanup_request_logs", + "schedule": crontab(hour=0, minute=0, day_of_week="*"), + "args": (LOG_OUTGOING_REQUESTS_MAX_AGE,), + }, } RETRY_SUBMISSIONS_TIME_LIMIT = config( diff --git a/src/openforms/fixtures/admin_index_unlisted.json b/src/openforms/fixtures/admin_index_unlisted.json index b882296070..2fd1ae184c 100644 --- a/src/openforms/fixtures/admin_index_unlisted.json +++ b/src/openforms/fixtures/admin_index_unlisted.json @@ -19,5 +19,5 @@ "forms.FormLogic", "of_authentication.AuthInfo", "of_authentication.RegistratorInfo", - "upgrades.VersionInfo" + "upgrades.VersionInfo", ] diff --git a/src/openforms/logging/tasks.py b/src/openforms/logging/tasks.py index 07bb81b5e9..b6256e70fb 100644 --- a/src/openforms/logging/tasks.py +++ b/src/openforms/logging/tasks.py @@ -1,6 +1,10 @@ -from datetime import datetime +from datetime import datetime, timedelta from typing import List, TypedDict +from django.utils import timezone + +from log_outgoing_requests.models import OutgoingRequestsLog + from openforms.celery import app from openforms.forms.models import FormLogic from openforms.typing import JSONObject, JSONValue @@ -12,6 +16,14 @@ class EvaluatedRuleDict(TypedDict): action_log_data: dict[int, JSONValue] +@app.task() +def cleanup_request_logs(log_requests_max_age: int): + local_time = timezone.localtime() + delta = timedelta(hours=log_requests_max_age) + + OutgoingRequestsLog.objects.filter(timestamp__lte=local_time - delta).delete() + + @app.task(ignore_result=True) def log_logic_evaluation( *, diff --git a/src/openforms/logging/tests/test_tasks.py b/src/openforms/logging/tests/test_tasks.py index 408c20e2c1..b79dac5eab 100644 --- a/src/openforms/logging/tests/test_tasks.py +++ b/src/openforms/logging/tests/test_tasks.py @@ -1,8 +1,12 @@ +from datetime import timedelta + from django.test import TestCase from django.utils import timezone +from log_outgoing_requests.models import OutgoingRequestsLog + from openforms.logging.models import TimelineLogProxy -from openforms.logging.tasks import log_logic_evaluation +from openforms.logging.tasks import cleanup_request_logs, log_logic_evaluation from openforms.submissions.tests.factories import SubmissionFactory @@ -20,3 +24,18 @@ def test_no_logged_rules(self): ) self.assertEqual(0, TimelineLogProxy.objects.count()) + + def test_cleanup_request_logs(self): + """Assert that logs are cleaned if and only if created before specified time""" + + log_requests_max_age = 1 # 1 hour + delta = timedelta(hours=log_requests_max_age + 1) + + OutgoingRequestsLog.objects.create(timestamp=timezone.localtime() - delta) + OutgoingRequestsLog.objects.create(timestamp=timezone.localtime()) + + self.assertEqual(OutgoingRequestsLog.objects.count(), 2) + + cleanup_request_logs(log_requests_max_age) + + self.assertEqual(OutgoingRequestsLog.objects.count(), 1)