diff --git a/requirements/base.in b/requirements/base.in index 022dd9f805..87b6380fc4 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -49,6 +49,7 @@ django-cors-headers django-decorator-include django-digid-eherkenning django-hijack +django-log-outgoing-requests django-modeltranslation django-ordered-model django-privates diff --git a/requirements/base.txt b/requirements/base.txt index 0b61c370f1..b052ff91fb 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: # # ./bin/compile_dependencies.sh # @@ -103,6 +103,7 @@ django==3.2.18 # django-filter # django-formtools # django-hijack + # django-log-outgoing-requests # django-modeltranslation # django-otp # django-phonenumber-field @@ -166,6 +167,8 @@ django-hijack==3.1.6 # via -r requirements/base.in django-ipware==3.0.1 # via django-axes +django-log-outgoing-requests==0.1.0 + # via -r requirements/base.in django-modeltranslation==0.18.5 # via -r requirements/base.in django-ordered-model==3.6 @@ -376,6 +379,7 @@ redis==4.5.4 requests==2.26.0 # via # django-camunda + # django-log-outgoing-requests # gemma-zds-client # maykin-python3-saml # mozilla-django-oidc diff --git a/requirements/ci.txt b/requirements/ci.txt index 632521e1c0..b2f82237d6 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: # # ./bin/compile_dependencies.sh # @@ -178,6 +178,7 @@ django==3.2.18 # django-formtools # django-hijack # django-jenkins + # django-log-outgoing-requests # django-modeltranslation # django-otp # django-phonenumber-field @@ -280,6 +281,10 @@ django-ipware==3.0.1 # django-axes django-jenkins==0.110.0 # via -r requirements/test-tools.in +django-log-outgoing-requests==0.1.0 + # via + # -c requirements/base.txt + # -r requirements/base.txt django-modeltranslation==0.18.5 # via # -c requirements/base.txt @@ -725,6 +730,7 @@ requests==2.26.0 # -c requirements/base.txt # -r requirements/base.txt # django-camunda + # django-log-outgoing-requests # gemma-zds-client # maykin-python3-saml # mozilla-django-oidc diff --git a/requirements/dev.txt b/requirements/dev.txt index 59fd4d7f61..2218afc147 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: # # ./bin/compile_dependencies.sh # @@ -205,6 +205,7 @@ django==3.2.18 # django-formtools # django-hijack # django-jenkins + # django-log-outgoing-requests # django-modeltranslation # django-otp # django-phonenumber-field @@ -315,6 +316,10 @@ django-jenkins==0.110.0 # via # -c requirements/ci.txt # -r requirements/ci.txt +django-log-outgoing-requests==0.1.0 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt django-modeltranslation==0.18.5 # via # -c requirements/ci.txt @@ -858,6 +863,7 @@ requests==2.26.0 # -r requirements/ci.txt # ddt-api-calls # django-camunda + # django-log-outgoing-requests # django-rosetta # django-silk # gemma-zds-client diff --git a/requirements/extensions.txt b/requirements/extensions.txt index 1274959fda..fb51f62482 100644 --- a/requirements/extensions.txt +++ b/requirements/extensions.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: # # ./bin/compile_dependencies.sh # @@ -142,6 +142,7 @@ django==3.2.18 # django-filter # django-formtools # django-hijack + # django-log-outgoing-requests # django-modeltranslation # django-otp # django-phonenumber-field @@ -240,6 +241,10 @@ django-ipware==3.0.1 # via # -r requirements/base.txt # django-axes +django-log-outgoing-requests==0.1.0 + # via + # -c requirements/base.in + # -r requirements/base.txt django-modeltranslation==0.18.5 # via # -c requirements/base.in @@ -579,6 +584,7 @@ requests==2.26.0 # via # -r requirements/base.txt # django-camunda + # django-log-outgoing-requests # gemma-zds-client # maykin-python3-saml # mozilla-django-oidc diff --git a/src/openforms/conf/base.py b/src/openforms/conf/base.py index 6f20750bf4..8b5456ade2 100644 --- a/src/openforms/conf/base.py +++ b/src/openforms/conf/base.py @@ -8,6 +8,7 @@ import sentry_sdk from celery.schedules import crontab from corsheaders.defaults import default_headers as default_cors_headers +from log_outgoing_requests.formatters import HttpFormatter from csp_post_processor.constants import NONCE_HTTP_HEADER @@ -185,6 +186,7 @@ "cspreports", "csp_post_processor", "django_camunda", + "log_outgoing_requests", # Project applications. "openforms.accounts", "openforms.analytics_tools", @@ -376,6 +378,7 @@ "performance": { "format": "%(asctime)s %(process)d | %(thread)d | %(message)s", }, + "outgoing_requests": {"()": HttpFormatter}, }, "filters": { "require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}, @@ -419,6 +422,15 @@ "maxBytes": 1024 * 1024 * 10, # 10 MB "backupCount": 10, }, + "log_outgoing_requests": { + "level": "DEBUG", + "formatter": "outgoing_requests", + "class": "logging.StreamHandler", + }, + "save_outgoing_requests": { + "level": "DEBUG", + "class": "log_outgoing_requests.handlers.DatabaseOutgoingRequestsHandler", + }, }, "loggers": { "openforms": { @@ -445,9 +457,19 @@ "handlers": ["project"] if not LOG_STDOUT else ["console"], "level": "DEBUG", }, + "requests": { + "handlers": ["log_outgoing_requests", "save_outgoing_requests"], + "level": "DEBUG", + "propagate": True, + }, }, } +LOG_OUTGOING_REQUESTS_DB_SAVE = True +LOG_OUTGOING_REQUESTS_MAX_AGE = config("LOG_OUTGOING_REQUESTS_MAX_AGE", default=7 * 24) +LOG_OUTGOING_REQUESTS_SAVE_BODY = True +LOG_OUTGOING_REQUESTS_LOG_BODY_TO_STDOUT = True + # # AUTH settings - user accounts, passwords, backends... # @@ -664,6 +686,11 @@ "task": "openforms.forms.admin.tasks.clear_forms_export", "schedule": crontab(hour=0, minute=0, day_of_week="sunday"), }, + "cleanup-outgoing-request-logs": { + "task": "openforms.logging.tasks.cleanup_request_logs", + "schedule": crontab(hour=0, minute=0, day_of_week="*"), + "args": (LOG_OUTGOING_REQUESTS_MAX_AGE,), + }, } RETRY_SUBMISSIONS_TIME_LIMIT = config( diff --git a/src/openforms/fixtures/admin_index_unlisted.json b/src/openforms/fixtures/admin_index_unlisted.json index b882296070..91789aadf9 100644 --- a/src/openforms/fixtures/admin_index_unlisted.json +++ b/src/openforms/fixtures/admin_index_unlisted.json @@ -19,5 +19,6 @@ "forms.FormLogic", "of_authentication.AuthInfo", "of_authentication.RegistratorInfo", - "upgrades.VersionInfo" + "upgrades.VersionInfo", + log_outgoing_requests.OutgoingRequestsLog, ] diff --git a/src/openforms/logging/migrations/0003_outgoingrequestslogconfig.py b/src/openforms/logging/migrations/0003_outgoingrequestslogconfig.py new file mode 100644 index 0000000000..3bf522af11 --- /dev/null +++ b/src/openforms/logging/migrations/0003_outgoingrequestslogconfig.py @@ -0,0 +1,37 @@ +# Generated by Django 3.2.18 on 2023-05-04 09:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("logging", "0002_avgtimelinelogproxy"), + ] + + operations = [ + migrations.CreateModel( + name="OutgoingRequestsLogConfig", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "save_to_db", + models.BooleanField( + help_text="Whether request logs should be saved to the database or sent to standard output (terminal) only. This overrides the setting defined via the environment variable.", + verbose_name="Save logs to database", + ), + ), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/src/openforms/logging/models.py b/src/openforms/logging/models.py index af30654286..5bf9d2572c 100644 --- a/src/openforms/logging/models.py +++ b/src/openforms/logging/models.py @@ -7,6 +7,7 @@ from django.utils.html import format_html from django.utils.translation import gettext, gettext_lazy as _ +from solo.models import SingletonModel from timeline_logger.models import TimelineLog from openforms.forms.models import Form @@ -176,3 +177,14 @@ class Meta: proxy = True verbose_name = _("avg timeline log entry") verbose_name_plural = _("avg timeline log entries") + + +class OutgoingRequestsLogConfig(SingletonModel): + save_to_db = models.BooleanField( + _("Save logs to database"), + help_text=_( + "Whether request logs should be saved to the database or sent to standard " + "output (terminal) only. This overrides the setting defined via the " + "environment variable." + ), + ) diff --git a/src/openforms/logging/tasks.py b/src/openforms/logging/tasks.py index 07bb81b5e9..b6256e70fb 100644 --- a/src/openforms/logging/tasks.py +++ b/src/openforms/logging/tasks.py @@ -1,6 +1,10 @@ -from datetime import datetime +from datetime import datetime, timedelta from typing import List, TypedDict +from django.utils import timezone + +from log_outgoing_requests.models import OutgoingRequestsLog + from openforms.celery import app from openforms.forms.models import FormLogic from openforms.typing import JSONObject, JSONValue @@ -12,6 +16,14 @@ class EvaluatedRuleDict(TypedDict): action_log_data: dict[int, JSONValue] +@app.task() +def cleanup_request_logs(log_requests_max_age: int): + local_time = timezone.localtime() + delta = timedelta(hours=log_requests_max_age) + + OutgoingRequestsLog.objects.filter(timestamp__lte=local_time - delta).delete() + + @app.task(ignore_result=True) def log_logic_evaluation( *, diff --git a/src/openforms/logging/tests/test_tasks.py b/src/openforms/logging/tests/test_tasks.py index 408c20e2c1..b79dac5eab 100644 --- a/src/openforms/logging/tests/test_tasks.py +++ b/src/openforms/logging/tests/test_tasks.py @@ -1,8 +1,12 @@ +from datetime import timedelta + from django.test import TestCase from django.utils import timezone +from log_outgoing_requests.models import OutgoingRequestsLog + from openforms.logging.models import TimelineLogProxy -from openforms.logging.tasks import log_logic_evaluation +from openforms.logging.tasks import cleanup_request_logs, log_logic_evaluation from openforms.submissions.tests.factories import SubmissionFactory @@ -20,3 +24,18 @@ def test_no_logged_rules(self): ) self.assertEqual(0, TimelineLogProxy.objects.count()) + + def test_cleanup_request_logs(self): + """Assert that logs are cleaned if and only if created before specified time""" + + log_requests_max_age = 1 # 1 hour + delta = timedelta(hours=log_requests_max_age + 1) + + OutgoingRequestsLog.objects.create(timestamp=timezone.localtime() - delta) + OutgoingRequestsLog.objects.create(timestamp=timezone.localtime()) + + self.assertEqual(OutgoingRequestsLog.objects.count(), 2) + + cleanup_request_logs(log_requests_max_age) + + self.assertEqual(OutgoingRequestsLog.objects.count(), 1)