[#2762] update settings: log outgoing requests

requirements/base.in

@@ -49,6 +49,7 @@ django-cors-headers
 django-decorator-include
 django-digid-eherkenning
 django-hijack
+django-log-outgoing-requests
 django-modeltranslation
 django-ordered-model
 django-privates

requirements/base.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
-# by the following command:
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
 #
 #    ./bin/compile_dependencies.sh
 #
@@ -103,6 +103,7 @@ django==3.2.19
     #   django-filter
     #   django-formtools
     #   django-hijack
+    #   django-log-outgoing-requests
     #   django-modeltranslation
     #   django-otp
     #   django-phonenumber-field
@@ -166,6 +167,8 @@ django-hijack==3.1.6
     # via -r requirements/base.in
 django-ipware==3.0.1
     # via django-axes
+django-log-outgoing-requests==0.1.0
+    # via -r requirements/base.in
 django-modeltranslation==0.18.5
     # via -r requirements/base.in
 django-ordered-model==3.6
@@ -376,6 +379,7 @@ redis==4.5.4
 requests==2.31.0
     # via
     #   django-camunda
+    #   django-log-outgoing-requests
     #   gemma-zds-client
     #   maykin-python3-saml
     #   mozilla-django-oidc

requirements/ci.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
-# by the following command:
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
 #
 #    ./bin/compile_dependencies.sh
 #
@@ -178,6 +178,7 @@ django==3.2.19
     #   django-formtools
     #   django-hijack
     #   django-jenkins
+    #   django-log-outgoing-requests
     #   django-modeltranslation
     #   django-otp
     #   django-phonenumber-field
@@ -280,6 +281,10 @@ django-ipware==3.0.1
     #   django-axes
 django-jenkins==0.110.0
     # via -r requirements/test-tools.in
+django-log-outgoing-requests==0.1.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
 django-modeltranslation==0.18.5
     # via
     #   -c requirements/base.txt
@@ -725,6 +730,7 @@ requests==2.31.0
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   django-camunda
+    #   django-log-outgoing-requests
     #   gemma-zds-client
     #   maykin-python3-saml
     #   mozilla-django-oidc

requirements/dev.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
-# by the following command:
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
 #
 #    ./bin/compile_dependencies.sh
 #
@@ -205,6 +205,7 @@ django==3.2.19
     #   django-formtools
     #   django-hijack
     #   django-jenkins
+    #   django-log-outgoing-requests
     #   django-modeltranslation
     #   django-otp
     #   django-phonenumber-field
@@ -315,6 +316,10 @@ django-jenkins==0.110.0
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+django-log-outgoing-requests==0.1.0
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
 django-modeltranslation==0.18.5
     # via
     #   -c requirements/ci.txt
@@ -858,6 +863,7 @@ requests==2.31.0
     #   -r requirements/ci.txt
     #   ddt-api-calls
     #   django-camunda
+    #   django-log-outgoing-requests
     #   django-rosetta
     #   django-silk
     #   gemma-zds-client

requirements/extensions.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
-# by the following command:
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
 #
 #    ./bin/compile_dependencies.sh
 #
@@ -142,6 +142,7 @@ django==3.2.19
     #   django-filter
     #   django-formtools
     #   django-hijack
+    #   django-log-outgoing-requests
     #   django-modeltranslation
     #   django-otp
     #   django-phonenumber-field
@@ -240,6 +241,10 @@ django-ipware==3.0.1
     # via
     #   -r requirements/base.txt
     #   django-axes
+django-log-outgoing-requests==0.1.0
+    # via
+    #   -c requirements/base.in
+    #   -r requirements/base.txt
 django-modeltranslation==0.18.5
     # via
     #   -c requirements/base.in
@@ -579,6 +584,7 @@ requests==2.31.0
     # via
     #   -r requirements/base.txt
     #   django-camunda
+    #   django-log-outgoing-requests
     #   gemma-zds-client
     #   maykin-python3-saml
     #   mozilla-django-oidc

src/openforms/conf/base.py

@@ -8,6 +8,7 @@
 import sentry_sdk
 from celery.schedules import crontab
 from corsheaders.defaults import default_headers as default_cors_headers
+from log_outgoing_requests.formatters import HttpFormatter
 
 from csp_post_processor.constants import NONCE_HTTP_HEADER
 
@@ -185,6 +186,7 @@
     "cspreports",
     "csp_post_processor",
     "django_camunda",
+    "log_outgoing_requests",
     # Project applications.
     "openforms.accounts",
     "openforms.analytics_tools",
@@ -376,6 +378,7 @@
         "performance": {
             "format": "%(asctime)s %(process)d | %(thread)d | %(message)s",
         },
+        "outgoing_requests": {"()": HttpFormatter},
     },
     "filters": {
         "require_debug_false": {"()": "django.utils.log.RequireDebugFalse"},
@@ -419,6 +422,15 @@
             "maxBytes": 1024 * 1024 * 10,  # 10 MB
             "backupCount": 10,
         },
+        "log_outgoing_requests": {
+            "level": "DEBUG",
+            "formatter": "outgoing_requests",
+            "class": "logging.StreamHandler",
+        },
+        "save_outgoing_requests": {
+            "level": "DEBUG",
+            "class": "log_outgoing_requests.handlers.DatabaseOutgoingRequestsHandler",
+        },
     },
     "loggers": {
         "openforms": {
@@ -445,9 +457,27 @@
             "handlers": ["project"] if not LOG_STDOUT else ["console"],
             "level": "DEBUG",
         },
+        "requests": {
+            "handlers": ["log_outgoing_requests", "save_outgoing_requests"],
+            "level": "DEBUG",
+            "propagate": True,
+        },
     },
 }
 
+LOG_OUTGOING_REQUESTS_DB_SAVE = True
+LOG_OUTGOING_REQUESTS_DB_SAVE_BODY = True
+LOG_OUTGOING_REQUESTS_CONTENT_TYPES = [
+    "text/*",
+    "application/json",
+    "application/xml",
+    "application/soap+xml",
+]
+LOG_OUTGOING_REQUESTS_EMIT_BODY = True
+LOG_OUTGOING_REQUESTS_MAX_AGE = config("LOG_OUTGOING_REQUESTS_MAX_AGE", default=7 * 24)
+LOG_OUTGOING_REQUESTS_MAX_CONTENT_LENGTH = 524_288  # 0.5MB
+LOG_OUTGOING_REQUESTS_LOG_BODY_TO_STDOUT = True
+
 #
 # AUTH settings - user accounts, passwords, backends...
 #
@@ -668,6 +698,11 @@
         "task": "openforms.forms.admin.tasks.clear_forms_export",
         "schedule": crontab(hour=0, minute=0, day_of_week="sunday"),
     },
+    "cleanup-outgoing-request-logs": {
+        "task": "openforms.logging.tasks.cleanup_request_logs",
+        "schedule": crontab(hour=0, minute=0, day_of_week="*"),
+        "args": (LOG_OUTGOING_REQUESTS_MAX_AGE,),
+    },
 }
 
 RETRY_SUBMISSIONS_TIME_LIMIT = config(

src/openforms/fixtures/admin_index_unlisted.json

@@ -19,5 +19,5 @@
     "forms.FormLogic",
     "of_authentication.AuthInfo",
     "of_authentication.RegistratorInfo",
-    "upgrades.VersionInfo"
+    "upgrades.VersionInfo",
 ]

src/openforms/logging/tasks.py

@@ -1,6 +1,10 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import List, TypedDict
 
+from django.utils import timezone
+
+from log_outgoing_requests.models import OutgoingRequestsLog
+
 from openforms.celery import app
 from openforms.forms.models import FormLogic
 from openforms.typing import JSONObject, JSONValue
@@ -12,6 +16,14 @@ class EvaluatedRuleDict(TypedDict):
     action_log_data: dict[int, JSONValue]
 
 
+@app.task()
+def cleanup_request_logs(log_requests_max_age: int):
+    local_time = timezone.localtime()
+    delta = timedelta(hours=log_requests_max_age)
+
+    OutgoingRequestsLog.objects.filter(timestamp__lte=local_time - delta).delete()
+
+
 @app.task(ignore_result=True)
 def log_logic_evaluation(
     *,

src/openforms/logging/tests/test_tasks.py

@@ -1,8 +1,12 @@
+from datetime import timedelta
+
 from django.test import TestCase
 from django.utils import timezone
 
+from log_outgoing_requests.models import OutgoingRequestsLog
+
 from openforms.logging.models import TimelineLogProxy
-from openforms.logging.tasks import log_logic_evaluation
+from openforms.logging.tasks import cleanup_request_logs, log_logic_evaluation
 from openforms.submissions.tests.factories import SubmissionFactory
 
 
@@ -20,3 +24,18 @@ def test_no_logged_rules(self):
         )
 
         self.assertEqual(0, TimelineLogProxy.objects.count())
+
+    def test_cleanup_request_logs(self):
+        """Assert that logs are cleaned if and only if created before specified time"""
+
+        log_requests_max_age = 1  # 1 hour
+        delta = timedelta(hours=log_requests_max_age + 1)
+
+        OutgoingRequestsLog.objects.create(timestamp=timezone.localtime() - delta)
+        OutgoingRequestsLog.objects.create(timestamp=timezone.localtime())
+
+        self.assertEqual(OutgoingRequestsLog.objects.count(), 2)
+
+        cleanup_request_logs(log_requests_max_age)
+
+        self.assertEqual(OutgoingRequestsLog.objects.count(), 1)