[#2762] update settings: log outgoing requests

open-formulieren · May 10, 2023 · 931b7eb · 931b7eb
1 parent 78e7ad6
commit 931b7eb
Show file tree

Hide file tree

Showing 11 changed files with 142 additions and 11 deletions.
diff --git a/requirements/base.in b/requirements/base.in
@@ -49,6 +49,7 @@ django-cors-headers
 django-decorator-include
 django-digid-eherkenning
 django-hijack
+django-log-outgoing-requests
 django-modeltranslation
 django-ordered-model
 django-privates

diff --git a/requirements/base.txt b/requirements/base.txt
@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
-# by the following command:
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
 #
 #    ./bin/compile_dependencies.sh
 #
@@ -103,6 +103,7 @@ django==3.2.18
     #   django-filter
     #   django-formtools
     #   django-hijack
+    #   django-log-outgoing-requests
     #   django-modeltranslation
     #   django-otp
     #   django-phonenumber-field
@@ -166,6 +167,8 @@ django-hijack==3.1.6
     # via -r requirements/base.in
 django-ipware==3.0.1
     # via django-axes
+django-log-outgoing-requests==0.1.0
+    # via -r requirements/base.in
 django-modeltranslation==0.18.5
     # via -r requirements/base.in
 django-ordered-model==3.6
@@ -376,6 +379,7 @@ redis==4.5.4
 requests==2.26.0
     # via
     #   django-camunda
+    #   django-log-outgoing-requests
     #   gemma-zds-client
     #   maykin-python3-saml
     #   mozilla-django-oidc

diff --git a/requirements/ci.txt b/requirements/ci.txt
@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
-# by the following command:
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
 #
 #    ./bin/compile_dependencies.sh
 #
@@ -178,6 +178,7 @@ django==3.2.18
     #   django-formtools
     #   django-hijack
     #   django-jenkins
+    #   django-log-outgoing-requests
     #   django-modeltranslation
     #   django-otp
     #   django-phonenumber-field
@@ -280,6 +281,10 @@ django-ipware==3.0.1
     #   django-axes
 django-jenkins==0.110.0
     # via -r requirements/test-tools.in
+django-log-outgoing-requests==0.1.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
 django-modeltranslation==0.18.5
     # via
     #   -c requirements/base.txt
@@ -725,6 +730,7 @@ requests==2.26.0
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   django-camunda
+    #   django-log-outgoing-requests
     #   gemma-zds-client
     #   maykin-python3-saml
     #   mozilla-django-oidc

diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
-# by the following command:
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
 #
 #    ./bin/compile_dependencies.sh
 #
@@ -205,6 +205,7 @@ django==3.2.18
     #   django-formtools
     #   django-hijack
     #   django-jenkins
+    #   django-log-outgoing-requests
     #   django-modeltranslation
     #   django-otp
     #   django-phonenumber-field
@@ -315,6 +316,10 @@ django-jenkins==0.110.0
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+django-log-outgoing-requests==0.1.0
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
 django-modeltranslation==0.18.5
     # via
     #   -c requirements/ci.txt
@@ -858,6 +863,7 @@ requests==2.26.0
     #   -r requirements/ci.txt
     #   ddt-api-calls
     #   django-camunda
+    #   django-log-outgoing-requests
     #   django-rosetta
     #   django-silk
     #   gemma-zds-client

diff --git a/requirements/extensions.txt b/requirements/extensions.txt
@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
-# by the following command:
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
 #
 #    ./bin/compile_dependencies.sh
 #
@@ -142,6 +142,7 @@ django==3.2.18
     #   django-filter
     #   django-formtools
     #   django-hijack
+    #   django-log-outgoing-requests
     #   django-modeltranslation
     #   django-otp
     #   django-phonenumber-field
@@ -240,6 +241,10 @@ django-ipware==3.0.1
     # via
     #   -r requirements/base.txt
     #   django-axes
+django-log-outgoing-requests==0.1.0
+    # via
+    #   -c requirements/base.in
+    #   -r requirements/base.txt
 django-modeltranslation==0.18.5
     # via
     #   -c requirements/base.in
@@ -579,6 +584,7 @@ requests==2.26.0
     # via
     #   -r requirements/base.txt
     #   django-camunda
+    #   django-log-outgoing-requests
     #   gemma-zds-client
     #   maykin-python3-saml
     #   mozilla-django-oidc

diff --git a/src/openforms/conf/base.py b/src/openforms/conf/base.py
@@ -8,6 +8,7 @@
 import sentry_sdk
 from celery.schedules import crontab
 from corsheaders.defaults import default_headers as default_cors_headers
+from log_outgoing_requests.formatters import HttpFormatter
 
 from csp_post_processor.constants import NONCE_HTTP_HEADER
 
@@ -185,6 +186,7 @@
     "cspreports",
     "csp_post_processor",
     "django_camunda",
+    "log_outgoing_requests",
     # Project applications.
     "openforms.accounts",
     "openforms.analytics_tools",
@@ -376,6 +378,7 @@
         "performance": {
             "format": "%(asctime)s %(process)d | %(thread)d | %(message)s",
         },
+        "outgoing_requests": {"()": HttpFormatter},
     },
     "filters": {
         "require_debug_false": {"()": "django.utils.log.RequireDebugFalse"},
@@ -419,6 +422,15 @@
             "maxBytes": 1024 * 1024 * 10,  # 10 MB
             "backupCount": 10,
         },
+        "log_outgoing_requests": {
+            "level": "DEBUG",
+            "formatter": "outgoing_requests",
+            "class": "logging.StreamHandler",
+        },
+        "save_outgoing_requests": {
+            "level": "DEBUG",
+            "class": "log_outgoing_requests.handlers.DatabaseOutgoingRequestsHandler",
+        },
     },
     "loggers": {
         "openforms": {
@@ -445,9 +457,19 @@
             "handlers": ["project"] if not LOG_STDOUT else ["console"],
             "level": "DEBUG",
         },
+        "requests": {
+            "handlers": ["log_outgoing_requests", "save_outgoing_requests"],
+            "level": "DEBUG",
+            "propagate": True,
+        },
     },
 }
 
+LOG_OUTGOING_REQUESTS_DB_SAVE = True
+LOG_OUTGOING_REQUESTS_MAX_AGE = config("LOG_OUTGOING_REQUESTS_MAX_AGE", default=7 * 24)
+LOG_OUTGOING_REQUESTS_SAVE_BODY = True
+LOG_OUTGOING_REQUESTS_LOG_BODY_TO_STDOUT = True
+
 #
 # AUTH settings - user accounts, passwords, backends...
 #
@@ -664,6 +686,11 @@
         "task": "openforms.forms.admin.tasks.clear_forms_export",
         "schedule": crontab(hour=0, minute=0, day_of_week="sunday"),
     },
+    "cleanup-outgoing-request-logs": {
+        "task": "openforms.logging.tasks.cleanup_request_logs",
+        "schedule": crontab(hour=0, minute=0, day_of_week="*"),
+        "args": (LOG_OUTGOING_REQUESTS_MAX_AGE,),
+    },
 }
 
 RETRY_SUBMISSIONS_TIME_LIMIT = config(

diff --git a/src/openforms/fixtures/admin_index_unlisted.json b/src/openforms/fixtures/admin_index_unlisted.json
@@ -19,5 +19,6 @@
     "forms.FormLogic",
     "of_authentication.AuthInfo",
     "of_authentication.RegistratorInfo",
-    "upgrades.VersionInfo"
+    "upgrades.VersionInfo",
+    log_outgoing_requests.OutgoingRequestsLog,
 ]
diff --git a/src/openforms/logging/migrations/0003_outgoingrequestslogconfig.py b/src/openforms/logging/migrations/0003_outgoingrequestslogconfig.py
@@ -0,0 +1,37 @@
+# Generated by Django 3.2.18 on 2023-05-04 09:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("logging", "0002_avgtimelinelogproxy"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="OutgoingRequestsLogConfig",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "save_to_db",
+                    models.BooleanField(
+                        help_text="Whether request logs should be saved to the database or sent to standard output (terminal) only. This overrides the setting defined via the environment variable.",
+                        verbose_name="Save logs to database",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]
diff --git a/src/openforms/logging/models.py b/src/openforms/logging/models.py
@@ -7,6 +7,7 @@
 from django.utils.html import format_html
 from django.utils.translation import gettext, gettext_lazy as _
 
+from solo.models import SingletonModel
 from timeline_logger.models import TimelineLog
 
 from openforms.forms.models import Form
@@ -176,3 +177,14 @@ class Meta:
         proxy = True
         verbose_name = _("avg timeline log entry")
         verbose_name_plural = _("avg timeline log entries")
+
+
+class OutgoingRequestsLogConfig(SingletonModel):
+    save_to_db = models.BooleanField(
+        _("Save logs to database"),
+        help_text=_(
+            "Whether request logs should be saved to the database or sent to standard "
+            "output (terminal) only. This overrides the setting defined via the "
+            "environment variable."
+        ),
+    )
diff --git a/src/openforms/logging/tasks.py b/src/openforms/logging/tasks.py
@@ -1,6 +1,10 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import List, TypedDict
 
+from django.utils import timezone
+
+from log_outgoing_requests.models import OutgoingRequestsLog
+
 from openforms.celery import app
 from openforms.forms.models import FormLogic
 from openforms.typing import JSONObject, JSONValue
@@ -12,6 +16,14 @@ class EvaluatedRuleDict(TypedDict):
     action_log_data: dict[int, JSONValue]
 
 
+@app.task()
+def cleanup_request_logs(log_requests_max_age: int):
+    local_time = timezone.localtime()
+    delta = timedelta(hours=log_requests_max_age)
+
+    OutgoingRequestsLog.objects.filter(timestamp__lte=local_time - delta).delete()
+
+
 @app.task(ignore_result=True)
 def log_logic_evaluation(
     *,

diff --git a/src/openforms/logging/tests/test_tasks.py b/src/openforms/logging/tests/test_tasks.py
@@ -1,8 +1,12 @@
+from datetime import timedelta
+
 from django.test import TestCase
 from django.utils import timezone
 
+from log_outgoing_requests.models import OutgoingRequestsLog
+
 from openforms.logging.models import TimelineLogProxy
-from openforms.logging.tasks import log_logic_evaluation
+from openforms.logging.tasks import cleanup_request_logs, log_logic_evaluation
 from openforms.submissions.tests.factories import SubmissionFactory
 
 
@@ -20,3 +24,18 @@ def test_no_logged_rules(self):
         )
 
         self.assertEqual(0, TimelineLogProxy.objects.count())
+
+    def test_cleanup_request_logs(self):
+        """Assert that logs are cleaned if and only if created before specified time"""
+
+        log_requests_max_age = 1  # 1 hour
+        delta = timedelta(hours=log_requests_max_age + 1)
+
+        OutgoingRequestsLog.objects.create(timestamp=timezone.localtime() - delta)
+        OutgoingRequestsLog.objects.create(timestamp=timezone.localtime())
+
+        self.assertEqual(OutgoingRequestsLog.objects.count(), 2)
+
+        cleanup_request_logs(log_requests_max_age)
+
+        self.assertEqual(OutgoingRequestsLog.objects.count(), 1)