Skip to content

Commit

Permalink
[#2762] log outgoing requests
Browse files Browse the repository at this point in the history
  • Loading branch information
pi-sigma committed May 8, 2023
1 parent c819217 commit 43bce54
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 1 deletion.
26 changes: 26 additions & 0 deletions src/openforms/conf/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import sentry_sdk
from celery.schedules import crontab
from corsheaders.defaults import default_headers as default_cors_headers
from log_outgoing_requests.formatters import HttpFormatter

from csp_post_processor.constants import NONCE_HTTP_HEADER

Expand Down Expand Up @@ -185,6 +186,7 @@
"cspreports",
"csp_post_processor",
"django_camunda",
"log_outgoing_requests",
# Project applications.
"openforms.accounts",
"openforms.analytics_tools",
Expand Down Expand Up @@ -376,6 +378,7 @@
"performance": {
"format": "%(asctime)s %(process)d | %(thread)d | %(message)s",
},
"outgoing_requests": {"()": HttpFormatter},
},
"filters": {
"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"},
Expand Down Expand Up @@ -419,6 +422,15 @@
"maxBytes": 1024 * 1024 * 10, # 10 MB
"backupCount": 10,
},
"log_outgoing_requests": {
"level": "DEBUG",
"formatter": "outgoing_requests",
"class": "logging.StreamHandler",
},
"save_outgoing_requests": {
"level": "DEBUG",
"class": "log_outgoing_requests.handlers.DatabaseOutgoingRequestsHandler",
},
},
"loggers": {
"openforms": {
Expand All @@ -445,9 +457,18 @@
"handlers": ["project"] if not LOG_STDOUT else ["console"],
"level": "DEBUG",
},
"requests": {
"handlers": ["log_outgoing_requests", "save_outgoing_requests"],
"level": "DEBUG",
"propagate": True,
},
},
}

LOG_OUTGOING_REQUESTS_DB_SAVE = True
LOG_OUTGOING_REQUESTS_MAX_AGE = config("LOG_OUTGOING_REQUESTS_MAX_AGE", default=7 * 24)
LOG_OUTGOING_REQUESTS_SAVE_BODY = True

#
# AUTH settings - user accounts, passwords, backends...
#
Expand Down Expand Up @@ -664,6 +685,11 @@
"task": "openforms.forms.admin.tasks.clear_forms_export",
"schedule": crontab(hour=0, minute=0, day_of_week="sunday"),
},
"cleanup-outgoing-request-logs": {
"task": "openforms.logging.tasks.cleanup_request_logs",
"schedule": crontab(hour=0, minute=0, day_of_week="*"),
"args": (LOG_OUTGOING_REQUESTS_MAX_AGE,),
},
}

RETRY_SUBMISSIONS_TIME_LIMIT = config(
Expand Down
37 changes: 37 additions & 0 deletions src/openforms/logging/migrations/0003_outgoingrequestslogconfig.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Generated by Django 3.2.18 on 2023-05-04 09:18

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
("logging", "0002_avgtimelinelogproxy"),
]

operations = [
migrations.CreateModel(
name="OutgoingRequestsLogConfig",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"save_to_db",
models.BooleanField(
help_text="Whether request logs should be saved to the database or sent to standard output (terminal) only. This overrides the setting defined via the environment variable.",
verbose_name="Save logs to database",
),
),
],
options={
"abstract": False,
},
),
]
12 changes: 12 additions & 0 deletions src/openforms/logging/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from django.utils.html import format_html
from django.utils.translation import gettext, gettext_lazy as _

from solo.models import SingletonModel
from timeline_logger.models import TimelineLog

from openforms.forms.models import Form
Expand Down Expand Up @@ -176,3 +177,14 @@ class Meta:
proxy = True
verbose_name = _("avg timeline log entry")
verbose_name_plural = _("avg timeline log entries")


class OutgoingRequestsLogConfig(SingletonModel):
save_to_db = models.BooleanField(
_("Save logs to database"),
help_text=_(
"Whether request logs should be saved to the database or sent to standard "
"output (terminal) only. This overrides the setting defined via the "
"environment variable."
),
)
14 changes: 13 additions & 1 deletion src/openforms/logging/tasks.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
from datetime import datetime
from datetime import timedelta
from typing import List, TypedDict

from django.utils import timezone

from log_outgoing_requests.models import OutgoingRequestsLog

from openforms.celery import app
from openforms.forms.models import FormLogic
from openforms.typing import JSONObject, JSONValue
Expand All @@ -12,6 +16,14 @@ class EvaluatedRuleDict(TypedDict):
action_log_data: dict[int, JSONValue]


@app.task()
def cleanup_request_logs(log_requests_max_age: int):
local_time = timezone.localtime()
delta = timedelta(hours=log_requests_max_age)

OutgoingRequestsLog.objects.filter(timestamp__lte=local_time - delta).delete()


@app.task(ignore_result=True)
def log_logic_evaluation(
*,
Expand Down

0 comments on commit 43bce54

Please sign in to comment.