From a7740479f4df2515e20571a3d1a034308d42c4d3 Mon Sep 17 00:00:00 2001 From: Marcel Waldvogel Date: Tue, 29 Sep 2020 13:51:18 +0200 Subject: [PATCH] :bug: Send out mail only once every force interval --- CHANGELOG.md | 3 +++ Makefile | 2 +- autoblockchainify/commit.py | 35 +++++++++++++++++++---------------- autoblockchainify/mail.py | 36 ++++++++++++++++++++++++++---------- 4 files changed, 49 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 946b4c6..91a1b25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## Fixed - Handle that `.gnupg` also is in the volume now +- Send out mail only every force interval. Also, do not trigger immediate + addition of a mail reply only, avoiding double commits/timestamps every + force interval on idle repositories. ## Changed diff --git a/Makefile b/Makefile index 6b710a1..09203fe 100644 --- a/Makefile +++ b/Makefile @@ -109,7 +109,7 @@ gen-docker-dev:python-package (echo "### THIS FILE WAS AUTOGENERATED, CHANGES WILL BE LOST ###" && \ sed -e 's/^##DEVONLY## *//' -e '/##PRODONLY##$$/d' \ < autoblockchainify/$$i ) > autoblockchainify-dev/$$i; done - for i in health.sh; do \ + for i in run-autoblockchainify.sh health.sh; do \ (head -1 autoblockchainify/$$i && \ echo "### THIS FILE WAS AUTOGENERATED, CHANGES WILL BE LOST ###" && \ sed -e 's/^##DEVONLY## *//' -e '/##PRODONLY##$$/d' \ diff --git a/autoblockchainify/commit.py b/autoblockchainify/commit.py index c3d57ac..95ee523 100644 --- a/autoblockchainify/commit.py +++ b/autoblockchainify/commit.py @@ -20,8 +20,9 @@ # Committing to git and obtaining timestamps -import datetime +from datetime import datetime, timezone import logging as _logging +from pathlib import Path import subprocess import sys import threading @@ -53,12 +54,15 @@ def cross_timestamp(repo, branch, server): % (branch, server)) -def has_changes(repo): +def has_user_changes(repo): """Check whether there are uncommitted changes, i.e., whether - `git status -z` has any output.""" + `git status -z` has any output. A modification of only `pgp-timestamp.sig` + is ignored, as it is neither necessary nor desirable to trigger on it: + (a) our own timestamp is not really needed on it and + (b) it would cause an unnecessary second timestamp per idle force period.""" ret = subprocess.run(['git', 'status', '-z'], cwd=repo, capture_output=True, check=True) - return len(ret.stdout) > 0 + return len(ret.stdout) > 0 and ret.stdout != b' M pgp-timestamp.sig\0' def pending_merge(repo): @@ -69,7 +73,7 @@ def pending_merge(repo): def commit_current_state(repo): """Force a commit; will be called only if a commit has to be made. I.e., if there really are changes or the force duration has expired.""" - now = datetime.datetime.now(datetime.timezone.utc) + now = datetime.now(timezone.utc) nowstr = now.strftime('%Y-%m-%d %H:%M:%S UTC') subprocess.run(['git', 'add', '.'], cwd=repo, check=True) @@ -85,9 +89,9 @@ def head_older_than(repo, duration): r = git.Repository(repo) if r.head_is_unborn: return False - now = datetime.datetime.utcnow() - if datetime.datetime.utcfromtimestamp(r.head.peel().commit_time) + duration < now: - return True + now = datetime.utcnow() + return datetime.utcfromtimestamp( + r.head.peel().commit_time) + duration < now def do_commit(): @@ -108,18 +112,18 @@ def do_commit(): # Allow 5% of an interval tolerance, such that small timing differences # will not lead to lengthening the duration by one commit_interval force_interval = (autoblockchainify.config.arg.commit_interval - * (autoblockchainify.config.arg.force_after_intervals - 0.05)) + * (autoblockchainify.config.arg.force_after_intervals - 0.05)) try: repo = autoblockchainify.config.arg.repository # If a merge (a manual process on the repository) is detected, # try to not interfere with the manual process and wait for the # next forced update - if ((has_changes(repo) and not pending_merge(repo)) + if ((has_user_changes(repo) and not pending_merge(repo)) or head_older_than(repo, force_interval)): # 1. Commit commit_current_state(repo) - # 2. Timestamp using Zeitgitter + # 2. Timestamp (synchronously) using Zeitgitter repositories = autoblockchainify.config.arg.push_repository branches = autoblockchainify.config.arg.push_branch for r in autoblockchainify.config.arg.zeitgitter_servers: @@ -132,15 +136,14 @@ def do_commit(): logging.info("Pushing upstream to %s" % r) push_upstream(repo, r, branches) - # 4. Timestamp by mail (asynchronous) + # 4. Timestamp by mail (asynchronously) if autoblockchainify.config.arg.stamper_own_address: - logging.info("cross-timestamping by mail") - autoblockchainify.mail.async_email_timestamp() + autoblockchainify.mail.async_email_timestamp(wait=force_interval) - logging.info("do_commit done") + logging.info("do_commit done") except Exception as e: logging.error("Unhandled exception in do_commit() thread: %s: %s" % - (e, ''.join(traceback.format_tb(sys.exc_info()[2])))) + (e, ''.join(traceback.format_tb(sys.exc_info()[2])))) def loop(): diff --git a/autoblockchainify/mail.py b/autoblockchainify/mail.py index 2680142..c5db99f 100644 --- a/autoblockchainify/mail.py +++ b/autoblockchainify/mail.py @@ -314,7 +314,19 @@ def wait_for_receive(logfile): logging.error("No response received, giving up") -def async_email_timestamp(resume=False): +def not_modified_in(file, wait): + """Has `logfile` not been modified in ~`wait` seconds? + Non-existent file is considered to *not* fulfill this.""" + try: + stat = file.stat() + mtime = datetime.utcfromtimestamp(stat.st_mtime) + now = datetime.utcnow() + return mtime + wait < now + except FileNotFoundError: + return False + + +def async_email_timestamp(resume=False, wait=None): """If called with `resume=True`, tries to resume waiting for the mail""" path = autoblockchainify.config.arg.repository repo = git.Repository(path) @@ -326,6 +338,7 @@ def async_email_timestamp(resume=False): return head = repo.head logfile = Path(path, 'pgp-timestamp.tmp') + sigfile = Path(path, 'pgp-timestamp.sig') if resume: if not logfile.is_file(): logging.info("Not resuming mail timestamp: No pending mail reply") @@ -336,12 +349,15 @@ def async_email_timestamp(resume=False): logging.info("Not resuming mail timestamp: No revision info") return else: # Fresh request - new_rev = ("git commit %s\nTimestamp requested at %s\n" % - (head.target.hex, - strftime("%Y-%m-%d %H:%M:%S UTC", gmtime()))) - with serialize_create: - with logfile.open('w') as f: - f.write(new_rev) - send(new_rev) - threading.Thread(target=wait_for_receive, args=(logfile,), - daemon=True).start() + # No recent attempts or results for mail timestamping + if (not sigfile.is_file() or not_modified_in(logfile, wait) + or not_modified_in(sigfile, wait)): + new_rev = ("git commit %s\nTimestamp requested at %s\n" % + (head.target.hex, + strftime("%Y-%m-%d %H:%M:%S UTC", gmtime()))) + with serialize_create: + with logfile.open('w') as f: + f.write(new_rev) + send(new_rev) + threading.Thread(target=wait_for_receive, args=(logfile,), + daemon=True).start()