From c5937356955825c940507938ce48be6accf86608 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 24 Jan 2024 13:25:57 -0800 Subject: [PATCH 1/4] Switch outdir to a keyward argument instead of positional --- .../cdsa/management/commands/cdsa_records.py | 3 ++- primed/cdsa/tests/test_commands.py | 20 ++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/primed/cdsa/management/commands/cdsa_records.py b/primed/cdsa/management/commands/cdsa_records.py index 1f049343..78db2704 100644 --- a/primed/cdsa/management/commands/cdsa_records.py +++ b/primed/cdsa/management/commands/cdsa_records.py @@ -12,8 +12,9 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - "outdir", + "--outdir", help="""Output directory where reports should be written. This directory will be created.""", + required=True, ) def _export_table(self, table, filename): diff --git a/primed/cdsa/tests/test_commands.py b/primed/cdsa/tests/test_commands.py index e5ad8ebe..fdfc5454 100644 --- a/primed/cdsa/tests/test_commands.py +++ b/primed/cdsa/tests/test_commands.py @@ -20,12 +20,12 @@ def setUp(self): def test_output(self): out = StringIO() - call_command("cdsa_records", self.outdir, "--no-color", stdout=out) + call_command("cdsa_records", "--outdir", self.outdir, "--no-color", stdout=out) self.assertIn("generating reports... done!", out.getvalue()) def test_files_created(self): out = StringIO() - call_command("cdsa_records", self.outdir, "--no-color", stdout=out) + call_command("cdsa_records", "--outdir", self.outdir, "--no-color", stdout=out) self.assertTrue(isdir(self.outdir)) self.assertTrue(isfile(os.path.join(self.outdir, "representative_records.tsv"))) self.assertTrue(isfile(os.path.join(self.outdir, "study_records.tsv"))) @@ -34,7 +34,7 @@ def test_files_created(self): def test_representative_records_zero(self): out = StringIO() - call_command("cdsa_records", self.outdir, "--no-color", stdout=out) + call_command("cdsa_records", "--outdir", self.outdir, "--no-color", stdout=out) with open(os.path.join(self.outdir, "representative_records.tsv")) as f: lines = f.readlines() self.assertEqual(len(lines), 1) @@ -44,14 +44,14 @@ def test_representative_records_three(self): factories.DataAffiliateAgreementFactory.create() factories.NonDataAffiliateAgreementFactory.create() out = StringIO() - call_command("cdsa_records", self.outdir, "--no-color", stdout=out) + call_command("cdsa_records", "--outdir", self.outdir, "--no-color", stdout=out) with open(os.path.join(self.outdir, "representative_records.tsv")) as f: lines = f.readlines() self.assertEqual(len(lines), 4) def test_study_records_zero(self): out = StringIO() - call_command("cdsa_records", self.outdir, "--no-color", stdout=out) + call_command("cdsa_records", "--outdir", self.outdir, "--no-color", stdout=out) with open(os.path.join(self.outdir, "study_records.tsv")) as f: lines = f.readlines() self.assertEqual(len(lines), 1) @@ -61,14 +61,14 @@ def test_study_records_one(self): signed_agreement__is_primary=True ) out = StringIO() - call_command("cdsa_records", self.outdir, "--no-color", stdout=out) + call_command("cdsa_records", "--outdir", self.outdir, "--no-color", stdout=out) with open(os.path.join(self.outdir, "study_records.tsv")) as f: lines = f.readlines() self.assertEqual(len(lines), 2) def test_cdsa_workspace_records_zero(self): out = StringIO() - call_command("cdsa_records", self.outdir, "--no-color", stdout=out) + call_command("cdsa_records", "--outdir", self.outdir, "--no-color", stdout=out) with open(os.path.join(self.outdir, "workspace_records.tsv")) as f: lines = f.readlines() self.assertEqual(len(lines), 1) @@ -77,7 +77,7 @@ def test_cdsa_workspace_records_one(self): agreement = factories.DataAffiliateAgreementFactory.create() factories.CDSAWorkspaceFactory.create(study=agreement.study) out = StringIO() - call_command("cdsa_records", self.outdir, "--no-color", stdout=out) + call_command("cdsa_records", "--outdir", self.outdir, "--no-color", stdout=out) with open(os.path.join(self.outdir, "workspace_records.tsv")) as f: lines = f.readlines() self.assertEqual(len(lines), 2) @@ -86,5 +86,7 @@ def test_directory_exists(self): os.mkdir(self.outdir) out = StringIO() with self.assertRaises(CommandError) as e: - call_command("cdsa_records", self.outdir, "--no-color", stdout=out) + call_command( + "cdsa_records", "--outdir", self.outdir, "--no-color", stdout=out + ) self.assertIn("already exists", str(e.exception)) From 66ea743917eba6720db5674f6162645b1aed2f2a Mon Sep 17 00:00:00 2001 From: Jonas Carson Date: Wed, 24 Jan 2024 16:02:15 -0800 Subject: [PATCH 2/4] Add cdsa_records command. Stop redirecting stderr to cron log and set mailto so that we are notified when a job encounters errors --- primed_apps.cron | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/primed_apps.cron b/primed_apps.cron index ef7a8db7..99a331cf 100644 --- a/primed_apps.cron +++ b/primed_apps.cron @@ -1,6 +1,12 @@ -# PRIMED_APPS crontab +# PRIMED_APPS crontab - maintained in git primed_apps.cron +# Send errors to primedweb email +MAILTO="primedweb@uw.edu" + # nightly except sunday at 01:00 -0 1 * * MON-SAT . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py run_anvil_audit --email primedconsortium@uw.edu --errors-only >> cron.log 2>&1 +0 1 * * MON-SAT . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py run_anvil_audit --email primedconsortium@uw.edu --errors-only >> cron.log # sunday night at 01:00 -0 1 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py run_anvil_audit --email primedconsortium@uw.edu >> cron.log 2>&1 +0 1 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py run_anvil_audit --email primedconsortium@uw.edu >> cron.log + +# Weekly cdsa_records run Sundays at 03:00 - disabled until permissions issues are resolved +0 3 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py cdsa_records /projects/primed/records/cdsa/$(date +'\%Y-\%m-\%d') >> cron.log From 97be96b7127af578ac0fc4463eea5d287e3f2cb5 Mon Sep 17 00:00:00 2001 From: Jonas Carson Date: Wed, 24 Jan 2024 16:03:01 -0800 Subject: [PATCH 3/4] Update dev cron to have mailto and not redirect stderr --- primed_apps_dev.cron | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/primed_apps_dev.cron b/primed_apps_dev.cron index 99032ef3..b2e0e994 100644 --- a/primed_apps_dev.cron +++ b/primed_apps_dev.cron @@ -1,7 +1,8 @@ # PRIMED_APPS_DEV crontab - disabled by default +MAILTO="primedweb@uw.edu" # can be enabled for testing. # nightly except sunday at 01:00 -# 0 1 * * MON-SAT . /var/www/django/primed_apps_dev/primed-apps-dev-activate.sh; python manage.py run_anvil_audit --email primedconsortium@uw.edu --errors-only >> cron.log 2>&1 +# 0 1 * * MON-SAT . /var/www/django/primed_apps_dev/primed-apps-dev-activate.sh; python manage.py run_anvil_audit --email primedconsortium@uw.edu --errors-only >> cron.log # sunday night at 01:00 -# 0 1 * * SUN . /var/www/django/primed_apps_dev/primed-apps-dev-activate.sh; python manage.py run_anvil_audit --email primedconsortium@uw.edu >> cron.log 2>&1 +# 0 1 * * SUN . /var/www/django/primed_apps_dev/primed-apps-dev-activate.sh; python manage.py run_anvil_audit --email primedconsortium@uw.edu >> cron.log From b13c4d2e3a8819e5a211baafe8aaf9413c986e22 Mon Sep 17 00:00:00 2001 From: Jonas Carson Date: Mon, 29 Jan 2024 09:42:01 -0800 Subject: [PATCH 4/4] Update cron command to use --outdir --- primed_apps.cron | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primed_apps.cron b/primed_apps.cron index 99a331cf..cf4a4f93 100644 --- a/primed_apps.cron +++ b/primed_apps.cron @@ -9,4 +9,4 @@ MAILTO="primedweb@uw.edu" 0 1 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py run_anvil_audit --email primedconsortium@uw.edu >> cron.log # Weekly cdsa_records run Sundays at 03:00 - disabled until permissions issues are resolved -0 3 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py cdsa_records /projects/primed/records/cdsa/$(date +'\%Y-\%m-\%d') >> cron.log +0 3 * * SUN . /var/www/django/primed_apps/primed-apps-activate.sh; python manage.py cdsa_records --outdir /projects/primed/records/cdsa/$(date +'\%Y-\%m-\%d') >> cron.log