diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f6d54b14bb..1c44bb6f20 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -59,7 +59,7 @@ jobs: path: geckodriver.log - name: Upload Coverage Results to Codecov - uses: codecov/codecov-action@v4.3.1 + uses: codecov/codecov-action@v4.4.1 with: files: coverage.xml diff --git a/bin/daily b/bin/daily index 6adb16794d..b4ea339958 100755 --- a/bin/daily +++ b/bin/daily @@ -36,6 +36,3 @@ $DTDIR/ietf/manage.py populate_yang_model_dirs -v0 # Re-run yang checks on active documents $DTDIR/ietf/manage.py run_yang_model_checks -v0 - -# Purge older PersonApiKeyEvents -$DTDIR/ietf/manage.py purge_old_personal_api_key_events 14 diff --git a/bin/weekly b/bin/weekly deleted file mode 100755 index 8e01c273ca..0000000000 --- a/bin/weekly +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Weekly datatracker jobs. -# -# This script is expected to be triggered by cron from -# /etc/cron.d/datatracker -export LANG=en_US.UTF-8 -export PYTHONIOENCODING=utf-8 - -DTDIR=/a/www/ietf-datatracker/web -cd $DTDIR/ - -# Set up the virtual environment -source $DTDIR/env/bin/activate - -logger -p user.info -t cron "Running $DTDIR/bin/weekly" - - -# Send out weekly summaries of apikey usage - -$DTDIR/ietf/manage.py send_apikey_usage_emails - diff --git a/docker/README.md b/docker/README.md index bc9af7c212..2e4b50d8ff 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,7 +6,7 @@ > See the [IETF Tools Windows Dev guide](https://github.com/ietf-tools/.github/blob/main/docs/windows-dev.md) on how to get started when using Windows. -2. On Linux, you must also install [Docker Compose](https://docs.docker.com/compose/install/). Docker Desktop for Mac and Windows already include Docker Compose. +2. On Linux, you must [install Docker Compose manually](https://docs.docker.com/compose/install/linux/#install-the-plugin-manually) and not install Docker Desktop. On Mac and Windows install Docker Desktop which already includes Docker Compose. 2. If you have a copy of the datatracker code checked out already, simply `cd` to the top-level directory. @@ -183,3 +183,18 @@ The content of the source files will be copied into the target `.ics` files. Mak ### Missing assets in the data folder Because including all assets in the image would significantly increase the file size, they are not included by default. You can however fetch them by running the **Fetch assets via rsync** task in VS Code or run manually the script `docker/scripts/app-rsync-extras.sh` + + +### Linux file permissions leaking to the host system + +If on the host filesystem you have permissions that look like this, + +```bash +$ ls -la +total 4624 +drwxrwxr-x 2 100999 100999 4096 May 25 07:56 bin +drwxrwxr-x 5 100999 100999 4096 May 25 07:56 client +(etc...) +``` + +Try uninstalling Docker Desktop and installing Docker Compose manually. The Docker Compose bundled with Docker Desktop is incompatible with our software. See also [Rootless Docker: file ownership changes #3343](https://github.com/lando/lando/issues/3343), [Docker context desktop-linux has container permission issues #75](https://github.com/docker/desktop-linux/issues/75). \ No newline at end of file diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh index c8286b2429..b96b88f1f5 100755 --- a/docker/scripts/app-init.sh +++ b/docker/scripts/app-init.sh @@ -2,6 +2,12 @@ WORKSPACEDIR="/workspace" +# Handle Linux host mounting the workspace dir as root +if [ ! -O "${WORKSPACEDIR}/ietf" ]; then + sudo chown -R dev:dev $WORKSPACEDIR +fi + +# Start rsyslog service sudo service rsyslog start &>/dev/null # Add /workspace as a safe git directory diff --git a/ietf/api/tests.py b/ietf/api/tests.py index c4e627c52e..66bc7a3be7 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -30,7 +30,7 @@ from ietf.group.factories import RoleFactory from ietf.meeting.factories import MeetingFactory, SessionFactory from ietf.meeting.models import Session -from ietf.nomcom.models import Volunteer, NomCom +from ietf.nomcom.models import Volunteer from ietf.nomcom.factories import NomComFactory, nomcom_kwargs_for_year from ietf.person.factories import PersonFactory, random_faker, EmailFactory from ietf.person.models import Email, User @@ -828,7 +828,7 @@ def test_api_new_meeting_registration_nomcom_volunteer(self): 'reg_type': 'onsite', 'ticket_type': '', 'checkedin': 'False', - 'is_nomcom_volunteer': 'True', + 'is_nomcom_volunteer': 'False', } person = PersonFactory() reg['email'] = person.email().address @@ -842,16 +842,22 @@ def test_api_new_meeting_registration_nomcom_volunteer(self): # create appropriate group and nomcom objects nomcom = NomComFactory.create(is_accepting_volunteers=True, **nomcom_kwargs_for_year(year)) url = urlreverse('ietf.api.views.api_new_meeting_registration') - r = self.client.post(url, reg) - self.assertContains(r, 'Invalid apikey', status_code=403) oidcp = PersonFactory(user__is_staff=True) # Make sure 'oidcp' has an acceptable role RoleFactory(name_id='robot', person=oidcp, email=oidcp.email(), group__acronym='secretariat') key = PersonalApiKey.objects.create(person=oidcp, endpoint=url) reg['apikey'] = key.hash() + + # first test is_nomcom_volunteer False r = self.client.post(url, reg) - nomcom = NomCom.objects.last() self.assertContains(r, "Accepted, New registration", status_code=202) + # assert no Volunteers exists + self.assertEqual(Volunteer.objects.count(), 0) + + # test is_nomcom_volunteer True + reg['is_nomcom_volunteer'] = 'True' + r = self.client.post(url, reg) + self.assertContains(r, "Accepted, Updated registration", status_code=202) # assert Volunteer exists self.assertEqual(Volunteer.objects.count(), 1) volunteer = Volunteer.objects.last() diff --git a/ietf/api/views.py b/ietf/api/views.py index 6f97efbdb8..3c3ea0d5a9 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -212,7 +212,7 @@ def err(code, text): response += ", Email sent" # handle nomcom volunteer - if data['is_nomcom_volunteer'] and object.person: + if request.POST.get('is_nomcom_volunteer', 'false').lower() == 'true' and object.person: try: nomcom = NomCom.objects.get(is_accepting_volunteers=True) except (NomCom.DoesNotExist, NomCom.MultipleObjectsReturned): diff --git a/ietf/bin/aliases-from-json.py b/ietf/bin/aliases-from-json.py index 72fcb469f7..a0c383a1ac 100644 --- a/ietf/bin/aliases-from-json.py +++ b/ietf/bin/aliases-from-json.py @@ -61,6 +61,13 @@ def generate_files(records, adest, vdest, postconfirm, vdomain): shutil.move(vpath, vdest) +def directory_path(val): + p = Path(val) + if p.is_dir(): + return p + else: + raise argparse.ArgumentTypeError(f"{p} is not a directory") + if __name__ == "__main__": parser = argparse.ArgumentParser( description="Convert a JSON stream of draft alias definitions into alias / virtual alias files." @@ -73,7 +80,7 @@ def generate_files(records, adest, vdest, postconfirm, vdomain): parser.add_argument( "--output-dir", default="./", - type=Path, + type=directory_path, help="Destination for output files.", ) parser.add_argument( @@ -87,8 +94,6 @@ def generate_files(records, adest, vdest, postconfirm, vdomain): help=f"Virtual domain (defaults to {VDOMAIN}_", ) args = parser.parse_args() - if not args.output_dir.is_dir(): - sys.stderr.write("Error: output-dir must be a directory") data = json.load(sys.stdin) generate_files( data["aliases"], diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py index 9c9287dab2..e18b2abfd9 100644 --- a/ietf/doc/tests_ballot.py +++ b/ietf/doc/tests_ballot.py @@ -32,7 +32,7 @@ from ietf.utils.test_utils import TestCase, login_testing_unauthorized from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.text import unwrap -from ietf.utils.timezone import date_today +from ietf.utils.timezone import date_today, datetime_today class EditPositionTests(TestCase): @@ -529,6 +529,7 @@ def test_issue_ballot_warn_if_early(self): login_testing_unauthorized(self, "secretary", url) # expect warning about issuing a ballot before IETF Last Call is done + # No last call has yet been issued r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) @@ -536,6 +537,38 @@ def test_issue_ballot_warn_if_early(self): self.assertTrue(q('[class=text-danger]:contains("not completed IETF Last Call")')) self.assertTrue(q('[type=submit]:contains("Save")')) + # Last call exists but hasn't expired + LastCallDocEvent.objects.create( + doc=draft, + expires=datetime_today()+datetime.timedelta(days=14), + by=Person.objects.get(name="(System)") + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertTrue(q('[class=text-danger]:contains("not completed IETF Last Call")')) + + # Last call exists and has expired + LastCallDocEvent.objects.filter(doc=draft).update(expires=datetime_today()-datetime.timedelta(days=2)) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertFalse(q('[class=text-danger]:contains("not completed IETF Last Call")')) + + for state_slug in ["lc", "watching", "ad-eval"]: + draft.set_state(State.objects.get(type="draft-iesg",slug=state_slug)) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertTrue(q('[class=text-danger]:contains("It would be unexpected to issue a ballot while in this state.")')) + + draft.set_state(State.objects.get(type="draft-iesg",slug="writeupw")) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertFalse(q('[class=text-danger]:contains("It would be unexpected to issue a ballot while in this state.")')) + + def test_edit_approval_text(self): ad = Person.objects.get(user__username="ad") draft = WgDraftFactory(ad=ad,states=[('draft','active'),('draft-iesg','iesg-eva')],intended_std_level_id='ps',group__parent=Group.objects.get(acronym='farfut')) diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py index 9b0ccdcead..02b55249d6 100644 --- a/ietf/doc/views_ballot.py +++ b/ietf/doc/views_ballot.py @@ -38,6 +38,7 @@ from ietf.message.utils import infer_message from ietf.name.models import BallotPositionName, DocTypeName from ietf.person.models import Person +from ietf.utils.fields import ModelMultipleChoiceField from ietf.utils.mail import send_mail_text, send_mail_preformatted from ietf.utils.decorators import require_api_key from ietf.utils.response import permission_denied @@ -686,7 +687,8 @@ def ballot_writeupnotes(request, name): dict(doc=doc, back_url=doc.get_absolute_url(), ballot_issued=bool(doc.latest_event(type="sent_ballot_announcement")), - ballot_issue_danger=bool(prev_state.slug in ['ad-eval', 'lc']), + warn_lc = not doc.docevent_set.filter(lastcalldocevent__expires__date__lt=date_today(DEADLINE_TZINFO)).exists(), + warn_unexpected_state= prev_state if bool(prev_state.slug in ['watching', 'ad-eval', 'lc']) else None, ballot_writeup_form=form, need_intended_status=need_intended_status, )) @@ -931,7 +933,7 @@ def approve_ballot(request, name): class ApproveDownrefsForm(forms.Form): - checkboxes = forms.ModelMultipleChoiceField( + checkboxes = ModelMultipleChoiceField( widget = forms.CheckboxSelectMultiple, queryset = RelatedDocument.objects.none(), ) diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py index 1deca4503c..30175491da 100644 --- a/ietf/doc/views_draft.py +++ b/ietf/doc/views_draft.py @@ -52,6 +52,7 @@ from ietf.utils.mail import send_mail, send_mail_message, on_behalf_of from ietf.utils.textupload import get_cleaned_text_file_content from ietf.utils import log +from ietf.utils.fields import ModelMultipleChoiceField from ietf.utils.response import permission_denied from ietf.utils.timezone import datetime_today, DEADLINE_TZINFO @@ -390,9 +391,9 @@ def replaces(request, name): )) class SuggestedReplacesForm(forms.Form): - replaces = forms.ModelMultipleChoiceField(queryset=Document.objects.all(), - label="Suggestions", required=False, widget=forms.CheckboxSelectMultiple, - help_text="Select only the documents that are replaced by this document") + replaces = ModelMultipleChoiceField(queryset=Document.objects.all(), + label="Suggestions", required=False, widget=forms.CheckboxSelectMultiple, + help_text="Select only the documents that are replaced by this document") comment = forms.CharField(label="Optional comment", widget=forms.Textarea, required=False, strip=False) def __init__(self, suggested, *args, **kwargs): @@ -1601,7 +1602,7 @@ class ChangeStreamStateForm(forms.Form): new_state = forms.ModelChoiceField(queryset=State.objects.filter(used=True), label='State' ) weeks = forms.IntegerField(label='Expected weeks in state',required=False) comment = forms.CharField(widget=forms.Textarea, required=False, help_text="Optional comment for the document history.", strip=False) - tags = forms.ModelMultipleChoiceField(queryset=DocTagName.objects.filter(used=True), widget=forms.CheckboxSelectMultiple, required=False) + tags = ModelMultipleChoiceField(queryset=DocTagName.objects.filter(used=True), widget=forms.CheckboxSelectMultiple, required=False) def __init__(self, *args, **kwargs): doc = kwargs.pop("doc") diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py index 646b51b09c..bb9e56742d 100644 --- a/ietf/doc/views_review.py +++ b/ietf/doc/views_review.py @@ -52,7 +52,7 @@ from ietf.utils.textupload import get_cleaned_text_file_content from ietf.utils.mail import send_mail_message from ietf.mailtrigger.utils import gather_address_lists -from ietf.utils.fields import MultiEmailField +from ietf.utils.fields import ModelMultipleChoiceField, MultiEmailField from ietf.utils.http import is_ajax from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, DEADLINE_TZINFO @@ -68,7 +68,7 @@ def clean_doc_revision(doc, rev): return rev class RequestReviewForm(forms.ModelForm): - team = forms.ModelMultipleChoiceField(queryset=Group.objects.all(), widget=forms.CheckboxSelectMultiple) + team = ModelMultipleChoiceField(queryset=Group.objects.all(), widget=forms.CheckboxSelectMultiple) deadline = DatepickerDateField(date_format="yyyy-mm-dd", picker_settings={ "autoclose": "1", "start-date": "+0d" }) class Meta: diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 422e38f7dd..2ef4ee83e6 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -69,6 +69,7 @@ from ietf.person.models import Person from ietf.person.utils import get_active_ads from ietf.utils.draft_search import normalize_draftname +from ietf.utils.fields import ModelMultipleChoiceField from ietf.utils.log import log from ietf.doc.utils_search import prepare_document_table, doc_type, doc_state, doc_type_name, AD_WORKLOAD from ietf.ietfauth.utils import has_role @@ -100,7 +101,7 @@ class SearchForm(forms.Form): ("ad", "AD"), ("-ad", "AD (desc)"), ), required=False, widget=forms.HiddenInput) - doctypes = forms.ModelMultipleChoiceField(queryset=DocTypeName.objects.filter(used=True).exclude(slug__in=('draft', 'rfc', 'bcp', 'std', 'fyi', 'liai-att')).order_by('name'), required=False) + doctypes = ModelMultipleChoiceField(queryset=DocTypeName.objects.filter(used=True).exclude(slug__in=('draft', 'rfc', 'bcp', 'std', 'fyi', 'liai-att')).order_by('name'), required=False) def __init__(self, *args, **kwargs): super(SearchForm, self).__init__(*args, **kwargs) diff --git a/ietf/ietfauth/management/__init__.py b/ietf/ietfauth/management/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/ietf/ietfauth/management/commands/__init__.py b/ietf/ietfauth/management/commands/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/ietf/ietfauth/management/commands/send_apikey_usage_emails.py b/ietf/ietfauth/management/commands/send_apikey_usage_emails.py deleted file mode 100644 index d3fce1bcc2..0000000000 --- a/ietf/ietfauth/management/commands/send_apikey_usage_emails.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright The IETF Trust 2017-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -import datetime - -from textwrap import dedent - -from django.conf import settings -from django.core.management.base import BaseCommand -from django.utils import timezone - -import debug # pyflakes:ignore - -from ietf.person.models import PersonalApiKey, PersonApiKeyEvent -from ietf.utils.mail import send_mail - - -class Command(BaseCommand): - """ - Send out emails to all persons who have personal API keys about usage. - - Usage is show over the given period, where the default period is 7 days. - """ - - help = dedent(__doc__).strip() - - def add_arguments(self, parser): - parser.add_argument('-d', '--days', dest='days', type=int, default=7, - help='The period over which to show usage.') - - def handle(self, *filenames, **options): - """ - """ - - self.verbosity = int(options.get('verbosity')) - days = options.get('days') - - keys = PersonalApiKey.objects.filter(valid=True) - for key in keys: - earliest = timezone.now() - datetime.timedelta(days=days) - events = PersonApiKeyEvent.objects.filter(key=key, time__gt=earliest) - count = events.count() - events = events[:32] - if count: - key_name = key.hash()[:8] - subject = "API key usage for key '%s' for the last %s days" %(key_name, days) - to = key.person.email_address() - frm = settings.DEFAULT_FROM_EMAIL - send_mail(None, to, frm, subject, 'utils/apikey_usage_report.txt', {'person':key.person, - 'days':days, 'key':key, 'key_name':key_name, 'count':count, 'events':events, } ) - diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py index 29afa56d78..503c091a85 100644 --- a/ietf/ietfauth/tests.py +++ b/ietf/ietfauth/tests.py @@ -41,6 +41,7 @@ from ietf.nomcom.factories import NomComFactory from ietf.person.factories import PersonFactory, EmailFactory, UserFactory, PersonalApiKeyFactory from ietf.person.models import Person, Email, PersonalApiKey +from ietf.person.tasks import send_apikey_usage_emails_task from ietf.review.factories import ReviewRequestFactory, ReviewAssignmentFactory from ietf.review.models import ReviewWish, UnavailablePeriod from ietf.stats.models import MeetingRegistration @@ -50,8 +51,6 @@ from ietf.utils.timezone import date_today -import ietf.ietfauth.views - if os.path.exists(settings.HTPASSWD_COMMAND): skip_htpasswd_command = False skip_message = "" @@ -83,30 +82,30 @@ def tearDown(self): super().tearDown() def test_index(self): - self.assertEqual(self.client.get(urlreverse(ietf.ietfauth.views.index)).status_code, 200) + self.assertEqual(self.client.get(urlreverse("ietf.ietfauth.views.index")).status_code, 200) def test_login_and_logout(self): PersonFactory(user__username='plain') # try logging in without a next - r = self.client.get(urlreverse(ietf.ietfauth.views.login)) + r = self.client.get(urlreverse("ietf.ietfauth.views.login")) self.assertEqual(r.status_code, 200) - r = self.client.post(urlreverse(ietf.ietfauth.views.login), {"username":"plain", "password":"plain+password"}) + r = self.client.post(urlreverse("ietf.ietfauth.views.login"), {"username":"plain", "password":"plain+password"}) self.assertEqual(r.status_code, 302) - self.assertEqual(urlsplit(r["Location"])[2], urlreverse(ietf.ietfauth.views.profile)) + self.assertEqual(urlsplit(r["Location"])[2], urlreverse("ietf.ietfauth.views.profile")) # try logging out r = self.client.post(urlreverse('django.contrib.auth.views.logout'), {}) self.assertEqual(r.status_code, 200) self.assertNotContains(r, "accounts/logout") - r = self.client.get(urlreverse(ietf.ietfauth.views.profile)) + r = self.client.get(urlreverse("ietf.ietfauth.views.profile")) self.assertEqual(r.status_code, 302) - self.assertEqual(urlsplit(r["Location"])[2], urlreverse(ietf.ietfauth.views.login)) + self.assertEqual(urlsplit(r["Location"])[2], urlreverse("ietf.ietfauth.views.login")) # try logging in with a next - r = self.client.post(urlreverse(ietf.ietfauth.views.login) + "?next=/foobar", {"username":"plain", "password":"plain+password"}) + r = self.client.post(urlreverse("ietf.ietfauth.views.login") + "?next=/foobar", {"username":"plain", "password":"plain+password"}) self.assertEqual(r.status_code, 302) self.assertEqual(urlsplit(r["Location"])[2], "/foobar") @@ -137,19 +136,19 @@ def _test_login(url): # try with a trivial next _test_login("/") # try with a next that requires login - _test_login(urlreverse(ietf.ietfauth.views.profile)) + _test_login(urlreverse("ietf.ietfauth.views.profile")) def test_login_with_different_email(self): person = PersonFactory(user__username='plain') email = EmailFactory(person=person) # try logging in without a next - r = self.client.get(urlreverse(ietf.ietfauth.views.login)) + r = self.client.get(urlreverse("ietf.ietfauth.views.login")) self.assertEqual(r.status_code, 200) - r = self.client.post(urlreverse(ietf.ietfauth.views.login), {"username":email, "password":"plain+password"}) + r = self.client.post(urlreverse("ietf.ietfauth.views.login"), {"username":email, "password":"plain+password"}) self.assertEqual(r.status_code, 302) - self.assertEqual(urlsplit(r["Location"])[2], urlreverse(ietf.ietfauth.views.profile)) + self.assertEqual(urlsplit(r["Location"])[2], urlreverse("ietf.ietfauth.views.profile")) def extract_confirm_url(self, confirm_email): # dig out confirm_email link @@ -176,7 +175,7 @@ def username_in_htpasswd_file(self, username): # For the lowered barrier to account creation period, we are disabling this kind of failure # def test_create_account_failure(self): - # url = urlreverse(ietf.ietfauth.views.create_account) + # url = urlreverse("ietf.ietfauth.views.create_account") # # get # r = self.client.get(url) @@ -195,7 +194,7 @@ def test_create_account_failure_template(self): self.assertTrue("Additional Assistance Required" in r) def register(self, email): - url = urlreverse(ietf.ietfauth.views.create_account) + url = urlreverse("ietf.ietfauth.views.create_account") # register email empty_outbox() @@ -240,7 +239,7 @@ def test_create_existing_account(self): note = get_payload_text(outbox[-1]) self.assertIn(email, note) self.assertIn("A datatracker account for that email already exists", note) - self.assertIn(urlreverse(ietf.ietfauth.views.password_reset), note) + self.assertIn(urlreverse("ietf.ietfauth.views.password_reset"), note) def test_ietfauth_profile(self): EmailFactory(person__user__username='plain') @@ -249,7 +248,7 @@ def test_ietfauth_profile(self): username = "plain" email_address = Email.objects.filter(person__user__username=username).first().address - url = urlreverse(ietf.ietfauth.views.profile) + url = urlreverse("ietf.ietfauth.views.profile") login_testing_unauthorized(self, username, url) @@ -400,7 +399,7 @@ def test_ietfauth_profile(self): def test_email_case_insensitive_protection(self): EmailFactory(address="TestAddress@example.net") person = PersonFactory() - url = urlreverse(ietf.ietfauth.views.profile) + url = urlreverse("ietf.ietfauth.views.profile") login_testing_unauthorized(self, person.user.username, url) data = { @@ -441,7 +440,7 @@ def test_nomcom_dressing_on_profile(self): def test_reset_password(self): - url = urlreverse(ietf.ietfauth.views.password_reset) + url = urlreverse("ietf.ietfauth.views.password_reset") email = 'someone@example.com' password = 'foobar' @@ -507,7 +506,7 @@ def test_reset_password(self): self.assertEqual(len(outbox), 1) confirm_url = self.extract_confirm_url(outbox[-1]) - r = self.client.post(urlreverse(ietf.ietfauth.views.login), {'username': email, 'password': password}) + r = self.client.post(urlreverse("ietf.ietfauth.views.login"), {'username': email, 'password': password}) r = self.client.get(confirm_url) self.assertEqual(r.status_code, 404) @@ -589,7 +588,7 @@ def test_review_overview(self): availability="unavailable", ) - url = urlreverse(ietf.ietfauth.views.review_overview) + url = urlreverse("ietf.ietfauth.views.review_overview") login_testing_unauthorized(self, reviewer.user.username, url) @@ -633,10 +632,9 @@ def test_htpasswd_file_with_htpasswd_binary(self): def test_change_password(self): - - chpw_url = urlreverse(ietf.ietfauth.views.change_password) - prof_url = urlreverse(ietf.ietfauth.views.profile) - login_url = urlreverse(ietf.ietfauth.views.login) + chpw_url = urlreverse("ietf.ietfauth.views.change_password") + prof_url = urlreverse("ietf.ietfauth.views.profile") + login_url = urlreverse("ietf.ietfauth.views.login") redir_url = '%s?next=%s' % (login_url, chpw_url) # get without logging in @@ -681,9 +679,9 @@ def test_change_password(self): def test_change_username(self): - chun_url = urlreverse(ietf.ietfauth.views.change_username) - prof_url = urlreverse(ietf.ietfauth.views.profile) - login_url = urlreverse(ietf.ietfauth.views.login) + chun_url = urlreverse("ietf.ietfauth.views.change_username") + prof_url = urlreverse("ietf.ietfauth.views.profile") + login_url = urlreverse("ietf.ietfauth.views.login") redir_url = '%s?next=%s' % (login_url, chun_url) # get without logging in @@ -856,9 +854,6 @@ def test_apikey_errors(self): key2.delete() def test_send_apikey_report(self): - from ietf.ietfauth.management.commands.send_apikey_usage_emails import Command - from ietf.utils.mail import outbox, empty_outbox - person = RoleFactory(name_id='secr', group__acronym='secretariat').person url = urlreverse('ietf.ietfauth.views.apikey_create') @@ -883,9 +878,8 @@ def test_send_apikey_report(self): date = str(date_today()) empty_outbox() - cmd = Command() - cmd.handle(verbosity=0, days=7) - + send_apikey_usage_emails_task(days=7) + self.assertEqual(len(outbox), len(endpoints)) for mail in outbox: body = get_payload_text(mail) diff --git a/ietf/ietfauth/urls.py b/ietf/ietfauth/urls.py index 30e639ad65..7493fe5c97 100644 --- a/ietf/ietfauth/urls.py +++ b/ietf/ietfauth/urls.py @@ -14,7 +14,7 @@ url(r'^confirmnewemail/(?P[^/]+)/$', views.confirm_new_email), url(r'^create/$', views.create_account), url(r'^create/confirm/(?P[^/]+)/$', views.confirm_account), - url(r'^login/$', views.login), + url(r'^login/$', views.AnyEmailLoginView.as_view(), name="ietf.ietfauth.views.login"), url(r'^logout/$', LogoutView.as_view(), name="django.contrib.auth.views.logout"), url(r'^password/$', views.change_password), url(r'^profile/$', views.profile), diff --git a/ietf/ietfauth/views.py b/ietf/ietfauth/views.py index 8c61b8356a..7c3f72108e 100644 --- a/ietf/ietfauth/views.py +++ b/ietf/ietfauth/views.py @@ -45,7 +45,7 @@ from django import forms from django.contrib import messages from django.conf import settings -from django.contrib.auth import update_session_auth_hash, logout, authenticate +from django.contrib.auth import logout, update_session_auth_hash from django.contrib.auth.decorators import login_required from django.contrib.auth.forms import AuthenticationForm from django.contrib.auth.hashers import identify_hasher @@ -752,53 +752,57 @@ def change_username(request): return render(request, 'registration/change_username.html', {'form': form}) - -def login(request, extra_context=None): - """ - This login function is a wrapper around django's login() for the purpose - of providing a notification if the user's password has been cleared. The - warning will be triggered if the password field has been set to something - which is not recognized as a valid password hash. +class AnyEmailAuthenticationForm(AuthenticationForm): + """AuthenticationForm that allows any email address as the username + + Also performs a check for a cleared password field and provides a helpful error message + if that applies to the user attempting to log in. """ - - if request.method == "POST": - form = AuthenticationForm(request, data=request.POST) - username = form.data.get('username') - user = User.objects.filter(username__iexact=username).first() # Consider _never_ actually looking for the User username and only looking at Email - if not user: - # try to find user ID from the email address + _unauthenticated_user = None + + def clean_username(self): + username = self.cleaned_data.get("username", None) + if username is None: + raise self.get_invalid_login_error() + user = User.objects.filter(username__iexact=username).first() + if user is None: email = Email.objects.filter(address=username).first() - if email and email.person and email.person.user: - u2 = email.person.user - # be conservative, only accept this if login is valid - if u2: - pw = form.data.get('password') - au = authenticate(request, username=u2.username, password=pw) - if au: - # kludge to change the querydict - q2 = request.POST.copy() - q2['username'] = u2.username - request.POST = q2 - user = u2 - # - if user: - try: - identify_hasher(user.password) + if email and email.person: + user = email.person.user # might be None + if user is None: + raise self.get_invalid_login_error() + self._unauthenticated_user = user # remember this for the clean() method + return user.username + + def clean(self): + if self._unauthenticated_user is not None: + try: + identify_hasher(self._unauthenticated_user.password) except ValueError: - extra_context = {"alert": - "Note: Your password has been cleared because " - "of possible password leakage. " - "Please use the password reset link below " - "to set a new password for your account.", - } - response = LoginView.as_view(extra_context=extra_context)(request) - if isinstance(response, HttpResponseRedirect) and user and user.is_authenticated: - try: - user.person - except Person.DoesNotExist: - logout(request) - response = render(request, 'registration/missing_person.html') - return response + self.add_error( + "password", + 'Your password has been cleared because of possible password leakage. ' + 'Please use the "Forgot your password?" button below to set a new password ' + 'for your account.', + ) + return super().clean() + + +class AnyEmailLoginView(LoginView): + """LoginView that allows any email address as the username + + Redirects to the missing_person page instead of logging in if the user does not have a Person + """ + form_class = AnyEmailAuthenticationForm + + def form_valid(self, form): + """Security check complete. Log the user in if they have a Person.""" + user = form.get_user() # user has authenticated at this point + if not hasattr(user, "person"): + logout(self.request) # should not be logged in yet, but just in case... + return render(self.request, "registration/missing_person.html") + return super().form_valid(form) + @login_required @person_required diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py index 0a6974e5bb..1d91041b25 100644 --- a/ietf/liaisons/forms.py +++ b/ietf/liaisons/forms.py @@ -32,7 +32,7 @@ from ietf.person.models import Email from ietf.person.fields import SearchableEmailField from ietf.doc.models import Document -from ietf.utils.fields import DatepickerDateField +from ietf.utils.fields import DatepickerDateField, ModelMultipleChoiceField from ietf.utils.timezone import date_today, datetime_from_date, DEADLINE_TZINFO from functools import reduce @@ -200,7 +200,7 @@ def get_results(self): return results -class CustomModelMultipleChoiceField(forms.ModelMultipleChoiceField): +class CustomModelMultipleChoiceField(ModelMultipleChoiceField): '''If value is a QuerySet, return it as is (for use in widget.render)''' def prepare_value(self, value): if isinstance(value, QuerySetAny): @@ -215,12 +215,12 @@ def prepare_value(self, value): class LiaisonModelForm(forms.ModelForm): '''Specify fields which require a custom widget or that are not part of the model. ''' - from_groups = forms.ModelMultipleChoiceField(queryset=Group.objects.all(),label='Groups',required=False) + from_groups = ModelMultipleChoiceField(queryset=Group.objects.all(),label='Groups',required=False) from_groups.widget.attrs["class"] = "select2-field" from_groups.widget.attrs['data-minimum-input-length'] = 0 from_contact = forms.EmailField() # type: Union[forms.EmailField, SearchableEmailField] to_contacts = forms.CharField(label="Contacts", widget=forms.Textarea(attrs={'rows':'3', }), strip=False) - to_groups = forms.ModelMultipleChoiceField(queryset=Group.objects,label='Groups',required=False) + to_groups = ModelMultipleChoiceField(queryset=Group.objects,label='Groups',required=False) to_groups.widget.attrs["class"] = "select2-field" to_groups.widget.attrs['data-minimum-input-length'] = 0 deadline = DatepickerDateField(date_format="yyyy-mm-dd", picker_settings={"autoclose": "1" }, label='Deadline', required=True) diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py index 2cec669db9..b31ffb6cd7 100644 --- a/ietf/meeting/forms.py +++ b/ietf/meeting/forms.py @@ -28,7 +28,13 @@ from ietf.message.models import Message from ietf.name.models import TimeSlotTypeName, SessionPurposeName from ietf.person.models import Person -from ietf.utils.fields import DatepickerDateField, DurationField, MultiEmailField, DatepickerSplitDateTimeWidget +from ietf.utils.fields import ( + DatepickerDateField, + DatepickerSplitDateTimeWidget, + DurationField, + ModelMultipleChoiceField, + MultiEmailField, +) from ietf.utils.validators import ( validate_file_size, validate_mime_type, validate_file_extension, validate_no_html_frame) @@ -551,7 +557,7 @@ class SwapTimeslotsForm(forms.Form): queryset=TimeSlot.objects.none(), # default to none, fill in when we have a meeting widget=forms.TextInput, ) - rooms = forms.ModelMultipleChoiceField( + rooms = ModelMultipleChoiceField( required=True, queryset=Room.objects.none(), # default to none, fill in when we have a meeting widget=CsvModelPkInput, @@ -617,7 +623,7 @@ class TimeSlotCreateForm(forms.Form): ) duration = TimeSlotDurationField() show_location = forms.BooleanField(required=False, initial=True) - locations = forms.ModelMultipleChoiceField( + locations = ModelMultipleChoiceField( queryset=Room.objects.none(), widget=forms.CheckboxSelectMultiple, ) diff --git a/ietf/nomcom/forms.py b/ietf/nomcom/forms.py index 7db5009121..5987b22637 100644 --- a/ietf/nomcom/forms.py +++ b/ietf/nomcom/forms.py @@ -21,6 +21,7 @@ from ietf.person.models import Email from ietf.person.fields import (SearchableEmailField, SearchableEmailsField, SearchablePersonField, SearchablePersonsField ) +from ietf.utils.fields import ModelMultipleChoiceField from ietf.utils.mail import send_mail from ietf.mailtrigger.utils import gather_address_lists @@ -719,9 +720,9 @@ def set_nomcom(self, nomcom, person, instances=None): required= self.feedback_type.slug != 'comment', help_text='Hold down "Control", or "Command" on a Mac, to select more than one.') if self.feedback_type.slug == 'comment': - self.fields['topic'] = forms.ModelMultipleChoiceField(queryset=self.nomcom.topic_set.all(), - help_text='Hold down "Control" or "Command" on a Mac, to select more than one.', - required=False,) + self.fields['topic'] = ModelMultipleChoiceField(queryset=self.nomcom.topic_set.all(), + help_text='Hold down "Control" or "Command" on a Mac, to select more than one.', + required=False,) else: self.fields['position'] = forms.ModelChoiceField(queryset=Position.objects.get_by_nomcom(self.nomcom).filter(is_open=True), label="Position") self.fields['searched_email'] = SearchableEmailField(only_users=False,help_text="Try to find the candidate you are classifying with this field first. Only use the name and email fields below if this search does not find the candidate.",label="Candidate",required=False) @@ -847,7 +848,7 @@ class Meta: class NominationResponseCommentForm(forms.Form): comments = forms.CharField(widget=forms.Textarea,required=False,help_text="Any comments provided will be encrypted and will only be visible to the NomCom.", strip=False) -class NomcomVolunteerMultipleChoiceField(forms.ModelMultipleChoiceField): +class NomcomVolunteerMultipleChoiceField(ModelMultipleChoiceField): def label_from_instance(self, obj): year = obj.year() return f'Volunteer for the {year}/{year+1} Nominating Committee' diff --git a/ietf/person/management/commands/purge_old_personal_api_key_events.py b/ietf/person/management/commands/purge_old_personal_api_key_events.py deleted file mode 100644 index 66b9d2c33e..0000000000 --- a/ietf/person/management/commands/purge_old_personal_api_key_events.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright The IETF Trust 2021, All Rights Reserved -# -*- coding: utf-8 -*- - -from datetime import timedelta -from django.core.management.base import BaseCommand, CommandError -from django.db.models import Max, Min -from django.utils import timezone - -from ietf.person.models import PersonApiKeyEvent - - -class Command(BaseCommand): - help = 'Purge PersonApiKeyEvent instances older than KEEP_DAYS days' - - def add_arguments(self, parser): - parser.add_argument('keep_days', type=int, - help='Delete events older than this many days') - parser.add_argument('-n', '--dry-run', action='store_true', default=False, - help="Don't delete events, just show what would be done") - - - def handle(self, *args, **options): - keep_days = options['keep_days'] - dry_run = options['dry_run'] - verbosity = options.get("verbosity", 1) - - def _format_count(count, unit='day'): - return '{} {}{}'.format(count, unit, ('' if count == 1 else 's')) - - if keep_days < 0: - raise CommandError('Negative keep_days not allowed ({} was specified)'.format(keep_days)) - - if verbosity > 1: - self.stdout.write('purge_old_personal_api_key_events: Finding events older than {}\n'.format(_format_count(keep_days))) - if dry_run: - self.stdout.write('Dry run requested, records will not be deleted\n') - self.stdout.flush() - - now = timezone.now() - old_events = PersonApiKeyEvent.objects.filter( - time__lt=now - timedelta(days=keep_days) - ) - - stats = old_events.aggregate(Min('time'), Max('time')) - old_count = old_events.count() - if old_count == 0: - if verbosity > 1: - self.stdout.write('No events older than {} found\n'.format(_format_count(keep_days))) - return - - oldest_date = stats['time__min'] - oldest_ago = now - oldest_date - newest_date = stats['time__max'] - newest_ago = now - newest_date - - action_fmt = 'Would delete {}\n' if dry_run else 'Deleting {}\n' - if verbosity > 1: - self.stdout.write(action_fmt.format(_format_count(old_count, 'event'))) - self.stdout.write(' Oldest at {} ({} ago)\n'.format(oldest_date, _format_count(oldest_ago.days))) - self.stdout.write(' Most recent at {} ({} ago)\n'.format(newest_date, _format_count(newest_ago.days))) - self.stdout.flush() - - if not dry_run: - old_events.delete() diff --git a/ietf/person/management/commands/tests.py b/ietf/person/management/commands/tests.py deleted file mode 100644 index 38d770a588..0000000000 --- a/ietf/person/management/commands/tests.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright The IETF Trust 2021, All Rights Reserved -# -*- coding: utf-8 -*- - -import datetime -from io import StringIO - -from django.core.management import call_command, CommandError -from django.utils import timezone - -from ietf.person.factories import PersonApiKeyEventFactory -from ietf.person.models import PersonApiKeyEvent, PersonEvent -from ietf.utils.test_utils import TestCase - - -class CommandTests(TestCase): - @staticmethod - def _call_command(command_name, *args, **options): - out = StringIO() - options['stdout'] = out - call_command(command_name, *args, **options) - return out.getvalue() - - def _assert_purge_results(self, cmd_output, expected_delete_count, expected_kept_events): - self.assertNotIn('Dry run requested', cmd_output) - if expected_delete_count == 0: - delete_text = 'No events older than' - else: - delete_text = 'Deleting {} event'.format(expected_delete_count) - self.assertIn(delete_text, cmd_output) - self.assertCountEqual( - PersonApiKeyEvent.objects.all(), - expected_kept_events, - 'Wrong events were deleted' - ) - - def _assert_purge_dry_run_results(self, cmd_output, expected_delete_count, expected_kept_events): - self.assertIn('Dry run requested', cmd_output) - if expected_delete_count == 0: - delete_text = 'No events older than' - else: - delete_text = 'Would delete {} event'.format(expected_delete_count) - self.assertIn(delete_text, cmd_output) - self.assertCountEqual( - PersonApiKeyEvent.objects.all(), - expected_kept_events, - 'Events were deleted when dry-run option was used' - ) - - def test_purge_old_personal_api_key_events(self): - keep_days = 10 - - # Remember how many PersonEvents were present so we can verify they're cleaned up properly. - personevents_before = PersonEvent.objects.count() - - now = timezone.now() - # The first of these events will be timestamped a fraction of a second more than keep_days - # days ago by the time we call the management command, so will just barely chosen for purge. - old_events = [ - PersonApiKeyEventFactory(time=now - datetime.timedelta(days=n)) - for n in range(keep_days, 2 * keep_days + 1) - ] - num_old_events = len(old_events) - - recent_events = [ - PersonApiKeyEventFactory(time=now - datetime.timedelta(days=n)) - for n in range(0, keep_days) - ] - # We did not create recent_event timestamped exactly keep_days ago because it would - # be treated as an old_event by the management command. Create an event a few seconds - # on the "recent" side of keep_days old to test the threshold. - recent_events.append( - PersonApiKeyEventFactory( - time=now + datetime.timedelta(seconds=3) - datetime.timedelta(days=keep_days) - ) - ) - num_recent_events = len(recent_events) - - # call with dry run - output = self._call_command('purge_old_personal_api_key_events', str(keep_days), '--dry-run', '-v2') - self._assert_purge_dry_run_results(output, num_old_events, old_events + recent_events) - - # call for real - output = self._call_command('purge_old_personal_api_key_events', str(keep_days), '-v2') - self._assert_purge_results(output, num_old_events, recent_events) - self.assertEqual(PersonEvent.objects.count(), personevents_before + num_recent_events, - 'PersonEvents were not cleaned up properly') - - # repeat - there should be nothing left to delete - output = self._call_command('purge_old_personal_api_key_events', '--dry-run', str(keep_days), '-v2') - self._assert_purge_dry_run_results(output, 0, recent_events) - - output = self._call_command('purge_old_personal_api_key_events', str(keep_days), '-v2') - self._assert_purge_results(output, 0, recent_events) - self.assertEqual(PersonEvent.objects.count(), personevents_before + num_recent_events, - 'PersonEvents were not cleaned up properly') - - # and now delete the remaining events - output = self._call_command('purge_old_personal_api_key_events', '0', '-v2') - self._assert_purge_results(output, num_recent_events, []) - self.assertEqual(PersonEvent.objects.count(), personevents_before, - 'PersonEvents were not cleaned up properly') - - def test_purge_old_personal_api_key_events_rejects_invalid_arguments(self): - """The purge_old_personal_api_key_events command should reject invalid arguments""" - event = PersonApiKeyEventFactory(time=timezone.now() - datetime.timedelta(days=30)) - - with self.assertRaises(CommandError): - self._call_command('purge_old_personal_api_key_events') - - with self.assertRaises(CommandError): - self._call_command('purge_old_personal_api_key_events', '-15') - - with self.assertRaises(CommandError): - self._call_command('purge_old_personal_api_key_events', '15.3') - - with self.assertRaises(CommandError): - self._call_command('purge_old_personal_api_key_events', '15', '15') - - with self.assertRaises(CommandError): - self._call_command('purge_old_personal_api_key_events', 'abc', '15') - - self.assertCountEqual(PersonApiKeyEvent.objects.all(), [event]) diff --git a/ietf/person/tasks.py b/ietf/person/tasks.py new file mode 100644 index 0000000000..f0c979fa26 --- /dev/null +++ b/ietf/person/tasks.py @@ -0,0 +1,59 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +# +# Celery task definitions +# +import datetime + +from celery import shared_task + +from django.conf import settings +from django.utils import timezone + +from ietf.utils import log +from ietf.utils.mail import send_mail +from .models import PersonalApiKey, PersonApiKeyEvent + + +@shared_task +def send_apikey_usage_emails_task(days): + """Send usage emails to Persons who have API keys""" + earliest = timezone.now() - datetime.timedelta(days=days) + keys = PersonalApiKey.objects.filter( + valid=True, + personapikeyevent__time__gt=earliest, + ).distinct() + for key in keys: + events = PersonApiKeyEvent.objects.filter(key=key, time__gt=earliest) + count = events.count() + events = events[:32] + if count: + key_name = key.hash()[:8] + subject = "API key usage for key '%s' for the last %s days" % ( + key_name, + days, + ) + to = key.person.email_address() + frm = settings.DEFAULT_FROM_EMAIL + send_mail( + None, + to, + frm, + subject, + "utils/apikey_usage_report.txt", + { + "person": key.person, + "days": days, + "key": key, + "key_name": key_name, + "count": count, + "events": events, + }, + ) + +@shared_task +def purge_personal_api_key_events_task(keep_days): + keep_since = timezone.now() - datetime.timedelta(days=keep_days) + old_events = PersonApiKeyEvent.objects.filter(time__lt=keep_since) + count = len(old_events) + old_events.delete() + log.log(f"Deleted {count} PersonApiKeyEvents older than {keep_since}") diff --git a/ietf/person/tests.py b/ietf/person/tests.py index e5bc855a29..9da201b707 100644 --- a/ietf/person/tests.py +++ b/ietf/person/tests.py @@ -4,6 +4,7 @@ import datetime import json +import mock from io import StringIO, BytesIO from PIL import Image @@ -25,8 +26,9 @@ from ietf.nomcom.models import NomCom from ietf.nomcom.test_data import nomcom_test_data from ietf.nomcom.factories import NomComFactory, NomineeFactory, NominationFactory, FeedbackFactory, PositionFactory -from ietf.person.factories import EmailFactory, PersonFactory -from ietf.person.models import Person, Alias +from ietf.person.factories import EmailFactory, PersonFactory, PersonApiKeyEventFactory +from ietf.person.models import Person, Alias, PersonApiKeyEvent +from ietf.person.tasks import purge_personal_api_key_events_task from ietf.person.utils import (merge_persons, determine_merge_order, send_merge_notification, handle_users, get_extra_primary, dedupe_aliases, move_related_objects, merge_nominees, handle_reviewer_settings, get_dots) @@ -450,3 +452,16 @@ def test_dots(self): self.assertEqual(get_dots(ncmember),['nomcom']) ncchair = RoleFactory(group__acronym='nomcom2020',group__type_id='nomcom',name_id='chair').person self.assertEqual(get_dots(ncchair),['nomcom']) + + +class TaskTests(TestCase): + @mock.patch("ietf.person.tasks.log.log") + def test_purge_personal_api_key_events_task(self, mock_log): + now = timezone.now() + old_event = PersonApiKeyEventFactory(time=now - datetime.timedelta(days=1, minutes=1)) + young_event = PersonApiKeyEventFactory(time=now - datetime.timedelta(days=1, minutes=-1)) + purge_personal_api_key_events_task(keep_days=1) + self.assertFalse(PersonApiKeyEvent.objects.filter(pk=old_event.pk).exists()) + self.assertTrue(PersonApiKeyEvent.objects.filter(pk=young_event.pk).exists()) + self.assertTrue(mock_log.called) + self.assertIn("Deleted 1", mock_log.call_args[0][0]) diff --git a/ietf/secr/sreq/forms.py b/ietf/secr/sreq/forms.py index 1100bc7c8a..4a0f449b2a 100644 --- a/ietf/secr/sreq/forms.py +++ b/ietf/secr/sreq/forms.py @@ -13,6 +13,7 @@ from ietf.meeting.models import ResourceAssociation, Constraint from ietf.person.fields import SearchablePersonsField from ietf.person.models import Person +from ietf.utils.fields import ModelMultipleChoiceField from ietf.utils.html import clean_text_field from ietf.utils import log @@ -57,7 +58,7 @@ def __init__(self,*args,**kwargs): self.fields['group'].widget.choices = choices -class NameModelMultipleChoiceField(forms.ModelMultipleChoiceField): +class NameModelMultipleChoiceField(ModelMultipleChoiceField): def label_from_instance(self, name): return name.desc @@ -159,7 +160,7 @@ def __init__(self, group, meeting, data=None, *args, **kwargs): self.fields['resources'].widget = forms.MultipleHiddenInput() self.fields['timeranges'].widget = forms.MultipleHiddenInput() # and entirely replace bethere - no need to support searching if input is hidden - self.fields['bethere'] = forms.ModelMultipleChoiceField( + self.fields['bethere'] = ModelMultipleChoiceField( widget=forms.MultipleHiddenInput, required=False, queryset=Person.objects.all(), ) diff --git a/ietf/settings.py b/ietf/settings.py index 8bb264bd67..95a8ad4e45 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -817,6 +817,9 @@ def skip_unreadable_post(record): # Max time to allow for validation before a submission is subject to cancellation IDSUBMIT_MAX_VALIDATION_TIME = datetime.timedelta(minutes=20) +# Age at which a submission expires if not posted +IDSUBMIT_EXPIRATION_AGE = datetime.timedelta(days=14) + IDSUBMIT_MANUAL_STAGING_DIR = '/tmp/' IDSUBMIT_FILE_TYPES = ( diff --git a/ietf/submit/forms.py b/ietf/submit/forms.py index f857ac9fd8..4e5644b36e 100644 --- a/ietf/submit/forms.py +++ b/ietf/submit/forms.py @@ -39,6 +39,7 @@ from ietf.submit.parsers.xml_parser import XMLParser from ietf.utils import log from ietf.utils.draft import PlaintextDraft +from ietf.utils.fields import ModelMultipleChoiceField from ietf.utils.text import normalize_text from ietf.utils.timezone import date_today from ietf.utils.xmldraft import InvalidXMLError, XMLDraft, XMLParseError @@ -793,7 +794,7 @@ class EditSubmissionForm(forms.ModelForm): rev = forms.CharField(label='Revision', max_length=2, required=True) document_date = forms.DateField(required=True) pages = forms.IntegerField(required=True) - formal_languages = forms.ModelMultipleChoiceField(queryset=FormalLanguageName.objects.filter(used=True), widget=forms.CheckboxSelectMultiple, required=False) + formal_languages = ModelMultipleChoiceField(queryset=FormalLanguageName.objects.filter(used=True), widget=forms.CheckboxSelectMultiple, required=False) abstract = forms.CharField(widget=forms.Textarea, required=True, strip=False) note = forms.CharField(label=mark_safe('Comment to the Secretariat'), widget=forms.Textarea, required=False, strip=False) diff --git a/ietf/submit/tasks.py b/ietf/submit/tasks.py index 382bff7fae..9a13268bce 100644 --- a/ietf/submit/tasks.py +++ b/ietf/submit/tasks.py @@ -37,19 +37,34 @@ def process_and_accept_uploaded_submission_task(submission_id): @shared_task def cancel_stale_submissions(): now = timezone.now() - stale_submissions = Submission.objects.filter( + # first check for submissions gone stale awaiting validation + stale_unvalidated_submissions = Submission.objects.filter( state_id='validating', ).annotate( submitted_at=Min('submissionevent__time'), ).filter( submitted_at__lt=now - settings.IDSUBMIT_MAX_VALIDATION_TIME, ) - for subm in stale_submissions: + for subm in stale_unvalidated_submissions: age = now - subm.submitted_at log.log(f'Canceling stale submission (id={subm.id}, age={age})') cancel_submission(subm) create_submission_event(None, subm, 'Submission canceled: validation checks took too long') + # now check for expired submissions + expired_submissions = Submission.objects.exclude( + state_id__in=["posted", "cancel"], + ).annotate( + submitted_at=Min("submissionevent__time"), + ).filter( + submitted_at__lt=now - settings.IDSUBMIT_EXPIRATION_AGE, + ) + for subm in expired_submissions: + age = now - subm.submitted_at + log.log(f'Canceling expired submission (id={subm.id}, age={age})') + cancel_submission(subm) + create_submission_event(None, subm, 'Submission canceled: expired without being posted') + @shared_task(bind=True) def poke(self): diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 58a47aef8b..618f237e4d 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -42,7 +42,7 @@ from ietf.group.utils import setup_default_community_list_for_group from ietf.meeting.models import Meeting from ietf.meeting.factories import MeetingFactory -from ietf.name.models import FormalLanguageName +from ietf.name.models import DraftSubmissionStateName, FormalLanguageName from ietf.person.models import Person from ietf.person.factories import UserFactory, PersonFactory, EmailFactory from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactory @@ -3136,28 +3136,59 @@ def test_status_of_validating_submission(self): self.assertContains(r, s.name) self.assertContains(r, 'This submission is being processed and validated.', status_code=200) - @override_settings(IDSUBMIT_MAX_VALIDATION_TIME=datetime.timedelta(minutes=30)) + @override_settings( + IDSUBMIT_MAX_VALIDATION_TIME=datetime.timedelta(minutes=30), + IDSUBMIT_EXPIRATION_AGE=datetime.timedelta(minutes=90), + ) def test_cancel_stale_submissions(self): + # these will be lists of (Submission, "state_id") pairs + submissions_to_skip = [] + submissions_to_cancel = [] + + # submissions in the validating state fresh_submission = SubmissionFactory(state_id='validating') fresh_submission.submissionevent_set.create( desc='fake created event', time=timezone.now() - datetime.timedelta(minutes=15), ) + submissions_to_skip.append((fresh_submission, "validating")) + stale_submission = SubmissionFactory(state_id='validating') stale_submission.submissionevent_set.create( desc='fake created event', time=timezone.now() - datetime.timedelta(minutes=30, seconds=1), ) + submissions_to_cancel.append((stale_submission, "validating")) + + # submissions in other states + for state in DraftSubmissionStateName.objects.filter(used=True).exclude(slug="validating"): + to_skip = SubmissionFactory(state_id=state.pk) + to_skip.submissionevent_set.create( + desc="fake created event", + time=timezone.now() - datetime.timedelta(minutes=45), # would be canceled if it were "validating" + ) + submissions_to_skip.append((to_skip, state.pk)) + to_expire = SubmissionFactory(state_id=state.pk) + to_expire.submissionevent_set.create( + desc="fake created event", + time=timezone.now() - datetime.timedelta(minutes=90, seconds=1), + ) + if state.pk in ["posted", "cancel"]: + submissions_to_skip.append((to_expire, state.pk)) # these ones should not be expired regardless of age + else: + submissions_to_cancel.append(((to_expire, state.pk))) cancel_stale_submissions() - fresh_submission = Submission.objects.get(pk=fresh_submission.pk) - self.assertEqual(fresh_submission.state_id, 'validating') - self.assertEqual(fresh_submission.submissionevent_set.count(), 1) + for _subm, original_state_id in submissions_to_skip: + subm = Submission.objects.get(pk=_subm.pk) + self.assertEqual(subm.state_id, original_state_id) + self.assertEqual(subm.submissionevent_set.count(), 1) - stale_submission = Submission.objects.get(pk=stale_submission.pk) - self.assertEqual(stale_submission.state_id, 'cancel') - self.assertEqual(stale_submission.submissionevent_set.count(), 2) + for _subm, _ in submissions_to_cancel: + subm = Submission.objects.get(pk=_subm.pk) + self.assertEqual(subm.state_id, "cancel") + self.assertEqual(subm.submissionevent_set.count(), 2) class ApiSubmitTests(BaseSubmitTestCase): diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index a19c42ecf8..770352b4cd 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -661,7 +661,7 @@ def move_files_to_repository(submission): os.link(dest, ftp_dest) elif dest.exists(): log.log("Intended to move '%s' to '%s', but found source missing while destination exists.") - elif ext in submission.file_types.split(','): + elif f".{ext}" in submission.file_types.split(','): raise ValueError("Intended to move '%s' to '%s', but found source and destination missing.") diff --git a/ietf/templates/doc/ballot/writeupnotes.html b/ietf/templates/doc/ballot/writeupnotes.html index 925387d28d..8e985c15c7 100644 --- a/ietf/templates/doc/ballot/writeupnotes.html +++ b/ietf/templates/doc/ballot/writeupnotes.html @@ -15,11 +15,16 @@

{% bootstrap_form ballot_writeup_form %}
Technical summary, Working Group summary, document quality, personnel, IANA note. This text will be appended to all announcements and messages to the IRTF or RFC Editor. - {% if ballot_issue_danger %} + {% if warn_lc %}

This document has not completed IETF Last Call. Please do not issue the ballot early without good reason.

{% endif %} + {% if warn_unexpected_state %} +

+ This document is in an IESG state of "{{warn_unexpected_state}}". It would be unexpected to issue a ballot while in this state. +

+ {% endif %}

{{ meeting.number }} {% if meeting.interim_meeting_cancelled %}Cancelled{% endif %} {% else %} - IETF-{{ meeting.number }} + {% if meeting.get_number > 28 %} + IETF-{{ meeting.number }} + {% else %} + IETF-{{ meeting.number }} + {% endif %} {% endif %} diff --git a/ietf/templates/meeting/session_details_panel.html b/ietf/templates/meeting/session_details_panel.html index df0f57cae0..d053ba1c1c 100644 --- a/ietf/templates/meeting/session_details_panel.html +++ b/ietf/templates/meeting/session_details_panel.html @@ -137,8 +137,8 @@

Chatlog and polls

{% url 'ietf.doc.views_doc.document_main' name=pres.document.name as url %} - {{ pres.document.title }} - ({{ pres.document.name }}) + {{ pres.document.title }} + ( as json ) {% endfor %} diff --git a/ietf/utils/fields.py b/ietf/utils/fields.py index 95d8a2aa7e..3e6f56d45e 100644 --- a/ietf/utils/fields.py +++ b/ietf/utils/fields.py @@ -14,7 +14,7 @@ from django import forms from django.db import models # pyflakes:ignore -from django.core.validators import validate_email +from django.core.validators import ProhibitNullCharactersValidator, validate_email from django.core.exceptions import ValidationError from django.utils.dateparse import parse_duration @@ -353,3 +353,20 @@ def update_dimension_fields(self, *args, **kwargs): super().update_dimension_fields(*args, **kwargs) except FileNotFoundError: pass # don't do anything if the file has gone missing + + +class ModelMultipleChoiceField(forms.ModelMultipleChoiceField): + """ModelMultipleChoiceField that rejects null characters cleanly""" + validate_no_nulls = ProhibitNullCharactersValidator() + + def clean(self, value): + try: + for item in value: + self.validate_no_nulls(item) + except TypeError: + # A TypeError probably means value is not iterable, which most commonly comes up + # with None as a value. If it's something more exotic, we don't know how to test + # for null characters anyway. Either way, trust the superclass clean() method to + # handle it. + pass + return super().clean(value) diff --git a/ietf/utils/management/commands/periodic_tasks.py b/ietf/utils/management/commands/periodic_tasks.py index 792eb0068b..7f0c988dcd 100644 --- a/ietf/utils/management/commands/periodic_tasks.py +++ b/ietf/utils/management/commands/periodic_tasks.py @@ -241,6 +241,28 @@ def create_default_tasks(self): ), ) + PeriodicTask.objects.get_or_create( + name="Send personal API key usage emails", + task="ietf.person.tasks.send_apikey_usage_emails_task", + kwargs=json.dumps(dict(days=7)), + defaults=dict( + enabled=False, + crontab=self.crontabs["weekly"], + description="Send personal API key usage summary emails for the past week", + ), + ) + + PeriodicTask.objects.get_or_create( + name="Purge old personal API key events", + task="ietf.person.tasks.purge_personal_api_key_events_task", + kwargs=json.dumps(dict(keep_days=14)), + defaults=dict( + enabled=False, + crontab=self.crontabs["daily"], + description="Purge PersonApiKeyEvent instances older than 14 days", + ), + ) + def show_tasks(self): for label, crontab in self.crontabs.items(): tasks = PeriodicTask.objects.filter(crontab=crontab).order_by(