ci: merge main to release (#7504)

ci: merge main to release

.github/workflows/tests.yml

@@ -59,7 +59,7 @@ jobs:
         path: geckodriver.log
 
     - name: Upload Coverage Results to Codecov
-      uses: codecov/codecov-action@v4.3.1
+      uses: codecov/codecov-action@v4.4.1
       with:
         files: coverage.xml
 

bin/daily

@@ -36,6 +36,3 @@ $DTDIR/ietf/manage.py populate_yang_model_dirs -v0
 
 # Re-run yang checks on active documents
 $DTDIR/ietf/manage.py run_yang_model_checks -v0
-
-# Purge older PersonApiKeyEvents
-$DTDIR/ietf/manage.py purge_old_personal_api_key_events 14

docker/README.md

@@ -6,7 +6,7 @@
 
 > See the [IETF Tools Windows Dev guide](https://github.com/ietf-tools/.github/blob/main/docs/windows-dev.md) on how to get started when using Windows.
 
-2. On Linux, you must also install [Docker Compose](https://docs.docker.com/compose/install/). Docker Desktop for Mac and Windows already include Docker Compose.
+2. On Linux, you must [install Docker Compose manually](https://docs.docker.com/compose/install/linux/#install-the-plugin-manually) and not install Docker Desktop. On Mac and Windows install Docker Desktop which already includes Docker Compose.
 
 2. If you have a copy of the datatracker code checked out already, simply `cd` to the top-level directory.
 
@@ -183,3 +183,18 @@ The content of the source files will be copied into the target `.ics` files. Mak
 ### Missing assets in the data folder
 
 Because including all assets in the image would significantly increase the file size, they are not included by default. You can however fetch them by running the **Fetch assets via rsync** task in VS Code or run manually the script `docker/scripts/app-rsync-extras.sh`
+
+
+### Linux file permissions leaking to the host system
+
+If on the host filesystem you have permissions that look like this,
+
+```bash
+$ ls -la
+total 4624
+drwxrwxr-x   2  100999  100999    4096 May 25 07:56 bin
+drwxrwxr-x   5  100999  100999    4096 May 25 07:56 client
+(etc...)
+```
+
+Try uninstalling Docker Desktop and installing Docker Compose manually. The Docker Compose bundled with Docker Desktop is incompatible with our software. See also [Rootless Docker: file ownership changes #3343](https://github.com/lando/lando/issues/3343), [Docker context desktop-linux has container permission issues #75](https://github.com/docker/desktop-linux/issues/75).

docker/scripts/app-init.sh

@@ -2,6 +2,12 @@
 
 WORKSPACEDIR="/workspace"
 
+# Handle Linux host mounting the workspace dir as root
+if [ ! -O "${WORKSPACEDIR}/ietf" ]; then
+    sudo chown -R dev:dev $WORKSPACEDIR
+fi
+
+# Start rsyslog service
 sudo service rsyslog start &>/dev/null
 
 # Add /workspace as a safe git directory

ietf/api/tests.py

@@ -30,7 +30,7 @@
 from ietf.group.factories import RoleFactory
 from ietf.meeting.factories import MeetingFactory, SessionFactory
 from ietf.meeting.models import Session
-from ietf.nomcom.models import Volunteer, NomCom
+from ietf.nomcom.models import Volunteer
 from ietf.nomcom.factories import NomComFactory, nomcom_kwargs_for_year
 from ietf.person.factories import PersonFactory, random_faker, EmailFactory
 from ietf.person.models import Email, User
@@ -828,7 +828,7 @@ def test_api_new_meeting_registration_nomcom_volunteer(self):
             'reg_type': 'onsite',
             'ticket_type': '',
             'checkedin': 'False',
-            'is_nomcom_volunteer': 'True',
+            'is_nomcom_volunteer': 'False',
         }
         person = PersonFactory()
         reg['email'] = person.email().address
@@ -842,16 +842,22 @@ def test_api_new_meeting_registration_nomcom_volunteer(self):
         # create appropriate group and nomcom objects
         nomcom = NomComFactory.create(is_accepting_volunteers=True, **nomcom_kwargs_for_year(year))
         url = urlreverse('ietf.api.views.api_new_meeting_registration')
-        r = self.client.post(url, reg)
-        self.assertContains(r, 'Invalid apikey', status_code=403)
         oidcp = PersonFactory(user__is_staff=True)
         # Make sure 'oidcp' has an acceptable role
         RoleFactory(name_id='robot', person=oidcp, email=oidcp.email(), group__acronym='secretariat')
         key = PersonalApiKey.objects.create(person=oidcp, endpoint=url)
         reg['apikey'] = key.hash()
+
+        # first test is_nomcom_volunteer False
         r = self.client.post(url, reg)
-        nomcom = NomCom.objects.last()
         self.assertContains(r, "Accepted, New registration", status_code=202)
+        # assert no Volunteers exists
+        self.assertEqual(Volunteer.objects.count(), 0)
+
+        # test is_nomcom_volunteer True
+        reg['is_nomcom_volunteer'] = 'True'
+        r = self.client.post(url, reg)
+        self.assertContains(r, "Accepted, Updated registration", status_code=202)
         # assert Volunteer exists
         self.assertEqual(Volunteer.objects.count(), 1)
         volunteer = Volunteer.objects.last()

ietf/api/views.py

@@ -212,7 +212,7 @@ def err(code, text):
                 response += ", Email sent"
 
             # handle nomcom volunteer
-            if data['is_nomcom_volunteer'] and object.person:
+            if request.POST.get('is_nomcom_volunteer', 'false').lower() == 'true' and object.person:
                 try:
                     nomcom = NomCom.objects.get(is_accepting_volunteers=True)
                 except (NomCom.DoesNotExist, NomCom.MultipleObjectsReturned):

ietf/bin/aliases-from-json.py

@@ -61,6 +61,13 @@ def generate_files(records, adest, vdest, postconfirm, vdomain):
         shutil.move(vpath, vdest)
 
 
+def directory_path(val):
+    p = Path(val)
+    if p.is_dir():
+        return p
+    else:
+        raise argparse.ArgumentTypeError(f"{p} is not a directory")
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Convert a JSON stream of draft alias definitions into alias / virtual alias files."
@@ -73,7 +80,7 @@ def generate_files(records, adest, vdest, postconfirm, vdomain):
     parser.add_argument(
         "--output-dir",
         default="./",
-        type=Path,
+        type=directory_path,
         help="Destination for output files.",
     )
     parser.add_argument(
@@ -87,8 +94,6 @@ def generate_files(records, adest, vdest, postconfirm, vdomain):
         help=f"Virtual domain (defaults to {VDOMAIN}_",
     )
     args = parser.parse_args()
-    if not args.output_dir.is_dir():
-        sys.stderr.write("Error: output-dir must be a directory")
     data = json.load(sys.stdin)
     generate_files(
         data["aliases"],

ietf/doc/tests_ballot.py

@@ -32,7 +32,7 @@
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.text import unwrap
-from ietf.utils.timezone import date_today
+from ietf.utils.timezone import date_today, datetime_today
 
 
 class EditPositionTests(TestCase):
@@ -529,13 +529,46 @@ def test_issue_ballot_warn_if_early(self):
         login_testing_unauthorized(self, "secretary", url)
 
         # expect warning about issuing a ballot before IETF Last Call is done
+        # No last call has yet been issued
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertEqual(len(q('textarea[name=ballot_writeup]')), 1)
         self.assertTrue(q('[class=text-danger]:contains("not completed IETF Last Call")'))
         self.assertTrue(q('[type=submit]:contains("Save")'))
 
+        # Last call exists but hasn't expired
+        LastCallDocEvent.objects.create(
+            doc=draft,
+            expires=datetime_today()+datetime.timedelta(days=14),
+            by=Person.objects.get(name="(System)")
+        )
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        self.assertTrue(q('[class=text-danger]:contains("not completed IETF Last Call")'))
+
+        # Last call exists and has expired
+        LastCallDocEvent.objects.filter(doc=draft).update(expires=datetime_today()-datetime.timedelta(days=2))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        self.assertFalse(q('[class=text-danger]:contains("not completed IETF Last Call")'))
+
+        for state_slug in ["lc", "watching", "ad-eval"]:
+            draft.set_state(State.objects.get(type="draft-iesg",slug=state_slug))
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            self.assertTrue(q('[class=text-danger]:contains("It would be unexpected to issue a ballot while in this state.")'))
+
+        draft.set_state(State.objects.get(type="draft-iesg",slug="writeupw"))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        self.assertFalse(q('[class=text-danger]:contains("It would be unexpected to issue a ballot while in this state.")'))         
+
+
     def test_edit_approval_text(self):
         ad = Person.objects.get(user__username="ad")
         draft = WgDraftFactory(ad=ad,states=[('draft','active'),('draft-iesg','iesg-eva')],intended_std_level_id='ps',group__parent=Group.objects.get(acronym='farfut'))

ietf/doc/views_ballot.py

@@ -38,6 +38,7 @@
 from ietf.message.utils import infer_message
 from ietf.name.models import BallotPositionName, DocTypeName
 from ietf.person.models import Person
+from ietf.utils.fields import ModelMultipleChoiceField
 from ietf.utils.mail import send_mail_text, send_mail_preformatted
 from ietf.utils.decorators import require_api_key
 from ietf.utils.response import permission_denied
@@ -686,7 +687,8 @@ def ballot_writeupnotes(request, name):
                               dict(doc=doc,
                                    back_url=doc.get_absolute_url(),
                                    ballot_issued=bool(doc.latest_event(type="sent_ballot_announcement")),
-                                   ballot_issue_danger=bool(prev_state.slug in ['ad-eval', 'lc']),
+                                   warn_lc = not doc.docevent_set.filter(lastcalldocevent__expires__date__lt=date_today(DEADLINE_TZINFO)).exists(),
+                                   warn_unexpected_state= prev_state if bool(prev_state.slug in ['watching', 'ad-eval', 'lc']) else None,
                                    ballot_writeup_form=form,
                                    need_intended_status=need_intended_status,
                                    ))
@@ -931,7 +933,7 @@ def approve_ballot(request, name):
 
 
 class ApproveDownrefsForm(forms.Form):
-    checkboxes = forms.ModelMultipleChoiceField(
+    checkboxes = ModelMultipleChoiceField(
         widget = forms.CheckboxSelectMultiple,
         queryset =  RelatedDocument.objects.none(), )
 

ietf/doc/views_draft.py

@@ -52,6 +52,7 @@
 from ietf.utils.mail import send_mail, send_mail_message, on_behalf_of
 from ietf.utils.textupload import get_cleaned_text_file_content
 from ietf.utils import log
+from ietf.utils.fields import ModelMultipleChoiceField
 from ietf.utils.response import permission_denied
 from ietf.utils.timezone import datetime_today, DEADLINE_TZINFO
 
@@ -390,9 +391,9 @@ def replaces(request, name):
                    ))
 
 class SuggestedReplacesForm(forms.Form):
-    replaces = forms.ModelMultipleChoiceField(queryset=Document.objects.all(),
-                                              label="Suggestions", required=False, widget=forms.CheckboxSelectMultiple,
-                                              help_text="Select only the documents that are replaced by this document")
+    replaces = ModelMultipleChoiceField(queryset=Document.objects.all(),
+                                        label="Suggestions", required=False, widget=forms.CheckboxSelectMultiple,
+                                        help_text="Select only the documents that are replaced by this document")
     comment = forms.CharField(label="Optional comment", widget=forms.Textarea, required=False, strip=False)
 
     def __init__(self, suggested, *args, **kwargs):
@@ -1601,7 +1602,7 @@ class ChangeStreamStateForm(forms.Form):
     new_state = forms.ModelChoiceField(queryset=State.objects.filter(used=True), label='State' )
     weeks = forms.IntegerField(label='Expected weeks in state',required=False)
     comment = forms.CharField(widget=forms.Textarea, required=False, help_text="Optional comment for the document history.", strip=False)
-    tags = forms.ModelMultipleChoiceField(queryset=DocTagName.objects.filter(used=True), widget=forms.CheckboxSelectMultiple, required=False)
+    tags = ModelMultipleChoiceField(queryset=DocTagName.objects.filter(used=True), widget=forms.CheckboxSelectMultiple, required=False)
 
     def __init__(self, *args, **kwargs):
         doc = kwargs.pop("doc")

ietf/doc/views_review.py

@@ -52,7 +52,7 @@
 from ietf.utils.textupload import get_cleaned_text_file_content
 from ietf.utils.mail import send_mail_message
 from ietf.mailtrigger.utils import gather_address_lists
-from ietf.utils.fields import MultiEmailField
+from ietf.utils.fields import ModelMultipleChoiceField, MultiEmailField
 from ietf.utils.http import is_ajax
 from ietf.utils.response import permission_denied
 from ietf.utils.timezone import date_today, DEADLINE_TZINFO
@@ -68,7 +68,7 @@ def clean_doc_revision(doc, rev):
     return rev
 
 class RequestReviewForm(forms.ModelForm):
-    team = forms.ModelMultipleChoiceField(queryset=Group.objects.all(), widget=forms.CheckboxSelectMultiple)
+    team = ModelMultipleChoiceField(queryset=Group.objects.all(), widget=forms.CheckboxSelectMultiple)
     deadline = DatepickerDateField(date_format="yyyy-mm-dd", picker_settings={ "autoclose": "1", "start-date": "+0d" })
 
     class Meta:

ietf/doc/views_search.py

@@ -69,6 +69,7 @@
 from ietf.person.models import Person
 from ietf.person.utils import get_active_ads
 from ietf.utils.draft_search import normalize_draftname
+from ietf.utils.fields import ModelMultipleChoiceField
 from ietf.utils.log import log
 from ietf.doc.utils_search import prepare_document_table, doc_type, doc_state, doc_type_name, AD_WORKLOAD
 from ietf.ietfauth.utils import has_role
@@ -100,7 +101,7 @@ class SearchForm(forms.Form):
             ("ad", "AD"), ("-ad", "AD (desc)"), ),
         required=False, widget=forms.HiddenInput)
 
-    doctypes = forms.ModelMultipleChoiceField(queryset=DocTypeName.objects.filter(used=True).exclude(slug__in=('draft', 'rfc', 'bcp', 'std', 'fyi', 'liai-att')).order_by('name'), required=False)
+    doctypes = ModelMultipleChoiceField(queryset=DocTypeName.objects.filter(used=True).exclude(slug__in=('draft', 'rfc', 'bcp', 'std', 'fyi', 'liai-att')).order_by('name'), required=False)
 
     def __init__(self, *args, **kwargs):
         super(SearchForm, self).__init__(*args, **kwargs)