ci: merge main to release (#7560)

ci: merge main to release

bin/daily

@@ -5,7 +5,6 @@
 # This script is expected to be triggered by cron from
 # /etc/cron.d/datatracker
 export LANG=en_US.UTF-8
-export PYTHONIOENCODING=utf-8
 
 # Make sure we stop if something goes wrong:
 program=${0##*/}
@@ -17,10 +16,6 @@ cd $DTDIR/
 
 logger -p user.info -t cron "Running $DTDIR/bin/daily"
 
-# Set up the virtual environment
-source $DTDIR/env/bin/activate
-
-
 # Get IANA-registered yang models
 #YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR')
 # Hardcode the rsync target to avoid any unwanted deletes:
@@ -30,9 +25,3 @@ rsync -avzq --delete /a/www/ietf-ftp/iana/yang-parameters/ /a/www/ietf-ftp/yang/
 # Get Yang models from Yangcatalog.
 #rsync -avzq rsync://rsync.yangcatalog.org:10873/yangdeps /a/www/ietf-ftp/yang/catalogmod/
 /a/www/ietf-datatracker/scripts/sync_to_yangcatalog
-
-# Populate the yang repositories
-$DTDIR/ietf/manage.py populate_yang_model_dirs -v0
-
-# Re-run yang checks on active documents
-$DTDIR/ietf/manage.py run_yang_model_checks -v0

ietf/checks.py

@@ -28,61 +28,6 @@ def already_ran():
         checks_run.append(name)
         return False
 
-
-@checks.register('files')
-def check_group_email_aliases_exists(app_configs, **kwargs):
-    from ietf.group.views import check_group_email_aliases
-    #
-    if already_ran():
-        return []
-    #
-    errors = []
-    try:
-        ok = check_group_email_aliases()
-        if not ok:
-            errors.append(checks.Error(
-                "Found no aliases in the group email aliases file\n'%s'."%settings.GROUP_ALIASES_PATH,
-                hint="These should be created by the infrastructure using ietf/bin/aliases-from-json.py.",
-                obj=None,
-                id="datatracker.E0002",
-            ))
-    except IOError as e:
-        errors.append(checks.Error(
-            "Could not read group email aliases:\n   %s" % e,
-            hint="These should be created by the infrastructure using ietf/bin/aliases-from-json.py.",
-            obj=None,
-            id="datatracker.E0003",
-        ))
-
-    return errors
-
-@checks.register('files')
-def check_doc_email_aliases_exists(app_configs, **kwargs):
-    from ietf.doc.views_doc import check_doc_email_aliases
-    #
-    if already_ran():
-        return []
-    #
-    errors = []
-    try:
-        ok = check_doc_email_aliases()
-        if not ok:
-            errors.append(checks.Error(
-                "Found no aliases in the document email aliases file\n'%s'."%settings.DRAFT_VIRTUAL_PATH,
-                hint="These should be created by the infrastructure using ietf/bin/aliases-from-json.py.",
-                obj=None,
-                id="datatracker.E0004",
-            ))
-    except IOError as e:
-        errors.append(checks.Error(
-            "Could not read document email aliases:\n   %s" % e,
-            hint="These should be created by the infrastructure using ietf/bin/aliases-from-json.py.",
-            obj=None,
-            id="datatracker.E0005",
-        ))
-
-    return errors
-
 @checks.register('directories')
 def check_id_submission_directories(app_configs, **kwargs):
     #

ietf/doc/tests.py

@@ -16,7 +16,6 @@
 from pathlib import Path
 from pyquery import PyQuery
 from urllib.parse import urlparse, parse_qs
-from tempfile import NamedTemporaryFile
 from collections import defaultdict
 from zoneinfo import ZoneInfo
 
@@ -51,6 +50,7 @@
     DraftAliasGenerator,
     generate_idnits2_rfc_status,
     generate_idnits2_rfcs_obsoleted,
+    get_doc_email_aliases,
 )
 from ietf.group.models import Group, Role
 from ietf.group.factories import GroupFactory, RoleFactory
@@ -2169,24 +2169,6 @@ def test_references(self):
         self.assertContains(r, doc1.name)
 
 class GenerateDraftAliasesTests(TestCase):
-    def setUp(self):
-        super().setUp()
-        self.doc_aliases_file = NamedTemporaryFile(delete=False, mode="w+")
-        self.doc_aliases_file.close()
-        self.doc_virtual_file = NamedTemporaryFile(delete=False, mode="w+")
-        self.doc_virtual_file.close()
-        self.saved_draft_aliases_path = settings.DRAFT_ALIASES_PATH
-        self.saved_draft_virtual_path = settings.DRAFT_VIRTUAL_PATH
-        settings.DRAFT_ALIASES_PATH = self.doc_aliases_file.name
-        settings.DRAFT_VIRTUAL_PATH = self.doc_virtual_file.name
-
-    def tearDown(self):
-        settings.DRAFT_ALIASES_PATH = self.saved_draft_aliases_path
-        settings.DRAFT_VIRTUAL_PATH = self.saved_draft_virtual_path
-        os.unlink(self.doc_aliases_file.name)
-        os.unlink(self.doc_virtual_file.name)
-        super().tearDown()
-
     @override_settings(TOOLS_SERVER="tools.example.org", DRAFT_ALIAS_DOMAIN="draft.example.org")
     def test_generator_class(self):
         """The DraftAliasGenerator should generate the same lists as the old mgmt cmd"""
@@ -2286,6 +2268,28 @@ def test_generator_class(self):
             {k: sorted(v) for k, v in expected_dict.items()},
         )
 
+        # check single name
+        output = [(alias, alist) for alias, alist in DraftAliasGenerator(Document.objects.filter(name=doc1.name))]
+        alias_dict = dict(output)
+        self.assertEqual(len(alias_dict), len(output))  # no duplicate aliases
+        expected_dict = {
+            doc1.name: [author1.email_address()],
+            doc1.name + ".ad": [ad.email_address()],
+            doc1.name + ".authors": [author1.email_address()],
+            doc1.name + ".shepherd": [shepherd.email_address()],
+            doc1.name
+            + ".all": [
+                author1.email_address(),
+                ad.email_address(),
+                shepherd.email_address(),
+            ],
+        }
+        # Sort lists for comparison
+        self.assertEqual(
+            {k: sorted(v) for k, v in alias_dict.items()},
+            {k: sorted(v) for k, v in expected_dict.items()},
+        )
+
     @override_settings(TOOLS_SERVER="tools.example.org", DRAFT_ALIAS_DOMAIN="draft.example.org")
     def test_get_draft_notify_emails(self):
         ad = PersonFactory()
@@ -2336,37 +2340,20 @@ def setUp(self):
         WgDraftFactory(name='draft-ietf-mars-test',group__acronym='mars')
         WgDraftFactory(name='draft-ietf-ames-test',group__acronym='ames')
         RoleFactory(group__type_id='review', group__acronym='yangdoctors', name_id='secr')
-        self.doc_alias_file = NamedTemporaryFile(delete=False, mode='w+')
-        self.doc_alias_file.write("""# Generated by hand at 2015-02-12_16:26:45
-virtual.ietf.org anything
-[email protected]              xfilter-draft-ietf-mars-test
-[email protected]  [email protected], [email protected]
-[email protected]      xfilter-draft-ietf-mars-test.authors
-[email protected]  [email protected], [email protected]
-[email protected]      xfilter-draft-ietf-mars-test.chairs
-[email protected]  [email protected]
-[email protected]      xfilter-draft-ietf-mars-test.all
-[email protected]  [email protected], [email protected], [email protected]
-[email protected]              xfilter-draft-ietf-ames-test
-[email protected]  [email protected], [email protected]
-[email protected]      xfilter-draft-ietf-ames-test.authors
-[email protected]  [email protected], [email protected]
-[email protected]      xfilter-draft-ietf-ames-test.chairs
-[email protected]  [email protected]
-[email protected]      xfilter-draft-ietf-ames-test.all
-[email protected]  [email protected], [email protected], [email protected]
-
-""")
-        self.doc_alias_file.close()
-        self.saved_draft_virtual_path = settings.DRAFT_VIRTUAL_PATH
-        settings.DRAFT_VIRTUAL_PATH = self.doc_alias_file.name
-
-    def tearDown(self):
-        settings.DRAFT_VIRTUAL_PATH = self.saved_draft_virtual_path
-        os.unlink(self.doc_alias_file.name)
-        super().tearDown()
-
-    def testAliases(self):
+
+
+    @mock.patch("ietf.doc.views_doc.get_doc_email_aliases")
+    def testAliases(self, mock_get_aliases):
+        mock_get_aliases.return_value = [
+            {"doc_name": "draft-ietf-mars-test", "alias_type": "", "expansion": "[email protected], [email protected]"},
+            {"doc_name": "draft-ietf-mars-test", "alias_type": ".authors", "expansion": "[email protected], [email protected]"},
+            {"doc_name": "draft-ietf-mars-test", "alias_type": ".chairs", "expansion": "[email protected]"},
+            {"doc_name": "draft-ietf-mars-test", "alias_type": ".all", "expansion": "[email protected], [email protected], [email protected]"},
+            {"doc_name": "draft-ietf-ames-test", "alias_type": "", "expansion": "[email protected], [email protected]"},
+            {"doc_name": "draft-ietf-ames-test", "alias_type": ".authors", "expansion": "[email protected], [email protected]"},
+            {"doc_name": "draft-ietf-ames-test", "alias_type": ".chairs", "expansion": "[email protected]"},
+            {"doc_name": "draft-ietf-ames-test", "alias_type": ".all", "expansion": "[email protected], [email protected], [email protected]"},
+        ]
         PersonFactory(user__username='plain')
         url = urlreverse('ietf.doc.urls.redirect.document_email', kwargs=dict(name="draft-ietf-mars-test"))
         r = self.client.get(url)
@@ -2376,16 +2363,70 @@ def testAliases(self):
         login_testing_unauthorized(self, "plain", url)
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
+        self.assertEqual(mock_get_aliases.call_args, mock.call())
         self.assertTrue(all([x in unicontent(r) for x in ['mars-test@','mars-test.authors@','mars-test.chairs@']]))
         self.assertTrue(all([x in unicontent(r) for x in ['ames-test@','ames-test.authors@','ames-test.chairs@']]))
 
-    def testExpansions(self):
+
+    @mock.patch("ietf.doc.views_doc.get_doc_email_aliases")
+    def testExpansions(self, mock_get_aliases):
+        mock_get_aliases.return_value = [
+            {"doc_name": "draft-ietf-mars-test", "alias_type": "", "expansion": "[email protected], [email protected]"},
+            {"doc_name": "draft-ietf-mars-test", "alias_type": ".authors", "expansion": "[email protected], [email protected]"},
+            {"doc_name": "draft-ietf-mars-test", "alias_type": ".chairs", "expansion": "[email protected]"},
+            {"doc_name": "draft-ietf-mars-test", "alias_type": ".all", "expansion": "[email protected], [email protected], [email protected]"},
+        ]
         url = urlreverse('ietf.doc.views_doc.document_email', kwargs=dict(name="draft-ietf-mars-test"))
         r = self.client.get(url)
+        self.assertEqual(mock_get_aliases.call_args, mock.call("draft-ietf-mars-test"))
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, '[email protected]')
         self.assertContains(r, 'iesg_ballot_saved')
+
+    @mock.patch("ietf.doc.utils.DraftAliasGenerator")
+    def test_get_doc_email_aliases(self, mock_alias_gen_cls):
+        mock_alias_gen_cls.return_value = [
+            ("draft-something-or-other.some-type", ["[email protected]"]),
+            ("draft-something-or-other", ["[email protected]"]),
+            ("draft-nothing-at-all", ["[email protected]"]),
+            ("draft-nothing-at-all.some-type", ["[email protected]"]),
+        ]
+        # order is important in the response - should be sorted by doc name and otherwise left
+        # in order
+        self.assertEqual(
+            get_doc_email_aliases(),
+            [
+                {
+                    "doc_name": "draft-nothing-at-all",
+                    "alias_type": "",
+                    "expansion": "[email protected]",
+                },
+                {
+                    "doc_name": "draft-nothing-at-all",
+                    "alias_type": ".some-type",
+                    "expansion": "[email protected]",
+                },
+                {
+                    "doc_name": "draft-something-or-other",
+                    "alias_type": ".some-type",
+                    "expansion": "[email protected]",
+                },
+                {
+                    "doc_name": "draft-something-or-other",
+                    "alias_type": "",
+                    "expansion": "[email protected]",
+                },
+            ],
+        )
+        self.assertEqual(mock_alias_gen_cls.call_args, mock.call(None))
 
+        # Repeat with a name, no need to re-test that the alias list is actually passed through, just
+        # check that the DraftAliasGenerator is called correctly
+        draft = WgDraftFactory()
+        get_doc_email_aliases(draft.name)
+        self.assertQuerySetEqual(mock_alias_gen_cls.call_args[0][0], Document.objects.filter(pk=draft.pk))
+
+
 class DocumentMeetingTests(TestCase):
 
     def setUp(self):

ietf/doc/utils.py

@@ -14,7 +14,7 @@
 from collections import defaultdict, namedtuple, Counter
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Iterator, Union
+from typing import Iterator, Optional, Union
 from zoneinfo import ZoneInfo
 
 from django.conf import settings
@@ -1265,6 +1265,12 @@ def bibxml_for_draft(doc, rev=None):
 class DraftAliasGenerator:
     days = 2 * 365
 
+    def __init__(self, draft_queryset=None):
+        if draft_queryset is not None:
+            self.draft_queryset = draft_queryset.filter(type_id="draft")  # only drafts allowed
+        else:
+            self.draft_queryset = Document.objects.filter(type_id="draft")
+
     def get_draft_ad_emails(self, doc):
         """Get AD email addresses for the given draft, if any."""
         from ietf.group.utils import get_group_ad_emails  # avoid circular import
@@ -1333,7 +1339,7 @@ def get_draft_notify_emails(self, doc):
     def __iter__(self) -> Iterator[tuple[str, list[str]]]:
         # Internet-Drafts with active status or expired within self.days
         show_since = timezone.now() - datetime.timedelta(days=self.days)
-        drafts = Document.objects.filter(type_id="draft")
+        drafts = self.draft_queryset
         active_drafts = drafts.filter(states__slug='active')
         inactive_recent_drafts = drafts.exclude(states__slug='active').filter(expires__gte=show_since)
         interesting_drafts = active_drafts | inactive_recent_drafts
@@ -1384,6 +1390,22 @@ def __iter__(self) -> Iterator[tuple[str, list[str]]]:
             if all:
                 yield alias + ".all", list(all)
 
+
+def get_doc_email_aliases(name: Optional[str] = None):
+    aliases = []
+    for (alias, alist) in DraftAliasGenerator(
+        Document.objects.filter(type_id="draft", name=name) if name else None
+    ):
+        # alias is draft-name.alias_type
+        doc_name, _dot, alias_type = alias.partition(".")
+        aliases.append({
+            "doc_name": doc_name,
+            "alias_type": f".{alias_type}" if alias_type else "",
+            "expansion": ", ".join(sorted(alist)),
+        })
+    return sorted(aliases, key=lambda a: (a["doc_name"]))
+
+
 def investigate_fragment(name_fragment):
     can_verify = set()
     for root in [settings.INTERNET_DRAFT_PATH, settings.INTERNET_DRAFT_ARCHIVE_DIR]: