Update CI files

.ci/ansible/Containerfile.j2

@@ -9,6 +9,7 @@ ADD ./{{ item.name }} ./{{ item.name }}
 # S3 botocore needs to be patched to handle responses from minio during 0-byte uploads
 # Hacking botocore (https://github.com/boto/botocore/pull/1990)
 
+# This MUST be the ONLY call to pip install in inside the container.
 RUN pip3 install --upgrade pip setuptools wheel && \
   rm -rf /root/.cache/pip && \
   pip3 install

.ci/scripts/check_release.py

@@ -64,6 +64,7 @@ def main():
         for branch in branches:
             if branch != DEFAULT_BRANCH:
                 # Check if a Z release is needed
+                reasons = []
                 changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/")
                 z_changelog = False
                 for change in changes.split("\n"):
@@ -76,23 +77,27 @@ def main():
                         )
                     elif ext in Z_CHANGELOG_EXTS:
                         z_changelog = True
+                if z_changelog:
+                    reasons.append("Backports")
 
                 last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}")
                 req_txt_diff = repo.git.diff(
                     f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt"
                 )
-                if z_changelog or req_txt_diff:
+                if req_txt_diff:
+                    reasons.append("requirements.txt")
+
+                if reasons:
                     curr_version = Version(last_tag)
                     assert curr_version.base_version.startswith(
                         branch
                     ), "Current-version has to belong to the current branch!"
                     next_version = Version(f"{branch}.{curr_version.micro + 1}")
-                    reason = "CHANGES" if z_changelog else "requirements.txt"
                     print(
                         f"A Z-release is needed for {branch}, "
                         f"Prev: {last_tag}, "
                         f"Next: {next_version.base_version}, "
-                        f"Reason: {reason}"
+                        f"Reason: {','.join(reasons)}"
                     )
                     releases.append(next_version)
             else:

.ci/scripts/collect_changes.py

@@ -103,7 +103,7 @@ def main():
             for change in main_changes:
                 fp.write(change[1])
 
-        repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE)
+        repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE)
 
 
 if __name__ == "__main__":

.ci/scripts/pr_labels.py

@@ -0,0 +1,60 @@
+#!/bin/env python3
+
+# This script is running with elevated privileges from the main branch against pull requests.
+
+import re
+import sys
+import tomllib
+from pathlib import Path
+
+from git import Repo
+
+
+def main():
+    assert len(sys.argv) == 3
+
+    with open("pyproject.toml", "rb") as fp:
+        PYPROJECT_TOML = tomllib.load(fp)
+    BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT")
+    ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)")
+    CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$")
+    try:
+        CHANGELOG_EXTS = {
+            f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"]
+        }
+    except KeyError:
+        CHANGELOG_EXTS = {".feature", ".bugfix", ".doc", ".removal", ".misc"}
+
+    repo = Repo(".")
+
+    base_commit = repo.commit(sys.argv[1])
+    head_commit = repo.commit(sys.argv[2])
+
+    pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}"))
+
+    labels = {
+        "multi-commit": len(pr_commits) > 1,
+        "cherry-pick": False,
+        "no-issue": False,
+        "no-changelog": False,
+        "wip": False,
+    }
+    for commit in pr_commits:
+        labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None
+        no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None
+        labels["no-issue"] |= no_issue
+        cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None
+        labels["cherry-pick"] |= cherry_pick
+        changelog_snippets = [
+            k
+            for k in commit.stats.files
+            if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS
+        ]
+        labels["no-changelog"] |= not changelog_snippets
+
+    print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v)))
+    print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v)))
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/validate_commit_message.py

@@ -13,7 +13,6 @@
 import warnings
 from github import Github
 
-NO_ISSUE = "[noissue]"
 CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"]
 KEYWORDS = ["fixes", "closes"]
 
@@ -60,15 +59,5 @@ def __check_changelog(issue):
     for issue in pattern.findall(message):
         __check_status(issue)
         __check_changelog(issue)
-else:
-    if NO_ISSUE in message:
-        print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE))
-    elif "Merge" in message and "cherry picked from commit" in message:
-        pass
-    else:
-        sys.exit(
-            "Error: no attached issues found for {sha}. If this was intentional, add "
-            " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE)
-        )
 
 print("Commit message for {sha} passed.".format(sha=sha[0:7]))

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-379-gece49c9
+2021.08.26-388-g624de1a

.github/workflows/create-branch.yml

@@ -93,10 +93,8 @@ jobs:
           branch: minor-version-bump
           base: main
           title: Bump minor version
-          body: '[noissue]'
           commit-message: |
             Bump minor version
-            [noissue]
           delete-branch: true
 
       - name: Push release branch

.github/workflows/nightly.yml

@@ -179,4 +179,4 @@ jobs:
           docker logs pulp || true
           docker exec pulp ls -latr /etc/yum.repos.d/ || true
           docker exec pulp cat /etc/yum.repos.d/* || true
-          docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree"
+          docker exec pulp bash -c "pip3 list" || true

.github/workflows/pr_checks.yml

@@ -6,57 +6,62 @@
 # For more info visit https://github.com/pulp/plugin_template
 
 ---
-name: File PR static checks
+name: "File PR static checks"
 on:
   pull_request_target:
-    types: [opened, synchronize, reopened]
+    types: ["opened", "synchronize", "reopened"]
 
 # This workflow runs with elevated permissions.
 # Do not even think about running a single bit of code from the PR.
 # Static analysis should be fine however.
 
 concurrency:
-  group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
+  group: "${{ github.event.pull_request.number }}-${{ github.workflow }}"
   cancel-in-progress: true
 
 jobs:
-  single_commit:
-    runs-on: ubuntu-latest
-    name: Label multiple commit PR
+  apply_labels:
+    runs-on: "ubuntu-latest"
+    name: "Label PR"
     permissions:
-      pull-requests: write
+      pull-requests: "write"
     steps:
       - uses: "actions/checkout@v4"
         with:
           fetch-depth: 0
-      - name: Commit Count Check
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.11"
+      - name: "Determine PR labels"
         run: |
+          pip install GitPython==3.1.42
           git fetch origin ${{ github.event.pull_request.head.sha }}
-          echo "COMMIT_COUNT=$(git log  --oneline --no-merges origin/${{ github.base_ref }}..${{ github.event.pull_request.head.sha }} | wc -l)" >> "$GITHUB_ENV"
-      - uses: actions/github-script@v7
+          python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV"
+      - uses: "actions/github-script@v7"
+        name: "Apply PR Labels"
         with:
           script: |
-            const labelName = "multi-commit";
-            const { COMMIT_COUNT } = process.env;
+            const { ADD_LABELS, REMOVE_LABELS } = process.env;
 
-            if (COMMIT_COUNT == 1)
-            {
-              try {
-                await github.rest.issues.removeLabel({
-                  issue_number: context.issue.number,
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  name: labelName,
-                });
-              } catch(err) {
+            if (REMOVE_LABELS.length) {
+              for await (const labelName of REMOVE_LABELS.split(",")) {
+                try {
+                  await github.rest.issues.removeLabel({
+                    issue_number: context.issue.number,
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    name: labelName,
+                  });
+                } catch(err) {
+                }
               }
             }
-            else
-            {
+            if (ADD_LABELS.length) {
               await github.rest.issues.addLabels({
                 issue_number: context.issue.number,
                 owner: context.repo.owner,
                 repo: context.repo.repo,
-                labels: [labelName],
+                labels: ADD_LABELS.split(","),
               });
             }
+...

.github/workflows/scripts/install.sh

@@ -166,5 +166,5 @@ if [[ "$TEST" = "azure" ]]; then
 fi
 
 echo ::group::PIP_LIST
-cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree"
+cmd_prefix bash -c "pip3 list"
 echo ::endgroup::

.github/workflows/scripts/script.sh

@@ -46,12 +46,14 @@ pushd ../pulp-openapi-generator
 
   # Workaround: Domains are not supported by the published bindings.
   # Sadly: Different pulpcore-versions aren't either...
-  # So we exclude the prebuilt ones only for domains disabled.
-  if [ "$(jq -r '.domain_enabled' <<<"${REPORTED_STATUS}")" = "true" ] || [ "$(jq -r '.online_workers[0].pulp_href|startswith("/pulp/api/v3/")' <<< "${REPORTED_STATUS}")" = "false" ]
+  # * In the 'pulp' scenario we use the published/prebuilt bindings, so we can test it.
+  # * In other scenarios we generate new bindings from server spec, so we have a more
+  #   reliable client.
+  if [ "$TEST" = "pulp" ]
   then
-    BUILT_CLIENTS=""
-  else
     BUILT_CLIENTS=" file "
+  else
+    BUILT_CLIENTS=""
   fi
 
   for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}")

.github/workflows/test.yml

@@ -147,5 +147,5 @@ jobs:
           docker logs pulp || true
           docker exec pulp ls -latr /etc/yum.repos.d/ || true
           docker exec pulp cat /etc/yum.repos.d/* || true
-          docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree"
+          docker exec pulp bash -c "pip3 list" || true
 ...

.github/workflows/update_ci.yml

@@ -62,13 +62,8 @@ jobs:
           committer: "pulpbot <[email protected]>"
           author: "pulpbot <[email protected]>"
           title: "Update CI files for branch main"
-          body: ""
           branch: "update-ci/main"
           base: "main"
-          commit-message: |
-            Update CI files
-
-            [noissue]
           delete-branch: true
       - uses: "actions/checkout@v4"
         with:
@@ -89,13 +84,8 @@ jobs:
           committer: "pulpbot <[email protected]>"
           author: "pulpbot <[email protected]>"
           title: "Update CI files for branch 1.10"
-          body: ""
           branch: "update-ci/1.10"
           base: "1.10"
-          commit-message: |
-            Update CI files
-
-            [noissue]
           delete-branch: true
       - uses: "actions/checkout@v4"
         with:
@@ -116,13 +106,8 @@ jobs:
           committer: "pulpbot <[email protected]>"
           author: "pulpbot <[email protected]>"
           title: "Update CI files for branch 1.11"
-          body: ""
           branch: "update-ci/1.11"
           base: "1.11"
-          commit-message: |
-            Update CI files
-
-            [noissue]
           delete-branch: true
       - uses: "actions/checkout@v4"
         with:
@@ -143,13 +128,8 @@ jobs:
           committer: "pulpbot <[email protected]>"
           author: "pulpbot <[email protected]>"
           title: "Update CI files for branch 1.12"
-          body: ""
           branch: "update-ci/1.12"
           base: "1.12"
-          commit-message: |
-            Update CI files
-
-            [noissue]
           delete-branch: true
       - uses: "actions/checkout@v4"
         with:
@@ -170,13 +150,8 @@ jobs:
           committer: "pulpbot <[email protected]>"
           author: "pulpbot <[email protected]>"
           title: "Update CI files for branch 1.13"
-          body: ""
           branch: "update-ci/1.13"
           base: "1.13"
-          commit-message: |
-            Update CI files
-
-            [noissue]
           delete-branch: true
       - uses: "actions/checkout@v4"
         with:
@@ -197,11 +172,7 @@ jobs:
           committer: "pulpbot <[email protected]>"
           author: "pulpbot <[email protected]>"
           title: "Update CI files for branch 1.14"
-          body: ""
           branch: "update-ci/1.14"
           base: "1.14"
-          commit-message: |
-            Update CI files
-
-            [noissue]
           delete-branch: true
+...

MANIFEST.in

@@ -9,3 +9,4 @@ include functest_requirements.txt
 include test_requirements.txt
 include unittest_requirements.txt
 recursive-exclude pulp_file/tests/functional *
+exclude releasing.md

releasing.md

@@ -0,0 +1,27 @@
+[//]: # "WARNING: DO NOT EDIT!"
+[//]: # ""
+[//]: # "This file was generated by plugin_template, and is managed by it. Please use"
+[//]: # "'./plugin-template --github pulp_file' to update this file."
+[//]: # ""
+[//]: # "For more info visit https://github.com/pulp/plugin_template"
+# Releasing (For Internal Use)
+
+This document outlines the steps to perform a release.
+
+### Determine if a Release is Required
+- Run the release checker script:
+  ```
+  python3 .ci/scripts/check_release.py
+  ```
+
+### Create a New Y-branch (e.g., 3.23)
+- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_file/actions/workflows/create-branch.yml) job via the GitHub Actions.
+- Look for the "Bump minor version" pull request and merge it.
+
+### Release a New Z-version (Patch Release) (e.g., 3.23.1, 3.22.12)
+- Trigger a [Release Pipeline](https://github.com/pulp/pulp_file/actions/workflows/release.yml) job by specifying the release branch (X.Y) via the GitHub Actions.
+
+### Final Steps (Optional but Recommended)
+- Ensure the new version appears on PyPI.
+- Verify that the changelog has been updated by looking for the "Update Changelog" pull request.
+- Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/).