diff --git a/.github/publish.yaml b/.github/publish.yaml index 71d0f22..fe81e64 100644 --- a/.github/publish.yaml +++ b/.github/publish.yaml @@ -2,12 +2,12 @@ pypi: versions: - - version_tag - - version_branch + - tag + - stabilization_branch + - default_branch packages: - {} docker: - auto_login: true images: - name: camptocamp/tag-publish helm: diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 6b0fabe..96f3d1c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -80,6 +80,10 @@ jobs: - run: docker build --tag camptocamp/tag-publish tests + - name: Publish dry-run + run: poetry run tag-publish --dry-run + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Publish run: poetry run tag-publish env: diff --git a/README.md b/README.md index dd5334f..ff7b2ce 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ The main goals of Tag Publish offer the commands to publish the project, Using a tag, a stabilization branch, a feature branch or a pull request. -When possible it can do a secret-less publishing, if it's not possible the login should be done before the publishing. +When possible it can do a secret-less publishing (privileged in defaults), if it's not possible the login should be done before the publishing. See the [documentation](https://github.com/camptocamp/c2cciutils/wiki/Publishing). @@ -62,8 +62,9 @@ To create a patch version you should just create tag. This tool can publish on different kind of versions: -- `version_tag`: Related to a Git tag. -- `version_branch`: Related to a stabilization branch (including the default branch). +- `tag`: Related to a Git tag. +- `default_branch`: Related to the default branch. +- `stabilization_branch`: Related to a stabilization branch (including the default branch). - `feature_branch`: Related to a feature branch or a pull request. We can also publish on different kind of versions like `rebuild` by using the `--type` argument. @@ -108,7 +109,7 @@ https://docs.github.com/en/actions/security-for-github-actions/security-hardenin By default the package will be published only on tag, if you want to publish on stabilization branch you should add a `versions` key with the list of versions you want to publish, that can be: -`rebuild` (specified with --type), `version_tag`, `version_branch`, `feature_branch`, `feature_tag` (for pull request) +`rebuild` (specified with --type), `tag`, `default_branch`, `stabilization_branch`, `feature_branch`, `pull_request` (for pull request merge: number) It we have a `setup.py` file, we will be in legacy mode: When publishing, the version computed from arguments or `GITHUB_REF` is put in environment variable `VERSION`, thus you should use it in `setup.py`, example: @@ -127,8 +128,10 @@ enable = true vcs = "git" pattern = "^(?P\\d+(\\.\\d+)*)" format-jinja = """ -{%- if env.get("VERSION_TYPE") == "version_branch" -%} -{{serialize_pep440(bump_version(base, 1 if env.get("IS_MASTER") == "TRUE" else 2), dev=distance)}} +{%- if env.get("VERSION_TYPE") == "default_branch" -%} +{{serialize_pep440(bump_version(base, 1), dev=distance)}} +{%- elif env.get("VERSION_TYPE") == "stabilization_branch" -%} +{{serialize_pep440(bump_version(base, 2), dev=distance)}} {%- elif distance == 0 -%} {{serialize_pep440(base)}} {%- else -%} @@ -138,7 +141,7 @@ format-jinja = """ ``` -Note that we can access to the environment variables `VERSION`,`VERSION_TYPE` and `IS_MASTER`. +Note that we can access to the environment variables `VERSION`,`VERSION_TYPE`. Then by default: @@ -199,13 +202,13 @@ docker: If you want to use the GitHub token to be logged in on ghcr you should set `auto_login` to `True`, the requires the permissions are `packages: write`. -With that the image initially named `camptocamp/tag-publish:latest` will be published on GitHub CHCR and on Docker hub. +With that the image initially named `camptocamp/tag-publish:latest` will be published on GitHub GHCR and on Docker hub. The full config is like this: ```yaml docker: - auto_login: False + github_oidc_login: True latest: True images: - # The base name of the image we want to publish @@ -215,7 +218,7 @@ docker: # The fqdn name of the server if not Docker hub server: # List of kinds of versions you want to publish, that can be: rebuild (specified using --type), - # version_tag, version_branch, feature_branch, feature_tag (for pull request) + # tag, stabilization_branch, feature_branch, pull_request (for pull request merge: number) version: # List of tags we want to publish interpreted with `format(version=version)` # e.g. if you use `{version}-lite` when you publish the version `1.2.3` the source tag @@ -293,7 +296,7 @@ This will create a repository dispatch of type `published` on own repository wit ```json { "version": "1.2.3", - "version_type": "version_tag", + "version_type": "tag", "repository": "camptocamp/tag-publish", "items": [ { diff --git a/config.md b/config.md index fc33362..63d2450 100644 --- a/config.md +++ b/config.md @@ -4,9 +4,10 @@ _Tag Publish configuration file (.github/publish.yaml)_ ## Properties -- **`version`** _(object)_: The version configurations. - - **`branch_to_version_re`**: Refer to _[#/definitions/version_transform](#definitions/version_transform)_. - - **`tag_to_version_re`**: Refer to _[#/definitions/version_transform](#definitions/version_transform)_. +- **`transformers`** _(object)_: The version transform configurations. Default: `{"pull_request_to_version_re": [{"to": "pr-\\1"}]}`. + - **`branch_to_version`**: Refer to _[#/definitions/transform](#definitions/transform)_. + - **`tag_to_version`**: Refer to _[#/definitions/transform](#definitions/transform)_. + - **`pull_request_to_version`**: Refer to _[#/definitions/transform](#definitions/transform)_. - **`docker`**: Refer to _[#/definitions/docker](#definitions/docker)_. - **`pypi`**: Refer to _[#/definitions/pypi](#definitions/pypi)_. - **`node`**: Refer to _[#/definitions/node](#definitions/node)_. @@ -14,7 +15,7 @@ _Tag Publish configuration file (.github/publish.yaml)_ - **`dispatch`** _(array)_: Default: `[]`. - **Items** _(object)_: Send a dispatch event to an other repository. Default: `{}`. - **`repository`** _(string)_: The repository name to be triggered. Default: `"camptocamp/argocd-gs-gmf-apps"`. - - **`event-type`** _(string)_: The event type to be triggered. Default: `"published"`. + - **`event_type`** _(string)_: The event type to be triggered. Default: `"published"`. ## Definitions @@ -26,12 +27,12 @@ _Tag Publish configuration file (.github/publish.yaml)_ - **`name`** _(string)_: The image name. - **`tags`** _(array)_: The tag name, will be formatted with the version=, the image with version=latest should be present when we call the tag-publish script. Default: `["{version}"]`. - **Items** _(string)_ - - **`repository`** _(object)_: The repository where we should publish the images. Can contain additional properties. Default: `{"github": {"server": "ghcr.io", "versions": ["version_tag", "version_branch", "rebuild"]}}`. + - **`repository`** _(object)_: The repository where we should publish the images. Can contain additional properties. Default: `{"github": {"host": "ghcr.io", "versions_type": ["tag", "default_branch", "stabilization_branch", "rebuild"]}}`. - **Additional properties** _(object)_ - - **`server`** _(string)_: The server URL. - - **`versions`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["version_tag", "version_branch", "rebuild", "feature_branch"]`. + - **`host`** _(string)_: The host of the repository URL. + - **`versions_type`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["tag", "default_branch", "stabilization_branch", "rebuild", "feature_branch", "pull_request"]`. - **Items** _(string)_ - - **`auto_login`** _(boolean)_: Auto login to the GitHub Docker registry. Default: `false`. + - **`github_oidc_login`** _(boolean)_: Auto login to the GitHub Docker registry. Default: `true`. - **`pypi`** _(object)_: Configuration to publish on pypi. - **`packages`** _(array)_: The configuration of packages that will be published. - **Items** _(object)_: The configuration of package that will be published. @@ -39,18 +40,18 @@ _Tag Publish configuration file (.github/publish.yaml)_ - **`folder`** _(string)_: The folder of the pypi package. Default: `"."`. - **`build_command`** _(array)_: The command used to do the build. - **Items** _(string)_ - - **`versions`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["version_tag"]`. + - **`versions_type`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["tag"]`. - **Items** _(string)_ - **`node`** _(object)_: Configuration to publish on node. - **`packages`** _(array)_: The configuration of packages that will be published. - **Items** _(object)_: The configuration of package that will be published. - **`group`** _(string)_: The image is in the group, should be used with the --group option of tag-publish script. Default: `"default"`. - **`folder`** _(string)_: The folder of the node package. Default: `"."`. - - **`versions`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["version_tag"]`. + - **`versions_type`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["tag"]`. - **Items** _(string)_ - - **`repository`** _(object)_: The packages repository where we should publish the packages. Can contain additional properties. Default: `{"github": {"server": "npm.pkg.github.com"}}`. + - **`repository`** _(object)_: The packages repository where we should publish the packages. Can contain additional properties. Default: `{"github": {"host": "npm.pkg.github.com"}}`. - **Additional properties** _(object)_ - - **`server`** _(string)_: The server URL. + - **`host`** _(string)_: The host of the repository URL. - **`args`** _(array)_: The arguments to pass to the publish command. Default: `["--provenance"]`. - **Items** _(string)_ - **`helm`** _(object)_: Configuration to publish Helm charts on GitHub release. @@ -58,9 +59,9 @@ _Tag Publish configuration file (.github/publish.yaml)_ - **Items** _(object)_: The configuration of package that will be published. - **`group`** _(string)_: The image is in the group, should be used with the --group option of tag-publish script. Default: `"default"`. - **`folder`** _(string)_: The folder of the pypi package. Default: `"."`. - - **`versions`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["version_tag"]`. + - **`versions_type`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["tag"]`. - **Items** _(string)_ -- **`version_transform`** _(array)_: A version transformer definition. +- **`transform`** _(array)_: A version transformer definition. Default: `[]`. - **Items** _(object)_ - - **`from`** _(string)_: The from regular expression. - - **`to`** _(string)_: The expand regular expression: https://docs.python.org/3/library/re.html#re.Match.expand. + - **`from_re`** _(string)_: The from regular expression. Default: `"(.+)"`. + - **`to`** _(string)_: The expand regular expression: https://docs.python.org/3/library/re.html#re.Match.expand. Default: `"\\1"`. diff --git a/pyproject.toml b/pyproject.toml index 9bb44b6..faceff8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,8 +93,10 @@ enable = true vcs = "git" pattern = "^(?P\\d+(\\.\\d+)*)" format-jinja = """ -{%- if env.get("VERSION_TYPE") == "version_branch" -%} -{{serialize_pep440(bump_version(base, 1 if env.get("IS_MASTER") == "TRUE" else 2), dev=distance)}} +{%- if env.get("VERSION_TYPE") == "default_branch" -%} +{{serialize_pep440(bump_version(base, 1), dev=distance)}} +{%- elif env.get("VERSION_TYPE") == "stabilization_branch" -%} +{{serialize_pep440(bump_version(base, 2), dev=distance)}} {%- elif distance == 0 -%} {{serialize_pep440(base)}} {%- else -%} diff --git a/tag_publish/__init__.py b/tag_publish/__init__.py index e3d4e20..4360f2c 100644 --- a/tag_publish/__init__.py +++ b/tag_publish/__init__.py @@ -13,7 +13,6 @@ import applications_download import github import jsonschema_validator -import requests import ruamel.yaml import security_md import yaml @@ -59,17 +58,27 @@ def __init__(self) -> None: self.default_branch = self.repo.default_branch -def get_security_md(gh: GH) -> security_md.Security: +def get_security_md(gh: GH, local: bool) -> security_md.Security: """ Get the SECURITY.md file. Arguments: gh: The GitHub helper + local: If we should use the local file """ + if local: + if os.path.exists("SECURITY.md"): + print("Using the local SECURITY.md file") + with open("SECURITY.md", encoding="utf-8") as open_file: + return security_md.Security(open_file.read()) + print("No local SECURITY.md file") + return security_md.Security("") + try: security_file = gh.repo.get_contents("SECURITY.md") assert isinstance(security_file, github.ContentFile.ContentFile) + print("Using SECURITY.md file from the default branch") return security_md.Security(security_file.decoded_content.decode("utf-8")) except github.GithubException as exception: if exception.status == 404: @@ -79,27 +88,7 @@ def get_security_md(gh: GH) -> security_md.Security: raise exception -def merge(default_config: Any, config: Any) -> Any: - """ - Deep merge the dictionaries (on dictionaries only, not on arrays). - - Arguments: - default_config: The default config that will be applied - config: The base config, will be modified - - """ - if not isinstance(default_config, dict) or not isinstance(config, dict): - return config - - for key in default_config: - if key not in config: - config[key] = default_config[key] - else: - merge(default_config[key], config[key]) - return config - - -def get_config(gh: GH) -> tag_publish.configuration.Configuration: +def get_config() -> tag_publish.configuration.Configuration: """ Get the configuration, with project and auto detections. """ @@ -114,21 +103,6 @@ def get_config(gh: GH) -> tag_publish.configuration.Configuration: config = yaml_.load(open_file) jsonschema_validator.validate(".github/publish.yaml", cast(dict[str, Any], config), schema) - merge( - { - "version": { - "tag_to_version_re": [ - {"from": r"([0-9]+.[0-9]+.[0-9]+)", "to": r"\1"}, - ], - "branch_to_version_re": [ - {"from": r"([0-9]+.[0-9]+)", "to": r"\1"}, - {"from": gh.default_branch, "to": gh.default_branch}, - ], - } - }, - config, - ) - return config @@ -148,12 +122,10 @@ def get_value(matched: Optional[Match[str]], config: Optional[VersionTransform], Return the value """ - return matched.expand(config.get("to", r"\1")) if matched is not None and config is not None else value + return matched.expand(config["to"]) if matched is not None and config is not None else value -def compile_re( - config: tag_publish.configuration.VersionTransform, prefix: str = "" -) -> list[VersionTransform]: +def compile_re(config: tag_publish.configuration.Transform) -> list[VersionTransform]: """ Compile the from as a regular expression of a dictionary of the config list. @@ -161,21 +133,21 @@ def compile_re( Arguments: config: The transform config - prefix: The version prefix Return the compiled transform config. """ result = [] for conf in config: - new_conf = cast(VersionTransform, dict(conf)) + new_conf = cast( + VersionTransform, {"to": conf.get("to", tag_publish.configuration.TRANSFORM_TO_DEFAULT)} + ) - from_re = conf.get("from", r"(.*)") - if from_re[0] == "^": - from_re = from_re[1:] + from_re = conf.get("from_re", tag_publish.configuration.TRANSFORM_FROM_DEFAULT) + if from_re[0] != "^": + from_re = f"^{from_re}" if from_re[-1] != "$": from_re += "$" - from_re = f"^{re.escape(prefix)}{from_re}" new_conf["from"] = re.compile(from_re) result.append(new_conf) @@ -204,53 +176,6 @@ def match( return None, None, value -def does_match(value: str, config: list[VersionTransform]) -> bool: - """ - Check if the version match with the config patterns. - - Arguments: - --------- - value: That we want to match with - config: The result of `compile` - - Returns True it it does match else False - - """ - matched, _, _ = match(value, config) - return matched is not None - - -def check_response(response: requests.Response, raise_for_status: bool = True) -> Any: - """ - Check the response and raise an exception if it's not ok. - - Also print the X-Ratelimit- headers to get information about the rate limiting. - """ - for header in response.headers: - if header.lower().startswith("x-ratelimit-"): - print(f"{header}: {response.headers[header]}") - if raise_for_status: - response.raise_for_status() - - -def add_authorization_header(headers: dict[str, str]) -> dict[str, str]: - """ - Add the Authorization header needed to be authenticated on GitHub. - - Arguments: - headers: The headers - - Return the headers (to be chained) - - """ - try: - token = os.environ["GITHUB_TOKEN"].strip() - headers["Authorization"] = f"Bearer {token}" - return headers - except FileNotFoundError: - return headers - - @overload def download_application(application_name: str, binary_filename: str) -> str: ... diff --git a/tag_publish/cli.py b/tag_publish/cli.py index 8f9bae8..3d981e1 100644 --- a/tag_publish/cli.py +++ b/tag_publish/cli.py @@ -39,28 +39,6 @@ def match(tpe: str, base_re: str) -> Optional[Match[str]]: return re.match(f"^refs/{tpe}/{base_re}", os.environ["GITHUB_REF"]) -def to_version(full_config: tag_publish.configuration.Configuration, value: str, kind: str) -> str: - """ - Compute publish version from branch name or tag. - - Arguments: - full_config: The full configuration - value: The value to be transformed - kind: The name of the transformer in the configuration - - """ - item_re = tag_publish.compile_re( - cast( - tag_publish.configuration.VersionTransform, - full_config["version"].get(kind + "_to_version_re", []), - ) - ) - value_match = tag_publish.match(value, item_re) - if value_match[0] is not None: - return tag_publish.get_value(*value_match) - return value - - def main() -> None: """ Run the publish. @@ -72,15 +50,14 @@ def main() -> None: "--docker-versions", help="The versions to publish on Docker registry, comma separated, ex: 'x,x.y,x.y.z,latest'.", ) - parser.add_argument("--branch", help="The branch from which to compute the version") - parser.add_argument("--tag", help="The tag from which to compute the version") parser.add_argument("--dry-run", action="store_true", help="Don't do the publish") parser.add_argument("--dry-run-tag", help="Don't do the publish, on a tag") parser.add_argument("--dry-run-branch", help="Don't do the publish, on a branch") + parser.add_argument("--dry-run-pull", help="Don't do the publish, on a pull request") parser.add_argument( "--type", help="The type of version, if no argument provided auto-determinate, can be: " - "rebuild (in case of rebuild), version_tag, version_branch, feature_branch, feature_tag " + "rebuild (in case of rebuild), tag, default_branch, stabilization, feature_branch, pull_request " "(for pull request)", ) args = parser.parse_args() @@ -91,95 +68,84 @@ def main() -> None: if args.dry_run_branch is not None: args.dry_run = True os.environ["GITHUB_REF"] = f"refs/heads/{args.dry_run_branch}" + if args.dry_run_pull is not None: + args.dry_run = True + os.environ["GITHUB_REF"] = f"refs/pull/{args.dry_run_pull}" - github = tag_publish.GH() - config = tag_publish.get_config(github) + config = tag_publish.get_config() - # Describe the kind of release we do: rebuild (specified with --type), version_tag, version_branch, - # feature_branch, feature_tag (for pull request) + # Describe the kind of release we do: rebuild (specified with --type), tag, default_branch, + # stabilization_branch, feature_branch, pull_request (merge, number) version: str = "" ref = os.environ.get("GITHUB_REF", "refs/heads/fake-local") local = "GITHUB_REF" not in os.environ - if len([e for e in [args.version, args.branch, args.tag] if e is not None]) > 1: - print("::error::you specified more than one of the arguments --version, --branch or --tag") + if args.type is not None and args.version is None: + print("::error::you specified the argument --type but not the --version") sys.exit(1) version_type = args.type - - tag_match = tag_publish.match( - ref, - tag_publish.compile_re(config["version"].get("tag_to_version_re", []), "refs/tags/"), - ) - branch_match = tag_publish.match( - ref, - tag_publish.compile_re(config["version"].get("branch_to_version_re", []), "refs/heads/"), + github = tag_publish.GH() + security = tag_publish.get_security_md(github, local) + transformers = config.get( + "transformers", + cast(tag_publish.configuration.Transformers, tag_publish.configuration.TRANSFORMERS_DEFAULT), ) - ref_match = re.match(r"refs/pull/(.*)/merge", ref) if args.version is not None: version = args.version - elif args.branch is not None: - version = to_version(config, args.branch, "branch") - elif args.tag is not None: - version = to_version(config, args.tag, "tag") - elif tag_match[0] is not None: - if version_type is None: - version_type = "version_tag" - else: - print("::warning::you specified the argument --type but not one of --version, --branch or --tag") + elif ref.startswith("refs/tags/"): + version_type = "tag" + tag_match = tag_publish.match( + ref.split("/", 2)[2], + tag_publish.compile_re( + transformers.get("tag_to_version", cast(tag_publish.configuration.Transform, [{}])) + ), + ) version = tag_publish.get_value(*tag_match) - elif branch_match[0] is not None: - if version_type is None: - version_type = "version_branch" - else: - print("::warning::you specified the argument --type but not one of --version, --branch or --tag") - version = tag_publish.get_value(*branch_match) - elif ref_match is not None: - version = tag_publish.get_value(ref_match, {}, ref) - if version_type is None: - version_type = "feature_branch" elif ref.startswith("refs/heads/"): - if version_type is None: - version_type = "feature_branch" + branch = ref.split("/", 2)[2] + if branch == github.repo.default_branch: + version_type = "default_branch" + elif branch in security.branches(): + version_type = "stabilization_branch" else: - print("::warning::you specified the argument --type but not one of --version, --branch or --tag") - # By the way we replace '/' by '_' because it isn't supported by Docker - version = "_".join(ref.split("/")[2:]) - elif ref.startswith("refs/tags/"): - if version_type is None: - version_type = "feature_tag" + version_type = "feature_branch" + + if version_type in ("default_branch", "stabilization_branch"): + branch_match = tag_publish.match( + ref.split("/", 2)[2], + tag_publish.compile_re( + transformers.get("branch_to_version", cast(tag_publish.configuration.Transform, [{}])) + ), + ) + version = tag_publish.get_value(*branch_match) else: - print("::warning::you specified the argument --type but not one of --version, --branch or --tag") - # By the way we replace '/' by '_' because it isn't supported by Docker + version = branch.replace("/", "_") + elif ref.startswith("refs/pull/"): + version_type = "pull_request" + pull_match = tag_publish.match( + ref.split("/", 2)[2], + tag_publish.compile_re( + transformers.get("pull_request_to_version", cast(tag_publish.configuration.Transform, [{}])) + ), + ) + version = tag_publish.get_value(*pull_match) version = "_".join(ref.split("/")[2:]) else: print( - f"WARNING: {ref} is not supported, only ref starting with 'refs/heads/' or 'refs/tags/' " - "are supported, ignoring" + f"WARNING: {ref} is not supported, only ref starting with 'refs/heads/', 'refs/tags/' " + "or 'refs/pull/' are supported, ignoring" ) sys.exit(0) - if version_type is None: - print( - "::error::you specified one of the arguments --version, --branch or --tag but not the --type, " - f"GitHub ref is: {ref}" - ) - sys.exit(1) - - if version_type is not None: - if args.dry_run: - print(f"Create release type {version_type}: {version} (dry run)") - else: - print(f"Create release type {version_type}: {version}") - - github = tag_publish.GH() + print(f"Create release type {version_type}: {version}" + (" (dry run)" if args.dry_run else "")) success = True published_payload: list[tag_publish.PublishedPayload] = [] success &= _handle_pypi_publish( - args.group, args.dry_run, config, version, version_type, github, published_payload + args.group, args.dry_run, config, version, version_type, published_payload ) success &= _handle_node_publish( args.group, args.dry_run, config, version, version_type, published_payload @@ -191,9 +157,8 @@ def main() -> None: config, version, version_type, - github, published_payload, - local, + security, ) success &= _handle_helm_publish( args.group, args.dry_run, config, version, version_type, github, published_payload @@ -210,7 +175,6 @@ def _handle_pypi_publish( config: tag_publish.configuration.Configuration, version: str, version_type: str, - github: tag_publish.GH, published_payload: list[tag_publish.PublishedPayload], ) -> bool: success = True @@ -222,13 +186,13 @@ def _handle_pypi_publish( for package in pypi_config.get("packages", []): if package.get("group", tag_publish.configuration.PIP_PACKAGE_GROUP_DEFAULT) == group: publish = version_type in pypi_config.get( - "versions", tag_publish.configuration.PYPI_VERSIONS_DEFAULT + "versions_type", tag_publish.configuration.PYPI_VERSIONS_DEFAULT ) folder = package.get("folder", tag_publish.configuration.PYPI_PACKAGE_FOLDER_DEFAULT) if dry_run: print(f"{'Publishing' if publish else 'Checking'} '{folder}' to pypi, skipping (dry run)") else: - success &= tag_publish.publish.pip(package, version, version_type, publish, github) + success &= tag_publish.publish.pip(package, version, version_type, publish) if publish: published_payload.append({"type": "pypi", "folder": folder}) return success @@ -245,7 +209,7 @@ def _handle_node_publish( success = True node_config = config.get("node", {}) if node_config: - if version_type == "version_branch": + if version_type in ("default_branch", "stable_branch"): last_tag = ( subprocess.run( ["git", "describe", "--abbrev=0", "--tags"], check=True, stdout=subprocess.PIPE @@ -264,7 +228,7 @@ def _handle_node_publish( for package in node_config.get("packages", []): if package.get("group", tag_publish.configuration.NODE_PACKAGE_GROUP_DEFAULT) == group: publish = version_type in node_config.get( - "versions", tag_publish.configuration.NODE_VERSIONS_DEFAULT + "versions_type", tag_publish.configuration.NODE_VERSIONS_DEFAULT ) folder = package.get("folder", tag_publish.configuration.NODE_PACKAGE_FOLDER_DEFAULT) for repo_name, repo_config in node_config.get( @@ -300,16 +264,15 @@ def _handle_docker_publish( config: tag_publish.configuration.Configuration, version: str, version_type: str, - github: tag_publish.GH, published_payload: list[tag_publish.PublishedPayload], - local: bool, + security: security_md.Security, ) -> bool: success = True docker_config = config.get("docker", {}) if docker_config: sys.stdout.flush() sys.stderr.flush() - if docker_config.get("auto_login", tag_publish.configuration.DOCKER_AUTO_LOGIN_DEFAULT): + if docker_config.get("github_oidc_login", tag_publish.configuration.DOCKER_AUTO_LOGIN_DEFAULT): subprocess.run( [ "docker", @@ -320,16 +283,6 @@ def _handle_docker_publish( ], check=True, ) - security_text = "" - if local: - if os.path.exists("SECURITY.md"): - with open("SECURITY.md", encoding="utf-8") as security_file: - security_text = security_file.read() - security = security_md.Security(security_text) - else: - security = security_md.Security("") - else: - security = tag_publish.get_security_md(github) version_index = security.version_index alternate_tag_index = security.alternate_tag_index @@ -376,7 +329,7 @@ def _handle_docker_publish( ).items(): for docker_version in versions: if version_type in conf.get( - "versions", + "versions_type", tag_publish.configuration.DOCKER_REPOSITORY_VERSIONS_DEFAULT, ): tags = [ @@ -464,7 +417,7 @@ def _handle_helm_publish( .stdout.strip() .decode() ) - if version_type == "version_branch": + if version_type in ("default_branch", "stabilization_branch"): last_tag = ( subprocess.run( ["git", "describe", "--abbrev=0", "--tags"], check=True, stdout=subprocess.PIPE @@ -482,7 +435,9 @@ def _handle_helm_publish( for package in helm_config["packages"]: if package.get("group", tag_publish.configuration.HELM_PACKAGE_GROUP_DEFAULT) == group: - versions_type = helm_config.get("versions", tag_publish.configuration.HELM_VERSIONS_DEFAULT) + versions_type = helm_config.get( + "versions_type", tag_publish.configuration.HELM_VERSIONS_DEFAULT + ) publish = version_type in versions_type folder = package.get("folder", tag_publish.configuration.HELM_PACKAGE_FOLDER_DEFAULT) if publish: @@ -509,7 +464,7 @@ def _trigger_dispatch_events( ) -> None: for dispatch_config in config.get("dispatch", []): repository = dispatch_config.get("repository") - event_type = dispatch_config.get("event-type", tag_publish.configuration.DISPATCH_EVENT_TYPE_DEFAULT) + event_type = dispatch_config.get("event_type", tag_publish.configuration.DISPATCH_EVENT_TYPE_DEFAULT) published = { "version": version, diff --git a/tag_publish/configuration.py b/tag_publish/configuration.py index 1412340..965cadf 100644 --- a/tag_publish/configuration.py +++ b/tag_publish/configuration.py @@ -12,11 +12,15 @@ class Configuration(TypedDict, total=False): Tag Publish configuration file (.github/publish.yaml) """ - version: "Version" + transformers: "Transformers" """ - Version. + Transformers. - The version configurations + The version transform configurations. + + default: + pull_request_to_version_re: + - to: pr-\1 """ docker: "Docker" @@ -65,15 +69,15 @@ class Configuration(TypedDict, total=False): DISPATCH_EVENT_TYPE_DEFAULT = "published" -""" Default value of the field path 'dispatch config event-type' """ +""" Default value of the field path 'dispatch config event_type' """ DISPATCH_REPOSITORY_DEFAULT = "camptocamp/argocd-gs-gmf-apps" """ Default value of the field path 'dispatch config repository' """ -DOCKER_AUTO_LOGIN_DEFAULT = False -""" Default value of the field path 'Docker auto_login' """ +DOCKER_AUTO_LOGIN_DEFAULT = True +""" Default value of the field path 'Docker github_oidc_login' """ DOCKER_IMAGE_GROUP_DEFAULT = "default" @@ -89,39 +93,52 @@ class Configuration(TypedDict, total=False): DOCKER_REPOSITORY_DEFAULT = { - "github": {"server": "ghcr.io", "versions": ["version_tag", "version_branch", "rebuild"]} + "github": { + "host": "ghcr.io", + "versions_type": ["tag", "default_branch", "stabilization_branch", "rebuild"], + } } """ Default value of the field path 'Docker repository' """ -DOCKER_REPOSITORY_VERSIONS_DEFAULT = ["version_tag", "version_branch", "rebuild", "feature_branch"] -""" Default value of the field path 'Docker repository versions' """ - - -# | dispatch config. -# | -# | Send a dispatch event to an other repository -# | -# | default: -# | {} -DispatchConfig = TypedDict( - "DispatchConfig", - { - # | Dispatch repository. - # | - # | The repository name to be triggered - # | - # | default: camptocamp/argocd-gs-gmf-apps - "repository": str, - # | Dispatch event type. - # | - # | The event type to be triggered - # | - # | default: published - "event-type": str, - }, - total=False, -) +DOCKER_REPOSITORY_VERSIONS_DEFAULT = [ + "tag", + "default_branch", + "stabilization_branch", + "rebuild", + "feature_branch", + "pull_request", +] +""" Default value of the field path 'Docker repository versions_type' """ + + +class DispatchConfig(TypedDict, total=False): + """ + dispatch config. + + Send a dispatch event to an other repository + + default: + {} + """ + + repository: str + """ + Dispatch repository. + + The repository name to be triggered + + default: camptocamp/argocd-gs-gmf-apps + """ + + event_type: str + """ + Dispatch event type. + + The event type to be triggered + + default: published + """ class Docker(TypedDict, total=False): @@ -151,20 +168,21 @@ class Docker(TypedDict, total=False): default: github: - server: ghcr.io - versions: - - version_tag - - version_branch + host: ghcr.io + versions_type: + - tag + - default_branch + - stabilization_branch - rebuild """ - auto_login: bool + github_oidc_login: bool """ Docker auto login. Auto login to the GitHub Docker registry - default: False + default: True """ @@ -197,20 +215,22 @@ class DockerImage(TypedDict, total=False): class DockerRepository(TypedDict, total=False): """Docker repository.""" - server: str - """ The server URL """ + host: str + """ The host of the repository URL """ - versions: List[str] + versions_type: List[str] """ Docker repository versions. The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script default: - - version_tag - - version_branch + - tag + - default_branch + - stabilization_branch - rebuild - feature_branch + - pull_request """ @@ -222,8 +242,8 @@ class DockerRepository(TypedDict, total=False): """ Default value of the field path 'helm package group' """ -HELM_VERSIONS_DEFAULT = ["version_tag"] -""" Default value of the field path 'helm versions' """ +HELM_VERSIONS_DEFAULT = ["tag"] +""" Default value of the field path 'helm versions_type' """ class Helm(TypedDict, total=False): @@ -236,14 +256,14 @@ class Helm(TypedDict, total=False): packages: List["HelmPackage"] """ The configuration of packages that will be published """ - versions: List[str] + versions_type: List[str] """ helm versions. The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script default: - - version_tag + - tag """ @@ -285,12 +305,12 @@ class HelmPackage(TypedDict, total=False): """ Default value of the field path 'node package group' """ -NODE_REPOSITORY_DEFAULT = {"github": {"server": "npm.pkg.github.com"}} +NODE_REPOSITORY_DEFAULT = {"github": {"host": "npm.pkg.github.com"}} """ Default value of the field path 'node repository' """ -NODE_VERSIONS_DEFAULT = ["version_tag"] -""" Default value of the field path 'node versions' """ +NODE_VERSIONS_DEFAULT = ["tag"] +""" Default value of the field path 'node versions_type' """ class Node(TypedDict, total=False): @@ -303,14 +323,14 @@ class Node(TypedDict, total=False): packages: List["NodePackage"] """ The configuration of packages that will be published """ - versions: List[str] + versions_type: List[str] """ node versions. The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script default: - - version_tag + - tag """ repository: Dict[str, "NodeRepository"] @@ -321,7 +341,7 @@ class Node(TypedDict, total=False): default: github: - server: npm.pkg.github.com + host: npm.pkg.github.com """ args: List[str] @@ -364,8 +384,8 @@ class NodePackage(TypedDict, total=False): class NodeRepository(TypedDict, total=False): """Node repository.""" - server: str - """ The server URL """ + host: str + """ The host of the repository URL """ PIP_PACKAGE_GROUP_DEFAULT = "default" @@ -376,8 +396,8 @@ class NodeRepository(TypedDict, total=False): """ Default value of the field path 'pypi package folder' """ -PYPI_VERSIONS_DEFAULT = ["version_tag"] -""" Default value of the field path 'pypi versions' """ +PYPI_VERSIONS_DEFAULT = ["tag"] +""" Default value of the field path 'pypi versions_type' """ class Pypi(TypedDict, total=False): @@ -390,14 +410,14 @@ class Pypi(TypedDict, total=False): packages: List["PypiPackage"] """ The configuration of packages that will be published """ - versions: List[str] + versions_type: List[str] """ pypi versions. The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script default: - - version_tag + - tag """ @@ -430,43 +450,92 @@ class PypiPackage(TypedDict, total=False): """ The command used to do the build """ -class Version(TypedDict, total=False): +TRANSFORMERS_DEFAULT = {"pull_request_to_version_re": [{"to": "pr-\\1"}]} +""" Default value of the field path 'Tag publish configuration transformers' """ + + +TRANSFORM_DEFAULT: List[Any] = [] +""" Default value of the field path 'transform' """ + + +TRANSFORM_FROM_DEFAULT = "(.+)" +""" Default value of the field path 'Version transform from_re' """ + + +TRANSFORM_TO_DEFAULT = "\\1" +""" Default value of the field path 'Version transform to' """ + + +Transform = List["VersionTransform"] +""" +transform. + +A version transformer definition + +default: + [] +""" + + +class Transformers(TypedDict, total=False): """ - Version. + Transformers. - The version configurations + The version transform configurations. + + default: + pull_request_to_version_re: + - to: pr-\1 """ - branch_to_version_re: "VersionTransform" + branch_to_version: "Transform" """ - Version transform. + transform. A version transformer definition + + default: + [] """ - tag_to_version_re: "VersionTransform" + tag_to_version: "Transform" """ - Version transform. + transform. A version transformer definition + + default: + [] """ + pull_request_to_version: "Transform" + """ + transform. -VersionTransform = List["_VersionTransformItem"] -""" -Version transform. + A version transformer definition + + default: + [] + """ -A version transformer definition -""" +class VersionTransform(TypedDict, total=False): + """Version transform.""" + + from_re: str + """ + transform from. -_VersionTransformItem = TypedDict( - "_VersionTransformItem", - { - # | The from regular expression - "from": str, - # | The expand regular expression: https://docs.python.org/3/library/re.html#re.Match.expand - "to": str, - }, - total=False, -) + The from regular expression + + default: (.+) + """ + + to: str + """ + transform to. + + The expand regular expression: https://docs.python.org/3/library/re.html#re.Match.expand + + default: \1 + """ diff --git a/tag_publish/publish.py b/tag_publish/publish.py index ecc5e4e..539ff3f 100644 --- a/tag_publish/publish.py +++ b/tag_publish/publish.py @@ -22,18 +22,16 @@ def pip( version: str, version_type: str, publish: bool, - github: tag_publish.GH, ) -> bool: """ Publish to pypi. Arguments: version: The version that will be published - version_type: Describe the kind of release we do: rebuild (specified using --type), version_tag, - version_branch, feature_branch, feature_tag (for pull request) + version_type: Describe the kind of release we do: rebuild (specified using --type), tag, + default_branch, stabilization_branch, feature_branch, pull_request (merge, number) publish: If False only check the package package: The package configuration - github: The GitHub helper """ folder = package.get("folder", tag_publish.configuration.PYPI_PACKAGE_FOLDER_DEFAULT) @@ -45,9 +43,6 @@ def pip( env = {} env["VERSION"] = version env["VERSION_TYPE"] = version_type - default_branch = github.repo.default_branch - is_master = default_branch == version - env["IS_MASTER"] = "TRUE" if is_master else "FALSE" cwd = os.path.abspath(folder) @@ -58,7 +53,7 @@ def pip( cmd = ["python3", "./setup.py", "egg_info", "--no-date"] cmd += ( ["--tag-build=dev" + datetime.datetime.now().strftime("%Y%m%d%H%M%S")] - if version_type in ("version_branch", "rebuild") + if version_type in ("default_branch", "stabilization_branch", "rebuild") else [] ) cmd.append("bdist_wheel") @@ -117,8 +112,8 @@ def node( Arguments: version: The version that will be published - version_type: Describe the kind of release we do: rebuild (specified using --type), version_tag, - version_branch, feature_branch, feature_tag (for pull request) + version_type: Describe the kind of release we do: rebuild (specified using --type), tag, + default_branch, stabilization_branch, feature_branch, pull_request (merge, number) repo_config: The repository configuration publish: If False only check the package package: The package configuration @@ -132,7 +127,7 @@ def node( sys.stderr.flush() try: - if version_type == "version_tag": + if version_type == "tag": with open(os.path.join(folder, "package.json"), encoding="utf-8") as open_file: package_json = json.loads(open_file.read()) package_json["version"] = version @@ -141,7 +136,7 @@ def node( cwd = os.path.abspath(folder) - is_github = repo_config["server"] == "npm.pkg.github.com" + is_github = repo_config["host"] == "npm.pkg.github.com" old_npmrc = None npmrc_filename = os.path.expanduser("~/.npmrc") if is_github: @@ -151,7 +146,7 @@ def node( old_npmrc = open_file.read() with open(npmrc_filename, "w", encoding="utf-8") as open_file: open_file.write(f"//npm.pkg.github.com/:_authToken={os.environ['GITHUB_TOKEN']}\n") - open_file.write(f"registry=https://{repo_config['server']}\n") + open_file.write(f"registry=https://{repo_config['host']}\n") open_file.write("always-auth=true\n") subprocess.run(["npm", "publish", *([] if publish else ["--dry-run"]), *args], cwd=cwd, check=True) @@ -196,8 +191,8 @@ def docker( tag_src: The source tag (usually latest) dst_tags: Publish using the provided tags images_full: The list of published images (with tag), used to build the dispatch event - version_type: Describe the kind of release we do: rebuild (specified using --type), version_tag, - version_branch, feature_branch, feature_tag (for pull request) + version_type: Describe the kind of release we do: rebuild (specified using --type), tag, + default_branch, stabilization_branch, feature_branch, pull_request (merge, number) published: The list of published artifacts to be filled """ @@ -210,23 +205,23 @@ def docker( try: new_images_full = [] - if "server" in config: + if "host" in config: for tag in dst_tags: subprocess.run( [ "docker", "tag", f"{image_config['name']}:{tag_src}", - f"{config['server']}/{image_config['name']}:{tag}", + f"{config['host']}/{image_config['name']}:{tag}", ], check=True, ) - new_images_full.append(f"{config['server']}/{image_config['name']}:{tag}") + new_images_full.append(f"{config['host']}/{image_config['name']}:{tag}") if published is not None: published.append( { "type": "docker", - "repository": config["server"], + "repository": config["host"], "image": image_config["name"], "tag": tag, } diff --git a/tag_publish/schema.json b/tag_publish/schema.json index 1f3b7e8..72bf83c 100644 --- a/tag_publish/schema.json +++ b/tag_publish/schema.json @@ -51,8 +51,8 @@ "description": "The repository where we should publish the images", "default": { "github": { - "server": "ghcr.io", - "versions": ["version_tag", "version_branch", "rebuild"] + "host": "ghcr.io", + "versions_type": ["tag", "default_branch", "stabilization_branch", "rebuild"] } }, "type": "object", @@ -60,15 +60,22 @@ "title": "Docker repository", "type": "object", "properties": { - "server": { - "description": "The server URL", + "host": { + "description": "The host of the repository URL", "type": "string" }, - "versions": { + "versions_type": { "description": "The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script", "title": "Docker repository versions", "type": "array", - "default": ["version_tag", "version_branch", "rebuild", "feature_branch"], + "default": [ + "tag", + "default_branch", + "stabilization_branch", + "rebuild", + "feature_branch", + "pull_request" + ], "items": { "type": "string" } @@ -76,11 +83,11 @@ } } }, - "auto_login": { + "github_oidc_login": { "title": "Docker auto login", "description": "Auto login to the GitHub Docker registry", "type": "boolean", - "default": false + "default": true } } }, @@ -119,11 +126,11 @@ } } }, - "versions": { + "versions_type": { "title": "pypi versions", "description": "The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script", "type": "array", - "default": ["version_tag"], + "default": ["tag"], "items": { "type": "string" } @@ -158,11 +165,11 @@ } } }, - "versions": { + "versions_type": { "title": "node versions", "description": "The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script", "type": "array", - "default": ["version_tag"], + "default": ["tag"], "items": { "type": "string" } @@ -172,7 +179,7 @@ "description": "The packages repository where we should publish the packages", "default": { "github": { - "server": "npm.pkg.github.com" + "host": "npm.pkg.github.com" } }, "type": "object", @@ -180,8 +187,8 @@ "title": "Node repository", "type": "object", "properties": { - "server": { - "description": "The server URL", + "host": { + "description": "The host of the repository URL", "type": "string" } } @@ -226,44 +233,54 @@ } } }, - "versions": { + "versions_type": { "title": "helm versions", "description": "The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script", "type": "array", - "default": ["version_tag"], + "default": ["tag"], "items": { "type": "string" } } } }, - "version_transform": { - "title": "Version transform", + "transform": { + "title": "transform", "description": "A version transformer definition", "type": "array", + "default": [], "items": { + "title": "Version transform", "type": "object", "properties": { - "from": { + "from_re": { + "title": "transform from", "description": "The from regular expression", - "type": "string" + "type": "string", + "default": "(.+)" }, "to": { + "title": "transform to", "description": "The expand regular expression: https://docs.python.org/3/library/re.html#re.Match.expand", - "type": "string" + "type": "string", + "default": "\\1" } } } } }, "properties": { - "version": { - "title": "Version", - "description": "The version configurations", + "transformers": { + "title": "Transformers", + "description": "The version transform configurations.", "type": "object", + "default": { + "pull_request_to_version_re": [{ "to": "pr-\\1" }] + }, "properties": { - "branch_to_version_re": { "$ref": "#/definitions/version_transform" }, - "tag_to_version_re": { "$ref": "#/definitions/version_transform" } + "branch_to_version": { "$ref": "#/definitions/transform" }, + "tag_to_version": { "$ref": "#/definitions/transform" }, + "pull_request_to_version": { "$ref": "#/definitions/transform" } } }, "docker": { "$ref": "#/definitions/docker" }, @@ -287,7 +304,7 @@ "default": "camptocamp/argocd-gs-gmf-apps", "type": "string" }, - "event-type": { + "event_type": { "title": "Dispatch event type", "description": "The event type to be triggered", "default": "published",