From acb9b684e5357c533f7e873386640cd9436968a6 Mon Sep 17 00:00:00 2001 From: Clayton Parnell <42805768+claytonparnell@users.noreply.github.com> Date: Mon, 26 Feb 2024 11:47:42 -0500 Subject: [PATCH] feature: Support automated releases (#179) --- .github/workflows/PR_TEMPLATE.md | 4 + .github/workflows/build-image.yml | 101 ++++++++++++++++++++ .github/workflows/monthly-minor-release.yml | 44 +++++++++ .github/workflows/weekly-patch-release.yml | 46 +++++++++ src/main.py | 10 +- 5 files changed, 200 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/PR_TEMPLATE.md create mode 100644 .github/workflows/build-image.yml create mode 100644 .github/workflows/monthly-minor-release.yml create mode 100644 .github/workflows/weekly-patch-release.yml diff --git a/.github/workflows/PR_TEMPLATE.md b/.github/workflows/PR_TEMPLATE.md new file mode 100644 index 00000000..f6bcb2e4 --- /dev/null +++ b/.github/workflows/PR_TEMPLATE.md @@ -0,0 +1,4 @@ +This pull request was created by GitHub Actions/AWS CodeBuild! Before merging, please do the following: +- [ ] Review changelog/staleness report. +- [ ] Review build/test results by clicking *Build Logs* in CI Report (be patient, tests take ~4hr). +- [ ] Review ECR Scan results. \ No newline at end of file diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 00000000..764b59cd --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,101 @@ +name: Build New Image Version +on: + # Manually call + workflow_dispatch: + inputs: + release-type: + required: true + type: choice + description: Type of release + options: + - patch + - minor + - major + base-version: + required: true + description: Base version= + # Call from other workflow + workflow_call: + inputs: + release-type: + type: string + required: true + base-version: + type: string + required: true +defaults: + run: + shell: bash -l {0} +jobs: + open-pr: + name: Open Pull Request + runs-on: ubuntu-latest + if: github.repository == 'aws/sagemaker-distribution' + permissions: + pull-requests: write + contents: write + outputs: + pr_id: ${{ steps.get_pr_id.outputs.pr_id }} + target_version: ${{ steps.calc_target.outputs.target_version }} + steps: + - uses: actions/checkout@v4 + - uses: mamba-org/setup-micromamba@v1 + with: + environment-file: ./environment.yml + environment-name: sagemaker-distribution + init-shell: bash + - name: Free up disk space + run: rm -rf /opt/hostedtoolcache + - name: Activate sagemaker-distribution + run: micromamba activate sagemaker-distribution + - name: Calculate target version + id: calc_target + run: | + TARGET_VERSION=$(python -c 'import semver; print(semver.bump_${{ inputs.release-type }}("${{ inputs.base-version }}"))') + echo "target_version=$TARGET_VERSION" >> $GITHUB_OUTPUT + - name: Create new branch + # Note - CodeBuild depends on this branch name. Don't change without corresponding backend change. + run: git checkout -b release-${{ steps.calc_target.outputs.target_version }} + - name: Generate artifacts + run: python ./src/main.py create-${{ inputs.release-type }}-version-artifacts --base-patch-version ${{ inputs.base-version }} + - name: Commit .in artifacts to branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config --local user.email "github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + git add ./build_artifacts + git commit -m 'chore: Generate build artifacts for ${{ steps.calc_target.outputs.target_version }} release' + git push --set-upstream origin release-${{ steps.calc_target.outputs.target_version }} + - name: Open pull request + id: get_pr_id + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Note - CodeBuild depends on this PR title. Don't change without corresponding backend change. + run: | + URL=$(gh pr create -H release-${{ steps.calc_target.outputs.target_version }} \ + --title 'release: v${{ steps.calc_target.outputs.target_version }}' -F ./.github/workflows/PR_TEMPLATE.md) + PR=$(echo $URL | sed 's:.*/::') + echo "pr_id=$PR" >> $GITHUB_OUTPUT + call-codebuild-project: + runs-on: ubuntu-latest + needs: open-pr + permissions: + pull-requests: write + contents: write + id-token: write + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::700843992353:role/codebuild-start-build-role + aws-region: us-west-2 + # CodeBuild timeout of 8 hours + role-duration-seconds: 3840 + - name: Run CodeBuild + uses: dark-mechanicum/aws-codebuild@v1 + env: + CODEBUILD__sourceVersion: 'pr/${{ needs.open-pr.outputs.pr_id }}' + with: + projectName: build-test-public-image + buildspec: '{"imageOverride": "aws/codebuild/standard:7.0"}' diff --git a/.github/workflows/monthly-minor-release.yml b/.github/workflows/monthly-minor-release.yml new file mode 100644 index 00000000..7b91b75c --- /dev/null +++ b/.github/workflows/monthly-minor-release.yml @@ -0,0 +1,44 @@ +name: Generate monthly minor release(s) +on: + # Run first day of the month at 8AM PST / 9AM PDT + schedule: + - cron: '0 16 1 * *' +jobs: + generate-version-matrix: + name: Generate version matrix + runs-on: ubuntu-latest + if: github.repository == 'aws/sagemaker-distribution' + outputs: + matrix: ${{ steps.gen-mat.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - name: Generate base patch version matrix + id: gen-mat + # Output looks like: matrix={"version":["0.11.0","1.4.1"]} + # For each major, get highest minor version. Then get highest patch + # for that minor. Use this patch as base version. + run: | + versions=("{\"version\":[") + for major in build_artifacts/v*; do + highest_minor=$(ls $major | sort -t. -k2n | tail -n1) + highest_patch=$(ls $major/$highest_minor | sort -t. -k2n | tail -n1) + versions+="\"${highest_patch#v}\"" + versions+="," + done + versions=${versions::-1} + versions+="]}" + echo "matrix=$versions" >> $GITHUB_OUTPUT + start-monthly-minor: + name: Start monthly minor release + needs: generate-version-matrix + permissions: + pull-requests: write + contents: write + id-token: write + strategy: + matrix: ${{ fromJson(needs.generate-version-matrix.outputs.matrix) }} + fail-fast: false + uses: aws/sagemaker-distribution/.github/workflows/build-image.yml@main + with: + release-type: "minor" + base-version: ${{ matrix.version }} \ No newline at end of file diff --git a/.github/workflows/weekly-patch-release.yml b/.github/workflows/weekly-patch-release.yml new file mode 100644 index 00000000..8de5b130 --- /dev/null +++ b/.github/workflows/weekly-patch-release.yml @@ -0,0 +1,46 @@ +name: Generate weekly patch release(s) +on: + # Run every Monday at 6AM PST / 7AM PDT + # Run before monthly, so we don't immediately patch a new minor version + schedule: + - cron: '0 14 * * MON' +jobs: + generate-version-matrix: + name: Generate-matrix + runs-on: ubuntu-latest + if: github.repository == 'aws/sagemaker-distribution' + outputs: + matrix: ${{ steps.gen-mat.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - name: Generate patch version matrix + id: gen-mat + # Output looks like :matrix={"version":["0.0.2","0.1.2",..."1.4.1"]} + # For each minor, get highest patch version. Use each patch + # as base version. Use this patch as base version. + run: | + versions=("{\"version\":[") + for minor in build_artifacts/v*/*; do + minor_version="${minor##*/}" + highest_patch=$(ls $minor | sort -t. -k3,3n | tail -n1) + versions+="\"${highest_patch#v}\"" + versions+="," + done + versions=${versions::-1} + versions+="]}" + echo "matrix=$versions" >> $GITHUB_OUTPUT + + start-weekly-patch: + name: Start weekly patch release + needs: generate-version-matrix + permissions: + pull-requests: write + contents: write + id-token: write + strategy: + matrix: ${{ fromJson(needs.generate-version-matrix.outputs.matrix) }} + fail-fast: false + uses: aws/sagemaker-distribution/.github/workflows/build-image.yml@main + with: + release-type: "patch" + base-version: ${{ matrix.version }} diff --git a/src/main.py b/src/main.py index 9cf42db6..8108bc7d 100644 --- a/src/main.py +++ b/src/main.py @@ -193,7 +193,7 @@ def _push_images_upstream(image_versions_to_push: list[dict[str, str]], region: def _test_local_images(image_ids_to_test: list[str], target_version: str): assert len(image_ids_to_test) == len(_image_generator_configs) for (image_id, config) in zip(image_ids_to_test, _image_generator_configs): - exit_code = pytest.main(['-n', '2', '-m', config['image_type'], '--local-image-version', + exit_code = pytest.main(['-n', 'auto', '-m', config['image_type'], '--local-image-version', target_version, *config['pytest_flags']]) assert exit_code == 0, f'Tests failed with exit code: {exit_code} against: {image_id}' @@ -262,10 +262,10 @@ def _build_local_images(target_version: Version, target_ecr_repo_list: list[str] for t in image_tags_to_apply: image.tag(target_ecr_repo, tag=t) generated_image_versions.append({'repository': target_ecr_repo, 'tag': t}) - else: - # Tag the image for testing - image.tag('localhost/sagemaker-distribution', - config['image_tag_generator'].format(image_version=str(target_version))) + + # Tag the image for testing + image.tag('localhost/sagemaker-distribution', + config['image_tag_generator'].format(image_version=str(target_version))) return generated_image_ids, generated_image_versions