From 4ec78772590fc0afa7d76d7c9ff3b9e45df964c4 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Fri, 6 Oct 2023 10:12:01 -0400 Subject: [PATCH] refactor diff generation --- .github/scripts/codegen-diff-revisions.py | 78 +++++++------- .github/workflows/codegen-preview.yml | 119 +++++++++++++--------- 2 files changed, 103 insertions(+), 94 deletions(-) diff --git a/.github/scripts/codegen-diff-revisions.py b/.github/scripts/codegen-diff-revisions.py index 871be4017ff..eda9877bbb6 100755 --- a/.github/scripts/codegen-diff-revisions.py +++ b/.github/scripts/codegen-diff-revisions.py @@ -33,8 +33,6 @@ import tempfile import shlex -HEAD_BRANCH_NAME = "__tmp-localonly-head" -BASE_BRANCH_NAME = "__tmp-localonly-base" OUTPUT_PATH = "tmp-codegen-diff/" COMMIT_AUTHOR_NAME = "GitHub Action (generated codegen diff)" @@ -138,9 +136,11 @@ def write_html_template(title, subtitle, tmp_file): tmp_file.flush() -def make_diff(title, path_to_diff, base_sha, head_sha, suffix, ignore_whitespace): +def make_diff(opts, title, path_to_diff, suffix, ignore_whitespace): + base_sha = opts.base_sha + head_sha = opts.head_sha ws_flag = "-b" if ignore_whitespace else "" - diff_exists = get_cmd_status(f"git diff --quiet {ws_flag} {BASE_BRANCH_NAME} {HEAD_BRANCH_NAME} -- {path_to_diff}") + diff_exists = get_cmd_status(f"git diff --quiet {ws_flag} {opts.base_branch} {opts.head_branch} -- {path_to_diff}") if diff_exists == 0: eprint(f"No diff output for {base_sha}..{head_sha}") @@ -156,7 +156,7 @@ def make_diff(title, path_to_diff, base_sha, head_sha, suffix, ignore_whitespace # All arguments after the first `--` go to the `git diff` command. diff_cmd = f"diff2html -s line -f html -d word -i command --hwt " \ f"{tmp_file.name} -F {OUTPUT_PATH}/{dest_path} -- " \ - f"-U20 {ws_flag} {BASE_BRANCH_NAME} {HEAD_BRANCH_NAME} -- {path_to_diff}" + f"-U20 {ws_flag} {opts.base_branch} {opts.head_branch} -- {path_to_diff}" eprint(f"Running diff cmd: {diff_cmd}") run(diff_cmd) return dest_path @@ -169,16 +169,24 @@ def diff_link(diff_text, empty_diff_text, diff_location, alternate_text, alterna return f"[{diff_text}]({CDN_URL}/codegen-diff/{diff_location}) ([{alternate_text}]({CDN_URL}/codegen-diff/{alternate_location}))" -def make_diffs(base_sha, head_sha): - sdk_ws = make_diff('AWS SDK', f'{OUTPUT_PATH}/services', base_sha, head_sha, 'aws-sdk', ignore_whitespace=False) - sdk_no_ws = make_diff('AWS SDK', f'{OUTPUT_PATH}/services', base_sha, head_sha, 'aws-sdk-ignore-ws', - ignore_whitespace=True) - +def make_diffs(opts): + sdk_ws = make_diff(opts, 'AWS SDK', f'{OUTPUT_PATH}/services', 'aws-sdk', ignore_whitespace=False) + sdk_no_ws = make_diff(opts, 'AWS SDK', f'{OUTPUT_PATH}/services', 'aws-sdk-ignore-ws', ignore_whitespace=True) sdk_links = diff_link('AWS SDK', 'No codegen difference in the AWS SDK', sdk_ws, 'ignoring whitespace', sdk_no_ws) return f'A new generated diff is ready to view.\\n\\n- {sdk_links}\\n' +def _codegen_cmd(opts): + generate_and_commit_generated_code(opts.head_sha, opts.bootstrap) + + +def _generate_diffs_cmd(opts): + bot_message = make_diffs(opts) + with open(f"{OUTPUT_PATH}/bot-message", 'w') as f: + f.write(bot_message) + + def create_cli(): parser = argparse.ArgumentParser( prog="codegen-diff-revisions", @@ -186,12 +194,21 @@ def create_cli(): formatter_class=argparse.ArgumentDefaultsHelpFormatter ) - parser.add_argument("repo_root", help="repository root") - parser.add_argument("base_sha", help="base commit to diff against (SHA-like)") - parser.add_argument("--bootstrap", help="services to pass to bootstrap and include in diff output", - default="+dynamodb,+codebuild,+sts,+ec2,+polly,+s3") parser.add_argument("--head-sha", help="head commit to use (defaults to whatever current HEAD) is") + subparsers = parser.add_subparsers() + codegen = subparsers.add_parser("codegen", help="generate and commit generated code") + codegen.add_argument("--bootstrap", help="services to pass to bootstrap and include in diff output", + default="+dynamodb,+codebuild,+sts,+ec2,+polly,+s3") + codegen.set_defaults(cmd=_codegen_cmd) + + generate_diffs = subparsers.add_parser("generate-diffs", + help="generate diffs between two branches and output bot message") + generate_diffs.add_argument("--base-sha", help="base commit to diff against (SHA-like)") + generate_diffs.add_argument("base_branch", help="name of the base branch to diff against") + generate_diffs.add_argument("head_branch", help="name of the head branch to diff against") + generate_diffs.set_defaults(cmd=_generate_diffs_cmd) + return parser @@ -200,39 +217,12 @@ def main(): opts = cli.parse_args() print(opts) - os.chdir(opts.repo_root) - if opts.head_sha is None: - head_sha = get_cmd_output("git rev-parse HEAD") - else: - head_sha = opts.head_sha - - print(f"using head sha is {head_sha}") - - # Make sure the working tree is clean - if get_cmd_status("git diff --quiet") != 0: - eprint("working tree is not clean. aborting") - sys.exit(1) + opts.head_sha = get_cmd_output("git rev-parse HEAD") - # Generate code for HEAD - print(f"Creating temporary branch with generated code for the HEAD revision {head_sha}") - run(f"git checkout {head_sha} -b {HEAD_BRANCH_NAME}") - generate_and_commit_generated_code(head_sha, opts.bootstrap) - - # Generate code for base - print(f"Creating temporary branch with generated code for the base revision {opts.base_sha}") - run(f"git checkout {opts.base_sha} -b {BASE_BRANCH_NAME}") - generate_and_commit_generated_code(opts.base_sha, opts.bootstrap) - - bot_message = make_diffs(opts.base_sha, head_sha) - with open(f"{OUTPUT_PATH}/bot-message", 'w') as f: - f.write(bot_message) + print(f"using head sha: {opts.head_sha}") - # cleanup - if not running_in_github_action(): - run(f"git checkout main") - run(f"git branch -D {BASE_BRANCH_NAME}") - run(f"git branch -D {HEAD_BRANCH_NAME}") + opts.cmd(opts) if __name__ == '__main__': diff --git a/.github/workflows/codegen-preview.yml b/.github/workflows/codegen-preview.yml index d93983be327..d0e45e0acb8 100644 --- a/.github/workflows/codegen-preview.yml +++ b/.github/workflows/codegen-preview.yml @@ -31,6 +31,8 @@ env: # - @restJson1: polly # - @restXml: s3 PREVIEW_SERVICES: +dynamodb,+codebuild,+sts,+ec2,+polly,+s3 + HEAD_BRANCH_NAME: __tmp-localonly-head + BASE_BRANCH_NAME: __tmp-localonly-base jobs: @@ -71,63 +73,80 @@ jobs: run: | npm install -g diff2html-cli@${{ env.DIFF2HTML_VERSION }} env | sort - - name: Generate diff for head ref + - name: Generate code for head ref run: | REPO_TOOLS=$GITHUB_WORKSPACE/aws-kotlin-repo-tools SMITHY_KOTLIN_DIR=$GITHUB_WORKSPACE/smithy-kotlin SDK_DIR=$GITHUB_WORKSPACE/aws-sdk-kotlin branch=$(python3 $REPO_TOOLS/scripts/ci.py get-branch $SDK_DIR) echo "using branch $branch" - python3 $REPO_TOOLS/scripts/ci.py set-branch --branch branch $SMITHY_KOTLIN_DIR + python3 $REPO_TOOLS/scripts/ci.py set-branch --branch $branch $SMITHY_KOTLIN_DIR + pushd $SDK_DIR + git checkout -b $HEAD_BRANCH_NAME + .github/scripts/codegen-diff-revisions.py codegen --bootstrap ${{ env.PREVIEW_SERVICES }} + popd -# - name: Generate diff -# id: generate-diff -# run: | -# # codegen-diff-revisions requires a clean index, set-upstream-versions.py can modify local repo state -# # we don't push these branches/commits anywhere so just commit it if necessary and move on -# if ! git diff --quiet gradle.properties -# then -# echo "gradle.properties is dirty, committing before generating diffs" -# git add gradle.properties -# PRE_COMMIT_ALLOW_NO_CONFIG=1 git \ -# -c "user.name=GitHub Action (generated code preview)" \ -# -c "user.email=generated-code-action@github.com" \ -# commit -m "codegen diff autocommit" --allow-empty -# fi -# .github/scripts/codegen-diff-revisions.py \ -# --bootstrap ${{ env.PREVIEW_SERVICES }} \ -# --head-sha ${{ github.event.pull_request.head.sha }} \ -# . ${{ github.event.pull_request.base.sha }} -# echo "codegen-diff-msg<> $GITHUB_OUTPUT -# cat ./tmp-codegen-diff/bot-message) >> $GITHUB_OUTPUT -# echo "EOF" >> $GITHUB_OUTPUT -# - name: Configure AWS Credentials -# uses: aws-actions/configure-aws-credentials@v2 -# with: -# role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} -# aws-region: us-west-2 -# - name: Upload diff to S3 -# run: | -# if [[ -d ./tmp-codegen-diff/${{ github.event.pull_request.base.sha }} ]]; then -# aws s3 cp ./tmp-codegen-diff/${{ github.event.pull_request.base.sha }} \ -# "s3://${{ secrets.CDN_S3_BUCKET_NAME }}/codegen-diff/${{ github.event.pull_request.base.sha }}" --recursive -# fi + - name: Generate code for base ref + run: | + REPO_TOOLS=$GITHUB_WORKSPACE/aws-kotlin-repo-tools + SMITHY_KOTLIN_DIR=$GITHUB_WORKSPACE/smithy-kotlin + SDK_DIR=$GITHUB_WORKSPACE/aws-sdk-kotlin + branch=$GITHUB_BASE_REF + echo "checkout smithy-kotlin at $branch" + pushd $SMITHY_KOTLIN_DIR + git switch -f main + python3 $REPO_TOOLS/scripts/ci.py set-branch --branch $branch + popd + echo "resetting aws-sdk-kotlin" + pushd $SDK_DIR + git switch -f main + python3 $REPO_TOOLS/scripts/ci.py set-branch --branch $branch + git checkout -b $BASE_BRANCH_NAME + .github/scripts/codegen-diff-revisions.py codegen --bootstrap ${{ env.PREVIEW_SERVICES }} + popd + + - name: Generate diffs + run: | + SDK_DIR=$GITHUB_WORKSPACE/aws-sdk-kotlin + pushd $SDK_DIR + .github/scripts/codegen-diff-revisions.py generate-diffs \ + --head-sha ${{ github.event.pull_request.head.sha }} \ + --base-sha ${{ github.event.pull_request.base.sha }} \ + $BASE_BRANCH_NAME $HEAD_BRANCH_NAME + echo "codegen-diff-msg<> $GITHUB_OUTPUT + cat ./tmp-codegen-diff/bot-message) >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + popd + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} + aws-region: us-west-2 + + - name: Upload diff to S3 + run: | + SDK_DIR=$GITHUB_WORKSPACE/aws-sdk-kotlin + pushd $SDK_DIR + if [[ -d ./tmp-codegen-diff/${{ github.event.pull_request.base.sha }} ]]; then + aws s3 cp ./tmp-codegen-diff/${{ github.event.pull_request.base.sha }} \ + "s3://${{ secrets.CDN_S3_BUCKET_NAME }}/codegen-diff/${{ github.event.pull_request.base.sha }}" --recursive + fi # TODO - generate doc preview for N services and upload and link as well -# post-bot-comment: -# name: Post bot comment -# runs-on: ubuntu-latest -# needs: -# - generate-codegen-diff -# steps: -# - name: Post bot comment -# uses: actions/github-script@v5 -# with: -# script: | -# await github.rest.issues.createComment({ -# issue_number: ${{ github.event.number }}, -# owner: context.repo.owner, -# repo: context.repo.repo, -# body: '${{ needs.generate-codegen-diff.outputs.bot-message }}\n\n' -# }) + post-bot-comment: + name: Post bot comment + runs-on: ubuntu-latest + needs: + - generate-codegen-diff + steps: + - name: Post bot comment + uses: actions/github-script@v5 + with: + script: | + await github.rest.issues.createComment({ + issue_number: ${{ github.event.number }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: '${{ needs.generate-codegen-diff.outputs.bot-message }}\n\n' + }) \ No newline at end of file