refactor diff generation

.github/scripts/codegen-diff-revisions.py

@@ -33,8 +33,6 @@
 import tempfile
 import shlex
 
-HEAD_BRANCH_NAME = "__tmp-localonly-head"
-BASE_BRANCH_NAME = "__tmp-localonly-base"
 OUTPUT_PATH = "tmp-codegen-diff/"
 
 COMMIT_AUTHOR_NAME = "GitHub Action (generated codegen diff)"
@@ -138,9 +136,11 @@ def write_html_template(title, subtitle, tmp_file):
     tmp_file.flush()
 
 
-def make_diff(title, path_to_diff, base_sha, head_sha, suffix, ignore_whitespace):
+def make_diff(opts, title, path_to_diff, suffix, ignore_whitespace):
+    base_sha = opts.base_sha
+    head_sha = opts.head_sha
     ws_flag = "-b" if ignore_whitespace else ""
-    diff_exists = get_cmd_status(f"git diff --quiet {ws_flag} {BASE_BRANCH_NAME} {HEAD_BRANCH_NAME} -- {path_to_diff}")
+    diff_exists = get_cmd_status(f"git diff --quiet {ws_flag} {opts.base_branch} {opts.head_branch} -- {path_to_diff}")
 
     if diff_exists == 0:
         eprint(f"No diff output for {base_sha}..{head_sha}")
@@ -156,7 +156,7 @@ def make_diff(title, path_to_diff, base_sha, head_sha, suffix, ignore_whitespace
         # All arguments after the first `--` go to the `git diff` command.
         diff_cmd = f"diff2html -s line -f html -d word -i command --hwt " \
                    f"{tmp_file.name} -F {OUTPUT_PATH}/{dest_path} -- " \
-                   f"-U20 {ws_flag} {BASE_BRANCH_NAME} {HEAD_BRANCH_NAME} -- {path_to_diff}"
+                   f"-U20 {ws_flag} {opts.base_branch} {opts.head_branch} -- {path_to_diff}"
         eprint(f"Running diff cmd: {diff_cmd}")
         run(diff_cmd)
     return dest_path
@@ -169,29 +169,46 @@ def diff_link(diff_text, empty_diff_text, diff_location, alternate_text, alterna
     return f"[{diff_text}]({CDN_URL}/codegen-diff/{diff_location}) ([{alternate_text}]({CDN_URL}/codegen-diff/{alternate_location}))"
 
 
-def make_diffs(base_sha, head_sha):
-    sdk_ws = make_diff('AWS SDK', f'{OUTPUT_PATH}/services', base_sha, head_sha, 'aws-sdk', ignore_whitespace=False)
-    sdk_no_ws = make_diff('AWS SDK', f'{OUTPUT_PATH}/services', base_sha, head_sha, 'aws-sdk-ignore-ws',
-                          ignore_whitespace=True)
-
+def make_diffs(opts):
+    sdk_ws = make_diff(opts, 'AWS SDK', f'{OUTPUT_PATH}/services', 'aws-sdk', ignore_whitespace=False)
+    sdk_no_ws = make_diff(opts, 'AWS SDK', f'{OUTPUT_PATH}/services', 'aws-sdk-ignore-ws', ignore_whitespace=True)
     sdk_links = diff_link('AWS SDK', 'No codegen difference in the AWS SDK', sdk_ws, 'ignoring whitespace', sdk_no_ws)
 
     return f'A new generated diff is ready to view.\\n\\n- {sdk_links}\\n'
 
 
+def _codegen_cmd(opts):
+    generate_and_commit_generated_code(opts.head_sha, opts.bootstrap)
+
+
+def _generate_diffs_cmd(opts):
+    bot_message = make_diffs(opts)
+    with open(f"{OUTPUT_PATH}/bot-message", 'w') as f:
+        f.write(bot_message)
+
+
 def create_cli():
     parser = argparse.ArgumentParser(
         prog="codegen-diff-revisions",
         description="Generate HTML diffs of codegen output",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
-    parser.add_argument("repo_root", help="repository root")
-    parser.add_argument("base_sha", help="base commit to diff against (SHA-like)")
-    parser.add_argument("--bootstrap", help="services to pass to bootstrap and include in diff output",
-                        default="+dynamodb,+codebuild,+sts,+ec2,+polly,+s3")
     parser.add_argument("--head-sha", help="head commit to use (defaults to whatever current HEAD) is")
 
+    subparsers = parser.add_subparsers()
+    codegen = subparsers.add_parser("codegen", help="generate and commit generated code")
+    codegen.add_argument("--bootstrap", help="services to pass to bootstrap and include in diff output",
+                         default="+dynamodb,+codebuild,+sts,+ec2,+polly,+s3")
+    codegen.set_defaults(cmd=_codegen_cmd)
+
+    generate_diffs = subparsers.add_parser("generate-diffs",
+                                           help="generate diffs between two branches and output bot message")
+    generate_diffs.add_argument("--base-sha", help="base commit to diff against (SHA-like)")
+    generate_diffs.add_argument("base_branch", help="name of the base branch to diff against")
+    generate_diffs.add_argument("head_branch", help="name of the head branch to diff against")
+    generate_diffs.set_defaults(cmd=_generate_diffs_cmd)
+
     return parser
 
 
@@ -200,39 +217,12 @@ def main():
     opts = cli.parse_args()
     print(opts)
 
-    os.chdir(opts.repo_root)
-
     if opts.head_sha is None:
-        head_sha = get_cmd_output("git rev-parse HEAD")
-    else:
-        head_sha = opts.head_sha
-
-    print(f"using head sha is {head_sha}")
-
-    # Make sure the working tree is clean
-    if get_cmd_status("git diff --quiet") != 0:
-        eprint("working tree is not clean. aborting")
-        sys.exit(1)
+        opts.head_sha = get_cmd_output("git rev-parse HEAD")
 
-    # Generate code for HEAD
-    print(f"Creating temporary branch with generated code for the HEAD revision {head_sha}")
-    run(f"git checkout {head_sha} -b {HEAD_BRANCH_NAME}")
-    generate_and_commit_generated_code(head_sha, opts.bootstrap)
-
-    # Generate code for base
-    print(f"Creating temporary branch with generated code for the base revision {opts.base_sha}")
-    run(f"git checkout {opts.base_sha} -b {BASE_BRANCH_NAME}")
-    generate_and_commit_generated_code(opts.base_sha, opts.bootstrap)
-
-    bot_message = make_diffs(opts.base_sha, head_sha)
-    with open(f"{OUTPUT_PATH}/bot-message", 'w') as f:
-        f.write(bot_message)
+    print(f"using head sha: {opts.head_sha}")
 
-    # cleanup
-    if not running_in_github_action():
-        run(f"git checkout main")
-        run(f"git branch -D {BASE_BRANCH_NAME}")
-        run(f"git branch -D {HEAD_BRANCH_NAME}")
+    opts.cmd(opts)
 
 
 if __name__ == '__main__':

.github/workflows/codegen-preview.yml

@@ -31,6 +31,8 @@ env:
   #  - @restJson1: polly
   #  - @restXml: s3
   PREVIEW_SERVICES: +dynamodb,+codebuild,+sts,+ec2,+polly,+s3
+  HEAD_BRANCH_NAME: __tmp-localonly-head
+  BASE_BRANCH_NAME: __tmp-localonly-base
 
 jobs:
 
@@ -71,63 +73,80 @@ jobs:
         run: |
           npm install -g diff2html-cli@${{ env.DIFF2HTML_VERSION }}
           env | sort
-      - name: Generate diff for head ref
+      - name: Generate code for head ref
         run: |
           REPO_TOOLS=$GITHUB_WORKSPACE/aws-kotlin-repo-tools
           SMITHY_KOTLIN_DIR=$GITHUB_WORKSPACE/smithy-kotlin
           SDK_DIR=$GITHUB_WORKSPACE/aws-sdk-kotlin
           branch=$(python3 $REPO_TOOLS/scripts/ci.py get-branch $SDK_DIR)
           echo "using branch $branch"
-          python3 $REPO_TOOLS/scripts/ci.py set-branch --branch branch $SMITHY_KOTLIN_DIR
+          python3 $REPO_TOOLS/scripts/ci.py set-branch --branch $branch $SMITHY_KOTLIN_DIR
+          pushd $SDK_DIR
+          git checkout -b $HEAD_BRANCH_NAME
+          .github/scripts/codegen-diff-revisions.py codegen --bootstrap ${{ env.PREVIEW_SERVICES }}
+          popd
 
-#      - name: Generate diff
-#        id: generate-diff
-#        run: |
-#          # codegen-diff-revisions requires a clean index, set-upstream-versions.py can modify local repo state
-#          # we don't push these branches/commits anywhere so just commit it if necessary and move on
-#          if ! git diff --quiet gradle.properties
-#          then
-#            echo "gradle.properties is dirty, committing before generating diffs"
-#            git add gradle.properties
-#            PRE_COMMIT_ALLOW_NO_CONFIG=1 git \
-#            -c "user.name=GitHub Action (generated code preview)" \
-#            -c "[email protected]" \
-#            commit -m "codegen diff autocommit" --allow-empty
-#          fi
-#          .github/scripts/codegen-diff-revisions.py \
-#            --bootstrap ${{ env.PREVIEW_SERVICES }} \
-#            --head-sha ${{ github.event.pull_request.head.sha }} \
-#            . ${{ github.event.pull_request.base.sha }}
-#          echo "codegen-diff-msg<<EOF" >> $GITHUB_OUTPUT
-#          cat ./tmp-codegen-diff/bot-message) >> $GITHUB_OUTPUT
-#          echo "EOF" >> $GITHUB_OUTPUT
-#      - name: Configure AWS Credentials
-#        uses: aws-actions/configure-aws-credentials@v2
-#        with:
-#          role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
-#          aws-region: us-west-2
-#      - name: Upload diff to S3
-#        run: |
-#          if [[ -d ./tmp-codegen-diff/${{ github.event.pull_request.base.sha }} ]]; then
-#            aws s3 cp ./tmp-codegen-diff/${{ github.event.pull_request.base.sha }} \
-#                "s3://${{ secrets.CDN_S3_BUCKET_NAME }}/codegen-diff/${{ github.event.pull_request.base.sha }}" --recursive
-#          fi
+      - name: Generate code for base ref
+        run: |
+          REPO_TOOLS=$GITHUB_WORKSPACE/aws-kotlin-repo-tools
+          SMITHY_KOTLIN_DIR=$GITHUB_WORKSPACE/smithy-kotlin
+          SDK_DIR=$GITHUB_WORKSPACE/aws-sdk-kotlin
+          branch=$GITHUB_BASE_REF
+          echo "checkout smithy-kotlin at $branch"
+          pushd $SMITHY_KOTLIN_DIR
+          git switch -f main
+          python3 $REPO_TOOLS/scripts/ci.py set-branch --branch $branch
+          popd
+          echo "resetting aws-sdk-kotlin"
+          pushd $SDK_DIR
+          git switch -f main
+          python3 $REPO_TOOLS/scripts/ci.py set-branch --branch $branch 
+          git checkout -b $BASE_BRANCH_NAME
+          .github/scripts/codegen-diff-revisions.py codegen --bootstrap ${{ env.PREVIEW_SERVICES }}
+          popd
+
+      - name: Generate diffs
+        run: |
+          SDK_DIR=$GITHUB_WORKSPACE/aws-sdk-kotlin
+          pushd $SDK_DIR
+          .github/scripts/codegen-diff-revisions.py generate-diffs \
+            --head-sha ${{ github.event.pull_request.head.sha }} \
+            --base-sha ${{ github.event.pull_request.base.sha }} \
+            $BASE_BRANCH_NAME $HEAD_BRANCH_NAME
+          echo "codegen-diff-msg<<EOF" >> $GITHUB_OUTPUT
+          cat ./tmp-codegen-diff/bot-message) >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          popd
 
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
+          aws-region: us-west-2
+
+      - name: Upload diff to S3
+        run: |
+          SDK_DIR=$GITHUB_WORKSPACE/aws-sdk-kotlin
+          pushd $SDK_DIR
+          if [[ -d ./tmp-codegen-diff/${{ github.event.pull_request.base.sha }} ]]; then
+            aws s3 cp ./tmp-codegen-diff/${{ github.event.pull_request.base.sha }} \
+                "s3://${{ secrets.CDN_S3_BUCKET_NAME }}/codegen-diff/${{ github.event.pull_request.base.sha }}" --recursive
+          fi
 
   # TODO - generate doc preview for N services and upload and link as well
-#  post-bot-comment:
-#    name: Post bot comment
-#    runs-on: ubuntu-latest
-#    needs:
-#      - generate-codegen-diff
-#    steps:
-#      - name: Post bot comment
-#        uses: actions/github-script@v5
-#        with:
-#          script: |
-#            await github.rest.issues.createComment({
-#              issue_number: ${{ github.event.number }},
-#              owner: context.repo.owner,
-#              repo: context.repo.repo,
-#              body: '${{ needs.generate-codegen-diff.outputs.bot-message }}\n\n'
-#            })
+  post-bot-comment:
+    name: Post bot comment
+    runs-on: ubuntu-latest
+    needs:
+      - generate-codegen-diff
+    steps:
+      - name: Post bot comment
+        uses: actions/github-script@v5
+        with:
+          script: |
+            await github.rest.issues.createComment({
+              issue_number: ${{ github.event.number }},
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: '${{ needs.generate-codegen-diff.outputs.bot-message }}\n\n'
+            })