diff --git a/.github/scripts/codegen-diff-revisions.py b/.github/scripts/codegen-diff-revisions.py index 871be4017ff..eda9877bbb6 100755 --- a/.github/scripts/codegen-diff-revisions.py +++ b/.github/scripts/codegen-diff-revisions.py @@ -33,8 +33,6 @@ import tempfile import shlex -HEAD_BRANCH_NAME = "__tmp-localonly-head" -BASE_BRANCH_NAME = "__tmp-localonly-base" OUTPUT_PATH = "tmp-codegen-diff/" COMMIT_AUTHOR_NAME = "GitHub Action (generated codegen diff)" @@ -138,9 +136,11 @@ def write_html_template(title, subtitle, tmp_file): tmp_file.flush() -def make_diff(title, path_to_diff, base_sha, head_sha, suffix, ignore_whitespace): +def make_diff(opts, title, path_to_diff, suffix, ignore_whitespace): + base_sha = opts.base_sha + head_sha = opts.head_sha ws_flag = "-b" if ignore_whitespace else "" - diff_exists = get_cmd_status(f"git diff --quiet {ws_flag} {BASE_BRANCH_NAME} {HEAD_BRANCH_NAME} -- {path_to_diff}") + diff_exists = get_cmd_status(f"git diff --quiet {ws_flag} {opts.base_branch} {opts.head_branch} -- {path_to_diff}") if diff_exists == 0: eprint(f"No diff output for {base_sha}..{head_sha}") @@ -156,7 +156,7 @@ def make_diff(title, path_to_diff, base_sha, head_sha, suffix, ignore_whitespace # All arguments after the first `--` go to the `git diff` command. diff_cmd = f"diff2html -s line -f html -d word -i command --hwt " \ f"{tmp_file.name} -F {OUTPUT_PATH}/{dest_path} -- " \ - f"-U20 {ws_flag} {BASE_BRANCH_NAME} {HEAD_BRANCH_NAME} -- {path_to_diff}" + f"-U20 {ws_flag} {opts.base_branch} {opts.head_branch} -- {path_to_diff}" eprint(f"Running diff cmd: {diff_cmd}") run(diff_cmd) return dest_path @@ -169,16 +169,24 @@ def diff_link(diff_text, empty_diff_text, diff_location, alternate_text, alterna return f"[{diff_text}]({CDN_URL}/codegen-diff/{diff_location}) ([{alternate_text}]({CDN_URL}/codegen-diff/{alternate_location}))" -def make_diffs(base_sha, head_sha): - sdk_ws = make_diff('AWS SDK', f'{OUTPUT_PATH}/services', base_sha, head_sha, 'aws-sdk', ignore_whitespace=False) - sdk_no_ws = make_diff('AWS SDK', f'{OUTPUT_PATH}/services', base_sha, head_sha, 'aws-sdk-ignore-ws', - ignore_whitespace=True) - +def make_diffs(opts): + sdk_ws = make_diff(opts, 'AWS SDK', f'{OUTPUT_PATH}/services', 'aws-sdk', ignore_whitespace=False) + sdk_no_ws = make_diff(opts, 'AWS SDK', f'{OUTPUT_PATH}/services', 'aws-sdk-ignore-ws', ignore_whitespace=True) sdk_links = diff_link('AWS SDK', 'No codegen difference in the AWS SDK', sdk_ws, 'ignoring whitespace', sdk_no_ws) return f'A new generated diff is ready to view.\\n\\n- {sdk_links}\\n' +def _codegen_cmd(opts): + generate_and_commit_generated_code(opts.head_sha, opts.bootstrap) + + +def _generate_diffs_cmd(opts): + bot_message = make_diffs(opts) + with open(f"{OUTPUT_PATH}/bot-message", 'w') as f: + f.write(bot_message) + + def create_cli(): parser = argparse.ArgumentParser( prog="codegen-diff-revisions", @@ -186,12 +194,21 @@ def create_cli(): formatter_class=argparse.ArgumentDefaultsHelpFormatter ) - parser.add_argument("repo_root", help="repository root") - parser.add_argument("base_sha", help="base commit to diff against (SHA-like)") - parser.add_argument("--bootstrap", help="services to pass to bootstrap and include in diff output", - default="+dynamodb,+codebuild,+sts,+ec2,+polly,+s3") parser.add_argument("--head-sha", help="head commit to use (defaults to whatever current HEAD) is") + subparsers = parser.add_subparsers() + codegen = subparsers.add_parser("codegen", help="generate and commit generated code") + codegen.add_argument("--bootstrap", help="services to pass to bootstrap and include in diff output", + default="+dynamodb,+codebuild,+sts,+ec2,+polly,+s3") + codegen.set_defaults(cmd=_codegen_cmd) + + generate_diffs = subparsers.add_parser("generate-diffs", + help="generate diffs between two branches and output bot message") + generate_diffs.add_argument("--base-sha", help="base commit to diff against (SHA-like)") + generate_diffs.add_argument("base_branch", help="name of the base branch to diff against") + generate_diffs.add_argument("head_branch", help="name of the head branch to diff against") + generate_diffs.set_defaults(cmd=_generate_diffs_cmd) + return parser @@ -200,39 +217,12 @@ def main(): opts = cli.parse_args() print(opts) - os.chdir(opts.repo_root) - if opts.head_sha is None: - head_sha = get_cmd_output("git rev-parse HEAD") - else: - head_sha = opts.head_sha - - print(f"using head sha is {head_sha}") - - # Make sure the working tree is clean - if get_cmd_status("git diff --quiet") != 0: - eprint("working tree is not clean. aborting") - sys.exit(1) + opts.head_sha = get_cmd_output("git rev-parse HEAD") - # Generate code for HEAD - print(f"Creating temporary branch with generated code for the HEAD revision {head_sha}") - run(f"git checkout {head_sha} -b {HEAD_BRANCH_NAME}") - generate_and_commit_generated_code(head_sha, opts.bootstrap) - - # Generate code for base - print(f"Creating temporary branch with generated code for the base revision {opts.base_sha}") - run(f"git checkout {opts.base_sha} -b {BASE_BRANCH_NAME}") - generate_and_commit_generated_code(opts.base_sha, opts.bootstrap) - - bot_message = make_diffs(opts.base_sha, head_sha) - with open(f"{OUTPUT_PATH}/bot-message", 'w') as f: - f.write(bot_message) + print(f"using head sha: {opts.head_sha}") - # cleanup - if not running_in_github_action(): - run(f"git checkout main") - run(f"git branch -D {BASE_BRANCH_NAME}") - run(f"git branch -D {HEAD_BRANCH_NAME}") + opts.cmd(opts) if __name__ == '__main__': diff --git a/.github/workflows/codegen-preview.yml b/.github/workflows/codegen-preview.yml index ed9b9963118..17329dac0a5 100644 --- a/.github/workflows/codegen-preview.yml +++ b/.github/workflows/codegen-preview.yml @@ -20,10 +20,6 @@ concurrency: env: JAVA_VERSION: 11 - BUILDER_VERSION: v0.8.22 - BUILDER_SOURCE: releases - BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net - PACKAGE_NAME: aws-sdk-kotlin RUN: ${{ github.run_id }}-${{ github.run_number }} DIFF2HTML_VERSION: 5.2.5 # Below is the set of services that are generated for codegen preview @@ -35,6 +31,8 @@ env: # - @restJson1: polly # - @restXml: s3 PREVIEW_SERVICES: +dynamodb,+codebuild,+sts,+ec2,+polly,+s3 + HEAD_BRANCH_NAME: __tmp-localonly-head + BASE_BRANCH_NAME: __tmp-localonly-base jobs: @@ -44,9 +42,20 @@ jobs: outputs: bot-message: ${{ steps.generate-diff.outputs.codegen-diff-msg }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 + path: 'aws-sdk-kotlin' + - uses: actions/checkout@v4 + with: + repository: 'awslabs/aws-kotlin-repo-tools' + ref: 'ci-utils' + path: 'aws-kotlin-repo-tools' + - uses: actions/checkout@v4 + with: + repository: 'awslabs/smithy-kotlin' + fetch-depth: 0 + path: 'smithy-kotlin' - uses: actions/cache@v2 name: Gradle Cache with: @@ -60,47 +69,87 @@ jobs: uses: actions/setup-java@v1 with: java-version: ${{ env.JAVA_VERSION }} - - name: Install and build deps - # abuse crt builder to download and build upstream dependencies like smithy-kotlin + + - name: Install deps and setup environment run: | npm install -g diff2html-cli@${{ env.DIFF2HTML_VERSION }} - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - chmod a+x builder.pyz - ./builder.pyz build -p ${{ env.PACKAGE_NAME }} --variant codegen-preview - - name: Generate diff - id: generate-diff + env | sort + # store off a copy of head ref of ci.py, otherwise base ref generation will use a different version of this script + CODEGEN_DIFF_REVISIONS=${{ runner.temp }}/codegen-diff-revisions.py + cp $GITHUB_WORKSPACE/aws-sdk-kotlin/.github/scripts/codegen-diff-revisions.py $CODEGEN_DIFF_REVISIONS + echo "CODEGEN_DIFF_REVISIONS=$CODEGEN_DIFF_REVISIONS" >> "$GITHUB_ENV" + echo "REPO_TOOLS=$GITHUB_WORKSPACE/aws-kotlin-repo-tools" >> "$GITHUB_ENV" + echo "SMITHY_KOTLIN_DIR=$GITHUB_WORKSPACE/smithy-kotlin" >> "$GITHUB_ENV" + echo "SDK_DIR=$GITHUB_WORKSPACE/aws-sdk-kotlin" >> "$GITHUB_ENV" + + - name: Generate code for head ref run: | - # codegen-diff-revisions requires a clean index, set-upstream-versions.py can modify local repo state - # we don't push these branches/commits anywhere so just commit it if necessary and move on - if ! git diff --quiet gradle.properties + branch=$(python3 $REPO_TOOLS/scripts/ci.py get-branch $SDK_DIR) + pushd $SMITHY_KOTLIN_DIR + if git ls-remote --quiet --exit-code --heads origin refs/heads/$branch then - echo "gradle.properties is dirty, committing before generating diffs" - git add gradle.properties - PRE_COMMIT_ALLOW_NO_CONFIG=1 git \ - -c "user.name=GitHub Action (generated code preview)" \ - -c "user.email=generated-code-action@github.com" \ - commit -m "codegen diff autocommit" --allow-empty + echo "using smithy-kotlin branch $branch for head ref codegen" + else + # in the case of feature branches with tentacles we may or may not have a corresponding base ref in smithy-kotlin + # if we do let's use that, otherwise fallback to main + echo "$branch does not exist in smithy-kotlin, will attempt base ref $GITHUB_BASE_REF" + branch=$GITHUB_BASE_REF fi - .github/scripts/codegen-diff-revisions.py \ - --bootstrap ${{ env.PREVIEW_SERVICES }} \ + python3 $REPO_TOOLS/scripts/ci.py -v set-branch --branch $branch + popd + pushd $SDK_DIR + git checkout -b $HEAD_BRANCH_NAME + $CODEGEN_DIFF_REVISIONS codegen --bootstrap ${{ env.PREVIEW_SERVICES }} + popd + + - name: Generate code for base ref + run: | + branch=$GITHUB_BASE_REF + echo "checkout smithy-kotlin at base ref: $branch" + pushd $SMITHY_KOTLIN_DIR + git switch -f main + python3 $REPO_TOOLS/scripts/ci.py -v set-branch --branch $branch + popd + echo "resetting aws-sdk-kotlin" + pushd $SDK_DIR + git switch -f main + python3 $REPO_TOOLS/scripts/ci.py -v set-branch --branch $branch + git checkout -b $BASE_BRANCH_NAME + $CODEGEN_DIFF_REVISIONS codegen --bootstrap ${{ env.PREVIEW_SERVICES }} + popd + + - name: Generate diffs + id: generate-diff + run: | + pushd $SDK_DIR + $CODEGEN_DIFF_REVISIONS \ --head-sha ${{ github.event.pull_request.head.sha }} \ - . ${{ github.event.pull_request.base.sha }} - echo "codegen-diff-msg<> $GITHUB_OUTPUT - cat ./tmp-codegen-diff/bot-message) >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT + generate-diffs \ + --base-sha ${{ github.event.pull_request.base.sha }} \ + $BASE_BRANCH_NAME $HEAD_BRANCH_NAME + cat ./tmp-codegen-diff/bot-message + { + echo 'codegen-diff-msg<> "$GITHUB_OUTPUT" + echo $GITHUB_OUTPUT + - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} aws-region: us-west-2 + - name: Upload diff to S3 run: | + SDK_DIR=$GITHUB_WORKSPACE/aws-sdk-kotlin + pushd $SDK_DIR if [[ -d ./tmp-codegen-diff/${{ github.event.pull_request.base.sha }} ]]; then aws s3 cp ./tmp-codegen-diff/${{ github.event.pull_request.base.sha }} \ "s3://${{ secrets.CDN_S3_BUCKET_NAME }}/codegen-diff/${{ github.event.pull_request.base.sha }}" --recursive fi - # TODO - generate doc preview for N services and upload and link as well post-bot-comment: name: Post bot comment @@ -117,4 +166,4 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, body: '${{ needs.generate-codegen-diff.outputs.bot-message }}\n\n' - }) + }) \ No newline at end of file diff --git a/builder.json b/builder.json index c82d45657c4..3e77b666dca 100644 --- a/builder.json +++ b/builder.json @@ -57,11 +57,6 @@ "!test_steps": [ "{gradlew} test allTests" ] - }, - "codegen-preview": { - "!imports": [], - "!build_steps": [], - "!test_steps": [] } } }