refactor: split AWS SDK specific customizations into new module (#1151)

First step to separating AWS protocol support from AWS SDK specific customizations. Specifically we introduce a new module aws-sdk-codegen which is everything AWS SDK for Kotlin specific. The existing smithy-aws-kotlin-codegen now only contains AWS protocol support. refactor: Split out AWS SDK for Kotlin specific customizations into a new module aws-sdk-codegen refactor: Use the software.amazon.smithy.kotlin.codegen.aws namespace for AWS protocol support instead of aws.sdk.kotlin.codegen to better align with the rest of the code generation in smithy-kotlin refactor: Remove S3Generator in favor of overriding sections fix: Fix route53 customization to only apply to the operation it is meant to refactor: Migrate flow utils to smithy-kotlin runtime.
awslabs · Dec 15, 2023 · 1e99683 · 1e99683
1 parent 0cfd96a
commit 1e99683
Show file tree

Hide file tree

Showing 119 changed files with 824 additions and 1,247 deletions.
diff --git a/.github/scripts/codegen-diff-revisions.py b/.github/scripts/codegen-diff-revisions.py
@@ -5,24 +5,45 @@
 #
 # This script can be run and tested locally. To do so, you should check out
 # a second aws-sdk-kotlin repository so that you can work on the script and still
-# run it without it immediately bailing for an unclean working tree (and to avoid creating
-# temporary branches).
+# run it without it immediately bailing for an unclean working tree (and to avoid mutating
+# your local repository).
 #
 # Example:
-# `aws-sdk-kotlin/` - the main repo you're working out of
+# `aws-sdk-kotlin/` - the main repo you're working out of <path-to-working-dir> below
 # `/tmp/aws-sdk-kotlin/` - the repo you're testing against
 #
+# Generate code using the current HEAD of the repository.
+
 # ```
-# $ cd test/aws-sdk-kotlin
-# $ ../../aws-sdk-kotlin/.github/scripts/codegen-diff-revisions.py . <some commit hash to diff against>
+# $ cd /tmp/aws-sdk-kotlin
+# $ <path-to-working-dir>/.github/scripts/codegen-diff-revisions.py codegen --bootstrap +s3,+dynamodb
 # ```
 #
-# It will diff the generated code from HEAD against any commit hash you feed it. If you want to test
-# a specific range, change the HEAD of the test repository.
 #
-# This script requires `diff2html-cli` to be installed from NPM:
+# Generate diffs
+#
 # ```
-# $ npm install -g [email protected]
+# $ cd /tmp/aws-sdk-kotlin
+# HEAD_SHA=$(git rev-parse <my-head-sha>)
+# BASE_SHA=$(git rev-parse <my-base-branch>)
+# HEAD_BRANCH_NAME="__tmp-localonly-head"
+# BASE_BRANCH_NAME="__tmp-localonly-base"
+#
+# <path-to-working-dir>/.github/scripts/codegen-diff-revisions.py --head-sha $HEAD_SHA generate-diffs --base-sha $BASE_SHA $BASE_BRANCH_NAME $HEAD_BRANCH_NAME
+# ```
+#
+# This script requires `difftags` to be installed from the smithy-rs repository. This requires installing a working
+# rust toolchain (cargo, rustc, etc).
+# ```
+# curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
+# ```
+#
+# Install difftags from smithy-rs
+# ```
+# git clone --depth 1 https://github.com/smithy-lang/smithy-rs.git
+# cd smithy-rs/tools/ci-build/difftags
+# cargo install --path .
+# difftags -h
 # ```
 # Make sure the local version matches the version referenced from the GitHub Actions workflow.
 
@@ -33,7 +54,7 @@
 import tempfile
 import shlex
 
-OUTPUT_PATH = "tmp-codegen-diff/"
+OUTPUT_PATH = "tmp-codegen-diff"
 
 COMMIT_AUTHOR_NAME = "GitHub Action (generated codegen diff)"
 COMMIT_AUTHOR_EMAIL = "[email protected]"
@@ -103,40 +124,7 @@ def generate_and_commit_generated_code(sha, services_to_bootstrap):
     run(f"git -c 'user.name={COMMIT_AUTHOR_NAME}' -c 'user.email={COMMIT_AUTHOR_EMAIL}' commit --no-verify -m 'Generated code for {sha}' --allow-empty")
 
 
-# Writes an HTML template for diff2html so that we can add contextual information
-def write_html_template(title, subtitle, tmp_file):
-    tmp_file.writelines(map(lambda line: line.encode(), [
-        "<!doctype html>",
-        "<html>",
-        "<head>",
-        '  <metadata charset="utf-8">',
-        f'  <title>Codegen diff for the {title}: {subtitle}</title>',
-        '  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.6.0/styles/github.min.css" / >',
-        '  <!--diff2html-css-->',
-        '  <!--diff2html-js-ui-->',
-        '  <script>',
-        '  document.addEventListener("DOMContentLoaded", () => {',
-        '    const targetElement = document.getElementById("diff");',
-        '    const diff2htmlUi = new Diff2HtmlUI(targetElement);',
-        '    //diff2html-fileListToggle',
-        '    //diff2html-synchronisedScroll',
-        '    //diff2html-highlightCode',
-        '  });',
-        '  </script>',
-        "</head>",
-        '<body style="font-family: sans-serif;">',
-        f"  <h1>Codegen diff for the {title}</h1>",
-        f"  <p>{subtitle}</p>",
-        '  <div id="diff">',
-        '    <!--diff2html-diff-->',
-        '  </div>',
-        "</body>",
-        "</html>",
-    ]))
-    tmp_file.flush()
-
-
-def make_diff(opts, title, path_to_diff, suffix, ignore_whitespace):
+def make_diff(opts, title, path_to_diff, outsubdir, ignore_whitespace):
     base_sha = opts.base_sha
     head_sha = opts.head_sha
     ws_flag = "-b" if ignore_whitespace else ""
@@ -146,32 +134,36 @@ def make_diff(opts, title, path_to_diff, suffix, ignore_whitespace):
         eprint(f"No diff output for {base_sha}..{head_sha}")
         return None
 
-    run(f'mkdir -p {OUTPUT_PATH}/{base_sha}/{head_sha}')
-    dest_path = f"{base_sha}/{head_sha}/diff-{suffix}.html"
+    dest_path = f"{base_sha}/{head_sha}/{outsubdir}"
+    run(f'mkdir -p {OUTPUT_PATH}/{dest_path}')
     whitespace_context = "(ignoring whitespace)" if ignore_whitespace else ""
-    with tempfile.NamedTemporaryFile() as tmp_file:
-        write_html_template(title, f"rev. {head_sha} {whitespace_context}", tmp_file)
-
-        # Generate HTML diff. This uses the diff2html-cli, which defers to `git diff` under the hood.
-        # All arguments after the first `--` go to the `git diff` command.
-        diff_cmd = f"diff2html -s line -f html -d word -i command --hwt " \
-                   f"{tmp_file.name} -F {OUTPUT_PATH}/{dest_path} -- " \
-                   f"-U20 {ws_flag} {opts.base_branch} {opts.head_branch} -- {path_to_diff}"
+
+    with tempfile.NamedTemporaryFile(mode='w') as tmp_file:
+        diff_cmd = f"git diff -U30 {ws_flag} {opts.base_branch} {opts.head_branch} -- {path_to_diff}"
         eprint(f"Running diff cmd: {diff_cmd}")
-        run(diff_cmd)
+        output = get_cmd_output(diff_cmd)
+        tmp_file.write(output)
+
+        subtitle = f"rev. {base_sha}..{head_sha} {whitespace_context}"
+        generate_html_cmd = f"difftags --title \"{title}\" --subtitle \"{subtitle}\" --output-dir {OUTPUT_PATH}/{dest_path} {tmp_file.name}"
+        eprint(f"Running generate html cmd: {generate_html_cmd}")
+        run(generate_html_cmd)
+
     return dest_path
 
 
 def diff_link(diff_text, empty_diff_text, diff_location, alternate_text, alternate_location):
     if diff_location is None:
         return empty_diff_text
 
-    return f"[{diff_text}]({CDN_URL}/codegen-diff/{diff_location}) ([{alternate_text}]({CDN_URL}/codegen-diff/{alternate_location}))"
+    return f"[{diff_text}]({CDN_URL}/codegen-diff/{diff_location}/index.html) ([{alternate_text}]({CDN_URL}/codegen-diff/{alternate_location}/index.html))"
 
 
 def make_diffs(opts):
-    sdk_ws = make_diff(opts, 'AWS SDK', f'{OUTPUT_PATH}/services', 'aws-sdk', ignore_whitespace=False)
-    sdk_no_ws = make_diff(opts, 'AWS SDK', f'{OUTPUT_PATH}/services', 'aws-sdk-ignore-ws', ignore_whitespace=True)
+    path_to_diff = f"{OUTPUT_PATH}/services"
+
+    sdk_ws = make_diff(opts, 'AWS SDK', path_to_diff, "aws-sdk", ignore_whitespace=False)
+    sdk_no_ws = make_diff(opts, 'AWS SDK', path_to_diff, "aws-sdk-ignore-ws", ignore_whitespace=True)
     sdk_links = diff_link('AWS SDK', 'No codegen difference in the AWS SDK', sdk_ws, 'ignoring whitespace', sdk_no_ws)
 
     return f'A new generated diff is ready to view.\\n\\n- {sdk_links}\\n'

diff --git a/.github/workflows/codegen-preview.yml b/.github/workflows/codegen-preview.yml
@@ -55,6 +55,12 @@ jobs:
           repository: 'awslabs/smithy-kotlin'
           fetch-depth: 0
           path: 'smithy-kotlin'
+      - uses: actions/checkout@v4
+        with:
+          repository: 'smithy-lang/smithy-rs'
+          path: 'smithy-rs'
+          sparse-checkout: | 
+            tools/ci-build/difftags
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:
@@ -63,15 +69,18 @@ jobs:
           cache: 'gradle'
       - name: Install deps and setup environment
         run: |
-          npm install -g diff2html-cli@${{ env.DIFF2HTML_VERSION }}
-          env | sort
           # store off a copy of head ref of ci.py, otherwise base ref generation will use a different version of this script
           CODEGEN_DIFF_REVISIONS=${{ runner.temp }}/codegen-diff-revisions.py
           cp $GITHUB_WORKSPACE/aws-sdk-kotlin/.github/scripts/codegen-diff-revisions.py $CODEGEN_DIFF_REVISIONS
           echo "CODEGEN_DIFF_REVISIONS=$CODEGEN_DIFF_REVISIONS" >> "$GITHUB_ENV"
           echo "REPO_TOOLS=$GITHUB_WORKSPACE/aws-kotlin-repo-tools" >> "$GITHUB_ENV"
           echo "SMITHY_KOTLIN_DIR=$GITHUB_WORKSPACE/smithy-kotlin" >> "$GITHUB_ENV"
           echo "SDK_DIR=$GITHUB_WORKSPACE/aws-sdk-kotlin" >> "$GITHUB_ENV"
+          echo "installing smithy-rs diff tooling"
+          pushd $GITHUB_WORKSPACE/smithy-rs/tools/ci-build/difftags
+          cargo install --path .
+          difftags --version
+          popd
 
       - name: Generate code for head ref
         run: |

diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts
@@ -101,7 +101,7 @@ codegen {
             transforms = listOf(
                 """
                 {
-                    "name": "awsSdkKotlinIncludeOperations",
+                    "name": "awsSmithyKotlinIncludeOperations",
                     "args": {
                         "operations": [
                             "com.amazonaws.sts#AssumeRole",
@@ -138,7 +138,7 @@ codegen {
             transforms = listOf(
                 """
                 {
-                    "name": "awsSdkKotlinIncludeOperations",
+                    "name": "awsSmithyKotlinIncludeOperations",
                     "args": {
                         "operations": [
                             "com.amazonaws.sso#GetRoleCredentials"
@@ -173,7 +173,7 @@ codegen {
             transforms = listOf(
                 """
             {
-                "name": "awsSdkKotlinIncludeOperations",
+                "name": "awsSmithyKotlinIncludeOperations",
                 "args": {
                     "operations": [
                         "com.amazonaws.ssooidc#CreateToken"

diff --git a/aws-runtime/aws-core/api/aws-core.api b/aws-runtime/aws-core/api/aws-core.api
@@ -26,9 +26,6 @@ public class aws/sdk/kotlin/runtime/ConfigurationException : aws/sdk/kotlin/runt
 	public fun <init> (Ljava/lang/Throwable;)V
 }
 
-public final class aws/sdk/kotlin/runtime/FlowUtilKt {
-}
-
 public abstract interface annotation class aws/sdk/kotlin/runtime/InternalSdkApi : java/lang/annotation/Annotation {
 }
 

diff --git a/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/FlowUtil.kt b/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/FlowUtil.kt
diff --git a/aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/FlowUtilTest.kt b/aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/FlowUtilTest.kt
-Original file line number
+Diff line change
@@ Expand Up @@
     	public fun <init> (Ljava/lang/Throwable;)V
     }
-    public final class aws/sdk/kotlin/runtime/FlowUtilKt {
-    }
     public abstract interface annotation class aws/sdk/kotlin/runtime/InternalSdkApi : java/lang/annotation/Annotation {
     }
@@ Expand Down @@