From 15e6cc5f9b406f234a29d866f51d37cb75dff9a0 Mon Sep 17 00:00:00 2001 From: Aaron Todd Date: Wed, 11 Oct 2023 10:37:47 -0400 Subject: [PATCH] chore: refactor ci workflows (#974) --- .github/workflows/continuous-integration.yml | 147 ++++++++++-------- build.gradle.kts | 29 ++-- .../tests/SigningSuiteTestBaseJVM.kt | 6 +- runtime/build.gradle.kts | 6 + settings.gradle.kts | 8 + 5 files changed, 120 insertions(+), 76 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 663932a83..65f4b5bff 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -4,87 +4,104 @@ on: push: branches: [ main ] pull_request: - branches: - - main - - 'feat-*' workflow_dispatch: +# Allow one instance of this workflow per pull request, and cancel older runs when new changes are pushed +concurrency: + group: ci-pr-${{ github.ref }} + cancel-in-progress: true + env: - BUILDER_VERSION: v0.8.22 - BUILDER_SOURCE: releases - # host owned by CRT team to host aws-crt-builder releases. Contact their on-call with any issues - BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net - PACKAGE_NAME: smithy-kotlin - LINUX_BASE_IMAGE: ubuntu-16-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} + GRADLE_OPTS: "-Dorg.gradle.daemon=false -Dkotlin.incremental=false" jobs: - linux-compat: + jvm: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # we build with a specific JDK version but source/target compatibility should ensure the jar is usable by + # the target versions we want to support + java-version: + - 8 + - 11 + - 17 + - 21 steps: - - name: Checkout sources - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} - restore-keys: | - ${{ runner.os }}-gradle- - - name: Build and Test ${{ env.PACKAGE_NAME }} - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - chmod a+x builder.pyz - echo "kotlinWarningsAsErrors=true" >> $GITHUB_WORKSPACE/local.properties - ./builder.pyz build -p ${{ env.PACKAGE_NAME }} - - macos-compat: - runs-on: macos-latest - steps: - - name: Checkout sources - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} - restore-keys: | - ${{ runner.os }}-gradle- - - name: Build and Test ${{ env.PACKAGE_NAME }} - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - chmod a+x builder.pyz - echo "kotlinWarningsAsErrors=true" >> $GITHUB_WORKSPACE/local.properties - ./builder.pyz build -p ${{ env.PACKAGE_NAME }} + - name: Checkout sources + uses: actions/checkout@v4 + - name: Configure JDK + uses: actions/setup-java@v3 + with: + distribution: 'corretto' + java-version: 17 + cache: 'gradle' + - name: Test + shell: bash + run: | + ./gradlew -Ptest.java.version=${{ matrix.java-version }} jvmTest --stacktrace - windows-compat: - runs-on: windows-latest + all-platforms: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, macos-latest, windows-latest ] steps: - name: Checkout sources - uses: actions/checkout@v2 - - name: Build and Test ${{ env.PACKAGE_NAME }} + uses: actions/checkout@v4 + - name: Configure JDK + uses: actions/setup-java@v3 + with: + distribution: 'corretto' + java-version: 17 + cache: 'gradle' + - name: Test + shell: bash run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python3 builder.pyz build -p ${{ env.PACKAGE_NAME }} + echo "kotlinWarningsAsErrors=true" >> $GITHUB_WORKSPACE/local.properties + ./gradlew apiCheck + ./gradlew test allTests + - name: Save Test Reports + if: failure() + uses: actions/upload-artifact@v3 + with: + name: test-reports + path: '**/build/reports' downstream: runs-on: ubuntu-latest steps: - name: Checkout sources - uses: actions/checkout@v2 - - uses: actions/cache@v2 + uses: actions/checkout@v4 + with: + path: 'smithy-kotlin' + - name: Checkout tools + uses: actions/checkout@v4 + with: + path: 'aws-kotlin-repo-tools' + repository: 'awslabs/aws-kotlin-repo-tools' + ref: '0.2.3' + sparse-checkout: | + .github + - name: Checkout aws-sdk-kotlin + uses: ./aws-kotlin-repo-tools/.github/actions/checkout-head + with: + # smithy-kotlin is checked out as a sibling dir which will automatically make it an included build + path: 'aws-sdk-kotlin' + repository: 'awslabs/aws-sdk-kotlin' + - name: Configure JDK + uses: actions/setup-java@v3 with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} - restore-keys: | - ${{ runner.os }}-gradle- - - name: Build and Test ${{ env.PACKAGE_NAME }} Downstream Consumers + distribution: 'corretto' + java-version: 17 + cache: 'gradle' + - name: Build and Test aws-sdk-kotlin downstream run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - echo "kotlinWarningsAsErrors=true" >> $GITHUB_WORKSPACE/local.properties - ./builder build -p ${{ env.PACKAGE_NAME }} --spec downstream + # TODO - JVM only + cd $GITHUB_WORKSPACE/smithy-kotlin + ./gradlew --parallel publishToMavenLocal + cd $GITHUB_WORKSPACE/aws-sdk-kotlin + ./gradlew --parallel publishToMavenLocal + ./gradlew testAllProtocols diff --git a/build.gradle.kts b/build.gradle.kts index ccd3c008b..ada6c0d5e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -30,13 +30,13 @@ plugins { // configures (KMP) subprojects with our own KMP conventions and some default dependencies apply(plugin = "aws.sdk.kotlin.kmp") -allprojects { - repositories { - mavenLocal() - mavenCentral() - google() - } +val testJavaVersion = typedProp("test.java.version")?.let { + JavaLanguageVersion.of(it) +}?.also { + println("configuring tests to run with jdk $it") +} +allprojects { tasks.withType().configureEach { val sdkVersion: String by project moduleVersion.set(sdkVersion) @@ -61,14 +61,25 @@ allprojects { ) pluginsMapConfiguration.set(pluginConfigMap) } -} -if (project.typedProp("kotlinWarningsAsErrors") == true) { - subprojects { + if (rootProject.typedProp("kotlinWarningsAsErrors") == true) { tasks.withType { kotlinOptions.allWarningsAsErrors = true } } + + if (testJavaVersion != null) { + tasks.withType { + // JDK8 tests fail with out of memory sometimes, not sure why... + maxHeapSize = "2g" + val toolchains = project.extensions.getByType() + javaLauncher.set( + toolchains.launcherFor { + languageVersion.set(testJavaVersion) + }, + ) + } + } } // configure the root multimodule docs diff --git a/runtime/auth/aws-signing-tests/jvm/src/aws/smithy/kotlin/runtime/auth/awssigning/tests/SigningSuiteTestBaseJVM.kt b/runtime/auth/aws-signing-tests/jvm/src/aws/smithy/kotlin/runtime/auth/awssigning/tests/SigningSuiteTestBaseJVM.kt index 97994a259..e994760df 100644 --- a/runtime/auth/aws-signing-tests/jvm/src/aws/smithy/kotlin/runtime/auth/awssigning/tests/SigningSuiteTestBaseJVM.kt +++ b/runtime/auth/aws-signing-tests/jvm/src/aws/smithy/kotlin/runtime/auth/awssigning/tests/SigningSuiteTestBaseJVM.kt @@ -37,11 +37,11 @@ import org.junit.jupiter.params.provider.MethodSource import java.nio.file.FileSystems import java.nio.file.Files import java.nio.file.Path +import java.util.stream.Collectors import kotlin.io.path.exists import kotlin.io.path.isDirectory import kotlin.io.path.name import kotlin.io.path.readText -import kotlin.streams.toList import kotlin.test.* import kotlin.time.Duration.Companion.seconds @@ -94,7 +94,9 @@ public actual abstract class SigningSuiteTestBase : HasSigner { private val testDirPaths: List by lazy { Files .walk(testSuitePath) - .toList() + // Due to https://youtrack.jetbrains.com/issue/KT-47039 setting jvmTarget compatibility isn't enough + // ignore the toList() extension in-favor of something that should work JDK8+ even if we compile with JDK17+ + .collect(Collectors.toList()) .filter { !it.isDirectory() && it.name == "request.txt" } .filterNot { it.parent.name in disabledTests } .map { it.parent } diff --git a/runtime/build.gradle.kts b/runtime/build.gradle.kts index ff8a7d963..2d920d377 100644 --- a/runtime/build.gradle.kts +++ b/runtime/build.gradle.kts @@ -63,4 +63,10 @@ subprojects { dependencies { dokkaPlugin(project(":dokka-smithy")) } + + tasks.withType { + kotlinOptions { + jvmTarget = "1.8" + } + } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 79f9be3da..d0e741f99 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -20,6 +20,14 @@ pluginManagement { } } +dependencyResolutionManagement { + repositories { + mavenLocal() + mavenCentral() + google() + } +} + sourceControl { gitRepository(java.net.URI("https://github.com/awslabs/aws-kotlin-repo-tools.git")) { producesModule("aws.sdk.kotlin:build-plugins")