From f2e02d410de24edbdf7b2b729b76b87e8e5edc08 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Fri, 29 Nov 2024 20:17:32 +0000 Subject: [PATCH 1/5] fix release build scripts (p1) - mv wildcard statement failed due to collisions (src and dst dir both had "deps" subfolders). Fix by being more targeted, only moving lib/ folder - Fix some s3 cp commands. --include has no effect unless you also have --exclude "*" --- codebuild/cd/generic-unix-build.sh | 16 +++++++++++++--- codebuild/cd/linux-aarch64-fips-build.sh | 15 +++++++++++++-- codebuild/cd/musl-linux-build.sh | 8 +++++++- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/codebuild/cd/generic-unix-build.sh b/codebuild/cd/generic-unix-build.sh index 2dd4174d4..ee72af627 100755 --- a/codebuild/cd/generic-unix-build.sh +++ b/codebuild/cd/generic-unix-build.sh @@ -29,10 +29,20 @@ chmod a+x builder GIT_TAG=$(git describe --tags) ./builder build -p aws-crt-java --target=$AWS_CRT_TARGET run_tests=false -# Builder corss-compiles the shared lib to `target/cmake-build/aws-crt-java/`, move it to the expected path for mvn to generate the jar. -mv target/cmake-build/aws-crt-java/* target/cmake-build/ + +# When cross-compiling with builder, the shared lib gets an extra "/aws-crt-java/" in its path. +# Move it to expected location. +if [ -d target/cmake-build/aws-crt-java/lib ]; then + mv target/cmake-build/aws-crt-java/lib target/cmake-build/lib +fi + +# Double check that shared lib is where we expect +if ! find target/cmake-build/lib -type f -name "*.so" | grep -q .; then + echo "No .so files found" + exit 1 +fi JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 mvn -B package -DskipTests -Dshared-lib.skip=true -Dcrt.classifier=$CLASSIFIER -aws s3 cp --recursive --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib +aws s3 cp --recursive --exclude "*" --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib aws s3 cp target/ s3://aws-crt-java-pipeline/${GIT_TAG}/jar/ --recursive --exclude "*" --include "aws-crt*.jar" diff --git a/codebuild/cd/linux-aarch64-fips-build.sh b/codebuild/cd/linux-aarch64-fips-build.sh index edece0f05..15f4df370 100644 --- a/codebuild/cd/linux-aarch64-fips-build.sh +++ b/codebuild/cd/linux-aarch64-fips-build.sh @@ -18,8 +18,19 @@ chmod a+x builder GIT_TAG=$(git describe --tags) ./builder build -p aws-crt-java run_tests=false --target=linux-arm64 --cmake-extra=-DCRT_FIPS=ON -mv target/cmake-build/aws-crt-java/* target/cmake-build/ + +# When cross-compiling with builder, the shared lib gets an extra "/aws-crt-java/" in its path. +# Move it to expected location. +if [ -d target/cmake-build/aws-crt-java/lib ]; then + mv target/cmake-build/aws-crt-java/lib target/cmake-build/lib +fi + +# Double check that shared lib is where we expect +if ! find target/cmake-build/lib -type f -name "*.so" | grep -q .; then + echo "No .so files found" + exit 1 +fi JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 mvn -B package -DskipTests -Dshared-lib.skip=true -Dcrt.classifier=linux-aarch_64-fips -aws s3 cp --recursive --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/fips_lib +aws s3 cp --recursive --exclude "*" --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/fips_lib diff --git a/codebuild/cd/musl-linux-build.sh b/codebuild/cd/musl-linux-build.sh index 4613c18ce..83731105a 100755 --- a/codebuild/cd/musl-linux-build.sh +++ b/codebuild/cd/musl-linux-build.sh @@ -23,5 +23,11 @@ docker container prune -f # Upload the artifacts to S3 export GIT_TAG=$(git describe --tags) -aws s3 cp --recursive --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib +# Double check that shared lib is where we expect +if ! find target/cmake-build/lib -type f -name "*.so" | grep -q .; then + echo "No .so files found" + exit 1 +fi + +aws s3 cp --recursive --exclude "*" --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib aws s3 cp target/ s3://aws-crt-java-pipeline/${GIT_TAG}/jar/ --recursive --exclude "*" --include "aws-crt*.jar" From 8af5313c91ce5f056596b37a46232275674f5a05 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Fri, 29 Nov 2024 16:08:11 -0800 Subject: [PATCH 2/5] Install golang according to official instructions Apparently you're supposed to set $GOROOT if you install it somewhere arbitrary, so let's put it in the official /usr/local/go --- codebuild/cd/manylinux-x64-fips-build.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/codebuild/cd/manylinux-x64-fips-build.yml b/codebuild/cd/manylinux-x64-fips-build.yml index e42a090d4..b3a76f242 100644 --- a/codebuild/cd/manylinux-x64-fips-build.yml +++ b/codebuild/cd/manylinux-x64-fips-build.yml @@ -11,9 +11,10 @@ phases: - git submodule update --init # double check aws-lc is the FIPS approved branch. - bash ./codebuild/cd/test-fips-branch.sh - - curl -OL https://go.dev/dl/go1.21.6.linux-amd64.tar.gz && mkdir ./go - - tar -C ./go -xvf go1.21.6.linux-amd64.tar.gz - - export PATH=$PATH:./go/go/bin + # install recent Golang, following these instructions: https://go.dev/doc/install + - curl -L -o /tmp/go.tar.gz https://go.dev/dl/go1.21.13.linux-amd64.tar.gz + - rm -rf /usr/local/go && tar -C /usr/local -xzf /tmp/go.tar.gz + - export PATH=$PATH:/usr/local/go/bin - mvn -B package -DskipTests -Dcrt.classifier=linux-x86_64-fips -Dcmake.crt_fips=ON post_build: From 2a4439001f5d42ad729bed4e0776f5ac97b6d80c Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Fri, 29 Nov 2024 16:19:44 -0800 Subject: [PATCH 3/5] not sure what this GO_PATH stuff was about It existed briefly in builder.json in the PR that added this line, https://github.com/awslabs/aws-crt-java/pull/772/commits/ed809e43c20f7c72e0ff2c8e93db2f551427dcaf#, but it was removed from builder.json before that PR was merged --- CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8f49fd86e..ac91773ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,11 +67,6 @@ if (BUILD_DEPS) if (CRT_FIPS) list(APPEND AWSLC_CMAKE_ARGUMENTS -DFIPS=ON) list(APPEND AWSLC_CMAKE_ARGUMENTS -DDISABLE_PERL=OFF) - - if (DEFINED ENV{GO_PATH}) - list(APPEND AWSLC_CMAKE_ARGUMENTS -DGO_EXECUTABLE=$ENV{GO_PATH}/go) - message(STATUS "Overriding GO_EXECUTABLE to ${GO_EXECUTABLE}") - endif() else() list(APPEND AWSLC_CMAKE_ARGUMENTS -DDISABLE_PERL=ON) # Disable codegen list(APPEND AWSLC_CMAKE_ARGUMENTS -DDISABLE_GO=ON) # Disable codegen From 0d9e994aa8f53f942bd135828d816f7d18b0eafa Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Mon, 2 Dec 2024 13:30:48 -0800 Subject: [PATCH 4/5] put back GO_PATH, with comments explaining it --- CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index ac91773ff..11e0c451f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,6 +67,14 @@ if (BUILD_DEPS) if (CRT_FIPS) list(APPEND AWSLC_CMAKE_ARGUMENTS -DFIPS=ON) list(APPEND AWSLC_CMAKE_ARGUMENTS -DDISABLE_PERL=OFF) + + # Pick up GO_PATH env-var, set by aws-crt-builder when cross-compiling, see: + # https://github.com/awslabs/aws-crt-builder/blob/31307c808ed9f2ea1eb16503b25a9b582f886481/builder/imports/golang.py#L84 + # https://github.com/awslabs/aws-crt-builder/blob/31307c808ed9f2ea1eb16503b25a9b582f886481/builder/actions/cmake.py#L110 + if (DEFINED ENV{GO_PATH}) + list(APPEND AWSLC_CMAKE_ARGUMENTS -DGO_EXECUTABLE=$ENV{GO_PATH}/go) + message(STATUS "Overriding GO_EXECUTABLE to ${GO_EXECUTABLE}") + endif() else() list(APPEND AWSLC_CMAKE_ARGUMENTS -DDISABLE_PERL=ON) # Disable codegen list(APPEND AWSLC_CMAKE_ARGUMENTS -DDISABLE_GO=ON) # Disable codegen From 3bf458090ae1b53338b6a2c1cccdc2e95cd47dd3 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Mon, 2 Dec 2024 13:31:12 -0800 Subject: [PATCH 5/5] install golang with via package manager because it's simple --- codebuild/cd/manylinux-x64-fips-build.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/codebuild/cd/manylinux-x64-fips-build.yml b/codebuild/cd/manylinux-x64-fips-build.yml index b3a76f242..5e2027f20 100644 --- a/codebuild/cd/manylinux-x64-fips-build.yml +++ b/codebuild/cd/manylinux-x64-fips-build.yml @@ -11,10 +11,8 @@ phases: - git submodule update --init # double check aws-lc is the FIPS approved branch. - bash ./codebuild/cd/test-fips-branch.sh - # install recent Golang, following these instructions: https://go.dev/doc/install - - curl -L -o /tmp/go.tar.gz https://go.dev/dl/go1.21.13.linux-amd64.tar.gz - - rm -rf /usr/local/go && tar -C /usr/local -xzf /tmp/go.tar.gz - - export PATH=$PATH:/usr/local/go/bin + # aws-lc FIPS build requires golang for codegen + - yum install -y golang - mvn -B package -DskipTests -Dcrt.classifier=linux-x86_64-fips -Dcmake.crt_fips=ON post_build: