From 7c54ab0f18e5d582248c5b19511942d7dfad92bc Mon Sep 17 00:00:00 2001 From: Mike Cobbett <77053+techcobweb@users.noreply.github.com> Date: Mon, 21 Oct 2024 12:11:06 +0100 Subject: [PATCH 1/2] etcd client upgraded, gson level upgraded Signed-off-by: Mike Cobbett <77053+techcobweb@users.noreply.github.com> --- calculate-transitive-dependencies.sh | 206 ++++++++++++++++++ .../dev.galasa.cps.etcd/bnd.bnd | 78 ++++--- .../dev.galasa.cps.etcd/build.gradle | 63 +++++- 3 files changed, 301 insertions(+), 46 deletions(-) create mode 100755 calculate-transitive-dependencies.sh diff --git a/calculate-transitive-dependencies.sh b/calculate-transitive-dependencies.sh new file mode 100755 index 00000000..f6b5a2a4 --- /dev/null +++ b/calculate-transitive-dependencies.sh @@ -0,0 +1,206 @@ +#! /usr/bin/env bash + +# +# Copyright contributors to the Galasa project +# +# SPDX-License-Identifier: EPL-2.0 +# +#----------------------------------------------------------------------------------------- +# +# Objectives: Calculate transient dependencies so we can include them in the bnd bundle +# +#----------------------------------------------------------------------------------------- + +# Where is this script executing from ? +BASEDIR=$(dirname "$0");pushd $BASEDIR 2>&1 >> /dev/null ;BASEDIR=$(pwd);popd 2>&1 >> /dev/null +# echo "Running from directory ${BASEDIR}" +export ORIGINAL_DIR=$(pwd) +cd "${BASEDIR}" + +cd "${BASEDIR}/.." +WORKSPACE_DIR=$(pwd) +cd "${BASEDIR}" + +#----------------------------------------------------------------------------------------- +# +# Set Colors +# +#----------------------------------------------------------------------------------------- +bold=$(tput bold) +underline=$(tput sgr 0 1) +reset=$(tput sgr0) +red=$(tput setaf 1) +green=$(tput setaf 76) +white=$(tput setaf 7) +tan=$(tput setaf 202) +blue=$(tput setaf 25) + +#----------------------------------------------------------------------------------------- +# +# Headers and Logging +# +#----------------------------------------------------------------------------------------- +underline() { printf "${underline}${bold}%s${reset}\n" "$@" ;} +h1() { printf "\n${underline}${bold}${blue}%s${reset}\n" "$@" ;} +h2() { printf "\n${underline}${bold}${white}%s${reset}\n" "$@" ;} +debug() { printf "${white}%s${reset}\n" "$@" ;} +info() { printf "${white}➜ %s${reset}\n" "$@" ;} +success() { printf "${green}✔ %s${reset}\n" "$@" ;} +error() { printf "${red}✖ %s${reset}\n" "$@" ;} +warn() { printf "${tan}➜ %s${reset}\n" "$@" ;} +bold() { printf "${bold}%s${reset}\n" "$@" ;} +note() { printf "\n${underline}${bold}${blue}Note:${reset} ${blue}%s${reset}\n" "$@" ;} + +#----------------------------------------------------------------------------------------- +# Functions +#----------------------------------------------------------------------------------------- +function usage { + info "Syntax: calculate-dependencies.sh [OPTIONS]" + cat << EOF +Options are: + + +Environment variables used: + + +EOF +} + + +function create_temp_project { + h2 "Creating a temporary project so we can calculate the dependencies it has" + mkdir -p ${BASEDIR}/temp + cat << EOF > ${BASEDIR}/temp/pom.xml + + 4.0.0 + + dev.galasa + dependency-finder + 1.0-SNAPSHOT + jar + + + + io.etcd + jetcd-core + 0.5.9 + + + + +EOF + success "OK" +} + + +function list_transient_dependencies_using_maven { + + cd ${BASEDIR}/temp + h2 "This might be good in the bnd file of the cps extension:" + mvn dependency:tree | sed "s/\[INFO\]//g" | grep -v "BUILD SUCCESS" | grep -v "\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-" | grep -v "Total time" | grep -v "Finished at" \ + | grep -v "Scanning for projects" | grep -v "Building dependency-finder" | grep -v "maven-dependency-plugin" | grep -v "dev.galasa:dependency-finder" \ + | grep -v "^[ \t]*$" | sed "s/^[|+ \t\\\-]*//" | sed "s/:compile.*//g" | sed "s/:runtime.*//g" \ + | cut -d ':' -f2,4 \ + | sed "s/:/-/g" | sed "s/$/.jar; lib:=true,\\\/" | sed "s/^/ /" \ + | sort | uniq \ + > $BASEDIR/temp/dependencies_maven.txt + + info "See $BASEDIR/temp/dependencies_maven.txt" + success "OK" + + + h2 "This might be good in the gradle file of the cps extension:" + mvn dependency:tree | sed "s/\[INFO\]//g" | grep -v "BUILD SUCCESS" | grep -v "\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-" | grep -v "Total time" | grep -v "Finished at" \ + | grep -v "Scanning for projects" | grep -v "Building dependency-finder" | grep -v "maven-dependency-plugin" | grep -v "dev.galasa:dependency-finder" \ + | grep -v "^[ \t]*$" | sed "s/^[|+ \t\\\-]*//" | sed "s/:compile.*//g" | sed "s/:runtime.*//g" \ + | sed "s/^/ implementation ('/" \ + | sed "s/$/')/" \ + | sed "s/:jar:/:/" \ + | sort | uniq \ + > $BASEDIR/temp/dependencies_maven_imports.txt + + info "See $BASEDIR/temp/dependencies_maven_imports.txt" + success "OK" + + + # mvn dependency:tree | sed "s/\[INFO\]//g" | grep -v "BUILD SUCCESS" | grep -v "\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-" | grep -v "Total time" | grep -v "Finished at" \ + # | grep -v "Scanning for projects" | grep -v "Building dependency-finder" | grep -v "maven-dependency-plugin" | grep -v "dev.galasa:dependency-finder" +} + + + +function list_transient_dependencies_using_gradle { + cd ${BASEDIR}/*parent + h2 "This might be good in the bnd file of the cps extension:" + gradle dev.galasa.cps.etcd:dependencies --configuration runtimeClasspath > $BASEDIR/temp/dependencies.txt + sed '/Indicates repeated occurrences/,$d' $BASEDIR/temp/dependencies.txt > $BASEDIR/temp/dependencies2.txt + cd ${BASEDIR}/temp + csplit $BASEDIR/temp/dependencies2.txt "/Runtime classpath of source set/" > /dev/null + mv $BASEDIR/temp/xx01 $BASEDIR/temp/dependencies3.txt + cat $BASEDIR/temp/dependencies3.txt | grep -v "runtimeClasspath - Runtime classpath of source set" \ + | sed "s/^[+ \\|\-]*//" \ + | cut -d ':' -f2,3 \ + | sed "s/:[a-zA-Z0-9.\\-]* -> /:/" \ + | sed "s/ [(][*][)]$//" \ + | sed "s/:/-/" \ + | grep -v "^[ \t]*$" \ + | sed "s/$/.jar/" \ + | sed "s/^/ /" \ + | sed "s/$/; lib:=true,\\\/" \ + | grep -v "dev.galasa" \ + | grep -v "bcel" \ + | grep -v "commons-io-.*.jar" \ + | grep -v "commons-lang3-.*.jar" \ + | grep -v "commons-logging-" \ + | grep -v "easymock-.*.jar" \ + | grep -v "log4j-api-.*.jar" \ + | grep -v "log4j-core-.*.jar" \ + | grep -v "log4j-slf4j-impl-.*.jar" \ + | grep -v "org.apache.felix.bundlerepository-.*.jar" \ + | grep -v "org.osgi" \ + | sort | uniq \ + > $BASEDIR/temp/dependencies_gradle.txt + # cat $BASEDIR/temp/dependencies_gradle.txt +# + rm $BASEDIR/temp/dependencies2.txt $BASEDIR/temp/dependencies3.txt + info "See $BASEDIR/temp/dependencies_gradle.txt" + success "OK" + + cd ${BASEDIR}/*parent + h2 "This might be good in the gradle file of the cps extension:" + gradle dev.galasa.cps.etcd:dependencies --configuration runtimeClasspath > $BASEDIR/temp/dependencies.txt + sed '/Indicates repeated occurrences/,$d' $BASEDIR/temp/dependencies.txt > $BASEDIR/temp/dependencies2.txt + cd ${BASEDIR}/temp + csplit $BASEDIR/temp/dependencies2.txt "/Runtime classpath of source set/" > /dev/null + mv $BASEDIR/temp/xx01 $BASEDIR/temp/dependencies3.txt + cat $BASEDIR/temp/dependencies3.txt | grep -v "runtimeClasspath - Runtime classpath of source set" \ + | sed "s/^[+ \\|\-]*//" \ + | sed "s/:[a-zA-Z0-9.\\-]* -> /:/" \ + | sed "s/ [(][*][)]$//" \ + | grep -v "dev.galasa" \ + | grep -v "bcel" \ + | grep -v "commons-io" \ + | grep -v "commons-lang3" \ + | grep -v "commons-logging" \ + | grep -v "easymock" \ + | grep -v "org.apache.felix.bundlerepository" \ + | grep -v "org.osgi" \ + | sed "s/^/ implementation('/" \ + | sed "s/$/')/" \ + | grep -v " implementation('')" \ + | sort | uniq \ + > $BASEDIR/temp/dependencies_gradle_imports.txt + + # | grep -v "log4j-api" \ + # | grep -v "log4j-core" \ + # | grep -v "log4j-slf4j-impl" \ + + info "See $BASEDIR/temp/dependencies_gradle_imports.txt" + success "OK" +} + +create_temp_project +# list_transient_dependencies_using_maven +list_transient_dependencies_using_gradle \ No newline at end of file diff --git a/galasa-extensions-parent/dev.galasa.cps.etcd/bnd.bnd b/galasa-extensions-parent/dev.galasa.cps.etcd/bnd.bnd index 84589f85..6d8da9ef 100644 --- a/galasa-extensions-parent/dev.galasa.cps.etcd/bnd.bnd +++ b/galasa-extensions-parent/dev.galasa.cps.etcd/bnd.bnd @@ -12,39 +12,47 @@ Import-Package: \ javax.security.cert Embed-Transitive: true Embed-Dependency: *;scope=compile|runtime --includeresource: gson-2.10.1.jar; lib:=true,\ - jetcd-core-0.5.9.jar; lib:=true,\ - jetcd-common-0.5.9.jar; lib:=true,\ - grpc-core-1.39.0.jar; lib:=true,\ - grpc-api-1.39.0.jar; lib:=true,\ - grpc-context-1.39.0.jar; lib:=true,\ - error_prone_annotations-2.5.1.jar; lib:=true,\ - jsr305-3.0.2.jar; lib:=true,\ - animal-sniffer-annotations-1.19.jar; lib:=true,\ - annotations-4.1.1.4.jar; lib:=true,\ - perfmark-api-0.23.0.jar; lib:=true,\ - grpc-netty-1.39.0.jar; lib:=true,\ - netty-codec-http2-4.1.86.Final.jar; lib:=true,\ - netty-common-4.1.86.Final.jar; lib:=true,\ - netty-buffer-4.1.86.Final.jar; lib:=true,\ - netty-transport-4.1.86.Final.jar; lib:=true,\ - netty-resolver-4.1.86.Final.jar; lib:=true,\ - netty-codec-4.1.86.Final.jar; lib:=true,\ - netty-handler-4.1.86.Final.jar; lib:=true,\ - netty-codec-http-4.1.86.Final.jar; lib:=true,\ - netty-handler-proxy-4.1.86.Final.jar; lib:=true,\ - netty-codec-socks-4.1.86.Final.jar; lib:=true,\ - grpc-protobuf-1.39.0.jar; lib:=true,\ - protobuf-java-3.17.2.jar; lib:=true,\ - guava-30.1.1-jre.jar; lib:=true,\ - failureaccess-1.0.1.jar; lib:=true,\ - checker-qual-3.8.0.jar; lib:=true,\ - j2objc-annotations-1.3.jar; lib:=true,\ - proto-google-common-protos-2.0.1.jar; lib:=true,\ - grpc-protobuf-lite-1.39.0.jar; lib:=true,\ - grpc-stub-1.39.0.jar; lib:=true,\ - grpc-grpclb-1.39.0.jar; lib:=true,\ - protobuf-java-util-3.17.2.jar; lib:=true,\ - slf4j-api-1.7.32.jar; lib:=true,\ - failsafe-2.4.1.jar; lib:=true +-includeresource: \ + checker-qual-3.42.0.jar; lib:=true,\ + failsafe-3.3.2.jar; lib:=true,\ + failureaccess-1.0.2.jar; lib:=true,\ + grpc-api-1.66.0.jar; lib:=true,\ + grpc-context-1.66.0.jar; lib:=true,\ + grpc-core-1.66.0.jar; lib:=true,\ + grpc-grpclb-1.66.0.jar; lib:=true,\ + grpc-netty-1.66.0.jar; lib:=true,\ + grpc-protobuf-1.66.0.jar; lib:=true,\ + grpc-protobuf-lite-1.66.0.jar; lib:=true,\ + grpc-stub-1.66.0.jar; lib:=true,\ + grpc-util-1.66.0.jar; lib:=true,\ + gson-2.10.1.jar; lib:=true,\ + guava-33.2.1-jre.jar; lib:=true,\ + jackson-core-2.16.1.jar; lib:=true,\ + javax.annotation-api-1.3.2.jar; lib:=true,\ + jetcd-api-0.8.3.jar; lib:=true,\ + jetcd-common-0.8.3.jar; lib:=true,\ + jetcd-core-0.8.3.jar; lib:=true,\ + jetcd-grpc-0.8.3.jar; lib:=true,\ + listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar; lib:=true,\ + netty-buffer-4.1.111.Final.jar; lib:=true,\ + netty-codec-4.1.111.Final.jar; lib:=true,\ + netty-codec-dns-4.1.111.Final.jar; lib:=true,\ + netty-codec-http-4.1.111.Final.jar; lib:=true,\ + netty-codec-http2-4.1.111.Final.jar; lib:=true,\ + netty-codec-socks-4.1.111.Final.jar; lib:=true,\ + netty-common-4.1.111.Final.jar; lib:=true,\ + netty-handler-4.1.111.Final.jar; lib:=true,\ + netty-handler-proxy-4.1.111.Final.jar; lib:=true,\ + netty-resolver-4.1.111.Final.jar; lib:=true,\ + netty-resolver-dns-4.1.111.Final.jar; lib:=true,\ + netty-transport-4.1.111.Final.jar; lib:=true,\ + netty-transport-native-unix-common-4.1.111.Final.jar; lib:=true,\ + perfmark-api-0.27.0.jar; lib:=true,\ + proto-google-common-protos-2.41.0.jar; lib:=true,\ + protobuf-java-3.25.3.jar; lib:=true,\ + protobuf-java-util-3.25.3.jar; lib:=true,\ + slf4j-api-2.0.16.jar; lib:=true,\ + validation-api-2.0.1.Final.jar; lib:=true,\ + vertx-core-4.5.9.jar; lib:=true,\ + vertx-grpc-4.5.9.jar; lib:=true -fixupmessages "Classes found in the wrong directory"; restrict:=error; is:=warning diff --git a/galasa-extensions-parent/dev.galasa.cps.etcd/build.gradle b/galasa-extensions-parent/dev.galasa.cps.etcd/build.gradle index 627a0b1c..4cf92f12 100644 --- a/galasa-extensions-parent/dev.galasa.cps.etcd/build.gradle +++ b/galasa-extensions-parent/dev.galasa.cps.etcd/build.gradle @@ -8,19 +8,60 @@ description = 'Galasa etcd3 for CPS, DSS and Credentials - Provides the CPS, DSS version = '0.38.0' dependencies { - implementation ('io.etcd:jetcd-core:0.5.9') + implementation ('io.etcd:jetcd-core:0.8.3') + // { + // exclude group: com.google.code.gson ,module: gson + // } + // We don't want to use their gson implementation. + implementation ('com.google.code.gson:gson:2.10.1') // Not required for compile, but required to force the download of the jars to embed by bnd - implementation ('com.google.code.gson:gson:2.10.1') - implementation ('org.codehaus.mojo:animal-sniffer-annotations:1.19') - implementation ('com.google.android:annotations:4.1.1.4') - implementation ('io.perfmark:perfmark-api:0.23.0') - implementation ('io.netty:netty-handler-proxy:4.1.86.Final') - implementation ('io.netty:netty-codec-socks:4.1.86.Final') - implementation ('io.netty:netty-codec-http2:4.1.86.Final') - implementation ('com.google.protobuf:protobuf-java-util:3.17.2') - implementation ('com.google.guava:guava:30.1.1-jre') - implementation ('com.google.guava:failureaccess:1.0.1') + implementation('com.fasterxml.jackson.core:jackson-core:2.16.1') + implementation('com.google.api.grpc:proto-google-common-protos:2.41.0') + implementation('com.google.code.gson:gson:2.10.1') + implementation('com.google.guava:failureaccess:1.0.2') + implementation('com.google.guava:guava:33.2.1-jre') + implementation('com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava') + implementation('com.google.protobuf:protobuf-java-util:3.25.3') + implementation('com.google.protobuf:protobuf-java:3.25.3') + implementation('dev.failsafe:failsafe:3.3.2') + implementation('io.etcd:jetcd-api:0.8.3') + implementation('io.etcd:jetcd-common:0.8.3') + implementation('io.etcd:jetcd-core:0.8.3') + implementation('io.etcd:jetcd-grpc:0.8.3') + implementation('io.grpc:grpc-api:1.66.0') + implementation('io.grpc:grpc-context:1.66.0') + implementation('io.grpc:grpc-core:1.66.0') + implementation('io.grpc:grpc-grpclb:1.66.0') + implementation('io.grpc:grpc-netty:1.66.0') + implementation('io.grpc:grpc-protobuf-lite:1.66.0') + implementation('io.grpc:grpc-protobuf:1.66.0') + implementation('io.grpc:grpc-stub:1.66.0') + implementation('io.grpc:grpc-util:1.66.0') + implementation('io.netty:netty-buffer:4.1.111.Final') + implementation('io.netty:netty-codec-dns:4.1.111.Final') + implementation('io.netty:netty-codec-http2:4.1.111.Final') + implementation('io.netty:netty-codec-http:4.1.111.Final') + implementation('io.netty:netty-codec-socks:4.1.111.Final') + implementation('io.netty:netty-codec:4.1.111.Final') + implementation('io.netty:netty-common:4.1.111.Final') + implementation('io.netty:netty-handler-proxy:4.1.111.Final') + implementation('io.netty:netty-handler:4.1.111.Final') + implementation('io.netty:netty-resolver-dns:4.1.111.Final') + implementation('io.netty:netty-resolver:4.1.111.Final') + implementation('io.netty:netty-transport-native-unix-common:4.1.111.Final') + implementation('io.netty:netty-transport:4.1.111.Final') + implementation('io.perfmark:perfmark-api:0.27.0') + implementation('io.vertx:vertx-core:4.5.9') + implementation('io.vertx:vertx-grpc:4.5.9') + implementation('javax.annotation:javax.annotation-api:1.3.2') + implementation('javax.validation:validation-api:2.0.1.Final') + // logging is already embedded inside the framework bundle.... + // implementation('org.apache.logging.log4j:log4j-api:2.17.1') + // implementation('org.apache.logging.log4j:log4j-core:2.17.1') + // implementation('org.apache.logging.log4j:log4j-slf4j-impl:2.17.1') + implementation('org.checkerframework:checker-qual:3.42.0') + implementation('org.slf4j:slf4j-api:2.0.16') } // Note: These values are consumed by the parent build process From 1d97c8bd5be206c051d59e7535562c1c7b2e377c Mon Sep 17 00:00:00 2001 From: Mike Cobbett <77053+techcobweb@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:15:44 +0100 Subject: [PATCH 2/2] Added readme.md to explain how to uplift etcd transitive dependencies Signed-off-by: Mike Cobbett <77053+techcobweb@users.noreply.github.com> --- .../dev.galasa.cps.etcd/README.md | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 galasa-extensions-parent/dev.galasa.cps.etcd/README.md diff --git a/galasa-extensions-parent/dev.galasa.cps.etcd/README.md b/galasa-extensions-parent/dev.galasa.cps.etcd/README.md new file mode 100644 index 00000000..51f37c24 --- /dev/null +++ b/galasa-extensions-parent/dev.galasa.cps.etcd/README.md @@ -0,0 +1,29 @@ +# The cps etcd extension + +This extension provides access to an implementation of a CPS store based based on remotely connecting to an etcd server. + + +## How the packaging works + +This extension is perhaps slightly strange in that it doesn't every want to go out to maven central, so any dependencies it needs to run +must be included in the osgi bundle. + +The extension therefore needs to get hold of the transitive dependencies and include them in a 'fat jar' bundle. + +If you look at the built bundle (in the `galasa-extensions-parent/dev.galasa.cps.etcd/build/libs` folder) and unpack the jar, you will see that it packs up the dependndency jars in the root of the bundle jar. + +There are 2 parts to packaging this extension: +- The `build.gradle` file downloads everything that it needed to compile, but it also downloads all the transitive dependencies +- The `bnd.bnd` file which controls which jars are included in the 'fat jar'. + +The gradle plugin `biz.aQute.bnd.builder` is used to do the packing-up of the dependency jars, based on the content of the `bnd.bnd` file. + +## How to upgrade the dependencies + +- Remove the dependencies from the `build.gradle` file which are only there to make sure things are downloaded which can be bundled into the fat jar. +- Upgrade what you need to in the `build.gradle`. For example, bump the value for the `io.etcd:jetcd-core` bundle might be what you are trying to do. +- Run the `calculate-transitive-dependencies.sh` script (in the root of this module). This script calculates the transitive depdendencies of everything which is in the build.gradle file. That's why we need to clean the `build.gradle` to contain only the minimum to start with. +- Move the contents of `temp/dependencies_gradle_imports.txt` to the `build.gradle` file. +- Move the contents of `temp/dependencies_gradle.txt` into the `bnd.bnd` file. + +This should result in a long list of jars which are first downloaded, then packaged into the extension bundle jar.