diff --git a/.github/dependabot.yml b/.github/dependabot.yml index af83f8b2edac2..e6a1f74299fb5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,7 +11,8 @@ updates: open-pull-requests-limit: 1 package-ecosystem: "github-actions" schedule: - interval: "weekly" + interval: "daily" + time: "21:10" labels: - "dependabot" - "dependencies" diff --git a/build.gradle b/build.gradle index 55b31ca816214..d35a5898de7d0 100644 --- a/build.gradle +++ b/build.gradle @@ -38,6 +38,7 @@ import org.apache.tools.ant.taskdefs.condition.Os import org.opensearch.gradle.BuildPlugin import org.opensearch.gradle.Version import org.opensearch.gradle.VersionProperties +import org.opensearch.gradle.VersionPropertiesGenerator import org.opensearch.gradle.info.BuildParams import org.opensearch.gradle.plugin.PluginBuildPlugin import org.opensearch.gradle.tar.SymbolicLinkPreservingTar @@ -733,6 +734,30 @@ tasks.register('checkCompatibility', CheckCompatibilityTask) { description('Checks the compatibility with child components') } +def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) { + destinationFile = file("${buildDir}/version.properties") + comment = 'Generated version properties' + + def libsTomlFile = file("gradle/libs.versions.toml") + def propertiesFile = file("${buildDir}/versions.properties") + + doLast { + def properties = org.opensearch.gradle.VersionPropertiesGenerator.generateVersionProperties(libsTomlFile) + + propertiesFile.withWriter('UTF-8') { writer -> + properties.store(writer, "Generated by Gradle") + } + + println "Generated versions.properties at: ${propertiesFile.absolutePath}" + } +} + +if (project != rootProject) { + processResources { + from(generateVersionProperties) + } +} + allprojects { project -> project.afterEvaluate { if (project.tasks.findByName('publishToMavenLocal')) { diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index b984ef3800490..43a62ca42c39b 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -57,19 +57,9 @@ if (project == rootProject) { // we update the version property to reflect if we are building a snapshot or a release build // we write this back out below to load it in the Build.java which will be shown in rest main action // to indicate this being a snapshot build or a release build. -Properties props = VersionPropertiesLoader.loadBuildSrcVersion(project.file('version.properties')) +Properties props = VersionPropertiesLoader.loadBuildSrcVersion(project.file('../gradle/libs.versions.toml')) version = props.getProperty("opensearch") -def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) { - destinationFile = file("${buildDir}/version.properties") - comment = 'Generated version properties' - properties(props) -} - -processResources { - from(generateVersionProperties) -} - /***************************************************************************** * Java version * *****************************************************************************/ @@ -123,6 +113,7 @@ dependencies { api 'org.jruby.joni:joni:2.2.1' api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}" api "org.ajoberstar.grgit:grgit-core:5.2.1" + implementation 'org.tomlj:tomlj:1.1.1' testFixturesApi "junit:junit:${props.getProperty('junit')}" testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}" @@ -297,6 +288,10 @@ class VersionPropertiesLoader { } finally { is.close() } + props.forEach((key, value) -> { + String newValue = value.toString().replace("\"", "") + props.setProperty(key.toString(), newValue) + }); loadBuildSrcVersion(props, System.getProperties()) return props } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java b/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java index 4d8b62d95dff1..583f00ba4a766 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java @@ -34,11 +34,15 @@ import org.apache.commons.lang3.StringUtils; import java.io.IOException; -import java.io.InputStream; +import java.nio.file.Path; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import org.tomlj.Toml; +import org.tomlj.TomlParseResult; +import org.tomlj.TomlTable; + /** * Accessor for shared dependency versions used by opensearch, namely the opensearch and lucene versions. */ @@ -123,17 +127,30 @@ public static Map getVersions() { } } - private static Properties getVersionProperties() { - Properties props = new Properties(); - try (InputStream propsStream = VersionProperties.class.getResourceAsStream("/version.properties")) { - if (propsStream == null) { - throw new IllegalStateException("/version.properties resource missing"); + private static void flattenToml(TomlTable tomlTable, Properties properties, String prefix) { + for (String key : tomlTable.keySet()) { + String fullKey = prefix.isEmpty() ? key : prefix + "." + key; + Object value = tomlTable.get(key); + + if (value instanceof TomlTable) { + flattenToml((TomlTable) value, properties, fullKey); + } else { + properties.setProperty(key, value.toString()); } - props.load(propsStream); + } + } + + private static Properties getVersionProperties() { + TomlParseResult toml = null; + try { + toml = Toml.parse(Path.of("gradle/libs.versions.toml")); } catch (IOException e) { - throw new IllegalStateException("Failed to load version properties", e); + throw new RuntimeException(e); } - return props; + + Properties properties = new Properties(); + flattenToml(toml, properties, ""); + return properties; } public static boolean isOpenSearchSnapshot() { diff --git a/buildSrc/src/main/java/org/opensearch/gradle/VersionPropertiesGenerator.java b/buildSrc/src/main/java/org/opensearch/gradle/VersionPropertiesGenerator.java new file mode 100644 index 0000000000000..2e68f4f4a5ac7 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/VersionPropertiesGenerator.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle; + +import java.io.File; +import java.io.FileReader; +import java.util.Properties; + +import org.tomlj.Toml; +import org.tomlj.TomlParseResult; + +/** + * Generator for shared dependency versions used by opensearch, namely the opensearch and lucene versions. + */ +public class VersionPropertiesGenerator { + + public static Properties generateVersionProperties(File tomlFile) throws Exception { + TomlParseResult toml = Toml.parse(new FileReader(tomlFile)); + Properties properties = new Properties(); + + toml.getTable("versions").keySet().forEach(key -> { + String version = toml.getString("versions." + key); + properties.setProperty(key, version); + }); + + return properties; + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml new file mode 100644 index 0000000000000..ea65d32237d20 --- /dev/null +++ b/gradle/libs.versions.toml @@ -0,0 +1,79 @@ +[versions] +opensearch = "3.0.0" +lucene = "9.12.0" + +bundled_jdk_vendor = "adoptium" +bundled_jdk = "21.0.4+7" + +# optional dependencies +spatial4j = "0.7" +jts = "1.15.0" +jackson = "2.17.2" +jackson_databind = "2.17.2" +snakeyaml = "2.1" +icu4j = "75.1" +supercsv = "2.4.0" +log4j = "2.21.0" +slf4j = "1.7.36" +asm = "9.7" +jettison = "1.5.4" +woodstox = "6.4.0" +kotlin = "1.7.10" +antlr4 = "4.13.1" +guava = "32.1.1-jre" +protobuf = "3.25.5" +jakarta_annotation = "1.3.5" +google_http_client = "1.44.1" +tdigest = "3.3" +hdrhistogram = "2.2.2" + +# when updating the JNA version, also update the version in buildSrc/build.gradle +jna = "5.13.0" + +netty = "4.1.114.Final" +joda = "2.12.7" + +# project reactor +reactor_netty = "1.1.22" +reactor = "3.5.20" + +# client dependencies +httpclient5 = "5.3.1" +httpcore5 = "5.2.5" +httpclient = "4.5.14" +httpcore = "4.4.16" +httpasyncclient = "4.1.5" +commonslogging = "1.2" +commonscodec = "1.16.1" +commonslang = "3.14.0" +commonscompress = "1.26.1" +commonsio = "2.16.0" +# plugin dependencies +aws = "2.20.86" +reactivestreams = "1.0.4" + +# when updating this version, you need to ensure compatibility with: +# - plugins/ingest-attachment (transitive dependency, check the upstream POM) +# - distribution/tools/plugin-cli +bouncycastle="1.78" +# test dependencies +randomizedrunner = "2.7.1" +junit = "4.13.2" +hamcrest = "2.1" +mockito = "5.12.0" +objenesis = "3.2" +bytebuddy = "1.14.9" + +# benchmark dependencies +jmh = "1.35" + +# compression +zstd = "1.5.5-5" + +jzlib = "1.1.3" + +resteasy = "6.2.4.Final" + +# opentelemetry dependencies +opentelemetry = "1.41.0" +opentelemetrysemconv = "1.27.0-alpha" diff --git a/settings.gradle b/settings.gradle index 8412d198a2a29..86fb73b7cf7fb 100644 --- a/settings.gradle +++ b/settings.gradle @@ -139,6 +139,14 @@ include projects.toArray(new String[0]) project(':build-tools').projectDir = new File(rootProject.projectDir, 'buildSrc') project(':build-tools:reaper').projectDir = new File(rootProject.projectDir, 'buildSrc/reaper') +//dependencyResolutionManagement { +// versionCatalogs { +// libs { +// from(files("gradle/libs.versions.toml")) +// } +// } +//} + project(":libs").children.each { libsProject -> libsProject.name = "opensearch-${libsProject.name}" } diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index bc63362980bea..352d37bd2c646 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -74,7 +74,7 @@ dependencies { api 'org.apache.zookeeper:zookeeper:3.9.2' api "org.apache.commons:commons-text:1.12.0" api "commons-net:commons-net:3.11.1" - api "ch.qos.logback:logback-core:1.5.8" + api "ch.qos.logback:logback-core:1.5.9" api "ch.qos.logback:logback-classic:1.2.13" api "org.jboss.xnio:xnio-nio:3.8.16.Final" api 'org.jline:jline:3.27.0'