Update Gradle wrapper to 8.8 (elastic#108021)

Fix incompatibility with 8.8 and our internal api usages - Update ospackage to a version that contains a fix we provided - Tweak build logic to avoid deprecation warnings - Use newer permission api - Use custom shadowplugin - Rework ElasticsearchDistribution dependencies resolution - Update Gradle wrapper to 8.8 (cherry picked from commit 8ac3e3d) # Conflicts: # build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java # build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/HdfsUtils.java # build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java # build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java # distribution/archives/build.gradle # distribution/build.gradle # distribution/packages/build.gradle # gradle/build.versions.toml # gradle/verification-metadata.xml # server/build.gradle
breskeby · Jun 17, 2024 · ec8baad · ec8baad
1 parent 551011c
commit ec8baad
Show file tree

Hide file tree

Showing 32 changed files with 429 additions and 207 deletions.
diff --git a/build-conventions/build.gradle b/build-conventions/build.gradle
@@ -8,6 +8,16 @@
 
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 
+
+buildscript {
+  repositories {
+    maven {
+      url 'https://jitpack.io'
+    }
+    mavenCentral()
+  }
+}
+
 plugins {
     id 'java-gradle-plugin'
     id 'java-test-fixtures'
@@ -58,6 +68,10 @@ gradlePlugin {
 }
 
 repositories {
+    maven {
+      url 'https://jitpack.io'
+    }
+
     mavenCentral()
     gradlePluginPortal()
 }

diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle
@@ -218,6 +218,9 @@ tasks.named('licenseHeaders').configure {
  *****************************************************************************/
 
 repositories {
+  maven {
+    url 'https://jitpack.io'
+  }
   mavenCentral()
   gradlePluginPortal()
 }

diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=194717442575a6f96e1c1befa2c30e9a4fc90f701d7aee33eb879b79e7ff05c0
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip
+distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

diff --git a/build-tools-internal/settings.gradle b/build-tools-internal/settings.gradle
@@ -1,5 +1,13 @@
 pluginManagement {
-    includeBuild "../build-conventions"
+  repositories {
+    maven {
+      url 'https://jitpack.io'
+    }
+    mavenCentral()
+    gradlePluginPortal()
+  }
+
+  includeBuild "../build-conventions"
     includeBuild "../build-tools"
 }
 
@@ -9,4 +17,4 @@ dependencyResolutionManagement {
             from(files("../gradle/build.versions.toml"))
         }
     }
-}
+}
diff --git a/...c/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java b/...c/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java
@@ -99,8 +99,8 @@ private void configureGeneralTaskDefaults(Project project) {
         project.getTasks().withType(AbstractCopyTask.class).configureEach(t -> {
             t.dependsOn(project.getTasks().withType(EmptyDirTask.class));
             t.setIncludeEmptyDirs(true);
-            t.setDirMode(0755);
-            t.setFileMode(0644);
+            t.dirPermissions(permissions -> permissions.unix(0755));
+            t.filePermissions(permissions -> permissions.unix(0644));
         });
 
         // common config across all archives

diff --git a/...l/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java b/...l/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java
@@ -23,16 +23,17 @@
 import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.GradleException;
-import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.artifacts.dsl.DependencyHandler;
 import org.gradle.api.provider.Provider;
 
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.function.Function;
 
-import static org.elasticsearch.gradle.util.GradleUtils.projectDependency;
-
 /**
  * An internal elasticsearch build plugin that registers additional
  * distribution resolution strategies to the 'elasticsearch.download-distribution' plugin
@@ -64,18 +65,18 @@ public void apply(Project project) {
      * <p>
      * BWC versions are resolved as project to projects under `:distribution:bwc`.
      */
-    private void registerInternalDistributionResolutions(NamedDomainObjectContainer<DistributionResolution> resolutions) {
-        resolutions.register("localBuild", distributionResolution -> distributionResolution.setResolver((project, distribution) -> {
-            if (VersionProperties.getElasticsearch().equals(distribution.getVersion())) {
+    private void registerInternalDistributionResolutions(List<DistributionResolution> resolutions) {
+        resolutions.add(new DistributionResolution("local-build", (project, distribution) -> {
+            if (isCurrentVersion(distribution)) {
                 // non-external project, so depend on local build
                 return new ProjectBasedDistributionDependency(
-                    config -> projectDependency(project, distributionProjectPath(distribution), config)
+                    config -> projectDependency(project.getDependencies(), distributionProjectPath(distribution), config)
                 );
             }
             return null;
         }));
 
-        resolutions.register("bwc", distributionResolution -> distributionResolution.setResolver((project, distribution) -> {
+        resolutions.add(new DistributionResolution("bwc", (project, distribution) -> {
             BwcVersions.UnreleasedVersionInfo unreleasedInfo = BuildParams.getBwcVersions()
                 .unreleasedInfo(Version.fromString(distribution.getVersion()));
             if (unreleasedInfo != null) {
@@ -89,13 +90,19 @@ private void registerInternalDistributionResolutions(NamedDomainObjectContainer<
                 }
                 String projectConfig = getProjectConfig(distribution, unreleasedInfo);
                 return new ProjectBasedDistributionDependency(
-                    (config) -> projectDependency(project, unreleasedInfo.gradleProjectPath, projectConfig)
+                    (config) -> projectDependency(project.getDependencies(), unreleasedInfo.gradleProjectPath, projectConfig)
                 );
             }
             return null;
         }));
     }
 
+    private boolean isCurrentVersion(ElasticsearchDistribution distribution) {
+        Version currentVersionNumber = Version.fromString(VersionProperties.getElasticsearch());
+        Version parsedDistVersionNumber = Version.fromString(distribution.getVersion());
+        return currentVersionNumber.equals(parsedDistVersionNumber);
+    }
+
     /**
      * Will be removed once this is backported to all unreleased branches.
      */
@@ -110,6 +117,13 @@ private static String getProjectConfig(ElasticsearchDistribution distribution, B
         }
     }
 
+    private static Dependency projectDependency(DependencyHandler dependencyHandler, String projectPath, String projectConfig) {
+        Map<String, Object> depConfig = new HashMap<>();
+        depConfig.put("path", projectPath);
+        depConfig.put("configuration", projectConfig);
+        return dependencyHandler.project(depConfig);
+    }
+
     private static String distributionProjectPath(ElasticsearchDistribution distribution) {
         String projectPath = ":distribution";
         if (distribution.getType() == ElasticsearchDistributionTypes.INTEG_TEST_ZIP) {

diff --git a/...s-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java b/...s-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java
@@ -152,7 +152,7 @@ private void visitSymbolicLink(final FileCopyDetailsInternal details) {
                 visitedSymbolicLinks.add(details.getFile());
                 final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString(), TarConstants.LF_SYMLINK);
                 entry.setModTime(getModTime(details));
-                entry.setMode(UnixStat.LINK_FLAG | details.getMode());
+                entry.setMode(UnixStat.LINK_FLAG | details.getPermissions().toUnixNumeric());
                 try {
                     entry.setLinkName(Files.readSymbolicLink(details.getFile().toPath()).toString());
                     tar.putArchiveEntry(entry);
@@ -165,7 +165,7 @@ private void visitSymbolicLink(final FileCopyDetailsInternal details) {
             private void visitDirectory(final FileCopyDetailsInternal details) {
                 final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString() + "/");
                 entry.setModTime(getModTime(details));
-                entry.setMode(UnixStat.DIR_FLAG | details.getMode());
+                entry.setMode(UnixStat.DIR_FLAG | details.getPermissions().toUnixNumeric());
                 try {
                     tar.putArchiveEntry(entry);
                     tar.closeArchiveEntry();
@@ -177,7 +177,7 @@ private void visitDirectory(final FileCopyDetailsInternal details) {
             private void visitFile(final FileCopyDetailsInternal details) {
                 final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString());
                 entry.setModTime(getModTime(details));
-                entry.setMode(UnixStat.FILE_FLAG | details.getMode());
+                entry.setMode(UnixStat.FILE_FLAG | details.getPermissions().toUnixNumeric());
                 entry.setSize(details.getSize());
                 try {
                     tar.putArchiveEntry(entry);

diff --git a/...-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java b/...-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java
@@ -262,7 +262,7 @@ private List<JavaHome> getAvailableJavaVersions() {
     private Stream<InstallationLocation> getAvailableJavaInstallationLocationSteam() {
         return Stream.concat(
             javaInstallationRegistry.toolchains().stream().map(metadata -> metadata.location),
-            Stream.of(new InstallationLocation(Jvm.current().getJavaHome(), "Current JVM"))
+            Stream.of(InstallationLocation.userDefined(Jvm.current().getJavaHome(), "Current JVM"))
         );
     }
 

diff --git a/...n/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java b/...n/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java
@@ -12,27 +12,23 @@
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
-import org.gradle.api.artifacts.Configuration;
-import org.gradle.api.artifacts.ProjectDependency;
+import org.gradle.api.artifacts.component.ComponentIdentifier;
+import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
 import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.specs.Spec;
 import org.gradle.api.tasks.TaskProvider;
 
 public class DependencyLicensesPrecommitPlugin extends PrecommitPlugin {
+    private static Spec<ComponentIdentifier> COMPONENT_FILTER = identifier -> (identifier instanceof ModuleComponentIdentifier)
+        && ((ModuleComponentIdentifier) identifier).getGroup().startsWith("org.elasticsearch") == false;
 
     @Override
     public TaskProvider<? extends Task> createTask(Project project) {
         project.getPlugins().apply(CompileOnlyResolvePlugin.class);
-        TaskProvider<DependencyLicensesTask> dependencyLicenses = project.getTasks()
-            .register("dependencyLicenses", DependencyLicensesTask.class);
-
-        // only require dependency licenses for non-elasticsearch deps
-        dependencyLicenses.configure(t -> {
-            Configuration runtimeClasspath = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
-            Configuration compileOnly = project.getConfigurations()
-                .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
-            t.setDependencies(
-                runtimeClasspath.fileCollection(dependency -> dependency instanceof ProjectDependency == false).minus(compileOnly)
-            );
+        var dependencyLicenses = project.getTasks().register("dependencyLicenses", DependencyLicensesTask.class, t -> {
+            var runtimeClasspath = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
+            var compileOnly = project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
+            t.configureDependencies(runtimeClasspath, compileOnly, COMPONENT_FILTER);
         });
         return dependencyLicenses;
     }

diff --git a/...nal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java b/...nal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java
@@ -11,14 +11,18 @@
 import org.gradle.api.DefaultTask;
 import org.gradle.api.GradleException;
 import org.gradle.api.InvalidUserDataException;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.component.ComponentIdentifier;
 import org.gradle.api.file.Directory;
 import org.gradle.api.file.DirectoryProperty;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
+import org.gradle.api.specs.Spec;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
 import org.gradle.api.tasks.InputFiles;
@@ -41,6 +45,8 @@
 
 import javax.inject.Inject;
 
+import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView;
+
 /**
  * A task to check licenses for dependencies.
  * <p>
@@ -83,7 +89,7 @@
  * for the dependency. This artifact will be redistributed by us with the release to
  * comply with the license terms.
  */
-public class DependencyLicensesTask extends DefaultTask {
+public abstract class DependencyLicensesTask extends DefaultTask {
 
     private final Pattern regex = Pattern.compile("-v?\\d+.*");
 
@@ -183,6 +189,10 @@ public void ignoreFile(String file) {
         ignoreFiles.add(file);
     }
 
+    @Input
+    @Optional
+    public abstract Property<Spec<ComponentIdentifier>> getComponentFilter();
+
     @TaskAction
     public void checkDependencies() {
         if (dependencies == null) {
@@ -297,7 +307,6 @@ private String getFileName(String name, Map<String, ?> counters, String type) {
             // try the other suffix...TODO: get rid of this, just support ending in .txt
             return fileName + ".txt";
         }
-
         return fileName;
     }
 
@@ -312,4 +321,15 @@ public LinkedHashMap<String, String> getMappings() {
         return new LinkedHashMap<>(mappings);
     }
 
+    /**
+     * Convencience method for configuring dependencies to be checked and ignoring transitive dependencies for now.
+     * */
+    public void configureDependencies(
+        Configuration plusConfiguration,
+        Configuration minusConfiguration,
+        Spec<ComponentIdentifier> componentFilter
+    ) {
+        setDependencies(createFileCollectionFromNonTransitiveArtifactsView(plusConfiguration, componentFilter).minus(minusConfiguration));
+    }
+
 }
diff --git a/...main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java b/...main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
@@ -15,11 +15,14 @@
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
 import org.gradle.api.tasks.TaskProvider;
 
 import java.io.File;
 import java.nio.file.Path;
 
+import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView;
+
 public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
 
     public static final String JDK_JAR_HELL_CONFIG_NAME = "jdkJarHell";
@@ -54,12 +57,14 @@ public TaskProvider<? extends Task> createTask(Project project) {
             Configuration compileOnly = project.getConfigurations()
                 .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
             t.setClasspath(runtimeConfiguration.plus(compileOnly));
-            t.getJarsToScan().from(runtimeConfiguration.fileCollection(dep -> {
-                // These are SelfResolvingDependency, and some of them backed by file collections, like the Gradle API files,
-                // or dependencies added as `files(...)`, we can't be sure if those are third party or not.
-                // err on the side of scanning these to make sure we don't miss anything
-                return dep.getGroup() != null && dep.getGroup().startsWith("org.elasticsearch") == false;
-            }));
+            t.getJarsToScan()
+                .from(
+                    createFileCollectionFromNonTransitiveArtifactsView(
+                        runtimeConfiguration,
+                        identifier -> identifier instanceof ModuleComponentIdentifier
+                            && ((ModuleComponentIdentifier) identifier).getGroup().startsWith("org.elasticsearch") == false
+                    )
+                );
             t.dependsOn(resourcesTask);
             if (BuildParams.getIsRuntimeJavaHomeSet()) {
                 t.getJavaHome().set(project.provider(BuildParams::getRuntimeJavaHome).map(File::getPath));

diff --git a/...tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java b/...tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java
@@ -123,7 +123,6 @@ public void apply(Project project) {
             String taskname = destructiveDistroTestTaskName(distribution);
             TaskProvider<?> depsTask = project.getTasks().register(taskname + "#deps");
             // explicitly depend on the archive not on the implicit extracted distribution
-            depsTask.configure(t -> t.dependsOn(distribution.getArchiveDependencies()));
             depsTask.configure(t -> t.dependsOn(examplePlugin.getDependencies()));
             depsTasks.put(taskname, depsTask);
             TaskProvider<Test> destructiveTask = configureTestTask(project, taskname, distribution, t -> {
@@ -447,6 +446,7 @@ private static ElasticsearchDistribution createDistro(
             if (isDocker == false) {
                 d.setBundledJdk(bundledJdk);
             }
+            d.setPreferArchive(true);
             d.setVersion(version);
         });