From 55ed2de0dc9bdbdece9fe778919f1f71a3a66491 Mon Sep 17 00:00:00 2001 From: Hannes Wellmann Date: Sun, 3 Sep 2023 23:58:41 +0200 Subject: [PATCH] Support assembling and archiving repositories in parallel Assembling and arching repositories is a relatively long running task and repository-projects are usually build as one of the last projects in the reactor. So the chances are relatively high, that (if multiple projects are present) multiple repositories are assembled at the same. Therefore it should be possible to build multiple repositories in parallel and there should not be one global lock for each repository-related mojo. --- .../p2/repository/AbstractRepositoryMojo.java | 22 +++ .../p2/repository/ArchiveRepositoryMojo.java | 27 ++-- .../p2/repository/AssembleRepositoryMojo.java | 147 +++++++++--------- .../FixArtifactsMetadataMetadataMojo.java | 24 ++- .../RemapArtifactToMavenRepositoriesMojo.java | 10 +- 5 files changed, 122 insertions(+), 108 deletions(-) diff --git a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AbstractRepositoryMojo.java b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AbstractRepositoryMojo.java index 295089b6e8..8e5712123b 100644 --- a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AbstractRepositoryMojo.java +++ b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AbstractRepositoryMojo.java @@ -13,11 +13,33 @@ package org.eclipse.tycho.plugins.p2.repository; import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.annotations.Parameter; import org.eclipse.tycho.core.maven.AbstractP2Mojo; public abstract class AbstractRepositoryMojo extends AbstractP2Mojo { + private static final Map REPOSITORY_LOCK = new ConcurrentHashMap<>(); + + interface Locking extends AutoCloseable { + @Override + void close(); // Don't throw an Exception + } + + static Locking lockFor(File file) throws MojoExecutionException { + try { + Lock lock = REPOSITORY_LOCK.computeIfAbsent(file.getCanonicalFile(), k -> new ReentrantLock()); + lock.lock(); + return lock::unlock; + } catch (IOException e) { + throw new MojoExecutionException("Error while canonicalizing repository location", e); + } + } @Parameter private File repositoryLocation; diff --git a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/ArchiveRepositoryMojo.java b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/ArchiveRepositoryMojo.java index edf83d6e73..e42ec26942 100644 --- a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/ArchiveRepositoryMojo.java +++ b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/ArchiveRepositoryMojo.java @@ -33,7 +33,6 @@ */ @Mojo(name = "archive-repository", defaultPhase = LifecyclePhase.PACKAGE, threadSafe = true) public final class ArchiveRepositoryMojo extends AbstractRepositoryMojo { - private static final Object LOCK = new Object(); @Component(role = Archiver.class, hint = "zip") private Archiver inflater; @@ -57,22 +56,18 @@ public void execute() throws MojoExecutionException, MojoFailureException { if (skipArchive) { return; } - - synchronized (LOCK) { - File destFile = getBuildDirectory().getChild(finalName + ".zip"); - - try { - inflater.addFileSet(DefaultFileSet.fileSet(getAssemblyRepositoryLocation()).prefixed("")); - inflater.setDestFile(destFile); - inflater.createArchive(); - } catch (ArchiverException e) { - throw new MojoExecutionException("Error packing p2 repository", e); - } catch (IOException e) { - throw new MojoExecutionException("Error packing p2 repository", e); - } - - getProject().getArtifact().setFile(destFile); + File repositoryLocation = getAssemblyRepositoryLocation(); + File destFile = getBuildDirectory().getChild(finalName + ".zip"); + try (var repoLock = lockFor(repositoryLocation); var destLock = lockFor(destFile);) { + inflater.addFileSet(DefaultFileSet.fileSet(repositoryLocation).prefixed("")); + inflater.setDestFile(destFile); + inflater.createArchive(); + } catch (ArchiverException e) { + throw new MojoExecutionException("Error packing p2 repository", e); + } catch (IOException e) { + throw new MojoExecutionException("Error packing p2 repository", e); } + getProject().getArtifact().setFile(destFile); } } diff --git a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java index 6461550af3..7987c279e5 100644 --- a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java +++ b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java @@ -79,7 +79,6 @@ public static class RepositoryReferenceFilter { List exclude = List.of(); } - private static final Object LOCK = new Object(); /** *

* By default, this goal creates a p2 repository. Set this to false if only a p2 @@ -290,87 +289,85 @@ public static class RepositoryReferenceFilter { @Override public void execute() throws MojoExecutionException, MojoFailureException { - synchronized (LOCK) { - try { - File destination = getAssemblyRepositoryLocation(); - destination.mkdirs(); - copyResources(destination); - - final ReactorProject reactorProject = getReactorProject(); - Collection projectSeeds = TychoProjectUtils.getDependencySeeds(reactorProject); - if (projectSeeds.isEmpty()) { - getLog().warn("No content specified for p2 repository"); - return; - } + File destination = getAssemblyRepositoryLocation(); + try (var locking = lockFor(destination)) { + destination.mkdirs(); + copyResources(destination); + + final ReactorProject reactorProject = getReactorProject(); + Collection projectSeeds = TychoProjectUtils.getDependencySeeds(reactorProject); + if (projectSeeds.isEmpty()) { + getLog().warn("No content specified for p2 repository"); + return; + } - reactorProject.setContextValue(TychoConstants.CTX_METADATA_ARTIFACT_LOCATION, categoriesDirectory); - RepositoryReferences sources = repositoryReferenceTool.getVisibleRepositories(getProject(), - getSession(), RepositoryReferenceTool.REPOSITORIES_INCLUDE_CURRENT_MODULE); - sources.setTargetPlatform(TychoProjectUtils.getTargetPlatform(getReactorProject())); - - List repositoryReferences = getCategories(categoriesDirectory).stream()// - .map(Category::getRepositoryReferences)// - .flatMap(List::stream)// - .map(ref -> new RepositoryReference(ref.getName(), ref.getLocation(), ref.isEnabled()))// - .collect(Collectors.toCollection(ArrayList::new)); - Predicate autoReferencesFilter = buildRepositoryReferenceLocationFilter(); - if (addPomRepositoryReferences) { - getProject().getRepositories().stream() // - .filter(pomRepo -> "p2".equals(pomRepo.getLayout())) - .filter(pomRepo -> autoReferencesFilter.test(pomRepo.getUrl())) - .map(pomRepo -> new RepositoryReference(pomRepo.getName(), pomRepo.getUrl(), true)) - .forEach(repositoryReferences::add); - } - if (addIUTargetRepositoryReferences) { - projectManager.getTargetPlatformConfiguration(getProject()).getTargets().stream() - .flatMap(tpFile -> tpFile.getLocations().stream()) - .filter(InstallableUnitLocation.class::isInstance).map(InstallableUnitLocation.class::cast) - .flatMap(iu -> iu.getRepositories().stream()) - .filter(iuRepo -> autoReferencesFilter.test(iuRepo.getLocation())) - .map(iuRepo -> new RepositoryReference(null, iuRepo.getLocation(), true)) - .forEach(repositoryReferences::add); + reactorProject.setContextValue(TychoConstants.CTX_METADATA_ARTIFACT_LOCATION, categoriesDirectory); + RepositoryReferences sources = repositoryReferenceTool.getVisibleRepositories(getProject(), getSession(), + RepositoryReferenceTool.REPOSITORIES_INCLUDE_CURRENT_MODULE); + sources.setTargetPlatform(TychoProjectUtils.getTargetPlatform(getReactorProject())); + + List repositoryReferences = getCategories(categoriesDirectory).stream()// + .map(Category::getRepositoryReferences)// + .flatMap(List::stream)// + .map(ref -> new RepositoryReference(ref.getName(), ref.getLocation(), ref.isEnabled()))// + .collect(Collectors.toCollection(ArrayList::new)); + Predicate autoReferencesFilter = buildRepositoryReferenceLocationFilter(); + if (addPomRepositoryReferences) { + getProject().getRepositories().stream() // + .filter(pomRepo -> "p2".equals(pomRepo.getLayout())) + .filter(pomRepo -> autoReferencesFilter.test(pomRepo.getUrl())) + .map(pomRepo -> new RepositoryReference(pomRepo.getName(), pomRepo.getUrl(), true)) + .forEach(repositoryReferences::add); + } + if (addIUTargetRepositoryReferences) { + projectManager.getTargetPlatformConfiguration(getProject()).getTargets().stream() + .flatMap(tpFile -> tpFile.getLocations().stream()) + .filter(InstallableUnitLocation.class::isInstance).map(InstallableUnitLocation.class::cast) + .flatMap(iu -> iu.getRepositories().stream()) + .filter(iuRepo -> autoReferencesFilter.test(iuRepo.getLocation())) + .map(iuRepo -> new RepositoryReference(null, iuRepo.getLocation(), true)) + .forEach(repositoryReferences::add); + } + DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(destination, + repositoryName, compress, xzCompress, keepNonXzIndexFiles, !createArtifactRepository, true, + extraArtifactRepositoryProperties, repositoryReferences); + mirrorApp.mirrorReactor(sources, destinationRepoDescriptor, projectSeeds, getBuildContext(), + includeAllDependencies, includeAllSources, includeRequiredPlugins, includeRequiredFeatures, + filterProvided, profileProperties); + if (generateOSGiRepository) { + XMLResourceGenerator resourceGenerator = new XMLResourceGenerator(); + resourceGenerator.name(repositoryName); + resourceGenerator.base(destination.toURI()); + File plugins = new File(destination, "plugins"); + if (plugins.isDirectory()) { + File[] files = plugins.listFiles(path -> path.getName().endsWith(".jar") && path.isFile()); + try { + resourceGenerator.repository(new FileSetRepository("plugins", Arrays.asList(files))); + } catch (Exception e) { + throw new MojoExecutionException("Could not read p2 repository plugins", e); + } } - DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor( - destination, repositoryName, compress, xzCompress, keepNonXzIndexFiles, - !createArtifactRepository, true, extraArtifactRepositoryProperties, repositoryReferences); - mirrorApp.mirrorReactor(sources, destinationRepoDescriptor, projectSeeds, getBuildContext(), - includeAllDependencies, includeAllSources, includeRequiredPlugins, includeRequiredFeatures, - filterProvided, profileProperties); - if (generateOSGiRepository) { - XMLResourceGenerator resourceGenerator = new XMLResourceGenerator(); - resourceGenerator.name(repositoryName); - resourceGenerator.base(destination.toURI()); - File plugins = new File(destination, "plugins"); - if (plugins.isDirectory()) { - File[] files = plugins.listFiles(path -> path.getName().endsWith(".jar") && path.isFile()); + File features = new File(destination, "features"); + if (features.isDirectory()) { + File[] files = features.listFiles(path -> path.getName().endsWith(".jar") && path.isFile()); + for (File featureFile : files) { try { - resourceGenerator.repository(new FileSetRepository("plugins", Arrays.asList(files))); - } catch (Exception e) { - throw new MojoExecutionException("Could not read p2 repository plugins", e); - } - } - File features = new File(destination, "features"); - if (features.isDirectory()) { - File[] files = features.listFiles(path -> path.getName().endsWith(".jar") && path.isFile()); - for (File featureFile : files) { - try { - Feature feature = Feature.readJar(featureFile); - feature.toResource().forEach(resourceGenerator::resource); - } catch (IOException e) { - throw new MojoExecutionException("Could not read feature " + featureFile, e); - } + Feature feature = Feature.readJar(featureFile); + feature.toResource().forEach(resourceGenerator::resource); + } catch (IOException e) { + throw new MojoExecutionException("Could not read feature " + featureFile, e); } } - try { - String filename = compress ? repositoryFileName + ".gz" : repositoryFileName; - resourceGenerator.save(new File(destination, filename)); - } catch (IOException e) { - throw new MojoExecutionException("Could not write OSGi Repository!", e); - } } - } catch (FacadeException e) { - throw new MojoExecutionException("Could not assemble p2 repository", e); + try { + String filename = compress ? repositoryFileName + ".gz" : repositoryFileName; + resourceGenerator.save(new File(destination, filename)); + } catch (IOException e) { + throw new MojoExecutionException("Could not write OSGi Repository!", e); + } } + } catch (FacadeException e) { + throw new MojoExecutionException("Could not assemble p2 repository", e); } } diff --git a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/FixArtifactsMetadataMetadataMojo.java b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/FixArtifactsMetadataMetadataMojo.java index b63d97b0df..dc1a12475a 100644 --- a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/FixArtifactsMetadataMetadataMojo.java +++ b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/FixArtifactsMetadataMetadataMojo.java @@ -36,7 +36,7 @@ */ @Mojo(name = "fix-artifacts-metadata", defaultPhase = LifecyclePhase.PREPARE_PACKAGE, threadSafe = true) public class FixArtifactsMetadataMetadataMojo extends AbstractRepositoryMojo { - private static final Object LOCK = new Object(); + @Parameter(defaultValue = "${project.name}") private String repositoryName; @@ -64,19 +64,17 @@ public class FixArtifactsMetadataMetadataMojo extends AbstractRepositoryMojo { @Override public void execute() throws MojoExecutionException, MojoFailureException { - synchronized (LOCK) { - try { - File destination = getAssemblyRepositoryLocation(); - if (!destination.isDirectory()) { - throw new MojoExecutionException( - "Could not update p2 repository, directory does not exist: " + destination); - } - DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor( - destination, repositoryName, true, xzCompress, keepNonXzIndexFiles, false, true); - mirrorApp.recreateArtifactRepository(destinationRepoDescriptor); - } catch (FacadeException e) { - throw new MojoExecutionException("Could not update p2 repository", e); + File destination = getAssemblyRepositoryLocation(); + try (var locking = lockFor(destination)) { + if (!destination.isDirectory()) { + throw new MojoExecutionException( + "Could not update p2 repository, directory does not exist: " + destination); } + DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(destination, + repositoryName, true, xzCompress, keepNonXzIndexFiles, false, true); + mirrorApp.recreateArtifactRepository(destinationRepoDescriptor); + } catch (FacadeException e) { + throw new MojoExecutionException("Could not update p2 repository", e); } } diff --git a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/RemapArtifactToMavenRepositoriesMojo.java b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/RemapArtifactToMavenRepositoriesMojo.java index 0600d4b007..fe3d515ecd 100644 --- a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/RemapArtifactToMavenRepositoriesMojo.java +++ b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/RemapArtifactToMavenRepositoriesMojo.java @@ -9,6 +9,7 @@ *******************************************************************************/ package org.eclipse.tycho.plugins.p2.repository; +import java.io.File; import java.net.URI; import org.apache.maven.artifact.repository.ArtifactRepository; @@ -25,7 +26,7 @@ * artifacts the can be resolved to Maven repositories so the URL under Maven repository is used for * fetching and artifact is not duplicated inside this repo. */ -@Mojo(name = "remap-artifacts-to-m2-repo", defaultPhase = LifecyclePhase.PREPARE_PACKAGE) +@Mojo(name = "remap-artifacts-to-m2-repo", defaultPhase = LifecyclePhase.PREPARE_PACKAGE, threadSafe = true) public class RemapArtifactToMavenRepositoriesMojo extends AbstractRepositoryMojo { @Component() @@ -33,9 +34,10 @@ public class RemapArtifactToMavenRepositoriesMojo extends AbstractRepositoryMojo @Override public void execute() throws MojoExecutionException, MojoFailureException { - try { - mirrorApp.addMavenMappingRules(getAssemblyRepositoryLocation(), - getProject().getRemoteArtifactRepositories().stream() // + File location = getAssemblyRepositoryLocation(); + try (var locking = lockFor(location)) { + mirrorApp.addMavenMappingRules( // + location, getProject().getRemoteArtifactRepositories().stream() // .filter(artifactRepo -> artifactRepo.getLayout().getId().equals("default")) // .map(ArtifactRepository::getUrl) // .map(URI::create) //