From 80cd493af4804cd2b485c4283cfde47a49b3660a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20L=C3=A4ubrich?= Date: Sat, 28 Dec 2024 18:02:55 +0100 Subject: [PATCH] Add support for reading artifacts from global bundle pools Since a while P2 offers a utility method to get access to the shared bundlepools that allow to reuse artifacts already downloaded. This now adds a first implementation of ArtifactDownloadProvider that make use of this shared pools to provide artifacts from the local pools instead of download them through remote sites. --- ...efaultSimpleArtifactRepositoryFactory.java | 36 +++ .../BundlePoolArtifactDownloadProvider.java | 205 ++++++++++++++++++ src/site/markdown/SystemProperties.md | 3 + .../org/eclipse/tycho/TychoConstants.java | 3 + .../tycho/helper/MavenPropertyHelper.java | 14 +- 5 files changed, 260 insertions(+), 1 deletion(-) create mode 100644 p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/repository/DefaultSimpleArtifactRepositoryFactory.java create mode 100644 p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/BundlePoolArtifactDownloadProvider.java diff --git a/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/repository/DefaultSimpleArtifactRepositoryFactory.java b/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/repository/DefaultSimpleArtifactRepositoryFactory.java new file mode 100644 index 0000000000..6b29eef6f4 --- /dev/null +++ b/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/repository/DefaultSimpleArtifactRepositoryFactory.java @@ -0,0 +1,36 @@ +/******************************************************************************* + * Copyright (c) 2024 Christoph Läubrich and others. + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Christoph Läubrich - initial API and implementation + *******************************************************************************/ +package org.eclipse.tycho.p2maven.repository; + +import javax.inject.Inject; +import javax.inject.Named; + +import org.eclipse.equinox.internal.p2.artifact.repository.simple.SimpleArtifactRepositoryFactory; +import org.eclipse.equinox.p2.core.IProvisioningAgent; + +@Named +public class DefaultSimpleArtifactRepositoryFactory extends SimpleArtifactRepositoryFactory { + + private IProvisioningAgent agent; + + @Inject + public DefaultSimpleArtifactRepositoryFactory(IProvisioningAgent agent) { + this.agent = agent; + } + + @Override + protected IProvisioningAgent getAgent() { + return agent; + } + +} diff --git a/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/BundlePoolArtifactDownloadProvider.java b/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/BundlePoolArtifactDownloadProvider.java new file mode 100644 index 0000000000..1b74488f34 --- /dev/null +++ b/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/BundlePoolArtifactDownloadProvider.java @@ -0,0 +1,205 @@ +/******************************************************************************* + * Copyright (c) 2024 Christoph Läubrich and others. + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Christoph Läubrich - initial API and implementation + *******************************************************************************/ +package org.eclipse.tycho.p2maven.transport; + +import java.io.OutputStream; +import java.net.URI; +import java.nio.file.Path; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Stream; + +import javax.inject.Inject; +import javax.inject.Named; + +import org.apache.commons.io.FileUtils; +import org.codehaus.plexus.logging.Logger; +import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.Status; +import org.eclipse.equinox.internal.p2.artifact.repository.simple.SimpleArtifactRepositoryFactory; +import org.eclipse.equinox.internal.p2.repository.DownloadStatus; +import org.eclipse.equinox.internal.p2.repository.helpers.ChecksumHelper; +import org.eclipse.equinox.internal.p2.repository.helpers.RepositoryHelper; +import org.eclipse.equinox.p2.core.ProvisionException; +import org.eclipse.equinox.p2.repository.artifact.IArtifactDescriptor; +import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository; +import org.eclipse.tycho.TychoConstants; +import org.eclipse.tycho.helper.MavenPropertyHelper; +import org.eclipse.tycho.transport.ArtifactDownloadProvider; + +/** + * Provides artifacts already available from the users bundle pools + */ +@Named +public class BundlePoolArtifactDownloadProvider implements ArtifactDownloadProvider { + + private SimpleArtifactRepositoryFactory artifactRepositoryFactory; + private Map repositoryMap = new ConcurrentHashMap<>(); + private TransportCacheConfig cacheConfig; + private Logger logger; + private boolean useSharedPools; + private boolean useWorkspacePools; + private int priority; + + @Inject + public BundlePoolArtifactDownloadProvider(SimpleArtifactRepositoryFactory artifactRepositoryFactory, + TransportCacheConfig cacheConfig, Logger logger, MavenPropertyHelper propertyHelper) { + this.artifactRepositoryFactory = artifactRepositoryFactory; + this.cacheConfig = cacheConfig; + this.logger = logger; + useSharedPools = propertyHelper.getGlobalBooleanProperty("tycho.p2.transport.bundlepools.shared", true); + useWorkspacePools = propertyHelper.getGlobalBooleanProperty("tycho.p2.transport.bundlepools.workspace", true); + priority = propertyHelper.getGlobalIntProperty("tycho.p2.transport.bundlepools.priority", 100); + + } + + @Override + public IStatus downloadArtifact(URI source, OutputStream target, IArtifactDescriptor originalDescriptor) { + return pools().parallel().map(path -> { + IArtifactRepository repository = getRepository(path); + if (repository != null) { + IArtifactDescriptor[] descriptors = repository + .getArtifactDescriptors(originalDescriptor.getArtifactKey()); + for (IArtifactDescriptor descriptor : descriptors) { + if (isMatching(descriptor, originalDescriptor)) { + return new RepositoryCandidate(path, repository, descriptor); + } + } + } + return null; + }).filter(Objects::nonNull).findAny().map(candidate -> { + IArtifactRepository repository = candidate.repository(); + Path path = candidate.path(); + IArtifactDescriptor descriptor = candidate.descriptor(); + if (cacheConfig.isInteractive()) { + logger.info("Reading from " + path + ": " + descriptor.getArtifactKey()); + } + IStatus status = repository.getArtifact(descriptor, target, null); + if (status instanceof DownloadStatus dls) { + if (cacheConfig.isInteractive()) { + logger.info(String.format("Read from %s (%s): %s (%s at %s/s)", repository.getName(), path, + descriptor.getArtifactKey().toExternalForm(), + FileUtils.byteCountToDisplaySize(dls.getFileSize()), + FileUtils.byteCountToDisplaySize(dls.getTransferRate()))); + } + // we don't want to let the mirror getting a high rating just because we have + // actually downloaded this from the disk and not the supplied URL + dls.setTransferRate(0); + } + return status; + }).orElse(Status.CANCEL_STATUS); + } + + /** + * Test if two descriptors have at least one matching hashsum in which case we + * assume they are describing the same artifact and not only have the same + * version/id, this should not happen usually, but as we use global pools here + * it is better to be safe than sorry. + * + * @param repositoryDescriptor the descriptor of the repository we want to use + * @param originalDescriptor the original descriptor queried + * @return true if at least one hashsum matches in both descriptors + */ + private boolean isMatching(IArtifactDescriptor repositoryDescriptor, IArtifactDescriptor originalDescriptor) { + for (Entry entry : originalDescriptor.getProperties().entrySet()) { + String key = entry.getKey(); + if (key.startsWith(TychoConstants.PROP_DOWNLOAD_CHECKSUM_PREFIX)) { + String property = repositoryDescriptor.getProperty(key); + if (property != null) { + String value = entry.getValue(); + return value.equals(property); + } + } + } + if (fileSizeMatch(repositoryDescriptor, originalDescriptor)) { + // if we are here, then it means no download checksums where present for + // comparison and we need to generate one ourself + for (Entry entry : originalDescriptor.getProperties().entrySet()) { + String key = entry.getKey(); + if (key.startsWith(TychoConstants.PROP_DOWNLOAD_CHECKSUM_PREFIX)) { + try { + String algorithm = key.substring(TychoConstants.PROP_DOWNLOAD_CHECKSUM_PREFIX.length()) + .toUpperCase(); + MessageDigest md = MessageDigest.getInstance(algorithm); + try (DigestOutputStream outputStream = new DigestOutputStream(OutputStream.nullOutputStream(), + md)) { + IArtifactRepository repository = repositoryDescriptor.getRepository(); + IStatus status = repository.getArtifact(repositoryDescriptor, outputStream, null); + if (!status.isOK()) { + continue; + } + } + return ChecksumHelper.toHexString(md.digest()).equals(entry.getValue()); + } catch (Exception e) { + // can't check... + } + } + } + } + return false; + } + + private boolean fileSizeMatch(IArtifactDescriptor repositoryDescriptor, IArtifactDescriptor originalDescriptor) { + String originalSize = originalDescriptor.getProperty(IArtifactDescriptor.DOWNLOAD_SIZE); + if (originalSize != null) { + String property = repositoryDescriptor.getProperty(IArtifactDescriptor.DOWNLOAD_SIZE); + if (property != null) { + return originalSize.equals(property); + } + } + // assume true for further processing + return true; + } + + private Stream pools() { + if (useSharedPools) { + if (useWorkspacePools) { + List sharedBundlePools = RepositoryHelper.getSharedBundlePools(); + List workspaceBundlePools = RepositoryHelper.getWorkspaceBundlePools(); + return Stream.concat(sharedBundlePools.stream(), workspaceBundlePools.stream()).distinct(); + } else { + return RepositoryHelper.getSharedBundlePools().stream(); + } + } else if (useWorkspacePools) { + return RepositoryHelper.getWorkspaceBundlePools().stream(); + } else { + return Stream.empty(); + } + } + + private IArtifactRepository getRepository(Path path) { + return repositoryMap.computeIfAbsent(path, p -> { + try { + return artifactRepositoryFactory.load(path.toUri(), 0, null); + } catch (ProvisionException e) { + return null; + } + }); + } + + private static record RepositoryCandidate(Path path, IArtifactRepository repository, + IArtifactDescriptor descriptor) { + + } + + @Override + public int getPriority() { + return priority; + } + +} diff --git a/src/site/markdown/SystemProperties.md b/src/site/markdown/SystemProperties.md index 97371ac5d8..85fcb0ffc3 100644 --- a/src/site/markdown/SystemProperties.md +++ b/src/site/markdown/SystemProperties.md @@ -46,3 +46,6 @@ tycho.p2.transport.cache | file path | local maven repository | Specify the loca tycho.p2.transport.debug | true/false | false | enable debugging of the Tycho Transport tycho.p2.transport.max-download-threads | number | 4 | maximum number of threads that should be used to download artifacts in parallel tycho.p2.transport.min-cache-minutes | number | 60 | Number of minutes that a cache entry is assumed to be fresh and is not fetched again from the server +tycho.p2.transport.bundlepools.priority | number | 100 | priority used for bundle pools +tycho.p2.transport.bundlepools.shared | true/false | true | query shared bundle pools for artifacts before downloading them from remote servers +tycho.p2.transport.bundlepools.workspace | true/false | true | query Workspace bundle pools for artifacts before downloading them from remote servers diff --git a/tycho-api/src/main/java/org/eclipse/tycho/TychoConstants.java b/tycho-api/src/main/java/org/eclipse/tycho/TychoConstants.java index 4595fac7a6..bb40fde78b 100644 --- a/tycho-api/src/main/java/org/eclipse/tycho/TychoConstants.java +++ b/tycho-api/src/main/java/org/eclipse/tycho/TychoConstants.java @@ -17,6 +17,8 @@ import java.io.File; import java.util.regex.Pattern; +import org.eclipse.equinox.p2.repository.artifact.IArtifactDescriptor; + public interface TychoConstants { String USER_HOME = System.getProperty("user.home"); @@ -152,4 +154,5 @@ public interface TychoConstants { String SUFFIX_QUALIFIER = ".qualifier"; String SUFFIX_SNAPSHOT = "-SNAPSHOT"; + String PROP_DOWNLOAD_CHECKSUM_PREFIX = IArtifactDescriptor.DOWNLOAD_CHECKSUM + "."; } diff --git a/tycho-spi/src/main/java/org/eclipse/tycho/helper/MavenPropertyHelper.java b/tycho-spi/src/main/java/org/eclipse/tycho/helper/MavenPropertyHelper.java index dbd6db4847..480f01d409 100644 --- a/tycho-spi/src/main/java/org/eclipse/tycho/helper/MavenPropertyHelper.java +++ b/tycho-spi/src/main/java/org/eclipse/tycho/helper/MavenPropertyHelper.java @@ -90,7 +90,19 @@ public String getGlobalProperty(String key, String defaultValue) { return systemProperty; } } - // java sysem properties last + // java system properties last return System.getProperty(key, defaultValue); } + + public boolean getGlobalBooleanProperty(String key, boolean defaultValue) { + return Boolean.parseBoolean(getGlobalProperty(key, Boolean.toString(defaultValue))); + } + + public int getGlobalIntProperty(String key, int defaultValue) { + try { + return Integer.parseInt(getGlobalProperty(key, Integer.toString(defaultValue))); + } catch (NumberFormatException e) { + return defaultValue; + } + } }