diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java index 8481a0e76c..fc91096441 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java @@ -101,7 +101,7 @@ public void mirrorStandalone(RepositoryReferences sources, DestinationRepository Collection seedIUs, MirrorOptions mirrorOptions, BuildDirectory tempDirectory) throws FacadeException { agent.getService(IArtifactRepositoryManager.class); //force init of framework if not already done! - final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent); + final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent, logger); mirrorApp.setSlicingOptions(createSlicingOptions(mirrorOptions)); mirrorApp.setIgnoreErrors(mirrorOptions.isIgnoreErrors()); try { @@ -176,7 +176,7 @@ public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDes boolean includeAllSource, boolean includeRequiredBundles, boolean includeRequiredFeatures, boolean filterProvided, boolean addOnlyProvidingRepoReferences, Map filterProperties) throws FacadeException { - final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent); + final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent, logger); // mirror scope: seed units... try { @@ -250,8 +250,8 @@ public void recreateArtifactRepository(DestinationRepositoryDescriptor destinati } private static TychoMirrorApplication createMirrorApplication(RepositoryReferences sources, - DestinationRepositoryDescriptor destination, IProvisioningAgent agent) { - final TychoMirrorApplication mirrorApp = new TychoMirrorApplication(agent, destination); + DestinationRepositoryDescriptor destination, IProvisioningAgent agent, Logger logger) { + final TychoMirrorApplication mirrorApp = new TychoMirrorApplication(agent, destination, logger); mirrorApp.setRaw(false); List sourceDescriptors = createSourceDescriptors(sources); diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java index 4b0add9a8b..f00bcdfa0f 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java @@ -16,11 +16,9 @@ import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.mapping; -import static java.util.stream.Collectors.toList; import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.HashMap; @@ -32,6 +30,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import org.codehaus.plexus.logging.Logger; import org.eclipse.core.runtime.IProgressMonitor; import org.eclipse.equinox.internal.p2.metadata.IRequiredCapability; import org.eclipse.equinox.internal.p2.metadata.InstallableUnit; @@ -62,12 +61,9 @@ import org.eclipse.tycho.p2tools.copiedfromp2.PermissiveSlicer; import org.eclipse.tycho.p2tools.copiedfromp2.RepositoryDescriptor; import org.eclipse.tycho.p2tools.copiedfromp2.Slicer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class TychoMirrorApplication extends org.eclipse.tycho.p2tools.copiedfromp2.MirrorApplication { - private static final Logger LOGGER = LoggerFactory.getLogger(TychoMirrorApplication.class); private static final String SOURCE_SUFFIX = ".source"; private static final String FEATURE_GROUP = ".feature.group"; private final DestinationRepositoryDescriptor destination; @@ -77,10 +73,13 @@ public class TychoMirrorApplication extends org.eclipse.tycho.p2tools.copiedfrom private boolean filterProvided; private boolean addOnlyProvidingRepoReferences; private TargetPlatform targetPlatform; + private Logger logger; - public TychoMirrorApplication(IProvisioningAgent agent, DestinationRepositoryDescriptor destination) { + public TychoMirrorApplication(IProvisioningAgent agent, DestinationRepositoryDescriptor destination, + Logger logger) { super(agent); this.destination = destination; + this.logger = logger; this.removeAddedRepositories = false; } @@ -274,8 +273,8 @@ protected void finalizeRepositories() { if (repository != null) { Collection references = repository.getReferences(); if (!references.isEmpty()) { - LOGGER.info("Adding references to the following repositories:"); - references.stream().map(r -> r.getLocation()).distinct().forEach(loc -> LOGGER.info(" {}", loc)); + logger.info("Adding references to the following repositories:"); + references.stream().map(r -> r.getLocation()).distinct().forEach(loc -> logger.info(" " + loc)); } } super.finalizeRepositories(); @@ -296,11 +295,12 @@ protected Set collectUnits(IQueryable slice, throws ProvisionException { Set units = super.collectUnits(slice, monitor); if (isFilterProvidedItems()) { - Map> fullRepositoryContent = units.stream() - .collect(groupingBy(IInstallableUnit::getId, mapping(IInstallableUnit::getVersion, toList()))); + Map> fullRepositoryContent = units.stream().collect( + groupingBy(IInstallableUnit::getId, mapping(IInstallableUnit::getVersion, Collectors.toSet()))); - List> metadataRepositories = removeProvidedItems(units, - getMetadataRepositoryManager(), IRepository.TYPE_METADATA, monitor); + IMetadataRepositoryManager manager = getMetadataRepositoryManager(); + Map> metadataRepositories = removeProvidedItems(units, manager, + IRepository.TYPE_METADATA, monitor); if (addOnlyProvidingRepoReferences) { Set removableReferences = destination.getFilterableRepositoryReferences().stream() @@ -309,7 +309,8 @@ protected Set collectUnits(IQueryable slice, .forEach(removableReferences::remove); // keep reference if explicitly added to the repository if (!removableReferences.isEmpty()) { // Assume that for all units that correspond to artifacts the metadata either has a co-located artifact repository or a references to to one that contains it. - removeNotProvidingReferences(fullRepositoryContent, metadataRepositories, removableReferences); + removeNotProvidingReferences(fullRepositoryContent, metadataRepositories, removableReferences, + manager); } } } @@ -320,9 +321,9 @@ private boolean isFilterProvidedItems() { return filterProvided && !destinationMetadataRepository.getReferences().isEmpty(); } - private List> removeProvidedItems(Collection allElements, IRepositoryManager repoManager, - int repositoryType, IProgressMonitor monitor) throws ProvisionException { - List> referencedRepositories = new ArrayList<>(); + private Map> removeProvidedItems(Collection allElements, + IRepositoryManager repoManager, int repositoryType, IProgressMonitor monitor) throws ProvisionException { + Map> referencedRepositories = new HashMap<>(); for (IRepositoryReference reference : destinationMetadataRepository.getReferences()) { if (reference.getType() != repositoryType) { continue; @@ -330,7 +331,8 @@ private List> removeProvidedItems(Collection allElements, try { URI location = reference.getLocation(); IRepository repository = repoManager.loadRepository(location, monitor); - referencedRepositories.add(repository); + //We need to retain the location in the map as the repo manager might rewrite it to a mirrored location + referencedRepositories.put(location, repository); } catch (IllegalArgumentException e) { if (e.getCause() instanceof URISyntaxException uriException) { throw new ProvisionException("Can't parse referenced URI!", uriException); @@ -339,28 +341,46 @@ private List> removeProvidedItems(Collection allElements, } } } - allElements.removeIf(e -> referencedRepositories.stream().anyMatch(repo -> repo.contains(e))); + allElements.removeIf(e -> referencedRepositories.values().stream().anyMatch(repo -> repo.contains(e))); return referencedRepositories; } - private void removeNotProvidingReferences(Map> fullRepositoryContent, - List> metadataRepositories, Set removableReferenceURIs) { + private void removeNotProvidingReferences(Map> fullRepositoryContent, + Map> metadataRepositories, Set removableReferenceURIs, + IMetadataRepositoryManager manager) { Map> usedRepositoryItems = new HashMap<>(); - for (IRepository repo : metadataRepositories) { - Set usedRepoContent = repo.query(QueryUtil.ALL_UNITS, null).stream() - .filter(a -> fullRepositoryContent.getOrDefault(a.getId(), List.of()).contains(a.getVersion())) + Map> providedItems = new HashMap<>(); + for (Entry> repo : metadataRepositories.entrySet()) { + Set content = getRepositoryContent(repo.getKey(), repo.getValue(), new HashSet<>(), + manager).collect(Collectors.toSet()); + Set usedRepoContent = content.stream() + .filter(a -> fullRepositoryContent.getOrDefault(a.getId(), Set.of()).contains(a.getVersion())) .collect(Collectors.toSet()); - usedRepositoryItems.put(repo.getLocation(), usedRepoContent); + usedRepositoryItems.put(repo.getKey(), usedRepoContent); + providedItems.put(repo.getKey(), content); } // Remove filterable references that contribute nothing or whose relevant content is also provided by another repo usedRepositoryItems.entrySet().removeIf(repo -> { - if (!removableReferenceURIs.contains(repo.getKey())) { - return false; + if (removableReferenceURIs.contains(repo.getKey())) { + Set usedContent = repo.getValue(); + if (usedContent.isEmpty()) { + logger.info( + "Remove reference " + repo.getKey() + " because no units are contained in the repository."); + return true; + } + for (Entry> entry : usedRepositoryItems.entrySet()) { + if (entry == repo) { + continue; + } + Set other = providedItems.getOrDefault(entry.getKey(), Set.of()); + if (!other.isEmpty() && other.containsAll(usedContent)) { + logger.info("Remove reference " + repo.getKey() + + " because all units are also contained in reference " + entry.getKey() + " already."); + return true; + } + } } - Set usedContent = repo.getValue(); - return usedContent.isEmpty() - || usedRepositoryItems.entrySet().stream().filter(e -> e != repo).map(Entry::getValue) - .anyMatch(other -> other.size() >= usedContent.size() && other.containsAll(usedContent)); + return false; }); IMetadataRepository repository = getDestinationMetadataRepository(); List discardedReferences = repository.getReferences().stream() @@ -368,6 +388,34 @@ private void removeNotProvidingReferences(Map> fullReposit repository.removeReferences(discardedReferences); } + private Stream getRepositoryContent(URI uri, IRepository repo, Set visited, + IMetadataRepositoryManager manager) { + if (visited.add(uri)) { + Stream stream = repo.query(QueryUtil.ALL_UNITS, null).stream(); + if (repo instanceof IMetadataRepository meta) { + Collection references = meta.getReferences(); + for (IRepositoryReference reference : references) { + if (reference.getType() == IRepository.TYPE_METADATA && isEnabled(reference)) { + try { + URI referenceLocation = reference.getLocation(); + IMetadataRepository referenceRepository = manager.loadRepository(referenceLocation, null); + stream = Stream.concat(stream, + getRepositoryContent(referenceLocation, referenceRepository, visited, manager)); + } catch (ProvisionException e) { + //can't use then... + } + } + } + } + return stream; + } + return Stream.empty(); + } + + private boolean isEnabled(IRepositoryReference reference) { + return (reference.getOptions() & IRepository.ENABLED) != 0; + } + public void setIncludeSources(boolean includeAllSource, TargetPlatform targetPlatform) { this.includeAllSource = includeAllSource; this.targetPlatform = targetPlatform;