Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[tycho-4.0.x] Use original URL to decide if a IU is provided by reference #3854

Merged
merged 1 commit into from
May 17, 2024
Merged

[tycho-4.0.x] Use original URL to decide if a IU is provided by reference #3854

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public void mirrorStandalone(RepositoryReferences sources, DestinationRepository
Collection<IUDescription> seedIUs, MirrorOptions mirrorOptions, BuildDirectory tempDirectory)
throws FacadeException {
agent.getService(IArtifactRepositoryManager.class); //force init of framework if not already done!
final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent);
final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent, logger);
mirrorApp.setSlicingOptions(createSlicingOptions(mirrorOptions));
mirrorApp.setIgnoreErrors(mirrorOptions.isIgnoreErrors());
try {
Expand Down Expand Up @@ -176,7 +176,7 @@ public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDes
boolean includeAllSource, boolean includeRequiredBundles, boolean includeRequiredFeatures,
boolean filterProvided, boolean addOnlyProvidingRepoReferences, Map<String, String> filterProperties)
throws FacadeException {
final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent);
final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent, logger);

// mirror scope: seed units...
try {
Expand Down Expand Up @@ -250,8 +250,8 @@ public void recreateArtifactRepository(DestinationRepositoryDescriptor destinati
}

private static TychoMirrorApplication createMirrorApplication(RepositoryReferences sources,
DestinationRepositoryDescriptor destination, IProvisioningAgent agent) {
final TychoMirrorApplication mirrorApp = new TychoMirrorApplication(agent, destination);
DestinationRepositoryDescriptor destination, IProvisioningAgent agent, Logger logger) {
final TychoMirrorApplication mirrorApp = new TychoMirrorApplication(agent, destination, logger);
mirrorApp.setRaw(false);

List<RepositoryDescriptor> sourceDescriptors = createSourceDescriptors(sources);
Expand Down
108 changes: 78 additions & 30 deletions tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,9 @@

import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.mapping;
import static java.util.stream.Collectors.toList;

import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
Expand All @@ -32,6 +30,7 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.codehaus.plexus.logging.Logger;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.equinox.internal.p2.metadata.IRequiredCapability;
import org.eclipse.equinox.internal.p2.metadata.InstallableUnit;
Expand Down Expand Up @@ -62,12 +61,9 @@
import org.eclipse.tycho.p2tools.copiedfromp2.PermissiveSlicer;
import org.eclipse.tycho.p2tools.copiedfromp2.RepositoryDescriptor;
import org.eclipse.tycho.p2tools.copiedfromp2.Slicer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TychoMirrorApplication extends org.eclipse.tycho.p2tools.copiedfromp2.MirrorApplication {

private static final Logger LOGGER = LoggerFactory.getLogger(TychoMirrorApplication.class);
private static final String SOURCE_SUFFIX = ".source";
private static final String FEATURE_GROUP = ".feature.group";
private final DestinationRepositoryDescriptor destination;
Expand All @@ -77,10 +73,13 @@ public class TychoMirrorApplication extends org.eclipse.tycho.p2tools.copiedfrom
private boolean filterProvided;
private boolean addOnlyProvidingRepoReferences;
private TargetPlatform targetPlatform;
private Logger logger;

public TychoMirrorApplication(IProvisioningAgent agent, DestinationRepositoryDescriptor destination) {
public TychoMirrorApplication(IProvisioningAgent agent, DestinationRepositoryDescriptor destination,
Logger logger) {
super(agent);
this.destination = destination;
this.logger = logger;
this.removeAddedRepositories = false;
}

Expand Down Expand Up @@ -274,8 +273,8 @@ protected void finalizeRepositories() {
if (repository != null) {
Collection<IRepositoryReference> references = repository.getReferences();
if (!references.isEmpty()) {
LOGGER.info("Adding references to the following repositories:");
references.stream().map(r -> r.getLocation()).distinct().forEach(loc -> LOGGER.info(" {}", loc));
logger.info("Adding references to the following repositories:");
references.stream().map(r -> r.getLocation()).distinct().forEach(loc -> logger.info(" " + loc));
}
}
super.finalizeRepositories();
Expand All @@ -296,11 +295,12 @@ protected Set<IInstallableUnit> collectUnits(IQueryable<IInstallableUnit> slice,
throws ProvisionException {
Set<IInstallableUnit> units = super.collectUnits(slice, monitor);
if (isFilterProvidedItems()) {
Map<String, List<Version>> fullRepositoryContent = units.stream()
.collect(groupingBy(IInstallableUnit::getId, mapping(IInstallableUnit::getVersion, toList())));
Map<String, Set<Version>> fullRepositoryContent = units.stream().collect(
groupingBy(IInstallableUnit::getId, mapping(IInstallableUnit::getVersion, Collectors.toSet())));

List<IRepository<IInstallableUnit>> metadataRepositories = removeProvidedItems(units,
getMetadataRepositoryManager(), IRepository.TYPE_METADATA, monitor);
IMetadataRepositoryManager manager = getMetadataRepositoryManager();
Map<URI, IRepository<IInstallableUnit>> metadataRepositories = removeProvidedItems(units, manager,
IRepository.TYPE_METADATA, monitor);

if (addOnlyProvidingRepoReferences) {
Set<URI> removableReferences = destination.getFilterableRepositoryReferences().stream()
Expand All @@ -309,7 +309,8 @@ protected Set<IInstallableUnit> collectUnits(IQueryable<IInstallableUnit> slice,
.forEach(removableReferences::remove); // keep reference if explicitly added to the repository
if (!removableReferences.isEmpty()) {
// Assume that for all units that correspond to artifacts the metadata either has a co-located artifact repository or a references to to one that contains it.
removeNotProvidingReferences(fullRepositoryContent, metadataRepositories, removableReferences);
removeNotProvidingReferences(fullRepositoryContent, metadataRepositories, removableReferences,
manager);
}
}
}
Expand All @@ -320,17 +321,18 @@ private boolean isFilterProvidedItems() {
return filterProvided && !destinationMetadataRepository.getReferences().isEmpty();
}

private <T> List<IRepository<T>> removeProvidedItems(Collection<T> allElements, IRepositoryManager<T> repoManager,
int repositoryType, IProgressMonitor monitor) throws ProvisionException {
List<IRepository<T>> referencedRepositories = new ArrayList<>();
private <T> Map<URI, IRepository<T>> removeProvidedItems(Collection<T> allElements,
IRepositoryManager<T> repoManager, int repositoryType, IProgressMonitor monitor) throws ProvisionException {
Map<URI, IRepository<T>> referencedRepositories = new HashMap<>();
for (IRepositoryReference reference : destinationMetadataRepository.getReferences()) {
if (reference.getType() != repositoryType) {
continue;
}
try {
URI location = reference.getLocation();
IRepository<T> repository = repoManager.loadRepository(location, monitor);
referencedRepositories.add(repository);
//We need to retain the location in the map as the repo manager might rewrite it to a mirrored location
referencedRepositories.put(location, repository);
} catch (IllegalArgumentException e) {
if (e.getCause() instanceof URISyntaxException uriException) {
throw new ProvisionException("Can't parse referenced URI!", uriException);
Expand All @@ -339,35 +341,81 @@ private <T> List<IRepository<T>> removeProvidedItems(Collection<T> allElements,
}
}
}
allElements.removeIf(e -> referencedRepositories.stream().anyMatch(repo -> repo.contains(e)));
allElements.removeIf(e -> referencedRepositories.values().stream().anyMatch(repo -> repo.contains(e)));
return referencedRepositories;
}

private void removeNotProvidingReferences(Map<String, List<Version>> fullRepositoryContent,
List<IRepository<IInstallableUnit>> metadataRepositories, Set<URI> removableReferenceURIs) {
private void removeNotProvidingReferences(Map<String, Set<Version>> fullRepositoryContent,
Map<URI, IRepository<IInstallableUnit>> metadataRepositories, Set<URI> removableReferenceURIs,
IMetadataRepositoryManager manager) {
Map<URI, Set<IInstallableUnit>> usedRepositoryItems = new HashMap<>();
for (IRepository<IInstallableUnit> repo : metadataRepositories) {
Set<IInstallableUnit> usedRepoContent = repo.query(QueryUtil.ALL_UNITS, null).stream()
.filter(a -> fullRepositoryContent.getOrDefault(a.getId(), List.of()).contains(a.getVersion()))
Map<URI, Set<IInstallableUnit>> providedItems = new HashMap<>();
for (Entry<URI, IRepository<IInstallableUnit>> repo : metadataRepositories.entrySet()) {
Set<IInstallableUnit> content = getRepositoryContent(repo.getKey(), repo.getValue(), new HashSet<>(),
manager).collect(Collectors.toSet());
Set<IInstallableUnit> usedRepoContent = content.stream()
.filter(a -> fullRepositoryContent.getOrDefault(a.getId(), Set.of()).contains(a.getVersion()))
.collect(Collectors.toSet());
usedRepositoryItems.put(repo.getLocation(), usedRepoContent);
usedRepositoryItems.put(repo.getKey(), usedRepoContent);
providedItems.put(repo.getKey(), content);
}
// Remove filterable references that contribute nothing or whose relevant content is also provided by another repo
usedRepositoryItems.entrySet().removeIf(repo -> {
if (!removableReferenceURIs.contains(repo.getKey())) {
return false;
if (removableReferenceURIs.contains(repo.getKey())) {
Set<IInstallableUnit> usedContent = repo.getValue();
if (usedContent.isEmpty()) {
logger.info(
"Remove reference " + repo.getKey() + " because no units are contained in the repository.");
return true;
}
for (Entry<URI, Set<IInstallableUnit>> entry : usedRepositoryItems.entrySet()) {
if (entry == repo) {
continue;
}
Set<IInstallableUnit> other = providedItems.getOrDefault(entry.getKey(), Set.of());
if (!other.isEmpty() && other.containsAll(usedContent)) {
logger.info("Remove reference " + repo.getKey()
+ " because all units are also contained in reference " + entry.getKey() + " already.");
return true;
}
}
}
Set<IInstallableUnit> usedContent = repo.getValue();
return usedContent.isEmpty()
|| usedRepositoryItems.entrySet().stream().filter(e -> e != repo).map(Entry::getValue)
.anyMatch(other -> other.size() >= usedContent.size() && other.containsAll(usedContent));
return false;
});
IMetadataRepository repository = getDestinationMetadataRepository();
List<IRepositoryReference> discardedReferences = repository.getReferences().stream()
.filter(rr -> !usedRepositoryItems.keySet().contains(rr.getLocation())).toList();
repository.removeReferences(discardedReferences);
}

private Stream<IInstallableUnit> getRepositoryContent(URI uri, IRepository<IInstallableUnit> repo, Set<URI> visited,
IMetadataRepositoryManager manager) {
if (visited.add(uri)) {
Stream<IInstallableUnit> stream = repo.query(QueryUtil.ALL_UNITS, null).stream();
if (repo instanceof IMetadataRepository meta) {
Collection<IRepositoryReference> references = meta.getReferences();
for (IRepositoryReference reference : references) {
if (reference.getType() == IRepository.TYPE_METADATA && isEnabled(reference)) {
try {
URI referenceLocation = reference.getLocation();
IMetadataRepository referenceRepository = manager.loadRepository(referenceLocation, null);
stream = Stream.concat(stream,
getRepositoryContent(referenceLocation, referenceRepository, visited, manager));
} catch (ProvisionException e) {
//can't use then...
}
}
}
}
return stream;
}
return Stream.empty();
}

private boolean isEnabled(IRepositoryReference reference) {
return (reference.getOptions() & IRepository.ENABLED) != 0;
}

public void setIncludeSources(boolean includeAllSource, TargetPlatform targetPlatform) {
this.includeAllSource = includeAllSource;
this.targetPlatform = targetPlatform;
Expand Down
Loading