Skip to content

Commit

Permalink
Allow multiple cache entries per checksum
Browse files Browse the repository at this point in the history
  • Loading branch information
@reda-alaoui
reda-alaoui committed Aug 20, 2024
1 parent c2e4c0b commit 443d29d
Show file tree
Hide file tree
Showing 22 changed files with 452 additions and 139 deletions.
15 changes: 12 additions & 3 deletions src/main/java/org/apache/maven/buildcache/BuildCacheMojosExecutionStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ public void execute(
// execute clean bound goals before restoring to not interfere/slowdown clean
CacheState cacheState = DISABLED;
CacheResult result = CacheResult.empty();
boolean restorable = false;
boolean skipCache = cacheConfig.isSkipCache() || MavenProjectInput.isSkipCache(project);
boolean cacheIsDisabled = MavenProjectInput.isCacheDisabled(project);
// Forked execution should be thought as a part of originating mojo internal implementation
Expand All @@ -125,11 +126,17 @@ public void execute(
"Cache is explicitly disabled on project level for {}", getVersionlessProjectKey(project));
}
if (cacheState == INITIALIZED || skipCache) {
result = cacheController.findCachedBuild(session, project, mojoExecutions, skipCache);
for (Zone zone : cacheConfig.getInputZones()) {
result = cacheController.findCachedBuild(session, project, mojoExecutions, zone, skipCache);
restorable = result.isSuccess() || result.isPartialSuccess();
if (!restorable) {
continue;
}
break;
}
}
}

boolean restorable = result.isSuccess() || result.isPartialSuccess();
boolean restored = false; // if partially restored need to save increment
if (restorable) {
CacheRestorationStatus cacheRestorationStatus =
Expand Down Expand Up @@ -157,7 +164,9 @@ public void execute(
LOGGER.info("Cache storing is skipped since there was no \"clean\" phase.");
} else {
final Map<String, MojoExecutionEvent> executionEvents = mojoListener.getProjectExecutions(project);
cacheController.save(result, mojoExecutions, executionEvents);
for (Zone zone : cacheConfig.getOutputZones()) {
cacheController.save(result, mojoExecutions, executionEvents, zone);
}
}
}

Expand Down
9 changes: 7 additions & 2 deletions src/main/java/org/apache/maven/buildcache/CacheController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,19 @@
public interface CacheController {

CacheResult findCachedBuild(
MavenSession session, MavenProject project, List<MojoExecution> mojoExecutions, boolean skipCache);
MavenSession session,
MavenProject project,
List<MojoExecution> mojoExecutions,
Zone inputZone,
boolean skipCache);

ArtifactRestorationReport restoreProjectArtifacts(CacheResult cacheResult);

void save(
CacheResult cacheResult,
List<MojoExecution> mojoExecutions,
Map<String, MojoExecutionEvent> executionEvents);
Map<String, MojoExecutionEvent> executionEvents,
Zone outputZone);

boolean isForcedExecution(MavenProject project, MojoExecution execution);

Expand Down
83 changes: 48 additions & 35 deletions src/main/java/org/apache/maven/buildcache/CacheControllerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,11 @@ public CacheControllerImpl(
@Override
@Nonnull
public CacheResult findCachedBuild(
MavenSession session, MavenProject project, List<MojoExecution> mojoExecutions, boolean skipCache) {
MavenSession session,
MavenProject project,
List<MojoExecution> mojoExecutions,
Zone inputZone,
boolean skipCache) {
final String highestPhase = lifecyclePhasesHelper.resolveHighestLifecyclePhase(project, mojoExecutions);

if (!lifecyclePhasesHelper.isLaterPhaseThanClean(highestPhase)) {
Expand All @@ -177,30 +181,33 @@ public CacheResult findCachedBuild(

String projectName = getVersionlessProjectKey(project);

ProjectsInputInfo inputInfo = projectInputCalculator.calculateInput(project);
ProjectsInputInfo inputInfo = projectInputCalculator.calculateInput(project, inputZone);

final CacheContext context = new CacheContext(project, inputInfo, session);

CacheResult result = empty(context);
CacheResult result = empty(context, inputZone);
if (!skipCache) {

LOGGER.info("Attempting to restore project {} from build cache", projectName);
LOGGER.info("Attempting to restore project {} from build cache zone {}", projectName, inputZone);

// remote build first
if (cacheConfig.isRemoteCacheEnabled()) {
result = findCachedBuild(mojoExecutions, context);
result = findCachedBuild(mojoExecutions, context, inputZone);
if (!result.isSuccess() && result.getContext() != null) {
LOGGER.info("Remote cache is incomplete or missing, trying local build for {}", projectName);
}
}

if (!result.isSuccess() && result.getContext() != null) {
CacheResult localBuild = findLocalBuild(mojoExecutions, context);
CacheResult localBuild = findLocalBuild(mojoExecutions, context, inputZone);
if (localBuild.isSuccess() || (localBuild.isPartialSuccess() && !result.isPartialSuccess())) {
result = localBuild;
} else {
LOGGER.info(
"Local build was not found by checksum {} for {}", inputInfo.getChecksum(), projectName);
"Local build was not found by checksum {} for {} and zone {}",
inputInfo.getChecksum(),
projectName,
inputZone);
}
}
} else {
Expand All @@ -212,39 +219,43 @@ public CacheResult findCachedBuild(
return result;
}

private CacheResult findCachedBuild(List<MojoExecution> mojoExecutions, CacheContext context) {
private CacheResult findCachedBuild(List<MojoExecution> mojoExecutions, CacheContext context, Zone inputZone) {
Optional<Build> cachedBuild = Optional.empty();
try {
cachedBuild = localCache.findBuild(context);
cachedBuild = localCache.findBuild(context, inputZone);
if (cachedBuild.isPresent()) {
return analyzeResult(context, mojoExecutions, cachedBuild.get());
return analyzeResult(context, mojoExecutions, inputZone, cachedBuild.get());
}
} catch (Exception e) {
LOGGER.error("Cannot read cached remote build", e);
}
return cachedBuild.map(build -> failure(build, context)).orElseGet(() -> empty(context));
return cachedBuild.map(build -> failure(build, context, inputZone)).orElseGet(() -> empty(context, inputZone));
}

private CacheResult findLocalBuild(List<MojoExecution> mojoExecutions, CacheContext context) {
private CacheResult findLocalBuild(List<MojoExecution> mojoExecutions, CacheContext context, Zone inputZone) {
Optional<Build> localBuild = Optional.empty();
try {
localBuild = localCache.findLocalBuild(context);
localBuild = localCache.findLocalBuild(context, inputZone);
if (localBuild.isPresent()) {
return analyzeResult(context, mojoExecutions, localBuild.get());
return analyzeResult(context, mojoExecutions, inputZone, localBuild.get());
}
} catch (Exception e) {
LOGGER.error("Cannot read local build", e);
}
return localBuild.map(build -> failure(build, context)).orElseGet(() -> empty(context));
return localBuild.map(build -> failure(build, context, inputZone)).orElseGet(() -> empty(context, inputZone));
}

private CacheResult analyzeResult(CacheContext context, List<MojoExecution> mojoExecutions, Build build) {
private CacheResult analyzeResult(
CacheContext context, List<MojoExecution> mojoExecutions, Zone inputZone, Build build) {
try {
final ProjectsInputInfo inputInfo = context.getInputInfo();
String projectName = getVersionlessProjectKey(context.getProject());

LOGGER.info(
"Found cached build, restoring {} from cache by checksum {}", projectName, inputInfo.getChecksum());
"Found cached build, restoring {} from cache zone {} by checksum {}",
projectName,
inputZone,
inputInfo.getChecksum());
LOGGER.debug("Cached build details: {}", build);

final String cacheImplementationVersion = build.getCacheImplementationVersion();
Expand All @@ -263,12 +274,12 @@ private CacheResult analyzeResult(CacheContext context, List<MojoExecution> mojo
"Cached build doesn't contains all requested plugin executions "
+ "(missing: {}), cannot restore",
missingMojos);
return failure(build, context);
return failure(build, context, inputZone);
}

if (!isCachedSegmentPropertiesPresent(context.getProject(), build, cachedSegment)) {
LOGGER.info("Cached build violates cache rules, cannot restore");
return failure(build, context);
return failure(build, context, inputZone);
}

final String highestRequestPhase =
Expand All @@ -281,15 +292,15 @@ private CacheResult analyzeResult(CacheContext context, List<MojoExecution> mojo
projectName,
build.getHighestCompletedGoal(),
highestRequestPhase);
return partialSuccess(build, context);
return partialSuccess(build, context, inputZone);
}

return success(build, context);
return success(build, context, inputZone);

} catch (Exception e) {
LOGGER.error("Failed to restore project", e);
localCache.clearCache(context);
return failure(build, context);
localCache.clearCache(context, inputZone);
return failure(build, context, inputZone);
}
}

Expand Down Expand Up @@ -389,8 +400,8 @@ public ArtifactRestorationReport restoreProjectArtifacts(CacheResult cacheResult
// Set this value before trying the restoration, to keep a trace of the attempt if it fails
restorationReport.setRestoredFilesInProjectDirectory(true);
// generated sources artifact
final Path attachedArtifactFile =
localCache.getArtifactFile(context, cacheResult.getSource(), attachedArtifactInfo);
final Path attachedArtifactFile = localCache.getArtifactFile(
context, cacheResult.getSource(), cacheResult.getInputZone(), attachedArtifactInfo);
restoreGeneratedSources(attachedArtifactInfo, attachedArtifactFile, project);
}
} else {
Expand Down Expand Up @@ -462,7 +473,8 @@ private Future<File> createDownloadTask(
String originalVersion) {
final FutureTask<File> downloadTask = new FutureTask<>(() -> {
LOGGER.debug("Downloading artifact {}", artifact.getArtifactId());
final Path artifactFile = localCache.getArtifactFile(context, cacheResult.getSource(), artifact);
final Path artifactFile =
localCache.getArtifactFile(context, cacheResult.getSource(), cacheResult.getInputZone(), artifact);

if (!Files.exists(artifactFile)) {
throw new FileNotFoundException("Missing file for cached build, cannot restore. File: " + artifactFile);
Expand All @@ -482,7 +494,8 @@ private Future<File> createDownloadTask(
public void save(
CacheResult cacheResult,
List<MojoExecution> mojoExecutions,
Map<String, MojoExecutionEvent> executionEvents) {
Map<String, MojoExecutionEvent> executionEvents,
Zone outputZone) {
CacheContext context = cacheResult.getContext();

if (context == null || context.getInputInfo() == null) {
Expand Down Expand Up @@ -526,19 +539,19 @@ public void save(
build.getDto().set_final(cacheConfig.isSaveToRemoteFinal());
cacheResults.put(getVersionlessProjectKey(project), rebuilded(cacheResult, build));

localCache.beforeSave(context);
localCache.beforeSave(context, outputZone);

// if package phase presence means new artifacts were packaged
if (project.hasLifecyclePhase("package")) {
if (projectArtifact.getFile() != null) {
localCache.saveArtifactFile(cacheResult, projectArtifact);
localCache.saveArtifactFile(cacheResult, outputZone, projectArtifact);
}
for (org.apache.maven.artifact.Artifact attachedArtifact : attachedArtifacts) {
if (attachedArtifact.getFile() != null) {
boolean storeArtifact =
isOutputArtifact(attachedArtifact.getFile().getName());
if (storeArtifact) {
localCache.saveArtifactFile(cacheResult, attachedArtifact);
localCache.saveArtifactFile(cacheResult, outputZone, attachedArtifact);
} else {
LOGGER.debug(
"Skipping attached project artifact '{}' = "
Expand All @@ -549,7 +562,7 @@ public void save(
}
}

localCache.saveBuildInfo(cacheResult, build);
localCache.saveBuildInfo(cacheResult, outputZone, build);

if (cacheConfig.isBaselineDiffEnabled()) {
produceDiffReport(cacheResult, build);
Expand All @@ -558,7 +571,7 @@ public void save(
} catch (Exception e) {
LOGGER.error("Failed to save project, cleaning cache. Project: {}", project, e);
try {
localCache.clearCache(context);
localCache.clearCache(context, outputZone);
} catch (Exception ex) {
LOGGER.error("Failed to clean cache due to unexpected error:", ex);
}
Expand All @@ -567,7 +580,7 @@ public void save(

public void produceDiffReport(CacheResult cacheResult, Build build) {
MavenProject project = cacheResult.getContext().getProject();
Optional<Build> baselineHolder = remoteCache.findBaselineBuild(project);
Optional<Build> baselineHolder = remoteCache.findBaselineBuild(project, cacheResult.getInputZone());
if (baselineHolder.isPresent()) {
Build baseline = baselineHolder.get();
String outputDirectory = project.getBuild().getDirectory();
Expand Down Expand Up @@ -841,10 +854,10 @@ public void saveCacheReport(MavenSession session) {
projectReport.setLifecycleMatched(checksumMatched && result.isSuccess());
projectReport.setSource(String.valueOf(result.getSource()));
if (result.getSource() == CacheSource.REMOTE) {
projectReport.setUrl(remoteCache.getResourceUrl(context, BUILDINFO_XML));
projectReport.setUrl(remoteCache.getResourceUrl(context, result.getInputZone(), BUILDINFO_XML));
} else if (result.getSource() == CacheSource.BUILD && cacheConfig.isSaveToRemote()) {
projectReport.setSharedToRemote(true);
projectReport.setUrl(remoteCache.getResourceUrl(context, BUILDINFO_XML));
projectReport.setUrl(remoteCache.getResourceUrl(context, result.getInputZone(), BUILDINFO_XML));
}
cacheReport.addProject(projectReport);
}
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/org/apache/maven/buildcache/CacheRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@
public interface CacheRepository {

@Nonnull
Optional<Build> findBuild(CacheContext context) throws IOException;
Optional<Build> findBuild(CacheContext context, Zone inputZone) throws IOException;

void saveBuildInfo(CacheResult cacheResult, Build build) throws IOException;
void saveBuildInfo(CacheResult cacheResult, Zone outputZone, Build build) throws IOException;

void saveArtifactFile(CacheResult cacheResult, Artifact artifact) throws IOException;
void saveArtifactFile(CacheResult cacheResult, Zone outputZone, Artifact artifact) throws IOException;

void saveCacheReport(String buildId, MavenSession session, CacheReport cacheReport) throws IOException;
}
37 changes: 24 additions & 13 deletions src/main/java/org/apache/maven/buildcache/CacheResult.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,49 +31,56 @@ public class CacheResult {
private final RestoreStatus status;
private final Build build;
private final CacheContext context;
private final Zone inputZone;

private CacheResult(RestoreStatus status, Build build, CacheContext context) {
private CacheResult(RestoreStatus status, Build build, CacheContext context, Zone inputZone) {
this.status = requireNonNull(status);
this.build = build;
this.context = context;
this.inputZone = inputZone;
}

public static CacheResult empty(CacheContext context) {
public static CacheResult empty(CacheContext context, Zone inputZone) {
requireNonNull(context);
return new CacheResult(RestoreStatus.EMPTY, null, context);
requireNonNull(inputZone);
return new CacheResult(RestoreStatus.EMPTY, null, context, inputZone);
}

public static CacheResult empty() {
return new CacheResult(RestoreStatus.EMPTY, null, null);
return new CacheResult(RestoreStatus.EMPTY, null, null, null);
}

public static CacheResult failure(Build build, CacheContext context) {
public static CacheResult failure(Build build, CacheContext context, Zone inputZone) {
requireNonNull(build);
requireNonNull(context);
return new CacheResult(RestoreStatus.FAILURE, build, context);
requireNonNull(inputZone);
return new CacheResult(RestoreStatus.FAILURE, build, context, inputZone);
}

public static CacheResult success(Build build, CacheContext context) {
public static CacheResult success(Build build, CacheContext context, Zone inputZone) {
requireNonNull(build);
requireNonNull(context);
return new CacheResult(RestoreStatus.SUCCESS, build, context);
requireNonNull(inputZone);
return new CacheResult(RestoreStatus.SUCCESS, build, context, inputZone);
}

public static CacheResult partialSuccess(Build build, CacheContext context) {
public static CacheResult partialSuccess(Build build, CacheContext context, Zone inputZone) {
requireNonNull(build);
requireNonNull(context);
return new CacheResult(RestoreStatus.PARTIAL, build, context);
requireNonNull(inputZone);
return new CacheResult(RestoreStatus.PARTIAL, build, context, inputZone);
}

public static CacheResult failure(CacheContext context) {
public static CacheResult failure(CacheContext context, Zone inputZone) {
requireNonNull(context);
return new CacheResult(RestoreStatus.FAILURE, null, context);
requireNonNull(inputZone);
return new CacheResult(RestoreStatus.FAILURE, null, context, inputZone);
}

public static CacheResult rebuilded(CacheResult orig, Build build) {
requireNonNull(orig);
requireNonNull(build);
return new CacheResult(orig.status, build, orig.context);
return new CacheResult(orig.status, build, orig.context, orig.inputZone);
}

public boolean isSuccess() {
Expand Down Expand Up @@ -103,4 +110,8 @@ public RestoreStatus getStatus() {
public boolean isFinal() {
return build != null && build.getDto().is_final();
}

public Zone getInputZone() {
return inputZone;
}
}
Loading

0 comments on commit 443d29d

Please sign in to comment.