Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rework maven code for faster calculation of path config #1661

Merged
merged 21 commits into from
Nov 26, 2024
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
1fc579d
Added fast path to avoid starting maven just to install dependencies
DavyLandman Oct 3, 2022
0e8b61a
Rewrote to using internal maven engine instead of forking jvm
DavyLandman Oct 4, 2022
f574853
Rewrote to calling maven without changing global properties (but sadl…
DavyLandman Oct 4, 2022
fb3c978
Merge branch 'main' into reduce-maven-overhead
rodinaarssen Jul 22, 2024
6dc6715
Fixed path normalization for embedded maven
rodinaarssen Jul 22, 2024
a2ce678
Updated maven library version
rodinaarssen Jul 22, 2024
df9a698
Added another necessary dependency for maven-embedder
rodinaarssen Jul 30, 2024
d5cff33
Configured maven-shade-plugin to correctly shade required files for d…
rodinaarssen Aug 20, 2024
0bcadda
Fixed regular expression to obtain the classpath from Maven's output
rodinaarssen Aug 20, 2024
cb9851d
Layout
rodinaarssen Aug 20, 2024
c9fc81e
Merge branch 'main' into reduce-maven-overhead
rodinaarssen Aug 20, 2024
d5c655b
No longer manipulating System.out and System.err streams, instead wri…
rodinaarssen Aug 20, 2024
2aab716
Correctly extracting class path from maven output
rodinaarssen Aug 27, 2024
d374969
Merge branch 'main' into reduce-maven-overhead
rodinaarssen Aug 27, 2024
90271d3
Changed slf4j dependency to suppress maven output to the REPL
rodinaarssen Sep 10, 2024
0529f26
Removed now-redundant constant
rodinaarssen Sep 10, 2024
d30a696
Created separate class for Maven interaction
rodinaarssen Sep 11, 2024
d181d88
Fully qualified dependency plugin; removed quiet argument from maven …
rodinaarssen Sep 17, 2024
badd6e0
Make sure that maven-dependency-plugin is downloaded when not yet pre…
rodinaarssen Sep 17, 2024
dd327ad
Merge branch 'main' into reduce-maven-overhead
rodinaarssen Nov 25, 2024
c82fbc9
Added missing license header
rodinaarssen Nov 25, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 29 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<lucence-version>7.5.0</lucence-version>
<maven-version>3.9.8</maven-version>
<exec.mainClass>org.rascalmpl.shell.RascalShell</exec.mainClass>
<rascal.test.memory>2</rascal.test.memory>
<maven.compiler.release>11</maven.compiler.release>
Expand Down Expand Up @@ -317,9 +318,10 @@
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>org/rascalmpl/uri/resolvers.config</resource>
<resource>io/usethesource/vallang/type/types.config </resource>
<resource>META-INF/sisu/javax.inject.Named</resource> <!-- Needed for dependency injection in MavenCli -->
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer">
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ComponentsXmlResourceTransformer"/> <!-- Needed for dependency injection in MavenCli -->
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
<addHeader>false</addHeader>
</transformer>
Expand Down Expand Up @@ -514,5 +516,30 @@
<artifactId>icu4j</artifactId>
<version>74.2</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-embedder</artifactId>
<version>${maven-version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.7.1</version>
</dependency>
<dependency> <!-- needed for maven-embedder-->
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>1.7.36</version>
</dependency>
<dependency> <!-- needed for maven-embedder-->
<groupId>org.apache.maven</groupId>
<artifactId>maven-compat</artifactId>
<version>${maven-version}</version>
</dependency>
<dependency> <!-- needed for maven-embedder-->
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>${maven-version}</version>
</dependency>
</dependencies>
</project>
109 changes: 38 additions & 71 deletions src/org/rascalmpl/library/util/PathConfig.java
Original file line number Diff line number Diff line change
@@ -1,21 +1,25 @@
package org.rascalmpl.library.util;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.io.StringWriter;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Pattern;

import org.apache.maven.cli.CliRequest;
import org.apache.maven.cli.MavenCli;
import org.codehaus.plexus.classworlds.ClassWorld;
import org.rascalmpl.interpreter.Configuration;
import org.rascalmpl.interpreter.utils.RascalManifest;
import org.rascalmpl.uri.ILogicalSourceLocationResolver;
Expand Down Expand Up @@ -650,83 +654,46 @@
return vf.list();
}

String mvnCommand = computeMavenCommandName();

installNecessaryMavenPlugins(mvnCommand);

// Note how we try to do this "offline" using the "-o" flag
ProcessBuilder processBuilder = new ProcessBuilder(mvnCommand,
"--batch-mode",
"-o",
"dependency:build-classpath",
"-DincludeScope=compile",
trustStoreFix()
);

processBuilder.directory(new File(manifestRoot.getPath()));
processBuilder.environment().put("JAVA_HOME", System.getProperty("java.home", System.getenv("JAVA_HOME")));

Process process = processBuilder.start();
var maven = new MavenCli();
var tempFile = Files.createTempFile("rascal-classpath-", ".tmp");

maven.doMain(buildRequest(new String[] {"-quiet", "-o", "dependency:build-classpath", "-DincludeScope=compile", "-Dmdep.outputFile=" + tempFile.toString()}, manifestRoot));
PieterOlivier marked this conversation as resolved.
Show resolved Hide resolved
PieterOlivier marked this conversation as resolved.
Show resolved Hide resolved

var foundClassPath = Files.readAllLines(tempFile).get(0);

Check warning on line 662 in src/org/rascalmpl/library/util/PathConfig.java

View check run for this annotation

Codecov / codecov/patch

src/org/rascalmpl/library/util/PathConfig.java#L662

Added line #L662 was not covered by tests

try (BufferedReader processOutputReader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
return processOutputReader.lines()
.filter(line -> !line.startsWith("["))
.filter(line -> !line.contains("-----"))
.flatMap(line -> Arrays.stream(line.split(File.pathSeparator)))
.filter(fileName -> new File(fileName).exists())
.map(elem -> {
try {
return URIUtil.createFileLocation(elem);
}
catch (URISyntaxException e) {
return null;
}
})
.filter(e -> e != null)
.collect(vf.listWriter());
}
return Arrays.stream(foundClassPath.split(File.pathSeparator))
.filter(fileName -> new File(fileName).exists())
.map(elem -> {

Check warning on line 666 in src/org/rascalmpl/library/util/PathConfig.java

View check run for this annotation

Codecov / codecov/patch

src/org/rascalmpl/library/util/PathConfig.java#L664-L666

Added lines #L664 - L666 were not covered by tests
try {
return URIUtil.createFileLocation(elem);

Check warning on line 668 in src/org/rascalmpl/library/util/PathConfig.java

View check run for this annotation

Codecov / codecov/patch

src/org/rascalmpl/library/util/PathConfig.java#L668

Added line #L668 was not covered by tests
}
catch (URISyntaxException e) {
return null;

Check warning on line 671 in src/org/rascalmpl/library/util/PathConfig.java

View check run for this annotation

Codecov / codecov/patch

src/org/rascalmpl/library/util/PathConfig.java#L670-L671

Added lines #L670 - L671 were not covered by tests
}
})
.filter(Objects::nonNull)
.collect(vf.listWriter());

Check warning on line 675 in src/org/rascalmpl/library/util/PathConfig.java

View check run for this annotation

Codecov / codecov/patch

src/org/rascalmpl/library/util/PathConfig.java#L674-L675

Added lines #L674 - L675 were not covered by tests
}
catch (IOException e) {
catch (IOException | RuntimeException | ReflectiveOperationException e) {
return vf.list();
}
}

private static boolean isWindows() {
return System.getProperty("os.name").toLowerCase().contains("win");
private static void setField(CliRequest req, String fieldName, Object value) throws ReflectiveOperationException {
var field = CliRequest.class.getDeclaredField(fieldName);
field.setAccessible(true);
field.set(req, value);
}

private static String computeMavenCommandName() {
if (System.getProperty("os.name", "generic").startsWith("Windows")) {
return "mvn.cmd";
}
else {
return "mvn";
}
}

private static void installNecessaryMavenPlugins(String mvnCommand) {
try {
ProcessBuilder processBuilder = new ProcessBuilder(mvnCommand,
"-q",
"dependency:get",
"-DgroupId=org.apache.maven.plugins",
"-DartifactId=maven-dependency-plugin",
"-Dversion=2.8",
trustStoreFix());
processBuilder.environment().put("JAVA_HOME", System.getProperty("java.home", System.getenv("JAVA_HOME")));

Process process = processBuilder.start();
if (process.waitFor() != 0) {
throw new IOException("mvn dependency:get returned non-zero");
}
}
catch (IOException | InterruptedException e) {
System.err.println("[WARNING] Could not install exec-maven-plugin; classpath resolution may be incomplete hereafter: " + e.getMessage());
}
}

private static String trustStoreFix() {
return isWindows() ? WINDOWS_ROOT_TRUSTSTORE_TYPE_DEFINITION : "-Dnothing_to_see_here";
private static CliRequest buildRequest(String[] args, ISourceLocation manifestRoot) throws ReflectiveOperationException {
// we need to set a field that the default class doesn't set
// it's a work around around a bug in the MavenCli code
var cons = CliRequest.class.getDeclaredConstructor(String[].class, ClassWorld.class);
cons.setAccessible(true);
var result = cons.newInstance(args, null);
setField(result, "workingDirectory", new File(manifestRoot.getPath()).getPath());
setField(result, "multiModuleProjectDirectory", new File(manifestRoot.getPath()));
return result;
}

public ISourceLocation getBin() {
Expand Down
Loading