Skip to content

Commit

Permalink
[GR-60266] Added --max-test-time to unittest command.
Browse files Browse the repository at this point in the history 
PullRequest: mx/1858
  • Loading branch information
@dougxc
dougxc committed Jan 3, 2025
2 parents 1be0ca2 + 733c012 commit b1f5550
Show file tree
Hide file tree
Showing 5 changed files with 94 additions and 17 deletions.
14 changes: 7 additions & 7 deletions common.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,13 @@

"oraclejdk23": {"name": "jpg-jdk", "version": "23", "build_id": "jdk-23+37", "platformspecific": true, "extrabundles": ["static-libs"]},

"oraclejdk-latest": {"name": "jpg-jdk", "version": "25", "build_id": "jdk-25+2", "platformspecific": true, "extrabundles": ["static-libs"]},
"labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-25+2-jvmci-b01", "platformspecific": true },
"labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-25+2-jvmci-b01-debug", "platformspecific": true },
"labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-25+2-jvmci-b01-sulong", "platformspecific": true },
"labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-25+2-jvmci-b01", "platformspecific": true },
"labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-25+2-jvmci-b01-debug", "platformspecific": true },
"labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-25+2-jvmci-b01-sulong", "platformspecific": true }
"oraclejdk-latest": {"name": "jpg-jdk", "version": "25", "build_id": "jdk-25+3", "platformspecific": true, "extrabundles": ["static-libs"]},
"labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-25+3-jvmci-b01", "platformspecific": true },
"labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-25+3-jvmci-b01-debug", "platformspecific": true },
"labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-25+3-jvmci-b01-sulong", "platformspecific": true },
"labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-25+3-jvmci-b01", "platformspecific": true },
"labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-25+3-jvmci-b01-debug", "platformspecific": true },
"labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-25+3-jvmci-b01-sulong", "platformspecific": true }
},

"eclipse": {
Expand Down
34 changes: 29 additions & 5 deletions java/com.oracle.mxtool.junit/src/com/oracle/mxtool/junit/MxJUnitWrapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,11 @@ public static class MxJUnitConfig {
* This field is ignored and is only retained for backwards compatibility.
*/
public String jsonResultTags;

/**
* Max time in seconds allowed for a single test.
*/
public Long maxTestTime;
}

private static class RepeatingRunner extends Runner {
Expand Down Expand Up @@ -173,6 +178,8 @@ public static void main(String... args) {
config.maxClassFailures = 1;
} else if (each.contentEquals("-JUnitEnableTiming")) {
config.enableTiming = true;
} else if (each.contentEquals("-JUnitMaxTestTime")) {
config.maxTestTime = (long) parseIntArg(system, expandedArgs, each, ++i);
} else if (each.contentEquals("-JUnitColor")) {
config.color = true;
} else if (each.contentEquals("-JUnitEagerStackTrace")) {
Expand Down Expand Up @@ -293,8 +300,9 @@ public static Result runRequest(JUnitCore junitCore, JUnitSystem system, MxJUnit
} else {
textListener = new TextRunListener(system);
}
TimingAndDiskUsageDecorator timings = config.enableTiming ? new TimingAndDiskUsageDecorator(textListener) : null;
MxRunListener mxListener = config.enableTiming ? timings : textListener;
List<MxJUnitWrapper.Timing<Description>> maxTestTimeExceeded = new ArrayList<>();
TimingAndDiskUsageDecorator timings = config.enableTiming || config.maxTestTime != null ? new TimingAndDiskUsageDecorator(textListener, config.maxTestTime, maxTestTimeExceeded) : null;
MxRunListener mxListener = timings != null ? timings : textListener;
ResultCollectorDecorator resultLoggerDecorator = null;

final boolean failingFast;
Expand Down Expand Up @@ -352,7 +360,7 @@ public void schedule(Runnable childStatement) {
}
}
}
if (failureCount < config.maxClassFailures) {
if (failureCount + maxTestTimeExceeded.size() < config.maxClassFailures) {
childStatement.run();
}
}
Expand Down Expand Up @@ -386,7 +394,23 @@ public void finished() {
}
}));

return junitCore.run(request);
Result result = junitCore.run(request);
if (!maxTestTimeExceeded.isEmpty()) {
maxTestTimeExceeded.sort(Collections.reverseOrder());
// Match output format of org.junit.internal.TextListener.printFooter
system.out().println();
system.out().println("FAILURES!!!");
system.out().printf("Tests exceeded max time of %d seconds (specified by -JUnitMaxTestTime or --max-test-time): %d%n", config.maxTestTime, maxTestTimeExceeded.size());
for (MxJUnitWrapper.Timing<Description> timing : maxTestTimeExceeded) {
system.out().printf(" %,10d ms %s%n", timing.value, timing.subject);
}

// Add a failure to the final result so that it reports false for Report.wasSuccessful()
RuntimeException ex = new RuntimeException(maxTestTimeExceeded.size() + " tests exceeded max time");
Failure failure = new Failure(Description.createTestDescription("<all classes>", "<all methods>"), ex);
result.getFailures().add(failure);
}
return result;
}

private static Set<String> findTestClasses(Description desc, Set<String> classes) {
Expand All @@ -409,7 +433,7 @@ private static PrintStream getResultStream(JUnitSystem system, String file) {
return openFile(system, file);
}

private static class Timing<T> implements Comparable<Timing<T>> {
public static class Timing<T> implements Comparable<Timing<T>> {
final T subject;
final long value;

Expand Down
60 changes: 56 additions & 4 deletions java/com.oracle.mxtool.junit/src/com/oracle/mxtool/junit/TimingAndDiskUsageDecorator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@
import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import org.junit.runner.Description;
import org.junit.runner.notification.Failure;

/**
* Timing and disk usage support for JUnit test runs.
Expand All @@ -41,18 +43,45 @@ class TimingAndDiskUsageDecorator extends MxRunListenerDecorator {
private long startTime;
private long classStartTime;
private Description currentTest;

/**
* Time in milliseconds per test class.
*/
final Map<Class<?>, Long> classTimes;

/**
* Time in milliseconds per test.
*/
final Map<Description, Long> testTimes;

/**
* Time taken by the last test in milliseconds.
*/
private Long testTimeMS;

/**
* Max time in seconds for a passing test before it is added to {@link #maxTestTimeExceeded}. If
* null, then no max time has been set.
*/
private final Long maxTestTime;

/**
* Collects the tests that pass but run longer than {@link #maxTestTime}.
*/
private final List<MxJUnitWrapper.Timing<Description>> maxTestTimeExceeded;

private final FileStore fileStore;
private final String totalDiskSpace;

TimingAndDiskUsageDecorator(MxRunListener l) {
TimingAndDiskUsageDecorator(TextRunListener l, Long maxTestTime, List<MxJUnitWrapper.Timing<Description>> maxTestTimeExceeded) {
super(l);
this.classTimes = new ConcurrentHashMap<>();
this.testTimes = new ConcurrentHashMap<>();
FileStore fs = initFileStore();
this.fileStore = fs;
this.totalDiskSpace = fs == null ? null : initTotalDiskSpace(fs);
this.maxTestTime = maxTestTime;
this.maxTestTimeExceeded = maxTestTimeExceeded;
}

@Override
Expand All @@ -78,15 +107,38 @@ public void testStarted(Description description) {
super.testStarted(description);
}

@Override
public void testFailed(Failure failure) {
stopTestTiming(failure.getDescription());
super.testFailed(failure);
}

@Override
public void testSucceeded(Description description) {
long timeMS = stopTestTiming(description);
if (maxTestTime != null) {
long maxTestTimeMS = maxTestTime * 1000;
if (timeMS > maxTestTimeMS) {
maxTestTimeExceeded.add(new MxJUnitWrapper.Timing<>(description, timeMS));
}
}
super.testSucceeded(description);
}

private long stopTestTiming(Description description) {
testTimeMS = (System.nanoTime() - startTime) / 1_000_000;
testTimes.put(description, testTimeMS);
return testTimeMS;
}

@Override
public void testFinished(Description description) {
long totalTime = System.nanoTime() - startTime;
super.testFinished(description);
if (beVerbose()) {
getWriter().print(" " + valueToString(totalTime));
getWriter().print(" " + testTimeMS + " ms");
}
currentTest = null;
testTimes.put(description, totalTime / 1_000_000);
testTimeMS = null;
}

static String valueToString(long valueNS) {
Expand Down
2 changes: 1 addition & 1 deletion src/mx/_impl/mx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18367,7 +18367,7 @@ def alarm_handler(signum, frame):
_CACHE_DIR = get_env('MX_CACHE_DIR', join(dot_mx_dir(), 'cache'))

# The version must be updated for every PR (checked in CI) and the comment should reflect the PR's issue
version = VersionSpec("7.36.5") # 60664 - provide latest checkstyle, version 10.21.0
version = VersionSpec("7.37.0") # GR-60266 Implement time limit for mx unit tests

_mx_start_datetime = datetime.utcnow()

Expand Down
1 change: 1 addition & 0 deletions src/mx/_impl/mx_unittest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -656,6 +656,7 @@ def __call__(self, parser, namespace, values, option_string=None):
parser.add_argument('--very-verbose', help='enable very verbose JUnit output', dest='JUnitVeryVerbose', action=MxJUnitWrapperBoolArg)
parser.add_argument('--max-class-failures', help='stop after N test classes that have a failure (default is no limit)', type=is_strictly_positive, metavar='<N>', dest='JUnitMaxClassFailures', action=MxJUnitWrapperArg)
parser.add_argument('--fail-fast', help='alias for --max-class-failures=1', dest='JUnitFailFast', action=MxJUnitWrapperBoolArg)
parser.add_argument('--max-test-time', help='max time in seconds allowed for a test (default is no limit)', type=is_strictly_positive, metavar='<N>', dest='JUnitMaxTestTime', action=MxJUnitWrapperArg)
parser.add_argument('--enable-timing', help='enable JUnit test timing (always on - option retained for compatibility)', dest='JUnitEnableTiming', action=MxJUnitWrapperBoolArg)
parser.add_argument('--regex', help='run only testcases matching a regular expression', metavar='<regex>')
parser.add_argument('--color', help='enable color output', dest='JUnitColor', action=MxJUnitWrapperBoolArg)
Expand Down

0 comments on commit b1f5550

Please sign in to comment.