Sort input files in the test runner. (#168)

While the order of the input files _shouldn't_ affect behavior, it sometimes does. Let's use a consistent order so that we at least don't see behavior differences between local and CI runs: #165 (comment)
jspecify · Apr 2, 2024 · c6f916d · c6f916d
1 parent 75aa1de
commit c6f916d
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 10 deletions.
diff --git a/conformance-test-framework/src/main/java/org/jspecify/conformance/ConformanceTestRunner.java b/conformance-test-framework/src/main/java/org/jspecify/conformance/ConformanceTestRunner.java
@@ -15,18 +15,20 @@
 package org.jspecify.conformance;
 
 import static com.google.common.base.Strings.nullToEmpty;
-import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet;
 import static com.google.common.collect.Lists.partition;
 import static com.google.common.io.MoreFiles.asCharSink;
 import static com.google.common.io.MoreFiles.asCharSource;
 import static com.google.common.truth.Truth.assertThat;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.nio.file.Files.walk;
 import static java.util.Arrays.stream;
+import static java.util.Comparator.naturalOrder;
 import static java.util.stream.Collectors.toList;
 
 import com.google.common.base.Ascii;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSortedSet;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
@@ -48,7 +50,7 @@ public interface Analyzer {
      * @return the facts reported by the analysis
      */
     Iterable<ReportedFact> analyze(
-        Path testDirectory, ImmutableList<Path> files, ImmutableList<Path> testDeps);
+        Path testDirectory, ImmutableSortedSet<Path> files, ImmutableList<Path> testDeps);
   }
 
   private final Analyzer analyzer;
@@ -72,26 +74,28 @@ public ConformanceTestReport runTests(Path testDirectory, ImmutableList<Path> te
           .filter(path -> path.toFile().isDirectory())
           .flatMap(
               directory -> {
-                Stream<ImmutableList<Path>> groups = javaFileGroups(directory);
+                Stream<ImmutableSortedSet<Path>> groups = javaFileGroups(directory);
                 return directory.equals(testDirectory)
-                    ? groups.flatMap(files -> partition(files, 1).stream())
+                    ? groups.flatMap(files -> partition(files.asList(), 1).stream())
                     : groups;
               })
-          .map(ImmutableList::copyOf)
+          .map(ImmutableSortedSet::copyOf)
           .forEach(
               files -> report.addFiles(files, analyzer.analyze(testDirectory, files, testDeps)));
       return report.build();
     }
   }
 
-  private static Stream<ImmutableList<Path>> javaFileGroups(Path directory) {
-    ImmutableList<Path> files = javaFilesInDirectory(directory);
+  private static Stream<ImmutableSortedSet<Path>> javaFileGroups(Path directory) {
+    ImmutableSortedSet<Path> files = javaFilesInDirectory(directory);
     return files.isEmpty() ? Stream.empty() : Stream.of(files);
   }
 
-  private static ImmutableList<Path> javaFilesInDirectory(Path directory) {
+  private static ImmutableSortedSet<Path> javaFilesInDirectory(Path directory) {
     try (Stream<Path> files = Files.list(directory)) {
-      return files.filter(f -> f.toString().endsWith(".java")).collect(toImmutableList());
+      return files
+          .filter(f -> f.toString().endsWith(".java"))
+          .collect(toImmutableSortedSet(naturalOrder()));
     } catch (IOException e) {
       throw new UncheckedIOException(e);
     }

diff --git a/src/test/java/tests/ConformanceTest.java b/src/test/java/tests/ConformanceTest.java
@@ -24,6 +24,7 @@
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
 import com.google.jspecify.nullness.NullSpecChecker;
 import java.io.IOException;
 import java.nio.file.Path;
@@ -130,7 +131,7 @@ private static Path systemPropertyPath(String key, String description) {
   }
 
   private static ImmutableSet<ReportedFact> analyze(
-      Path testDirectory, ImmutableList<Path> files, ImmutableList<Path> testDeps) {
+      Path testDirectory, ImmutableSortedSet<Path> files, ImmutableList<Path> testDeps) {
     TestConfiguration config =
         TestConfigurationBuilder.buildDefaultConfiguration(
             null,