From c6f916d96021370bbf472c57c31550b473db52cc Mon Sep 17 00:00:00 2001
From: Chris Povirk <cpovirk@google.com>
Date: Tue, 2 Apr 2024 15:44:44 -0400
Subject: [PATCH] Sort input files in the test runner. (#168)

While the order of the input files _shouldn't_ affect behavior, it
sometimes does. Let's use a consistent order so that we at least don't
see behavior differences between local and CI runs:

https://github.com/jspecify/jspecify-reference-checker/pull/165#issuecomment-2032659286
---
 .../conformance/ConformanceTestRunner.java    | 22 +++++++++++--------
 src/test/java/tests/ConformanceTest.java      |  3 ++-
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/conformance-test-framework/src/main/java/org/jspecify/conformance/ConformanceTestRunner.java b/conformance-test-framework/src/main/java/org/jspecify/conformance/ConformanceTestRunner.java
index 364b240..1d980f6 100644
--- a/conformance-test-framework/src/main/java/org/jspecify/conformance/ConformanceTestRunner.java
+++ b/conformance-test-framework/src/main/java/org/jspecify/conformance/ConformanceTestRunner.java
@@ -15,7 +15,7 @@
 package org.jspecify.conformance;
 
 import static com.google.common.base.Strings.nullToEmpty;
-import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet;
 import static com.google.common.collect.Lists.partition;
 import static com.google.common.io.MoreFiles.asCharSink;
 import static com.google.common.io.MoreFiles.asCharSource;
@@ -23,10 +23,12 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.nio.file.Files.walk;
 import static java.util.Arrays.stream;
+import static java.util.Comparator.naturalOrder;
 import static java.util.stream.Collectors.toList;
 
 import com.google.common.base.Ascii;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSortedSet;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
@@ -48,7 +50,7 @@ public interface Analyzer {
      * @return the facts reported by the analysis
      */
     Iterable<ReportedFact> analyze(
-        Path testDirectory, ImmutableList<Path> files, ImmutableList<Path> testDeps);
+        Path testDirectory, ImmutableSortedSet<Path> files, ImmutableList<Path> testDeps);
   }
 
   private final Analyzer analyzer;
@@ -72,26 +74,28 @@ public ConformanceTestReport runTests(Path testDirectory, ImmutableList<Path> te
           .filter(path -> path.toFile().isDirectory())
           .flatMap(
               directory -> {
-                Stream<ImmutableList<Path>> groups = javaFileGroups(directory);
+                Stream<ImmutableSortedSet<Path>> groups = javaFileGroups(directory);
                 return directory.equals(testDirectory)
-                    ? groups.flatMap(files -> partition(files, 1).stream())
+                    ? groups.flatMap(files -> partition(files.asList(), 1).stream())
                     : groups;
               })
-          .map(ImmutableList::copyOf)
+          .map(ImmutableSortedSet::copyOf)
           .forEach(
               files -> report.addFiles(files, analyzer.analyze(testDirectory, files, testDeps)));
       return report.build();
     }
   }
 
-  private static Stream<ImmutableList<Path>> javaFileGroups(Path directory) {
-    ImmutableList<Path> files = javaFilesInDirectory(directory);
+  private static Stream<ImmutableSortedSet<Path>> javaFileGroups(Path directory) {
+    ImmutableSortedSet<Path> files = javaFilesInDirectory(directory);
     return files.isEmpty() ? Stream.empty() : Stream.of(files);
   }
 
-  private static ImmutableList<Path> javaFilesInDirectory(Path directory) {
+  private static ImmutableSortedSet<Path> javaFilesInDirectory(Path directory) {
     try (Stream<Path> files = Files.list(directory)) {
-      return files.filter(f -> f.toString().endsWith(".java")).collect(toImmutableList());
+      return files
+          .filter(f -> f.toString().endsWith(".java"))
+          .collect(toImmutableSortedSet(naturalOrder()));
     } catch (IOException e) {
       throw new UncheckedIOException(e);
     }
diff --git a/src/test/java/tests/ConformanceTest.java b/src/test/java/tests/ConformanceTest.java
index 275a364..8e40198 100644
--- a/src/test/java/tests/ConformanceTest.java
+++ b/src/test/java/tests/ConformanceTest.java
@@ -24,6 +24,7 @@
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
 import com.google.jspecify.nullness.NullSpecChecker;
 import java.io.IOException;
 import java.nio.file.Path;
@@ -130,7 +131,7 @@ private static Path systemPropertyPath(String key, String description) {
   }
 
   private static ImmutableSet<ReportedFact> analyze(
-      Path testDirectory, ImmutableList<Path> files, ImmutableList<Path> testDeps) {
+      Path testDirectory, ImmutableSortedSet<Path> files, ImmutableList<Path> testDeps) {
     TestConfiguration config =
         TestConfigurationBuilder.buildDefaultConfiguration(
             null,