Use conformance test artifacts (#112)

Instead of looking in the `jspecify` sibling project directory for conformance test inputs and dependencies, use the distributed artifacts produced by it. Depends on jspecify/jspecify#420. Part of #107.
jspecify · Dec 5, 2023 · f3a17f3 · f3a17f3
1 parent 7fc5ec8
commit f3a17f3
Show file tree

Hide file tree

Showing 6 changed files with 59 additions and 17 deletions.
diff --git a/build.gradle b/build.gradle
@@ -15,6 +15,12 @@ repositories {
 
 configurations {
     errorproneJavac
+    conformanceTestAssertions {
+        attributes {
+            attribute(DocsType.DOCS_TYPE_ATTRIBUTE, objects.named(DocsType, DocsType.SOURCES))
+        }
+    }
+    conformanceTestDeps
 }
 
 ext {
@@ -37,9 +43,23 @@ dependencies {
     testImplementation libs.checkerFramework.framework.test
     testImplementation libs.guava
     testImplementation libs.junit
-    testImplementation libs.jspecify.conformance
+    testImplementation libs.jspecify.conformanceTestFramework
     testRuntimeOnly libs.jsr305 // jsr305 annotations are in some of the samples
 
+    conformanceTestAssertions("org.jspecify.conformance:conformance-tests:0.0.0-SNAPSHOT") {
+        capabilities {
+            // Depend on the assertion sources
+            requireCapability "org.jspecify.conformance:conformance-tests-assertions"
+        }
+    }
+
+    conformanceTestDeps("org.jspecify.conformance:conformance-tests:0.0.0-SNAPSHOT") {
+        capabilities {
+            // Depend on the assertion dependencies
+            requireCapability "org.jspecify.conformance:conformance-tests-deps"
+        }
+    }
+
     errorproneJavac libs.errorProne.javac
     errorprone libs.errorProne.core
 }
@@ -113,7 +133,9 @@ tasks.register('jspecifySamplesTest', Test) {
 }
 
 TaskProvider<Test> conformanceTest(String name, String testDirectory, String reportFile) {
+    check.dependsOn(name)
     return tasks.register(name, Test) {
+        group = 'verification'
         include '**/ConformanceTest.class'
         inputs.dir(testDirectory)
         inputs.files(reportFile)
@@ -122,7 +144,18 @@ TaskProvider<Test> conformanceTest(String name, String testDirectory, String rep
     }
 }
 
-conformanceTest('conformanceTest', "${jspecify.projectDir}/conformance", 'tests/ConformanceTest-report.txt')
+tasks.register('unzipConformanceTestAssertions', Copy) {
+    dependsOn configurations.conformanceTestAssertions
+    from zipTree(configurations.conformanceTestAssertions.singleFile)
+    exclude 'META-INF/'
+    into "${buildDir}/conformanceTests"
+}
+
+conformanceTest('conformanceTest', "${buildDir}/conformanceTests/org/jspecify/conformance/tests", 'tests/ConformanceTest-report.txt').configure {
+    dependsOn 'unzipConformanceTestAssertions', configurations.conformanceTestDeps
+    inputs.files(configurations.conformanceTestDeps)
+    systemProperty("JSpecifyConformanceTest.testDeps", configurations.conformanceTestDeps.join(":"))
+}
 
 conformanceTest('conformanceTestOnSamples', "${jspecify.projectDir}/samples", 'tests/ConformanceTestOnSamples-report.txt')
 

diff --git a/conformance/build.gradle → conformance-test-framework/build.gradle b/conformance/build.gradle → conformance-test-framework/build.gradle
@@ -2,7 +2,7 @@ plugins {
     id 'java-library'
 }
 
-group 'org.jspecify'
+group 'org.jspecify.conformance'
 version '0.0.0-SNAPSHOT'
 
 repositories {
@@ -18,7 +18,3 @@ dependencies {
     implementation libs.jspecify
     implementation libs.truth
 }
-
-test {
-    useJUnitPlatform()
-}
diff --git a/.../conformance/AbstractConformanceTest.java → .../conformance/AbstractConformanceTest.java b/.../conformance/AbstractConformanceTest.java → .../conformance/AbstractConformanceTest.java
@@ -30,6 +30,7 @@
 import static org.jspecify.conformance.AbstractConformanceTest.ExpectedFact.readExpectedFact;
 
 import com.google.common.base.Ascii;
+import com.google.common.base.Splitter;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableListMultimap;
@@ -81,11 +82,14 @@
 public abstract class AbstractConformanceTest {
 
   private final Path testDirectory;
+  private final ImmutableList<Path> testDeps;
   private final CharSource testReportSource;
   private final CharSink testReportSink;
 
-  protected AbstractConformanceTest(Path testDirectory, Path testReport) {
+  protected AbstractConformanceTest(
+      Path testDirectory, ImmutableList<Path> testDeps, Path testReport) {
     this.testDirectory = testDirectory;
+    this.testDeps = testDeps;
     this.testReportSource = asCharSource(testReport, UTF_8);
     this.testReportSink = asCharSink(testReport, UTF_8);
   }
@@ -95,11 +99,17 @@ protected AbstractConformanceTest() {
         systemPropertyPath(
             "JSpecifyConformanceTest.sourceDirectory",
             "the location of the JSpecify conformance test sources"),
+        Stream.ofNullable(System.getProperty("JSpecifyConformanceTest.testDeps"))
+            .flatMap(COLON::splitToStream)
+            .map(Paths::get)
+            .collect(toImmutableList()),
         systemPropertyPath(
             "JSpecifyConformanceTest.report",
             "the location of the JSpecify conformance test report"));
   }
 
+  private static final Splitter COLON = Splitter.on(':');
+
   /** Returns the directory that is the root of all test inputs. */
   protected final Path getTestDirectory() {
     return testDirectory;
@@ -117,7 +127,7 @@ private ConformanceTestReport runTests() throws IOException {
 
   private Stream<ConformanceTestReport> analyzeFiles(List<Path> files) {
     ImmutableListMultimap<Path, ReportedFact> reportedFactsByFile =
-        index(analyze(ImmutableList.copyOf(files)), ReportedFact::getFile);
+        index(analyze(ImmutableList.copyOf(files), testDeps), ReportedFact::getFile);
     return files.stream()
         .map(testDirectory::relativize)
         .map(
@@ -127,11 +137,13 @@ private Stream<ConformanceTestReport> analyzeFiles(List<Path> files) {
   }
 
   /**
-   * Analyzes a nonempty set of Java source {@code files} that may refer to each other.
+   * Analyzes a nonempty set of Java source {@code files} that may refer to each other, along with a
+   * classpath containing symbols the files may depend on.
    *
    * @return the facts reported by the analysis
    */
-  protected abstract Iterable<ReportedFact> analyze(ImmutableList<Path> files);
+  protected abstract Iterable<ReportedFact> analyze(
+      ImmutableList<Path> files, ImmutableList<Path> testDeps);
 
   /** Reads {@link ExpectedFact}s from comments in a file. */
   private ImmutableList<ExpectedFact> readExpectedFacts(Path file) {

diff --git a/...fy/conformance/ConformanceTestReport.java → ...fy/conformance/ConformanceTestReport.java b/...fy/conformance/ConformanceTestReport.java → ...fy/conformance/ConformanceTestReport.java
diff --git a/settings.gradle b/settings.gradle
@@ -1,8 +1,8 @@
 // Project name is read-only in build scripts, and defaults to directory name.
 rootProject.name = "jspecify-reference-checker"
-include 'conformance'
+include 'conformance-test-framework'
 
-// Lets the main build depend on the conformance subproject as org.jspecify:conformance.
+// Lets the main build depend on the conformance subproject as org.jspecify:conformance-test-framework.
 // See https://docs.gradle.org/current/userguide/composite_builds.html#included_build_declaring_substitutions
 includeBuild(".")
 
@@ -38,7 +38,7 @@ dependencyResolutionManagement {
             library("errorProne-javac", "com.google.errorprone:javac:9+181-r4173-1")
             library("guava", "com.google.guava:guava:31.1-jre")
             library("jspecify", "org.jspecify:jspecify:0.0.0-SNAPSHOT")
-            library("jspecify-conformance", "org.jspecify:conformance:0.0.0-SNAPSHOT")
+            library("jspecify-conformanceTestFramework", "org.jspecify.conformance:conformance-test-framework:0.0.0-SNAPSHOT")
             library("jsr305", "com.google.code.findbugs:jsr305:3.0.2")
             library("junit", "junit:junit:4.12")
             library("truth", "com.google.truth:truth:1.1.3")

diff --git a/src/test/java/tests/ConformanceTest.java b/src/test/java/tests/ConformanceTest.java
@@ -15,8 +15,8 @@
 package tests;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static java.util.Collections.emptyList;
 import static java.util.Objects.requireNonNullElse;
 import static java.util.stream.Collectors.joining;
 import static org.jspecify.conformance.AbstractConformanceTest.ExpectedFact.cannotConvert;
@@ -59,12 +59,13 @@ public final class ConformanceTest extends AbstractConformanceTest {
           "-AshowTypes");
 
   @Override
-  protected Iterable<ReportedFact> analyze(ImmutableList<Path> files) {
+  protected Iterable<ReportedFact> analyze(
+      ImmutableList<Path> files, ImmutableList<Path> testDeps) {
     TestConfiguration config =
         TestConfigurationBuilder.buildDefaultConfiguration(
             null,
             files.stream().map(Path::toFile).collect(toImmutableSet()),
-            emptyList(),
+            testDeps.stream().map(Path::toString).collect(toImmutableList()),
             ImmutableList.of(NullSpecChecker.class.getName()),
             OPTIONS,
             TestUtilities.getShouldEmitDebugInfo());