Fast tests script (#1102)

planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java

@@ -124,7 +124,7 @@ public static PlanetilerConfig from(Arguments arguments) {
     int renderMaxzoom =
       arguments.getInteger("render_maxzoom", "maximum rendering zoom level up to " + MAX_MAXZOOM,
         Math.max(maxzoom, DEFAULT_MAXZOOM));
-    Path tmpDir = arguments.file("tmpdir", "temp directory", Path.of("data", "tmp"));
+    Path tmpDir = arguments.file("tmpdir|tmp", "temp directory", Path.of("data", "tmp"));
 
     return new PlanetilerConfig(
       arguments,

planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java

@@ -62,8 +62,8 @@ public class Wikidata {
 
   private static final ObjectMapper objectMapper = new ObjectMapper();
   private static final Logger LOGGER = LoggerFactory.getLogger(Wikidata.class);
-  private static final Pattern wikidataIRIMatcher = Pattern.compile("http://www.wikidata.org/entity/Q([0-9]+)");
-  private static final Pattern qidPattern = Pattern.compile("Q([0-9]+)");
+  private static final Pattern wikidataIRIMatcher = Pattern.compile("http://www.wikidata.org/entity/Q(\\d+)");
+  private static final Pattern qidPattern = Pattern.compile("Q(\\d+)");
   private final Counter.Readable blocks = Counter.newMultiThreadCounter();
   private final Counter.Readable nodes = Counter.newMultiThreadCounter();
   private final Counter.Readable ways = Counter.newMultiThreadCounter();
@@ -128,7 +128,7 @@ public static void fetch(OsmInputFile infile, Path outfile, PlanetilerConfig con
 
     var timer = stats.startStage("wikidata");
     int processThreads = Math.max(1, config.threads() - 1);
-    LOGGER.info("Starting with " + processThreads + " process threads");
+    LOGGER.info("Starting with {} process threads", processThreads);
 
     WikidataTranslations oldMappings = load(outfile, maxAge, updateLimit);
     try (
@@ -166,7 +166,7 @@ public static void fetch(OsmInputFile infile, Path outfile, PlanetilerConfig con
         .addPipelineStats(pipeline);
 
       pipeline.awaitAndLog(loggers, config.logInterval());
-      LOGGER.info("DONE fetched:" + fetcher.wikidatas.get());
+      LOGGER.info("DONE fetched: {}", fetcher.wikidatas.get());
     } catch (IOException e) {
       throw new UncheckedIOException(e);
     }
@@ -192,10 +192,14 @@ private static WikidataTranslations load(Path path, Duration maxAge, int updateL
       try (BufferedReader fis = Files.newBufferedReader(path)) {
         WikidataTranslations result = load(fis, maxAge, updateLimit, Clock.systemUTC());
         LOGGER.info(
-          "loaded from " + result.getAll().size() + " mappings from " + path.toAbsolutePath() + " in " + timer.stop());
+          "loaded from {} mappings from {} in {}",
+          result.getAll().size(),
+          path.toAbsolutePath(),
+          timer.stop()
+        );
         return result;
       } catch (IOException e) {
-        LOGGER.info("error loading " + path.toAbsolutePath() + ": " + e);
+        LOGGER.info("error loading {}: {}", path.toAbsolutePath(), e);
         return new WikidataTranslations();
       }
     }
@@ -247,7 +251,7 @@ private static long extractIdFromWikidataIRI(String iri) {
       String idText = matcher.group(1);
       return Long.parseLong(idText);
     } else {
-      throw new RuntimeException("Unexpected response IRI: " + iri);
+      throw new IllegalStateException("Unexpected response IRI: " + iri);
     }
   }
 
@@ -277,14 +281,13 @@ private void filter(Iterable<OsmBlockSource.Block> prev, Consumer<Long> next) {
           blockRelations++;
         }
         Object wikidata = elem.getString("wikidata");
-        if (wikidata instanceof String wikidataString) {
-          if (profile.caresAboutWikidataTranslation(elem)) {
-            long qid = parseQid(wikidataString);
-            if (qid > 0) {
-              next.accept(qid);
-            }
+        if (wikidata instanceof String wikidataString && profile.caresAboutWikidataTranslation(elem)) {
+          long qid = parseQid(wikidataString);
+          if (qid > 0) {
+            next.accept(qid);
           }
         }
+
       }
       blocks.inc();
       nodes.incBy(blockNodes);
@@ -356,12 +359,12 @@ private LongObjectMap<Map<String, String>> queryWikidata(List<Long> qidsToFetch)
       } catch (IOException e) {
         boolean lastTry = i == config.httpRetries();
         if (!lastTry) {
-          LOGGER.warn("sparql query failed, retrying: " + e);
+          LOGGER.warn("sparql query failed, retrying: {}", e.toString());
         } else {
-          LOGGER.error("sparql query failed, exhausted retries: " + e);
+          LOGGER.error("sparql query failed, exhausted retries: {}", e.toString());
           throw e;
         }
-        Thread.sleep(config.httpRetryWait());
+        sleep(config.httpRetryWait());
       }
     }
 
@@ -372,11 +375,15 @@ private LongObjectMap<Map<String, String>> queryWikidata(List<Long> qidsToFetch)
     }
   }
 
+  protected void sleep(Duration duration) throws InterruptedException {
+    Thread.sleep(duration.toMillis());
+  }
+
   void loadExisting(WikidataTranslations oldMappings) throws IOException {
     LongObjectMap<Map<String, String>> alreadyHave = oldMappings.getAll();
     LongObjectMap<Instant> alreadyHaveUpdateTimes = oldMappings.getUpdateTimes();
     if (!alreadyHave.isEmpty()) {
-      LOGGER.info("skipping " + alreadyHave.size() + " mappings we already have");
+      LOGGER.info("skipping {} mappings we already have", alreadyHave.size());
       writeTranslations(alreadyHave, alreadyHaveUpdateTimes);
       for (LongObjectCursor<Map<String, String>> cursor : alreadyHave) {
         visited.add(cursor.key);
@@ -405,12 +412,12 @@ private void writeTranslations(LongObjectMap<Map<String, String>> results, LongO
   interface Client {
 
     static Client wrap(HttpClient client) {
-      return (req) -> {
+      return req -> {
         var response = client.send(req, BodyHandlers.ofInputStream());
         if (response.statusCode() >= 400) {
           String body;
-          try {
-            body = new String(response.body().readAllBytes(), StandardCharsets.UTF_8);
+          try (var responseBody = response.body()) {
+            body = new String(responseBody.readAllBytes(), StandardCharsets.UTF_8);
           } catch (IOException e) {
             body = "Error reading body: " + e;
           }
@@ -435,8 +442,6 @@ public static class WikidataTranslations implements Translations.TranslationProv
     private final LongObjectMap<Map<String, String>> data = Hppc.newLongObjectHashMap();
     private final LongObjectMap<Instant> updateTimes = Hppc.newLongObjectHashMap();
 
-    public WikidataTranslations() {}
-
     /** Returns a map from language code to translated name for {@code qid}. */
     public Map<String, String> get(long qid) {
       return data.get(qid);

planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java

@@ -638,6 +638,7 @@ void testExtractConnectedComponents() {
     );
   }
 
+  @Slow
   @ParameterizedTest
   @CsvSource({
     "bostonbuildings.mbtiles, 2477, 3028, 13, 1141",

planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java

@@ -72,6 +72,7 @@
 /**
  * In-memory tests with fake data and profiles to ensure all features work end-to-end.
  */
+@Slow
 class PlanetilerTests {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(PlanetilerTests.class);

planetiler-core/src/test/java/com/onthegomap/planetiler/Slow.java

@@ -0,0 +1,14 @@
+package com.onthegomap.planetiler;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.junit.jupiter.api.Tag;
+
+/** Add to any junit test classes or methods to exclude when run with {@code -Pfast} maven argument. */
+@Target({ElementType.TYPE, ElementType.METHOD})
+@Tag("slow")
+@Retention(RetentionPolicy.RUNTIME)
+public @interface Slow {
+}

planetiler-core/src/test/java/com/onthegomap/planetiler/collection/AppendStoreTest.java

@@ -3,22 +3,29 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import java.io.IOException;
 import java.nio.file.Path;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
-class AppendStoreTest {
+abstract class AppendStoreTest {
 
-  static abstract class IntsTest {
+  abstract static class IntsTest {
 
     protected AppendStore.Ints store;
 
+    @AfterEach
+    void close() throws IOException {
+      store.close();
+    }
+
     @ParameterizedTest
     @ValueSource(ints = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
-    public void writeThenRead(int num) {
+    void writeThenRead(int num) {
       for (int i = 0; i < num; i++) {
         store.appendInt(i + 1);
       }
@@ -30,7 +37,7 @@ public void writeThenRead(int num) {
     }
 
     @Test
-    public void readBig() {
+    void readBig() {
       store.appendInt(Integer.MAX_VALUE);
       store.appendInt(Integer.MAX_VALUE - 1);
       store.appendInt(Integer.MAX_VALUE - 2);
@@ -40,13 +47,18 @@ public void readBig() {
     }
   }
 
-  static abstract class LongsTest {
+  abstract static class LongsTest {
 
     protected AppendStore.Longs store;
 
+    @AfterEach
+    void close() throws IOException {
+      store.close();
+    }
+
     @ParameterizedTest
     @ValueSource(ints = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
-    public void writeThenRead(int num) {
+    void writeThenRead(int num) {
       for (int i = 0; i < num; i++) {
         store.appendLong(i + 1);
       }
@@ -57,87 +69,88 @@ public void writeThenRead(int num) {
       assertThrows(IndexOutOfBoundsException.class, () -> store.getLong(num + 1));
     }
 
-    private static final long maxInt = Integer.MAX_VALUE;
+    private static final long MAX_INT = Integer.MAX_VALUE;
 
     @ParameterizedTest
-    @ValueSource(longs = {maxInt - 1, maxInt, maxInt + 1, 2 * maxInt - 1, 2 * maxInt, 5 * maxInt - 1, 5 * maxInt + 1})
-    public void readBig(long value) {
+    @ValueSource(longs = {MAX_INT - 1,
+      MAX_INT, MAX_INT + 1, 2 * MAX_INT - 1, 2 * MAX_INT, 5 * MAX_INT - 1, 5 * MAX_INT + 1})
+    void readBig(long value) {
       store.appendLong(value);
       assertEquals(value, store.getLong(0));
     }
 
   }
 
-  static class RamInt extends IntsTest {
+  static class RamIntTest extends IntsTest {
 
     @BeforeEach
-    public void setup() {
+    void setup() {
       this.store = new AppendStoreRam.Ints(false, 4 << 2);
     }
   }
 
-  static class MMapInt extends IntsTest {
+  static class MMapIntTest extends IntsTest {
 
     @BeforeEach
-    public void setup(@TempDir Path path) {
+    void setup(@TempDir Path path) {
       this.store = new AppendStoreMmap.Ints(path.resolve("ints"), 4 << 2, true);
     }
   }
 
-  static class DirectInt extends IntsTest {
+  static class DirectIntTest extends IntsTest {
 
     @BeforeEach
-    public void setup() {
+    void setup() {
       this.store = new AppendStoreRam.Ints(true, 4 << 2);
     }
   }
 
-  static class RamLong extends LongsTest {
+  static class RamLongTest extends LongsTest {
 
     @BeforeEach
-    public void setup() {
+    void setup() {
       this.store = new AppendStoreRam.Longs(false, 4 << 2);
     }
   }
 
-  static class MMapLong extends LongsTest {
+  static class MMapLongTest extends LongsTest {
 
     @BeforeEach
-    public void setup(@TempDir Path path) {
+    void setup(@TempDir Path path) {
       this.store = new AppendStoreMmap.Longs(path.resolve("longs"), 4 << 2, true);
     }
   }
 
-  static class DirectLong extends LongsTest {
+  static class DirectLongTest extends LongsTest {
 
     @BeforeEach
-    public void setup() {
+    void setup() {
       this.store = new AppendStoreRam.Longs(true, 4 << 2);
     }
   }
 
-  static class MMapSmallLong extends LongsTest {
+  static class MMapSmallLongTest extends LongsTest {
 
     @BeforeEach
-    public void setup(@TempDir Path path) {
+    void setup(@TempDir Path path) {
       this.store = new AppendStore.SmallLongs(
-        (i) -> new AppendStoreMmap.Ints(path.resolve("smalllongs" + i), 4 << 2, true));
+        i -> new AppendStoreMmap.Ints(path.resolve("smalllongs" + i), 4 << 2, true));
     }
   }
 
-  static class RamSmallLong extends LongsTest {
+  static class RamSmallLongTest extends LongsTest {
 
     @BeforeEach
-    public void setup() {
-      this.store = new AppendStore.SmallLongs((i) -> new AppendStoreRam.Ints(false, 4 << 2));
+    void setup() {
+      this.store = new AppendStore.SmallLongs(i -> new AppendStoreRam.Ints(false, 4 << 2));
     }
   }
 
-  static class DirectSmallLong extends LongsTest {
+  static class DirectSmallLongTest extends LongsTest {
 
     @BeforeEach
-    public void setup() {
-      this.store = new AppendStore.SmallLongs((i) -> new AppendStoreRam.Ints(true, 4 << 2));
+    void setup() {
+      this.store = new AppendStore.SmallLongs(i -> new AppendStoreRam.Ints(true, 4 << 2));
     }
   }
 }