fix: Feed generates error to authorize upload (#1633)

core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java

@@ -42,6 +42,8 @@ public abstract class GtfsInput implements Closeable {
   public static final String invalidInputMessage =
       "At least 1 GTFS file is in a subfolder. All GTFS files must reside at the root level directly.";
 
+  public static final String USER_AGENT_PREFIX = "MobilityData GTFS-Validator";
+
   /**
    * Creates a specific GtfsInput to read data from the given path.
    *
@@ -92,18 +94,6 @@ public static boolean hasSubfolderWithGtfsFile(Path path) throws IOException {
     return containsGtfsFileInSubfolder(zipInputStream);
   }
 
-  /**
-   * Check if an input zip file from an URL contains a subfolder with GTFS files
-   *
-   * @param url
-   * @return
-   * @throws IOException
-   */
-  public static boolean hasSubfolderWithGtfsFile(URL url) throws IOException {
-    ZipInputStream zipInputStream = new ZipInputStream(new BufferedInputStream(url.openStream()));
-    return containsGtfsFileInSubfolder(zipInputStream);
-  }
-
   /**
    * Common method used by two overloaded hasSubfolderWithGtfsFile methods
    *
@@ -143,20 +133,21 @@ private static boolean containsGtfsFileInSubfolder(ZipInputStream zipInputStream
    * @param sourceUrl the fully qualified URL to download of the resource to download
    * @param targetPath the path to store the downloaded GTFS archive
    * @param noticeContainer
+   * @param validatorVersion
    * @return the {@code GtfsInput} created after download of the GTFS archive
    * @throws IOException if GTFS archive cannot be stored at the specified location
    * @throws URISyntaxException if URL is malformed
    */
   public static GtfsInput createFromUrl(
-      URL sourceUrl, Path targetPath, NoticeContainer noticeContainer)
+      URL sourceUrl, Path targetPath, NoticeContainer noticeContainer, String validatorVersion)
       throws IOException, URISyntaxException {
     // getParent() may return null if there is no parent, so call toAbsolutePath() first.
     Path targetDirectory = targetPath.toAbsolutePath().getParent();
     if (!Files.isDirectory(targetDirectory)) {
       Files.createDirectories(targetDirectory);
     }
     try (OutputStream outputStream = Files.newOutputStream(targetPath)) {
-      loadFromUrl(sourceUrl, outputStream);
+      loadFromUrl(sourceUrl, outputStream, validatorVersion);
     }
     return createFromPath(targetPath, noticeContainer);
   }
@@ -171,14 +162,15 @@ public static GtfsInput createFromUrl(
    * @throws IOException if no file could not be found at the specified location
    * @throws URISyntaxException if URL is malformed
    */
-  public static GtfsInput createFromUrlInMemory(URL sourceUrl, NoticeContainer noticeContainer)
+  public static GtfsInput createFromUrlInMemory(
+      URL sourceUrl, NoticeContainer noticeContainer, String validatorVersion)
       throws IOException, URISyntaxException {
     try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
-      loadFromUrl(sourceUrl, outputStream);
+      loadFromUrl(sourceUrl, outputStream, validatorVersion);
       File zipFile = new File(sourceUrl.toString());
       String fileName = zipFile.getName().replace(".zip", "");
-      if (hasSubfolderWithGtfsFile(sourceUrl)) {
-
+      if (containsGtfsFileInSubfolder(
+          new ZipInputStream(new ByteArrayInputStream(outputStream.toByteArray())))) {
         noticeContainer.addValidationNotice(
             new InvalidInputFilesInSubfolderNotice(invalidInputMessage));
       }
@@ -192,19 +184,35 @@ public static GtfsInput createFromUrlInMemory(URL sourceUrl, NoticeContainer not
    *
    * @param sourceUrl the fully qualified URL
    * @param outputStream the output stream
+   * @param validatorVersion
    * @throws IOException if no file could not be found at the specified location
    * @throws URISyntaxException if URL is malformed
    */
-  private static void loadFromUrl(URL sourceUrl, OutputStream outputStream)
+  private static void loadFromUrl(URL sourceUrl, OutputStream outputStream, String validatorVersion)
       throws IOException, URISyntaxException {
     try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
       HttpGet httpGet = new HttpGet(sourceUrl.toURI());
+      httpGet.setHeader("User-Agent", getUserAgent(validatorVersion));
       try (CloseableHttpResponse httpResponse = httpClient.execute(httpGet)) {
         httpResponse.getEntity().writeTo(outputStream);
       }
     }
   }
 
+  /**
+   * @param validatorVersion version of the validator
+   * @return the user agent string in the format: "MobilityData GTFS-Validator/{validatorVersion}
+   *     (Java {java version})"
+   */
+  private static String getUserAgent(String validatorVersion) {
+    return USER_AGENT_PREFIX
+        + "/"
+        + (validatorVersion != null ? validatorVersion : "")
+        + " (Java "
+        + System.getProperty("java.version")
+        + ")";
+  }
+
   /**
    * Lists all files inside the GTFS dataset, even if they are not CSV and do not have .txt
    * extension.

core/src/test/java/org/mobilitydata/gtfsvalidator/input/GtfsInputTest.java

@@ -77,17 +77,14 @@ public void zipInput() throws IOException {
     }
   }
 
-  @Test
-  public void urlInputHasNoSubfolderWithGtfsFile() throws IOException {
-    URL url = new URL(VALID_URL);
-    assertFalse(GtfsInput.hasSubfolderWithGtfsFile(url));
-  }
-
   @Test
   public void createFromUrl_valid_success() throws IOException, URISyntaxException {
     try (GtfsInput underTest =
         GtfsInput.createFromUrl(
-            new URL(VALID_URL), tmpDir.getRoot().toPath().resolve("storage"), noticeContainer)) {
+            new URL(VALID_URL),
+            tmpDir.getRoot().toPath().resolve("storage"),
+            noticeContainer,
+            "1.0.1")) {
       assertThat(underTest instanceof GtfsZipFileInput);
     }
   }
@@ -100,13 +97,14 @@ public void createFromUrl_invalid_throwsException() {
             GtfsInput.createFromUrl(
                 new URL(INVALID_URL),
                 tmpDir.getRoot().toPath().resolve("storage"),
-                noticeContainer));
+                noticeContainer,
+                "1.0.1"));
   }
 
   @Test
   public void createFromUrlInMemory_valid_success() throws IOException, URISyntaxException {
     try (GtfsInput underTest =
-        GtfsInput.createFromUrlInMemory(new URL(VALID_URL), noticeContainer)) {
+        GtfsInput.createFromUrlInMemory(new URL(VALID_URL), noticeContainer, "1.0.1")) {
       assertThat(underTest instanceof GtfsZipFileInput);
     }
   }
@@ -115,6 +113,6 @@ public void createFromUrlInMemory_valid_success() throws IOException, URISyntaxE
   public void createFromUrlInMemory_invalid_throwsException() {
     assertThrows(
         IOException.class,
-        () -> GtfsInput.createFromUrlInMemory(new URL(INVALID_URL), noticeContainer));
+        () -> GtfsInput.createFromUrlInMemory(new URL(INVALID_URL), noticeContainer, "1.0.1"));
   }
 }

main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java

@@ -102,7 +102,7 @@ public Status run(ValidationRunnerConfig config) {
     GtfsFeedContainer feedContainer;
     GtfsInput gtfsInput = null;
     try {
-      gtfsInput = createGtfsInput(config);
+      gtfsInput = createGtfsInput(config, versionInfo.currentVersion().get());
     } catch (IOException e) {
       logger.atSevere().withCause(e).log("Cannot load GTFS feed");
       noticeContainer.addSystemError(new IOError(e));
@@ -305,24 +305,26 @@ public static void exportReport(
    * Creates a {@code GtfsInput}
    *
    * @param config used to retrieve information needed to the creation of the {@code GtfsInput}
+   * @param validatorVersion version of the validator
    * @return the {@code GtfsInput} generated after
    * @throws IOException in case of error while loading a file
    * @throws URISyntaxException in case of error in the {@code URL} syntax
    */
-  public static GtfsInput createGtfsInput(ValidationRunnerConfig config)
+  public static GtfsInput createGtfsInput(ValidationRunnerConfig config, String validatorVersion)
       throws IOException, URISyntaxException {
     URI source = config.gtfsSource();
     if (source.getScheme().equals("file")) {
       return GtfsInput.createFromPath(Paths.get(source), noticeContainer);
     }
 
     if (config.storageDirectory().isEmpty()) {
-      return GtfsInput.createFromUrlInMemory(source.toURL(), noticeContainer);
+      return GtfsInput.createFromUrlInMemory(source.toURL(), noticeContainer, validatorVersion);
     } else {
       return GtfsInput.createFromUrl(
           source.toURL(),
           config.storageDirectory().get().resolve(GTFS_ZIP_FILENAME),
-          noticeContainer);
+          noticeContainer,
+          validatorVersion);
     }
   }
 }

main/src/test/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerTest.java

@@ -30,7 +30,8 @@ public void createGtfsInput_WindowsPath_valid() throws IOException, URISyntaxExc
 
     // We are testing path parsing here only. We expect a FileNotFoundException but NOT a
     // InvalidPathException. This should catch issues such as #1158.
-    assertThrows(FileNotFoundException.class, () -> ValidationRunner.createGtfsInput(config));
+    assertThrows(
+        FileNotFoundException.class, () -> ValidationRunner.createGtfsInput(config, "1.1.0"));
   }
 
   @Test
@@ -39,6 +40,7 @@ public void createGtfsInput_LinuxPath_valid() throws IOException, URISyntaxExcep
 
     // We are testing path parsing here only. We expect a FileNotFoundException but NOT a
     // InvalidPathException. This should catch issues such as #1158.
-    assertThrows(FileNotFoundException.class, () -> ValidationRunner.createGtfsInput(config));
+    assertThrows(
+        FileNotFoundException.class, () -> ValidationRunner.createGtfsInput(config, "1.1.0"));
   }
 }