Write Properties files in a reproducible way

eclipse-tycho · Jan 4, 2025 · 51a7c6c · 51a7c6c
1 parent 4ca8978
commit 51a7c6c
Show file tree

Hide file tree

Showing 13 changed files with 125 additions and 84 deletions.
diff --git a/...aven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java b/...aven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java
@@ -40,6 +40,7 @@
 import org.codehaus.plexus.component.annotations.Requirement;
 import org.codehaus.plexus.logging.Logger;
 import org.eclipse.equinox.internal.p2.repository.AuthenticationFailedException;
+import org.eclipse.tycho.ReproducibleUtils;
 
 @Component(role = HttpCache.class)
 public class SharedHttpCacheStorage implements HttpCache {
@@ -392,11 +393,9 @@ protected void updateHeader(Headers response, int code) throws IOException, File
 				}
 			}
 			FileUtils.forceMkdir(file.getParentFile());
-			try (OutputStream out = new BufferedOutputStream(new FileOutputStream(headerFile))) {
-				// we store the header here, this might be a 404 response or (permanent)
-				// redirect we probably need to work with later on
-				header.store(out, null);
-			}
+			// we store the header here, this might be a 404 response or (permanent)
+			// redirect we probably need to work with later on
+			ReproducibleUtils.storeProperties(header, headerFile);
 		}
 
 		private synchronized Date pareHttpDate(String input) {

diff --git a/...n/java/org/eclipse/sisu/equinox/launching/internal/DefaultEquinoxInstallationFactory.java b/...n/java/org/eclipse/sisu/equinox/launching/internal/DefaultEquinoxInstallationFactory.java
@@ -12,12 +12,9 @@
  *******************************************************************************/
 package org.eclipse.sisu.equinox.launching.internal;
 
-import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -45,6 +42,7 @@
 import org.eclipse.sisu.equinox.launching.EquinoxInstallation;
 import org.eclipse.sisu.equinox.launching.EquinoxInstallationDescription;
 import org.eclipse.sisu.equinox.launching.EquinoxInstallationFactory;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.TychoConstants;
 import org.osgi.framework.Constants;
 
@@ -150,9 +148,7 @@ public EquinoxInstallation createInstallation(EquinoxInstallationDescription des
             File configIni = new File(location, TychoConstants.CONFIG_INI_PATH);
             File configurationLocation = configIni.getParentFile();
             configurationLocation.mkdirs();
-            try (OutputStream fos = new BufferedOutputStream(new FileOutputStream(configIni))) {
-                p.store(fos, null);
-            }
+            ReproducibleUtils.storeProperties(p, configIni);
 
             return new DefaultEquinoxInstallation(description, location, configurationLocation);
         } catch (IOException e) {
@@ -210,9 +206,7 @@ private String createDevProperties(File location, Map<String, String> devEntries
         File file = new File(location, "dev.properties");
         Properties properties = new Properties();
         properties.putAll(devEntries);
-        try (OutputStream os = new BufferedOutputStream(new FileOutputStream(file))) {
-            properties.store(os, null);
-        }
+        ReproducibleUtils.storeProperties(properties, file);
         return file.toURI().toURL().toExternalForm();
     }
 

diff --git a/tycho-api/src/main/java/org/eclipse/tycho/ReproducibleUtils.java b/tycho-api/src/main/java/org/eclipse/tycho/ReproducibleUtils.java
@@ -0,0 +1,50 @@
+/*******************************************************************************
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.tycho;
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Properties;
+import java.util.stream.Collectors;
+
+/**
+ * Utility methods for reproducible builds.
+ */
+public class ReproducibleUtils {
+    private ReproducibleUtils() {
+    }
+
+    /**
+     * Writes the property list to the output stream in a reproducible way. The java.util.Properties
+     * class writes the lines in a non-reproducible order, adds a non-reproducible timestamp and
+     * uses platform-dependent new line characters.
+     * 
+     * @param properties
+     *            the properties object to write to the file.
+     * @param file
+     *            the file to write to.
+     * @throws IOException
+     *             if writing the property list to the specified output stream throws an
+     *             IOException.
+     */
+    public static void storeProperties(Properties properties, File file) throws IOException {
+        final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        properties.store(baos, null);
+        final String content = baos.toString(StandardCharsets.ISO_8859_1).lines().filter(line -> !line.startsWith("#"))
+                .sorted().collect(Collectors.joining("\n", "", "\n"));
+        try (OutputStream os = new BufferedOutputStream(new FileOutputStream(file))) {
+            os.write(content.getBytes(StandardCharsets.ISO_8859_1));
+        }
+    }
+}
diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2/repository/module/ModuleArtifactMap.java b/tycho-core/src/main/java/org/eclipse/tycho/p2/repository/module/ModuleArtifactMap.java
@@ -14,12 +14,9 @@
 
 import static org.eclipse.tycho.p2.repository.BundleConstants.BUNDLE_ID;
 
-import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.OutputStream;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -29,6 +26,7 @@
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.Status;
 import org.eclipse.equinox.p2.core.ProvisionException;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.TychoConstants;
 import org.eclipse.tycho.p2.repository.MavenRepositoryCoordinates;
 import org.eclipse.tycho.p2.repository.RepositoryReader;
@@ -151,7 +149,7 @@ private void store() throws ProvisionException {
         }
 
         try {
-            writeProperties(outputProperties, mapFile);
+            ReproducibleUtils.storeProperties(outputProperties, mapFile);
         } catch (IOException e) {
             String message = "I/O error while writing repository to " + mapFile;
             int code = ProvisionException.REPOSITORY_FAILED_WRITE;
@@ -160,10 +158,4 @@ private void store() throws ProvisionException {
         }
 
     }
-
-    private static void writeProperties(Properties properties, File outputFile) throws IOException {
-        try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(outputFile))) {
-            properties.store(outputStream, null);
-        }
-    }
 }
diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/P2GeneratorImpl.java b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/P2GeneratorImpl.java
@@ -13,12 +13,9 @@
  *******************************************************************************/
 package org.eclipse.tycho.p2resolver;
 
-import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -69,6 +66,7 @@
 import org.eclipse.tycho.OptionalResolutionAction;
 import org.eclipse.tycho.PackagingType;
 import org.eclipse.tycho.ReactorProject;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.TargetEnvironment;
 import org.eclipse.tycho.TychoConstants;
 import org.eclipse.tycho.core.osgitools.BundleReader;
@@ -483,13 +481,7 @@ static void writeArtifactLocations(File outputFile, Map<String, File> artifactLo
             }
         }
 
-        writeProperties(outputProperties, outputFile);
-    }
-
-    private static void writeProperties(Properties properties, File outputFile) throws IOException {
-        try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(outputFile))) {
-            properties.store(outputStream, null);
-        }
+        ReproducibleUtils.storeProperties(outputProperties, outputFile);
     }
 
     /**

diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java
@@ -17,7 +17,6 @@
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.OutputStream;
 import java.net.HttpURLConnection;
 import java.net.URI;
 import java.net.URLConnection;
@@ -72,6 +71,7 @@
 import org.eclipse.tycho.ArtifactType;
 import org.eclipse.tycho.BuildDirectory;
 import org.eclipse.tycho.DependencySeed;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.core.shared.StatusTool;
 import org.eclipse.tycho.p2.repository.GAV;
 import org.eclipse.tycho.p2.repository.RepositoryLayoutHelper;
@@ -523,9 +523,8 @@ private void writeP2Index(File repositoryDestination) throws FacadeException {
         properties.setProperty("version", "1");
         properties.setProperty("artifact.repository.factory.order", "artifacts.xml,!");
         properties.setProperty("metadata.repository.factory.order", "content.xml,!");
-        try (OutputStream stream = new BufferedOutputStream(
-                new FileOutputStream(new File(repositoryDestination, P2_INDEX_FILE)))) {
-            properties.store(stream, null);
+        try {
+            ReproducibleUtils.storeProperties(properties, new File(repositoryDestination, P2_INDEX_FILE));
         } catch (IOException e) {
             throw new FacadeException("writing index file failed", e);
         }

diff --git a/tycho-its/src/test/java/org/eclipse/tycho/test/reproducible/ReproducibleBuildTest.java b/tycho-its/src/test/java/org/eclipse/tycho/test/reproducible/ReproducibleBuildTest.java
@@ -9,6 +9,7 @@
 package org.eclipse.tycho.test.reproducible;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
@@ -25,26 +26,40 @@
 import org.junit.Assert;
 import org.junit.Test;
 
+/**
+ * Tests that the build artifacts produced by Tycho are reproducible.
+ */
 public class ReproducibleBuildTest extends AbstractTychoIntegrationTest {
 	// The ZipEntry.getLastModifiedTime() method uses the default timezone to
 	// convert date and time fields to Instant, so we also use the default timezone
 	// for the expected timestamp here.
 	private static final String EXPECTED_TIMESTAMP_STRING = "2023-01-01T00:00:00";
 	private static final Instant EXPECTED_TIMESTAMP_INSTANT = LocalDateTime.parse(EXPECTED_TIMESTAMP_STRING)
 			.toInstant(OffsetDateTime.now().getOffset());
+	Verifier verifier;
 
 	/**
-	 * Check that the build is reproducible.
+	 * Run the maven integration tests related to reproducible builds.
+	 * 
+	 * @throws Exception
 	 */
 	@Test
-	public void test() throws Exception {
-		Verifier verifier = getVerifier("reproducible-build");
+	public void testReproducible() throws Exception {
+		verifier = getVerifier("reproducible-build");
 		verifier.executeGoals(List.of("clean", "verify"));
 		verifier.verifyErrorFreeLog();
 
-		// Check that the timestamp of the files inside the produced archives is equal
-		// to the one specified in the "project.build.outputTimestamp" property of the
-		// pom file.
+		checkArchiveTimestamps();
+		testBuildQualifier();
+		testPropertiesFiles();
+	}
+
+	/**
+	 * Checks that the timestamp of the files inside the produced archives is equal
+	 * to the one specified in the "project.build.outputTimestamp" property of the
+	 * pom file.
+	 */
+	private void checkArchiveTimestamps() throws Exception {
 		checkTimestamps(verifier.getBasedir() + "/reproducible.bundle/target/reproducible.bundle-1.0.0.jar");
 		checkTimestamps(verifier.getBasedir() + "/reproducible.bundle/target/reproducible.bundle-1.0.0-attached.jar");
 		checkTimestamps(verifier.getBasedir() + "/reproducible.bundle/target/reproducible.bundle-1.0.0-sources.jar");
@@ -55,11 +70,8 @@ public void test() throws Exception {
 		checkTimestamps(verifier.getBasedir() + "/reproducible.iu/target/reproducible.iu-1.0.0.zip");
 		checkTimestamps(verifier.getBasedir() + "/reproducible.repository/target/reproducible.repository-1.0.0.zip");
 		checkTimestamps(verifier.getBasedir() + "/reproducible.repository/target/p2-site.zip");
-
-		// Check that the build qualifier uses the timestamp specified in the
-		// "project.build.outputTimestamp" property of the pom file.
-		checkBuildQualifier(verifier.getBasedir()
-				+ "/reproducible.buildqualifier/target/reproducible.buildqualifier-1.0.0-SNAPSHOT.jar");
+		checkTimestamps(
+				verifier.getBasedir() + "/reproducible.repository/target/products/main.product.id-linux.gtk.x86.zip");
 	}
 
 	private void checkTimestamps(String file) throws IOException {
@@ -72,11 +84,33 @@ private void checkTimestamps(String file) throws IOException {
 		}
 	}
 
-	private void checkBuildQualifier(String file) throws IOException {
+	/**
+	 * Checks that the build qualifier uses the timestamp specified in the
+	 * "project.build.outputTimestamp" property of the pom file.
+	 * 
+	 * @throws IOException
+	 */
+	private void testBuildQualifier() throws IOException {
+		final String file = verifier.getBasedir()
+				+ "/reproducible.buildqualifier/target/reproducible.buildqualifier-1.0.0-SNAPSHOT.jar";
 		try (FileSystem fileSystem = FileSystems.newFileSystem(Path.of(file))) {
-			Path manifest = fileSystem.getPath("META-INF/MANIFEST.MF");
-			List<String> lines = Files.readAllLines(manifest);
+			final Path manifest = fileSystem.getPath("META-INF/MANIFEST.MF");
+			final List<String> lines = Files.readAllLines(manifest);
 			Assert.assertTrue(lines.stream().anyMatch(l -> l.equals("Bundle-Version: 1.0.0.202301010000")));
 		}
 	}
+
+	/**
+	 * Checks that the generated properties files are reproducible.
+	 * 
+	 * @throws IOException
+	 */
+	private void testPropertiesFiles() throws IOException {
+		final String file = verifier.getBasedir() + "/reproducible.bundle/target/reproducible.bundle-1.0.0-sources.jar";
+		try (FileSystem fileSystem = FileSystems.newFileSystem(Path.of(file))) {
+			final Path propFile = fileSystem.getPath("OSGI-INF/l10n/bundle-src.properties");
+			final String content = Files.readString(propFile, StandardCharsets.ISO_8859_1);
+			Assert.assertEquals("bundleName=Reproducible-bundle Source\n" + "bundleVendor=unknown\n", content);
+		}
+	}
 }
diff --git a/...ository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/MavenP2SiteMojo.java b/...ository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/MavenP2SiteMojo.java
@@ -72,6 +72,7 @@
 import org.eclipse.equinox.p2.core.IProvisioningAgent;
 import org.eclipse.equinox.p2.repository.artifact.IArtifactRepositoryManager;
 import org.eclipse.tycho.PackagingType;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.TychoConstants;
 import org.eclipse.tycho.core.PGPService;
 import org.eclipse.tycho.p2maven.tools.TychoFeaturesAndBundlesPublisherApplication;
@@ -484,9 +485,7 @@ protected File createMavenAdvice(Artifact artifact) throws MojoExecutionExceptio
             addProvidesAndProperty(properties, TychoConstants.PROP_EXTENSION, artifact.getType(), cnt++);
             addProvidesAndProperty(properties, TychoConstants.PROP_CLASSIFIER, artifact.getClassifier(), cnt++);
             addProvidesAndProperty(properties, "maven-scope", artifact.getScope(), cnt++);
-            try (OutputStream os = new BufferedOutputStream(new FileOutputStream(p2))) {
-                properties.store(os, null);
-            }
+            ReproducibleUtils.storeProperties(properties, p2);
             return p2;
         } catch (IOException e) {
             throw new MojoExecutionException("failed to generate p2.inf", e);

diff --git a/...va/org/eclipse/tycho/packaging/reverseresolve/MavenCentralArtifactCoordinateResolver.java b/...va/org/eclipse/tycho/packaging/reverseresolve/MavenCentralArtifactCoordinateResolver.java
@@ -12,13 +12,10 @@
  *******************************************************************************/
 package org.eclipse.tycho.packaging.reverseresolve;
 
-import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.OutputStream;
 import java.nio.file.Files;
 import java.security.MessageDigest;
 import java.util.List;
@@ -36,6 +33,7 @@
 import org.codehaus.plexus.component.annotations.Component;
 import org.codehaus.plexus.component.annotations.Requirement;
 import org.codehaus.plexus.logging.Logger;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.core.shared.MavenContext;
 import org.eclipse.tycho.p2.repository.GAV;
 import org.eclipse.tycho.p2.repository.RepositoryLayoutHelper;
@@ -160,9 +158,7 @@ private void cacheResult(File cacheFile, Dependency dependency) {
 			properties.setProperty(KEY_ARTIFACT_ID, dependency.getArtifactId());
 			properties.setProperty(KEY_VERSION, dependency.getVersion());
 			properties.setProperty(KEY_TYPE, dependency.getType());
-			try (OutputStream stream = new BufferedOutputStream(new FileOutputStream(cacheFile))) {
-				properties.store(stream, null);
-			}
+			ReproducibleUtils.storeProperties(properties, cacheFile);
 		} catch (IOException e) {
 			// can't create cache file then...
 		}