Adding SWH-compatible hash for source files of package versions

infrastructure/loader/pom.xml

@@ -75,6 +75,11 @@
             <artifactId>sources-provider</artifactId>
             <version>0.0.12-SNAPSHOT</version>
         </dependency>
+        <dependency>
+            <groupId>eu.fasten-project</groupId>
+            <artifactId>swh-inserter</artifactId>
+            <version>0.0.12-SNAPSHOT</version>
+        </dependency>
 
         <!-- actual dependencies -->
         <dependency>

plugins/pom.xml

@@ -18,6 +18,7 @@
         <module>integration-tests</module>
         <module>ingested-artifact-completion</module>
         <module>sources-provider</module>
+        <module>swh-inserter</module>
     </modules>
 
     <dependencies>

plugins/sources-provider/src/main/java/eu/f4sten/sourcesprovider/data/SourcePayload.java

@@ -15,12 +15,23 @@
  */
 package eu.f4sten.sourcesprovider.data;
 
+import static org.apache.commons.lang3.builder.ToStringStyle.MULTI_LINE_STYLE;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+
 public class SourcePayload {
+
     private String forge;
     private String product;
     private String version;
     private String sourcePath;
 
+    public SourcePayload() {
+        // for object mappers
+    }
+
     public SourcePayload(String forge, String product, String version, String sourcePath) {
         setForge(forge);
         setProduct(product);
@@ -59,4 +70,19 @@ public String getSourcePath() {
     public void setSourcePath(String sourcePath) {
         this.sourcePath = sourcePath;
     }
-}
+
+    @Override
+    public boolean equals(Object obj) {
+        return EqualsBuilder.reflectionEquals(this, obj);
+    }
+
+    @Override
+    public int hashCode() {
+        return HashCodeBuilder.reflectionHashCode(this);
+    }
+
+    @Override
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this, MULTI_LINE_STYLE);
+    }
+}

plugins/sources-provider/src/test/java/eu/f4sten/sourcesprovider/data/SourcePayloadTest.java

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2022 Delft University of Technology
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package eu.f4sten.sourcesprovider.data;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+public class SourcePayloadTest {
+
+    @Test
+    public void defaults1() {
+        var sut = new SourcePayload();
+        assertNull(sut.getForge());
+        assertNull(sut.getProduct());
+        assertNull(sut.getVersion());
+        assertNull(sut.getSourcePath());
+    }
+
+    @Test
+    public void defaults2() {
+        var sut = new SourcePayload("f", "p:q", "1.2.3", "/a/b/c");
+        assertEquals("f", sut.getForge());
+        assertEquals("p:q", sut.getProduct());
+        assertEquals("1.2.3", sut.getVersion());
+        assertEquals("/a/b/c", sut.getSourcePath());
+    }
+
+    @Test
+    public void setForge() {
+        var sut = new SourcePayload();
+        sut.setForge("f");
+        assertEquals("f", sut.getForge());
+    }
+
+    @Test
+    public void setProduct() {
+        var sut = new SourcePayload();
+        sut.setProduct("p:q");
+        assertEquals("p:q", sut.getProduct());
+    }
+
+    @Test
+    public void setVersion() {
+        var sut = new SourcePayload();
+        sut.setVersion("1.2.3");
+        assertEquals("1.2.3", sut.getVersion());
+    }
+
+    @Test
+    public void setSourcePath() {
+        var sut = new SourcePayload();
+        sut.setSourcePath("/a/b/c");
+        assertEquals("/a/b/c", sut.getSourcePath());
+    }
+
+    @Test
+    public void equalityDefault() {
+        var a = new SourcePayload();
+        var b = new SourcePayload();
+        assertEquals(a, b);
+        assertEquals(a.hashCode(), b.hashCode());
+    }
+
+    @Test
+    public void equalityNonDefault() {
+        var a = new SourcePayload("f", "p:q", "1.2.3", "/a/b/c");
+        var b = new SourcePayload("f", "p:q", "1.2.3", "/a/b/c");
+        assertEquals(a, b);
+        assertEquals(a.hashCode(), b.hashCode());
+    }
+
+    @Test
+    public void toStringIsImplemented() {
+        var actual = new SourcePayload().toString();
+        assertTrue(actual.contains(SourcePayload.class.getSimpleName()));
+        assertTrue(actual.contains("\n"));
+        assertTrue(actual.contains("forge"));
+        assertTrue(actual.contains("@"));
+    }
+}

plugins/swh-inserter/pom.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>plugins</artifactId>
+        <groupId>eu.fasten-project</groupId>
+        <version>0.0.12-SNAPSHOT</version>
+    </parent>
+    <artifactId>swh-inserter</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>eu.fasten-project</groupId>
+            <artifactId>sources-provider</artifactId>
+            <version>0.0.12-SNAPSHOT</version>
+            <scope>compile</scope>
+        </dependency>
+    </dependencies>
+</project>

plugins/swh-inserter/src/main/java/eu/f4sten/swhinserter/DatabaseUtils.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2022 Delft University of Technology
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package eu.f4sten.swhinserter;
+
+import java.util.List;
+
+import org.jooq.DSLContext;
+import org.jooq.JSONB;
+import org.jooq.exception.DataAccessException;
+import org.json.JSONObject;
+
+import eu.fasten.core.data.metadatadb.codegen.tables.Files;
+import eu.fasten.core.data.metadatadb.codegen.tables.PackageVersions;
+import eu.fasten.core.data.metadatadb.codegen.tables.Packages;
+import eu.fasten.core.exceptions.UnrecoverableError;
+
+public class DatabaseUtils {
+
+    private final DSLContext context;
+
+    public DatabaseUtils(DSLContext context) {
+        this.context = context;
+    }
+
+    public Long getPkgVersionID(String pkgName, String version) {
+        try {
+            var pkgVerID = context.select(PackageVersions.PACKAGE_VERSIONS.ID)
+                    .from(Packages.PACKAGES, PackageVersions.PACKAGE_VERSIONS)
+                    .where(Packages.PACKAGES.PACKAGE_NAME.eq(pkgName)
+                            .and(PackageVersions.PACKAGE_VERSIONS.PACKAGE_ID.eq(Packages.PACKAGES.ID))
+                            .and(PackageVersions.PACKAGE_VERSIONS.VERSION.eq(version)))
+                    .fetchOne();
+            // May produce null pointer exception
+            return pkgVerID.component1();
+        } catch (DataAccessException e) {
+            throw new UnrecoverableError(e);
+        }
+    }
+
+    public List<String> getFilePaths4PkgVersion(Long pkgVersionID) {
+        try {
+            var filePaths = context.select(Files.FILES.PATH).from(Files.FILES)
+                    .where(Files.FILES.PACKAGE_VERSION_ID.eq(pkgVersionID)).fetch();
+            return filePaths.getValues(Files.FILES.PATH);
+        } catch (DataAccessException e) {
+            throw new UnrecoverableError(e);
+        }
+    }
+
+    public String addFileHash(Long pkgVersionID, String filePath, String fileHash) {
+        try {
+            var fileMetadata = JSONB.valueOf(String.valueOf(new JSONObject().put("swh_checksum", fileHash)));
+            return context.update(Files.FILES).set(Files.FILES.METADATA, fileMetadata)
+                    .where(Files.FILES.PACKAGE_VERSION_ID.eq(pkgVersionID).and(Files.FILES.PATH.eq(filePath)))
+                    .returningResult(Files.FILES.PATH).fetchOne().getValue(Files.FILES.PATH);
+        } catch (DataAccessException e) {
+            throw new UnrecoverableError(e);
+        }
+    }
+}

plugins/swh-inserter/src/main/java/eu/f4sten/swhinserter/Main.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2022 Delft University of Technology
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package eu.f4sten.swhinserter;
+
+import java.io.File;
+import java.nio.file.Path;
+
+import javax.inject.Inject;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import eu.f4sten.infra.AssertArgs;
+import eu.f4sten.infra.Plugin;
+import eu.f4sten.infra.kafka.Kafka;
+import eu.f4sten.infra.kafka.Lane;
+import eu.f4sten.infra.utils.IoUtils;
+import eu.f4sten.sourcesprovider.data.SourcePayload;
+
+public class Main implements Plugin {
+
+    private static final Logger LOG = LoggerFactory.getLogger(Main.class);
+
+    private final SwhInserterArgs args;
+    private final Kafka kafka;
+    private final DatabaseUtils db;
+    private final IoUtils io;
+
+    private final SwhHashCalculator calc;
+
+    @Inject
+    public Main(SwhInserterArgs args, Kafka kafka, DatabaseUtils db, IoUtils io, SwhHashCalculator calc) {
+        this.args = args;
+        this.kafka = kafka;
+        this.db = db;
+        this.io = io;
+        this.calc = calc;
+    }
+
+    @Override
+    public void run() {
+
+        try {
+            AssertArgs.assertFor(args)//
+                    .notNull(a -> a.kafkaIn, "kafka input topic"); //
+
+            LOG.info("Subscribing to '{}'", args.kafkaIn);
+            kafka.subscribe(args.kafkaIn, SourcePayload.class, this::consume);
+            while (true) {
+                LOG.debug("Polling ...");
+                kafka.poll();
+            }
+        } finally {
+            kafka.stop();
+        }
+    }
+
+    public void consume(SourcePayload payload, Lane lane) {
+        LOG.info("Consuming next {} record ...", lane);
+        var pkgName = payload.getProduct();
+        var ver = payload.getVersion();
+
+        var basePath = getBasePath(payload.getForge(), pkgName, ver);
+
+        var pkgVerID = db.getPkgVersionID(pkgName, ver);
+        var paths = db.getFilePaths4PkgVersion(pkgVerID);
+
+        paths.forEach(path -> {
+            var hash = calc.calc(basePath, path);
+            db.addFileHash(pkgVerID, path, hash);
+            LOG.info("Added file hash for {}", path);
+        });
+    }
+
+    private File getBasePath(String forge, String pkgName, String version) {
+        String[] ga = pkgName.split(":");
+        var groupID = ga[0];
+        var artifactID = ga[1];
+        var baseDir = io.getBaseFolder().getPath();
+        var firstChar = Character.toString(groupID.charAt(0));
+        var basePath = Path.of(baseDir, "sources", forge, firstChar, groupID, artifactID, version).toFile();
+        return basePath;
+    }
+}