Merge pull request #96 from keeps/jgomes-dev

Added snapshot workflow to github actions, parse of sip-s empty folders

.github/workflows/snapshot.yaml

@@ -0,0 +1,31 @@
+name: Snapshot
+on:
+  push:
+    branches:
+      - "master"
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Chache
+        uses: actions/[email protected]
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-maven-
+      - name: Build Jar
+        run: mvn --batch-mode clean package -P cli
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Deploy Jar
+        run:  mvn --batch-mode deploy -Djacoco.skip=true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

src/main/java/org/roda_project/commons_ip2/model/IPFileShallow.java

@@ -11,6 +11,11 @@
  */
 public class IPFileShallow implements IPFileInterface {
 
+
+  public static IPFileShallow createEmptyFolder(final List<String> emptyFolderPath) {
+    return new IPFileShallow(emptyFolderPath);
+  }
+
   private URI fileLocation;
   private FileType fileType;
   private List<String> relativeFolders;

src/main/java/org/roda_project/commons_ip2/model/impl/eark/EARKUtils.java

@@ -11,7 +11,9 @@
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -758,12 +760,14 @@ protected static void processRepresentationFiles(IPInterface ip, MetsWrapper rep
                   }
                 } else {
                   // treat as a SIP shallow
-                  Optional<IPFileInterface> ipFileInterface = validateFileShallow(ip, fLocat, filePath, fileType);
+                  Optional<IPFileInterface> ipFileInterface = validateFileShallow(ip, fLocat, filePath, fileType,
+                    Collections.emptyList());
                   ipFileInterface.ifPresent(representation::addFile);
                 }
               } else {
                 // treat as a SIP shallow
-                Optional<IPFileInterface> ipFileInterface = validateFileShallow(ip, fLocat, filePath, fileType);
+                Optional<IPFileInterface> ipFileInterface = validateFileShallow(ip, fLocat, filePath, fileType,
+                  Collections.emptyList());
                 ipFileInterface.ifPresent(representation::addFile);
               }
             } else {
@@ -774,6 +778,13 @@ protected static void processRepresentationFiles(IPInterface ip, MetsWrapper rep
         }
       }
 
+      for (DivType subDiv : representationMetsWrapper.getDataDiv().getDiv()) {
+        final List<String> subDivRelativePath = new ArrayList<>();
+        subDivRelativePath.add(subDiv.getLABEL());
+        processRepresentationFilesSubDivs(ip, representationMetsWrapper, representation, representationBasePath, subDiv,
+                subDivRelativePath);
+      }
+
       // post-process validations
       if (representation.getData().isEmpty()) {
         ValidationUtils.addIssue(ip.getValidationReport(), ValidationConstants.REPRESENTATION_HAS_NO_FILES,
@@ -783,12 +794,76 @@ protected static void processRepresentationFiles(IPInterface ip, MetsWrapper rep
     }
   }
 
+  protected static void processRepresentationFilesSubDivs(IPInterface ip, MetsWrapper representationMetsWrapper,
+    IPRepresentation representation, Path representationBasePath, DivType div, List<String> relativePath)
+    throws IPException {
+
+    final List<Fptr> fptrs = div.getFptr();
+    if (fptrs != null && !fptrs.isEmpty()) {
+      for (Fptr fptr : fptrs) {
+        final Object object = fptr.getFILEID();
+        if (object instanceof FileGrpType) {
+          final FileGrpType fileGrp = (FileGrpType) object;
+          for (FileType fileType : fileGrp.getFile()) {
+            if (fileType != null && fileType.getFLocat() != null) {
+              final FLocat fLocat = fileType.getFLocat().get(0);
+              final String href = Utils.extractedRelativePathFromHref(fLocat.getHref());
+              final Path filePath = representationBasePath.resolve(href);
+
+              // Verify that when protocol is file:/// the file is inside the SIP or not
+              if (filePath.startsWith(representationBasePath)) {
+                // treat as a SIP (generic behaviour)
+                if (Files.exists(filePath)) {
+                  final List<String> fileRelativeFolders = Utils
+                    .getFileRelativeFolders(representationBasePath.resolve(IPConstants.DATA), filePath);
+                  final Optional<IPFileInterface> file = validateFile(ip, filePath, fileType, fileRelativeFolders);
+
+                  if (file.isPresent()) {
+                    representation.addFile(file.get());
+                    ValidationUtils.addInfo(ip.getValidationReport(),
+                      ValidationConstants.REPRESENTATION_FILE_FOUND_WITH_MATCHING_CHECKSUMS, ip.getBasePath(),
+                      filePath);
+                  }
+                } else {
+                  // treat as a SIP shallow
+                  final Optional<IPFileInterface> ipFileInterface = validateFileShallow(ip, fLocat, filePath, fileType,
+                    relativePath);
+                  ipFileInterface.ifPresent(representation::addFile);
+                }
+              } else {
+                // treat as a SIP shallow
+                final Optional<IPFileInterface> ipFileInterface = validateFileShallow(ip, fLocat, filePath, fileType,
+                  relativePath);
+                ipFileInterface.ifPresent(representation::addFile);
+              }
+            } else {
+              ValidationUtils.addIssue(ip.getValidationReport(), ValidationConstants.REPRESENTATION_FILE_HAS_NO_FLOCAT,
+                ValidationEntry.LEVEL.ERROR, fileType, ip.getBasePath(), representationMetsWrapper.getMetsPath());
+            }
+          }
+        }
+      }
+    } else if (div.getDiv().isEmpty()) {
+      // This is a empty folder, add an empty folder representation in form of a
+      // IPFileShallow
+      representation.addFile(IPFileShallow.createEmptyFolder(relativePath));
+    }
+
+    for (DivType subDiv : div.getDiv()) {
+      final List<String> subDivRelativePath = new ArrayList<>(relativePath);
+      subDivRelativePath.add(subDiv.getLABEL());
+      processRepresentationFilesSubDivs(ip, representationMetsWrapper, representation, representationBasePath, subDiv,
+        subDivRelativePath);
+    }
+
+  }
+
   private static Optional<IPFileInterface> validateFileShallow(IPInterface ip, FLocat fLocat, Path filePath,
-    FileType fileType) {
+    FileType fileType, List<String> relativeFolders) {
     Optional<IPFileInterface> file = Optional.empty();
 
     if (URI.create(fLocat.getHref()).getScheme() != null) {
-      file = Optional.of(new IPFileShallow(URI.create(fLocat.getHref()), fileType));
+      file = Optional.of(new IPFileShallow(URI.create(fLocat.getHref()), fileType, relativeFolders));
     } else {
       ValidationUtils.addIssue(ip.getValidationReport(), ValidationConstants.REPRESENTATION_SCHEME_NOT_FOUND,
         ValidationEntry.LEVEL.ERROR, ip.getBasePath(), filePath);

src/test/java/org/roda_project/commons_ip2/model/impl/eark/ParseEARK2STest.java

@@ -0,0 +1,51 @@
+package org.roda_project.commons_ip2.model.impl.eark;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.roda_project.commons_ip.model.ParseException;
+import org.roda_project.commons_ip.utils.IPException;
+import org.roda_project.commons_ip2.model.IPFileInterface;
+import org.roda_project.commons_ip2.model.SIP;
+import org.roda_project.commons_ip2.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+
+/**
+ * {@author João Gomes <[email protected]>}.
+ */
+public class ParseEARK2STest {
+  private static final String REPRESENTATION_STATUS_NORMALIZED = "NORMALIZED";
+  private static final Logger LOGGER = LoggerFactory.getLogger(ParseEARK2STest.class);
+
+  private static Path tempFolder;
+
+  @BeforeClass
+  public static void setup() throws IOException {
+    tempFolder = Files.createTempDirectory("temp");
+  }
+
+  @AfterClass
+  public static void cleanup() throws Exception {
+    Utils.deletePath(tempFolder);
+  }
+
+  @Test
+  public void buildEARKSip2withFolders() throws IPException, InterruptedException, ParseException {
+    LOGGER.info("Creating EARK SIP 2");
+    Path zipSIPS = Paths.get("src/test/resources/SIP-S/shallowFileFolderAndEmptyFolder.zip");
+    // 1) invoke static method parse and that's it
+    SIP earkSIP = EARKSIP.parse(zipSIPS, tempFolder);
+
+    List<IPFileInterface> files = earkSIP.getRepresentations().get(0).getData();
+    Assert.assertEquals(4, files.size());
+    LOGGER.info("Done creating full E-ARK SIP-S");
+  }
+}