Merge pull request #185 from NASA-PDS/i184

Update exception handling for expected null values
NASA-PDS · Jan 22, 2025 · e32990d · e32990d
2 parents c6c50c0 + 769a0ff
commit e32990d
Show file tree

Hide file tree

Showing 11 changed files with 141 additions and 83 deletions.
diff --git a/catalog-legacy/src/main/java/gov/nasa/pds/citool/CIToolIngester.java b/catalog-legacy/src/main/java/gov/nasa/pds/citool/CIToolIngester.java
@@ -90,8 +90,7 @@ public void processVolume(List<Label> catLabels) throws Exception
     	{	
     		if (obj.getIsLocal()) 
     		{
-    			catIngester.ingest(obj);	
-    			report.record(obj.getLabel().getLabelURI(), obj.getLabel().getProblems());
+              catIngester.ingest(obj);
     		}    		
        	} 
 
@@ -117,6 +116,7 @@ public void processVolume(List<Label> catLabels) throws Exception
     		if(obj.getIsLocal())
     		{
     			catIngester.publishObject(obj);
+                report.record(obj.getLabel().getLabelURI(), obj.getLabel().getProblems());
     		}
     	}
     }
@@ -138,8 +138,7 @@ private CatalogVolumeIngester createCatalogVolumeIngester(List<Label> catLabels)
 		{
 			CatalogObject catObj = new CatalogObject(this.report);
 			boolean validFile = catObj.processLabel(lbl);
-			if (validFile) 
-			{
+            if (validFile) {
 				if(catObj.getCatObjType().equalsIgnoreCase("VOLUME"))
 				{
 					pointerFiles = catObj.getPointerFiles();

diff --git a/catalog-legacy/src/main/java/gov/nasa/pds/citool/ingestor/CatalogObject.java b/catalog-legacy/src/main/java/gov/nasa/pds/citool/ingestor/CatalogObject.java
@@ -1,31 +1,30 @@
 package gov.nasa.pds.citool.ingestor;
 
-import java.net.MalformedURLException;
 import java.io.File;
-import java.util.List;
+import java.net.MalformedURLException;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
-import java.util.Map;
 import java.util.HashMap;
 import java.util.Iterator;
-import java.text.SimpleDateFormat;
-
-import gov.nasa.pds.tools.label.Label;
-import gov.nasa.pds.tools.label.Scalar;
-import gov.nasa.pds.tools.label.Set;
-import gov.nasa.pds.tools.label.Sequence;
-import gov.nasa.pds.tools.label.ObjectStatement;
-import gov.nasa.pds.tools.label.AttributeStatement;
-import gov.nasa.pds.tools.label.PointerStatement;
-import gov.nasa.pds.tools.label.Value;
-import gov.nasa.pds.tools.containers.FileReference;
-
-import gov.nasa.pds.citool.report.IngestReport;
-import gov.nasa.pds.citool.util.References;
+import java.util.List;
+import java.util.Map;
 import gov.nasa.pds.citool.file.FileObject;
 import gov.nasa.pds.citool.file.MD5Checksum;
 import gov.nasa.pds.citool.registry.model.Metadata;
 import gov.nasa.pds.citool.registry.model.RegistryObject;
+import gov.nasa.pds.citool.report.IngestReport;
+import gov.nasa.pds.citool.util.References;
+import gov.nasa.pds.tools.containers.FileReference;
+import gov.nasa.pds.tools.label.AttributeStatement;
+import gov.nasa.pds.tools.label.Label;
+import gov.nasa.pds.tools.label.ObjectStatement;
+import gov.nasa.pds.tools.label.PointerStatement;
+import gov.nasa.pds.tools.label.Scalar;
+import gov.nasa.pds.tools.label.Sequence;
+import gov.nasa.pds.tools.label.Set;
+import gov.nasa.pds.tools.label.Value;
 
 
 /**
@@ -162,7 +161,6 @@ protected Map<String, AttributeStatement> getCatalogObj(
 
 		// first level catalog object
 		for (ObjectStatement objSmt : objList) {
-			//System.out.println("CATALOG object type = " + objType);
 			if (objType.equalsIgnoreCase("VOLUME")) {
 				List<ObjectStatement> objList2 = objSmt.getObjects();
 
@@ -174,11 +172,13 @@ protected Map<String, AttributeStatement> getCatalogObj(
 						for (PointerStatement ptSmt : ptList) {
 							if (ptSmt.hasMultipleReferences()) {
 								List<FileReference> refFiles = ptSmt.getFileRefs();
-								for (FileReference fileRef : refFiles) {
-									_pointerFiles.add(fileRef.getPath());
+                                for (FileReference fileRef : refFiles) {
+                                  _pointerFiles
+                                      .add(Paths.get(fileRef.getPath()).getFileName().toString());
 								}
 							} else {
-								_pointerFiles.add(ptSmt.getValue().toString());
+                              _pointerFiles.add(
+                                  Paths.get(ptSmt.getValue().toString()).getFileName().toString());
 							}
 						}
 					}
@@ -189,7 +189,6 @@ protected Map<String, AttributeStatement> getCatalogObj(
 			String keyDesc = null;
 			for (AttributeStatement attrSmt : attrList) {
 				pdsLabelMap.put(attrSmt.getElementIdentifier(), attrSmt);
-				//System.out.println(attrSmt.getElementIdentifier() + " = " + attrSmt.getValue().toString());
 
 				// multivalues
 				if (attrSmt.getValue() instanceof Set) {
@@ -200,7 +199,6 @@ protected Map<String, AttributeStatement> getCatalogObj(
 				}
 				else {
 					_metadata.addMetadata(attrSmt.getElementIdentifier(), attrSmt.getValue().toString());
-					//System.out.println(attrSmt.getElementIdentifier() + " is added into the metadata...");
 				    if (attrSmt.getElementIdentifier().equals("REFERENCE_KEY_ID"))
 				    	keyId = attrSmt.getValue().toString();
 
@@ -225,7 +223,6 @@ protected Map<String, AttributeStatement> getCatalogObj(
 					List<AttributeStatement> objAttr = smt2.getAttributes();
 					lblMap = new HashMap<String, AttributeStatement>(pdsLabelMap);
 
-					//System.out.println("2nd object name = " + smt2.getIdentifier());
 					if (objType.equalsIgnoreCase("DATA_SET_HOUSEKEEPING") &&
 						smt2.getIdentifier().toString().equals("RESOURCE_INFORMATION")) {
 						_resrcObjs.add(smt2);
@@ -235,11 +232,12 @@ protected Map<String, AttributeStatement> getCatalogObj(
 					for (AttributeStatement attrSmt : objAttr) {
 						lblMap.put(attrSmt.getElementIdentifier(), attrSmt);
 
-						//System.out.println("attrSmt.getElementIdentifier() = " + attrSmt.getElementIdentifier());
 						if (attrSmt.getElementIdentifier().toString().equals("TARGET_TYPE")) {
-							String targetName = objSmt.getAttribute("TARGET_NAME").getValue().toString();
-							//System.out.println("target type = " + attrSmt.getValue() + "  target name = " + targetName);
-							_targetInfos.put(targetName, attrSmt.getValue().toString());
+                          if (objSmt.getAttribute("TARGET_NAME") != null) {
+                            String targetName =
+                                objSmt.getAttribute("TARGET_NAME").getValue().toString();
+                            _targetInfos.put(targetName, attrSmt.getValue().toString());
+                          }
 						}
 
 						if (attrSmt.getValue() instanceof Set) {

diff --git a/catalog-legacy/src/main/java/gov/nasa/pds/citool/ingestor/CatalogVolumeIngester.java b/catalog-legacy/src/main/java/gov/nasa/pds/citool/ingestor/CatalogVolumeIngester.java
@@ -1,31 +1,32 @@
 package gov.nasa.pds.citool.ingestor;
 
-import gov.nasa.pds.citool.ingestor.Constants;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 import gov.nasa.pds.citool.registry.client.RegistryClient;
 import gov.nasa.pds.citool.registry.client.RegistryClientManager;
 import gov.nasa.pds.citool.registry.model.FileInfo;
 import gov.nasa.pds.citool.registry.model.Metadata;
 import gov.nasa.pds.citool.registry.model.RegistryObject;
 import gov.nasa.pds.citool.registry.model.Slots;
 import gov.nasa.pds.citool.search.DocGenerator;
+import gov.nasa.pds.citool.search.DocGeneratorException;
 import gov.nasa.pds.citool.util.ReferenceUtils;
 import gov.nasa.pds.citool.util.Utility;
-
 import gov.nasa.pds.tools.LabelParserException;
 import gov.nasa.pds.tools.constants.Constants.ProblemType;
 
-import java.util.List;
-import java.util.ArrayList;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.Map;
-import java.util.HashMap;
-
 
 public class CatalogVolumeIngester 
 {	
 	public static int fileObjCount = 0;
 	public static int registryCount = 0;
+    public static int solrDocCount = 0;
 	public static int failCount = 0;
 
 	private String archiveStatus = null;
@@ -63,7 +64,7 @@ public boolean labelExists(String filename)
     	for (CatalogObject aCatObj: catObjs) 
     	{
     		String tmpFilename = aCatObj.getFilename();
-            tmpFilename = tmpFilename.substring(tmpFilename.lastIndexOf('/') + 1);
+            tmpFilename = tmpFilename.substring(tmpFilename.lastIndexOf(File.separator) + 1);
 
             if (tmpFilename.equalsIgnoreCase(filename)) 
             {
@@ -412,13 +413,15 @@ public void publishObject(CatalogObject obj)
 		RegistryObject ro = obj.getExtrinsicObject();
 		if(ro == null) return;
 
-		try
-		{
-			DocGenerator.getInstance().addDoc(ro);
-		} 
-		catch(Exception ex) 
-		{
-			log.log(Level.SEVERE, "Could not publish catalog object.", ex);
-		}  		
+		try {
+          DocGenerator.getInstance().addDoc(ro);
+          solrDocCount++;
+        } catch (DocGeneratorException e) {
+          LabelParserException lp = new LabelParserException(obj.getLabel().getLabelURI(), null,
+              null, "ingest.warning.skipFile", ProblemType.INVALID_LABEL_WARNING, e.getMessage());
+          obj.getLabel().addProblem(lp);
+        } catch (IOException ex) {
+          log.log(Level.SEVERE, "Unexpected error trying to generate Solr Doc.", ex);
+        }
 	}
 }
diff --git a/...log-legacy/src/main/java/gov/nasa/pds/citool/registry/client/RegistryClientException.java b/...log-legacy/src/main/java/gov/nasa/pds/citool/registry/client/RegistryClientException.java
@@ -0,0 +1,19 @@
+package gov.nasa.pds.citool.registry.client;
+
+/**
+ * 
+ */
+public class RegistryClientException extends Exception {
+
+  private static final long serialVersionUID = -3864429373208213612L;
+
+  /**
+   * Constructs a new {@code DocGeneratorException} with specified detail message.
+   *
+   * @param message the error message
+   */
+  public RegistryClientException(String message) {
+    super(message);
+  }
+
+}
diff --git a/catalog-legacy/src/main/java/gov/nasa/pds/citool/registry/client/RegistryClientSolr.java b/catalog-legacy/src/main/java/gov/nasa/pds/citool/registry/client/RegistryClientSolr.java
@@ -1,10 +1,10 @@
 package gov.nasa.pds.citool.registry.client;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.logging.Logger;
-
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
@@ -14,7 +14,6 @@
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.util.NamedList;
-
 import gov.nasa.pds.citool.registry.model.FileInfo;
 import gov.nasa.pds.citool.registry.model.RegistryObject;
 
@@ -42,7 +41,7 @@ public String publishObject(RegistryObject obj) throws Exception
 	}
 
 
-	public boolean publishFile(FileInfo fi) throws Exception
+    public boolean publishFile(FileInfo fi) throws IOException
 	{
 		// A file with this MD5 hash already exists
 		if(md5Exists(fi))
@@ -64,7 +63,7 @@ public boolean publishFile(FileInfo fi) throws Exception
 	        Object error = resp.get("error"); 
 	        if(error != null)
 	        {
-	        	log.warning("Blob: " + blobName + ": " + error.toString());
+              throw new RegistryClientException("Blob: " + blobName + ": " + error.toString());
 	        }
 	    }
 		catch (HttpSolrClient.RemoteSolrException ex)
@@ -134,7 +133,7 @@ public List<String> getResourceIds(String dataSetId) throws Exception
 	}
 
 
-	private boolean md5Exists(FileInfo fi) throws Exception
+    private boolean md5Exists(FileInfo fi)
 	{
 		// If md5 starts with '0' Solr 7.7.x strips it out. 
 		// Is it a Solr bug?

diff --git a/catalog-legacy/src/main/java/gov/nasa/pds/citool/report/IngestReport.java b/catalog-legacy/src/main/java/gov/nasa/pds/citool/report/IngestReport.java
@@ -123,20 +123,21 @@ private void printProblem(PrintWriter writer,
   @Override
   protected void printFooter(PrintWriter writer) {
       int totalFiles = getNumPassed() + getNumFailed() + getNumSkipped();
-      int totalValidated = getNumPassed() + getNumFailed();
       writer.println();
       writer.println("Summary:");
       writer.println();
-      writer.println("  " + gov.nasa.pds.citool.ingestor.CatalogVolumeIngester.fileObjCount + " of " 
-    		  + totalFiles + " file(s) ingested, "
-              + this.getNumSkipped() + " skipped");
-      //writer.println("  " + this.getNumPassed() + " of " + totalValidated
-      //        + " passed");
+      writer.println("  Number of files processed: " + totalFiles);
+      writer.println("  Number of files skipped: " + this.getNumSkipped());
+      writer.println("  Number of Solr Docs generated: "
+          + gov.nasa.pds.citool.ingestor.CatalogVolumeIngester.solrDocCount);
+
       writer.println();  
       if (printDetails) {
-    	  writer.println("  Number of successful file object ingestion: " + gov.nasa.pds.citool.ingestor.CatalogVolumeIngester.fileObjCount);
-    	  writer.println("  Number of successful registry ingestion: " + gov.nasa.pds.citool.ingestor.CatalogVolumeIngester.registryCount);
-//    	  writer.println("  Name of the registry package: " + gov.nasa.pds.citool.ingestor.CatalogRegistryIngester.registryPackageName);
+        writer.println("Technical Summary:");
+        writer.println("  Number of successful local file object ingestions: "
+            + gov.nasa.pds.citool.ingestor.CatalogVolumeIngester.fileObjCount);
+        writer.println("  Number of successful registry ingestions: "
+            + gov.nasa.pds.citool.ingestor.CatalogVolumeIngester.registryCount);
     	  writer.println();
       }
       writer.println("End of Report\n");

diff --git a/catalog-legacy/src/main/java/gov/nasa/pds/citool/report/Report.java b/catalog-legacy/src/main/java/gov/nasa/pds/citool/report/Report.java
@@ -3,11 +3,6 @@
 
 package gov.nasa.pds.citool.report;
 
-import gov.nasa.pds.tools.LabelParserException;
-import gov.nasa.pds.tools.constants.Constants;
-import gov.nasa.pds.tools.constants.Constants.Severity;
-import gov.nasa.pds.citool.status.Status;
-
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
@@ -18,6 +13,10 @@
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
+import gov.nasa.pds.citool.status.Status;
+import gov.nasa.pds.tools.LabelParserException;
+import gov.nasa.pds.tools.constants.Constants;
+import gov.nasa.pds.tools.constants.Constants.Severity;
 
 /**
  * Abstract class that represents a Report for the Vtool command line API. This
@@ -170,6 +169,10 @@ public Status record(URI sourceUri, final List<LabelParserException> problems) {
           numErrors++;
         }
       } else if (problem.getType().getSeverity() == Constants.Severity.WARNING) {
+        if (problem.getType().equals(
+            Constants.ProblemType.INVALID_LABEL_WARNING)) {
+          status = Status.SKIP;
+        }
         if (Constants.Severity.WARNING.getValue() <= this.level.getValue()) {
           numWarnings++;
         }