update the example to include schema

src/main/java/gov/nasa/pds/harvest/dao/MetadataWriter.java

@@ -22,7 +22,7 @@ public class MetadataWriter implements Closeable
     private final static String WARN_SKIP_PRE = "Skipping registered product ";
     private final static String WARN_SKIP_POST = " (LIDVID/LID already exists in registry database)";
     private final static int ES_DOC_BATCH_SIZE = 50;
-
+    private final ConnectionFactory conFact;
     private Logger log;
 
     private RegistryDao registryDao;
@@ -42,6 +42,7 @@ public class MetadataWriter implements Closeable
      */
     public MetadataWriter(ConnectionFactory conFact, RegistryDao dao, Counter counter) throws Exception
     {
+      this.conFact = conFact;
         log = LogManager.getLogger(this.getClass());
         loader = new DataLoader(conFact);
         docBatch = new RegistryDocBatch();
@@ -60,7 +61,7 @@ public void setOverwriteExisting(boolean b)
 
     public void write(Metadata meta) throws Exception
     {
-        docBatch.write(meta, jobId);
+        docBatch.write(this.conFact, meta, jobId);
 
         if(docBatch.size() % ES_DOC_BATCH_SIZE == 0)
         {

src/main/java/gov/nasa/pds/harvest/dao/RegistryDocBatch.java

@@ -1,8 +1,12 @@
 package gov.nasa.pds.harvest.dao;
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
-
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import gov.nasa.pds.registry.common.ConnectionFactory;
+import gov.nasa.pds.registry.common.RestClient;
 import gov.nasa.pds.registry.common.meta.Metadata;
 import gov.nasa.pds.registry.common.util.json.RegistryDocBuilder;
 
@@ -20,7 +24,8 @@ public static class NJsonItem
         public String dataJson;     // Data JSON (line 2)
 
     }
-
+    final private static HashSet<String> alreadyLearned = new HashSet<String>();
+    final private Logger log = LogManager.getLogger(RegistryDocBatch.class);
     private List<NJsonItem> items;
 
 
@@ -32,15 +37,34 @@ public RegistryDocBatch()
         items = new ArrayList<>();
     }
 
-
-    public void write(Metadata meta, String jobId) throws Exception
+    /* hack for PDS-NASA/harvest#127
+     * search the JSON string for any ref_lid_ and add if necessary to index so that it is searchable
+     */
+    private void updateIndex(ConnectionFactory conFact, String json) {
+      int begin_index = json.indexOf("ref_lid_"), end_index;
+      String name;
+      while (-1 < begin_index) {
+        end_index = json.indexOf('"', begin_index+5);
+        name = json.substring(begin_index, end_index);
+        System.out.println(name);
+        if (!alreadyLearned.contains(name)) {
+          try (RestClient client = conFact.createRestClient()) {
+            // FIXME: how to add text or keyword to index to make it searchable??
+          } catch (Exception e) {
+            log.error("Unexpected error (should not have made it here) while updating index with " + name,e);
+          }
+        }
+        alreadyLearned.add(name);
+      }
+    }
+    public void write(ConnectionFactory conFact, Metadata meta, String jobId) throws Exception
     {
         NJsonItem item = new NJsonItem();
         item.lidvid = meta.lidvid;
         item.prodClass = meta.prodClass;
         item.pkJson = RegistryDocBuilder.createPKJson(meta);
         item.dataJson = RegistryDocBuilder.createDataJson(meta, jobId);
-
+        this.updateIndex(conFact, item.dataJson);
         items.add(item);
     }
 

src/main/resources/conf/examples/bundles.xml

@@ -17,7 +17,9 @@
   *     PSA      - Planetary Science Archive
   *     JAXA     - Japan Aerospace Exploration Agency
 -->
-<harvest>
+<harvest
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"  
+  xsi:schemaLocation="https://github.com/NASA-PDS/harvest/blob/main/src/main/resources/conf/configuration.xsd">
   <nodeName>CHANGE_ME</nodeName>
   <!-- Registry configuration -->
   <!-- UPDATE with your registry information -->

src/main/resources/conf/examples/directories.xml

@@ -14,7 +14,9 @@
   *     PSA      - Planetary Science Archive
   *     JAXA     - Japan Aerospace Exploration Agency
 -->
-<harvest>
+<harvest
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"  
+  xsi:schemaLocation="https://github.com/NASA-PDS/harvest/blob/main/src/main/resources/conf/configuration.xsd">
   <nodeName>CHANGE_ME</nodeName>
 
   <!-- Registry configuration -->

src/main/resources/conf/examples/files.xml

@@ -14,7 +14,9 @@
   *     PSA      - Planetary Science Archive
   *     JAXA     - Japan Aerospace Exploration Agency
 -->
-<harvest>
+<harvest
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"  
+  xsi:schemaLocation="https://github.com/NASA-PDS/harvest/blob/main/src/main/resources/conf/configuration.xsd">
   <nodeName>CHANGE_ME</nodeName>
 
   <!-- Registry configuration -->

src/main/resources/conf/examples/xpaths.xml

@@ -1,6 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
-<harvest>
+<harvest
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"  
+  xsi:schemaLocation="https://github.com/NASA-PDS/harvest/blob/main/src/main/resources/conf/configuration.xsd">
   <nodeName>CHANGE_ME</nodeName>
   <load>
     <directories>