Update Harvest to support multi-tenancy and OpenSearch Serverless (#152)

* updates to match new common API The changes remove the local understanding of data and push it to a facade with a series of impls and wrappers to hide the actual understanding of opensearch in registry-common. * move to the connection URL This pulls all of the understanding of connecting to opensearch out of harvest and into registry-common. It uses the same URL style as the command line registry-mgr. * update all of the examples These now reflect the new harvest configuration as well as the connections. Use xmllint to verify that harvest configuration is valid. They point at the schema here at github for now.
NASA-PDS · Jul 5, 2024 · 354a58a · 354a58a
1 parent 02ea033
commit 354a58a
Show file tree

Hide file tree

Showing 19 changed files with 179 additions and 457 deletions.
diff --git a/pom.xml b/pom.xml
@@ -113,7 +113,7 @@
     <dependency>
       <groupId>gov.nasa.pds</groupId>
       <artifactId>registry-common</artifactId>
-      <version>1.5.1</version>
+      <version>1.6.0-SNAPSHOT</version>
     </dependency>
     <!-- rich XML parsing -->
     <dependency>
@@ -128,8 +128,8 @@
       <version>5.7.0</version>
       <scope>test</scope>
     </dependency>
-    <!-- two artifacts for complete configuration parsing -->
-  <dependency>
+    <!-- three (3) artifacts for complete configuration parsing -->
+    <dependency>
       <groupId>jakarta.xml.bind</groupId>
       <artifactId>jakarta.xml.bind-api</artifactId>
       <version>4.0.1</version>

diff --git a/src/main/java/gov/nasa/pds/harvest/cfg/ConfigManager.java b/src/main/java/gov/nasa/pds/harvest/cfg/ConfigManager.java
@@ -6,7 +6,8 @@
 import jakarta.xml.bind.JAXBContext;
 import jakarta.xml.bind.JAXBException;
 import org.glassfish.jaxb.runtime.v2.JAXBContextFactory;
-import gov.nasa.pds.registry.common.cfg.RegistryCfg;
+import gov.nasa.pds.registry.common.ConnectionFactory;
+import gov.nasa.pds.registry.common.EstablishConnectionFactory;
 import gov.nasa.pds.registry.common.meta.cfg.FileRefRule;
 
 /**
@@ -41,19 +42,8 @@ static public List<String> exchangeLidvids (List<CollectionType> ids) {
       }
       return lidvids;
     }
-    /**
-     * FIXME: This is a hack to keep changes limited to harvest.
-     *        Replace the whole registry initiation with something different
-     *        when moving to multitenancy 
-     * @param xml2bean
-     * @return
-     */
-    static public RegistryCfg exchangeRegistry (RegistryType xml) {
-      RegistryCfg bean = new RegistryCfg();
-      bean.authFile = xml.getAuth();
-      bean.indexName = xml.getIndex();
-      bean.url = xml.getServerUrl();
-      return bean;
+    static public ConnectionFactory exchangeRegistry (RegistryType xml) throws Exception {
+      return EstablishConnectionFactory.from (xml.getValue(), xml.getAuth());
     }
     static public HarvestConfigurationType read(File file) throws JAXBException {
       JAXBContext jaxbContext = new JAXBContextFactory().createContext(new Class[]{Harvest.class}, null);

diff --git a/src/main/java/gov/nasa/pds/harvest/cfg/Harvest.java b/src/main/java/gov/nasa/pds/harvest/cfg/Harvest.java
@@ -14,6 +14,14 @@
 
 
 /**
+ * 
+ *           This terrible construct is so that xjc can autodetect this as the
+ *           root node for processing. Many things would be better but this is
+ *           the most workable solution especially if the making of the binding
+ *           code is automated in the pom. The only other real solution is to
+ *           modify one of the classes generated by hand.
+ *         
+ * 
  * <p>Java class for anonymous complex type.
  * 
  * <p>The following schema fragment specifies the expected content contained within this class.

diff --git a/src/main/java/gov/nasa/pds/harvest/cfg/HarvestConfigurationType.java b/src/main/java/gov/nasa/pds/harvest/cfg/HarvestConfigurationType.java
@@ -9,8 +9,8 @@
 
 import jakarta.xml.bind.annotation.XmlAccessType;
 import jakarta.xml.bind.annotation.XmlAccessorType;
-import jakarta.xml.bind.annotation.XmlAttribute;
 import jakarta.xml.bind.annotation.XmlElement;
+import jakarta.xml.bind.annotation.XmlSchemaType;
 import jakarta.xml.bind.annotation.XmlSeeAlso;
 import jakarta.xml.bind.annotation.XmlType;
 
@@ -19,11 +19,11 @@
  * 
  *         These are the basic options for the harvest configuration file.
  * 
- *         @nodeName: the PDS node that this harvest run applies to
  * 
  *         autogenFields: should not be used except in development testing
- *         do: tells where and how to harvest PDS4 labels
+ *         load: tells where and how to harvest PDS4 labels
  *         fileInfo: option allowing filename prefixes to be replaced
+ *         nodeName: the PDS node that this harvest run applies to
  *         productFilter: should not be used except in development testing
  *         registry: define the server harvest should use
  *         xpathMaps: allow constraints in the PDS4 label to control harvesting
@@ -41,12 +41,12 @@
  *         <element name="autogenFields" type="{}autogen_fields_type" minOccurs="0"/>
  *         <element name="load" type="{}load_type"/>
  *         <element name="fileInfo" type="{}file_info_type" minOccurs="0"/>
+ *         <element name="nodeName" type="{}node_name_enum"/>
  *         <element name="productFilter" type="{}filter_type" minOccurs="0"/>
  *         <element name="references" type="{}references_type" minOccurs="0"/>
  *         <element name="registry" type="{}registry_type"/>
  *         <element name="xpathMaps" type="{}xpath_maps_type" minOccurs="0"/>
  *       </all>
- *       <attribute name="nodeName" use="required" type="{}node_name_enum" />
  *     </restriction>
  *   </complexContent>
  * </complexType>
@@ -67,13 +67,14 @@ public class HarvestConfigurationType {
     @XmlElement(required = true)
     protected LoadType load;
     protected FileInfoType fileInfo;
+    @XmlElement(required = true)
+    @XmlSchemaType(name = "normalizedString")
+    protected NodeNameEnum nodeName;
     protected FilterType productFilter;
     protected ReferencesType references;
     @XmlElement(required = true)
     protected RegistryType registry;
     protected XpathMapsType xpathMaps;
-    @XmlAttribute(name = "nodeName", required = true)
-    protected NodeNameEnum nodeName;
 
     /**
      * Gets the value of the autogenFields property.
@@ -147,6 +148,30 @@ public void setFileInfo(FileInfoType value) {
         this.fileInfo = value;
     }
 
+    /**
+     * Gets the value of the nodeName property.
+     * 
+     * @return
+     *     possible object is
+     *     {@link NodeNameEnum }
+     *     
+     */
+    public NodeNameEnum getNodeName() {
+        return nodeName;
+    }
+
+    /**
+     * Sets the value of the nodeName property.
+     * 
+     * @param value
+     *     allowed object is
+     *     {@link NodeNameEnum }
+     *     
+     */
+    public void setNodeName(NodeNameEnum value) {
+        this.nodeName = value;
+    }
+
     /**
      * Gets the value of the productFilter property.
      * 
@@ -243,28 +268,4 @@ public void setXpathMaps(XpathMapsType value) {
         this.xpathMaps = value;
     }
 
-    /**
-     * Gets the value of the nodeName property.
-     * 
-     * @return
-     *     possible object is
-     *     {@link NodeNameEnum }
-     *     
-     */
-    public NodeNameEnum getNodeName() {
-        return nodeName;
-    }
-
-    /**
-     * Sets the value of the nodeName property.
-     * 
-     * @param value
-     *     allowed object is
-     *     {@link NodeNameEnum }
-     *     
-     */
-    public void setNodeName(NodeNameEnum value) {
-        this.nodeName = value;
-    }
-
 }
diff --git a/src/main/java/gov/nasa/pds/harvest/cfg/RegistryType.java b/src/main/java/gov/nasa/pds/harvest/cfg/RegistryType.java
@@ -10,120 +10,76 @@
 import jakarta.xml.bind.annotation.XmlAccessType;
 import jakarta.xml.bind.annotation.XmlAccessorType;
 import jakarta.xml.bind.annotation.XmlAttribute;
-import jakarta.xml.bind.annotation.XmlElement;
 import jakarta.xml.bind.annotation.XmlSchemaType;
 import jakarta.xml.bind.annotation.XmlType;
+import jakarta.xml.bind.annotation.XmlValue;
 import jakarta.xml.bind.annotation.adapters.NormalizedStringAdapter;
 import jakarta.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 
 
 /**
  * 
  *         Define the connection to the registry, security for the connection, and
- *         the index within the registry.
+ *         the index within the registry. The value of this tag is a pointer to
+ *         a registry connection like app://known/direct/localhost.xml or
+ *         app://known/cognito/first_test.xml
  * 
  *         @auth: a java property file containing a username and password
- *         @index: the index to be used by harvest whose default is registry
- *         @trust_self_signed: all self signed certificates for https
- * 
- *         cognito_client_id: the cognito client ID for AWS based instances of opensearch
- *         server_url: the opensearch URL when not using AWS services
- *       
+ *        
  * 
  * <p>Java class for registry_type complex type.
  * 
  * <p>The following schema fragment specifies the expected content contained within this class.
  * 
  * <pre>{@code
  * <complexType name="registry_type">
- *   <complexContent>
- *     <restriction base="{http://www.w3.org/2001/XMLSchema}anyType">
- *       <choice>
- *         <element name="cognito_client_id" type="{http://www.w3.org/2001/XMLSchema}normalizedString"/>
- *         <element name="server_url" type="{http://www.w3.org/2001/XMLSchema}normalizedString"/>
- *       </choice>
+ *   <simpleContent>
+ *     <extension base="<http://www.w3.org/2001/XMLSchema>normalizedString">
  *       <attribute name="auth" use="required" type="{http://www.w3.org/2001/XMLSchema}normalizedString" />
- *       <attribute name="index" type="{http://www.w3.org/2001/XMLSchema}normalizedString" default="registry" />
- *       <attribute name="trust_self_signed" type="{http://www.w3.org/2001/XMLSchema}boolean" default="false" />
- *     </restriction>
- *   </complexContent>
+ *     </extension>
+ *   </simpleContent>
  * </complexType>
  * }</pre>
  * 
  * 
  */
 @XmlAccessorType(XmlAccessType.FIELD)
 @XmlType(name = "registry_type", propOrder = {
-    "cognitoClientId",
-    "serverUrl"
+    "value"
 })
 public class RegistryType {
 
-    @XmlElement(name = "cognito_client_id")
-    @XmlJavaTypeAdapter(NormalizedStringAdapter.class)
-    @XmlSchemaType(name = "normalizedString")
-    protected String cognitoClientId;
-    @XmlElement(name = "server_url")
+    @XmlValue
     @XmlJavaTypeAdapter(NormalizedStringAdapter.class)
     @XmlSchemaType(name = "normalizedString")
-    protected String serverUrl;
+    protected String value;
     @XmlAttribute(name = "auth", required = true)
     @XmlJavaTypeAdapter(NormalizedStringAdapter.class)
     @XmlSchemaType(name = "normalizedString")
     protected String auth;
-    @XmlAttribute(name = "index")
-    @XmlJavaTypeAdapter(NormalizedStringAdapter.class)
-    @XmlSchemaType(name = "normalizedString")
-    protected String index;
-    @XmlAttribute(name = "trust_self_signed")
-    protected Boolean trustSelfSigned;
 
     /**
-     * Gets the value of the cognitoClientId property.
+     * Gets the value of the value property.
      * 
      * @return
      *     possible object is
      *     {@link String }
      *     
      */
-    public String getCognitoClientId() {
-        return cognitoClientId;
+    public String getValue() {
+        return value;
     }
 
     /**
-     * Sets the value of the cognitoClientId property.
+     * Sets the value of the value property.
      * 
      * @param value
      *     allowed object is
      *     {@link String }
      *     
      */
-    public void setCognitoClientId(String value) {
-        this.cognitoClientId = value;
-    }
-
-    /**
-     * Gets the value of the serverUrl property.
-     * 
-     * @return
-     *     possible object is
-     *     {@link String }
-     *     
-     */
-    public String getServerUrl() {
-        return serverUrl;
-    }
-
-    /**
-     * Sets the value of the serverUrl property.
-     * 
-     * @param value
-     *     allowed object is
-     *     {@link String }
-     *     
-     */
-    public void setServerUrl(String value) {
-        this.serverUrl = value;
+    public void setValue(String value) {
+        this.value = value;
     }
 
     /**
@@ -150,60 +106,4 @@ public void setAuth(String value) {
         this.auth = value;
     }
 
-    /**
-     * Gets the value of the index property.
-     * 
-     * @return
-     *     possible object is
-     *     {@link String }
-     *     
-     */
-    public String getIndex() {
-        if (index == null) {
-            return "registry";
-        } else {
-            return index;
-        }
-    }
-
-    /**
-     * Sets the value of the index property.
-     * 
-     * @param value
-     *     allowed object is
-     *     {@link String }
-     *     
-     */
-    public void setIndex(String value) {
-        this.index = value;
-    }
-
-    /**
-     * Gets the value of the trustSelfSigned property.
-     * 
-     * @return
-     *     possible object is
-     *     {@link Boolean }
-     *     
-     */
-    public boolean isTrustSelfSigned() {
-        if (trustSelfSigned == null) {
-            return false;
-        } else {
-            return trustSelfSigned;
-        }
-    }
-
-    /**
-     * Sets the value of the trustSelfSigned property.
-     * 
-     * @param value
-     *     allowed object is
-     *     {@link Boolean }
-     *     
-     */
-    public void setTrustSelfSigned(Boolean value) {
-        this.trustSelfSigned = value;
-    }
-
 }