From b8878a8c1528f2fbac90bb4f69d5a58beef26341 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Thu, 3 Oct 2024 12:12:39 -0600 Subject: [PATCH 1/4] fix a couple NPEs --- src/main/java/org/ecocean/Encounter.java | 2 +- src/main/java/org/ecocean/Util.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index cb8e7128b3..4b40d4a405 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -2746,7 +2746,7 @@ public void setTissueSamples(List samps) { public Set getTissueSampleIDs() { Set ids = new HashSet(); - for (TissueSample ts : tissueSamples) { + if (tissueSamples != null) for (TissueSample ts : tissueSamples) { ids.add(ts.getSampleID()); } return ids; diff --git a/src/main/java/org/ecocean/Util.java b/src/main/java/org/ecocean/Util.java index a74b2cb9fb..c56f44344a 100644 --- a/src/main/java/org/ecocean/Util.java +++ b/src/main/java/org/ecocean/Util.java @@ -1177,6 +1177,7 @@ public static long getVersionFromModified(final String modified) { } public static String getISO8601Date(final String date) { + if (date == null) return null; String iso8601 = date.replace(" ", "T"); if (iso8601.length() == 10) iso8601 += "T00:00:00"; From e968b7ef2867759b37db022adedfa1fa2067a802 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Thu, 3 Oct 2024 12:14:02 -0600 Subject: [PATCH 2/4] do indiv cascaded indexing of encounters in background (so opensearchIndexDeep() will not hang) --- .../java/org/ecocean/MarkedIndividual.java | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 37e750aec8..e7154a5c0e 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2865,11 +2865,30 @@ public void mergeAndThrowawayIndividual(MarkedIndividual other, String username, public void opensearchIndexDeep() throws IOException { - if (this.encounters != null) - for (Encounter enc : this.encounters) { - enc.opensearchIndex(); - } this.opensearchIndex(); + Vector encs = this.encounters; + if ((encs != null) && (encs.size() > 0)) { + int total = encs.size(); + System.out.println("opensearchIndexDeep() backgrounding " + encs.size() + " encs for " + this); + String indId = this.getId(); + Runnable rn = new Runnable() { + public void run() { + int ct = 0; + for (Encounter enc : encs) { + ct++; + System.out.println("opensearchIndexDeep() background indexing " + enc.getId() + " via " + indId + " [" + ct + "/" + total + "]"); + try { + enc.opensearchIndex(); + } catch (Exception ex) { + System.out.println("opensearchIndexDeep() background indexing " + enc.getId() + " FAILED: " + ex.toString()); + ex.printStackTrace(); + } + } + System.out.println("opensearchIndexDeep() backgrounding " + indId + " finished."); + } + }; + new Thread(rn).start(); + } } public String toString() { From cc2488be358b86c1ff3c2e99a47e3f7c808fe1f1 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Fri, 25 Oct 2024 12:41:44 -0600 Subject: [PATCH 3/4] create new shepherd for backgrounding; use ExecutorService --- .../java/org/ecocean/MarkedIndividual.java | 64 ++++++++++++------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 3bfb590e2f..7f6ee756ea 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -7,6 +7,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.concurrent.Executors; +import java.util.concurrent.ExecutorService; + import javax.jdo.Query; import javax.servlet.http.HttpServletRequest; import org.apache.commons.lang3.builder.ToStringBuilder; @@ -44,7 +47,7 @@ public class MarkedIndividual extends Base implements java.io.Serializable { private static HashMap NAMES_KEY_CACHE = new HashMap(); private String alternateid; - private String legacyIndividualID; + private String legacyIndividualID; // additional comments added by researchers private String comments = "None"; @@ -1832,7 +1835,6 @@ public List getAllEmailsToUpdate() { // process encounters for (int i = 0; i < numEncounters; i++) { Encounter enc = (Encounter)encounters.get(i); - List allUsers = new ArrayList(); if (enc.getSubmitters() != null) allUsers.addAll(enc.getSubmitters()); if (enc.getPhotographers() != null) allUsers.addAll(enc.getPhotographers()); @@ -2594,29 +2596,45 @@ public void mergeAndThrowawayIndividual(MarkedIndividual other, String username, public void opensearchIndexDeep() throws IOException { this.opensearchIndex(); - Vector encs = this.encounters; - if ((encs != null) && (encs.size() > 0)) { - int total = encs.size(); - System.out.println("opensearchIndexDeep() backgrounding " + encs.size() + " encs for " + this); - String indId = this.getId(); - Runnable rn = new Runnable() { - public void run() { - int ct = 0; - for (Encounter enc : encs) { - ct++; - System.out.println("opensearchIndexDeep() background indexing " + enc.getId() + " via " + indId + " [" + ct + "/" + total + "]"); - try { - enc.opensearchIndex(); - } catch (Exception ex) { - System.out.println("opensearchIndexDeep() background indexing " + enc.getId() + " FAILED: " + ex.toString()); - ex.printStackTrace(); - } + + final String indivId = this.getId(); + ExecutorService executor = Executors.newFixedThreadPool(4); + Runnable rn = new Runnable() { + public void run() { + Shepherd bgShepherd = new Shepherd("context0"); + bgShepherd.setAction("MarkedIndividual.opensearchIndexDeep"); + bgShepherd.beginDBTransaction(); + MarkedIndividual indiv = bgShepherd.getMarkedIndividual(indivId); + if ((indiv == null) || (indiv.getEncounters() == null)) { + bgShepherd.rollbackAndClose(); + executor.shutdown(); + return; + } + int total = indiv.getNumEncounters(); + int ct = 0; + for (Encounter enc : indiv.getEncounters()) { + ct++; + System.out.println("opensearchIndexDeep() background indexing " + enc.getId() + + " via " + indivId + " [" + ct + "/" + total + "]"); + try { + enc.opensearchIndex(); + } catch (Exception ex) { + System.out.println("opensearchIndexDeep() background indexing " + + enc.getId() + " FAILED: " + ex.toString()); + ex.printStackTrace(); } - System.out.println("opensearchIndexDeep() backgrounding " + indId + " finished."); } - }; - new Thread(rn).start(); - } + System.out.println("opensearchIndexDeep() backgrounding MarkedIndividual " + + indivId + " finished."); + bgShepherd.rollbackAndClose(); + executor.shutdown(); + } + }; + System.out.println("opensearchIndexDeep() begin backgrounding " + this.getNumEncounters() + + " encs for " + this); + executor.execute(rn); + System.out.println("opensearchIndexDeep() [foreground] finished for MarkedIndividual " + + indivId); } public String toString() { From 9e3566b43168024447860ecaea94e2d5646f9561 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Fri, 25 Oct 2024 14:19:59 -0600 Subject: [PATCH 4/4] try/finally wrapper --- .../java/org/ecocean/MarkedIndividual.java | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 7f6ee756ea..858a3f4fba 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2604,29 +2604,32 @@ public void run() { Shepherd bgShepherd = new Shepherd("context0"); bgShepherd.setAction("MarkedIndividual.opensearchIndexDeep"); bgShepherd.beginDBTransaction(); - MarkedIndividual indiv = bgShepherd.getMarkedIndividual(indivId); - if ((indiv == null) || (indiv.getEncounters() == null)) { - bgShepherd.rollbackAndClose(); - executor.shutdown(); - return; - } - int total = indiv.getNumEncounters(); - int ct = 0; - for (Encounter enc : indiv.getEncounters()) { - ct++; - System.out.println("opensearchIndexDeep() background indexing " + enc.getId() + - " via " + indivId + " [" + ct + "/" + total + "]"); - try { - enc.opensearchIndex(); - } catch (Exception ex) { + try { + MarkedIndividual indiv = bgShepherd.getMarkedIndividual(indivId); + if ((indiv == null) || (indiv.getEncounters() == null)) { + bgShepherd.rollbackAndClose(); + executor.shutdown(); + return; + } + int total = indiv.getNumEncounters(); + int ct = 0; + for (Encounter enc : indiv.getEncounters()) { + ct++; System.out.println("opensearchIndexDeep() background indexing " + - enc.getId() + " FAILED: " + ex.toString()); - ex.printStackTrace(); + enc.getId() + " via " + indivId + " [" + ct + "/" + total + "]"); + try { + enc.opensearchIndex(); + } catch (Exception ex) { + System.out.println("opensearchIndexDeep() background indexing " + + enc.getId() + " FAILED: " + ex.toString()); + ex.printStackTrace(); + } } + } finally { + bgShepherd.rollbackAndClose(); } System.out.println("opensearchIndexDeep() backgrounding MarkedIndividual " + indivId + " finished."); - bgShepherd.rollbackAndClose(); executor.shutdown(); } };