From 92c81afe6de7b7faa3243a00b87a8b23e23c3596 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Mon, 25 Sep 2023 22:22:58 -0700 Subject: [PATCH 1/9] Refine collab check --- src/main/java/org/ecocean/servlet/ServletUtilities.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ecocean/servlet/ServletUtilities.java b/src/main/java/org/ecocean/servlet/ServletUtilities.java index fd94637a3a..844afd1d55 100644 --- a/src/main/java/org/ecocean/servlet/ServletUtilities.java +++ b/src/main/java/org/ecocean/servlet/ServletUtilities.java @@ -521,7 +521,8 @@ public static boolean isUserAuthorizedForImportTask(ImportTask occ, HttpServletR //quick collaboration check between current user and bulk import owner //if(occ.getCreator() !=null && Collaboration.canCollaborate(request.getUserPrincipal().getName(), occ.getCreator().getUsername(), myShepherd.getContext()))return true; - if(Collaboration.collaborationBetweenUsers(myShepherd, request.getUserPrincipal().getName(), occ.getCreator().getUsername())!=null)return true; + Collaboration collab = Collaboration.collaborationBetweenUsers(myShepherd, request.getUserPrincipal().getName(), occ.getCreator().getUsername()); + if(collab!=null && collab.getState()!=null && (collab.getState().equals(Collaboration.STATE_EDIT_PRIV)||collab.getState().equals(Collaboration.STATE_APPROVED)) )return true; //quick orgAdminCheck From c7c608227e461ca7fb27556686171d9da7f83664 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Tue, 3 Oct 2023 15:22:41 -0700 Subject: [PATCH 2/9] Make sure we get UTF-8 support and unescaped HTML into Excel export --- .../export/EncounterSearchExportExcelFile.java | 14 +++++++++----- .../export/EncounterSearchExportMetadataExcel.java | 8 +++++++- .../org/ecocean/servlet/export/ExportColumn.java | 3 ++- .../OccurrenceSearchExportMetadataExcel.java | 8 ++++++-- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportExcelFile.java b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportExcelFile.java index 967b972ae6..d1d120dd72 100755 --- a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportExcelFile.java +++ b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportExcelFile.java @@ -5,6 +5,7 @@ import java.io.*; import java.util.*; +import org.apache.commons.text.StringEscapeUtils; import org.ecocean.*; import org.ecocean.genetics.*; import org.ecocean.servlet.ServletUtilities; @@ -16,6 +17,8 @@ import jxl.write.*; import jxl.Workbook; +import jxl.WorkbookSettings; + import java.lang.Boolean; @@ -43,7 +46,6 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) thr context=ServletUtilities.getContext(request); Shepherd myShepherd = new Shepherd(context); myShepherd.setAction("EncounterSearchExportExcelFile.class"); - Vector rEncounters = new Vector(); @@ -91,7 +93,9 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) thr WritableCellFormat integerFormat = new WritableCellFormat(NumberFormats.INTEGER); //let's write out headers for the OBIS export file - WritableWorkbook workbookOBIS = Workbook.createWorkbook(excelFile); + WorkbookSettings ws = new WorkbookSettings(); + ws.setEncoding( "UTF-8" ); + WritableWorkbook workbookOBIS = Workbook.createWorkbook(excelFile,ws); WritableSheet sheet = workbookOBIS.createSheet("Search Results", 0); Label label0 = new Label(0, 0, "Date Last Modified"); sheet.addCell(label0); @@ -245,7 +249,7 @@ else if(CommonConfiguration.getProperty("genusSpecies0",context)!=null){ sheet.addCell(lNumberx22); } - Label lNumberx23 = new Label(22, count, enc.getLocation()); + Label lNumberx23 = new Label(22, count, StringEscapeUtils.unescapeHtml4(enc.getLocation())); sheet.addCell(lNumberx23); //check for available locale coordinates @@ -282,7 +286,7 @@ else if(CommonConfiguration.getProperty("genusSpecies0",context)!=null){ sheet.addCell(lSex); } if(enc.getComments()!=null){ - Label lNumberx26 = new Label(26, count, enc.getComments().replaceAll("
", ". ").replaceAll("\n", "").replaceAll("\r", "")); + Label lNumberx26 = new Label(26, count, StringEscapeUtils.unescapeHtml4(enc.getComments().replaceAll("
", ". ").replaceAll("\n", "").replaceAll("\r", ""))); sheet.addCell(lNumberx26); } if(enc.getSizeAsDouble()!=null){ @@ -290,7 +294,7 @@ else if(CommonConfiguration.getProperty("genusSpecies0",context)!=null){ sheet.addCell(lNumberx27); } if (enc.getIndividual()!=null) { - Label lNumberx28 = new Label(28, count, enc.getIndividual().getDisplayName(request, myShepherd)); + Label lNumberx28 = new Label(28, count, StringEscapeUtils.unescapeHtml4(enc.getIndividual().getDisplayName(request, myShepherd))); sheet.addCell(lNumberx28); } if (enc.getLocationCode() != null) { diff --git a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportMetadataExcel.java b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportMetadataExcel.java index 85dec85ff3..1a99713684 100755 --- a/src/main/java/org/ecocean/servlet/export/EncounterSearchExportMetadataExcel.java +++ b/src/main/java/org/ecocean/servlet/export/EncounterSearchExportMetadataExcel.java @@ -4,6 +4,8 @@ import java.io.*; import java.util.*; + +import org.apache.commons.text.StringEscapeUtils; import org.ecocean.*; import org.ecocean.media.*; import org.ecocean.genetics.*; @@ -13,10 +15,12 @@ import java.lang.StringBuffer; import jxl.write.*; import jxl.Workbook; +import jxl.WorkbookSettings; import java.lang.reflect.Method; import java.lang.reflect.InvocationTargetException; + public class EncounterSearchExportMetadataExcel extends HttpServlet { @@ -132,7 +136,9 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) thr // business logic start here - WritableWorkbook excelWorkbook = Workbook.createWorkbook(excelFile); + WorkbookSettings ws = new WorkbookSettings(); + ws.setEncoding( "UTF-8" ); + WritableWorkbook excelWorkbook = Workbook.createWorkbook(excelFile,ws); WritableSheet sheet = excelWorkbook.createSheet("Search Results", 0); List columns = new ArrayList(); diff --git a/src/main/java/org/ecocean/servlet/export/ExportColumn.java b/src/main/java/org/ecocean/servlet/export/ExportColumn.java index 3be68c7024..5f558e8a1c 100644 --- a/src/main/java/org/ecocean/servlet/export/ExportColumn.java +++ b/src/main/java/org/ecocean/servlet/export/ExportColumn.java @@ -16,6 +16,7 @@ import java.lang.reflect.Method; import java.lang.reflect.InvocationTargetException; +import org.apache.commons.text.StringEscapeUtils; public class ExportColumn { @@ -63,7 +64,7 @@ public String getStringValue(Object obj) throws InvocationTargetException, Illeg if (value == null){ return null; } - return value.toString(); + return StringEscapeUtils.unescapeHtml4(value.toString()); } public int getMeasurementNum() {return measureNum;} diff --git a/src/main/java/org/ecocean/servlet/export/OccurrenceSearchExportMetadataExcel.java b/src/main/java/org/ecocean/servlet/export/OccurrenceSearchExportMetadataExcel.java index 18a6378358..63d8bbf376 100755 --- a/src/main/java/org/ecocean/servlet/export/OccurrenceSearchExportMetadataExcel.java +++ b/src/main/java/org/ecocean/servlet/export/OccurrenceSearchExportMetadataExcel.java @@ -16,6 +16,8 @@ import jxl.write.*; import jxl.Workbook; +import jxl.WorkbookSettings; +import org.apache.commons.text.StringEscapeUtils; public class OccurrenceSearchExportMetadataExcel extends HttpServlet { @@ -121,7 +123,9 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) thr WritableCellFormat integerFormat = new WritableCellFormat(NumberFormats.INTEGER); //let's write out headers for the OBIS export file - WritableWorkbook workbookOBIS = Workbook.createWorkbook(excelFile); + WorkbookSettings ws = new WorkbookSettings(); + ws.setEncoding( "UTF-8" ); + WritableWorkbook workbookOBIS = Workbook.createWorkbook(excelFile,ws); sheet = workbookOBIS.createSheet("Search Results", 0); String[] colHeaders = new String[]{ @@ -188,7 +192,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) thr writeCell("taxonomies", occ.getAllSpecies()); // the getAllSpecies List toStrings nicely writeCell("individualCount", occ.getIndividualCount()); writeCell("groupBehavior", occ.getGroupBehavior()); - writeCell("commments", occ.getComments()); + writeCell("commments", StringEscapeUtils.unescapeHtml4(occ.getComments())); writeCell("modified", occ.getDWCDateLastModified()); writeCell("dateTimeCreated", occ.getDateTimeCreated()); writeCell("fieldStudySite", occ.getFieldStudySite()); From 8de12982067b8dfd374660806cc4f77ddcd76a70 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Sun, 15 Oct 2023 07:06:39 -0700 Subject: [PATCH 3/9] Persist these annot acmID changes --- src/main/java/org/ecocean/acm/AcmUtil.java | 4 +++- src/main/java/org/ecocean/ia/plugin/WildbookIAM.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/ecocean/acm/AcmUtil.java b/src/main/java/org/ecocean/acm/AcmUtil.java index f705a75484..3eea3c510c 100644 --- a/src/main/java/org/ecocean/acm/AcmUtil.java +++ b/src/main/java/org/ecocean/acm/AcmUtil.java @@ -36,7 +36,7 @@ public static int rectifyMediaAssetIds(List mas, List acmIds return numChanged; } - public static int rectifyAnnotationIds(List anns, List acmIds) { + public static int rectifyAnnotationIds(List anns, List acmIds, Shepherd myShepherd) { if ((anns == null) || (acmIds == null) || (anns.size() != acmIds.size())) { IA.log("ERROR: AcmUtil.rectifyAnnotationIds() has invalid lists passed; failing"); return -1; @@ -56,6 +56,8 @@ public static int rectifyAnnotationIds(List anns, List acmId numChanged++; } } + //persist this + if(numChanged>0)myShepherd.updateDBTransaction(); return numChanged; } } diff --git a/src/main/java/org/ecocean/ia/plugin/WildbookIAM.java b/src/main/java/org/ecocean/ia/plugin/WildbookIAM.java index 89c8549b9d..609b75c868 100755 --- a/src/main/java/org/ecocean/ia/plugin/WildbookIAM.java +++ b/src/main/java/org/ecocean/ia/plugin/WildbookIAM.java @@ -285,7 +285,7 @@ public JSONObject sendAnnotations(ArrayList anns, boolean checkFirst if (acmIds == null) { IA.log("WARNING: WildbookIAM.sendAnnotations() could not get list of acmIds from response: " + rtn); } else { - int numChanged = AcmUtil.rectifyAnnotationIds(acmList, acmIds); + int numChanged = AcmUtil.rectifyAnnotationIds(acmList, acmIds, myShepherd); IA.log("INFO: WildbookIAM.sendAnnotations() updated " + numChanged + " Annotation(s) acmId(s) via rectifyAnnotationIds()"); } return rtn; From eb2b3dc6b246f8baae33ef58941c1bdadb24a2af Mon Sep 17 00:00:00 2001 From: holmbergius Date: Tue, 24 Oct 2023 14:31:40 -0700 Subject: [PATCH 4/9] Fix 1 tighter encounter security --- src/main/webapp/encounters/encounter.jsp | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/main/webapp/encounters/encounter.jsp b/src/main/webapp/encounters/encounter.jsp index 54437cc30f..69712fe458 100755 --- a/src/main/webapp/encounters/encounter.jsp +++ b/src/main/webapp/encounters/encounter.jsp @@ -693,7 +693,7 @@ $(function() { <% //set a default date if we cann - if(enc.getDateInMilliseconds()!=null){ + if(isOwner && enc.getDateInMilliseconds()!=null){ //LocalDateTime jodaTime = new LocalDateTime(enc.getDateInMilliseconds()); @@ -867,10 +867,9 @@ else { <% -if(enc.getLocation()!=null){ +if(isOwner && enc.getLocation()!=null){ %> - -<%=encprops.getProperty("locationDescription")%> <%=enc.getLocation()%> + <%=encprops.getProperty("locationDescription")%> <%=enc.getLocation()%> <% } %> @@ -887,11 +886,14 @@ if(enc.getLocation()!=null){ List hier=LocationID.getIDForChildAndParents(enc.getLocationID(), null); int sizeHier=hier.size(); String displayPath=""; - for(int q=0;q <%=displayPath %> @@ -908,7 +910,7 @@ if(CommonConfiguration.showProperty("showCountry",context)){ %> <% - if(enc.getCountry()!=null){ + if(isOwner && enc.getCountry()!=null){ %> : <%=enc.getCountry()%> <% @@ -3632,7 +3634,7 @@ else { <%}%>

- <%if(enc.getDateInMilliseconds()!=null && visible){ %> + <%if(isOwner && visible && enc.getDateInMilliseconds()!=null){ %> <%=enc.getDate()%> @@ -3649,7 +3651,7 @@ else {
<%=encprops.getProperty("verbatimEventDate")%>: <% - if(enc.getVerbatimEventDate()!=null){ + if(isOwner && enc.getVerbatimEventDate()!=null){ %> <%=enc.getVerbatimEventDate()%> <% From 832af7a55207f3c62cd117c91ec539b9bbe6e584 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Thu, 26 Oct 2023 13:13:24 -0700 Subject: [PATCH 5/9] Remove blocking semicolon JDOQL --- src/main/java/org/ecocean/EncounterQueryProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/ecocean/EncounterQueryProcessor.java b/src/main/java/org/ecocean/EncounterQueryProcessor.java index 73ba6de10d..2c722c64d1 100755 --- a/src/main/java/org/ecocean/EncounterQueryProcessor.java +++ b/src/main/java/org/ecocean/EncounterQueryProcessor.java @@ -274,7 +274,7 @@ public static String queryStringBuilder(HttpServletRequest request, StringBuffer jdoqlVariableDeclaration += ";org.ecocean.Annotation annot46;"; } else { - jdoqlVariableDeclaration=" VARIABLES org.ecocean.Annotation annot46;"; + jdoqlVariableDeclaration=" VARIABLES org.ecocean.Annotation annot46"; } } From 74a668ff4e298660321b089967d2595e524b2e8a Mon Sep 17 00:00:00 2001 From: holmbergius Date: Thu, 26 Oct 2023 13:28:15 -0700 Subject: [PATCH 6/9] Further lock down occurrence page --- src/main/webapp/occurrence.jsp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/occurrence.jsp b/src/main/webapp/occurrence.jsp index fd18577b48..17ce0eedea 100755 --- a/src/main/webapp/occurrence.jsp +++ b/src/main/webapp/occurrence.jsp @@ -551,6 +551,7 @@ if (!Util.collectionIsEmptyOrNull(occ.getInformOthers())) { int total = dateSortedEncs.length; for (int i = 0; i < total; i++) { Encounter enc = dateSortedEncs[i]; + if(ServletUtilities.isUserAuthorizedForEncounter(enc, request, myShepherd)){ %> @@ -626,7 +627,11 @@ if (!Util.collectionIsEmptyOrNull(occ.getInformOthers())) { <%}%> - <%} //End of loop iterating over encounters. %> + <% + + }//end if user is authorized + + } //End of loop iterating over encounters. %> From aa5b7d4a1c74a85627894eaba1ec3a0b7e0c4392 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Tue, 24 Oct 2023 14:31:40 -0700 Subject: [PATCH 7/9] Fix 1 tighter encounter security --- src/main/webapp/encounters/encounter.jsp | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/main/webapp/encounters/encounter.jsp b/src/main/webapp/encounters/encounter.jsp index 54437cc30f..69712fe458 100755 --- a/src/main/webapp/encounters/encounter.jsp +++ b/src/main/webapp/encounters/encounter.jsp @@ -693,7 +693,7 @@ $(function() { <% //set a default date if we cann - if(enc.getDateInMilliseconds()!=null){ + if(isOwner && enc.getDateInMilliseconds()!=null){ //LocalDateTime jodaTime = new LocalDateTime(enc.getDateInMilliseconds()); @@ -867,10 +867,9 @@ else { <% -if(enc.getLocation()!=null){ +if(isOwner && enc.getLocation()!=null){ %> - -<%=encprops.getProperty("locationDescription")%> <%=enc.getLocation()%> + <%=encprops.getProperty("locationDescription")%> <%=enc.getLocation()%> <% } %> @@ -887,11 +886,14 @@ if(enc.getLocation()!=null){ List hier=LocationID.getIDForChildAndParents(enc.getLocationID(), null); int sizeHier=hier.size(); String displayPath=""; - for(int q=0;q <%=displayPath %> @@ -908,7 +910,7 @@ if(CommonConfiguration.showProperty("showCountry",context)){ %> <% - if(enc.getCountry()!=null){ + if(isOwner && enc.getCountry()!=null){ %> : <%=enc.getCountry()%> <% @@ -3632,7 +3634,7 @@ else { <%}%>

- <%if(enc.getDateInMilliseconds()!=null && visible){ %> + <%if(isOwner && visible && enc.getDateInMilliseconds()!=null){ %> <%=enc.getDate()%> @@ -3649,7 +3651,7 @@ else {
<%=encprops.getProperty("verbatimEventDate")%>: <% - if(enc.getVerbatimEventDate()!=null){ + if(isOwner && enc.getVerbatimEventDate()!=null){ %> <%=enc.getVerbatimEventDate()%> <% From 1fcd4c5bce9ea09ed510131ad527b9ecf6cd118f Mon Sep 17 00:00:00 2001 From: holmbergius Date: Thu, 26 Oct 2023 13:38:02 -0700 Subject: [PATCH 8/9] More PII privacy in some collab cases --- src/main/webapp/encounters/encounter.jsp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/encounters/encounter.jsp b/src/main/webapp/encounters/encounter.jsp index 69712fe458..9d34334dd8 100755 --- a/src/main/webapp/encounters/encounter.jsp +++ b/src/main/webapp/encounters/encounter.jsp @@ -2333,7 +2333,7 @@ function checkIdDisplay() {

<%=encprops.getProperty("submitter") %>

<% - if(enc.getSubmitters()!=null){ + if(isOwner && enc.getSubmitters()!=null){ %> @@ -2417,7 +2417,7 @@ function checkIdDisplay() {

<%=encprops.getProperty("photographer") %>

<% - if(enc.getPhotographers()!=null){ + if(isOwner && enc.getPhotographers()!=null){ %>
From 5aea77405a0aba2633bb7580c2a9850b15921d92 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Thu, 26 Oct 2023 14:10:08 -0700 Subject: [PATCH 9/9] Expose list of bad media assets --- src/main/webapp/import.jsp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/main/webapp/import.jsp b/src/main/webapp/import.jsp index 825655784c..9b9a1c54ad 100644 --- a/src/main/webapp/import.jsp +++ b/src/main/webapp/import.jsp @@ -616,6 +616,7 @@ try{ int numWithACMID=0; int numAllowedIA=0; int numDetectionComplete=0; + ArrayList invalidMediaAssets=new ArrayList(); for(MediaAsset asset:allAssets){ if(asset.getAcmId()!=null)numWithACMID++; @@ -625,6 +626,10 @@ try{ } else if(asset.validateSourceImage()){numAllowedIA++;myShepherd.updateDBTransaction();} + if(asset.isValidImageForIA() == null || !asset.isValidImageForIA().booleanValue()){ + invalidMediaAssets.add(asset); + } + if(asset.getDetectionStatus()!=null && (asset.getDetectionStatus().equals("complete")||asset.getDetectionStatus().equals("pending"))) numDetectionComplete++; } @@ -634,6 +639,24 @@ try{
  • Number with acmIDs: <%=numWithACMID %>
  • Number valid for image analysis: <%=numAllowedIA %>
    • + + <% + if("complete".equals(itask.getStatus()) && invalidMediaAssets.size()>0){ + %> +
  • Number invalid for image analysis: <%=invalidMediaAssets.size()%> +
      + <% + for(MediaAsset inv_asset:invalidMediaAssets){ + %> +
    1. <%=inv_asset.getId() %>
      2. + <% + } + %> +
    +
    • + <% + } + %>
  • Number that have completed detection: <%=numDetectionComplete %>