Merge pull request #51 from o19s/metrics

Calculating metrics
o19s · Dec 9, 2024 · c8643c9 · c8643c9
2 parents a8517ed + 99ffc09
commit c8643c9
Show file tree

Hide file tree

Showing 22 changed files with 554 additions and 539 deletions.
diff --git a/opensearch-search-quality-evaluation-plugin/build.gradle b/opensearch-search-quality-evaluation-plugin/build.gradle
@@ -57,7 +57,6 @@ dependencies {
     compileOnly "org.apache.httpcomponents:httpcore:${versions.httpcore}"
     compileOnly "org.apache.httpcomponents:httpclient:${versions.httpclient}"
     compileOnly "commons-logging:commons-logging:${versions.commonslogging}"
-    implementation "org.apache.commons:commons-lang3:3.17.0"
     implementation "com.google.code.gson:gson:2.11.0"
     yamlRestTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
 }

diff --git a/opensearch-search-quality-evaluation-plugin/docker-compose.yaml b/opensearch-search-quality-evaluation-plugin/docker-compose.yaml
@@ -10,7 +10,7 @@ services:
       logger.level: info
       OPENSEARCH_INITIAL_ADMIN_PASSWORD: SuperSecretPassword_123
       http.max_content_length: 500mb
-      OPENSEARCH_JAVA_OPTS: "-Xms8g -Xmx8g"
+      OPENSEARCH_JAVA_OPTS: "-Xms16g -Xmx16g"
     ulimits:
       memlock:
         soft: -1

diff --git a/opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-3.17.0.jar.sha1 b/opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-3.17.0.jar.sha1
diff --git a/opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-LICENSE.txt b/opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-LICENSE.txt
diff --git a/opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-NOTICE.txt b/opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-NOTICE.txt
diff --git a/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh b/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh
@@ -3,5 +3,19 @@
 echo "Deleting existing judgments index..."
 curl -s -X DELETE http://localhost:9200/judgments
 
+echo "Creating judgments index..."
+curl -s -X PUT http://localhost:9200/judgments -H 'Content-Type: application/json' -d'
+                                              {
+                                                "mappings": {
+                                                  "properties": {
+                                                    "judgments_id": { "type": "keyword" },
+                                                    "query_id": { "type": "keyword" },
+                                                    "query": { "type": "keyword" },
+                                                    "document_id": { "type": "keyword" },
+                                                    "judgment": { "type": "double" }
+                                                  }
+                                                }
+                                              }'
+
 echo "Creating judgments..."
 curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=20"
diff --git a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh
@@ -1,21 +1,57 @@
 #!/bin/bash -e
 
 QUERY_SET_ID="${1}"
-JUDGMENTS_ID="669fc8aa-3fe7-418f-952b-df7354af8f37"
+JUDGMENTS_ID="9183599e-46dd-49e0-9584-df816164a4c2"
 INDEX="ecommerce"
 ID_FIELD="asin"
-K="10"
+K="20"
+THRESHOLD="1.0" # Default value
 
 curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets_run_results"
 
+# Keyword search
 curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}" \
    -H "Content-Type: application/json" \
     --data-binary '{
-                    "query": {
-                      "match": {
-                        "title": {
-                          "query": "#$query##"
-                        }
+                      "multi_match": {
+                        "query": "#$query##",
+                        "fields": ["id", "title", "category", "bullets", "description", "attrs.Brand", "attrs.Color"]
                       }
-                    }
                   }'
+
+## Neural search
+#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}&search_pipeline=neural-search-pipeline" \
+#   -H "Content-Type: application/json" \
+#    --data-binary '{
+#                      "neural": {
+#                        "title_embedding": {
+#                          "query_text": ""#$query##",
+#                          "k": "50"
+#                        }
+#                      }
+#                  }'
+
+# Hybrid search
+#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}&search_pipeline=hybrid-search-pipeline" \
+#   -H "Content-Type: application/json" \
+#    --data-binary '{
+#                      "hybrid": {
+#                        "queries": [
+#                          {
+#                            "match": {
+#                              "title": {
+#                                "query": "#$query##"
+#                              }
+#                            }
+#                          },
+#                          {
+#                            "neural": {
+#                              "title_embedding": {
+#                                "query_text": "#$query##",
+#                                "k": "50"
+#                              }
+#                            }
+#                          }
+#                        ]
+#                      }
+#                  }'
diff --git a/...evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java b/...evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java
@@ -8,7 +8,6 @@
  */
 package org.opensearch.eval;
 
-import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.index.IndexRequest;
@@ -113,7 +112,7 @@ public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionC
                     final long startTime = System.currentTimeMillis();
                     final long judgments;
 
-                    if(StringUtils.equalsIgnoreCase(searchQualityEvaluationJobParameter.getClickModel(), "coec")) {
+                    if("coec".equalsIgnoreCase(searchQualityEvaluationJobParameter.getClickModel())) {
 
                         LOGGER.info("Beginning implicit judgment generation using clicks-over-expected-clicks.");
                         final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(true, searchQualityEvaluationJobParameter.getMaxRank());