Adding search pipeline parameter.

o19s · Dec 6, 2024 · b72c52a · b72c52a
1 parent 5616289
commit b72c52a
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 8 deletions.
diff --git a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh
@@ -8,12 +8,49 @@ K="20"
 
 curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets_run_results"
 
+# Keyword search
 curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}" \
    -H "Content-Type: application/json" \
     --data-binary '{
-                      "match": {
-                        "description": {
-                          "query": "#$query##"
-                        }
+                      "multi_match": {
+                        "query": "#$query##",
+                        "fields": ["id", "title", "category", "bullets", "description", "attrs.Brand", "attrs.Color"]
                       }
                   }'
+
+## Neural search
+#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}&search_pipeline=neural-search-pipeline" \
+#   -H "Content-Type: application/json" \
+#    --data-binary '{
+#                      "neural": {
+#                        "title_embedding": {
+#                          "query_text": ""#$query##",
+#                          "k": "50"
+#                        }
+#                      }
+#                  }'
+
+# Hybrid search
+#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}&search_pipeline=hybrid-search-pipeline" \
+#   -H "Content-Type: application/json" \
+#    --data-binary '{
+#                      "hybrid": {
+#                        "queries": [
+#                          {
+#                            "match": {
+#                              "title": {
+#                                "query": "#$query##"
+#                              }
+#                            }
+#                          },
+#                          {
+#                            "neural": {
+#                              "title_embedding": {
+#                                "query_text": "#$query##",
+#                                "k": "50"
+#                              }
+#                            }
+#                          }
+#                        ]
+#                      }
+#                  }'
diff --git a/...aluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/...aluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java
@@ -156,6 +156,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
             final String querySetId = request.param("id");
             final String judgmentsId = request.param("judgments_id");
             final String index = request.param("index");
+            final String searchPipeline = request.param("search_pipeline", null);
             final String idField = request.param("id_field", "_id");
             final int k = Integer.parseInt(request.param("k", "10"));
 
@@ -182,7 +183,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
             try {
 
                 final OpenSearchQuerySetRunner openSearchQuerySetRunner = new OpenSearchQuerySetRunner(client);
-                final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId, judgmentsId, index, idField, query, k);
+                final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId, judgmentsId, index, searchPipeline, idField, query, k);
                 openSearchQuerySetRunner.save(querySetRunResult);
 
             } catch (Exception ex) {

diff --git a/...y-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/...y-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java
@@ -43,12 +43,14 @@ public AbstractQuerySetRunner(final Client client) {
      * @param querySetId The ID of the query set to run.
      * @param judgmentsId The ID of the judgments set to use for search metric calculation.
      * @param index The name of the index to run the query sets against.
+     * @param searchPipeline The name of the search pipeline to use, or <code>null</code> to not use a search pipeline.
      * @param idField The field in the index that is used to uniquely identify a document.
      * @param query The query that will be used to run the query set.
      * @param k The k used for metrics calculation, i.e. DCG@k.
      * @return The query set {@link QuerySetRunResult results} and calculated metrics.
      */
-    abstract QuerySetRunResult run(String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k) throws Exception;
+    abstract QuerySetRunResult run(String querySetId, final String judgmentsId, final String index, final String searchPipeline,
+                                   final String idField, final String query, final int k) throws Exception;
 
     /**
      * Saves the query set results to a persistent store, which may be the search engine itself.
@@ -141,6 +143,10 @@ public void onResponse(SearchResponse searchResponse) {
                         judgment[0] = (Double) j.get("judgment");
                     }
 
+                    if(judgment[0] > 0) {
+                        LOGGER.info("Found a nonzero judgment! = {}", judgment[0]);
+                    }
+
                 } else {
 
                     // LOGGER.info("No judgments found for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId);
@@ -192,7 +198,7 @@ public List<Double> getRelevanceScores(final String judgmentsId, final String qu
 
         }
 
-        LOGGER.info("----- scores size: " + scores.size());
+       // LOGGER.info("----- scores size: " + scores.size());
 
         //final String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", "));
         //LOGGER.info("Got relevance scores: size = {}: scores = {}", listOfScores.length(), listOfScores);

diff --git a/...evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/...evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java
@@ -49,7 +49,9 @@ public OpenSearchQuerySetRunner(final Client client) {
     }
 
     @Override
-    public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k) throws Exception {
+    public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index,
+                                 final String searchPipeline, final String idField, final String query,
+                                 final int k) throws Exception {
 
         final Collection<Map<String, Long>> querySet = getQuerySet(querySetId);
         LOGGER.info("Found {} queries in query set {}", querySet.size(), querySetId);
@@ -77,6 +79,10 @@ public QuerySetRunResult run(final String querySetId, final String judgmentsId,
                     final String[] excludeFields = new String[]{};
                     searchSourceBuilder.fetchSource(includeFields, excludeFields);
 
+                    if(searchPipeline != null) {
+                        searchSourceBuilder.pipeline(searchPipeline);
+                    }
+
                     final SearchRequest searchRequest = new SearchRequest(index);
                     searchRequest.source(searchSourceBuilder);