From b72c52ac3b31a1c689d8da3a55925d0503c87adc Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 11:55:20 -0500 Subject: [PATCH] Adding search pipeline parameter. --- .../scripts/run-query-set.sh | 45 +++++++++++++++++-- .../SearchQualityEvaluationRestHandler.java | 3 +- .../eval/runners/AbstractQuerySetRunner.java | 10 ++++- .../runners/OpenSearchQuerySetRunner.java | 8 +++- 4 files changed, 58 insertions(+), 8 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh index 4e46628..9befe60 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh @@ -8,12 +8,49 @@ K="20" curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets_run_results" +# Keyword search curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}" \ -H "Content-Type: application/json" \ --data-binary '{ - "match": { - "description": { - "query": "#$query##" - } + "multi_match": { + "query": "#$query##", + "fields": ["id", "title", "category", "bullets", "description", "attrs.Brand", "attrs.Color"] } }' + +## Neural search +#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}&search_pipeline=neural-search-pipeline" \ +# -H "Content-Type: application/json" \ +# --data-binary '{ +# "neural": { +# "title_embedding": { +# "query_text": ""#$query##", +# "k": "50" +# } +# } +# }' + +# Hybrid search +#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}&search_pipeline=hybrid-search-pipeline" \ +# -H "Content-Type: application/json" \ +# --data-binary '{ +# "hybrid": { +# "queries": [ +# { +# "match": { +# "title": { +# "query": "#$query##" +# } +# } +# }, +# { +# "neural": { +# "title_embedding": { +# "query_text": "#$query##", +# "k": "50" +# } +# } +# } +# ] +# } +# }' diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java index 2915702..69577fa 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java @@ -156,6 +156,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli final String querySetId = request.param("id"); final String judgmentsId = request.param("judgments_id"); final String index = request.param("index"); + final String searchPipeline = request.param("search_pipeline", null); final String idField = request.param("id_field", "_id"); final int k = Integer.parseInt(request.param("k", "10")); @@ -182,7 +183,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli try { final OpenSearchQuerySetRunner openSearchQuerySetRunner = new OpenSearchQuerySetRunner(client); - final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId, judgmentsId, index, idField, query, k); + final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId, judgmentsId, index, searchPipeline, idField, query, k); openSearchQuerySetRunner.save(querySetRunResult); } catch (Exception ex) { diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index 0f5ad10..d930928 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -43,12 +43,14 @@ public AbstractQuerySetRunner(final Client client) { * @param querySetId The ID of the query set to run. * @param judgmentsId The ID of the judgments set to use for search metric calculation. * @param index The name of the index to run the query sets against. + * @param searchPipeline The name of the search pipeline to use, or null to not use a search pipeline. * @param idField The field in the index that is used to uniquely identify a document. * @param query The query that will be used to run the query set. * @param k The k used for metrics calculation, i.e. DCG@k. * @return The query set {@link QuerySetRunResult results} and calculated metrics. */ - abstract QuerySetRunResult run(String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k) throws Exception; + abstract QuerySetRunResult run(String querySetId, final String judgmentsId, final String index, final String searchPipeline, + final String idField, final String query, final int k) throws Exception; /** * Saves the query set results to a persistent store, which may be the search engine itself. @@ -141,6 +143,10 @@ public void onResponse(SearchResponse searchResponse) { judgment[0] = (Double) j.get("judgment"); } + if(judgment[0] > 0) { + LOGGER.info("Found a nonzero judgment! = {}", judgment[0]); + } + } else { // LOGGER.info("No judgments found for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId); @@ -192,7 +198,7 @@ public List getRelevanceScores(final String judgmentsId, final String qu } - LOGGER.info("----- scores size: " + scores.size()); + // LOGGER.info("----- scores size: " + scores.size()); //final String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", ")); //LOGGER.info("Got relevance scores: size = {}: scores = {}", listOfScores.length(), listOfScores); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index 8c1913d..b145901 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -49,7 +49,9 @@ public OpenSearchQuerySetRunner(final Client client) { } @Override - public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k) throws Exception { + public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index, + final String searchPipeline, final String idField, final String query, + final int k) throws Exception { final Collection> querySet = getQuerySet(querySetId); LOGGER.info("Found {} queries in query set {}", querySet.size(), querySetId); @@ -77,6 +79,10 @@ public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String[] excludeFields = new String[]{}; searchSourceBuilder.fetchSource(includeFields, excludeFields); + if(searchPipeline != null) { + searchSourceBuilder.pipeline(searchPipeline); + } + final SearchRequest searchRequest = new SearchRequest(index); searchRequest.source(searchSourceBuilder);