From e632e15a11bf328386b2d9e395770d6a58bd461c Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 07:40:52 -0500 Subject: [PATCH 01/14] New branch for metrics. --- .../SearchQualityEvaluationRestHandler.java | 5 +- .../clickmodel/coec/CoecClickModel.java | 2 + .../eval/runners/AbstractQuerySetRunner.java | 55 ++++++++++++++++--- ...ner.java => OpenSearchQuerySetRunner.java} | 44 +++++---------- 4 files changed, 66 insertions(+), 40 deletions(-) rename opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/{OpenSearchAbstractQuerySetRunner.java => OpenSearchQuerySetRunner.java} (79%) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java index 83ac507..ba606c9 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java @@ -22,7 +22,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel; import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModelParameters; -import org.opensearch.eval.runners.OpenSearchAbstractQuerySetRunner; +import org.opensearch.eval.runners.OpenSearchQuerySetRunner; import org.opensearch.eval.runners.QuerySetRunResult; import org.opensearch.eval.samplers.AllQueriesQuerySampler; import org.opensearch.eval.samplers.AllQueriesQuerySamplerParameters; @@ -41,7 +41,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.concurrent.atomic.AtomicBoolean; public class SearchQualityEvaluationRestHandler extends BaseRestHandler { @@ -182,7 +181,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli try { - final OpenSearchAbstractQuerySetRunner openSearchQuerySetRunner = new OpenSearchAbstractQuerySetRunner(client); + final OpenSearchQuerySetRunner openSearchQuerySetRunner = new OpenSearchQuerySetRunner(client); final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId, judgmentsId, index, idField, query, k); openSearchQuerySetRunner.save(querySetRunResult); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java index 7afac0c..e1480f9 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java @@ -57,6 +57,7 @@ public class CoecClickModel extends ClickModel { // UBI event names. public static final String EVENT_CLICK = "click"; + // TODO: Change "view" to "impression". public static final String EVENT_VIEW = "view"; private final CoecClickModelParameters parameters; @@ -244,6 +245,7 @@ private Map> getClickthroughRate(final int maxRank .source(searchSourceBuilder) .scroll(scroll); + // TODO Don't use .get() SearchResponse searchResponse = client.search(searchRequest).get(); String scrollId = searchResponse.getScrollId(); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index db14ce2..d2b27c9 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -18,11 +18,12 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.builder.SearchSourceBuilder; +import java.util.Collection; import java.util.Map; /** * Base class for query set runners. Classes that extend this class - * should be specific to a search engine. See the {@link OpenSearchAbstractQuerySetRunner} for an example. + * should be specific to a search engine. See the {@link OpenSearchQuerySetRunner} for an example. */ public abstract class AbstractQuerySetRunner { @@ -52,9 +53,47 @@ public AbstractQuerySetRunner(final Client client) { */ abstract void save(QuerySetRunResult result) throws Exception; + /** + * Gets a query set from the index. + * @param querySetId The ID of the query set to get. + * @return The query set as a collection of maps of query to frequency + * @throws Exception Thrown if the query set cannot be retrieved. + */ + public final Collection> getQuerySet(final String querySetId) throws Exception { + + // Get the query set. + final SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.query(QueryBuilders.matchQuery("_id", querySetId)); + + // Will be at most one match. + sourceBuilder.from(0); + sourceBuilder.size(1); + sourceBuilder.trackTotalHits(true); + + final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.QUERY_SETS_INDEX_NAME).source(sourceBuilder); + + // TODO: Don't use .get() + final SearchResponse searchResponse = client.search(searchRequest).get(); + + if(searchResponse.getHits().getTotalHits().value > 0) { + + // The queries from the query set that will be run. + return (Collection>) searchResponse.getHits().getAt(0).getSourceAsMap().get("queries"); + + } else { + + LOGGER.error("Unable to get query set with ID {}", querySetId); + + // The query set was not found. + throw new RuntimeException("The query set with ID " + querySetId + " was not found."); + + } + + } + /** * Get a judgment from the index. - * @param judgmentsId The judgements ID the judgment to find belongs to. + * @param judgmentsId The ID of the judgments to find. * @param query The user query. * @param documentId The document ID. * @return The value of the judgment, or NaN if the judgment cannot be found. @@ -87,15 +126,17 @@ public Double getJudgment(final String judgmentsId, final String query, final St // TODO: Don't use .get() final SearchResponse searchResponse = client.search(searchRequest).get(); - if(searchResponse.getHits().getHits().length == 0) { + if(searchResponse.getHits().getHits().length > 0) { - // The judgments_id is probably not valid. - return Double.NaN; + final Map j = searchResponse.getHits().getAt(0).getSourceAsMap(); + return Double.parseDouble(j.get("judgment").toString()); } else { - final Map j = searchResponse.getHits().getAt(0).getSourceAsMap(); - return Double.parseDouble(j.get("judgment").toString()); + LOGGER.warn("Unable to find judgments with ID {} for query {} and document ID {}", judgmentsId, query, documentId); + + // TODO: What to return here when there is no judgment? + return Double.NaN; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchAbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java similarity index 79% rename from opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchAbstractQuerySetRunner.java rename to opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index b5ffd9b..1abe94c 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchAbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -17,10 +17,7 @@ import org.opensearch.client.Client; import org.opensearch.core.action.ActionListener; import org.opensearch.eval.SearchQualityEvaluationPlugin; -import org.opensearch.eval.judgments.model.Judgment; import org.opensearch.eval.metrics.DcgSearchMetric; -import org.opensearch.eval.metrics.NdcgSearchMetric; -import org.opensearch.eval.metrics.PrecisionSearchMetric; import org.opensearch.eval.metrics.SearchMetric; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; @@ -38,63 +35,50 @@ /** * A {@link AbstractQuerySetRunner} for Amazon OpenSearch. */ -public class OpenSearchAbstractQuerySetRunner extends AbstractQuerySetRunner { +public class OpenSearchQuerySetRunner extends AbstractQuerySetRunner { - private static final Logger LOGGER = LogManager.getLogger(OpenSearchAbstractQuerySetRunner.class); + private static final Logger LOGGER = LogManager.getLogger(OpenSearchQuerySetRunner.class); /** * Creates a new query set runner + * * @param client An OpenSearch {@link Client}. */ - public OpenSearchAbstractQuerySetRunner(final Client client) { + public OpenSearchQuerySetRunner(final Client client) { super(client); } @Override public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k) throws Exception { - // Get the query set. - final SearchSourceBuilder getQuerySetSearchSourceBuilder = new SearchSourceBuilder(); - getQuerySetSearchSourceBuilder.query(QueryBuilders.matchQuery("_id", querySetId)); - getQuerySetSearchSourceBuilder.from(0); - // TODO: Need to page through to make sure we get all of the queries. - getQuerySetSearchSourceBuilder.size(500); - - final SearchRequest getQuerySetSearchRequest = new SearchRequest(SearchQualityEvaluationPlugin.QUERY_SETS_INDEX_NAME); - getQuerySetSearchRequest.source(getQuerySetSearchSourceBuilder); + final Collection> querySet = getQuerySet(querySetId); try { - // TODO: Don't use .get() - final SearchResponse searchResponse = client.search(getQuerySetSearchRequest).get(); - - // The queries from the query set that will be run. - final Collection> queries = (Collection>) searchResponse.getHits().getAt(0).getSourceAsMap().get("queries"); - // The results of each query. final List queryResults = new ArrayList<>(); - for(Map queryMap : queries) { + for (Map queryMap : querySet) { // Loop over each query in the map and run each one. for (final String userQuery : queryMap.keySet()) { // Replace the query placeholder with the user query. - final String q = query.replace(QUERY_PLACEHOLDER, userQuery); + final String parsedQuery = query.replace(QUERY_PLACEHOLDER, userQuery); // Build the query from the one that was passed in. final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.wrapperQuery(q)); + searchSourceBuilder.query(QueryBuilders.wrapperQuery(parsedQuery)); searchSourceBuilder.from(0); searchSourceBuilder.size(k); - String[] includeFields = new String[] {idField}; - String[] excludeFields = new String[] {}; + String[] includeFields = new String[]{idField}; + String[] excludeFields = new String[]{}; searchSourceBuilder.fetchSource(includeFields, excludeFields); // TODO: Allow for setting this index name. final SearchRequest searchRequest = new SearchRequest(index); - getQuerySetSearchRequest.source(searchSourceBuilder); + searchRequest.source(searchSourceBuilder); client.search(searchRequest, new ActionListener<>() { @@ -165,7 +149,7 @@ public void save(final QuerySetRunResult result) throws Exception { results.put("query_results", result.getQueryResultsAsMap()); // Calculate and add each metric to the object to index. - for(final SearchMetric searchMetric : result.getSearchMetrics()) { + for (final SearchMetric searchMetric : result.getSearchMetrics()) { results.put(searchMetric.getName(), searchMetric.calculate()); } @@ -192,7 +176,7 @@ public List getRelevanceScores(final String query, final List or final List scores = new ArrayList<>(); // Go through each document up to k and get the score. - for(int i = 0; i < k; i++) { + for (int i = 0; i < k; i++) { final String documentId = orderedDocumentIds.get(i); @@ -201,7 +185,7 @@ public List getRelevanceScores(final String query, final List or scores.add(judgment); - if(i == orderedDocumentIds.size()) { + if (i == orderedDocumentIds.size()) { // k is greater than the actual length of documents. break; } From 5f99cfe5a446d440c77823a270cf12468e2712d2 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 07:49:38 -0500 Subject: [PATCH 02/14] Adding error handling. --- .../eval/runners/AbstractQuerySetRunner.java | 69 +++++++++++++++---- .../runners/OpenSearchQuerySetRunner.java | 40 ++--------- 2 files changed, 61 insertions(+), 48 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index d2b27c9..1a25e0d 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -18,8 +18,11 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.builder.SearchSourceBuilder; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Base class for query set runners. Classes that extend this class @@ -97,9 +100,8 @@ public final Collection> getQuerySet(final String querySetId) * @param query The user query. * @param documentId The document ID. * @return The value of the judgment, or NaN if the judgment cannot be found. - * @throws Exception Thrown if the indexed cannot be queried for the judgment. */ - public Double getJudgment(final String judgmentsId, final String query, final String documentId) throws Exception { + public Double getJudgmentValue(final String judgmentsId, final String query, final String documentId) { // Find a judgment that matches the judgments_id, query_id, and document_id fields in the index. @@ -116,30 +118,71 @@ public Double getJudgment(final String judgmentsId, final String query, final St searchSourceBuilder.size(1); // Only include the judgment field. - String[] includeFields = new String[] {"judgment"}; - String[] excludeFields = new String[] {}; + final String[] includeFields = new String[] {"judgment"}; + final String[] excludeFields = new String[] {}; searchSourceBuilder.fetchSource(includeFields, excludeFields); final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME); searchRequest.source(searchSourceBuilder); - // TODO: Don't use .get() - final SearchResponse searchResponse = client.search(searchRequest).get(); + try { - if(searchResponse.getHits().getHits().length > 0) { + // TODO: Don't use .get() + final SearchResponse searchResponse = client.search(searchRequest).get(); - final Map j = searchResponse.getHits().getAt(0).getSourceAsMap(); - return Double.parseDouble(j.get("judgment").toString()); + if (searchResponse.getHits().getHits().length > 0) { - } else { + final Map j = searchResponse.getHits().getAt(0).getSourceAsMap(); + return Double.parseDouble(j.get("judgment").toString()); + + } else { + + LOGGER.warn("Unable to find judgments with ID {} for query {} and document ID {}", judgmentsId, query, documentId); + + // TODO: What to return here when there is no judgment? + return Double.NaN; + + } + + } catch (Exception ex) { + + LOGGER.error("Unable to run query to find judgment for judgmentsIs {}, query = {}, documentId = {}", judgmentsId, query, documentId, ex); + throw new RuntimeException("Unable to run query to find judgment.", ex); + + } + + } + + public List getRelevanceScores(final String judgmentsId, final String query, final List orderedDocumentIds, final int k) { + + // Ordered list of scores. + final List scores = new ArrayList<>(); - LOGGER.warn("Unable to find judgments with ID {} for query {} and document ID {}", judgmentsId, query, documentId); + // Go through each document up to k and get the score. + for (int i = 0; i < k; i++) { - // TODO: What to return here when there is no judgment? - return Double.NaN; + final String documentId = orderedDocumentIds.get(i); + + // Find the judgment value for this combination of query and documentId from the index. + final double judgmentValue = getJudgmentValue(judgmentsId, query, documentId); + + // If a judgment for this query/doc pair is not found, Double.NaN will be returned. + if(!Double.isNaN(judgmentValue)) { + scores.add(judgmentValue); + } + + if (i == orderedDocumentIds.size()) { + // k is greater than the actual length of documents. + break; + } } + String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", ")); + LOGGER.info("Got relevance scores: {}", listOfScores); + + return scores; + } } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index 1abe94c..ea7219d 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -28,7 +28,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.opensearch.eval.SearchQualityEvaluationRestHandler.QUERY_PLACEHOLDER; @@ -72,8 +71,8 @@ public QuerySetRunResult run(final String querySetId, final String judgmentsId, searchSourceBuilder.from(0); searchSourceBuilder.size(k); - String[] includeFields = new String[]{idField}; - String[] excludeFields = new String[]{}; + final String[] includeFields = new String[]{idField}; + final String[] excludeFields = new String[]{}; searchSourceBuilder.fetchSource(includeFields, excludeFields); // TODO: Allow for setting this index name. @@ -97,7 +96,7 @@ public void onResponse(final SearchResponse searchResponse) { } // TODO: Use getJudgment() to get the judgment for this document. - final List relevanceScores = getRelevanceScores(query, orderedDocumentIds, k); + final List relevanceScores = getRelevanceScores(judgmentsId, userQuery, orderedDocumentIds, k); final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores); // TODO: Add these metrics in, too. @@ -112,7 +111,7 @@ public void onResponse(final SearchResponse searchResponse) { @Override public void onFailure(Exception ex) { - LOGGER.error("Unable to search for query: {}", query, ex); + LOGGER.error("Unable to search using query: {}", query, ex); } }); @@ -122,7 +121,7 @@ public void onFailure(Exception ex) { // TODO: Calculate the search metrics for the entire query set given the results and the judgments. final List orderedDocumentIds = new ArrayList<>(); - final List relevanceScores = getRelevanceScores(query, orderedDocumentIds, k); + final List relevanceScores = getRelevanceScores(judgmentsId, "TODO", orderedDocumentIds, k); final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores); // TODO: Add these metrics in, too. //final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores, idealRelevanceScores); @@ -170,33 +169,4 @@ public void onFailure(Exception ex) { } - public List getRelevanceScores(final String query, final List orderedDocumentIds, final int k) { - - // Ordered list of scores. - final List scores = new ArrayList<>(); - - // Go through each document up to k and get the score. - for (int i = 0; i < k; i++) { - - final String documentId = orderedDocumentIds.get(i); - - // TODO: Find the judgment value for this combination of query and documentId from the index. - final double judgment = 0.1; - - scores.add(judgment); - - if (i == orderedDocumentIds.size()) { - // k is greater than the actual length of documents. - break; - } - - } - - String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", ")); - LOGGER.info("Got relevance scores: {}", listOfScores); - - return scores; - - } - } From 73074f12e99d3dbea27743762b83e636873faa72 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 09:21:28 -0500 Subject: [PATCH 03/14] Creating judgments mapping and updating judgment search. --- .../scripts/create-judgments-now.sh | 13 +++++++++++ .../scripts/run-query-set.sh | 6 ++--- .../SearchQualityEvaluationRestHandler.java | 2 +- .../eval/judgments/model/Judgment.java | 2 +- .../eval/runners/AbstractQuerySetRunner.java | 23 +++++++++---------- .../runners/OpenSearchQuerySetRunner.java | 11 +++++++-- .../eval/runners/QuerySetRunResult.java | 6 ++--- 7 files changed, 40 insertions(+), 23 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh b/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh index c353326..ed5ada0 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh @@ -3,5 +3,18 @@ echo "Deleting existing judgments index..." curl -s -X DELETE http://localhost:9200/judgments +echo "Creating judgments index..." +curl -s -X PUT http://localhost:9200/judgments -H 'Content-Type: application/json' -d' + { + "mappings": { + "properties": { + "judgment_id": { "type": "keyword" }, + "query": { "type": "keyword" }, + "document_id": { "type": "keyword" }, + "judgment": { "type": "double" } + } + } + }' + echo "Creating judgments..." curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=20" diff --git a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh index 9a4843e..6d5cfe1 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh @@ -1,7 +1,7 @@ #!/bin/bash -e QUERY_SET_ID="${1}" -JUDGMENTS_ID="669fc8aa-3fe7-418f-952b-df7354af8f37" +JUDGMENTS_ID="a42ebf21-718b-402e-9d3a-259c555cbaed" INDEX="ecommerce" ID_FIELD="asin" K="10" @@ -11,11 +11,9 @@ curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets_run_resu curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}" \ -H "Content-Type: application/json" \ --data-binary '{ - "query": { "match": { - "title": { + "description": { "query": "#$query##" } } - } }' diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java index ba606c9..2915702 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java @@ -186,7 +186,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli openSearchQuerySetRunner.save(querySetRunResult); } catch (Exception ex) { - LOGGER.error("Unable to run query set.", ex); + LOGGER.error("Unable to run query set. Verify query set and judgments exist.", ex); return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, ex.getMessage())); } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java index 514b5dd..15f9b5d 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java @@ -53,7 +53,7 @@ public Map getJudgmentAsMap() { final Map judgmentMap = new HashMap<>(); judgmentMap.put("query_id", queryId); judgmentMap.put("query", query); - judgmentMap.put("document", document); + judgmentMap.put("document_id", document); judgmentMap.put("judgment", judgment); return judgmentMap; diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index 1a25e0d..62c77ca 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -106,9 +106,9 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin // Find a judgment that matches the judgments_id, query_id, and document_id fields in the index. final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.matchQuery("judgments_id", judgmentsId)); - boolQueryBuilder.must(QueryBuilders.matchQuery("query", query)); - boolQueryBuilder.must(QueryBuilders.matchQuery("document_id", documentId)); + boolQueryBuilder.must(QueryBuilders.termQuery("judgment_id", judgmentsId)); + boolQueryBuilder.must(QueryBuilders.termQuery("query", query)); + boolQueryBuilder.must(QueryBuilders.termQuery("document_id", documentId)); final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(boolQueryBuilder); @@ -117,13 +117,14 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin searchSourceBuilder.from(0); searchSourceBuilder.size(1); - // Only include the judgment field. + // Only include the judgment field in the response. final String[] includeFields = new String[] {"judgment"}; final String[] excludeFields = new String[] {}; searchSourceBuilder.fetchSource(includeFields, excludeFields); - final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME); - searchRequest.source(searchSourceBuilder); + LOGGER.info("query: " + boolQueryBuilder.toString()); + + final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME).source(searchSourceBuilder); try { @@ -137,9 +138,7 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin } else { - LOGGER.warn("Unable to find judgments with ID {} for query {} and document ID {}", judgmentsId, query, documentId); - - // TODO: What to return here when there is no judgment? + // No judgment for this query/doc pair exists. return Double.NaN; } @@ -159,7 +158,7 @@ public List getRelevanceScores(final String judgmentsId, final String qu final List scores = new ArrayList<>(); // Go through each document up to k and get the score. - for (int i = 0; i < k; i++) { + for (int i = 0; i < k && i < orderedDocumentIds.size(); i++) { final String documentId = orderedDocumentIds.get(i); @@ -178,8 +177,8 @@ public List getRelevanceScores(final String judgmentsId, final String qu } - String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", ")); - LOGGER.info("Got relevance scores: {}", listOfScores); + final String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", ")); + LOGGER.info("Got relevance scores: size = {}: scores = {}", listOfScores.length(), listOfScores); return scores; diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index ea7219d..9cabaa4 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import static org.opensearch.eval.SearchQualityEvaluationRestHandler.QUERY_PLACEHOLDER; @@ -79,6 +80,8 @@ public QuerySetRunResult run(final String querySetId, final String judgmentsId, final SearchRequest searchRequest = new SearchRequest(index); searchRequest.source(searchSourceBuilder); + LOGGER.info("Doing search for: {}", userQuery); + client.search(searchRequest, new ActionListener<>() { @Override @@ -111,7 +114,7 @@ public void onResponse(final SearchResponse searchResponse) { @Override public void onFailure(Exception ex) { - LOGGER.error("Unable to search using query: {}", query, ex); + LOGGER.error("Unable to search using query: {}", parsedQuery, ex); } }); @@ -128,8 +131,12 @@ public void onFailure(Exception ex) { //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores); final Collection searchMetrics = List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric); + final String querySetRunId = UUID.randomUUID().toString(); + final QuerySetRunResult querySetRunResult = new QuerySetRunResult(querySetRunId, queryResults, searchMetrics); + + LOGGER.info("Query set run complete: {}", querySetRunId); - return new QuerySetRunResult(queryResults, searchMetrics); + return querySetRunResult; } catch (Exception ex) { throw new RuntimeException("Unable to run query set.", ex); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java index 0c97e67..4e9a046 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java @@ -15,7 +15,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; /** * The results of a query set run. @@ -28,11 +27,12 @@ public class QuerySetRunResult { /** * Creates a new query set run result. A random UUID is generated as the run ID. + * @param runId A unique identifier for this query set run. * @param queryResults A collection of {@link QueryResult} that contains the queries and search results. * @param metrics The {@link SearchMetric metrics} calculated from the search results. */ - public QuerySetRunResult(final List queryResults, final Collection metrics) { - this.runId = UUID.randomUUID().toString(); + public QuerySetRunResult(final String runId, final List queryResults, final Collection metrics) { + this.runId = runId; this.queryResults = queryResults; this.metrics = metrics; } From 561628975df9b1630183f532047e0dc4fe873ace Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 11:39:27 -0500 Subject: [PATCH 04/14] Working on metrics calculation. Signed-off-by: jzonthemtn --- .../docker-compose.yaml | 2 +- .../scripts/create-judgments-now.sh | 3 +- .../scripts/run-query-set.sh | 4 +- .../clickmodel/coec/CoecClickModel.java | 10 +-- .../opensearch/OpenSearchHelper.java | 1 + .../opensearch/eval/metrics/SearchMetric.java | 12 ++++ .../eval/runners/AbstractQuerySetRunner.java | 65 ++++++++++++------- .../runners/OpenSearchQuerySetRunner.java | 23 +++++-- 8 files changed, 80 insertions(+), 40 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/docker-compose.yaml b/opensearch-search-quality-evaluation-plugin/docker-compose.yaml index 8938320..c2ab3b7 100644 --- a/opensearch-search-quality-evaluation-plugin/docker-compose.yaml +++ b/opensearch-search-quality-evaluation-plugin/docker-compose.yaml @@ -10,7 +10,7 @@ services: logger.level: info OPENSEARCH_INITIAL_ADMIN_PASSWORD: SuperSecretPassword_123 http.max_content_length: 500mb - OPENSEARCH_JAVA_OPTS: "-Xms8g -Xmx8g" + OPENSEARCH_JAVA_OPTS: "-Xms16g -Xmx16g" ulimits: memlock: soft: -1 diff --git a/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh b/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh index ed5ada0..b35ddc8 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh @@ -8,7 +8,8 @@ curl -s -X PUT http://localhost:9200/judgments -H 'Content-Type: application/jso { "mappings": { "properties": { - "judgment_id": { "type": "keyword" }, + "judgments_id": { "type": "keyword" }, + "query_id": { "type": "keyword" }, "query": { "type": "keyword" }, "document_id": { "type": "keyword" }, "judgment": { "type": "double" } diff --git a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh index 6d5cfe1..4e46628 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh @@ -1,10 +1,10 @@ #!/bin/bash -e QUERY_SET_ID="${1}" -JUDGMENTS_ID="a42ebf21-718b-402e-9d3a-259c555cbaed" +JUDGMENTS_ID="9183599e-46dd-49e0-9584-df816164a4c2" INDEX="ecommerce" ID_FIELD="asin" -K="10" +K="20" curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets_run_results" diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java index e1480f9..1dfb18d 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java @@ -9,7 +9,6 @@ package org.opensearch.eval.judgments.clickmodel.coec; import com.google.gson.Gson; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.search.SearchRequest; @@ -24,8 +23,8 @@ import org.opensearch.eval.judgments.model.Judgment; import org.opensearch.eval.judgments.model.ubi.event.UbiEvent; import org.opensearch.eval.judgments.opensearch.OpenSearchHelper; -import org.opensearch.eval.judgments.util.MathUtils; import org.opensearch.eval.judgments.util.IncrementalUserQueryHash; +import org.opensearch.eval.judgments.util.MathUtils; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -272,7 +271,7 @@ private Map> getClickthroughRate(final int maxRank // Get the ClickthroughRate object for the object that was interacted with. final ClickthroughRate clickthroughRate = clickthroughRates.stream().filter(p -> p.getObjectId().equals(ubiEvent.getEventAttributes().getObject().getObjectId())).findFirst().orElse(new ClickthroughRate(ubiEvent.getEventAttributes().getObject().getObjectId())); - if (StringUtils.equalsIgnoreCase(ubiEvent.getActionName(), EVENT_CLICK)) { + if (EVENT_CLICK.equalsIgnoreCase(ubiEvent.getActionName())) { //LOGGER.info("Logging a CLICK on " + ubiEvent.getEventAttributes().getObject().getObjectId()); clickthroughRate.logClick(); } else { @@ -341,10 +340,11 @@ public Map getRankAggregatedClickThrough() throws Exception { final Terms actionTerms = searchResponse.getAggregations().get("By_Action"); final Collection actionBuckets = actionTerms.getBuckets(); + for(final Terms.Bucket actionBucket : actionBuckets) { // Handle the "click" bucket. - if(StringUtils.equalsIgnoreCase(actionBucket.getKey().toString(), EVENT_CLICK)) { + if(EVENT_CLICK.equalsIgnoreCase(actionBucket.getKey().toString())) { final Terms positionTerms = actionBucket.getAggregations().get("By_Position"); final Collection positionBuckets = positionTerms.getBuckets(); @@ -356,7 +356,7 @@ public Map getRankAggregatedClickThrough() throws Exception { } // Handle the "view" bucket. - if(StringUtils.equalsIgnoreCase(actionBucket.getKey().toString(), EVENT_VIEW)) { + if(EVENT_VIEW.equalsIgnoreCase(actionBucket.getKey().toString())) { final Terms positionTerms = actionBucket.getAggregations().get("By_Position"); final Collection positionBuckets = positionTerms.getBuckets(); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java index 23ea955..d268933 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java @@ -328,6 +328,7 @@ public String indexJudgments(final Collection judgments) throws Except for(final Judgment judgment : judgments) { + // TODO: Add a timestamp. final Map j = judgment.getJudgmentAsMap(); j.put("judgments_id", judgmentsId); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java index 658c202..2b3044b 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java @@ -21,6 +21,8 @@ public abstract class SearchMetric { public abstract double calculate(); + private Double value = Double.NaN; + public SearchMetric(final int k) { this.k = k; } @@ -29,4 +31,14 @@ public int getK() { return k; } + public double getValue() { + + if(Double.isNaN(value)) { + this.value = calculate(); + } + + return value; + + } + } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index 62c77ca..0f5ad10 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -13,6 +13,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; +import org.opensearch.core.action.ActionListener; import org.opensearch.eval.SearchQualityEvaluationPlugin; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -22,7 +23,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; /** * Base class for query set runners. Classes that extend this class @@ -106,7 +106,7 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin // Find a judgment that matches the judgments_id, query_id, and document_id fields in the index. final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.termQuery("judgment_id", judgmentsId)); + boolQueryBuilder.must(QueryBuilders.termQuery("judgments_id", judgmentsId)); boolQueryBuilder.must(QueryBuilders.termQuery("query", query)); boolQueryBuilder.must(QueryBuilders.termQuery("document_id", documentId)); @@ -122,38 +122,51 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin final String[] excludeFields = new String[] {}; searchSourceBuilder.fetchSource(includeFields, excludeFields); - LOGGER.info("query: " + boolQueryBuilder.toString()); - final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME).source(searchSourceBuilder); - try { + final Double[] judgment = new Double[1]; + judgment[0] = Double.NaN; + + client.search(searchRequest, new ActionListener<>() { + + @Override + public void onResponse(SearchResponse searchResponse) { - // TODO: Don't use .get() - final SearchResponse searchResponse = client.search(searchRequest).get(); + if (searchResponse.getHits().getHits().length > 0) { - if (searchResponse.getHits().getHits().length > 0) { + final Map j = searchResponse.getHits().getAt(0).getSourceAsMap(); - final Map j = searchResponse.getHits().getAt(0).getSourceAsMap(); - return Double.parseDouble(j.get("judgment").toString()); + // TODO: Why does this not exist in some cases? + if(j.containsKey("judgment")) { + judgment[0] = (Double) j.get("judgment"); + } - } else { + } else { - // No judgment for this query/doc pair exists. - return Double.NaN; + // LOGGER.info("No judgments found for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId); + + // No judgment for this query/doc pair exists. + judgment[0] = Double.NaN; + + } } - } catch (Exception ex) { + @Override + public void onFailure(Exception ex) { + LOGGER.error("Unable to get judgment for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId, ex); + } - LOGGER.error("Unable to run query to find judgment for judgmentsIs {}, query = {}, documentId = {}", judgmentsId, query, documentId, ex); - throw new RuntimeException("Unable to run query to find judgment.", ex); + }); - } + return judgment[0]; } public List getRelevanceScores(final String judgmentsId, final String query, final List orderedDocumentIds, final int k) { + // LOGGER.info("Getting relevance scores for query: {}, k = {}, docIds size = {}", query, k, orderedDocumentIds.size()); + // Ordered list of scores. final List scores = new ArrayList<>(); @@ -163,22 +176,26 @@ public List getRelevanceScores(final String judgmentsId, final String qu final String documentId = orderedDocumentIds.get(i); // Find the judgment value for this combination of query and documentId from the index. - final double judgmentValue = getJudgmentValue(judgmentsId, query, documentId); + final Double judgmentValue = getJudgmentValue(judgmentsId, query, documentId); + + // LOGGER.info("Got judgment value: {}", judgmentValue); // If a judgment for this query/doc pair is not found, Double.NaN will be returned. if(!Double.isNaN(judgmentValue)) { scores.add(judgmentValue); } - if (i == orderedDocumentIds.size()) { - // k is greater than the actual length of documents. - break; - } +// if (i == orderedDocumentIds.size()) { +// // k is greater than the actual length of documents. +// break; +// } } - final String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", ")); - LOGGER.info("Got relevance scores: size = {}: scores = {}", listOfScores.length(), listOfScores); + LOGGER.info("----- scores size: " + scores.size()); + + //final String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", ")); + //LOGGER.info("Got relevance scores: size = {}: scores = {}", listOfScores.length(), listOfScores); return scores; diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index 9cabaa4..8c1913d 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -52,6 +52,7 @@ public OpenSearchQuerySetRunner(final Client client) { public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k) throws Exception { final Collection> querySet = getQuerySet(querySetId); + LOGGER.info("Found {} queries in query set {}", querySet.size(), querySetId); try { @@ -76,11 +77,12 @@ public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String[] excludeFields = new String[]{}; searchSourceBuilder.fetchSource(includeFields, excludeFields); - // TODO: Allow for setting this index name. final SearchRequest searchRequest = new SearchRequest(index); searchRequest.source(searchSourceBuilder); - LOGGER.info("Doing search for: {}", userQuery); + // This is to keep OpenSearch from rejecting queries. + // TODO: Look at using the Workload Management in 2.18.0. + Thread.sleep(50); client.search(searchRequest, new ActionListener<>() { @@ -89,16 +91,21 @@ public void onResponse(final SearchResponse searchResponse) { final List orderedDocumentIds = new ArrayList<>(); + // LOGGER.info("Found {} results for query {}", searchResponse.getHits().getTotalHits().value, userQuery); + for (final SearchHit hit : searchResponse.getHits().getHits()) { final Map sourceAsMap = hit.getSourceAsMap(); final String documentId = sourceAsMap.get(idField).toString(); + //LOGGER.info(" -- query {}, documentId: {}", userQuery, documentId); orderedDocumentIds.add(documentId); } - // TODO: Use getJudgment() to get the judgment for this document. + //LOGGER.info("Number of documents: " + orderedDocumentIds.size()); + + // TODO: If no hits are returned, there's no need to get the relevance scores. final List relevanceScores = getRelevanceScores(judgmentsId, userQuery, orderedDocumentIds, k); final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores); @@ -106,6 +113,8 @@ public void onResponse(final SearchResponse searchResponse) { //final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores, idealRelevanceScores); //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores); + LOGGER.info("query set dcg = " + dcgSearchMetric.getValue()); + final Collection searchMetrics = List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric); queryResults.add(new QueryResult(userQuery, orderedDocumentIds, k, searchMetrics)); @@ -123,14 +132,14 @@ public void onFailure(Exception ex) { } // TODO: Calculate the search metrics for the entire query set given the results and the judgments. - final List orderedDocumentIds = new ArrayList<>(); + /*final List orderedDocumentIds = new ArrayList<>(); final List relevanceScores = getRelevanceScores(judgmentsId, "TODO", orderedDocumentIds, k); final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores); // TODO: Add these metrics in, too. //final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores, idealRelevanceScores); - //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores); + //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores);*/ - final Collection searchMetrics = List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric); + final Collection searchMetrics = new ArrayList<>(); // List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric); final String querySetRunId = UUID.randomUUID().toString(); final QuerySetRunResult querySetRunResult = new QuerySetRunResult(querySetRunId, queryResults, searchMetrics); @@ -147,7 +156,7 @@ public void onFailure(Exception ex) { @Override public void save(final QuerySetRunResult result) throws Exception { - // Index the results into OpenSearch. + // Index the query results into OpenSearch. final Map results = new HashMap<>(); From b72c52ac3b31a1c689d8da3a55925d0503c87adc Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 11:55:20 -0500 Subject: [PATCH 05/14] Adding search pipeline parameter. --- .../scripts/run-query-set.sh | 45 +++++++++++++++++-- .../SearchQualityEvaluationRestHandler.java | 3 +- .../eval/runners/AbstractQuerySetRunner.java | 10 ++++- .../runners/OpenSearchQuerySetRunner.java | 8 +++- 4 files changed, 58 insertions(+), 8 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh index 4e46628..9befe60 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh @@ -8,12 +8,49 @@ K="20" curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets_run_results" +# Keyword search curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}" \ -H "Content-Type: application/json" \ --data-binary '{ - "match": { - "description": { - "query": "#$query##" - } + "multi_match": { + "query": "#$query##", + "fields": ["id", "title", "category", "bullets", "description", "attrs.Brand", "attrs.Color"] } }' + +## Neural search +#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}&search_pipeline=neural-search-pipeline" \ +# -H "Content-Type: application/json" \ +# --data-binary '{ +# "neural": { +# "title_embedding": { +# "query_text": ""#$query##", +# "k": "50" +# } +# } +# }' + +# Hybrid search +#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}&search_pipeline=hybrid-search-pipeline" \ +# -H "Content-Type: application/json" \ +# --data-binary '{ +# "hybrid": { +# "queries": [ +# { +# "match": { +# "title": { +# "query": "#$query##" +# } +# } +# }, +# { +# "neural": { +# "title_embedding": { +# "query_text": "#$query##", +# "k": "50" +# } +# } +# } +# ] +# } +# }' diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java index 2915702..69577fa 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java @@ -156,6 +156,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli final String querySetId = request.param("id"); final String judgmentsId = request.param("judgments_id"); final String index = request.param("index"); + final String searchPipeline = request.param("search_pipeline", null); final String idField = request.param("id_field", "_id"); final int k = Integer.parseInt(request.param("k", "10")); @@ -182,7 +183,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli try { final OpenSearchQuerySetRunner openSearchQuerySetRunner = new OpenSearchQuerySetRunner(client); - final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId, judgmentsId, index, idField, query, k); + final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId, judgmentsId, index, searchPipeline, idField, query, k); openSearchQuerySetRunner.save(querySetRunResult); } catch (Exception ex) { diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index 0f5ad10..d930928 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -43,12 +43,14 @@ public AbstractQuerySetRunner(final Client client) { * @param querySetId The ID of the query set to run. * @param judgmentsId The ID of the judgments set to use for search metric calculation. * @param index The name of the index to run the query sets against. + * @param searchPipeline The name of the search pipeline to use, or null to not use a search pipeline. * @param idField The field in the index that is used to uniquely identify a document. * @param query The query that will be used to run the query set. * @param k The k used for metrics calculation, i.e. DCG@k. * @return The query set {@link QuerySetRunResult results} and calculated metrics. */ - abstract QuerySetRunResult run(String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k) throws Exception; + abstract QuerySetRunResult run(String querySetId, final String judgmentsId, final String index, final String searchPipeline, + final String idField, final String query, final int k) throws Exception; /** * Saves the query set results to a persistent store, which may be the search engine itself. @@ -141,6 +143,10 @@ public void onResponse(SearchResponse searchResponse) { judgment[0] = (Double) j.get("judgment"); } + if(judgment[0] > 0) { + LOGGER.info("Found a nonzero judgment! = {}", judgment[0]); + } + } else { // LOGGER.info("No judgments found for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId); @@ -192,7 +198,7 @@ public List getRelevanceScores(final String judgmentsId, final String qu } - LOGGER.info("----- scores size: " + scores.size()); + // LOGGER.info("----- scores size: " + scores.size()); //final String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", ")); //LOGGER.info("Got relevance scores: size = {}: scores = {}", listOfScores.length(), listOfScores); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index 8c1913d..b145901 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -49,7 +49,9 @@ public OpenSearchQuerySetRunner(final Client client) { } @Override - public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k) throws Exception { + public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index, + final String searchPipeline, final String idField, final String query, + final int k) throws Exception { final Collection> querySet = getQuerySet(querySetId); LOGGER.info("Found {} queries in query set {}", querySet.size(), querySetId); @@ -77,6 +79,10 @@ public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String[] excludeFields = new String[]{}; searchSourceBuilder.fetchSource(includeFields, excludeFields); + if(searchPipeline != null) { + searchSourceBuilder.pipeline(searchPipeline); + } + final SearchRequest searchRequest = new SearchRequest(index); searchRequest.source(searchSourceBuilder); From b4c29a203b2b66fef5f5ea273f9d4740e397806b Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 12:29:44 -0500 Subject: [PATCH 06/14] Updating dcg metrics. --- .../eval/metrics/DcgSearchMetric.java | 13 +++-- .../opensearch/eval/metrics/SearchMetric.java | 2 +- .../eval/runners/AbstractQuerySetRunner.java | 51 ++++++++----------- .../runners/OpenSearchQuerySetRunner.java | 25 +++++---- 4 files changed, 47 insertions(+), 44 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java index 6d59d3a..7100ad7 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java @@ -28,10 +28,17 @@ public String getName() { public double calculate() { double dcg = 0.0; - for(int i = 0; i < relevanceScores.size(); i++) { - double relevance = relevanceScores.get(i); - dcg += relevance / Math.log(i + 2); // Add 2 to avoid log(1) = 0 + for(int i = 1; i <= relevanceScores.size(); i++) { + + final double relevanceScore = relevanceScores.get(i - 1); + final double numerator = Math.pow(2, relevanceScore) - 1.0; + final double denominator = Math.log(i) / Math.log(i + 2); + + LOGGER.info("numerator = {}, denominator = {}", numerator, denominator); + dcg += (numerator / denominator); + } + return dcg; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java index 2b3044b..9aee190 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java @@ -13,7 +13,7 @@ public abstract class SearchMetric { - private static final Logger LOGGER = LogManager.getLogger(SearchMetric.class); + protected static final Logger LOGGER = LogManager.getLogger(SearchMetric.class); protected int k; diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index d930928..22bef51 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -103,7 +103,7 @@ public final Collection> getQuerySet(final String querySetId) * @param documentId The document ID. * @return The value of the judgment, or NaN if the judgment cannot be found. */ - public Double getJudgmentValue(final String judgmentsId, final String query, final String documentId) { + public Double getJudgmentValue(final String judgmentsId, final String query, final String documentId) throws Exception { // Find a judgment that matches the judgments_id, query_id, and document_id fields in the index. @@ -126,50 +126,38 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME).source(searchSourceBuilder); - final Double[] judgment = new Double[1]; - judgment[0] = Double.NaN; + Double judgment = Double.NaN; - client.search(searchRequest, new ActionListener<>() { - - @Override - public void onResponse(SearchResponse searchResponse) { - - if (searchResponse.getHits().getHits().length > 0) { - - final Map j = searchResponse.getHits().getAt(0).getSourceAsMap(); - - // TODO: Why does this not exist in some cases? - if(j.containsKey("judgment")) { - judgment[0] = (Double) j.get("judgment"); - } - - if(judgment[0] > 0) { - LOGGER.info("Found a nonzero judgment! = {}", judgment[0]); - } + final SearchResponse searchResponse = client.search(searchRequest).get(); - } else { + if (searchResponse.getHits().getHits().length > 0) { - // LOGGER.info("No judgments found for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId); + final Map j = searchResponse.getHits().getAt(0).getSourceAsMap(); - // No judgment for this query/doc pair exists. - judgment[0] = Double.NaN; + // TODO: Why does this not exist in some cases? + if(j.containsKey("judgment")) { + judgment = (Double) j.get("judgment"); + if(judgment > 0) { + LOGGER.info("Found a nonzero judgment! = {}, {}", judgment, query); } } - @Override - public void onFailure(Exception ex) { - LOGGER.error("Unable to get judgment for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId, ex); - } + } else { + + // LOGGER.info("No judgments found for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId); - }); + // No judgment for this query/doc pair exists. + judgment = Double.NaN; + + } - return judgment[0]; + return judgment; } - public List getRelevanceScores(final String judgmentsId, final String query, final List orderedDocumentIds, final int k) { + public List getRelevanceScores(final String judgmentsId, final String query, final List orderedDocumentIds, final int k) throws Exception { // LOGGER.info("Getting relevance scores for query: {}, k = {}, docIds size = {}", query, k, orderedDocumentIds.size()); @@ -188,6 +176,7 @@ public List getRelevanceScores(final String judgmentsId, final String qu // If a judgment for this query/doc pair is not found, Double.NaN will be returned. if(!Double.isNaN(judgmentValue)) { + //LOGGER.info("Adding score {} for query {}", judgmentValue, query); scores.add(judgmentValue); } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index b145901..889d331 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -111,19 +111,26 @@ public void onResponse(final SearchResponse searchResponse) { //LOGGER.info("Number of documents: " + orderedDocumentIds.size()); - // TODO: If no hits are returned, there's no need to get the relevance scores. - final List relevanceScores = getRelevanceScores(judgmentsId, userQuery, orderedDocumentIds, k); + try { - final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores); - // TODO: Add these metrics in, too. - //final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores, idealRelevanceScores); - //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores); + // TODO: If no hits are returned, there's no need to get the relevance scores. + final List relevanceScores = getRelevanceScores(judgmentsId, userQuery, orderedDocumentIds, k); - LOGGER.info("query set dcg = " + dcgSearchMetric.getValue()); + final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores); + // TODO: Add these metrics in, too. + //final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores, idealRelevanceScores); + //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores); - final Collection searchMetrics = List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric); + //LOGGER.info("size list for query {}: {}", userQuery, relevanceScores.size()); + //LOGGER.info("query set ({}) dcg = {}", userQuery, dcgSearchMetric.getValue()); - queryResults.add(new QueryResult(userQuery, orderedDocumentIds, k, searchMetrics)); + final Collection searchMetrics = List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric); + + queryResults.add(new QueryResult(userQuery, orderedDocumentIds, k, searchMetrics)); + + } catch (Exception ex) { + LOGGER.error("Unable to get relevance scores.", ex); + } } From 8208f25c1668f3f62838f1aaa82f6dc71f39d102 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 13:59:48 -0500 Subject: [PATCH 07/14] Adding ndcg calculations. --- .../eval/metrics/DcgSearchMetric.java | 4 ++++ .../eval/metrics/NdcgSearchMetric.java | 19 +++++-------------- .../runners/OpenSearchQuerySetRunner.java | 9 +++------ 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java index 7100ad7..9bd14fb 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java @@ -26,6 +26,10 @@ public String getName() { @Override public double calculate() { + return calculateDcg(relevanceScores); + } + + protected double calculateDcg(final List relevanceScores) { double dcg = 0.0; for(int i = 1; i <= relevanceScores.size(); i++) { diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/NdcgSearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/NdcgSearchMetric.java index b62aed7..d396f87 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/NdcgSearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/NdcgSearchMetric.java @@ -12,11 +12,8 @@ public class NdcgSearchMetric extends DcgSearchMetric { - private final List idealRelevanceScores; - - public NdcgSearchMetric(final int k, final List relevanceScores, final List idealRelevanceScores) { + public NdcgSearchMetric(final int k, final List relevanceScores) { super(k, relevanceScores); - this.idealRelevanceScores = idealRelevanceScores; } @Override @@ -27,17 +24,11 @@ public String getName() { @Override public double calculate() { - double dcg = super.calculate(); - - double idcg = 0.0; - for(int i = 0; i < idealRelevanceScores.size(); i++) { - double relevance = idealRelevanceScores.get(i); - idcg += relevance / Math.log(i + 2); // Add 2 to avoid log(1) = 0 - } + // Make the ideal relevance scores by sorting the relevance scores largest to smallest. + relevanceScores.sort(Double::compare); - if(idcg == 0) { - return 0; - } + double dcg = super.calculate(); + double idcg = super.calculateDcg(relevanceScores); return dcg / idcg; diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index 889d331..8b514e9 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -18,6 +18,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.eval.SearchQualityEvaluationPlugin; import org.opensearch.eval.metrics.DcgSearchMetric; +import org.opensearch.eval.metrics.NdcgSearchMetric; import org.opensearch.eval.metrics.SearchMetric; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; @@ -97,28 +98,24 @@ public void onResponse(final SearchResponse searchResponse) { final List orderedDocumentIds = new ArrayList<>(); - // LOGGER.info("Found {} results for query {}", searchResponse.getHits().getTotalHits().value, userQuery); - for (final SearchHit hit : searchResponse.getHits().getHits()) { final Map sourceAsMap = hit.getSourceAsMap(); final String documentId = sourceAsMap.get(idField).toString(); - //LOGGER.info(" -- query {}, documentId: {}", userQuery, documentId); orderedDocumentIds.add(documentId); } - //LOGGER.info("Number of documents: " + orderedDocumentIds.size()); - try { // TODO: If no hits are returned, there's no need to get the relevance scores. final List relevanceScores = getRelevanceScores(judgmentsId, userQuery, orderedDocumentIds, k); final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores); + final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores); + // TODO: Add these metrics in, too. - //final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores, idealRelevanceScores); //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores); //LOGGER.info("size list for query {}: {}", userQuery, relevanceScores.size()); From 4b2c2d6e6595e1619726b1bcca38b9f06eaff9bb Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 14:05:51 -0500 Subject: [PATCH 08/14] Clean up, adding javadocs, removing unnecessary logging. --- .../eval/metrics/PrecisionSearchMetric.java | 2 +- .../eval/runners/AbstractQuerySetRunner.java | 29 +++++++------------ .../runners/OpenSearchQuerySetRunner.java | 12 +++----- 3 files changed, 15 insertions(+), 28 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java index 9ae1e0c..6269e50 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java @@ -27,7 +27,7 @@ public String getName() { @Override public double calculate() { - // TODO: Implement this. + // TODO: Implement precision calculation. return 0.0; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index 22bef51..ea475ab 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -13,7 +13,6 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; -import org.opensearch.core.action.ActionListener; import org.opensearch.eval.SearchQualityEvaluationPlugin; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -146,8 +145,6 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin } else { - // LOGGER.info("No judgments found for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId); - // No judgment for this query/doc pair exists. judgment = Double.NaN; @@ -157,9 +154,16 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin } - public List getRelevanceScores(final String judgmentsId, final String query, final List orderedDocumentIds, final int k) throws Exception { - - // LOGGER.info("Getting relevance scores for query: {}, k = {}, docIds size = {}", query, k, orderedDocumentIds.size()); + /** + * Gets the judgments for a query / document pairs. + * @param judgmentsId The judgments collection for which the judgment to retrieve belongs. + * @param query The user query. + * @param orderedDocumentIds A list of document IDs returned for the user query. + * @param k The k used for metrics calculation, i.e. DCG@k. + * @return An ordered list of relevance scores for the query / document pairs. + * @throws Exception Thrown if a judgment cannot be retrieved. + */ + protected List getRelevanceScores(final String judgmentsId, final String query, final List orderedDocumentIds, final int k) throws Exception { // Ordered list of scores. final List scores = new ArrayList<>(); @@ -172,26 +176,13 @@ public List getRelevanceScores(final String judgmentsId, final String qu // Find the judgment value for this combination of query and documentId from the index. final Double judgmentValue = getJudgmentValue(judgmentsId, query, documentId); - // LOGGER.info("Got judgment value: {}", judgmentValue); - // If a judgment for this query/doc pair is not found, Double.NaN will be returned. if(!Double.isNaN(judgmentValue)) { - //LOGGER.info("Adding score {} for query {}", judgmentValue, query); scores.add(judgmentValue); } -// if (i == orderedDocumentIds.size()) { -// // k is greater than the actual length of documents. -// break; -// } - } - // LOGGER.info("----- scores size: " + scores.size()); - - //final String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", ")); - //LOGGER.info("Got relevance scores: size = {}: scores = {}", listOfScores.length(), listOfScores); - return scores; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index 8b514e9..eb323c2 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -19,6 +19,7 @@ import org.opensearch.eval.SearchQualityEvaluationPlugin; import org.opensearch.eval.metrics.DcgSearchMetric; import org.opensearch.eval.metrics.NdcgSearchMetric; +import org.opensearch.eval.metrics.PrecisionSearchMetric; import org.opensearch.eval.metrics.SearchMetric; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; @@ -109,19 +110,14 @@ public void onResponse(final SearchResponse searchResponse) { try { - // TODO: If no hits are returned, there's no need to get the relevance scores. final List relevanceScores = getRelevanceScores(judgmentsId, userQuery, orderedDocumentIds, k); + // Calculate the metrics for this query. final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores); final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores); + final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores); - // TODO: Add these metrics in, too. - //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores); - - //LOGGER.info("size list for query {}: {}", userQuery, relevanceScores.size()); - //LOGGER.info("query set ({}) dcg = {}", userQuery, dcgSearchMetric.getValue()); - - final Collection searchMetrics = List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric); + final Collection searchMetrics = List.of(dcgSearchMetric, ndcgSearchmetric, precisionSearchMetric); queryResults.add(new QueryResult(userQuery, orderedDocumentIds, k, searchMetrics)); From 41ca4a40a0ef6ddca0ead894108646e16c98b9ce Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 16:30:11 -0500 Subject: [PATCH 09/14] Adding threshold value for calculating precision. Signed-off-by: jzonthemtn --- .../scripts/run-query-set.sh | 1 + .../opensearch/eval/SearchQualityEvaluationRestHandler.java | 3 ++- .../org/opensearch/eval/metrics/PrecisionSearchMetric.java | 4 +++- .../org/opensearch/eval/runners/AbstractQuerySetRunner.java | 6 +++++- .../opensearch/eval/runners/OpenSearchQuerySetRunner.java | 4 ++-- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh index 9befe60..477dbd9 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh @@ -5,6 +5,7 @@ JUDGMENTS_ID="9183599e-46dd-49e0-9584-df816164a4c2" INDEX="ecommerce" ID_FIELD="asin" K="20" +THRESHOLD="1.0" # Default value curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets_run_results" diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java index 69577fa..48f4840 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java @@ -159,6 +159,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli final String searchPipeline = request.param("search_pipeline", null); final String idField = request.param("id_field", "_id"); final int k = Integer.parseInt(request.param("k", "10")); + final double threshold = Double.parseDouble(request.param("threshold", "1.0")); if(querySetId == null || querySetId.isEmpty() || judgmentsId == null || judgmentsId.isEmpty() || index == null || index.isEmpty()) { return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Missing required parameters.\"}")); @@ -183,7 +184,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli try { final OpenSearchQuerySetRunner openSearchQuerySetRunner = new OpenSearchQuerySetRunner(client); - final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId, judgmentsId, index, searchPipeline, idField, query, k); + final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId, judgmentsId, index, searchPipeline, idField, query, k, threshold); openSearchQuerySetRunner.save(querySetRunResult); } catch (Exception ex) { diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java index 6269e50..a1bcbcd 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java @@ -12,10 +12,12 @@ public class PrecisionSearchMetric extends SearchMetric { + private final double threshold; private final List relevanceScores; - public PrecisionSearchMetric(final int k, final List relevanceScores) { + public PrecisionSearchMetric(final int k, final double threshold, final List relevanceScores) { super(k); + this.threshold = threshold; this.relevanceScores = relevanceScores; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index ea475ab..f64b4ac 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -46,10 +46,14 @@ public AbstractQuerySetRunner(final Client client) { * @param idField The field in the index that is used to uniquely identify a document. * @param query The query that will be used to run the query set. * @param k The k used for metrics calculation, i.e. DCG@k. + * @param threshold The cutoff for binary judgments. A judgment score greater than or equal + * to this value will be assigned a binary judgment value of 1. A judgment score + * less than this value will be assigned a binary judgment value of 0. * @return The query set {@link QuerySetRunResult results} and calculated metrics. */ abstract QuerySetRunResult run(String querySetId, final String judgmentsId, final String index, final String searchPipeline, - final String idField, final String query, final int k) throws Exception; + final String idField, final String query, final int k, + final double threshold) throws Exception; /** * Saves the query set results to a persistent store, which may be the search engine itself. diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index eb323c2..1124a23 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -53,7 +53,7 @@ public OpenSearchQuerySetRunner(final Client client) { @Override public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index, final String searchPipeline, final String idField, final String query, - final int k) throws Exception { + final int k, final double threshold) throws Exception { final Collection> querySet = getQuerySet(querySetId); LOGGER.info("Found {} queries in query set {}", querySet.size(), querySetId); @@ -115,7 +115,7 @@ public void onResponse(final SearchResponse searchResponse) { // Calculate the metrics for this query. final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores); final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores); - final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores); + final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, threshold, relevanceScores); final Collection searchMetrics = List.of(dcgSearchMetric, ndcgSearchmetric, precisionSearchMetric); From 2628b0973bd1c28b0ef1626ee96135a00b083366 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 16:37:30 -0500 Subject: [PATCH 10/14] Adding javadocs. Signed-off-by: jzonthemtn --- .../eval/metrics/DcgSearchMetric.java | 8 ++++++ .../eval/metrics/NdcgSearchMetric.java | 8 ++++++ .../eval/metrics/PrecisionSearchMetric.java | 19 ++++++++++++++ .../opensearch/eval/metrics/SearchMetric.java | 26 +++++++++++++++++++ 4 files changed, 61 insertions(+) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java index 9bd14fb..55fa60a 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java @@ -10,10 +10,18 @@ import java.util.List; +/** + * Subclass of {@link SearchMetric} that calculates Discounted Cumulative Gain @ k. + */ public class DcgSearchMetric extends SearchMetric { protected final List relevanceScores; + /** + * Creates new DCG metrics. + * @param k The k value. + * @param relevanceScores A list of relevance scores. + */ public DcgSearchMetric(final int k, final List relevanceScores) { super(k); this.relevanceScores = relevanceScores; diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/NdcgSearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/NdcgSearchMetric.java index d396f87..5476d6c 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/NdcgSearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/NdcgSearchMetric.java @@ -10,8 +10,16 @@ import java.util.List; +/** + * Subclass of {@link SearchMetric} that calculates Normalized Discounted Cumulative Gain @ k. + */ public class NdcgSearchMetric extends DcgSearchMetric { + /** + * Creates new NDCG metrics. + * @param k The k value. + * @param relevanceScores A list of relevancy scores. + */ public NdcgSearchMetric(final int k, final List relevanceScores) { super(k, relevanceScores); } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java index a1bcbcd..4ef559d 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java @@ -10,11 +10,22 @@ import java.util.List; +/** + * Subclass of {@link SearchMetric} that calculates Precision @ k. + */ public class PrecisionSearchMetric extends SearchMetric { private final double threshold; private final List relevanceScores; + /** + * Creates new precision metrics. + * @param k The k value. + * @param threshold The threshold for assigning binary relevancy scores to non-binary scores. + * Scores greater than or equal to this value will be assigned a relevancy score of 1 (relevant). + * Scores less than this value will be assigned a relevancy score of 0 (not relevant). + * @param relevanceScores A list of relevance scores. + */ public PrecisionSearchMetric(final int k, final double threshold, final List relevanceScores) { super(k); this.threshold = threshold; @@ -34,4 +45,12 @@ public double calculate() { } + /** + * Gets the threshold value. + * @return The threshold value. + */ + public double threshold() { + return threshold; + } + } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java index 9aee190..f3a9d66 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java @@ -11,26 +11,52 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +/** + * Base class for search metrics. + */ public abstract class SearchMetric { protected static final Logger LOGGER = LogManager.getLogger(SearchMetric.class); protected int k; + /** + * Gets the name of the metric, i.e. ndcg. + * @return The name of the metric. + */ public abstract String getName(); + /** + * Calculates the metric. + * @return The value of the metric. + */ public abstract double calculate(); private Double value = Double.NaN; + /** + * Creates the metric. + * @param k The k value. + */ public SearchMetric(final int k) { this.k = k; } + /** + * Gets the k value. + * @return The k value. + */ public int getK() { return k; } + /** + * Gets the value of the metric. If the metric has not yet been calculated, + * the metric will first be calculated by calling calculate. This + * function should be used in cases where repeated access to the metric's value is + * needed without recalculating the metric's value. + * @return The value of the metric. + */ public double getValue() { if(Double.isNaN(value)) { From c3d823dd86178ba1c18845a23ad0810a602ccb44 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Sat, 7 Dec 2024 15:08:55 -0500 Subject: [PATCH 11/14] Adding precision calculation. --- .../eval/metrics/PrecisionSearchMetric.java | 11 +++++++++-- .../org/opensearch/eval/metrics/SearchMetric.java | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java index 4ef559d..fc4b80c 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java @@ -40,8 +40,15 @@ public String getName() { @Override public double calculate() { - // TODO: Implement precision calculation. - return 0.0; + int numberOfRelevantItems = 0; + + for(final double relevanceScore : relevanceScores) { + if(relevanceScore >= threshold) { + numberOfRelevantItems++; + } + } + + return (double) numberOfRelevantItems / (double) k; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java index f3a9d66..acd580a 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java @@ -53,8 +53,8 @@ public int getK() { /** * Gets the value of the metric. If the metric has not yet been calculated, * the metric will first be calculated by calling calculate. This - * function should be used in cases where repeated access to the metric's value is - * needed without recalculating the metric's value. + * function should be used in cases where repeated access to the metrics value is + * needed without recalculating the metrics value. * @return The value of the metric. */ public double getValue() { From e4e56cd439d542390825f8b2f7c293ff3eff2968 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Sat, 7 Dec 2024 15:18:56 -0500 Subject: [PATCH 12/14] Calculating the overall query set metrics. --- .../runners/OpenSearchQuerySetRunner.java | 33 +++++++++++-------- .../eval/runners/QuerySetRunResult.java | 12 +++---- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index 1124a23..1f44e2f 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -137,17 +137,24 @@ public void onFailure(Exception ex) { } - // TODO: Calculate the search metrics for the entire query set given the results and the judgments. - /*final List orderedDocumentIds = new ArrayList<>(); - final List relevanceScores = getRelevanceScores(judgmentsId, "TODO", orderedDocumentIds, k); - final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores); - // TODO: Add these metrics in, too. - //final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores, idealRelevanceScores); - //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores);*/ - - final Collection searchMetrics = new ArrayList<>(); // List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric); + // Calculate the search metrics for the entire query set given the individual query set metrics. + // Sum up the metrics for each query per metric type. + final int querySetSize = queryResults.size(); + final Map sumOfMetrics = new HashMap<>(); + for(final QueryResult queryResult : queryResults) { + for(final SearchMetric searchMetric : queryResult.getSearchMetrics()) { + sumOfMetrics.merge(searchMetric.getName(), searchMetric.getValue(), Double::sum); + } + } + + // Now divide by the number of queries. + final Map querySetMetrics = new HashMap<>(); + for(final String metric : sumOfMetrics.keySet()) { + querySetMetrics.put(metric, sumOfMetrics.get(metric) / querySetSize); + } + final String querySetRunId = UUID.randomUUID().toString(); - final QuerySetRunResult querySetRunResult = new QuerySetRunResult(querySetRunId, queryResults, searchMetrics); + final QuerySetRunResult querySetRunResult = new QuerySetRunResult(querySetRunId, queryResults, querySetMetrics); LOGGER.info("Query set run complete: {}", querySetRunId); @@ -169,9 +176,9 @@ public void save(final QuerySetRunResult result) throws Exception { results.put("run_id", result.getRunId()); results.put("query_results", result.getQueryResultsAsMap()); - // Calculate and add each metric to the object to index. - for (final SearchMetric searchMetric : result.getSearchMetrics()) { - results.put(searchMetric.getName(), searchMetric.calculate()); + // Add each metric to the object to index. + for (final String metric : result.getSearchMetrics().keySet()) { + results.put(metric, result.getSearchMetrics().get(metric)); } final IndexRequest indexRequest = new IndexRequest(SearchQualityEvaluationPlugin.QUERY_SETS_RUN_RESULTS_INDEX_NAME) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java index 4e9a046..0279121 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java @@ -23,15 +23,15 @@ public class QuerySetRunResult { private final String runId; private final List queryResults; - private final Collection metrics; + private final Map metrics; /** * Creates a new query set run result. A random UUID is generated as the run ID. * @param runId A unique identifier for this query set run. * @param queryResults A collection of {@link QueryResult} that contains the queries and search results. - * @param metrics The {@link SearchMetric metrics} calculated from the search results. + * @param metrics A map of metric name to value. */ - public QuerySetRunResult(final String runId, final List queryResults, final Collection metrics) { + public QuerySetRunResult(final String runId, final List queryResults, final Map metrics) { this.runId = runId; this.queryResults = queryResults; this.metrics = metrics; @@ -46,10 +46,10 @@ public String getRunId() { } /** - * Gets the {@link SearchMetric metrics} calculated from the run. - * @return The {@link SearchMetric metrics} calculated from the run. + * Gets the search metrics. + * @return The search metrics. */ - public Collection getSearchMetrics() { + public Map getSearchMetrics() { return metrics; } From 33846a41b2aa5f73ccdcae7e03d991b1fd485f27 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Sat, 7 Dec 2024 15:24:35 -0500 Subject: [PATCH 13/14] Removing commons lang3 dependency. --- .../build.gradle | 1 - .../licenses/commons-lang3-3.17.0.jar.sha1 | 1 - .../licenses/commons-lang3-LICENSE.txt | 202 ------------------ .../licenses/commons-lang3-NOTICE.txt | 0 .../SearchQualityEvaluationJobRunner.java | 3 +- .../SearchQualityEvaluationRestHandler.java | 3 +- .../judgments/model/ClickthroughRate.java | 13 -- .../eval/judgments/model/Judgment.java | 17 -- .../judgments/model/ubi/query/UbiQuery.java | 16 -- .../eval/runners/AbstractQuerySetRunner.java | 3 +- 10 files changed, 4 insertions(+), 255 deletions(-) delete mode 100644 opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-3.17.0.jar.sha1 delete mode 100644 opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-LICENSE.txt delete mode 100644 opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-NOTICE.txt diff --git a/opensearch-search-quality-evaluation-plugin/build.gradle b/opensearch-search-quality-evaluation-plugin/build.gradle index 0e0ae61..5c6c101 100644 --- a/opensearch-search-quality-evaluation-plugin/build.gradle +++ b/opensearch-search-quality-evaluation-plugin/build.gradle @@ -57,7 +57,6 @@ dependencies { compileOnly "org.apache.httpcomponents:httpcore:${versions.httpcore}" compileOnly "org.apache.httpcomponents:httpclient:${versions.httpclient}" compileOnly "commons-logging:commons-logging:${versions.commonslogging}" - implementation "org.apache.commons:commons-lang3:3.17.0" implementation "com.google.code.gson:gson:2.11.0" yamlRestTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}" } diff --git a/opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-3.17.0.jar.sha1 b/opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-3.17.0.jar.sha1 deleted file mode 100644 index 073922f..0000000 --- a/opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-3.17.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b17d2136f0460dcc0d2016ceefca8723bdf4ee70 \ No newline at end of file diff --git a/opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-LICENSE.txt b/opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-LICENSE.txt deleted file mode 100644 index 7a4a3ea..0000000 --- a/opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-NOTICE.txt b/opensearch-search-quality-evaluation-plugin/licenses/commons-lang3-NOTICE.txt deleted file mode 100644 index e69de29..0000000 diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java index 4b850d5..1503df0 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java @@ -8,7 +8,6 @@ */ package org.opensearch.eval; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.index.IndexRequest; @@ -113,7 +112,7 @@ public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionC final long startTime = System.currentTimeMillis(); final long judgments; - if(StringUtils.equalsIgnoreCase(searchQualityEvaluationJobParameter.getClickModel(), "coec")) { + if("coec".equalsIgnoreCase(searchQualityEvaluationJobParameter.getClickModel())) { LOGGER.info("Beginning implicit judgment generation using clicks-over-expected-clicks."); final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(true, searchQualityEvaluationJobParameter.getMaxRank()); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java index 48f4840..094f6a5 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java @@ -35,6 +35,7 @@ import org.opensearch.rest.RestResponse; import java.io.IOException; +import java.nio.charset.Charset; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.HashMap; @@ -174,7 +175,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli } // Get the query JSON from the content. - final String query = new String(BytesReference.toBytes(request.content())); + final String query = new String(BytesReference.toBytes(request.content()), Charset.defaultCharset()); // Validate the query has a QUERY_PLACEHOLDER. if(!query.contains(QUERY_PLACEHOLDER)) { diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ClickthroughRate.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ClickthroughRate.java index 542affb..7d9efa4 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ClickthroughRate.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ClickthroughRate.java @@ -8,7 +8,6 @@ */ package org.opensearch.eval.judgments.model; -import org.apache.commons.lang3.builder.EqualsBuilder; import org.opensearch.eval.judgments.util.MathUtils; /** @@ -42,18 +41,6 @@ public ClickthroughRate(final String objectId, final int clicks, final int event this.events = events; } - @Override - public boolean equals(Object obj) { - return EqualsBuilder.reflectionEquals(this, obj); - } - - @Override - public int hashCode() { - int result = 17; - result = 29 * result + objectId.hashCode(); - return result; - } - @Override public String toString() { return "object_id: " + objectId + ", clicks: " + clicks + ", events: " + events + ", ctr: " + MathUtils.round(getClickthroughRate()); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java index 15f9b5d..abda38e 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java @@ -8,8 +8,6 @@ */ package org.opensearch.eval.judgments.model; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.eval.judgments.util.MathUtils; @@ -106,21 +104,6 @@ public String toString() { return "query_id: " + queryId + ", query: " + query + ", document: " + document + ", judgment: " + MathUtils.round(judgment); } - @Override - public boolean equals(Object obj) { - return EqualsBuilder.reflectionEquals(this, obj); - } - - @Override - public int hashCode() { - return new HashCodeBuilder(17, 37). - append(queryId). - append(query). - append(document). - append(judgment). - toHashCode(); - } - /** * Gets the judgment's query ID. * @return The judgment's query ID. diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/query/UbiQuery.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/query/UbiQuery.java index 52d48e7..0b7ca0b 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/query/UbiQuery.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/query/UbiQuery.java @@ -9,8 +9,6 @@ package org.opensearch.eval.judgments.model.ubi.query; import com.google.gson.annotations.SerializedName; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; import java.util.Map; @@ -47,20 +45,6 @@ public UbiQuery() { } - @Override - public boolean equals(Object obj) { - return EqualsBuilder.reflectionEquals(this, obj); - } - - @Override - public int hashCode() { - return new HashCodeBuilder(17, 37). - append(queryId). - append(userQuery). - append(clientId). - toHashCode(); - } - /** * Gets the timestamp for the query. * @return The timestamp for the query. diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index f64b4ac..adecfd1 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -76,14 +76,13 @@ public final Collection> getQuerySet(final String querySetId) // Will be at most one match. sourceBuilder.from(0); sourceBuilder.size(1); - sourceBuilder.trackTotalHits(true); final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.QUERY_SETS_INDEX_NAME).source(sourceBuilder); // TODO: Don't use .get() final SearchResponse searchResponse = client.search(searchRequest).get(); - if(searchResponse.getHits().getTotalHits().value > 0) { + if(searchResponse.getHits().getHits().length > 0) { // The queries from the query set that will be run. return (Collection>) searchResponse.getHits().getAt(0).getSourceAsMap().get("queries"); From 99ffc0949c516ae9e50d245f98ad1d40b8df7cfe Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Sat, 7 Dec 2024 15:49:44 -0500 Subject: [PATCH 14/14] Indexing data as expected by the dashboards. --- .../runners/OpenSearchQuerySetRunner.java | 47 ++++++++++++++++++- .../eval/runners/QuerySetRunResult.java | 12 ++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index 1f44e2f..0e684fa 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -10,6 +10,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchRequest; @@ -154,7 +156,7 @@ public void onFailure(Exception ex) { } final String querySetRunId = UUID.randomUUID().toString(); - final QuerySetRunResult querySetRunResult = new QuerySetRunResult(querySetRunId, queryResults, querySetMetrics); + final QuerySetRunResult querySetRunResult = new QuerySetRunResult(querySetRunId, querySetId, queryResults, querySetMetrics); LOGGER.info("Query set run complete: {}", querySetRunId); @@ -196,6 +198,49 @@ public void onFailure(Exception ex) { } }); + // TODO: Index the metrics as expected by the dashboards. + + // See https://github.com/o19s/opensearch-search-quality-evaluation/blob/main/opensearch-dashboard-prototyping/METRICS_SCHEMA.md + // See https://github.com/o19s/opensearch-search-quality-evaluation/blob/main/opensearch-dashboard-prototyping/sample_data.ndjson + + final BulkRequest bulkRequest = new BulkRequest(); + + for(final QueryResult queryResult : result.getQueryResults()) { + + for(final SearchMetric searchMetric : queryResult.getSearchMetrics()) { + + // TODO: Make sure all of these items have values. + final Map metrics = new HashMap<>(); + metrics.put("datetime", "2024-09-01T00:00:00"); + metrics.put("search_config", "research_1"); + metrics.put("query_set_id", result.getQuerySetId()); + metrics.put("query", queryResult.getQuery()); + metrics.put("metric", searchMetric.getName()); + metrics.put("value", searchMetric.getValue()); + metrics.put("application", "sample_data"); + metrics.put("evaluation_id", result.getRunId()); + + // TODO: This is using the index name from the sample data. + bulkRequest.add(new IndexRequest("sqe_metrics_sample_data").source(metrics)); + + } + + } + + client.bulk(bulkRequest, new ActionListener<>() { + + @Override + public void onResponse(BulkResponse bulkItemResponses) { + LOGGER.info("Successfully indexed {} metrics.", bulkItemResponses.getItems().length); + } + + @Override + public void onFailure(Exception ex) { + LOGGER.error("Unable to bulk index metrics.", ex); + } + + }); + } } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java index 0279121..7bc86c2 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java @@ -22,6 +22,7 @@ public class QuerySetRunResult { private final String runId; + private final String querySetId; private final List queryResults; private final Map metrics; @@ -31,8 +32,9 @@ public class QuerySetRunResult { * @param queryResults A collection of {@link QueryResult} that contains the queries and search results. * @param metrics A map of metric name to value. */ - public QuerySetRunResult(final String runId, final List queryResults, final Map metrics) { + public QuerySetRunResult(final String runId, final String querySetId, final List queryResults, final Map metrics) { this.runId = runId; + this.querySetId = querySetId; this.queryResults = queryResults; this.metrics = metrics; } @@ -45,6 +47,14 @@ public String getRunId() { return runId; } + /** + * Gets the query set ID. + * @return The query set ID. + */ + public String getQuerySetId() { + return querySetId; + } + /** * Gets the search metrics. * @return The search metrics.