Calculating the overall query set metrics.

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java

@@ -137,17 +137,24 @@ public void onFailure(Exception ex) {
 
             }
 
-            // TODO: Calculate the search metrics for the entire query set given the results and the judgments.
-            /*final List<String> orderedDocumentIds = new ArrayList<>();
-            final List<Double> relevanceScores = getRelevanceScores(judgmentsId, "TODO", orderedDocumentIds, k);
-            final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores);
-            // TODO: Add these metrics in, too.
-            //final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores, idealRelevanceScores);
-            //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores);*/
-
-            final Collection<SearchMetric> searchMetrics = new ArrayList<>(); // List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric);
+            // Calculate the search metrics for the entire query set given the individual query set metrics.
+            // Sum up the metrics for each query per metric type.
+            final int querySetSize = queryResults.size();
+            final Map<String, Double> sumOfMetrics = new HashMap<>();
+            for(final QueryResult queryResult : queryResults) {
+                for(final SearchMetric searchMetric : queryResult.getSearchMetrics()) {
+                    sumOfMetrics.merge(searchMetric.getName(), searchMetric.getValue(), Double::sum);
+                }
+            }
+
+            // Now divide by the number of queries.
+            final Map<String, Double> querySetMetrics = new HashMap<>();
+            for(final String metric : sumOfMetrics.keySet()) {
+                querySetMetrics.put(metric, sumOfMetrics.get(metric) / querySetSize);
+            }
+
             final String querySetRunId = UUID.randomUUID().toString();
-            final QuerySetRunResult querySetRunResult = new QuerySetRunResult(querySetRunId, queryResults, searchMetrics);
+            final QuerySetRunResult querySetRunResult = new QuerySetRunResult(querySetRunId, queryResults, querySetMetrics);
 
             LOGGER.info("Query set run complete: {}", querySetRunId);
 
@@ -169,9 +176,9 @@ public void save(final QuerySetRunResult result) throws Exception {
         results.put("run_id", result.getRunId());
         results.put("query_results", result.getQueryResultsAsMap());
 
-        // Calculate and add each metric to the object to index.
-        for (final SearchMetric searchMetric : result.getSearchMetrics()) {
-            results.put(searchMetric.getName(), searchMetric.calculate());
+        // Add each metric to the object to index.
+        for (final String metric : result.getSearchMetrics().keySet()) {
+            results.put(metric, result.getSearchMetrics().get(metric));
         }
 
         final IndexRequest indexRequest = new IndexRequest(SearchQualityEvaluationPlugin.QUERY_SETS_RUN_RESULTS_INDEX_NAME)

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java

@@ -23,15 +23,15 @@ public class QuerySetRunResult {
 
     private final String runId;
     private final List<QueryResult> queryResults;
-    private final Collection<SearchMetric> metrics;
+    private final Map<String, Double> metrics;
 
     /**
      * Creates a new query set run result. A random UUID is generated as the run ID.
      * @param runId A unique identifier for this query set run.
      * @param queryResults A collection of {@link QueryResult} that contains the queries and search results.
-     * @param metrics The {@link SearchMetric metrics} calculated from the search results.
+     * @param metrics A map of metric name to value.
      */
-    public QuerySetRunResult(final String runId, final List<QueryResult> queryResults, final Collection<SearchMetric> metrics) {
+    public QuerySetRunResult(final String runId, final List<QueryResult> queryResults, final Map<String, Double> metrics) {
         this.runId = runId;
         this.queryResults = queryResults;
         this.metrics = metrics;
@@ -46,10 +46,10 @@ public String getRunId() {
     }
 
     /**
-     * Gets the {@link SearchMetric metrics} calculated from the run.
-     * @return The {@link SearchMetric metrics} calculated from the run.
+     * Gets the search metrics.
+     * @return The search metrics.
      */
-    public Collection<SearchMetric> getSearchMetrics() {
+    public Map<String, Double> getSearchMetrics() {
         return metrics;
     }