Updating dcg metrics.

o19s · Dec 6, 2024 · b4c29a2 · b4c29a2
1 parent b72c52a
commit b4c29a2
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 44 deletions.
diff --git a/...-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java b/...-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java
@@ -28,10 +28,17 @@ public String getName() {
     public double calculate() {
 
         double dcg = 0.0;
-        for(int i = 0; i < relevanceScores.size(); i++) {
-            double relevance = relevanceScores.get(i);
-            dcg += relevance / Math.log(i + 2); // Add 2 to avoid log(1) = 0
+        for(int i = 1; i <= relevanceScores.size(); i++) {
+
+            final double relevanceScore = relevanceScores.get(i - 1);
+            final double numerator = Math.pow(2, relevanceScore) - 1.0;
+            final double denominator = Math.log(i) / Math.log(i + 2);
+
+            LOGGER.info("numerator = {}, denominator = {}", numerator, denominator);
+            dcg += (numerator / denominator);
+
         }
+
         return dcg;
 
     }

diff --git a/...rch-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java b/...rch-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java
@@ -13,7 +13,7 @@
 
 public abstract class SearchMetric {
 
-    private static final Logger LOGGER = LogManager.getLogger(SearchMetric.class);
+    protected static final Logger LOGGER = LogManager.getLogger(SearchMetric.class);
 
     protected int k;
 

diff --git a/...y-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/...y-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java
@@ -103,7 +103,7 @@ public final Collection<Map<String, Long>> getQuerySet(final String querySetId)
      * @param documentId The document ID.
      * @return The value of the judgment, or <code>NaN</code> if the judgment cannot be found.
      */
-    public Double getJudgmentValue(final String judgmentsId, final String query, final String documentId) {
+    public Double getJudgmentValue(final String judgmentsId, final String query, final String documentId) throws Exception {
 
         // Find a judgment that matches the judgments_id, query_id, and document_id fields in the index.
 
@@ -126,50 +126,38 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin
 
         final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME).source(searchSourceBuilder);
 
-        final Double[] judgment = new Double[1];
-        judgment[0] = Double.NaN;
+        Double judgment = Double.NaN;
 
-        client.search(searchRequest, new ActionListener<>() {
-
-            @Override
-            public void onResponse(SearchResponse searchResponse) {
-
-                if (searchResponse.getHits().getHits().length > 0) {
-
-                    final Map<String, Object> j = searchResponse.getHits().getAt(0).getSourceAsMap();
-
-                    // TODO: Why does this not exist in some cases?
-                    if(j.containsKey("judgment")) {
-                        judgment[0] = (Double) j.get("judgment");
-                    }
-
-                    if(judgment[0] > 0) {
-                        LOGGER.info("Found a nonzero judgment! = {}", judgment[0]);
-                    }
+        final SearchResponse searchResponse = client.search(searchRequest).get();
 
-                } else {
+        if (searchResponse.getHits().getHits().length > 0) {
 
-                    // LOGGER.info("No judgments found for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId);
+            final Map<String, Object> j = searchResponse.getHits().getAt(0).getSourceAsMap();
 
-                    // No judgment for this query/doc pair exists.
-                    judgment[0] = Double.NaN;
+            // TODO: Why does this not exist in some cases?
+            if(j.containsKey("judgment")) {
+                judgment = (Double) j.get("judgment");
 
+                if(judgment > 0) {
+                    LOGGER.info("Found a nonzero judgment! = {}, {}", judgment, query);
                 }
 
             }
 
-            @Override
-            public void onFailure(Exception ex) {
-                LOGGER.error("Unable to get judgment for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId, ex);
-            }
+        } else {
+
+            // LOGGER.info("No judgments found for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId);
 
-        });
+            // No judgment for this query/doc pair exists.
+            judgment = Double.NaN;
+
+        }
 
-        return judgment[0];
+        return judgment;
 
     }
 
-    public List<Double> getRelevanceScores(final String judgmentsId, final String query, final List<String> orderedDocumentIds, final int k) {
+    public List<Double> getRelevanceScores(final String judgmentsId, final String query, final List<String> orderedDocumentIds, final int k) throws Exception {
 
        //  LOGGER.info("Getting relevance scores for query: {}, k = {}, docIds size = {}", query, k, orderedDocumentIds.size());
 
@@ -188,6 +176,7 @@ public List<Double> getRelevanceScores(final String judgmentsId, final String qu
 
             // If a judgment for this query/doc pair is not found, Double.NaN will be returned.
             if(!Double.isNaN(judgmentValue)) {
+                //LOGGER.info("Adding score {} for query {}", judgmentValue, query);
                 scores.add(judgmentValue);
             }
 

diff --git a/...evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/...evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java
@@ -111,19 +111,26 @@ public void onResponse(final SearchResponse searchResponse) {
 
                             //LOGGER.info("Number of documents: " + orderedDocumentIds.size());
 
-                            // TODO: If no hits are returned, there's no need to get the relevance scores.
-                            final List<Double> relevanceScores = getRelevanceScores(judgmentsId, userQuery, orderedDocumentIds, k);
+                            try {
 
-                            final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores);
-                            // TODO: Add these metrics in, too.
-                            //final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores, idealRelevanceScores);
-                            //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores);
+                                // TODO: If no hits are returned, there's no need to get the relevance scores.
+                                final List<Double> relevanceScores = getRelevanceScores(judgmentsId, userQuery, orderedDocumentIds, k);
 
-                            LOGGER.info("query set dcg = " + dcgSearchMetric.getValue());
+                                final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores);
+                                // TODO: Add these metrics in, too.
+                                //final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores, idealRelevanceScores);
+                                //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores);
 
-                            final Collection<SearchMetric> searchMetrics = List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric);
+                                //LOGGER.info("size list for query {}: {}", userQuery, relevanceScores.size());
+                                //LOGGER.info("query set ({}) dcg = {}", userQuery, dcgSearchMetric.getValue());
 
-                            queryResults.add(new QueryResult(userQuery, orderedDocumentIds, k, searchMetrics));
+                                final Collection<SearchMetric> searchMetrics = List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric);
+
+                                queryResults.add(new QueryResult(userQuery, orderedDocumentIds, k, searchMetrics));
+
+                            } catch (Exception ex) {
+                                LOGGER.error("Unable to get relevance scores.", ex);
+                            }
 
                         }