From b4c29a203b2b66fef5f5ea273f9d4740e397806b Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 12:29:44 -0500 Subject: [PATCH] Updating dcg metrics. --- .../eval/metrics/DcgSearchMetric.java | 13 +++-- .../opensearch/eval/metrics/SearchMetric.java | 2 +- .../eval/runners/AbstractQuerySetRunner.java | 51 ++++++++----------- .../runners/OpenSearchQuerySetRunner.java | 25 +++++---- 4 files changed, 47 insertions(+), 44 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java index 6d59d3a..7100ad7 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java @@ -28,10 +28,17 @@ public String getName() { public double calculate() { double dcg = 0.0; - for(int i = 0; i < relevanceScores.size(); i++) { - double relevance = relevanceScores.get(i); - dcg += relevance / Math.log(i + 2); // Add 2 to avoid log(1) = 0 + for(int i = 1; i <= relevanceScores.size(); i++) { + + final double relevanceScore = relevanceScores.get(i - 1); + final double numerator = Math.pow(2, relevanceScore) - 1.0; + final double denominator = Math.log(i) / Math.log(i + 2); + + LOGGER.info("numerator = {}, denominator = {}", numerator, denominator); + dcg += (numerator / denominator); + } + return dcg; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java index 2b3044b..9aee190 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetric.java @@ -13,7 +13,7 @@ public abstract class SearchMetric { - private static final Logger LOGGER = LogManager.getLogger(SearchMetric.class); + protected static final Logger LOGGER = LogManager.getLogger(SearchMetric.class); protected int k; diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index d930928..22bef51 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -103,7 +103,7 @@ public final Collection> getQuerySet(final String querySetId) * @param documentId The document ID. * @return The value of the judgment, or NaN if the judgment cannot be found. */ - public Double getJudgmentValue(final String judgmentsId, final String query, final String documentId) { + public Double getJudgmentValue(final String judgmentsId, final String query, final String documentId) throws Exception { // Find a judgment that matches the judgments_id, query_id, and document_id fields in the index. @@ -126,50 +126,38 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME).source(searchSourceBuilder); - final Double[] judgment = new Double[1]; - judgment[0] = Double.NaN; + Double judgment = Double.NaN; - client.search(searchRequest, new ActionListener<>() { - - @Override - public void onResponse(SearchResponse searchResponse) { - - if (searchResponse.getHits().getHits().length > 0) { - - final Map j = searchResponse.getHits().getAt(0).getSourceAsMap(); - - // TODO: Why does this not exist in some cases? - if(j.containsKey("judgment")) { - judgment[0] = (Double) j.get("judgment"); - } - - if(judgment[0] > 0) { - LOGGER.info("Found a nonzero judgment! = {}", judgment[0]); - } + final SearchResponse searchResponse = client.search(searchRequest).get(); - } else { + if (searchResponse.getHits().getHits().length > 0) { - // LOGGER.info("No judgments found for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId); + final Map j = searchResponse.getHits().getAt(0).getSourceAsMap(); - // No judgment for this query/doc pair exists. - judgment[0] = Double.NaN; + // TODO: Why does this not exist in some cases? + if(j.containsKey("judgment")) { + judgment = (Double) j.get("judgment"); + if(judgment > 0) { + LOGGER.info("Found a nonzero judgment! = {}, {}", judgment, query); } } - @Override - public void onFailure(Exception ex) { - LOGGER.error("Unable to get judgment for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId, ex); - } + } else { + + // LOGGER.info("No judgments found for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId); - }); + // No judgment for this query/doc pair exists. + judgment = Double.NaN; + + } - return judgment[0]; + return judgment; } - public List getRelevanceScores(final String judgmentsId, final String query, final List orderedDocumentIds, final int k) { + public List getRelevanceScores(final String judgmentsId, final String query, final List orderedDocumentIds, final int k) throws Exception { // LOGGER.info("Getting relevance scores for query: {}, k = {}, docIds size = {}", query, k, orderedDocumentIds.size()); @@ -188,6 +176,7 @@ public List getRelevanceScores(final String judgmentsId, final String qu // If a judgment for this query/doc pair is not found, Double.NaN will be returned. if(!Double.isNaN(judgmentValue)) { + //LOGGER.info("Adding score {} for query {}", judgmentValue, query); scores.add(judgmentValue); } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index b145901..889d331 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -111,19 +111,26 @@ public void onResponse(final SearchResponse searchResponse) { //LOGGER.info("Number of documents: " + orderedDocumentIds.size()); - // TODO: If no hits are returned, there's no need to get the relevance scores. - final List relevanceScores = getRelevanceScores(judgmentsId, userQuery, orderedDocumentIds, k); + try { - final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores); - // TODO: Add these metrics in, too. - //final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores, idealRelevanceScores); - //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores); + // TODO: If no hits are returned, there's no need to get the relevance scores. + final List relevanceScores = getRelevanceScores(judgmentsId, userQuery, orderedDocumentIds, k); - LOGGER.info("query set dcg = " + dcgSearchMetric.getValue()); + final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores); + // TODO: Add these metrics in, too. + //final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores, idealRelevanceScores); + //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores); - final Collection searchMetrics = List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric); + //LOGGER.info("size list for query {}: {}", userQuery, relevanceScores.size()); + //LOGGER.info("query set ({}) dcg = {}", userQuery, dcgSearchMetric.getValue()); - queryResults.add(new QueryResult(userQuery, orderedDocumentIds, k, searchMetrics)); + final Collection searchMetrics = List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric); + + queryResults.add(new QueryResult(userQuery, orderedDocumentIds, k, searchMetrics)); + + } catch (Exception ex) { + LOGGER.error("Unable to get relevance scores.", ex); + } }