From 7587a59e650beedd45fc059218d6593ae89e09c3 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 17:09:32 -0500 Subject: [PATCH 1/2] Not using track total hits. Signed-off-by: jzonthemtn --- .../eval/judgments/opensearch/OpenSearchHelper.java | 5 ++++- .../org/opensearch/eval/runners/AbstractQuerySetRunner.java | 5 ++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java index d268933..afe86a7 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java @@ -204,7 +204,10 @@ public long getCountOfQueriesForUserQueryHavingResultInRankR(final String userQu //LOGGER.info("Query ID: {} --- Count of {} having {} at rank {} = {}", queryId, userQuery, objectId, rank, response.getHits().getTotalHits().value); - countOfTimesShownAtRank += response.getHits().getTotalHits().value; + // Won't be null as long as trackTotalHits is true. + if(response.getHits().getTotalHits() != null) { + countOfTimesShownAtRank += response.getHits().getTotalHits().value; + } } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index f64b4ac..b5d3c1e 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -76,14 +76,13 @@ public final Collection> getQuerySet(final String querySetId) // Will be at most one match. sourceBuilder.from(0); sourceBuilder.size(1); - sourceBuilder.trackTotalHits(true); final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.QUERY_SETS_INDEX_NAME).source(sourceBuilder); // TODO: Don't use .get() final SearchResponse searchResponse = client.search(searchRequest).get(); - if(searchResponse.getHits().getTotalHits().value > 0) { + if (searchResponse.getHits().getHits().length > 0) { // The queries from the query set that will be run. return (Collection>) searchResponse.getHits().getAt(0).getSourceAsMap().get("queries"); @@ -93,7 +92,7 @@ public final Collection> getQuerySet(final String querySetId) LOGGER.error("Unable to get query set with ID {}", querySetId); // The query set was not found. - throw new RuntimeException("The query set with ID " + querySetId + " was not found."); + throw new RuntimeException("The query set with ID " + querySetId + " was not found."); } From 986f2ef8270b78692a73b34b033e627fb7e0856f Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 9 Dec 2024 08:42:20 -0500 Subject: [PATCH 2/2] Renaming view to impression. --- .../judgments/clickmodel/coec/CoecClickModel.java | 11 +++++------ .../eval/judgments/opensearch/OpenSearchHelper.java | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java index 1dfb18d..d8c0a64 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java @@ -56,8 +56,7 @@ public class CoecClickModel extends ClickModel { // UBI event names. public static final String EVENT_CLICK = "click"; - // TODO: Change "view" to "impression". - public static final String EVENT_VIEW = "view"; + public static final String EVENT_IMPRESSION = "impression"; private final CoecClickModelParameters parameters; @@ -192,7 +191,7 @@ private Map> getClickthroughRate(final int maxRank * }, * { * "term": { - * "action_name": "view" + * "action_name": "impression" * } * } * ], @@ -219,7 +218,7 @@ private Map> getClickthroughRate(final int maxRank " },\n" + " {\n" + " \"term\": {\n" + - " \"action_name\": \"view\"\n" + + " \"action_name\": \"impression\"\n" + " }\n" + " }\n" + " ],\n" + @@ -355,8 +354,8 @@ public Map getRankAggregatedClickThrough() throws Exception { } - // Handle the "view" bucket. - if(EVENT_VIEW.equalsIgnoreCase(actionBucket.getKey().toString())) { + // Handle the "impression" bucket. + if(EVENT_IMPRESSION.equalsIgnoreCase(actionBucket.getKey().toString())) { final Terms positionTerms = actionBucket.getAggregations().get("By_Position"); final Collection positionBuckets = positionTerms.getBuckets(); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java index afe86a7..af178a6 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java @@ -145,7 +145,7 @@ public long getCountOfQueriesForUserQueryHavingResultInRankR(final String userQu // Get all query IDs matching this user query. final Collection queryIds = getQueryIdsHavingUserQuery(userQuery); - // For each query ID, get the events with action_name = "view" having a match on objectId and rank (position). + // For each query ID, get the events with action_name = "impression" having a match on objectId and rank (position). for(final String queryId : queryIds) { //LOGGER.info("userQuery = {}; queryId = {}; objectId = {}; rank = {}", userQuery, queryId, objectId, rank); @@ -160,7 +160,7 @@ public long getCountOfQueriesForUserQueryHavingResultInRankR(final String userQu " },\n" + " {\n" + " \"term\": {\n" + - " \"action_name\": \"view\"\n" + + " \"action_name\": \"impression\"\n" + " }\n" + " },\n" + " {\n" +