diff --git a/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-by-id.sh b/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh similarity index 86% rename from opensearch-search-quality-evaluation-plugin/scripts/create-query-set-by-id.sh rename to opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh index c04886a..5f9f928 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-by-id.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh @@ -1,7 +1,7 @@ #!/bin/bash -e #QUERY_SET=`curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss" | jq .query_set | tr -d '"'` -curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=none&max_queries=500" +curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss&max_queries=500" #echo ${QUERY_SET} diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobParameter.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobParameter.java index b8024e5..2ea5379 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobParameter.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobParameter.java @@ -17,9 +17,24 @@ public class SearchQualityEvaluationJobParameter implements ScheduledJobParameter { + /** + * The name of the parameter for providing a name for the scheduled job. + */ public static final String NAME_FIELD = "name"; + + /** + * The name of the parameter for creating a job as enabled or disabled. + */ public static final String ENABLED_FILED = "enabled"; + + /** + * The name of the parameter for specifying when the job was last updated. + */ public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; + + /** + * The name of the parameter for specifying a readable time for when the job was last updated. + */ public static final String LAST_UPDATE_TIME_FIELD_READABLE = "last_update_time_field"; public static final String SCHEDULE_FIELD = "schedule"; public static final String ENABLED_TIME_FILED = "enabled_time"; @@ -27,8 +42,14 @@ public class SearchQualityEvaluationJobParameter implements ScheduledJobParamete public static final String LOCK_DURATION_SECONDS = "lock_duration_seconds"; public static final String JITTER = "jitter"; - // Custom properties. + /** + * The name of the parameter that allows for specifying the type of click model to use. + */ public static final String CLICK_MODEL = "click_model"; + + /** + * The name of the parameter that allows for setting a max rank value to use during judgment generation. + */ public static final String MAX_RANK = "max_rank"; // Properties from ScheduledJobParameter. @@ -136,46 +157,90 @@ public Double getJitter() { return jitter; } + /** + * Sets the name of the job. + * @param jobName The name of the job. + */ public void setJobName(String jobName) { this.jobName = jobName; } + /** + * Sets when the job was last updated. + * @param lastUpdateTime An {@link Instant} of when the job was last updated. + */ public void setLastUpdateTime(Instant lastUpdateTime) { this.lastUpdateTime = lastUpdateTime; } + /** + * Sets when the job was enabled. + * @param enabledTime An {@link Instant} of when the job was enabled. + */ public void setEnabledTime(Instant enabledTime) { this.enabledTime = enabledTime; } + /** + * Sets whether the job is enabled. + * @param enabled A boolean representing whether the job is enabled. + */ public void setEnabled(boolean enabled) { this.enabled = enabled; } + /** + * Sets the schedule for the job. + * @param schedule A {@link Schedule} for the job. + */ public void setSchedule(Schedule schedule) { this.schedule = schedule; } + /** + * Sets the lock duration for the cluster when running the job. + * @param lockDurationSeconds The lock duration in seconds. + */ public void setLockDurationSeconds(Long lockDurationSeconds) { this.lockDurationSeconds = lockDurationSeconds; } + /** + * Sets the jitter for the job. + * @param jitter The jitter for the job. + */ public void setJitter(Double jitter) { this.jitter = jitter; } + /** + * Gets the type of click model to use for implicit judgment generation. + * @return The type of click model to use for implicit judgment generation. + */ public String getClickModel() { return clickModel; } + /** + * Sets the click model type to use for implicit judgment generation. + * @param clickModel The click model type to use for implicit judgment generation. + */ public void setClickModel(String clickModel) { this.clickModel = clickModel; } + /** + * Gets the max rank to use when generating implicit judgments. + * @return The max rank to use when generating implicit judgments. + */ public int getMaxRank() { return maxRank; } + /** + * Sets the max rank to use when generating implicit judgments. + * @param maxRank The max rank to use when generating implicit judgments. + */ public void setMaxRank(int maxRank) { this.maxRank = maxRank; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java index 784343c..e971f8f 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java @@ -28,12 +28,19 @@ import java.util.Map; import java.util.UUID; +/** + * Job runner for scheduled implicit judgments jobs. + */ public class SearchQualityEvaluationJobRunner implements ScheduledJobRunner { private static final Logger LOGGER = LogManager.getLogger(SearchQualityEvaluationJobRunner.class); private static SearchQualityEvaluationJobRunner INSTANCE; + /** + * Gets a singleton instance of this class. + * @return A {@link SearchQualityEvaluationJobRunner}. + */ public static SearchQualityEvaluationJobRunner getJobRunnerInstance() { LOGGER.info("Getting job runner instance"); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java index 69eb84a..199eb26 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java @@ -44,14 +44,36 @@ import java.util.List; import java.util.function.Supplier; +/** + * Main class for the Search Quality Evaluation plugin. + */ public class SearchQualityEvaluationPlugin extends Plugin implements ActionPlugin, JobSchedulerExtension { private static final Logger LOGGER = LogManager.getLogger(SearchQualityEvaluationPlugin.class); + /** + * The name of the UBI index containing the queries. This should not be changed. + */ public static final String UBI_QUERIES_INDEX_NAME = "ubi_queries"; + /** + * The name of the UBI index containing the events. This should not be changed. + */ + public static final String UBI_EVENTS_INDEX_NAME = "ubi_events"; + + /** + * The name of the index to store the scheduled jobs to create implicit judgments. + */ public static final String SCHEDULED_JOBS_INDEX_NAME = "search_quality_eval_scheduled_jobs"; + + /** + * The name of the index to store the completed jobs to create implicit judgments. + */ public static final String COMPLETED_JOBS_INDEX_NAME = "search_quality_eval_completed_jobs"; + + /** + * The name of the index that stores the query sets. + */ public static final String QUERY_SETS_INDEX_NAME = "search_quality_eval_query_sets"; @Override diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java index d073922..83129ad 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java @@ -350,9 +350,7 @@ public void onFailure(Exception e) { } } else { - return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, "{\"error\": \"" + request.path() + " was not found.\"}")); - } } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/ClickModel.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/ClickModel.java index d0d7779..d5b38fc 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/ClickModel.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/ClickModel.java @@ -8,11 +8,16 @@ */ package org.opensearch.eval.judgments.clickmodel; -public abstract class ClickModel { - - public static final String INDEX_UBI_EVENTS = "ubi_events"; - public static final String INDEX_UBI_QUERIES = "ubi_queries"; +/** + * Base class for creating click models. + */ +public abstract class ClickModel { + /** + * Calculate implicit judgments. + * @return The number of implicit judgments created. + * @throws Exception Thrown if the judgments cannot be created. + */ public abstract long calculateJudgments() throws Exception; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java index e317417..5415361 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java @@ -18,13 +18,14 @@ import org.opensearch.client.Client; import org.opensearch.client.Requests; import org.opensearch.common.unit.TimeValue; +import org.opensearch.eval.SearchQualityEvaluationPlugin; import org.opensearch.eval.judgments.clickmodel.ClickModel; import org.opensearch.eval.judgments.model.ClickthroughRate; import org.opensearch.eval.judgments.model.Judgment; import org.opensearch.eval.judgments.model.ubi.event.UbiEvent; import org.opensearch.eval.judgments.opensearch.OpenSearchHelper; import org.opensearch.eval.judgments.util.MathUtils; -import org.opensearch.eval.judgments.util.UserQueryHash; +import org.opensearch.eval.judgments.util.IncrementalUserQueryHash; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -46,7 +47,7 @@ import java.util.Map; import java.util.Set; -public class CoecClickModel extends ClickModel { +public class CoecClickModel extends ClickModel { // OpenSearch indexes. public static final String INDEX_RANK_AGGREGATED_CTR = "rank_aggregated_ctr"; @@ -61,7 +62,7 @@ public class CoecClickModel extends ClickModel { private final OpenSearchHelper openSearchHelper; - private final UserQueryHash userQueryHash = new UserQueryHash(); + private final IncrementalUserQueryHash incrementalUserQueryHash = new IncrementalUserQueryHash(); private final Gson gson = new Gson(); private final Client client; @@ -144,7 +145,7 @@ public long calculateCoec(final Map rankAggregatedClickThrough, final double judgment = totalNumberClicksForQueryResult / denominatorSum; // Hash the user query to get a query ID. - final int queryId = userQueryHash.getHash(userQuery); + final int queryId = incrementalUserQueryHash.getHash(userQuery); // Add the judgment to the list. // TODO: What to do for query ID when the values are per user_query instead? @@ -238,7 +239,7 @@ private Map> getClickthroughRate(final int maxRank final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(10L)); final SearchRequest searchRequest = Requests - .searchRequest(INDEX_UBI_EVENTS) + .searchRequest(SearchQualityEvaluationPlugin.UBI_EVENTS_INDEX_NAME) .source(searchSourceBuilder) .scroll(scroll); @@ -329,7 +330,7 @@ public Map getRankAggregatedClickThrough() throws Exception { searchSourceBuilder.from(0); searchSourceBuilder.size(100); - final SearchRequest searchRequest = new SearchRequest(INDEX_UBI_EVENTS).source(searchSourceBuilder); + final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.UBI_EVENTS_INDEX_NAME).source(searchSourceBuilder); final SearchResponse searchResponse = client.search(searchRequest).get(); final Map clickCounts = new HashMap<>(); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModelParameters.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModelParameters.java index 3f2f00b..7f75222 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModelParameters.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModelParameters.java @@ -10,31 +10,57 @@ import org.opensearch.eval.judgments.clickmodel.ClickModelParameters; +/** + * The parameters for the {@link CoecClickModel}. + */ public class CoecClickModelParameters extends ClickModelParameters { private final boolean persist; private final int maxRank; private int roundingDigits = 3; + /** + * Creates new parameters. + * @param persist Whether to persist the calculated judgments. + * @param maxRank The max rank to use when calculating the judgments. + */ public CoecClickModelParameters(boolean persist, final int maxRank) { this.persist = persist; this.maxRank = maxRank; } + /** + * Creates new parameters. + * @param persist Whether to persist the calculated judgments. + * @param maxRank The max rank to use when calculating the judgments. + * @param roundingDigits The number of decimal places to round calculated values to. + */ public CoecClickModelParameters(boolean persist, final int maxRank, final int roundingDigits) { this.persist = persist; this.maxRank = maxRank; this.roundingDigits = roundingDigits; } + /** + * Gets whether to persist the calculated judgments. + * @return Whether to persist the calculated judgments. + */ public boolean isPersist() { return persist; } + /** + * Gets the max rank for the implicit judgments calculation. + * @return The max rank for the implicit judgments calculation. + */ public int getMaxRank() { return maxRank; } + /** + * Gets the number of rounding digits to use for judgments. + * @return The number of rounding digits to use for judgments. + */ public int getRoundingDigits() { return roundingDigits; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ClickthroughRate.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ClickthroughRate.java index 3cd249d..542affb 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ClickthroughRate.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ClickthroughRate.java @@ -20,12 +20,22 @@ public class ClickthroughRate { private int clicks; private int events; + /** + * Creates a new clickthrough rate for an object. + * @param objectId The ID of the object. + */ public ClickthroughRate(final String objectId) { this.objectId = objectId; this.clicks = 0; this.events = 0; } + /** + * Creates a new clickthrough rate for an object given counts of clicks and events. + * @param objectId The object ID. + * @param clicks The count of clicks. + * @param events The count of events. + */ public ClickthroughRate(final String objectId, final int clicks, final int events) { this.objectId = objectId; this.clicks = clicks; @@ -49,27 +59,50 @@ public String toString() { return "object_id: " + objectId + ", clicks: " + clicks + ", events: " + events + ", ctr: " + MathUtils.round(getClickthroughRate()); } + /** + * Log a click to this object. + * This increments clicks and events. + */ public void logClick() { clicks++; events++; } + /** + * Log an event to this object. + */ public void logEvent() { events++; } + /** + * Calculate the clickthrough rate. + * @return The clickthrough rate as clicks divided by events. + */ public double getClickthroughRate() { return (double) clicks / events; } + /** + * Gets the count of clicks. + * @return The count of clicks. + */ public int getClicks() { return clicks; } + /** + * Gets the count of events. + * @return The count of events. + */ public int getEvents() { return events; } + /** + * Gets the object ID. + * @return The object ID. + */ public String getObjectId() { return objectId; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java index 4c025e5..6b083bd 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java @@ -28,6 +28,13 @@ public class Judgment { private final String document; private final double judgment; + /** + * Creates a new judgment. + * @param queryId The query ID for the judgment. + * @param query The query for the judgment. + * @param document The document in the jdugment. + * @param judgment The judgment value. + */ public Judgment(final String queryId, final String query, final String document, final double judgment) { this.queryId = queryId; this.query = query; @@ -39,6 +46,10 @@ public String toJudgmentString() { return queryId + ", " + query + ", " + document + ", " + MathUtils.round(judgment); } + /** + * A convenience function to output the judgments. + * @param judgments A collection of {@link Judgment}. + */ public static void showJudgments(final Collection judgments) { LOGGER.info("query_id, query, document, judgment"); @@ -69,18 +80,34 @@ public int hashCode() { toHashCode(); } + /** + * Gets the judgment's query ID. + * @return The judgment's query ID. + */ public String getQueryId() { return queryId; } + /** + * Gets the judgment's query. + * @return The judgment's query. + */ public String getQuery() { return query; } + /** + * Gets the judgment's document. + * @return The judgment's document. + */ public String getDocument() { return document; } + /** + * Gets the judgment's value. + * @return The judgment's value. + */ public double getJudgment() { return judgment; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/UserQueryHash.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/UserQueryHash.java new file mode 100644 index 0000000..0def477 --- /dev/null +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/UserQueryHash.java @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.eval.judgments.model; + +/** + * In interface for creating hashes of user queries. + */ +public interface UserQueryHash { + + /** + * Creates a unique integer given a user query. + * @param userQuery The user query. + * @return A unique integer representing the user query. + */ + int getHash(String userQuery); + +} diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/EventAttributes.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/EventAttributes.java index 2d36afa..cf09444 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/EventAttributes.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/EventAttributes.java @@ -10,6 +10,9 @@ import com.google.gson.annotations.SerializedName; +/** + * Attributes on an UBI event. + */ public class EventAttributes { @SerializedName("object") @@ -21,26 +24,57 @@ public class EventAttributes { @SerializedName("position") private Position position; + /** + * Creates a new object. + */ + public EventAttributes() { + + } + + /** + * Gets the {@link EventObject} for the event. + * @return A {@link EventObject}. + */ public EventObject getObject() { return object; } + /** + * Sets the {@link EventObject} for the event. + * @param object A {@link EventObject}. + */ public void setObject(EventObject object) { this.object = object; } + /** + * Gets the session ID for the event. + * @return The session ID for the event. + */ public String getSessionId() { return sessionId; } + /** + * Sets the session ID for the event. + * @param sessionId The session ID for the evnet. + */ public void setSessionId(String sessionId) { this.sessionId = sessionId; } + /** + * Gets the {@link Position} associated with the event. + * @return The {@link Position} associated with the event. + */ public Position getPosition() { return position; } + /** + * Sets the {@link Position} associated with the event. + * @param position The {@link Position} associated with the event. + */ public void setPosition(Position position) { this.position = position; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/EventObject.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/EventObject.java index 6a07b95..55595ba 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/EventObject.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/EventObject.java @@ -23,18 +23,34 @@ public String toString() { return "[" + objectIdField + ", " + objectId + "]"; } + /** + * Gets the object ID. + * @return The object ID. + */ public String getObjectId() { return objectId; } + /** + * Sets the object ID. + * @param objectId The object ID. + */ public void setObjectId(String objectId) { this.objectId = objectId; } + /** + * Gets the object ID field. + * @return The object ID field. + */ public String getObjectIdField() { return objectIdField; } + /** + * Sets the object ID field. + * @param objectIdField The object ID field. + */ public void setObjectIdField(String objectIdField) { this.objectIdField = objectIdField; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/Position.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/Position.java index b585312..3f0f4b3 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/Position.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/Position.java @@ -10,6 +10,9 @@ import com.google.gson.annotations.SerializedName; +/** + * A position represents the location of a search result in an event. + */ public class Position { @SerializedName("index") @@ -20,10 +23,18 @@ public String toString() { return String.valueOf(index); } + /** + * Gets the index of the position. + * @return The index of the position. + */ public int getIndex() { return index; } + /** + * Sets the index of the position. + * @param index The index of the position. + */ public void setIndex(int index) { this.index = index; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/UbiEvent.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/UbiEvent.java index 0d76fb4..fad79d6 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/UbiEvent.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/UbiEvent.java @@ -32,22 +32,42 @@ public String toString() { return actionName + ", " + clientId + ", " + queryId + ", " + eventAttributes.getObject().toString() + ", " + eventAttributes.getPosition().getIndex(); } + /** + * Gets the name of the action. + * @return The name of the action. + */ public String getActionName() { return actionName; } + /** + * Gets the client ID. + * @return The client ID. + */ public String getClientId() { return clientId; } + /** + * Gets the query ID. + * @return The query ID. + */ public String getQueryId() { return queryId; } + /** + * Gets the event attributes. + * @return The {@link EventAttributes}. + */ public EventAttributes getEventAttributes() { return eventAttributes; } + /** + * Sets the event attributes. + * @param eventAttributes The {@link EventAttributes}. + */ public void setEventAttributes(EventAttributes eventAttributes) { this.eventAttributes = eventAttributes; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/query/UbiQuery.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/query/UbiQuery.java index e6c61fb..52d48e7 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/query/UbiQuery.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/query/UbiQuery.java @@ -15,7 +15,7 @@ import java.util.Map; /** - * A UBI query. + * Represents a UBI query. */ public class UbiQuery { @@ -40,6 +40,13 @@ public class UbiQuery { @SerializedName("query_response") private QueryResponse queryResponse; + /** + * Creates a new UBI query object. + */ + public UbiQuery() { + + } + @Override public boolean equals(Object obj) { return EqualsBuilder.reflectionEquals(this, obj); @@ -54,58 +61,114 @@ public int hashCode() { toHashCode(); } + /** + * Gets the timestamp for the query. + * @return The timestamp for the query. + */ public String getTimestamp() { return timestamp; } + /** + * Sets the timestamp for the query. + * @param timestamp The timestamp for the query. + */ public void setTimestamp(String timestamp) { this.timestamp = timestamp; } + /** + * Gets the query ID. + * @return The query ID. + */ public String getQueryId() { return queryId; } + /** + * Sets the query ID. + * @param queryId The query ID. + */ public void setQueryId(String queryId) { this.queryId = queryId; } + /** + * Sets the client ID. + * @param clientId The client ID. + */ public void setClientId(String clientId) { this.clientId = clientId; } + /** + * Gets the client ID. + * @return The client ID. + */ public String getClientId() { return clientId; } + /** + * Gets the user query. + * @return The user query. + */ public String getUserQuery() { return userQuery; } + /** + * Sets the user query. + * @param userQuery The user query. + */ public void setUserQuery(String userQuery) { this.userQuery = userQuery; } + /** + * Gets the query. + * @return The query. + */ public String getQuery() { return query; } + /** + * Sets the query. + * @param query The query. + */ public void setQuery(String query) { this.query = query; } + /** + * Sets the query attributes. + * @return The query attributes. + */ public Map getQueryAttributes() { return queryAttributes; } + /** + * Sets the query attributes. + * @param queryAttributes The query attributes. + */ public void setQueryAttributes(Map queryAttributes) { this.queryAttributes = queryAttributes; } + /** + * Gets the query responses. + * @return The query responses. + */ public QueryResponse getQueryResponse() { return queryResponse; } + /** + * Sets the query responses. + * @param queryResponse The query responses. + */ public void setQueryResponse(QueryResponse queryResponse) { this.queryResponse = queryResponse; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java index 9f68c61..9058623 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java @@ -34,8 +34,8 @@ import java.util.Set; import java.util.UUID; -import static org.opensearch.eval.judgments.clickmodel.ClickModel.INDEX_UBI_EVENTS; -import static org.opensearch.eval.judgments.clickmodel.ClickModel.INDEX_UBI_QUERIES; +import static org.opensearch.eval.SearchQualityEvaluationPlugin.UBI_EVENTS_INDEX_NAME; +import static org.opensearch.eval.SearchQualityEvaluationPlugin.UBI_QUERIES_INDEX_NAME; import static org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel.INDEX_JUDGMENTS; import static org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel.INDEX_QUERY_DOC_CTR; import static org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel.INDEX_RANK_AGGREGATED_CTR; @@ -79,6 +79,7 @@ public String getUserQuery(final String queryId) throws Exception { * Gets the query object for a given query ID. * @param queryId The query ID. * @return A {@link UbiQuery} object for the given query ID. + * @throws Exception Thrown if the query cannot be retrieved. */ public UbiQuery getQueryFromQueryId(final String queryId) throws Exception { @@ -93,7 +94,7 @@ public UbiQuery getQueryFromQueryId(final String queryId) throws Exception { searchSourceBuilder.from(0); searchSourceBuilder.size(1); - final String[] indexes = {INDEX_UBI_QUERIES}; + final String[] indexes = {UBI_QUERIES_INDEX_NAME}; final SearchRequest searchRequest = new SearchRequest(indexes, searchSourceBuilder); final SearchResponse response = client.search(searchRequest).get(); @@ -115,7 +116,7 @@ private Collection getQueryIdsHavingUserQuery(final String userQuery) th final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(qb); - final String[] indexes = {INDEX_UBI_QUERIES}; + final String[] indexes = {UBI_QUERIES_INDEX_NAME}; final SearchRequest searchRequest = new SearchRequest(indexes, searchSourceBuilder); final SearchResponse response = client.search(searchRequest).get(); @@ -180,7 +181,7 @@ public long getCountOfQueriesForUserQueryHavingResultInRankR(final String userQu searchSourceBuilder.trackTotalHits(true); searchSourceBuilder.size(0); - final String[] indexes = {INDEX_UBI_EVENTS}; + final String[] indexes = {UBI_EVENTS_INDEX_NAME}; final SearchRequest searchRequest = new SearchRequest(indexes, searchSourceBuilder); final SearchResponse response = client.search(searchRequest).get(); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/util/UserQueryHash.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/util/IncrementalUserQueryHash.java similarity index 73% rename from opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/util/UserQueryHash.java rename to opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/util/IncrementalUserQueryHash.java index c0ccc6f..d09b6f0 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/util/UserQueryHash.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/util/IncrementalUserQueryHash.java @@ -8,18 +8,27 @@ */ package org.opensearch.eval.judgments.util; +import org.opensearch.eval.judgments.model.UserQueryHash; + import java.util.HashMap; import java.util.Map; -public class UserQueryHash { +/** + * Facilitates the hashing of user queries. + */ +public class IncrementalUserQueryHash implements UserQueryHash { private final Map userQueries; private int count = 1; - public UserQueryHash() { + /** + * Creates a new instance of this class. + */ + public IncrementalUserQueryHash() { this.userQueries = new HashMap<>(); } + @Override public int getHash(String userQuery) { final int hash; diff --git a/opensearch-search-quality-implicit-judgments/aggs.sh b/opensearch-search-quality-implicit-judgments/aggs.sh deleted file mode 100755 index ade861b..0000000 --- a/opensearch-search-quality-implicit-judgments/aggs.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e - -curl -X GET http://localhost:9200/ubi_events/_search -H "Content-Type: application/json" -d' -{ - "size": 0, - "aggs": { - "By_Action": { - "terms": { - "field": "action_name" - }, - "aggs": { - "By_Position": { - "terms": { - "field": "event_attributes.position.index" - } - } - } - } - } -}' | jq \ No newline at end of file diff --git a/opensearch-search-quality-implicit-judgments/build.gradle b/opensearch-search-quality-implicit-judgments/build.gradle deleted file mode 100644 index 5507280..0000000 --- a/opensearch-search-quality-implicit-judgments/build.gradle +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -apply plugin: 'java' -apply plugin: 'java-library-distribution' -apply plugin: 'application' - -description = "opensearch-search-quality-implicit-judgments" - -repositories { - mavenLocal() - mavenCentral() - maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } -} - -test { - useJUnitPlatform() -} - -dependencies { - implementation "org.opensearch.client:opensearch-rest-high-level-client:2.17.1" - implementation 'org.apache.httpcomponents.client5:httpclient5:5.4' - compileOnly "org.apache.logging.log4j:log4j-core:2.24.0" - implementation "org.apache.commons:commons-lang3:3.17.0" - implementation "com.google.code.gson:gson:2.11.0" - - testImplementation "org.mockito:mockito-core:5.14.2" - testImplementation 'org.junit.jupiter:junit-jupiter-api:5.11.3' - testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.11.3' -} - -tasks.withType(Tar){ - duplicatesStrategy = DuplicatesStrategy.EXCLUDE -} - -tasks.withType(Zip){ - duplicatesStrategy = DuplicatesStrategy.EXCLUDE -} \ No newline at end of file diff --git a/opensearch-search-quality-implicit-judgments/coec.png b/opensearch-search-quality-implicit-judgments/coec.png deleted file mode 100644 index 65e297a..0000000 Binary files a/opensearch-search-quality-implicit-judgments/coec.png and /dev/null differ diff --git a/opensearch-search-quality-implicit-judgments/coec_definition.png b/opensearch-search-quality-implicit-judgments/coec_definition.png deleted file mode 100644 index e5ed3f9..0000000 Binary files a/opensearch-search-quality-implicit-judgments/coec_definition.png and /dev/null differ diff --git a/opensearch-search-quality-implicit-judgments/queries.txt b/opensearch-search-quality-implicit-judgments/queries.txt deleted file mode 100644 index 9ad9c30..0000000 --- a/opensearch-search-quality-implicit-judgments/queries.txt +++ /dev/null @@ -1,11 +0,0 @@ -GET ubi_queries/_search - -GET ubi_events/_search - -GET rank_aggregated_ctr/_search - -GET click_through_rates/_search - -GET judgments/_search - -DELETE rank_aggregated_ctr,click_through_rates,judgments diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/App.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/App.java deleted file mode 100644 index 588d0b3..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/App.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.opensearch.qef; - -import org.apache.http.HttpHost; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.client.RestClient; -import org.opensearch.client.RestClientBuilder; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.qef.clickmodel.coec.CoecClickModel; -import org.opensearch.qef.clickmodel.coec.CoecClickModelParameters; -import org.opensearch.qef.model.Judgment; - -import java.util.Collection; - -/** - * Entry point for the OpenSearch Evaluation Framework standalone app. - */ -public class App { - - private static final Logger LOGGER = LogManager.getLogger(App.class.getName()); - - public static void main(String[] args) throws Exception { - - final RestClientBuilder builder = RestClient.builder(new HttpHost("localhost", 9200, "http")); - final RestHighLevelClient restHighLevelClient = new RestHighLevelClient(builder); - - final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(false, 20); - final CoecClickModel coecClickModel = new CoecClickModel(coecClickModelParameters); - - final Collection judgments = coecClickModel.calculateJudgments(); - Judgment.showJudgments(judgments); - - } - -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/clickmodel/ClickModel.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/clickmodel/ClickModel.java deleted file mode 100644 index a164b9e..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/clickmodel/ClickModel.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.opensearch.qef.clickmodel; - -import org.opensearch.qef.model.Judgment; - -import java.io.IOException; -import java.util.Collection; - -public abstract class ClickModel { - - public static final String INDEX_UBI_EVENTS = "ubi_events"; - public static final String INDEX_UBI_QUERIES = "ubi_queries"; - - public abstract Collection calculateJudgments() throws IOException; - -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/clickmodel/ClickModelParameters.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/clickmodel/ClickModelParameters.java deleted file mode 100644 index a03b414..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/clickmodel/ClickModelParameters.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.opensearch.qef.clickmodel; - -public abstract class ClickModelParameters { - -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/clickmodel/coec/CoecClickModel.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/clickmodel/coec/CoecClickModel.java deleted file mode 100644 index 62dc496..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/clickmodel/coec/CoecClickModel.java +++ /dev/null @@ -1,357 +0,0 @@ -package org.opensearch.qef.clickmodel.coec; - -import com.google.gson.Gson; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.search.SearchScrollRequest; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.Requests; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.index.query.WrapperQueryBuilder; -import org.opensearch.qef.clickmodel.ClickModel; -import org.opensearch.qef.engine.opensearch.OpenSearchHelper; -import org.opensearch.qef.model.ClickthroughRate; -import org.opensearch.qef.model.Judgment; -import org.opensearch.qef.model.ubi.event.UbiEvent; -import org.opensearch.qef.util.MathUtils; -import org.opensearch.qef.util.UserQueryHash; -import org.opensearch.search.Scroll; -import org.opensearch.search.SearchHit; -import org.opensearch.search.aggregations.AggregationBuilders; -import org.opensearch.search.aggregations.bucket.terms.Terms; -import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.opensearch.search.builder.SearchSourceBuilder; - -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.Map; -import java.util.Set; - -public class CoecClickModel extends ClickModel { - - // OpenSearch indexes. - public static final String INDEX_RANK_AGGREGATED_CTR = "rank_aggregated_ctr"; - public static final String INDEX_QUERY_DOC_CTR = "click_through_rates"; - public static final String INDEX_JUDGMENT = "judgments"; - - // UBI event names. - public static final String EVENT_CLICK = "click"; - public static final String EVENT_VIEW = "view"; - - private final CoecClickModelParameters parameters; - - private final OpenSearchHelper openSearchHelper; - - private final UserQueryHash userQueryHash = new UserQueryHash(); - private final Gson gson = new Gson(); - private final RestHighLevelClient client; - - private static final Logger LOGGER = LogManager.getLogger(CoecClickModel.class.getName()); - - public CoecClickModel(final CoecClickModelParameters parameters) { - - this.parameters = parameters; - this.openSearchHelper = new OpenSearchHelper(parameters.getRestHighLevelClient()); - this.client = parameters.getRestHighLevelClient(); - - } - - @Override - public Collection calculateJudgments() throws IOException { - - final int maxRank = parameters.getMaxRank(); - - // Calculate and index the rank-aggregated click-through. - final Map rankAggregatedClickThrough = getRankAggregatedClickThrough(); - LOGGER.info("Rank-aggregated clickthrough positions: {}", rankAggregatedClickThrough.size()); - showRankAggregatedClickThrough(rankAggregatedClickThrough); - - // Calculate and index the click-through rate for query/doc pairs. - final Map> clickthroughRates = getClickthroughRate(maxRank); - LOGGER.info("Clickthrough rates for number of queries: {}", clickthroughRates.size()); - showClickthroughRates(clickthroughRates); - - // Generate and index the implicit judgments. - final Collection judgments = calculateCoec(rankAggregatedClickThrough, clickthroughRates); - LOGGER.info("Number of judgments: {}", judgments.size()); - - return judgments; - - } - - public Collection calculateCoec(final Map rankAggregatedClickThrough, - final Map> clickthroughRates) throws IOException { - - // Calculate the COEC. - // Numerator is the total number of clicks received by a query/result pair. - // Denominator is the expected clicks (EC) that an average result would receive after being impressed i times at rank r, - // and CTR is the average CTR for each position in the results page (up to R) computed over all queries and results. - - // Format: query_id, query, document, judgment - final Collection judgments = new LinkedList<>(); - - // Up to Rank R - final int maxRank = 20; - - for(final String userQuery : clickthroughRates.keySet()) { - - // The clickthrough rates for this query. - final Collection ctrs = clickthroughRates.get(userQuery); - - for(final ClickthroughRate ctr : ctrs) { - - double denominatorSum = 0; - - for(int r = 0; r < maxRank; r++) { - - final double meanCtrAtRank = rankAggregatedClickThrough.getOrDefault(r, 0.0); - final int countOfTimesShownAtRank = openSearchHelper.getCountOfQueriesForUserQueryHavingResultInRankR(userQuery, ctr.getObjectId(), r); - -// System.out.println("rank = " + r); -// System.out.println("\tmeanCtrAtRank = " + meanCtrAtRank); -// System.out.println("\tcountOfTimesShownAtRank = " + countOfTimesShownAtRank); - - denominatorSum += (meanCtrAtRank * countOfTimesShownAtRank); - - } - - // Numerator is sum of clicks at all ranks up to the maxRank. - final int totalNumberClicksForQueryResult = ctr.getClicks(); - -// System.out.println("numerator = " + totalNumberClicksForQueryResult); -// System.out.println("denominator = " + denominatorSum); - - // Divide the numerator by the denominator (value). - final double judgment = totalNumberClicksForQueryResult / denominatorSum; - - // Hash the user query to get a query ID. - final int queryId = userQueryHash.getHash(userQuery); - - // Add the judgment to the list. - // TODO: What to do for query ID when the values are per user_query instead? - judgments.add(new Judgment(String.valueOf(queryId), userQuery, ctr.getObjectId(), judgment)); - - } - - } - - if(parameters.isPersist()) { - openSearchHelper.indexJudgments(judgments); - } - - return judgments; - - } - - /** - * Gets the clickthrough rates for each query and its results. - * @param maxRank The maximum rank position to consider. - * @return A map of user_query to the clickthrough rate for each query result. - * @throws IOException Thrown when a problem accessing OpenSearch. - */ - private Map> getClickthroughRate(final int maxRank) throws IOException { - - // For each query: - // - Get each document returned in that query (in the QueryResponse object). - // - Calculate the click-through rate for the document. (clicks/impressions) - - // TODO: Use maxRank in place of the hardcoded 20. - // TODO: Allow for a time period and for a specific application. - - final String query = "{\n" + - " \"bool\": {\n" + - " \"should\": [\n" + - " {\n" + - " \"term\": {\n" + - " \"action_name\": \"click\"\n" + - " }\n" + - " },\n" + - " {\n" + - " \"term\": {\n" + - " \"action_name\": \"view\"\n" + - " }\n" + - " }\n" + - " ],\n" + - " \"must\": [\n" + - " {\n" + - " \"range\": {\n" + - " \"event_attributes.position.index\": {\n" + - " \"lte\": 20\n" + - " }\n" + - " }\n" + - " }\n" + - " ]\n" + - " }\n" + - " }"; - - final BoolQueryBuilder queryBuilder = new BoolQueryBuilder().must(new WrapperQueryBuilder(query)); - final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(queryBuilder).size(1000); - final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(10L)); - - final SearchRequest searchRequest = Requests - .searchRequest(INDEX_UBI_EVENTS) - .source(searchSourceBuilder) - .scroll(scroll); - - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - String scrollId = searchResponse.getScrollId(); - SearchHit[] searchHits = searchResponse.getHits().getHits(); - - final Map> queriesToClickthroughRates = new HashMap<>(); - - while (searchHits != null && searchHits.length > 0) { - - for (final SearchHit hit : searchHits) { - - final UbiEvent ubiEvent = gson.fromJson(hit.getSourceAsString(), UbiEvent.class); - - // We need to the hash of the query_id because two users can both search - // for "computer" and those searches will have different query IDs, but they are the same search. - final String userQuery = openSearchHelper.getUserQuery(ubiEvent.getQueryId()); - // LOGGER.debug("user_query = {}", userQuery); - - // Get the clicks for this queryId from the map, or an empty list if this is a new query. - final Set clickthroughRates = queriesToClickthroughRates.getOrDefault(userQuery, new LinkedHashSet<>()); - - // Get the ClickthroughRate object for the object that was interacted with. - final ClickthroughRate clickthroughRate = clickthroughRates.stream().filter(p -> p.getObjectId().equals(ubiEvent.getEventAttributes().getObject().getObjectId())).findFirst().orElse(new ClickthroughRate(ubiEvent.getEventAttributes().getObject().getObjectId())); - - if (StringUtils.equalsIgnoreCase(ubiEvent.getActionName(), EVENT_CLICK)) { - clickthroughRate.logClick(); - } else { - clickthroughRate.logEvent(); - } - - clickthroughRates.add(clickthroughRate); - queriesToClickthroughRates.put(userQuery, clickthroughRates); - // LOGGER.debug("clickthroughRate = {}", queriesToClickthroughRates.size()); - - } - - final SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); - scrollRequest.scroll(scroll); - - searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); - scrollId = searchResponse.getScrollId(); - - searchHits = searchResponse.getHits().getHits(); - - } - - if(parameters.isPersist()) { - openSearchHelper.indexClickthroughRates(queriesToClickthroughRates); - } - - return queriesToClickthroughRates; - - } - - /** - * Calculate the rank-aggregated click through from the UBI events. - * @return A map of positions to clickthrough rates. - * @throws IOException Thrown when a problem accessing OpenSearch. - */ - public Map getRankAggregatedClickThrough() throws IOException { - - final Map rankAggregatedClickThrough = new HashMap<>(); - - // TODO: Allow for a time period and for a specific application. - - final QueryBuilder findRangeNumber = QueryBuilders.rangeQuery("event_attributes.position.index").lte(parameters.getMaxRank()); - final QueryBuilder queryBuilder = new BoolQueryBuilder().must(findRangeNumber); - - final TermsAggregationBuilder positionsAggregator = AggregationBuilders.terms("By_Position").field("event_attributes.position.index").size(parameters.getMaxRank()); - final TermsAggregationBuilder actionNameAggregation = AggregationBuilders.terms("By_Action").field("action_name").subAggregation(positionsAggregator).size(parameters.getMaxRank()); - - final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(queryBuilder); - searchSourceBuilder.aggregation(actionNameAggregation); - searchSourceBuilder.from(0); - searchSourceBuilder.size(100); - - final SearchRequest searchRequest = new SearchRequest(INDEX_UBI_EVENTS).source(searchSourceBuilder); - final SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - - final Map clickCounts = new HashMap<>(); - final Map viewCounts = new HashMap<>(); - - final Terms actionTerms = searchResponse.getAggregations().get("By_Action"); - final Collection actionBuckets = actionTerms.getBuckets(); - for(final Terms.Bucket actionBucket : actionBuckets) { - - // Handle the "click" bucket. - if(StringUtils.equalsIgnoreCase(actionBucket.getKey().toString(), EVENT_CLICK)) { - - final Terms positionTerms = actionBucket.getAggregations().get("By_Position"); - final Collection positionBuckets = positionTerms.getBuckets(); - - for(final Terms.Bucket positionBucket : positionBuckets) { - clickCounts.put(Integer.valueOf(positionBucket.getKey().toString()), (double) positionBucket.getDocCount()); - } - - } - - // Handle the "view" bucket. - if(StringUtils.equalsIgnoreCase(actionBucket.getKey().toString(), EVENT_VIEW)) { - - final Terms positionTerms = actionBucket.getAggregations().get("By_Position"); - final Collection positionBuckets = positionTerms.getBuckets(); - - for(final Terms.Bucket positionBucket : positionBuckets) { - viewCounts.put(Integer.valueOf(positionBucket.getKey().toString()), (double) positionBucket.getDocCount()); - } - - } - - } - - for(final Integer x : clickCounts.keySet()) { - //System.out.println("Position = " + x + ", Click Count = " + clickCounts.get(x) + ", Event Count = " + viewCounts.get(x)); - rankAggregatedClickThrough.put(x, clickCounts.get(x) / viewCounts.get(x)); - } - - if(parameters.isPersist()) { - openSearchHelper.indexRankAggregatedClickthrough(rankAggregatedClickThrough); - } - - return rankAggregatedClickThrough; - - } - - private void showClickthroughRates(final Map> clickthroughRates) { - - for(final String userQuery : clickthroughRates.keySet()) { - - LOGGER.info("user_query: {}", userQuery); - - for(final ClickthroughRate clickthroughRate : clickthroughRates.get(userQuery)) { - - LOGGER.info("\t - {}", clickthroughRate.toString()); - - } - - } - - } - - private void showRankAggregatedClickThrough(final Map rankAggregatedClickThrough) { - - for(final int position : rankAggregatedClickThrough.keySet()) { - - LOGGER.info("Position: {}, # ctr: {}", position, MathUtils.round(rankAggregatedClickThrough.get(position), parameters.getRoundingDigits())); - - } - - } - -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/clickmodel/coec/CoecClickModelParameters.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/clickmodel/coec/CoecClickModelParameters.java deleted file mode 100644 index b72dc58..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/clickmodel/coec/CoecClickModelParameters.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.opensearch.qef.clickmodel.coec; - -import org.opensearch.client.RestClient; -import org.opensearch.client.RestClientBuilder; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.qef.clickmodel.ClickModelParameters; - -public class CoecClickModelParameters extends ClickModelParameters { - - private final RestHighLevelClient restHighLevelClient; - private final boolean persist; - private final int maxRank; - private int roundingDigits = 3; - - public CoecClickModelParameters(boolean persist, final int maxRank) { - - final RestClientBuilder builder = RestClient.builder("http://localhost:9200"); - this.restHighLevelClient = new RestHighLevelClient(builder); - - this.persist = persist; - this.maxRank = maxRank; - } - - public CoecClickModelParameters(boolean persist, final int maxRank, final int roundingDigits) { - - final RestClientBuilder builder = RestClient.builder("http://localhost:9200"); - this.restHighLevelClient = new RestHighLevelClient(builder); - - this.persist = persist; - this.maxRank = maxRank; - this.roundingDigits = roundingDigits; - } - - public RestHighLevelClient getRestHighLevelClient() { - return restHighLevelClient; - } - - public boolean isPersist() { - return persist; - } - - public int getMaxRank() { - return maxRank; - } - - public int getRoundingDigits() { - return roundingDigits; - } - -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/engine/opensearch/OpenSearchHelper.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/engine/opensearch/OpenSearchHelper.java deleted file mode 100644 index bbfe38c..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/engine/opensearch/OpenSearchHelper.java +++ /dev/null @@ -1,222 +0,0 @@ -package org.opensearch.qef.engine.opensearch; - -import com.google.gson.Gson; -import org.opensearch.action.bulk.BulkRequest; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.index.query.WrapperQueryBuilder; -import org.opensearch.qef.model.ClickthroughRate; -import org.opensearch.qef.model.Judgment; -import org.opensearch.qef.model.ubi.query.UbiQuery; -import org.opensearch.search.SearchHit; -import org.opensearch.search.builder.SearchSourceBuilder; - -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - -import static org.opensearch.qef.clickmodel.coec.CoecClickModel.INDEX_JUDGMENT; -import static org.opensearch.qef.clickmodel.coec.CoecClickModel.INDEX_QUERY_DOC_CTR; -import static org.opensearch.qef.clickmodel.coec.CoecClickModel.INDEX_RANK_AGGREGATED_CTR; -import static org.opensearch.qef.clickmodel.coec.CoecClickModel.INDEX_UBI_QUERIES; - -public class OpenSearchHelper { - - private final RestHighLevelClient client; - private final Gson gson = new Gson(); - - // Used to cache the query ID->user_query to avoid unnecessary lookups to OpenSearch. - private static final Map userQueryCache = new HashMap<>(); - - public OpenSearchHelper(final RestHighLevelClient client) { - this.client = client; - } - - /** - * Gets the user query for a given query ID. - * @param queryId The query ID. - * @return The user query. - * @throws IOException Thrown when there is a problem accessing OpenSearch. - */ - public String getUserQuery(final String queryId) throws IOException { - - // If it's in the cache just get it and return it. - if(userQueryCache.containsKey(queryId)) { - return userQueryCache.get(queryId); - } - - // Cache it and return it. - final UbiQuery ubiQuery = getQueryFromQueryId(queryId); - userQueryCache.put(queryId, ubiQuery.getUserQuery()); - - return ubiQuery.getUserQuery(); - - } - - /** - * Gets the query object for a given query ID. - * @param queryId The query ID. - * @return A {@link UbiQuery} object for the given query ID. - */ - public UbiQuery getQueryFromQueryId(final String queryId) throws IOException { - - final String query = "{\"match\": {\"query_id\": \"" + queryId + "\" }}"; - final WrapperQueryBuilder qb = QueryBuilders.wrapperQuery(query); - - // The query_id should be unique anyway, but we are limiting it to a single result anyway. - final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(qb); - searchSourceBuilder.from(0); - searchSourceBuilder.size(1); - - final String[] indexes = {INDEX_UBI_QUERIES}; - - final SearchRequest searchRequest = new SearchRequest(indexes, searchSourceBuilder); - final SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT); - - // Will only be a single result. - final SearchHit hit = response.getHits().getHits()[0]; - - return gson.fromJson(hit.getSourceAsString(), UbiQuery.class); - - } - - public int getCountOfQueriesForUserQueryHavingResultInRankR(final String userQuery, final String objectId, final int rank) throws IOException { - - int countOfTimesShownAtRank = 0; - - final String query = "{\"match\": {\"user_query\": \"" + userQuery + "\" }}"; - final WrapperQueryBuilder qb = QueryBuilders.wrapperQuery(query); - - final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(qb); - - final String[] indexes = {INDEX_UBI_QUERIES}; - - final SearchRequest searchRequest = new SearchRequest(indexes, searchSourceBuilder); - final SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT); - - for(final SearchHit searchHit : response.getHits().getHits()) { - - final List queryResponseObjectIds = (List) searchHit.getSourceAsMap().get("query_response_object_ids"); - - if(queryResponseObjectIds.get(rank).equals(objectId)) { - countOfTimesShownAtRank++; - } - - } - - return countOfTimesShownAtRank; - - } - - /** - * Index the rank-aggregated clickthrough values. - * @param rankAggregatedClickThrough A map of position to clickthrough values. - * @throws IOException Thrown when there is a problem accessing OpenSearch. - */ - public void indexRankAggregatedClickthrough(final Map rankAggregatedClickThrough) throws IOException { - - if(!rankAggregatedClickThrough.isEmpty()) { - - // TODO: Split this into multiple bulk insert requests. - - final BulkRequest request = new BulkRequest(); - - for (final int position : rankAggregatedClickThrough.keySet()) { - - final Map jsonMap = new HashMap<>(); - jsonMap.put("position", position); - jsonMap.put("ctr", rankAggregatedClickThrough.get(position)); - - final IndexRequest indexRequest = new IndexRequest(INDEX_RANK_AGGREGATED_CTR).id(UUID.randomUUID().toString()).source(jsonMap); - - request.add(indexRequest); - - } - - client.bulk(request, RequestOptions.DEFAULT); - - } - - } - - /** - * Index the clickthrough rates. - * @param clickthroughRates A map of query IDs to a collection of {@link ClickthroughRate} objects. - * @throws IOException Thrown when there is a problem accessing OpenSearch. - */ - public void indexClickthroughRates(final Map> clickthroughRates) throws IOException { - - if(!clickthroughRates.isEmpty()) { - - // TODO: Split this into multiple bulk insert requests. - - final BulkRequest request = new BulkRequest(); - - for(final String queryId : clickthroughRates.keySet()) { - - for(final ClickthroughRate clickthroughRate : clickthroughRates.get(queryId)) { - - final Map jsonMap = new HashMap<>(); - jsonMap.put("query_id", queryId); - jsonMap.put("clicks", clickthroughRate.getClicks()); - jsonMap.put("events", clickthroughRate.getEvents()); - jsonMap.put("ctr", clickthroughRate.getClickthroughRate()); - - final IndexRequest indexRequest = new IndexRequest(INDEX_QUERY_DOC_CTR).id(UUID.randomUUID().toString()).source(jsonMap); - - request.add(indexRequest); - - } - - } - - client.bulk(request, RequestOptions.DEFAULT); - - } - - } - - /** - * Index the judgments. - * @param judgments A collection of {@link Judgment judgments}. - * @throws IOException Thrown when there is a problem accessing OpenSearch. - */ - public void indexJudgments(final Collection judgments) throws IOException { - - if(!judgments.isEmpty()) { - - // TODO: Split this into multiple bulk insert requests. - - final BulkRequest request = new BulkRequest(); - - for (final Judgment judgment : judgments) { - - final Map jsonMap = new HashMap<>(); - jsonMap.put("query_id", judgment.getQueryId()); - jsonMap.put("query", judgment.getQuery()); - jsonMap.put("document", judgment.getDocument()); - jsonMap.put("judgment", judgment.getJudgment()); - - final IndexRequest indexRequest = new IndexRequest(INDEX_JUDGMENT).id(UUID.randomUUID().toString()).source(jsonMap); - - request.add(indexRequest); - - } - - client.bulk(request, RequestOptions.DEFAULT); - - } - - } - -} \ No newline at end of file diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ClickthroughRate.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ClickthroughRate.java deleted file mode 100644 index d987498..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ClickthroughRate.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.opensearch.qef.model; - -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.opensearch.qef.util.MathUtils; - -/** - * A query result and its number of clicks and total events. - */ -public class ClickthroughRate { - - private final String objectId; - private int clicks; - private int events; - - public ClickthroughRate(final String objectId) { - this.objectId = objectId; - this.clicks = 0; - this.events = 0; - } - - public ClickthroughRate(final String objectId, final int clicks, final int events) { - this.objectId = objectId; - this.clicks = clicks; - this.events = events; - } - - @Override - public boolean equals(Object obj) { - return EqualsBuilder.reflectionEquals(this, obj); - } - - @Override - public int hashCode() { - int result = 17; - result = 29 * result + objectId.hashCode(); - return result; - } - - @Override - public String toString() { - return "object_id: " + objectId + ", clicks: " + clicks + ", events: " + events + ", ctr: " + MathUtils.round(getClickthroughRate()); - } - - public void logClick() { - clicks++; - events++; - } - - public void logEvent() { - events++; - } - - public double getClickthroughRate() { - return (double) clicks / events; - } - - public int getClicks() { - return clicks; - } - - public int getEvents() { - return events; - } - - public String getObjectId() { - return objectId; - } - -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/Judgment.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/Judgment.java deleted file mode 100644 index 54067ad..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/Judgment.java +++ /dev/null @@ -1,80 +0,0 @@ -package org.opensearch.qef.model; - -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.qef.util.MathUtils; - -import java.util.Collection; - -/** - * A judgment of a search result's quality for a given query. - */ -public class Judgment { - - private static final Logger LOGGER = LogManager.getLogger(Judgment.class.getName()); - - private final String queryId; - private final String query; - private final String document; - private final double judgment; - - public Judgment(final String queryId, final String query, final String document, final double judgment) { - this.queryId = queryId; - this.query = query; - this.document = document; - this.judgment = judgment; - } - - public String toJudgmentString() { - return queryId + ", " + query + ", " + document + ", " + MathUtils.round(judgment); - } - - public static void showJudgments(final Collection judgments) { - - LOGGER.info("query_id, query, document, judgment"); - - for(final Judgment judgment : judgments) { - LOGGER.info(judgment.toJudgmentString()); - } - - } - - @Override - public String toString() { - return "query_id: " + queryId + ", query: " + query + ", document: " + document + ", judgment: " + MathUtils.round(judgment); - } - - @Override - public boolean equals(Object obj) { - return EqualsBuilder.reflectionEquals(this, obj); - } - - @Override - public int hashCode() { - return new HashCodeBuilder(17, 37). - append(queryId). - append(query). - append(document). - append(judgment). - toHashCode(); - } - - public String getQueryId() { - return queryId; - } - - public String getQuery() { - return query; - } - - public String getDocument() { - return document; - } - - public double getJudgment() { - return judgment; - } - -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/event/EventAttributes.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/event/EventAttributes.java deleted file mode 100644 index 47bd901..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/event/EventAttributes.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.opensearch.qef.model.ubi.event; - -import com.google.gson.annotations.SerializedName; - -public class EventAttributes { - - @SerializedName("object") - private EventObject object; - - @SerializedName("session_id") - private String sessionId; - - @SerializedName("position") - private Position position; - - public EventObject getObject() { - return object; - } - - public void setObject(EventObject object) { - this.object = object; - } - - public String getSessionId() { - return sessionId; - } - - public void setSessionId(String sessionId) { - this.sessionId = sessionId; - } - - public Position getPosition() { - return position; - } - - public void setPosition(Position position) { - this.position = position; - } - -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/event/EventObject.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/event/EventObject.java deleted file mode 100644 index 57ae60f..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/event/EventObject.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.opensearch.qef.model.ubi.event; - -import com.google.gson.annotations.SerializedName; - -public class EventObject { - - @SerializedName("object_id_field") - private String objectIdField; - - @SerializedName("object_id") - private String objectId; - - public String getObjectId() { - return objectId; - } - - public void setObjectId(String objectId) { - this.objectId = objectId; - } - - public String getObjectIdField() { - return objectIdField; - } - - public void setObjectIdField(String objectIdField) { - this.objectIdField = objectIdField; - } - -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/event/Position.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/event/Position.java deleted file mode 100644 index 1e619c4..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/event/Position.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.opensearch.qef.model.ubi.event; - -import com.google.gson.annotations.SerializedName; - -public class Position { - - @SerializedName("index") - private int index; - - public int getIndex() { - return index; - } - - public void setIndex(int index) { - this.index = index; - } - -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/event/UbiEvent.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/event/UbiEvent.java deleted file mode 100644 index 74f28f5..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/event/UbiEvent.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.opensearch.qef.model.ubi.event; - -import com.google.gson.annotations.SerializedName; - -/** - * A UBI event. - */ -public class UbiEvent { - - @SerializedName("action_name") - private String actionName; - - @SerializedName("client_id") - private String clientId; - - @SerializedName("query_id") - private String queryId; - - @SerializedName("event_attributes") - private EventAttributes eventAttributes; - - @Override - public String toString() { - return actionName + ", " + clientId + ", " + queryId + ", " + eventAttributes.getObject() + ", " + eventAttributes.getPosition().getIndex(); - } - - public String getActionName() { - return actionName; - } - - public String getClientId() { - return clientId; - } - - public String getQueryId() { - return queryId; - } - - public EventAttributes getEventAttributes() { - return eventAttributes; - } - - public void setEventAttributes(EventAttributes eventAttributes) { - this.eventAttributes = eventAttributes; - } -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/query/QueryResponse.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/query/QueryResponse.java deleted file mode 100644 index 016db80..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/query/QueryResponse.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.opensearch.qef.model.ubi.query; - -import java.util.List; - -/** - * A query response for a {@link UbiQuery query}. - */ -public class QueryResponse { - - private final String queryId; - private final String queryResponseId; - private final List queryResponseObjectIds; - - /** - * Creates a query response. - * @param queryId The ID of the query. - * @param queryResponseId The ID of the query response. - * @param queryResponseObjectIds A list of IDs for the hits in the query. - */ - public QueryResponse(final String queryId, final String queryResponseId, final List queryResponseObjectIds) { - this.queryId = queryId; - this.queryResponseId = queryResponseId; - this.queryResponseObjectIds = queryResponseObjectIds; - } - - /** - * Gets the query ID. - * @return The query ID. - */ - public String getQueryId() { - return queryId; - } - - /** - * Gets the query response ID. - * @return The query response ID. - */ - public String getQueryResponseId() { - return queryResponseId; - } - - /** - * Gets the list of query response hit IDs. - * @return A list of query response hit IDs. - */ - public List getQueryResponseObjectIds() { - return queryResponseObjectIds; - } - -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/query/UbiQuery.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/query/UbiQuery.java deleted file mode 100644 index c00905b..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/model/ubi/query/UbiQuery.java +++ /dev/null @@ -1,105 +0,0 @@ -package org.opensearch.qef.model.ubi.query; - -import com.google.gson.annotations.SerializedName; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; - -import java.util.Map; - -/** - * A UBI query. - */ -public class UbiQuery { - - @SerializedName("timestamp") - private long timestamp; - - @SerializedName("query_id") - private String queryId; - - @SerializedName("client_id") - private String clientId; - - @SerializedName("user_query") - private String userQuery; - - @SerializedName("query") - private String query; - - @SerializedName("query_attributes") - private Map queryAttributes; - - @SerializedName("query_response") - private QueryResponse queryResponse; - - @Override - public boolean equals(Object obj) { - return EqualsBuilder.reflectionEquals(this, obj); - } - - @Override - public int hashCode() { - return new HashCodeBuilder(17, 37). - append(queryId). - append(userQuery). - append(clientId). - toHashCode(); - } - - public long getTimestamp() { - return timestamp; - } - - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - - public String getQueryId() { - return queryId; - } - - public void setQueryId(String queryId) { - this.queryId = queryId; - } - - public void setClientId(String clientId) { - this.clientId = clientId; - } - - public String getClientId() { - return clientId; - } - - public String getUserQuery() { - return userQuery; - } - - public void setUserQuery(String userQuery) { - this.userQuery = userQuery; - } - - public String getQuery() { - return query; - } - - public void setQuery(String query) { - this.query = query; - } - - public Map getQueryAttributes() { - return queryAttributes; - } - - public void setQueryAttributes(Map queryAttributes) { - this.queryAttributes = queryAttributes; - } - - public QueryResponse getQueryResponse() { - return queryResponse; - } - - public void setQueryResponse(QueryResponse queryResponse) { - this.queryResponse = queryResponse; - } - -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/util/MathUtils.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/util/MathUtils.java deleted file mode 100644 index 9fc945c..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/util/MathUtils.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.opensearch.qef.util; - -public class MathUtils { - - private MathUtils() { - - } - - public static String round(final double value, final int decimalPlaces) { - double factor = Math.pow(10, decimalPlaces); - return String.valueOf(Math.round(value * factor) / factor); - } - - public static String round(final double value) { - return round(value, 3); - } - -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/util/UserQueryHash.java b/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/util/UserQueryHash.java deleted file mode 100644 index 3b3f8cd..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/java/org/opensearch/qef/util/UserQueryHash.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.opensearch.qef.util; - -import java.util.HashMap; -import java.util.Map; - -public class UserQueryHash { - - private final Map userQueries; - private int count = 1; - - public UserQueryHash() { - this.userQueries = new HashMap<>(); - } - - public int getHash(String userQuery) { - - final int hash; - - if(userQueries.containsKey(userQuery)) { - - return userQueries.get(userQuery); - - } else { - - userQueries.put(userQuery, count); - hash = count; - count++; - - - } - - return hash; - - } - -} diff --git a/opensearch-search-quality-implicit-judgments/src/main/resources/events-mapping.json b/opensearch-search-quality-implicit-judgments/src/main/resources/events-mapping.json deleted file mode 100644 index cdb1393..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/resources/events-mapping.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "properties": { - "application": { "type": "keyword", "ignore_above": 256 }, - "action_name": { "type": "keyword", "ignore_above": 100 }, - "client_id": { "type": "keyword", "ignore_above": 100 }, - "message": { "type": "keyword", "ignore_above": 1024 }, - "message_type": { "type": "keyword", "ignore_above": 100 }, - "timestamp": { - "type": "date", - "format":"strict_date_time", - "ignore_malformed": true, - "doc_values": true - }, - "event_attributes": { - "dynamic": true, - "properties": { - "position": { - "properties": { - "ordinal": { "type": "integer" }, - "x": { "type": "integer" }, - "y": { "type": "integer" }, - "page_depth": { "type": "integer" }, - "scroll_depth": { "type": "integer" }, - "trail": { "type": "text", - "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } - } - } - } - }, - "object": { - "properties": { - "internal_id": { "type": "keyword" }, - "object_id": { "type": "keyword", "ignore_above": 256 }, - "object_id_field": { "type": "keyword", "ignore_above": 100 }, - "name": { "type": "keyword", "ignore_above": 256 }, - "description": { "type": "text", - "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } } - }, - "object_detail": { "type": "object" } - } - } - } - } - } -} \ No newline at end of file diff --git a/opensearch-search-quality-implicit-judgments/src/main/resources/log4j2.xml b/opensearch-search-quality-implicit-judgments/src/main/resources/log4j2.xml deleted file mode 100644 index eea5a4c..0000000 --- a/opensearch-search-quality-implicit-judgments/src/main/resources/log4j2.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/opensearch-search-quality-implicit-judgments/src/test/java/org/opensearch/qef/clickmodel/coec/CoecClickModelIT.java b/opensearch-search-quality-implicit-judgments/src/test/java/org/opensearch/qef/clickmodel/coec/CoecClickModelIT.java deleted file mode 100644 index cdce2cf..0000000 --- a/opensearch-search-quality-implicit-judgments/src/test/java/org/opensearch/qef/clickmodel/coec/CoecClickModelIT.java +++ /dev/null @@ -1,159 +0,0 @@ -package org.opensearch.qef.clickmodel.coec; - -import org.apache.http.HttpHost; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.opensearch.action.bulk.BulkRequest; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.RestClient; -import org.opensearch.client.RestClientBuilder; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.qef.engine.opensearch.OpenSearchHelper; -import org.opensearch.qef.model.Judgment; - -import java.io.IOException; -import java.util.Collection; -import java.util.UUID; - -public class CoecClickModelIT { - - private static final Logger LOGGER = LogManager.getLogger(CoecClickModelIT.class.getName()); - - @Disabled - @Test - public void calculateJudgmentForDoc1() throws IOException { - - final RestClientBuilder builder = RestClient.builder(new HttpHost("localhost", 9200, "http")); - final RestHighLevelClient restHighLevelClient = new RestHighLevelClient(builder); - -// // Remove any existing indexes. -// final boolean exists = restHighLevelClient.indices().exists(new GetIndexRequest("ubi_events"), RequestOptions.DEFAULT); -// if(exists) { -// restHighLevelClient.indices().delete(new DeleteIndexRequest("ubi_events"), RequestOptions.DEFAULT); -// } -// -// // Create the ubi_events index. -// final CreateIndexRequest createIndexRequest = new CreateIndexRequest("ubi_events").mapping(getResourceFile("/events-mapping.json")); -// restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT); - - final int numberOfViews = 250; - final int numberOfClicks = 110; - - final BulkRequest bulkRequest = new BulkRequest(); - - // Index the view. - for(int x = 1; x <= numberOfViews; x++) { - - final String event = "{\n" + - " \"action_name\" : \"view\",\n" + - " \"client_id\" : \"" + UUID.randomUUID() + "\",\n" + - " \"query_id\" : \"" + UUID.randomUUID() + "\",\n" + - " \"message_type\" : null,\n" + - " \"message\" : null,\n" + - " \"timestamp\" : 1.7276472197111678E9,\n" + - " \"event_attributes\" : {\n" + - " \"object\" : {\n" + - " \"object_id_field\" : \"primary_ean\",\n" + - " \"object_id\" : \"0731304258193\",\n" + - " \"description\" : \"APC IT Power Distribution Module 3 Pole 5 Wire 32A IEC309 620cm power distribution unit (PDU)\"\n" + - " },\n" + - " \"position\" : {\n" + - " \"index\" : 7\n" + - " },\n" + - " \"session_id\" : \"d4ed2513-aaa9-48c1-bcb9-e936a4e903a9\"\n" + - " }" + - "}"; - - final IndexRequest indexRequest = new IndexRequest(CoecClickModel.INDEX_UBI_EVENTS) - .id(String.valueOf(x)) - .source(event, XContentType.JSON); - - bulkRequest.add(indexRequest); - - } - - // Index the clicks. - for(int x = 1; x <= numberOfClicks; x++) { - -// final String query = "{\n" + -// " \"user_query\" : \"computer\",\n" + -// " \"query_id\" : \"" + UUID.randomUUID() + "\",\n" + -// " \"message_type\" : null,\n" + -// " \"message\" : null,\n" + -// " \"timestamp\" : 1.7276472197111678E9,\n" + -// " \"event_attributes\" : {\n" + -// " \"object\" : {\n" + -// " \"object_id_field\" : \"primary_ean\",\n" + -// " \"object_id\" : \"0731304258193\",\n" + -// " \"description\" : \"APC IT Power Distribution Module 3 Pole 5 Wire 32A IEC309 620cm power distribution unit (PDU)\"\n" + -// " },\n" + -// " \"position\" : {\n" + -// " \"index\" : 7\n" + -// " },\n" + -// " \"session_id\" : \"" + UUID.randomUUID() + "\"\n" + -// " }" + -// "}"; -// -// final IndexRequest queryIndexRequest = new IndexRequest(CoecClickModel.INDEX_UBI_QUERIES) -// .id(String.valueOf(x)) -// .source(query, XContentType.JSON); -// bulkRequest.add(queryIndexRequest); - - final String event = "{\n" + - " \"action_name\" : \"click\",\n" + - " \"client_id\" : \"" + UUID.randomUUID() + "\",\n" + - " \"query_id\" : \"" + UUID.randomUUID() + "\",\n" + - " \"message_type\" : null,\n" + - " \"message\" : null,\n" + - " \"timestamp\" : 1.7276472197111678E9,\n" + - " \"event_attributes\" : {\n" + - " \"object\" : {\n" + - " \"object_id_field\" : \"primary_ean\",\n" + - " \"object_id\" : \"0731304258193\",\n" + - " \"description\" : \"APC IT Power Distribution Module 3 Pole 5 Wire 32A IEC309 620cm power distribution unit (PDU)\"\n" + - " },\n" + - " \"position\" : {\n" + - " \"index\" : 7\n" + - " },\n" + - " \"session_id\" : \"" + UUID.randomUUID() + "\"\n" + - " }" + - "}"; - - final IndexRequest eventIndexRequest = new IndexRequest(CoecClickModel.INDEX_UBI_EVENTS) - .id(String.valueOf(x)) - .source(event, XContentType.JSON); - bulkRequest.add(eventIndexRequest); - - } - - restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT); - - final OpenSearchHelper openSearchHelper = new OpenSearchHelper(restHighLevelClient); - - final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(true, 20); - final CoecClickModel coecClickModel = new CoecClickModel(coecClickModelParameters); - - final Collection judgments = coecClickModel.calculateJudgments(); - Judgment.showJudgments(judgments); - - } -// -// private XContentBuilder getResourceFile(final String fileName) { -// try (InputStream is = CoecClickModelIT.class.getResourceAsStream(fileName)) { -// ByteArrayOutputStream out = new ByteArrayOutputStream(); -// Streams.copy(is.readAllBytes(), out); -// -// final String message = out.toString(StandardCharsets.UTF_8); -// -// return XContentFactory.jsonBuilder().value(message); -// -// } catch (IOException e) { -// throw new IllegalStateException("Unable to get mapping from resource [" + fileName + "]", e); -// } -// } - -} diff --git a/opensearch-search-quality-implicit-judgments/src/test/java/org/opensearch/qef/clickmodel/coec/CoecClickModelTest.java b/opensearch-search-quality-implicit-judgments/src/test/java/org/opensearch/qef/clickmodel/coec/CoecClickModelTest.java deleted file mode 100644 index 53df640..0000000 --- a/opensearch-search-quality-implicit-judgments/src/test/java/org/opensearch/qef/clickmodel/coec/CoecClickModelTest.java +++ /dev/null @@ -1,119 +0,0 @@ -package org.opensearch.qef.clickmodel.coec; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.qef.engine.opensearch.OpenSearchHelper; -import org.opensearch.qef.model.ClickthroughRate; -import org.opensearch.qef.model.Judgment; - -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.when; - -public class CoecClickModelTest { - - private static final Logger LOGGER = LogManager.getLogger(CoecClickModelTest.class.getName()); - - @Disabled - @Test - public void calculateJudgmentForDoc1() throws IOException { - - final RestHighLevelClient restHighLevelClient = Mockito.mock(RestHighLevelClient.class); - final OpenSearchHelper openSearchHelper = Mockito.mock(OpenSearchHelper.class); - - when(openSearchHelper.getCountOfQueriesForUserQueryHavingResultInRankR(anyString(), anyString(), anyInt())).thenReturn(250); - - final CoecClickModelParameters parameters = new CoecClickModelParameters(false, 20); - final CoecClickModel coecClickModel = new CoecClickModel(parameters); - - final Map rankAggregatedClickThrough = new HashMap<>(); - rankAggregatedClickThrough.put(1, 0.450); - - final Set ctrs = new HashSet<>(); - ctrs.add(new ClickthroughRate("object_id_1", 110, 250)); - - final Map> clickthroughRates = new HashMap<>(); - clickthroughRates.put("computer", ctrs); - - final Collection< Judgment> judgments = coecClickModel.calculateCoec(rankAggregatedClickThrough, clickthroughRates); - - Judgment.showJudgments(judgments); - - Assertions.assertEquals(1, judgments.size()); - Assertions.assertEquals(0.9777777777777777, judgments.iterator().next().getJudgment(), 0.01); - - } - - @Disabled - @Test - public void calculateJudgmentForDoc2() throws IOException { - - final RestHighLevelClient restHighLevelClient = Mockito.mock(RestHighLevelClient.class); - final OpenSearchHelper openSearchHelper = Mockito.mock(OpenSearchHelper.class); - - when(openSearchHelper.getCountOfQueriesForUserQueryHavingResultInRankR(anyString(), anyString(), anyInt())).thenReturn(124); - - final CoecClickModelParameters parameters = new CoecClickModelParameters(false, 20); - final CoecClickModel coecClickModel = new CoecClickModel(parameters); - - final Map rankAggregatedClickThrough = new HashMap<>(); - rankAggregatedClickThrough.put(2, 0.175); - - final Set ctrs = new HashSet<>(); - ctrs.add(new ClickthroughRate("object_id_2", 31, 124)); - - final Map> clickthroughRates = new HashMap<>(); - clickthroughRates.put("computer", ctrs); - - final Collection< Judgment> judgments = coecClickModel.calculateCoec(rankAggregatedClickThrough, clickthroughRates); - - Judgment.showJudgments(judgments); - - Assertions.assertEquals(1, judgments.size()); - Assertions.assertEquals(1.4285714285714286, judgments.iterator().next().getJudgment(), 0.01); - - } - - @Disabled - @Test - public void calculateJudgmentForDoc3() throws IOException { - - final RestHighLevelClient restHighLevelClient = Mockito.mock(RestHighLevelClient.class); - final OpenSearchHelper openSearchHelper = Mockito.mock(OpenSearchHelper.class); - - when(openSearchHelper.getCountOfQueriesForUserQueryHavingResultInRankR(anyString(), anyString(), anyInt())).thenReturn(240); - - final CoecClickModelParameters parameters = new CoecClickModelParameters(false, 20); - final CoecClickModel coecClickModel = new CoecClickModel(parameters); - - final Map rankAggregatedClickThrough = new HashMap<>(); - rankAggregatedClickThrough.put(3, 0.075); - - final Set ctrs = new HashSet<>(); - ctrs.add(new ClickthroughRate("object_id_3", 30, 240)); - - final Map> clickthroughRates = new HashMap<>(); - clickthroughRates.put("computer", ctrs); - - final Collection< Judgment> judgments = coecClickModel.calculateCoec(rankAggregatedClickThrough, clickthroughRates); - - Judgment.showJudgments(judgments); - - Assertions.assertEquals(1, judgments.size()); - Assertions.assertEquals(1.6666666666666667, judgments.iterator().next().getJudgment(), 0.01); - - } - -} diff --git a/opensearch-search-quality-implicit-judgments/src/test/java/org/opensearch/qef/model/ubi/UbiEventTest.java b/opensearch-search-quality-implicit-judgments/src/test/java/org/opensearch/qef/model/ubi/UbiEventTest.java deleted file mode 100644 index 866ccf7..0000000 --- a/opensearch-search-quality-implicit-judgments/src/test/java/org/opensearch/qef/model/ubi/UbiEventTest.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.opensearch.qef.model.ubi; - -public class UbiEventTest { - -} diff --git a/opensearch-search-quality-implicit-judgments/src/test/resources/log4j2.xml b/opensearch-search-quality-implicit-judgments/src/test/resources/log4j2.xml deleted file mode 100644 index eea5a4c..0000000 --- a/opensearch-search-quality-implicit-judgments/src/test/resources/log4j2.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/opensearch-search-quality-implicit-judgments/src/test/resources/ubi_event.json b/opensearch-search-quality-implicit-judgments/src/test/resources/ubi_event.json deleted file mode 100644 index d68e4e6..0000000 --- a/opensearch-search-quality-implicit-judgments/src/test/resources/ubi_event.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "_index" : "ubi_events", - "_id" : "520ff171-e6b4-4900-9c63-dba48a9753f7", - "_score" : 1.0, - "_ignored" : [ - "timestamp" - ], - "_source" : { - "action_name" : "click", - "client_id" : "2ce5eece-53cf-4b0c-9c55-bbeb57ad8642", - "query_id" : "efbeb66a-5b6b-48bd-89a9-33b171f95b2b", - "message_type" : null, - "message" : null, - "timestamp" : 1.7276472197111678E9, - "event_attributes" : { - "object" : { - "object_id_field" : "primary_ean", - "object_id" : "0731304258193", - "description" : "APC IT Power Distribution Module 3 Pole 5 Wire 32A IEC309 620cm power distribution unit (PDU)" - }, - "position" : { - "index" : 7 - }, - "session_id" : "d4ed2513-aaa9-48c1-bcb9-e936a4e903a9" - } - } -} \ No newline at end of file