diff --git a/opensearch-search-quality-evaluation-framework/build.gradle b/opensearch-search-quality-evaluation-framework/build.gradle index 069bf34..08bdcbc 100644 --- a/opensearch-search-quality-evaluation-framework/build.gradle +++ b/opensearch-search-quality-evaluation-framework/build.gradle @@ -6,12 +6,6 @@ apply plugin: 'java' apply plugin: 'idea' -ext { - projectSubstitutions = [:] - licenseFile = rootProject.file('LICENSE.txt') - noticeFile = rootProject.file('NOTICE.txt') -} - test { include "**/Test*.class" include "**/*Test.class" @@ -29,16 +23,11 @@ buildscript { mavenCentral() maven { url "https://plugins.gradle.org/m2/" } } - - dependencies { - classpath "org.opensearch.gradle:build-tools:${opensearchVersion}" - } } repositories { mavenLocal() mavenCentral() - maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } } dependencies { diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobParameter.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobParameter.java deleted file mode 100644 index 2ea5379..0000000 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobParameter.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.eval; - -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.jobscheduler.spi.ScheduledJobParameter; -import org.opensearch.jobscheduler.spi.schedule.Schedule; - -import java.io.IOException; -import java.time.Instant; - -public class SearchQualityEvaluationJobParameter implements ScheduledJobParameter { - - /** - * The name of the parameter for providing a name for the scheduled job. - */ - public static final String NAME_FIELD = "name"; - - /** - * The name of the parameter for creating a job as enabled or disabled. - */ - public static final String ENABLED_FILED = "enabled"; - - /** - * The name of the parameter for specifying when the job was last updated. - */ - public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; - - /** - * The name of the parameter for specifying a readable time for when the job was last updated. - */ - public static final String LAST_UPDATE_TIME_FIELD_READABLE = "last_update_time_field"; - public static final String SCHEDULE_FIELD = "schedule"; - public static final String ENABLED_TIME_FILED = "enabled_time"; - public static final String ENABLED_TIME_FILED_READABLE = "enabled_time_field"; - public static final String LOCK_DURATION_SECONDS = "lock_duration_seconds"; - public static final String JITTER = "jitter"; - - /** - * The name of the parameter that allows for specifying the type of click model to use. - */ - public static final String CLICK_MODEL = "click_model"; - - /** - * The name of the parameter that allows for setting a max rank value to use during judgment generation. - */ - public static final String MAX_RANK = "max_rank"; - - // Properties from ScheduledJobParameter. - private String jobName; - private Instant lastUpdateTime; - private Instant enabledTime; - private boolean enabled; - private Schedule schedule; - private Long lockDurationSeconds; - private Double jitter; - - // Custom properties. - private String clickModel; - private int maxRank; - - public SearchQualityEvaluationJobParameter() { - - } - - public SearchQualityEvaluationJobParameter(final String name, final Schedule schedule, - final Long lockDurationSeconds, final Double jitter, - final String clickModel, final int maxRank) { - this.jobName = name; - this.schedule = schedule; - this.enabled = true; - this.lockDurationSeconds = lockDurationSeconds; - this.jitter = jitter; - - final Instant now = Instant.now(); - this.enabledTime = now; - this.lastUpdateTime = now; - - // Custom properties. - this.clickModel = clickModel; - this.maxRank = maxRank; - - } - - @Override - public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { - - builder.startObject(); - - builder - .field(NAME_FIELD, this.jobName) - .field(ENABLED_FILED, this.enabled) - .field(SCHEDULE_FIELD, this.schedule) - .field(CLICK_MODEL, this.clickModel) - .field(MAX_RANK, this.maxRank); - - if (this.enabledTime != null) { - builder.timeField(ENABLED_TIME_FILED, ENABLED_TIME_FILED_READABLE, this.enabledTime.toEpochMilli()); - } - - if (this.lastUpdateTime != null) { - builder.timeField(LAST_UPDATE_TIME_FIELD, LAST_UPDATE_TIME_FIELD_READABLE, this.lastUpdateTime.toEpochMilli()); - } - - if (this.lockDurationSeconds != null) { - builder.field(LOCK_DURATION_SECONDS, this.lockDurationSeconds); - } - - if (this.jitter != null) { - builder.field(JITTER, this.jitter); - } - - builder.endObject(); - - return builder; - - } - - @Override - public String getName() { - return this.jobName; - } - - @Override - public Instant getLastUpdateTime() { - return this.lastUpdateTime; - } - - @Override - public Instant getEnabledTime() { - return this.enabledTime; - } - - @Override - public Schedule getSchedule() { - return this.schedule; - } - - @Override - public boolean isEnabled() { - return this.enabled; - } - - @Override - public Long getLockDurationSeconds() { - return this.lockDurationSeconds; - } - - @Override - public Double getJitter() { - return jitter; - } - - /** - * Sets the name of the job. - * @param jobName The name of the job. - */ - public void setJobName(String jobName) { - this.jobName = jobName; - } - - /** - * Sets when the job was last updated. - * @param lastUpdateTime An {@link Instant} of when the job was last updated. - */ - public void setLastUpdateTime(Instant lastUpdateTime) { - this.lastUpdateTime = lastUpdateTime; - } - - /** - * Sets when the job was enabled. - * @param enabledTime An {@link Instant} of when the job was enabled. - */ - public void setEnabledTime(Instant enabledTime) { - this.enabledTime = enabledTime; - } - - /** - * Sets whether the job is enabled. - * @param enabled A boolean representing whether the job is enabled. - */ - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - /** - * Sets the schedule for the job. - * @param schedule A {@link Schedule} for the job. - */ - public void setSchedule(Schedule schedule) { - this.schedule = schedule; - } - - /** - * Sets the lock duration for the cluster when running the job. - * @param lockDurationSeconds The lock duration in seconds. - */ - public void setLockDurationSeconds(Long lockDurationSeconds) { - this.lockDurationSeconds = lockDurationSeconds; - } - - /** - * Sets the jitter for the job. - * @param jitter The jitter for the job. - */ - public void setJitter(Double jitter) { - this.jitter = jitter; - } - - /** - * Gets the type of click model to use for implicit judgment generation. - * @return The type of click model to use for implicit judgment generation. - */ - public String getClickModel() { - return clickModel; - } - - /** - * Sets the click model type to use for implicit judgment generation. - * @param clickModel The click model type to use for implicit judgment generation. - */ - public void setClickModel(String clickModel) { - this.clickModel = clickModel; - } - - /** - * Gets the max rank to use when generating implicit judgments. - * @return The max rank to use when generating implicit judgments. - */ - public int getMaxRank() { - return maxRank; - } - - /** - * Sets the max rank to use when generating implicit judgments. - * @param maxRank The max rank to use when generating implicit judgments. - */ - public void setMaxRank(int maxRank) { - this.maxRank = maxRank; - } - -} diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java deleted file mode 100644 index 442ae4c..0000000 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.eval; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.action.index.IndexResponse; -import org.opensearch.action.support.WriteRequest; -import org.opensearch.client.Client; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.core.action.ActionListener; -import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel; -import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModelParameters; -import org.opensearch.jobscheduler.spi.JobExecutionContext; -import org.opensearch.jobscheduler.spi.ScheduledJobParameter; -import org.opensearch.jobscheduler.spi.ScheduledJobRunner; -import org.opensearch.jobscheduler.spi.utils.LockService; -import org.opensearch.threadpool.ThreadPool; - -import java.util.HashMap; -import java.util.Map; - -/** - * Job runner for scheduled implicit judgments jobs. - */ -public class SearchQualityEvaluationJobRunner implements ScheduledJobRunner { - - private static final Logger LOGGER = LogManager.getLogger(SearchQualityEvaluationJobRunner.class); - - private static SearchQualityEvaluationJobRunner INSTANCE; - - /** - * Gets a singleton instance of this class. - * @return A {@link SearchQualityEvaluationJobRunner}. - */ - public static SearchQualityEvaluationJobRunner getJobRunnerInstance() { - - LOGGER.info("Getting job runner instance"); - - if (INSTANCE != null) { - return INSTANCE; - } - - synchronized (SearchQualityEvaluationJobRunner.class) { - if (INSTANCE == null) { - INSTANCE = new SearchQualityEvaluationJobRunner(); - } - return INSTANCE; - } - - } - - private ClusterService clusterService; - private ThreadPool threadPool; - private Client client; - - private SearchQualityEvaluationJobRunner() { - - } - - public void setClusterService(ClusterService clusterService) { - this.clusterService = clusterService; - } - - public void setThreadPool(ThreadPool threadPool) { - this.threadPool = threadPool; - } - - public void setClient(Client client) { - this.client = client; - } - - @Override - public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionContext context) { - - if(!(jobParameter instanceof SearchQualityEvaluationJobParameter)) { - throw new IllegalStateException( - "Job parameter is not instance of SampleJobParameter, type: " + jobParameter.getClass().getCanonicalName() - ); - } - - if(this.clusterService == null) { - throw new IllegalStateException("ClusterService is not initialized."); - } - - if(this.threadPool == null) { - throw new IllegalStateException("ThreadPool is not initialized."); - } - - final LockService lockService = context.getLockService(); - - final Runnable runnable = () -> { - - if (jobParameter.getLockDurationSeconds() != null) { - - lockService.acquireLock(jobParameter, context, ActionListener.wrap(lock -> { - - if (lock == null) { - return; - } - - final SearchQualityEvaluationJobParameter searchQualityEvaluationJobParameter = (SearchQualityEvaluationJobParameter) jobParameter; - - final long startTime = System.currentTimeMillis(); - final String judgmentsId; - - if("coec".equalsIgnoreCase(searchQualityEvaluationJobParameter.getClickModel())) { - - LOGGER.info("Beginning implicit judgment generation using clicks-over-expected-clicks."); - final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(searchQualityEvaluationJobParameter.getMaxRank()); - final CoecClickModel coecClickModel = new CoecClickModel(client, coecClickModelParameters); - - judgmentsId = coecClickModel.calculateJudgments(); - - } else { - - // Invalid click model. - throw new IllegalArgumentException("Invalid click model: " + searchQualityEvaluationJobParameter.getClickModel()); - - } - - final long elapsedTime = System.currentTimeMillis() - startTime; - LOGGER.info("Implicit judgment generation completed in {} ms", elapsedTime); - - final Map job = new HashMap<>(); - job.put("name", searchQualityEvaluationJobParameter.getName()); - job.put("click_model", searchQualityEvaluationJobParameter.getClickModel()); - job.put("started", startTime); - job.put("duration", elapsedTime); - job.put("judgments", judgmentsId); - job.put("invocation", "scheduled"); - job.put("max_rank", searchQualityEvaluationJobParameter.getMaxRank()); - - final IndexRequest indexRequest = new IndexRequest() - .index(SearchQualityEvaluationPlugin.COMPLETED_JOBS_INDEX_NAME) - .id(judgmentsId) - .source(job) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - - client.index(indexRequest, new ActionListener<>() { - @Override - public void onResponse(IndexResponse indexResponse) { - LOGGER.info("Successfully indexed implicit judgments {}", judgmentsId); - } - - @Override - public void onFailure(Exception ex) { - LOGGER.error("Unable to index implicit judgments", ex); - } - }); - - }, exception -> { throw new IllegalStateException("Failed to acquire lock."); })); - } - - }; - - threadPool.generic().submit(runnable); - - } - -} diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java index f5192a7..d6c3a6a 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java @@ -16,7 +16,7 @@ import org.opensearch.action.search.SearchScrollRequest; import org.opensearch.client.Client; import org.opensearch.client.Requests; -import org.opensearch.common.unit.TimeValue; +import org.opensearch.eval.Constants; import org.opensearch.eval.judgments.clickmodel.ClickModel; import org.opensearch.eval.judgments.model.ClickthroughRate; import org.opensearch.eval.judgments.model.Judgment; @@ -216,7 +216,7 @@ private Map> getClickthroughRate() throws Exceptio final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(10L)); final SearchRequest searchRequest = Requests - .searchRequest(SearchQualityEvaluationPlugin.UBI_EVENTS_INDEX_NAME) + .searchRequest(Constants.UBI_EVENTS_INDEX_NAME) .source(searchSourceBuilder) .scroll(scroll); @@ -313,7 +313,7 @@ public Map getRankAggregatedClickThrough() throws Exception { .from(0) .size(0); - final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.UBI_EVENTS_INDEX_NAME).source(searchSourceBuilder); + final SearchRequest searchRequest = new SearchRequest(Constants.UBI_EVENTS_INDEX_NAME).source(searchSourceBuilder); final SearchResponse searchResponse = client.search(searchRequest).get(); final Map clickCounts = new HashMap<>(); diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java index 3c391b3..c6ea7d7 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java @@ -17,7 +17,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; -import org.opensearch.core.action.ActionListener; +import org.opensearch.eval.Constants; import org.opensearch.eval.judgments.model.ClickthroughRate; import org.opensearch.eval.judgments.model.Judgment; import org.opensearch.eval.judgments.model.ubi.query.UbiQuery; @@ -28,7 +28,6 @@ import org.opensearch.search.builder.SearchSourceBuilder; import java.io.IOException; -import java.security.AccessController; import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Collection; @@ -37,9 +36,6 @@ import java.util.Set; import java.util.UUID; -import static org.opensearch.eval.SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME; -import static org.opensearch.eval.SearchQualityEvaluationPlugin.UBI_EVENTS_INDEX_NAME; -import static org.opensearch.eval.SearchQualityEvaluationPlugin.UBI_QUERIES_INDEX_NAME; import static org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel.INDEX_QUERY_DOC_CTR; import static org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel.INDEX_RANK_AGGREGATED_CTR; @@ -110,7 +106,7 @@ public UbiQuery getQueryFromQueryId(final String queryId) throws Exception { searchSourceBuilder.from(0); searchSourceBuilder.size(1); - final String[] indexes = {UBI_QUERIES_INDEX_NAME}; + final String[] indexes = {Constants.UBI_QUERIES_INDEX_NAME}; final SearchRequest searchRequest = new SearchRequest(indexes, searchSourceBuilder); final SearchResponse response = client.search(searchRequest).get(); @@ -119,7 +115,7 @@ public UbiQuery getQueryFromQueryId(final String queryId) throws Exception { if(response.getHits().getHits() != null & response.getHits().getHits().length > 0) { final SearchHit hit = response.getHits().getHits()[0]; - return AccessController.doPrivileged((PrivilegedAction) () -> gson.fromJson(hit.getSourceAsString(), UbiQuery.class)); + return gson.fromJson(hit.getSourceAsString(), UbiQuery.class); } else { @@ -138,7 +134,7 @@ private Collection getQueryIdsHavingUserQuery(final String userQuery) th final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(qb); - final String[] indexes = {UBI_QUERIES_INDEX_NAME}; + final String[] indexes = {Constants.UBI_QUERIES_INDEX_NAME}; final SearchRequest searchRequest = new SearchRequest(indexes, searchSourceBuilder); final SearchResponse response = client.search(searchRequest).get(); @@ -198,7 +194,7 @@ public long getCountOfQueriesForUserQueryHavingResultInRankR(final String userQu searchSourceBuilder.trackTotalHits(true); searchSourceBuilder.size(0); - final String[] indexes = {UBI_EVENTS_INDEX_NAME}; + final String[] indexes = {Constants.UBI_EVENTS_INDEX_NAME}; final SearchRequest searchRequest = new SearchRequest(indexes, searchSourceBuilder); final SearchResponse response = client.search(searchRequest).get(); @@ -324,7 +320,7 @@ public String indexJudgments(final Collection judgments) throws Except j.put("judgments_id", judgmentsId); j.put("timestamp", timestamp); - final IndexRequest indexRequest = new IndexRequest(JUDGMENTS_INDEX_NAME) + final IndexRequest indexRequest = new IndexRequest(Constants.JUDGMENTS_INDEX_NAME) .id(UUID.randomUUID().toString()) .source(j); diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index 52c3322..356e72b 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -10,7 +10,12 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.client.Client; +import org.opensearch.client.OpenSearchClient; +import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.eval.Constants; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.builder.SearchSourceBuilder; import java.util.ArrayList; import java.util.Collection; @@ -25,9 +30,9 @@ public abstract class AbstractQuerySetRunner { private static final Logger LOGGER = LogManager.getLogger(AbstractQuerySetRunner.class); - protected final Client client; + protected final OpenSearchClient client; - public AbstractQuerySetRunner(final Client client) { + public AbstractQuerySetRunner(final OpenSearchClient client) { this.client = client; } @@ -71,7 +76,7 @@ public final Collection> getQuerySet(final String querySetId) sourceBuilder.from(0); sourceBuilder.size(1); - final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.QUERY_SETS_INDEX_NAME).source(sourceBuilder); + final SearchRequest searchRequest = new SearchRequest(Constants.QUERY_SETS_INDEX_NAME).source(sourceBuilder); // TODO: Don't use .get() final SearchResponse searchResponse = client.search(searchRequest).get(); @@ -120,7 +125,7 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin final String[] excludeFields = new String[] {}; searchSourceBuilder.fetchSource(includeFields, excludeFields); - final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME).source(searchSourceBuilder); + final SearchRequest searchRequest = new SearchRequest(Constants.JUDGMENTS_INDEX_NAME).source(searchSourceBuilder); Double judgment = Double.NaN;