From 17dcece0f48ad8e0a0da663e1fb839d34accd96b Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 17 Dec 2024 08:49:27 -0500 Subject: [PATCH] Working on standalone implementation. Signed-off-by: jzonthemtn --- .../build.gradle | 1 + .../java/org/opensearch/eval/Constants.java | 45 ++++ .../eval/SearchQualityEvaluationPlugin.java | 213 ------------------ .../clickmodel/coec/CoecClickModel.java | 1 - .../eval/runners/AbstractQuerySetRunner.java | 6 - .../runners/OpenSearchQuerySetRunner.java | 1 - .../eval/samplers/AbstractQuerySampler.java | 4 +- .../eval/samplers/AllQueriesQuerySampler.java | 4 +- ...roportionalToSizeAbstractQuerySampler.java | 4 +- 9 files changed, 52 insertions(+), 227 deletions(-) create mode 100644 opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/Constants.java delete mode 100644 opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java diff --git a/opensearch-search-quality-evaluation-framework/build.gradle b/opensearch-search-quality-evaluation-framework/build.gradle index bdd0586..069bf34 100644 --- a/opensearch-search-quality-evaluation-framework/build.gradle +++ b/opensearch-search-quality-evaluation-framework/build.gradle @@ -42,6 +42,7 @@ repositories { } dependencies { + implementation 'org.opensearch.client:opensearch-java:2.19.0' implementation 'org.apache.logging.log4j:log4j-core:2.24.3' implementation 'com.fasterxml.jackson.core:jackson-annotations:2.18.2' implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.2' diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/Constants.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/Constants.java new file mode 100644 index 0000000..6d0eccb --- /dev/null +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/Constants.java @@ -0,0 +1,45 @@ +package org.opensearch.eval; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class Constants { + + private static final Logger LOGGER = LogManager.getLogger(Constants.class); + + /** + * The name of the UBI index containing the queries. This should not be changed. + */ + public static final String UBI_QUERIES_INDEX_NAME = "ubi_queries"; + + /** + * The name of the UBI index containing the events. This should not be changed. + */ + public static final String UBI_EVENTS_INDEX_NAME = "ubi_events"; + + /** + * The name of the index to store the scheduled jobs to create implicit judgments. + */ + public static final String SCHEDULED_JOBS_INDEX_NAME = "search_quality_eval_scheduled_jobs"; + + /** + * The name of the index to store the completed jobs to create implicit judgments. + */ + public static final String COMPLETED_JOBS_INDEX_NAME = "search_quality_eval_completed_jobs"; + + /** + * The name of the index that stores the query sets. + */ + public static final String QUERY_SETS_INDEX_NAME = "search_quality_eval_query_sets"; + + /** + * The name of the index that stores the metrics for the dashboard. + */ + public static final String DASHBOARD_METRICS_INDEX_NAME = "sqe_metrics_sample_data"; + + /** + * The name of the index that stores the implicit judgments. + */ + public static final String JUDGMENTS_INDEX_NAME = "judgments"; + +} diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java deleted file mode 100644 index 6a7b581..0000000 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.eval; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.client.Client; -import org.opensearch.cluster.metadata.IndexNameExpressionResolver; -import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.IndexScopedSettings; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.settings.SettingsFilter; -import org.opensearch.core.common.io.stream.NamedWriteableRegistry; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.core.xcontent.XContentParserUtils; -import org.opensearch.env.Environment; -import org.opensearch.env.NodeEnvironment; -import org.opensearch.jobscheduler.spi.JobSchedulerExtension; -import org.opensearch.jobscheduler.spi.ScheduledJobParser; -import org.opensearch.jobscheduler.spi.ScheduledJobRunner; -import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; -import org.opensearch.plugins.ActionPlugin; -import org.opensearch.plugins.Plugin; -import org.opensearch.repositories.RepositoriesService; -import org.opensearch.rest.RestController; -import org.opensearch.rest.RestHandler; -import org.opensearch.script.ScriptService; -import org.opensearch.threadpool.ThreadPool; -import org.opensearch.watcher.ResourceWatcherService; - -import java.io.IOException; -import java.time.Instant; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.function.Supplier; - -/** - * Main class for the Search Quality Evaluation plugin. - */ -public class SearchQualityEvaluationPlugin extends Plugin implements ActionPlugin, JobSchedulerExtension { - - private static final Logger LOGGER = LogManager.getLogger(SearchQualityEvaluationPlugin.class); - - /** - * The name of the UBI index containing the queries. This should not be changed. - */ - public static final String UBI_QUERIES_INDEX_NAME = "ubi_queries"; - - /** - * The name of the UBI index containing the events. This should not be changed. - */ - public static final String UBI_EVENTS_INDEX_NAME = "ubi_events"; - - /** - * The name of the index to store the scheduled jobs to create implicit judgments. - */ - public static final String SCHEDULED_JOBS_INDEX_NAME = "search_quality_eval_scheduled_jobs"; - - /** - * The name of the index to store the completed jobs to create implicit judgments. - */ - public static final String COMPLETED_JOBS_INDEX_NAME = "search_quality_eval_completed_jobs"; - - /** - * The name of the index that stores the query sets. - */ - public static final String QUERY_SETS_INDEX_NAME = "search_quality_eval_query_sets"; - - /** - * The name of the index that stores the metrics for the dashboard. - */ - public static final String DASHBOARD_METRICS_INDEX_NAME = "sqe_metrics_sample_data"; - - /** - * The name of the index that stores the implicit judgments. - */ - public static final String JUDGMENTS_INDEX_NAME = "judgments"; - - @Override - public Collection createComponents( - final Client client, - final ClusterService clusterService, - final ThreadPool threadPool, - final ResourceWatcherService resourceWatcherService, - final ScriptService scriptService, - final NamedXContentRegistry xContentRegistry, - final Environment environment, - final NodeEnvironment nodeEnvironment, - final NamedWriteableRegistry namedWriteableRegistry, - final IndexNameExpressionResolver indexNameExpressionResolver, - final Supplier repositoriesServiceSupplier - ) { - - LOGGER.info("Creating search evaluation framework components"); - final SearchQualityEvaluationJobRunner jobRunner = SearchQualityEvaluationJobRunner.getJobRunnerInstance(); - jobRunner.setClusterService(clusterService); - jobRunner.setThreadPool(threadPool); - jobRunner.setClient(client); - - return Collections.emptyList(); - - } - - @Override - public String getJobType() { - return "scheduler_search_quality_eval"; - } - - @Override - public String getJobIndex() { - LOGGER.info("Getting job index name"); - return SCHEDULED_JOBS_INDEX_NAME; - } - - @Override - public ScheduledJobRunner getJobRunner() { - LOGGER.info("Creating job runner"); - return SearchQualityEvaluationJobRunner.getJobRunnerInstance(); - } - - @Override - public ScheduledJobParser getJobParser() { - - return (parser, id, jobDocVersion) -> { - - final SearchQualityEvaluationJobParameter jobParameter = new SearchQualityEvaluationJobParameter(); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); - - while (!parser.nextToken().equals(XContentParser.Token.END_OBJECT)) { - - final String fieldName = parser.currentName(); - - parser.nextToken(); - - switch (fieldName) { - case SearchQualityEvaluationJobParameter.NAME_FIELD: - jobParameter.setJobName(parser.text()); - break; - case SearchQualityEvaluationJobParameter.ENABLED_FILED: - jobParameter.setEnabled(parser.booleanValue()); - break; - case SearchQualityEvaluationJobParameter.ENABLED_TIME_FILED: - jobParameter.setEnabledTime(parseInstantValue(parser)); - break; - case SearchQualityEvaluationJobParameter.LAST_UPDATE_TIME_FIELD: - jobParameter.setLastUpdateTime(parseInstantValue(parser)); - break; - case SearchQualityEvaluationJobParameter.SCHEDULE_FIELD: - jobParameter.setSchedule(ScheduleParser.parse(parser)); - break; - case SearchQualityEvaluationJobParameter.LOCK_DURATION_SECONDS: - jobParameter.setLockDurationSeconds(parser.longValue()); - break; - case SearchQualityEvaluationJobParameter.JITTER: - jobParameter.setJitter(parser.doubleValue()); - break; - case SearchQualityEvaluationJobParameter.CLICK_MODEL: - jobParameter.setClickModel(parser.text()); - break; - case SearchQualityEvaluationJobParameter.MAX_RANK: - jobParameter.setMaxRank(parser.intValue()); - break; - default: - XContentParserUtils.throwUnknownToken(parser.currentToken(), parser.getTokenLocation()); - } - - } - - return jobParameter; - - }; - - } - - private Instant parseInstantValue(final XContentParser parser) throws IOException { - - if (XContentParser.Token.VALUE_NULL.equals(parser.currentToken())) { - return null; - } - - if (parser.currentToken().isValue()) { - return Instant.ofEpochMilli(parser.longValue()); - } - - XContentParserUtils.throwUnknownToken(parser.currentToken(), parser.getTokenLocation()); - return null; - - } - - @Override - public List getRestHandlers( - final Settings settings, - final RestController restController, - final ClusterSettings clusterSettings, - final IndexScopedSettings indexScopedSettings, - final SettingsFilter settingsFilter, - final IndexNameExpressionResolver indexNameExpressionResolver, - final Supplier nodesInCluster - ) { - return Collections.singletonList(new SearchQualityEvaluationRestHandler()); - } - -} diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java index f2e8aa8..f5192a7 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java @@ -17,7 +17,6 @@ import org.opensearch.client.Client; import org.opensearch.client.Requests; import org.opensearch.common.unit.TimeValue; -import org.opensearch.eval.SearchQualityEvaluationPlugin; import org.opensearch.eval.judgments.clickmodel.ClickModel; import org.opensearch.eval.judgments.model.ClickthroughRate; import org.opensearch.eval.judgments.model.Judgment; diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index 7ca0ad6..52c3322 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -10,13 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; -import org.opensearch.eval.SearchQualityEvaluationPlugin; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.search.builder.SearchSourceBuilder; import java.util.ArrayList; import java.util.Collection; diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index a1f0c4f..52f162b 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -21,7 +21,6 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.core.action.ActionListener; -import org.opensearch.eval.SearchQualityEvaluationPlugin; import org.opensearch.eval.metrics.DcgSearchMetric; import org.opensearch.eval.metrics.NdcgSearchMetric; import org.opensearch.eval.metrics.PrecisionSearchMetric; diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java index 3c70f0a..4246780 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java @@ -15,7 +15,7 @@ import org.opensearch.action.support.WriteRequest; import org.opensearch.client.node.NodeClient; import org.opensearch.core.action.ActionListener; -import org.opensearch.eval.SearchQualityEvaluationPlugin; +import org.opensearch.eval.Constants; import org.opensearch.eval.utils.TimeUtils; import java.util.ArrayList; @@ -73,7 +73,7 @@ protected String indexQuerySet(final NodeClient client, final String name, final final String querySetId = UUID.randomUUID().toString(); // TODO: Create a mapping for the query set index. - final IndexRequest indexRequest = new IndexRequest().index(SearchQualityEvaluationPlugin.QUERY_SETS_INDEX_NAME) + final IndexRequest indexRequest = new IndexRequest().index(Constants.QUERY_SETS_INDEX_NAME) .id(querySetId) .source(querySet) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AllQueriesQuerySampler.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AllQueriesQuerySampler.java index 263d70a..e9d4401 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AllQueriesQuerySampler.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AllQueriesQuerySampler.java @@ -11,7 +11,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.node.NodeClient; -import org.opensearch.eval.SearchQualityEvaluationPlugin; +import org.opensearch.eval.Constants; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; @@ -53,7 +53,7 @@ public String sample() throws Exception { searchSourceBuilder.from(0); searchSourceBuilder.size(parameters.getQuerySetSize()); - final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.UBI_QUERIES_INDEX_NAME).source(searchSourceBuilder); + final SearchRequest searchRequest = new SearchRequest(Constants.UBI_QUERIES_INDEX_NAME).source(searchSourceBuilder); // TODO: Don't use .get() final SearchResponse searchResponse = client.search(searchRequest).get(); diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java index 79f2c7c..112494c 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java @@ -15,7 +15,7 @@ import org.opensearch.action.search.SearchScrollRequest; import org.opensearch.client.node.NodeClient; import org.opensearch.common.unit.TimeValue; -import org.opensearch.eval.SearchQualityEvaluationPlugin; +import org.opensearch.eval.Constants; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.Scroll; import org.opensearch.search.SearchHit; @@ -69,7 +69,7 @@ public String sample() throws Exception { searchSourceBuilder.size(10000); final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(10L)); - final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.UBI_QUERIES_INDEX_NAME); + final SearchRequest searchRequest = new SearchRequest(Constants.UBI_QUERIES_INDEX_NAME); searchRequest.scroll(scroll); searchRequest.source(searchSourceBuilder);