diff --git a/opensearch-search-quality-evaluation-plugin/Dockerfile b/opensearch-search-quality-evaluation-plugin/Dockerfile index 2101888..786d077 100644 --- a/opensearch-search-quality-evaluation-plugin/Dockerfile +++ b/opensearch-search-quality-evaluation-plugin/Dockerfile @@ -1,4 +1,6 @@ FROM opensearchproject/opensearch:2.17.1 +RUN /usr/share/opensearch/bin/opensearch-plugin install --batch https://github.com/opensearch-project/user-behavior-insights/releases/download/2.17.1.0/opensearch-ubi-2.17.1.0.zip + ADD ./build/distributions/search-quality-evaluation-plugin-2.17.1.0.zip /tmp/search-quality-evaluation-plugin.zip RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/search-quality-evaluation-plugin.zip diff --git a/opensearch-search-quality-evaluation-plugin/README.md b/opensearch-search-quality-evaluation-plugin/README.md index e9a3aec..4a5bd7d 100644 --- a/opensearch-search-quality-evaluation-plugin/README.md +++ b/opensearch-search-quality-evaluation-plugin/README.md @@ -38,5 +38,11 @@ curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?id See the created job: ``` -curl -s http://localhost:9200/.scheduler_search_quality_eval/_search +curl -s http://localhost:9200/search_quality_eval_scheduler/_search +``` + +See the judgments: + +``` +curl -s http://localhost:9200/judgments/ ``` \ No newline at end of file diff --git a/opensearch-search-quality-evaluation-plugin/create-schedule.sh b/opensearch-search-quality-evaluation-plugin/create-schedule.sh index 719a944..72a79ed 100755 --- a/opensearch-search-quality-evaluation-plugin/create-schedule.sh +++ b/opensearch-search-quality-evaluation-plugin/create-schedule.sh @@ -1,5 +1,6 @@ #!/bin/bash -e -curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?id=1&index=ubi&job_name=test2&interval=1" | jq +curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/schedule?id=1&click_model=coec&max_rank=20&job_name=test2&interval=10" | jq -curl -s "http://localhost:9200/.scheduler_search_quality_eval/_search" | jq +echo "Scheduled jobs:" +curl -s "http://localhost:9200/search_quality_eval_scheduler/_search" | jq diff --git a/opensearch-search-quality-evaluation-plugin/docker-compose.yaml b/opensearch-search-quality-evaluation-plugin/docker-compose.yaml index 9b1cdd3..9744165 100644 --- a/opensearch-search-quality-evaluation-plugin/docker-compose.yaml +++ b/opensearch-search-quality-evaluation-plugin/docker-compose.yaml @@ -1,8 +1,8 @@ services: - opensearch: + opensearch_sef: build: . - container_name: opensearch + container_name: opensearch_sef environment: discovery.type: single-node node.name: opensearch @@ -21,24 +21,24 @@ services: - "9600:9600" networks: - opensearch-net - volumes: - - opensearch-data1:/usr/share/opensearch/data +# volumes: +# - opensearch-data1:/usr/share/opensearch/data -# opensearch_dashboards: +# opensearch_sef_dashboards: # image: opensearchproject/opensearch-dashboards:2.16.0 -# container_name: opensearch_dashboards +# container_name: opensearch_sef_dashboards # ports: # - "5601:5601" # environment: # OPENSEARCH_HOSTS: '["http://opensearch:9200"]' # DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true" # depends_on: -# - opensearch +# - opensearch_sef # networks: # - opensearch-net -volumes: - opensearch-data1: +#volumes: +# opensearch-data1: networks: opensearch-net: diff --git a/opensearch-search-quality-evaluation-plugin/generate-judgments-now.sh b/opensearch-search-quality-evaluation-plugin/generate-judgments-now.sh new file mode 100755 index 0000000..caa7113 --- /dev/null +++ b/opensearch-search-quality-evaluation-plugin/generate-judgments-now.sh @@ -0,0 +1,3 @@ +#!/bin/bash -e + +curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=20" diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobParameter.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobParameter.java index 4cc6550..b8024e5 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobParameter.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobParameter.java @@ -27,6 +27,10 @@ public class SearchQualityEvaluationJobParameter implements ScheduledJobParamete public static final String LOCK_DURATION_SECONDS = "lock_duration_seconds"; public static final String JITTER = "jitter"; + // Custom properties. + public static final String CLICK_MODEL = "click_model"; + public static final String MAX_RANK = "max_rank"; + // Properties from ScheduledJobParameter. private String jobName; private Instant lastUpdateTime; @@ -36,15 +40,17 @@ public class SearchQualityEvaluationJobParameter implements ScheduledJobParamete private Long lockDurationSeconds; private Double jitter; - // Custom properties for this job. - private String indexToWatch; + // Custom properties. + private String clickModel; + private int maxRank; public SearchQualityEvaluationJobParameter() { } - public SearchQualityEvaluationJobParameter(final String name, final String indexToWatch, final Schedule schedule, - final Long lockDurationSeconds, final Double jitter) { + public SearchQualityEvaluationJobParameter(final String name, final Schedule schedule, + final Long lockDurationSeconds, final Double jitter, + final String clickModel, final int maxRank) { this.jobName = name; this.schedule = schedule; this.enabled = true; @@ -55,7 +61,43 @@ public SearchQualityEvaluationJobParameter(final String name, final String index this.enabledTime = now; this.lastUpdateTime = now; - this.indexToWatch = indexToWatch; + // Custom properties. + this.clickModel = clickModel; + this.maxRank = maxRank; + + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + + builder.startObject(); + + builder + .field(NAME_FIELD, this.jobName) + .field(ENABLED_FILED, this.enabled) + .field(SCHEDULE_FIELD, this.schedule) + .field(CLICK_MODEL, this.clickModel) + .field(MAX_RANK, this.maxRank); + + if (this.enabledTime != null) { + builder.timeField(ENABLED_TIME_FILED, ENABLED_TIME_FILED_READABLE, this.enabledTime.toEpochMilli()); + } + + if (this.lastUpdateTime != null) { + builder.timeField(LAST_UPDATE_TIME_FIELD, LAST_UPDATE_TIME_FIELD_READABLE, this.lastUpdateTime.toEpochMilli()); + } + + if (this.lockDurationSeconds != null) { + builder.field(LOCK_DURATION_SECONDS, this.lockDurationSeconds); + } + + if (this.jitter != null) { + builder.field(JITTER, this.jitter); + } + + builder.endObject(); + + return builder; } @@ -114,10 +156,6 @@ public void setSchedule(Schedule schedule) { this.schedule = schedule; } - public void setIndexToWatch(String indexToWatch) { - this.indexToWatch = indexToWatch; - } - public void setLockDurationSeconds(Long lockDurationSeconds) { this.lockDurationSeconds = lockDurationSeconds; } @@ -126,36 +164,20 @@ public void setJitter(Double jitter) { this.jitter = jitter; } - @Override - public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { - - builder.startObject(); - - builder - .field(NAME_FIELD, this.jobName) - .field(ENABLED_FILED, this.enabled) - .field(SCHEDULE_FIELD, this.schedule); - - if (this.enabledTime != null) { - builder.timeField(ENABLED_TIME_FILED, ENABLED_TIME_FILED_READABLE, this.enabledTime.toEpochMilli()); - } - - if (this.lastUpdateTime != null) { - builder.timeField(LAST_UPDATE_TIME_FIELD, LAST_UPDATE_TIME_FIELD_READABLE, this.lastUpdateTime.toEpochMilli()); - } - - if (this.lockDurationSeconds != null) { - builder.field(LOCK_DURATION_SECONDS, this.lockDurationSeconds); - } - - if (this.jitter != null) { - builder.field(JITTER, this.jitter); - } + public String getClickModel() { + return clickModel; + } - builder.endObject(); + public void setClickModel(String clickModel) { + this.clickModel = clickModel; + } - return builder; + public int getMaxRank() { + return maxRank; + } + public void setMaxRank(int maxRank) { + this.maxRank = maxRank; } } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java index acb1f70..31fccca 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java @@ -8,6 +8,7 @@ */ package org.opensearch.eval; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.client.Client; @@ -70,19 +71,17 @@ public void setClient(Client client) { @Override public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionContext context) { - LOGGER.info("Running custom job! = {}", jobParameter.getName()); - - if (!(jobParameter instanceof SearchQualityEvaluationJobParameter)) { + if(!(jobParameter instanceof SearchQualityEvaluationJobParameter)) { throw new IllegalStateException( "Job parameter is not instance of SampleJobParameter, type: " + jobParameter.getClass().getCanonicalName() ); } - if (this.clusterService == null) { + if(this.clusterService == null) { throw new IllegalStateException("ClusterService is not initialized."); } - if (this.threadPool == null) { + if(this.threadPool == null) { throw new IllegalStateException("ThreadPool is not initialized."); } @@ -93,20 +92,28 @@ public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionC if (jobParameter.getLockDurationSeconds() != null) { lockService.acquireLock(jobParameter, context, ActionListener.wrap(lock -> { + if (lock == null) { return; } final SearchQualityEvaluationJobParameter searchQualityEvaluationJobParameter = (SearchQualityEvaluationJobParameter) jobParameter; - LOGGER.info("Message from inside the job."); + final long startTime = System.currentTimeMillis(); + + if(StringUtils.equalsIgnoreCase(searchQualityEvaluationJobParameter.getClickModel(), "coec")) { + + LOGGER.info("Beginning implicit judgment generation using clicks-over-expected-clicks."); + final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(true, searchQualityEvaluationJobParameter.getMaxRank()); + final CoecClickModel coecClickModel = new CoecClickModel(client, coecClickModelParameters); + + coecClickModel.calculateJudgments(); + + } - final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(true, 20); - final CoecClickModel coecClickModel = new CoecClickModel(client, coecClickModelParameters); - final Collection judgments = coecClickModel.calculateJudgments(); + LOGGER.info("Implicit judgment generation completed in {} ms", System.currentTimeMillis() - startTime); - lockService.release( - lock, + lockService.release(lock, ActionListener.wrap(released -> LOGGER.info("Released lock for job {}", jobParameter.getName()), exception -> { throw new IllegalStateException("Failed to release lock."); }) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java index 0f28ed0..2236055 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java @@ -48,7 +48,7 @@ public class SearchQualityEvaluationPlugin extends Plugin implements ActionPlugi private static final Logger LOGGER = LogManager.getLogger(SearchQualityEvaluationPlugin.class); - public static final String SCHEDULED_JOBS_INDEX_NAME = ".scheduler_search_quality_eval"; + public static final String SCHEDULED_JOBS_INDEX_NAME = "search_quality_eval_scheduler"; @Override public Collection createComponents( @@ -97,8 +97,6 @@ public ScheduledJobParser getJobParser() { return (parser, id, jobDocVersion) -> { - LOGGER.info("Getting job parser"); - final SearchQualityEvaluationJobParameter jobParameter = new SearchQualityEvaluationJobParameter(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); @@ -130,6 +128,12 @@ public ScheduledJobParser getJobParser() { case SearchQualityEvaluationJobParameter.JITTER: jobParameter.setJitter(parser.doubleValue()); break; + case SearchQualityEvaluationJobParameter.CLICK_MODEL: + jobParameter.setClickModel(parser.text()); + break; + case SearchQualityEvaluationJobParameter.MAX_RANK: + jobParameter.setMaxRank(parser.intValue()); + break; default: XContentParserUtils.throwUnknownToken(parser.currentToken(), parser.getTokenLocation()); } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java index 87b78eb..a84608a 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java @@ -8,6 +8,7 @@ */ package org.opensearch.eval; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.delete.DeleteRequest; @@ -19,6 +20,8 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; +import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel; +import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModelParameters; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; @@ -29,107 +32,178 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; +import java.util.UUID; public class SearchQualityEvaluationRestHandler extends BaseRestHandler { private static final Logger LOGGER = LogManager.getLogger(SearchQualityEvaluationRestHandler.class); /** - * URL for the implicit judgment generation. + * URL for the implicit judgment scheduling. */ - public static final String SEARCH_QUALITY_EVAL_JUDGMENTS_URL = "/_plugins/search_quality_eval/judgments"; + public static final String SCHEDULING_URL = "/_plugins/search_quality_eval/schedule"; + + /** + * URL for on-demand implicit judgment generation. + */ + public static final String IMPLICIT_JUDGMENTS_URL = "/_plugins/search_quality_eval/judgments"; + @Override public String getName() { - return "Search Quality Evaluation"; + return "Search Quality Evaluation Framework"; } @Override public List routes() { return List.of( - new Route(RestRequest.Method.POST, SEARCH_QUALITY_EVAL_JUDGMENTS_URL), - new Route(RestRequest.Method.DELETE, SEARCH_QUALITY_EVAL_JUDGMENTS_URL)); + new Route(RestRequest.Method.POST, IMPLICIT_JUDGMENTS_URL), + new Route(RestRequest.Method.POST, SCHEDULING_URL), + new Route(RestRequest.Method.DELETE, SCHEDULING_URL)); } @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - if (request.method().equals(RestRequest.Method.POST)) { + if(StringUtils.equalsIgnoreCase(request.path(), IMPLICIT_JUDGMENTS_URL)) { + + if (request.method().equals(RestRequest.Method.POST)) { + + final String clickModel = request.param("click_model"); + final int maxRank = Integer.parseInt(request.param("max_rank", "20")); + + if (StringUtils.equalsIgnoreCase(clickModel, "coec")) { + + final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(true, maxRank); + final CoecClickModel coecClickModel = new CoecClickModel(client, coecClickModelParameters); - // Get the job parameters from the request. - final String id = request.param("id"); - final String indexName = request.param("index"); - final String jobName = request.param("job_name"); - final String interval = request.param("interval"); - final String lockDurationSecondsString = request.param("lock_duration_seconds", "30"); - final Long lockDurationSeconds = lockDurationSecondsString != null ? Long.parseLong(lockDurationSecondsString) : null; - final String jitterString = request.param("jitter"); - final Double jitter = jitterString != null ? Double.parseDouble(jitterString) : null; + try { + coecClickModel.calculateJudgments(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "Implicit judgment generation initiated.")); + + } else { + + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "Invalid click_model.")); + + } + + } else { + + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, request.method() + " is not allowed.")); - if (id == null || indexName == null) { - throw new IllegalArgumentException("Must specify id and index parameter!"); } - final SearchQualityEvaluationJobParameter jobParameter = new SearchQualityEvaluationJobParameter( - jobName, - indexName, - new IntervalSchedule(Instant.ofEpochMilli(1730171983000L), Integer.parseInt(interval), ChronoUnit.MINUTES), - lockDurationSeconds, - jitter - ); - - final IndexRequest indexRequest = new IndexRequest().index(SearchQualityEvaluationPlugin.SCHEDULED_JOBS_INDEX_NAME) - .id(id) - .source(jobParameter.toXContent(JsonXContent.contentBuilder(), null)) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - - return restChannel -> { - - // index the job parameter - client.index(indexRequest, new ActionListener<>() { - @Override - public void onResponse(final IndexResponse indexResponse) { - try { - final RestResponse restResponse = new BytesRestResponse( - RestStatus.OK, - indexResponse.toXContent(JsonXContent.contentBuilder(), null) - ); - LOGGER.info("Created implicit judgments schedule."); - restChannel.sendResponse(restResponse); - } catch (IOException e) { + } else if(StringUtils.equalsIgnoreCase(request.path(), SCHEDULING_URL)) { + + if (request.method().equals(RestRequest.Method.POST)) { + + // Get the job parameters from the request. + final String id = request.param("id"); + final String jobName = request.param("job_name", UUID.randomUUID().toString()); + final String lockDurationSecondsString = request.param("lock_duration_seconds", "600"); + final Long lockDurationSeconds = lockDurationSecondsString != null ? Long.parseLong(lockDurationSecondsString) : null; + final String jitterString = request.param("jitter"); + final Double jitter = jitterString != null ? Double.parseDouble(jitterString) : null; + final String clickModel = request.param("click_model"); + final int maxRank = Integer.parseInt(request.param("max_rank", "20")); + + // Validate the request parameters. + if (id == null || clickModel == null) { + throw new IllegalArgumentException("The id and click_model parameters must be provided."); + } + + // Read the start_time. + final Instant startTime; + if (StringUtils.isEmpty(request.param("start_time"))) { + startTime = Instant.now(); + } else { + startTime = Instant.ofEpochMilli(Long.parseLong(request.param("start_time"))); + } + + // Read the interval. + final int interval; + if (StringUtils.isEmpty(request.param("interval"))) { + // Default to every 24 hours. + interval = 1440; + } else { + interval = Integer.parseInt(request.param("interval")); + } + + final SearchQualityEvaluationJobParameter jobParameter = new SearchQualityEvaluationJobParameter( + jobName, new IntervalSchedule(startTime, interval, ChronoUnit.MINUTES), lockDurationSeconds, + jitter, clickModel, maxRank + ); + + final IndexRequest indexRequest = new IndexRequest().index(SearchQualityEvaluationPlugin.SCHEDULED_JOBS_INDEX_NAME) + .id(id) + .source(jobParameter.toXContent(JsonXContent.contentBuilder(), null)) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + return restChannel -> { + + // index the job parameter + client.index(indexRequest, new ActionListener<>() { + + @Override + public void onResponse(final IndexResponse indexResponse) { + + try { + + final RestResponse restResponse = new BytesRestResponse( + RestStatus.OK, + indexResponse.toXContent(JsonXContent.contentBuilder(), null) + ); + LOGGER.info("Created implicit judgments schedule for click-model {}: Job name {}, running every {} minutes starting {}", clickModel, jobName, interval, startTime); + + restChannel.sendResponse(restResponse); + + } catch (IOException e) { + restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage())); + } + + } + + @Override + public void onFailure(Exception e) { restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage())); } - } + }); - @Override - public void onFailure(Exception e) { - restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage())); - } - }); - }; + }; - } else if (request.method().equals(RestRequest.Method.DELETE)) { + } else if (request.method().equals(RestRequest.Method.DELETE)) { - final String id = request.param("id"); - final DeleteRequest deleteRequest = new DeleteRequest().index(SearchQualityEvaluationPlugin.SCHEDULED_JOBS_INDEX_NAME).id(id); + final String id = request.param("id"); + final DeleteRequest deleteRequest = new DeleteRequest().index(SearchQualityEvaluationPlugin.SCHEDULED_JOBS_INDEX_NAME).id(id); - return restChannel -> { - client.delete(deleteRequest, new ActionListener<>() { - @Override - public void onResponse(final DeleteResponse deleteResponse) { - restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "Job deleted.")); - } + return restChannel -> { + client.delete(deleteRequest, new ActionListener<>() { + @Override + public void onResponse(final DeleteResponse deleteResponse) { + restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "Scheduled job deleted.")); + } + + @Override + public void onFailure(Exception e) { + restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage())); + } + }); + }; + + } else { + + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, request.method() + " is not allowed.")); + + } - @Override - public void onFailure(Exception e) { - restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage())); - } - }); - }; } else { - return restChannel -> { - restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, request.method() + " is not allowed.")); - }; + + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, request.path() + " is not found.")); + } } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/ClickModel.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/ClickModel.java index 6543b0e..cef9f49 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/ClickModel.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/ClickModel.java @@ -8,16 +8,11 @@ */ package org.opensearch.eval.judgments.clickmodel; -import org.opensearch.eval.judgments.model.Judgment; - -import java.io.IOException; -import java.util.Collection; - public abstract class ClickModel { public static final String INDEX_UBI_EVENTS = "ubi_events"; public static final String INDEX_UBI_QUERIES = "ubi_queries"; - public abstract Collection calculateJudgments() throws Exception; + public abstract void calculateJudgments() throws Exception; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java index 89787d1..cdab8ac 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java @@ -37,6 +37,8 @@ import org.opensearch.search.builder.SearchSourceBuilder; import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashSet; @@ -74,29 +76,29 @@ public CoecClickModel(final Client client, final CoecClickModelParameters parame } @Override - public Collection calculateJudgments() throws Exception { + public void calculateJudgments() throws Exception { final int maxRank = parameters.getMaxRank(); // Calculate and index the rank-aggregated click-through. + LOGGER.info("Beginning calculation of rank-aggregated click-through."); final Map rankAggregatedClickThrough = getRankAggregatedClickThrough(); LOGGER.info("Rank-aggregated clickthrough positions: {}", rankAggregatedClickThrough.size()); showRankAggregatedClickThrough(rankAggregatedClickThrough); // Calculate and index the click-through rate for query/doc pairs. + LOGGER.info("Beginning calculation of clickthrough rates."); final Map> clickthroughRates = getClickthroughRate(maxRank); LOGGER.info("Clickthrough rates for number of queries: {}", clickthroughRates.size()); showClickthroughRates(clickthroughRates); // Generate and index the implicit judgments. - final Collection judgments = calculateCoec(rankAggregatedClickThrough, clickthroughRates); - LOGGER.info("Number of judgments: {}", judgments.size()); - - return judgments; + LOGGER.info("Beginning calculation of implicit judgments."); + calculateCoec(rankAggregatedClickThrough, clickthroughRates); } - public Collection calculateCoec(final Map rankAggregatedClickThrough, + public void calculateCoec(final Map rankAggregatedClickThrough, final Map> clickthroughRates) throws Exception { // Calculate the COEC. @@ -156,7 +158,7 @@ public Collection calculateCoec(final Map rankAggrega openSearchHelper.indexJudgments(judgments); } - return judgments; + LOGGER.info("Persisted number of judgments: {}", judgments.size()); } @@ -221,7 +223,7 @@ private Map> getClickthroughRate(final int maxRank for (final SearchHit hit : searchHits) { - final UbiEvent ubiEvent = gson.fromJson(hit.getSourceAsString(), UbiEvent.class); + final UbiEvent ubiEvent = AccessController.doPrivileged((PrivilegedAction) () -> gson.fromJson(hit.getSourceAsString(), UbiEvent.class)); // We need to the hash of the query_id because two users can both search // for "computer" and those searches will have different query IDs, but they are the same search. diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/UbiEvent.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/UbiEvent.java index f7e1dcc..57809cc 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/UbiEvent.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/event/UbiEvent.java @@ -51,4 +51,5 @@ public EventAttributes getEventAttributes() { public void setEventAttributes(EventAttributes eventAttributes) { this.eventAttributes = eventAttributes; } + } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java index 014abb8..2f6fe49 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java @@ -23,6 +23,8 @@ import org.opensearch.search.builder.SearchSourceBuilder; import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -92,7 +94,7 @@ public UbiQuery getQueryFromQueryId(final String queryId) throws Exception { // Will only be a single result. final SearchHit hit = response.getHits().getHits()[0]; - return gson.fromJson(hit.getSourceAsString(), UbiQuery.class); + return AccessController.doPrivileged((PrivilegedAction) () -> gson.fromJson(hit.getSourceAsString(), UbiQuery.class)); } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/plugin-metadata/plugin-security.policy b/opensearch-search-quality-evaluation-plugin/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000..eb1558f --- /dev/null +++ b/opensearch-search-quality-evaluation-plugin/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,4 @@ +grant { + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + permission java.lang.RuntimePermission "accessDeclaredMembers"; +};