Merge pull request #58 from o19s/demo-changes

Clean up and changes for the demo
o19s · Dec 10, 2024 · efaa6dd · efaa6dd
2 parents ecdcb60 + babaf00
commit efaa6dd
Show file tree

Hide file tree

Showing 23 changed files with 41,477 additions and 217 deletions.
diff --git a/.gitignore b/.gitignore
@@ -29,5 +29,3 @@ gradle-app.setting
 .project
 .classpath
 
-transformed_data.json
-*.ndjson
diff --git a/data/esci/ubi_queries_events.ndjson b/data/esci/ubi_queries_events.ndjson
diff --git a/data/ubi-chorus-data-generator/transformed_data.json b/data/ubi-chorus-data-generator/transformed_data.json
diff --git a/opensearch-search-quality-evaluation-plugin/scripts/cleanup.sh b/opensearch-search-quality-evaluation-plugin/scripts/cleanup.sh
@@ -0,0 +1,6 @@
+#!/bin/bash -e
+
+curl -s -X DELETE "http://localhost:9200/judgments,search_quality_eval_completed_jobs,search_quality_eval_query_sets_run_results" | jq
+curl -s -X DELETE "http://localhost:9200/search_quality_eval_completed_jobs" | jq
+curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets_run_results" | jq
+curl -s -X DELETE "http://localhost:9200/ubi_queries,ubi_events" | jq
diff --git a/opensearch-search-quality-evaluation-plugin/scripts/walkthrough.sh b/opensearch-search-quality-evaluation-plugin/scripts/walkthrough.sh
diff --git a/...evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java b/...evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java
@@ -26,7 +26,6 @@
 
 import java.util.HashMap;
 import java.util.Map;
-import java.util.UUID;
 
 /**
  * Job runner for scheduled implicit judgments jobs.
@@ -110,15 +109,15 @@ public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionC
                     final SearchQualityEvaluationJobParameter searchQualityEvaluationJobParameter = (SearchQualityEvaluationJobParameter) jobParameter;
 
                     final long startTime = System.currentTimeMillis();
-                    final long judgments;
+                    final String judgmentsId;
 
                     if("coec".equalsIgnoreCase(searchQualityEvaluationJobParameter.getClickModel())) {
 
                         LOGGER.info("Beginning implicit judgment generation using clicks-over-expected-clicks.");
-                        final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(true, searchQualityEvaluationJobParameter.getMaxRank());
+                        final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(searchQualityEvaluationJobParameter.getMaxRank());
                         final CoecClickModel coecClickModel = new CoecClickModel(client, coecClickModelParameters);
 
-                        judgments = coecClickModel.calculateJudgments();
+                        judgmentsId = coecClickModel.calculateJudgments();
 
                     } else {
 
@@ -135,12 +134,10 @@ public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionC
                     job.put("click_model", searchQualityEvaluationJobParameter.getClickModel());
                     job.put("started", startTime);
                     job.put("duration", elapsedTime);
-                    job.put("judgments", judgments);
+                    job.put("judgments", judgmentsId);
                     job.put("invocation", "scheduled");
                     job.put("max_rank", searchQualityEvaluationJobParameter.getMaxRank());
 
-                    final String judgmentsId = UUID.randomUUID().toString();
-
                     final IndexRequest indexRequest = new IndexRequest()
                             .index(SearchQualityEvaluationPlugin.COMPLETED_JOBS_INDEX_NAME)
                             .id(judgmentsId)

diff --git a/...aluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/...aluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java
@@ -32,6 +32,7 @@
 import org.opensearch.eval.samplers.AllQueriesQuerySamplerParameters;
 import org.opensearch.eval.samplers.ProbabilityProportionalToSizeAbstractQuerySampler;
 import org.opensearch.eval.samplers.ProbabilityProportionalToSizeParameters;
+import org.opensearch.eval.utils.TimeUtils;
 import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule;
 import org.opensearch.rest.BaseRestHandler;
 import org.opensearch.rest.BytesRestResponse;
@@ -216,56 +217,59 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
 
                 if (CoecClickModel.CLICK_MODEL_NAME.equalsIgnoreCase(clickModel)) {
 
-                    final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(true, maxRank);
+                    final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(maxRank);
                     final CoecClickModel coecClickModel = new CoecClickModel(client, coecClickModelParameters);
 
+                    final String judgmentsId;
+
                     // TODO: Run this in a separate thread.
                     try {
 
-                        judgmentCount = coecClickModel.calculateJudgments();
+                        judgmentsId = coecClickModel.calculateJudgments();
 
-                        if(judgmentCount == 0) {
+                        // judgmentsId will be null if no judgments were created (and indexed).
+                        if(judgmentsId == null) {
                             // TODO: Is Bad Request the appropriate error? Perhaps Conflict is more appropriate?
                             return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"No judgments were created. Check the queries and events data.\"}"));
                         }
 
+                        final long elapsedTime = System.currentTimeMillis() - startTime;
+
+                        final Map<String, Object> job = new HashMap<>();
+                        job.put("name", "manual_generation");
+                        job.put("click_model", clickModel);
+                        job.put("started", startTime);
+                        job.put("duration", elapsedTime);
+                        job.put("invocation", "on_demand");
+                        job.put("judgments_id", judgmentsId);
+                        job.put("max_rank", maxRank);
+
+                        final String jobId = UUID.randomUUID().toString();
+
+                        final IndexRequest indexRequest = new IndexRequest()
+                                .index(SearchQualityEvaluationPlugin.COMPLETED_JOBS_INDEX_NAME)
+                                .id(jobId)
+                                .source(job)
+                                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+
+                        client.index(indexRequest, new ActionListener<>() {
+                            @Override
+                            public void onResponse(final IndexResponse indexResponse) {
+                                LOGGER.debug("Click model job completed successfully: {}", jobId);
+                            }
+
+                            @Override
+                            public void onFailure(final Exception ex) {
+                                LOGGER.error("Unable to run job with ID {}", jobId, ex);
+                                throw new RuntimeException("Unable to run job", ex);
+                            }
+                        });
+
                     } catch (Exception ex) {
                         throw new RuntimeException("Unable to generate judgments.", ex);
                     }
 
-                    final long elapsedTime = System.currentTimeMillis() - startTime;
-
-                    final Map<String, Object> job = new HashMap<>();
-                    job.put("name", "manual_generation");
-                    job.put("click_model", clickModel);
-                    job.put("started", startTime);
-                    job.put("duration", elapsedTime);
-                    job.put("judgment_count", judgmentCount);
-                    job.put("invocation", "on_demand");
-                    job.put("max_rank", maxRank);
-
-                    final String jobId = UUID.randomUUID().toString();
-
-                    final IndexRequest indexRequest = new IndexRequest()
-                            .index(SearchQualityEvaluationPlugin.COMPLETED_JOBS_INDEX_NAME)
-                            .id(jobId)
-                            .source(job)
-                            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
-
-                    client.index(indexRequest, new ActionListener<>() {
-                        @Override
-                        public void onResponse(final IndexResponse indexResponse) {
-                            LOGGER.debug("Click model job completed successfully: {}", jobId);
-                        }
-
-                        @Override
-                        public void onFailure(final Exception ex) {
-                            LOGGER.error("Unable to run job with ID {}", jobId, ex);
-                            throw new RuntimeException("Unable to run job", ex);
-                        }
-                    });
-
-                    return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"judgments_id\": \"" + jobId + "\"}"));
+                    return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"judgments_id\": \"" + judgmentsId + "\"}"));
 
                 } else {
                     return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Invalid click model.\"}"));
@@ -394,14 +398,15 @@ public void onResponse(final IndicesExistsResponse indicesExistsResponse) {
 
                 if(!indicesExistsResponse.isExists()) {
 
-                    // TODO: Read this from a resource file instead.
+                    // TODO: Read this mapping from a resource file instead.
                     final String mapping = "{\n" +
                             "                                                  \"properties\": {\n" +
                             "                                                    \"judgments_id\": { \"type\": \"keyword\" },\n" +
                             "                                                    \"query_id\": { \"type\": \"keyword\" },\n" +
                             "                                                    \"query\": { \"type\": \"keyword\" },\n" +
                             "                                                    \"document_id\": { \"type\": \"keyword\" },\n" +
-                            "                                                    \"judgment\": { \"type\": \"double\" }\n" +
+                            "                                                    \"judgment\": { \"type\": \"double\" },\n" +
+                            "                                                    \"timestamp\": { \"type\": \"date\", \"format\": \"basic_date_time\" }\n" +
                             "                                                  }\n" +
                             "                                              }";
 
@@ -412,7 +417,7 @@ public void onResponse(final IndicesExistsResponse indicesExistsResponse) {
                     client.admin().indices().create(createIndexRequest, new ActionListener<>() {
                         @Override
                         public void onResponse(CreateIndexResponse createIndexResponse) {
-                            LOGGER.info("Judgments index created: {}", JUDGMENTS_INDEX_NAME);
+                            LOGGER.debug("Judgments index created: {}", JUDGMENTS_INDEX_NAME);
                         }
 
                         @Override

diff --git a/...-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/ClickModel.java b/...-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/ClickModel.java
@@ -15,9 +15,9 @@ public abstract class ClickModel {
 
     /**
      * Calculate implicit judgments.
-     * @return The number of implicit judgments created.
+     * @return The judgments ID.
      * @throws Exception Thrown if the judgments cannot be created.
      */
-    public abstract long calculateJudgments() throws Exception;
+    public abstract String calculateJudgments() throws Exception;
 
 }