Merge branch 'main' of github.com:o19s/opensearch-search-quality-eval…

…uation

opensearch-search-quality-evaluation-plugin/Dockerfile

@@ -1,4 +1,6 @@
 FROM opensearchproject/opensearch:2.17.1
 
+RUN /usr/share/opensearch/bin/opensearch-plugin install --batch https://github.com/opensearch-project/user-behavior-insights/releases/download/2.17.1.0/opensearch-ubi-2.17.1.0.zip
+
 ADD ./build/distributions/search-quality-evaluation-plugin-2.17.1.0.zip  /tmp/search-quality-evaluation-plugin.zip
 RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/search-quality-evaluation-plugin.zip

opensearch-search-quality-evaluation-plugin/README.md

@@ -38,5 +38,11 @@ curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?id
 See the created job:
 
 ```
-curl -s http://localhost:9200/.scheduler_search_quality_eval/_search
+curl -s http://localhost:9200/search_quality_eval_scheduler/_search
+```
+
+See the judgments:
+
+```
+curl -s http://localhost:9200/judgments/
 ```

opensearch-search-quality-evaluation-plugin/create-schedule.sh

@@ -1,5 +1,6 @@
 #!/bin/bash -e
 
-curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?id=1&index=ubi&job_name=test2&interval=1" | jq
+curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/schedule?id=1&click_model=coec&max_rank=20&job_name=test2&interval=10" | jq
 
-curl -s "http://localhost:9200/.scheduler_search_quality_eval/_search" | jq
+echo "Scheduled jobs:"
+curl -s "http://localhost:9200/search_quality_eval_scheduler/_search" | jq

opensearch-search-quality-evaluation-plugin/docker-compose.yaml

@@ -1,8 +1,8 @@
 services:
 
-  opensearch:
+  opensearch_sef:
     build: .
-    container_name: opensearch
+    container_name: opensearch_sef
     environment:
       discovery.type: single-node
       node.name: opensearch
@@ -21,24 +21,24 @@ services:
       - "9600:9600"
     networks:
       - opensearch-net
-    volumes:
-      - opensearch-data1:/usr/share/opensearch/data
+#    volumes:
+#      - opensearch-data1:/usr/share/opensearch/data
 
-#  opensearch_dashboards:
+#  opensearch_sef_dashboards:
 #    image: opensearchproject/opensearch-dashboards:2.16.0
-#    container_name: opensearch_dashboards
+#    container_name: opensearch_sef_dashboards
 #    ports:
 #      - "5601:5601"
 #    environment:
 #      OPENSEARCH_HOSTS: '["http://opensearch:9200"]'
 #      DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true"
 #    depends_on:
-#      - opensearch
+#      - opensearch_sef
 #    networks:
 #      - opensearch-net
 
-volumes:
-  opensearch-data1:
+#volumes:
+#  opensearch-data1:
 
 networks:
   opensearch-net:

opensearch-search-quality-evaluation-plugin/generate-judgments-now.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -e
+
+curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=20"

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobParameter.java

@@ -27,6 +27,10 @@ public class SearchQualityEvaluationJobParameter implements ScheduledJobParamete
     public static final String LOCK_DURATION_SECONDS = "lock_duration_seconds";
     public static final String JITTER = "jitter";
 
+    // Custom properties.
+    public static final String CLICK_MODEL = "click_model";
+    public static final String MAX_RANK = "max_rank";
+
     // Properties from ScheduledJobParameter.
     private String jobName;
     private Instant lastUpdateTime;
@@ -36,15 +40,17 @@ public class SearchQualityEvaluationJobParameter implements ScheduledJobParamete
     private Long lockDurationSeconds;
     private Double jitter;
 
-    // Custom properties for this job.
-    private String indexToWatch;
+    // Custom properties.
+    private String clickModel;
+    private int maxRank;
 
     public SearchQualityEvaluationJobParameter() {
 
     }
 
-    public SearchQualityEvaluationJobParameter(final String name, final String indexToWatch, final Schedule schedule,
-                                               final Long lockDurationSeconds, final Double jitter) {
+    public SearchQualityEvaluationJobParameter(final String name, final Schedule schedule,
+                                               final Long lockDurationSeconds, final Double jitter,
+                                               final String clickModel, final int maxRank) {
         this.jobName = name;
         this.schedule = schedule;
         this.enabled = true;
@@ -55,7 +61,43 @@ public SearchQualityEvaluationJobParameter(final String name, final String index
         this.enabledTime = now;
         this.lastUpdateTime = now;
 
-        this.indexToWatch = indexToWatch;
+        // Custom properties.
+        this.clickModel = clickModel;
+        this.maxRank = maxRank;
+
+    }
+
+    @Override
+    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+
+        builder.startObject();
+
+        builder
+                .field(NAME_FIELD, this.jobName)
+                .field(ENABLED_FILED, this.enabled)
+                .field(SCHEDULE_FIELD, this.schedule)
+                .field(CLICK_MODEL, this.clickModel)
+                .field(MAX_RANK, this.maxRank);
+
+        if (this.enabledTime != null) {
+            builder.timeField(ENABLED_TIME_FILED, ENABLED_TIME_FILED_READABLE, this.enabledTime.toEpochMilli());
+        }
+
+        if (this.lastUpdateTime != null) {
+            builder.timeField(LAST_UPDATE_TIME_FIELD, LAST_UPDATE_TIME_FIELD_READABLE, this.lastUpdateTime.toEpochMilli());
+        }
+
+        if (this.lockDurationSeconds != null) {
+            builder.field(LOCK_DURATION_SECONDS, this.lockDurationSeconds);
+        }
+
+        if (this.jitter != null) {
+            builder.field(JITTER, this.jitter);
+        }
+
+        builder.endObject();
+
+        return builder;
 
     }
 
@@ -114,10 +156,6 @@ public void setSchedule(Schedule schedule) {
         this.schedule = schedule;
     }
 
-    public void setIndexToWatch(String indexToWatch) {
-        this.indexToWatch = indexToWatch;
-    }
-
     public void setLockDurationSeconds(Long lockDurationSeconds) {
         this.lockDurationSeconds = lockDurationSeconds;
     }
@@ -126,36 +164,20 @@ public void setJitter(Double jitter) {
         this.jitter = jitter;
     }
 
-    @Override
-    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
-
-        builder.startObject();
-
-        builder
-            .field(NAME_FIELD, this.jobName)
-            .field(ENABLED_FILED, this.enabled)
-            .field(SCHEDULE_FIELD, this.schedule);
-
-        if (this.enabledTime != null) {
-            builder.timeField(ENABLED_TIME_FILED, ENABLED_TIME_FILED_READABLE, this.enabledTime.toEpochMilli());
-        }
-
-        if (this.lastUpdateTime != null) {
-            builder.timeField(LAST_UPDATE_TIME_FIELD, LAST_UPDATE_TIME_FIELD_READABLE, this.lastUpdateTime.toEpochMilli());
-        }
-
-        if (this.lockDurationSeconds != null) {
-            builder.field(LOCK_DURATION_SECONDS, this.lockDurationSeconds);
-        }
-
-        if (this.jitter != null) {
-            builder.field(JITTER, this.jitter);
-        }
+    public String getClickModel() {
+        return clickModel;
+    }
 
-        builder.endObject();
+    public void setClickModel(String clickModel) {
+        this.clickModel = clickModel;
+    }
 
-        return builder;
+    public int getMaxRank() {
+        return maxRank;
+    }
 
+    public void setMaxRank(int maxRank) {
+        this.maxRank = maxRank;
     }
 
 }

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java

@@ -8,6 +8,7 @@
  */
 package org.opensearch.eval;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.client.Client;
@@ -70,19 +71,17 @@ public void setClient(Client client) {
     @Override
     public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionContext context) {
 
-        LOGGER.info("Running custom job! = {}", jobParameter.getName());
-
-        if (!(jobParameter instanceof SearchQualityEvaluationJobParameter)) {
+        if(!(jobParameter instanceof SearchQualityEvaluationJobParameter)) {
             throw new IllegalStateException(
                 "Job parameter is not instance of SampleJobParameter, type: " + jobParameter.getClass().getCanonicalName()
             );
         }
 
-        if (this.clusterService == null) {
+        if(this.clusterService == null) {
             throw new IllegalStateException("ClusterService is not initialized.");
         }
 
-        if (this.threadPool == null) {
+        if(this.threadPool == null) {
             throw new IllegalStateException("ThreadPool is not initialized.");
         }
 
@@ -93,20 +92,28 @@ public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionC
             if (jobParameter.getLockDurationSeconds() != null) {
 
                 lockService.acquireLock(jobParameter, context, ActionListener.wrap(lock -> {
+
                     if (lock == null) {
                         return;
                     }
 
                     final SearchQualityEvaluationJobParameter searchQualityEvaluationJobParameter = (SearchQualityEvaluationJobParameter) jobParameter;
 
-                    LOGGER.info("Message from inside the job.");
+                    final long startTime = System.currentTimeMillis();
+
+                    if(StringUtils.equalsIgnoreCase(searchQualityEvaluationJobParameter.getClickModel(), "coec")) {
+
+                        LOGGER.info("Beginning implicit judgment generation using clicks-over-expected-clicks.");
+                        final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(true, searchQualityEvaluationJobParameter.getMaxRank());
+                        final CoecClickModel coecClickModel = new CoecClickModel(client, coecClickModelParameters);
+
+                        coecClickModel.calculateJudgments();
+
+                    }
 
-                    final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(true, 20);
-                    final CoecClickModel coecClickModel = new CoecClickModel(client, coecClickModelParameters);
-                    final Collection<Judgment> judgments = coecClickModel.calculateJudgments();
+                    LOGGER.info("Implicit judgment generation completed in {} ms", System.currentTimeMillis() - startTime);
 
-                    lockService.release(
-                        lock,
+                    lockService.release(lock,
                         ActionListener.wrap(released -> LOGGER.info("Released lock for job {}", jobParameter.getName()), exception -> {
                             throw new IllegalStateException("Failed to release lock.");
                         })

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java

@@ -48,7 +48,7 @@ public class SearchQualityEvaluationPlugin extends Plugin implements ActionPlugi
 
     private static final Logger LOGGER = LogManager.getLogger(SearchQualityEvaluationPlugin.class);
 
-    public static final String SCHEDULED_JOBS_INDEX_NAME = ".scheduler_search_quality_eval";
+    public static final String SCHEDULED_JOBS_INDEX_NAME = "search_quality_eval_scheduler";
 
     @Override
     public Collection<Object> createComponents(
@@ -97,8 +97,6 @@ public ScheduledJobParser getJobParser() {
 
         return (parser, id, jobDocVersion) -> {
 
-            LOGGER.info("Getting job parser");
-
             final SearchQualityEvaluationJobParameter jobParameter = new SearchQualityEvaluationJobParameter();
             XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
 
@@ -130,6 +128,12 @@ public ScheduledJobParser getJobParser() {
                     case SearchQualityEvaluationJobParameter.JITTER:
                         jobParameter.setJitter(parser.doubleValue());
                         break;
+                    case SearchQualityEvaluationJobParameter.CLICK_MODEL:
+                        jobParameter.setClickModel(parser.text());
+                        break;
+                    case SearchQualityEvaluationJobParameter.MAX_RANK:
+                        jobParameter.setMaxRank(parser.intValue());
+                        break;
                     default:
                         XContentParserUtils.throwUnknownToken(parser.currentToken(), parser.getTokenLocation());
                 }