diff --git a/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh b/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh index c8744c6..94757ea 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh @@ -18,4 +18,4 @@ echo "Creating judgments index..." # }' echo "Creating judgments..." -curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=20" +curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=1" diff --git a/opensearch-search-quality-evaluation-plugin/scripts/sample-events.sh b/opensearch-search-quality-evaluation-plugin/scripts/sample-events.sh new file mode 100755 index 0000000..65c4b21 --- /dev/null +++ b/opensearch-search-quality-evaluation-plugin/scripts/sample-events.sh @@ -0,0 +1,267 @@ +curl -s -X DELETE http://localhost:9200/ubi_events,ubi_queries + +curl -s -X POST http://localhost:9200/_plugins/ubi/initialize + +curl -s -X PUT http://localhost:9200/ubi_events/_doc/1 -H "Content-Type: application/json" -d' +{ + "application": "esci_ubi_sample", + "action_name": "view", + "query_id": "bd43b377-67ff-4165-8753-58bbdb3392c5", + "session_id": "fdb13692-d42c-4d1d-950b-b8814c963de2", + "client_id": "28ccfb32-fbd7-4514-9051-cea719db42de", + "timestamp": "2024-12-11T04:56:49.419Z", + "user_query": "tv", + "message_type": null, + "message": null, + "event_attributes": { + "object": { + "object_id": "B07JW53H22", + "object_id_field": "product_id" + }, + "position": { + "index": 1 + } + } +}' | jq + +curl -s -X PUT http://localhost:9200/ubi_events/_doc/2 -H "Content-Type: application/json" -d' +{ + "application": "esci_ubi_sample", + "action_name": "view", + "query_id": "bd43b377-67ff-4165-8753-58bbdb3392c5", + "session_id": "fdb13692-d42c-4d1d-950b-b8814c963de2", + "client_id": "28ccfb32-fbd7-4514-9051-cea719db42de", + "timestamp": "2024-12-11T04:56:49.419Z", + "user_query": "tv", + "message_type": null, + "message": null, + "event_attributes": { + "object": { + "object_id": "B07JW53H22", + "object_id_field": "product_id" + }, + "position": { + "index": 1 + } + } +}' | jq + +curl -s -X PUT http://localhost:9200/ubi_events/_doc/3 -H "Content-Type: application/json" -d' +{ + "application": "esci_ubi_sample", + "action_name": "view", + "query_id": "bd43b377-67ff-4165-8753-58bbdb3392c5", + "session_id": "fdb13692-d42c-4d1d-950b-b8814c963de2", + "client_id": "28ccfb32-fbd7-4514-9051-cea719db42de", + "timestamp": "2024-12-11T04:56:49.419Z", + "user_query": "tv", + "message_type": null, + "message": null, + "event_attributes": { + "object": { + "object_id": "B07JW53H22", + "object_id_field": "product_id" + }, + "position": { + "index": 1 + } + } +}' | jq + +curl -s -X PUT http://localhost:9200/ubi_events/_doc/4 -H "Content-Type: application/json" -d' +{ + "application": "esci_ubi_sample", + "action_name": "view", + "query_id": "bd43b377-67ff-4165-8753-58bbdb3392c5", + "session_id": "fdb13692-d42c-4d1d-950b-b8814c963de2", + "client_id": "28ccfb32-fbd7-4514-9051-cea719db42de", + "timestamp": "2024-12-11T04:56:49.419Z", + "user_query": "tv", + "message_type": null, + "message": null, + "event_attributes": { + "object": { + "object_id": "B07JW53H22", + "object_id_field": "product_id" + }, + "position": { + "index": 1 + } + } +}' | jq + +curl -s -X PUT http://localhost:9200/ubi_events/_doc/5 -H "Content-Type: application/json" -d' +{ + "application": "esci_ubi_sample", + "action_name": "click", + "query_id": "bd43b377-67ff-4165-8753-58bbdb3392c5", + "session_id": "fdb13692-d42c-4d1d-950b-b8814c963de2", + "client_id": "28ccfb32-fbd7-4514-9051-cea719db42de", + "timestamp": "2024-12-11T04:56:49.419Z", + "user_query": "tv", + "message_type": null, + "message": null, + "event_attributes": { + "object": { + "object_id": "B07JW53H22", + "object_id_field": "product_id" + }, + "position": { + "index": 1 + } + } +}' | jq + +curl -s -X PUT http://localhost:9200/ubi_events/_doc/6 -H "Content-Type: application/json" -d' +{ + "application": "esci_ubi_sample", + "action_name": "view", + "query_id": "dc6872a3-1f4c-46b2-ad84-7add603b4c73", + "session_id": "a8f7d668-12b9-4cf3-a56f-22700b9e9b89", + "client_id": "a654b87b-a8cd-423b-996f-a169de13d4fb", + "timestamp": "2024-12-11T00:16:42.278Z", + "user_query": "airpods", + "message_type": null, + "message": null, + "event_attributes": { + "object": { + "object_id": "B088FVYG44", + "object_id_field": "product_id" + }, + "position": { + "index": 1 + } + } +}' | jq + +curl -s -X PUT http://localhost:9200/ubi_events/_doc/7 -H "Content-Type: application/json" -d' +{ + "application": "esci_ubi_sample", + "action_name": "view", + "query_id": "dc6872a3-1f4c-46b2-ad84-7add603b4c73", + "session_id": "a8f7d668-12b9-4cf3-a56f-22700b9e9b89", + "client_id": "a654b87b-a8cd-423b-996f-a169de13d4fb", + "timestamp": "2024-12-11T00:16:42.278Z", + "user_query": "airpods", + "message_type": null, + "message": null, + "event_attributes": { + "object": { + "object_id": "B088FVYG44", + "object_id_field": "product_id" + }, + "position": { + "index": 1 + } + } +}' | jq + +curl -s -X PUT http://localhost:9200/ubi_events/_doc/8 -H "Content-Type: application/json" -d' +{ + "application": "esci_ubi_sample", + "action_name": "view", + "query_id": "dc6872a3-1f4c-46b2-ad84-7add603b4c73", + "session_id": "a8f7d668-12b9-4cf3-a56f-22700b9e9b89", + "client_id": "a654b87b-a8cd-423b-996f-a169de13d4fb", + "timestamp": "2024-12-11T00:16:42.278Z", + "user_query": "airpods", + "message_type": null, + "message": null, + "event_attributes": { + "object": { + "object_id": "B088FVYG44", + "object_id_field": "product_id" + }, + "position": { + "index": 1 + } + } +}' | jq + +curl -s -X PUT http://localhost:9200/ubi_events/_doc/9 -H "Content-Type: application/json" -d' +{ + "application": "esci_ubi_sample", + "action_name": "view", + "query_id": "dc6872a3-1f4c-46b2-ad84-7add603b4c73", + "session_id": "a8f7d668-12b9-4cf3-a56f-22700b9e9b89", + "client_id": "a654b87b-a8cd-423b-996f-a169de13d4fb", + "timestamp": "2024-12-11T00:16:42.278Z", + "user_query": "airpods", + "message_type": null, + "message": null, + "event_attributes": { + "object": { + "object_id": "B088FVYG44", + "object_id_field": "product_id" + }, + "position": { + "index": 1 + } + } +}' | jq + +curl -s -X PUT http://localhost:9200/ubi_events/_doc/10 -H "Content-Type: application/json" -d' +{ + "application": "esci_ubi_sample", + "action_name": "click", + "query_id": "dc6872a3-1f4c-46b2-ad84-7add603b4c73", + "session_id": "a8f7d668-12b9-4cf3-a56f-22700b9e9b89", + "client_id": "a654b87b-a8cd-423b-996f-a169de13d4fb", + "timestamp": "2024-12-11T00:16:42.278Z", + "user_query": "airpods", + "message_type": null, + "message": null, + "event_attributes": { + "object": { + "object_id": "B088FVYG44", + "object_id_field": "product_id" + }, + "position": { + "index": 1 + } + } +}' | jq + +curl -s -X PUT http://localhost:9200/ubi_events/_doc/11 -H "Content-Type: application/json" -d' +{ + "application": "esci_ubi_sample", + "action_name": "click", + "query_id": "dc6872a3-1f4c-46b2-ad84-7add603b4c73", + "session_id": "a8f7d668-12b9-4cf3-a56f-22700b9e9b89", + "client_id": "a654b87b-a8cd-423b-996f-a169de13d4fb", + "timestamp": "2024-12-11T00:16:42.278Z", + "user_query": "airpods", + "message_type": null, + "message": null, + "event_attributes": { + "object": { + "object_id": "B088FVYG44", + "object_id_field": "product_id" + }, + "position": { + "index": 1 + } + } +}' | jq + +curl -s -X PUT http://localhost:9200/ubi_events/_doc/12 -H "Content-Type: application/json" -d' +{ + "application": "esci_ubi_sample", + "action_name": "click", + "query_id": "dc6872a3-1f4c-46b2-ad84-7add603b4c73", + "session_id": "a8f7d668-12b9-4cf3-a56f-22700b9e9b89", + "client_id": "a654b87b-a8cd-423b-996f-a169de13d4fb", + "timestamp": "2024-12-11T00:16:42.278Z", + "user_query": "airpods", + "message_type": null, + "message": null, + "event_attributes": { + "object": { + "object_id": "B088FVYG44", + "object_id_field": "product_id" + }, + "position": { + "index": 1 + } + } +}' | jq \ No newline at end of file diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java index ab76efe..58f8395 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java @@ -181,37 +181,8 @@ private Map> getClickthroughRate(final int maxRank // - Get each document returned in that query (in the QueryResponse object). // - Calculate the click-through rate for the document. (clicks/impressions) - // TODO: Use maxRank in place of the hardcoded 20. // TODO: Allow for a time period and for a specific application. - /* - * { - * "bool": { - * "should": [ - * { - * "term": { - * "action_name": "click" - * } - * }, - * { - * "term": { - * "action_name": "impression" - * } - * } - * ], - * "must": [ - * { - * "range": { - * "event_attributes.position.ordinal": { - * "lte": 20 - * } - * } - * } - * ] - * } - * } - */ - final String query = "{\n" + " \"bool\": {\n" + " \"should\": [\n" + @@ -230,7 +201,7 @@ private Map> getClickthroughRate(final int maxRank " {\n" + " \"range\": {\n" + " \"event_attributes.position.ordinal\": {\n" + - " \"lte\": 20\n" + + " \"lte\": " + maxRank + "\n" + " }\n" + " }\n" + " }\n" +