Skip to content

Commit

Permalink
Merge pull request #56 from o19s/max_rank
Browse files Browse the repository at this point in the history 
Not hardcoding max_rank
  • Loading branch information
@jzonthemtn
jzonthemtn authored Dec 9, 2024
2 parents 62c1d0c + 8172730 commit afc496d
Show file tree
Hide file tree
Showing 3 changed files with 269 additions and 31 deletions.
2 changes: 1 addition & 1 deletion opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@ echo "Creating judgments index..."
# }'

echo "Creating judgments..."
curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=20"
curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=1"
267 changes: 267 additions & 0 deletions opensearch-search-quality-evaluation-plugin/scripts/sample-events.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,267 @@
curl -s -X DELETE http://localhost:9200/ubi_events,ubi_queries

curl -s -X POST http://localhost:9200/_plugins/ubi/initialize

curl -s -X PUT http://localhost:9200/ubi_events/_doc/1 -H "Content-Type: application/json" -d'
{
"application": "esci_ubi_sample",
"action_name": "view",
"query_id": "bd43b377-67ff-4165-8753-58bbdb3392c5",
"session_id": "fdb13692-d42c-4d1d-950b-b8814c963de2",
"client_id": "28ccfb32-fbd7-4514-9051-cea719db42de",
"timestamp": "2024-12-11T04:56:49.419Z",
"user_query": "tv",
"message_type": null,
"message": null,
"event_attributes": {
"object": {
"object_id": "B07JW53H22",
"object_id_field": "product_id"
},
"position": {
"index": 1
}
}
}' | jq

curl -s -X PUT http://localhost:9200/ubi_events/_doc/2 -H "Content-Type: application/json" -d'
{
"application": "esci_ubi_sample",
"action_name": "view",
"query_id": "bd43b377-67ff-4165-8753-58bbdb3392c5",
"session_id": "fdb13692-d42c-4d1d-950b-b8814c963de2",
"client_id": "28ccfb32-fbd7-4514-9051-cea719db42de",
"timestamp": "2024-12-11T04:56:49.419Z",
"user_query": "tv",
"message_type": null,
"message": null,
"event_attributes": {
"object": {
"object_id": "B07JW53H22",
"object_id_field": "product_id"
},
"position": {
"index": 1
}
}
}' | jq

curl -s -X PUT http://localhost:9200/ubi_events/_doc/3 -H "Content-Type: application/json" -d'
{
"application": "esci_ubi_sample",
"action_name": "view",
"query_id": "bd43b377-67ff-4165-8753-58bbdb3392c5",
"session_id": "fdb13692-d42c-4d1d-950b-b8814c963de2",
"client_id": "28ccfb32-fbd7-4514-9051-cea719db42de",
"timestamp": "2024-12-11T04:56:49.419Z",
"user_query": "tv",
"message_type": null,
"message": null,
"event_attributes": {
"object": {
"object_id": "B07JW53H22",
"object_id_field": "product_id"
},
"position": {
"index": 1
}
}
}' | jq

curl -s -X PUT http://localhost:9200/ubi_events/_doc/4 -H "Content-Type: application/json" -d'
{
"application": "esci_ubi_sample",
"action_name": "view",
"query_id": "bd43b377-67ff-4165-8753-58bbdb3392c5",
"session_id": "fdb13692-d42c-4d1d-950b-b8814c963de2",
"client_id": "28ccfb32-fbd7-4514-9051-cea719db42de",
"timestamp": "2024-12-11T04:56:49.419Z",
"user_query": "tv",
"message_type": null,
"message": null,
"event_attributes": {
"object": {
"object_id": "B07JW53H22",
"object_id_field": "product_id"
},
"position": {
"index": 1
}
}
}' | jq

curl -s -X PUT http://localhost:9200/ubi_events/_doc/5 -H "Content-Type: application/json" -d'
{
"application": "esci_ubi_sample",
"action_name": "click",
"query_id": "bd43b377-67ff-4165-8753-58bbdb3392c5",
"session_id": "fdb13692-d42c-4d1d-950b-b8814c963de2",
"client_id": "28ccfb32-fbd7-4514-9051-cea719db42de",
"timestamp": "2024-12-11T04:56:49.419Z",
"user_query": "tv",
"message_type": null,
"message": null,
"event_attributes": {
"object": {
"object_id": "B07JW53H22",
"object_id_field": "product_id"
},
"position": {
"index": 1
}
}
}' | jq

curl -s -X PUT http://localhost:9200/ubi_events/_doc/6 -H "Content-Type: application/json" -d'
{
"application": "esci_ubi_sample",
"action_name": "view",
"query_id": "dc6872a3-1f4c-46b2-ad84-7add603b4c73",
"session_id": "a8f7d668-12b9-4cf3-a56f-22700b9e9b89",
"client_id": "a654b87b-a8cd-423b-996f-a169de13d4fb",
"timestamp": "2024-12-11T00:16:42.278Z",
"user_query": "airpods",
"message_type": null,
"message": null,
"event_attributes": {
"object": {
"object_id": "B088FVYG44",
"object_id_field": "product_id"
},
"position": {
"index": 1
}
}
}' | jq

curl -s -X PUT http://localhost:9200/ubi_events/_doc/7 -H "Content-Type: application/json" -d'
{
"application": "esci_ubi_sample",
"action_name": "view",
"query_id": "dc6872a3-1f4c-46b2-ad84-7add603b4c73",
"session_id": "a8f7d668-12b9-4cf3-a56f-22700b9e9b89",
"client_id": "a654b87b-a8cd-423b-996f-a169de13d4fb",
"timestamp": "2024-12-11T00:16:42.278Z",
"user_query": "airpods",
"message_type": null,
"message": null,
"event_attributes": {
"object": {
"object_id": "B088FVYG44",
"object_id_field": "product_id"
},
"position": {
"index": 1
}
}
}' | jq

curl -s -X PUT http://localhost:9200/ubi_events/_doc/8 -H "Content-Type: application/json" -d'
{
"application": "esci_ubi_sample",
"action_name": "view",
"query_id": "dc6872a3-1f4c-46b2-ad84-7add603b4c73",
"session_id": "a8f7d668-12b9-4cf3-a56f-22700b9e9b89",
"client_id": "a654b87b-a8cd-423b-996f-a169de13d4fb",
"timestamp": "2024-12-11T00:16:42.278Z",
"user_query": "airpods",
"message_type": null,
"message": null,
"event_attributes": {
"object": {
"object_id": "B088FVYG44",
"object_id_field": "product_id"
},
"position": {
"index": 1
}
}
}' | jq

curl -s -X PUT http://localhost:9200/ubi_events/_doc/9 -H "Content-Type: application/json" -d'
{
"application": "esci_ubi_sample",
"action_name": "view",
"query_id": "dc6872a3-1f4c-46b2-ad84-7add603b4c73",
"session_id": "a8f7d668-12b9-4cf3-a56f-22700b9e9b89",
"client_id": "a654b87b-a8cd-423b-996f-a169de13d4fb",
"timestamp": "2024-12-11T00:16:42.278Z",
"user_query": "airpods",
"message_type": null,
"message": null,
"event_attributes": {
"object": {
"object_id": "B088FVYG44",
"object_id_field": "product_id"
},
"position": {
"index": 1
}
}
}' | jq

curl -s -X PUT http://localhost:9200/ubi_events/_doc/10 -H "Content-Type: application/json" -d'
{
"application": "esci_ubi_sample",
"action_name": "click",
"query_id": "dc6872a3-1f4c-46b2-ad84-7add603b4c73",
"session_id": "a8f7d668-12b9-4cf3-a56f-22700b9e9b89",
"client_id": "a654b87b-a8cd-423b-996f-a169de13d4fb",
"timestamp": "2024-12-11T00:16:42.278Z",
"user_query": "airpods",
"message_type": null,
"message": null,
"event_attributes": {
"object": {
"object_id": "B088FVYG44",
"object_id_field": "product_id"
},
"position": {
"index": 1
}
}
}' | jq

curl -s -X PUT http://localhost:9200/ubi_events/_doc/11 -H "Content-Type: application/json" -d'
{
"application": "esci_ubi_sample",
"action_name": "click",
"query_id": "dc6872a3-1f4c-46b2-ad84-7add603b4c73",
"session_id": "a8f7d668-12b9-4cf3-a56f-22700b9e9b89",
"client_id": "a654b87b-a8cd-423b-996f-a169de13d4fb",
"timestamp": "2024-12-11T00:16:42.278Z",
"user_query": "airpods",
"message_type": null,
"message": null,
"event_attributes": {
"object": {
"object_id": "B088FVYG44",
"object_id_field": "product_id"
},
"position": {
"index": 1
}
}
}' | jq

curl -s -X PUT http://localhost:9200/ubi_events/_doc/12 -H "Content-Type: application/json" -d'
{
"application": "esci_ubi_sample",
"action_name": "click",
"query_id": "dc6872a3-1f4c-46b2-ad84-7add603b4c73",
"session_id": "a8f7d668-12b9-4cf3-a56f-22700b9e9b89",
"client_id": "a654b87b-a8cd-423b-996f-a169de13d4fb",
"timestamp": "2024-12-11T00:16:42.278Z",
"user_query": "airpods",
"message_type": null,
"message": null,
"event_attributes": {
"object": {
"object_id": "B088FVYG44",
"object_id_field": "product_id"
},
"position": {
"index": 1
}
}
}' | jq
31 changes: 1 addition & 30 deletions ...on-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,37 +181,8 @@ private Map<String, Set<ClickthroughRate>> getClickthroughRate(final int maxRank
// - Get each document returned in that query (in the QueryResponse object).
// - Calculate the click-through rate for the document. (clicks/impressions)

// TODO: Use maxRank in place of the hardcoded 20.
// TODO: Allow for a time period and for a specific application.

/*
* {
* "bool": {
* "should": [
* {
* "term": {
* "action_name": "click"
* }
* },
* {
* "term": {
* "action_name": "impression"
* }
* }
* ],
* "must": [
* {
* "range": {
* "event_attributes.position.ordinal": {
* "lte": 20
* }
* }
* }
* ]
* }
* }
*/

final String query = "{\n" +
" \"bool\": {\n" +
" \"should\": [\n" +
Expand All @@ -230,7 +201,7 @@ private Map<String, Set<ClickthroughRate>> getClickthroughRate(final int maxRank
" {\n" +
" \"range\": {\n" +
" \"event_attributes.position.ordinal\": {\n" +
" \"lte\": 20\n" +
" \"lte\": " + maxRank + "\n" +
" }\n" +
" }\n" +
" }\n" +
Expand Down

0 comments on commit afc496d

Please sign in to comment.