From d62ca4d93179aa425f8c01a0d6f8437f6ec84684 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 10 Dec 2024 11:54:58 -0500 Subject: [PATCH] Adding todo for shortcircuit in PPTSS. Signed-off-by: jzonthemtn --- .../scripts/create-query-set-no-sampling.sh | 11 ++--------- .../create-query-set-using-pptss-sampling.sh | 14 +------------- .../eval/samplers/AllQueriesQuerySampler.java | 1 + ...lityProportionalToSizeAbstractQuerySampler.java | 9 +++++---- 4 files changed, 9 insertions(+), 26 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-no-sampling.sh b/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-no-sampling.sh index fc053d2..ace0404 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-no-sampling.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-no-sampling.sh @@ -1,12 +1,5 @@ #!/bin/bash -e -#QUERY_SET=`curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss" | jq .query_set | tr -d '"'` -curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=none&query_set_size=500" - -#echo ${QUERY_SET} - -#curl -s http://localhost:9200/search_quality_eval_query_sets/_search | jq - -# Run the query set now. -#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET}" | jq +curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets" +curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=none&query_set_size=10" diff --git a/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh b/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh index 613aabf..572bc8d 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh @@ -2,16 +2,4 @@ curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets" -curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss&query_set_size=100" - - -#QUERY_SET=`curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss" | jq .query_set | tr -d '"'` -#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss&query_set_size=100" - -#echo ${QUERY_SET} - -#curl -s -X GET http://localhost:9200/search_quality_eval_query_sets/_doc/${QUERY_SET} | jq - -# Run the query set now. -#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET}" | jq - +curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss&query_set_size=20" diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/AllQueriesQuerySampler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/AllQueriesQuerySampler.java index 7d76af6..23af3df 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/AllQueriesQuerySampler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/AllQueriesQuerySampler.java @@ -56,6 +56,7 @@ public String sample() throws Exception { final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.UBI_QUERIES_INDEX_NAME); searchRequest.source(searchSourceBuilder); + // TODO: Don't use .get() final SearchResponse searchResponse = client.search(searchRequest).get(); final Map queries = new HashMap<>(); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java index 457c06a..79f2c7c 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java @@ -83,8 +83,6 @@ public String sample() throws Exception { while (searchHits != null && searchHits.length > 0) { - LOGGER.info("search hits size = {}", searchHits.length); - for(final SearchHit hit : searchHits) { final Map fields = hit.getSourceAsMap(); userQueries.add(fields.get("user_query").toString()); @@ -137,7 +135,10 @@ public String sample() throws Exception { // This may require generating more random numbers than what was requested // because removing duplicate user queries will require randomly picking more queries. int count = 1; - while(querySet.size() < parameters.getQuerySetSize() && count < userQueries.size()) { + + // TODO: How to short-circuit this such that if the same query gets picked over and over, the loop will never end. + final int max = 5000; + while(querySet.size() < parameters.getQuerySetSize() && count < max) { // Make a random number not yet used. double random; @@ -160,7 +161,7 @@ public String sample() throws Exception { querySet.put(closestQuery, weights.get(closestQuery)); count++; - // LOGGER.info("Generated random value: {}; Smallest delta = {}; Closest query = {}", random, smallestDelta, closestQuery); + //LOGGER.info("Generated random value: {}; Smallest delta = {}; Closest query = {}", random, smallestDelta, closestQuery); }