diff --git a/CHANGELOG.md b/CHANGELOG.md index 5334190bce754..94ee083721ef1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,6 +130,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Make MultiBucketConsumerService thread safe to use across slices during search ([#9047](https://github.com/opensearch-project/OpenSearch/pull/9047)) - Change shard_size and shard_min_doc_count evaluation to happen in shard level reduce phase ([#9085](https://github.com/opensearch-project/OpenSearch/pull/9085)) - Add attributes to startSpan methods ([#9199](https://github.com/opensearch-project/OpenSearch/pull/9199)) +- Add support for query profiler with concurrent aggregation ([#9248](https://github.com/opensearch-project/OpenSearch/pull/9248)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java index 67ab062c0e3ca..c89bc584243ee 100644 --- a/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java +++ b/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java @@ -80,6 +80,7 @@ public Map toBreakdownMap() { for (T timingType : this.timingTypes) { map.put(timingType.toString(), this.timings[timingType.ordinal()].getApproximateTiming()); map.put(timingType + TIMING_TYPE_COUNT_SUFFIX, this.timings[timingType.ordinal()].getCount()); + map.put(timingType + TIMING_TYPE_START_TIME_SUFFIX, this.timings[timingType.ordinal()].getEarliestTimerStartTime()); } return Collections.unmodifiableMap(map); } @@ -91,7 +92,7 @@ protected Map toDebugMap() { return emptyMap(); } - public final long toNodeTime() { + public long toNodeTime() { long total = 0; for (T timingType : timingTypes) { total += timings[timingType.ordinal()].getApproximateTiming(); diff --git a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java index 6f0c78e8b307d..8fef4778f6a9b 100644 --- a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java +++ b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java @@ -23,6 +23,10 @@ * @opensearch.internal */ public final class ConcurrentQueryProfileBreakdown extends ContextualProfileBreakdown { + public static final String MAX_END_TIME_SUFFIX = "_max_end_time"; + public static final String MIN_START_TIME_SUFFIX = "_min_start_time"; + private static long breakdownMaxEndTime = 0; + private static long breakdownMinStartTime = Long.MAX_VALUE; private final Map> contexts = new ConcurrentHashMap<>(); /** Sole constructor. */ @@ -47,11 +51,51 @@ public Map toBreakdownMap() { final Map map = new HashMap<>(super.toBreakdownMap()); for (final AbstractProfileBreakdown context : contexts.values()) { - for (final Map.Entry entry : context.toBreakdownMap().entrySet()) { - map.merge(entry.getKey(), entry.getValue(), Long::sum); + Map breakdown = context.toBreakdownMap(); + for (final Map.Entry entry : breakdown.entrySet()) { + if (entry.getKey().contains(TIMING_TYPE_COUNT_SUFFIX)) { + map.merge(entry.getKey(), entry.getValue(), Long::sum); + } } + addMaxEndTimeAndMinStartTime(map, breakdown); } + return buildQueryProfileBreakdownMap(map); + } - return map; + static void addMaxEndTimeAndMinStartTime(Map map, Map breakdown) { + for (QueryTimingType queryTimingType : QueryTimingType.values()) { + String timingType = queryTimingType.toString(); + String timingTypeStartTime = timingType + AbstractProfileBreakdown.TIMING_TYPE_START_TIME_SUFFIX; + long currentMaxEndTime = map.getOrDefault(timingType + MAX_END_TIME_SUFFIX, 0L); + long currentMinStartTime = map.getOrDefault(timingType + MIN_START_TIME_SUFFIX, Long.MAX_VALUE); + // Only "create_weight" is performed at the query level + Map source = timingType.equals(QueryTimingType.CREATE_WEIGHT.toString()) ? map : breakdown; + long maxEndTime = Math.max(currentMaxEndTime, source.get(timingTypeStartTime) + source.get(timingType)); + long minStartTime = Math.min(currentMinStartTime, source.get(timingTypeStartTime)); + map.put(timingType + MAX_END_TIME_SUFFIX, maxEndTime); + map.put(timingType + MIN_START_TIME_SUFFIX, minStartTime); + } + } + + static Map buildQueryProfileBreakdownMap(Map map) { + final Map breakdownMap = new HashMap<>(); + for (QueryTimingType timingType : QueryTimingType.values()) { + String type = timingType.toString(); + String timingTypeStartTime = type + AbstractProfileBreakdown.TIMING_TYPE_START_TIME_SUFFIX; + Long timingTypeMaxEndTime = map.getOrDefault(type + MAX_END_TIME_SUFFIX, map.get(timingTypeStartTime) + map.get(type)); + Long timingTypeMinStartTime = map.getOrDefault(type + MIN_START_TIME_SUFFIX, map.get(timingTypeStartTime)); + breakdownMap.put(type, timingTypeMaxEndTime - timingTypeMinStartTime); + breakdownMap.put(type + TIMING_TYPE_COUNT_SUFFIX, map.get(type + TIMING_TYPE_COUNT_SUFFIX)); + breakdownMaxEndTime = Math.max(breakdownMaxEndTime, timingTypeMaxEndTime); + if (timingTypeMaxEndTime != 0L && timingTypeMinStartTime != 0L) { + breakdownMinStartTime = Math.min(breakdownMinStartTime, timingTypeMinStartTime); + } + } + return breakdownMap; + } + + @Override + public long toNodeTime() { + return breakdownMaxEndTime - breakdownMinStartTime; } } diff --git a/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java b/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java new file mode 100644 index 0000000000000..32d5464282c74 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java @@ -0,0 +1,142 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.profile.query; + +import org.opensearch.test.OpenSearchTestCase; + +import java.util.HashMap; +import java.util.Map; + +public class ConcurrentQueryProfileBreakdownTests extends OpenSearchTestCase { + + public void testBuildQueryProfileBreakdownMap() { + Map testMap = new HashMap<>(); + testMap.put("set_min_competitive_score_count", 0L); + testMap.put("match_count", 0L); + testMap.put("score_start_time", 0L); + testMap.put("shallow_advance_count", 0L); + testMap.put("create_weight_start_time", 1598347679188617L); + testMap.put("next_doc", 0L); + testMap.put("compute_max_score_start_time", 0L); + testMap.put("shallow_advance_min_start_time", 0L); + testMap.put("score_count", 5L); + testMap.put("compute_max_score_count", 0L); + testMap.put("advance_start_time", 0L); + testMap.put("advance", 0L); + testMap.put("advance_count", 3L); + testMap.put("compute_max_score_min_start_time", 0L); + testMap.put("score", 0L); + testMap.put("next_doc_max_end_time", 1598347688298671L); + testMap.put("advance_max_end_time", 1598347688280739L); + testMap.put("next_doc_start_time", 0L); + testMap.put("shallow_advance", 0L); + testMap.put("build_scorer_max_end_time", 1598347688270123L); + testMap.put("create_weight_count", 1L); + testMap.put("create_weight_max_end_time", 1598347679558960L); + testMap.put("match_min_start_time", 0L); + testMap.put("build_scorer", 0L); + testMap.put("compute_max_score_max_end_time", 0L); + testMap.put("next_doc_min_start_time", 1598347688110288L); + testMap.put("set_min_competitive_score", 0L); + testMap.put("set_min_competitive_score_start_time", 0L); + testMap.put("match", 0L); + testMap.put("set_min_competitive_score_max_end_time", 0L); + testMap.put("match_start_time", 0L); + testMap.put("shallow_advance_max_end_time", 0L); + testMap.put("build_scorer_start_time", 0L); + testMap.put("next_doc_count", 5L); + testMap.put("shallow_advance_start_time", 0L); + testMap.put("set_min_competitive_score_min_start_time", 0L); + testMap.put("compute_max_score", 0L); + testMap.put("create_weight_min_start_time", 1598347679188617L); + testMap.put("build_scorer_count", 6L); + testMap.put("create_weight", 370343L); + testMap.put("score_min_start_time", 1598347688018500L); + testMap.put("match_max_end_time", 0L); + testMap.put("advance_min_start_time", 1598347687991984L); + testMap.put("score_max_end_time", 1598347688282743L); + testMap.put("build_scorer_min_start_time", 1598347686448701L); + Map breakdownMap = ConcurrentQueryProfileBreakdown.buildQueryProfileBreakdownMap(testMap); + assertEquals(18, breakdownMap.size()); + assertEquals( + "{set_min_competitive_score_count=0, match_count=0, shallow_advance_count=0, set_min_competitive_score=0, next_doc=188383, match=0, next_doc_count=5, score_count=5, compute_max_score_count=0, compute_max_score=0, advance=288755, advance_count=3, score=264243, build_scorer_count=6, create_weight=370343, shallow_advance=0, create_weight_count=1, build_scorer=1821422}", + breakdownMap.toString() + ); + } + + public void testAddMaxEndTimeAndMinStartTime() { + Map map = new HashMap<>(); + map.put("set_min_competitive_score_count", 0L); + map.put("match_count", 0L); + map.put("score_start_time", 0L); + map.put("shallow_advance_count", 0L); + map.put("create_weight_start_time", 1629732014278990L); + map.put("next_doc", 0L); + map.put("compute_max_score_start_time", 0L); + map.put("score_count", 2L); + map.put("compute_max_score_count", 0L); + map.put("advance_start_time", 0L); + map.put("advance", 0L); + map.put("advance_count", 1L); + map.put("score", 0L); + map.put("next_doc_start_time", 0L); + map.put("shallow_advance", 0L); + map.put("create_weight_count", 1L); + map.put("build_scorer", 0L); + map.put("set_min_competitive_score", 0L); + map.put("set_min_competitive_score_start_time", 0L); + map.put("match", 0L); + map.put("match_start_time", 0L); + map.put("build_scorer_start_time", 0L); + map.put("next_doc_count", 2L); + map.put("shallow_advance_start_time", 0L); + map.put("compute_max_score", 0L); + map.put("build_scorer_count", 2L); + map.put("create_weight", 201692L); + Map breakdown = new HashMap<>(); + breakdown.put("set_min_competitive_score_count", 0L); + breakdown.put("match_count", 0L); + breakdown.put("score_start_time", 1629732030778977L); + breakdown.put("shallow_advance_count", 0L); + breakdown.put("create_weight_start_time", 0L); + breakdown.put("next_doc", 1150L); + breakdown.put("compute_max_score_start_time", 0L); + breakdown.put("score_count", 2L); + breakdown.put("compute_max_score_count", 0L); + breakdown.put("advance_start_time", 1629732030776129L); + breakdown.put("advance", 920L); + breakdown.put("advance_count", 1L); + breakdown.put("score", 1050L); + breakdown.put("next_doc_start_time", 1629732030806446L); + breakdown.put("shallow_advance", 0L); + breakdown.put("create_weight_count", 0L); + breakdown.put("build_scorer", 9649L); + breakdown.put("set_min_competitive_score", 0L); + breakdown.put("set_min_competitive_score_start_time", 0L); + breakdown.put("match", 0L); + breakdown.put("match_start_time", 0L); + breakdown.put("build_scorer_start_time", 1629732030749745L); + breakdown.put("next_doc_count", 2L); + breakdown.put("shallow_advance_start_time", 0L); + breakdown.put("compute_max_score", 0L); + breakdown.put("build_scorer_count", 2L); + breakdown.put("create_weight", 0L); + ConcurrentQueryProfileBreakdown.addMaxEndTimeAndMinStartTime(map, breakdown); + assertEquals(45, map.size()); + assertEquals( + "{set_min_competitive_score_count=0, match_count=0, score_start_time=0, shallow_advance_count=0, create_weight_start_time=1629732014278990, next_doc=0, compute_max_score_start_time=0, shallow_advance_min_start_time=0, score_count=2, compute_max_score_count=0, advance_start_time=0, advance=0, advance_count=1, compute_max_score_min_start_time=0, score=0, next_doc_max_end_time=1629732030807596, advance_max_end_time=1629732030777049, next_doc_start_time=0, shallow_advance=0, build_scorer_max_end_time=1629732030759394, create_weight_count=1, create_weight_max_end_time=1629732014480682, match_min_start_time=0, build_scorer=0, compute_max_score_max_end_time=0, next_doc_min_start_time=1629732030806446, set_min_competitive_score=0, set_min_competitive_score_start_time=0, match=0, set_min_competitive_score_max_end_time=0, match_start_time=0, shallow_advance_max_end_time=0, build_scorer_start_time=0, next_doc_count=2, shallow_advance_start_time=0, set_min_competitive_score_min_start_time=0, compute_max_score=0, create_weight_min_start_time=1629732014278990, build_scorer_count=2, create_weight=201692, score_min_start_time=1629732030778977, match_max_end_time=0, advance_min_start_time=1629732030776129, score_max_end_time=1629732030780027, build_scorer_min_start_time=1629732030749745}", + map.toString() + ); + } +}