Skip to content

Commit

Permalink
Add support for query profiler with concurrent aggregation
Browse files Browse the repository at this point in the history 
Signed-off-by: Ticheng Lin <[email protected]>
  • Loading branch information
@ticheng-aws
ticheng-aws committed Aug 11, 2023
1 parent d352f2c commit 777a7bb
Show file tree
Hide file tree
Showing 3 changed files with 190 additions and 4 deletions.
3 changes: 2 additions & 1 deletion server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ public Map<String, Long> toBreakdownMap() {
for (T timingType : this.timingTypes) {
map.put(timingType.toString(), this.timings[timingType.ordinal()].getApproximateTiming());
map.put(timingType + TIMING_TYPE_COUNT_SUFFIX, this.timings[timingType.ordinal()].getCount());
map.put(timingType + TIMING_TYPE_START_TIME_SUFFIX, this.timings[timingType.ordinal()].getEarliestTimerStartTime());
}
return Collections.unmodifiableMap(map);
}
Expand All @@ -91,7 +92,7 @@ protected Map<String, Object> toDebugMap() {
return emptyMap();
}

public final long toNodeTime() {
public long toNodeTime() {
long total = 0;
for (T timingType : timingTypes) {
total += timings[timingType.ordinal()].getApproximateTiming();
Expand Down
49 changes: 46 additions & 3 deletions ...er/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@
* @opensearch.internal
*/
public final class ConcurrentQueryProfileBreakdown extends ContextualProfileBreakdown<QueryTimingType> {
public static final String MAX_END_TIME_SUFFIX = "_max_end_time";
public static final String MIN_START_TIME_SUFFIX = "_min_start_time";
private static long breakdownMaxEndTime = 0;
private static long breakdownMinStartTime = Long.MAX_VALUE;
private final Map<Object, AbstractProfileBreakdown<QueryTimingType>> contexts = new ConcurrentHashMap<>();

/** Sole constructor. */
Expand All @@ -47,11 +51,50 @@ public Map<String, Long> toBreakdownMap() {
final Map<String, Long> map = new HashMap<>(super.toBreakdownMap());

for (final AbstractProfileBreakdown<QueryTimingType> context : contexts.values()) {
for (final Map.Entry<String, Long> entry : context.toBreakdownMap().entrySet()) {
map.merge(entry.getKey(), entry.getValue(), Long::sum);
Map<String, Long> breakdown = context.toBreakdownMap();
for (final Map.Entry<String, Long> entry : breakdown.entrySet()) {
if (entry.getKey().contains(TIMING_TYPE_COUNT_SUFFIX)) {
map.merge(entry.getKey(), entry.getValue(), Long::sum);
}
}
addMaxEndTimeAndMinStartTime(map, breakdown);
}
return buildQueryProfileBreakdownMap(map);
}

return map;
static void addMaxEndTimeAndMinStartTime(Map<String, Long> map, Map<String, Long> breakdown) {
for (QueryTimingType queryTimingType : QueryTimingType.values()) {
String timingType = queryTimingType.toString();
String timingTypeStartTime = timingType + AbstractProfileBreakdown.TIMING_TYPE_START_TIME_SUFFIX;
long currentMaxEndTime = map.getOrDefault(timingType + MAX_END_TIME_SUFFIX, 0L);
long currentMinStartTime = map.getOrDefault(timingType + MIN_START_TIME_SUFFIX, Long.MAX_VALUE);
// Only "create_weight" is performed at the query level
Map<String, Long> source = timingType.equals(QueryTimingType.CREATE_WEIGHT.toString()) ? map : breakdown;
long maxEndTime = Math.max(currentMaxEndTime, source.get(timingTypeStartTime) + source.get(timingType));
long minStartTime = Math.min(currentMinStartTime, source.get(timingTypeStartTime));
map.put(timingType + MAX_END_TIME_SUFFIX, maxEndTime);
map.put(timingType + MIN_START_TIME_SUFFIX, minStartTime);
}
}

static Map<String, Long> buildQueryProfileBreakdownMap(Map<String, Long> map) {
final Map<String, Long> breakdownMap = new HashMap<>();
for (QueryTimingType timingType : QueryTimingType.values()) {
String type = timingType.toString();
Long timingTypeMaxEndTime = map.get(type + MAX_END_TIME_SUFFIX);
Long timingTypeMinStartTime = map.get(type + MIN_START_TIME_SUFFIX);
breakdownMap.put(type, timingTypeMaxEndTime - timingTypeMinStartTime);
breakdownMap.put(type + TIMING_TYPE_COUNT_SUFFIX, map.get(type + TIMING_TYPE_COUNT_SUFFIX));
breakdownMaxEndTime = Math.max(breakdownMaxEndTime, timingTypeMaxEndTime);
if (timingTypeMaxEndTime != 0L && timingTypeMinStartTime != 0L) {
breakdownMinStartTime = Math.min(breakdownMinStartTime, timingTypeMinStartTime);
}
}
return breakdownMap;
}

@Override
public long toNodeTime() {
return breakdownMaxEndTime - breakdownMinStartTime;
}
}
142 changes: 142 additions & 0 deletions ...c/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

/*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/

package org.opensearch.search.profile.query;

import org.opensearch.test.OpenSearchTestCase;

import java.util.HashMap;
import java.util.Map;

public class ConcurrentQueryProfileBreakdownTests extends OpenSearchTestCase {

public void testBuildQueryProfileBreakdownMap() {
Map<String, Long> testMap = new HashMap<>();
testMap.put("set_min_competitive_score_count", 0L);
testMap.put("match_count", 0L);
testMap.put("score_start_time", 0L);
testMap.put("shallow_advance_count", 0L);
testMap.put("create_weight_start_time", 1598347679188617L);
testMap.put("next_doc", 0L);
testMap.put("compute_max_score_start_time", 0L);
testMap.put("shallow_advance_min_start_time", 0L);
testMap.put("score_count", 5L);
testMap.put("compute_max_score_count", 0L);
testMap.put("advance_start_time", 0L);
testMap.put("advance", 0L);
testMap.put("advance_count", 3L);
testMap.put("compute_max_score_min_start_time", 0L);
testMap.put("score", 0L);
testMap.put("next_doc_max_end_time", 1598347688298671L);
testMap.put("advance_max_end_time", 1598347688280739L);
testMap.put("next_doc_start_time", 0L);
testMap.put("shallow_advance", 0L);
testMap.put("build_scorer_max_end_time", 1598347688270123L);
testMap.put("create_weight_count", 1L);
testMap.put("create_weight_max_end_time", 1598347679558960L);
testMap.put("match_min_start_time", 0L);
testMap.put("build_scorer", 0L);
testMap.put("compute_max_score_max_end_time", 0L);
testMap.put("next_doc_min_start_time", 1598347688110288L);
testMap.put("set_min_competitive_score", 0L);
testMap.put("set_min_competitive_score_start_time", 0L);
testMap.put("match", 0L);
testMap.put("set_min_competitive_score_max_end_time", 0L);
testMap.put("match_start_time", 0L);
testMap.put("shallow_advance_max_end_time", 0L);
testMap.put("build_scorer_start_time", 0L);
testMap.put("next_doc_count", 5L);
testMap.put("shallow_advance_start_time", 0L);
testMap.put("set_min_competitive_score_min_start_time", 0L);
testMap.put("compute_max_score", 0L);
testMap.put("create_weight_min_start_time", 1598347679188617L);
testMap.put("build_scorer_count", 6L);
testMap.put("create_weight", 370343L);
testMap.put("score_min_start_time", 1598347688018500L);
testMap.put("match_max_end_time", 0L);
testMap.put("advance_min_start_time", 1598347687991984L);
testMap.put("score_max_end_time", 1598347688282743L);
testMap.put("build_scorer_min_start_time", 1598347686448701L);
Map<String, Long> breakdownMap = ConcurrentQueryProfileBreakdown.buildQueryProfileBreakdownMap(testMap);
assertEquals(18, breakdownMap.size());
assertEquals(
"{set_min_competitive_score_count=0, match_count=0, shallow_advance_count=0, set_min_competitive_score=0, next_doc=188383, match=0, next_doc_count=5, score_count=5, compute_max_score_count=0, compute_max_score=0, advance=288755, advance_count=3, score=264243, build_scorer_count=6, create_weight=370343, shallow_advance=0, create_weight_count=1, build_scorer=1821422}",
breakdownMap.toString()
);
}

public void testAddMaxEndTimeAndMinStartTime() {
Map<String, Long> map = new HashMap<>();
map.put("set_min_competitive_score_count", 0L);
map.put("match_count", 0L);
map.put("score_start_time", 0L);
map.put("shallow_advance_count", 0L);
map.put("create_weight_start_time", 1629732014278990L);
map.put("next_doc", 0L);
map.put("compute_max_score_start_time", 0L);
map.put("score_count", 2L);
map.put("compute_max_score_count", 0L);
map.put("advance_start_time", 0L);
map.put("advance", 0L);
map.put("advance_count", 1L);
map.put("score", 0L);
map.put("next_doc_start_time", 0L);
map.put("shallow_advance", 0L);
map.put("create_weight_count", 1L);
map.put("build_scorer", 0L);
map.put("set_min_competitive_score", 0L);
map.put("set_min_competitive_score_start_time", 0L);
map.put("match", 0L);
map.put("match_start_time", 0L);
map.put("build_scorer_start_time", 0L);
map.put("next_doc_count", 2L);
map.put("shallow_advance_start_time", 0L);
map.put("compute_max_score", 0L);
map.put("build_scorer_count", 2L);
map.put("create_weight", 201692L);
Map<String, Long> breakdown = new HashMap<>();
breakdown.put("set_min_competitive_score_count", 0L);
breakdown.put("match_count", 0L);
breakdown.put("score_start_time", 1629732030778977L);
breakdown.put("shallow_advance_count", 0L);
breakdown.put("create_weight_start_time", 0L);
breakdown.put("next_doc", 1150L);
breakdown.put("compute_max_score_start_time", 0L);
breakdown.put("score_count", 2L);
breakdown.put("compute_max_score_count", 0L);
breakdown.put("advance_start_time", 1629732030776129L);
breakdown.put("advance", 920L);
breakdown.put("advance_count", 1L);
breakdown.put("score", 1050L);
breakdown.put("next_doc_start_time", 1629732030806446L);
breakdown.put("shallow_advance", 0L);
breakdown.put("create_weight_count", 0L);
breakdown.put("build_scorer", 9649L);
breakdown.put("set_min_competitive_score", 0L);
breakdown.put("set_min_competitive_score_start_time", 0L);
breakdown.put("match", 0L);
breakdown.put("match_start_time", 0L);
breakdown.put("build_scorer_start_time", 1629732030749745L);
breakdown.put("next_doc_count", 2L);
breakdown.put("shallow_advance_start_time", 0L);
breakdown.put("compute_max_score", 0L);
breakdown.put("build_scorer_count", 2L);
breakdown.put("create_weight", 0L);
ConcurrentQueryProfileBreakdown.addMaxEndTimeAndMinStartTime(map, breakdown);
assertEquals(45, map.size());
assertEquals(
"{set_min_competitive_score_count=0, match_count=0, score_start_time=0, shallow_advance_count=0, create_weight_start_time=1629732014278990, next_doc=0, compute_max_score_start_time=0, shallow_advance_min_start_time=0, score_count=2, compute_max_score_count=0, advance_start_time=0, advance=0, advance_count=1, compute_max_score_min_start_time=0, score=0, next_doc_max_end_time=1629732030807596, advance_max_end_time=1629732030777049, next_doc_start_time=0, shallow_advance=0, build_scorer_max_end_time=1629732030759394, create_weight_count=1, create_weight_max_end_time=1629732014480682, match_min_start_time=0, build_scorer=0, compute_max_score_max_end_time=0, next_doc_min_start_time=1629732030806446, set_min_competitive_score=0, set_min_competitive_score_start_time=0, match=0, set_min_competitive_score_max_end_time=0, match_start_time=0, shallow_advance_max_end_time=0, build_scorer_start_time=0, next_doc_count=2, shallow_advance_start_time=0, set_min_competitive_score_min_start_time=0, compute_max_score=0, create_weight_min_start_time=1629732014278990, build_scorer_count=2, create_weight=201692, score_min_start_time=1629732030778977, match_max_end_time=0, advance_min_start_time=1629732030776129, score_max_end_time=1629732030780027, build_scorer_min_start_time=1629732030749745}",
map.toString()
);
}
}

0 comments on commit 777a7bb

Please sign in to comment.