Skip to content

Commit

Permalink
Created QueryShapeService
Browse files Browse the repository at this point in the history
Signed-off-by: David Zane <[email protected]>
  • Loading branch information
dzane17 committed Jul 24, 2024
1 parent f6879c8 commit 2f995db
Show file tree
Hide file tree
Showing 2 changed files with 147 additions and 116 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.plugin.insights.core.service.categorizer;

import org.opensearch.index.query.QueryBuilder;
import org.opensearch.search.aggregations.AggregationBuilder;
import org.opensearch.search.aggregations.AggregatorFactories;
import org.opensearch.search.aggregations.PipelineAggregationBuilder;
import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.search.sort.FieldSortBuilder;
import org.opensearch.search.sort.SortBuilder;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;

class QueryShapeService {
static final String TWO_SPACE_INDENT = " ";

static String buildShape(SearchSourceBuilder source, Boolean showFields) {
StringBuilder shape = new StringBuilder();
shape.append(buildQueryShape(source.query()));
shape.append(buildAggregationShape(source.aggregations(), showFields));
shape.append(buildSortShape(source.sorts(), showFields));
return shape.toString();
}

static String buildQueryShape(QueryBuilder queryBuilder) {
if (queryBuilder == null) {
return "";
}
QueryShapeVisitor shapeVisitor = new QueryShapeVisitor();
queryBuilder.visit(shapeVisitor);
return shapeVisitor.prettyPrintTree("");
}

static String buildAggregationShape(AggregatorFactories.Builder aggregationsBuilder, Boolean showFields) {
if (aggregationsBuilder == null) {
return "";
}
StringBuilder aggregationShape = recursiveAggregationShapeBuilder(
aggregationsBuilder.getAggregatorFactories(),
aggregationsBuilder.getPipelineAggregatorFactories(),
new StringBuilder(),
0,
showFields
);
return aggregationShape.toString();
}

static StringBuilder recursiveAggregationShapeBuilder(
Collection<AggregationBuilder> aggregationBuilders,
Collection<PipelineAggregationBuilder> pipelineAggregations,
StringBuilder outputBuilder,
int indentCount,
Boolean showFields
) {
String baseIndent = TWO_SPACE_INDENT.repeat(indentCount);

//// Normal Aggregations ////
if (aggregationBuilders.isEmpty() == false) {
outputBuilder.append(baseIndent).append("aggregation:").append("\n");
}
List<String> aggShapeStrings = new ArrayList<>();
for (AggregationBuilder agg : aggregationBuilders) {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(baseIndent).append(TWO_SPACE_INDENT).append(agg.getType());
if (showFields) {
if (agg instanceof ValuesSourceAggregationBuilder) {
stringBuilder.append(" [").append(((ValuesSourceAggregationBuilder<?>) agg).field()).append("]");
} else {
stringBuilder.append(" []");
}
}
stringBuilder.append("\n");

if (agg.getSubAggregations().isEmpty() == false) {
// Recursive call on sub-aggregations
recursiveAggregationShapeBuilder(
agg.getSubAggregations(),
agg.getPipelineAggregations(),
stringBuilder,
indentCount + 2,
showFields
);
}
aggShapeStrings.add(stringBuilder.toString());
}

// Sort aggregations
Collections.sort(aggShapeStrings);
for (String shapeString : aggShapeStrings) {
outputBuilder.append(shapeString);
}

//// Pipeline Aggregation (cannot have sub-aggregations) ////
if (pipelineAggregations.isEmpty() == false) {
outputBuilder.append(baseIndent).append("pipeline aggregation:").append("\n");
}
for (PipelineAggregationBuilder pipelineAgg : pipelineAggregations) {
outputBuilder.append(baseIndent).append(TWO_SPACE_INDENT).append(pipelineAgg.getName()).append("\n");
}

return outputBuilder;
}

static String buildSortShape(List<SortBuilder<?>> sortBuilderList, Boolean showFields) {
if (sortBuilderList == null || sortBuilderList.isEmpty()) {
return "";
}
StringBuilder sortShape = new StringBuilder();
sortShape.append("sort:\n");

List<String> shapeStrings = new ArrayList<>();
for (SortBuilder<?> sortBuilder : sortBuilderList) {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(TWO_SPACE_INDENT).append(sortBuilder.order());
if (showFields) {
if (sortBuilder instanceof FieldSortBuilder) {
stringBuilder.append(" [").append(((FieldSortBuilder) sortBuilder).getFieldName()).append("]");
} else {
stringBuilder.append(" []");
}
}
shapeStrings.add(stringBuilder.toString());
}

Collections.sort(shapeStrings);
for (String line : shapeStrings) {
sortShape.append(line).append("\n");
}
return sortShape.toString();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,12 @@
import org.opensearch.plugin.insights.rules.model.Attribute;
import org.opensearch.plugin.insights.rules.model.MetricType;
import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
import org.opensearch.search.aggregations.AggregationBuilder;
import org.opensearch.search.aggregations.AggregatorFactories;
import org.opensearch.search.aggregations.PipelineAggregationBuilder;
import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.search.sort.FieldSortBuilder;
import org.opensearch.search.sort.SortBuilder;
import org.opensearch.telemetry.metrics.MetricsRegistry;
import org.opensearch.telemetry.metrics.tags.Tags;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;

Expand All @@ -38,7 +31,7 @@
public final class SearchQueryCategorizer {

private static final Logger logger = LogManager.getLogger(SearchQueryCategorizer.class);
public static final String TWO_SPACE_INDENT = " ";

/**
* Contains all the search query counters
*/
Expand Down Expand Up @@ -91,13 +84,12 @@ public void categorize(SearchQueryRecord record) {
SearchSourceBuilder source = (SearchSourceBuilder) record.getAttributes().get(Attribute.SOURCE);
Map<MetricType, Number> measurements = record.getMeasurements();

QueryBuilder topLevelQueryBuilder = source.query();
logQueryShape(topLevelQueryBuilder);
incrementQueryTypeCounters(topLevelQueryBuilder, measurements);
incrementQueryTypeCounters(source.query(), measurements);
incrementQueryAggregationCounters(source.aggregations(), measurements);
logAggregationsShape(source.aggregations(), true);
incrementQuerySortCounters(source.sorts(), measurements);
logSortShape(source.sorts(), true);

String searchShape = QueryShapeService.buildShape(source, true);
logger.trace(searchShape);
}

private void incrementQuerySortCounters(List<SortBuilder<?>> sorts, Map<MetricType, Number> measurements) {
Expand Down Expand Up @@ -125,59 +117,6 @@ private void incrementQueryTypeCounters(QueryBuilder topLevelQueryBuilder, Map<M
topLevelQueryBuilder.visit(searchQueryVisitor);
}

private void logQueryShape(QueryBuilder topLevelQueryBuilder) {
if (logger.isTraceEnabled()) {
if (topLevelQueryBuilder == null) {
return;
}
QueryShapeVisitor shapeVisitor = new QueryShapeVisitor();
topLevelQueryBuilder.visit(shapeVisitor);
logger.trace(shapeVisitor.prettyPrintTree(TWO_SPACE_INDENT));
}
}

private void logAggregationsShape(AggregatorFactories.Builder aggregationsBuilder, Boolean showFields) {
if (aggregationsBuilder == null) {
return;
}
StringBuilder aggregationShape = getAggregationShape(
aggregationsBuilder.getAggregatorFactories(),
aggregationsBuilder.getPipelineAggregatorFactories(),
new StringBuilder(),
0,
showFields
);
logger.trace(aggregationShape.toString());
}

private void logSortShape(List<SortBuilder<?>> sortBuilderList, Boolean showFields) {
if (sortBuilderList == null || sortBuilderList.isEmpty()) {
return;
}
StringBuilder sortShape = new StringBuilder();
sortShape.append("sort:\n");

List<String> shapeStrings = new ArrayList<>();
for (SortBuilder<?> sortBuilder : sortBuilderList) {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(TWO_SPACE_INDENT).append(sortBuilder.order());
if (showFields) {
if (sortBuilder instanceof FieldSortBuilder) {
stringBuilder.append(" [").append(((FieldSortBuilder) sortBuilder).getFieldName()).append("]");
} else {
stringBuilder.append(" []");
}
}
shapeStrings.add(stringBuilder.toString());
}

Collections.sort(shapeStrings);
for (String line : shapeStrings) {
sortShape.append(line).append("\n");
}
logger.trace(sortShape.toString());
}

/**
* Get search query counters
* @return search query counters
Expand All @@ -194,54 +133,4 @@ public void reset() {
instance = null;
}
}

public StringBuilder getAggregationShape(
Collection<AggregationBuilder> aggregationBuilders,
Collection<PipelineAggregationBuilder> pipelineAggregations,
StringBuilder outputBuilder,
int indentCount,
Boolean showFields
) {
String baseIndent = TWO_SPACE_INDENT.repeat(indentCount);

//// Normal Aggregations ////
if (aggregationBuilders.isEmpty() == false) {
outputBuilder.append(baseIndent).append("aggregation:").append("\n");
}
List<String> aggShapeStrings = new ArrayList<>();
for (AggregationBuilder agg : aggregationBuilders) {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(baseIndent).append(TWO_SPACE_INDENT).append(agg.getType());
if (showFields) {
if (agg instanceof ValuesSourceAggregationBuilder) {
stringBuilder.append(" [").append(((ValuesSourceAggregationBuilder<?>) agg).field()).append("]");
} else {
stringBuilder.append(" []");
}
}
stringBuilder.append("\n");

if (agg.getSubAggregations().isEmpty() == false) {
// Recursive call on sub-aggregations
stringBuilder = getAggregationShape(agg.getSubAggregations(), agg.getPipelineAggregations(), stringBuilder, indentCount + 2, showFields);
}
aggShapeStrings.add(stringBuilder.toString());
}

// Sort aggregations
Collections.sort(aggShapeStrings);
for (String shapeString : aggShapeStrings) {
outputBuilder.append(shapeString);
}

//// Pipeline Aggregation (cannot have sub-aggregations) ////
if (pipelineAggregations.isEmpty() == false) {
outputBuilder.append(baseIndent).append("pipeline aggregation:").append("\n");
}
for (PipelineAggregationBuilder pipelineAgg : pipelineAggregations) {
outputBuilder.append(baseIndent).append(TWO_SPACE_INDENT).append(pipelineAgg.getName()).append("\n");
}

return outputBuilder;
}
}

0 comments on commit 2f995db

Please sign in to comment.