+
+
+
+ Coverage Report > org.opensearch.search.aggregations.bucket.terms
+
+
+
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms |
+
+
+ 27.3%
+
+
+ (47/172)
+
+ |
+
+
+ 14%
+
+
+ (164/1175)
+
+ |
+
+
+ 15.5%
+
+
+ (607/3916)
+
+ |
+
+
+
+
+
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms |
+
+
+ 27.3%
+
+
+ (47/172)
+
+ |
+
+
+ 14%
+
+
+ (164/1175)
+
+ |
+
+
+ 15.5%
+
+
+ (607/3916)
+
+ |
+
+
+
+
+
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms |
+
+
+ 27.3%
+
+
+ (47/172)
+
+ |
+
+
+ 14%
+
+
+ (164/1175)
+
+ |
+
+
+ 15.5%
+
+
+ (607/3916)
+
+ |
+
+
+
+
+
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms |
+
+
+ 27.3%
+
+
+ (47/172)
+
+ |
+
+
+ 14%
+
+
+ (164/1175)
+
+ |
+
+
+ 15.5%
+
+
+ (607/3916)
+
+ |
+
+
+
+
+
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms |
+
+
+ 27.3%
+
+
+ (47/172)
+
+ |
+
+
+ 14%
+
+
+ (164/1175)
+
+ |
+
+
+ 15.5%
+
+
+ (607/3916)
+
+ |
+
+
+
+
+
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms |
+
+
+ 27.3%
+
+
+ (47/172)
+
+ |
+
+
+ 14%
+
+
+ (164/1175)
+
+ |
+
+
+ 15.5%
+
+
+ (607/3916)
+
+ |
+
+
+
+
+
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms |
+
+
+ 27.3%
+
+
+ (47/172)
+
+ |
+
+
+ 14%
+
+
+ (164/1175)
+
+ |
+
+
+ 15.5%
+
+
+ (607/3916)
+
+ |
+
+
+
+
+
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms |
+
+
+ 27.3%
+
+
+ (47/172)
+
+ |
+
+
+ 14%
+
+
+ (164/1175)
+
+ |
+
+
+ 15.5%
+
+
+ (607/3916)
+
+ |
+
+
+
+
+
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms |
+
+
+ 27.3%
+
+
+ (47/172)
+
+ |
+
+
+ 14%
+
+
+ (164/1175)
+
+ |
+
+
+ 15.5%
+
+
+ (607/3916)
+
+ |
+
+
+
+
+
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms |
+
+
+ 27.3%
+
+
+ (47/172)
+
+ |
+
+
+ 14%
+
+
+ (164/1175)
+
+ |
+
+
+ 15.5%
+
+
+ (607/3916)
+
+ |
+
+
+
+
+
+
+
+
+
+
+
+
Coverage Summary for Class: AbstractRareTermsAggregator (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | AbstractRareTermsAggregator |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ 0%
+
+
+ (0/7)
+
+ |
+
+
+ 0%
+
+
+ (0/26)
+
+ |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.common.util.SetBackedScalingCuckooFilter;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.bucket.DeferableBucketAggregator;
+ import org.opensearch.search.aggregations.bucket.DeferringBucketCollector;
+ import org.opensearch.search.aggregations.bucket.MergingBucketsDeferringCollector;
+ import org.opensearch.search.aggregations.bucket.nested.NestedAggregator;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.util.Map;
+ import java.util.Random;
+
+ /**
+ * Base class to Aggregate all docs that contain rare terms
+ *
+ * @opensearch.internal
+ */
+ public abstract class AbstractRareTermsAggregator extends DeferableBucketAggregator {
+
+ static final BucketOrder ORDER = BucketOrder.compound(BucketOrder.count(true), BucketOrder.key(true)); // sort by count ascending
+
+ protected final long maxDocCount;
+ private final double precision;
+ protected final DocValueFormat format;
+ private final int filterSeed;
+
+ protected MergingBucketsDeferringCollector deferringCollector;
+
+ AbstractRareTermsAggregator(
+ String name,
+ AggregatorFactories factories,
+ SearchContext context,
+ Aggregator parent,
+ Map<String, Object> metadata,
+ long maxDocCount,
+ double precision,
+ DocValueFormat format
+ ) throws IOException {
+ super(name, factories, context, parent, metadata);
+
+ this.maxDocCount = maxDocCount;
+ this.precision = precision;
+ this.format = format;
+ // We seed the rng with the ShardID so results are deterministic and don't change randomly
+ this.filterSeed = context.indexShard().shardId().hashCode();
+ String scoringAgg = subAggsNeedScore();
+ String nestedAgg = descendsFromNestedAggregator(parent);
+ if (scoringAgg != null && nestedAgg != null) {
+ /*
+ * Terms agg would force the collect mode to depth_first here, because
+ * we need to access the score of nested documents in a sub-aggregation
+ * and we are not able to generate this score while replaying deferred documents.
+ *
+ * But the RareTerms agg _must_ execute in breadth first since it relies on
+ * deferring execution, so we just have to throw up our hands and refuse
+ */
+ throw new IllegalStateException(
+ "RareTerms agg ["
+ + name()
+ + "] is the child of the nested agg ["
+ + nestedAgg
+ + "], and also has a scoring child agg ["
+ + scoringAgg
+ + "]. This combination is not supported because "
+ + "it requires executing in [depth_first] mode, which the RareTerms agg cannot do."
+ );
+ }
+ }
+
+ protected SetBackedScalingCuckooFilter newFilter() {
+ SetBackedScalingCuckooFilter filter = new SetBackedScalingCuckooFilter(10000, new Random(filterSeed), precision);
+ filter.registerBreaker(this::addRequestCircuitBreakerBytes);
+ return filter;
+ }
+
+ @Override
+ protected boolean shouldDefer(Aggregator aggregator) {
+ return true;
+ }
+
+ @Override
+ public DeferringBucketCollector getDeferringCollector() {
+ deferringCollector = new MergingBucketsDeferringCollector(context, descendsFromGlobalAggregator(parent()));
+ return deferringCollector;
+ }
+
+ private String subAggsNeedScore() {
+ for (Aggregator subAgg : subAggregators) {
+ if (subAgg.scoreMode().needsScores()) {
+ return subAgg.name();
+ }
+ }
+ return null;
+ }
+
+ private String descendsFromNestedAggregator(Aggregator parent) {
+ while (parent != null) {
+ if (parent.getClass() == NestedAggregator.class) {
+ return parent.name();
+ }
+ parent = parent.parent();
+ }
+ return null;
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: LongKeyedBucketOrds (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | LongKeyedBucketOrds |
+
+
+ 100%
+
+
+ (2/2)
+
+ |
+
+
+ 100%
+
+
+ (2/2)
+
+ |
+
+
+ | LongKeyedBucketOrds$BucketOrdsEnum |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ | LongKeyedBucketOrds$BucketOrdsEnum$1 |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ | LongKeyedBucketOrds$FromMany |
+
+
+ 0%
+
+
+ (0/9)
+
+ |
+
+
+ 0%
+
+
+ (0/17)
+
+ |
+
+
+ | LongKeyedBucketOrds$FromMany$1 |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ | LongKeyedBucketOrds$FromSingle |
+
+
+ 50%
+
+
+ (5/10)
+
+ |
+
+
+ 53.3%
+
+
+ (8/15)
+
+ |
+
+
+ | LongKeyedBucketOrds$FromSingle$1 |
+
+
+ 100%
+
+
+ (4/4)
+
+ |
+
+
+ 100%
+
+
+ (9/9)
+
+ |
+
+
+ | Total |
+
+
+ 32.4%
+
+
+ (11/34)
+
+ |
+
+
+ 32.8%
+
+
+ (19/58)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.common.lease.Releasable;
+ import org.opensearch.common.util.BigArrays;
+ import org.opensearch.common.util.LongLongHash;
+ import org.opensearch.common.util.ReorganizingLongHash;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+
+ /**
+ * Maps long bucket keys to bucket ordinals.
+ *
+ * @opensearch.internal
+ */
+ public abstract class LongKeyedBucketOrds implements Releasable {
+ /**
+ * Build a {@link LongKeyedBucketOrds}.
+ */
+ public static LongKeyedBucketOrds build(BigArrays bigArrays, CardinalityUpperBound cardinality) {
+ return cardinality.map(estimate -> estimate < 2 ? new FromSingle(bigArrays) : new FromMany(bigArrays));
+ }
+
+ private LongKeyedBucketOrds() {}
+
+ /**
+ * Add the {@code owningBucketOrd, value} pair. Return the ord for
+ * their bucket if they have yet to be added, or {@code -1-ord}
+ * if they were already present.
+ */
+ public abstract long add(long owningBucketOrd, long value);
+
+ /**
+ * Count the buckets in {@code owningBucketOrd}.
+ * <p>
+ * Some aggregations expect this to be fast but most wouldn't
+ * mind particularly if it weren't.
+ */
+ public abstract long bucketsInOrd(long owningBucketOrd);
+
+ /**
+ * Find the {@code owningBucketOrd, value} pair. Return the ord for
+ * their bucket if they have been added or {@code -1} if they haven't.
+ */
+ public abstract long find(long owningBucketOrd, long value);
+
+ /**
+ * Returns the value currently associated with the bucket ordinal
+ */
+ public abstract long get(long ordinal);
+
+ /**
+ * The number of collected buckets.
+ */
+ public abstract long size();
+
+ /**
+ * The maximum possible used {@code owningBucketOrd}.
+ */
+ public abstract long maxOwningBucketOrd();
+
+ /**
+ * Build an iterator for buckets inside {@code owningBucketOrd} in order
+ * of increasing ord.
+ * <p>
+ * When this is first returns it is "unpositioned" and you must call
+ * {@link BucketOrdsEnum#next()} to move it to the first value.
+ */
+ public abstract BucketOrdsEnum ordsEnum(long owningBucketOrd);
+
+ /**
+ * An iterator for buckets inside a particular {@code owningBucketOrd}.
+ *
+ * @opensearch.internal
+ */
+ public interface BucketOrdsEnum {
+ /**
+ * Advance to the next value.
+ * @return {@code true} if there *is* a next value,
+ * {@code false} if there isn't
+ */
+ boolean next();
+
+ /**
+ * The ordinal of the current value.
+ */
+ long ord();
+
+ /**
+ * The current value.
+ */
+ long value();
+
+ /**
+ * An {@linkplain BucketOrdsEnum} that is empty.
+ */
+ BucketOrdsEnum EMPTY = new BucketOrdsEnum() {
+ @Override
+ public boolean next() {
+ return false;
+ }
+
+ @Override
+ public long ord() {
+ return 0;
+ }
+
+ @Override
+ public long value() {
+ return 0;
+ }
+ };
+ }
+
+ /**
+ * Implementation that only works if it is collecting from a single bucket.
+ *
+ * @opensearch.internal
+ */
+ public static class FromSingle extends LongKeyedBucketOrds {
+ private final ReorganizingLongHash ords;
+
+ public FromSingle(BigArrays bigArrays) {
+ ords = new ReorganizingLongHash(bigArrays);
+ }
+
+ @Override
+ public long add(long owningBucketOrd, long value) {
+ // This is in the critical path for collecting most aggs. Be careful of performance.
+ assert owningBucketOrd == 0;
+ return ords.add(value);
+ }
+
+ @Override
+ public long find(long owningBucketOrd, long value) {
+ assert owningBucketOrd == 0;
+ return ords.find(value);
+ }
+
+ @Override
+ public long get(long ordinal) {
+ return ords.get(ordinal);
+ }
+
+ @Override
+ public long bucketsInOrd(long owningBucketOrd) {
+ assert owningBucketOrd == 0;
+ return ords.size();
+ }
+
+ @Override
+ public long size() {
+ return ords.size();
+ }
+
+ @Override
+ public long maxOwningBucketOrd() {
+ return 0;
+ }
+
+ @Override
+ public BucketOrdsEnum ordsEnum(long owningBucketOrd) {
+ assert owningBucketOrd == 0;
+ return new BucketOrdsEnum() {
+ private long ord = -1;
+ private long value;
+
+ @Override
+ public boolean next() {
+ ord++;
+ if (ord >= ords.size()) {
+ return false;
+ }
+ value = ords.get(ord);
+ return true;
+ }
+
+ @Override
+ public long value() {
+ return value;
+ }
+
+ @Override
+ public long ord() {
+ return ord;
+ }
+ };
+ }
+
+ @Override
+ public void close() {
+ ords.close();
+ }
+ }
+
+ /**
+ * Implementation that works properly when collecting from many buckets.
+ *
+ * @opensearch.internal
+ */
+ public static class FromMany extends LongKeyedBucketOrds {
+ private final LongLongHash ords;
+
+ public FromMany(BigArrays bigArrays) {
+ ords = new LongLongHash(2, bigArrays);
+ }
+
+ @Override
+ public long add(long owningBucketOrd, long value) {
+ // This is in the critical path for collecting most aggs. Be careful of performance.
+ return ords.add(owningBucketOrd, value);
+ }
+
+ @Override
+ public long find(long owningBucketOrd, long value) {
+ return ords.find(owningBucketOrd, value);
+ }
+
+ @Override
+ public long get(long ordinal) {
+ return ords.getKey2(ordinal);
+ }
+
+ @Override
+ public long bucketsInOrd(long owningBucketOrd) {
+ // TODO it'd be faster to count the number of buckets in a list of these ords rather than one at a time
+ long count = 0;
+ for (long i = 0; i < ords.size(); i++) {
+ if (ords.getKey1(i) == owningBucketOrd) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ @Override
+ public long size() {
+ return ords.size();
+ }
+
+ @Override
+ public long maxOwningBucketOrd() {
+ // TODO this is fairly expensive to compute. Can we avoid needing it?
+ long max = -1;
+ for (long i = 0; i < ords.size(); i++) {
+ max = Math.max(max, ords.getKey1(i));
+ }
+ return max;
+ }
+
+ @Override
+ public BucketOrdsEnum ordsEnum(long owningBucketOrd) {
+ // TODO it'd be faster to iterate many ords at once rather than one at a time
+ return new BucketOrdsEnum() {
+ private long ord = -1;
+ private long value;
+
+ @Override
+ public boolean next() {
+ while (true) {
+ ord++;
+ if (ord >= ords.size()) {
+ return false;
+ }
+ if (ords.getKey1(ord) == owningBucketOrd) {
+ value = ords.getKey2(ord);
+ return true;
+ }
+ }
+ }
+
+ @Override
+ public long value() {
+ return value;
+ }
+
+ @Override
+ public long ord() {
+ return ord;
+ }
+ };
+ }
+
+ @Override
+ public void close() {
+ ords.close();
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: LongRareTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | LongRareTerms |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ | LongRareTerms$Bucket |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/15)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/20)
+
+ |
+
+
+ 0%
+
+
+ (0/25)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.common.util.SetBackedScalingCuckooFilter;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.InternalAggregations;
+
+ import java.io.IOException;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+
+ /**
+ * Result of the RareTerms aggregation when the field is some kind of whole number like a integer, long, or a date.
+ *
+ * @opensearch.internal
+ */
+ public class LongRareTerms extends InternalMappedRareTerms<LongRareTerms, LongRareTerms.Bucket> {
+ public static final String NAME = "lrareterms";
+
+ /**
+ * Bucket for rare long valued terms
+ *
+ * @opensearch.internal
+ */
+ public static class Bucket extends InternalRareTerms.Bucket<Bucket> {
+ long term;
+
+ public Bucket(long term, long docCount, InternalAggregations aggregations, DocValueFormat format) {
+ super(docCount, aggregations, format);
+ this.term = term;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public Bucket(StreamInput in, DocValueFormat format) throws IOException {
+ super(in, format);
+ term = in.readLong();
+ }
+
+ @Override
+ protected void writeTermTo(StreamOutput out) throws IOException {
+ out.writeLong(term);
+ }
+
+ @Override
+ public String getKeyAsString() {
+ return format.format(term).toString();
+ }
+
+ @Override
+ public Object getKey() {
+ return term;
+ }
+
+ @Override
+ public Number getKeyAsNumber() {
+ return term;
+ }
+
+ @Override
+ public int compareKey(Bucket other) {
+ return Long.compare(term, other.term);
+ }
+
+ @Override
+ protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ builder.field(CommonFields.KEY.getPreferredName(), term);
+ if (format != DocValueFormat.RAW) {
+ builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term).toString());
+ }
+ return builder;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj) && Objects.equals(term, ((Bucket) obj).term);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), term);
+ }
+ }
+
+ LongRareTerms(
+ String name,
+ BucketOrder order,
+ Map<String, Object> metadata,
+ DocValueFormat format,
+ List<LongRareTerms.Bucket> buckets,
+ long maxDocCount,
+ SetBackedScalingCuckooFilter filter
+ ) {
+ super(name, order, metadata, format, buckets, maxDocCount, filter);
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public LongRareTerms(StreamInput in) throws IOException {
+ super(in, LongRareTerms.Bucket::new);
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public LongRareTerms create(List<LongRareTerms.Bucket> buckets) {
+ return new LongRareTerms(name, order, metadata, format, buckets, maxDocCount, filter);
+ }
+
+ @Override
+ public LongRareTerms.Bucket createBucket(InternalAggregations aggregations, LongRareTerms.Bucket prototype) {
+ return new LongRareTerms.Bucket(prototype.term, prototype.getDocCount(), aggregations, prototype.format);
+ }
+
+ @Override
+ protected LongRareTerms createWithFilter(String name, List<LongRareTerms.Bucket> buckets, SetBackedScalingCuckooFilter filter) {
+ return new LongRareTerms(name, order, getMetadata(), format, buckets, maxDocCount, filter);
+ }
+
+ @Override
+ protected LongRareTerms.Bucket[] createBucketsArray(int size) {
+ return new LongRareTerms.Bucket[size];
+ }
+
+ @Override
+ public boolean containsTerm(SetBackedScalingCuckooFilter filter, LongRareTerms.Bucket bucket) {
+ return filter.mightContain((long) bucket.getKey());
+ }
+
+ @Override
+ public void addToFilter(SetBackedScalingCuckooFilter filter, LongRareTerms.Bucket bucket) {
+ filter.add((long) bucket.getKey());
+ }
+
+ @Override
+ Bucket createBucket(long docCount, InternalAggregations aggs, LongRareTerms.Bucket prototype) {
+ return new Bucket(prototype.term, docCount, aggs, format);
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: LongRareTermsAggregator (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | LongRareTermsAggregator |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ 0%
+
+
+ (0/46)
+
+ |
+
+
+ | LongRareTermsAggregator$1 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/16)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ 0%
+
+
+ (0/62)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.index.LeafReaderContext;
+ import org.apache.lucene.index.SortedNumericDocValues;
+ import org.opensearch.common.lease.Releasables;
+ import org.opensearch.common.util.LongHash;
+ import org.opensearch.common.util.SetBackedScalingCuckooFilter;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.LeafBucketCollector;
+ import org.opensearch.search.aggregations.LeafBucketCollectorBase;
+ import org.opensearch.search.aggregations.support.ValuesSource;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.List;
+ import java.util.Map;
+
+ import static java.util.Collections.emptyList;
+
+ /**
+ * An aggregator that finds "rare" string values (e.g. terms agg that orders ascending)
+ *
+ * @opensearch.internal
+ */
+ public class LongRareTermsAggregator extends AbstractRareTermsAggregator {
+ private final ValuesSource.Numeric valuesSource;
+ private final IncludeExclude.LongFilter filter;
+ private final LongKeyedBucketOrds bucketOrds;
+
+ LongRareTermsAggregator(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource.Numeric valuesSource,
+ DocValueFormat format,
+ SearchContext aggregationContext,
+ Aggregator parent,
+ IncludeExclude.LongFilter filter,
+ int maxDocCount,
+ double precision,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+ super(name, factories, aggregationContext, parent, metadata, maxDocCount, precision, format);
+ this.valuesSource = valuesSource;
+ this.filter = filter;
+ this.bucketOrds = LongKeyedBucketOrds.build(context.bigArrays(), cardinality);
+ }
+
+ protected SortedNumericDocValues getValues(ValuesSource.Numeric valuesSource, LeafReaderContext ctx) throws IOException {
+ return valuesSource.longValues(ctx);
+ }
+
+ @Override
+ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
+ SortedNumericDocValues values = getValues(valuesSource, ctx);
+ return new LeafBucketCollectorBase(sub, values) {
+ @Override
+ public void collect(int docId, long owningBucketOrd) throws IOException {
+ if (false == values.advanceExact(docId)) {
+ return;
+ }
+ int valuesCount = values.docValueCount();
+ long previous = Long.MAX_VALUE;
+ for (int i = 0; i < valuesCount; ++i) {
+ long val = values.nextValue();
+ if (i == 0 && previous == val) {
+ continue;
+ }
+ previous = val;
+ if (filter != null && false == filter.accept(val)) {
+ continue;
+ }
+ long bucketOrdinal = bucketOrds.add(owningBucketOrd, val);
+ if (bucketOrdinal < 0) { // already seen
+ bucketOrdinal = -1 - bucketOrdinal;
+ collectExistingBucket(sub, docId, bucketOrdinal);
+ } else {
+ collectBucket(sub, docId, bucketOrdinal);
+ }
+ }
+ }
+ };
+ }
+
+ @Override
+ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException {
+ /*
+ * Collect the list of buckets, populate the filter with terms
+ * that are too frequent, and figure out how to merge sub-buckets.
+ */
+ LongRareTerms.Bucket[][] rarestPerOrd = new LongRareTerms.Bucket[owningBucketOrds.length][];
+ SetBackedScalingCuckooFilter[] filters = new SetBackedScalingCuckooFilter[owningBucketOrds.length];
+ long keepCount = 0;
+ long[] mergeMap = new long[(int) bucketOrds.size()];
+ Arrays.fill(mergeMap, -1);
+ long offset = 0;
+ for (int owningOrdIdx = 0; owningOrdIdx < owningBucketOrds.length; owningOrdIdx++) {
+ try (LongHash bucketsInThisOwningBucketToCollect = new LongHash(1, context.bigArrays())) {
+ filters[owningOrdIdx] = newFilter();
+ List<LongRareTerms.Bucket> builtBuckets = new ArrayList<>();
+ LongKeyedBucketOrds.BucketOrdsEnum collectedBuckets = bucketOrds.ordsEnum(owningBucketOrds[owningOrdIdx]);
+ while (collectedBuckets.next()) {
+ long docCount = bucketDocCount(collectedBuckets.ord());
+ // if the key is below threshold, reinsert into the new ords
+ if (docCount <= maxDocCount) {
+ LongRareTerms.Bucket bucket = new LongRareTerms.Bucket(collectedBuckets.value(), docCount, null, format);
+ bucket.bucketOrd = offset + bucketsInThisOwningBucketToCollect.add(collectedBuckets.value());
+ mergeMap[(int) collectedBuckets.ord()] = bucket.bucketOrd;
+ builtBuckets.add(bucket);
+ keepCount++;
+ } else {
+ filters[owningOrdIdx].add(collectedBuckets.value());
+ }
+ }
+ rarestPerOrd[owningOrdIdx] = builtBuckets.toArray(new LongRareTerms.Bucket[0]);
+ offset += bucketsInThisOwningBucketToCollect.size();
+ }
+ }
+
+ /*
+ * Only merge/delete the ordinals if we have actually deleted one,
+ * to save on some redundant work.
+ */
+ if (keepCount != mergeMap.length) {
+ mergeBuckets(mergeMap, offset);
+ if (deferringCollector != null) {
+ deferringCollector.mergeBuckets(mergeMap);
+ }
+ }
+
+ /*
+ * Now build the results!
+ */
+ buildSubAggsForAllBuckets(rarestPerOrd, b -> b.bucketOrd, (b, aggs) -> b.aggregations = aggs);
+ InternalAggregation[] result = new InternalAggregation[owningBucketOrds.length];
+ for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) {
+ Arrays.sort(rarestPerOrd[ordIdx], ORDER.comparator());
+ result[ordIdx] = new LongRareTerms(
+ name,
+ ORDER,
+ metadata(),
+ format,
+ Arrays.asList(rarestPerOrd[ordIdx]),
+ maxDocCount,
+ filters[ordIdx]
+ );
+ }
+ return result;
+ }
+
+ @Override
+ public InternalAggregation buildEmptyAggregation() {
+ return new LongRareTerms(name, ORDER, metadata(), format, emptyList(), 0, newFilter());
+ }
+
+ @Override
+ public void doClose() {
+ Releasables.close(bucketOrds);
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: LongTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | LongTerms |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/44)
+
+ |
+
+
+ | LongTerms$Bucket |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/21)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/20)
+
+ |
+
+
+ 0%
+
+
+ (0/65)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.InternalAggregations;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+
+ /**
+ * Result of the {@link TermsAggregator} when the field is some kind of whole number like a integer, long, or a date.
+ *
+ * @opensearch.internal
+ */
+ public class LongTerms extends InternalMappedTerms<LongTerms, LongTerms.Bucket> {
+ public static final String NAME = "lterms";
+
+ /**
+ * Bucket for long terms
+ *
+ * @opensearch.internal
+ */
+ public static class Bucket extends InternalTerms.Bucket<Bucket> {
+ long term;
+
+ public Bucket(
+ long term,
+ long docCount,
+ InternalAggregations aggregations,
+ boolean showDocCountError,
+ long docCountError,
+ DocValueFormat format
+ ) {
+ super(docCount, aggregations, showDocCountError, docCountError, format);
+ this.term = term;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public Bucket(StreamInput in, DocValueFormat format, boolean showDocCountError) throws IOException {
+ super(in, format, showDocCountError);
+ term = in.readLong();
+ }
+
+ @Override
+ protected void writeTermTo(StreamOutput out) throws IOException {
+ out.writeLong(term);
+ }
+
+ @Override
+ public String getKeyAsString() {
+ return format.format(term).toString();
+ }
+
+ @Override
+ public Object getKey() {
+ if (format == DocValueFormat.UNSIGNED_LONG_SHIFTED) {
+ return format.format(term);
+ } else {
+ return term;
+ }
+ }
+
+ @Override
+ public Number getKeyAsNumber() {
+ if (format == DocValueFormat.UNSIGNED_LONG_SHIFTED) {
+ return (Number) format.format(term);
+ } else {
+ return term;
+ }
+ }
+
+ @Override
+ public int compareKey(Bucket other) {
+ return Long.compare(term, other.term);
+ }
+
+ @Override
+ protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ if (format == DocValueFormat.UNSIGNED_LONG_SHIFTED) {
+ builder.field(CommonFields.KEY.getPreferredName(), format.format(term));
+ } else {
+ builder.field(CommonFields.KEY.getPreferredName(), term);
+ }
+ if (format != DocValueFormat.RAW && format != DocValueFormat.UNSIGNED_LONG_SHIFTED) {
+ builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term).toString());
+ }
+ return builder;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj) && Objects.equals(term, ((Bucket) obj).term);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), term);
+ }
+ }
+
+ public LongTerms(
+ String name,
+ BucketOrder reduceOrder,
+ BucketOrder order,
+ Map<String, Object> metadata,
+ DocValueFormat format,
+ int shardSize,
+ boolean showTermDocCountError,
+ long otherDocCount,
+ List<Bucket> buckets,
+ long docCountError,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds
+ ) {
+ super(
+ name,
+ reduceOrder,
+ order,
+ metadata,
+ format,
+ shardSize,
+ showTermDocCountError,
+ otherDocCount,
+ buckets,
+ docCountError,
+ bucketCountThresholds
+ );
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public LongTerms(StreamInput in) throws IOException {
+ super(in, Bucket::new);
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public LongTerms create(List<Bucket> buckets) {
+ return new LongTerms(
+ name,
+ reduceOrder,
+ order,
+ metadata,
+ format,
+ shardSize,
+ showTermDocCountError,
+ otherDocCount,
+ buckets,
+ docCountError,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) {
+ return new Bucket(
+ prototype.term,
+ prototype.docCount,
+ aggregations,
+ prototype.showDocCountError,
+ prototype.docCountError,
+ prototype.format
+ );
+ }
+
+ @Override
+ protected LongTerms create(String name, List<Bucket> buckets, BucketOrder reduceOrder, long docCountError, long otherDocCount) {
+ return new LongTerms(
+ name,
+ reduceOrder,
+ order,
+ getMetadata(),
+ format,
+ shardSize,
+ showTermDocCountError,
+ otherDocCount,
+ buckets,
+ docCountError,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ protected Bucket[] createBucketsArray(int size) {
+ return new Bucket[size];
+ }
+
+ @Override
+ public InternalAggregation reduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+ boolean unsignedLongFormat = false;
+ boolean rawFormat = false;
+ for (InternalAggregation agg : aggregations) {
+ if (agg instanceof DoubleTerms) {
+ return agg.reduce(aggregations, reduceContext);
+ }
+ if (agg instanceof LongTerms) {
+ if (((LongTerms) agg).format == DocValueFormat.RAW) {
+ rawFormat = true;
+ } else if (((LongTerms) agg).format == DocValueFormat.UNSIGNED_LONG_SHIFTED) {
+ unsignedLongFormat = true;
+ } else if (((LongTerms) agg).format == DocValueFormat.UNSIGNED_LONG) {
+ unsignedLongFormat = true;
+ }
+ }
+ }
+ if (rawFormat && unsignedLongFormat) { // if we have mixed formats, convert results to double format
+ List<InternalAggregation> newAggs = new ArrayList<>(aggregations.size());
+ for (InternalAggregation agg : aggregations) {
+ if (agg instanceof LongTerms) {
+ DoubleTerms dTerms = LongTerms.convertLongTermsToDouble((LongTerms) agg, format);
+ newAggs.add(dTerms);
+ } else {
+ newAggs.add(agg);
+ }
+ }
+ return newAggs.get(0).reduce(newAggs, reduceContext);
+ }
+ return super.reduce(aggregations, reduceContext);
+ }
+
+ @Override
+ Bucket createBucket(long docCount, InternalAggregations aggs, long docCountError, LongTerms.Bucket prototype) {
+ return new Bucket(prototype.term, docCount, aggs, prototype.showDocCountError, docCountError, format);
+ }
+
+ /**
+ * Converts a {@link LongTerms} into a {@link DoubleTerms}, returning the value of the specified long terms as doubles.
+ */
+ static DoubleTerms convertLongTermsToDouble(LongTerms longTerms, DocValueFormat decimalFormat) {
+ List<LongTerms.Bucket> buckets = longTerms.getBuckets();
+ List<DoubleTerms.Bucket> newBuckets = new ArrayList<>();
+ for (Terms.Bucket bucket : buckets) {
+ newBuckets.add(
+ new DoubleTerms.Bucket(
+ bucket.getKeyAsNumber().doubleValue(),
+ bucket.getDocCount(),
+ (InternalAggregations) bucket.getAggregations(),
+ longTerms.showTermDocCountError,
+ longTerms.showTermDocCountError ? bucket.getDocCountError() : 0,
+ decimalFormat
+ )
+ );
+ }
+ return new DoubleTerms(
+ longTerms.getName(),
+ longTerms.reduceOrder,
+ longTerms.order,
+ longTerms.metadata,
+ longTerms.format,
+ longTerms.shardSize,
+ longTerms.showTermDocCountError,
+ longTerms.otherDocCount,
+ newBuckets,
+ longTerms.docCountError,
+ longTerms.bucketCountThresholds
+ );
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: MapStringTermsAggregator (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | MapStringTermsAggregator |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/23)
+
+ |
+
+
+ | MapStringTermsAggregator$CollectConsumer |
+
+
+ | MapStringTermsAggregator$CollectorSource |
+
+
+ | MapStringTermsAggregator$ResultStrategy |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/31)
+
+ |
+
+
+ | MapStringTermsAggregator$SignificantTermsResults |
+
+
+ 0%
+
+
+ (0/14)
+
+ |
+
+
+ 0%
+
+
+ (0/28)
+
+ |
+
+
+ | MapStringTermsAggregator$SignificantTermsResults$1 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ | MapStringTermsAggregator$StandardTermsResults |
+
+
+ 0%
+
+
+ (0/14)
+
+ |
+
+
+ 0%
+
+
+ (0/36)
+
+ |
+
+
+ | MapStringTermsAggregator$ValuesSourceCollectorSource |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ | MapStringTermsAggregator$ValuesSourceCollectorSource$1 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/13)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/48)
+
+ |
+
+
+ 0%
+
+
+ (0/141)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.index.LeafReaderContext;
+ import org.apache.lucene.search.ScoreMode;
+ import org.apache.lucene.util.BytesRef;
+ import org.apache.lucene.util.BytesRefBuilder;
+ import org.apache.lucene.util.PriorityQueue;
+ import org.opensearch.common.lease.Releasable;
+ import org.opensearch.common.lease.Releasables;
+ import org.opensearch.common.util.LongArray;
+ import org.opensearch.index.fielddata.SortedBinaryDocValues;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.InternalMultiBucketAggregation;
+ import org.opensearch.search.aggregations.InternalOrder;
+ import org.opensearch.search.aggregations.LeafBucketCollector;
+ import org.opensearch.search.aggregations.LeafBucketCollectorBase;
+ import org.opensearch.search.aggregations.bucket.LocalBucketCountThresholds;
+ import org.opensearch.search.aggregations.bucket.terms.SignificanceLookup.BackgroundFrequencyForBytes;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+ import org.opensearch.search.aggregations.support.ValuesSource;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.util.Arrays;
+ import java.util.Map;
+ import java.util.function.BiConsumer;
+ import java.util.function.Function;
+ import java.util.function.LongConsumer;
+ import java.util.function.Supplier;
+
+ import static org.opensearch.search.aggregations.InternalOrder.isKeyOrder;
+
+ /**
+ * An aggregator of string values that hashes the strings on the fly rather
+ * than up front like the {@link GlobalOrdinalsStringTermsAggregator}.
+ *
+ * @opensearch.internal
+ */
+ public class MapStringTermsAggregator extends AbstractStringTermsAggregator {
+ private final CollectorSource collectorSource;
+ private final ResultStrategy<?, ?> resultStrategy;
+ private final BytesKeyedBucketOrds bucketOrds;
+ private final IncludeExclude.StringFilter includeExclude;
+
+ public MapStringTermsAggregator(
+ String name,
+ AggregatorFactories factories,
+ CollectorSource collectorSource,
+ Function<MapStringTermsAggregator, ResultStrategy<?, ?>> resultStrategy,
+ BucketOrder order,
+ DocValueFormat format,
+ BucketCountThresholds bucketCountThresholds,
+ IncludeExclude.StringFilter includeExclude,
+ SearchContext context,
+ Aggregator parent,
+ SubAggCollectionMode collectionMode,
+ boolean showTermDocCountError,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+ super(name, factories, context, parent, order, format, bucketCountThresholds, collectionMode, showTermDocCountError, metadata);
+ this.collectorSource = collectorSource;
+ this.resultStrategy = resultStrategy.apply(this); // ResultStrategy needs a reference to the Aggregator to do its job.
+ this.includeExclude = includeExclude;
+ bucketOrds = BytesKeyedBucketOrds.build(context.bigArrays(), cardinality);
+ }
+
+ @Override
+ public ScoreMode scoreMode() {
+ if (collectorSource.needsScores()) {
+ return ScoreMode.COMPLETE;
+ }
+ return super.scoreMode();
+ }
+
+ @Override
+ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
+ return resultStrategy.wrapCollector(
+ collectorSource.getLeafCollector(
+ includeExclude,
+ ctx,
+ sub,
+ this::addRequestCircuitBreakerBytes,
+ (s, doc, owningBucketOrd, bytes) -> {
+ long bucketOrdinal = bucketOrds.add(owningBucketOrd, bytes);
+ if (bucketOrdinal < 0) { // already seen
+ bucketOrdinal = -1 - bucketOrdinal;
+ collectExistingBucket(s, doc, bucketOrdinal);
+ } else {
+ collectBucket(s, doc, bucketOrdinal);
+ }
+ }
+ )
+ );
+ }
+
+ @Override
+ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException {
+ return resultStrategy.buildAggregations(owningBucketOrds);
+ }
+
+ @Override
+ public InternalAggregation buildEmptyAggregation() {
+ return resultStrategy.buildEmptyResult();
+ }
+
+ @Override
+ public void collectDebugInfo(BiConsumer<String, Object> add) {
+ super.collectDebugInfo(add);
+ add.accept("total_buckets", bucketOrds.size());
+ add.accept("result_strategy", resultStrategy.describe());
+ }
+
+ @Override
+ public void doClose() {
+ Releasables.close(collectorSource, resultStrategy, bucketOrds);
+ }
+
+ /**
+ * Abstaction on top of building collectors to fetch values.
+ *
+ * @opensearch.internal
+ */
+ public interface CollectorSource extends Releasable {
+ boolean needsScores();
+
+ LeafBucketCollector getLeafCollector(
+ IncludeExclude.StringFilter includeExclude,
+ LeafReaderContext ctx,
+ LeafBucketCollector sub,
+ LongConsumer addRequestCircuitBreakerBytes,
+ CollectConsumer consumer
+ ) throws IOException;
+ }
+
+ /**
+ * Consumer for the collector
+ *
+ * @opensearch.internal
+ */
+ @FunctionalInterface
+ public interface CollectConsumer {
+ void accept(LeafBucketCollector sub, int doc, long owningBucketOrd, BytesRef bytes) throws IOException;
+ }
+
+ /**
+ * Fetch values from a {@link ValuesSource}.
+ *
+ * @opensearch.internal
+ */
+ public static class ValuesSourceCollectorSource implements CollectorSource {
+ private final ValuesSource valuesSource;
+
+ public ValuesSourceCollectorSource(ValuesSource valuesSource) {
+ this.valuesSource = valuesSource;
+ }
+
+ @Override
+ public boolean needsScores() {
+ return valuesSource.needsScores();
+ }
+
+ @Override
+ public LeafBucketCollector getLeafCollector(
+ IncludeExclude.StringFilter includeExclude,
+ LeafReaderContext ctx,
+ LeafBucketCollector sub,
+ LongConsumer addRequestCircuitBreakerBytes,
+ CollectConsumer consumer
+ ) throws IOException {
+ SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
+ return new LeafBucketCollectorBase(sub, values) {
+ final BytesRefBuilder previous = new BytesRefBuilder();
+
+ @Override
+ public void collect(int doc, long owningBucketOrd) throws IOException {
+ if (false == values.advanceExact(doc)) {
+ return;
+ }
+ int valuesCount = values.docValueCount();
+
+ // SortedBinaryDocValues don't guarantee uniqueness so we
+ // need to take care of dups
+ previous.clear();
+ for (int i = 0; i < valuesCount; ++i) {
+ BytesRef bytes = values.nextValue();
+ if (includeExclude != null && false == includeExclude.accept(bytes)) {
+ continue;
+ }
+ if (i > 0 && previous.get().equals(bytes)) {
+ continue;
+ }
+ previous.copyBytes(bytes);
+ consumer.accept(sub, doc, owningBucketOrd, bytes);
+ }
+ }
+ };
+ }
+
+ @Override
+ public void close() {}
+ }
+
+ /**
+ * Strategy for building results.
+ */
+ abstract class ResultStrategy<R extends InternalAggregation, B extends InternalMultiBucketAggregation.InternalBucket>
+ implements
+ Releasable {
+
+ private InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException {
+ LocalBucketCountThresholds localBucketCountThresholds = context.asLocalBucketCountThresholds(bucketCountThresholds);
+ B[][] topBucketsPerOrd = buildTopBucketsPerOrd(owningBucketOrds.length);
+ long[] otherDocCounts = new long[owningBucketOrds.length];
+ for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) {
+ collectZeroDocEntriesIfNeeded(owningBucketOrds[ordIdx]);
+ int size = (int) Math.min(bucketOrds.size(), localBucketCountThresholds.getRequiredSize());
+
+ PriorityQueue<B> ordered = buildPriorityQueue(size);
+ B spare = null;
+ BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrds[ordIdx]);
+ Supplier<B> emptyBucketBuilder = emptyBucketBuilder(owningBucketOrds[ordIdx]);
+ while (ordsEnum.next()) {
+ long docCount = bucketDocCount(ordsEnum.ord());
+ otherDocCounts[ordIdx] += docCount;
+ if (docCount < localBucketCountThresholds.getMinDocCount()) {
+ continue;
+ }
+ if (spare == null) {
+ spare = emptyBucketBuilder.get();
+ }
+ updateBucket(spare, ordsEnum, docCount);
+ spare = ordered.insertWithOverflow(spare);
+ }
+
+ topBucketsPerOrd[ordIdx] = buildBuckets(ordered.size());
+ for (int i = ordered.size() - 1; i >= 0; --i) {
+ topBucketsPerOrd[ordIdx][i] = ordered.pop();
+ otherDocCounts[ordIdx] -= topBucketsPerOrd[ordIdx][i].getDocCount();
+ finalizeBucket(topBucketsPerOrd[ordIdx][i]);
+ }
+ }
+
+ buildSubAggs(topBucketsPerOrd);
+ InternalAggregation[] result = new InternalAggregation[owningBucketOrds.length];
+ for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) {
+ result[ordIdx] = buildResult(owningBucketOrds[ordIdx], otherDocCounts[ordIdx], topBucketsPerOrd[ordIdx]);
+ }
+ return result;
+ }
+
+ /**
+ * Short description of the collection mechanism added to the profile
+ * output to help with debugging.
+ */
+ abstract String describe();
+
+ /**
+ * Wrap the "standard" numeric terms collector to collect any more
+ * information that this result type may need.
+ */
+ abstract LeafBucketCollector wrapCollector(LeafBucketCollector primary);
+
+ /**
+ * Collect extra entries for "zero" hit documents if they were requested
+ * and required.
+ */
+ abstract void collectZeroDocEntriesIfNeeded(long owningBucketOrd) throws IOException;
+
+ /**
+ * Build an empty temporary bucket.
+ */
+ abstract Supplier<B> emptyBucketBuilder(long owningBucketOrd);
+
+ /**
+ * Build a {@link PriorityQueue} to sort the buckets. After we've
+ * collected all of the buckets we'll collect all entries in the queue.
+ */
+ abstract PriorityQueue<B> buildPriorityQueue(int size);
+
+ /**
+ * Update fields in {@code spare} to reflect information collected for
+ * this bucket ordinal.
+ */
+ abstract void updateBucket(B spare, BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum, long docCount) throws IOException;
+
+ /**
+ * Build an array to hold the "top" buckets for each ordinal.
+ */
+ abstract B[][] buildTopBucketsPerOrd(int size);
+
+ /**
+ * Build an array of buckets for a particular ordinal to collect the
+ * results. The populated list is passed to {@link #buildResult}.
+ */
+ abstract B[] buildBuckets(int size);
+
+ /**
+ * Finalize building a bucket. Called once we know that the bucket will
+ * be included in the results.
+ */
+ abstract void finalizeBucket(B bucket);
+
+ /**
+ * Build the sub-aggregations into the buckets. This will usually
+ * delegate to {@link #buildSubAggsForAllBuckets}.
+ */
+ abstract void buildSubAggs(B[][] topBucketsPerOrd) throws IOException;
+
+ /**
+ * Turn the buckets into an aggregation result.
+ */
+ abstract R buildResult(long owningBucketOrd, long otherDocCount, B[] topBuckets);
+
+ /**
+ * Build an "empty" result. Only called if there isn't any data on this
+ * shard.
+ */
+ abstract R buildEmptyResult();
+ }
+
+ /**
+ * Builds results for the standard {@code terms} aggregation.
+ */
+ class StandardTermsResults extends ResultStrategy<StringTerms, StringTerms.Bucket> {
+ private final ValuesSource valuesSource;
+
+ StandardTermsResults(ValuesSource valuesSource) {
+ this.valuesSource = valuesSource;
+ }
+
+ @Override
+ String describe() {
+ return "terms";
+ }
+
+ @Override
+ LeafBucketCollector wrapCollector(LeafBucketCollector primary) {
+ return primary;
+ }
+
+ @Override
+ void collectZeroDocEntriesIfNeeded(long owningBucketOrd) throws IOException {
+ if (bucketCountThresholds.getMinDocCount() != 0) {
+ return;
+ }
+ if (InternalOrder.isCountDesc(order) && bucketOrds.bucketsInOrd(owningBucketOrd) >= bucketCountThresholds.getRequiredSize()) {
+ return;
+ }
+ // we need to fill-in the blanks
+ for (LeafReaderContext ctx : context.searcher().getTopReaderContext().leaves()) {
+ SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
+ // brute force
+ for (int docId = 0; docId < ctx.reader().maxDoc(); ++docId) {
+ if (values.advanceExact(docId)) {
+ int valueCount = values.docValueCount();
+ for (int i = 0; i < valueCount; ++i) {
+ BytesRef term = values.nextValue();
+ if (includeExclude == null || includeExclude.accept(term)) {
+ bucketOrds.add(owningBucketOrd, term);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ Supplier<StringTerms.Bucket> emptyBucketBuilder(long owningBucketOrd) {
+ return () -> new StringTerms.Bucket(new BytesRef(), 0, null, showTermDocCountError, 0, format);
+ }
+
+ @Override
+ PriorityQueue<StringTerms.Bucket> buildPriorityQueue(int size) {
+ return new BucketPriorityQueue<>(size, partiallyBuiltBucketComparator);
+ }
+
+ @Override
+ void updateBucket(StringTerms.Bucket spare, BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum, long docCount) throws IOException {
+ ordsEnum.readValue(spare.termBytes);
+ spare.docCount = docCount;
+ spare.bucketOrd = ordsEnum.ord();
+ }
+
+ @Override
+ StringTerms.Bucket[][] buildTopBucketsPerOrd(int size) {
+ return new StringTerms.Bucket[size][];
+ }
+
+ @Override
+ StringTerms.Bucket[] buildBuckets(int size) {
+ return new StringTerms.Bucket[size];
+ }
+
+ @Override
+ void finalizeBucket(StringTerms.Bucket bucket) {
+ /*
+ * termBytes contains a reference to the bytes held by the
+ * bucketOrds which will be invalid once the aggregation is
+ * closed so we have to copy it.
+ */
+ bucket.termBytes = BytesRef.deepCopyOf(bucket.termBytes);
+ }
+
+ @Override
+ void buildSubAggs(StringTerms.Bucket[][] topBucketsPerOrd) throws IOException {
+ buildSubAggsForAllBuckets(topBucketsPerOrd, b -> b.bucketOrd, (b, a) -> b.aggregations = a);
+ }
+
+ @Override
+ StringTerms buildResult(long owningBucketOrd, long otherDocCount, StringTerms.Bucket[] topBuckets) {
+ final BucketOrder reduceOrder;
+ if (isKeyOrder(order) == false) {
+ reduceOrder = InternalOrder.key(true);
+ Arrays.sort(topBuckets, reduceOrder.comparator());
+ } else {
+ reduceOrder = order;
+ }
+ return new StringTerms(
+ name,
+ reduceOrder,
+ order,
+ metadata(),
+ format,
+ bucketCountThresholds.getShardSize(),
+ showTermDocCountError,
+ otherDocCount,
+ Arrays.asList(topBuckets),
+ 0,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ StringTerms buildEmptyResult() {
+ return buildEmptyTermsAggregation();
+ }
+
+ @Override
+ public void close() {}
+ }
+
+ /**
+ * Builds results for the {@code significant_terms} aggregation.
+ */
+ class SignificantTermsResults extends ResultStrategy<SignificantStringTerms, SignificantStringTerms.Bucket> {
+ private final BackgroundFrequencyForBytes backgroundFrequencies;
+ private final long supersetSize;
+ private final SignificanceHeuristic significanceHeuristic;
+
+ private LongArray subsetSizes = context.bigArrays().newLongArray(1, true);
+
+ SignificantTermsResults(
+ SignificanceLookup significanceLookup,
+ SignificanceHeuristic significanceHeuristic,
+ CardinalityUpperBound cardinality
+ ) {
+ backgroundFrequencies = significanceLookup.bytesLookup(context.bigArrays(), cardinality);
+ supersetSize = significanceLookup.supersetSize();
+ this.significanceHeuristic = significanceHeuristic;
+ }
+
+ @Override
+ String describe() {
+ return "significant_terms";
+ }
+
+ @Override
+ LeafBucketCollector wrapCollector(LeafBucketCollector primary) {
+ return new LeafBucketCollectorBase(primary, null) {
+ @Override
+ public void collect(int doc, long owningBucketOrd) throws IOException {
+ super.collect(doc, owningBucketOrd);
+ subsetSizes = context.bigArrays().grow(subsetSizes, owningBucketOrd + 1);
+ subsetSizes.increment(owningBucketOrd, 1);
+ }
+ };
+ }
+
+ @Override
+ void collectZeroDocEntriesIfNeeded(long owningBucketOrd) throws IOException {}
+
+ @Override
+ Supplier<SignificantStringTerms.Bucket> emptyBucketBuilder(long owningBucketOrd) {
+ long subsetSize = subsetSizes.get(owningBucketOrd);
+ return () -> new SignificantStringTerms.Bucket(new BytesRef(), 0, subsetSize, 0, 0, null, format, 0);
+ }
+
+ @Override
+ PriorityQueue<SignificantStringTerms.Bucket> buildPriorityQueue(int size) {
+ return new BucketSignificancePriorityQueue<>(size);
+ }
+
+ @Override
+ void updateBucket(SignificantStringTerms.Bucket spare, BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum, long docCount)
+ throws IOException {
+
+ ordsEnum.readValue(spare.termBytes);
+ spare.bucketOrd = ordsEnum.ord();
+ spare.subsetDf = docCount;
+ spare.supersetDf = backgroundFrequencies.freq(spare.termBytes);
+ spare.supersetSize = supersetSize;
+ /*
+ * During shard-local down-selection we use subset/superset stats
+ * that are for this shard only. Back at the central reducer these
+ * properties will be updated with global stats.
+ */
+ spare.updateScore(significanceHeuristic);
+ }
+
+ @Override
+ SignificantStringTerms.Bucket[][] buildTopBucketsPerOrd(int size) {
+ return new SignificantStringTerms.Bucket[size][];
+ }
+
+ @Override
+ SignificantStringTerms.Bucket[] buildBuckets(int size) {
+ return new SignificantStringTerms.Bucket[size];
+ }
+
+ @Override
+ void finalizeBucket(SignificantStringTerms.Bucket bucket) {
+ /*
+ * termBytes contains a reference to the bytes held by the
+ * bucketOrds which will be invalid once the aggregation is
+ * closed so we have to copy it.
+ */
+ bucket.termBytes = BytesRef.deepCopyOf(bucket.termBytes);
+ }
+
+ @Override
+ void buildSubAggs(SignificantStringTerms.Bucket[][] topBucketsPerOrd) throws IOException {
+ buildSubAggsForAllBuckets(topBucketsPerOrd, b -> b.bucketOrd, (b, a) -> b.aggregations = a);
+ }
+
+ @Override
+ SignificantStringTerms buildResult(long owningBucketOrd, long otherDocCount, SignificantStringTerms.Bucket[] topBuckets) {
+ return new SignificantStringTerms(
+ name,
+ metadata(),
+ format,
+ subsetSizes.get(owningBucketOrd),
+ supersetSize,
+ significanceHeuristic,
+ Arrays.asList(topBuckets),
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ SignificantStringTerms buildEmptyResult() {
+ return buildEmptySignificantTermsAggregation(0, significanceHeuristic);
+ }
+
+ @Override
+ public void close() {
+ Releasables.close(backgroundFrequencies, subsetSizes);
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: MultiTermsAggregationBuilder (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | MultiTermsAggregationBuilder |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 9.7%
+
+
+ (3/31)
+
+ |
+
+
+ 18.8%
+
+
+ (22/117)
+
+ |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+ import org.opensearch.core.ParseField;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.index.query.QueryShardContext;
+ import org.opensearch.search.aggregations.AbstractAggregationBuilder;
+ import org.opensearch.search.aggregations.AggregationBuilder;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.AggregatorFactory;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.InternalOrder;
+ import org.opensearch.search.aggregations.support.MultiTermsValuesSourceConfig;
+ import org.opensearch.search.aggregations.support.ValuesSourceRegistry;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+
+ import static org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS;
+
+ /**
+ * Multi-terms aggregation supports collecting terms from multiple fields in the same document.
+ *
+ * <p>
+ * For example, using the multi-terms aggregation to group by two fields region, host, calculate max cpu, and sort by max cpu.
+ * </p>
+ * <pre>
+ * GET test_000001/_search
+ * {
+ * "size": 0,
+ * "aggs": {
+ * "hot": {
+ * "multi_terms": {
+ * "terms": [{
+ * "field": "region"
+ * },{
+ * "field": "host"
+ * }],
+ * "order": {"max-cpu": "desc"}
+ * },
+ * "aggs": {
+ * "max-cpu": { "max": { "field": "cpu" } }
+ * }
+ * }
+ * }
+ * }
+ * </pre>
+ *
+ * <p>
+ * The aggregation result contains
+ * - key: a list of value extract from multiple fields in the same doc.
+ * </p>
+ * <pre>
+ * {
+ * "hot": {
+ * "doc_count_error_upper_bound": 0,
+ * "sum_other_doc_count": 0,
+ * "buckets": [
+ * {
+ * "key": [
+ * "dub",
+ * "h1"
+ * ],
+ * "key_as_string": "dub|h1",
+ * "doc_count": 2,
+ * "max-cpu": {
+ * "value": 90.0
+ * }
+ * },
+ * {
+ * "key": [
+ * "dub",
+ * "h2"
+ * ],
+ * "key_as_string": "dub|h2",
+ * "doc_count": 2,
+ * "max-cpu": {
+ * "value": 70.0
+ * }
+ * }
+ * ]
+ * }
+ * }
+ * </pre>
+ *
+ * <p>
+ * <b>Notes:</b> The current implementation focuses on adding new type aggregates. Performance (latency) is not good，mainly because of
+ * simply encoding/decoding a list of values as bucket keys.
+ * </p>
+ *
+ * @opensearch.internal
+ */
+ public class MultiTermsAggregationBuilder extends AbstractAggregationBuilder<MultiTermsAggregationBuilder> {
+ public static final String NAME = "multi_terms";
+ public static final ObjectParser<MultiTermsAggregationBuilder, String> PARSER = ObjectParser.fromBuilder(
+ NAME,
+ MultiTermsAggregationBuilder::new
+ );
+
+ public static final ParseField TERMS_FIELD = new ParseField("terms");
+ public static final ParseField SHARD_SIZE_FIELD_NAME = new ParseField("shard_size");
+ public static final ParseField MIN_DOC_COUNT_FIELD_NAME = new ParseField("min_doc_count");
+ public static final ParseField SHARD_MIN_DOC_COUNT_FIELD_NAME = new ParseField("shard_min_doc_count");
+ public static final ParseField REQUIRED_SIZE_FIELD_NAME = new ParseField("size");
+ public static final ParseField SHOW_TERM_DOC_COUNT_ERROR = new ParseField("show_term_doc_count_error");
+ public static final ParseField ORDER_FIELD = new ParseField("order");
+
+ @Override
+ public String getType() {
+ return NAME;
+ }
+
+ static {
+ final ObjectParser<MultiTermsValuesSourceConfig.Builder, Void> parser = MultiTermsValuesSourceConfig.PARSER.apply(
+ true,
+ true,
+ true,
+ true
+ );
+ PARSER.declareObjectArray(MultiTermsAggregationBuilder::terms, (p, c) -> parser.parse(p, null).build(), TERMS_FIELD);
+
+ PARSER.declareBoolean(MultiTermsAggregationBuilder::showTermDocCountError, SHOW_TERM_DOC_COUNT_ERROR);
+
+ PARSER.declareInt(MultiTermsAggregationBuilder::shardSize, SHARD_SIZE_FIELD_NAME);
+
+ PARSER.declareLong(MultiTermsAggregationBuilder::minDocCount, MIN_DOC_COUNT_FIELD_NAME);
+
+ PARSER.declareLong(MultiTermsAggregationBuilder::shardMinDocCount, SHARD_MIN_DOC_COUNT_FIELD_NAME);
+
+ PARSER.declareInt(MultiTermsAggregationBuilder::size, REQUIRED_SIZE_FIELD_NAME);
+
+ PARSER.declareObjectArray(MultiTermsAggregationBuilder::order, (p, c) -> InternalOrder.Parser.parseOrderParam(p), ORDER_FIELD);
+
+ PARSER.declareField(
+ MultiTermsAggregationBuilder::collectMode,
+ (p, c) -> Aggregator.SubAggCollectionMode.parse(p.text(), LoggingDeprecationHandler.INSTANCE),
+ Aggregator.SubAggCollectionMode.KEY,
+ ObjectParser.ValueType.STRING
+ );
+ }
+
+ public static final ValuesSourceRegistry.RegistryKey<MultiTermsAggregationFactory.InternalValuesSourceSupplier> REGISTRY_KEY =
+ new ValuesSourceRegistry.RegistryKey<>(
+ MultiTermsAggregationBuilder.NAME,
+ MultiTermsAggregationFactory.InternalValuesSourceSupplier.class
+ );
+
+ private List<MultiTermsValuesSourceConfig> terms;
+
+ private BucketOrder order = BucketOrder.compound(BucketOrder.count(false)); // automatically adds tie-breaker key asc order
+ private Aggregator.SubAggCollectionMode collectMode = null;
+ private TermsAggregator.BucketCountThresholds bucketCountThresholds = new TermsAggregator.BucketCountThresholds(
+ DEFAULT_BUCKET_COUNT_THRESHOLDS
+ );
+ private boolean showTermDocCountError = false;
+
+ public MultiTermsAggregationBuilder(String name) {
+ super(name);
+ }
+
+ protected MultiTermsAggregationBuilder(
+ MultiTermsAggregationBuilder clone,
+ AggregatorFactories.Builder factoriesBuilder,
+ Map<String, Object> metadata
+ ) {
+ super(clone, factoriesBuilder, metadata);
+ this.terms = new ArrayList<>(clone.terms);
+ this.order = clone.order;
+ this.collectMode = clone.collectMode;
+ this.bucketCountThresholds = new TermsAggregator.BucketCountThresholds(clone.bucketCountThresholds);
+ this.showTermDocCountError = clone.showTermDocCountError;
+ }
+
+ @Override
+ protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map<String, Object> metadata) {
+ return new MultiTermsAggregationBuilder(this, factoriesBuilder, metadata);
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public MultiTermsAggregationBuilder(StreamInput in) throws IOException {
+ super(in);
+ terms = in.readList(MultiTermsValuesSourceConfig::new);
+ bucketCountThresholds = new TermsAggregator.BucketCountThresholds(in);
+ collectMode = in.readOptionalWriteable(Aggregator.SubAggCollectionMode::readFromStream);
+ order = InternalOrder.Streams.readOrder(in);
+ showTermDocCountError = in.readBoolean();
+ }
+
+ @Override
+ protected void doWriteTo(StreamOutput out) throws IOException {
+ out.writeList(terms);
+ bucketCountThresholds.writeTo(out);
+ out.writeOptionalWriteable(collectMode);
+ order.writeTo(out);
+ out.writeBoolean(showTermDocCountError);
+ }
+
+ @Override
+ protected AggregatorFactory doBuild(
+ QueryShardContext queryShardContext,
+ AggregatorFactory parent,
+ AggregatorFactories.Builder subfactoriesBuilder
+ ) throws IOException {
+ return new MultiTermsAggregationFactory(
+ name,
+ queryShardContext,
+ parent,
+ subfactoriesBuilder,
+ metadata,
+ terms,
+ order,
+ collectMode,
+ bucketCountThresholds,
+ showTermDocCountError
+ );
+ }
+
+ @Override
+ protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject();
+ if (terms != null) {
+ builder.field(TERMS_FIELD.getPreferredName(), terms);
+ }
+ bucketCountThresholds.toXContent(builder, params);
+ builder.field(SHOW_TERM_DOC_COUNT_ERROR.getPreferredName(), showTermDocCountError);
+ builder.field(ORDER_FIELD.getPreferredName());
+ order.toXContent(builder, params);
+ if (collectMode != null) {
+ builder.field(Aggregator.SubAggCollectionMode.KEY.getPreferredName(), collectMode.parseField().getPreferredName());
+ }
+ builder.endObject();
+ return builder;
+ }
+
+ /**
+ * Set the terms.
+ */
+ public MultiTermsAggregationBuilder terms(List<MultiTermsValuesSourceConfig> terms) {
+ if (terms == null) {
+ throw new IllegalArgumentException("[terms] must not be null. Found null terms in [" + name + "]");
+ }
+ if (terms.size() < 2) {
+ throw new IllegalArgumentException(
+ "multi term aggregation must has at least 2 terms. Found ["
+ + terms.size()
+ + "] in"
+ + " ["
+ + name
+ + "]"
+ + (terms.size() == 1 ? " Use terms aggregation for single term aggregation" : "")
+ );
+ }
+ this.terms = terms;
+ return this;
+ }
+
+ /**
+ * Sets the size - indicating how many term buckets should be returned
+ * (defaults to 10)
+ */
+ public MultiTermsAggregationBuilder size(int size) {
+ if (size <= 0) {
+ throw new IllegalArgumentException("[size] must be greater than 0. Found [" + size + "] in [" + name + "]");
+ }
+ bucketCountThresholds.setRequiredSize(size);
+ return this;
+ }
+
+ /**
+ * Returns the number of term buckets currently configured
+ */
+ public int size() {
+ return bucketCountThresholds.getRequiredSize();
+ }
+
+ /**
+ * Sets the shard_size - indicating the number of term buckets each shard
+ * will return to the coordinating node (the node that coordinates the
+ * search execution). The higher the shard size is, the more accurate the
+ * results are.
+ */
+ public MultiTermsAggregationBuilder shardSize(int shardSize) {
+ if (shardSize <= 0) {
+ throw new IllegalArgumentException("[shardSize] must be greater than 0. Found [" + shardSize + "] in [" + name + "]");
+ }
+ bucketCountThresholds.setShardSize(shardSize);
+ return this;
+ }
+
+ /**
+ * Returns the number of term buckets per shard that are currently configured
+ */
+ public int shardSize() {
+ return bucketCountThresholds.getShardSize();
+ }
+
+ /**
+ * Set the minimum document count terms should have in order to appear in
+ * the response.
+ */
+ public MultiTermsAggregationBuilder minDocCount(long minDocCount) {
+ if (minDocCount < 0) {
+ throw new IllegalArgumentException(
+ "[minDocCount] must be greater than or equal to 0. Found [" + minDocCount + "] in [" + name + "]"
+ );
+ }
+ bucketCountThresholds.setMinDocCount(minDocCount);
+ return this;
+ }
+
+ /**
+ * Returns the minimum document count required per term
+ */
+ public long minDocCount() {
+ return bucketCountThresholds.getMinDocCount();
+ }
+
+ /**
+ * Set the minimum document count terms should have on the shard in order to
+ * appear in the response.
+ */
+ public MultiTermsAggregationBuilder shardMinDocCount(long shardMinDocCount) {
+ if (shardMinDocCount < 0) {
+ throw new IllegalArgumentException(
+ "[shardMinDocCount] must be greater than or equal to 0. Found [" + shardMinDocCount + "] in [" + name + "]"
+ );
+ }
+ bucketCountThresholds.setShardMinDocCount(shardMinDocCount);
+ return this;
+ }
+
+ /**
+ * Returns the minimum document count required per term, per shard
+ */
+ public long shardMinDocCount() {
+ return bucketCountThresholds.getShardMinDocCount();
+ }
+
+ /** Set a new order on this builder and return the builder so that calls
+ * can be chained. A tie-breaker may be added to avoid non-deterministic ordering. */
+ public MultiTermsAggregationBuilder order(BucketOrder order) {
+ if (order == null) {
+ throw new IllegalArgumentException("[order] must not be null: [" + name + "]");
+ }
+ if (order instanceof InternalOrder.CompoundOrder || InternalOrder.isKeyOrder(order)) {
+ this.order = order; // if order already contains a tie-breaker we are good to go
+ } else { // otherwise add a tie-breaker by using a compound order
+ this.order = BucketOrder.compound(order);
+ }
+ return this;
+ }
+
+ /**
+ * Sets the order in which the buckets will be returned. A tie-breaker may be added to avoid non-deterministic
+ * ordering.
+ */
+ public MultiTermsAggregationBuilder order(List<BucketOrder> orders) {
+ if (orders == null) {
+ throw new IllegalArgumentException("[orders] must not be null: [" + name + "]");
+ }
+ // if the list only contains one order use that to avoid inconsistent xcontent
+ order(orders.size() > 1 ? BucketOrder.compound(orders) : orders.get(0));
+ return this;
+ }
+
+ /**
+ * Gets the order in which the buckets will be returned.
+ */
+ public BucketOrder order() {
+ return order;
+ }
+
+ /**
+ * Expert: set the collection mode.
+ */
+ public MultiTermsAggregationBuilder collectMode(Aggregator.SubAggCollectionMode collectMode) {
+ if (collectMode == null) {
+ throw new IllegalArgumentException("[collectMode] must not be null: [" + name + "]");
+ }
+ this.collectMode = collectMode;
+ return this;
+ }
+
+ /**
+ * Expert: get the collection mode.
+ */
+ public Aggregator.SubAggCollectionMode collectMode() {
+ return collectMode;
+ }
+
+ /**
+ * Get whether doc count error will be return for individual terms
+ */
+ public boolean showTermDocCountError() {
+ return showTermDocCountError;
+ }
+
+ /**
+ * Set whether doc count error will be return for individual terms
+ */
+ public MultiTermsAggregationBuilder showTermDocCountError(boolean showTermDocCountError) {
+ this.showTermDocCountError = showTermDocCountError;
+ return this;
+ }
+
+ @Override
+ public BucketCardinality bucketCardinality() {
+ return BucketCardinality.MANY;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), bucketCountThresholds, collectMode, order, showTermDocCountError);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+ MultiTermsAggregationBuilder other = (MultiTermsAggregationBuilder) obj;
+ return Objects.equals(terms, other.terms)
+ && Objects.equals(bucketCountThresholds, other.bucketCountThresholds)
+ && Objects.equals(collectMode, other.collectMode)
+ && Objects.equals(order, other.order)
+ && Objects.equals(showTermDocCountError, other.showTermDocCountError);
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: MultiTermsAggregationFactory (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | MultiTermsAggregationFactory |
+
+
+ 14.3%
+
+
+ (1/7)
+
+ |
+
+
+ 8%
+
+
+ (4/50)
+
+ |
+
+
+ | MultiTermsAggregationFactory$InternalValuesSourceSupplier |
+
+
+ | Total |
+
+
+ 14.3%
+
+
+ (1/7)
+
+ |
+
+
+ 8%
+
+
+ (4/50)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.common.collect.Tuple;
+ import org.opensearch.index.query.QueryShardContext;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.AggregatorFactory;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.InternalOrder;
+ import org.opensearch.search.aggregations.bucket.BucketUtils;
+ import org.opensearch.search.aggregations.support.CoreValuesSourceType;
+ import org.opensearch.search.aggregations.support.MultiTermsValuesSourceConfig;
+ import org.opensearch.search.aggregations.support.ValuesSource;
+ import org.opensearch.search.aggregations.support.ValuesSourceConfig;
+ import org.opensearch.search.aggregations.support.ValuesSourceRegistry;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.stream.Collectors;
+
+ import static org.opensearch.search.aggregations.bucket.terms.MultiTermsAggregationBuilder.REGISTRY_KEY;
+
+ /**
+ * Factory of {@link MultiTermsAggregator}.
+ *
+ * @opensearch.internal
+ */
+ public class MultiTermsAggregationFactory extends AggregatorFactory {
+
+ private final List<Tuple<ValuesSourceConfig, IncludeExclude>> configs;
+ private final List<DocValueFormat> formats;
+ /**
+ * Fields inherent from Terms Aggregation Factory.
+ */
+ private final BucketOrder order;
+ private final Aggregator.SubAggCollectionMode collectMode;
+ private final TermsAggregator.BucketCountThresholds bucketCountThresholds;
+ private final boolean showTermDocCountError;
+
+ public static void registerAggregators(ValuesSourceRegistry.Builder builder) {
+ builder.register(REGISTRY_KEY, List.of(CoreValuesSourceType.BYTES, CoreValuesSourceType.IP), config -> {
+ final IncludeExclude.StringFilter filter = config.v2() == null ? null : config.v2().convertToStringFilter(config.v1().format());
+ return MultiTermsAggregator.InternalValuesSourceFactory.bytesValuesSource(config.v1().getValuesSource(), filter);
+ }, true);
+
+ builder.register(
+ REGISTRY_KEY,
+ List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.BOOLEAN, CoreValuesSourceType.DATE),
+ config -> {
+ ValuesSourceConfig valuesSourceConfig = config.v1();
+ IncludeExclude includeExclude = config.v2();
+ ValuesSource.Numeric valuesSource = ((ValuesSource.Numeric) valuesSourceConfig.getValuesSource());
+ IncludeExclude.LongFilter longFilter = null;
+ if (valuesSource.isFloatingPoint()) {
+ if (includeExclude != null) {
+ longFilter = includeExclude.convertToDoubleFilter();
+ }
+ return MultiTermsAggregator.InternalValuesSourceFactory.doubleValueSource(valuesSource, longFilter);
+ } else if (valuesSource.isBigInteger()) {
+ if (includeExclude != null) {
+ longFilter = includeExclude.convertToDoubleFilter();
+ }
+ return MultiTermsAggregator.InternalValuesSourceFactory.unsignedLongValuesSource(valuesSource, longFilter);
+ } else {
+ if (includeExclude != null) {
+ longFilter = includeExclude.convertToLongFilter(valuesSourceConfig.format());
+ }
+ return MultiTermsAggregator.InternalValuesSourceFactory.longValuesSource(valuesSource, longFilter);
+ }
+ },
+ true
+ );
+
+ builder.registerUsage(MultiTermsAggregationBuilder.NAME);
+ }
+
+ public MultiTermsAggregationFactory(
+ String name,
+ QueryShardContext queryShardContext,
+ AggregatorFactory parent,
+ AggregatorFactories.Builder subFactoriesBuilder,
+ Map<String, Object> metadata,
+ List<MultiTermsValuesSourceConfig> multiTermConfigs,
+ BucketOrder order,
+ Aggregator.SubAggCollectionMode collectMode,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ boolean showTermDocCountError
+ ) throws IOException {
+ super(name, queryShardContext, parent, subFactoriesBuilder, metadata);
+ this.configs = multiTermConfigs.stream()
+ .map(
+ c -> new Tuple<ValuesSourceConfig, IncludeExclude>(
+ ValuesSourceConfig.resolveUnregistered(
+ queryShardContext,
+ c.getUserValueTypeHint(),
+ c.getFieldName(),
+ c.getScript(),
+ c.getMissing(),
+ c.getTimeZone(),
+ c.getFormat(),
+ CoreValuesSourceType.BYTES
+ ),
+ c.getIncludeExclude()
+ )
+ )
+ .collect(Collectors.toList());
+ this.formats = this.configs.stream().map(c -> c.v1().format()).collect(Collectors.toList());
+ this.order = order;
+ this.collectMode = collectMode;
+ this.bucketCountThresholds = bucketCountThresholds;
+ this.showTermDocCountError = showTermDocCountError;
+ }
+
+ @Override
+ protected Aggregator createInternal(
+ SearchContext searchContext,
+ Aggregator parent,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+ TermsAggregator.BucketCountThresholds bucketCountThresholds = new TermsAggregator.BucketCountThresholds(this.bucketCountThresholds);
+ if (InternalOrder.isKeyOrder(order) == false
+ && bucketCountThresholds.getShardSize() == TermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) {
+ // The user has not made a shardSize selection. Use default
+ // heuristic to avoid any wrong-ranking caused by distributed
+ // counting
+ bucketCountThresholds.setShardSize(BucketUtils.suggestShardSideQueueSize(bucketCountThresholds.getRequiredSize()));
+ }
+ bucketCountThresholds.ensureValidity();
+ return new MultiTermsAggregator(
+ name,
+ factories,
+ showTermDocCountError,
+ configs.stream()
+ .map(config -> queryShardContext.getValuesSourceRegistry().getAggregator(REGISTRY_KEY, config.v1()).build(config))
+ .collect(Collectors.toList()),
+ configs.stream().map(c -> c.v1().format()).collect(Collectors.toList()),
+ order,
+ collectMode,
+ bucketCountThresholds,
+ searchContext,
+ parent,
+ cardinality,
+ metadata
+ );
+ }
+
+ @Override
+ protected boolean supportsConcurrentSegmentSearch() {
+ return true;
+ }
+
+ /**
+ * Supplier for internal values source
+ *
+ * @opensearch.internal
+ */
+ public interface InternalValuesSourceSupplier {
+ MultiTermsAggregator.InternalValuesSource build(Tuple<ValuesSourceConfig, IncludeExclude> config);
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: MultiTermsAggregator (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | MultiTermsAggregator |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/91)
+
+ |
+
+
+ | MultiTermsAggregator$1 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ | MultiTermsAggregator$InternalValuesSource |
+
+
+ | MultiTermsAggregator$InternalValuesSourceCollector |
+
+
+ | MultiTermsAggregator$InternalValuesSourceFactory |
+
+
+ 0%
+
+
+ (0/13)
+
+ |
+
+
+ 0%
+
+
+ (0/64)
+
+ |
+
+
+ | MultiTermsAggregator$MultiTermsValuesSource |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ 0%
+
+
+ (0/9)
+
+ |
+
+
+ | MultiTermsAggregator$MultiTermsValuesSource$1 |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ 0%
+
+
+ (0/18)
+
+ |
+
+
+ | MultiTermsAggregator$MultiTermsValuesSourceCollector |
+
+
+ | MultiTermsAggregator$TermValue |
+
+
+ 0%
+
+
+ (0/7)
+
+ |
+
+
+ 0%
+
+
+ (0/12)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/38)
+
+ |
+
+
+ 0%
+
+
+ (0/202)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.index.LeafReaderContext;
+ import org.apache.lucene.index.SortedNumericDocValues;
+ import org.apache.lucene.util.BytesRef;
+ import org.apache.lucene.util.NumericUtils;
+ import org.apache.lucene.util.PriorityQueue;
+ import org.opensearch.ExceptionsHelper;
+ import org.opensearch.common.CheckedSupplier;
+ import org.opensearch.common.Numbers;
+ import org.opensearch.common.io.stream.BytesStreamOutput;
+ import org.opensearch.common.lease.Releasable;
+ import org.opensearch.common.lease.Releasables;
+ import org.opensearch.core.common.bytes.BytesArray;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.common.io.stream.Writeable;
+ import org.opensearch.index.fielddata.SortedBinaryDocValues;
+ import org.opensearch.index.fielddata.SortedNumericDoubleValues;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.InternalOrder;
+ import org.opensearch.search.aggregations.LeafBucketCollector;
+ import org.opensearch.search.aggregations.bucket.DeferableBucketAggregator;
+ import org.opensearch.search.aggregations.bucket.LocalBucketCountThresholds;
+ import org.opensearch.search.aggregations.support.AggregationPath;
+ import org.opensearch.search.aggregations.support.ValuesSource;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.math.BigInteger;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.Collections;
+ import java.util.Comparator;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Set;
+
+ import static org.opensearch.search.aggregations.InternalOrder.isKeyOrder;
+ import static org.opensearch.search.aggregations.bucket.terms.TermsAggregator.descendsFromNestedAggregator;
+
+ /**
+ * An aggregator that aggregate with multi_terms.
+ *
+ * @opensearch.internal
+ */
+ public class MultiTermsAggregator extends DeferableBucketAggregator {
+
+ private final BytesKeyedBucketOrds bucketOrds;
+ private final MultiTermsValuesSource multiTermsValue;
+ private final boolean showTermDocCountError;
+ private final List<DocValueFormat> formats;
+ private final TermsAggregator.BucketCountThresholds bucketCountThresholds;
+ private final BucketOrder order;
+ private final Comparator<InternalMultiTerms.Bucket> partiallyBuiltBucketComparator;
+ private final SubAggCollectionMode collectMode;
+ private final Set<Aggregator> aggsUsedForSorting = new HashSet<>();
+
+ public MultiTermsAggregator(
+ String name,
+ AggregatorFactories factories,
+ boolean showTermDocCountError,
+ List<InternalValuesSource> internalValuesSources,
+ List<DocValueFormat> formats,
+ BucketOrder order,
+ SubAggCollectionMode collectMode,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ SearchContext context,
+ Aggregator parent,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+ super(name, factories, context, parent, metadata);
+ this.bucketOrds = BytesKeyedBucketOrds.build(context.bigArrays(), cardinality);
+ this.multiTermsValue = new MultiTermsValuesSource(internalValuesSources);
+ this.showTermDocCountError = showTermDocCountError;
+ this.formats = formats;
+ this.bucketCountThresholds = bucketCountThresholds;
+ this.order = order;
+ this.partiallyBuiltBucketComparator = order == null ? null : order.partiallyBuiltBucketComparator(b -> b.bucketOrd, this);
+ // Todo, copy from TermsAggregator. need to remove duplicate code.
+ if (subAggsNeedScore() && descendsFromNestedAggregator(parent)) {
+ /*
+ Force the execution to depth_first because we need to access the score of
+ nested documents in a sub-aggregation and we are not able to generate this score
+ while replaying deferred documents.
+ */
+ this.collectMode = SubAggCollectionMode.DEPTH_FIRST;
+ } else {
+ this.collectMode = collectMode;
+ }
+ // Don't defer any child agg if we are dependent on it for pruning results
+ if (order instanceof InternalOrder.Aggregation) {
+ AggregationPath path = ((InternalOrder.Aggregation) order).path();
+ aggsUsedForSorting.add(path.resolveTopmostAggregator(this));
+ } else if (order instanceof InternalOrder.CompoundOrder) {
+ InternalOrder.CompoundOrder compoundOrder = (InternalOrder.CompoundOrder) order;
+ for (BucketOrder orderElement : compoundOrder.orderElements()) {
+ if (orderElement instanceof InternalOrder.Aggregation) {
+ AggregationPath path = ((InternalOrder.Aggregation) orderElement).path();
+ aggsUsedForSorting.add(path.resolveTopmostAggregator(this));
+ }
+ }
+ }
+ }
+
+ @Override
+ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException {
+ LocalBucketCountThresholds localBucketCountThresholds = context.asLocalBucketCountThresholds(bucketCountThresholds);
+ InternalMultiTerms.Bucket[][] topBucketsPerOrd = new InternalMultiTerms.Bucket[owningBucketOrds.length][];
+ long[] otherDocCounts = new long[owningBucketOrds.length];
+ for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) {
+ collectZeroDocEntriesIfNeeded(owningBucketOrds[ordIdx]);
+ long bucketsInOrd = bucketOrds.bucketsInOrd(owningBucketOrds[ordIdx]);
+
+ int size = (int) Math.min(bucketsInOrd, localBucketCountThresholds.getRequiredSize());
+ PriorityQueue<InternalMultiTerms.Bucket> ordered = new BucketPriorityQueue<>(size, partiallyBuiltBucketComparator);
+ InternalMultiTerms.Bucket spare = null;
+ BytesRef dest = null;
+ BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrds[ordIdx]);
+ CheckedSupplier<InternalMultiTerms.Bucket, IOException> emptyBucketBuilder = () -> InternalMultiTerms.Bucket.EMPTY(
+ showTermDocCountError,
+ formats
+ );
+ while (ordsEnum.next()) {
+ long docCount = bucketDocCount(ordsEnum.ord());
+ otherDocCounts[ordIdx] += docCount;
+ if (docCount < localBucketCountThresholds.getMinDocCount()) {
+ continue;
+ }
+ if (spare == null) {
+ spare = emptyBucketBuilder.get();
+ dest = new BytesRef();
+ }
+
+ ordsEnum.readValue(dest);
+
+ spare.termValues = decode(dest);
+ spare.docCount = docCount;
+ spare.bucketOrd = ordsEnum.ord();
+ spare = ordered.insertWithOverflow(spare);
+ }
+
+ // Get the top buckets
+ InternalMultiTerms.Bucket[] bucketsForOrd = new InternalMultiTerms.Bucket[ordered.size()];
+ topBucketsPerOrd[ordIdx] = bucketsForOrd;
+ for (int b = ordered.size() - 1; b >= 0; --b) {
+ topBucketsPerOrd[ordIdx][b] = ordered.pop();
+ otherDocCounts[ordIdx] -= topBucketsPerOrd[ordIdx][b].getDocCount();
+ }
+ }
+
+ buildSubAggsForAllBuckets(topBucketsPerOrd, b -> b.bucketOrd, (b, aggs) -> b.aggregations = aggs);
+
+ InternalAggregation[] result = new InternalAggregation[owningBucketOrds.length];
+ for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) {
+ result[ordIdx] = buildResult(owningBucketOrds[ordIdx], otherDocCounts[ordIdx], topBucketsPerOrd[ordIdx]);
+ }
+ return result;
+ }
+
+ InternalMultiTerms buildResult(long owningBucketOrd, long otherDocCount, InternalMultiTerms.Bucket[] topBuckets) {
+ BucketOrder reduceOrder;
+ if (isKeyOrder(order) == false) {
+ reduceOrder = InternalOrder.key(true);
+ Arrays.sort(topBuckets, reduceOrder.comparator());
+ } else {
+ reduceOrder = order;
+ }
+ return new InternalMultiTerms(
+ name,
+ reduceOrder,
+ order,
+ metadata(),
+ bucketCountThresholds.getShardSize(),
+ showTermDocCountError,
+ otherDocCount,
+ 0,
+ formats,
+ List.of(topBuckets),
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ public InternalAggregation buildEmptyAggregation() {
+ return new InternalMultiTerms(
+ name,
+ order,
+ order,
+ metadata(),
+ bucketCountThresholds.getShardSize(),
+ showTermDocCountError,
+ 0,
+ 0,
+ formats,
+ Collections.emptyList(),
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
+ MultiTermsValuesSourceCollector collector = multiTermsValue.getValues(ctx);
+ return new LeafBucketCollector() {
+ @Override
+ public void collect(int doc, long owningBucketOrd) throws IOException {
+ for (BytesRef compositeKey : collector.apply(doc)) {
+ long bucketOrd = bucketOrds.add(owningBucketOrd, compositeKey);
+ if (bucketOrd < 0) {
+ bucketOrd = -1 - bucketOrd;
+ collectExistingBucket(sub, doc, bucketOrd);
+ } else {
+ collectBucket(sub, doc, bucketOrd);
+ }
+ }
+ }
+ };
+ }
+
+ @Override
+ protected void doClose() {
+ Releasables.close(bucketOrds, multiTermsValue);
+ }
+
+ private static List<Object> decode(BytesRef bytesRef) {
+ try (StreamInput input = new BytesArray(bytesRef).streamInput()) {
+ return input.readList(StreamInput::readGenericValue);
+ } catch (IOException e) {
+ throw ExceptionsHelper.convertToRuntime(e);
+ }
+ }
+
+ private boolean subAggsNeedScore() {
+ for (Aggregator subAgg : subAggregators) {
+ if (subAgg.scoreMode().needsScores()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ protected boolean shouldDefer(Aggregator aggregator) {
+ return collectMode == Aggregator.SubAggCollectionMode.BREADTH_FIRST && !aggsUsedForSorting.contains(aggregator);
+ }
+
+ private void collectZeroDocEntriesIfNeeded(long owningBucketOrd) throws IOException {
+ if (bucketCountThresholds.getMinDocCount() != 0) {
+ return;
+ }
+ if (InternalOrder.isCountDesc(order) && bucketOrds.bucketsInOrd(owningBucketOrd) >= bucketCountThresholds.getRequiredSize()) {
+ return;
+ }
+ // we need to fill-in the blanks
+ for (LeafReaderContext ctx : context.searcher().getTopReaderContext().leaves()) {
+ MultiTermsValuesSourceCollector collector = multiTermsValue.getValues(ctx);
+ // brute force
+ for (int docId = 0; docId < ctx.reader().maxDoc(); ++docId) {
+ for (BytesRef compositeKey : collector.apply(docId)) {
+ bucketOrds.add(owningBucketOrd, compositeKey);
+ }
+ }
+ }
+ }
+
+ /**
+ * A multi_terms collector which collect values on each doc,
+ */
+ @FunctionalInterface
+ interface MultiTermsValuesSourceCollector {
+ /**
+ * Collect a list values of multi_terms on each doc.
+ * Each terms could have multi_values, so the result is the cartesian product of each term's values.
+ */
+ List<BytesRef> apply(int doc) throws IOException;
+ }
+
+ @FunctionalInterface
+ interface InternalValuesSource {
+ /**
+ * Create {@link InternalValuesSourceCollector} from existing {@link LeafReaderContext}.
+ */
+ InternalValuesSourceCollector apply(LeafReaderContext ctx) throws IOException;
+ }
+
+ /**
+ * A terms collector which collect values on each doc,
+ */
+ @FunctionalInterface
+ interface InternalValuesSourceCollector {
+ /**
+ * Collect a list values of a term on specific doc.
+ */
+ List<TermValue<?>> apply(int doc) throws IOException;
+ }
+
+ /**
+ * Represents an individual term value.
+ */
+ static class TermValue<T> implements Writeable {
+ private static final Writer<BytesRef> BYTES_REF_WRITER = StreamOutput.getWriter(BytesRef.class);
+ private static final Writer<Long> LONG_WRITER = StreamOutput.getWriter(Long.class);
+ private static final Writer<BigInteger> BIG_INTEGER_WRITER = StreamOutput.getWriter(BigInteger.class);
+ private static final Writer<Double> DOUBLE_WRITER = StreamOutput.getWriter(Double.class);
+
+ private final T value;
+ private final Writer<T> writer;
+
+ private TermValue(T value, Writer<T> writer) {
+ this.value = value;
+ this.writer = writer;
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ writer.write(out, value);
+ }
+
+ public static TermValue<BytesRef> of(BytesRef value) {
+ return new TermValue<>(value, BYTES_REF_WRITER);
+ }
+
+ public static TermValue<Long> of(Long value) {
+ return new TermValue<>(value, LONG_WRITER);
+ }
+
+ public static TermValue<BigInteger> of(BigInteger value) {
+ return new TermValue<>(value, BIG_INTEGER_WRITER);
+ }
+
+ public static TermValue<Double> of(Double value) {
+ return new TermValue<>(value, DOUBLE_WRITER);
+ }
+ }
+
+ /**
+ * Multi_Term ValuesSource, it is a collection of {@link InternalValuesSource}
+ *
+ * @opensearch.internal
+ */
+ static class MultiTermsValuesSource implements Releasable {
+ private final List<InternalValuesSource> valuesSources;
+ private final BytesStreamOutput scratch = new BytesStreamOutput();
+
+ public MultiTermsValuesSource(List<InternalValuesSource> valuesSources) {
+ this.valuesSources = valuesSources;
+ }
+
+ public MultiTermsValuesSourceCollector getValues(LeafReaderContext ctx) throws IOException {
+ List<InternalValuesSourceCollector> collectors = new ArrayList<>();
+ for (InternalValuesSource valuesSource : valuesSources) {
+ collectors.add(valuesSource.apply(ctx));
+ }
+ return new MultiTermsValuesSourceCollector() {
+ @Override
+ public List<BytesRef> apply(int doc) throws IOException {
+ List<List<TermValue<?>>> collectedValues = new ArrayList<>();
+ for (InternalValuesSourceCollector collector : collectors) {
+ collectedValues.add(collector.apply(doc));
+ }
+ List<BytesRef> result = new ArrayList<>();
+ scratch.seek(0);
+ scratch.writeVInt(collectors.size()); // number of fields per composite key
+ cartesianProduct(result, scratch, collectedValues, 0);
+ return result;
+ }
+
+ /**
+ * Cartesian product using depth first search.
+ *
+ * <p>
+ * Composite keys are encoded to a {@link BytesRef} in a format compatible with {@link StreamOutput::writeGenericValue},
+ * but reuses the encoding of the shared prefixes from the previous levels to avoid wasteful work.
+ */
+ private void cartesianProduct(
+ List<BytesRef> compositeKeys,
+ BytesStreamOutput scratch,
+ List<List<TermValue<?>>> collectedValues,
+ int index
+ ) throws IOException {
+ if (collectedValues.size() == index) {
+ compositeKeys.add(BytesRef.deepCopyOf(scratch.bytes().toBytesRef()));
+ return;
+ }
+
+ long position = scratch.position();
+ for (TermValue<?> value : collectedValues.get(index)) {
+ value.writeTo(scratch); // encode the value
+ cartesianProduct(compositeKeys, scratch, collectedValues, index + 1); // dfs
+ scratch.seek(position); // backtrack
+ }
+ }
+ };
+ }
+
+ @Override
+ public void close() {
+ scratch.close();
+ }
+ }
+
+ /**
+ * Factory for construct {@link InternalValuesSource}.
+ *
+ * @opensearch.internal
+ */
+ static class InternalValuesSourceFactory {
+ static InternalValuesSource bytesValuesSource(ValuesSource valuesSource, IncludeExclude.StringFilter includeExclude) {
+ return ctx -> {
+ SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
+ return doc -> {
+ if (false == values.advanceExact(doc)) {
+ return Collections.emptyList();
+ }
+ int valuesCount = values.docValueCount();
+ List<TermValue<?>> termValues = new ArrayList<>(valuesCount);
+
+ // SortedBinaryDocValues don't guarantee uniqueness so we
+ // need to take care of dups
+ BytesRef previous = null;
+ for (int i = 0; i < valuesCount; ++i) {
+ BytesRef bytes = values.nextValue();
+ if (includeExclude != null && false == includeExclude.accept(bytes)) {
+ continue;
+ }
+ if (i > 0 && bytes.equals(previous)) {
+ continue;
+ }
+ BytesRef copy = BytesRef.deepCopyOf(bytes);
+ termValues.add(TermValue.of(copy));
+ previous = copy;
+ }
+ return termValues;
+ };
+ };
+ }
+
+ static InternalValuesSource unsignedLongValuesSource(ValuesSource.Numeric valuesSource, IncludeExclude.LongFilter longFilter) {
+ return ctx -> {
+ SortedNumericDocValues values = valuesSource.longValues(ctx);
+ return doc -> {
+ if (values.advanceExact(doc)) {
+ int valuesCount = values.docValueCount();
+
+ BigInteger previous = Numbers.MAX_UNSIGNED_LONG_VALUE;
+ List<TermValue<?>> termValues = new ArrayList<>(valuesCount);
+ for (int i = 0; i < valuesCount; ++i) {
+ BigInteger val = Numbers.toUnsignedBigInteger(values.nextValue());
+ if (previous.compareTo(val) != 0 || i == 0) {
+ if (longFilter == null || longFilter.accept(NumericUtils.doubleToSortableLong(val.doubleValue()))) {
+ termValues.add(TermValue.of(val));
+ }
+ previous = val;
+ }
+ }
+ return termValues;
+ }
+ return Collections.emptyList();
+ };
+ };
+ }
+
+ static InternalValuesSource longValuesSource(ValuesSource.Numeric valuesSource, IncludeExclude.LongFilter longFilter) {
+ return ctx -> {
+ SortedNumericDocValues values = valuesSource.longValues(ctx);
+ return doc -> {
+ if (values.advanceExact(doc)) {
+ int valuesCount = values.docValueCount();
+
+ long previous = Long.MAX_VALUE;
+ List<TermValue<?>> termValues = new ArrayList<>(valuesCount);
+ for (int i = 0; i < valuesCount; ++i) {
+ long val = values.nextValue();
+ if (previous != val || i == 0) {
+ if (longFilter == null || longFilter.accept(val)) {
+ termValues.add(TermValue.of(val));
+ }
+ previous = val;
+ }
+ }
+ return termValues;
+ }
+ return Collections.emptyList();
+ };
+ };
+ }
+
+ static InternalValuesSource doubleValueSource(ValuesSource.Numeric valuesSource, IncludeExclude.LongFilter longFilter) {
+ return ctx -> {
+ SortedNumericDoubleValues values = valuesSource.doubleValues(ctx);
+ return doc -> {
+ if (values.advanceExact(doc)) {
+ int valuesCount = values.docValueCount();
+
+ double previous = Double.MAX_VALUE;
+ List<TermValue<?>> termValues = new ArrayList<>(valuesCount);
+ for (int i = 0; i < valuesCount; ++i) {
+ double val = values.nextValue();
+ if (previous != val || i == 0) {
+ if (longFilter == null || longFilter.accept(NumericUtils.doubleToSortableLong(val))) {
+ termValues.add(TermValue.of(val));
+ }
+ previous = val;
+ }
+ }
+ return termValues;
+ }
+ return Collections.emptyList();
+ };
+ };
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: NumericTermsAggregator (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | NumericTermsAggregator |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ 0%
+
+
+ (0/17)
+
+ |
+
+
+ | NumericTermsAggregator$1 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/14)
+
+ |
+
+
+ | NumericTermsAggregator$DoubleTermsResults |
+
+
+ 0%
+
+
+ (0/9)
+
+ |
+
+
+ 0%
+
+
+ (0/24)
+
+ |
+
+
+ | NumericTermsAggregator$LongTermsResults |
+
+
+ 0%
+
+
+ (0/9)
+
+ |
+
+
+ 0%
+
+
+ (0/24)
+
+ |
+
+
+ | NumericTermsAggregator$ResultStrategy |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/32)
+
+ |
+
+
+ | NumericTermsAggregator$SignificantLongTermsResults |
+
+
+ 0%
+
+
+ (0/14)
+
+ |
+
+
+ 0%
+
+
+ (0/34)
+
+ |
+
+
+ | NumericTermsAggregator$SignificantLongTermsResults$1 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ | NumericTermsAggregator$StandardTermsResultStrategy |
+
+
+ 0%
+
+
+ (0/7)
+
+ |
+
+
+ 0%
+
+
+ (0/19)
+
+ |
+
+
+ | NumericTermsAggregator$UnsignedLongTermsResults |
+
+
+ 0%
+
+
+ (0/9)
+
+ |
+
+
+ 0%
+
+
+ (0/24)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/62)
+
+ |
+
+
+ 0%
+
+
+ (0/192)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.index.IndexReader;
+ import org.apache.lucene.index.LeafReaderContext;
+ import org.apache.lucene.index.SortedNumericDocValues;
+ import org.apache.lucene.search.ScoreMode;
+ import org.apache.lucene.util.NumericUtils;
+ import org.apache.lucene.util.PriorityQueue;
+ import org.opensearch.common.Numbers;
+ import org.opensearch.common.lease.Releasable;
+ import org.opensearch.common.lease.Releasables;
+ import org.opensearch.common.util.LongArray;
+ import org.opensearch.index.fielddata.FieldData;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.InternalMultiBucketAggregation;
+ import org.opensearch.search.aggregations.InternalOrder;
+ import org.opensearch.search.aggregations.LeafBucketCollector;
+ import org.opensearch.search.aggregations.LeafBucketCollectorBase;
+ import org.opensearch.search.aggregations.bucket.LocalBucketCountThresholds;
+ import org.opensearch.search.aggregations.bucket.terms.IncludeExclude.LongFilter;
+ import org.opensearch.search.aggregations.bucket.terms.LongKeyedBucketOrds.BucketOrdsEnum;
+ import org.opensearch.search.aggregations.bucket.terms.SignificanceLookup.BackgroundFrequencyForLong;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+ import org.opensearch.search.aggregations.support.ValuesSource;
+ import org.opensearch.search.internal.ContextIndexSearcher;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.math.BigInteger;
+ import java.util.Arrays;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.function.BiConsumer;
+ import java.util.function.Function;
+ import java.util.function.Supplier;
+
+ import static java.util.Collections.emptyList;
+ import static org.opensearch.search.aggregations.InternalOrder.isKeyOrder;
+
+ /**
+ * Aggregate all docs that contain numeric terms
+ *
+ * @opensearch.internal
+ */
+ public class NumericTermsAggregator extends TermsAggregator {
+ private final ResultStrategy<?, ?> resultStrategy;
+ private final ValuesSource.Numeric valuesSource;
+ private final LongKeyedBucketOrds bucketOrds;
+ private final LongFilter longFilter;
+
+ public NumericTermsAggregator(
+ String name,
+ AggregatorFactories factories,
+ Function<NumericTermsAggregator, ResultStrategy<?, ?>> resultStrategy,
+ ValuesSource.Numeric valuesSource,
+ DocValueFormat format,
+ BucketOrder order,
+ BucketCountThresholds bucketCountThresholds,
+ SearchContext aggregationContext,
+ Aggregator parent,
+ SubAggCollectionMode subAggCollectMode,
+ IncludeExclude.LongFilter longFilter,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+ super(name, factories, aggregationContext, parent, bucketCountThresholds, order, format, subAggCollectMode, metadata);
+ this.resultStrategy = resultStrategy.apply(this); // ResultStrategy needs a reference to the Aggregator to do its job.
+ this.valuesSource = valuesSource;
+ this.longFilter = longFilter;
+ bucketOrds = LongKeyedBucketOrds.build(context.bigArrays(), cardinality);
+ }
+
+ @Override
+ public ScoreMode scoreMode() {
+ if (valuesSource != null && valuesSource.needsScores()) {
+ return ScoreMode.COMPLETE;
+ }
+ return super.scoreMode();
+ }
+
+ @Override
+ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
+ SortedNumericDocValues values = resultStrategy.getValues(ctx);
+ return resultStrategy.wrapCollector(new LeafBucketCollectorBase(sub, values) {
+ @Override
+ public void collect(int doc, long owningBucketOrd) throws IOException {
+ if (values.advanceExact(doc)) {
+ int valuesCount = values.docValueCount();
+
+ long previous = Long.MAX_VALUE;
+ for (int i = 0; i < valuesCount; ++i) {
+ long val = values.nextValue();
+ if (previous != val || i == 0) {
+ if ((longFilter == null) || (longFilter.accept(val))) {
+ long bucketOrdinal = bucketOrds.add(owningBucketOrd, val);
+ if (bucketOrdinal < 0) { // already seen
+ bucketOrdinal = -1 - bucketOrdinal;
+ collectExistingBucket(sub, doc, bucketOrdinal);
+ } else {
+ collectBucket(sub, doc, bucketOrdinal);
+ }
+ }
+
+ previous = val;
+ }
+ }
+ }
+ }
+ });
+ }
+
+ @Override
+ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException {
+ return resultStrategy.buildAggregations(owningBucketOrds);
+ }
+
+ @Override
+ public InternalAggregation buildEmptyAggregation() {
+ return resultStrategy.buildEmptyResult();
+ }
+
+ @Override
+ public void doClose() {
+ Releasables.close(super::doClose, bucketOrds, resultStrategy);
+ }
+
+ @Override
+ public void collectDebugInfo(BiConsumer<String, Object> add) {
+ super.collectDebugInfo(add);
+ add.accept("result_strategy", resultStrategy.describe());
+ add.accept("total_buckets", bucketOrds.size());
+ }
+
+ /**
+ * Strategy for building results.
+ */
+ abstract class ResultStrategy<R extends InternalAggregation, B extends InternalMultiBucketAggregation.InternalBucket>
+ implements
+ Releasable {
+ private InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException {
+ LocalBucketCountThresholds localBucketCountThresholds = context.asLocalBucketCountThresholds(bucketCountThresholds);
+ B[][] topBucketsPerOrd = buildTopBucketsPerOrd(owningBucketOrds.length);
+ long[] otherDocCounts = new long[owningBucketOrds.length];
+ for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) {
+ collectZeroDocEntriesIfNeeded(owningBucketOrds[ordIdx]);
+ long bucketsInOrd = bucketOrds.bucketsInOrd(owningBucketOrds[ordIdx]);
+
+ int size = (int) Math.min(bucketsInOrd, localBucketCountThresholds.getRequiredSize());
+ PriorityQueue<B> ordered = buildPriorityQueue(size);
+ B spare = null;
+ BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrds[ordIdx]);
+ Supplier<B> emptyBucketBuilder = emptyBucketBuilder(owningBucketOrds[ordIdx]);
+ while (ordsEnum.next()) {
+ long docCount = bucketDocCount(ordsEnum.ord());
+ otherDocCounts[ordIdx] += docCount;
+ if (docCount < localBucketCountThresholds.getMinDocCount()) {
+ continue;
+ }
+ if (spare == null) {
+ spare = emptyBucketBuilder.get();
+ }
+ updateBucket(spare, ordsEnum, docCount);
+ spare = ordered.insertWithOverflow(spare);
+ }
+
+ // Get the top buckets
+ B[] bucketsForOrd = buildBuckets(ordered.size());
+ topBucketsPerOrd[ordIdx] = bucketsForOrd;
+ for (int b = ordered.size() - 1; b >= 0; --b) {
+ topBucketsPerOrd[ordIdx][b] = ordered.pop();
+ otherDocCounts[ordIdx] -= topBucketsPerOrd[ordIdx][b].getDocCount();
+ }
+ }
+
+ buildSubAggs(topBucketsPerOrd);
+
+ InternalAggregation[] result = new InternalAggregation[owningBucketOrds.length];
+ for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) {
+ result[ordIdx] = buildResult(owningBucketOrds[ordIdx], otherDocCounts[ordIdx], topBucketsPerOrd[ordIdx]);
+ }
+ return result;
+ }
+
+ /**
+ * Short description of the collection mechanism added to the profile
+ * output to help with debugging.
+ */
+ abstract String describe();
+
+ /**
+ * Resolve the doc values to collect results of this type.
+ */
+ abstract SortedNumericDocValues getValues(LeafReaderContext ctx) throws IOException;
+
+ /**
+ * Wrap the "standard" numeric terms collector to collect any more
+ * information that this result type may need.
+ */
+ abstract LeafBucketCollector wrapCollector(LeafBucketCollector primary);
+
+ /**
+ * Build an array to hold the "top" buckets for each ordinal.
+ */
+ abstract B[][] buildTopBucketsPerOrd(int size);
+
+ /**
+ * Build an array of buckets for a particular ordinal. These arrays
+ * are asigned to the value returned by {@link #buildTopBucketsPerOrd}.
+ */
+ abstract B[] buildBuckets(int size);
+
+ /**
+ * Build a {@linkplain Supplier} that can be used to build "empty"
+ * buckets. Those buckets will then be {@link #updateBucket updated}
+ * for each collected bucket.
+ */
+ abstract Supplier<B> emptyBucketBuilder(long owningBucketOrd);
+
+ /**
+ * Update fields in {@code spare} to reflect information collected for
+ * this bucket ordinal.
+ */
+ abstract void updateBucket(B spare, BucketOrdsEnum ordsEnum, long docCount) throws IOException;
+
+ /**
+ * Build a {@link PriorityQueue} to sort the buckets. After we've
+ * collected all of the buckets we'll collect all entries in the queue.
+ */
+ abstract PriorityQueue<B> buildPriorityQueue(int size);
+
+ /**
+ * Build the sub-aggregations into the buckets. This will usually
+ * delegate to {@link #buildSubAggsForAllBuckets}.
+ */
+ abstract void buildSubAggs(B[][] topBucketsPerOrd) throws IOException;
+
+ /**
+ * Collect extra entries for "zero" hit documents if they were requested
+ * and required.
+ */
+ abstract void collectZeroDocEntriesIfNeeded(long owningBucketOrd) throws IOException;
+
+ /**
+ * Turn the buckets into an aggregation result.
+ */
+ abstract R buildResult(long owningBucketOrd, long otherDocCounts, B[] topBuckets);
+
+ /**
+ * Build an "empty" result. Only called if there isn't any data on this
+ * shard.
+ */
+ abstract R buildEmptyResult();
+ }
+
+ abstract class StandardTermsResultStrategy<R extends InternalMappedTerms<R, B>, B extends InternalTerms.Bucket<B>> extends
+ ResultStrategy<R, B> {
+ protected final boolean showTermDocCountError;
+
+ StandardTermsResultStrategy(boolean showTermDocCountError) {
+ this.showTermDocCountError = showTermDocCountError;
+ }
+
+ @Override
+ final LeafBucketCollector wrapCollector(LeafBucketCollector primary) {
+ return primary;
+ }
+
+ @Override
+ final PriorityQueue<B> buildPriorityQueue(int size) {
+ return new BucketPriorityQueue<>(size, partiallyBuiltBucketComparator);
+ }
+
+ @Override
+ final void buildSubAggs(B[][] topBucketsPerOrd) throws IOException {
+ buildSubAggsForAllBuckets(topBucketsPerOrd, b -> b.bucketOrd, (b, aggs) -> b.aggregations = aggs);
+ }
+
+ @Override
+ Supplier<B> emptyBucketBuilder(long owningBucketOrd) {
+ return this::buildEmptyBucket;
+ }
+
+ abstract B buildEmptyBucket();
+
+ @Override
+ final void collectZeroDocEntriesIfNeeded(long owningBucketOrd) throws IOException {
+ if (bucketCountThresholds.getMinDocCount() != 0) {
+ return;
+ }
+ if (InternalOrder.isCountDesc(order) && bucketOrds.bucketsInOrd(owningBucketOrd) >= bucketCountThresholds.getRequiredSize()) {
+ return;
+ }
+ // we need to fill-in the blanks
+ for (LeafReaderContext ctx : context.searcher().getTopReaderContext().leaves()) {
+ SortedNumericDocValues values = getValues(ctx);
+ for (int docId = 0; docId < ctx.reader().maxDoc(); ++docId) {
+ if (values.advanceExact(docId)) {
+ int valueCount = values.docValueCount();
+ for (int v = 0; v < valueCount; ++v) {
+ long value = values.nextValue();
+ if (longFilter == null || longFilter.accept(value)) {
+ bucketOrds.add(owningBucketOrd, value);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public final void close() {}
+ }
+
+ class LongTermsResults extends StandardTermsResultStrategy<LongTerms, LongTerms.Bucket> {
+ LongTermsResults(boolean showTermDocCountError) {
+ super(showTermDocCountError);
+ }
+
+ @Override
+ String describe() {
+ return "long_terms";
+ }
+
+ @Override
+ SortedNumericDocValues getValues(LeafReaderContext ctx) throws IOException {
+ return valuesSource.longValues(ctx);
+ }
+
+ @Override
+ LongTerms.Bucket[][] buildTopBucketsPerOrd(int size) {
+ return new LongTerms.Bucket[size][];
+ }
+
+ @Override
+ LongTerms.Bucket[] buildBuckets(int size) {
+ return new LongTerms.Bucket[size];
+ }
+
+ @Override
+ LongTerms.Bucket buildEmptyBucket() {
+ return new LongTerms.Bucket(0, 0, null, showTermDocCountError, 0, format);
+ }
+
+ @Override
+ void updateBucket(LongTerms.Bucket spare, BucketOrdsEnum ordsEnum, long docCount) {
+ spare.term = ordsEnum.value();
+ spare.docCount = docCount;
+ spare.bucketOrd = ordsEnum.ord();
+ }
+
+ @Override
+ LongTerms buildResult(long owningBucketOrd, long otherDocCount, LongTerms.Bucket[] topBuckets) {
+ final BucketOrder reduceOrder;
+ if (isKeyOrder(order) == false) {
+ reduceOrder = InternalOrder.key(true);
+ Arrays.sort(topBuckets, reduceOrder.comparator());
+ } else {
+ reduceOrder = order;
+ }
+ return new LongTerms(
+ name,
+ reduceOrder,
+ order,
+ metadata(),
+ format,
+ bucketCountThresholds.getShardSize(),
+ showTermDocCountError,
+ otherDocCount,
+ List.of(topBuckets),
+ 0,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ LongTerms buildEmptyResult() {
+ return new LongTerms(
+ name,
+ order,
+ order,
+ metadata(),
+ format,
+ bucketCountThresholds.getShardSize(),
+ showTermDocCountError,
+ 0,
+ emptyList(),
+ 0,
+ bucketCountThresholds
+ );
+ }
+ }
+
+ class DoubleTermsResults extends StandardTermsResultStrategy<DoubleTerms, DoubleTerms.Bucket> {
+
+ DoubleTermsResults(boolean showTermDocCountError) {
+ super(showTermDocCountError);
+ }
+
+ @Override
+ String describe() {
+ return "double_terms";
+ }
+
+ @Override
+ SortedNumericDocValues getValues(LeafReaderContext ctx) throws IOException {
+ return FieldData.toSortableLongBits(valuesSource.doubleValues(ctx));
+ }
+
+ @Override
+ DoubleTerms.Bucket[][] buildTopBucketsPerOrd(int size) {
+ return new DoubleTerms.Bucket[size][];
+ }
+
+ @Override
+ DoubleTerms.Bucket[] buildBuckets(int size) {
+ return new DoubleTerms.Bucket[size];
+ }
+
+ @Override
+ DoubleTerms.Bucket buildEmptyBucket() {
+ return new DoubleTerms.Bucket(0, 0, null, showTermDocCountError, 0, format);
+ }
+
+ @Override
+ void updateBucket(DoubleTerms.Bucket spare, BucketOrdsEnum ordsEnum, long docCount) {
+ spare.term = NumericUtils.sortableLongToDouble(ordsEnum.value());
+ spare.docCount = docCount;
+ spare.bucketOrd = ordsEnum.ord();
+ }
+
+ @Override
+ DoubleTerms buildResult(long owningBucketOrd, long otherDocCount, DoubleTerms.Bucket[] topBuckets) {
+ final BucketOrder reduceOrder;
+ if (isKeyOrder(order) == false) {
+ reduceOrder = InternalOrder.key(true);
+ Arrays.sort(topBuckets, reduceOrder.comparator());
+ } else {
+ reduceOrder = order;
+ }
+ return new DoubleTerms(
+ name,
+ reduceOrder,
+ order,
+ metadata(),
+ format,
+ bucketCountThresholds.getShardSize(),
+ showTermDocCountError,
+ otherDocCount,
+ List.of(topBuckets),
+ 0,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ DoubleTerms buildEmptyResult() {
+ return new DoubleTerms(
+ name,
+ order,
+ order,
+ metadata(),
+ format,
+ bucketCountThresholds.getShardSize(),
+ showTermDocCountError,
+ 0,
+ emptyList(),
+ 0,
+ bucketCountThresholds
+ );
+ }
+ }
+
+ class UnsignedLongTermsResults extends StandardTermsResultStrategy<UnsignedLongTerms, UnsignedLongTerms.Bucket> {
+ UnsignedLongTermsResults(boolean showTermDocCountError) {
+ super(showTermDocCountError);
+ }
+
+ @Override
+ String describe() {
+ return "unsigned_long_terms";
+ }
+
+ @Override
+ SortedNumericDocValues getValues(LeafReaderContext ctx) throws IOException {
+ return valuesSource.longValues(ctx);
+ }
+
+ @Override
+ UnsignedLongTerms.Bucket[][] buildTopBucketsPerOrd(int size) {
+ return new UnsignedLongTerms.Bucket[size][];
+ }
+
+ @Override
+ UnsignedLongTerms.Bucket[] buildBuckets(int size) {
+ return new UnsignedLongTerms.Bucket[size];
+ }
+
+ @Override
+ UnsignedLongTerms.Bucket buildEmptyBucket() {
+ return new UnsignedLongTerms.Bucket(BigInteger.ZERO, 0, null, showTermDocCountError, 0, format);
+ }
+
+ @Override
+ void updateBucket(UnsignedLongTerms.Bucket spare, BucketOrdsEnum ordsEnum, long docCount) {
+ spare.term = Numbers.toUnsignedBigInteger(ordsEnum.value());
+ spare.docCount = docCount;
+ spare.bucketOrd = ordsEnum.ord();
+ }
+
+ @Override
+ UnsignedLongTerms buildResult(long owningBucketOrd, long otherDocCount, UnsignedLongTerms.Bucket[] topBuckets) {
+ final BucketOrder reduceOrder;
+ if (isKeyOrder(order) == false) {
+ reduceOrder = InternalOrder.key(true);
+ Arrays.sort(topBuckets, reduceOrder.comparator());
+ } else {
+ reduceOrder = order;
+ }
+ return new UnsignedLongTerms(
+ name,
+ reduceOrder,
+ order,
+ metadata(),
+ format,
+ bucketCountThresholds.getShardSize(),
+ showTermDocCountError,
+ otherDocCount,
+ List.of(topBuckets),
+ 0,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ UnsignedLongTerms buildEmptyResult() {
+ return new UnsignedLongTerms(
+ name,
+ order,
+ order,
+ metadata(),
+ format,
+ bucketCountThresholds.getShardSize(),
+ showTermDocCountError,
+ 0,
+ emptyList(),
+ 0,
+ bucketCountThresholds
+ );
+ }
+ }
+
+ class SignificantLongTermsResults extends ResultStrategy<SignificantLongTerms, SignificantLongTerms.Bucket> {
+ private final BackgroundFrequencyForLong backgroundFrequencies;
+ private final long supersetSize;
+ private final SignificanceHeuristic significanceHeuristic;
+ private LongArray subsetSizes;
+
+ SignificantLongTermsResults(
+ SignificanceLookup significanceLookup,
+ SignificanceHeuristic significanceHeuristic,
+ CardinalityUpperBound cardinality
+ ) {
+ backgroundFrequencies = significanceLookup.longLookup(context.bigArrays(), cardinality);
+ supersetSize = significanceLookup.supersetSize();
+ this.significanceHeuristic = significanceHeuristic;
+ subsetSizes = context.bigArrays().newLongArray(1, true);
+ }
+
+ @Override
+ SortedNumericDocValues getValues(LeafReaderContext ctx) throws IOException {
+ return valuesSource.longValues(ctx);
+ }
+
+ @Override
+ String describe() {
+ return "significant_terms";
+ }
+
+ @Override
+ LeafBucketCollector wrapCollector(LeafBucketCollector primary) {
+ return new LeafBucketCollectorBase(primary, null) {
+ @Override
+ public void collect(int doc, long owningBucketOrd) throws IOException {
+ super.collect(doc, owningBucketOrd);
+ subsetSizes = context.bigArrays().grow(subsetSizes, owningBucketOrd + 1);
+ subsetSizes.increment(owningBucketOrd, 1);
+ }
+ };
+ }
+
+ @Override
+ SignificantLongTerms.Bucket[][] buildTopBucketsPerOrd(int size) {
+ return new SignificantLongTerms.Bucket[size][];
+ }
+
+ @Override
+ SignificantLongTerms.Bucket[] buildBuckets(int size) {
+ return new SignificantLongTerms.Bucket[size];
+ }
+
+ @Override
+ Supplier<SignificantLongTerms.Bucket> emptyBucketBuilder(long owningBucketOrd) {
+ long subsetSize = subsetSizes.get(owningBucketOrd);
+ return () -> new SignificantLongTerms.Bucket(0, subsetSize, 0, supersetSize, 0, null, format, 0);
+ }
+
+ @Override
+ void updateBucket(SignificantLongTerms.Bucket spare, BucketOrdsEnum ordsEnum, long docCount) throws IOException {
+ spare.term = ordsEnum.value();
+ spare.subsetDf = docCount;
+ spare.supersetDf = backgroundFrequencies.freq(spare.term);
+ spare.bucketOrd = ordsEnum.ord();
+ // During shard-local down-selection we use subset/superset stats that are for this shard only
+ // Back at the central reducer these properties will be updated with global stats
+ spare.updateScore(significanceHeuristic);
+ }
+
+ @Override
+ PriorityQueue<SignificantLongTerms.Bucket> buildPriorityQueue(int size) {
+ return new BucketSignificancePriorityQueue<>(size);
+ }
+
+ @Override
+ void buildSubAggs(SignificantLongTerms.Bucket[][] topBucketsPerOrd) throws IOException {
+ buildSubAggsForAllBuckets(topBucketsPerOrd, b -> b.bucketOrd, (b, aggs) -> b.aggregations = aggs);
+ }
+
+ @Override
+ void collectZeroDocEntriesIfNeeded(long owningBucketOrd) throws IOException {}
+
+ @Override
+ SignificantLongTerms buildResult(long owningBucketOrd, long otherDocCoun, SignificantLongTerms.Bucket[] topBuckets) {
+ SignificantLongTerms significantLongTerms = new SignificantLongTerms(
+ name,
+ metadata(),
+ format,
+ subsetSizes.get(owningBucketOrd),
+ supersetSize,
+ significanceHeuristic,
+ List.of(topBuckets),
+ bucketCountThresholds
+ );
+ return significantLongTerms;
+ }
+
+ @Override
+ SignificantLongTerms buildEmptyResult() {
+ // We need to account for the significance of a miss in our global stats - provide corpus size as context
+ ContextIndexSearcher searcher = context.searcher();
+ IndexReader topReader = searcher.getIndexReader();
+ int supersetSize = topReader.numDocs();
+ return new SignificantLongTerms(
+ name,
+ metadata(),
+ format,
+ 0,
+ supersetSize,
+ significanceHeuristic,
+ emptyList(),
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ public void close() {
+ Releasables.close(backgroundFrequencies, subsetSizes);
+ }
+ }
+
+ }
+
+
+
+
+
+
Coverage Summary for Class: ParsedDoubleTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ParsedDoubleTerms |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ | ParsedDoubleTerms$ParsedBucket |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ 0%
+
+
+ (0/14)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/22)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+
+ import java.io.IOException;
+
+ /**
+ * A double numeric terms result parsed between nodes
+ *
+ * @opensearch.internal
+ */
+ public class ParsedDoubleTerms extends ParsedTerms {
+
+ @Override
+ public String getType() {
+ return DoubleTerms.NAME;
+ }
+
+ private static final ObjectParser<ParsedDoubleTerms, Void> PARSER = new ObjectParser<>(
+ ParsedDoubleTerms.class.getSimpleName(),
+ true,
+ ParsedDoubleTerms::new
+ );
+ static {
+ declareParsedTermsFields(PARSER, ParsedBucket::fromXContent);
+ }
+
+ public static ParsedDoubleTerms fromXContent(XContentParser parser, String name) throws IOException {
+ ParsedDoubleTerms aggregation = PARSER.parse(parser, null);
+ aggregation.setName(name);
+ return aggregation;
+ }
+
+ /**
+ * Parsed bucket for double terms
+ *
+ * @opensearch.internal
+ */
+ public static class ParsedBucket extends ParsedTerms.ParsedBucket {
+
+ private Double key;
+
+ @Override
+ public Object getKey() {
+ return key;
+ }
+
+ @Override
+ public String getKeyAsString() {
+ String keyAsString = super.getKeyAsString();
+ if (keyAsString != null) {
+ return keyAsString;
+ }
+ if (key != null) {
+ return Double.toString(key);
+ }
+ return null;
+ }
+
+ public Number getKeyAsNumber() {
+ return key;
+ }
+
+ @Override
+ protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ builder.field(CommonFields.KEY.getPreferredName(), key);
+ if (super.getKeyAsString() != null) {
+ builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString());
+ }
+ return builder;
+ }
+
+ static ParsedBucket fromXContent(XContentParser parser) throws IOException {
+ return parseTermsBucketXContent(parser, ParsedBucket::new, (p, bucket) -> bucket.key = p.doubleValue());
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: ParsedLongRareTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ParsedLongRareTerms |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ | ParsedLongRareTerms$ParsedBucket |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ 0%
+
+
+ (0/14)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/22)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+
+ import java.io.IOException;
+
+ /**
+ * A long numeric term result parsed between nodes
+ *
+ * @opensearch.internal
+ */
+ public class ParsedLongRareTerms extends ParsedRareTerms {
+ @Override
+ public String getType() {
+ return LongRareTerms.NAME;
+ }
+
+ private static final ObjectParser<ParsedLongRareTerms, Void> PARSER = new ObjectParser<>(
+ ParsedLongRareTerms.class.getSimpleName(),
+ true,
+ ParsedLongRareTerms::new
+ );
+
+ static {
+ declareParsedTermsFields(PARSER, ParsedBucket::fromXContent);
+ }
+
+ public static ParsedLongRareTerms fromXContent(XContentParser parser, String name) throws IOException {
+ ParsedLongRareTerms aggregation = PARSER.parse(parser, null);
+ aggregation.setName(name);
+ return aggregation;
+ }
+
+ /**
+ * Parsed bucket for rare long values
+ *
+ * @opensearch.internal
+ */
+ public static class ParsedBucket extends ParsedRareTerms.ParsedBucket {
+
+ private Long key;
+
+ @Override
+ public Object getKey() {
+ return key;
+ }
+
+ @Override
+ public String getKeyAsString() {
+ String keyAsString = super.getKeyAsString();
+ if (keyAsString != null) {
+ return keyAsString;
+ }
+ if (key != null) {
+ return Long.toString(key);
+ }
+ return null;
+ }
+
+ public Number getKeyAsNumber() {
+ return key;
+ }
+
+ @Override
+ protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ builder.field(CommonFields.KEY.getPreferredName(), key);
+ if (super.getKeyAsString() != null) {
+ builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString());
+ }
+ return builder;
+ }
+
+ static ParsedLongRareTerms.ParsedBucket fromXContent(XContentParser parser) throws IOException {
+ return parseRareTermsBucketXContent(parser, ParsedLongRareTerms.ParsedBucket::new, (p, bucket) -> bucket.key = p.longValue());
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: ParsedLongTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ParsedLongTerms |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ | ParsedLongTerms$ParsedBucket |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ 0%
+
+
+ (0/14)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/22)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+
+ import java.io.IOException;
+
+ /**
+ * A long term agg result parsed between nodes
+ *
+ * @opensearch.internal
+ */
+ public class ParsedLongTerms extends ParsedTerms {
+
+ @Override
+ public String getType() {
+ return LongTerms.NAME;
+ }
+
+ private static final ObjectParser<ParsedLongTerms, Void> PARSER = new ObjectParser<>(
+ ParsedLongTerms.class.getSimpleName(),
+ true,
+ ParsedLongTerms::new
+ );
+ static {
+ declareParsedTermsFields(PARSER, ParsedBucket::fromXContent);
+ }
+
+ public static ParsedLongTerms fromXContent(XContentParser parser, String name) throws IOException {
+ ParsedLongTerms aggregation = PARSER.parse(parser, null);
+ aggregation.setName(name);
+ return aggregation;
+ }
+
+ /**
+ * Parsed bucket for long term values
+ *
+ * @opensearch.internal
+ */
+ public static class ParsedBucket extends ParsedTerms.ParsedBucket {
+
+ private Long key;
+
+ @Override
+ public Object getKey() {
+ return key;
+ }
+
+ @Override
+ public String getKeyAsString() {
+ String keyAsString = super.getKeyAsString();
+ if (keyAsString != null) {
+ return keyAsString;
+ }
+ if (key != null) {
+ return Long.toString(key);
+ }
+ return null;
+ }
+
+ public Number getKeyAsNumber() {
+ return key;
+ }
+
+ @Override
+ protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ builder.field(CommonFields.KEY.getPreferredName(), key);
+ if (super.getKeyAsString() != null) {
+ builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString());
+ }
+ return builder;
+ }
+
+ static ParsedBucket fromXContent(XContentParser parser) throws IOException {
+ return parseTermsBucketXContent(parser, ParsedBucket::new, (p, bucket) -> bucket.key = p.longValue());
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: ParsedMultiTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ParsedMultiTerms |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ | ParsedMultiTerms$ParsedBucket |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ 0%
+
+
+ (0/14)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/22)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+
+ import java.io.IOException;
+ import java.util.List;
+
+ /**
+ * A multi terms result parsed between nodes
+ *
+ * @opensearch.internal
+ */
+ public class ParsedMultiTerms extends ParsedTerms {
+ @Override
+ public String getType() {
+ return MultiTermsAggregationBuilder.NAME;
+ }
+
+ private static final ObjectParser<ParsedMultiTerms, Void> PARSER = new ObjectParser<>(
+ ParsedMultiTerms.class.getSimpleName(),
+ true,
+ ParsedMultiTerms::new
+ );
+ static {
+ declareParsedTermsFields(PARSER, ParsedBucket::fromXContent);
+ }
+
+ public static ParsedMultiTerms fromXContent(XContentParser parser, String name) throws IOException {
+ ParsedMultiTerms aggregation = PARSER.parse(parser, null);
+ aggregation.setName(name);
+ return aggregation;
+ }
+
+ /**
+ * Parsed bucket for multi terms
+ *
+ * @opensearch.internal
+ */
+ public static class ParsedBucket extends ParsedTerms.ParsedBucket {
+
+ private List<Object> key;
+
+ @Override
+ public List<Object> getKey() {
+ return key;
+ }
+
+ @Override
+ public String getKeyAsString() {
+ String keyAsString = super.getKeyAsString();
+ if (keyAsString != null) {
+ return keyAsString;
+ }
+ if (key != null) {
+ return key.toString();
+ }
+ return null;
+ }
+
+ public Number getKeyAsNumber() {
+ throw new UnsupportedOperationException("not implemented");
+ }
+
+ @Override
+ protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ builder.field(CommonFields.KEY.getPreferredName(), key);
+ if (super.getKeyAsString() != null) {
+ builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString());
+ }
+ return builder;
+ }
+
+ static ParsedBucket fromXContent(XContentParser parser) throws IOException {
+ return parseTermsBucketXContent(parser, ParsedBucket::new, (p, bucket) -> { bucket.key = p.list(); });
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: ParsedRareTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ParsedRareTerms |
+
+
+ 0%
+
+
+ (0/5)
+
+ |
+
+
+ 0%
+
+
+ (0/14)
+
+ |
+
+
+ | ParsedRareTerms$ParsedBucket |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ 0%
+
+
+ (0/25)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ 0%
+
+
+ (0/39)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.common.CheckedBiConsumer;
+ import org.opensearch.common.CheckedFunction;
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+ import org.opensearch.core.xcontent.XContentParserUtils;
+ import org.opensearch.search.aggregations.Aggregation;
+ import org.opensearch.search.aggregations.Aggregations;
+ import org.opensearch.search.aggregations.ParsedMultiBucketAggregation;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.List;
+ import java.util.function.Supplier;
+
+ /**
+ * A rare term result parsed between nodes
+ *
+ * @opensearch.internal
+ */
+ public abstract class ParsedRareTerms extends ParsedMultiBucketAggregation<ParsedRareTerms.ParsedBucket> implements RareTerms {
+ @Override
+ public List<? extends RareTerms.Bucket> getBuckets() {
+ return buckets;
+ }
+
+ @Override
+ public RareTerms.Bucket getBucketByKey(String term) {
+ for (RareTerms.Bucket bucket : getBuckets()) {
+ if (bucket.getKeyAsString().equals(term)) {
+ return bucket;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+ builder.startArray(CommonFields.BUCKETS.getPreferredName());
+ for (RareTerms.Bucket bucket : getBuckets()) {
+ bucket.toXContent(builder, params);
+ }
+ builder.endArray();
+ return builder;
+ }
+
+ static void declareParsedTermsFields(
+ final ObjectParser<? extends ParsedRareTerms, Void> objectParser,
+ final CheckedFunction<XContentParser, ParsedBucket, IOException> bucketParser
+ ) {
+ declareMultiBucketAggregationFields(objectParser, bucketParser::apply, bucketParser::apply);
+ }
+
+ /**
+ * Parsed Bucket for rare term values
+ *
+ * @opensearch.internal
+ */
+ public abstract static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements RareTerms.Bucket {
+
+ @Override
+ public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject();
+ keyToXContent(builder);
+ builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+ getAggregations().toXContentInternal(builder, params);
+ builder.endObject();
+ return builder;
+ }
+
+ static <B extends ParsedBucket> B parseRareTermsBucketXContent(
+ final XContentParser parser,
+ final Supplier<B> bucketSupplier,
+ final CheckedBiConsumer<XContentParser, B, IOException> keyConsumer
+ ) throws IOException {
+
+ final B bucket = bucketSupplier.get();
+ final List<Aggregation> aggregations = new ArrayList<>();
+
+ XContentParser.Token token;
+ String currentFieldName = parser.currentName();
+ while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+ if (token == XContentParser.Token.FIELD_NAME) {
+ currentFieldName = parser.currentName();
+ } else if (token.isValue()) {
+ if (CommonFields.KEY_AS_STRING.getPreferredName().equals(currentFieldName)) {
+ bucket.setKeyAsString(parser.text());
+ } else if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) {
+ keyConsumer.accept(parser, bucket);
+ } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) {
+ bucket.setDocCount(parser.longValue());
+ }
+ } else if (token == XContentParser.Token.START_OBJECT) {
+ XContentParserUtils.parseTypedKeysObject(
+ parser,
+ Aggregation.TYPED_KEYS_DELIMITER,
+ Aggregation.class,
+ aggregations::add
+ );
+ }
+ }
+ bucket.setAggregations(new Aggregations(aggregations));
+ return bucket;
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: ParsedSignificantLongTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ParsedSignificantLongTerms |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ | ParsedSignificantLongTerms$ParsedBucket |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ 0%
+
+
+ (0/12)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/18)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+
+ import java.io.IOException;
+
+ /**
+ * A significant long terms result parsed between nodes
+ *
+ * @opensearch.internal
+ */
+ public class ParsedSignificantLongTerms extends ParsedSignificantTerms {
+
+ @Override
+ public String getType() {
+ return SignificantLongTerms.NAME;
+ }
+
+ private static final ObjectParser<ParsedSignificantLongTerms, Void> PARSER = new ObjectParser<>(
+ ParsedSignificantLongTerms.class.getSimpleName(),
+ true,
+ ParsedSignificantLongTerms::new
+ );
+ static {
+ declareParsedSignificantTermsFields(PARSER, ParsedBucket::fromXContent);
+ }
+
+ public static ParsedSignificantLongTerms fromXContent(XContentParser parser, String name) throws IOException {
+ return parseSignificantTermsXContent(() -> PARSER.parse(parser, null), name);
+ }
+
+ /**
+ * Parsed bucket for significant long values
+ *
+ * @opensearch.internal
+ */
+ public static class ParsedBucket extends ParsedSignificantTerms.ParsedBucket {
+
+ private Long key;
+
+ @Override
+ public Object getKey() {
+ return key;
+ }
+
+ @Override
+ public String getKeyAsString() {
+ String keyAsString = super.getKeyAsString();
+ if (keyAsString != null) {
+ return keyAsString;
+ }
+ return Long.toString(key);
+ }
+
+ public Number getKeyAsNumber() {
+ return key;
+ }
+
+ @Override
+ protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ builder.field(CommonFields.KEY.getPreferredName(), key);
+ if (super.getKeyAsString() != null) {
+ builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString());
+ }
+ return builder;
+ }
+
+ static ParsedBucket fromXContent(XContentParser parser) throws IOException {
+ return parseSignificantTermsBucketXContent(parser, new ParsedBucket(), (p, bucket) -> bucket.key = p.longValue());
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: ParsedSignificantStringTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ParsedSignificantStringTerms |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ | ParsedSignificantStringTerms$ParsedBucket |
+
+
+ 0%
+
+
+ (0/7)
+
+ |
+
+
+ 0%
+
+
+ (0/13)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/11)
+
+ |
+
+
+ 0%
+
+
+ (0/19)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.util.BytesRef;
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+
+ import java.io.IOException;
+ import java.nio.CharBuffer;
+
+ /**
+ * A significant string terms result parsed between nodes
+ *
+ * @opensearch.internal
+ */
+ public class ParsedSignificantStringTerms extends ParsedSignificantTerms {
+
+ @Override
+ public String getType() {
+ return SignificantStringTerms.NAME;
+ }
+
+ private static final ObjectParser<ParsedSignificantStringTerms, Void> PARSER = new ObjectParser<>(
+ ParsedSignificantStringTerms.class.getSimpleName(),
+ true,
+ ParsedSignificantStringTerms::new
+ );
+ static {
+ declareParsedSignificantTermsFields(PARSER, ParsedBucket::fromXContent);
+ }
+
+ public static ParsedSignificantStringTerms fromXContent(XContentParser parser, String name) throws IOException {
+ return parseSignificantTermsXContent(() -> PARSER.parse(parser, null), name);
+ }
+
+ /**
+ * Parsed bucket for significant string values
+ *
+ * @opensearch.internal
+ */
+ public static class ParsedBucket extends ParsedSignificantTerms.ParsedBucket {
+
+ private BytesRef key;
+
+ @Override
+ public Object getKey() {
+ return getKeyAsString();
+ }
+
+ @Override
+ public String getKeyAsString() {
+ String keyAsString = super.getKeyAsString();
+ if (keyAsString != null) {
+ return keyAsString;
+ }
+ return key.utf8ToString();
+ }
+
+ public Number getKeyAsNumber() {
+ return Double.parseDouble(key.utf8ToString());
+ }
+
+ @Override
+ protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ return builder.field(CommonFields.KEY.getPreferredName(), getKey());
+ }
+
+ static ParsedBucket fromXContent(XContentParser parser) throws IOException {
+ return parseSignificantTermsBucketXContent(parser, new ParsedBucket(), (p, bucket) -> {
+ CharBuffer cb = p.charBufferOrNull();
+ if (cb == null) {
+ bucket.key = null;
+ } else {
+ bucket.key = new BytesRef(cb);
+ }
+ });
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: AbstractStringTermsAggregator (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | AbstractStringTermsAggregator |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 16.7%
+
+
+ (2/12)
+
+ |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.index.IndexReader;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+ import org.opensearch.search.internal.ContextIndexSearcher;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.util.Map;
+
+ import static java.util.Collections.emptyList;
+
+ /**
+ * Base Aggregator to collect all docs that contain significant terms
+ *
+ * @opensearch.internal
+ */
+ abstract class AbstractStringTermsAggregator extends TermsAggregator {
+
+ protected final boolean showTermDocCountError;
+
+ AbstractStringTermsAggregator(
+ String name,
+ AggregatorFactories factories,
+ SearchContext context,
+ Aggregator parent,
+ BucketOrder order,
+ DocValueFormat format,
+ BucketCountThresholds bucketCountThresholds,
+ SubAggCollectionMode subAggCollectMode,
+ boolean showTermDocCountError,
+ Map<String, Object> metadata
+ ) throws IOException {
+ super(name, factories, context, parent, bucketCountThresholds, order, format, subAggCollectMode, metadata);
+ this.showTermDocCountError = showTermDocCountError;
+ }
+
+ protected StringTerms buildEmptyTermsAggregation() {
+ return new StringTerms(
+ name,
+ order,
+ order,
+ metadata(),
+ format,
+ bucketCountThresholds.getShardSize(),
+ showTermDocCountError,
+ 0,
+ emptyList(),
+ 0,
+ bucketCountThresholds
+ );
+ }
+
+ protected SignificantStringTerms buildEmptySignificantTermsAggregation(long subsetSize, SignificanceHeuristic significanceHeuristic) {
+ // We need to account for the significance of a miss in our global stats - provide corpus size as context
+ ContextIndexSearcher searcher = context.searcher();
+ IndexReader topReader = searcher.getIndexReader();
+ int supersetSize = topReader.numDocs();
+ return new SignificantStringTerms(
+ name,
+ metadata(),
+ format,
+ subsetSize,
+ supersetSize,
+ significanceHeuristic,
+ emptyList(),
+ bucketCountThresholds
+ );
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: ParsedSignificantTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ParsedSignificantTerms |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/27)
+
+ |
+
+
+ | ParsedSignificantTerms$ParsedBucket |
+
+
+ 0%
+
+
+ (0/9)
+
+ |
+
+
+ 0%
+
+
+ (0/38)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/19)
+
+ |
+
+
+ 0%
+
+
+ (0/65)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.common.CheckedBiConsumer;
+ import org.opensearch.common.CheckedFunction;
+ import org.opensearch.common.CheckedSupplier;
+ import org.opensearch.core.ParseField;
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+ import org.opensearch.core.xcontent.XContentParserUtils;
+ import org.opensearch.search.aggregations.Aggregation;
+ import org.opensearch.search.aggregations.Aggregations;
+ import org.opensearch.search.aggregations.ParsedMultiBucketAggregation;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.function.Function;
+ import java.util.stream.Collectors;
+
+ /**
+ * A significant terms result parsed between nodes
+ *
+ * @opensearch.internal
+ */
+ public abstract class ParsedSignificantTerms extends ParsedMultiBucketAggregation<ParsedSignificantTerms.ParsedBucket>
+ implements
+ SignificantTerms {
+
+ private Map<String, ParsedBucket> bucketMap;
+ protected long subsetSize;
+ protected long supersetSize;
+
+ protected long getSubsetSize() {
+ return subsetSize;
+ }
+
+ protected long getSupersetSize() {
+ return supersetSize;
+ }
+
+ @Override
+ public List<? extends SignificantTerms.Bucket> getBuckets() {
+ return buckets;
+ }
+
+ @Override
+ public SignificantTerms.Bucket getBucketByKey(String term) {
+ if (bucketMap == null) {
+ bucketMap = buckets.stream().collect(Collectors.toMap(SignificantTerms.Bucket::getKeyAsString, Function.identity()));
+ }
+ return bucketMap.get(term);
+ }
+
+ @Override
+ public Iterator<SignificantTerms.Bucket> iterator() {
+ return buckets.stream().map(bucket -> (SignificantTerms.Bucket) bucket).collect(Collectors.toList()).iterator();
+ }
+
+ @Override
+ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+ builder.field(CommonFields.DOC_COUNT.getPreferredName(), subsetSize);
+ builder.field(InternalMappedSignificantTerms.BG_COUNT, supersetSize);
+ builder.startArray(CommonFields.BUCKETS.getPreferredName());
+ for (SignificantTerms.Bucket bucket : buckets) {
+ bucket.toXContent(builder, params);
+ }
+ builder.endArray();
+ return builder;
+ }
+
+ static <T extends ParsedSignificantTerms> T parseSignificantTermsXContent(
+ final CheckedSupplier<T, IOException> aggregationSupplier,
+ final String name
+ ) throws IOException {
+ T aggregation = aggregationSupplier.get();
+ aggregation.setName(name);
+ for (ParsedBucket bucket : aggregation.buckets) {
+ bucket.subsetSize = aggregation.subsetSize;
+ bucket.supersetSize = aggregation.supersetSize;
+ }
+ return aggregation;
+ }
+
+ static void declareParsedSignificantTermsFields(
+ final ObjectParser<? extends ParsedSignificantTerms, Void> objectParser,
+ final CheckedFunction<XContentParser, ParsedSignificantTerms.ParsedBucket, IOException> bucketParser
+ ) {
+ declareMultiBucketAggregationFields(objectParser, bucketParser::apply, bucketParser::apply);
+ objectParser.declareLong((parsedTerms, value) -> parsedTerms.subsetSize = value, CommonFields.DOC_COUNT);
+ objectParser.declareLong(
+ (parsedTerms, value) -> parsedTerms.supersetSize = value,
+ new ParseField(InternalMappedSignificantTerms.BG_COUNT)
+ );
+ }
+
+ /**
+ * Parsed bucket for significant values
+ *
+ * @opensearch.internal
+ */
+ public abstract static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements SignificantTerms.Bucket {
+
+ protected long subsetDf;
+ protected long subsetSize;
+ protected long supersetDf;
+ protected long supersetSize;
+ protected double score;
+
+ @Override
+ public long getDocCount() {
+ return getSubsetDf();
+ }
+
+ @Override
+ public long getSubsetDf() {
+ return subsetDf;
+ }
+
+ @Override
+ public long getSupersetDf() {
+ return supersetDf;
+ }
+
+ @Override
+ public double getSignificanceScore() {
+ return score;
+ }
+
+ @Override
+ public long getSupersetSize() {
+ return supersetSize;
+ }
+
+ @Override
+ public long getSubsetSize() {
+ return subsetSize;
+ }
+
+ @Override
+ public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject();
+ keyToXContent(builder);
+ builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+ builder.field(InternalSignificantTerms.SCORE, getSignificanceScore());
+ builder.field(InternalSignificantTerms.BG_COUNT, getSupersetDf());
+ getAggregations().toXContentInternal(builder, params);
+ builder.endObject();
+ return builder;
+ }
+
+ @Override
+ protected abstract XContentBuilder keyToXContent(XContentBuilder builder) throws IOException;
+
+ static <B extends ParsedBucket> B parseSignificantTermsBucketXContent(
+ final XContentParser parser,
+ final B bucket,
+ final CheckedBiConsumer<XContentParser, B, IOException> keyConsumer
+ ) throws IOException {
+
+ final List<Aggregation> aggregations = new ArrayList<>();
+ XContentParser.Token token;
+ String currentFieldName = parser.currentName();
+ while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+ if (token == XContentParser.Token.FIELD_NAME) {
+ currentFieldName = parser.currentName();
+ } else if (token.isValue()) {
+ if (CommonFields.KEY_AS_STRING.getPreferredName().equals(currentFieldName)) {
+ bucket.setKeyAsString(parser.text());
+ } else if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) {
+ keyConsumer.accept(parser, bucket);
+ } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) {
+ long value = parser.longValue();
+ bucket.subsetDf = value;
+ bucket.setDocCount(value);
+ } else if (InternalSignificantTerms.SCORE.equals(currentFieldName)) {
+ bucket.score = parser.doubleValue();
+ } else if (InternalSignificantTerms.BG_COUNT.equals(currentFieldName)) {
+ bucket.supersetDf = parser.longValue();
+ }
+ } else if (token == XContentParser.Token.START_OBJECT) {
+ XContentParserUtils.parseTypedKeysObject(
+ parser,
+ Aggregation.TYPED_KEYS_DELIMITER,
+ Aggregation.class,
+ aggregations::add
+ );
+ }
+ }
+ bucket.setAggregations(new Aggregations(aggregations));
+ return bucket;
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: ParsedStringRareTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ParsedStringRareTerms |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ | ParsedStringRareTerms$ParsedBucket |
+
+
+ 0%
+
+
+ (0/7)
+
+ |
+
+
+ 0%
+
+
+ (0/17)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/11)
+
+ |
+
+
+ 0%
+
+
+ (0/25)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.util.BytesRef;
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+
+ import java.io.IOException;
+ import java.nio.CharBuffer;
+
+ /**
+ * A significant rare result parsed between nodes
+ *
+ * @opensearch.internal
+ */
+ public class ParsedStringRareTerms extends ParsedRareTerms {
+ @Override
+ public String getType() {
+ return StringRareTerms.NAME;
+ }
+
+ private static final ObjectParser<ParsedStringRareTerms, Void> PARSER = new ObjectParser<>(
+ ParsedStringRareTerms.class.getSimpleName(),
+ true,
+ ParsedStringRareTerms::new
+ );
+
+ static {
+ declareParsedTermsFields(PARSER, ParsedBucket::fromXContent);
+ }
+
+ public static ParsedStringRareTerms fromXContent(XContentParser parser, String name) throws IOException {
+ ParsedStringRareTerms aggregation = PARSER.parse(parser, null);
+ aggregation.setName(name);
+ return aggregation;
+ }
+
+ /**
+ * Parsed bucket for rare string terms
+ *
+ * @opensearch.internal
+ */
+ public static class ParsedBucket extends ParsedRareTerms.ParsedBucket {
+
+ private BytesRef key;
+
+ @Override
+ public Object getKey() {
+ return getKeyAsString();
+ }
+
+ @Override
+ public String getKeyAsString() {
+ String keyAsString = super.getKeyAsString();
+ if (keyAsString != null) {
+ return keyAsString;
+ }
+ if (key != null) {
+ return key.utf8ToString();
+ }
+ return null;
+ }
+
+ public Number getKeyAsNumber() {
+ if (key != null) {
+ return Double.parseDouble(key.utf8ToString());
+ }
+ return null;
+ }
+
+ @Override
+ protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ return builder.field(CommonFields.KEY.getPreferredName(), getKey());
+ }
+
+ static ParsedStringRareTerms.ParsedBucket fromXContent(XContentParser parser) throws IOException {
+ return parseRareTermsBucketXContent(parser, ParsedStringRareTerms.ParsedBucket::new, (p, bucket) -> {
+ CharBuffer cb = p.charBufferOrNull();
+ if (cb == null) {
+ bucket.key = null;
+ } else {
+ bucket.key = new BytesRef(cb);
+ }
+ });
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: ParsedStringTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ParsedStringTerms |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ | ParsedStringTerms$ParsedBucket |
+
+
+ 0%
+
+
+ (0/7)
+
+ |
+
+
+ 0%
+
+
+ (0/17)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/11)
+
+ |
+
+
+ 0%
+
+
+ (0/25)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.util.BytesRef;
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+
+ import java.io.IOException;
+ import java.nio.CharBuffer;
+
+ /**
+ * A string result parsed between nodes
+ *
+ * @opensearch.internal
+ */
+ public class ParsedStringTerms extends ParsedTerms {
+
+ @Override
+ public String getType() {
+ return StringTerms.NAME;
+ }
+
+ private static final ObjectParser<ParsedStringTerms, Void> PARSER = new ObjectParser<>(
+ ParsedStringTerms.class.getSimpleName(),
+ true,
+ ParsedStringTerms::new
+ );
+ static {
+ declareParsedTermsFields(PARSER, ParsedBucket::fromXContent);
+ }
+
+ public static ParsedStringTerms fromXContent(XContentParser parser, String name) throws IOException {
+ ParsedStringTerms aggregation = PARSER.parse(parser, null);
+ aggregation.setName(name);
+ return aggregation;
+ }
+
+ /**
+ * Parsed bucket for string values
+ *
+ * @opensearch.internal
+ */
+ public static class ParsedBucket extends ParsedTerms.ParsedBucket {
+
+ private BytesRef key;
+
+ @Override
+ public Object getKey() {
+ return getKeyAsString();
+ }
+
+ @Override
+ public String getKeyAsString() {
+ String keyAsString = super.getKeyAsString();
+ if (keyAsString != null) {
+ return keyAsString;
+ }
+ if (key != null) {
+ return key.utf8ToString();
+ }
+ return null;
+ }
+
+ public Number getKeyAsNumber() {
+ if (key != null) {
+ return Double.parseDouble(key.utf8ToString());
+ }
+ return null;
+ }
+
+ @Override
+ protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ return builder.field(CommonFields.KEY.getPreferredName(), getKey());
+ }
+
+ static ParsedBucket fromXContent(XContentParser parser) throws IOException {
+ return parseTermsBucketXContent(parser, ParsedBucket::new, (p, bucket) -> {
+ CharBuffer cb = p.charBufferOrNull();
+ if (cb == null) {
+ bucket.key = null;
+ } else {
+ bucket.key = new BytesRef(cb);
+ }
+ });
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: ParsedTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ParsedTerms |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ 0%
+
+
+ (0/21)
+
+ |
+
+
+ | ParsedTerms$ParsedBucket |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/33)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/12)
+
+ |
+
+
+ 0%
+
+
+ (0/54)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.common.CheckedBiConsumer;
+ import org.opensearch.common.CheckedFunction;
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+ import org.opensearch.core.xcontent.XContentParserUtils;
+ import org.opensearch.search.aggregations.Aggregation;
+ import org.opensearch.search.aggregations.Aggregations;
+ import org.opensearch.search.aggregations.ParsedMultiBucketAggregation;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.List;
+ import java.util.function.Supplier;
+
+ import static org.opensearch.search.aggregations.bucket.terms.InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME;
+ import static org.opensearch.search.aggregations.bucket.terms.InternalTerms.SUM_OF_OTHER_DOC_COUNTS;
+
+ /**
+ * A terms result parsed between nodes
+ *
+ * @opensearch.internal
+ */
+ public abstract class ParsedTerms extends ParsedMultiBucketAggregation<ParsedTerms.ParsedBucket> implements Terms {
+
+ protected long docCountErrorUpperBound;
+ protected long sumOtherDocCount;
+
+ @Override
+ public long getDocCountError() {
+ return docCountErrorUpperBound;
+ }
+
+ @Override
+ public long getSumOfOtherDocCounts() {
+ return sumOtherDocCount;
+ }
+
+ @Override
+ public List<? extends Terms.Bucket> getBuckets() {
+ return buckets;
+ }
+
+ @Override
+ public Terms.Bucket getBucketByKey(String term) {
+ for (Terms.Bucket bucket : getBuckets()) {
+ if (bucket.getKeyAsString().equals(term)) {
+ return bucket;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+ builder.field(DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName(), getDocCountError());
+ builder.field(SUM_OF_OTHER_DOC_COUNTS.getPreferredName(), getSumOfOtherDocCounts());
+ builder.startArray(CommonFields.BUCKETS.getPreferredName());
+ for (Terms.Bucket bucket : getBuckets()) {
+ bucket.toXContent(builder, params);
+ }
+ builder.endArray();
+ return builder;
+ }
+
+ static void declareParsedTermsFields(
+ final ObjectParser<? extends ParsedTerms, Void> objectParser,
+ final CheckedFunction<XContentParser, ParsedBucket, IOException> bucketParser
+ ) {
+ declareMultiBucketAggregationFields(objectParser, bucketParser::apply, bucketParser::apply);
+ objectParser.declareLong(
+ (parsedTerms, value) -> parsedTerms.docCountErrorUpperBound = value,
+ DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME
+ );
+ objectParser.declareLong((parsedTerms, value) -> parsedTerms.sumOtherDocCount = value, SUM_OF_OTHER_DOC_COUNTS);
+ }
+
+ /**
+ * Base parsed bucket
+ *
+ * @opensearch.internal
+ */
+ public abstract static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Terms.Bucket {
+
+ boolean showDocCountError = false;
+ protected long docCountError;
+
+ @Override
+ public long getDocCountError() {
+ return docCountError;
+ }
+
+ @Override
+ public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject();
+ keyToXContent(builder);
+ builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+ if (showDocCountError) {
+ builder.field(DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName(), getDocCountError());
+ }
+ getAggregations().toXContentInternal(builder, params);
+ builder.endObject();
+ return builder;
+ }
+
+ static <B extends ParsedBucket> B parseTermsBucketXContent(
+ final XContentParser parser,
+ final Supplier<B> bucketSupplier,
+ final CheckedBiConsumer<XContentParser, B, IOException> keyConsumer
+ ) throws IOException {
+
+ final B bucket = bucketSupplier.get();
+ final List<Aggregation> aggregations = new ArrayList<>();
+
+ XContentParser.Token token;
+ String currentFieldName = parser.currentName();
+ while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+ // field value could be list, e.g. multi_terms aggregation.
+ if ((token.isValue() || token == XContentParser.Token.START_ARRAY)
+ && CommonFields.KEY.getPreferredName().equals(currentFieldName)) {
+ keyConsumer.accept(parser, bucket);
+ }
+ if (token == XContentParser.Token.FIELD_NAME) {
+ currentFieldName = parser.currentName();
+ } else if (token.isValue()) {
+ if (CommonFields.KEY_AS_STRING.getPreferredName().equals(currentFieldName)) {
+ bucket.setKeyAsString(parser.text());
+ } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) {
+ bucket.setDocCount(parser.longValue());
+ } else if (DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName().equals(currentFieldName)) {
+ bucket.docCountError = parser.longValue();
+ bucket.showDocCountError = true;
+ }
+ } else if (token == XContentParser.Token.START_OBJECT) {
+ XContentParserUtils.parseTypedKeysObject(
+ parser,
+ Aggregation.TYPED_KEYS_DELIMITER,
+ Aggregation.class,
+ aggregations::add
+ );
+ }
+ }
+ bucket.setAggregations(new Aggregations(aggregations));
+ return bucket;
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: ParsedUnsignedLongTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ParsedUnsignedLongTerms |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ | ParsedUnsignedLongTerms$ParsedBucket |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ 0%
+
+
+ (0/14)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/22)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+
+ import java.io.IOException;
+ import java.math.BigInteger;
+
+ /**
+ * A long term agg result parsed between nodes
+ *
+ * @opensearch.internal
+ */
+ public class ParsedUnsignedLongTerms extends ParsedTerms {
+
+ @Override
+ public String getType() {
+ return UnsignedLongTerms.NAME;
+ }
+
+ private static final ObjectParser<ParsedUnsignedLongTerms, Void> PARSER = new ObjectParser<>(
+ ParsedUnsignedLongTerms.class.getSimpleName(),
+ true,
+ ParsedUnsignedLongTerms::new
+ );
+ static {
+ declareParsedTermsFields(PARSER, ParsedBucket::fromXContent);
+ }
+
+ public static ParsedUnsignedLongTerms fromXContent(XContentParser parser, String name) throws IOException {
+ ParsedUnsignedLongTerms aggregation = PARSER.parse(parser, null);
+ aggregation.setName(name);
+ return aggregation;
+ }
+
+ /**
+ * Parsed bucket for long term values
+ *
+ * @opensearch.internal
+ */
+ public static class ParsedBucket extends ParsedTerms.ParsedBucket {
+
+ private BigInteger key;
+
+ @Override
+ public Object getKey() {
+ return key;
+ }
+
+ @Override
+ public String getKeyAsString() {
+ String keyAsString = super.getKeyAsString();
+ if (keyAsString != null) {
+ return keyAsString;
+ }
+ if (key != null) {
+ return key.toString();
+ }
+ return null;
+ }
+
+ public Number getKeyAsNumber() {
+ return key;
+ }
+
+ @Override
+ protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ builder.field(CommonFields.KEY.getPreferredName(), key);
+ if (super.getKeyAsString() != null) {
+ builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString());
+ }
+ return builder;
+ }
+
+ static ParsedBucket fromXContent(XContentParser parser) throws IOException {
+ return parseTermsBucketXContent(parser, ParsedBucket::new, (p, bucket) -> bucket.key = p.bigIntegerValue());
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: RareTermsAggregationBuilder (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | RareTermsAggregationBuilder |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 8.7%
+
+
+ (2/23)
+
+ |
+
+
+ 17.2%
+
+
+ (10/58)
+
+ |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.Version;
+ import org.opensearch.core.ParseField;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.index.query.QueryShardContext;
+ import org.opensearch.search.aggregations.AggregationBuilder;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.AggregatorFactory;
+ import org.opensearch.search.aggregations.support.CoreValuesSourceType;
+ import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder;
+ import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+ import org.opensearch.search.aggregations.support.ValuesSourceConfig;
+ import org.opensearch.search.aggregations.support.ValuesSourceRegistry;
+ import org.opensearch.search.aggregations.support.ValuesSourceType;
+
+ import java.io.IOException;
+ import java.util.Map;
+ import java.util.Objects;
+
+ /**
+ * Aggregation Builder for rare_terms agg
+ *
+ * @opensearch.internal
+ */
+ public class RareTermsAggregationBuilder extends ValuesSourceAggregationBuilder<RareTermsAggregationBuilder> {
+ public static final String NAME = "rare_terms";
+ public static final ValuesSourceRegistry.RegistryKey<RareTermsAggregatorSupplier> REGISTRY_KEY = new ValuesSourceRegistry.RegistryKey<>(
+ NAME,
+ RareTermsAggregatorSupplier.class
+ );
+
+ private static final ParseField MAX_DOC_COUNT_FIELD_NAME = new ParseField("max_doc_count");
+ private static final ParseField PRECISION = new ParseField("precision");
+
+ private static final int MAX_MAX_DOC_COUNT = 100;
+ public static final ObjectParser<RareTermsAggregationBuilder, String> PARSER = ObjectParser.fromBuilder(
+ NAME,
+ RareTermsAggregationBuilder::new
+ );
+ static {
+ ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false);
+ PARSER.declareLong(RareTermsAggregationBuilder::maxDocCount, MAX_DOC_COUNT_FIELD_NAME);
+
+ PARSER.declareField(
+ (b, v) -> b.includeExclude(IncludeExclude.merge(v, b.includeExclude())),
+ IncludeExclude::parseInclude,
+ IncludeExclude.INCLUDE_FIELD,
+ ObjectParser.ValueType.OBJECT_ARRAY_OR_STRING
+ );
+
+ PARSER.declareField(
+ (b, v) -> b.includeExclude(IncludeExclude.merge(b.includeExclude(), v)),
+ IncludeExclude::parseExclude,
+ IncludeExclude.EXCLUDE_FIELD,
+ ObjectParser.ValueType.STRING_ARRAY
+ );
+
+ PARSER.declareDouble(RareTermsAggregationBuilder::setPrecision, PRECISION);
+ }
+
+ public static void registerAggregators(ValuesSourceRegistry.Builder builder) {
+ RareTermsAggregatorFactory.registerAggregators(builder);
+ }
+
+ private IncludeExclude includeExclude = null;
+ private int maxDocCount = 1;
+ private double precision = 0.001;
+
+ public RareTermsAggregationBuilder(String name) {
+ super(name);
+ }
+
+ private RareTermsAggregationBuilder(
+ RareTermsAggregationBuilder clone,
+ AggregatorFactories.Builder factoriesBuilder,
+ Map<String, Object> metadata
+ ) {
+ super(clone, factoriesBuilder, metadata);
+ this.includeExclude = clone.includeExclude;
+ }
+
+ @Override
+ protected ValuesSourceType defaultValueSourceType() {
+ return CoreValuesSourceType.BYTES;
+ }
+
+ @Override
+ protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map<String, Object> metadata) {
+ return new RareTermsAggregationBuilder(this, factoriesBuilder, metadata);
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public RareTermsAggregationBuilder(StreamInput in) throws IOException {
+ super(in);
+ includeExclude = in.readOptionalWriteable(IncludeExclude::new);
+ maxDocCount = in.readVInt();
+ }
+
+ @Override
+ protected boolean serializeTargetValueType(Version version) {
+ return true;
+ }
+
+ @Override
+ protected void innerWriteTo(StreamOutput out) throws IOException {
+ out.writeOptionalWriteable(includeExclude);
+ out.writeVInt(maxDocCount);
+ }
+
+ /**
+ * Set the maximum document count terms should have in order to appear in
+ * the response.
+ */
+ public RareTermsAggregationBuilder maxDocCount(long maxDocCount) {
+ if (maxDocCount <= 0) {
+ throw new IllegalArgumentException(
+ "["
+ + MAX_DOC_COUNT_FIELD_NAME.getPreferredName()
+ + "] must be greater than 0. Found ["
+ + maxDocCount
+ + "] in ["
+ + name
+ + "]"
+ );
+ }
+ // TODO review: what size cap should we put on this?
+ if (maxDocCount > MAX_MAX_DOC_COUNT) {
+ throw new IllegalArgumentException(
+ "[" + MAX_DOC_COUNT_FIELD_NAME.getPreferredName() + "] must be smaller" + "than " + MAX_MAX_DOC_COUNT + "in [" + name + "]"
+ );
+ }
+ this.maxDocCount = (int) maxDocCount;
+ return this;
+ }
+
+ /**
+ * Set terms to include and exclude from the aggregation results
+ */
+ public RareTermsAggregationBuilder includeExclude(IncludeExclude includeExclude) {
+ this.includeExclude = includeExclude;
+ return this;
+ }
+
+ /**
+ * Get terms to include and exclude from the aggregation results
+ */
+ public IncludeExclude includeExclude() {
+ return includeExclude;
+ }
+
+ /**
+ * Get the current false positive rate for individual cuckoo filters.
+ */
+ public double getPrecision() {
+ return precision;
+ }
+
+ /**
+ * Set's the false-positive rate for individual cuckoo filters. Does not dictate the overall fpp rate
+ * since we use a "scaling" cuckoo filter which adds more filters as required, and the overall
+ * error rate grows differently than individual filters
+ * <p>
+ * This value does, however, affect the overall space usage of the filter. Coarser precisions provide
+ * more compact filters. The default is 0.01
+ */
+ public void setPrecision(double precision) {
+ if (precision < 0.00001) {
+ throw new IllegalArgumentException("[precision] must be greater than 0.00001");
+ }
+ this.precision = precision;
+ }
+
+ @Override
+ public BucketCardinality bucketCardinality() {
+ return BucketCardinality.MANY;
+ }
+
+ @Override
+ protected ValuesSourceAggregatorFactory innerBuild(
+ QueryShardContext queryShardContext,
+ ValuesSourceConfig config,
+ AggregatorFactory parent,
+ AggregatorFactories.Builder subFactoriesBuilder
+ ) throws IOException {
+ return new RareTermsAggregatorFactory(
+ name,
+ config,
+ includeExclude,
+ queryShardContext,
+ parent,
+ subFactoriesBuilder,
+ metadata,
+ maxDocCount,
+ precision
+ );
+ }
+
+ @Override
+ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+ if (includeExclude != null) {
+ includeExclude.toXContent(builder, params);
+ }
+ builder.field(MAX_DOC_COUNT_FIELD_NAME.getPreferredName(), maxDocCount);
+ builder.field(PRECISION.getPreferredName(), precision);
+ return builder;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), includeExclude, maxDocCount, precision);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+ RareTermsAggregationBuilder other = (RareTermsAggregationBuilder) obj;
+ return Objects.equals(includeExclude, other.includeExclude)
+ && Objects.equals(maxDocCount, other.maxDocCount)
+ && Objects.equals(precision, other.precision);
+ }
+
+ @Override
+ public String getType() {
+ return NAME;
+ }
+
+ @Override
+ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
+ return REGISTRY_KEY;
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: RareTermsAggregatorFactory (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | RareTermsAggregatorFactory |
+
+
+ 42.9%
+
+
+ (3/7)
+
+ |
+
+
+ 40%
+
+
+ (8/20)
+
+ |
+
+
+ | RareTermsAggregatorFactory$1 |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 20%
+
+
+ (1/5)
+
+ |
+
+
+ | RareTermsAggregatorFactory$2 |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 11.1%
+
+
+ (1/9)
+
+ |
+
+
+ | RareTermsAggregatorFactory$3 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ | RareTermsAggregatorFactory$ExecutionMode |
+
+
+ 0%
+
+
+ (0/5)
+
+ |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ | RareTermsAggregatorFactory$ExecutionMode$1 |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ 0%
+
+
+ (0/7)
+
+ |
+
+
+ | Total |
+
+
+ 23.8%
+
+
+ (5/21)
+
+ |
+
+
+ 19.6%
+
+
+ (10/51)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.common.logging.DeprecationLogger;
+ import org.opensearch.core.ParseField;
+ import org.opensearch.index.query.QueryShardContext;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.AggregatorFactory;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.NonCollectingAggregator;
+ import org.opensearch.search.aggregations.support.CoreValuesSourceType;
+ import org.opensearch.search.aggregations.support.ValuesSource;
+ import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+ import org.opensearch.search.aggregations.support.ValuesSourceConfig;
+ import org.opensearch.search.aggregations.support.ValuesSourceRegistry;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.util.Arrays;
+ import java.util.Map;
+
+ /**
+ * Aggregation Factory for rare_terms agg
+ *
+ * @opensearch.internal
+ */
+ public class RareTermsAggregatorFactory extends ValuesSourceAggregatorFactory {
+ private final IncludeExclude includeExclude;
+ private final int maxDocCount;
+ private final double precision;
+
+ static void registerAggregators(ValuesSourceRegistry.Builder builder) {
+ builder.register(
+ RareTermsAggregationBuilder.REGISTRY_KEY,
+ Arrays.asList(CoreValuesSourceType.BYTES, CoreValuesSourceType.IP),
+ RareTermsAggregatorFactory.bytesSupplier(),
+ true
+ );
+
+ builder.register(
+ RareTermsAggregationBuilder.REGISTRY_KEY,
+ Arrays.asList(CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN, CoreValuesSourceType.NUMERIC),
+ RareTermsAggregatorFactory.numericSupplier(),
+ true
+ );
+ }
+
+ /**
+ * This supplier is used for all the field types that should be aggregated as bytes/strings,
+ * including those that need global ordinals
+ */
+ private static RareTermsAggregatorSupplier bytesSupplier() {
+ return new RareTermsAggregatorSupplier() {
+ @Override
+ public Aggregator build(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ DocValueFormat format,
+ int maxDocCount,
+ double precision,
+ IncludeExclude includeExclude,
+ SearchContext context,
+ Aggregator parent,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+
+ ExecutionMode execution = ExecutionMode.MAP; // TODO global ords not implemented yet, only supports "map"
+
+ if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) {
+ throw new IllegalArgumentException(
+ "Aggregation ["
+ + name
+ + "] cannot support "
+ + "regular expression style include/exclude settings as they can only be applied to string fields. "
+ + "Use an array of values for include/exclude clauses"
+ );
+ }
+
+ return execution.create(
+ name,
+ factories,
+ valuesSource,
+ format,
+ includeExclude,
+ context,
+ parent,
+ metadata,
+ maxDocCount,
+ precision,
+ cardinality
+ );
+
+ }
+ };
+ }
+
+ /**
+ * This supplier is used for all fields that expect to be aggregated as a numeric value.
+ * This includes floating points, and formatted types that use numerics internally for storage (date, boolean, etc)
+ */
+ private static RareTermsAggregatorSupplier numericSupplier() {
+ return new RareTermsAggregatorSupplier() {
+ @Override
+ public Aggregator build(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ DocValueFormat format,
+ int maxDocCount,
+ double precision,
+ IncludeExclude includeExclude,
+ SearchContext context,
+ Aggregator parent,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+
+ if ((includeExclude != null) && (includeExclude.isRegexBased())) {
+ throw new IllegalArgumentException(
+ "Aggregation ["
+ + name
+ + "] cannot support regular expression "
+ + "style include/exclude settings as they can only be applied to string fields. Use an array of numeric "
+ + "values for include/exclude clauses used to filter numeric fields"
+ );
+ }
+
+ IncludeExclude.LongFilter longFilter = null;
+ if (((ValuesSource.Numeric) valuesSource).isFloatingPoint()) {
+ throw new IllegalArgumentException("RareTerms aggregation does not support floating point fields.");
+ }
+ if (includeExclude != null) {
+ longFilter = includeExclude.convertToLongFilter(format);
+ }
+ return new LongRareTermsAggregator(
+ name,
+ factories,
+ (ValuesSource.Numeric) valuesSource,
+ format,
+ context,
+ parent,
+ longFilter,
+ maxDocCount,
+ precision,
+ cardinality,
+ metadata
+ );
+ }
+ };
+ }
+
+ RareTermsAggregatorFactory(
+ String name,
+ ValuesSourceConfig config,
+ IncludeExclude includeExclude,
+ QueryShardContext queryShardContext,
+ AggregatorFactory parent,
+ AggregatorFactories.Builder subFactoriesBuilder,
+ Map<String, Object> metadata,
+ int maxDocCount,
+ double precision
+ ) throws IOException {
+ super(name, config, queryShardContext, parent, subFactoriesBuilder, metadata);
+ this.includeExclude = includeExclude;
+ this.maxDocCount = maxDocCount;
+ this.precision = precision;
+ }
+
+ @Override
+ protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, Map<String, Object> metadata) throws IOException {
+ final InternalAggregation aggregation = new UnmappedRareTerms(name, metadata);
+ return new NonCollectingAggregator(name, searchContext, parent, factories, metadata) {
+ @Override
+ public InternalAggregation buildEmptyAggregation() {
+ return aggregation;
+ }
+ };
+ }
+
+ @Override
+ protected Aggregator doCreateInternal(
+ SearchContext searchContext,
+ Aggregator parent,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+ return queryShardContext.getValuesSourceRegistry()
+ .getAggregator(RareTermsAggregationBuilder.REGISTRY_KEY, config)
+ .build(
+ name,
+ factories,
+ config.getValuesSource(),
+ config.format(),
+ maxDocCount,
+ precision,
+ includeExclude,
+ searchContext,
+ parent,
+ cardinality,
+ metadata
+ );
+ }
+
+ @Override
+ protected boolean supportsConcurrentSegmentSearch() {
+ return true;
+ }
+
+ /**
+ * Execution mode for rare terms agg
+ *
+ * @opensearch.internal
+ */
+ public enum ExecutionMode {
+
+ MAP(new ParseField("map")) {
+
+ @Override
+ Aggregator create(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ DocValueFormat format,
+ IncludeExclude includeExclude,
+ SearchContext context,
+ Aggregator parent,
+ Map<String, Object> metadata,
+ long maxDocCount,
+ double precision,
+ CardinalityUpperBound cardinality
+ ) throws IOException {
+ int maxRegexLength = context.getQueryShardContext().getIndexSettings().getMaxRegexLength();
+ final IncludeExclude.StringFilter filter = includeExclude == null
+ ? null
+ : includeExclude.convertToStringFilter(format, maxRegexLength);
+ return new StringRareTermsAggregator(
+ name,
+ factories,
+ (ValuesSource.Bytes) valuesSource,
+ format,
+ filter,
+ context,
+ parent,
+ metadata,
+ maxDocCount,
+ precision,
+ cardinality
+ );
+ }
+
+ @Override
+ boolean needsGlobalOrdinals() {
+ return false;
+ }
+
+ };
+
+ public static ExecutionMode fromString(String value, final DeprecationLogger deprecationLogger) {
+ switch (value) {
+ case "map":
+ return MAP;
+ default:
+ throw new IllegalArgumentException("Unknown `execution_hint`: [" + value + "], expected any of [map]");
+ }
+ }
+
+ private final ParseField parseField;
+
+ ExecutionMode(ParseField parseField) {
+ this.parseField = parseField;
+ }
+
+ abstract Aggregator create(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ DocValueFormat format,
+ IncludeExclude includeExclude,
+ SearchContext context,
+ Aggregator parent,
+ Map<String, Object> metadata,
+ long maxDocCount,
+ double precision,
+ CardinalityUpperBound cardinality
+ ) throws IOException;
+
+ abstract boolean needsGlobalOrdinals();
+
+ @Override
+ public String toString() {
+ return parseField.getPreferredName();
+ }
+ }
+
+ }
+
+
+
+
+
+
Coverage Summary for Class: SignificanceLookup (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | SignificanceLookup |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ 0%
+
+
+ (0/30)
+
+ |
+
+
+ | SignificanceLookup$1 |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ | SignificanceLookup$2 |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ 0%
+
+
+ (0/11)
+
+ |
+
+
+ | SignificanceLookup$3 |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ | SignificanceLookup$4 |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ 0%
+
+
+ (0/11)
+
+ |
+
+
+ | SignificanceLookup$BackgroundFrequencyForBytes |
+
+
+ | SignificanceLookup$BackgroundFrequencyForLong |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/20)
+
+ |
+
+
+ 0%
+
+
+ (0/58)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.index.IndexReader;
+ import org.apache.lucene.index.PostingsEnum;
+ import org.apache.lucene.index.Term;
+ import org.apache.lucene.index.TermsEnum;
+ import org.apache.lucene.search.BooleanClause.Occur;
+ import org.apache.lucene.search.BooleanQuery;
+ import org.apache.lucene.search.IndexSearcher;
+ import org.apache.lucene.search.Query;
+ import org.apache.lucene.search.TermQuery;
+ import org.apache.lucene.util.BytesRef;
+ import org.opensearch.common.lease.Releasable;
+ import org.opensearch.common.lease.Releasables;
+ import org.opensearch.common.lucene.index.FilterableTermsEnum;
+ import org.opensearch.common.util.BigArrays;
+ import org.opensearch.common.util.BytesRefHash;
+ import org.opensearch.common.util.LongArray;
+ import org.opensearch.common.util.LongHash;
+ import org.opensearch.index.mapper.MappedFieldType;
+ import org.opensearch.index.query.QueryBuilder;
+ import org.opensearch.index.query.QueryShardContext;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+
+ import java.io.IOException;
+
+ /**
+ * Looks up values used for {@link SignificanceHeuristic}s.
+ *
+ * @opensearch.internal
+ */
+ class SignificanceLookup {
+ /**
+ * Lookup frequencies for {@link BytesRef} terms.
+ */
+ interface BackgroundFrequencyForBytes extends Releasable {
+ long freq(BytesRef term) throws IOException;
+ }
+
+ /**
+ * Lookup frequencies for {@code long} terms.
+ */
+ interface BackgroundFrequencyForLong extends Releasable {
+ long freq(long term) throws IOException;
+ }
+
+ private final QueryShardContext context;
+ private final MappedFieldType fieldType;
+ private final DocValueFormat format;
+ private final Query backgroundFilter;
+ private final int supersetNumDocs;
+ private TermsEnum termsEnum;
+
+ SignificanceLookup(QueryShardContext context, MappedFieldType fieldType, DocValueFormat format, QueryBuilder backgroundFilter)
+ throws IOException {
+ this.context = context;
+ this.fieldType = fieldType;
+ this.format = format;
+ this.backgroundFilter = backgroundFilter == null ? null : backgroundFilter.toQuery(context);
+ /*
+ * We need to use a superset size that includes deleted docs or we
+ * could end up blowing up with bad statistics that cause us to blow
+ * up later on.
+ */
+ IndexSearcher searcher = context.searcher();
+ supersetNumDocs = backgroundFilter == null ? searcher.getIndexReader().maxDoc() : searcher.count(this.backgroundFilter);
+ }
+
+ /**
+ * Get the number of docs in the superset.
+ */
+ long supersetSize() {
+ return supersetNumDocs;
+ }
+
+ /**
+ * Get the background frequency of a {@link BytesRef} term.
+ */
+ BackgroundFrequencyForBytes bytesLookup(BigArrays bigArrays, CardinalityUpperBound cardinality) {
+ if (cardinality == CardinalityUpperBound.ONE) {
+ return new BackgroundFrequencyForBytes() {
+ @Override
+ public long freq(BytesRef term) throws IOException {
+ return getBackgroundFrequency(term);
+ }
+
+ @Override
+ public void close() {}
+ };
+ }
+ return new BackgroundFrequencyForBytes() {
+ private final BytesRefHash termToPosition = new BytesRefHash(bigArrays);
+ private LongArray positionToFreq = bigArrays.newLongArray(1, false);
+
+ @Override
+ public long freq(BytesRef term) throws IOException {
+ long position = termToPosition.add(term);
+ if (position < 0) {
+ return positionToFreq.get(-1 - position);
+ }
+ long freq = getBackgroundFrequency(term);
+ positionToFreq = bigArrays.grow(positionToFreq, position + 1);
+ positionToFreq.set(position, freq);
+ return freq;
+ }
+
+ @Override
+ public void close() {
+ Releasables.close(termToPosition, positionToFreq);
+ }
+ };
+ }
+
+ /**
+ * Get the background frequency of a {@link BytesRef} term.
+ */
+ private long getBackgroundFrequency(BytesRef term) throws IOException {
+ return getBackgroundFrequency(fieldType.termQuery(format.format(term).toString(), context));
+ }
+
+ /**
+ * Get the background frequency of a {@code long} term.
+ */
+ BackgroundFrequencyForLong longLookup(BigArrays bigArrays, CardinalityUpperBound cardinality) {
+ if (cardinality == CardinalityUpperBound.ONE) {
+ return new BackgroundFrequencyForLong() {
+ @Override
+ public long freq(long term) throws IOException {
+ return getBackgroundFrequency(term);
+ }
+
+ @Override
+ public void close() {}
+ };
+ }
+ return new BackgroundFrequencyForLong() {
+ private final LongHash termToPosition = new LongHash(1, bigArrays);
+ private LongArray positionToFreq = bigArrays.newLongArray(1, false);
+
+ @Override
+ public long freq(long term) throws IOException {
+ long position = termToPosition.add(term);
+ if (position < 0) {
+ return positionToFreq.get(-1 - position);
+ }
+ long freq = getBackgroundFrequency(term);
+ positionToFreq = bigArrays.grow(positionToFreq, position + 1);
+ positionToFreq.set(position, freq);
+ return freq;
+ }
+
+ @Override
+ public void close() {
+ Releasables.close(termToPosition, positionToFreq);
+ }
+ };
+ }
+
+ /**
+ * Get the background frequency of a {@code long} term.
+ */
+ private long getBackgroundFrequency(long term) throws IOException {
+ return getBackgroundFrequency(fieldType.termQuery(format.format(term).toString(), context));
+ }
+
+ private long getBackgroundFrequency(Query query) throws IOException {
+ if (query instanceof TermQuery) {
+ // for types that use the inverted index, we prefer using a terms
+ // enum that will do a better job at reusing index inputs
+ Term term = ((TermQuery) query).getTerm();
+ TermsEnum termsEnum = getTermsEnum(term.field());
+ if (termsEnum.seekExact(term.bytes())) {
+ return termsEnum.docFreq();
+ }
+ return 0;
+ }
+ // otherwise do it the naive way
+ if (backgroundFilter != null) {
+ query = new BooleanQuery.Builder().add(query, Occur.FILTER).add(backgroundFilter, Occur.FILTER).build();
+ }
+ return context.searcher().count(query);
+ }
+
+ private TermsEnum getTermsEnum(String field) throws IOException {
+ // TODO this method helps because of asMultiBucketAggregator. Once we remove it we can move this logic into the aggregators.
+ if (termsEnum != null) {
+ return termsEnum;
+ }
+ IndexReader reader = context.getIndexReader();
+ termsEnum = new FilterableTermsEnum(reader, fieldType.name(), PostingsEnum.NONE, backgroundFilter);
+ return termsEnum;
+ }
+
+ }
+
+
+
+
+
+
Coverage Summary for Class: SignificantLongTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | SignificantLongTerms |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ | SignificantLongTerms$Bucket |
+
+
+ 0%
+
+
+ (0/9)
+
+ |
+
+
+ 0%
+
+
+ (0/23)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/17)
+
+ |
+
+
+ 0%
+
+
+ (0/33)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.InternalAggregations;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+
+ import java.io.IOException;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+
+ /**
+ * Result of the running the significant terms aggregation on a numeric field.
+ *
+ * @opensearch.internal
+ */
+ public class SignificantLongTerms extends InternalMappedSignificantTerms<SignificantLongTerms, SignificantLongTerms.Bucket> {
+ public static final String NAME = "siglterms";
+
+ /**
+ * Bucket for significant long values
+ *
+ * @opensearch.internal
+ */
+ static class Bucket extends InternalSignificantTerms.Bucket<Bucket> {
+
+ long term;
+
+ Bucket(
+ long subsetDf,
+ long subsetSize,
+ long supersetDf,
+ long supersetSize,
+ long term,
+ InternalAggregations aggregations,
+ DocValueFormat format,
+ double score
+ ) {
+ super(subsetDf, subsetSize, supersetDf, supersetSize, aggregations, format);
+ this.term = term;
+ this.score = score;
+ }
+
+ Bucket(StreamInput in, long subsetSize, long supersetSize, DocValueFormat format) throws IOException {
+ super(in, subsetSize, supersetSize, format);
+ subsetDf = in.readVLong();
+ supersetDf = in.readVLong();
+ term = in.readLong();
+ score = in.readDouble();
+ aggregations = InternalAggregations.readFrom(in);
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeVLong(subsetDf);
+ out.writeVLong(supersetDf);
+ out.writeLong(term);
+ out.writeDouble(getSignificanceScore());
+ aggregations.writeTo(out);
+ }
+
+ @Override
+ public Object getKey() {
+ return term;
+ }
+
+ @Override
+ public String getKeyAsString() {
+ return format.format(term).toString();
+ }
+
+ @Override
+ public Number getKeyAsNumber() {
+ return term;
+ }
+
+ @Override
+ protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ builder.field(CommonFields.KEY.getPreferredName(), term);
+ if (format != DocValueFormat.RAW) {
+ builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term));
+ }
+ return builder;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj) && Objects.equals(term, ((Bucket) obj).term);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), term);
+ }
+ }
+
+ public SignificantLongTerms(
+ String name,
+ Map<String, Object> metadata,
+ DocValueFormat format,
+ long subsetSize,
+ long supersetSize,
+ SignificanceHeuristic significanceHeuristic,
+ List<Bucket> buckets,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds
+ ) {
+ super(name, metadata, format, subsetSize, supersetSize, significanceHeuristic, buckets, bucketCountThresholds);
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public SignificantLongTerms(StreamInput in) throws IOException {
+ super(in, Bucket::new);
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public SignificantLongTerms create(List<SignificantLongTerms.Bucket> buckets) {
+ return new SignificantLongTerms(
+ name,
+ metadata,
+ format,
+ subsetSize,
+ supersetSize,
+ significanceHeuristic,
+ buckets,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ public Bucket createBucket(InternalAggregations aggregations, SignificantLongTerms.Bucket prototype) {
+ return new Bucket(
+ prototype.subsetDf,
+ prototype.subsetSize,
+ prototype.supersetDf,
+ prototype.supersetSize,
+ prototype.term,
+ aggregations,
+ prototype.format,
+ prototype.score
+ );
+ }
+
+ @Override
+ protected SignificantLongTerms create(long subsetSize, long supersetSize, List<Bucket> buckets) {
+ return new SignificantLongTerms(
+ getName(),
+ getMetadata(),
+ format,
+ subsetSize,
+ supersetSize,
+ significanceHeuristic,
+ buckets,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ protected Bucket[] createBucketsArray(int size) {
+ return new Bucket[size];
+ }
+
+ @Override
+ Bucket createBucket(
+ long subsetDf,
+ long subsetSize,
+ long supersetDf,
+ long supersetSize,
+ InternalAggregations aggregations,
+ SignificantLongTerms.Bucket prototype
+ ) {
+ return new Bucket(subsetDf, subsetSize, supersetDf, supersetSize, prototype.term, aggregations, format, prototype.score);
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: SignificantStringTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | SignificantStringTerms |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ | SignificantStringTerms$Bucket |
+
+
+ 0%
+
+
+ (0/9)
+
+ |
+
+
+ 0%
+
+
+ (0/23)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/17)
+
+ |
+
+
+ 0%
+
+
+ (0/33)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.util.BytesRef;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.InternalAggregations;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+
+ import java.io.IOException;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+
+ /**
+ * Result of the running the significant terms aggregation on a String field.
+ *
+ * @opensearch.internal
+ */
+ public class SignificantStringTerms extends InternalMappedSignificantTerms<SignificantStringTerms, SignificantStringTerms.Bucket> {
+ public static final String NAME = "sigsterms";
+
+ /**
+ * Bucket for significant string values
+ *
+ * @opensearch.internal
+ */
+ public static class Bucket extends InternalSignificantTerms.Bucket<Bucket> {
+
+ BytesRef termBytes;
+
+ public Bucket(
+ BytesRef term,
+ long subsetDf,
+ long subsetSize,
+ long supersetDf,
+ long supersetSize,
+ InternalAggregations aggregations,
+ DocValueFormat format,
+ double score
+ ) {
+ super(subsetDf, subsetSize, supersetDf, supersetSize, aggregations, format);
+ this.termBytes = term;
+ this.score = score;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public Bucket(StreamInput in, long subsetSize, long supersetSize, DocValueFormat format) throws IOException {
+ super(in, subsetSize, supersetSize, format);
+ termBytes = in.readBytesRef();
+ subsetDf = in.readVLong();
+ supersetDf = in.readVLong();
+ score = in.readDouble();
+ aggregations = InternalAggregations.readFrom(in);
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeBytesRef(termBytes);
+ out.writeVLong(subsetDf);
+ out.writeVLong(supersetDf);
+ out.writeDouble(getSignificanceScore());
+ aggregations.writeTo(out);
+ }
+
+ @Override
+ public Number getKeyAsNumber() {
+ // this method is needed for scripted numeric aggregations
+ return Double.parseDouble(termBytes.utf8ToString());
+ }
+
+ @Override
+ public String getKeyAsString() {
+ return format.format(termBytes).toString();
+ }
+
+ @Override
+ public String getKey() {
+ return getKeyAsString();
+ }
+
+ @Override
+ protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ return builder.field(CommonFields.KEY.getPreferredName(), getKeyAsString());
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+
+ return super.equals(obj) && Objects.equals(termBytes, ((SignificantStringTerms.Bucket) obj).termBytes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), termBytes);
+ }
+ }
+
+ public SignificantStringTerms(
+ String name,
+ Map<String, Object> metadata,
+ DocValueFormat format,
+ long subsetSize,
+ long supersetSize,
+ SignificanceHeuristic significanceHeuristic,
+ List<Bucket> buckets,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds
+ ) {
+ super(name, metadata, format, subsetSize, supersetSize, significanceHeuristic, buckets, bucketCountThresholds);
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public SignificantStringTerms(StreamInput in) throws IOException {
+ super(in, Bucket::new);
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public SignificantStringTerms create(List<SignificantStringTerms.Bucket> buckets) {
+ return new SignificantStringTerms(
+ name,
+ metadata,
+ format,
+ subsetSize,
+ supersetSize,
+ significanceHeuristic,
+ buckets,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ public Bucket createBucket(InternalAggregations aggregations, SignificantStringTerms.Bucket prototype) {
+ return new Bucket(
+ prototype.termBytes,
+ prototype.subsetDf,
+ prototype.subsetSize,
+ prototype.supersetDf,
+ prototype.supersetSize,
+ aggregations,
+ prototype.format,
+ prototype.score
+ );
+ }
+
+ @Override
+ protected SignificantStringTerms create(long subsetSize, long supersetSize, List<Bucket> buckets) {
+ return new SignificantStringTerms(
+ getName(),
+ getMetadata(),
+ format,
+ subsetSize,
+ supersetSize,
+ significanceHeuristic,
+ buckets,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ protected Bucket[] createBucketsArray(int size) {
+ return new Bucket[size];
+ }
+
+ @Override
+ Bucket createBucket(
+ long subsetDf,
+ long subsetSize,
+ long supersetDf,
+ long supersetSize,
+ InternalAggregations aggregations,
+ SignificantStringTerms.Bucket prototype
+ ) {
+ return new Bucket(prototype.termBytes, subsetDf, subsetSize, supersetDf, supersetSize, aggregations, format, prototype.score);
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: SignificantTermsAggregationBuilder (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | SignificantTermsAggregationBuilder |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 5.6%
+
+
+ (2/36)
+
+ |
+
+
+ 13.9%
+
+
+ (16/115)
+
+ |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.Version;
+ import org.opensearch.core.ParseField;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+ import org.opensearch.index.query.QueryBuilder;
+ import org.opensearch.index.query.QueryRewriteContext;
+ import org.opensearch.index.query.QueryShardContext;
+ import org.opensearch.search.aggregations.AggregationBuilder;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.AggregatorFactory;
+ import org.opensearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.JLHScore;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+ import org.opensearch.search.aggregations.support.CoreValuesSourceType;
+ import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder;
+ import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+ import org.opensearch.search.aggregations.support.ValuesSourceConfig;
+ import org.opensearch.search.aggregations.support.ValuesSourceRegistry;
+ import org.opensearch.search.aggregations.support.ValuesSourceType;
+
+ import java.io.IOException;
+ import java.util.Map;
+ import java.util.Objects;
+
+ import static org.opensearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
+
+ /**
+ * Aggregation Builder for significant terms agg
+ *
+ * @opensearch.internal
+ */
+ public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationBuilder<SignificantTermsAggregationBuilder> {
+ public static final String NAME = "significant_terms";
+ public static final ValuesSourceRegistry.RegistryKey<SignificantTermsAggregatorSupplier> REGISTRY_KEY =
+ new ValuesSourceRegistry.RegistryKey<>(NAME, SignificantTermsAggregatorSupplier.class);
+
+ static final ParseField BACKGROUND_FILTER = new ParseField("background_filter");
+ static final ParseField HEURISTIC = new ParseField("significance_heuristic");
+
+ static final TermsAggregator.BucketCountThresholds DEFAULT_BUCKET_COUNT_THRESHOLDS = new TermsAggregator.BucketCountThresholds(
+ 3,
+ 0,
+ 10,
+ -1
+ );
+ static final SignificanceHeuristic DEFAULT_SIGNIFICANCE_HEURISTIC = new JLHScore();
+
+ private static final ObjectParser<SignificantTermsAggregationBuilder, Void> PARSER = new ObjectParser<>(
+ SignificantTermsAggregationBuilder.NAME,
+ SignificanceHeuristic.class,
+ SignificantTermsAggregationBuilder::significanceHeuristic,
+ null
+ );
+ static {
+ ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false);
+
+ PARSER.declareInt(SignificantTermsAggregationBuilder::shardSize, TermsAggregationBuilder.SHARD_SIZE_FIELD_NAME);
+
+ PARSER.declareLong(SignificantTermsAggregationBuilder::minDocCount, TermsAggregationBuilder.MIN_DOC_COUNT_FIELD_NAME);
+
+ PARSER.declareLong(SignificantTermsAggregationBuilder::shardMinDocCount, TermsAggregationBuilder.SHARD_MIN_DOC_COUNT_FIELD_NAME);
+
+ PARSER.declareInt(SignificantTermsAggregationBuilder::size, TermsAggregationBuilder.REQUIRED_SIZE_FIELD_NAME);
+
+ PARSER.declareString(SignificantTermsAggregationBuilder::executionHint, TermsAggregationBuilder.EXECUTION_HINT_FIELD_NAME);
+
+ PARSER.declareObject(
+ SignificantTermsAggregationBuilder::backgroundFilter,
+ (p, context) -> parseInnerQueryBuilder(p),
+ SignificantTermsAggregationBuilder.BACKGROUND_FILTER
+ );
+
+ PARSER.declareField(
+ (b, v) -> b.includeExclude(IncludeExclude.merge(v, b.includeExclude())),
+ IncludeExclude::parseInclude,
+ IncludeExclude.INCLUDE_FIELD,
+ ObjectParser.ValueType.OBJECT_ARRAY_OR_STRING
+ );
+
+ PARSER.declareField(
+ (b, v) -> b.includeExclude(IncludeExclude.merge(b.includeExclude(), v)),
+ IncludeExclude::parseExclude,
+ IncludeExclude.EXCLUDE_FIELD,
+ ObjectParser.ValueType.STRING_ARRAY
+ );
+ }
+
+ public static SignificantTermsAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
+ return PARSER.parse(parser, new SignificantTermsAggregationBuilder(aggregationName), null);
+ }
+
+ public static void registerAggregators(ValuesSourceRegistry.Builder builder) {
+ SignificantTermsAggregatorFactory.registerAggregators(builder);
+ }
+
+ private IncludeExclude includeExclude = null;
+ private String executionHint = null;
+ private QueryBuilder filterBuilder = null;
+ private TermsAggregator.BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(DEFAULT_BUCKET_COUNT_THRESHOLDS);
+ private SignificanceHeuristic significanceHeuristic = DEFAULT_SIGNIFICANCE_HEURISTIC;
+
+ public SignificantTermsAggregationBuilder(String name) {
+ super(name);
+ }
+
+ /**
+ * Read from a Stream.
+ */
+ public SignificantTermsAggregationBuilder(StreamInput in) throws IOException {
+ super(in);
+ bucketCountThresholds = new BucketCountThresholds(in);
+ executionHint = in.readOptionalString();
+ filterBuilder = in.readOptionalNamedWriteable(QueryBuilder.class);
+ includeExclude = in.readOptionalWriteable(IncludeExclude::new);
+ significanceHeuristic = in.readNamedWriteable(SignificanceHeuristic.class);
+ }
+
+ protected SignificantTermsAggregationBuilder(
+ SignificantTermsAggregationBuilder clone,
+ AggregatorFactories.Builder factoriesBuilder,
+ Map<String, Object> metadata
+ ) {
+ super(clone, factoriesBuilder, metadata);
+ this.bucketCountThresholds = new BucketCountThresholds(clone.bucketCountThresholds);
+ this.executionHint = clone.executionHint;
+ this.filterBuilder = clone.filterBuilder;
+ this.includeExclude = clone.includeExclude;
+ this.significanceHeuristic = clone.significanceHeuristic;
+ }
+
+ @Override
+ protected ValuesSourceType defaultValueSourceType() {
+ return CoreValuesSourceType.BYTES;
+ }
+
+ @Override
+ protected SignificantTermsAggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map<String, Object> metadata) {
+ return new SignificantTermsAggregationBuilder(this, factoriesBuilder, metadata);
+ }
+
+ protected AggregationBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+ if (filterBuilder != null) {
+ QueryBuilder rewrittenFilter = filterBuilder.rewrite(queryShardContext);
+ if (rewrittenFilter != filterBuilder) {
+ SignificantTermsAggregationBuilder rewritten = shallowCopy(factoriesBuilder, metadata);
+ rewritten.backgroundFilter(rewrittenFilter);
+ return rewritten;
+ }
+ }
+ return super.doRewrite(queryShardContext);
+ }
+
+ @Override
+ protected void innerWriteTo(StreamOutput out) throws IOException {
+ bucketCountThresholds.writeTo(out);
+ out.writeOptionalString(executionHint);
+ out.writeOptionalNamedWriteable(filterBuilder);
+ out.writeOptionalWriteable(includeExclude);
+ out.writeNamedWriteable(significanceHeuristic);
+ }
+
+ @Override
+ protected boolean serializeTargetValueType(Version version) {
+ return true;
+ }
+
+ protected TermsAggregator.BucketCountThresholds getBucketCountThresholds() {
+ return new TermsAggregator.BucketCountThresholds(bucketCountThresholds);
+ }
+
+ public TermsAggregator.BucketCountThresholds bucketCountThresholds() {
+ return bucketCountThresholds;
+ }
+
+ public SignificantTermsAggregationBuilder bucketCountThresholds(TermsAggregator.BucketCountThresholds bucketCountThresholds) {
+ if (bucketCountThresholds == null) {
+ throw new IllegalArgumentException("[bucketCountThresholds] must not be null: [" + name + "]");
+ }
+ this.bucketCountThresholds = bucketCountThresholds;
+ return this;
+ }
+
+ /**
+ * Sets the size - indicating how many term buckets should be returned
+ * (defaults to 10)
+ */
+ public SignificantTermsAggregationBuilder size(int size) {
+ if (size <= 0) {
+ throw new IllegalArgumentException("[size] must be greater than 0. Found [" + size + "] in [" + name + "]");
+ }
+ bucketCountThresholds.setRequiredSize(size);
+ return this;
+ }
+
+ /**
+ * Sets the shard_size - indicating the number of term buckets each shard
+ * will return to the coordinating node (the node that coordinates the
+ * search execution). The higher the shard size is, the more accurate the
+ * results are.
+ */
+ public SignificantTermsAggregationBuilder shardSize(int shardSize) {
+ if (shardSize <= 0) {
+ throw new IllegalArgumentException("[shardSize] must be greater than 0. Found [" + shardSize + "] in [" + name + "]");
+ }
+ bucketCountThresholds.setShardSize(shardSize);
+ return this;
+ }
+
+ /**
+ * Set the minimum document count terms should have in order to appear in
+ * the response.
+ */
+ public SignificantTermsAggregationBuilder minDocCount(long minDocCount) {
+ if (minDocCount < 0) {
+ throw new IllegalArgumentException(
+ "[minDocCount] must be greater than or equal to 0. Found [" + minDocCount + "] in [" + name + "]"
+ );
+ }
+ bucketCountThresholds.setMinDocCount(minDocCount);
+ return this;
+ }
+
+ /**
+ * Set the minimum document count terms should have on the shard in order to
+ * appear in the response.
+ */
+ public SignificantTermsAggregationBuilder shardMinDocCount(long shardMinDocCount) {
+ if (shardMinDocCount < 0) {
+ throw new IllegalArgumentException(
+ "[shardMinDocCount] must be greater than or equal to 0. Found [" + shardMinDocCount + "] in [" + name + "]"
+ );
+ }
+ bucketCountThresholds.setShardMinDocCount(shardMinDocCount);
+ return this;
+ }
+
+ /**
+ * Expert: sets an execution hint to the aggregation.
+ */
+ public SignificantTermsAggregationBuilder executionHint(String executionHint) {
+ this.executionHint = executionHint;
+ return this;
+ }
+
+ /**
+ * Expert: gets an execution hint to the aggregation.
+ */
+ public String executionHint() {
+ return executionHint;
+ }
+
+ public SignificantTermsAggregationBuilder backgroundFilter(QueryBuilder backgroundFilter) {
+ if (backgroundFilter == null) {
+ throw new IllegalArgumentException("[backgroundFilter] must not be null: [" + name + "]");
+ }
+ this.filterBuilder = backgroundFilter;
+ return this;
+ }
+
+ public QueryBuilder backgroundFilter() {
+ return filterBuilder;
+ }
+
+ /**
+ * Set terms to include and exclude from the aggregation results
+ */
+ public SignificantTermsAggregationBuilder includeExclude(IncludeExclude includeExclude) {
+ this.includeExclude = includeExclude;
+ return this;
+ }
+
+ /**
+ * Get terms to include and exclude from the aggregation results
+ */
+ public IncludeExclude includeExclude() {
+ return includeExclude;
+ }
+
+ public SignificantTermsAggregationBuilder significanceHeuristic(SignificanceHeuristic significanceHeuristic) {
+ if (significanceHeuristic == null) {
+ throw new IllegalArgumentException("[significanceHeuristic] must not be null: [" + name + "]");
+ }
+ this.significanceHeuristic = significanceHeuristic;
+ return this;
+ }
+
+ public SignificanceHeuristic significanceHeuristic() {
+ return significanceHeuristic;
+ }
+
+ @Override
+ public BucketCardinality bucketCardinality() {
+ return BucketCardinality.MANY;
+ }
+
+ @Override
+ protected ValuesSourceAggregatorFactory innerBuild(
+ QueryShardContext queryShardContext,
+ ValuesSourceConfig config,
+ AggregatorFactory parent,
+ AggregatorFactories.Builder subFactoriesBuilder
+ ) throws IOException {
+ SignificanceHeuristic executionHeuristic = this.significanceHeuristic.rewrite(queryShardContext);
+ return new SignificantTermsAggregatorFactory(
+ name,
+ config,
+ includeExclude,
+ executionHint,
+ filterBuilder,
+ bucketCountThresholds,
+ executionHeuristic,
+ queryShardContext,
+ parent,
+ subFactoriesBuilder,
+ metadata
+ );
+ }
+
+ @Override
+ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+ bucketCountThresholds.toXContent(builder, params);
+ if (executionHint != null) {
+ builder.field(TermsAggregationBuilder.EXECUTION_HINT_FIELD_NAME.getPreferredName(), executionHint);
+ }
+ if (filterBuilder != null) {
+ builder.field(BACKGROUND_FILTER.getPreferredName(), filterBuilder);
+ }
+ if (includeExclude != null) {
+ includeExclude.toXContent(builder, params);
+ }
+ significanceHeuristic.toXContent(builder, params);
+ return builder;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), bucketCountThresholds, executionHint, filterBuilder, includeExclude, significanceHeuristic);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+ SignificantTermsAggregationBuilder other = (SignificantTermsAggregationBuilder) obj;
+ return Objects.equals(bucketCountThresholds, other.bucketCountThresholds)
+ && Objects.equals(executionHint, other.executionHint)
+ && Objects.equals(filterBuilder, other.filterBuilder)
+ && Objects.equals(includeExclude, other.includeExclude)
+ && Objects.equals(significanceHeuristic, other.significanceHeuristic);
+ }
+
+ @Override
+ public String getType() {
+ return NAME;
+ }
+
+ @Override
+ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
+ return REGISTRY_KEY;
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: SignificantTermsAggregatorFactory (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | SignificantTermsAggregatorFactory |
+
+
+ 50%
+
+
+ (4/8)
+
+ |
+
+
+ 27.3%
+
+
+ (9/33)
+
+ |
+
+
+ | SignificantTermsAggregatorFactory$1 |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 9.1%
+
+
+ (1/11)
+
+ |
+
+
+ | SignificantTermsAggregatorFactory$2 |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 9.1%
+
+
+ (1/11)
+
+ |
+
+
+ | SignificantTermsAggregatorFactory$3 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ | SignificantTermsAggregatorFactory$ExecutionMode |
+
+
+ 0%
+
+
+ (0/5)
+
+ |
+
+
+ 0%
+
+
+ (0/14)
+
+ |
+
+
+ | SignificantTermsAggregatorFactory$ExecutionMode$1 |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ 0%
+
+
+ (0/7)
+
+ |
+
+
+ | SignificantTermsAggregatorFactory$ExecutionMode$2 |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ | Total |
+
+
+ 23.1%
+
+
+ (6/26)
+
+ |
+
+
+ 12.5%
+
+
+ (11/88)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.common.logging.DeprecationLogger;
+ import org.opensearch.core.ParseField;
+ import org.opensearch.index.query.QueryBuilder;
+ import org.opensearch.index.query.QueryShardContext;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.AggregatorFactory;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.NonCollectingAggregator;
+ import org.opensearch.search.aggregations.bucket.BucketUtils;
+ import org.opensearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+ import org.opensearch.search.aggregations.support.CoreValuesSourceType;
+ import org.opensearch.search.aggregations.support.ValuesSource;
+ import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+ import org.opensearch.search.aggregations.support.ValuesSourceConfig;
+ import org.opensearch.search.aggregations.support.ValuesSourceRegistry;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.util.Arrays;
+ import java.util.Map;
+
+ /**
+ * Aggregation Factory for significant_terms agg
+ *
+ * @opensearch.internal
+ */
+ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFactory {
+ private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(SignificantTermsAggregatorFactory.class);
+
+ static void registerAggregators(ValuesSourceRegistry.Builder builder) {
+ builder.register(
+ SignificantTermsAggregationBuilder.REGISTRY_KEY,
+ Arrays.asList(CoreValuesSourceType.BYTES, CoreValuesSourceType.IP),
+ SignificantTermsAggregatorFactory.bytesSupplier(),
+ true
+ );
+
+ builder.register(
+ SignificantTermsAggregationBuilder.REGISTRY_KEY,
+ Arrays.asList(CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN, CoreValuesSourceType.NUMERIC),
+ SignificantTermsAggregatorFactory.numericSupplier(),
+ true
+ );
+ }
+
+ /**
+ * This supplier is used for all the field types that should be aggregated as bytes/strings,
+ * including those that need global ordinals
+ */
+ private static SignificantTermsAggregatorSupplier bytesSupplier() {
+ return new SignificantTermsAggregatorSupplier() {
+ @Override
+ public Aggregator build(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ DocValueFormat format,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ IncludeExclude includeExclude,
+ String executionHint,
+ SearchContext context,
+ Aggregator parent,
+ SignificanceHeuristic significanceHeuristic,
+ SignificanceLookup lookup,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+
+ ExecutionMode execution = null;
+ if (executionHint != null) {
+ execution = ExecutionMode.fromString(executionHint, deprecationLogger);
+ }
+ if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals == false) {
+ execution = ExecutionMode.MAP;
+ }
+ if (execution == null) {
+ execution = ExecutionMode.GLOBAL_ORDINALS;
+ }
+
+ if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) {
+ throw new IllegalArgumentException(
+ "Aggregation ["
+ + name
+ + "] cannot support regular expression style "
+ + "include/exclude settings as they can only be applied to string fields. Use an array of values for "
+ + "include/exclude clauses"
+ );
+ }
+
+ return execution.create(
+ name,
+ factories,
+ valuesSource,
+ format,
+ bucketCountThresholds,
+ includeExclude,
+ context,
+ parent,
+ significanceHeuristic,
+ lookup,
+ cardinality,
+ metadata
+ );
+ }
+ };
+ }
+
+ /**
+ * This supplier is used for all fields that expect to be aggregated as a numeric value.
+ * This includes floating points, and formatted types that use numerics internally for storage (date, boolean, etc)
+ */
+ private static SignificantTermsAggregatorSupplier numericSupplier() {
+ return new SignificantTermsAggregatorSupplier() {
+ @Override
+ public Aggregator build(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ DocValueFormat format,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ IncludeExclude includeExclude,
+ String executionHint,
+ SearchContext context,
+ Aggregator parent,
+ SignificanceHeuristic significanceHeuristic,
+ SignificanceLookup lookup,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+
+ if ((includeExclude != null) && (includeExclude.isRegexBased())) {
+ throw new IllegalArgumentException(
+ "Aggregation ["
+ + name
+ + "] cannot support regular expression style include/exclude "
+ + "settings as they can only be applied to string fields. Use an array of numeric "
+ + "values for include/exclude clauses used to filter numeric fields"
+ );
+ }
+
+ ValuesSource.Numeric numericValuesSource = (ValuesSource.Numeric) valuesSource;
+ if (numericValuesSource.isFloatingPoint()) {
+ throw new UnsupportedOperationException("No support for examining floating point numerics");
+ }
+
+ IncludeExclude.LongFilter longFilter = null;
+ if (includeExclude != null) {
+ longFilter = includeExclude.convertToLongFilter(format);
+ }
+
+ return new NumericTermsAggregator(
+ name,
+ factories,
+ agg -> agg.new SignificantLongTermsResults(lookup, significanceHeuristic, cardinality),
+ numericValuesSource,
+ format,
+ null,
+ bucketCountThresholds,
+ context,
+ parent,
+ SubAggCollectionMode.BREADTH_FIRST,
+ longFilter,
+ cardinality,
+ metadata
+ );
+ }
+ };
+ }
+
+ private final IncludeExclude includeExclude;
+ private final String executionHint;
+ private final QueryBuilder backgroundFilter;
+ private final TermsAggregator.BucketCountThresholds bucketCountThresholds;
+ private final SignificanceHeuristic significanceHeuristic;
+
+ SignificantTermsAggregatorFactory(
+ String name,
+ ValuesSourceConfig config,
+ IncludeExclude includeExclude,
+ String executionHint,
+ QueryBuilder backgroundFilter,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ SignificanceHeuristic significanceHeuristic,
+ QueryShardContext queryShardContext,
+ AggregatorFactory parent,
+ AggregatorFactories.Builder subFactoriesBuilder,
+ Map<String, Object> metadata
+ ) throws IOException {
+ super(name, config, queryShardContext, parent, subFactoriesBuilder, metadata);
+
+ if (config.hasValues()) {
+ if (config.fieldContext().fieldType().isSearchable() == false) {
+ throw new IllegalArgumentException(
+ "SignificantText aggregation requires fields to be searchable, but ["
+ + config.fieldContext().fieldType().name()
+ + "] is not"
+ );
+ }
+ }
+
+ this.includeExclude = includeExclude;
+ this.executionHint = executionHint;
+ this.backgroundFilter = backgroundFilter;
+ this.bucketCountThresholds = bucketCountThresholds;
+ this.significanceHeuristic = significanceHeuristic;
+ }
+
+ @Override
+ protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, Map<String, Object> metadata) throws IOException {
+ final InternalAggregation aggregation = new UnmappedSignificantTerms(name, bucketCountThresholds, metadata);
+ return new NonCollectingAggregator(name, searchContext, parent, factories, metadata) {
+ @Override
+ public InternalAggregation buildEmptyAggregation() {
+ return aggregation;
+ }
+ };
+ }
+
+ @Override
+ protected Aggregator doCreateInternal(
+ SearchContext searchContext,
+ Aggregator parent,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+ SignificantTermsAggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry()
+ .getAggregator(SignificantTermsAggregationBuilder.REGISTRY_KEY, config);
+
+ BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(this.bucketCountThresholds);
+ if (bucketCountThresholds.getShardSize() == SignificantTermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) {
+ // The user has not made a shardSize selection .
+ // Use default heuristic to avoid any wrong-ranking caused by
+ // distributed counting
+ // but request double the usual amount.
+ // We typically need more than the number of "top" terms requested
+ // by other aggregations
+ // as the significance algorithm is in less of a position to
+ // down-select at shard-level -
+ // some of the things we want to find have only one occurrence on
+ // each shard and as
+ // such are impossible to differentiate from non-significant terms
+ // at that early stage.
+ bucketCountThresholds.setShardSize(2 * BucketUtils.suggestShardSideQueueSize(bucketCountThresholds.getRequiredSize()));
+ }
+
+ SignificanceLookup lookup = new SignificanceLookup(
+ queryShardContext,
+ config.fieldContext().fieldType(),
+ config.format(),
+ backgroundFilter
+ );
+
+ return aggregatorSupplier.build(
+ name,
+ factories,
+ config.getValuesSource(),
+ config.format(),
+ bucketCountThresholds,
+ includeExclude,
+ executionHint,
+ searchContext,
+ parent,
+ significanceHeuristic,
+ lookup,
+ cardinality,
+ metadata
+ );
+ }
+
+ @Override
+ protected boolean supportsConcurrentSegmentSearch() {
+ return true;
+ }
+
+ /**
+ * The execution mode for the significant terms agg
+ *
+ * @opensearch.internal
+ */
+ public enum ExecutionMode {
+
+ MAP(new ParseField("map")) {
+
+ @Override
+ Aggregator create(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ DocValueFormat format,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ IncludeExclude includeExclude,
+ SearchContext aggregationContext,
+ Aggregator parent,
+ SignificanceHeuristic significanceHeuristic,
+ SignificanceLookup lookup,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+ int maxRegexLength = aggregationContext.getQueryShardContext().getIndexSettings().getMaxRegexLength();
+ final IncludeExclude.StringFilter filter = includeExclude == null
+ ? null
+ : includeExclude.convertToStringFilter(format, maxRegexLength);
+ return new MapStringTermsAggregator(
+ name,
+ factories,
+ new MapStringTermsAggregator.ValuesSourceCollectorSource(valuesSource),
+ a -> a.new SignificantTermsResults(lookup, significanceHeuristic, cardinality),
+ null,
+ format,
+ bucketCountThresholds,
+ filter,
+ aggregationContext,
+ parent,
+ SubAggCollectionMode.BREADTH_FIRST,
+ false,
+ cardinality,
+ metadata
+ );
+
+ }
+
+ },
+ GLOBAL_ORDINALS(new ParseField("global_ordinals")) {
+
+ @Override
+ Aggregator create(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ DocValueFormat format,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ IncludeExclude includeExclude,
+ SearchContext aggregationContext,
+ Aggregator parent,
+ SignificanceHeuristic significanceHeuristic,
+ SignificanceLookup lookup,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+ int maxRegexLength = aggregationContext.getQueryShardContext().getIndexSettings().getMaxRegexLength();
+ final IncludeExclude.OrdinalsFilter filter = includeExclude == null
+ ? null
+ : includeExclude.convertToOrdinalsFilter(format, maxRegexLength);
+ boolean remapGlobalOrd = true;
+ if (cardinality == CardinalityUpperBound.ONE && factories == AggregatorFactories.EMPTY && includeExclude == null) {
+ /*
+ * We don't need to remap global ords iff this aggregator:
+ * - collects from a single bucket AND
+ * - has no include/exclude rules AND
+ * - has no sub-aggregator
+ */
+ remapGlobalOrd = false;
+ }
+
+ return new GlobalOrdinalsStringTermsAggregator(
+ name,
+ factories,
+ a -> a.new SignificantTermsResults(lookup, significanceHeuristic, cardinality),
+ (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource,
+ null,
+ format,
+ bucketCountThresholds,
+ filter,
+ aggregationContext,
+ parent,
+ remapGlobalOrd,
+ SubAggCollectionMode.BREADTH_FIRST,
+ false,
+ cardinality,
+ metadata
+ );
+ }
+ };
+
+ public static ExecutionMode fromString(String value, final DeprecationLogger deprecationLogger) {
+ if ("global_ordinals".equals(value)) {
+ return GLOBAL_ORDINALS;
+ } else if ("global_ordinals_hash".equals(value)) {
+ deprecationLogger.deprecate(
+ "global_ordinals_hash",
+ "global_ordinals_hash is deprecated. Please use [global_ordinals] instead."
+ );
+ return GLOBAL_ORDINALS;
+ } else if ("map".equals(value)) {
+ return MAP;
+ }
+ throw new IllegalArgumentException("Unknown `execution_hint`: [" + value + "], expected any of [map, global_ordinals]");
+ }
+
+ private final ParseField parseField;
+
+ ExecutionMode(ParseField parseField) {
+ this.parseField = parseField;
+ }
+
+ abstract Aggregator create(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ DocValueFormat format,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ IncludeExclude includeExclude,
+ SearchContext aggregationContext,
+ Aggregator parent,
+ SignificanceHeuristic significanceHeuristic,
+ SignificanceLookup lookup,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException;
+
+ @Override
+ public String toString() {
+ return parseField.getPreferredName();
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: SignificantTextAggregationBuilder (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | SignificantTextAggregationBuilder |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ 0%
+
+
+ (0/34)
+
+ |
+
+
+ 0%
+
+
+ (0/131)
+
+ |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.ParseField;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+ import org.opensearch.index.query.AbstractQueryBuilder;
+ import org.opensearch.index.query.QueryBuilder;
+ import org.opensearch.index.query.QueryShardContext;
+ import org.opensearch.search.aggregations.AbstractAggregationBuilder;
+ import org.opensearch.search.aggregations.AggregationBuilder;
+ import org.opensearch.search.aggregations.AggregationInitializationException;
+ import org.opensearch.search.aggregations.AggregatorFactories.Builder;
+ import org.opensearch.search.aggregations.AggregatorFactory;
+ import org.opensearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+
+ import java.io.IOException;
+ import java.util.Arrays;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+
+ /**
+ * Aggregation Builder for significant text agg
+ *
+ * @opensearch.internal
+ */
+ public class SignificantTextAggregationBuilder extends AbstractAggregationBuilder<SignificantTextAggregationBuilder> {
+ public static final String NAME = "significant_text";
+
+ static final ParseField FIELD_NAME = new ParseField("field");
+ static final ParseField SOURCE_FIELDS_NAME = new ParseField("source_fields");
+ static final ParseField FILTER_DUPLICATE_TEXT_FIELD_NAME = new ParseField("filter_duplicate_text");
+
+ static final TermsAggregator.BucketCountThresholds DEFAULT_BUCKET_COUNT_THRESHOLDS =
+ SignificantTermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS;
+ static final SignificanceHeuristic DEFAULT_SIGNIFICANCE_HEURISTIC = SignificantTermsAggregationBuilder.DEFAULT_SIGNIFICANCE_HEURISTIC;
+
+ private String fieldName = null;
+ private String[] sourceFieldNames = null;
+ private boolean filterDuplicateText = false;
+ private IncludeExclude includeExclude = null;
+ private QueryBuilder filterBuilder = null;
+ private TermsAggregator.BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(DEFAULT_BUCKET_COUNT_THRESHOLDS);
+ private SignificanceHeuristic significanceHeuristic = DEFAULT_SIGNIFICANCE_HEURISTIC;
+
+ private static final ObjectParser<SignificantTextAggregationBuilder, Void> PARSER = new ObjectParser<>(
+ SignificantTextAggregationBuilder.NAME,
+ SignificanceHeuristic.class,
+ SignificantTextAggregationBuilder::significanceHeuristic,
+ null
+ );
+ static {
+ PARSER.declareInt(SignificantTextAggregationBuilder::shardSize, TermsAggregationBuilder.SHARD_SIZE_FIELD_NAME);
+
+ PARSER.declareLong(SignificantTextAggregationBuilder::minDocCount, TermsAggregationBuilder.MIN_DOC_COUNT_FIELD_NAME);
+
+ PARSER.declareLong(SignificantTextAggregationBuilder::shardMinDocCount, TermsAggregationBuilder.SHARD_MIN_DOC_COUNT_FIELD_NAME);
+
+ PARSER.declareInt(SignificantTextAggregationBuilder::size, TermsAggregationBuilder.REQUIRED_SIZE_FIELD_NAME);
+
+ PARSER.declareString(SignificantTextAggregationBuilder::fieldName, FIELD_NAME);
+
+ PARSER.declareStringArray(SignificantTextAggregationBuilder::sourceFieldNames, SOURCE_FIELDS_NAME);
+
+ PARSER.declareBoolean(SignificantTextAggregationBuilder::filterDuplicateText, FILTER_DUPLICATE_TEXT_FIELD_NAME);
+
+ PARSER.declareObject(
+ SignificantTextAggregationBuilder::backgroundFilter,
+ (p, context) -> AbstractQueryBuilder.parseInnerQueryBuilder(p),
+ SignificantTermsAggregationBuilder.BACKGROUND_FILTER
+ );
+
+ PARSER.declareField(
+ (b, v) -> b.includeExclude(IncludeExclude.merge(v, b.includeExclude())),
+ IncludeExclude::parseInclude,
+ IncludeExclude.INCLUDE_FIELD,
+ ObjectParser.ValueType.OBJECT_ARRAY_OR_STRING
+ );
+
+ PARSER.declareField(
+ (b, v) -> b.includeExclude(IncludeExclude.merge(b.includeExclude(), v)),
+ IncludeExclude::parseExclude,
+ IncludeExclude.EXCLUDE_FIELD,
+ ObjectParser.ValueType.STRING_ARRAY
+ );
+ }
+
+ public static SignificantTextAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
+ return PARSER.parse(parser, new SignificantTextAggregationBuilder(aggregationName, null), null);
+ }
+
+ protected SignificantTextAggregationBuilder(
+ SignificantTextAggregationBuilder clone,
+ Builder factoriesBuilder,
+ Map<String, Object> metadata
+ ) {
+ super(clone, factoriesBuilder, metadata);
+ this.bucketCountThresholds = new BucketCountThresholds(clone.bucketCountThresholds);
+ this.fieldName = clone.fieldName;
+ this.filterBuilder = clone.filterBuilder;
+ this.filterDuplicateText = clone.filterDuplicateText;
+ this.includeExclude = clone.includeExclude;
+ this.significanceHeuristic = clone.significanceHeuristic;
+ this.sourceFieldNames = clone.sourceFieldNames;
+ }
+
+ @Override
+ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map<String, Object> metadata) {
+ return new SignificantTextAggregationBuilder(this, factoriesBuilder, metadata);
+ }
+
+ protected TermsAggregator.BucketCountThresholds getBucketCountThresholds() {
+ return new TermsAggregator.BucketCountThresholds(bucketCountThresholds);
+ }
+
+ public TermsAggregator.BucketCountThresholds bucketCountThresholds() {
+ return bucketCountThresholds;
+ }
+
+ @Override
+ public SignificantTextAggregationBuilder subAggregations(Builder subFactories) {
+ throw new AggregationInitializationException(
+ "Aggregator [" + name + "] of type [" + getType() + "] cannot accept sub-aggregations"
+ );
+ }
+
+ @Override
+ public SignificantTextAggregationBuilder subAggregation(AggregationBuilder aggregation) {
+ throw new AggregationInitializationException(
+ "Aggregator [" + name + "] of type [" + getType() + "] cannot accept sub-aggregations"
+ );
+ }
+
+ public SignificantTextAggregationBuilder bucketCountThresholds(TermsAggregator.BucketCountThresholds bucketCountThresholds) {
+ if (bucketCountThresholds == null) {
+ throw new IllegalArgumentException("[bucketCountThresholds] must not be null: [" + name + "]");
+ }
+ this.bucketCountThresholds = bucketCountThresholds;
+ return this;
+ }
+
+ /**
+ * Sets the size - indicating how many term buckets should be returned
+ * (defaults to 10)
+ */
+ public SignificantTextAggregationBuilder size(int size) {
+ if (size <= 0) {
+ throw new IllegalArgumentException("[size] must be greater than 0. Found [" + size + "] in [" + name + "]");
+ }
+ bucketCountThresholds.setRequiredSize(size);
+ return this;
+ }
+
+ /**
+ * Sets the shard_size - indicating the number of term buckets each shard
+ * will return to the coordinating node (the node that coordinates the
+ * search execution). The higher the shard size is, the more accurate the
+ * results are.
+ */
+ public SignificantTextAggregationBuilder shardSize(int shardSize) {
+ if (shardSize <= 0) {
+ throw new IllegalArgumentException("[shardSize] must be greater than 0. Found [" + shardSize + "] in [" + name + "]");
+ }
+ bucketCountThresholds.setShardSize(shardSize);
+ return this;
+ }
+
+ /**
+ * Sets the name of the text field that will be the subject of this
+ * aggregation.
+ */
+ public SignificantTextAggregationBuilder fieldName(String fieldName) {
+ this.fieldName = fieldName;
+ return this;
+ }
+
+ /**
+ * Selects the fields to load from _source JSON and analyze.
+ * If none are specified, the indexed "fieldName" value is assumed
+ * to also be the name of the JSON field holding the value
+ */
+ public SignificantTextAggregationBuilder sourceFieldNames(List<String> names) {
+ this.sourceFieldNames = names.toArray(new String[0]);
+ return this;
+ }
+
+ /**
+ * Control if duplicate paragraphs of text should try be filtered from the
+ * statistical text analysis. Can improve results but slows down analysis.
+ * Default is false.
+ */
+ public SignificantTextAggregationBuilder filterDuplicateText(boolean filterDuplicateText) {
+ this.filterDuplicateText = filterDuplicateText;
+ return this;
+ }
+
+ /**
+ * Set the minimum document count terms should have in order to appear in
+ * the response.
+ */
+ public SignificantTextAggregationBuilder minDocCount(long minDocCount) {
+ if (minDocCount < 0) {
+ throw new IllegalArgumentException(
+ "[minDocCount] must be greater than or equal to 0. Found [" + minDocCount + "] in [" + name + "]"
+ );
+ }
+ bucketCountThresholds.setMinDocCount(minDocCount);
+ return this;
+ }
+
+ /**
+ * Set the minimum document count terms should have on the shard in order to
+ * appear in the response.
+ */
+ public SignificantTextAggregationBuilder shardMinDocCount(long shardMinDocCount) {
+ if (shardMinDocCount < 0) {
+ throw new IllegalArgumentException(
+ "[shardMinDocCount] must be greater than or equal to 0. Found [" + shardMinDocCount + "] in [" + name + "]"
+ );
+ }
+ bucketCountThresholds.setShardMinDocCount(shardMinDocCount);
+ return this;
+ }
+
+ public SignificantTextAggregationBuilder backgroundFilter(QueryBuilder backgroundFilter) {
+ if (backgroundFilter == null) {
+ throw new IllegalArgumentException("[backgroundFilter] must not be null: [" + name + "]");
+ }
+ this.filterBuilder = backgroundFilter;
+ return this;
+ }
+
+ public QueryBuilder backgroundFilter() {
+ return filterBuilder;
+ }
+
+ /**
+ * Set terms to include and exclude from the aggregation results
+ */
+ public SignificantTextAggregationBuilder includeExclude(IncludeExclude includeExclude) {
+ this.includeExclude = includeExclude;
+ return this;
+ }
+
+ /**
+ * Get terms to include and exclude from the aggregation results
+ */
+ public IncludeExclude includeExclude() {
+ return includeExclude;
+ }
+
+ public SignificantTextAggregationBuilder significanceHeuristic(SignificanceHeuristic significanceHeuristic) {
+ if (significanceHeuristic == null) {
+ throw new IllegalArgumentException("[significanceHeuristic] must not be null: [" + name + "]");
+ }
+ this.significanceHeuristic = significanceHeuristic;
+ return this;
+ }
+
+ public SignificanceHeuristic significanceHeuristic() {
+ return significanceHeuristic;
+ }
+
+ /**
+ * @param name
+ * the name of this aggregation
+ * @param fieldName
+ * the name of the text field that will be the subject of this
+ * aggregation
+ *
+ */
+ public SignificantTextAggregationBuilder(String name, String fieldName) {
+ super(name);
+ this.fieldName = fieldName;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public SignificantTextAggregationBuilder(StreamInput in) throws IOException {
+ super(in);
+ fieldName = in.readString();
+ filterDuplicateText = in.readBoolean();
+ bucketCountThresholds = new BucketCountThresholds(in);
+ filterBuilder = in.readOptionalNamedWriteable(QueryBuilder.class);
+ includeExclude = in.readOptionalWriteable(IncludeExclude::new);
+ significanceHeuristic = in.readNamedWriteable(SignificanceHeuristic.class);
+ sourceFieldNames = in.readOptionalStringArray();
+ }
+
+ @Override
+ protected void doWriteTo(StreamOutput out) throws IOException {
+ out.writeString(fieldName);
+ out.writeBoolean(filterDuplicateText);
+ bucketCountThresholds.writeTo(out);
+ out.writeOptionalNamedWriteable(filterBuilder);
+ out.writeOptionalWriteable(includeExclude);
+ out.writeNamedWriteable(significanceHeuristic);
+ out.writeOptionalStringArray(sourceFieldNames);
+ }
+
+ @Override
+ public BucketCardinality bucketCardinality() {
+ return BucketCardinality.MANY;
+ }
+
+ @Override
+ protected AggregatorFactory doBuild(QueryShardContext queryShardContext, AggregatorFactory parent, Builder subFactoriesBuilder)
+ throws IOException {
+ SignificanceHeuristic executionHeuristic = this.significanceHeuristic.rewrite(queryShardContext);
+
+ return new SignificantTextAggregatorFactory(
+ name,
+ includeExclude,
+ filterBuilder,
+ bucketCountThresholds,
+ executionHeuristic,
+ queryShardContext,
+ parent,
+ subFactoriesBuilder,
+ fieldName,
+ sourceFieldNames,
+ filterDuplicateText,
+ metadata
+ );
+ }
+
+ @Override
+ protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject();
+ bucketCountThresholds.toXContent(builder, params);
+ if (fieldName != null) {
+ builder.field(FIELD_NAME.getPreferredName(), fieldName);
+ }
+ if (sourceFieldNames != null) {
+ builder.array(SOURCE_FIELDS_NAME.getPreferredName(), sourceFieldNames);
+ }
+
+ if (filterDuplicateText) {
+ builder.field(FILTER_DUPLICATE_TEXT_FIELD_NAME.getPreferredName(), filterDuplicateText);
+ }
+ if (filterBuilder != null) {
+ builder.field(SignificantTermsAggregationBuilder.BACKGROUND_FILTER.getPreferredName(), filterBuilder);
+ }
+ if (includeExclude != null) {
+ includeExclude.toXContent(builder, params);
+ }
+ significanceHeuristic.toXContent(builder, params);
+
+ builder.endObject();
+ return builder;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ super.hashCode(),
+ bucketCountThresholds,
+ fieldName,
+ filterDuplicateText,
+ filterBuilder,
+ includeExclude,
+ significanceHeuristic,
+ Arrays.hashCode(sourceFieldNames)
+ );
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+ SignificantTextAggregationBuilder other = (SignificantTextAggregationBuilder) obj;
+ return Objects.equals(bucketCountThresholds, other.bucketCountThresholds)
+ && Objects.equals(fieldName, other.fieldName)
+ && Arrays.equals(sourceFieldNames, other.sourceFieldNames)
+ && filterDuplicateText == other.filterDuplicateText
+ && Objects.equals(filterBuilder, other.filterBuilder)
+ && Objects.equals(includeExclude, other.includeExclude)
+ && Objects.equals(significanceHeuristic, other.significanceHeuristic);
+ }
+
+ @Override
+ public String getType() {
+ return NAME;
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: SignificantTextAggregatorFactory (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | SignificantTextAggregatorFactory |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/25)
+
+ |
+
+
+ | SignificantTextAggregatorFactory$SignificantTextCollectorSource |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/9)
+
+ |
+
+
+ | SignificantTextAggregatorFactory$SignificantTextCollectorSource$1 |
+
+
+ 0%
+
+
+ (0/5)
+
+ |
+
+
+ 0%
+
+
+ (0/56)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/13)
+
+ |
+
+
+ 0%
+
+
+ (0/90)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.analysis.Analyzer;
+ import org.apache.lucene.analysis.TokenStream;
+ import org.apache.lucene.analysis.miscellaneous.DeDuplicatingTokenFilter;
+ import org.apache.lucene.analysis.miscellaneous.DuplicateByteSequenceSpotter;
+ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+ import org.apache.lucene.index.LeafReaderContext;
+ import org.apache.lucene.util.BytesRef;
+ import org.apache.lucene.util.BytesRefBuilder;
+ import org.opensearch.common.lease.Releasables;
+ import org.opensearch.common.util.BigArrays;
+ import org.opensearch.common.util.BytesRefHash;
+ import org.opensearch.common.util.ObjectArray;
+ import org.opensearch.index.mapper.MappedFieldType;
+ import org.opensearch.index.query.QueryBuilder;
+ import org.opensearch.index.query.QueryShardContext;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.AggregatorFactory;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.LeafBucketCollector;
+ import org.opensearch.search.aggregations.LeafBucketCollectorBase;
+ import org.opensearch.search.aggregations.bucket.BucketUtils;
+ import org.opensearch.search.aggregations.bucket.terms.IncludeExclude.StringFilter;
+ import org.opensearch.search.aggregations.bucket.terms.MapStringTermsAggregator.CollectConsumer;
+ import org.opensearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+ import org.opensearch.search.internal.SearchContext;
+ import org.opensearch.search.lookup.SourceLookup;
+
+ import java.io.IOException;
+ import java.util.Iterator;
+ import java.util.Map;
+ import java.util.function.LongConsumer;
+
+ /**
+ * Aggregation Factory for significant_text agg
+ *
+ * @opensearch.internal
+ */
+ public class SignificantTextAggregatorFactory extends AggregatorFactory {
+ private static final int MEMORY_GROWTH_REPORTING_INTERVAL_BYTES = 5000;
+
+ private final IncludeExclude includeExclude;
+ private final String indexedFieldName;
+ private final MappedFieldType fieldType;
+ private final String[] sourceFieldNames;
+ private final QueryBuilder backgroundFilter;
+ private final TermsAggregator.BucketCountThresholds bucketCountThresholds;
+ private final SignificanceHeuristic significanceHeuristic;
+ private final boolean filterDuplicateText;
+
+ public SignificantTextAggregatorFactory(
+ String name,
+ IncludeExclude includeExclude,
+ QueryBuilder backgroundFilter,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ SignificanceHeuristic significanceHeuristic,
+ QueryShardContext queryShardContext,
+ AggregatorFactory parent,
+ AggregatorFactories.Builder subFactoriesBuilder,
+ String fieldName,
+ String[] sourceFieldNames,
+ boolean filterDuplicateText,
+ Map<String, Object> metadata
+ ) throws IOException {
+ super(name, queryShardContext, parent, subFactoriesBuilder, metadata);
+
+ // Note that if the field is unmapped (its field type is null), we don't fail,
+ // and just use the given field name as a placeholder.
+ this.fieldType = queryShardContext.fieldMapper(fieldName);
+ if (fieldType != null && fieldType.indexAnalyzer() == null) {
+ throw new IllegalArgumentException(
+ "Field [" + fieldType.name() + "] has no analyzer, but SignificantText " + "requires an analyzed field"
+ );
+ }
+ this.indexedFieldName = fieldType != null ? fieldType.name() : fieldName;
+ this.sourceFieldNames = sourceFieldNames == null ? new String[] { indexedFieldName } : sourceFieldNames;
+
+ this.includeExclude = includeExclude;
+ this.backgroundFilter = backgroundFilter;
+ this.filterDuplicateText = filterDuplicateText;
+ this.bucketCountThresholds = bucketCountThresholds;
+ this.significanceHeuristic = significanceHeuristic;
+ }
+
+ @Override
+ protected Aggregator createInternal(
+ SearchContext searchContext,
+ Aggregator parent,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+ BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(this.bucketCountThresholds);
+ if (bucketCountThresholds.getShardSize() == SignificantTextAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) {
+ // The user has not made a shardSize selection.
+ // Use default heuristic to avoid any wrong-ranking caused by
+ // distributed counting but request double the usual amount.
+ // We typically need more than the number of "top" terms requested
+ // by other aggregations as the significance algorithm is in less
+ // of a position to down-select at shard-level - some of the things
+ // we want to find have only one occurrence on each shard and as
+ // such are impossible to differentiate from non-significant terms
+ // at that early stage.
+ bucketCountThresholds.setShardSize(2 * BucketUtils.suggestShardSideQueueSize(bucketCountThresholds.getRequiredSize()));
+ }
+
+ // TODO - need to check with mapping that this is indeed a text field....
+
+ int maxRegexLength = searchContext.getQueryShardContext().getIndexSettings().getMaxRegexLength();
+ IncludeExclude.StringFilter incExcFilter = includeExclude == null
+ ? null
+ : includeExclude.convertToStringFilter(DocValueFormat.RAW, maxRegexLength);
+
+ MapStringTermsAggregator.CollectorSource collectorSource = new SignificantTextCollectorSource(
+ queryShardContext.bigArrays(),
+ fieldType,
+ sourceFieldNames,
+ filterDuplicateText
+ );
+ SignificanceLookup lookup = new SignificanceLookup(queryShardContext, fieldType, DocValueFormat.RAW, backgroundFilter);
+ return new MapStringTermsAggregator(
+ name,
+ factories,
+ collectorSource,
+ a -> a.new SignificantTermsResults(lookup, significanceHeuristic, cardinality),
+ null,
+ DocValueFormat.RAW,
+ bucketCountThresholds,
+ incExcFilter,
+ searchContext,
+ parent,
+ SubAggCollectionMode.BREADTH_FIRST,
+ false,
+ cardinality,
+ metadata
+ );
+ }
+
+ /**
+ * Collects significant text
+ *
+ * @opensearch.internal
+ */
+ private static class SignificantTextCollectorSource implements MapStringTermsAggregator.CollectorSource {
+ private final SourceLookup sourceLookup;
+ private final BigArrays bigArrays;
+ private final MappedFieldType fieldType;
+ private final String[] sourceFieldNames;
+ private ObjectArray<DuplicateByteSequenceSpotter> dupSequenceSpotters;
+
+ SignificantTextCollectorSource(
+ BigArrays bigArrays,
+ MappedFieldType fieldType,
+ String[] sourceFieldNames,
+ boolean filterDuplicateText
+ ) {
+ // Create a new SourceLookup instance per aggregator instead of use the shared one from SearchLookup. This is fine because it
+ // will only be accessed by this Aggregator instance and not anywhere else.
+ this.sourceLookup = new SourceLookup();
+ this.bigArrays = bigArrays;
+ this.fieldType = fieldType;
+ this.sourceFieldNames = sourceFieldNames;
+ dupSequenceSpotters = filterDuplicateText ? bigArrays.newObjectArray(1) : null;
+ }
+
+ @Override
+ public boolean needsScores() {
+ return false;
+ }
+
+ @Override
+ public LeafBucketCollector getLeafCollector(
+ StringFilter includeExclude,
+ LeafReaderContext ctx,
+ LeafBucketCollector sub,
+ LongConsumer addRequestCircuitBreakerBytes,
+ CollectConsumer consumer
+ ) throws IOException {
+ return new LeafBucketCollectorBase(sub, null) {
+ private final BytesRefBuilder scratch = new BytesRefBuilder();
+
+ @Override
+ public void collect(int doc, long owningBucketOrd) throws IOException {
+ if (dupSequenceSpotters == null) {
+ collectFromSource(doc, owningBucketOrd, null);
+ return;
+ }
+ dupSequenceSpotters = bigArrays.grow(dupSequenceSpotters, owningBucketOrd + 1);
+ DuplicateByteSequenceSpotter spotter = dupSequenceSpotters.get(owningBucketOrd);
+ if (spotter == null) {
+ spotter = new DuplicateByteSequenceSpotter();
+ dupSequenceSpotters.set(owningBucketOrd, spotter);
+ }
+ collectFromSource(doc, owningBucketOrd, spotter);
+ spotter.startNewSequence();
+ }
+
+ private void collectFromSource(int doc, long owningBucketOrd, DuplicateByteSequenceSpotter spotter) throws IOException {
+ sourceLookup.setSegmentAndDocument(ctx, doc);
+ BytesRefHash inDocTerms = new BytesRefHash(256, bigArrays);
+
+ try {
+ for (String sourceField : sourceFieldNames) {
+ Iterator<String> itr = sourceLookup.extractRawValues(sourceField).stream().map(obj -> {
+ if (obj == null) {
+ return null;
+ }
+ if (obj instanceof BytesRef) {
+ return fieldType.valueForDisplay(obj).toString();
+ }
+ return obj.toString();
+ }).iterator();
+ Analyzer analyzer = fieldType.indexAnalyzer();
+ while (itr.hasNext()) {
+ TokenStream ts = analyzer.tokenStream(fieldType.name(), itr.next());
+ processTokenStream(doc, owningBucketOrd, ts, inDocTerms, spotter);
+ }
+ }
+ } finally {
+ Releasables.close(inDocTerms);
+ }
+ }
+
+ private void processTokenStream(
+ int doc,
+ long owningBucketOrd,
+ TokenStream ts,
+ BytesRefHash inDocTerms,
+ DuplicateByteSequenceSpotter spotter
+ ) throws IOException {
+ long lastTrieSize = 0;
+ if (spotter != null) {
+ lastTrieSize = spotter.getEstimatedSizeInBytes();
+ ts = new DeDuplicatingTokenFilter(ts, spotter);
+ }
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+ ts.reset();
+ try {
+ while (ts.incrementToken()) {
+ if (spotter != null) {
+ long newTrieSize = spotter.getEstimatedSizeInBytes();
+ long growth = newTrieSize - lastTrieSize;
+ // Only update the circuitbreaker after
+ if (growth > MEMORY_GROWTH_REPORTING_INTERVAL_BYTES) {
+ addRequestCircuitBreakerBytes.accept(growth);
+ lastTrieSize = newTrieSize;
+ }
+ }
+
+ scratch.clear();
+ scratch.copyChars(termAtt);
+ BytesRef bytes = scratch.get();
+ if (includeExclude != null && false == includeExclude.accept(bytes)) {
+ continue;
+ }
+ if (inDocTerms.add(bytes) < 0) {
+ continue;
+ }
+ consumer.accept(sub, doc, owningBucketOrd, bytes);
+ }
+ } finally {
+ ts.close();
+ }
+ if (spotter != null) {
+ long growth = spotter.getEstimatedSizeInBytes() - lastTrieSize;
+ if (growth > 0) {
+ addRequestCircuitBreakerBytes.accept(growth);
+ }
+ }
+ }
+ };
+ }
+
+ @Override
+ public void close() {
+ Releasables.close(dupSequenceSpotters);
+ }
+ }
+
+ @Override
+ protected boolean supportsConcurrentSegmentSearch() {
+ return true;
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: StringRareTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | StringRareTerms |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ | StringRareTerms$Bucket |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/14)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/20)
+
+ |
+
+
+ 0%
+
+
+ (0/24)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.util.BytesRef;
+ import org.opensearch.common.util.SetBackedScalingCuckooFilter;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.InternalAggregations;
+
+ import java.io.IOException;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+
+ /**
+ * Implementation of string rare terms
+ *
+ * @opensearch.internal
+ */
+ public class StringRareTerms extends InternalMappedRareTerms<StringRareTerms, StringRareTerms.Bucket> {
+ public static final String NAME = "srareterms";
+
+ /**
+ * Bucket for rare string terms
+ *
+ * @opensearch.internal
+ */
+ public static class Bucket extends InternalRareTerms.Bucket<Bucket> {
+ BytesRef termBytes;
+
+ public Bucket(BytesRef term, long docCount, InternalAggregations aggregations, DocValueFormat format) {
+ super(docCount, aggregations, format);
+ this.termBytes = term;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public Bucket(StreamInput in, DocValueFormat format) throws IOException {
+ super(in, format);
+ termBytes = in.readBytesRef();
+ }
+
+ @Override
+ protected void writeTermTo(StreamOutput out) throws IOException {
+ out.writeBytesRef(termBytes);
+ }
+
+ @Override
+ public Object getKey() {
+ return getKeyAsString();
+ }
+
+ // this method is needed for scripted numeric aggs
+ @Override
+ public Number getKeyAsNumber() {
+ /*
+ * If the term is a long greater than 2^52 then parsing as a double would lose accuracy. Therefore, we first parse as a long and
+ * if this fails then we attempt to parse the term as a double.
+ */
+ try {
+ return Long.parseLong(termBytes.utf8ToString());
+ } catch (final NumberFormatException ignored) {
+ return Double.parseDouble(termBytes.utf8ToString());
+ }
+ }
+
+ @Override
+ public String getKeyAsString() {
+ return format.format(termBytes).toString();
+ }
+
+ @Override
+ public int compareKey(Bucket other) {
+ return termBytes.compareTo(other.termBytes);
+ }
+
+ @Override
+ protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ return builder.field(CommonFields.KEY.getPreferredName(), getKeyAsString());
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj) && Objects.equals(termBytes, ((Bucket) obj).termBytes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), termBytes);
+ }
+ }
+
+ StringRareTerms(
+ String name,
+ BucketOrder order,
+ Map<String, Object> metadata,
+ DocValueFormat format,
+ List<StringRareTerms.Bucket> buckets,
+ long maxDocCount,
+ SetBackedScalingCuckooFilter filter
+ ) {
+ super(name, order, metadata, format, buckets, maxDocCount, filter);
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public StringRareTerms(StreamInput in) throws IOException {
+ super(in, StringRareTerms.Bucket::new);
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public StringRareTerms create(List<StringRareTerms.Bucket> buckets) {
+ return new StringRareTerms(name, order, metadata, format, buckets, maxDocCount, filter);
+ }
+
+ @Override
+ public StringRareTerms.Bucket createBucket(InternalAggregations aggregations, StringRareTerms.Bucket prototype) {
+ return new StringRareTerms.Bucket(prototype.termBytes, prototype.getDocCount(), aggregations, prototype.format);
+ }
+
+ @Override
+ protected StringRareTerms createWithFilter(
+ String name,
+ List<StringRareTerms.Bucket> buckets,
+ SetBackedScalingCuckooFilter filterFilter
+ ) {
+ return new StringRareTerms(name, order, metadata, format, buckets, maxDocCount, filterFilter);
+ }
+
+ @Override
+ protected StringRareTerms.Bucket[] createBucketsArray(int size) {
+ return new StringRareTerms.Bucket[size];
+ }
+
+ @Override
+ public boolean containsTerm(SetBackedScalingCuckooFilter filter, StringRareTerms.Bucket bucket) {
+ return filter.mightContain(bucket.termBytes);
+ }
+
+ @Override
+ public void addToFilter(SetBackedScalingCuckooFilter filter, StringRareTerms.Bucket bucket) {
+ filter.add(bucket.termBytes);
+ }
+
+ @Override
+ Bucket createBucket(long docCount, InternalAggregations aggs, StringRareTerms.Bucket prototype) {
+ return new Bucket(prototype.termBytes, docCount, aggs, format);
+ }
+
+ }
+
+
+
+
+
+
Coverage Summary for Class: StringRareTermsAggregator (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | StringRareTermsAggregator |
+
+
+ 0%
+
+
+ (0/5)
+
+ |
+
+
+ 0%
+
+
+ (0/47)
+
+ |
+
+
+ | StringRareTermsAggregator$1 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/17)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/7)
+
+ |
+
+
+ 0%
+
+
+ (0/64)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.index.LeafReaderContext;
+ import org.apache.lucene.util.BytesRef;
+ import org.apache.lucene.util.BytesRefBuilder;
+ import org.opensearch.common.lease.Releasables;
+ import org.opensearch.common.util.BytesRefHash;
+ import org.opensearch.common.util.SetBackedScalingCuckooFilter;
+ import org.opensearch.index.fielddata.SortedBinaryDocValues;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.LeafBucketCollector;
+ import org.opensearch.search.aggregations.LeafBucketCollectorBase;
+ import org.opensearch.search.aggregations.support.ValuesSource;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.List;
+ import java.util.Map;
+
+ import static java.util.Collections.emptyList;
+
+ /**
+ * An aggregator that finds "rare" string values (e.g. terms agg that orders ascending)
+ *
+ * @opensearch.internal
+ */
+ public class StringRareTermsAggregator extends AbstractRareTermsAggregator {
+ private final ValuesSource.Bytes valuesSource;
+ private final IncludeExclude.StringFilter filter;
+ private final BytesKeyedBucketOrds bucketOrds;
+
+ StringRareTermsAggregator(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource.Bytes valuesSource,
+ DocValueFormat format,
+ IncludeExclude.StringFilter filter,
+ SearchContext context,
+ Aggregator parent,
+ Map<String, Object> metadata,
+ long maxDocCount,
+ double precision,
+ CardinalityUpperBound cardinality
+ ) throws IOException {
+ super(name, factories, context, parent, metadata, maxDocCount, precision, format);
+ this.valuesSource = valuesSource;
+ this.filter = filter;
+ this.bucketOrds = BytesKeyedBucketOrds.build(context.bigArrays(), cardinality);
+ }
+
+ @Override
+ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException {
+ final SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
+ return new LeafBucketCollectorBase(sub, values) {
+ final BytesRefBuilder previous = new BytesRefBuilder();
+
+ @Override
+ public void collect(int docId, long owningBucketOrd) throws IOException {
+ if (false == values.advanceExact(docId)) {
+ return;
+ }
+ int valuesCount = values.docValueCount();
+ previous.clear();
+
+ // SortedBinaryDocValues don't guarantee uniqueness so we
+ // need to take care of dups
+ for (int i = 0; i < valuesCount; ++i) {
+ BytesRef bytes = values.nextValue();
+ if (filter != null && false == filter.accept(bytes)) {
+ continue;
+ }
+ if (i > 0 && previous.get().equals(bytes)) {
+ continue;
+ }
+ previous.copyBytes(bytes);
+ long bucketOrdinal = bucketOrds.add(owningBucketOrd, bytes);
+ if (bucketOrdinal < 0) { // already seen
+ bucketOrdinal = -1 - bucketOrdinal;
+ collectExistingBucket(sub, docId, bucketOrdinal);
+ } else {
+ collectBucket(sub, docId, bucketOrdinal);
+ }
+ }
+ }
+ };
+ }
+
+ @Override
+ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException {
+ /*
+ * Collect the list of buckets, populate the filter with terms
+ * that are too frequent, and figure out how to merge sub-buckets.
+ */
+ StringRareTerms.Bucket[][] rarestPerOrd = new StringRareTerms.Bucket[owningBucketOrds.length][];
+ SetBackedScalingCuckooFilter[] filters = new SetBackedScalingCuckooFilter[owningBucketOrds.length];
+ long keepCount = 0;
+ long[] mergeMap = new long[(int) bucketOrds.size()];
+ Arrays.fill(mergeMap, -1);
+ long offset = 0;
+ for (int owningOrdIdx = 0; owningOrdIdx < owningBucketOrds.length; owningOrdIdx++) {
+ try (BytesRefHash bucketsInThisOwningBucketToCollect = new BytesRefHash(context.bigArrays())) {
+ filters[owningOrdIdx] = newFilter();
+ List<StringRareTerms.Bucket> builtBuckets = new ArrayList<>();
+ BytesKeyedBucketOrds.BucketOrdsEnum collectedBuckets = bucketOrds.ordsEnum(owningBucketOrds[owningOrdIdx]);
+ BytesRef scratch = new BytesRef();
+ while (collectedBuckets.next()) {
+ collectedBuckets.readValue(scratch);
+ long docCount = bucketDocCount(collectedBuckets.ord());
+ // if the key is below threshold, reinsert into the new ords
+ if (docCount <= maxDocCount) {
+ StringRareTerms.Bucket bucket = new StringRareTerms.Bucket(BytesRef.deepCopyOf(scratch), docCount, null, format);
+ bucket.bucketOrd = offset + bucketsInThisOwningBucketToCollect.add(scratch);
+ mergeMap[(int) collectedBuckets.ord()] = bucket.bucketOrd;
+ builtBuckets.add(bucket);
+ keepCount++;
+ } else {
+ filters[owningOrdIdx].add(scratch);
+ }
+ }
+ rarestPerOrd[owningOrdIdx] = builtBuckets.toArray(new StringRareTerms.Bucket[0]);
+ offset += bucketsInThisOwningBucketToCollect.size();
+ }
+ }
+
+ /*
+ * Only merge/delete the ordinals if we have actually deleted one,
+ * to save on some redundant work.
+ */
+ if (keepCount != mergeMap.length) {
+ mergeBuckets(mergeMap, offset);
+ if (deferringCollector != null) {
+ deferringCollector.mergeBuckets(mergeMap);
+ }
+ }
+
+ /*
+ * Now build the results!
+ */
+ buildSubAggsForAllBuckets(rarestPerOrd, b -> b.bucketOrd, (b, aggs) -> b.aggregations = aggs);
+ InternalAggregation[] result = new InternalAggregation[owningBucketOrds.length];
+ for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) {
+ Arrays.sort(rarestPerOrd[ordIdx], ORDER.comparator());
+ result[ordIdx] = new StringRareTerms(
+ name,
+ ORDER,
+ metadata(),
+ format,
+ Arrays.asList(rarestPerOrd[ordIdx]),
+ maxDocCount,
+ filters[ordIdx]
+ );
+ }
+ return result;
+ }
+
+ @Override
+ public InternalAggregation buildEmptyAggregation() {
+ return new StringRareTerms(name, LongRareTermsAggregator.ORDER, metadata(), format, emptyList(), 0, newFilter());
+ }
+
+ @Override
+ public void doClose() {
+ Releasables.close(bucketOrds);
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: BucketPriorityQueue (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | BucketPriorityQueue |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 100%
+
+
+ (2/2)
+
+ |
+
+
+ 100%
+
+
+ (3/3)
+
+ |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.util.PriorityQueue;
+
+ import java.util.Comparator;
+
+ /**
+ * Internal priority queue for computing terms aggs
+ *
+ * @opensearch.internal
+ */
+ public class BucketPriorityQueue<B extends Terms.Bucket> extends PriorityQueue<B> {
+
+ private final Comparator<? super B> comparator;
+
+ public BucketPriorityQueue(int size, Comparator<? super B> comparator) {
+ super(size);
+ this.comparator = comparator;
+ }
+
+ @Override
+ protected boolean lessThan(B a, B b) {
+ return comparator.compare(a, b) > 0; // reverse, since we reverse again when adding to a list
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: StringTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | StringTerms |
+
+
+ 50%
+
+
+ (4/8)
+
+ |
+
+
+ 55.6%
+
+
+ (5/9)
+
+ |
+
+
+ | StringTerms$Bucket |
+
+
+ 30%
+
+
+ (3/10)
+
+ |
+
+
+ 28.6%
+
+
+ (4/14)
+
+ |
+
+
+ | Total |
+
+
+ 38.9%
+
+
+ (7/18)
+
+ |
+
+
+ 39.1%
+
+
+ (9/23)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.util.BytesRef;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.InternalAggregations;
+
+ import java.io.IOException;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+
+ /**
+ * Result of the {@link TermsAggregator} when the field is a String.
+ *
+ * @opensearch.internal
+ */
+ public class StringTerms extends InternalMappedTerms<StringTerms, StringTerms.Bucket> {
+ public static final String NAME = "sterms";
+
+ /**
+ * Bucket for string terms
+ *
+ * @opensearch.internal
+ */
+ public static class Bucket extends InternalTerms.Bucket<Bucket> {
+ BytesRef termBytes;
+
+ public Bucket(
+ BytesRef term,
+ long docCount,
+ InternalAggregations aggregations,
+ boolean showDocCountError,
+ long docCountError,
+ DocValueFormat format
+ ) {
+ super(docCount, aggregations, showDocCountError, docCountError, format);
+ this.termBytes = term;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public Bucket(StreamInput in, DocValueFormat format, boolean showDocCountError) throws IOException {
+ super(in, format, showDocCountError);
+ termBytes = in.readBytesRef();
+ }
+
+ @Override
+ protected void writeTermTo(StreamOutput out) throws IOException {
+ out.writeBytesRef(termBytes);
+ }
+
+ @Override
+ public Object getKey() {
+ return getKeyAsString();
+ }
+
+ // this method is needed for scripted numeric aggs
+ @Override
+ public Number getKeyAsNumber() {
+ /*
+ * If the term is a long greater than 2^52 then parsing as a double would lose accuracy. Therefore, we first parse as a long and
+ * if this fails then we attempt to parse the term as a double.
+ */
+ try {
+ return Long.parseLong(termBytes.utf8ToString());
+ } catch (final NumberFormatException ignored) {
+ return Double.parseDouble(termBytes.utf8ToString());
+ }
+ }
+
+ @Override
+ public String getKeyAsString() {
+ return format.format(termBytes).toString();
+ }
+
+ @Override
+ public int compareKey(Bucket other) {
+ return termBytes.compareTo(other.termBytes);
+ }
+
+ @Override
+ protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ return builder.field(CommonFields.KEY.getPreferredName(), getKeyAsString());
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj) && Objects.equals(termBytes, ((Bucket) obj).termBytes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), termBytes);
+ }
+ }
+
+ public StringTerms(
+ String name,
+ BucketOrder reduceOrder,
+ BucketOrder order,
+ Map<String, Object> metadata,
+ DocValueFormat format,
+ int shardSize,
+ boolean showTermDocCountError,
+ long otherDocCount,
+ List<Bucket> buckets,
+ long docCountError,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds
+ ) {
+ super(
+ name,
+ reduceOrder,
+ order,
+ metadata,
+ format,
+ shardSize,
+ showTermDocCountError,
+ otherDocCount,
+ buckets,
+ docCountError,
+ bucketCountThresholds
+ );
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public StringTerms(StreamInput in) throws IOException {
+ super(in, Bucket::new);
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public StringTerms create(List<Bucket> buckets) {
+ return new StringTerms(
+ name,
+ reduceOrder,
+ order,
+ metadata,
+ format,
+ shardSize,
+ showTermDocCountError,
+ otherDocCount,
+ buckets,
+ docCountError,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) {
+ return new Bucket(
+ prototype.termBytes,
+ prototype.docCount,
+ aggregations,
+ prototype.showDocCountError,
+ prototype.docCountError,
+ prototype.format
+ );
+ }
+
+ @Override
+ Bucket createBucket(long docCount, InternalAggregations aggs, long docCountError, StringTerms.Bucket prototype) {
+ return new Bucket(prototype.termBytes, docCount, aggs, prototype.showDocCountError, docCountError, format);
+ }
+
+ @Override
+ protected StringTerms create(String name, List<Bucket> buckets, BucketOrder reduceOrder, long docCountError, long otherDocCount) {
+ return new StringTerms(
+ name,
+ reduceOrder,
+ order,
+ getMetadata(),
+ format,
+ shardSize,
+ showTermDocCountError,
+ otherDocCount,
+ buckets,
+ docCountError,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ protected Bucket[] createBucketsArray(int size) {
+ return new Bucket[size];
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: TermsAggregationBuilder (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | TermsAggregationBuilder |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 30%
+
+
+ (12/40)
+
+ |
+
+
+ 34.6%
+
+
+ (45/130)
+
+ |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.Version;
+ import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+ import org.opensearch.core.ParseField;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.index.query.QueryRewriteContext;
+ import org.opensearch.index.query.QueryShardContext;
+ import org.opensearch.search.aggregations.AggregationBuilder;
+ import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.AggregatorFactory;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.InternalOrder;
+ import org.opensearch.search.aggregations.InternalOrder.CompoundOrder;
+ import org.opensearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
+ import org.opensearch.search.aggregations.support.CoreValuesSourceType;
+ import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder;
+ import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+ import org.opensearch.search.aggregations.support.ValuesSourceConfig;
+ import org.opensearch.search.aggregations.support.ValuesSourceRegistry;
+ import org.opensearch.search.aggregations.support.ValuesSourceType;
+
+ import java.io.IOException;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+
+ /**
+ * Aggregation Builder for terms agg
+ *
+ * @opensearch.internal
+ */
+ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<TermsAggregationBuilder> {
+ public static final String NAME = "terms";
+ public static final ValuesSourceRegistry.RegistryKey<TermsAggregatorSupplier> REGISTRY_KEY = new ValuesSourceRegistry.RegistryKey<>(
+ NAME,
+ TermsAggregatorSupplier.class
+ );
+
+ public static final ParseField EXECUTION_HINT_FIELD_NAME = new ParseField("execution_hint");
+ public static final ParseField SHARD_SIZE_FIELD_NAME = new ParseField("shard_size");
+ public static final ParseField MIN_DOC_COUNT_FIELD_NAME = new ParseField("min_doc_count");
+ public static final ParseField SHARD_MIN_DOC_COUNT_FIELD_NAME = new ParseField("shard_min_doc_count");
+ public static final ParseField REQUIRED_SIZE_FIELD_NAME = new ParseField("size");
+
+ static final TermsAggregator.BucketCountThresholds DEFAULT_BUCKET_COUNT_THRESHOLDS = new TermsAggregator.BucketCountThresholds(
+ 1,
+ 0,
+ 10,
+ -1
+ );
+ public static final ParseField SHOW_TERM_DOC_COUNT_ERROR = new ParseField("show_term_doc_count_error");
+ public static final ParseField ORDER_FIELD = new ParseField("order");
+
+ public static final ObjectParser<TermsAggregationBuilder, String> PARSER = ObjectParser.fromBuilder(NAME, TermsAggregationBuilder::new);
+ static {
+ ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false);
+
+ PARSER.declareBoolean(TermsAggregationBuilder::showTermDocCountError, TermsAggregationBuilder.SHOW_TERM_DOC_COUNT_ERROR);
+
+ PARSER.declareInt(TermsAggregationBuilder::shardSize, SHARD_SIZE_FIELD_NAME);
+
+ PARSER.declareLong(TermsAggregationBuilder::minDocCount, MIN_DOC_COUNT_FIELD_NAME);
+
+ PARSER.declareLong(TermsAggregationBuilder::shardMinDocCount, SHARD_MIN_DOC_COUNT_FIELD_NAME);
+
+ PARSER.declareInt(TermsAggregationBuilder::size, REQUIRED_SIZE_FIELD_NAME);
+
+ PARSER.declareString(TermsAggregationBuilder::executionHint, EXECUTION_HINT_FIELD_NAME);
+
+ PARSER.declareField(
+ TermsAggregationBuilder::collectMode,
+ (p, c) -> SubAggCollectionMode.parse(p.text(), LoggingDeprecationHandler.INSTANCE),
+ SubAggCollectionMode.KEY,
+ ObjectParser.ValueType.STRING
+ );
+
+ PARSER.declareObjectArray(
+ TermsAggregationBuilder::order,
+ (p, c) -> InternalOrder.Parser.parseOrderParam(p),
+ TermsAggregationBuilder.ORDER_FIELD
+ );
+
+ PARSER.declareField(
+ (b, v) -> b.includeExclude(IncludeExclude.merge(v, b.includeExclude())),
+ IncludeExclude::parseInclude,
+ IncludeExclude.INCLUDE_FIELD,
+ ObjectParser.ValueType.OBJECT_ARRAY_OR_STRING
+ );
+
+ PARSER.declareField(
+ (b, v) -> b.includeExclude(IncludeExclude.merge(b.includeExclude(), v)),
+ IncludeExclude::parseExclude,
+ IncludeExclude.EXCLUDE_FIELD,
+ ObjectParser.ValueType.STRING_ARRAY
+ );
+ }
+
+ public static void registerAggregators(ValuesSourceRegistry.Builder builder) {
+ TermsAggregatorFactory.registerAggregators(builder);
+ }
+
+ private BucketOrder order = BucketOrder.compound(BucketOrder.count(false)); // automatically adds tie-breaker key asc order
+ private IncludeExclude includeExclude = null;
+ private String executionHint = null;
+ private SubAggCollectionMode collectMode = null;
+ private TermsAggregator.BucketCountThresholds bucketCountThresholds = new TermsAggregator.BucketCountThresholds(
+ DEFAULT_BUCKET_COUNT_THRESHOLDS
+ );
+ private boolean showTermDocCountError = false;
+
+ public TermsAggregationBuilder(String name) {
+ super(name);
+ }
+
+ protected TermsAggregationBuilder(
+ TermsAggregationBuilder clone,
+ AggregatorFactories.Builder factoriesBuilder,
+ Map<String, Object> metadata
+ ) {
+ super(clone, factoriesBuilder, metadata);
+ this.order = clone.order;
+ this.executionHint = clone.executionHint;
+ this.includeExclude = clone.includeExclude;
+ this.collectMode = clone.collectMode;
+ this.bucketCountThresholds = new BucketCountThresholds(clone.bucketCountThresholds);
+ this.showTermDocCountError = clone.showTermDocCountError;
+ }
+
+ @Override
+ protected ValuesSourceType defaultValueSourceType() {
+ return CoreValuesSourceType.BYTES;
+ }
+
+ @Override
+ protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map<String, Object> metadata) {
+ return new TermsAggregationBuilder(this, factoriesBuilder, metadata);
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public TermsAggregationBuilder(StreamInput in) throws IOException {
+ super(in);
+ bucketCountThresholds = new BucketCountThresholds(in);
+ collectMode = in.readOptionalWriteable(SubAggCollectionMode::readFromStream);
+ executionHint = in.readOptionalString();
+ includeExclude = in.readOptionalWriteable(IncludeExclude::new);
+ order = InternalOrder.Streams.readOrder(in);
+ showTermDocCountError = in.readBoolean();
+ }
+
+ @Override
+ protected boolean serializeTargetValueType(Version version) {
+ return true;
+ }
+
+ @Override
+ protected void innerWriteTo(StreamOutput out) throws IOException {
+ bucketCountThresholds.writeTo(out);
+ out.writeOptionalWriteable(collectMode);
+ out.writeOptionalString(executionHint);
+ out.writeOptionalWriteable(includeExclude);
+ order.writeTo(out);
+ out.writeBoolean(showTermDocCountError);
+ }
+
+ /**
+ * Sets the size - indicating how many term buckets should be returned
+ * (defaults to 10)
+ */
+ public TermsAggregationBuilder size(int size) {
+ if (size <= 0) {
+ throw new IllegalArgumentException("[size] must be greater than 0. Found [" + size + "] in [" + name + "]");
+ }
+ bucketCountThresholds.setRequiredSize(size);
+ return this;
+ }
+
+ /**
+ * Returns the number of term buckets currently configured
+ */
+ public int size() {
+ return bucketCountThresholds.getRequiredSize();
+ }
+
+ /**
+ * Sets the shard_size - indicating the number of term buckets each shard
+ * will return to the coordinating node (the node that coordinates the
+ * search execution). The higher the shard size is, the more accurate the
+ * results are.
+ */
+ public TermsAggregationBuilder shardSize(int shardSize) {
+ if (shardSize <= 0) {
+ throw new IllegalArgumentException("[shardSize] must be greater than 0. Found [" + shardSize + "] in [" + name + "]");
+ }
+ bucketCountThresholds.setShardSize(shardSize);
+ return this;
+ }
+
+ /**
+ * Returns the number of term buckets per shard that are currently configured
+ */
+ public int shardSize() {
+ return bucketCountThresholds.getShardSize();
+ }
+
+ /**
+ * Set the minimum document count terms should have in order to appear in
+ * the response.
+ */
+ public TermsAggregationBuilder minDocCount(long minDocCount) {
+ if (minDocCount < 0) {
+ throw new IllegalArgumentException(
+ "[minDocCount] must be greater than or equal to 0. Found [" + minDocCount + "] in [" + name + "]"
+ );
+ }
+ bucketCountThresholds.setMinDocCount(minDocCount);
+ return this;
+ }
+
+ /**
+ * Returns the minimum document count required per term
+ */
+ public long minDocCount() {
+ return bucketCountThresholds.getMinDocCount();
+ }
+
+ /**
+ * Set the minimum document count terms should have on the shard in order to
+ * appear in the response.
+ */
+ public TermsAggregationBuilder shardMinDocCount(long shardMinDocCount) {
+ if (shardMinDocCount < 0) {
+ throw new IllegalArgumentException(
+ "[shardMinDocCount] must be greater than or equal to 0. Found [" + shardMinDocCount + "] in [" + name + "]"
+ );
+ }
+ bucketCountThresholds.setShardMinDocCount(shardMinDocCount);
+ return this;
+ }
+
+ /**
+ * Returns the minimum document count required per term, per shard
+ */
+ public long shardMinDocCount() {
+ return bucketCountThresholds.getShardMinDocCount();
+ }
+
+ /** Set a new order on this builder and return the builder so that calls
+ * can be chained. A tie-breaker may be added to avoid non-deterministic ordering. */
+ public TermsAggregationBuilder order(BucketOrder order) {
+ if (order == null) {
+ throw new IllegalArgumentException("[order] must not be null: [" + name + "]");
+ }
+ if (order instanceof CompoundOrder || InternalOrder.isKeyOrder(order)) {
+ this.order = order; // if order already contains a tie-breaker we are good to go
+ } else { // otherwise add a tie-breaker by using a compound order
+ this.order = BucketOrder.compound(order);
+ }
+ return this;
+ }
+
+ /**
+ * Sets the order in which the buckets will be returned. A tie-breaker may be added to avoid non-deterministic
+ * ordering.
+ */
+ public TermsAggregationBuilder order(List<BucketOrder> orders) {
+ if (orders == null) {
+ throw new IllegalArgumentException("[orders] must not be null: [" + name + "]");
+ }
+ // if the list only contains one order use that to avoid inconsistent xcontent
+ order(orders.size() > 1 ? BucketOrder.compound(orders) : orders.get(0));
+ return this;
+ }
+
+ /**
+ * Gets the order in which the buckets will be returned.
+ */
+ public BucketOrder order() {
+ return order;
+ }
+
+ /**
+ * Expert: sets an execution hint to the aggregation.
+ */
+ public TermsAggregationBuilder executionHint(String executionHint) {
+ this.executionHint = executionHint;
+ return this;
+ }
+
+ /**
+ * Expert: gets an execution hint to the aggregation.
+ */
+ public String executionHint() {
+ return executionHint;
+ }
+
+ /**
+ * Expert: set the collection mode.
+ */
+ public TermsAggregationBuilder collectMode(SubAggCollectionMode collectMode) {
+ if (collectMode == null) {
+ throw new IllegalArgumentException("[collectMode] must not be null: [" + name + "]");
+ }
+ this.collectMode = collectMode;
+ return this;
+ }
+
+ /**
+ * Expert: get the collection mode.
+ */
+ public SubAggCollectionMode collectMode() {
+ return collectMode;
+ }
+
+ /**
+ * Set terms to include and exclude from the aggregation results
+ */
+ public TermsAggregationBuilder includeExclude(IncludeExclude includeExclude) {
+ this.includeExclude = includeExclude;
+ return this;
+ }
+
+ /**
+ * Get terms to include and exclude from the aggregation results
+ */
+ public IncludeExclude includeExclude() {
+ return includeExclude;
+ }
+
+ /**
+ * Get whether doc count error will be return for individual terms
+ */
+ public boolean showTermDocCountError() {
+ return showTermDocCountError;
+ }
+
+ /**
+ * Set whether doc count error will be return for individual terms
+ */
+ public TermsAggregationBuilder showTermDocCountError(boolean showTermDocCountError) {
+ this.showTermDocCountError = showTermDocCountError;
+ return this;
+ }
+
+ @Override
+ public BucketCardinality bucketCardinality() {
+ return BucketCardinality.MANY;
+ }
+
+ @Override
+ protected ValuesSourceAggregatorFactory innerBuild(
+ QueryShardContext queryShardContext,
+ ValuesSourceConfig config,
+ AggregatorFactory parent,
+ AggregatorFactories.Builder subFactoriesBuilder
+ ) throws IOException {
+ return new TermsAggregatorFactory(
+ name,
+ config,
+ order,
+ includeExclude,
+ executionHint,
+ collectMode,
+ bucketCountThresholds,
+ showTermDocCountError,
+ queryShardContext,
+ parent,
+ subFactoriesBuilder,
+ metadata
+ );
+ }
+
+ @Override
+ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+ bucketCountThresholds.toXContent(builder, params);
+ builder.field(SHOW_TERM_DOC_COUNT_ERROR.getPreferredName(), showTermDocCountError);
+ if (executionHint != null) {
+ builder.field(TermsAggregationBuilder.EXECUTION_HINT_FIELD_NAME.getPreferredName(), executionHint);
+ }
+ builder.field(ORDER_FIELD.getPreferredName());
+ order.toXContent(builder, params);
+ if (collectMode != null) {
+ builder.field(SubAggCollectionMode.KEY.getPreferredName(), collectMode.parseField().getPreferredName());
+ }
+ if (includeExclude != null) {
+ includeExclude.toXContent(builder, params);
+ }
+ return builder;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ super.hashCode(),
+ bucketCountThresholds,
+ collectMode,
+ executionHint,
+ includeExclude,
+ order,
+ showTermDocCountError
+ );
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+ TermsAggregationBuilder other = (TermsAggregationBuilder) obj;
+ return Objects.equals(bucketCountThresholds, other.bucketCountThresholds)
+ && Objects.equals(collectMode, other.collectMode)
+ && Objects.equals(executionHint, other.executionHint)
+ && Objects.equals(includeExclude, other.includeExclude)
+ && Objects.equals(order, other.order)
+ && Objects.equals(showTermDocCountError, other.showTermDocCountError);
+ }
+
+ @Override
+ public String getType() {
+ return NAME;
+ }
+
+ @Override
+ protected AggregationBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+ return super.doRewrite(queryShardContext);
+ }
+
+ @Override
+ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
+ return REGISTRY_KEY;
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: TermsAggregator (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | TermsAggregator |
+
+
+ 50%
+
+
+ (2/4)
+
+ |
+
+
+ 37.9%
+
+
+ (11/29)
+
+ |
+
+
+ | TermsAggregator$BucketCountThresholds |
+
+
+ 56.2%
+
+
+ (9/16)
+
+ |
+
+
+ 36.2%
+
+
+ (17/47)
+
+ |
+
+
+ | TermsAggregator$CoordinatorBucketCountThresholds |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ | Total |
+
+
+ 47.8%
+
+
+ (11/23)
+
+ |
+
+
+ 35.4%
+
+
+ (28/79)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.OpenSearchException;
+ import org.opensearch.common.annotation.PublicApi;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.common.io.stream.Writeable;
+ import org.opensearch.core.xcontent.ToXContentFragment;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.AggregationExecutionException;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.InternalOrder.Aggregation;
+ import org.opensearch.search.aggregations.InternalOrder.CompoundOrder;
+ import org.opensearch.search.aggregations.bucket.DeferableBucketAggregator;
+ import org.opensearch.search.aggregations.bucket.nested.NestedAggregator;
+ import org.opensearch.search.aggregations.support.AggregationPath;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.util.Comparator;
+ import java.util.HashSet;
+ import java.util.Map;
+ import java.util.Objects;
+ import java.util.Set;
+
+ /**
+ * Base aggregator class to aggregate documents by terms
+ *
+ * @opensearch.internal
+ */
+ public abstract class TermsAggregator extends DeferableBucketAggregator {
+
+ /**
+ * Bucket count thresholds
+ *
+ * @opensearch.api
+ */
+ @PublicApi(since = "1.0.0")
+ public static class BucketCountThresholds implements Writeable, ToXContentFragment {
+ private long minDocCount;
+ private long shardMinDocCount;
+ private int requiredSize;
+ private int shardSize;
+
+ public BucketCountThresholds(long minDocCount, long shardMinDocCount, int requiredSize, int shardSize) {
+ this.minDocCount = minDocCount;
+ this.shardMinDocCount = shardMinDocCount;
+ this.requiredSize = requiredSize;
+ this.shardSize = shardSize;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public BucketCountThresholds(StreamInput in) throws IOException {
+ requiredSize = in.readInt();
+ shardSize = in.readInt();
+ minDocCount = in.readLong();
+ shardMinDocCount = in.readLong();
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeInt(requiredSize);
+ out.writeInt(shardSize);
+ out.writeLong(minDocCount);
+ out.writeLong(shardMinDocCount);
+ }
+
+ public BucketCountThresholds(BucketCountThresholds bucketCountThresholds) {
+ this(
+ bucketCountThresholds.minDocCount,
+ bucketCountThresholds.shardMinDocCount,
+ bucketCountThresholds.requiredSize,
+ bucketCountThresholds.shardSize
+ );
+ }
+
+ public void ensureValidity() {
+
+ // shard_size cannot be smaller than size as we need to at least fetch <size> entries from every shards in order to return
+ // <size>
+ if (shardSize < requiredSize) {
+ setShardSize(requiredSize);
+ }
+
+ // shard_min_doc_count should not be larger than min_doc_count because this can cause buckets to be removed that would match
+ // the min_doc_count criteria
+ if (shardMinDocCount > minDocCount) {
+ setShardMinDocCount(minDocCount);
+ }
+
+ if (requiredSize <= 0 || shardSize <= 0) {
+ throw new OpenSearchException("parameters [required_size] and [shard_size] must be >0 in terms aggregation.");
+ }
+
+ if (minDocCount < 0 || shardMinDocCount < 0) {
+ throw new OpenSearchException("parameter [min_doc_count] and [shardMinDocCount] must be >=0 in terms aggregation.");
+ }
+ }
+
+ public long getShardMinDocCount() {
+ return shardMinDocCount;
+ }
+
+ public void setShardMinDocCount(long shardMinDocCount) {
+ this.shardMinDocCount = shardMinDocCount;
+ }
+
+ public long getMinDocCount() {
+ return minDocCount;
+ }
+
+ public void setMinDocCount(long minDocCount) {
+ this.minDocCount = minDocCount;
+ }
+
+ public int getRequiredSize() {
+ return requiredSize;
+ }
+
+ public void setRequiredSize(int requiredSize) {
+ this.requiredSize = requiredSize;
+ }
+
+ public int getShardSize() {
+ return shardSize;
+ }
+
+ public void setShardSize(int shardSize) {
+ this.shardSize = shardSize;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.field(TermsAggregationBuilder.REQUIRED_SIZE_FIELD_NAME.getPreferredName(), requiredSize);
+ if (shardSize != -1) {
+ builder.field(TermsAggregationBuilder.SHARD_SIZE_FIELD_NAME.getPreferredName(), shardSize);
+ }
+ builder.field(TermsAggregationBuilder.MIN_DOC_COUNT_FIELD_NAME.getPreferredName(), minDocCount);
+ builder.field(TermsAggregationBuilder.SHARD_MIN_DOC_COUNT_FIELD_NAME.getPreferredName(), shardMinDocCount);
+ return builder;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(requiredSize, shardSize, minDocCount, shardMinDocCount);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ BucketCountThresholds other = (BucketCountThresholds) obj;
+ return Objects.equals(requiredSize, other.requiredSize)
+ && Objects.equals(shardSize, other.shardSize)
+ && Objects.equals(minDocCount, other.minDocCount)
+ && Objects.equals(shardMinDocCount, other.shardMinDocCount);
+ }
+ }
+
+ /**
+ * BucketCountThresholds type that throws an exception when shardMinDocCount or shardSize are accessed. This is used for
+ * deserialization on the coordinator during reduce as shardMinDocCount and shardSize should not be accessed this way on the
+ * coordinator.
+ *
+ * @opensearch.internal
+ */
+ public static class CoordinatorBucketCountThresholds extends BucketCountThresholds {
+
+ public CoordinatorBucketCountThresholds(long minDocCount, long shardMinDocCount, int requiredSize, int shardSize) {
+ super(minDocCount, shardMinDocCount, requiredSize, shardSize);
+ }
+
+ @Override
+ public long getShardMinDocCount() {
+ throw new AggregationExecutionException("shard_min_doc_count should not be accessed via CoordinatorBucketCountThresholds");
+ }
+
+ @Override
+ public int getShardSize() {
+ throw new AggregationExecutionException("shard_size should not be accessed via CoordinatorBucketCountThresholds");
+ }
+ }
+
+ protected final DocValueFormat format;
+ protected final BucketCountThresholds bucketCountThresholds;
+ protected final BucketOrder order;
+ protected final Comparator<InternalTerms.Bucket<?>> partiallyBuiltBucketComparator;
+ protected final Set<Aggregator> aggsUsedForSorting = new HashSet<>();
+ protected final SubAggCollectionMode collectMode;
+
+ public TermsAggregator(
+ String name,
+ AggregatorFactories factories,
+ SearchContext context,
+ Aggregator parent,
+ BucketCountThresholds bucketCountThresholds,
+ BucketOrder order,
+ DocValueFormat format,
+ SubAggCollectionMode collectMode,
+ Map<String, Object> metadata
+ ) throws IOException {
+ super(name, factories, context, parent, metadata);
+ this.bucketCountThresholds = bucketCountThresholds;
+ this.order = order;
+ partiallyBuiltBucketComparator = order == null ? null : order.partiallyBuiltBucketComparator(b -> b.bucketOrd, this);
+ this.format = format;
+ if (subAggsNeedScore() && descendsFromNestedAggregator(parent)) {
+ /*
+ Force the execution to depth_first because we need to access the score of
+ nested documents in a sub-aggregation and we are not able to generate this score
+ while replaying deferred documents.
+ */
+ this.collectMode = SubAggCollectionMode.DEPTH_FIRST;
+ } else {
+ this.collectMode = collectMode;
+ }
+ // Don't defer any child agg if we are dependent on it for pruning results
+ if (order instanceof Aggregation) {
+ AggregationPath path = ((Aggregation) order).path();
+ aggsUsedForSorting.add(path.resolveTopmostAggregator(this));
+ } else if (order instanceof CompoundOrder) {
+ CompoundOrder compoundOrder = (CompoundOrder) order;
+ for (BucketOrder orderElement : compoundOrder.orderElements()) {
+ if (orderElement instanceof Aggregation) {
+ AggregationPath path = ((Aggregation) orderElement).path();
+ aggsUsedForSorting.add(path.resolveTopmostAggregator(this));
+ }
+ }
+ }
+ }
+
+ static boolean descendsFromNestedAggregator(Aggregator parent) {
+ while (parent != null) {
+ if (parent.getClass() == NestedAggregator.class) {
+ return true;
+ }
+ parent = parent.parent();
+ }
+ return false;
+ }
+
+ private boolean subAggsNeedScore() {
+ for (Aggregator subAgg : subAggregators) {
+ if (subAgg.scoreMode().needsScores()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ protected boolean shouldDefer(Aggregator aggregator) {
+ return collectMode == SubAggCollectionMode.BREADTH_FIRST && !aggsUsedForSorting.contains(aggregator);
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: TermsAggregatorFactory (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | TermsAggregatorFactory |
+
+
+ 70%
+
+
+ (7/10)
+
+ |
+
+
+ 59.6%
+
+
+ (28/47)
+
+ |
+
+
+ | TermsAggregatorFactory$1 |
+
+
+ 100%
+
+
+ (2/2)
+
+ |
+
+
+ 78.6%
+
+
+ (11/14)
+
+ |
+
+
+ | TermsAggregatorFactory$2 |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 5.3%
+
+
+ (1/19)
+
+ |
+
+
+ | TermsAggregatorFactory$3 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ | TermsAggregatorFactory$ExecutionMode |
+
+
+ 100%
+
+
+ (5/5)
+
+ |
+
+
+ 80%
+
+
+ (8/10)
+
+ |
+
+
+ | TermsAggregatorFactory$ExecutionMode$1 |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 14.3%
+
+
+ (1/7)
+
+ |
+
+
+ | TermsAggregatorFactory$ExecutionMode$2 |
+
+
+ 75%
+
+
+ (3/4)
+
+ |
+
+
+ 60.9%
+
+
+ (14/23)
+
+ |
+
+
+ | Total |
+
+
+ 67.9%
+
+
+ (19/28)
+
+ |
+
+
+ 51.6%
+
+
+ (63/122)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.search.IndexSearcher;
+ import org.opensearch.core.ParseField;
+ import org.opensearch.index.query.QueryShardContext;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.AggregationExecutionException;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.AggregatorFactory;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.InternalOrder;
+ import org.opensearch.search.aggregations.InternalOrder.CompoundOrder;
+ import org.opensearch.search.aggregations.NonCollectingAggregator;
+ import org.opensearch.search.aggregations.bucket.BucketUtils;
+ import org.opensearch.search.aggregations.bucket.terms.NumericTermsAggregator.ResultStrategy;
+ import org.opensearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
+ import org.opensearch.search.aggregations.support.CoreValuesSourceType;
+ import org.opensearch.search.aggregations.support.ValuesSource;
+ import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+ import org.opensearch.search.aggregations.support.ValuesSourceConfig;
+ import org.opensearch.search.aggregations.support.ValuesSourceRegistry;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.util.Arrays;
+ import java.util.Map;
+ import java.util.function.Function;
+
+ /**
+ * Aggregation Factory for terms agg
+ *
+ * @opensearch.internal
+ */
+ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory {
+ static Boolean REMAP_GLOBAL_ORDS, COLLECT_SEGMENT_ORDS;
+
+ static void registerAggregators(ValuesSourceRegistry.Builder builder) {
+ builder.register(
+ TermsAggregationBuilder.REGISTRY_KEY,
+ Arrays.asList(CoreValuesSourceType.BYTES, CoreValuesSourceType.IP),
+ TermsAggregatorFactory.bytesSupplier(),
+ true
+ );
+
+ builder.register(
+ TermsAggregationBuilder.REGISTRY_KEY,
+ Arrays.asList(CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN, CoreValuesSourceType.NUMERIC),
+ TermsAggregatorFactory.numericSupplier(),
+ true
+ );
+ }
+
+ /**
+ * This supplier is used for all the field types that should be aggregated as bytes/strings,
+ * including those that need global ordinals
+ */
+ private static TermsAggregatorSupplier bytesSupplier() {
+ return new TermsAggregatorSupplier() {
+ @Override
+ public Aggregator build(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ BucketOrder order,
+ DocValueFormat format,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ IncludeExclude includeExclude,
+ String executionHint,
+ SearchContext context,
+ Aggregator parent,
+ SubAggCollectionMode subAggCollectMode,
+ boolean showTermDocCountError,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+ ExecutionMode execution = null;
+ if (executionHint != null) {
+ execution = ExecutionMode.fromString(executionHint);
+ }
+ // In some cases, using ordinals is just not supported: override it
+ if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals == false) {
+ execution = ExecutionMode.MAP;
+ }
+ if (execution == null) {
+ execution = ExecutionMode.GLOBAL_ORDINALS;
+ }
+ final long maxOrd = execution == ExecutionMode.GLOBAL_ORDINALS ? getMaxOrd(valuesSource, context.searcher()) : -1;
+ if (subAggCollectMode == null) {
+ subAggCollectMode = pickSubAggColectMode(factories, bucketCountThresholds.getShardSize(), maxOrd);
+ }
+
+ if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) {
+ // TODO this exception message is not really accurate for the string case. It's really disallowing regex + formatter
+ throw new AggregationExecutionException(
+ "Aggregation ["
+ + name
+ + "] cannot support regular expression style "
+ + "include/exclude settings as they can only be applied to string fields. Use an array of values for "
+ + "include/exclude clauses"
+ );
+ }
+
+ // TODO: [Zach] we might want refactor and remove ExecutionMode#create(), moving that logic outside the enum
+ return execution.create(
+ name,
+ factories,
+ valuesSource,
+ order,
+ format,
+ bucketCountThresholds,
+ includeExclude,
+ context,
+ parent,
+ subAggCollectMode,
+ showTermDocCountError,
+ cardinality,
+ metadata
+ );
+
+ }
+ };
+ }
+
+ /**
+ * This supplier is used for all fields that expect to be aggregated as a numeric value.
+ * This includes floating points, and formatted types that use numerics internally for storage (date, boolean, etc)
+ */
+ private static TermsAggregatorSupplier numericSupplier() {
+ return new TermsAggregatorSupplier() {
+ @Override
+ public Aggregator build(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ BucketOrder order,
+ DocValueFormat format,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ IncludeExclude includeExclude,
+ String executionHint,
+ SearchContext context,
+ Aggregator parent,
+ SubAggCollectionMode subAggCollectMode,
+ boolean showTermDocCountError,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+
+ if ((includeExclude != null) && (includeExclude.isRegexBased())) {
+ throw new AggregationExecutionException(
+ "Aggregation ["
+ + name
+ + "] cannot support regular expression style "
+ + "include/exclude settings as they can only be applied to string fields. Use an array of numeric values for "
+ + "include/exclude clauses used to filter numeric fields"
+ );
+ }
+
+ if (subAggCollectMode == null) {
+ subAggCollectMode = pickSubAggColectMode(factories, bucketCountThresholds.getShardSize(), -1);
+ }
+
+ ValuesSource.Numeric numericValuesSource = (ValuesSource.Numeric) valuesSource;
+ IncludeExclude.LongFilter longFilter = null;
+ Function<NumericTermsAggregator, ResultStrategy<?, ?>> resultStrategy;
+ if (numericValuesSource.isFloatingPoint()) {
+ if (includeExclude != null) {
+ longFilter = includeExclude.convertToDoubleFilter();
+ }
+ resultStrategy = agg -> agg.new DoubleTermsResults(showTermDocCountError);
+ } else if (numericValuesSource.isBigInteger()) {
+ if (includeExclude != null) {
+ longFilter = includeExclude.convertToDoubleFilter();
+ }
+ resultStrategy = agg -> agg.new UnsignedLongTermsResults(showTermDocCountError);
+ } else {
+ if (includeExclude != null) {
+ longFilter = includeExclude.convertToLongFilter(format);
+ }
+ resultStrategy = agg -> agg.new LongTermsResults(showTermDocCountError);
+ }
+ return new NumericTermsAggregator(
+ name,
+ factories,
+ resultStrategy,
+ numericValuesSource,
+ format,
+ order,
+ bucketCountThresholds,
+ context,
+ parent,
+ subAggCollectMode,
+ longFilter,
+ cardinality,
+ metadata
+ );
+ }
+ };
+ }
+
+ private final BucketOrder order;
+ private final IncludeExclude includeExclude;
+ private final String executionHint;
+ private final SubAggCollectionMode collectMode;
+ private final TermsAggregator.BucketCountThresholds bucketCountThresholds;
+ private final boolean showTermDocCountError;
+
+ TermsAggregatorFactory(
+ String name,
+ ValuesSourceConfig config,
+ BucketOrder order,
+ IncludeExclude includeExclude,
+ String executionHint,
+ SubAggCollectionMode collectMode,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ boolean showTermDocCountError,
+ QueryShardContext queryShardContext,
+ AggregatorFactory parent,
+ AggregatorFactories.Builder subFactoriesBuilder,
+ Map<String, Object> metadata
+ ) throws IOException {
+ super(name, config, queryShardContext, parent, subFactoriesBuilder, metadata);
+ this.order = order;
+ this.includeExclude = includeExclude;
+ this.executionHint = executionHint;
+ this.collectMode = collectMode;
+ this.bucketCountThresholds = bucketCountThresholds;
+ this.showTermDocCountError = showTermDocCountError;
+ }
+
+ @Override
+ protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, Map<String, Object> metadata) throws IOException {
+ final InternalAggregation aggregation = new UnmappedTerms(name, order, bucketCountThresholds, metadata);
+ Aggregator agg = new NonCollectingAggregator(name, searchContext, parent, factories, metadata) {
+ @Override
+ public InternalAggregation buildEmptyAggregation() {
+ return aggregation;
+ }
+ };
+ // even in the case of an unmapped aggregator, validate the order
+ order.validate(agg);
+ return agg;
+ }
+
+ private static boolean isAggregationSort(BucketOrder order) {
+ if (order instanceof InternalOrder.Aggregation) {
+ return true;
+ } else if (order instanceof InternalOrder.CompoundOrder) {
+ InternalOrder.CompoundOrder compoundOrder = (CompoundOrder) order;
+ return compoundOrder.orderElements().stream().anyMatch(TermsAggregatorFactory::isAggregationSort);
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ protected Aggregator doCreateInternal(
+ SearchContext searchContext,
+ Aggregator parent,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+ TermsAggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry()
+ .getAggregator(TermsAggregationBuilder.REGISTRY_KEY, config);
+ BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(this.bucketCountThresholds);
+ if (InternalOrder.isKeyOrder(order) == false
+ && bucketCountThresholds.getShardSize() == TermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) {
+ // The user has not made a shardSize selection. Use default
+ // heuristic to avoid any wrong-ranking caused by distributed
+ // counting
+ bucketCountThresholds.setShardSize(BucketUtils.suggestShardSideQueueSize(bucketCountThresholds.getRequiredSize()));
+ }
+ bucketCountThresholds.ensureValidity();
+
+ return aggregatorSupplier.build(
+ name,
+ factories,
+ config.getValuesSource(),
+ order,
+ config.format(),
+ bucketCountThresholds,
+ includeExclude,
+ executionHint,
+ searchContext,
+ parent,
+ collectMode,
+ showTermDocCountError,
+ cardinality,
+ metadata
+ );
+ }
+
+ /**
+ * Pick a {@link SubAggCollectionMode} based on heuristics about what
+ * we're collecting.
+ */
+ static SubAggCollectionMode pickSubAggColectMode(AggregatorFactories factories, int expectedSize, long maxOrd) {
+ if (factories.countAggregators() == 0) {
+ // Without sub-aggregations we pretty much ignore this field value so just pick something
+ return SubAggCollectionMode.DEPTH_FIRST;
+ }
+ if (expectedSize == Integer.MAX_VALUE) {
+ // We expect to return all buckets so delaying them won't save any time
+ return SubAggCollectionMode.DEPTH_FIRST;
+ }
+ if (maxOrd == -1 || maxOrd > expectedSize) {
+ /*
+ * We either don't know how many buckets we expect there to be
+ * (maxOrd == -1) or we expect there to be more buckets than
+ * we will collect from this shard. So delaying collection of
+ * the sub-buckets *should* save time.
+ */
+ return SubAggCollectionMode.BREADTH_FIRST;
+ }
+ // We expect to collect so many buckets that we may as well collect them all.
+ return SubAggCollectionMode.DEPTH_FIRST;
+ }
+
+ /**
+ * Get the maximum global ordinal value for the provided {@link ValuesSource} or -1
+ * if the values source is not an instance of {@link ValuesSource.Bytes.WithOrdinals}.
+ */
+ private static long getMaxOrd(ValuesSource source, IndexSearcher searcher) throws IOException {
+ if (source instanceof ValuesSource.Bytes.WithOrdinals) {
+ ValuesSource.Bytes.WithOrdinals valueSourceWithOrdinals = (ValuesSource.Bytes.WithOrdinals) source;
+ return valueSourceWithOrdinals.globalMaxOrd(searcher);
+ } else {
+ return -1;
+ }
+ }
+
+ /**
+ * The execution mode for the terms agg
+ *
+ * @opensearch.internal
+ */
+ public enum ExecutionMode {
+
+ MAP(new ParseField("map")) {
+
+ @Override
+ Aggregator create(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ BucketOrder order,
+ DocValueFormat format,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ IncludeExclude includeExclude,
+ SearchContext context,
+ Aggregator parent,
+ SubAggCollectionMode subAggCollectMode,
+ boolean showTermDocCountError,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+ int maxRegexLength = context.getQueryShardContext().getIndexSettings().getMaxRegexLength();
+ final IncludeExclude.StringFilter filter = includeExclude == null
+ ? null
+ : includeExclude.convertToStringFilter(format, maxRegexLength);
+ return new MapStringTermsAggregator(
+ name,
+ factories,
+ new MapStringTermsAggregator.ValuesSourceCollectorSource(valuesSource),
+ a -> a.new StandardTermsResults(valuesSource),
+ order,
+ format,
+ bucketCountThresholds,
+ filter,
+ context,
+ parent,
+ subAggCollectMode,
+ showTermDocCountError,
+ cardinality,
+ metadata
+ );
+ }
+ },
+ GLOBAL_ORDINALS(new ParseField("global_ordinals")) {
+
+ @Override
+ Aggregator create(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ BucketOrder order,
+ DocValueFormat format,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ IncludeExclude includeExclude,
+ SearchContext context,
+ Aggregator parent,
+ SubAggCollectionMode subAggCollectMode,
+ boolean showTermDocCountError,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+
+ final long maxOrd = getMaxOrd(valuesSource, context.searcher());
+ assert maxOrd != -1;
+ final double ratio = maxOrd / ((double) context.searcher().getIndexReader().numDocs());
+
+ assert valuesSource instanceof ValuesSource.Bytes.WithOrdinals;
+ ValuesSource.Bytes.WithOrdinals ordinalsValuesSource = (ValuesSource.Bytes.WithOrdinals) valuesSource;
+
+ if (factories == AggregatorFactories.EMPTY
+ && includeExclude == null
+ && cardinality == CardinalityUpperBound.ONE
+ && ordinalsValuesSource.supportsGlobalOrdinalsMapping()
+ &&
+ // we use the static COLLECT_SEGMENT_ORDS to allow tests to force specific optimizations
+ (COLLECT_SEGMENT_ORDS != null ? COLLECT_SEGMENT_ORDS.booleanValue() : ratio <= 0.5 && maxOrd <= 2048)) {
+ /*
+ * We can use the low cardinality execution mode iff this aggregator:
+ * - has no sub-aggregator AND
+ * - collects from a single bucket AND
+ * - has a values source that can map from segment to global ordinals
+ * - At least we reduce the number of global ordinals look-ups by half (ration <= 0.5) AND
+ * - the maximum global ordinal is less than 2048 (LOW_CARDINALITY has additional memory usage,
+ * which directly linked to maxOrd, so we need to limit).
+ */
+ return new GlobalOrdinalsStringTermsAggregator.LowCardinality(
+ name,
+ factories,
+ a -> a.new StandardTermsResults(),
+ ordinalsValuesSource,
+ order,
+ format,
+ bucketCountThresholds,
+ context,
+ parent,
+ false,
+ subAggCollectMode,
+ showTermDocCountError,
+ metadata
+ );
+
+ }
+ int maxRegexLength = context.getQueryShardContext().getIndexSettings().getMaxRegexLength();
+ final IncludeExclude.OrdinalsFilter filter = includeExclude == null
+ ? null
+ : includeExclude.convertToOrdinalsFilter(format, maxRegexLength);
+ boolean remapGlobalOrds;
+ if (cardinality == CardinalityUpperBound.ONE && REMAP_GLOBAL_ORDS != null) {
+ /*
+ * We use REMAP_GLOBAL_ORDS to allow tests to force
+ * specific optimizations but this particular one
+ * is only possible if we're collecting from a single
+ * bucket.
+ */
+ remapGlobalOrds = REMAP_GLOBAL_ORDS.booleanValue();
+ } else {
+ remapGlobalOrds = true;
+ if (includeExclude == null
+ && cardinality == CardinalityUpperBound.ONE
+ && (factories == AggregatorFactories.EMPTY
+ || (isAggregationSort(order) == false && subAggCollectMode == SubAggCollectionMode.BREADTH_FIRST))) {
+ /*
+ * We don't need to remap global ords iff this aggregator:
+ * - has no include/exclude rules AND
+ * - only collects from a single bucket AND
+ * - has no sub-aggregator or only sub-aggregator that can be deferred
+ * ({@link SubAggCollectionMode#BREADTH_FIRST}).
+ */
+ remapGlobalOrds = false;
+ }
+ }
+ return new GlobalOrdinalsStringTermsAggregator(
+ name,
+ factories,
+ a -> a.new StandardTermsResults(),
+ ordinalsValuesSource,
+ order,
+ format,
+ bucketCountThresholds,
+ filter,
+ context,
+ parent,
+ remapGlobalOrds,
+ subAggCollectMode,
+ showTermDocCountError,
+ cardinality,
+ metadata
+ );
+ }
+ };
+
+ public static ExecutionMode fromString(String value) {
+ switch (value) {
+ case "global_ordinals":
+ return GLOBAL_ORDINALS;
+ case "map":
+ return MAP;
+ default:
+ throw new IllegalArgumentException("Unknown `execution_hint`: [" + value + "], expected any of [map, global_ordinals]");
+ }
+ }
+
+ private final ParseField parseField;
+
+ ExecutionMode(ParseField parseField) {
+ this.parseField = parseField;
+ }
+
+ abstract Aggregator create(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ BucketOrder order,
+ DocValueFormat format,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ IncludeExclude includeExclude,
+ SearchContext context,
+ Aggregator parent,
+ SubAggCollectionMode subAggCollectMode,
+ boolean showTermDocCountError,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException;
+
+ @Override
+ public String toString() {
+ return parseField.getPreferredName();
+ }
+ }
+
+ @Override
+ protected boolean supportsConcurrentSegmentSearch() {
+ return true;
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: UnmappedRareTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | UnmappedRareTerms |
+
+
+ 0%
+
+
+ (0/15)
+
+ |
+
+
+ 0%
+
+
+ (0/15)
+
+ |
+
+
+ | UnmappedRareTerms$Bucket |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/16)
+
+ |
+
+
+ 0%
+
+
+ (0/16)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.common.util.SetBackedScalingCuckooFilter;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.InternalAggregations;
+
+ import java.io.IOException;
+ import java.util.Collections;
+ import java.util.List;
+ import java.util.Map;
+
+ import static java.util.Collections.emptyList;
+
+ /**
+ * Result of the RareTerms aggregation when the field is unmapped.
+ *
+ * @opensearch.internal
+ */
+ public class UnmappedRareTerms extends InternalRareTerms<UnmappedRareTerms, UnmappedRareTerms.Bucket> {
+ public static final String NAME = "umrareterms";
+
+ /**
+ * Bucket for unmapped rare values
+ *
+ * @opensearch.internal
+ */
+ protected abstract static class Bucket extends InternalRareTerms.Bucket<Bucket> {
+ private Bucket(long docCount, InternalAggregations aggregations, DocValueFormat formatter) {
+ super(docCount, aggregations, formatter);
+ }
+ }
+
+ UnmappedRareTerms(String name, Map<String, Object> metadata) {
+ super(name, LongRareTermsAggregator.ORDER, 0, metadata);
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public UnmappedRareTerms(StreamInput in) throws IOException {
+ super(in);
+ }
+
+ @Override
+ protected void writeTermTypeInfoTo(StreamOutput out) throws IOException {
+ // Nothing to write
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public String getType() {
+ return StringTerms.NAME;
+ }
+
+ @Override
+ public UnmappedRareTerms create(List<UnmappedRareTerms.Bucket> buckets) {
+ return new UnmappedRareTerms(name, metadata);
+ }
+
+ @Override
+ public UnmappedRareTerms.Bucket createBucket(InternalAggregations aggregations, UnmappedRareTerms.Bucket prototype) {
+ throw new UnsupportedOperationException("not supported for UnmappedRareTerms");
+ }
+
+ @Override
+ UnmappedRareTerms.Bucket createBucket(long docCount, InternalAggregations aggs, Bucket prototype) {
+ throw new UnsupportedOperationException("not supported for UnmappedRareTerms");
+ }
+
+ @Override
+ protected UnmappedRareTerms createWithFilter(String name, List<UnmappedRareTerms.Bucket> buckets, SetBackedScalingCuckooFilter filter) {
+ throw new UnsupportedOperationException("not supported for UnmappedRareTerms");
+ }
+
+ @Override
+ public InternalAggregation reduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+ return new UnmappedRareTerms(name, metadata);
+ }
+
+ @Override
+ public boolean isMapped() {
+ return false;
+ }
+
+ @Override
+ public final XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+ return doXContentCommon(builder, params, Collections.emptyList());
+ }
+
+ @Override
+ public List<UnmappedRareTerms.Bucket> getBuckets() {
+ return emptyList();
+ }
+
+ @Override
+ public UnmappedRareTerms.Bucket getBucketByKey(String term) {
+ return null;
+ }
+
+ @Override
+ protected UnmappedRareTerms.Bucket[] createBucketsArray(int size) {
+ return new UnmappedRareTerms.Bucket[size];
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: UnmappedSignificantTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | UnmappedSignificantTerms |
+
+
+ 0%
+
+
+ (0/19)
+
+ |
+
+
+ 0%
+
+
+ (0/20)
+
+ |
+
+
+ | UnmappedSignificantTerms$Bucket |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/20)
+
+ |
+
+
+ 0%
+
+
+ (0/21)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.util.BytesRef;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.InternalAggregations;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+
+ import java.io.IOException;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+
+ import static java.util.Collections.emptyIterator;
+ import static java.util.Collections.emptyList;
+
+ /**
+ * Result of the running the significant terms aggregation on an unmapped field.
+ *
+ * @opensearch.internal
+ */
+ public class UnmappedSignificantTerms extends InternalSignificantTerms<UnmappedSignificantTerms, UnmappedSignificantTerms.Bucket> {
+
+ public static final String NAME = "umsigterms";
+
+ /**
+ * Concrete type that can't be built because Java needs a concrete type so {@link InternalTerms.Bucket} can have a self type but
+ * {@linkplain UnmappedTerms} doesn't ever need to build it because it never returns any buckets.
+ *
+ * @opensearch.internal
+ */
+ protected abstract static class Bucket extends InternalSignificantTerms.Bucket<Bucket> {
+ private Bucket(
+ BytesRef term,
+ long subsetDf,
+ long subsetSize,
+ long supersetDf,
+ long supersetSize,
+ InternalAggregations aggregations,
+ DocValueFormat format
+ ) {
+ super(subsetDf, subsetSize, supersetDf, supersetSize, aggregations, format);
+ }
+ }
+
+ public UnmappedSignificantTerms(
+ String name,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ Map<String, Object> metadata
+ ) {
+ super(name, bucketCountThresholds, metadata);
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public UnmappedSignificantTerms(StreamInput in) throws IOException {
+ super(in);
+ }
+
+ @Override
+ protected void writeTermTypeInfoTo(StreamOutput out) throws IOException {
+ // Nothing to write
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public String getType() {
+ return SignificantStringTerms.NAME;
+ }
+
+ @Override
+ public UnmappedSignificantTerms create(List<Bucket> buckets) {
+ return new UnmappedSignificantTerms(name, bucketCountThresholds, metadata);
+ }
+
+ @Override
+ public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) {
+ throw new UnsupportedOperationException("not supported for UnmappedSignificantTerms");
+ }
+
+ @Override
+ protected UnmappedSignificantTerms create(long subsetSize, long supersetSize, List<Bucket> buckets) {
+ throw new UnsupportedOperationException("not supported for UnmappedSignificantTerms");
+ }
+
+ @Override
+ Bucket createBucket(
+ long subsetDf,
+ long subsetSize,
+ long supersetDf,
+ long supersetSize,
+ InternalAggregations aggregations,
+ Bucket prototype
+ ) {
+ throw new UnsupportedOperationException("not supported for UnmappedSignificantTerms");
+ }
+
+ @Override
+ public InternalAggregation reduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+ return new UnmappedSignificantTerms(name, bucketCountThresholds, metadata);
+ }
+
+ @Override
+ public boolean isMapped() {
+ return false;
+ }
+
+ @Override
+ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+ builder.startArray(CommonFields.BUCKETS.getPreferredName()).endArray();
+ return builder;
+ }
+
+ @Override
+ protected Bucket[] createBucketsArray(int size) {
+ return new Bucket[size];
+ }
+
+ @Override
+ public Iterator<SignificantTerms.Bucket> iterator() {
+ return emptyIterator();
+ }
+
+ @Override
+ public List<Bucket> getBuckets() {
+ return emptyList();
+ }
+
+ @Override
+ public SignificantTerms.Bucket getBucketByKey(String term) {
+ return null;
+ }
+
+ @Override
+ protected SignificanceHeuristic getSignificanceHeuristic() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected long getSubsetSize() {
+ return 0;
+ }
+
+ @Override
+ protected long getSupersetSize() {
+ return 0;
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: UnmappedTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | UnmappedTerms |
+
+
+ 0%
+
+
+ (0/19)
+
+ |
+
+
+ 0%
+
+
+ (0/19)
+
+ |
+
+
+ | UnmappedTerms$Bucket |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/20)
+
+ |
+
+
+ 0%
+
+
+ (0/20)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.InternalAggregations;
+
+ import java.io.IOException;
+ import java.util.Collections;
+ import java.util.List;
+ import java.util.Map;
+
+ import static java.util.Collections.emptyList;
+
+ /**
+ * Result of the {@link TermsAggregator} when the field is unmapped.
+ *
+ * @opensearch.internal
+ */
+ public class UnmappedTerms extends InternalTerms<UnmappedTerms, UnmappedTerms.Bucket> {
+ public static final String NAME = "umterms";
+
+ /**
+ * Concrete type that can't be built because Java needs a concrete type so {@link InternalTerms.Bucket} can have a self type but
+ * {@linkplain UnmappedTerms} doesn't ever need to build it because it never returns any buckets.
+ *
+ * @opensearch.internal
+ */
+ protected abstract static class Bucket extends InternalTerms.Bucket<Bucket> {
+ private Bucket(
+ long docCount,
+ InternalAggregations aggregations,
+ boolean showDocCountError,
+ long docCountError,
+ DocValueFormat formatter
+ ) {
+ super(docCount, aggregations, showDocCountError, docCountError, formatter);
+ }
+ }
+
+ public UnmappedTerms(
+ String name,
+ BucketOrder order,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ Map<String, Object> metadata
+ ) {
+ super(name, order, order, bucketCountThresholds, metadata);
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public UnmappedTerms(StreamInput in) throws IOException {
+ super(in);
+ }
+
+ @Override
+ protected void writeTermTypeInfoTo(StreamOutput out) throws IOException {
+ // Nothing to write
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public String getType() {
+ return StringTerms.NAME;
+ }
+
+ @Override
+ public UnmappedTerms create(List<Bucket> buckets) {
+ return new UnmappedTerms(name, order, bucketCountThresholds, metadata);
+ }
+
+ @Override
+ public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) {
+ throw new UnsupportedOperationException("not supported for UnmappedTerms");
+ }
+
+ @Override
+ Bucket createBucket(long docCount, InternalAggregations aggs, long docCountError, Bucket prototype) {
+ throw new UnsupportedOperationException("not supported for UnmappedTerms");
+ }
+
+ @Override
+ protected UnmappedTerms create(String name, List<Bucket> buckets, BucketOrder reduceOrder, long docCountError, long otherDocCount) {
+ throw new UnsupportedOperationException("not supported for UnmappedTerms");
+ }
+
+ @Override
+ public InternalAggregation reduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+ return new UnmappedTerms(name, order, bucketCountThresholds, metadata);
+ }
+
+ @Override
+ public boolean isMapped() {
+ return false;
+ }
+
+ @Override
+ public final XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+ return doXContentCommon(builder, params, 0, 0, Collections.emptyList());
+ }
+
+ @Override
+ protected void setDocCountError(long docCountError) {}
+
+ @Override
+ protected int getShardSize() {
+ return 0;
+ }
+
+ @Override
+ public long getDocCountError() {
+ return 0;
+ }
+
+ @Override
+ public long getSumOfOtherDocCounts() {
+ return 0;
+ }
+
+ @Override
+ public List<Bucket> getBuckets() {
+ return emptyList();
+ }
+
+ @Override
+ public Bucket getBucketByKey(String term) {
+ return null;
+ }
+
+ @Override
+ protected Bucket[] createBucketsArray(int size) {
+ return new Bucket[size];
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: UnsignedLongTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | UnsignedLongTerms |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/44)
+
+ |
+
+
+ | UnsignedLongTerms$Bucket |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/21)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/20)
+
+ |
+
+
+ 0%
+
+
+ (0/65)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.InternalAggregations;
+
+ import java.io.IOException;
+ import java.math.BigInteger;
+ import java.util.ArrayList;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+
+ /**
+ * Result of the {@link TermsAggregator} when the field is some kind of whole number like a integer,
+ * long, unsigned long or a date.
+ *
+ * @opensearch.internal
+ */
+ public class UnsignedLongTerms extends InternalMappedTerms<UnsignedLongTerms, UnsignedLongTerms.Bucket> {
+ public static final String NAME = "ulterms";
+
+ /**
+ * Bucket for long terms
+ *
+ * @opensearch.internal
+ */
+ public static class Bucket extends InternalTerms.Bucket<Bucket> {
+ BigInteger term;
+
+ public Bucket(
+ BigInteger term,
+ long docCount,
+ InternalAggregations aggregations,
+ boolean showDocCountError,
+ long docCountError,
+ DocValueFormat format
+ ) {
+ super(docCount, aggregations, showDocCountError, docCountError, format);
+ this.term = term;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public Bucket(StreamInput in, DocValueFormat format, boolean showDocCountError) throws IOException {
+ super(in, format, showDocCountError);
+ term = in.readBigInteger();
+ }
+
+ @Override
+ protected void writeTermTo(StreamOutput out) throws IOException {
+ out.writeBigInteger(term);
+ }
+
+ @Override
+ public String getKeyAsString() {
+ return format.format(term).toString();
+ }
+
+ @Override
+ public Object getKey() {
+ if (format == DocValueFormat.UNSIGNED_LONG_SHIFTED) {
+ return format.format(term);
+ } else {
+ return term;
+ }
+ }
+
+ @Override
+ public Number getKeyAsNumber() {
+ if (format == DocValueFormat.UNSIGNED_LONG_SHIFTED) {
+ return (Number) format.format(term);
+ } else {
+ return term;
+ }
+ }
+
+ @Override
+ public int compareKey(Bucket other) {
+ return term.compareTo(other.term);
+ }
+
+ @Override
+ protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ if (format == DocValueFormat.UNSIGNED_LONG_SHIFTED) {
+ builder.field(CommonFields.KEY.getPreferredName(), format.format(term));
+ } else {
+ builder.field(CommonFields.KEY.getPreferredName(), term);
+ }
+ if (format != DocValueFormat.RAW && format != DocValueFormat.UNSIGNED_LONG_SHIFTED && format != DocValueFormat.UNSIGNED_LONG) {
+ builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term).toString());
+ }
+ return builder;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj) && Objects.equals(term, ((Bucket) obj).term);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), term);
+ }
+ }
+
+ public UnsignedLongTerms(
+ String name,
+ BucketOrder reduceOrder,
+ BucketOrder order,
+ Map<String, Object> metadata,
+ DocValueFormat format,
+ int shardSize,
+ boolean showTermDocCountError,
+ long otherDocCount,
+ List<Bucket> buckets,
+ long docCountError,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds
+ ) {
+ super(
+ name,
+ reduceOrder,
+ order,
+ metadata,
+ format,
+ shardSize,
+ showTermDocCountError,
+ otherDocCount,
+ buckets,
+ docCountError,
+ bucketCountThresholds
+ );
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public UnsignedLongTerms(StreamInput in) throws IOException {
+ super(in, Bucket::new);
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public UnsignedLongTerms create(List<Bucket> buckets) {
+ return new UnsignedLongTerms(
+ name,
+ reduceOrder,
+ order,
+ metadata,
+ format,
+ shardSize,
+ showTermDocCountError,
+ otherDocCount,
+ buckets,
+ docCountError,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) {
+ return new Bucket(
+ prototype.term,
+ prototype.docCount,
+ aggregations,
+ prototype.showDocCountError,
+ prototype.docCountError,
+ prototype.format
+ );
+ }
+
+ @Override
+ protected UnsignedLongTerms create(String name, List<Bucket> buckets, BucketOrder reduceOrder, long docCountError, long otherDocCount) {
+ return new UnsignedLongTerms(
+ name,
+ reduceOrder,
+ order,
+ getMetadata(),
+ format,
+ shardSize,
+ showTermDocCountError,
+ otherDocCount,
+ buckets,
+ docCountError,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ protected Bucket[] createBucketsArray(int size) {
+ return new Bucket[size];
+ }
+
+ @Override
+ public InternalAggregation reduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+ boolean unsignedLongFormat = false;
+ boolean rawFormat = false;
+ for (InternalAggregation agg : aggregations) {
+ if (agg instanceof DoubleTerms) {
+ return agg.reduce(aggregations, reduceContext);
+ }
+ if (agg instanceof UnsignedLongTerms) {
+ if (((UnsignedLongTerms) agg).format == DocValueFormat.RAW) {
+ rawFormat = true;
+ } else if (((UnsignedLongTerms) agg).format == DocValueFormat.UNSIGNED_LONG_SHIFTED) {
+ unsignedLongFormat = true;
+ } else if (((UnsignedLongTerms) agg).format == DocValueFormat.UNSIGNED_LONG) {
+ unsignedLongFormat = true;
+ }
+ }
+ }
+ if (rawFormat && unsignedLongFormat) { // if we have mixed formats, convert results to double format
+ List<InternalAggregation> newAggs = new ArrayList<>(aggregations.size());
+ for (InternalAggregation agg : aggregations) {
+ if (agg instanceof UnsignedLongTerms) {
+ DoubleTerms dTerms = UnsignedLongTerms.convertUnsignedLongTermsToDouble((UnsignedLongTerms) agg, format);
+ newAggs.add(dTerms);
+ } else {
+ newAggs.add(agg);
+ }
+ }
+ return newAggs.get(0).reduce(newAggs, reduceContext);
+ }
+ return super.reduce(aggregations, reduceContext);
+ }
+
+ @Override
+ Bucket createBucket(long docCount, InternalAggregations aggs, long docCountError, UnsignedLongTerms.Bucket prototype) {
+ return new Bucket(prototype.term, docCount, aggs, prototype.showDocCountError, docCountError, format);
+ }
+
+ /**
+ * Converts a {@link UnsignedLongTerms} into a {@link DoubleTerms}, returning the value of the specified long terms as doubles.
+ */
+ static DoubleTerms convertUnsignedLongTermsToDouble(UnsignedLongTerms unsignedLongTerms, DocValueFormat decimalFormat) {
+ List<UnsignedLongTerms.Bucket> buckets = unsignedLongTerms.getBuckets();
+ List<DoubleTerms.Bucket> newBuckets = new ArrayList<>();
+ for (Terms.Bucket bucket : buckets) {
+ newBuckets.add(
+ new DoubleTerms.Bucket(
+ bucket.getKeyAsNumber().doubleValue(),
+ bucket.getDocCount(),
+ (InternalAggregations) bucket.getAggregations(),
+ unsignedLongTerms.showTermDocCountError,
+ unsignedLongTerms.showTermDocCountError ? bucket.getDocCountError() : 0,
+ decimalFormat
+ )
+ );
+ }
+ return new DoubleTerms(
+ unsignedLongTerms.getName(),
+ unsignedLongTerms.reduceOrder,
+ unsignedLongTerms.order,
+ unsignedLongTerms.metadata,
+ unsignedLongTerms.format,
+ unsignedLongTerms.shardSize,
+ unsignedLongTerms.showTermDocCountError,
+ unsignedLongTerms.otherDocCount,
+ newBuckets,
+ unsignedLongTerms.docCountError,
+ unsignedLongTerms.bucketCountThresholds
+ );
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: RareTermsAggregatorSupplier (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+
+ | RareTermsAggregatorSupplier |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.support.ValuesSource;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.util.Map;
+
+ /**
+ * Aggregator supplier interface for rare_terms agg
+ *
+ * @opensearch.internal
+ */
+ interface RareTermsAggregatorSupplier {
+ Aggregator build(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ DocValueFormat format,
+ int maxDocCount,
+ double precision,
+ IncludeExclude includeExclude,
+ SearchContext context,
+ Aggregator parent,
+ CardinalityUpperBound carinality,
+ Map<String, Object> metadata
+ ) throws IOException;
+ }
+
+
+
+
+
+
Coverage Summary for Class: SignificantTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+
+ | SignificantTerms$Bucket |
+
+
+ | Total |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
+
+ import java.util.List;
+
+ /**
+ * An aggregation that collects significant terms in comparison to a background set.
+ *
+ * @opensearch.internal
+ */
+ public interface SignificantTerms extends MultiBucketsAggregation, Iterable<SignificantTerms.Bucket> {
+
+ /**
+ * Bucket for significant terms
+ *
+ * @opensearch.internal
+ */
+ interface Bucket extends MultiBucketsAggregation.Bucket {
+
+ /**
+ * @return The significant score for the subset
+ */
+ double getSignificanceScore();
+
+ /**
+ * @return The number of docs in the subset containing a particular term.
+ * This number is equal to the document count of the bucket.
+ */
+ long getSubsetDf();
+
+ /**
+ * @return The numbers of docs in the subset (also known as "foreground set").
+ * This number is equal to the document count of the containing aggregation.
+ */
+ long getSubsetSize();
+
+ /**
+ * @return The number of docs in the superset containing a particular term (also
+ * known as the "background count" of the bucket)
+ */
+ long getSupersetDf();
+
+ /**
+ * @return The numbers of docs in the superset (ordinarily the background count
+ * of the containing aggregation).
+ */
+ long getSupersetSize();
+
+ /**
+ * @return The key, expressed as a number
+ */
+ Number getKeyAsNumber();
+ }
+
+ @Override
+ List<? extends Bucket> getBuckets();
+
+ /**
+ * Get the bucket for the given term, or null if there is no such bucket.
+ */
+ Bucket getBucketByKey(String term);
+
+ }
+
+
+
+
+
+
Coverage Summary for Class: TermsAggregatorSupplier (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+
+ | TermsAggregatorSupplier |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.support.ValuesSource;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.util.Map;
+
+ /**
+ * Aggregator supplier interface for terms agg
+ *
+ * @opensearch.internal
+ */
+ interface TermsAggregatorSupplier {
+ Aggregator build(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ BucketOrder order,
+ DocValueFormat format,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ IncludeExclude includeExclude,
+ String executionHint,
+ SearchContext context,
+ Aggregator parent,
+ Aggregator.SubAggCollectionMode subAggCollectMode,
+ boolean showTermDocCountError,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException;
+ }
+
+
+
+
+
+
Coverage Summary for Class: Terms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+
+ | Terms$Bucket |
+
+
+ | Total |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
+
+ import java.util.List;
+
+ /**
+ * A {@code terms} aggregation. Defines multiple bucket, each associated with a unique term for a specific field.
+ * All documents in a bucket has the bucket's term in that field.
+ *
+ * @opensearch.internal
+ */
+ public interface Terms extends MultiBucketsAggregation {
+
+ /**
+ * A bucket that is associated with a single term
+ *
+ * @opensearch.internal
+ */
+ interface Bucket extends MultiBucketsAggregation.Bucket {
+
+ Number getKeyAsNumber();
+
+ long getDocCountError();
+ }
+
+ /**
+ * Return the sorted list of the buckets in this terms aggregation.
+ */
+ @Override
+ List<? extends Bucket> getBuckets();
+
+ /**
+ * Get the bucket for the given term, or null if there is no such bucket.
+ */
+ Bucket getBucketByKey(String term);
+
+ /**
+ * Get an upper bound of the error on document counts in this aggregation.
+ */
+ long getDocCountError();
+
+ /**
+ * Return the sum of the document counts of all buckets that did not make
+ * it to the top buckets.
+ */
+ long getSumOfOtherDocCounts();
+ }
+
+
+
+
+
+
Coverage Summary for Class: RareTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+
+ | RareTerms$Bucket |
+
+
+ | Total |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
+
+ import java.util.List;
+
+ /**
+ * Rare terms interface
+ *
+ * @opensearch.internal
+ */
+ public interface RareTerms extends MultiBucketsAggregation {
+
+ /**
+ * A bucket that is associated with a single term
+ *
+ * @opensearch.internal
+ */
+ interface Bucket extends MultiBucketsAggregation.Bucket {
+
+ Number getKeyAsNumber();
+ }
+
+ /**
+ * Return the sorted list of the buckets in this terms aggregation.
+ */
+ @Override
+ List<? extends Bucket> getBuckets();
+
+ /**
+ * Get the bucket for the given term, or null if there is no such bucket.
+ */
+ Bucket getBucketByKey(String term);
+
+ }
+
+
+
+
+
+
Coverage Summary for Class: SignificantTermsAggregatorSupplier (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+
+ | SignificantTermsAggregatorSupplier |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+ import org.opensearch.search.aggregations.support.ValuesSource;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.util.Map;
+
+ /**
+ * Aggregator supplier interface for significant_terms agg
+ *
+ * @opensearch.internal
+ */
+ interface SignificantTermsAggregatorSupplier {
+ Aggregator build(
+ String name,
+ AggregatorFactories factories,
+ ValuesSource valuesSource,
+ DocValueFormat format,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ IncludeExclude includeExclude,
+ String executionHint,
+ SearchContext context,
+ Aggregator parent,
+ SignificanceHeuristic significanceHeuristic,
+ SignificanceLookup lookup,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException;
+ }
+
+
+
+
+
+
Coverage Summary for Class: BucketSignificancePriorityQueue (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | BucketSignificancePriorityQueue |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.util.PriorityQueue;
+
+ /**
+ * Priority queue for computing bucket significance
+ *
+ * @opensearch.internal
+ */
+ public class BucketSignificancePriorityQueue<B extends SignificantTerms.Bucket> extends PriorityQueue<B> {
+
+ public BucketSignificancePriorityQueue(int size) {
+ super(size);
+ }
+
+ @Override
+ protected boolean lessThan(SignificantTerms.Bucket o1, SignificantTerms.Bucket o2) {
+ return o1.getSignificanceScore() < o2.getSignificanceScore();
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: BytesKeyedBucketOrds (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | BytesKeyedBucketOrds |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ | BytesKeyedBucketOrds$BucketOrdsEnum |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ | BytesKeyedBucketOrds$BucketOrdsEnum$1 |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ | BytesKeyedBucketOrds$FromMany |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ 0%
+
+
+ (0/12)
+
+ |
+
+
+ | BytesKeyedBucketOrds$FromMany$1 |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ | BytesKeyedBucketOrds$FromSingle |
+
+
+ 0%
+
+
+ (0/7)
+
+ |
+
+
+ 0%
+
+
+ (0/9)
+
+ |
+
+
+ | BytesKeyedBucketOrds$FromSingle$1 |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/28)
+
+ |
+
+
+ 0%
+
+
+ (0/38)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.util.BytesRef;
+ import org.opensearch.common.lease.Releasable;
+ import org.opensearch.common.lease.Releasables;
+ import org.opensearch.common.util.BigArrays;
+ import org.opensearch.common.util.BytesRefHash;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+
+ /**
+ * Maps {@link BytesRef} bucket keys to bucket ordinals.
+ *
+ * @opensearch.internal
+ */
+ public abstract class BytesKeyedBucketOrds implements Releasable {
+ /**
+ * Build a {@link LongKeyedBucketOrds}.
+ */
+ public static BytesKeyedBucketOrds build(BigArrays bigArrays, CardinalityUpperBound cardinality) {
+ return cardinality.map(estimate -> estimate < 2 ? new FromSingle(bigArrays) : new FromMany(bigArrays));
+ }
+
+ private BytesKeyedBucketOrds() {}
+
+ /**
+ * Add the {@code owningBucketOrd, value} pair. Return the ord for
+ * their bucket if they have yet to be added, or {@code -1-ord}
+ * if they were already present.
+ */
+ public abstract long add(long owningBucketOrd, BytesRef value);
+
+ /**
+ * Count the buckets in {@code owningBucketOrd}.
+ */
+ public abstract long bucketsInOrd(long owningBucketOrd);
+
+ /**
+ * The number of collected buckets.
+ */
+ public abstract long size();
+
+ /**
+ * Build an iterator for buckets inside {@code owningBucketOrd} in order
+ * of increasing ord.
+ * <p>
+ * When this is first returns it is "unpositioned" and you must call
+ * {@link BucketOrdsEnum#next()} to move it to the first value.
+ */
+ public abstract BucketOrdsEnum ordsEnum(long owningBucketOrd);
+
+ /**
+ * An iterator for buckets inside a particular {@code owningBucketOrd}.
+ *
+ * @opensearch.internal
+ */
+ public interface BucketOrdsEnum {
+ /**
+ * Advance to the next value.
+ * @return {@code true} if there *is* a next value,
+ * {@code false} if there isn't
+ */
+ boolean next();
+
+ /**
+ * The ordinal of the current value.
+ */
+ long ord();
+
+ /**
+ * Read the current value.
+ */
+ void readValue(BytesRef dest);
+
+ /**
+ * An {@linkplain BucketOrdsEnum} that is empty.
+ */
+ BucketOrdsEnum EMPTY = new BucketOrdsEnum() {
+ @Override
+ public boolean next() {
+ return false;
+ }
+
+ @Override
+ public long ord() {
+ return 0;
+ }
+
+ @Override
+ public void readValue(BytesRef dest) {}
+ };
+ }
+
+ /**
+ * Implementation that only works if it is collecting from a single bucket.
+ *
+ * @opensearch.internal
+ */
+ private static class FromSingle extends BytesKeyedBucketOrds {
+ private final BytesRefHash ords;
+
+ private FromSingle(BigArrays bigArrays) {
+ ords = new BytesRefHash(bigArrays);
+ }
+
+ @Override
+ public long add(long owningBucketOrd, BytesRef value) {
+ assert owningBucketOrd == 0;
+ return ords.add(value);
+ }
+
+ @Override
+ public long bucketsInOrd(long owningBucketOrd) {
+ return ords.size();
+ }
+
+ @Override
+ public long size() {
+ return ords.size();
+ }
+
+ @Override
+ public BucketOrdsEnum ordsEnum(long owningBucketOrd) {
+ return new BucketOrdsEnum() {
+ private int ord = -1;
+
+ @Override
+ public boolean next() {
+ ord++;
+ return ord < ords.size();
+ }
+
+ @Override
+ public long ord() {
+ return ord;
+ }
+
+ @Override
+ public void readValue(BytesRef dest) {
+ ords.get(ord, dest);
+ }
+ };
+ }
+
+ @Override
+ public void close() {
+ ords.close();
+ }
+ }
+
+ /**
+ * Implementation that works properly when collecting from many buckets.
+ *
+ * @opensearch.internal
+ */
+ private static class FromMany extends BytesKeyedBucketOrds {
+ // TODO we can almost certainly do better here by building something fit for purpose rather than trying to lego together stuff
+ private final BytesRefHash bytesToLong;
+ private final LongKeyedBucketOrds longToBucketOrds;
+
+ private FromMany(BigArrays bigArrays) {
+ bytesToLong = new BytesRefHash(bigArrays);
+ longToBucketOrds = LongKeyedBucketOrds.build(bigArrays, CardinalityUpperBound.MANY);
+ }
+
+ @Override
+ public long add(long owningBucketOrd, BytesRef value) {
+ long l = bytesToLong.add(value);
+ if (l < 0) {
+ l = -1 - l;
+ }
+ return longToBucketOrds.add(owningBucketOrd, l);
+ }
+
+ @Override
+ public long bucketsInOrd(long owningBucketOrd) {
+ return longToBucketOrds.bucketsInOrd(owningBucketOrd);
+ }
+
+ @Override
+ public long size() {
+ return longToBucketOrds.size();
+ }
+
+ @Override
+ public BucketOrdsEnum ordsEnum(long owningBucketOrd) {
+ LongKeyedBucketOrds.BucketOrdsEnum delegate = longToBucketOrds.ordsEnum(owningBucketOrd);
+ return new BucketOrdsEnum() {
+ @Override
+ public boolean next() {
+ return delegate.next();
+ }
+
+ @Override
+ public long ord() {
+ return delegate.ord();
+ }
+
+ @Override
+ public void readValue(BytesRef dest) {
+ bytesToLong.get(delegate.value(), dest);
+ }
+ };
+ }
+
+ @Override
+ public void close() {
+ Releasables.close(bytesToLong, longToBucketOrds);
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: DoubleTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | DoubleTerms |
+
+
+ 0%
+
+
+ (0/9)
+
+ |
+
+
+ 0%
+
+
+ (0/32)
+
+ |
+
+
+ | DoubleTerms$Bucket |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ 0%
+
+
+ (0/15)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/19)
+
+ |
+
+
+ 0%
+
+
+ (0/47)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.InternalAggregations;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+
+ /**
+ * Result of the {@link TermsAggregator} when the field is some kind of decimal number like a float, double, or distance.
+ *
+ * @opensearch.internal
+ */
+ public class DoubleTerms extends InternalMappedTerms<DoubleTerms, DoubleTerms.Bucket> {
+ public static final String NAME = "dterms";
+
+ /**
+ * Bucket for a double terms agg
+ *
+ * @opensearch.internal
+ */
+ static class Bucket extends InternalTerms.Bucket<Bucket> {
+ double term;
+
+ Bucket(
+ double term,
+ long docCount,
+ InternalAggregations aggregations,
+ boolean showDocCountError,
+ long docCountError,
+ DocValueFormat format
+ ) {
+ super(docCount, aggregations, showDocCountError, docCountError, format);
+ this.term = term;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ Bucket(StreamInput in, DocValueFormat format, boolean showDocCountError) throws IOException {
+ super(in, format, showDocCountError);
+ term = in.readDouble();
+ }
+
+ @Override
+ protected void writeTermTo(StreamOutput out) throws IOException {
+ out.writeDouble(term);
+ }
+
+ @Override
+ public String getKeyAsString() {
+ return format.format(term).toString();
+ }
+
+ @Override
+ public Object getKey() {
+ return term;
+ }
+
+ @Override
+ public Number getKeyAsNumber() {
+ return term;
+ }
+
+ @Override
+ public int compareKey(Bucket other) {
+ return Double.compare(term, other.term);
+ }
+
+ @Override
+ protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ builder.field(CommonFields.KEY.getPreferredName(), term);
+ if (format != DocValueFormat.RAW) {
+ builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term).toString());
+ }
+ return builder;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj) && Objects.equals(term, ((Bucket) obj).term);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), term);
+ }
+ }
+
+ public DoubleTerms(
+ String name,
+ BucketOrder reduceOrder,
+ BucketOrder order,
+ Map<String, Object> metadata,
+ DocValueFormat format,
+ int shardSize,
+ boolean showTermDocCountError,
+ long otherDocCount,
+ List<Bucket> buckets,
+ long docCountError,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds
+ ) {
+ super(
+ name,
+ reduceOrder,
+ order,
+ metadata,
+ format,
+ shardSize,
+ showTermDocCountError,
+ otherDocCount,
+ buckets,
+ docCountError,
+ bucketCountThresholds
+ );
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public DoubleTerms(StreamInput in) throws IOException {
+ super(in, Bucket::new);
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public DoubleTerms create(List<Bucket> buckets) {
+ return new DoubleTerms(
+ name,
+ reduceOrder,
+ order,
+ metadata,
+ format,
+ shardSize,
+ showTermDocCountError,
+ otherDocCount,
+ buckets,
+ docCountError,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) {
+ return new Bucket(
+ prototype.term,
+ prototype.docCount,
+ aggregations,
+ prototype.showDocCountError,
+ prototype.docCountError,
+ prototype.format
+ );
+ }
+
+ @Override
+ protected DoubleTerms create(String name, List<Bucket> buckets, BucketOrder reduceOrder, long docCountError, long otherDocCount) {
+ return new DoubleTerms(
+ name,
+ reduceOrder,
+ order,
+ getMetadata(),
+ format,
+ shardSize,
+ showTermDocCountError,
+ otherDocCount,
+ buckets,
+ docCountError,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ protected Bucket[] createBucketsArray(int size) {
+ return new Bucket[size];
+ }
+
+ @Override
+ public InternalAggregation reduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+ boolean promoteToDouble = false;
+ for (InternalAggregation agg : aggregations) {
+ if (agg instanceof LongTerms
+ && (((LongTerms) agg).format == DocValueFormat.RAW || ((LongTerms) agg).format == DocValueFormat.UNSIGNED_LONG_SHIFTED)) {
+ /*
+ * this terms agg mixes longs and doubles, we must promote longs to doubles to make the internal aggs
+ * compatible
+ */
+ promoteToDouble = true;
+ break;
+ } else if (agg instanceof UnsignedLongTerms
+ && (((UnsignedLongTerms) agg).format == DocValueFormat.RAW
+ || ((UnsignedLongTerms) agg).format == DocValueFormat.UNSIGNED_LONG_SHIFTED
+ || ((UnsignedLongTerms) agg).format == DocValueFormat.UNSIGNED_LONG)) {
+ /*
+ * this terms agg mixes unsigned longs and doubles, we must promote unsigned longs to doubles to make the internal aggs
+ * compatible
+ */
+ promoteToDouble = true;
+ break;
+ }
+ }
+ if (promoteToDouble == false) {
+ return super.reduce(aggregations, reduceContext);
+ }
+ List<InternalAggregation> newAggs = new ArrayList<>(aggregations.size());
+ for (InternalAggregation agg : aggregations) {
+ if (agg instanceof LongTerms) {
+ DoubleTerms dTerms = LongTerms.convertLongTermsToDouble((LongTerms) agg, format);
+ newAggs.add(dTerms);
+ } else if (agg instanceof UnsignedLongTerms) {
+ DoubleTerms dTerms = UnsignedLongTerms.convertUnsignedLongTermsToDouble((UnsignedLongTerms) agg, format);
+ newAggs.add(dTerms);
+ } else {
+ newAggs.add(agg);
+ }
+ }
+ return newAggs.get(0).reduce(newAggs, reduceContext);
+ }
+
+ @Override
+ Bucket createBucket(long docCount, InternalAggregations aggs, long docCountError, DoubleTerms.Bucket prototype) {
+ return new Bucket(prototype.term, docCount, aggs, prototype.showDocCountError, docCountError, format);
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: GlobalOrdinalsStringTermsAggregator (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator |
+
+
+ 76.9%
+
+
+ (10/13)
+
+ |
+
+
+ 72.5%
+
+
+ (58/80)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$1 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$2 |
+
+
+ 100%
+
+
+ (2/2)
+
+ |
+
+
+ 100%
+
+
+ (5/5)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$3 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$4 |
+
+
+ 100%
+
+
+ (2/2)
+
+ |
+
+
+ 100%
+
+
+ (6/6)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$BucketInfoConsumer |
+
+
+ | GlobalOrdinalsStringTermsAggregator$BucketUpdater |
+
+
+ | GlobalOrdinalsStringTermsAggregator$CollectionStrategy |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$DenseGlobalOrds |
+
+
+ 62.5%
+
+
+ (5/8)
+
+ |
+
+
+ 75%
+
+
+ (12/16)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$GlobalOrdLookupFunction |
+
+
+ | GlobalOrdinalsStringTermsAggregator$LowCardinality |
+
+
+ 0%
+
+
+ (0/7)
+
+ |
+
+
+ 0%
+
+
+ (0/32)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$LowCardinality$1 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$LowCardinality$2 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$OrdBucket |
+
+
+ 28.6%
+
+
+ (2/7)
+
+ |
+
+
+ 28.6%
+
+
+ (2/7)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$RemapGlobalOrds |
+
+
+ 62.5%
+
+
+ (5/8)
+
+ |
+
+
+ 44.4%
+
+
+ (12/27)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$ResultStrategy |
+
+
+ 100%
+
+
+ (2/2)
+
+ |
+
+
+ 80.8%
+
+
+ (21/26)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$ResultStrategy$1 |
+
+
+ 100%
+
+
+ (2/2)
+
+ |
+
+
+ 100%
+
+
+ (8/8)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$SignificantTermsResults |
+
+
+ 0%
+
+
+ (0/17)
+
+ |
+
+
+ 0%
+
+
+ (0/36)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$SignificantTermsResults$1 |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ | GlobalOrdinalsStringTermsAggregator$StandardTermsResults |
+
+
+ 80%
+
+
+ (12/15)
+
+ |
+
+
+ 82.8%
+
+
+ (24/29)
+
+ |
+
+
+ | Total |
+
+
+ 45.7%
+
+
+ (43/94)
+
+ |
+
+
+ 50.2%
+
+
+ (149/297)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.index.DocValues;
+ import org.apache.lucene.index.IndexReader;
+ import org.apache.lucene.index.LeafReaderContext;
+ import org.apache.lucene.index.NumericDocValues;
+ import org.apache.lucene.index.SortedDocValues;
+ import org.apache.lucene.index.SortedSetDocValues;
+ import org.apache.lucene.index.Terms;
+ import org.apache.lucene.index.TermsEnum;
+ import org.apache.lucene.search.Weight;
+ import org.apache.lucene.util.ArrayUtil;
+ import org.apache.lucene.util.BytesRef;
+ import org.apache.lucene.util.PriorityQueue;
+ import org.opensearch.common.lease.Releasable;
+ import org.opensearch.common.lease.Releasables;
+ import org.opensearch.common.util.LongArray;
+ import org.opensearch.common.util.LongHash;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.index.mapper.DocCountFieldMapper;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.AggregationExecutionException;
+ import org.opensearch.search.aggregations.Aggregator;
+ import org.opensearch.search.aggregations.AggregatorFactories;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.CardinalityUpperBound;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.InternalMultiBucketAggregation;
+ import org.opensearch.search.aggregations.InternalOrder;
+ import org.opensearch.search.aggregations.LeafBucketCollector;
+ import org.opensearch.search.aggregations.LeafBucketCollectorBase;
+ import org.opensearch.search.aggregations.bucket.LocalBucketCountThresholds;
+ import org.opensearch.search.aggregations.bucket.terms.SignificanceLookup.BackgroundFrequencyForBytes;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+ import org.opensearch.search.aggregations.support.ValuesSource;
+ import org.opensearch.search.internal.SearchContext;
+
+ import java.io.IOException;
+ import java.util.Arrays;
+ import java.util.Map;
+ import java.util.function.BiConsumer;
+ import java.util.function.Function;
+ import java.util.function.LongPredicate;
+ import java.util.function.LongUnaryOperator;
+
+ import static org.opensearch.search.aggregations.InternalOrder.isKeyOrder;
+ import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
+ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
+ /**
+ * An aggregator of string values that relies on global ordinals in order to build buckets.
+ *
+ * @opensearch.internal
+ */
+ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggregator {
+ protected final ResultStrategy<?, ?, ?> resultStrategy;
+ protected final ValuesSource.Bytes.WithOrdinals valuesSource;
+
+ private final LongPredicate acceptedGlobalOrdinals;
+ private final long valueCount;
+ private final String fieldName;
+ private Weight weight;
+ private final GlobalOrdLookupFunction lookupGlobalOrd;
+ protected final CollectionStrategy collectionStrategy;
+ protected int segmentsWithSingleValuedOrds = 0;
+ protected int segmentsWithMultiValuedOrds = 0;
+
+ /**
+ * Lookup global ordinals
+ *
+ * @opensearch.internal
+ */
+ public interface GlobalOrdLookupFunction {
+ BytesRef apply(long ord) throws IOException;
+ }
+
+ public GlobalOrdinalsStringTermsAggregator(
+ String name,
+ AggregatorFactories factories,
+ Function<GlobalOrdinalsStringTermsAggregator, ResultStrategy<?, ?, ?>> resultStrategy,
+ ValuesSource.Bytes.WithOrdinals valuesSource,
+ BucketOrder order,
+ DocValueFormat format,
+ BucketCountThresholds bucketCountThresholds,
+ IncludeExclude.OrdinalsFilter includeExclude,
+ SearchContext context,
+ Aggregator parent,
+ boolean remapGlobalOrds,
+ SubAggCollectionMode collectionMode,
+ boolean showTermDocCountError,
+ CardinalityUpperBound cardinality,
+ Map<String, Object> metadata
+ ) throws IOException {
+ super(name, factories, context, parent, order, format, bucketCountThresholds, collectionMode, showTermDocCountError, metadata);
+ this.resultStrategy = resultStrategy.apply(this); // ResultStrategy needs a reference to the Aggregator to do its job.
+ this.valuesSource = valuesSource;
+ final IndexReader reader = context.searcher().getIndexReader();
+ final SortedSetDocValues values = reader.leaves().size() > 0
+ ? valuesSource.globalOrdinalsValues(context.searcher().getIndexReader().leaves().get(0))
+ : DocValues.emptySortedSet();
+ this.valueCount = values.getValueCount();
+ this.lookupGlobalOrd = values::lookupOrd;
+ this.acceptedGlobalOrdinals = includeExclude == null ? ALWAYS_TRUE : includeExclude.acceptedGlobalOrdinals(values)::get;
+ if (remapGlobalOrds) {
+ this.collectionStrategy = new RemapGlobalOrds(cardinality);
+ } else {
+ this.collectionStrategy = cardinality.map(estimate -> {
+ if (estimate > 1) {
+ throw new AggregationExecutionException("Dense ords don't know how to collect from many buckets");
+ }
+ return new DenseGlobalOrds();
+ });
+ }
+ this.fieldName = (valuesSource instanceof ValuesSource.Bytes.WithOrdinals.FieldData)
+ ? ((ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource).getIndexFieldName()
+ : null;
+ }
+
+ String descriptCollectionStrategy() {
+ return collectionStrategy.describe();
+ }
+
+ public void setWeight(Weight weight) {
+ this.weight = weight;
+ }
+
+ /**
+ Collects term frequencies for a given field from a LeafReaderContext directly from stored segment terms
+ @param ctx The LeafReaderContext to collect terms from
+ @param globalOrds The SortedSetDocValues for the field's ordinals
+ @param ordCountConsumer A consumer to accept collected term frequencies
+ @return A no-operation LeafBucketCollector implementation, since collection is complete
+ @throws IOException If an I/O error occurs during reading
+ */
+ LeafBucketCollector termDocFreqCollector(
+ LeafReaderContext ctx,
+ SortedSetDocValues globalOrds,
+ BiConsumer<Long, Integer> ordCountConsumer
+ ) throws IOException {
+ if (weight == null) {
+ // Weight not assigned - cannot use this optimization
+ return null;
+ } else {
+ if (weight.count(ctx) == 0) {
+ // No documents matches top level query on this segment, we can skip the segment entirely
+ return LeafBucketCollector.NO_OP_COLLECTOR;
+ } else if (weight.count(ctx) != ctx.reader().maxDoc()) {
+ // weight.count(ctx) == ctx.reader().maxDoc() implies there are no deleted documents and
+ // top-level query matches all docs in the segment
+ return null;
+ }
+ }
+
+ Terms segmentTerms = ctx.reader().terms(this.fieldName);
+ if (segmentTerms == null) {
+ // Field is not indexed.
+ return null;
+ }
+
+ NumericDocValues docCountValues = DocValues.getNumeric(ctx.reader(), DocCountFieldMapper.NAME);
+ if (docCountValues.nextDoc() != NO_MORE_DOCS) {
+ // This segment has at least one document with the _doc_count field.
+ return null;
+ }
+
+ TermsEnum indexTermsEnum = segmentTerms.iterator();
+ BytesRef indexTerm = indexTermsEnum.next();
+ TermsEnum globalOrdinalTermsEnum = globalOrds.termsEnum();
+ BytesRef ordinalTerm = globalOrdinalTermsEnum.next();
+
+ // Iterate over the terms in the segment, look for matches in the global ordinal terms,
+ // and increment bucket count when segment terms match global ordinal terms.
+ while (indexTerm != null && ordinalTerm != null) {
+ int compare = indexTerm.compareTo(ordinalTerm);
+ if (compare == 0) {
+ if (acceptedGlobalOrdinals.test(globalOrdinalTermsEnum.ord())) {
+ ordCountConsumer.accept(globalOrdinalTermsEnum.ord(), indexTermsEnum.docFreq());
+ }
+ indexTerm = indexTermsEnum.next();
+ ordinalTerm = globalOrdinalTermsEnum.next();
+ } else if (compare < 0) {
+ indexTerm = indexTermsEnum.next();
+ } else {
+ ordinalTerm = globalOrdinalTermsEnum.next();
+ }
+ }
+ return LeafBucketCollector.NO_OP_COLLECTOR;
+ }
+
+ @Override
+ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
+ SortedSetDocValues globalOrds = valuesSource.globalOrdinalsValues(ctx);
+ collectionStrategy.globalOrdsReady(globalOrds);
+
+ if (collectionStrategy instanceof DenseGlobalOrds
+ && this.resultStrategy instanceof StandardTermsResults
+ && sub == LeafBucketCollector.NO_OP_COLLECTOR) {
+ LeafBucketCollector termDocFreqCollector = termDocFreqCollector(
+ ctx,
+ globalOrds,
+ (ord, docCount) -> incrementBucketDocCount(collectionStrategy.globalOrdToBucketOrd(0, ord), docCount)
+ );
+ if (termDocFreqCollector != null) {
+ return termDocFreqCollector;
+ }
+ }
+
+ SortedDocValues singleValues = DocValues.unwrapSingleton(globalOrds);
+ if (singleValues != null) {
+ segmentsWithSingleValuedOrds++;
+ if (acceptedGlobalOrdinals == ALWAYS_TRUE) {
+ /*
+ * Optimize when there isn't a filter because that is very
+ * common and marginally faster.
+ */
+ return resultStrategy.wrapCollector(new LeafBucketCollectorBase(sub, globalOrds) {
+ @Override
+ public void collect(int doc, long owningBucketOrd) throws IOException {
+ if (false == singleValues.advanceExact(doc)) {
+ return;
+ }
+ int globalOrd = singleValues.ordValue();
+ collectionStrategy.collectGlobalOrd(owningBucketOrd, doc, globalOrd, sub);
+ }
+ });
+ }
+ return resultStrategy.wrapCollector(new LeafBucketCollectorBase(sub, globalOrds) {
+ @Override
+ public void collect(int doc, long owningBucketOrd) throws IOException {
+ if (false == singleValues.advanceExact(doc)) {
+ return;
+ }
+ int globalOrd = singleValues.ordValue();
+ if (false == acceptedGlobalOrdinals.test(globalOrd)) {
+ return;
+ }
+ collectionStrategy.collectGlobalOrd(owningBucketOrd, doc, globalOrd, sub);
+ }
+ });
+ }
+ segmentsWithMultiValuedOrds++;
+ if (acceptedGlobalOrdinals == ALWAYS_TRUE) {
+ /*
+ * Optimize when there isn't a filter because that is very
+ * common and marginally faster.
+ */
+ return resultStrategy.wrapCollector(new LeafBucketCollectorBase(sub, globalOrds) {
+ @Override
+ public void collect(int doc, long owningBucketOrd) throws IOException {
+ if (false == globalOrds.advanceExact(doc)) {
+ return;
+ }
+ for (long globalOrd = globalOrds.nextOrd(); globalOrd != NO_MORE_ORDS; globalOrd = globalOrds.nextOrd()) {
+ collectionStrategy.collectGlobalOrd(owningBucketOrd, doc, globalOrd, sub);
+ }
+ }
+ });
+ }
+ return resultStrategy.wrapCollector(new LeafBucketCollectorBase(sub, globalOrds) {
+ @Override
+ public void collect(int doc, long owningBucketOrd) throws IOException {
+ if (false == globalOrds.advanceExact(doc)) {
+ return;
+ }
+ for (long globalOrd = globalOrds.nextOrd(); globalOrd != NO_MORE_ORDS; globalOrd = globalOrds.nextOrd()) {
+ if (false == acceptedGlobalOrdinals.test(globalOrd)) {
+ continue;
+ }
+ collectionStrategy.collectGlobalOrd(owningBucketOrd, doc, globalOrd, sub);
+ }
+ }
+ });
+ }
+
+ @Override
+ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException {
+ return resultStrategy.buildAggregations(owningBucketOrds);
+ }
+
+ @Override
+ public InternalAggregation buildEmptyAggregation() {
+ return resultStrategy.buildEmptyResult();
+ }
+
+ @Override
+ public void collectDebugInfo(BiConsumer<String, Object> add) {
+ super.collectDebugInfo(add);
+ add.accept("collection_strategy", collectionStrategy.describe());
+ collectionStrategy.collectDebugInfo(add);
+ add.accept("result_strategy", resultStrategy.describe());
+ add.accept("segments_with_single_valued_ords", segmentsWithSingleValuedOrds);
+ add.accept("segments_with_multi_valued_ords", segmentsWithMultiValuedOrds);
+ add.accept("has_filter", acceptedGlobalOrdinals != ALWAYS_TRUE);
+ }
+
+ /**
+ * This is used internally only, just for compare using global ordinal instead of term bytes in the PQ
+ *
+ * @opensearch.internal
+ */
+ static class OrdBucket extends InternalTerms.Bucket<OrdBucket> {
+ long globalOrd;
+
+ OrdBucket(boolean showDocCountError, DocValueFormat format) {
+ super(0, null, showDocCountError, 0, format);
+ }
+
+ @Override
+ public int compareKey(OrdBucket other) {
+ return Long.compare(globalOrd, other.globalOrd);
+ }
+
+ @Override
+ public String getKeyAsString() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object getKey() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Number getKeyAsNumber() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected void writeTermTo(StreamOutput out) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ @Override
+ protected void doClose() {
+ Releasables.close(resultStrategy, collectionStrategy);
+ }
+
+ /**
+ * Variant of {@link GlobalOrdinalsStringTermsAggregator} that
+ * resolves global ordinals post segment collection instead of on the fly
+ * for each match.This is beneficial for low cardinality fields, because
+ * it can reduce the amount of look-ups significantly.
+ * <p>
+ * This is only supported for the standard {@code terms} aggregation and
+ * doesn't support {@code significant_terms} so this forces
+ * {@link StandardTermsResults}.
+ *
+ * @opensearch.internal
+ */
+ static class LowCardinality extends GlobalOrdinalsStringTermsAggregator {
+
+ private LongUnaryOperator mapping;
+ private LongArray segmentDocCounts;
+
+ LowCardinality(
+ String name,
+ AggregatorFactories factories,
+ Function<GlobalOrdinalsStringTermsAggregator, ResultStrategy<?, ?, ?>> resultStrategy,
+ ValuesSource.Bytes.WithOrdinals valuesSource,
+ BucketOrder order,
+ DocValueFormat format,
+ BucketCountThresholds bucketCountThresholds,
+ SearchContext context,
+ Aggregator parent,
+ boolean remapGlobalOrds,
+ SubAggCollectionMode collectionMode,
+ boolean showTermDocCountError,
+ Map<String, Object> metadata
+ ) throws IOException {
+ super(
+ name,
+ factories,
+ resultStrategy,
+ valuesSource,
+ order,
+ format,
+ bucketCountThresholds,
+ null,
+ context,
+ parent,
+ remapGlobalOrds,
+ collectionMode,
+ showTermDocCountError,
+ CardinalityUpperBound.ONE,
+ metadata
+ );
+ assert factories == null || factories.countAggregators() == 0;
+ this.segmentDocCounts = context.bigArrays().newLongArray(1, true);
+ }
+
+ @Override
+ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
+ if (mapping != null) {
+ mapSegmentCountsToGlobalCounts(mapping);
+ }
+ final SortedSetDocValues segmentOrds = valuesSource.ordinalsValues(ctx);
+ segmentDocCounts = context.bigArrays().grow(segmentDocCounts, 1 + segmentOrds.getValueCount());
+ assert sub == LeafBucketCollector.NO_OP_COLLECTOR;
+ mapping = valuesSource.globalOrdinalsMapping(ctx);
+
+ if (this.resultStrategy instanceof StandardTermsResults) {
+ LeafBucketCollector termDocFreqCollector = this.termDocFreqCollector(
+ ctx,
+ segmentOrds,
+ (ord, docCount) -> incrementBucketDocCount(mapping.applyAsLong(ord), docCount)
+ );
+ if (termDocFreqCollector != null) {
+ return termDocFreqCollector;
+ }
+ }
+
+ final SortedDocValues singleValues = DocValues.unwrapSingleton(segmentOrds);
+ if (singleValues != null) {
+ segmentsWithSingleValuedOrds++;
+ return resultStrategy.wrapCollector(new LeafBucketCollectorBase(sub, segmentOrds) {
+ @Override
+ public void collect(int doc, long owningBucketOrd) throws IOException {
+ assert owningBucketOrd == 0;
+ if (false == singleValues.advanceExact(doc)) {
+ return;
+ }
+ int ord = singleValues.ordValue();
+ long docCount = docCountProvider.getDocCount(doc);
+ segmentDocCounts.increment(ord + 1, docCount);
+ }
+ });
+ }
+ segmentsWithMultiValuedOrds++;
+ return resultStrategy.wrapCollector(new LeafBucketCollectorBase(sub, segmentOrds) {
+ @Override
+ public void collect(int doc, long owningBucketOrd) throws IOException {
+ assert owningBucketOrd == 0;
+ if (false == segmentOrds.advanceExact(doc)) {
+ return;
+ }
+ for (long segmentOrd = segmentOrds.nextOrd(); segmentOrd != NO_MORE_ORDS; segmentOrd = segmentOrds.nextOrd()) {
+ long docCount = docCountProvider.getDocCount(doc);
+ segmentDocCounts.increment(segmentOrd + 1, docCount);
+ }
+ }
+ });
+ }
+
+ @Override
+ protected void doPostCollection() throws IOException {
+ if (mapping != null) {
+ mapSegmentCountsToGlobalCounts(mapping);
+ mapping = null;
+ }
+ }
+
+ @Override
+ protected void doClose() {
+ Releasables.close(resultStrategy, segmentDocCounts, collectionStrategy);
+ }
+
+ private void mapSegmentCountsToGlobalCounts(LongUnaryOperator mapping) throws IOException {
+ for (long i = 1; i < segmentDocCounts.size(); i++) {
+ // We use set(...) here, because we need to reset the slow to 0.
+ // segmentDocCounts get reused over the segments and otherwise counts would be too high.
+ long inc = segmentDocCounts.set(i, 0);
+ if (inc == 0) {
+ continue;
+ }
+ long ord = i - 1; // remember we do +1 when counting
+ long globalOrd = mapping.applyAsLong(ord);
+ incrementBucketDocCount(collectionStrategy.globalOrdToBucketOrd(0, globalOrd), inc);
+ }
+ }
+ }
+
+ /**
+ * Strategy for collecting global ordinals.
+ * <p>
+ * The {@link GlobalOrdinalsStringTermsAggregator} uses one of these
+ * to collect the global ordinals by calling
+ * {@link CollectionStrategy#collectGlobalOrd} for each global ordinal
+ * that it hits and then calling {@link CollectionStrategy#forEach}
+ * once to iterate on the results.
+ */
+ abstract class CollectionStrategy implements Releasable {
+ /**
+ * Short description of the collection mechanism added to the profile
+ * output to help with debugging.
+ */
+ abstract String describe();
+
+ /**
+ * Collect debug information to add to the profiling results. This will
+ * only be called if the aggregation is being profiled.
+ */
+ abstract void collectDebugInfo(BiConsumer<String, Object> add);
+
+ /**
+ * Called when the global ordinals are ready.
+ */
+ abstract void globalOrdsReady(SortedSetDocValues globalOrds);
+
+ /**
+ * Called once per unique document, global ordinal combination to
+ * collect the bucket.
+ *
+ * @param owningBucketOrd the ordinal of the bucket that owns this collection
+ * @param doc the doc id in to collect
+ * @param globalOrd the global ordinal to collect
+ * @param sub the sub-aggregators that that will collect the bucket data
+ */
+ abstract void collectGlobalOrd(long owningBucketOrd, int doc, long globalOrd, LeafBucketCollector sub) throws IOException;
+
+ /**
+ * Convert a global ordinal into a bucket ordinal.
+ */
+ abstract long globalOrdToBucketOrd(long owningBucketOrd, long globalOrd);
+
+ /**
+ * Iterate all of the buckets. Implementations take into account
+ * the {@link BucketCountThresholds}. In particular,
+ * if the {@link BucketCountThresholds#getMinDocCount()} is 0 then
+ * they'll make sure to iterate a bucket even if it was never
+ * {{@link #collectGlobalOrd collected}.
+ * If {@link BucketCountThresholds#getMinDocCount()} is not 0 then
+ * they'll skip all global ords that weren't collected.
+ */
+ abstract void forEach(long owningBucketOrd, BucketInfoConsumer consumer) throws IOException;
+ }
+
+ interface BucketInfoConsumer {
+ void accept(long globalOrd, long bucketOrd, long docCount) throws IOException;
+ }
+
+ /**
+ * {@linkplain CollectionStrategy} that just uses the global ordinal as the
+ * bucket ordinal.
+ */
+ class DenseGlobalOrds extends CollectionStrategy {
+ @Override
+ String describe() {
+ return "dense";
+ }
+
+ @Override
+ void collectDebugInfo(BiConsumer<String, Object> add) {}
+
+ @Override
+ void globalOrdsReady(SortedSetDocValues globalOrds) {
+ grow(globalOrds.getValueCount());
+ }
+
+ @Override
+ void collectGlobalOrd(long owningBucketOrd, int doc, long globalOrd, LeafBucketCollector sub) throws IOException {
+ assert owningBucketOrd == 0;
+ collectExistingBucket(sub, doc, globalOrd);
+ }
+
+ @Override
+ long globalOrdToBucketOrd(long owningBucketOrd, long globalOrd) {
+ assert owningBucketOrd == 0;
+ return globalOrd;
+ }
+
+ @Override
+ void forEach(long owningBucketOrd, BucketInfoConsumer consumer) throws IOException {
+ assert owningBucketOrd == 0;
+ for (long globalOrd = 0; globalOrd < valueCount; globalOrd++) {
+ if (false == acceptedGlobalOrdinals.test(globalOrd)) {
+ continue;
+ }
+ long docCount = bucketDocCount(globalOrd);
+ if (bucketCountThresholds.getMinDocCount() == 0 || docCount > 0) {
+ consumer.accept(globalOrd, globalOrd, docCount);
+ }
+ }
+ }
+
+ @Override
+ public void close() {}
+ }
+
+ /**
+ * {@linkplain CollectionStrategy} that uses a {@link LongHash} to map the
+ * global ordinal into bucket ordinals. This uses more memory than
+ * {@link DenseGlobalOrds} when collecting every ordinal, but significantly
+ * less when collecting only a few.
+ */
+ private class RemapGlobalOrds extends CollectionStrategy {
+ private final LongKeyedBucketOrds bucketOrds;
+
+ private RemapGlobalOrds(CardinalityUpperBound cardinality) {
+ bucketOrds = LongKeyedBucketOrds.build(context.bigArrays(), cardinality);
+ }
+
+ @Override
+ String describe() {
+ return "remap";
+ }
+
+ @Override
+ void collectDebugInfo(BiConsumer<String, Object> add) {
+ add.accept("total_buckets", bucketOrds.size());
+ }
+
+ @Override
+ void globalOrdsReady(SortedSetDocValues globalOrds) {}
+
+ @Override
+ void collectGlobalOrd(long owningBucketOrd, int doc, long globalOrd, LeafBucketCollector sub) throws IOException {
+ long bucketOrd = bucketOrds.add(owningBucketOrd, globalOrd);
+ if (bucketOrd < 0) {
+ bucketOrd = -1 - bucketOrd;
+ collectExistingBucket(sub, doc, bucketOrd);
+ } else {
+ collectBucket(sub, doc, bucketOrd);
+ }
+ }
+
+ @Override
+ long globalOrdToBucketOrd(long owningBucketOrd, long globalOrd) {
+ return bucketOrds.find(owningBucketOrd, globalOrd);
+ }
+
+ @Override
+ void forEach(long owningBucketOrd, BucketInfoConsumer consumer) throws IOException {
+ if (bucketCountThresholds.getMinDocCount() == 0) {
+ for (long globalOrd = 0; globalOrd < valueCount; globalOrd++) {
+ if (false == acceptedGlobalOrdinals.test(globalOrd)) {
+ continue;
+ }
+ /*
+ * Use `add` instead of `find` here to assign an ordinal
+ * even if the global ord wasn't found so we can build
+ * sub-aggregations without trouble even though we haven't
+ * hit any documents for them. This is wasteful, but
+ * settings minDocCount == 0 is wasteful in general.....
+ */
+ long bucketOrd = bucketOrds.add(owningBucketOrd, globalOrd);
+ long docCount;
+ if (bucketOrd < 0) {
+ bucketOrd = -1 - bucketOrd;
+ docCount = bucketDocCount(bucketOrd);
+ } else {
+ docCount = 0;
+ }
+ consumer.accept(globalOrd, bucketOrd, docCount);
+ }
+ } else {
+ LongKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrd);
+ while (ordsEnum.next()) {
+ if (false == acceptedGlobalOrdinals.test(ordsEnum.value())) {
+ continue;
+ }
+ consumer.accept(ordsEnum.value(), ordsEnum.ord(), bucketDocCount(ordsEnum.ord()));
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ bucketOrds.close();
+ }
+ }
+
+ /**
+ * Strategy for building results.
+ */
+ abstract class ResultStrategy<
+ R extends InternalAggregation,
+ B extends InternalMultiBucketAggregation.InternalBucket,
+ TB extends InternalMultiBucketAggregation.InternalBucket> implements Releasable {
+
+ private InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException {
+ LocalBucketCountThresholds localBucketCountThresholds = context.asLocalBucketCountThresholds(bucketCountThresholds);
+ if (valueCount == 0) { // no context in this reader
+ InternalAggregation[] results = new InternalAggregation[owningBucketOrds.length];
+ for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) {
+ results[ordIdx] = buildNoValuesResult(owningBucketOrds[ordIdx]);
+ }
+ return results;
+ }
+
+ B[][] topBucketsPreOrd = buildTopBucketsPerOrd(owningBucketOrds.length);
+ long[] otherDocCount = new long[owningBucketOrds.length];
+ for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) {
+ final int size;
+ if (localBucketCountThresholds.getMinDocCount() == 0) {
+ // if minDocCount == 0 then we can end up with more buckets then maxBucketOrd() returns
+ size = (int) Math.min(valueCount, localBucketCountThresholds.getRequiredSize());
+ } else {
+ size = (int) Math.min(maxBucketOrd(), localBucketCountThresholds.getRequiredSize());
+ }
+ PriorityQueue<TB> ordered = buildPriorityQueue(size);
+ final int finalOrdIdx = ordIdx;
+ BucketUpdater<TB> updater = bucketUpdater(owningBucketOrds[ordIdx]);
+ collectionStrategy.forEach(owningBucketOrds[ordIdx], new BucketInfoConsumer() {
+ TB spare = null;
+
+ @Override
+ public void accept(long globalOrd, long bucketOrd, long docCount) throws IOException {
+ otherDocCount[finalOrdIdx] += docCount;
+ if (docCount >= localBucketCountThresholds.getMinDocCount()) {
+ if (spare == null) {
+ spare = buildEmptyTemporaryBucket();
+ }
+ updater.updateBucket(spare, globalOrd, bucketOrd, docCount);
+ spare = ordered.insertWithOverflow(spare);
+ }
+ }
+ });
+
+ // Get the top buckets
+ topBucketsPreOrd[ordIdx] = buildBuckets(ordered.size());
+ for (int i = ordered.size() - 1; i >= 0; --i) {
+ topBucketsPreOrd[ordIdx][i] = convertTempBucketToRealBucket(ordered.pop());
+ otherDocCount[ordIdx] -= topBucketsPreOrd[ordIdx][i].getDocCount();
+ }
+ }
+
+ buildSubAggs(topBucketsPreOrd);
+
+ InternalAggregation[] results = new InternalAggregation[owningBucketOrds.length];
+ for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) {
+ results[ordIdx] = buildResult(owningBucketOrds[ordIdx], otherDocCount[ordIdx], topBucketsPreOrd[ordIdx]);
+ }
+ return results;
+ }
+
+ /**
+ * Short description of the collection mechanism added to the profile
+ * output to help with debugging.
+ */
+ abstract String describe();
+
+ /**
+ * Wrap the "standard" numeric terms collector to collect any more
+ * information that this result type may need.
+ */
+ abstract LeafBucketCollector wrapCollector(LeafBucketCollector primary);
+
+ /**
+ * Build an empty temporary bucket.
+ */
+ abstract TB buildEmptyTemporaryBucket();
+
+ /**
+ * Update fields in {@code spare} to reflect information collected for
+ * this bucket ordinal.
+ */
+ abstract BucketUpdater<TB> bucketUpdater(long owningBucketOrd) throws IOException;
+
+ /**
+ * Build a {@link PriorityQueue} to sort the buckets. After we've
+ * collected all of the buckets we'll collect all entries in the queue.
+ */
+ abstract PriorityQueue<TB> buildPriorityQueue(int size);
+
+ /**
+ * Build an array to hold the "top" buckets for each ordinal.
+ */
+ abstract B[][] buildTopBucketsPerOrd(int size);
+
+ /**
+ * Build an array of buckets for a particular ordinal to collect the
+ * results. The populated list is passed to {@link #buildResult}.
+ */
+ abstract B[] buildBuckets(int size);
+
+ /**
+ * Convert a temporary bucket into a real bucket.
+ */
+ abstract B convertTempBucketToRealBucket(TB temp) throws IOException;
+
+ /**
+ * Build the sub-aggregations into the buckets. This will usually
+ * delegate to {@link #buildSubAggsForAllBuckets}.
+ */
+ abstract void buildSubAggs(B[][] topBucketsPreOrd) throws IOException;
+
+ /**
+ * Turn the buckets into an aggregation result.
+ */
+ abstract R buildResult(long owningBucketOrd, long otherDocCount, B[] topBuckets);
+
+ /**
+ * Build an "empty" result. Only called if there isn't any data on this
+ * shard.
+ */
+ abstract R buildEmptyResult();
+
+ /**
+ * Build an "empty" result for a particular bucket ordinal. Called when
+ * there aren't any values for the field on this shard.
+ */
+ abstract R buildNoValuesResult(long owningBucketOrdinal);
+ }
+
+ interface BucketUpdater<TB extends InternalMultiBucketAggregation.InternalBucket> {
+ void updateBucket(TB spare, long globalOrd, long bucketOrd, long docCount) throws IOException;
+ }
+
+ /**
+ * Builds results for the standard {@code terms} aggregation.
+ */
+ class StandardTermsResults extends ResultStrategy<StringTerms, StringTerms.Bucket, OrdBucket> {
+ @Override
+ String describe() {
+ return "terms";
+ }
+
+ @Override
+ LeafBucketCollector wrapCollector(LeafBucketCollector primary) {
+ return primary;
+ }
+
+ @Override
+ StringTerms.Bucket[][] buildTopBucketsPerOrd(int size) {
+ return new StringTerms.Bucket[size][];
+ }
+
+ @Override
+ StringTerms.Bucket[] buildBuckets(int size) {
+ return new StringTerms.Bucket[size];
+ }
+
+ @Override
+ OrdBucket buildEmptyTemporaryBucket() {
+ return new OrdBucket(showTermDocCountError, format);
+ }
+
+ @Override
+ BucketUpdater<OrdBucket> bucketUpdater(long owningBucketOrd) throws IOException {
+ return (spare, globalOrd, bucketOrd, docCount) -> {
+ spare.globalOrd = globalOrd;
+ spare.bucketOrd = bucketOrd;
+ spare.docCount = docCount;
+ };
+ }
+
+ @Override
+ PriorityQueue<OrdBucket> buildPriorityQueue(int size) {
+ return new BucketPriorityQueue<>(size, partiallyBuiltBucketComparator);
+ }
+
+ StringTerms.Bucket convertTempBucketToRealBucket(OrdBucket temp) throws IOException {
+ BytesRef term = BytesRef.deepCopyOf(lookupGlobalOrd.apply(temp.globalOrd));
+ StringTerms.Bucket result = new StringTerms.Bucket(term, temp.docCount, null, showTermDocCountError, 0, format);
+ result.bucketOrd = temp.bucketOrd;
+ result.docCountError = 0;
+ return result;
+ }
+
+ @Override
+ void buildSubAggs(StringTerms.Bucket[][] topBucketsPreOrd) throws IOException {
+ buildSubAggsForAllBuckets(topBucketsPreOrd, b -> b.bucketOrd, (b, aggs) -> b.aggregations = aggs);
+ }
+
+ @Override
+ StringTerms buildResult(long owningBucketOrd, long otherDocCount, StringTerms.Bucket[] topBuckets) {
+ final BucketOrder reduceOrder;
+ if (isKeyOrder(order) == false) {
+ reduceOrder = InternalOrder.key(true);
+ Arrays.sort(topBuckets, reduceOrder.comparator());
+ } else {
+ reduceOrder = order;
+ }
+ return new StringTerms(
+ name,
+ reduceOrder,
+ order,
+ metadata(),
+ format,
+ bucketCountThresholds.getShardSize(),
+ showTermDocCountError,
+ otherDocCount,
+ Arrays.asList(topBuckets),
+ 0,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ StringTerms buildEmptyResult() {
+ return buildEmptyTermsAggregation();
+ }
+
+ @Override
+ StringTerms buildNoValuesResult(long owningBucketOrdinal) {
+ return buildEmptyResult();
+ }
+
+ @Override
+ public void close() {}
+ }
+
+ /**
+ * Builds results for the {@code significant_terms} aggregation.
+ */
+ class SignificantTermsResults extends ResultStrategy<
+ SignificantStringTerms,
+ SignificantStringTerms.Bucket,
+ SignificantStringTerms.Bucket> {
+
+ private final BackgroundFrequencyForBytes backgroundFrequencies;
+ private final long supersetSize;
+ private final SignificanceHeuristic significanceHeuristic;
+
+ private LongArray subsetSizes = context.bigArrays().newLongArray(1, true);
+
+ SignificantTermsResults(
+ SignificanceLookup significanceLookup,
+ SignificanceHeuristic significanceHeuristic,
+ CardinalityUpperBound cardinality
+ ) {
+ backgroundFrequencies = significanceLookup.bytesLookup(context.bigArrays(), cardinality);
+ supersetSize = significanceLookup.supersetSize();
+ this.significanceHeuristic = significanceHeuristic;
+ }
+
+ @Override
+ String describe() {
+ return "significant_terms";
+ }
+
+ @Override
+ LeafBucketCollector wrapCollector(LeafBucketCollector primary) {
+ return new LeafBucketCollectorBase(primary, null) {
+ @Override
+ public void collect(int doc, long owningBucketOrd) throws IOException {
+ super.collect(doc, owningBucketOrd);
+ subsetSizes = context.bigArrays().grow(subsetSizes, owningBucketOrd + 1);
+ subsetSizes.increment(owningBucketOrd, 1);
+ }
+ };
+ }
+
+ @Override
+ SignificantStringTerms.Bucket[][] buildTopBucketsPerOrd(int size) {
+ return new SignificantStringTerms.Bucket[size][];
+ }
+
+ @Override
+ SignificantStringTerms.Bucket[] buildBuckets(int size) {
+ return new SignificantStringTerms.Bucket[size];
+ }
+
+ @Override
+ SignificantStringTerms.Bucket buildEmptyTemporaryBucket() {
+ return new SignificantStringTerms.Bucket(new BytesRef(), 0, 0, 0, 0, null, format, 0);
+ }
+
+ private long subsetSize(long owningBucketOrd) {
+ // if the owningBucketOrd is not in the array that means the bucket is empty so the size has to be 0
+ return owningBucketOrd < subsetSizes.size() ? subsetSizes.get(owningBucketOrd) : 0;
+ }
+
+ @Override
+ BucketUpdater<SignificantStringTerms.Bucket> bucketUpdater(long owningBucketOrd) throws IOException {
+ long subsetSize = subsetSize(owningBucketOrd);
+ return (spare, globalOrd, bucketOrd, docCount) -> {
+ spare.bucketOrd = bucketOrd;
+ oversizedCopy(lookupGlobalOrd.apply(globalOrd), spare.termBytes);
+ spare.subsetDf = docCount;
+ spare.subsetSize = subsetSize;
+ spare.supersetDf = backgroundFrequencies.freq(spare.termBytes);
+ spare.supersetSize = supersetSize;
+ /*
+ * During shard-local down-selection we use subset/superset stats
+ * that are for this shard only. Back at the central reducer these
+ * properties will be updated with global stats.
+ */
+ spare.updateScore(significanceHeuristic);
+ };
+ }
+
+ @Override
+ PriorityQueue<SignificantStringTerms.Bucket> buildPriorityQueue(int size) {
+ return new BucketSignificancePriorityQueue<>(size);
+ }
+
+ @Override
+ SignificantStringTerms.Bucket convertTempBucketToRealBucket(SignificantStringTerms.Bucket temp) throws IOException {
+ return temp;
+ }
+
+ @Override
+ void buildSubAggs(SignificantStringTerms.Bucket[][] topBucketsPreOrd) throws IOException {
+ buildSubAggsForAllBuckets(topBucketsPreOrd, b -> b.bucketOrd, (b, aggs) -> b.aggregations = aggs);
+ }
+
+ @Override
+ SignificantStringTerms buildResult(long owningBucketOrd, long otherDocCount, SignificantStringTerms.Bucket[] topBuckets) {
+ return new SignificantStringTerms(
+ name,
+ metadata(),
+ format,
+ subsetSize(owningBucketOrd),
+ supersetSize,
+ significanceHeuristic,
+ Arrays.asList(topBuckets),
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ SignificantStringTerms buildEmptyResult() {
+ return buildEmptySignificantTermsAggregation(0, significanceHeuristic);
+ }
+
+ @Override
+ SignificantStringTerms buildNoValuesResult(long owningBucketOrdinal) {
+ return buildEmptySignificantTermsAggregation(subsetSizes.get(owningBucketOrdinal), significanceHeuristic);
+ }
+
+ @Override
+ public void close() {
+ Releasables.close(backgroundFrequencies, subsetSizes);
+ }
+
+ /**
+ * Copies the bytes from {@code from} into {@code to}, oversizing
+ * the destination array if the bytes won't fit into the array.
+ * <p>
+ * This is fairly similar in spirit to
+ * {@link BytesRef#deepCopyOf(BytesRef)} in that it is a way to read
+ * bytes from a mutable {@link BytesRef} into
+ * <strong>something</strong> that won't mutate out from under you.
+ * Unlike {@linkplain BytesRef#deepCopyOf(BytesRef)} its designed to
+ * be run over and over again into the same destination. In particular,
+ * oversizing the destination bytes helps to keep from allocating
+ * a bunch of little arrays over and over and over again.
+ */
+ private void oversizedCopy(BytesRef from, BytesRef to) {
+ if (to.bytes.length < from.length) {
+ to.bytes = new byte[ArrayUtil.oversize(from.length, 1)];
+ }
+ to.offset = 0;
+ to.length = from.length;
+ System.arraycopy(from.bytes, from.offset, to.bytes, 0, from.length);
+ }
+ }
+
+ /**
+ * Predicate used for {@link #acceptedGlobalOrdinals} if there is no filter.
+ */
+ private static final LongPredicate ALWAYS_TRUE = l -> true;
+ }
+
+
+
+
+
+
Coverage Summary for Class: IncludeExclude (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | IncludeExclude |
+
+
+ 36.7%
+
+
+ (11/30)
+
+ |
+
+
+ 19.1%
+
+
+ (48/251)
+
+ |
+
+
+ | IncludeExclude$AutomatonBackedOrdinalsFilter |
+
+
+ 100%
+
+
+ (2/2)
+
+ |
+
+
+ 100%
+
+
+ (8/8)
+
+ |
+
+
+ | IncludeExclude$AutomatonBackedStringFilter |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ | IncludeExclude$DocValuesTerms |
+
+
+ 20%
+
+
+ (2/10)
+
+ |
+
+
+ 27.3%
+
+
+ (3/11)
+
+ |
+
+
+ | IncludeExclude$Filter |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ | IncludeExclude$LongFilter |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ | IncludeExclude$OrdinalsFilter |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ | IncludeExclude$PartitionedLongFilter |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ | IncludeExclude$PartitionedOrdinalsFilter |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/11)
+
+ |
+
+
+ | IncludeExclude$PartitionedStringFilter |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ | IncludeExclude$SetBackedLongFilter |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ | IncludeExclude$StringFilter |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ | IncludeExclude$TermListBackedOrdinalsFilter |
+
+
+ 100%
+
+
+ (2/2)
+
+ |
+
+
+ 100%
+
+
+ (19/19)
+
+ |
+
+
+ | IncludeExclude$TermListBackedStringFilter |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/4)
+
+ |
+
+
+ | Total |
+
+
+ 30.6%
+
+
+ (19/62)
+
+ |
+
+
+ 24.7%
+
+
+ (80/324)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.index.SortedSetDocValues;
+ import org.apache.lucene.index.Terms;
+ import org.apache.lucene.index.TermsEnum;
+ import org.apache.lucene.util.BytesRef;
+ import org.apache.lucene.util.LongBitSet;
+ import org.apache.lucene.util.NumericUtils;
+ import org.apache.lucene.util.StringHelper;
+ import org.apache.lucene.util.automaton.Automata;
+ import org.apache.lucene.util.automaton.Automaton;
+ import org.apache.lucene.util.automaton.ByteRunAutomaton;
+ import org.apache.lucene.util.automaton.CompiledAutomaton;
+ import org.apache.lucene.util.automaton.Operations;
+ import org.apache.lucene.util.automaton.RegExp;
+ import org.opensearch.OpenSearchParseException;
+ import org.opensearch.common.util.BitMixer;
+ import org.opensearch.core.ParseField;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.common.io.stream.Writeable;
+ import org.opensearch.core.xcontent.ToXContentFragment;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+ import org.opensearch.index.IndexSettings;
+ import org.opensearch.search.DocValueFormat;
+
+ import java.io.IOException;
+ import java.util.HashSet;
+ import java.util.Objects;
+ import java.util.Set;
+ import java.util.SortedSet;
+ import java.util.TreeSet;
+
+ /**
+ * Defines the include/exclude regular expression filtering for string terms aggregation. In this filtering logic,
+ * exclusion has precedence, where the {@code include} is evaluated first and then the {@code exclude}.
+ *
+ * @opensearch.internal
+ */
+ public class IncludeExclude implements Writeable, ToXContentFragment {
+ public static final ParseField INCLUDE_FIELD = new ParseField("include");
+ public static final ParseField EXCLUDE_FIELD = new ParseField("exclude");
+ public static final ParseField PARTITION_FIELD = new ParseField("partition");
+ public static final ParseField NUM_PARTITIONS_FIELD = new ParseField("num_partitions");
+ // Needed to add this seed for a deterministic term hashing policy
+ // otherwise tests fail to get expected results and worse, shards
+ // can disagree on which terms hash to the required partition.
+ private static final int HASH_PARTITIONING_SEED = 31;
+
+ /**
+ * The default length limit for a reg-ex string. The value is derived from {@link IndexSettings#MAX_REGEX_LENGTH_SETTING}.
+ * For context, see:
+ * https://github.com/opensearch-project/OpenSearch/issues/1992
+ * https://github.com/opensearch-project/OpenSearch/issues/2858
+ */
+ private static final int DEFAULT_MAX_REGEX_LENGTH = 1000;
+
+ // for parsing purposes only
+ // TODO: move all aggs to the same package so that this stuff could be pkg-private
+ public static IncludeExclude merge(IncludeExclude include, IncludeExclude exclude) {
+ if (include == null) {
+ return exclude;
+ }
+ if (exclude == null) {
+ return include;
+ }
+ if (include.isPartitionBased()) {
+ throw new IllegalArgumentException("Cannot specify any excludes when using a partition-based include");
+ }
+ String includeMethod = include.isRegexBased() ? "regex" : "set";
+ String excludeMethod = exclude.isRegexBased() ? "regex" : "set";
+ if (includeMethod.equals(excludeMethod) == false) {
+ throw new IllegalArgumentException(
+ "Cannot mix a " + includeMethod + "-based include with a " + excludeMethod + "-based method"
+ );
+ }
+ if (include.isRegexBased()) {
+ return new IncludeExclude(include.include, exclude.exclude);
+ } else {
+ return new IncludeExclude(include.includeValues, exclude.excludeValues);
+ }
+ }
+
+ public static IncludeExclude parseInclude(XContentParser parser) throws IOException {
+ XContentParser.Token token = parser.currentToken();
+ if (token == XContentParser.Token.VALUE_STRING) {
+ return new IncludeExclude(parser.text(), null);
+ } else if (token == XContentParser.Token.START_ARRAY) {
+ return new IncludeExclude(new TreeSet<>(parseArrayToSet(parser)), null);
+ } else if (token == XContentParser.Token.START_OBJECT) {
+ String currentFieldName = null;
+ Integer partition = null, numPartitions = null;
+ while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+ if (token == XContentParser.Token.FIELD_NAME) {
+ currentFieldName = parser.currentName();
+ } else if (NUM_PARTITIONS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ numPartitions = parser.intValue();
+ } else if (PARTITION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ partition = parser.intValue();
+ } else {
+ throw new OpenSearchParseException("Unknown parameter in Include/Exclude clause: " + currentFieldName);
+ }
+ }
+ if (partition == null) {
+ throw new IllegalArgumentException(
+ "Missing [" + PARTITION_FIELD.getPreferredName() + "] parameter for partition-based include"
+ );
+ }
+ if (numPartitions == null) {
+ throw new IllegalArgumentException(
+ "Missing [" + NUM_PARTITIONS_FIELD.getPreferredName() + "] parameter for partition-based include"
+ );
+ }
+ return new IncludeExclude(partition, numPartitions);
+ } else {
+ throw new IllegalArgumentException("Unrecognized token for an include [" + token + "]");
+ }
+ }
+
+ public static IncludeExclude parseExclude(XContentParser parser) throws IOException {
+ XContentParser.Token token = parser.currentToken();
+ if (token == XContentParser.Token.VALUE_STRING) {
+ return new IncludeExclude(null, parser.text());
+ } else if (token == XContentParser.Token.START_ARRAY) {
+ return new IncludeExclude(null, new TreeSet<>(parseArrayToSet(parser)));
+ } else {
+ throw new IllegalArgumentException("Unrecognized token for an exclude [" + token + "]");
+ }
+ }
+
+ /**
+ * Base filter class
+ *
+ * @opensearch.internal
+ */
+ public abstract static class Filter {}
+
+ /**
+ * The includeValue and excludeValue ByteRefs which are the result of the parsing
+ * process are converted into a LongFilter when used on numeric fields
+ * in the index.
+ *
+ * @opensearch.internal
+ */
+ public abstract static class LongFilter extends Filter {
+ public abstract boolean accept(long value);
+ }
+
+ /**
+ * Long filter that is partitioned
+ *
+ * @opensearch.internal
+ */
+ public class PartitionedLongFilter extends LongFilter {
+ @Override
+ public boolean accept(long value) {
+ // hash the value to keep even distributions
+ final long hashCode = BitMixer.mix64(value);
+ return Math.floorMod(hashCode, incNumPartitions) == incZeroBasedPartition;
+ }
+ }
+
+ /**
+ * Long filter backed by valid values
+ *
+ * @opensearch.internal
+ */
+ public static class SetBackedLongFilter extends LongFilter {
+ private Set<Long> valids;
+ private Set<Long> invalids;
+
+ private SetBackedLongFilter(int numValids, int numInvalids) {
+ if (numValids > 0) {
+ valids = new HashSet<>(numValids);
+ }
+ if (numInvalids > 0) {
+ invalids = new HashSet<>(numInvalids);
+ }
+ }
+
+ @Override
+ public boolean accept(long value) {
+ return ((valids == null) || (valids.contains(value))) && ((invalids == null) || (!invalids.contains(value)));
+ }
+
+ private void addAccept(long val) {
+ valids.add(val);
+ }
+
+ private void addReject(long val) {
+ invalids.add(val);
+ }
+ }
+
+ /**
+ * Only used for the 'map' execution mode (ie. scripts)
+ *
+ * @opensearch.internal
+ */
+ public abstract static class StringFilter extends Filter {
+ public abstract boolean accept(BytesRef value);
+ }
+
+ class PartitionedStringFilter extends StringFilter {
+ @Override
+ public boolean accept(BytesRef value) {
+ return Math.floorMod(StringHelper.murmurhash3_x86_32(value, HASH_PARTITIONING_SEED), incNumPartitions) == incZeroBasedPartition;
+ }
+ }
+
+ /**
+ * String filter backed by an automaton
+ *
+ * @opensearch.internal
+ */
+ static class AutomatonBackedStringFilter extends StringFilter {
+
+ private final ByteRunAutomaton runAutomaton;
+
+ private AutomatonBackedStringFilter(Automaton automaton) {
+ this.runAutomaton = new ByteRunAutomaton(automaton);
+ }
+
+ /**
+ * Returns whether the given value is accepted based on the {@code include} & {@code exclude} patterns.
+ */
+ @Override
+ public boolean accept(BytesRef value) {
+ return runAutomaton.run(value.bytes, value.offset, value.length);
+ }
+ }
+
+ /**
+ * String filter backed by a term list
+ *
+ * @opensearch.internal
+ */
+ static class TermListBackedStringFilter extends StringFilter {
+
+ private final Set<BytesRef> valids;
+ private final Set<BytesRef> invalids;
+
+ TermListBackedStringFilter(Set<BytesRef> includeValues, Set<BytesRef> excludeValues) {
+ this.valids = includeValues;
+ this.invalids = excludeValues;
+ }
+
+ /**
+ * Returns whether the given value is accepted based on the
+ * {@code include} & {@code exclude} sets.
+ */
+ @Override
+ public boolean accept(BytesRef value) {
+ return ((valids == null) || (valids.contains(value))) && ((invalids == null) || (!invalids.contains(value)));
+ }
+ }
+
+ /**
+ * An ordinals filter
+ *
+ * @opensearch.internal
+ */
+ public abstract static class OrdinalsFilter extends Filter {
+ public abstract LongBitSet acceptedGlobalOrdinals(SortedSetDocValues globalOrdinals) throws IOException;
+
+ }
+
+ class PartitionedOrdinalsFilter extends OrdinalsFilter {
+
+ @Override
+ public LongBitSet acceptedGlobalOrdinals(SortedSetDocValues globalOrdinals) throws IOException {
+ final long numOrds = globalOrdinals.getValueCount();
+ final LongBitSet acceptedGlobalOrdinals = new LongBitSet(numOrds);
+ final TermsEnum termEnum = globalOrdinals.termsEnum();
+
+ BytesRef term = termEnum.next();
+ while (term != null) {
+ if (Math.floorMod(
+ StringHelper.murmurhash3_x86_32(term, HASH_PARTITIONING_SEED),
+ incNumPartitions
+ ) == incZeroBasedPartition) {
+ acceptedGlobalOrdinals.set(termEnum.ord());
+ }
+ term = termEnum.next();
+ }
+ return acceptedGlobalOrdinals;
+ }
+ }
+
+ /**
+ * An ordinals filter backed by an automaton
+ *
+ * @opensearch.internal
+ */
+ static class AutomatonBackedOrdinalsFilter extends OrdinalsFilter {
+
+ private final CompiledAutomaton compiled;
+
+ private AutomatonBackedOrdinalsFilter(Automaton automaton) {
+ this.compiled = new CompiledAutomaton(automaton);
+ }
+
+ /**
+ * Computes which global ordinals are accepted by this IncludeExclude instance.
+ *
+ */
+ @Override
+ public LongBitSet acceptedGlobalOrdinals(SortedSetDocValues globalOrdinals) throws IOException {
+ LongBitSet acceptedGlobalOrdinals = new LongBitSet(globalOrdinals.getValueCount());
+ TermsEnum globalTermsEnum;
+ Terms globalTerms = new DocValuesTerms(globalOrdinals);
+ // TODO: specialize based on compiled.type: for ALL and prefixes (sinkState >= 0 ) we can avoid i/o and just set bits.
+ globalTermsEnum = compiled.getTermsEnum(globalTerms);
+ for (BytesRef term = globalTermsEnum.next(); term != null; term = globalTermsEnum.next()) {
+ acceptedGlobalOrdinals.set(globalTermsEnum.ord());
+ }
+ return acceptedGlobalOrdinals;
+ }
+
+ }
+
+ /**
+ * An ordinals filter backed by a terms list
+ *
+ * @opensearch.internal
+ */
+ static class TermListBackedOrdinalsFilter extends OrdinalsFilter {
+
+ private final SortedSet<BytesRef> includeValues;
+ private final SortedSet<BytesRef> excludeValues;
+
+ TermListBackedOrdinalsFilter(SortedSet<BytesRef> includeValues, SortedSet<BytesRef> excludeValues) {
+ this.includeValues = includeValues;
+ this.excludeValues = excludeValues;
+ }
+
+ @Override
+ public LongBitSet acceptedGlobalOrdinals(SortedSetDocValues globalOrdinals) throws IOException {
+ LongBitSet acceptedGlobalOrdinals = new LongBitSet(globalOrdinals.getValueCount());
+ if (includeValues != null) {
+ for (BytesRef term : includeValues) {
+ long ord = globalOrdinals.lookupTerm(term);
+ if (ord >= 0) {
+ acceptedGlobalOrdinals.set(ord);
+ }
+ }
+ } else if (acceptedGlobalOrdinals.length() > 0) {
+ // default to all terms being acceptable
+ acceptedGlobalOrdinals.set(0, acceptedGlobalOrdinals.length());
+ }
+ if (excludeValues != null) {
+ for (BytesRef term : excludeValues) {
+ long ord = globalOrdinals.lookupTerm(term);
+ if (ord >= 0) {
+ acceptedGlobalOrdinals.clear(ord);
+ }
+ }
+ }
+ return acceptedGlobalOrdinals;
+ }
+
+ }
+
+ private final String include, exclude;
+ private final SortedSet<BytesRef> includeValues, excludeValues;
+ private final int incZeroBasedPartition;
+ private final int incNumPartitions;
+
+ /**
+ * @param include The string or regular expression pattern for the terms to be included
+ * @param exclude The string or regular expression pattern for the terms to be excluded
+ */
+ public IncludeExclude(String include, String exclude) {
+ this.include = include;
+ this.exclude = exclude;
+ this.includeValues = null;
+ this.excludeValues = null;
+ this.incZeroBasedPartition = 0;
+ this.incNumPartitions = 0;
+ }
+
+ /**
+ * @param includeValues The terms to be included
+ * @param excludeValues The terms to be excluded
+ */
+ public IncludeExclude(SortedSet<BytesRef> includeValues, SortedSet<BytesRef> excludeValues) {
+ if (includeValues == null && excludeValues == null) {
+ throw new IllegalArgumentException();
+ }
+ this.include = null;
+ this.exclude = null;
+ this.incZeroBasedPartition = 0;
+ this.incNumPartitions = 0;
+ this.includeValues = includeValues;
+ this.excludeValues = excludeValues;
+ }
+
+ public IncludeExclude(String[] includeValues, String[] excludeValues) {
+ this(convertToBytesRefSet(includeValues), convertToBytesRefSet(excludeValues));
+ }
+
+ public IncludeExclude(double[] includeValues, double[] excludeValues) {
+ this(convertToBytesRefSet(includeValues), convertToBytesRefSet(excludeValues));
+ }
+
+ public IncludeExclude(long[] includeValues, long[] excludeValues) {
+ this(convertToBytesRefSet(includeValues), convertToBytesRefSet(excludeValues));
+ }
+
+ public IncludeExclude(int partition, int numPartitions) {
+ if (partition < 0 || partition >= numPartitions) {
+ throw new IllegalArgumentException("Partition must be >=0 and < numPartition which is " + numPartitions);
+ }
+ this.incZeroBasedPartition = partition;
+ this.incNumPartitions = numPartitions;
+ this.include = null;
+ this.exclude = null;
+ this.includeValues = null;
+ this.excludeValues = null;
+
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public IncludeExclude(StreamInput in) throws IOException {
+ if (in.readBoolean()) {
+ includeValues = null;
+ excludeValues = null;
+ incZeroBasedPartition = 0;
+ incNumPartitions = 0;
+ include = in.readOptionalString();
+ exclude = in.readOptionalString();
+ return;
+ }
+ include = null;
+ exclude = null;
+ if (in.readBoolean()) {
+ int size = in.readVInt();
+ includeValues = new TreeSet<>();
+ for (int i = 0; i < size; i++) {
+ includeValues.add(in.readBytesRef());
+ }
+ } else {
+ includeValues = null;
+ }
+ if (in.readBoolean()) {
+ int size = in.readVInt();
+ excludeValues = new TreeSet<>();
+ for (int i = 0; i < size; i++) {
+ excludeValues.add(in.readBytesRef());
+ }
+ } else {
+ excludeValues = null;
+ }
+ incNumPartitions = in.readVInt();
+ incZeroBasedPartition = in.readVInt();
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ boolean regexBased = isRegexBased();
+ out.writeBoolean(regexBased);
+ if (regexBased) {
+ out.writeOptionalString(include);
+ out.writeOptionalString(exclude);
+ } else {
+ boolean hasIncludes = includeValues != null;
+ out.writeBoolean(hasIncludes);
+ if (hasIncludes) {
+ out.writeVInt(includeValues.size());
+ for (BytesRef value : includeValues) {
+ out.writeBytesRef(value);
+ }
+ }
+ boolean hasExcludes = excludeValues != null;
+ out.writeBoolean(hasExcludes);
+ if (hasExcludes) {
+ out.writeVInt(excludeValues.size());
+ for (BytesRef value : excludeValues) {
+ out.writeBytesRef(value);
+ }
+ }
+ out.writeVInt(incNumPartitions);
+ out.writeVInt(incZeroBasedPartition);
+ }
+ }
+
+ private static SortedSet<BytesRef> convertToBytesRefSet(String[] values) {
+ SortedSet<BytesRef> returnSet = null;
+ if (values != null) {
+ returnSet = new TreeSet<>();
+ for (String value : values) {
+ returnSet.add(new BytesRef(value));
+ }
+ }
+ return returnSet;
+ }
+
+ private static SortedSet<BytesRef> convertToBytesRefSet(double[] values) {
+ SortedSet<BytesRef> returnSet = null;
+ if (values != null) {
+ returnSet = new TreeSet<>();
+ for (double value : values) {
+ returnSet.add(new BytesRef(String.valueOf(value)));
+ }
+ }
+ return returnSet;
+ }
+
+ private static SortedSet<BytesRef> convertToBytesRefSet(long[] values) {
+ SortedSet<BytesRef> returnSet = null;
+ if (values != null) {
+ returnSet = new TreeSet<>();
+ for (long value : values) {
+ returnSet.add(new BytesRef(String.valueOf(value)));
+ }
+ }
+ return returnSet;
+ }
+
+ /**
+ * Terms adapter around doc values.
+ *
+ * @opensearch.internal
+ */
+ private static class DocValuesTerms extends Terms {
+
+ private final SortedSetDocValues values;
+
+ DocValuesTerms(SortedSetDocValues values) {
+ this.values = values;
+ }
+
+ @Override
+ public TermsEnum iterator() throws IOException {
+ return values.termsEnum();
+ }
+
+ @Override
+ public long size() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public long getSumTotalTermFreq() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public long getSumDocFreq() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public int getDocCount() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public boolean hasFreqs() {
+ return false;
+ }
+
+ @Override
+ public boolean hasOffsets() {
+ return false;
+ }
+
+ @Override
+ public boolean hasPositions() {
+ return false;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return false;
+ }
+
+ }
+
+ private static Set<BytesRef> parseArrayToSet(XContentParser parser) throws IOException {
+ final Set<BytesRef> set = new HashSet<>();
+ if (parser.currentToken() != XContentParser.Token.START_ARRAY) {
+ throw new OpenSearchParseException("Missing start of array in include/exclude clause");
+ }
+ while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+ if (!parser.currentToken().isValue()) {
+ throw new OpenSearchParseException("Array elements in include/exclude clauses should be string values");
+ }
+ set.add(new BytesRef(parser.text()));
+ }
+ return set;
+ }
+
+ public boolean isRegexBased() {
+ return include != null || exclude != null;
+ }
+
+ public boolean isPartitionBased() {
+ return incNumPartitions > 0;
+ }
+
+ private Automaton toAutomaton(int maxRegExLength) {
+ Automaton a;
+ if (include != null) {
+ validateRegExpStringLength(include, maxRegExLength);
+ a = new RegExp(include).toAutomaton();
+ } else if (includeValues != null) {
+ a = Automata.makeStringUnion(includeValues);
+ } else {
+ a = Automata.makeAnyString();
+ }
+ if (exclude != null) {
+ validateRegExpStringLength(exclude, maxRegExLength);
+ Automaton excludeAutomaton = new RegExp(exclude).toAutomaton();
+ a = Operations.minus(a, excludeAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+ } else if (excludeValues != null) {
+ a = Operations.minus(a, Automata.makeStringUnion(excludeValues), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+ }
+ return a;
+ }
+
+ private static void validateRegExpStringLength(String source, int maxRegexLength) {
+ if (maxRegexLength > 0 && source.length() > maxRegexLength) {
+ throw new IllegalArgumentException(
+ "The length of regex ["
+ + source.length()
+ + "] used in the request has exceeded "
+ + "the allowed maximum of ["
+ + maxRegexLength
+ + "]. "
+ + "This maximum can be set by changing the ["
+ + IndexSettings.MAX_REGEX_LENGTH_SETTING.getKey()
+ + "] index level setting."
+ );
+ }
+ }
+
+ /**
+ * Wrapper method that imposes a default regex limit.
+ * See https://github.com/opensearch-project/OpenSearch/issues/2858
+ */
+ public StringFilter convertToStringFilter(DocValueFormat format) {
+ return convertToStringFilter(format, DEFAULT_MAX_REGEX_LENGTH);
+ }
+
+ public StringFilter convertToStringFilter(DocValueFormat format, int maxRegexLength) {
+ if (isRegexBased()) {
+ return new AutomatonBackedStringFilter(toAutomaton(maxRegexLength));
+ }
+ if (isPartitionBased()) {
+ return new PartitionedStringFilter();
+ }
+ return new TermListBackedStringFilter(parseForDocValues(includeValues, format), parseForDocValues(excludeValues, format));
+ }
+
+ private static SortedSet<BytesRef> parseForDocValues(SortedSet<BytesRef> endUserFormattedValues, DocValueFormat format) {
+ SortedSet<BytesRef> result = endUserFormattedValues;
+ if (endUserFormattedValues != null) {
+ if (format != DocValueFormat.RAW) {
+ result = new TreeSet<>();
+ for (BytesRef formattedVal : endUserFormattedValues) {
+ result.add(format.parseBytesRef(formattedVal.utf8ToString()));
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Wrapper method that imposes a default regex limit.
+ * See https://github.com/opensearch-project/OpenSearch/issues/2858
+ */
+ public OrdinalsFilter convertToOrdinalsFilter(DocValueFormat format) {
+ return convertToOrdinalsFilter(format, DEFAULT_MAX_REGEX_LENGTH);
+ }
+
+ public OrdinalsFilter convertToOrdinalsFilter(DocValueFormat format, int maxRegexLength) {
+
+ if (isRegexBased()) {
+ return new AutomatonBackedOrdinalsFilter(toAutomaton(maxRegexLength));
+ }
+ if (isPartitionBased()) {
+ return new PartitionedOrdinalsFilter();
+ }
+
+ return new TermListBackedOrdinalsFilter(parseForDocValues(includeValues, format), parseForDocValues(excludeValues, format));
+ }
+
+ public LongFilter convertToLongFilter(DocValueFormat format) {
+
+ if (isPartitionBased()) {
+ return new PartitionedLongFilter();
+ }
+
+ int numValids = includeValues == null ? 0 : includeValues.size();
+ int numInvalids = excludeValues == null ? 0 : excludeValues.size();
+ SetBackedLongFilter result = new SetBackedLongFilter(numValids, numInvalids);
+ if (includeValues != null) {
+ for (BytesRef val : includeValues) {
+ result.addAccept(format.parseLong(val.utf8ToString(), false, null));
+ }
+ }
+ if (excludeValues != null) {
+ for (BytesRef val : excludeValues) {
+ result.addReject(format.parseLong(val.utf8ToString(), false, null));
+ }
+ }
+ return result;
+ }
+
+ public LongFilter convertToDoubleFilter() {
+ if (isPartitionBased()) {
+ return new PartitionedLongFilter();
+ }
+
+ int numValids = includeValues == null ? 0 : includeValues.size();
+ int numInvalids = excludeValues == null ? 0 : excludeValues.size();
+ SetBackedLongFilter result = new SetBackedLongFilter(numValids, numInvalids);
+ if (includeValues != null) {
+ for (BytesRef val : includeValues) {
+ double dval = Double.parseDouble(val.utf8ToString());
+ result.addAccept(NumericUtils.doubleToSortableLong(dval));
+ }
+ }
+ if (excludeValues != null) {
+ for (BytesRef val : excludeValues) {
+ double dval = Double.parseDouble(val.utf8ToString());
+ result.addReject(NumericUtils.doubleToSortableLong(dval));
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ if (include != null) {
+ builder.field(INCLUDE_FIELD.getPreferredName(), include);
+ } else if (includeValues != null) {
+ builder.startArray(INCLUDE_FIELD.getPreferredName());
+ for (BytesRef value : includeValues) {
+ builder.value(value.utf8ToString());
+ }
+ builder.endArray();
+ } else if (isPartitionBased()) {
+ builder.startObject(INCLUDE_FIELD.getPreferredName());
+ builder.field(PARTITION_FIELD.getPreferredName(), incZeroBasedPartition);
+ builder.field(NUM_PARTITIONS_FIELD.getPreferredName(), incNumPartitions);
+ builder.endObject();
+ }
+ if (exclude != null) {
+ builder.field(EXCLUDE_FIELD.getPreferredName(), exclude);
+ } else if (excludeValues != null) {
+ builder.startArray(EXCLUDE_FIELD.getPreferredName());
+ for (BytesRef value : excludeValues) {
+ builder.value(value.utf8ToString());
+ }
+ builder.endArray();
+ }
+ return builder;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(include, exclude, includeValues, excludeValues, incZeroBasedPartition, incNumPartitions);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ IncludeExclude other = (IncludeExclude) obj;
+ return Objects.equals(include, other.include)
+ && Objects.equals(exclude, other.exclude)
+ && Objects.equals(includeValues, other.includeValues)
+ && Objects.equals(excludeValues, other.excludeValues)
+ && Objects.equals(incZeroBasedPartition, other.incZeroBasedPartition)
+ && Objects.equals(incNumPartitions, other.incNumPartitions);
+ }
+
+ }
+
+
+
+
+
+
Coverage Summary for Class: InternalMappedRareTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | InternalMappedRareTerms |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ 0%
+
+
+ (0/11)
+
+ |
+
+
+ 0%
+
+
+ (0/59)
+
+ |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.logging.log4j.LogManager;
+ import org.apache.logging.log4j.Logger;
+ import org.apache.lucene.util.CollectionUtil;
+ import org.opensearch.common.Randomness;
+ import org.opensearch.common.util.SetBackedScalingCuckooFilter;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.AggregationExecutionException;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.InternalAggregation;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+ import java.util.function.Function;
+ import java.util.stream.Collectors;
+
+ /**
+ * Implementation of mapped rare terms
+ *
+ * @opensearch.internal
+ */
+ public abstract class InternalMappedRareTerms<A extends InternalRareTerms<A, B>, B extends InternalRareTerms.Bucket<B>> extends
+ InternalRareTerms<A, B> {
+
+ protected DocValueFormat format;
+ protected List<B> buckets;
+ protected Map<String, B> bucketMap;
+
+ final SetBackedScalingCuckooFilter filter;
+
+ protected final Logger logger = LogManager.getLogger(getClass());
+
+ InternalMappedRareTerms(
+ String name,
+ BucketOrder order,
+ Map<String, Object> metadata,
+ DocValueFormat format,
+ List<B> buckets,
+ long maxDocCount,
+ SetBackedScalingCuckooFilter filter
+ ) {
+ super(name, order, maxDocCount, metadata);
+ this.format = format;
+ this.buckets = buckets;
+ this.filter = filter;
+ }
+
+ public long getMaxDocCount() {
+ return maxDocCount;
+ }
+
+ SetBackedScalingCuckooFilter getFilter() {
+ return filter;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ InternalMappedRareTerms(StreamInput in, Bucket.Reader<B> bucketReader) throws IOException {
+ super(in);
+ format = in.readNamedWriteable(DocValueFormat.class);
+ buckets = in.readList(stream -> bucketReader.read(stream, format));
+ filter = new SetBackedScalingCuckooFilter(in, Randomness.get());
+ }
+
+ @Override
+ protected void writeTermTypeInfoTo(StreamOutput out) throws IOException {
+ out.writeNamedWriteable(format);
+ out.writeList(buckets);
+ filter.writeTo(out);
+ }
+
+ @Override
+ public InternalAggregation reduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+ Map<Object, List<B>> buckets = new HashMap<>();
+ InternalRareTerms<A, B> referenceTerms = null;
+ SetBackedScalingCuckooFilter filter = null;
+
+ for (InternalAggregation aggregation : aggregations) {
+ // Unmapped rare terms don't have a cuckoo filter so we'll skip all this work
+ // and save some type casting headaches later.
+ if (aggregation.isMapped() == false) {
+ continue;
+ }
+
+ @SuppressWarnings("unchecked")
+ InternalRareTerms<A, B> terms = (InternalRareTerms<A, B>) aggregation;
+ if (referenceTerms == null && aggregation.getClass().equals(UnmappedRareTerms.class) == false) {
+ referenceTerms = terms;
+ }
+ if (referenceTerms != null
+ && referenceTerms.getClass().equals(terms.getClass()) == false
+ && terms.getClass().equals(UnmappedRareTerms.class) == false) {
+ // control gets into this loop when the same field name against which the query is executed
+ // is of different types in different indices.
+ throw new AggregationExecutionException(
+ "Merging/Reducing the aggregations failed when computing the aggregation ["
+ + referenceTerms.getName()
+ + "] because the field you gave in the aggregation query existed as two different "
+ + "types in two different indices"
+ );
+ }
+ for (B bucket : terms.getBuckets()) {
+ List<B> bucketList = buckets.computeIfAbsent(bucket.getKey(), k -> new ArrayList<>());
+ bucketList.add(bucket);
+ }
+
+ SetBackedScalingCuckooFilter otherFilter = ((InternalMappedRareTerms) aggregation).getFilter();
+ if (filter == null) {
+ filter = new SetBackedScalingCuckooFilter(otherFilter);
+ } else {
+ filter.merge(otherFilter);
+ }
+ }
+
+ final List<B> rare = new ArrayList<>();
+ for (List<B> sameTermBuckets : buckets.values()) {
+ final B b = reduceBucket(sameTermBuckets, reduceContext);
+ if ((b.getDocCount() <= maxDocCount && containsTerm(filter, b) == false)) {
+ rare.add(b);
+ reduceContext.consumeBucketsAndMaybeBreak(1);
+ } else if (b.getDocCount() > maxDocCount) {
+ // this term has gone over threshold while merging, so add it to the filter.
+ // Note this may happen during incremental reductions too
+ addToFilter(filter, b);
+ }
+ }
+ CollectionUtil.introSort(rare, order.comparator());
+ return createWithFilter(name, rare, filter);
+ }
+
+ public abstract boolean containsTerm(SetBackedScalingCuckooFilter filter, B bucket);
+
+ public abstract void addToFilter(SetBackedScalingCuckooFilter filter, B bucket);
+
+ @Override
+ public List<B> getBuckets() {
+ return buckets;
+ }
+
+ @Override
+ public B getBucketByKey(String term) {
+ if (bucketMap == null) {
+ bucketMap = buckets.stream().collect(Collectors.toMap(InternalRareTerms.Bucket::getKeyAsString, Function.identity()));
+ }
+ return bucketMap.get(term);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+ InternalMappedRareTerms<?, ?> that = (InternalMappedRareTerms<?, ?>) obj;
+ return Objects.equals(buckets, that.buckets) && Objects.equals(format, that.format) && Objects.equals(filter, that.filter);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), buckets, format, filter);
+ }
+
+ @Override
+ public final XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+ return doXContentCommon(builder, params, buckets);
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: InternalMappedSignificantTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | InternalMappedSignificantTerms |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ 0%
+
+
+ (0/12)
+
+ |
+
+
+ 0%
+
+
+ (0/43)
+
+ |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+
+ import java.io.IOException;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+ import java.util.function.Function;
+ import java.util.stream.Collectors;
+
+ /**
+ * Implementation of mapped significant terms
+ *
+ * @opensearch.internal
+ */
+ public abstract class InternalMappedSignificantTerms<
+ A extends InternalMappedSignificantTerms<A, B>,
+ B extends InternalSignificantTerms.Bucket<B>> extends InternalSignificantTerms<A, B> {
+
+ protected final DocValueFormat format;
+ protected final long subsetSize;
+ protected final long supersetSize;
+ protected final SignificanceHeuristic significanceHeuristic;
+ protected final List<B> buckets;
+ protected Map<String, B> bucketMap;
+
+ protected InternalMappedSignificantTerms(
+ String name,
+ Map<String, Object> metadata,
+ DocValueFormat format,
+ long subsetSize,
+ long supersetSize,
+ SignificanceHeuristic significanceHeuristic,
+ List<B> buckets,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds
+ ) {
+ super(name, bucketCountThresholds, metadata);
+ this.format = format;
+ this.buckets = buckets;
+ this.subsetSize = subsetSize;
+ this.supersetSize = supersetSize;
+ this.significanceHeuristic = significanceHeuristic;
+ }
+
+ protected InternalMappedSignificantTerms(StreamInput in, Bucket.Reader<B> bucketReader) throws IOException {
+ super(in);
+ format = in.readNamedWriteable(DocValueFormat.class);
+ subsetSize = in.readVLong();
+ supersetSize = in.readVLong();
+ significanceHeuristic = in.readNamedWriteable(SignificanceHeuristic.class);
+ buckets = in.readList(stream -> bucketReader.read(stream, subsetSize, supersetSize, format));
+ }
+
+ @Override
+ protected final void writeTermTypeInfoTo(StreamOutput out) throws IOException {
+ out.writeNamedWriteable(format);
+ out.writeVLong(subsetSize);
+ out.writeVLong(supersetSize);
+ out.writeNamedWriteable(significanceHeuristic);
+ out.writeList(buckets);
+ }
+
+ @Override
+ public Iterator<SignificantTerms.Bucket> iterator() {
+ return buckets.stream().map(bucket -> (SignificantTerms.Bucket) bucket).collect(Collectors.toList()).iterator();
+ }
+
+ @Override
+ public List<B> getBuckets() {
+ return buckets;
+ }
+
+ @Override
+ public B getBucketByKey(String term) {
+ if (bucketMap == null) {
+ bucketMap = buckets.stream().collect(Collectors.toMap(InternalSignificantTerms.Bucket::getKeyAsString, Function.identity()));
+ }
+ return bucketMap.get(term);
+ }
+
+ @Override
+ protected long getSubsetSize() {
+ return subsetSize;
+ }
+
+ @Override
+ protected long getSupersetSize() {
+ return supersetSize;
+ }
+
+ @Override
+ protected SignificanceHeuristic getSignificanceHeuristic() {
+ return significanceHeuristic;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+
+ InternalMappedSignificantTerms<?, ?> that = (InternalMappedSignificantTerms<?, ?>) obj;
+ return Objects.equals(format, that.format)
+ && subsetSize == that.subsetSize
+ && supersetSize == that.supersetSize
+ && Objects.equals(significanceHeuristic, that.significanceHeuristic)
+ && Objects.equals(buckets, that.buckets)
+ && Objects.equals(bucketMap, that.bucketMap);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), format, subsetSize, supersetSize, significanceHeuristic, buckets, bucketMap);
+ }
+
+ @Override
+ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+ builder.field(CommonFields.DOC_COUNT.getPreferredName(), subsetSize);
+ builder.field(BG_COUNT, supersetSize);
+ builder.startArray(CommonFields.BUCKETS.getPreferredName());
+ for (Bucket bucket : buckets) {
+ // There is a condition (presumably when only one shard has a bucket?) where reduce is not called
+ // and I end up with buckets that contravene the user's min_doc_count criteria in my reducer
+ if (bucket.subsetDf >= minDocCount) {
+ bucket.toXContent(builder, params);
+ }
+ }
+ builder.endArray();
+ return builder;
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: InternalMappedTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+ | Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | InternalMappedTerms |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 50%
+
+
+ (6/12)
+
+ |
+
+
+ 30%
+
+
+ (12/40)
+
+ |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.BucketOrder;
+
+ import java.io.IOException;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+ import java.util.function.Function;
+ import java.util.stream.Collectors;
+
+ /**
+ * Common superclass for results of the terms aggregation on mapped fields.
+ *
+ * @opensearch.internal
+ */
+ public abstract class InternalMappedTerms<A extends InternalTerms<A, B>, B extends InternalTerms.Bucket<B>> extends InternalTerms<A, B> {
+ protected final DocValueFormat format;
+ protected final int shardSize;
+ protected final boolean showTermDocCountError;
+ protected final long otherDocCount;
+ protected final List<B> buckets;
+ protected Map<String, B> bucketMap;
+
+ protected long docCountError;
+
+ protected InternalMappedTerms(
+ String name,
+ BucketOrder reduceOrder,
+ BucketOrder order,
+ Map<String, Object> metadata,
+ DocValueFormat format,
+ int shardSize,
+ boolean showTermDocCountError,
+ long otherDocCount,
+ List<B> buckets,
+ long docCountError,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds
+ ) {
+ super(name, reduceOrder, order, bucketCountThresholds, metadata);
+ this.format = format;
+ this.shardSize = shardSize;
+ this.showTermDocCountError = showTermDocCountError;
+ this.otherDocCount = otherDocCount;
+ this.docCountError = docCountError;
+ this.buckets = buckets;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ protected InternalMappedTerms(StreamInput in, Bucket.Reader<B> bucketReader) throws IOException {
+ super(in);
+ docCountError = in.readZLong();
+ format = in.readNamedWriteable(DocValueFormat.class);
+ shardSize = readSize(in);
+ showTermDocCountError = in.readBoolean();
+ otherDocCount = in.readVLong();
+ buckets = in.readList(stream -> bucketReader.read(stream, format, showTermDocCountError));
+ }
+
+ @Override
+ protected final void writeTermTypeInfoTo(StreamOutput out) throws IOException {
+ out.writeZLong(docCountError);
+ out.writeNamedWriteable(format);
+ writeSize(shardSize, out);
+ out.writeBoolean(showTermDocCountError);
+ out.writeVLong(otherDocCount);
+ out.writeList(buckets);
+ }
+
+ @Override
+ protected void setDocCountError(long docCountError) {
+ this.docCountError = docCountError;
+ }
+
+ @Override
+ protected int getShardSize() {
+ return shardSize;
+ }
+
+ @Override
+ public long getDocCountError() {
+ return docCountError;
+ }
+
+ @Override
+ public long getSumOfOtherDocCounts() {
+ return otherDocCount;
+ }
+
+ @Override
+ public List<B> getBuckets() {
+ return buckets;
+ }
+
+ @Override
+ public B getBucketByKey(String term) {
+ if (bucketMap == null) {
+ bucketMap = buckets.stream().collect(Collectors.toMap(Bucket::getKeyAsString, Function.identity()));
+ }
+ return bucketMap.get(term);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+
+ InternalMappedTerms<?, ?> that = (InternalMappedTerms<?, ?>) obj;
+ return Objects.equals(buckets, that.buckets)
+ && Objects.equals(format, that.format)
+ && Objects.equals(otherDocCount, that.otherDocCount)
+ && Objects.equals(showTermDocCountError, that.showTermDocCountError)
+ && Objects.equals(shardSize, that.shardSize)
+ && Objects.equals(docCountError, that.docCountError);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), buckets, format, otherDocCount, showTermDocCountError, shardSize);
+ }
+
+ @Override
+ public final XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+ return doXContentCommon(builder, params, docCountError, otherDocCount, buckets);
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: InternalMultiTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | InternalMultiTerms |
+
+
+ 0%
+
+
+ (0/19)
+
+ |
+
+
+ 0%
+
+
+ (0/65)
+
+ |
+
+
+ | InternalMultiTerms$Bucket |
+
+
+ 0%
+
+
+ (0/17)
+
+ |
+
+
+ 0%
+
+
+ (0/54)
+
+ |
+
+
+ | InternalMultiTerms$Bucket$BucketComparator |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/10)
+
+ |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/38)
+
+ |
+
+
+ 0%
+
+
+ (0/129)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.util.BytesRef;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.AggregationExecutionException;
+ import org.opensearch.search.aggregations.Aggregations;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.InternalAggregations;
+ import org.opensearch.search.aggregations.KeyComparable;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.Comparator;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+ import java.util.function.Function;
+ import java.util.stream.Collectors;
+
+ /**
+ * Result of the {@link MultiTermsAggregator}.
+ *
+ * @opensearch.internal
+ */
+ public class InternalMultiTerms extends InternalTerms<InternalMultiTerms, InternalMultiTerms.Bucket> {
+ /**
+ * Internal Multi Terms Bucket.
+ *
+ * @opensearch.internal
+ */
+ public static class Bucket extends InternalTerms.AbstractInternalBucket implements KeyComparable<Bucket> {
+
+ protected long bucketOrd;
+ /**
+ * list of terms values.
+ */
+ protected List<Object> termValues;
+ protected long docCount;
+ protected InternalAggregations aggregations;
+ protected boolean showDocCountError;
+ protected long docCountError;
+ /**
+ * A list of term's {@link DocValueFormat}.
+ */
+ protected final List<DocValueFormat> termFormats;
+
+ private static final String PIPE = "|";
+
+ /**
+ * Create default {@link Bucket}.
+ */
+ public static Bucket EMPTY(boolean showTermDocCountError, List<DocValueFormat> formats) {
+ return new Bucket(null, 0, null, showTermDocCountError, 0, formats);
+ }
+
+ public Bucket(
+ List<Object> values,
+ long docCount,
+ InternalAggregations aggregations,
+ boolean showDocCountError,
+ long docCountError,
+ List<DocValueFormat> formats
+ ) {
+ this.termValues = values;
+ this.docCount = docCount;
+ this.aggregations = aggregations;
+ this.showDocCountError = showDocCountError;
+ this.docCountError = docCountError;
+ this.termFormats = formats;
+ }
+
+ public Bucket(StreamInput in, List<DocValueFormat> formats, boolean showDocCountError) throws IOException {
+ this.termValues = in.readList(StreamInput::readGenericValue);
+ this.docCount = in.readVLong();
+ this.aggregations = InternalAggregations.readFrom(in);
+ this.showDocCountError = showDocCountError;
+ this.docCountError = -1;
+ if (showDocCountError) {
+ this.docCountError = in.readLong();
+ }
+ this.termFormats = formats;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject();
+ builder.field(CommonFields.KEY.getPreferredName(), getKey());
+ builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString());
+ builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+ if (showDocCountError) {
+ builder.field(DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName(), getDocCountError());
+ }
+ aggregations.toXContentInternal(builder, params);
+ builder.endObject();
+ return builder;
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeCollection(termValues, StreamOutput::writeGenericValue);
+ out.writeVLong(docCount);
+ aggregations.writeTo(out);
+ if (showDocCountError) {
+ out.writeLong(docCountError);
+ }
+ }
+
+ @Override
+ public List<Object> getKey() {
+ List<Object> keys = new ArrayList<>(termValues.size());
+ for (int i = 0; i < termValues.size(); i++) {
+ keys.add(formatObject(termValues.get(i), termFormats.get(i)));
+ }
+ return keys;
+ }
+
+ @Override
+ public String getKeyAsString() {
+ return getKey().stream().map(Object::toString).collect(Collectors.joining(PIPE));
+ }
+
+ @Override
+ public long getDocCount() {
+ return docCount;
+ }
+
+ @Override
+ public Aggregations getAggregations() {
+ return aggregations;
+ }
+
+ @Override
+ void setDocCountError(long docCountError) {
+ this.docCountError = docCountError;
+ }
+
+ @Override
+ public void setDocCountError(Function<Long, Long> updater) {
+ this.docCountError = updater.apply(this.docCountError);
+ }
+
+ @Override
+ public boolean showDocCountError() {
+ return showDocCountError;
+ }
+
+ @Override
+ public Number getKeyAsNumber() {
+ throw new IllegalArgumentException("getKeyAsNumber is not supported by [" + MultiTermsAggregationBuilder.NAME + "]");
+ }
+
+ @Override
+ public long getDocCountError() {
+ if (!showDocCountError) {
+ throw new IllegalStateException("show_terms_doc_count_error is false");
+ }
+ return docCountError;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ Bucket other = (Bucket) obj;
+ if (showDocCountError && docCountError != other.docCountError) {
+ return false;
+ }
+ return termValues.equals(other.termValues)
+ && docCount == other.docCount
+ && aggregations.equals(other.aggregations)
+ && showDocCountError == other.showDocCountError;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(termValues, docCount, aggregations, showDocCountError, showDocCountError ? docCountError : 0);
+ }
+
+ @Override
+ public int compareKey(Bucket other) {
+ return new BucketComparator().compare(this.termValues, other.termValues);
+ }
+
+ /**
+ * Visible for testing.
+ *
+ * @opensearch.internal
+ */
+ protected static class BucketComparator implements Comparator<List<Object>> {
+ @SuppressWarnings({ "unchecked" })
+ @Override
+ public int compare(List<Object> thisObjects, List<Object> thatObjects) {
+ if (thisObjects.size() != thatObjects.size()) {
+ throw new AggregationExecutionException(
+ "[" + MultiTermsAggregationBuilder.NAME + "] aggregations failed due to terms" + " size is different"
+ );
+ }
+ for (int i = 0; i < thisObjects.size(); i++) {
+ final Object thisObject = thisObjects.get(i);
+ final Object thatObject = thatObjects.get(i);
+ int ret = ((Comparable) thisObject).compareTo(thatObject);
+ if (ret != 0) {
+ return ret;
+ }
+ }
+ return 0;
+ }
+ }
+ }
+
+ private final int shardSize;
+ private final boolean showTermDocCountError;
+ private final long otherDocCount;
+ private final List<DocValueFormat> termFormats;
+ private final List<Bucket> buckets;
+ private Map<String, Bucket> bucketMap;
+
+ private long docCountError;
+
+ public InternalMultiTerms(
+ String name,
+ BucketOrder reduceOrder,
+ BucketOrder order,
+ Map<String, Object> metadata,
+ int shardSize,
+ boolean showTermDocCountError,
+ long otherDocCount,
+ long docCountError,
+ List<DocValueFormat> formats,
+ List<Bucket> buckets,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds
+ ) {
+ super(name, reduceOrder, order, bucketCountThresholds, metadata);
+ this.shardSize = shardSize;
+ this.showTermDocCountError = showTermDocCountError;
+ this.otherDocCount = otherDocCount;
+ this.termFormats = formats;
+ this.buckets = buckets;
+ this.docCountError = docCountError;
+ }
+
+ public InternalMultiTerms(StreamInput in) throws IOException {
+ super(in);
+ this.docCountError = in.readZLong();
+ this.termFormats = in.readList(stream -> stream.readNamedWriteable(DocValueFormat.class));
+ this.shardSize = readSize(in);
+ this.showTermDocCountError = in.readBoolean();
+ this.otherDocCount = in.readVLong();
+ this.buckets = in.readList(steam -> new Bucket(steam, termFormats, showTermDocCountError));
+ }
+
+ @Override
+ public String getWriteableName() {
+ return MultiTermsAggregationBuilder.NAME;
+ }
+
+ @Override
+ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+ return doXContentCommon(builder, params, docCountError, otherDocCount, buckets);
+ }
+
+ @Override
+ public InternalMultiTerms create(List<Bucket> buckets) {
+ return new InternalMultiTerms(
+ name,
+ reduceOrder,
+ order,
+ metadata,
+ shardSize,
+ showTermDocCountError,
+ otherDocCount,
+ docCountError,
+ termFormats,
+ buckets,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) {
+ return new Bucket(
+ prototype.termValues,
+ prototype.docCount,
+ aggregations,
+ prototype.showDocCountError,
+ prototype.docCountError,
+ prototype.termFormats
+ );
+ }
+
+ @Override
+ protected void writeTermTypeInfoTo(StreamOutput out) throws IOException {
+ out.writeZLong(docCountError);
+ out.writeCollection(termFormats, StreamOutput::writeNamedWriteable);
+ writeSize(shardSize, out);
+ out.writeBoolean(showTermDocCountError);
+ out.writeVLong(otherDocCount);
+ out.writeList(buckets);
+ }
+
+ @Override
+ public List<Bucket> getBuckets() {
+ return buckets;
+ }
+
+ @Override
+ public Bucket getBucketByKey(String term) {
+ if (bucketMap == null) {
+ bucketMap = buckets.stream().collect(Collectors.toMap(InternalMultiTerms.Bucket::getKeyAsString, Function.identity()));
+ }
+ return bucketMap.get(term);
+ }
+
+ @Override
+ public long getDocCountError() {
+ return docCountError;
+ }
+
+ @Override
+ public long getSumOfOtherDocCounts() {
+ return otherDocCount;
+ }
+
+ @Override
+ protected void setDocCountError(long docCountError) {
+ this.docCountError = docCountError;
+ }
+
+ @Override
+ protected int getShardSize() {
+ return shardSize;
+ }
+
+ @Override
+ protected InternalMultiTerms create(
+ String name,
+ List<Bucket> buckets,
+ BucketOrder reduceOrder,
+ long docCountError,
+ long otherDocCount
+ ) {
+ return new InternalMultiTerms(
+ name,
+ reduceOrder,
+ order,
+ metadata,
+ shardSize,
+ showTermDocCountError,
+ otherDocCount,
+ docCountError,
+ termFormats,
+ buckets,
+ bucketCountThresholds
+ );
+ }
+
+ @Override
+ protected Bucket[] createBucketsArray(int size) {
+ return new Bucket[size];
+ }
+
+ @Override
+ Bucket createBucket(long docCount, InternalAggregations aggs, long docCountError, Bucket prototype) {
+ return new Bucket(
+ prototype.termValues,
+ docCount,
+ aggs,
+ prototype.showDocCountError,
+ prototype.docCountError,
+ prototype.termFormats
+ );
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+ InternalMultiTerms that = (InternalMultiTerms) obj;
+
+ if (showTermDocCountError && docCountError != that.docCountError) {
+ return false;
+ }
+ return Objects.equals(buckets, that.buckets)
+ && Objects.equals(otherDocCount, that.otherDocCount)
+ && Objects.equals(showTermDocCountError, that.showTermDocCountError)
+ && Objects.equals(shardSize, that.shardSize)
+ && Objects.equals(docCountError, that.docCountError);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), buckets, otherDocCount, showTermDocCountError, shardSize);
+ }
+
+ /**
+ * Copy from InternalComposite
+ * <p>
+ * Format {@code obj} using the provided {@link DocValueFormat}.
+ * If the format is equals to {@link DocValueFormat#RAW}, the object is returned as is
+ * for numbers and a string for {@link BytesRef}s.
+ */
+ static Object formatObject(Object obj, DocValueFormat format) {
+ if (obj == null) {
+ return null;
+ }
+ if (obj.getClass() == BytesRef.class) {
+ BytesRef value = (BytesRef) obj;
+ if (format == DocValueFormat.RAW) {
+ return value.utf8ToString();
+ } else {
+ return format.format(value);
+ }
+ } else if (obj.getClass() == Long.class) {
+ long value = (long) obj;
+ if (format == DocValueFormat.RAW) {
+ return value;
+ } else {
+ return format.format(value);
+ }
+ } else if (obj.getClass() == Double.class) {
+ double value = (double) obj;
+ if (format == DocValueFormat.RAW) {
+ return value;
+ } else {
+ return format.format(value);
+ }
+ }
+ return obj;
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: InternalRareTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | InternalRareTerms |
+
+
+ 0%
+
+
+ (0/9)
+
+ |
+
+
+ 0%
+
+
+ (0/32)
+
+ |
+
+
+ | InternalRareTerms$Bucket |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ 0%
+
+
+ (0/24)
+
+ |
+
+
+ | InternalRareTerms$Bucket$Reader |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/17)
+
+ |
+
+
+ 0%
+
+
+ (0/56)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.common.util.SetBackedScalingCuckooFilter;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregations;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.InternalAggregations;
+ import org.opensearch.search.aggregations.InternalMultiBucketAggregation;
+ import org.opensearch.search.aggregations.InternalOrder;
+ import org.opensearch.search.aggregations.KeyComparable;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+
+ /**
+ * Implementation of rare terms
+ *
+ * @opensearch.internal
+ */
+ public abstract class InternalRareTerms<A extends InternalRareTerms<A, B>, B extends InternalRareTerms.Bucket<B>> extends
+ InternalMultiBucketAggregation<A, B>
+ implements
+ RareTerms {
+
+ /**
+ * Bucket for a rare terms agg
+ *
+ * @opensearch.internal
+ */
+ public abstract static class Bucket<B extends Bucket<B>> extends InternalMultiBucketAggregation.InternalBucket
+ implements
+ RareTerms.Bucket,
+ KeyComparable<B> {
+ /**
+ * Reads a bucket. Should be a constructor reference.
+ *
+ * @opensearch.internal
+ */
+ @FunctionalInterface
+ public interface Reader<B extends Bucket<B>> {
+ B read(StreamInput in, DocValueFormat format) throws IOException;
+ }
+
+ long bucketOrd;
+
+ protected long docCount;
+ protected InternalAggregations aggregations;
+ protected final DocValueFormat format;
+
+ protected Bucket(long docCount, InternalAggregations aggregations, DocValueFormat formatter) {
+ this.format = formatter;
+ this.docCount = docCount;
+ this.aggregations = aggregations;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ protected Bucket(StreamInput in, DocValueFormat formatter) throws IOException {
+ this.format = formatter;
+ docCount = in.readVLong();
+ aggregations = InternalAggregations.readFrom(in);
+ }
+
+ @Override
+ public final void writeTo(StreamOutput out) throws IOException {
+ out.writeVLong(getDocCount());
+ aggregations.writeTo(out);
+ writeTermTo(out);
+ }
+
+ protected abstract void writeTermTo(StreamOutput out) throws IOException;
+
+ @Override
+ public long getDocCount() {
+ return docCount;
+ }
+
+ @Override
+ public Aggregations getAggregations() {
+ return aggregations;
+ }
+
+ @Override
+ public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject();
+ keyToXContent(builder);
+ builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+ aggregations.toXContentInternal(builder, params);
+ builder.endObject();
+ return builder;
+ }
+
+ protected abstract XContentBuilder keyToXContent(XContentBuilder builder) throws IOException;
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ Bucket<?> that = (Bucket<?>) obj;
+ return Objects.equals(docCount, that.docCount) && Objects.equals(aggregations, that.aggregations);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getClass(), docCount, aggregations);
+ }
+ }
+
+ protected final BucketOrder order;
+ protected final long maxDocCount;
+
+ protected InternalRareTerms(String name, BucketOrder order, long maxDocCount, Map<String, Object> metadata) {
+ super(name, metadata);
+ this.order = order;
+ this.maxDocCount = maxDocCount;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ protected InternalRareTerms(StreamInput in) throws IOException {
+ super(in);
+ order = InternalOrder.Streams.readOrder(in);
+ maxDocCount = in.readVLong();
+ }
+
+ @Override
+ protected final void doWriteTo(StreamOutput out) throws IOException {
+ order.writeTo(out);
+ out.writeVLong(maxDocCount);
+ writeTermTypeInfoTo(out);
+ }
+
+ protected abstract void writeTermTypeInfoTo(StreamOutput out) throws IOException;
+
+ @Override
+ public abstract List<B> getBuckets();
+
+ @Override
+ public abstract B getBucketByKey(String term);
+
+ @Override
+ public InternalAggregation reduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+ throw new UnsupportedOperationException();
+ }
+
+ abstract B createBucket(long docCount, InternalAggregations aggs, B prototype);
+
+ @Override
+ protected B reduceBucket(List<B> buckets, ReduceContext context) {
+ assert buckets.size() > 0;
+ long docCount = 0;
+ List<InternalAggregations> aggregationsList = new ArrayList<>(buckets.size());
+ for (B bucket : buckets) {
+ docCount += bucket.docCount;
+ aggregationsList.add(bucket.aggregations);
+ }
+ InternalAggregations aggs = InternalAggregations.reduce(aggregationsList, context);
+ return createBucket(docCount, aggs, buckets.get(0));
+ }
+
+ protected abstract A createWithFilter(String name, List<B> buckets, SetBackedScalingCuckooFilter filter);
+
+ /**
+ * Create an array to hold some buckets. Used in collecting the results.
+ */
+ protected abstract B[] createBucketsArray(int size);
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+ InternalRareTerms<?, ?> that = (InternalRareTerms<?, ?>) obj;
+ return Objects.equals(maxDocCount, that.maxDocCount) && Objects.equals(order, that.order);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), maxDocCount, order);
+ }
+
+ protected static XContentBuilder doXContentCommon(XContentBuilder builder, Params params, List<? extends Bucket> buckets)
+ throws IOException {
+ builder.startArray(CommonFields.BUCKETS.getPreferredName());
+ for (Bucket bucket : buckets) {
+ bucket.toXContent(builder, params);
+ }
+ builder.endArray();
+ return builder;
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: InternalSignificantTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | InternalSignificantTerms |
+
+
+ 0%
+
+
+ (0/8)
+
+ |
+
+
+ 0%
+
+
+ (0/79)
+
+ |
+
+
+ | InternalSignificantTerms$Bucket |
+
+
+ 0%
+
+
+ (0/13)
+
+ |
+
+
+ 0%
+
+
+ (0/37)
+
+ |
+
+
+ | InternalSignificantTerms$Bucket$Reader |
+
+
+ | Total |
+
+
+ 0%
+
+
+ (0/21)
+
+ |
+
+
+ 0%
+
+
+ (0/116)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.Aggregations;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.InternalAggregations;
+ import org.opensearch.search.aggregations.InternalMultiBucketAggregation;
+ import org.opensearch.search.aggregations.bucket.LocalBucketCountThresholds;
+ import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+
+ /**
+ * Result of the significant terms aggregation.
+ *
+ * @opensearch.internal
+ */
+ public abstract class InternalSignificantTerms<A extends InternalSignificantTerms<A, B>, B extends InternalSignificantTerms.Bucket<B>>
+ extends InternalMultiBucketAggregation<A, B>
+ implements
+ SignificantTerms {
+
+ public static final String SCORE = "score";
+ public static final String BG_COUNT = "bg_count";
+
+ /**
+ * Bucket for a significant terms agg
+ *
+ * @opensearch.internal
+ */
+ @SuppressWarnings("PMD.ConstructorCallsOverridableMethod")
+ public abstract static class Bucket<B extends Bucket<B>> extends InternalMultiBucketAggregation.InternalBucket
+ implements
+ SignificantTerms.Bucket {
+ /**
+ * Reads a bucket. Should be a constructor reference.
+ *
+ * @opensearch.internal
+ */
+ @FunctionalInterface
+ public interface Reader<B extends Bucket<B>> {
+ B read(StreamInput in, long subsetSize, long supersetSize, DocValueFormat format) throws IOException;
+ }
+
+ long subsetDf;
+ long subsetSize;
+ long supersetDf;
+ long supersetSize;
+ long bucketOrd;
+ double score;
+ protected InternalAggregations aggregations;
+ final transient DocValueFormat format;
+
+ protected Bucket(
+ long subsetDf,
+ long subsetSize,
+ long supersetDf,
+ long supersetSize,
+ InternalAggregations aggregations,
+ DocValueFormat format
+ ) {
+ this.subsetSize = subsetSize;
+ this.supersetSize = supersetSize;
+ this.subsetDf = subsetDf;
+ this.supersetDf = supersetDf;
+ this.aggregations = aggregations;
+ this.format = format;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ protected Bucket(StreamInput in, long subsetSize, long supersetSize, DocValueFormat format) {
+ this.subsetSize = subsetSize;
+ this.supersetSize = supersetSize;
+ this.format = format;
+ }
+
+ @Override
+ public long getSubsetDf() {
+ return subsetDf;
+ }
+
+ @Override
+ public long getSupersetDf() {
+ return supersetDf;
+ }
+
+ @Override
+ public long getSupersetSize() {
+ return supersetSize;
+ }
+
+ @Override
+ public long getSubsetSize() {
+ return subsetSize;
+ }
+
+ // TODO we should refactor to remove this, since buckets should be immutable after they are generated.
+ // This can lead to confusing bugs if the bucket is re-created (via createBucket() or similar) without
+ // the score
+ void updateScore(SignificanceHeuristic significanceHeuristic) {
+ score = significanceHeuristic.getScore(subsetDf, subsetSize, supersetDf, supersetSize);
+ }
+
+ @Override
+ public long getDocCount() {
+ return subsetDf;
+ }
+
+ @Override
+ public Aggregations getAggregations() {
+ return aggregations;
+ }
+
+ @Override
+ public double getSignificanceScore() {
+ return score;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ Bucket<?> that = (Bucket<?>) o;
+ return bucketOrd == that.bucketOrd
+ && Double.compare(that.score, score) == 0
+ && Objects.equals(aggregations, that.aggregations)
+ && Objects.equals(format, that.format);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getClass(), bucketOrd, aggregations, score, format);
+ }
+
+ @Override
+ public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject();
+ keyToXContent(builder);
+ builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+ builder.field(SCORE, score);
+ builder.field(BG_COUNT, supersetDf);
+ aggregations.toXContentInternal(builder, params);
+ builder.endObject();
+ return builder;
+ }
+
+ protected abstract XContentBuilder keyToXContent(XContentBuilder builder) throws IOException;
+ }
+
+ protected final int requiredSize;
+ protected final long minDocCount;
+ protected final TermsAggregator.BucketCountThresholds bucketCountThresholds;
+
+ protected InternalSignificantTerms(
+ String name,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ Map<String, Object> metadata
+ ) {
+ super(name, metadata);
+ this.requiredSize = bucketCountThresholds.getRequiredSize();
+ this.minDocCount = bucketCountThresholds.getMinDocCount();
+ this.bucketCountThresholds = bucketCountThresholds;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ protected InternalSignificantTerms(StreamInput in) throws IOException {
+ super(in);
+ requiredSize = readSize(in);
+ minDocCount = in.readVLong();
+ // shardMinDocCount and shardSize are not used on the coordinator, so they are not deserialized. We use
+ // CoordinatorBucketCountThresholds which will throw an exception if they are accessed.
+ bucketCountThresholds = new TermsAggregator.CoordinatorBucketCountThresholds(minDocCount, -1, requiredSize, -1);
+ }
+
+ protected final void doWriteTo(StreamOutput out) throws IOException {
+ writeSize(requiredSize, out);
+ out.writeVLong(minDocCount);
+ writeTermTypeInfoTo(out);
+ }
+
+ protected abstract void writeTermTypeInfoTo(StreamOutput out) throws IOException;
+
+ @Override
+ public abstract List<B> getBuckets();
+
+ @Override
+ public InternalAggregation reduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+ LocalBucketCountThresholds localBucketCountThresholds = reduceContext.asLocalBucketCountThresholds(bucketCountThresholds);
+ long globalSubsetSize = 0;
+ long globalSupersetSize = 0;
+ // Compute the overall result set size and the corpus size using the
+ // top-level Aggregations from each shard
+ for (InternalAggregation aggregation : aggregations) {
+ @SuppressWarnings("unchecked")
+ InternalSignificantTerms<A, B> terms = (InternalSignificantTerms<A, B>) aggregation;
+ globalSubsetSize += terms.getSubsetSize();
+ // supersetSize is a shard level count, if we sum it across slices we would produce num_slices_with_bucket * supersetSize where
+ // num_slices_with_bucket is the number of segment slices that have collected a bucket for the key
+ if (reduceContext.isSliceLevel()) {
+ globalSupersetSize = terms.getSupersetSize();
+ } else {
+ globalSupersetSize += terms.getSupersetSize();
+ }
+ }
+ Map<String, List<B>> buckets = new HashMap<>();
+ for (InternalAggregation aggregation : aggregations) {
+ @SuppressWarnings("unchecked")
+ InternalSignificantTerms<A, B> terms = (InternalSignificantTerms<A, B>) aggregation;
+ for (B bucket : terms.getBuckets()) {
+ List<B> existingBuckets = buckets.get(bucket.getKeyAsString());
+ if (existingBuckets == null) {
+ existingBuckets = new ArrayList<>(aggregations.size());
+ buckets.put(bucket.getKeyAsString(), existingBuckets);
+ }
+ // Adjust the buckets with the global stats representing the
+ // total size of the pots from which the stats are drawn
+ existingBuckets.add(
+ createBucket(
+ bucket.getSubsetDf(),
+ globalSubsetSize,
+ bucket.getSupersetDf(),
+ globalSupersetSize,
+ bucket.aggregations,
+ bucket
+ )
+ );
+ }
+ }
+ SignificanceHeuristic heuristic = getSignificanceHeuristic().rewrite(reduceContext);
+ boolean isCoordinatorPartialReduce = reduceContext.isFinalReduce() == false && reduceContext.isSliceLevel() == false;
+ // Do not apply size threshold on coordinator partial reduce
+ final int size = !isCoordinatorPartialReduce
+ ? Math.min(localBucketCountThresholds.getRequiredSize(), buckets.size())
+ : buckets.size();
+ BucketSignificancePriorityQueue<B> ordered = new BucketSignificancePriorityQueue<>(size);
+ for (Map.Entry<String, List<B>> entry : buckets.entrySet()) {
+ List<B> sameTermBuckets = entry.getValue();
+ final B b = reduceBucket(sameTermBuckets, reduceContext);
+ b.updateScore(heuristic);
+ // For concurrent search case we do not apply bucket count thresholds in buildAggregation and instead is done here during
+ // reduce. However, the bucket score is only evaluated at the final coordinator reduce.
+ boolean meetsThresholds = (b.subsetDf >= localBucketCountThresholds.getMinDocCount())
+ && (((b.score > 0) || reduceContext.isSliceLevel()));
+ if (isCoordinatorPartialReduce || meetsThresholds) {
+ B removed = ordered.insertWithOverflow(b);
+ if (removed == null) {
+ reduceContext.consumeBucketsAndMaybeBreak(1);
+ } else {
+ reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(removed));
+ }
+ } else {
+ reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(b));
+ }
+ }
+ B[] list = createBucketsArray(ordered.size());
+ for (int i = ordered.size() - 1; i >= 0; i--) {
+ list[i] = ordered.pop();
+ }
+ return create(globalSubsetSize, globalSupersetSize, Arrays.asList(list));
+ }
+
+ @Override
+ protected B reduceBucket(List<B> buckets, ReduceContext context) {
+ assert buckets.size() > 0;
+ long subsetDf = 0;
+ long supersetDf = 0;
+ List<InternalAggregations> aggregationsList = new ArrayList<>(buckets.size());
+ for (B bucket : buckets) {
+ subsetDf += bucket.subsetDf;
+ // supersetDf is a shard level count, if we sum it across slices we would produce num_slices_with_bucket * supersetSize where
+ // num_slices_with_bucket is the number of segment slices that have collected a bucket for the key
+ if (context.isSliceLevel()) {
+ supersetDf = bucket.supersetDf;
+ } else {
+ supersetDf += bucket.supersetDf;
+ }
+ aggregationsList.add(bucket.aggregations);
+ }
+ InternalAggregations aggs = InternalAggregations.reduce(aggregationsList, context);
+ return createBucket(subsetDf, buckets.get(0).subsetSize, supersetDf, buckets.get(0).supersetSize, aggs, buckets.get(0));
+ }
+
+ abstract B createBucket(
+ long subsetDf,
+ long subsetSize,
+ long supersetDf,
+ long supersetSize,
+ InternalAggregations aggregations,
+ B prototype
+ );
+
+ protected abstract A create(long subsetSize, long supersetSize, List<B> buckets);
+
+ /**
+ * Create an array to hold some buckets. Used in collecting the results.
+ */
+ protected abstract B[] createBucketsArray(int size);
+
+ protected abstract long getSubsetSize();
+
+ protected abstract long getSupersetSize();
+
+ protected abstract SignificanceHeuristic getSignificanceHeuristic();
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), minDocCount, requiredSize);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+
+ InternalSignificantTerms<?, ?> that = (InternalSignificantTerms<?, ?>) obj;
+ return Objects.equals(minDocCount, that.minDocCount) && Objects.equals(requiredSize, that.requiredSize);
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: InternalTerms (org.opensearch.search.aggregations.bucket.terms)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | InternalTerms |
+
+
+ 53.8%
+
+
+ (7/13)
+
+ |
+
+
+ 58.6%
+
+
+ (112/191)
+
+ |
+
+
+ | InternalTerms$1 |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ | InternalTerms$AbstractInternalBucket |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ | InternalTerms$Bucket |
+
+
+ 41.7%
+
+
+ (5/12)
+
+ |
+
+
+ 23.8%
+
+
+ (10/42)
+
+ |
+
+
+ | InternalTerms$Bucket$Reader |
+
+
+ | Total |
+
+
+ 50%
+
+
+ (14/28)
+
+ |
+
+
+ 52.5%
+
+
+ (124/236)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms;
+
+ import org.apache.lucene.util.PriorityQueue;
+ import org.opensearch.core.ParseField;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.search.DocValueFormat;
+ import org.opensearch.search.aggregations.AggregationExecutionException;
+ import org.opensearch.search.aggregations.Aggregations;
+ import org.opensearch.search.aggregations.BucketOrder;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.InternalAggregations;
+ import org.opensearch.search.aggregations.InternalMultiBucketAggregation;
+ import org.opensearch.search.aggregations.InternalOrder;
+ import org.opensearch.search.aggregations.KeyComparable;
+ import org.opensearch.search.aggregations.bucket.IteratorAndCurrent;
+ import org.opensearch.search.aggregations.bucket.LocalBucketCountThresholds;
+ import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.Comparator;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+ import java.util.function.Function;
+
+ import static org.opensearch.search.aggregations.InternalOrder.isKeyAsc;
+ import static org.opensearch.search.aggregations.InternalOrder.isKeyOrder;
+
+ /**
+ * Implementation of terms
+ *
+ * @opensearch.internal
+ */
+ public abstract class InternalTerms<A extends InternalTerms<A, B>, B extends InternalTerms.AbstractInternalBucket> extends
+ InternalMultiBucketAggregation<A, B>
+ implements
+ Terms {
+
+ protected static final ParseField DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME = new ParseField("doc_count_error_upper_bound");
+ protected static final ParseField SUM_OF_OTHER_DOC_COUNTS = new ParseField("sum_other_doc_count");
+
+ /**
+ * Base internal multi bucket
+ *
+ * @opensearch.internal
+ */
+ public abstract static class AbstractInternalBucket extends InternalMultiBucketAggregation.InternalBucket implements Terms.Bucket {
+ abstract void setDocCountError(long docCountError);
+
+ abstract void setDocCountError(Function<Long, Long> updater);
+
+ abstract boolean showDocCountError();
+ }
+
+ /**
+ * Base bucket class
+ *
+ * @opensearch.internal
+ */
+ public abstract static class Bucket<B extends Bucket<B>> extends AbstractInternalBucket implements KeyComparable<B> {
+ /**
+ * Reads a bucket. Should be a constructor reference.
+ *
+ * @opensearch.internal
+ */
+ @FunctionalInterface
+ public interface Reader<B extends Bucket<B>> {
+ B read(StreamInput in, DocValueFormat format, boolean showDocCountError) throws IOException;
+ }
+
+ long bucketOrd;
+
+ protected long docCount;
+ protected long docCountError;
+ protected InternalAggregations aggregations;
+ protected final boolean showDocCountError;
+ protected final DocValueFormat format;
+
+ protected Bucket(
+ long docCount,
+ InternalAggregations aggregations,
+ boolean showDocCountError,
+ long docCountError,
+ DocValueFormat formatter
+ ) {
+ this.showDocCountError = showDocCountError;
+ this.format = formatter;
+ this.docCount = docCount;
+ this.aggregations = aggregations;
+ this.docCountError = docCountError;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ protected Bucket(StreamInput in, DocValueFormat formatter, boolean showDocCountError) throws IOException {
+ this.showDocCountError = showDocCountError;
+ this.format = formatter;
+ docCount = in.readVLong();
+ docCountError = -1;
+ if (showDocCountError) {
+ docCountError = in.readLong();
+ }
+ aggregations = InternalAggregations.readFrom(in);
+ }
+
+ @Override
+ public final void writeTo(StreamOutput out) throws IOException {
+ out.writeVLong(getDocCount());
+ if (showDocCountError) {
+ out.writeLong(docCountError);
+ }
+ aggregations.writeTo(out);
+ writeTermTo(out);
+ }
+
+ protected abstract void writeTermTo(StreamOutput out) throws IOException;
+
+ @Override
+ public long getDocCount() {
+ return docCount;
+ }
+
+ @Override
+ public long getDocCountError() {
+ if (!showDocCountError) {
+ throw new IllegalStateException("show_terms_doc_count_error is false");
+ }
+ return docCountError;
+ }
+
+ @Override
+ public void setDocCountError(long docCountError) {
+ this.docCountError = docCountError;
+ }
+
+ @Override
+ public void setDocCountError(Function<Long, Long> updater) {
+ this.docCountError = updater.apply(this.docCountError);
+ }
+
+ @Override
+ public boolean showDocCountError() {
+ return showDocCountError;
+ }
+
+ @Override
+ public Aggregations getAggregations() {
+ return aggregations;
+ }
+
+ @Override
+ public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject();
+ keyToXContent(builder);
+ builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+ if (showDocCountError) {
+ builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName(), getDocCountError());
+ }
+ aggregations.toXContentInternal(builder, params);
+ builder.endObject();
+ return builder;
+ }
+
+ protected abstract XContentBuilder keyToXContent(XContentBuilder builder) throws IOException;
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ Bucket<?> that = (Bucket<?>) obj;
+ // No need to take format and showDocCountError, they are attributes
+ // of the parent terms aggregation object that are only copied here
+ // for serialization purposes
+ return Objects.equals(docCount, that.docCount)
+ && Objects.equals(docCountError, that.docCountError)
+ && Objects.equals(aggregations, that.aggregations);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getClass(), docCount, docCountError, aggregations);
+ }
+ }
+
+ protected final BucketOrder reduceOrder;
+ protected final BucketOrder order;
+ protected final int requiredSize;
+ protected final long minDocCount;
+ protected final TermsAggregator.BucketCountThresholds bucketCountThresholds;
+ private boolean hasSliceLevelDocCountError = false;
+
+ /**
+ * Creates a new {@link InternalTerms}
+ * @param name The name of the aggregation
+ * @param reduceOrder The {@link BucketOrder} that should be used to merge shard results.
+ * @param order The {@link BucketOrder} that should be used to sort the final reduce.
+ * @param bucketCountThresholds Object containing values for minDocCount, shardMinDocCount, size, shardSize.
+ * @param metadata The metadata associated with the aggregation.
+ */
+ protected InternalTerms(
+ String name,
+ BucketOrder reduceOrder,
+ BucketOrder order,
+ TermsAggregator.BucketCountThresholds bucketCountThresholds,
+ Map<String, Object> metadata
+ ) {
+ super(name, metadata);
+ this.reduceOrder = reduceOrder;
+ this.order = order;
+ this.bucketCountThresholds = bucketCountThresholds;
+ this.requiredSize = bucketCountThresholds.getRequiredSize();
+ this.minDocCount = bucketCountThresholds.getMinDocCount();
+ }
+
+ /**
+ * Read from a stream.
+ */
+ protected InternalTerms(StreamInput in) throws IOException {
+ super(in);
+ reduceOrder = InternalOrder.Streams.readOrder(in);
+ order = InternalOrder.Streams.readOrder(in);
+ requiredSize = readSize(in);
+ minDocCount = in.readVLong();
+ // shardMinDocCount and shardSize are not used on the coordinator, so they are not deserialized. We use
+ // CoordinatorBucketCountThresholds which will throw an exception if they are accessed.
+ bucketCountThresholds = new TermsAggregator.CoordinatorBucketCountThresholds(minDocCount, -1, requiredSize, getShardSize());
+ }
+
+ @Override
+ protected final void doWriteTo(StreamOutput out) throws IOException {
+ reduceOrder.writeTo(out);
+ order.writeTo(out);
+ writeSize(requiredSize, out);
+ out.writeVLong(minDocCount);
+ writeTermTypeInfoTo(out);
+ }
+
+ protected abstract void writeTermTypeInfoTo(StreamOutput out) throws IOException;
+
+ @Override
+ public abstract List<B> getBuckets();
+
+ @Override
+ public abstract B getBucketByKey(String term);
+
+ private BucketOrder getReduceOrder(List<InternalAggregation> aggregations) {
+ BucketOrder thisReduceOrder = null;
+ for (InternalAggregation aggregation : aggregations) {
+ @SuppressWarnings("unchecked")
+ InternalTerms<A, B> terms = (InternalTerms<A, B>) aggregation;
+ if (terms.getBuckets().size() == 0) {
+ continue;
+ }
+ if (thisReduceOrder == null) {
+ thisReduceOrder = terms.reduceOrder;
+ } else if (thisReduceOrder.equals(terms.reduceOrder) == false) {
+ return order;
+ }
+ }
+ return thisReduceOrder != null ? thisReduceOrder : order;
+ }
+
+ private long getDocCountError(InternalTerms<?, ?> terms, ReduceContext reduceContext) {
+ int size = terms.getBuckets().size();
+ if (size == 0 || size < terms.getShardSize() || isKeyOrder(terms.order)) {
+ return 0;
+ } else if (InternalOrder.isCountDesc(terms.order)) {
+ if (terms.getDocCountError() > 0) {
+ // If there is an existing docCountError for this agg then
+ // use this as the error for this aggregation
+ return terms.getDocCountError();
+ } else {
+ // otherwise use the doc count of the last term in the
+ // aggregation
+ return terms.getBuckets().stream().mapToLong(MultiBucketsAggregation.Bucket::getDocCount).min().getAsLong();
+ }
+ } else {
+ return -1;
+ }
+ }
+
+ private List<B> reduceMergeSort(List<InternalAggregation> aggregations, BucketOrder thisReduceOrder, ReduceContext reduceContext) {
+ assert isKeyOrder(thisReduceOrder);
+ final Comparator<MultiBucketsAggregation.Bucket> cmp = thisReduceOrder.comparator();
+ final PriorityQueue<IteratorAndCurrent<B>> pq = new PriorityQueue<IteratorAndCurrent<B>>(aggregations.size()) {
+ @Override
+ protected boolean lessThan(IteratorAndCurrent<B> a, IteratorAndCurrent<B> b) {
+ return cmp.compare(a.current(), b.current()) < 0;
+ }
+ };
+ for (InternalAggregation aggregation : aggregations) {
+ @SuppressWarnings("unchecked")
+ InternalTerms<A, B> terms = (InternalTerms<A, B>) aggregation;
+ if (terms.getBuckets().isEmpty() == false) {
+ assert reduceOrder.equals(reduceOrder);
+ pq.add(new IteratorAndCurrent(terms.getBuckets().iterator()));
+ }
+ }
+ List<B> reducedBuckets = new ArrayList<>();
+ // list of buckets coming from different shards that have the same key
+ List<B> currentBuckets = new ArrayList<>();
+ B lastBucket = null;
+ while (pq.size() > 0) {
+ final IteratorAndCurrent<B> top = pq.top();
+ assert lastBucket == null || cmp.compare(top.current(), lastBucket) >= 0;
+ if (lastBucket != null && cmp.compare(top.current(), lastBucket) != 0) {
+ // the key changes, reduce what we already buffered and reset the buffer for current buckets
+ final B reduced = reduceBucket(currentBuckets, reduceContext);
+ reducedBuckets.add(reduced);
+ currentBuckets.clear();
+ }
+ lastBucket = top.current();
+ currentBuckets.add(top.current());
+ if (top.hasNext()) {
+ top.next();
+ assert cmp.compare(top.current(), lastBucket) > 0 : "shards must return data sorted by key";
+ pq.updateTop();
+ } else {
+ pq.pop();
+ }
+ }
+
+ if (currentBuckets.isEmpty() == false) {
+ final B reduced = reduceBucket(currentBuckets, reduceContext);
+ reducedBuckets.add(reduced);
+ }
+ return reducedBuckets;
+ }
+
+ private List<B> reduceLegacy(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+ Map<Object, List<B>> bucketMap = new HashMap<>();
+ for (InternalAggregation aggregation : aggregations) {
+ @SuppressWarnings("unchecked")
+ InternalTerms<A, B> terms = (InternalTerms<A, B>) aggregation;
+ if (terms.getBuckets().isEmpty() == false) {
+ for (B bucket : terms.getBuckets()) {
+ List<B> bucketList = bucketMap.get(bucket.getKey());
+ if (bucketList == null) {
+ bucketList = new ArrayList<>();
+ bucketMap.put(bucket.getKey(), bucketList);
+ }
+ bucketList.add(bucket);
+ }
+ }
+ }
+ List<B> reducedBuckets = new ArrayList<>();
+ for (List<B> sameTermBuckets : bucketMap.values()) {
+ final B b = reduceBucket(sameTermBuckets, reduceContext);
+ reducedBuckets.add(b);
+ }
+ return reducedBuckets;
+ }
+
+ public InternalAggregation reduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+ LocalBucketCountThresholds localBucketCountThresholds = reduceContext.asLocalBucketCountThresholds(bucketCountThresholds);
+ long sumDocCountError = 0;
+ long otherDocCount = 0;
+ InternalTerms<A, B> referenceTerms = null;
+ for (InternalAggregation aggregation : aggregations) {
+ @SuppressWarnings("unchecked")
+ InternalTerms<A, B> terms = (InternalTerms<A, B>) aggregation;
+ // For Concurrent Segment Search the aggregation will have a computed doc count error coming from the shards.
+ // We use the existence of this doc count error to determine whether or not doc count error originated from the slice level
+ // and if so we will maintain the doc count error for the 1 shard case at the coordinator level
+ if (aggregations.size() == 1 && terms.getDocCountError() > 0) {
+ hasSliceLevelDocCountError = true;
+ }
+ if (referenceTerms == null && aggregation.getClass().equals(UnmappedTerms.class) == false) {
+ referenceTerms = terms;
+ }
+ if (referenceTerms != null
+ && referenceTerms.getClass().equals(terms.getClass()) == false
+ && terms.getClass().equals(UnmappedTerms.class) == false) {
+ // control gets into this loop when the same field name against which the query is executed
+ // is of different types in different indices.
+ throw new AggregationExecutionException(
+ "Merging/Reducing the aggregations failed when computing the aggregation ["
+ + referenceTerms.getName()
+ + "] because the field you gave in the aggregation query existed as two different "
+ + "types in two different indices"
+ );
+ }
+ otherDocCount += terms.getSumOfOtherDocCounts();
+ final long thisAggDocCountError = getDocCountError(terms, reduceContext);
+ if (sumDocCountError != -1) {
+ if (thisAggDocCountError == -1) {
+ sumDocCountError = -1;
+ } else {
+ sumDocCountError += thisAggDocCountError;
+ }
+ }
+ setDocCountError(thisAggDocCountError);
+ for (B bucket : terms.getBuckets()) {
+ // If there is already a doc count error for this bucket
+ // subtract this aggs doc count error from it to make the
+ // new value for the bucket. This then means that when the
+ // final error for the bucket is calculated below we account
+ // for the existing error calculated in a previous reduce.
+ // Note that if the error is unbounded (-1) this will be fixed
+ // later in this method.
+ bucket.setDocCountError(docCountError -> docCountError - thisAggDocCountError);
+ }
+ }
+
+ final List<B> reducedBuckets;
+ /*
+ Buckets returned by a partial reduce or a shard response are sorted by key.
+ That allows to perform a merge sort when reducing multiple aggregations together.
+ For backward compatibility, we disable the merge sort and use ({@link InternalTerms#reduceLegacy} if any of
+ the provided aggregations use a different {@link InternalTerms#reduceOrder}.
+ */
+ BucketOrder thisReduceOrder = getReduceOrder(aggregations);
+ if (isKeyOrder(thisReduceOrder)) {
+ // extract the primary sort in case this is a compound order.
+ thisReduceOrder = InternalOrder.key(isKeyAsc(thisReduceOrder) ? true : false);
+ reducedBuckets = reduceMergeSort(aggregations, thisReduceOrder, reduceContext);
+ } else {
+ reducedBuckets = reduceLegacy(aggregations, reduceContext);
+ }
+ final B[] list;
+ if (reduceContext.isFinalReduce() || reduceContext.isSliceLevel()) {
+ final int size = Math.min(localBucketCountThresholds.getRequiredSize(), reducedBuckets.size());
+ // final comparator
+ final BucketPriorityQueue<B> ordered = new BucketPriorityQueue<>(size, order.comparator());
+ for (B bucket : reducedBuckets) {
+ if (sumDocCountError == -1) {
+ bucket.setDocCountError(-1);
+ } else {
+ final long finalSumDocCountError = sumDocCountError;
+ bucket.setDocCountError(docCountError -> docCountError + finalSumDocCountError);
+ }
+ if (bucket.getDocCount() >= localBucketCountThresholds.getMinDocCount()) {
+ B removed = ordered.insertWithOverflow(bucket);
+ if (removed != null) {
+ otherDocCount += removed.getDocCount();
+ reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(removed));
+ } else {
+ reduceContext.consumeBucketsAndMaybeBreak(1);
+ }
+ } else {
+ reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(bucket));
+ }
+ }
+ list = createBucketsArray(ordered.size());
+ for (int i = ordered.size() - 1; i >= 0; i--) {
+ list[i] = ordered.pop();
+ }
+ } else {
+ // we can prune the list on partial reduce if the aggregation is ordered by key
+ // and not filtered (minDocCount == 0)
+ int size = isKeyOrder(order) && localBucketCountThresholds.getMinDocCount() == 0
+ ? Math.min(localBucketCountThresholds.getRequiredSize(), reducedBuckets.size())
+ : reducedBuckets.size();
+ list = createBucketsArray(size);
+ for (int i = 0; i < size; i++) {
+ reduceContext.consumeBucketsAndMaybeBreak(1);
+ list[i] = reducedBuckets.get(i);
+ if (sumDocCountError == -1) {
+ list[i].setDocCountError(-1);
+ } else {
+ final long fSumDocCountError = sumDocCountError;
+ list[i].setDocCountError(docCountError -> docCountError + fSumDocCountError);
+ }
+ }
+ }
+ long docCountError;
+ if (sumDocCountError == -1) {
+ docCountError = -1;
+ } else {
+ if (hasSliceLevelDocCountError) {
+ docCountError = sumDocCountError;
+ } else {
+ docCountError = aggregations.size() == 1 ? 0 : sumDocCountError;
+ }
+ }
+
+ // Shards must return buckets sorted by key, so we apply the sort here in shard level reduce
+ if (reduceContext.isSliceLevel()) {
+ Arrays.sort(list, thisReduceOrder.comparator());
+ }
+ return create(name, Arrays.asList(list), reduceContext.isFinalReduce() ? order : thisReduceOrder, docCountError, otherDocCount);
+ }
+
+ @Override
+ protected B reduceBucket(List<B> buckets, ReduceContext context) {
+ assert !buckets.isEmpty();
+ long docCount = 0;
+ // For the per term doc count error we add up the errors from the
+ // shards that did not respond with the term. To do this we add up
+ // the errors from the shards that did respond with the terms and
+ // subtract that from the sum of the error from all shards
+ long docCountError = 0;
+ List<InternalAggregations> aggregationsList = new ArrayList<>(buckets.size());
+ for (B bucket : buckets) {
+ docCount += bucket.getDocCount();
+ if (docCountError != -1) {
+ if (bucket.showDocCountError() == false) {
+ docCountError = -1;
+ } else {
+ docCountError += bucket.getDocCountError();
+ }
+ }
+ aggregationsList.add((InternalAggregations) bucket.getAggregations());
+ }
+ InternalAggregations aggs = InternalAggregations.reduce(aggregationsList, context);
+ return createBucket(docCount, aggs, docCountError, buckets.get(0));
+ }
+
+ protected abstract void setDocCountError(long docCountError);
+
+ protected abstract int getShardSize();
+
+ protected abstract A create(String name, List<B> buckets, BucketOrder reduceOrder, long docCountError, long otherDocCount);
+
+ /**
+ * Create an array to hold some buckets. Used in collecting the results.
+ */
+ protected abstract B[] createBucketsArray(int size);
+
+ abstract B createBucket(long docCount, InternalAggregations aggs, long docCountError, B prototype);
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+
+ InternalTerms<?, ?> that = (InternalTerms<?, ?>) obj;
+ return Objects.equals(minDocCount, that.minDocCount)
+ && Objects.equals(reduceOrder, that.reduceOrder)
+ && Objects.equals(order, that.order)
+ && Objects.equals(requiredSize, that.requiredSize);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), minDocCount, reduceOrder, order, requiredSize);
+ }
+
+ protected static XContentBuilder doXContentCommon(
+ XContentBuilder builder,
+ Params params,
+ long docCountError,
+ long otherDocCount,
+ List<? extends AbstractInternalBucket> buckets
+ ) throws IOException {
+ builder.field(DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName(), docCountError);
+ builder.field(SUM_OF_OTHER_DOC_COUNTS.getPreferredName(), otherDocCount);
+ builder.startArray(CommonFields.BUCKETS.getPreferredName());
+ for (AbstractInternalBucket bucket : buckets) {
+ bucket.toXContent(builder, params);
+ }
+ builder.endArray();
+ return builder;
+ }
+ }
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms.heuristic |
+
+
+ 47.1%
+
+
+ (8/17)
+
+ |
+
+
+ 11.6%
+
+
+ (11/95)
+
+ |
+
+
+ 8.1%
+
+
+ (22/270)
+
+ |
+
+
+
+
+
+
+
+
+ |
+Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ChiSquare |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 10%
+
+
+ (1/10)
+
+ |
+
+
+ 12%
+
+
+ (3/25)
+
+ |
+
+
+ | GND |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 5.7%
+
+
+ (2/35)
+
+ |
+
+
+ | JLHScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 18.2%
+
+
+ (2/11)
+
+ |
+
+
+ 7.7%
+
+
+ (2/26)
+
+ |
+
+
+ | MutualInformation |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 9.1%
+
+
+ (1/11)
+
+ |
+
+
+ 11.1%
+
+
+ (4/36)
+
+ |
+
+
+ | NXYSignificanceHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 20%
+
+
+ (3/15)
+
+ |
+
+
+ 10.3%
+
+
+ (7/68)
+
+ |
+
+
+ | PercentageScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 4.5%
+
+
+ (1/22)
+
+ |
+
+
+ | ScriptHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 5%
+
+
+ (1/20)
+
+ |
+
+
+ 4.1%
+
+
+ (2/49)
+
+ |
+
+
+ | SignificanceHeuristic |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 25%
+
+
+ (1/4)
+
+ |
+
+
+ 11.1%
+
+
+ (1/9)
+
+ |
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms.heuristic |
+
+
+ 47.1%
+
+
+ (8/17)
+
+ |
+
+
+ 11.6%
+
+
+ (11/95)
+
+ |
+
+
+ 8.1%
+
+
+ (22/270)
+
+ |
+
+
+
+
+
+
+
+
+ |
+Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ChiSquare |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 10%
+
+
+ (1/10)
+
+ |
+
+
+ 12%
+
+
+ (3/25)
+
+ |
+
+
+ | GND |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 5.7%
+
+
+ (2/35)
+
+ |
+
+
+ | JLHScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 18.2%
+
+
+ (2/11)
+
+ |
+
+
+ 7.7%
+
+
+ (2/26)
+
+ |
+
+
+ | MutualInformation |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 9.1%
+
+
+ (1/11)
+
+ |
+
+
+ 11.1%
+
+
+ (4/36)
+
+ |
+
+
+ | NXYSignificanceHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 20%
+
+
+ (3/15)
+
+ |
+
+
+ 10.3%
+
+
+ (7/68)
+
+ |
+
+
+ | PercentageScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 4.5%
+
+
+ (1/22)
+
+ |
+
+
+ | ScriptHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 5%
+
+
+ (1/20)
+
+ |
+
+
+ 4.1%
+
+
+ (2/49)
+
+ |
+
+
+ | SignificanceHeuristic |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 25%
+
+
+ (1/4)
+
+ |
+
+
+ 11.1%
+
+
+ (1/9)
+
+ |
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms.heuristic |
+
+
+ 47.1%
+
+
+ (8/17)
+
+ |
+
+
+ 11.6%
+
+
+ (11/95)
+
+ |
+
+
+ 8.1%
+
+
+ (22/270)
+
+ |
+
+
+
+
+
+
+
+
+ |
+Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | SignificanceHeuristic |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 25%
+
+
+ (1/4)
+
+ |
+
+
+ 11.1%
+
+
+ (1/9)
+
+ |
+
+
+ | ScriptHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 5%
+
+
+ (1/20)
+
+ |
+
+
+ 4.1%
+
+
+ (2/49)
+
+ |
+
+
+ | PercentageScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 4.5%
+
+
+ (1/22)
+
+ |
+
+
+ | NXYSignificanceHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 20%
+
+
+ (3/15)
+
+ |
+
+
+ 10.3%
+
+
+ (7/68)
+
+ |
+
+
+ | MutualInformation |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 9.1%
+
+
+ (1/11)
+
+ |
+
+
+ 11.1%
+
+
+ (4/36)
+
+ |
+
+
+ | JLHScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 18.2%
+
+
+ (2/11)
+
+ |
+
+
+ 7.7%
+
+
+ (2/26)
+
+ |
+
+
+ | GND |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 5.7%
+
+
+ (2/35)
+
+ |
+
+
+ | ChiSquare |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 10%
+
+
+ (1/10)
+
+ |
+
+
+ 12%
+
+
+ (3/25)
+
+ |
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms.heuristic |
+
+
+ 47.1%
+
+
+ (8/17)
+
+ |
+
+
+ 11.6%
+
+
+ (11/95)
+
+ |
+
+
+ 8.1%
+
+
+ (22/270)
+
+ |
+
+
+
+
+
+
+
+
+ |
+Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | NXYSignificanceHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 20%
+
+
+ (3/15)
+
+ |
+
+
+ 10.3%
+
+
+ (7/68)
+
+ |
+
+
+ | ScriptHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 5%
+
+
+ (1/20)
+
+ |
+
+
+ 4.1%
+
+
+ (2/49)
+
+ |
+
+
+ | ChiSquare |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 10%
+
+
+ (1/10)
+
+ |
+
+
+ 12%
+
+
+ (3/25)
+
+ |
+
+
+ | GND |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 5.7%
+
+
+ (2/35)
+
+ |
+
+
+ | JLHScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 18.2%
+
+
+ (2/11)
+
+ |
+
+
+ 7.7%
+
+
+ (2/26)
+
+ |
+
+
+ | MutualInformation |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 9.1%
+
+
+ (1/11)
+
+ |
+
+
+ 11.1%
+
+
+ (4/36)
+
+ |
+
+
+ | PercentageScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 4.5%
+
+
+ (1/22)
+
+ |
+
+
+ | SignificanceHeuristic |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 25%
+
+
+ (1/4)
+
+ |
+
+
+ 11.1%
+
+
+ (1/9)
+
+ |
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms.heuristic |
+
+
+ 47.1%
+
+
+ (8/17)
+
+ |
+
+
+ 11.6%
+
+
+ (11/95)
+
+ |
+
+
+ 8.1%
+
+
+ (22/270)
+
+ |
+
+
+
+
+
+
+
+
+ |
+Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | SignificanceHeuristic |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 25%
+
+
+ (1/4)
+
+ |
+
+
+ 11.1%
+
+
+ (1/9)
+
+ |
+
+
+ | PercentageScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 4.5%
+
+
+ (1/22)
+
+ |
+
+
+ | MutualInformation |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 9.1%
+
+
+ (1/11)
+
+ |
+
+
+ 11.1%
+
+
+ (4/36)
+
+ |
+
+
+ | JLHScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 18.2%
+
+
+ (2/11)
+
+ |
+
+
+ 7.7%
+
+
+ (2/26)
+
+ |
+
+
+ | GND |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 5.7%
+
+
+ (2/35)
+
+ |
+
+
+ | ChiSquare |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 10%
+
+
+ (1/10)
+
+ |
+
+
+ 12%
+
+
+ (3/25)
+
+ |
+
+
+ | ScriptHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 5%
+
+
+ (1/20)
+
+ |
+
+
+ 4.1%
+
+
+ (2/49)
+
+ |
+
+
+ | NXYSignificanceHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 20%
+
+
+ (3/15)
+
+ |
+
+
+ 10.3%
+
+
+ (7/68)
+
+ |
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms.heuristic |
+
+
+ 47.1%
+
+
+ (8/17)
+
+ |
+
+
+ 11.6%
+
+
+ (11/95)
+
+ |
+
+
+ 8.1%
+
+
+ (22/270)
+
+ |
+
+
+
+
+
+
+
+
+ |
+Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ScriptHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 5%
+
+
+ (1/20)
+
+ |
+
+
+ 4.1%
+
+
+ (2/49)
+
+ |
+
+
+ | PercentageScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 4.5%
+
+
+ (1/22)
+
+ |
+
+
+ | GND |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 5.7%
+
+
+ (2/35)
+
+ |
+
+
+ | JLHScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 18.2%
+
+
+ (2/11)
+
+ |
+
+
+ 7.7%
+
+
+ (2/26)
+
+ |
+
+
+ | NXYSignificanceHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 20%
+
+
+ (3/15)
+
+ |
+
+
+ 10.3%
+
+
+ (7/68)
+
+ |
+
+
+ | MutualInformation |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 9.1%
+
+
+ (1/11)
+
+ |
+
+
+ 11.1%
+
+
+ (4/36)
+
+ |
+
+
+ | SignificanceHeuristic |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 25%
+
+
+ (1/4)
+
+ |
+
+
+ 11.1%
+
+
+ (1/9)
+
+ |
+
+
+ | ChiSquare |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 10%
+
+
+ (1/10)
+
+ |
+
+
+ 12%
+
+
+ (3/25)
+
+ |
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms.heuristic |
+
+
+ 47.1%
+
+
+ (8/17)
+
+ |
+
+
+ 11.6%
+
+
+ (11/95)
+
+ |
+
+
+ 8.1%
+
+
+ (22/270)
+
+ |
+
+
+
+
+
+
+
+
+ |
+Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ChiSquare |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 10%
+
+
+ (1/10)
+
+ |
+
+
+ 12%
+
+
+ (3/25)
+
+ |
+
+
+ | SignificanceHeuristic |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 25%
+
+
+ (1/4)
+
+ |
+
+
+ 11.1%
+
+
+ (1/9)
+
+ |
+
+
+ | MutualInformation |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 9.1%
+
+
+ (1/11)
+
+ |
+
+
+ 11.1%
+
+
+ (4/36)
+
+ |
+
+
+ | NXYSignificanceHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 20%
+
+
+ (3/15)
+
+ |
+
+
+ 10.3%
+
+
+ (7/68)
+
+ |
+
+
+ | JLHScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 18.2%
+
+
+ (2/11)
+
+ |
+
+
+ 7.7%
+
+
+ (2/26)
+
+ |
+
+
+ | GND |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 5.7%
+
+
+ (2/35)
+
+ |
+
+
+ | PercentageScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 4.5%
+
+
+ (1/22)
+
+ |
+
+
+ | ScriptHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 5%
+
+
+ (1/20)
+
+ |
+
+
+ 4.1%
+
+
+ (2/49)
+
+ |
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms.heuristic |
+
+
+ 47.1%
+
+
+ (8/17)
+
+ |
+
+
+ 11.6%
+
+
+ (11/95)
+
+ |
+
+
+ 8.1%
+
+
+ (22/270)
+
+ |
+
+
+
+
+
+
+
+
+ |
+Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ScriptHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 5%
+
+
+ (1/20)
+
+ |
+
+
+ 4.1%
+
+
+ (2/49)
+
+ |
+
+
+ | GND |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 5.7%
+
+
+ (2/35)
+
+ |
+
+
+ | PercentageScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 4.5%
+
+
+ (1/22)
+
+ |
+
+
+ | MutualInformation |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 9.1%
+
+
+ (1/11)
+
+ |
+
+
+ 11.1%
+
+
+ (4/36)
+
+ |
+
+
+ | ChiSquare |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 10%
+
+
+ (1/10)
+
+ |
+
+
+ 12%
+
+
+ (3/25)
+
+ |
+
+
+ | JLHScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 18.2%
+
+
+ (2/11)
+
+ |
+
+
+ 7.7%
+
+
+ (2/26)
+
+ |
+
+
+ | NXYSignificanceHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 20%
+
+
+ (3/15)
+
+ |
+
+
+ 10.3%
+
+
+ (7/68)
+
+ |
+
+
+ | SignificanceHeuristic |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 25%
+
+
+ (1/4)
+
+ |
+
+
+ 11.1%
+
+
+ (1/9)
+
+ |
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms.heuristic |
+
+
+ 47.1%
+
+
+ (8/17)
+
+ |
+
+
+ 11.6%
+
+
+ (11/95)
+
+ |
+
+
+ 8.1%
+
+
+ (22/270)
+
+ |
+
+
+
+
+
+
+
+
+ |
+Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | SignificanceHeuristic |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 25%
+
+
+ (1/4)
+
+ |
+
+
+ 11.1%
+
+
+ (1/9)
+
+ |
+
+
+ | NXYSignificanceHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 20%
+
+
+ (3/15)
+
+ |
+
+
+ 10.3%
+
+
+ (7/68)
+
+ |
+
+
+ | JLHScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 18.2%
+
+
+ (2/11)
+
+ |
+
+
+ 7.7%
+
+
+ (2/26)
+
+ |
+
+
+ | ChiSquare |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 10%
+
+
+ (1/10)
+
+ |
+
+
+ 12%
+
+
+ (3/25)
+
+ |
+
+
+ | MutualInformation |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 9.1%
+
+
+ (1/11)
+
+ |
+
+
+ 11.1%
+
+
+ (4/36)
+
+ |
+
+
+ | PercentageScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 4.5%
+
+
+ (1/22)
+
+ |
+
+
+ | GND |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 5.7%
+
+
+ (2/35)
+
+ |
+
+
+ | ScriptHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 5%
+
+
+ (1/20)
+
+ |
+
+
+ 4.1%
+
+
+ (2/49)
+
+ |
+
+
+
+
+
+
+Current scope:
all classes
+
|
+org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
Coverage Summary for Package: org.opensearch.search.aggregations.bucket.terms.heuristic
+
+
+ | Package |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | org.opensearch.search.aggregations.bucket.terms.heuristic |
+
+
+ 47.1%
+
+
+ (8/17)
+
+ |
+
+
+ 11.6%
+
+
+ (11/95)
+
+ |
+
+
+ 8.1%
+
+
+ (22/270)
+
+ |
+
+
+
+
+
+
+
+
+ |
+Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | SignificanceHeuristic |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 25%
+
+
+ (1/4)
+
+ |
+
+
+ 11.1%
+
+
+ (1/9)
+
+ |
+
+
+ | ScriptHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 5%
+
+
+ (1/20)
+
+ |
+
+
+ 4.1%
+
+
+ (2/49)
+
+ |
+
+
+ | PercentageScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 4.5%
+
+
+ (1/22)
+
+ |
+
+
+ | NXYSignificanceHeuristic |
+
+
+ 33.3%
+
+
+ (1/3)
+
+ |
+
+
+ 20%
+
+
+ (3/15)
+
+ |
+
+
+ 10.3%
+
+
+ (7/68)
+
+ |
+
+
+ | MutualInformation |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 9.1%
+
+
+ (1/11)
+
+ |
+
+
+ 11.1%
+
+
+ (4/36)
+
+ |
+
+
+ | JLHScore |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 18.2%
+
+
+ (2/11)
+
+ |
+
+
+ 7.7%
+
+
+ (2/26)
+
+ |
+
+
+ | GND |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 5.7%
+
+
+ (2/35)
+
+ |
+
+
+ | ChiSquare |
+
+
+ 50%
+
+
+ (1/2)
+
+ |
+
+
+ 10%
+
+
+ (1/10)
+
+ |
+
+
+ 12%
+
+
+ (3/25)
+
+ |
+
+
+
+
+
+
+
Coverage Summary for Class: ChiSquare (org.opensearch.search.aggregations.bucket.terms.heuristic)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ChiSquare |
+
+
+ 12.5%
+
+
+ (1/8)
+
+ |
+
+
+ 15%
+
+
+ (3/20)
+
+ |
+
+
+ | ChiSquare$ChiSquareBuilder |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/5)
+
+ |
+
+
+ | Total |
+
+
+ 10%
+
+
+ (1/10)
+
+ |
+
+
+ 12%
+
+
+ (3/25)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms.heuristic;
+
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.xcontent.ConstructingObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+
+ import java.io.IOException;
+
+ /**
+ * ChiSquare significance heuristic for significant terms agg
+ *
+ * @opensearch.internal
+ */
+ public class ChiSquare extends NXYSignificanceHeuristic {
+ public static final String NAME = "chi_square";
+ public static final ConstructingObjectParser<ChiSquare, Void> PARSER = new ConstructingObjectParser<>(
+ NAME,
+ buildFromParsedArgs(ChiSquare::new)
+ );
+ static {
+ NXYSignificanceHeuristic.declareParseFields(PARSER);
+ }
+
+ public ChiSquare(boolean includeNegatives, boolean backgroundIsSuperset) {
+ super(includeNegatives, backgroundIsSuperset);
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public ChiSquare(StreamInput in) throws IOException {
+ super(in);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof ChiSquare)) {
+ return false;
+ }
+ return super.equals(other);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = NAME.hashCode();
+ result = 31 * result + super.hashCode();
+ return result;
+ }
+
+ /**
+ * Calculates Chi^2
+ * see "Information Retrieval", Manning et al., Eq. 13.19
+ */
+ @Override
+ public double getScore(long subsetFreq, long subsetSize, long supersetFreq, long supersetSize) {
+ Frequencies frequencies = computeNxys(subsetFreq, subsetSize, supersetFreq, supersetSize, "ChiSquare");
+
+ // here we check if the term appears more often in subset than in background without subset.
+ if (!includeNegatives && frequencies.N11 / frequencies.N_1 < frequencies.N10 / frequencies.N_0) {
+ return Double.NEGATIVE_INFINITY;
+ }
+ return (frequencies.N * Math.pow((frequencies.N11 * frequencies.N00 - frequencies.N01 * frequencies.N10), 2.0) / ((frequencies.N_1)
+ * (frequencies.N1_) * (frequencies.N0_) * (frequencies.N_0)));
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(NAME);
+ super.build(builder);
+ builder.endObject();
+ return builder;
+ }
+
+ /**
+ * Builder for a chi squared heuristic
+ *
+ * @opensearch.internal
+ */
+ public static class ChiSquareBuilder extends NXYSignificanceHeuristic.NXYBuilder {
+ public ChiSquareBuilder(boolean includeNegatives, boolean backgroundIsSuperset) {
+ super(includeNegatives, backgroundIsSuperset);
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(NAME);
+ super.build(builder);
+ builder.endObject();
+ return builder;
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: GND (org.opensearch.search.aggregations.bucket.terms.heuristic)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | GND |
+
+
+ 10%
+
+
+ (1/10)
+
+ |
+
+
+ 6.7%
+
+
+ (2/30)
+
+ |
+
+
+ | GND$GNDBuilder |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/5)
+
+ |
+
+
+ | Total |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 5.7%
+
+
+ (2/35)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms.heuristic;
+
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.ConstructingObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+
+ import java.io.IOException;
+
+ import static org.opensearch.core.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+ /**
+ * GND significance heuristic for significant terms agg
+ *
+ * @opensearch.internal
+ */
+ public class GND extends NXYSignificanceHeuristic {
+ public static final String NAME = "gnd";
+ public static final ConstructingObjectParser<GND, Void> PARSER = new ConstructingObjectParser<>(NAME, args -> {
+ boolean backgroundIsSuperset = args[0] == null ? true : (boolean) args[0];
+ return new GND(backgroundIsSuperset);
+ });
+ static {
+ PARSER.declareBoolean(optionalConstructorArg(), BACKGROUND_IS_SUPERSET);
+ }
+
+ public GND(boolean backgroundIsSuperset) {
+ super(true, backgroundIsSuperset);
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public GND(StreamInput in) throws IOException {
+ super(true, in.readBoolean());
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeBoolean(backgroundIsSuperset);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof GND)) {
+ return false;
+ }
+ return super.equals(other);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = NAME.hashCode();
+ result = 31 * result + super.hashCode();
+ return result;
+ }
+
+ /**
+ * Calculates Google Normalized Distance, as described in "The Google Similarity Distance", Cilibrasi and Vitanyi, 2007
+ * link: http://arxiv.org/pdf/cs/0412098v3.pdf
+ */
+ @Override
+ public double getScore(long subsetFreq, long subsetSize, long supersetFreq, long supersetSize) {
+
+ Frequencies frequencies = computeNxys(subsetFreq, subsetSize, supersetFreq, supersetSize, "GND");
+ double fx = frequencies.N1_;
+ double fy = frequencies.N_1;
+ double fxy = frequencies.N11;
+ double N = frequencies.N;
+ if (fxy == 0) {
+ // no co-occurrence
+ return 0.0;
+ }
+ if ((fx == fy) && (fx == fxy)) {
+ // perfect co-occurrence
+ return 1.0;
+ }
+ double score = (Math.max(Math.log(fx), Math.log(fy)) - Math.log(fxy)) / (Math.log(N) - Math.min(Math.log(fx), Math.log(fy)));
+
+ // we must invert the order of terms because GND scores relevant terms low
+ score = Math.exp(-1.0d * score);
+ return score;
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(NAME);
+ builder.field(BACKGROUND_IS_SUPERSET.getPreferredName(), backgroundIsSuperset);
+ builder.endObject();
+ return builder;
+ }
+
+ /**
+ * Builder for a GND heuristic
+ *
+ * @opensearch.internal
+ */
+ public static class GNDBuilder extends NXYBuilder {
+ public GNDBuilder(boolean backgroundIsSuperset) {
+ super(true, backgroundIsSuperset);
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(NAME);
+ builder.field(BACKGROUND_IS_SUPERSET.getPreferredName(), backgroundIsSuperset);
+ builder.endObject();
+ return builder;
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: JLHScore (org.opensearch.search.aggregations.bucket.terms.heuristic)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | JLHScore |
+
+
+ 22.2%
+
+
+ (2/9)
+
+ |
+
+
+ 8.7%
+
+
+ (2/23)
+
+ |
+
+
+ | JLHScore$JLHScoreBuilder |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ | Total |
+
+
+ 18.2%
+
+
+ (2/11)
+
+ |
+
+
+ 7.7%
+
+
+ (2/26)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms.heuristic;
+
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+
+ import java.io.IOException;
+
+ /**
+ * JLHScore heuristic for significant terms agg
+ *
+ * @opensearch.internal
+ */
+ public class JLHScore extends SignificanceHeuristic {
+ public static final String NAME = "jlh";
+ public static final ObjectParser<JLHScore, Void> PARSER = new ObjectParser<>(NAME, JLHScore::new);
+
+ public JLHScore() {}
+
+ /**
+ * Read from a stream.
+ */
+ public JLHScore(StreamInput in) {
+ // Nothing to read.
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {}
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ /**
+ * Calculates the significance of a term in a sample against a background of
+ * normal distributions by comparing the changes in frequency. This is the heart
+ * of the significant terms feature.
+ */
+ @Override
+ public double getScore(long subsetFreq, long subsetSize, long supersetFreq, long supersetSize) {
+ checkFrequencyValidity(subsetFreq, subsetSize, supersetFreq, supersetSize, "JLHScore");
+ if ((subsetSize == 0) || (supersetSize == 0)) {
+ // avoid any divide by zero issues
+ return 0;
+ }
+ if (supersetFreq == 0) {
+ // If we are using a background context that is not a strict superset, a foreground
+ // term may be missing from the background, so for the purposes of this calculation
+ // we assume a value of 1 for our calculations which avoids returning an "infinity" result
+ supersetFreq = 1;
+ }
+ double subsetProbability = (double) subsetFreq / (double) subsetSize;
+ double supersetProbability = (double) supersetFreq / (double) supersetSize;
+
+ // Using absoluteProbabilityChange alone favours very common words e.g. you, we etc
+ // because a doubling in popularity of a common term is a big percent difference
+ // whereas a rare term would have to achieve a hundred-fold increase in popularity to
+ // achieve the same difference measure.
+ // In favouring common words as suggested features for search we would get high
+ // recall but low precision.
+ double absoluteProbabilityChange = subsetProbability - supersetProbability;
+ if (absoluteProbabilityChange <= 0) {
+ return 0;
+ }
+ // Using relativeProbabilityChange tends to favour rarer terms e.g.mis-spellings or
+ // unique URLs.
+ // A very low-probability term can very easily double in popularity due to the low
+ // numbers required to do so whereas a high-probability term would have to add many
+ // extra individual sightings to achieve the same shift.
+ // In favouring rare words as suggested features for search we would get high
+ // precision but low recall.
+ double relativeProbabilityChange = (subsetProbability / supersetProbability);
+
+ // A blend of the above metrics - favours medium-rare terms to strike a useful
+ // balance between precision and recall.
+ return absoluteProbabilityChange * relativeProbabilityChange;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(NAME).endObject();
+ return builder;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || obj.getClass() != getClass()) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return getClass().hashCode();
+ }
+
+ /**
+ * Builder for a JLH Score heuristic
+ *
+ * @opensearch.internal
+ */
+ public static class JLHScoreBuilder implements SignificanceHeuristicBuilder {
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(NAME).endObject();
+ return builder;
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: MutualInformation (org.opensearch.search.aggregations.bucket.terms.heuristic)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | MutualInformation |
+
+
+ 11.1%
+
+
+ (1/9)
+
+ |
+
+
+ 12.9%
+
+
+ (4/31)
+
+ |
+
+
+ | MutualInformation$MutualInformationBuilder |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/5)
+
+ |
+
+
+ | Total |
+
+
+ 9.1%
+
+
+ (1/11)
+
+ |
+
+
+ 11.1%
+
+
+ (4/36)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms.heuristic;
+
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.xcontent.ConstructingObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+
+ import java.io.IOException;
+
+ /**
+ * Mutual Information significance heuristic for significant terms agg
+ *
+ * @opensearch.internal
+ */
+ public class MutualInformation extends NXYSignificanceHeuristic {
+ public static final String NAME = "mutual_information";
+ public static final ConstructingObjectParser<MutualInformation, Void> PARSER = new ConstructingObjectParser<>(
+ NAME,
+ buildFromParsedArgs(MutualInformation::new)
+ );
+ static {
+ NXYSignificanceHeuristic.declareParseFields(PARSER);
+ }
+
+ private static final double log2 = Math.log(2.0);
+
+ public MutualInformation(boolean includeNegatives, boolean backgroundIsSuperset) {
+ super(includeNegatives, backgroundIsSuperset);
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public MutualInformation(StreamInput in) throws IOException {
+ super(in);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof MutualInformation)) {
+ return false;
+ }
+ return super.equals(other);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = NAME.hashCode();
+ result = 31 * result + super.hashCode();
+ return result;
+ }
+
+ /**
+ * Calculates mutual information
+ * see "Information Retrieval", Manning et al., Eq. 13.17
+ */
+ @Override
+ public double getScore(long subsetFreq, long subsetSize, long supersetFreq, long supersetSize) {
+ Frequencies frequencies = computeNxys(subsetFreq, subsetSize, supersetFreq, supersetSize, "MutualInformation");
+
+ double score = (getMITerm(frequencies.N00, frequencies.N0_, frequencies.N_0, frequencies.N) + getMITerm(
+ frequencies.N01,
+ frequencies.N0_,
+ frequencies.N_1,
+ frequencies.N
+ ) + getMITerm(frequencies.N10, frequencies.N1_, frequencies.N_0, frequencies.N) + getMITerm(
+ frequencies.N11,
+ frequencies.N1_,
+ frequencies.N_1,
+ frequencies.N
+ )) / log2;
+
+ if (Double.isNaN(score)) {
+ score = Double.NEGATIVE_INFINITY;
+ }
+ // here we check if the term appears more often in subset than in background without subset.
+ if (!includeNegatives && frequencies.N11 / frequencies.N_1 < frequencies.N10 / frequencies.N_0) {
+ score = Double.NEGATIVE_INFINITY;
+ }
+ return score;
+ }
+
+ /* make sure that
+ 0 * log(0/0) = 0
+ 0 * log(0) = 0
+ Else, this would be the score:
+ double score =
+ N11 / N * Math.log((N * N11) / (N1_ * N_1))
+ + N01 / N * Math.log((N * N01) / (N0_ * N_1))
+ + N10 / N * Math.log((N * N10) / (N1_ * N_0))
+ + N00 / N * Math.log((N * N00) / (N0_ * N_0));
+
+ but we get many NaN if we do not take case of the 0s */
+
+ double getMITerm(double Nxy, double Nx_, double N_y, double N) {
+ double numerator = Math.abs(N * Nxy);
+ double denominator = Math.abs(Nx_ * N_y);
+ double factor = Math.abs(Nxy / N);
+ if (numerator < 1.e-7 && factor < 1.e-7) {
+ return 0.0;
+ } else {
+ return factor * Math.log(numerator / denominator);
+ }
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(NAME);
+ super.build(builder);
+ builder.endObject();
+ return builder;
+ }
+
+ /**
+ * Builder for a Mutual Information heuristic
+ *
+ * @opensearch.internal
+ */
+ public static class MutualInformationBuilder extends NXYBuilder {
+ public MutualInformationBuilder(boolean includeNegatives, boolean backgroundIsSuperset) {
+ super(includeNegatives, backgroundIsSuperset);
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(NAME);
+ super.build(builder);
+ builder.endObject();
+ return builder;
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: NXYSignificanceHeuristic (org.opensearch.search.aggregations.bucket.terms.heuristic)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | NXYSignificanceHeuristic |
+
+
+ 25%
+
+
+ (3/12)
+
+ |
+
+
+ 11.7%
+
+
+ (7/60)
+
+ |
+
+
+ | NXYSignificanceHeuristic$Frequencies |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ 0%
+
+
+ (0/1)
+
+ |
+
+
+ | NXYSignificanceHeuristic$NXYBuilder |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/7)
+
+ |
+
+
+ | Total |
+
+
+ 20%
+
+
+ (3/15)
+
+ |
+
+
+ 10.3%
+
+
+ (7/68)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms.heuristic;
+
+ import org.opensearch.core.ParseField;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.ConstructingObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+
+ import java.io.IOException;
+ import java.util.function.BiFunction;
+ import java.util.function.Function;
+
+ import static org.opensearch.core.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+ /**
+ * NXY significance heuristic for significant terms agg
+ *
+ * @opensearch.internal
+ */
+ public abstract class NXYSignificanceHeuristic extends SignificanceHeuristic {
+
+ protected static final ParseField BACKGROUND_IS_SUPERSET = new ParseField("background_is_superset");
+
+ protected static final ParseField INCLUDE_NEGATIVES_FIELD = new ParseField("include_negatives");
+
+ protected static final String SCORE_ERROR_MESSAGE = ", does your background filter not include all documents in the bucket? "
+ + "If so and it is intentional, set \""
+ + BACKGROUND_IS_SUPERSET.getPreferredName()
+ + "\": false";
+
+ protected final boolean backgroundIsSuperset;
+
+ /**
+ * Some heuristics do not differentiate between terms that are descriptive for subset or for
+ * the background without the subset. We might want to filter out the terms that are appear much less often
+ * in the subset than in the background without the subset.
+ */
+ protected final boolean includeNegatives;
+
+ protected NXYSignificanceHeuristic(boolean includeNegatives, boolean backgroundIsSuperset) {
+ this.includeNegatives = includeNegatives;
+ this.backgroundIsSuperset = backgroundIsSuperset;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ protected NXYSignificanceHeuristic(StreamInput in) throws IOException {
+ includeNegatives = in.readBoolean();
+ backgroundIsSuperset = in.readBoolean();
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeBoolean(includeNegatives);
+ out.writeBoolean(backgroundIsSuperset);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null) return false;
+ if (getClass() != obj.getClass()) return false;
+ NXYSignificanceHeuristic other = (NXYSignificanceHeuristic) obj;
+ if (backgroundIsSuperset != other.backgroundIsSuperset) return false;
+ if (includeNegatives != other.includeNegatives) return false;
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = (includeNegatives ? 1 : 0);
+ result = 31 * result + (backgroundIsSuperset ? 1 : 0);
+ return result;
+ }
+
+ /**
+ * Frequencies for an NXY significance heuristic
+ *
+ * @opensearch.internal
+ */
+ protected static class Frequencies {
+ double N00, N01, N10, N11, N0_, N1_, N_0, N_1, N;
+ }
+
+ protected Frequencies computeNxys(long subsetFreq, long subsetSize, long supersetFreq, long supersetSize, String scoreFunctionName) {
+ checkFrequencies(subsetFreq, subsetSize, supersetFreq, supersetSize, scoreFunctionName);
+ Frequencies frequencies = new Frequencies();
+ if (backgroundIsSuperset) {
+ // documents not in class and do not contain term
+ frequencies.N00 = supersetSize - supersetFreq - (subsetSize - subsetFreq);
+ // documents in class and do not contain term
+ frequencies.N01 = (subsetSize - subsetFreq);
+ // documents not in class and do contain term
+ frequencies.N10 = supersetFreq - subsetFreq;
+ // documents in class and do contain term
+ frequencies.N11 = subsetFreq;
+ // documents that do not contain term
+ frequencies.N0_ = supersetSize - supersetFreq;
+ // documents that contain term
+ frequencies.N1_ = supersetFreq;
+ // documents that are not in class
+ frequencies.N_0 = supersetSize - subsetSize;
+ // documents that are in class
+ frequencies.N_1 = subsetSize;
+ // all docs
+ frequencies.N = supersetSize;
+ } else {
+ // documents not in class and do not contain term
+ frequencies.N00 = supersetSize - supersetFreq;
+ // documents in class and do not contain term
+ frequencies.N01 = subsetSize - subsetFreq;
+ // documents not in class and do contain term
+ frequencies.N10 = supersetFreq;
+ // documents in class and do contain term
+ frequencies.N11 = subsetFreq;
+ // documents that do not contain term
+ frequencies.N0_ = supersetSize - supersetFreq + subsetSize - subsetFreq;
+ // documents that contain term
+ frequencies.N1_ = supersetFreq + subsetFreq;
+ // documents that are not in class
+ frequencies.N_0 = supersetSize;
+ // documents that are in class
+ frequencies.N_1 = subsetSize;
+ // all docs
+ frequencies.N = supersetSize + subsetSize;
+ }
+ return frequencies;
+ }
+
+ protected void checkFrequencies(long subsetFreq, long subsetSize, long supersetFreq, long supersetSize, String scoreFunctionName) {
+ checkFrequencyValidity(subsetFreq, subsetSize, supersetFreq, supersetSize, scoreFunctionName);
+ if (backgroundIsSuperset) {
+ if (subsetFreq > supersetFreq) {
+ throw new IllegalArgumentException("subsetFreq > supersetFreq" + SCORE_ERROR_MESSAGE);
+ }
+ if (subsetSize > supersetSize) {
+ throw new IllegalArgumentException("subsetSize > supersetSize" + SCORE_ERROR_MESSAGE);
+ }
+ if (supersetFreq - subsetFreq > supersetSize - subsetSize) {
+ throw new IllegalArgumentException("supersetFreq - subsetFreq > supersetSize - subsetSize" + SCORE_ERROR_MESSAGE);
+ }
+ }
+ }
+
+ protected void build(XContentBuilder builder) throws IOException {
+ builder.field(INCLUDE_NEGATIVES_FIELD.getPreferredName(), includeNegatives)
+ .field(BACKGROUND_IS_SUPERSET.getPreferredName(), backgroundIsSuperset);
+ }
+
+ /**
+ * Set up and {@linkplain ConstructingObjectParser} to accept the standard arguments for an {@linkplain NXYSignificanceHeuristic}.
+ */
+ protected static void declareParseFields(ConstructingObjectParser<? extends NXYSignificanceHeuristic, ?> parser) {
+ parser.declareBoolean(optionalConstructorArg(), INCLUDE_NEGATIVES_FIELD);
+ parser.declareBoolean(optionalConstructorArg(), BACKGROUND_IS_SUPERSET);
+ }
+
+ /**
+ * Adapt a standard two argument ctor into one that consumes a {@linkplain ConstructingObjectParser}'s fields.
+ */
+ protected static <T> Function<Object[], T> buildFromParsedArgs(BiFunction<Boolean, Boolean, T> ctor) {
+ return args -> {
+ boolean includeNegatives = args[0] == null ? false : (boolean) args[0];
+ boolean backgroundIsSuperset = args[1] == null ? true : (boolean) args[1];
+ return ctor.apply(includeNegatives, backgroundIsSuperset);
+ };
+ }
+
+ /**
+ * Builder for a NXY Significance heuristic
+ *
+ * @opensearch.internal
+ */
+ protected abstract static class NXYBuilder implements SignificanceHeuristicBuilder {
+ protected boolean includeNegatives = true;
+ protected boolean backgroundIsSuperset = true;
+
+ public NXYBuilder(boolean includeNegatives, boolean backgroundIsSuperset) {
+ this.includeNegatives = includeNegatives;
+ this.backgroundIsSuperset = backgroundIsSuperset;
+ }
+
+ protected void build(XContentBuilder builder) throws IOException {
+ builder.field(INCLUDE_NEGATIVES_FIELD.getPreferredName(), includeNegatives)
+ .field(BACKGROUND_IS_SUPERSET.getPreferredName(), backgroundIsSuperset);
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: PercentageScore (org.opensearch.search.aggregations.bucket.terms.heuristic)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | PercentageScore |
+
+
+ 10%
+
+
+ (1/10)
+
+ |
+
+
+ 5.3%
+
+
+ (1/19)
+
+ |
+
+
+ | PercentageScore$PercentageScoreBuilder |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/3)
+
+ |
+
+
+ | Total |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 4.5%
+
+
+ (1/22)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms.heuristic;
+
+ import org.opensearch.OpenSearchParseException;
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.ObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.core.xcontent.XContentParser;
+ import org.opensearch.index.query.QueryShardException;
+
+ import java.io.IOException;
+
+ /**
+ * Percentage score significance heuristic for significant terms agg
+ *
+ * @opensearch.internal
+ */
+ public class PercentageScore extends SignificanceHeuristic {
+ public static final String NAME = "percentage";
+ public static final ObjectParser<PercentageScore, Void> PARSER = new ObjectParser<>(NAME, PercentageScore::new);
+
+ public PercentageScore() {}
+
+ public PercentageScore(StreamInput in) {
+ // Nothing to read.
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {}
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(NAME).endObject();
+ return builder;
+ }
+
+ public static SignificanceHeuristic parse(XContentParser parser) throws IOException, QueryShardException {
+ // move to the closing bracket
+ if (!parser.nextToken().equals(XContentParser.Token.END_OBJECT)) {
+ throw new OpenSearchParseException(
+ "failed to parse [percentage] significance heuristic. expected an empty object, " + "but got [{}] instead",
+ parser.currentToken()
+ );
+ }
+ return new PercentageScore();
+ }
+
+ /**
+ * Indicates the significance of a term in a sample by determining what percentage
+ * of all occurrences of a term are found in the sample.
+ */
+ @Override
+ public double getScore(long subsetFreq, long subsetSize, long supersetFreq, long supersetSize) {
+ checkFrequencyValidity(subsetFreq, subsetSize, supersetFreq, supersetSize, "PercentageScore");
+ if (supersetFreq == 0) {
+ // avoid a divide by zero issue
+ return 0;
+ }
+ return (double) subsetFreq / (double) supersetFreq;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || obj.getClass() != getClass()) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return getClass().hashCode();
+ }
+
+ /**
+ * Builder for a Percentage Score heuristic
+ *
+ * @opensearch.internal
+ */
+ public static class PercentageScoreBuilder implements SignificanceHeuristicBuilder {
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(NAME).endObject();
+ return builder;
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: ScriptHeuristic (org.opensearch.search.aggregations.bucket.terms.heuristic)
+
+
+
+
+ | Class |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | ScriptHeuristic |
+
+
+ 8.3%
+
+
+ (1/12)
+
+ |
+
+
+ 7.7%
+
+
+ (2/26)
+
+ |
+
+
+ | ScriptHeuristic$ExecutableScriptHeuristic |
+
+
+ 0%
+
+
+ (0/2)
+
+ |
+
+
+ 0%
+
+
+ (0/17)
+
+ |
+
+
+ | ScriptHeuristic$LongAccessor |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ 0%
+
+
+ (0/6)
+
+ |
+
+
+ | Total |
+
+
+ 5%
+
+
+ (1/20)
+
+ |
+
+
+ 4.1%
+
+
+ (2/49)
+
+ |
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms.heuristic;
+
+ import org.opensearch.core.common.io.stream.StreamInput;
+ import org.opensearch.core.common.io.stream.StreamOutput;
+ import org.opensearch.core.xcontent.ConstructingObjectParser;
+ import org.opensearch.core.xcontent.XContentBuilder;
+ import org.opensearch.index.query.QueryShardContext;
+ import org.opensearch.script.Script;
+ import org.opensearch.script.SignificantTermsHeuristicScoreScript;
+ import org.opensearch.search.aggregations.InternalAggregation;
+
+ import java.io.IOException;
+ import java.util.HashMap;
+ import java.util.Map;
+ import java.util.Objects;
+
+ import static org.opensearch.core.xcontent.ConstructingObjectParser.constructorArg;
+
+ /**
+ * Script significance heuristic for significant terms agg
+ *
+ * @opensearch.internal
+ */
+ public class ScriptHeuristic extends SignificanceHeuristic {
+ public static final String NAME = "script_heuristic";
+ public static final ConstructingObjectParser<ScriptHeuristic, Void> PARSER = new ConstructingObjectParser<>(
+ NAME,
+ args -> new ScriptHeuristic((Script) args[0])
+ );
+ static {
+ Script.declareScript(PARSER, constructorArg());
+ }
+
+ private final Script script;
+
+ /**
+ * This class holds an executable form of the script with private variables ready for execution
+ * on a single search thread.
+ *
+ * @opensearch.internal
+ */
+ static class ExecutableScriptHeuristic extends ScriptHeuristic {
+ private final LongAccessor subsetSizeHolder;
+ private final LongAccessor supersetSizeHolder;
+ private final LongAccessor subsetDfHolder;
+ private final LongAccessor supersetDfHolder;
+ private final SignificantTermsHeuristicScoreScript executableScript;
+ private final Map<String, Object> params = new HashMap<>();
+
+ ExecutableScriptHeuristic(Script script, SignificantTermsHeuristicScoreScript executableScript) {
+ super(script);
+ subsetSizeHolder = new LongAccessor();
+ supersetSizeHolder = new LongAccessor();
+ subsetDfHolder = new LongAccessor();
+ supersetDfHolder = new LongAccessor();
+ this.executableScript = executableScript;
+ params.putAll(script.getParams());
+ params.put("_subset_freq", subsetDfHolder);
+ params.put("_subset_size", subsetSizeHolder);
+ params.put("_superset_freq", supersetDfHolder);
+ params.put("_superset_size", supersetSizeHolder);
+ }
+
+ @Override
+ public double getScore(long subsetFreq, long subsetSize, long supersetFreq, long supersetSize) {
+ subsetSizeHolder.value = subsetSize;
+ supersetSizeHolder.value = supersetSize;
+ subsetDfHolder.value = subsetFreq;
+ supersetDfHolder.value = supersetFreq;
+ return executableScript.execute(params);
+ }
+ }
+
+ public ScriptHeuristic(Script script) {
+ this.script = script;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public ScriptHeuristic(StreamInput in) throws IOException {
+ this(new Script(in));
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ script.writeTo(out);
+ }
+
+ @Override
+ public SignificanceHeuristic rewrite(InternalAggregation.ReduceContext context) {
+ SignificantTermsHeuristicScoreScript.Factory factory = context.scriptService()
+ .compile(script, SignificantTermsHeuristicScoreScript.CONTEXT);
+ return new ExecutableScriptHeuristic(script, factory.newInstance());
+ }
+
+ @Override
+ public SignificanceHeuristic rewrite(QueryShardContext queryShardContext) {
+ SignificantTermsHeuristicScoreScript.Factory compiledScript = queryShardContext.compile(
+ script,
+ SignificantTermsHeuristicScoreScript.CONTEXT
+ );
+ return new ExecutableScriptHeuristic(script, compiledScript.newInstance());
+ }
+
+ /**
+ * Calculates score with a script
+ *
+ * @param subsetFreq The frequency of the term in the selected sample
+ * @param subsetSize The size of the selected sample (typically number of docs)
+ * @param supersetFreq The frequency of the term in the superset from which the sample was taken
+ * @param supersetSize The size of the superset from which the sample was taken (typically number of docs)
+ * @return a "significance" score
+ */
+ @Override
+ public double getScore(long subsetFreq, long subsetSize, long supersetFreq, long supersetSize) {
+ throw new UnsupportedOperationException(
+ "This scoring heuristic must have 'rewrite' called on it to provide a version ready " + "for use"
+ );
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params builderParams) throws IOException {
+ builder.startObject(NAME);
+ builder.field(Script.SCRIPT_PARSE_FIELD.getPreferredName());
+ script.toXContent(builder, builderParams);
+ builder.endObject();
+ return builder;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(script);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ ScriptHeuristic other = (ScriptHeuristic) obj;
+ return Objects.equals(script, other.script);
+ }
+
+ /**
+ * Accesses long values
+ *
+ * @opensearch.internal
+ */
+ public final class LongAccessor extends Number {
+ public long value;
+
+ @Override
+ public int intValue() {
+ return (int) value;
+ }
+
+ @Override
+ public long longValue() {
+ return value;
+ }
+
+ @Override
+ public float floatValue() {
+ return value;
+ }
+
+ @Override
+ public double doubleValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ return Long.toString(value);
+ }
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: SignificanceHeuristic (org.opensearch.search.aggregations.bucket.terms.heuristic)
+
+
+
+ | Class |
+
+ Class, %
+ |
+
+ Method, %
+ |
+
+ Line, %
+ |
+
+
+ | SignificanceHeuristic |
+
+
+ 100%
+
+
+ (1/1)
+
+ |
+
+
+ 25%
+
+
+ (1/4)
+
+ |
+
+
+ 11.1%
+
+
+ (1/9)
+
+ |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms.heuristic;
+
+ import org.opensearch.core.common.io.stream.NamedWriteable;
+ import org.opensearch.core.xcontent.ToXContentFragment;
+ import org.opensearch.index.query.QueryShardContext;
+ import org.opensearch.search.aggregations.InternalAggregation;
+ import org.opensearch.search.aggregations.bucket.terms.SignificantTerms;
+
+ /**
+ * Heuristic for that {@link SignificantTerms} uses to pick out significant terms.
+ *
+ * @opensearch.internal
+ */
+ public abstract class SignificanceHeuristic implements NamedWriteable, ToXContentFragment {
+ /**
+ * @param subsetFreq The frequency of the term in the selected sample
+ * @param subsetSize The size of the selected sample (typically number of docs)
+ * @param supersetFreq The frequency of the term in the superset from which the sample was taken
+ * @param supersetSize The size of the superset from which the sample was taken (typically number of docs)
+ * @return a "significance" score
+ */
+ public abstract double getScore(long subsetFreq, long subsetSize, long supersetFreq, long supersetSize);
+
+ protected void checkFrequencyValidity(
+ long subsetFreq,
+ long subsetSize,
+ long supersetFreq,
+ long supersetSize,
+ String scoreFunctionName
+ ) {
+ if (subsetFreq < 0 || subsetSize < 0 || supersetFreq < 0 || supersetSize < 0) {
+ throw new IllegalArgumentException(
+ "Frequencies of subset and superset must be positive in " + scoreFunctionName + ".getScore()"
+ );
+ }
+ if (subsetFreq > subsetSize) {
+ throw new IllegalArgumentException("subsetFreq > subsetSize, in " + scoreFunctionName);
+ }
+ if (supersetFreq > supersetSize) {
+ throw new IllegalArgumentException("supersetFreq > supersetSize, in " + scoreFunctionName);
+ }
+ }
+
+ /**
+ * Provides a hook for subclasses to provide a version of the heuristic
+ * prepared for execution on data on the coordinating node.
+ * @param reduceContext the reduce context on the coordinating node
+ * @return a version of this heuristic suitable for execution
+ */
+ public SignificanceHeuristic rewrite(InternalAggregation.ReduceContext reduceContext) {
+ return this;
+ }
+
+ /**
+ * Provides a hook for subclasses to provide a version of the heuristic
+ * prepared for execution on data on a shard.
+ * @param queryShardContext the shard context on the data node
+ * @return a version of this heuristic suitable for execution
+ */
+ public SignificanceHeuristic rewrite(QueryShardContext queryShardContext) {
+ return this;
+ }
+ }
+
+
+
+
+
+
Coverage Summary for Class: SignificanceHeuristicBuilder (org.opensearch.search.aggregations.bucket.terms.heuristic)
+
+
+
+ | Class |
+
+
+ | SignificanceHeuristicBuilder |
+
+
+
+
+
+
+
+
+
+ /*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+ /*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+ package org.opensearch.search.aggregations.bucket.terms.heuristic;
+
+ import org.opensearch.core.xcontent.ToXContentFragment;
+
+ /**
+ * Base builder class for significance heuristics
+ *
+ * @opensearch.internal
+ */
+ public interface SignificanceHeuristicBuilder extends ToXContentFragment {}
+
+
+