diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java index 0d4e35f7c3ab8..9bb4d15d72d12 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java @@ -35,7 +35,6 @@ import org.opensearch.index.mapper.CompositeMappedFieldType; import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.mapper.StarTreeMapper; import java.io.IOException; import java.util.ArrayList; @@ -221,12 +220,8 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, } // we have all the required fields to build composite fields if (compositeFieldSet.isEmpty()) { - for (CompositeMappedFieldType mappedType : compositeMappedFieldTypes) { - if (mappedType instanceof StarTreeMapper.StarTreeFieldType) { - try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService, fieldNumberAcrossCompositeFields)) { - starTreesBuilder.build(metaOut, dataOut, fieldProducerMap, composite99DocValuesConsumer); - } - } + try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService, fieldNumberAcrossCompositeFields)) { + starTreesBuilder.build(metaOut, dataOut, fieldProducerMap, composite99DocValuesConsumer); } } } @@ -295,16 +290,12 @@ private void mergeStarTreeFields(MergeState mergeState) throws IOException { if (compositeIndexValues instanceof StarTreeValues) { StarTreeValues starTreeValues = (StarTreeValues) compositeIndexValues; List fieldsList = starTreeSubsPerField.getOrDefault(fieldInfo.getField(), new ArrayList<>()); - if (starTreeField == null) { - starTreeField = starTreeValues.getStarTreeField(); - } + starTreeField = starTreeValues.getStarTreeField(); // assert star tree configuration is same across segments - else { - if (starTreeField.equals(starTreeValues.getStarTreeField()) == false) { - throw new IllegalArgumentException( - "star tree field configuration must match the configuration of the field being merged" - ); - } + if (starTreeField.equals(starTreeValues.getStarTreeField()) == false) { + throw new IllegalArgumentException( + "star tree field configuration must match the configuration of the field being merged" + ); } fieldsList.add(starTreeValues); starTreeSubsPerField.put(fieldInfo.getField(), fieldsList); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java index e665831b83d93..4f93d2b9c0ff2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java @@ -35,7 +35,7 @@ public class StarTreeIndexSettings { "index.composite_index.star_tree.max_fields", 1, 1, - 1, + 2, Setting.Property.IndexScope, Setting.Property.Final ); diff --git a/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java index 1c267c67e60ed..5aaf73008edf1 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java @@ -25,6 +25,7 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.tests.util.TestUtil; +import org.apache.lucene.util.NumericUtils; import org.opensearch.Version; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.metadata.IndexMetadata; @@ -61,6 +62,7 @@ import java.util.List; import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX; +import static org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING; import static org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils.assertStarTreeDocuments; /** @@ -68,7 +70,10 @@ */ @LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose") public class StarTreeDocValuesFormatTests extends BaseDocValuesFormatTestCase { - MapperService mapperService = null; + + final Logger logger = LogManager.getLogger(StarTreeDocValuesFormatTests.class); + Codec codec = null; + StarTreeFieldConfiguration.StarTreeBuildMode buildMode; public StarTreeDocValuesFormatTests(StarTreeFieldConfiguration.StarTreeBuildMode buildMode) { @@ -84,7 +89,7 @@ public static Collection parameters() { } @BeforeClass - public static void createMapper() throws Exception { + public static void createMapper() { FeatureFlags.initializeFeatureFlags(Settings.builder().put(STAR_TREE_INDEX, "true").build()); } @@ -94,24 +99,38 @@ public static void clearMapper() { } @After - public void teardown() throws IOException { - mapperService.close(); + public void teardown() throws Exception { + super.tearDown(); } @Override protected Codec getCodec() { - final Logger testLogger = LogManager.getLogger(StarTreeDocValuesFormatTests.class); - - try { - createMapperService(getExpandedMapping()); - } catch (IOException e) { - throw new RuntimeException(e); + if (codec == null) { + MapperService mapperService = null; + try { + mapperService = createMapperService(getExpandedMapping(), 1); + } catch (IOException e) { + throw new RuntimeException(e); + } + codec = new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger); } - Codec codec = new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, testLogger); + return codec; } + private void initialiseCodec(MapperService mapperService) { + codec = new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger); + Codec.setDefault(codec); + } + public void testStarTreeDocValues() throws IOException { + MapperService mapperService; + try { + mapperService = createMapperService(getExpandedMapping(), 1); + } catch (IOException e) { + throw new RuntimeException(e); + } + initialiseCodec(mapperService); Directory directory = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(null); conf.setMergePolicy(newLogMergePolicy()); @@ -207,6 +226,135 @@ public void testStarTreeDocValues() throws IOException { directory.close(); } + public void testStarTreeDocValues_allValuesNullInSecondTree() throws IOException { + MapperService mapperService; + try { + mapperService = createMapperService(getExpandedMappingV2(), 2); + } catch (IOException e) { + throw new RuntimeException(e); + } + initialiseCodec(mapperService); + Directory directory = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(null); + conf.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); + Document doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + doc.add(new SortedNumericDocValuesField("dv", 1)); + doc.add(new SortedNumericDocValuesField("field", -1)); + iw.addDocument(doc); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + doc.add(new SortedNumericDocValuesField("dv", 1)); + doc.add(new SortedNumericDocValuesField("field", -1)); + iw.addDocument(doc); + doc = new Document(); + if (random().nextBoolean()) { + iw.forceMerge(1); + } + doc.add(new SortedNumericDocValuesField("sndv", 2)); + doc.add(new SortedNumericDocValuesField("dv", 2)); + doc.add(new SortedNumericDocValuesField("field", -2)); + iw.addDocument(doc); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 2)); + doc.add(new SortedNumericDocValuesField("dv", 2)); + doc.add(new SortedNumericDocValuesField("field", -2)); + iw.addDocument(doc); + iw.forceMerge(1); + iw.close(); + + DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(directory)); + TestUtil.checkReader(ir); + assertEquals(1, ir.leaves().size()); + + // Segment documents + /** + * sndv dv field + * [1, 1, -1] + * [1, 1, -1] + * [2, 2, -2] + * [2, 2, -2] + */ + // Star tree docuements + /** + * sndv dv | [ sum, value_count, min, max[field]] , [ sum, value_count, min, max[sndv]], doc_count + * [1, 1] | [-2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0] + * [2, 2] | [-4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0] + * [null, 1] | [-2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0] + * [null, 2] | [-4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0] + */ + StarTreeDocument[] expectedStarTreeDocuments = new StarTreeDocument[4]; + expectedStarTreeDocuments[0] = new StarTreeDocument( + new Long[] { 1L, 1L }, + new Double[] { -2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0 } + ); + expectedStarTreeDocuments[1] = new StarTreeDocument( + new Long[] { 2L, 2L }, + new Double[] { -4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0 } + ); + expectedStarTreeDocuments[2] = new StarTreeDocument( + new Long[] { null, 1L }, + new Double[] { -2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0 } + ); + expectedStarTreeDocuments[3] = new StarTreeDocument( + new Long[] { null, 2L }, + new Double[] { -4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0 } + ); + + for (LeafReaderContext context : ir.leaves()) { + SegmentReader reader = Lucene.segmentReader(context.reader()); + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + List compositeIndexFields = starTreeDocValuesReader.getCompositeIndexFields(); + + for (CompositeIndexFieldInfo compositeIndexFieldInfo : compositeIndexFields) { + StarTreeValues starTreeValues = (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(compositeIndexFieldInfo); + if (starTreeValues.getStarTreeField().getName().equals("startree1")) { + StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + List.of( + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG + ), + reader.maxDoc() + ); + assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocuments); + } else if (starTreeValues.getStarTreeField().getName().equals("startree2")) { + StarTreeDocument[] resultStarTreeDocuments = new StarTreeDocument[4]; + resultStarTreeDocuments[0] = new StarTreeDocument(new Long[] { null, null, null }, new Double[] { 0.0, 4.0 }); + resultStarTreeDocuments[1] = new StarTreeDocument(new Long[] { null, null, null }, new Double[] { null, null }); + resultStarTreeDocuments[2] = new StarTreeDocument(new Long[] { null, null, null }, new Double[] { null, null }); + resultStarTreeDocuments[3] = new StarTreeDocument(new Long[] { null, null, null }, new Double[] { null, null }); + StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + List.of( + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG + ), + reader.maxDoc() + ); + assertStarTreeDocuments(starTreeDocuments, resultStarTreeDocuments); + } + } + } + ir.close(); + directory.close(); + } + private XContentBuilder getExpandedMapping() throws IOException { return topMapping(b -> { b.startObject("composite"); @@ -261,23 +409,278 @@ private XContentBuilder getExpandedMapping() throws IOException { }); } + public void testMultipleStarTrees() throws IOException { + MapperService mapperService; + try { + mapperService = createMapperService(getExpandedMappingV2(), 2); + } catch (IOException e) { + throw new RuntimeException(e); + } + initialiseCodec(mapperService); + Directory directory = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(null); + conf.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); + Document doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + doc.add(new SortedNumericDocValuesField("dv", 1)); + doc.add(new SortedNumericDocValuesField("field", -1)); + doc.add(new SortedNumericDocValuesField("fieldC", 1)); + doc.add(new SortedNumericDocValuesField("fieldB", 11)); + doc.add(new SortedNumericDocValuesField("fieldL", 21)); + doc.add(new SortedNumericDocValuesField("fieldI", NumericUtils.doubleToSortableLong(400.0))); + iw.addDocument(doc); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + doc.add(new SortedNumericDocValuesField("dv", 1)); + doc.add(new SortedNumericDocValuesField("field", -1)); + doc.add(new SortedNumericDocValuesField("fieldC", 1)); + doc.add(new SortedNumericDocValuesField("fieldB", 12)); + doc.add(new SortedNumericDocValuesField("fieldL", 22)); + doc.add(new SortedNumericDocValuesField("fieldI", NumericUtils.doubleToSortableLong(200.0))); + iw.addDocument(doc); + doc = new Document(); + if (random().nextBoolean()) { + iw.forceMerge(1); + } + doc.add(new SortedNumericDocValuesField("sndv", 2)); + doc.add(new SortedNumericDocValuesField("dv", 2)); + doc.add(new SortedNumericDocValuesField("field", -2)); + doc.add(new SortedNumericDocValuesField("fieldC", 2)); + doc.add(new SortedNumericDocValuesField("fieldB", 13)); + doc.add(new SortedNumericDocValuesField("fieldL", 23)); + doc.add(new SortedNumericDocValuesField("fieldI", NumericUtils.doubleToSortableLong(300.0))); + iw.addDocument(doc); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 2)); + doc.add(new SortedNumericDocValuesField("dv", 2)); + doc.add(new SortedNumericDocValuesField("field", -2)); + doc.add(new SortedNumericDocValuesField("fieldC", 2)); + doc.add(new SortedNumericDocValuesField("fieldB", 13)); + doc.add(new SortedNumericDocValuesField("fieldL", 21)); + doc.add(new SortedNumericDocValuesField("fieldI", NumericUtils.doubleToSortableLong(100.0))); + iw.addDocument(doc); + + doc = new Document(); + doc.add(new SortedNumericDocValuesField("fieldC", 3)); + doc.add(new SortedNumericDocValuesField("fieldB", 11)); + doc.add(new SortedNumericDocValuesField("fieldL", 21)); + doc.add(new SortedNumericDocValuesField("fieldI", NumericUtils.doubleToSortableLong(600.0))); + iw.addDocument(doc); + + doc = new Document(); + doc.add(new SortedNumericDocValuesField("fieldC", 3)); + doc.add(new SortedNumericDocValuesField("fieldB", 12)); + doc.add(new SortedNumericDocValuesField("fieldL", 23)); + doc.add(new SortedNumericDocValuesField("fieldI", NumericUtils.doubleToSortableLong(200.0))); + iw.addDocument(doc); + + doc = new Document(); + doc.add(new SortedNumericDocValuesField("fieldC", 3)); + doc.add(new SortedNumericDocValuesField("fieldB", 12)); + doc.add(new SortedNumericDocValuesField("fieldL", 21)); + doc.add(new SortedNumericDocValuesField("fieldI", NumericUtils.doubleToSortableLong(400.0))); + iw.addDocument(doc); + + iw.forceMerge(1); + iw.close(); + + DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(directory)); + TestUtil.checkReader(ir); + assertEquals(1, ir.leaves().size()); + + for (LeafReaderContext context : ir.leaves()) { + SegmentReader reader = Lucene.segmentReader(context.reader()); + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + List compositeIndexFields = starTreeDocValuesReader.getCompositeIndexFields(); + + for (CompositeIndexFieldInfo compositeIndexFieldInfo : compositeIndexFields) { + StarTreeValues starTreeValues = (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(compositeIndexFieldInfo); + if (starTreeValues.getStarTreeField().getName().equals("startree2")) { + StarTreeDocument[] expectedStarTreeDocuments = new StarTreeDocument[7]; + expectedStarTreeDocuments[0] = new StarTreeDocument(new Long[] { 1L, 11L, 21L }, new Object[] { 400.0, 1.0 }); + expectedStarTreeDocuments[1] = new StarTreeDocument(new Long[] { 1L, 12L, 22L }, new Object[] { 200.0, 1.0 }); + expectedStarTreeDocuments[2] = new StarTreeDocument(new Long[] { 2L, 13L, 21L }, new Object[] { 100.0, 1.0 }); + expectedStarTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 13L, 23L }, new Object[] { 300.0, 1.0 }); + expectedStarTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 11L, 21L }, new Object[] { 600.0, 1.0 }); + expectedStarTreeDocuments[5] = new StarTreeDocument(new Long[] { 3L, 12L, 21L }, new Object[] { 400.0, 1.0 }); + expectedStarTreeDocuments[6] = new StarTreeDocument(new Long[] { 3L, 12L, 23L }, new Object[] { 200.0, 1.0 }); + + StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + List.of(NumberFieldMapper.NumberType.DOUBLE, NumberFieldMapper.NumberType.LONG), + reader.maxDoc() + ); + assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocuments); + } else if (starTreeValues.getStarTreeField().getName().equals("startree1")) { + StarTreeDocument[] expectedStarTreeDocuments = new StarTreeDocument[7]; + expectedStarTreeDocuments[0] = new StarTreeDocument( + new Long[] { 1L, 1L }, + new Double[] { -2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0 } + ); + expectedStarTreeDocuments[1] = new StarTreeDocument( + new Long[] { 2L, 2L }, + new Double[] { -4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0 } + ); + expectedStarTreeDocuments[2] = new StarTreeDocument( + new Long[] { null, null }, + new Double[] { 0.0, 0.0, null, null, 0.0, 0.0, null, null, 3.0 } + ); + expectedStarTreeDocuments[3] = new StarTreeDocument( + new Long[] { null, 1L }, + new Double[] { -2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0 } + ); + expectedStarTreeDocuments[4] = new StarTreeDocument( + new Long[] { null, 2L }, + new Double[] { -4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0 } + ); + expectedStarTreeDocuments[5] = new StarTreeDocument( + new Long[] { null, null }, + new Double[] { 0.0, 0.0, null, null, 0.0, 0.0, null, null, 3.0 } + ); + expectedStarTreeDocuments[6] = new StarTreeDocument( + new Long[] { null, null }, + new Double[] { -6.0, 4.0, -2.0, -1.0, 6.0, 4.0, 1.0, 2.0, 7.0 } + ); + + StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + List.of( + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG + ), + reader.maxDoc() + ); + assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocuments); + } + } + } + ir.close(); + directory.close(); + } + + private XContentBuilder getExpandedMappingV2() throws IOException { + return topMapping(b -> { + b.startObject("composite"); + + // Startree 1 + b.startObject("startree1"); + b.field("type", "star_tree"); + b.startObject("config"); + b.field("max_leaf_docs", 1); + b.startArray("ordered_dimensions"); + b.startObject(); + b.field("name", "sndv"); + b.endObject(); + b.startObject(); + b.field("name", "dv"); + b.endObject(); + b.endArray(); + b.startArray("metrics"); + b.startObject(); + b.field("name", "field"); + b.startArray("stats"); + b.value("sum"); + b.value("value_count"); + b.value("avg"); + b.value("min"); + b.value("max"); + b.endArray(); + b.endObject(); + b.startObject(); + b.field("name", "sndv"); + b.startArray("stats"); + b.value("sum"); + b.value("value_count"); + b.value("avg"); + b.value("min"); + b.value("max"); + b.endArray(); + b.endObject(); + b.endArray(); + b.endObject(); + b.endObject(); + + // Startree 2 + b.startObject("startree2"); + b.field("type", "star_tree"); + b.startObject("config"); + b.field("max_leaf_docs", 1); + b.startArray("ordered_dimensions"); + b.startObject(); + b.field("name", "fieldC"); + b.endObject(); + b.startObject(); + b.field("name", "fieldB"); + b.endObject(); + b.startObject(); + b.field("name", "fieldL"); + b.endObject(); + b.endArray(); + b.startArray("metrics"); + b.startObject(); + b.field("name", "fieldI"); + b.startArray("stats"); + b.value("sum"); + b.endArray(); + b.endObject(); + b.endArray(); + b.endObject(); + b.endObject(); + b.endObject(); // End composite + + // Properties + b.startObject("properties"); + b.startObject("sndv"); + b.field("type", "integer"); + b.endObject(); + b.startObject("dv"); + b.field("type", "integer"); + b.endObject(); + b.startObject("field"); + b.field("type", "integer"); + b.endObject(); + b.startObject("fieldC"); + b.field("type", "integer"); + b.endObject(); + b.startObject("fieldB"); + b.field("type", "integer"); + b.endObject(); + b.startObject("fieldL"); + b.field("type", "integer"); + b.endObject(); + b.startObject("fieldI"); + b.field("type", "double"); + b.endObject(); + b.endObject(); + }); + } + private XContentBuilder topMapping(CheckedConsumer buildFields) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("_doc"); buildFields.accept(builder); return builder.endObject().endObject(); } - private void createMapperService(XContentBuilder builder) throws IOException { + private MapperService createMapperService(XContentBuilder builder, int noOfStarTrees) throws IOException { Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(STAR_TREE_MAX_FIELDS_SETTING.getKey(), noOfStarTrees) .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) .build(); IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).putMapping(builder.toString()).build(); IndicesModule indicesModule = new IndicesModule(Collections.emptyList()); - mapperService = MapperTestUtils.newMapperServiceWithHelperAnalyzer( + MapperService mapperService = MapperTestUtils.newMapperServiceWithHelperAnalyzer( new NamedXContentRegistry(ClusterModule.getNamedXWriteables()), createTempDir(), settings, @@ -285,5 +688,6 @@ private void createMapperService(XContentBuilder builder) throws IOException { "test" ); mapperService.merge(indexMetadata, MapperService.MergeReason.INDEX_TEMPLATE); + return mapperService; } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index 7cae1cd25ee93..5b85ec1795e37 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -8,6 +8,8 @@ package org.opensearch.index.compositeindex.datacube.startree; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.store.IndexInput; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; @@ -42,6 +44,8 @@ public class StarTreeTestUtils { + private static final Logger logger = LogManager.getLogger(StarTreeTestUtils.class); + public static StarTreeDocument[] getSegmentsStarTreeDocuments( List starTreeValuesSubs, List fieldValueConverters, @@ -109,10 +113,13 @@ public static StarTreeDocument getStarTreeDocument( public static Double toAggregatorValueType(Long value, FieldValueConverter fieldValueConverter) { try { - return fieldValueConverter.toDoubleValue(value); + if (value != null) { + return fieldValueConverter.toDoubleValue(value); + } } catch (Exception e) { - throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); + logger.warn("Cannot convert {} to sortable aggregation type", value, e); } + return null; } public static void assertStarTreeDocuments(StarTreeDocument[] starTreeDocuments, StarTreeDocument[] expectedStarTreeDocuments) {