Skip to content

Commit

Permalink
removing additional sorted numeric method
Browse files Browse the repository at this point in the history 
Signed-off-by: Sarthak Aggarwal <[email protected]>
  • Loading branch information
@sarthakaggarwal97
sarthakaggarwal97 committed Aug 20, 2024
1 parent 08b2ffe commit 8e956af
Show file tree
Hide file tree
Showing 6 changed files with 62 additions and 60 deletions.
9 changes: 0 additions & 9 deletions server/src/main/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducerWrapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@

import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SortedNumericDocValues;
import org.opensearch.index.codec.composite.DocValuesProvider;

import java.io.IOException;
Expand All @@ -25,7 +24,6 @@
public class Lucene90DocValuesProducerWrapper implements DocValuesProvider {

private final Lucene90DocValuesProducer lucene90DocValuesProducer;
private final SegmentReadState state;

public Lucene90DocValuesProducerWrapper(
SegmentReadState state,
Expand All @@ -35,13 +33,6 @@ public Lucene90DocValuesProducerWrapper(
String metaExtension
) throws IOException {
lucene90DocValuesProducer = new Lucene90DocValuesProducer(state, dataCodec, dataExtension, metaCodec, metaExtension);
this.state = state;
}

// returns the field doc id set iterator based on field name
@Override
public SortedNumericDocValues getSortedNumeric(String fieldName) throws IOException {
return this.lucene90DocValuesProducer.getSortedNumeric(state.fieldInfos.fieldInfo(fieldName));
}

@Override
Expand Down
19 changes: 8 additions & 11 deletions server/src/main/java/org/opensearch/index/codec/composite/DocValuesProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,6 @@
package org.opensearch.index.codec.composite;

import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.SortedNumericDocValues;

import java.io.IOException;

/**
* An interface that provides access to document values for a specific field.
Expand All @@ -20,14 +17,14 @@
*/
public interface DocValuesProvider {

/**
* Returns the sorted numeric document values for the specified field.
*
* @param fieldName The name of the field for which to retrieve the sorted numeric document values.
* @return The sorted numeric document values for the specified field.
* @throws IOException If an error occurs while retrieving the sorted numeric document values.
*/
SortedNumericDocValues getSortedNumeric(String fieldName) throws IOException;
// /**
// * Returns the sorted numeric document values for the specified field.
// *
// * @param fieldName The name of the field for which to retrieve the sorted numeric document values.
// * @return The sorted numeric document values for the specified field.
// * @throws IOException If an error occurs while retrieving the sorted numeric document values.
// */
// SortedNumericDocValues getSortedNumeric(String fieldName) throws IOException;

/**
* Returns the DocValuesProducer instance.
Expand Down
6 changes: 4 additions & 2 deletions ...er/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
package org.opensearch.index.codec.composite;

import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducerWrapper;
import org.apache.lucene.index.SegmentReadState;
import org.opensearch.index.codec.composite.composite99.Composite99Codec;
Expand All @@ -23,7 +24,7 @@
*/
public class LuceneDocValuesProducerFactory {

public static DocValuesProvider getDocValuesProducerForCompositeCodec(
public static DocValuesProducer getDocValuesProducerForCompositeCodec(
String compositeCodec,
SegmentReadState state,
String dataCodec,
Expand All @@ -34,7 +35,8 @@ public static DocValuesProvider getDocValuesProducerForCompositeCodec(

switch (compositeCodec) {
case Composite99Codec.COMPOSITE_INDEX_CODEC_NAME:
return new Lucene90DocValuesProducerWrapper(state, dataCodec, dataExtension, metaCodec, metaExtension);
return new Lucene90DocValuesProducerWrapper(state, dataCodec, dataExtension, metaCodec, metaExtension)
.getDocValuesProducer();
default:
throw new IllegalStateException("Invalid composite codec " + "[" + compositeCodec + "]");
}
Expand Down
20 changes: 7 additions & 13 deletions ...ain/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.index.codec.composite.CompositeIndexFieldInfo;
import org.opensearch.index.codec.composite.CompositeIndexReader;
import org.opensearch.index.codec.composite.DocValuesProvider;
import org.opensearch.index.codec.composite.LuceneDocValuesProducerFactory;
import org.opensearch.index.compositeindex.CompositeIndexMetadata;
import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.MetricEntry;
Expand Down Expand Up @@ -65,13 +64,12 @@ public class Composite99DocValuesReader extends DocValuesProducer implements Com
private final Map<String, IndexInput> compositeIndexInputMap = new LinkedHashMap<>();
private final Map<String, CompositeIndexMetadata> compositeIndexMetadataMap = new LinkedHashMap<>();
private final List<String> fields;
private DocValuesProvider compositeDocValuesProducer;
private DocValuesProducer compositeDocValuesProducer;
private final List<CompositeIndexFieldInfo> compositeFieldInfos = new ArrayList<>();
private final SegmentReadState readState;
private SegmentReadState readState;

public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState readState) throws IOException {
this.delegate = producer;
this.readState = readState;
this.fields = new ArrayList<>();

String metaFileName = IndexFileNames.segmentFileName(
Expand Down Expand Up @@ -178,18 +176,13 @@ public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState r
// populates the dummy list of field infos to fetch doc id set iterators for respective fields.
// the dummy field info is used to fetch the doc id set iterators for respective fields based on field name
FieldInfos fieldInfos = new FieldInfos(getFieldInfoList(fields));
SegmentReadState segmentReadState = new SegmentReadState(
readState.directory,
readState.segmentInfo,
fieldInfos,
readState.context
);
this.readState = new SegmentReadState(readState.directory, readState.segmentInfo, fieldInfos, readState.context);

// initialize star-tree doc values producer

compositeDocValuesProducer = LuceneDocValuesProducerFactory.getDocValuesProducerForCompositeCodec(
Composite99Codec.COMPOSITE_INDEX_CODEC_NAME,
segmentReadState,
this.readState,
Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC,
Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION,
Composite99DocValuesFormat.META_DOC_VALUES_CODEC,
Expand Down Expand Up @@ -246,7 +239,7 @@ public void close() throws IOException {
boolean success = false;
try {
IOUtils.close(metaIn, dataIn);
IOUtils.close(compositeDocValuesProducer.getDocValuesProducer());
IOUtils.close(compositeDocValuesProducer);
success = true;
} finally {
if (!success) {
Expand All @@ -273,7 +266,8 @@ public CompositeIndexValues getCompositeIndexValues(CompositeIndexFieldInfo comp
return new StarTreeValues(
compositeIndexMetadataMap.get(compositeIndexFieldInfo.getField()),
compositeIndexInputMap.get(compositeIndexFieldInfo.getField()),
compositeDocValuesProducer
compositeDocValuesProducer,
this.readState
);

default:
Expand Down
35 changes: 27 additions & 8 deletions ...main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,13 @@

package org.opensearch.index.compositeindex.datacube.startree.index;

import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.IndexInput;
import org.opensearch.common.annotation.ExperimentalApi;
import org.opensearch.index.codec.composite.DocValuesProvider;
import org.opensearch.index.compositeindex.CompositeIndexMetadata;
import org.opensearch.index.compositeindex.datacube.Dimension;
import org.opensearch.index.compositeindex.datacube.Metric;
Expand Down Expand Up @@ -66,7 +69,8 @@ public StarTreeValues(
public StarTreeValues(
CompositeIndexMetadata compositeIndexMetadata,
IndexInput compositeIndexIn,
DocValuesProvider compositeDocValuesProducer
DocValuesProducer compositeDocValuesProducer,
SegmentReadState readState
) throws IOException {

StarTreeMetadata starTreeMetadata = (StarTreeMetadata) compositeIndexMetadata;
Expand Down Expand Up @@ -110,13 +114,19 @@ public StarTreeValues(

// get doc id set iterators for dimensions
for (String dimension : starTreeMetadata.getDimensionFields()) {
SortedNumericDocValues dimensionSortedNumericDocValues = null;
if (readState != null) {
FieldInfo dimensionfieldInfo = readState.fieldInfos.fieldInfo(
fullyQualifiedFieldNameForStarTreeDimensionsDocValues(starTreeField.getName(), dimension)
);
if (dimensionfieldInfo != null) {
dimensionSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(dimensionfieldInfo);
}
}

dimensionDocValuesIteratorMap.put(
dimension,
getSortedNumericDocValues(
compositeDocValuesProducer.getSortedNumeric(
fullyQualifiedFieldNameForStarTreeDimensionsDocValues(starTreeField.getName(), dimension)
)
)
getSortedNumericDocValues(dimensionSortedNumericDocValues)
);
}

Expand All @@ -127,10 +137,19 @@ public StarTreeValues(
metricEntry.getMetricFieldName(),
metricEntry.getMetricStat().getTypeName()
);

SortedNumericDocValues metricSortedNumericDocValues = null;
if (readState != null) {
FieldInfo metricFieldInfo = readState.fieldInfos.fieldInfo(metricFullName);
if (metricFieldInfo != null) {
metricSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(metricFieldInfo);
}
}
metricDocValuesIteratorMap.put(
metricFullName,
getSortedNumericDocValues(compositeDocValuesProducer.getSortedNumeric(metricFullName))
getSortedNumericDocValues(metricSortedNumericDocValues)
);

}

// create star-tree attributes map
Expand Down
33 changes: 16 additions & 17 deletions ...ensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1760,7 +1760,7 @@ private void validateStarTreeFileFormats(
expectedStarTreeMetadata.getMetricEntries()
);

Lucene90DocValuesProducerWrapper compositeDocValuesProducer = (Lucene90DocValuesProducerWrapper) LuceneDocValuesProducerFactory
DocValuesProducer compositeDocValuesProducer = LuceneDocValuesProducerFactory
.getDocValuesProducerForCompositeCodec(
Composite99Codec.COMPOSITE_INDEX_CODEC_NAME,
readState,
Expand All @@ -1773,7 +1773,7 @@ private void validateStarTreeFileFormats(
IndexInput dataIn = readState.directory.openInput(dataFileName, IOContext.DEFAULT);
IndexInput metaIn = readState.directory.openInput(metaFileName, IOContext.DEFAULT);

StarTreeValues starTreeValues = new StarTreeValues(expectedStarTreeMetadata, dataIn, compositeDocValuesProducer);
StarTreeValues starTreeValues = new StarTreeValues(expectedStarTreeMetadata, dataIn, compositeDocValuesProducer, readState);
List<StarTreeNumericType> starTreeNumericTypes = new ArrayList<>();
builder.metricAggregatorInfos.forEach(
metricAggregatorInfo -> starTreeNumericTypes.add(metricAggregatorInfo.getValueAggregators().getAggregatedValueType())
Expand All @@ -1791,7 +1791,7 @@ private void validateStarTreeFileFormats(

dataIn.close();
metaIn.close();
compositeDocValuesProducer.getDocValuesProducer().close();
compositeDocValuesProducer.close();
}

private static Map<Integer, Map<Long, Double>> getExpectedDimToValueMap() {
Expand Down Expand Up @@ -2032,22 +2032,22 @@ public void testFlushFlowDimsReverse() throws IOException {
}

public void testFlushFlowBuild() throws IOException {
List<Long> dimList = new ArrayList<>(101);
List<Integer> docsWithField = new ArrayList<>(101);
List<Long> dimList = new ArrayList<>(100);
List<Integer> docsWithField = new ArrayList<>(100);
for (int i = 0; i < 100; i++) {
dimList.add((long) i);
docsWithField.add(i);
}

List<Long> dimList2 = new ArrayList<>(101);
List<Integer> docsWithField2 = new ArrayList<>(101);
List<Long> dimList2 = new ArrayList<>(100);
List<Integer> docsWithField2 = new ArrayList<>(100);
for (int i = 0; i < 100; i++) {
dimList2.add((long) i);
docsWithField2.add(i);
}

List<Long> metricsList = new ArrayList<>(101);
List<Integer> metricsWithField = new ArrayList<>(101);
List<Long> metricsList = new ArrayList<>(100);
List<Integer> metricsWithField = new ArrayList<>(100);
for (int i = 0; i < 100; i++) {
metricsList.add(getLongFromDouble(i * 10.0));
metricsWithField.add(i);
Expand All @@ -2064,7 +2064,7 @@ public void testFlushFlowBuild() throws IOException {
SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2);
SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField);

writeState = getWriteState(101);
writeState = getWriteState(100);
this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec(
Composite99Codec.COMPOSITE_INDEX_CODEC_NAME,
writeState,
Expand All @@ -2079,7 +2079,6 @@ public void testFlushFlowBuild() throws IOException {
DocValuesProducer d2vp = getDocValuesProducer(d2sndv);
DocValuesProducer m1vp = getDocValuesProducer(m1sndv);
Map<String, DocValuesProducer> fieldProducerMap = Map.of("field1", d1vp, "field3", d2vp, "field2", m1vp);
// builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer);
builder.build(fieldProducerMap, new AtomicInteger(), docValuesConsumer);
/**
* Asserting following dim / metrics [ dim1, dim2 / Sum [ metric] ]
Expand All @@ -2096,10 +2095,10 @@ public void testFlushFlowBuild() throws IOException {
*/
List<StarTreeDocument> starTreeDocuments = builder.getStarTreeDocuments();
for (StarTreeDocument starTreeDocument : starTreeDocuments) {
// assertEquals(
// starTreeDocument.dimensions[1] != null ? starTreeDocument.dimensions[1] * 10.0 : 49500.0,
// starTreeDocument.metrics[0]
// );
assertEquals(
starTreeDocument.dimensions[1] != null ? starTreeDocument.dimensions[1] * 10.0 : 49500.0,
starTreeDocument.metrics[0]
);
}
validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments());

Expand All @@ -2113,12 +2112,12 @@ public void testFlushFlowBuild() throws IOException {
mock(IndexInput.class),
List.of("field1", "field3"),
List.of(new MetricEntry("field2", MetricStat.SUM)),
101,
100,
1,
Set.of(),
getBuildMode(),
0,
6715
6781
);

validateStarTreeFileFormats(
Expand Down

0 comments on commit 8e956af

Please sign in to comment.