forked from opensearch-project/OpenSearch
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
dummy implementation of join on coordinator
Signed-off-by: bowenlan-amzn <[email protected]>
- Loading branch information
1 parent
130d554
commit 49b1284
Showing
8 changed files
with
318 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
112 changes: 112 additions & 0 deletions
112
server/src/main/java/org/opensearch/arrow/query/ArrowCollector.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.arrow.query; | ||
|
|
||
| import org.apache.arrow.vector.VarCharVector; | ||
| import org.apache.arrow.vector.VectorSchemaRoot; | ||
| import org.apache.lucene.index.LeafReaderContext; | ||
| import org.apache.lucene.index.SortedSetDocValues; | ||
| import org.apache.lucene.search.Collector; | ||
| import org.apache.lucene.search.FilterCollector; | ||
| import org.apache.lucene.search.LeafCollector; | ||
| import org.apache.lucene.search.Scorable; | ||
| import org.apache.lucene.search.ScoreMode; | ||
| import org.apache.lucene.search.Weight; | ||
| import org.apache.lucene.util.BytesRef; | ||
| import org.opensearch.arrow.ArrowStreamProvider; | ||
| import org.opensearch.common.annotation.ExperimentalApi; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map;@ExperimentalApi | ||
|
|
||
| public class ArrowCollector extends FilterCollector { | ||
|
|
||
| List<ProjectionField> projectionFields; | ||
| private final VectorSchemaRoot root; | ||
| private final ArrowStreamProvider.FlushSignal flushSignal; | ||
| private final int batchSize; | ||
|
|
||
| public ArrowCollector(Collector in, List<ProjectionField> projectionFields, VectorSchemaRoot root, int batchSize, ArrowStreamProvider.FlushSignal flushSignal) { | ||
| super(in); | ||
| this.projectionFields = projectionFields; | ||
| this.root = root; | ||
| this.batchSize = batchSize; | ||
| this.flushSignal = flushSignal; | ||
| } | ||
|
|
||
| @Override | ||
| public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { | ||
|
|
||
| Map<String, SortedSetDocValues> fieldValues = new HashMap<>(); | ||
| projectionFields.forEach(field -> { | ||
| try { | ||
| SortedSetDocValues dv = context.reader().getSortedSetDocValues(field.fieldName); | ||
| fieldValues.put(field.fieldName, dv); | ||
| } catch (IOException e) { | ||
| throw new RuntimeException(e); | ||
| } | ||
| }); | ||
|
|
||
| final int[] currentRow = {0}; | ||
| return new LeafCollector() { | ||
|
|
||
| @Override | ||
| public void collect(int docId) throws IOException { | ||
| // innerLeafCollector.collect(docId); | ||
|
|
||
| // read from the lucene field values | ||
| for (Map.Entry<String, SortedSetDocValues> entry : fieldValues.entrySet()) { | ||
| String field = entry.getKey(); | ||
| SortedSetDocValues dv = entry.getValue(); | ||
| VarCharVector vector = (VarCharVector) root.getVector(field); | ||
|
|
||
| if (dv.advanceExact(docId)) { | ||
| BytesRef keyword = dv.termsEnum().next(); | ||
| vector.setSafe(currentRow[0], keyword.utf8ToString().getBytes()); | ||
| } | ||
| } | ||
|
|
||
| currentRow[0]++; | ||
| if (currentRow[0] >= batchSize) { | ||
| root.setRowCount(batchSize); | ||
| flushSignal.awaitConsumption(); | ||
| currentRow[0] = 0; | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void finish() throws IOException { | ||
| if (currentRow[0] > 0) { | ||
| root.setRowCount(currentRow[0]); | ||
| flushSignal.awaitConsumption(); | ||
| currentRow[0] = 0; | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void setScorer(Scorable scorable) throws IOException { | ||
| // innerLeafCollector.setScorer(scorable); | ||
| } | ||
| }; | ||
| } | ||
|
|
||
| @Override | ||
| public ScoreMode scoreMode() { | ||
| return ScoreMode.COMPLETE_NO_SCORES; | ||
| } | ||
|
|
||
| @Override | ||
| public void setWeight(Weight weight) { | ||
| if (this.in != null) { | ||
| this.in.setWeight(weight); | ||
| } | ||
| } | ||
| } |
23 changes: 23 additions & 0 deletions
23
server/src/main/java/org/opensearch/arrow/query/ProjectionField.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.arrow.query; | ||
|
|
||
| import org.opensearch.common.annotation.ExperimentalApi; | ||
| import org.opensearch.index.fielddata.IndexNumericFieldData; | ||
|
|
||
| @ExperimentalApi | ||
| public class ProjectionField { | ||
| public String fieldName; | ||
| String type; | ||
|
|
||
| public ProjectionField(String fieldName, String type) { | ||
| this.fieldName = fieldName; | ||
| this.type = type; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.