Gbh pre 211

server/build.gradle

@@ -96,6 +96,12 @@ if (!isEclipse) {
     manifest.attributes('Multi-Release': 'true')
   }
 }
+repositories {
+  flatDir {
+    dirs 'lib'
+  }
+}
+
 
 dependencies {
 
@@ -107,7 +113,6 @@ dependencies {
   api project(":libs:opensearch-geo")
   api project(":libs:opensearch-telemetry")
 
-
   compileOnly project(':libs:opensearch-plugin-classloader')
   testRuntimeOnly project(':libs:opensearch-plugin-classloader')
 

server/src/main/java/org/opensearch/action/search/SearchStarTreeAction.java

@@ -0,0 +1,2 @@
+package org.opensearch.action.search;public class SearchStarTreeAction {
+}

server/src/main/java/org/opensearch/action/search/SearchStarTreeResponse.java

@@ -0,0 +1,2 @@
+package org.opensearch.action.search;public class SearchStarTreeResponse {
+}

server/src/main/java/org/opensearch/action/search/TransportSearchStarTreeAction.java

@@ -0,0 +1,133 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.action.search;
+
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.action.StepListener;
+import org.opensearch.action.support.ActionFilters;
+import org.opensearch.action.support.HandledTransportAction;
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.inject.Inject;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.search.SearchPhaseResult;
+import org.opensearch.search.internal.ShardSearchContextId;
+import org.opensearch.tasks.Task;
+import org.opensearch.transport.TransportRequest;
+import org.opensearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Transport action for creating PIT reader context
+ */
+public class TransportCreatePitAction extends HandledTransportAction<CreatePitRequest, CreatePitResponse> {
+
+    public static final String CREATE_PIT_ACTION = "create_pit";
+    private final TransportService transportService;
+    private final SearchTransportService searchTransportService;
+    private final ClusterService clusterService;
+    private final TransportSearchAction transportSearchAction;
+    private final NamedWriteableRegistry namedWriteableRegistry;
+    private final CreatePitController createPitController;
+
+    @Inject
+    public TransportCreatePitAction(
+        TransportService transportService,
+        ActionFilters actionFilters,
+        SearchTransportService searchTransportService,
+        ClusterService clusterService,
+        TransportSearchAction transportSearchAction,
+        NamedWriteableRegistry namedWriteableRegistry,
+        CreatePitController createPitController
+    ) {
+        super(CreatePitAction.NAME, transportService, actionFilters, in -> new CreatePitRequest(in));
+        this.transportService = transportService;
+        this.searchTransportService = searchTransportService;
+        this.clusterService = clusterService;
+        this.transportSearchAction = transportSearchAction;
+        this.namedWriteableRegistry = namedWriteableRegistry;
+        this.createPitController = createPitController;
+    }
+
+    @Override
+    protected void doExecute(Task task, CreatePitRequest request, ActionListener<CreatePitResponse> listener) {
+        final StepListener<SearchResponse> createPitListener = new StepListener<>();
+        final ActionListener<CreatePitResponse> updatePitIdListener = ActionListener.wrap(r -> listener.onResponse(r), e -> {
+            logger.error(
+                () -> new ParameterizedMessage(
+                    "PIT creation failed while updating PIT ID for indices [{}]",
+                    Arrays.toString(request.indices())
+                )
+            );
+            listener.onFailure(e);
+        });
+        createPitController.executeCreatePit(request, task, createPitListener, updatePitIdListener);
+    }
+
+    /**
+     * Request to create pit reader context with keep alive
+     */
+    public static class CreateReaderContextRequest extends TransportRequest {
+        private final ShardId shardId;
+        private final TimeValue keepAlive;
+
+        public CreateReaderContextRequest(ShardId shardId, TimeValue keepAlive) {
+            this.shardId = shardId;
+            this.keepAlive = keepAlive;
+        }
+
+        public ShardId getShardId() {
+            return shardId;
+        }
+
+        public TimeValue getKeepAlive() {
+            return keepAlive;
+        }
+
+        public CreateReaderContextRequest(StreamInput in) throws IOException {
+            super(in);
+            this.shardId = new ShardId(in);
+            this.keepAlive = in.readTimeValue();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            shardId.writeTo(out);
+            out.writeTimeValue(keepAlive);
+        }
+    }
+
+    /**
+     * Create pit reader context response which holds the contextId
+     */
+    public static class CreateReaderContextResponse extends SearchPhaseResult {
+        public CreateReaderContextResponse(ShardSearchContextId shardSearchContextId) {
+            this.contextId = shardSearchContextId;
+        }
+
+        public CreateReaderContextResponse(StreamInput in) throws IOException {
+            super(in);
+            contextId = new ShardSearchContextId(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            contextId.writeTo(out);
+        }
+    }
+
+}

server/src/main/java/org/opensearch/common/lucene/Lucene.java

@@ -1077,6 +1077,12 @@ public SortedDocValues getSortedDocValues(String field) {
                 return null;
             }
 
+            @Override
+            public Object getAggregatedDocValues()
+                throws IOException {
+                return null;
+            }
+
             public SortedNumericDocValues getSortedNumericDocValues(String field) {
                 return null;
             }

server/src/main/java/org/opensearch/index/codec/CodecService.java

@@ -39,6 +39,7 @@
 import org.opensearch.common.Nullable;
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.index.IndexSettings;
+import org.opensearch.index.codec.freshstartree.codec.StarTreeCodec;
 import org.opensearch.index.mapper.MapperService;
 
 import java.util.Map;
@@ -68,8 +69,8 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
         assert null != indexSettings;
         if (mapperService == null) {
-            codecs.put(DEFAULT_CODEC, new Lucene95Codec());
-            codecs.put(LZ4, new Lucene95Codec());
+            codecs.put(DEFAULT_CODEC, new StarTreeCodec());
+            codecs.put(LZ4, new StarTreeCodec());
             codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION));
             codecs.put(ZLIB, new Lucene95Codec(Mode.BEST_COMPRESSION));
         } else {
@@ -78,7 +79,7 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe
             codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
             codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
         }
-        codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
+        codecs.put(LUCENE_DEFAULT_CODEC, new StarTreeCodec());
         for (String codec : Codec.availableCodecs()) {
             codecs.put(codec, Codec.forName(codec));
         }
@@ -96,7 +97,7 @@ public CodecService(@Nullable MapperService mapperService, Logger logger) {
             codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
             codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
         }
-        codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
+        codecs.put(LUCENE_DEFAULT_CODEC, new StarTreeCodec());
         for (String codec : Codec.availableCodecs()) {
             codecs.put(codec, Codec.forName(codec));
         }

server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java

@@ -39,6 +39,7 @@
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
 import org.apache.lucene.codecs.lucene95.Lucene95Codec;
 import org.opensearch.common.lucene.Lucene;
+import org.opensearch.index.codec.freshstartree.codec.StarTreeDocValuesFormat;
 import org.opensearch.index.mapper.CompletionFieldMapper;
 import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.index.mapper.MapperService;
@@ -56,7 +57,7 @@
 public class PerFieldMappingPostingFormatCodec extends Lucene95Codec {
     private final Logger logger;
     private final MapperService mapperService;
-    private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat();
+    private final DocValuesFormat dvFormat = new StarTreeDocValuesFormat();
 
     static {
         assert Codec.forName(Lucene.LATEST_CODEC).getClass().isAssignableFrom(PerFieldMappingPostingFormatCodec.class)
@@ -84,4 +85,10 @@ public PostingsFormat getPostingsFormatForField(String field) {
     public DocValuesFormat getDocValuesFormatForField(String field) {
         return dvFormat;
     }
+
+
+    @Override
+    public final DocValuesFormat docValuesFormat() {
+        return dvFormat;
+    }
 }

server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/AggregationFunctionColumnPair.java

@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.opensearch.index.codec.freshstartree.aggregator;
+
+import java.util.Comparator;
+
+
+/** Aggregation function, doc values column pair */
+public class AggregationFunctionColumnPair implements Comparable<AggregationFunctionColumnPair> {
+    public static final String DELIMITER = "__";
+    public static final String STAR = "*";
+    public static final AggregationFunctionColumnPair COUNT_STAR =
+        new AggregationFunctionColumnPair(AggregationFunctionType.COUNT, STAR);
+
+    private final AggregationFunctionType _functionType;
+    private final String _column;
+
+    public AggregationFunctionColumnPair(AggregationFunctionType functionType, String column) {
+        _functionType = functionType;
+        if (functionType == AggregationFunctionType.COUNT) {
+            _column = STAR;
+        } else {
+            _column = column;
+        }
+    }
+
+    public AggregationFunctionType getFunctionType() {
+        return _functionType;
+    }
+
+    public String getColumn() {
+        return _column;
+    }
+
+    public String toColumnName() {
+        return toColumnName(_functionType, _column);
+    }
+
+    public static String toColumnName(AggregationFunctionType functionType, String column) {
+        return functionType.getName() + DELIMITER + column;
+    }
+
+    public static AggregationFunctionColumnPair fromColumnName(String columnName) {
+        String[] parts = columnName.split(DELIMITER, 2);
+        return fromFunctionAndColumnName(parts[0], parts[1]);
+    }
+
+    private static AggregationFunctionColumnPair fromFunctionAndColumnName(String functionName, String columnName) {
+        AggregationFunctionType functionType = AggregationFunctionType.getAggregationFunctionType(functionName);
+        if (functionType == AggregationFunctionType.COUNT) {
+            return COUNT_STAR;
+        } else {
+            return new AggregationFunctionColumnPair(functionType, columnName);
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * _functionType.hashCode() + _column.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj instanceof AggregationFunctionColumnPair) {
+            AggregationFunctionColumnPair anotherPair = (AggregationFunctionColumnPair) obj;
+            return _functionType == anotherPair._functionType && _column.equals(anotherPair._column);
+        }
+        return false;
+    }
+
+    @Override
+    public String toString() {
+        return toColumnName();
+    }
+
+    @Override
+    public int compareTo(AggregationFunctionColumnPair other) {
+        return Comparator.comparing((AggregationFunctionColumnPair o) -> o._column)
+            .thenComparing((AggregationFunctionColumnPair o) -> o._functionType).compare(this, other);
+    }
+}

server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/AggregationFunctionType.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.opensearch.index.codec.freshstartree.aggregator;
+
+/** Aggregated function type */
+public enum AggregationFunctionType {
+    COUNT("count"), SUM("sum");
+    // AVG("avg");
+
+    private String name;
+
+    AggregationFunctionType(String name) {
+        this.name = name;
+    }
+
+    public static AggregationFunctionType getAggregationFunctionType(String functionName) {
+        return AggregationFunctionType.valueOf(functionName);
+    }
+
+    public String getName() {
+        return name;
+    }
+}