Merge branch 'main' into 13531-range-agg

CHANGELOG.md

@@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Add support to disable flush based on translog reader count ([#14027](https://github.com/opensearch-project/OpenSearch/pull/14027))
 - [Query Insights] Add exporter support for top n queries ([#12982](https://github.com/opensearch-project/OpenSearch/pull/12982))
 - [Query Insights] Add X-Opaque-Id to search request metadata for top n queries ([#13374](https://github.com/opensearch-project/OpenSearch/pull/13374))
+- Add support for query level resource usage tracking ([#13172](https://github.com/opensearch-project/OpenSearch/pull/13172))
 - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865))
 
 ### Dependencies
@@ -41,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.apache.xmlbeans:xmlbeans` from 5.2.0 to 5.2.1 ([#13839](https://github.com/opensearch-project/OpenSearch/pull/13839))
 - Bump `actions/checkout` from 3 to 4 ([#13935](https://github.com/opensearch-project/OpenSearch/pull/13935))
 - Bump `com.netflix.nebula.ospackage-base` from 11.9.0 to 11.9.1 ([#13933](https://github.com/opensearch-project/OpenSearch/pull/13933))
+- Update to Apache Lucene 9.11.0 ([#14042](https://github.com/opensearch-project/OpenSearch/pull/14042))
 
 ### Changed
 - Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.11.0-snapshot-4be6531
+lucene            = 9.11.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.3+9

libs/core/licenses/lucene-core-9.11.0.jar.sha1

@@ -0,0 +1 @@
+2e487755a6814b2a1bc770c26569dcba86873dcf

libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceUsageInfo.java

@@ -104,6 +104,10 @@ public long getTotalValue() {
             return endValue.get() - startValue;
         }
 
+        public long getStartValue() {
+            return startValue;
+        }
+
         @Override
         public String toString() {
             return String.valueOf(getTotalValue());

libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/TaskResourceInfo.java

@@ -0,0 +1,225 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.core.tasks.resourcetracker;
+
+import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.core.ParseField;
+import org.opensearch.core.common.Strings;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ConstructingObjectParser;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.core.xcontent.ToXContentObject;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.opensearch.core.xcontent.ConstructingObjectParser.constructorArg;
+
+/**
+ * Task resource usage information with minimal information about the task
+ * <p>
+ * Writeable TaskResourceInfo objects are used to represent resource usage
+ * information of running tasks, which can be propagated to coordinator node
+ * to infer query-level resource usage
+ *
+ *  @opensearch.api
+ */
+@PublicApi(since = "2.15.0")
+public class TaskResourceInfo implements Writeable, ToXContentObject {
+    private final String action;
+    private final long taskId;
+    private final long parentTaskId;
+    private final String nodeId;
+    private final TaskResourceUsage taskResourceUsage;
+
+    private static final ParseField ACTION = new ParseField("action");
+    private static final ParseField TASK_ID = new ParseField("taskId");
+    private static final ParseField PARENT_TASK_ID = new ParseField("parentTaskId");
+    private static final ParseField NODE_ID = new ParseField("nodeId");
+    private static final ParseField TASK_RESOURCE_USAGE = new ParseField("taskResourceUsage");
+
+    public TaskResourceInfo(
+        final String action,
+        final long taskId,
+        final long parentTaskId,
+        final String nodeId,
+        final TaskResourceUsage taskResourceUsage
+    ) {
+        this.action = action;
+        this.taskId = taskId;
+        this.parentTaskId = parentTaskId;
+        this.nodeId = nodeId;
+        this.taskResourceUsage = taskResourceUsage;
+    }
+
+    public static final ConstructingObjectParser<TaskResourceInfo, Void> PARSER = new ConstructingObjectParser<>(
+        "task_resource_info",
+        a -> new Builder().setAction((String) a[0])
+            .setTaskId((Long) a[1])
+            .setParentTaskId((Long) a[2])
+            .setNodeId((String) a[3])
+            .setTaskResourceUsage((TaskResourceUsage) a[4])
+            .build()
+    );
+
+    static {
+        PARSER.declareString(constructorArg(), ACTION);
+        PARSER.declareLong(constructorArg(), TASK_ID);
+        PARSER.declareLong(constructorArg(), PARENT_TASK_ID);
+        PARSER.declareString(constructorArg(), NODE_ID);
+        PARSER.declareObject(constructorArg(), TaskResourceUsage.PARSER, TASK_RESOURCE_USAGE);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ACTION.getPreferredName(), this.action);
+        builder.field(TASK_ID.getPreferredName(), this.taskId);
+        builder.field(PARENT_TASK_ID.getPreferredName(), this.parentTaskId);
+        builder.field(NODE_ID.getPreferredName(), this.nodeId);
+        builder.startObject(TASK_RESOURCE_USAGE.getPreferredName());
+        this.taskResourceUsage.toXContent(builder, params);
+        builder.endObject();
+        builder.endObject();
+        return builder;
+    }
+
+    /**
+     * Builder for {@link TaskResourceInfo}
+     */
+    public static class Builder {
+        private TaskResourceUsage taskResourceUsage;
+        private String action;
+        private long taskId;
+        private long parentTaskId;
+        private String nodeId;
+
+        public Builder setTaskResourceUsage(final TaskResourceUsage taskResourceUsage) {
+            this.taskResourceUsage = taskResourceUsage;
+            return this;
+        }
+
+        public Builder setAction(final String action) {
+            this.action = action;
+            return this;
+        }
+
+        public Builder setTaskId(final long taskId) {
+            this.taskId = taskId;
+            return this;
+        }
+
+        public Builder setParentTaskId(final long parentTaskId) {
+            this.parentTaskId = parentTaskId;
+            return this;
+        }
+
+        public Builder setNodeId(final String nodeId) {
+            this.nodeId = nodeId;
+            return this;
+        }
+
+        public TaskResourceInfo build() {
+            return new TaskResourceInfo(action, taskId, parentTaskId, nodeId, taskResourceUsage);
+        }
+    }
+
+    /**
+     * Read task info from a stream.
+     *
+     * @param in StreamInput to read
+     * @return {@link TaskResourceInfo}
+     * @throws IOException IOException
+     */
+    public static TaskResourceInfo readFromStream(StreamInput in) throws IOException {
+        return new TaskResourceInfo.Builder().setAction(in.readString())
+            .setTaskId(in.readLong())
+            .setParentTaskId(in.readLong())
+            .setNodeId(in.readString())
+            .setTaskResourceUsage(TaskResourceUsage.readFromStream(in))
+            .build();
+    }
+
+    /**
+     * Get TaskResourceUsage
+     *
+     * @return taskResourceUsage
+     */
+    public TaskResourceUsage getTaskResourceUsage() {
+        return taskResourceUsage;
+    }
+
+    /**
+     * Get parent task id
+     *
+     * @return parent task id
+     */
+    public long getParentTaskId() {
+        return parentTaskId;
+    }
+
+    /**
+     * Get task id
+     * @return task id
+     */
+    public long getTaskId() {
+        return taskId;
+    }
+
+    /**
+     * Get node id
+     * @return node id
+     */
+    public String getNodeId() {
+        return nodeId;
+    }
+
+    /**
+     * Get task action
+     * @return task action
+     */
+    public String getAction() {
+        return action;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(action);
+        out.writeLong(taskId);
+        out.writeLong(parentTaskId);
+        out.writeString(nodeId);
+        taskResourceUsage.writeTo(out);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(MediaTypeRegistry.JSON, this);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || obj.getClass() != TaskResourceInfo.class) {
+            return false;
+        }
+        TaskResourceInfo other = (TaskResourceInfo) obj;
+        return action.equals(other.action)
+            && taskId == other.taskId
+            && parentTaskId == other.parentTaskId
+            && Objects.equals(nodeId, other.nodeId)
+            && taskResourceUsage.equals(other.taskResourceUsage);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(action, taskId, parentTaskId, nodeId, taskResourceUsage);
+    }
+}

modules/lang-expression/licenses/lucene-expressions-9.11.0.jar.sha1

@@ -0,0 +1 @@
+5e21d20edee0712472e7c6f605c9d97aeecf16c0

plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0.jar.sha1

@@ -0,0 +1 @@
+5c7f2d8eab0fca3fdc3d3e57a7f48a335dc7ac33

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0.jar.sha1

@@ -0,0 +1 @@
+efcf65dda1b4e9d7e83926fd5895a47e491cbf29

plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0.jar.sha1

@@ -0,0 +1 @@
+59599d7b8bed2e6bd27d0dad7935c078b98c39cc

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0.jar.sha1

@@ -0,0 +1 @@
+e55f83bb373ac139e313f64e80afe1eb0a75b8c0

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0.jar.sha1

@@ -0,0 +1 @@
+1be59d91c45a4de069611fb7f8aa3e8fd26020ec

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0.jar.sha1

@@ -0,0 +1 @@
+d5b5922acf3743b5a0c542959dd93fca8be333a7

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0.jar.sha1

@@ -0,0 +1 @@
+50fd7b471cbdd6648c4972169f3fc67fae9db7f6

server/licenses/lucene-analysis-common-9.11.0.jar.sha1

@@ -0,0 +1 @@
+75a0a333cf1e043102743066c929e65fe51cbcda

server/licenses/lucene-backward-codecs-9.11.0.jar.sha1

@@ -0,0 +1 @@
+db385446bc3fd70e7c6a744276c0a157bd60ee0a

server/licenses/lucene-core-9.11.0.jar.sha1

@@ -0,0 +1 @@
+2e487755a6814b2a1bc770c26569dcba86873dcf

server/licenses/lucene-grouping-9.11.0.jar.sha1

@@ -0,0 +1 @@
+882bdaf209b0acb332aa34836616424bcbecf462

server/licenses/lucene-highlighter-9.11.0.jar.sha1

@@ -0,0 +1 @@
+44accdc03c5482e602718f7bf91e5940ba4e4870

server/licenses/lucene-join-9.11.0.jar.sha1

@@ -0,0 +1 @@
+32a30ee03ed4f3e43bf63250270b2d4d53050045

server/licenses/lucene-memory-9.11.0.jar.sha1

@@ -0,0 +1 @@
+b3e80aa6aa3299118e76a23edc23b58f3ba5a515

server/licenses/lucene-misc-9.11.0.jar.sha1

@@ -0,0 +1 @@
+54fe308908194e1b0697a1157a45c5998c9e1083

server/licenses/lucene-queries-9.11.0.jar.sha1

@@ -0,0 +1 @@
+987d1286949ddf514b8405fd453ed47bebdfb12d

server/licenses/lucene-queryparser-9.11.0.jar.sha1

@@ -0,0 +1 @@
+e97fe1c0d102edb8d6e1c01454992fd2b8d80ae0

server/licenses/lucene-sandbox-9.11.0.jar.sha1

@@ -0,0 +1 @@
+5e46b790744bd9118ccc053f70235364213312a5

server/licenses/lucene-spatial-extras-9.11.0.jar.sha1

@@ -0,0 +1 @@
+079ca5aaf544a3acde84b8b88423ace6dedc23eb

server/licenses/lucene-spatial3d-9.11.0.jar.sha1

@@ -0,0 +1 @@
+564558818d70fc384db5b36fbc8a0ab27b107609

server/licenses/lucene-suggest-9.11.0.jar.sha1

@@ -0,0 +1 @@
+aa345db9b6caaf881e7890ea5b8911357d592167