Skip to content

Commit

Permalink
Merge branch 'main' into batched-psa
Browse files Browse the repository at this point in the history
Signed-off-by: Shivansh Arora <[email protected]>
  • Loading branch information
shiv0408 committed Oct 3, 2023
2 parents 3635240 + d5a95b8 commit 4a0a570
Show file tree
Hide file tree
Showing 415 changed files with 8,845 additions and 1,088 deletions.
1 change: 1 addition & 0 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ Resolves #[Issue number to be closed when this PR is merged]
- [ ] New functionality has javadoc added
- [ ] Commits are signed per the DCO using --signoff
- [ ] Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../blob/main/CONTRIBUTING.md#changelog))
- [ ] GitHub issue/PR created in [OpenSearch documentation repo](https://github.com/opensearch-project/documentation-website) for the required public documentation changes (#[Issue/PR number])

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).
2 changes: 1 addition & 1 deletion .github/workflows/auto-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
steps:
- name: GitHub App token
id: github_app_token
uses: tibdex/github-app-token@v1.5.0
uses: tibdex/github-app-token@v2.1.0
with:
app_id: ${{ secrets.APP_ID }}
private_key: ${{ secrets.APP_PRIVATE_KEY }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/backport.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ jobs:
steps:
- name: GitHub App token
id: github_app_token
uses: tibdex/github-app-token@v1.5.0
uses: tibdex/github-app-token@v2.1.0
with:
app_id: ${{ secrets.APP_ID }}
private_key: ${{ secrets.APP_PRIVATE_KEY }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/create-documentation-issue.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
steps:
- name: GitHub App token
id: github_app_token
uses: tibdex/github-app-token@v1.5.0
uses: tibdex/github-app-token@v2.1.0
with:
app_id: ${{ secrets.APP_ID }}
private_key: ${{ secrets.APP_PRIVATE_KEY }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/dependabot_pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
steps:
- name: GitHub App token
id: github_app_token
uses: tibdex/github-app-token@v1.5.0
uses: tibdex/github-app-token@v2.1.0
with:
app_id: ${{ secrets.APP_ID }}
private_key: ${{ secrets.APP_PRIVATE_KEY }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/gradle-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ jobs:
- name: Upload Coverage Report
if: success()
uses: codecov/codecov-action@v2
uses: codecov/codecov-action@v3
with:
files: ./codeCoverage.xml

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/lucene-snapshots.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
- name: Set hash
working-directory: ./lucene
run: |
echo "::set-output name=REVISION::$(git rev-parse --short HEAD)"
echo "REVISION=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
id: version

- name: Initialize gradle settings
Expand Down
4 changes: 0 additions & 4 deletions .github/workflows/precommit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,6 @@ jobs:
- name: Setup docker (missing on MacOS)
if: runner.os == 'macos'
run: |
# Workaround for https://github.com/actions/runner-images/issues/8104
brew remove --ignore-dependencies qemu
curl -o ./qemu.rb https://raw.githubusercontent.com/Homebrew/homebrew-core/f88e30b3a23ef3735580f9b05535ce5a0a03c9e3/Formula/qemu.rb
brew install ./qemu.rb
brew install docker
colima start
sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
Expand Down
28 changes: 28 additions & 0 deletions .github/workflows/stalled.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
name: Label Stalled PRs
on:
schedule:
- cron: '15 15 * * *' # Run every day at 15:15 UTC / 7:15 PST / 8:15 PDT
permissions:
pull-requests: write
jobs:
stale:
if: github.repository == 'opensearch-project/OpenSearch'
runs-on: ubuntu-latest
steps:
- name: GitHub App token
id: github_app_token
uses: tibdex/[email protected]
with:
app_id: ${{ secrets.APP_ID }}
private_key: ${{ secrets.APP_PRIVATE_KEY }}
installation_id: 22958780
- name: Stale PRs
uses: actions/stale@v8
with:
repo-token: ${{ steps.github_app_token.outputs.token }}
stale-pr-label: 'stalled'
stale-pr-message: 'This PR is stalled because it has been open for 30 days with no activity.'
days-before-pr-stale: 30
days-before-issue-stale: -1
days-before-pr-close: -1
days-before-issue-close: -1
8 changes: 4 additions & 4 deletions .github/workflows/version.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
steps:
- name: GitHub App token
id: github_app_token
uses: tibdex/github-app-token@v1.5.0
uses: tibdex/github-app-token@v2.1.0
with:
app_id: ${{ secrets.APP_ID }}
private_key: ${{ secrets.APP_PRIVATE_KEY }}
Expand Down Expand Up @@ -59,7 +59,7 @@ jobs:
sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" libs/core/src/main/java/org/opensearch/Version.java
- name: Create Pull Request
uses: peter-evans/create-pull-request@v3
uses: peter-evans/create-pull-request@v5
with:
token: ${{ steps.github_app_token.outputs.token }}
base: ${{ env.BASE }}
Expand All @@ -86,7 +86,7 @@ jobs:
sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
- name: Create Pull Request
uses: peter-evans/create-pull-request@v3
uses: peter-evans/create-pull-request@v5
with:
token: ${{ steps.github_app_token.outputs.token }}
base: ${{ env.BASE_X }}
Expand All @@ -113,7 +113,7 @@ jobs:
sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
- name: Create Pull Request
uses: peter-evans/create-pull-request@v3
uses: peter-evans/create-pull-request@v5
with:
token: ${{ steps.github_app_token.outputs.token }}
base: main
Expand Down
40 changes: 39 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854))
- Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107))
- Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679))
- Implement Visitor Design pattern in QueryBuilder to enable the capability to traverse through the complex QueryBuilder tree. ([#10110](https://github.com/opensearch-project/OpenSearch/pull/10110))
- Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
- Configurable merge policy for index with an option to choose from LogByteSize and Tiered merge policy ([#9992](https://github.com/opensearch-project/OpenSearch/pull/9992))

### Dependencies
- Bump `log4j-core` from 2.18.0 to 2.19.0
Expand Down Expand Up @@ -41,6 +44,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Bump `org.bouncycastle:bcmail-jdk15on` to `org.bouncycastle:bcmail-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247))
- Bump `org.bouncycastle:bcpkix-jdk15on` to `org.bouncycastle:bcpkix-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247))
- Bump JNA version from 5.5 to 5.13 ([#9963](https://github.com/opensearch-project/OpenSearch/pull/9963))
- Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822))
- Bump `org.eclipse.jgit` from 6.5.0 to 6.7.0 ([#10147](https://github.com/opensearch-project/OpenSearch/pull/10147))

### Changed
- [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
Expand Down Expand Up @@ -77,7 +82,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),

## [Unreleased 2.x]
### Added
- Add coordinator level stats for search latency ([#8386](https://github.com/opensearch-project/OpenSearch/issues/8386))
- Add metrics for thread_pool task wait time ([#9681](https://github.com/opensearch-project/OpenSearch/pull/9681))
- Async blob read support for S3 plugin ([#9694](https://github.com/opensearch-project/OpenSearch/pull/9694))
- [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter ([#9666](https://github.com/opensearch-project/OpenSearch/pull/9666))
- Async blob read support for encrypted containers ([#10131](https://github.com/opensearch-project/OpenSearch/pull/10131))
- Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189))
- Add Doc Status Counter for Indexing Engine ([#4562](https://github.com/opensearch-project/OpenSearch/issues/4562))
- Add unreferenced file cleanup count to merge stats ([#10204](https://github.com/opensearch-project/OpenSearch/pull/10204))

### Dependencies
- Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575))
Expand All @@ -87,20 +99,46 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Bump `com.google.cloud:google-cloud-core-http` from 2.21.1 to 2.23.0 ([#9971](https://github.com/opensearch-project/OpenSearch/pull/9971))
- Bump `mockito` from 5.4.0 to 5.5.0 ([#10022](https://github.com/opensearch-project/OpenSearch/pull/10022))
- Bump `bytebuddy` from 1.14.3 to 1.14.7 ([#10022](https://github.com/opensearch-project/OpenSearch/pull/10022))
- Bump `com.zaxxer:SparseBitSet` from 1.2 to 1.3 ([#10098](https://github.com/opensearch-project/OpenSearch/pull/10098))
- Bump `tibdex/github-app-token` from 1.5.0 to 2.1.0 ([#10125](https://github.com/opensearch-project/OpenSearch/pull/10125))
- Bump `org.wiremock:wiremock-standalone` from 2.35.0 to 3.1.0 ([#9752](https://github.com/opensearch-project/OpenSearch/pull/9752))
- Bump `com.google.http-client:google-http-client-jackson2` from 1.43.2 to 1.43.3 ([#10126](https://github.com/opensearch-project/OpenSearch/pull/10126))
- Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.4 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206))
- Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.0 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208))
- Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209))
- Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
- Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
- Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
- Bump Lucene from 9.7.0 to 9.8.0 ([10276](https://github.com/opensearch-project/OpenSearch/pull/10276))
- Bump `com.netflix.nebula.ospackage-base` from 11.4.0 to 11.5.0 ([#10295](https://github.com/opensearch-project/OpenSearch/pull/10295))
- Bump asm from 9.5 to 9.6 ([#10302](https://github.com/opensearch-project/OpenSearch/pull/10302))
- Bump netty from 4.1.97.Final to 4.1.99.Final ([#10303](https://github.com/opensearch-project/OpenSearch/pull/10303))
- Bump `peter-evans/create-pull-request` from 3 to 5 ([#10301](https://github.com/opensearch-project/OpenSearch/pull/10301))

### Changed
- Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415))
- Allow parameterization of tests with OpenSearchIntegTestCase.SuiteScopeTestCase annotation ([#9916](https://github.com/opensearch-project/OpenSearch/pull/9916))
- Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
- Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036))
- Add instrumentation in transport service. ([#10042](https://github.com/opensearch-project/OpenSearch/pull/10042))
- [Tracing Framework] Add support for SpanKind. ([#10122](https://github.com/opensearch-project/OpenSearch/pull/10122))
- Pass parent filter to inner query in nested query ([#10246](https://github.com/opensearch-project/OpenSearch/pull/10246))
- Disable concurrent segment search when terminate_after is used ([#10200](https://github.com/opensearch-project/OpenSearch/pull/10200))

### Deprecated

### Removed
- Remove spurious SGID bit on directories ([#9447](https://github.com/opensearch-project/OpenSearch/pull/9447))

### Fixed
- Fix ignore_missing parameter has no effect when using template snippet in rename ingest processor ([#9725](https://github.com/opensearch-project/OpenSearch/pull/9725))
- Fix broken backward compatibility from 2.7 for IndexSorted field indices ([#10045](https://github.com/opensearch-project/OpenSearch/pull/10045))
- Fix concurrent search NPE when track_total_hits, terminate_after and size=0 are used ([#10082](https://github.com/opensearch-project/OpenSearch/pull/10082))
- Fix remove ingest processor handing ignore_missing parameter not correctly ([10089](https://github.com/opensearch-project/OpenSearch/pull/10089))
- Fix circular dependency in Settings initialization ([10194](https://github.com/opensearch-project/OpenSearch/pull/10194))
- Fix registration and initialization of multiple extensions ([10256](https://github.com/opensearch-project/OpenSearch/pull/10256))

### Security

[Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x
[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x
6 changes: 5 additions & 1 deletion buildSrc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -128,11 +128,15 @@ dependencies {
testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
testFixturesApi gradleApi()
testFixturesApi gradleTestKit()
testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.35.0'
testImplementation 'org.wiremock:wiremock-standalone:3.1.0'
testImplementation "org.mockito:mockito-core:${props.getProperty('mockito')}"
integTestImplementation('org.spockframework:spock-core:2.3-groovy-3.0') {
exclude module: "groovy"
}
implementation('org.ajoberstar.grgit:grgit-core:5.2.0') {
exclude group: 'org.eclipse.jgit', module: 'org.eclipse.jgit'
}
implementation 'org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r'
}

configurations.all {
Expand Down
8 changes: 4 additions & 4 deletions buildSrc/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
opensearch = 3.0.0
lucene = 9.8.0-snapshot-4373c3b
lucene = 9.8.0

bundled_jdk_vendor = adoptium
bundled_jdk = 20.0.2+9
Expand All @@ -16,7 +16,7 @@ icu4j = 70.1
supercsv = 2.4.0
log4j = 2.20.0
slf4j = 1.7.36
asm = 9.5
asm = 9.6
jettison = 1.5.4
woodstox = 6.4.0
kotlin = 1.7.10
Expand All @@ -28,7 +28,7 @@ jakarta_annotation = 1.3.5
# when updating the JNA version, also update the version in buildSrc/build.gradle
jna = 5.13.0

netty = 4.1.97.Final
netty = 4.1.99.Final
joda = 2.12.2

# client dependencies
Expand All @@ -48,7 +48,7 @@ reactivestreams = 1.0.4
# when updating this version, you need to ensure compatibility with:
# - plugins/ingest-attachment (transitive dependency, check the upstream POM)
# - distribution/tools/plugin-cli
bouncycastle=1.75
bouncycastle=1.76
# test dependencies
randomizedrunner = 2.7.1
junit = 4.13.2
Expand Down
10 changes: 5 additions & 5 deletions distribution/packages/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ import java.util.regex.Pattern
*/

plugins {
id "com.netflix.nebula.ospackage-base" version "11.4.0"
id "com.netflix.nebula.ospackage-base" version "11.5.0"
}

void addProcessFilesTask(String type, boolean jdk) {
Expand Down Expand Up @@ -213,7 +213,7 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) {
configurationFile '/etc/opensearch/jvm.options'
configurationFile '/etc/opensearch/log4j2.properties'
from("${packagingFiles}") {
dirMode 02750
dirMode 0750
into('/etc')
permissionGroup 'opensearch'
includeEmptyDirs true
Expand All @@ -223,7 +223,7 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) {
}
from("${packagingFiles}/etc/opensearch") {
into('/etc/opensearch')
dirMode 02750
dirMode 0750
fileMode 0660
permissionGroup 'opensearch'
includeEmptyDirs true
Expand Down Expand Up @@ -281,8 +281,8 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) {
dirMode mode
}
}
copyEmptyDir('/var/log/opensearch', 'opensearch', 'opensearch', 02750)
copyEmptyDir('/var/lib/opensearch', 'opensearch', 'opensearch', 02750)
copyEmptyDir('/var/log/opensearch', 'opensearch', 'opensearch', 0750)
copyEmptyDir('/var/lib/opensearch', 'opensearch', 'opensearch', 0750)
copyEmptyDir('/usr/share/opensearch/plugins', 'root', 'root', 0755)

into '/usr/share/opensearch'
Expand Down
8 changes: 4 additions & 4 deletions distribution/packages/src/deb/lintian/opensearch
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ missing-dep-on-jarwrapper

# we prefer to not make our config and log files world readable
non-standard-file-perm etc/default/opensearch 0660 != 0644
non-standard-dir-perm etc/opensearch/ 2750 != 0755
non-standard-dir-perm etc/opensearch/jvm.options.d/ 2750 != 0755
non-standard-dir-perm etc/opensearch/ 0750 != 0755
non-standard-dir-perm etc/opensearch/jvm.options.d/ 0750 != 0755
non-standard-file-perm etc/opensearch/*
non-standard-dir-perm var/lib/opensearch/ 2750 != 0755
non-standard-dir-perm var/log/opensearch/ 2750 != 0755
non-standard-dir-perm var/lib/opensearch/ 0750 != 0755
non-standard-dir-perm var/log/opensearch/ 0750 != 0755
executable-is-not-world-readable etc/init.d/opensearch 0750
non-standard-file-permissions-for-etc-init.d-script etc/init.d/opensearch 0750 != 0755

Expand Down

This file was deleted.

1 change: 1 addition & 0 deletions libs/core/licenses/lucene-core-9.8.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5e8421c5f8573bcf22e9265fc7e19469545a775a
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ public class Index implements Writeable, ToXContentObject {
public static final Index[] EMPTY_ARRAY = new Index[0];
private static final String INDEX_UUID_KEY = "index_uuid";
private static final String INDEX_NAME_KEY = "index_name";
public static final String UNKNOWN_INDEX_NAME = "_unknown_";

private static final ObjectParser<Builder, Void> INDEX_PARSER = new ObjectParser<>("index", Builder::new);
static {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -527,6 +527,15 @@ public int getStatus() {
return status;
}

/**
* Get category class of a rest status code.
*
* @return Integer representing class category of the concrete rest status code
*/
public int getStatusFamilyCode() {
return status / 100;
}

public static RestStatus readFrom(StreamInput in) throws IOException {
return RestStatus.valueOf(in.readString());
}
Expand Down
Loading

0 comments on commit 4a0a570

Please sign in to comment.