Merge branch 'main' into batched-psa

.ci/bwcVersions

@@ -23,4 +23,7 @@ BWC_VERSION:
   - "2.9.0"
   - "2.9.1"
   - "2.10.0"
+  - "2.10.1"
   - "2.11.0"
+  - "2.11.1"
+  - "2.12.0"

.github/CODEOWNERS

@@ -1 +1 @@
-*   @reta @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @gbbafna @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @peternied @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @dbwiddis @sachinpkale @sohami @msfroh
+*   @abbashus @adnapibar @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @kartg @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @ryanbogan @sachinpkale @saratvemulapalli @setiah @shwetathareja @sohami @tlfeng @VachaShah

.github/dependabot.yml

@@ -711,6 +711,14 @@ updates:
     labels:
       - "dependabot"
       - "dependencies"
+  - directory: /modules/crypto/
+    open-pull-requests-limit: 1
+    package-ecosystem: gradle
+    schedule:
+      interval: weekly
+    labels:
+      - "dependabot"
+      - "dependencies"
   - directory: /plugins/
     open-pull-requests-limit: 1
     package-ecosystem: gradle

.github/pull_request_template.md

@@ -17,9 +17,10 @@ Resolves #[Issue number to be closed when this PR is merged]
   - [ ] All tests pass
 - [ ] New functionality has been documented.
   - [ ] New functionality has javadoc added
+- [ ] Failing checks are inspected and point to the corresponding known issue(s) (See: [Troubleshooting Failing Builds](../blob/main/CONTRIBUTING.md#troubleshooting-failing-builds))
 - [ ] Commits are signed per the DCO using --signoff
 - [ ] Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../blob/main/CONTRIBUTING.md#changelog))
-- [ ] GitHub issue/PR created in [OpenSearch documentation repo](https://github.com/opensearch-project/documentation-website) for the required public documentation changes (#[Issue/PR number])
+- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose)
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
 For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).

.github/workflows/add-untriaged.yml

@@ -9,7 +9,7 @@ jobs:
     if: github.repository == 'opensearch-project/OpenSearch'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/github-script@v6
+      - uses: actions/github-script@v7
         with:
           script: |
             github.rest.issues.addLabels({

.github/workflows/assemble.yml

@@ -0,0 +1,27 @@
+name: Gradle Assemble
+on: [pull_request]
+
+jobs:
+  assemble:
+    if: github.repository == 'opensearch-project/OpenSearch'
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        java: [ 11, 17, 21 ]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: temurin
+      - name: Setup docker (missing on MacOS)
+        if: runner.os == 'macos'
+        run: |
+          brew install docker
+          colima start
+          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
+      - name: Run Gradle (assemble)
+        run: |
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE

.github/workflows/dependabot_pr.yml

@@ -27,7 +27,7 @@ jobs:
           ./gradlew updateSHAs
 
       - name: Commit the changes
-        uses: stefanzweifel/git-auto-commit-action@v4.7.2
+        uses: stefanzweifel/git-auto-commit-action@v5
         with:
           commit_message: Updating SHAs
           branch: ${{ github.head_ref }}
@@ -40,7 +40,7 @@ jobs:
           ./gradlew spotlessApply
 
       - name: Commit the changes
-        uses: stefanzweifel/git-auto-commit-action@v4.7.2
+        uses: stefanzweifel/git-auto-commit-action@v5
         with:
           commit_message: Spotless formatting
           branch: ${{ github.head_ref }}
@@ -54,7 +54,7 @@ jobs:
           version: 'Unreleased 2.x'
 
       - name: Commit the changes
-        uses: stefanzweifel/git-auto-commit-action@v4
+        uses: stefanzweifel/git-auto-commit-action@v5
         with:
           commit_message: "Update changelog"
           branch: ${{ github.head_ref }}

.github/workflows/gradle-check.yml

@@ -82,10 +82,7 @@ jobs:
         with:
           issue-number: ${{ env.pr_number }}
           body: |
-            ### Gradle Check (Jenkins) Run Completed with:
-            * **RESULT:** ${{ env.result }} :white_check_mark:
-            * **URL:** ${{ env.workflow_url }}
-            * **CommitID:** ${{ env.pr_from_sha }}
+            :white_check_mark: Gradle check result for ${{ env.pr_from_sha }}: [${{ env.result }}](${{ env.workflow_url }})
 
       - name: Extract Test Failure
         if: ${{ github.event_name == 'pull_request_target' && env.result != 'SUCCESS' }}
@@ -108,10 +105,8 @@ jobs:
         with:
           issue-number: ${{ env.pr_number }}
           body: |
-            ### Gradle Check (Jenkins) Run Completed with:
-            * **RESULT:** ${{ env.result }} :grey_exclamation: ${{ env.test_failures }}
-            * **URL:** ${{ env.workflow_url }}
-            * **CommitID:** ${{ env.pr_from_sha }}
+            :grey_exclamation: Gradle check result for ${{ env.pr_from_sha }}: [${{ env.result }}](${{ env.workflow_url }}) ${{ env.test_failures }}
+
             Please review all [flaky tests](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) that succeeded after retry and create an issue if one does not already exist to track the flaky failure.
 
       - name: Create Comment Failure
@@ -120,12 +115,9 @@ jobs:
         with:
           issue-number: ${{ env.pr_number }}
           body: |
-            ### Gradle Check (Jenkins) Run Completed with:
-            * **RESULT:** ${{ env.result }} :x: ${{ env.test_failures }}
-            * **URL:** ${{ env.workflow_url }}
-            * **CommitID:** ${{ env.pr_from_sha }}
-            Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green.
-            Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
+            :x: Gradle check result for ${{ env.pr_from_sha }}: [${{ env.result }}](${{ env.workflow_url }})
+
+            Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green. Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
 
       - name: Create Issue On Push Failure
         if: ${{ github.event_name == 'push' && failure() }}

.github/workflows/lucene-snapshots.yml

@@ -7,7 +7,8 @@ on:
     # Inputs the workflow accepts.
     inputs:
       ref:
-        description:
+        description: 'Lucene ref in github.com/apache/lucene'
+        type: string
         required: false
         default: 'main'
 
@@ -21,40 +22,60 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/checkout@v4
-      - name: Set up JDK 17
-        uses: actions/setup-java@v3
-        with:
-          java-version: '17'
-          distribution: 'adopt'
-
-      - name: Checkout Lucene
+      - name: Checkout Lucene ref:${{ github.event.inputs.ref }}
         uses: actions/checkout@v4
         with:
           repository: 'apache/lucene'
-          path: lucene
           ref: ${{ github.event.inputs.ref }}
 
-      - name: Set hash
-        working-directory: ./lucene
+      - name: Get Java Min Version and Lucene Revision from Lucene Repository
         run: |
-          echo "REVISION=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-        id: version
+          java_version=`cat build.gradle | grep minJavaVersion | head -1 | grep -Eo '_[0-9]+$' | tr -d '_'`
+          echo "JAVA_VERSION=$java_version" >> $GITHUB_ENV
+          echo "REVISION=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+      - name: Setup JDK ${{ env.JAVA_VERSION }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: 'temurin'
 
       - name: Initialize gradle settings
-        working-directory: ./lucene
         run: ./gradlew localSettings
 
       - name: Publish Lucene to local maven repo.
-        working-directory: ./lucene
-        run: ./gradlew publishJarsPublicationToMavenLocal -Pversion.suffix=snapshot-${{ steps.version.outputs.REVISION }}
+        run: ./gradlew publishJarsPublicationToMavenLocal -Pversion.suffix=snapshot-${{ env.REVISION }}
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_SECRET_ROLE }}
+          aws-region: us-east-1
+
+      - name: Get S3 Bucket
+        id: get_s3_bucket
+        run: |
+          lucene_snapshots_bucket=`aws secretsmanager get-secret-value --secret-id jenkins-artifact-bucket-name --query SecretString --output text`
+          echo "::add-mask::$lucene_snapshots_bucket"
+          echo "LUCENE_SNAPSHOTS_BUCKET=$lucene_snapshots_bucket" >> $GITHUB_OUTPUT
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
+
+      - name: Copy files to S3 with the aws CLI (New)
+        run: |
+          aws s3 cp ~/.m2/repository/org/apache/lucene/ s3://${{ steps.get_s3_bucket.outputs.LUCENE_SNAPSHOTS_BUCKET }}/snapshots/lucene/org/apache/lucene/  --recursive --no-progress
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_ROLE }}
           aws-region: us-west-2
 
-      - name: Copy files to S3 with the aws CLI.
+        # We will remove this step once all the lucene snapshots old links are updated with the new one
+      - name: Copy files to S3 with the aws CLI (Old)
         run: |
           aws s3 cp ~/.m2/repository/org/apache/lucene/ s3://${{ secrets.LUCENE_SNAPSHOTS_BUCKET }}/snapshots/lucene/org/apache/lucene/  --recursive --no-progress

.github/workflows/precommit.yml

@@ -1,4 +1,4 @@
-name: Gradle Precommit and Assemble
+name: Gradle Precommit
 on: [pull_request]
 
 jobs:
@@ -7,24 +7,16 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        java: [ 11, 17, 21 ]
         os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
       - uses: actions/checkout@v4
-      - name: Set up JDK 11
+      - name: Set up JDK ${{ matrix.java }}
         uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: ${{ matrix.java }}
           distribution: temurin
           cache: gradle
       - name: Run Gradle (precommit)
         run: |
           ./gradlew javadoc precommit --parallel
-      - name: Setup docker (missing on MacOS)
-        if: runner.os == 'macos'
-        run: |
-          brew install docker
-          colima start
-          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
-      - name: Run Gradle (assemble)
-        run: |
-          ./gradlew assemble --parallel

.github/workflows/publish-maven-snapshots.yml

@@ -26,7 +26,7 @@ jobs:
           java-version: 17
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
           aws-region: us-east-1

.github/workflows/pull-request-checks.yml

@@ -0,0 +1,28 @@
+name: Pull Request Checks
+
+on:
+  pull_request:
+    types:
+      [
+        opened,
+        edited,
+        review_requested,
+        synchronize,
+        reopened,
+        ready_for_review,
+      ]
+
+jobs:
+  verify-description-checklist:
+    name: Verify Description Checklist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: peternied/check-pull-request-description-checklist@v1
+        with:
+          checklist-items: |
+            New functionality includes testing.
+            All tests pass
+            New functionality has been documented.
+            New functionality has javadoc added
+            Commits are signed per the DCO using --signoff
+            Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../blob/main/CONTRIBUTING.md#changelog))