iter

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>
harshavamsi · Aug 21, 2024 · b837be4 · b837be4
1 parent a5a9e57
commit b837be4
Show file tree

Hide file tree

Showing 675 changed files with 6,635 additions and 32,611 deletions.
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -36,5 +36,3 @@ BWC_VERSION:
   - "2.15.0"
   - "2.15.1"
   - "2.16.0"
-  - "2.16.1"
-  - "2.17.0"
diff --git a/.ci/java-versions.properties b/.ci/java-versions.properties
@@ -13,8 +13,7 @@
 # build and test OpenSearch for this branch. Valid Java versions
 # are 'java' or 'openjdk' followed by the major release number.
 
-# See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
-OPENSEARCH_BUILD_JAVA=openjdk17
+OPENSEARCH_BUILD_JAVA=openjdk11
 OPENSEARCH_RUNTIME_JAVA=java11
 GRADLE_TASK=build
 GRADLE_EXTRA_ARGS=
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -11,27 +11,17 @@
 #   3. Use the command palette to run the CODEOWNERS: Show owners of current file command, which will display all code owners for the current file.
 
 # Default ownership for all repo files
-* @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jainankitk @kotwanikunal @linuxpi @mch2 @msfroh @nknize @owaiskazi19  @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+* @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 
-/modules/lang-painless/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/modules/parent-join/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 /modules/transport-netty4/ @peternied
 
 /plugins/identity-shiro/ @peternied
 
-/server/src/internalClusterTest/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/server/src/internalClusterTest/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-
 /server/src/main/java/org/opensearch/extensions/ @peternied
 /server/src/main/java/org/opensearch/identity/ @peternied
-/server/src/main/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/server/src/main/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/server/src/main/java/org/opensearch/threadpool/ @jed326 @peternied
+/server/src/main/java/org/opensearch/threadpool/ @peternied
 /server/src/main/java/org/opensearch/transport/ @peternied
 
-/server/src/test/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/server/src/test/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-
-/.github/ @jed326 @peternied
+/.github/ @peternied
 
 /MAINTAINERS.md @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gaobinlong @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
diff --git a/.github/benchmark-configs.json b/.github/benchmark-configs.json
@@ -14,8 +14,7 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
+    }
   },
   "id_2": {
     "description": "Indexing only configuration for HTTP_LOGS workload",
@@ -31,8 +30,7 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
+    }
   },
   "id_3": {
     "description": "Search only test-procedure for NYC_TAXIS, uses snapshot to restore the data for OS-3.0.0",
@@ -42,14 +40,12 @@
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "nyc_taxis",
       "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"nyc_taxis_1_shard\"}",
-      "CAPTURE_NODE_STAT": "true",
-      "TEST_PROCEDURE": "restore-from-snapshot"
+      "CAPTURE_NODE_STAT": "true"
     },
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
+    }
   },
   "id_4": {
     "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-3.0.0",
@@ -59,31 +55,27 @@
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "http_logs",
       "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"http_logs_1_shard\"}",
-      "CAPTURE_NODE_STAT": "true",
-      "TEST_PROCEDURE": "restore-from-snapshot"
+      "CAPTURE_NODE_STAT": "true"
     },
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
+    }
   },
   "id_5": {
-    "description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-3.0.0",
+    "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-3.0.0",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
       "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}",
-      "CAPTURE_NODE_STAT": "true",
-      "TEST_PROCEDURE": "restore-from-snapshot"
+      "CAPTURE_NODE_STAT": "true"
     },
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
+    }
   },
   "id_6": {
     "description": "Search only test-procedure for NYC_TAXIS, uses snapshot to restore the data for OS-2.x",
@@ -93,14 +85,12 @@
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "nyc_taxis",
       "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"nyc_taxis_1_shard\"}",
-      "CAPTURE_NODE_STAT": "true",
-      "TEST_PROCEDURE": "restore-from-snapshot"
+      "CAPTURE_NODE_STAT": "true"
     },
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
+    }
   },
   "id_7": {
     "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-2.x",
@@ -110,31 +100,27 @@
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "http_logs",
       "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"http_logs_1_shard\"}",
-      "CAPTURE_NODE_STAT": "true",
-      "TEST_PROCEDURE": "restore-from-snapshot"
+      "CAPTURE_NODE_STAT": "true"
     },
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
+    }
   },
   "id_8": {
-    "description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-2.x",
+    "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-2.x",
     "supported_major_versions": ["2"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
       "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"big5_1_shard\"}",
-      "CAPTURE_NODE_STAT": "true",
-      "TEST_PROCEDURE": "restore-from-snapshot"
+      "CAPTURE_NODE_STAT": "true"
     },
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
+    }
   },
   "id_9": {
     "description": "Indexing and search configuration for pmc workload",
@@ -149,8 +135,7 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
+    }
   },
   "id_10": {
     "description": "Indexing only configuration for stack-overflow workload",
@@ -165,7 +150,6 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
+    }
   }
 }
diff --git a/.github/workflows/add-performance-comment.yml b/.github/workflows/add-performance-comment.yml
@@ -6,17 +6,14 @@ on:
 
 jobs:
   add-comment:
-    if: |
-      github.event.label.name == 'Performance' ||
-      github.event.label.name == 'Search:Performance' ||
-      github.event.label.name == 'Indexing:Performance'
+    if: github.event.label.name == 'Performance'
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
 
     steps:
       - name: Add comment to PR
-        uses: actions/github-script@v7
+        uses: actions/github-script@v6
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           script: |

diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml
@@ -16,17 +16,6 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
           distribution: temurin
-      - name: Set up JDK 17
-        # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
-        if: matrix.java == 11
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      - name: Set JAVA${{ matrix.java }}_HOME
-        shell: bash
-        run: |
-          echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
       - name: Setup docker (missing on MacOS)
         id: setup_docker
         if: runner.os == 'macos'
@@ -41,11 +30,10 @@ jobs:
           # Report success even if previous step failed (Docker on MacOS runner is very unstable)
           exit 0;
       - name: Run Gradle (assemble)
-        shell: bash
         if: runner.os != 'macos'
         run: |
-          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }} 
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
       - name: Run Gradle (assemble)
         if: runner.os == 'macos' && steps.setup_docker.outcome == 'success'
         run: |
-          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }}
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
diff --git a/.github/workflows/benchmark-pull-request.yml b/.github/workflows/benchmark-pull-request.yml
@@ -13,7 +13,7 @@ jobs:
       pull-requests: write
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
       - name: Set up required env vars
         run: |
           echo "PR_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV
@@ -25,7 +25,7 @@ jobs:
           echo "USER_TAGS=pull_request_number:${{ github.event.issue.number }},repository:OpenSearch" >> $GITHUB_ENV
       - name: Check comment format
         id: check_comment
-        uses: actions/github-script@v7
+        uses: actions/github-script@v6
         with:
           script: |
             const fs = require('fs');
@@ -60,12 +60,9 @@ jobs:
             for (const [key, value] of Object.entries(clusterBenchmarkConfigs)) {
               core.exportVariable(key, value);
             }
-            if (benchmarkConfigs[configId].hasOwnProperty('baseline_cluster_config')) {
-              core.exportVariable('BASELINE_CLUSTER_CONFIG', benchmarkConfigs[configId]['baseline_cluster_config']);
-            }
       - name: Post invalid format comment
         if: steps.check_comment.outputs.invalid == 'true'
-        uses: actions/github-script@v7
+        uses: actions/github-script@v6
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           script: |
@@ -80,6 +77,18 @@ jobs:
         run: |
           echo "Invalid comment format detected. Failing the workflow."
           exit 1
+      - id: get_approvers
+        run: |
+          echo "approvers=$(cat .github/CODEOWNERS | grep '^\*'  | tr -d '* ' | sed 's/@/,/g' | sed 's/,//1')" >> $GITHUB_OUTPUT
+      - uses: trstringer/manual-approval@v1
+        if: (!contains(steps.get_approvers.outputs.approvers, github.event.comment.user.login))
+        with:
+          secret: ${{ github.TOKEN }}
+          approvers: ${{ steps.get_approvers.outputs.approvers }}
+          minimum-approvals: 1
+          issue-title: 'Request to approve/deny benchmark run for PR #${{ env.PR_NUMBER }}'
+          issue-body: "Please approve or deny the benchmark run for PR #${{ env.PR_NUMBER }}"
+          exclude-workflow-initiator-as-approver: false
       - name: Get PR Details
         id: get_pr
         uses: actions/github-script@v7
@@ -97,51 +106,26 @@ jobs:
 
             return {
               "headRepoFullName": pull_request.head.repo.full_name,
-              "headRefSha": pull_request.head.sha
+              "headRef": pull_request.head.ref
             };
       - name: Set pr details env vars
         run: |
           echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRepoFullName'
-          echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRefSha'
+          echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRef'
           headRepo=$(echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRepoFullName')
-          headRefSha=$(echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRefSha')
+          headRef=$(echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRef')
           echo "prHeadRepo=$headRepo" >> $GITHUB_ENV
-          echo "prHeadRefSha=$headRefSha" >> $GITHUB_ENV
-      - id: get_approvers
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          result-encoding: string
-          script: |
-            // Get the collaborators - filtered to maintainer permissions
-            const maintainersResponse = await github.request('GET /repos/{owner}/{repo}/collaborators', {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              permission: 'maintain',
-              affiliation: 'all',
-              per_page: 100
-              });
-            return maintainersResponse.data.map(item => item.login).join(', ');
-      - uses: trstringer/manual-approval@v1
-        if: (!contains(steps.get_approvers.outputs.result, github.event.comment.user.login))
-        with:
-          secret: ${{ github.TOKEN }}
-          approvers: ${{ steps.get_approvers.outputs.result }}
-          minimum-approvals: 1
-          issue-title: 'Request to approve/deny benchmark run for PR #${{ env.PR_NUMBER }}'
-          issue-body: "Please approve or deny the benchmark run for PR #${{ env.PR_NUMBER }}"
-          exclude-workflow-initiator-as-approver: false
+          echo "prheadRef=$headRef" >> $GITHUB_ENV
       - name: Checkout PR Repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
         with:
           repository: ${{ env.prHeadRepo }}
-          ref: ${{ env.prHeadRefSha }}
+          ref: ${{ env.prHeadRef }}
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Setup Java
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v1
         with:
           java-version: 21
-          distribution: 'temurin'
       - name: Build and Assemble OpenSearch from PR
         run: |
           ./gradlew :distribution:archives:linux-tar:assemble -Dbuild.snapshot=false
@@ -166,7 +150,7 @@ jobs:
           cat $GITHUB_ENV
           bash opensearch-build/scripts/benchmark/benchmark-pull-request.sh ${{ secrets.JENKINS_PR_BENCHMARK_GENERIC_WEBHOOK_TOKEN }}
       - name: Update PR with Job Url
-        uses: actions/github-script@v7
+        uses: actions/github-script@v6
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |

diff --git a/.github/workflows/changelog_verifier.yml b/.github/workflows/changelog_verifier.yml
@@ -27,29 +27,18 @@ jobs:
         continue-on-error: true
       - run: |
           # The check was possibly skipped leading to success for both the jobs
-          has_backport_label=${{ contains(join(github.event.pull_request.labels.*.name, ', '), 'backport')}}
-          has_breaking_label=${{ contains(join(github.event.pull_request.labels.*.name, ', '), '>breaking')}}
-          if [[ $has_breaking_label == true && $has_backport_label == true ]]; then
-            echo "error: Please make sure that the PR does not have a backport label associated with it when making breaking changes"
-            exit 1
-          fi
-
           if [[ ${{ steps.verify-changelog-3x.outcome }} == 'success' && ${{ steps.verify-changelog.outcome }} == 'success' ]]; then
             exit 0
           fi
-
+          
           if [[ ${{ steps.verify-changelog-3x.outcome }} == 'failure' && ${{ steps.verify-changelog.outcome }} == 'failure' ]]; then
             echo "error: Please ensure a changelog entry exists in CHANGELOG.md or CHANGELOG-3.0.md"
             exit 1
           fi
-
+          
           # Concatenates the labels and checks if the string contains "backport"
+          has_backport_label=${{ contains(join(github.event.pull_request.labels.*.name, ', '), 'backport')}}
           if [[ ${{ steps.verify-changelog.outcome }} == 'success' && $has_backport_label == false ]]; then
             echo "error: Please make sure that the PR has a backport label associated with it when making an entry to the CHANGELOG.md file"
             exit 1
           fi
-
-          if [[ ${{ steps.verify-changelog-3x.outcome }} == 'success' && $has_backport_label == true ]]; then
-            echo "error: Please make sure that the PR does not have a backport label associated with it when making an entry to the CHANGELOG-3.0.md file"
-            exit 1
-          fi