Updates references to security demo configuration (opensearch-project…

…#480) * Updates github action reference to secruity demo configuration Signed-off-by: Darshit Chanpura <[email protected]> * Updates abstractions README to reflect changes with demo configuration Signed-off-by: Darshit Chanpura <[email protected]> * Updates references to admin password Signed-off-by: Darshit Chanpura <[email protected]> * Wider test range Signed-off-by: Thomas Farr <[email protected]> * Set initial admin password Signed-off-by: Thomas Farr <[email protected]> * Fix disabling SSL Signed-off-by: Thomas Farr <[email protected]> * Improve demo config condition Signed-off-by: Thomas Farr <[email protected]> * Use random password on >=2.12 Signed-off-by: Thomas Farr <[email protected]> --------- Signed-off-by: Darshit Chanpura <[email protected]> Signed-off-by: Thomas Farr <[email protected]> Co-authored-by: Thomas Farr <[email protected]> (cherry picked from commit dd2e674)
Xtansia · Jan 18, 2024 · 25f1f4a · 25f1f4a
1 parent d888669
commit 25f1f4a
Show file tree

Hide file tree

Showing 13 changed files with 376 additions and 35 deletions.
diff --git a/.github/actions/run-released-opensearch/action.yml b/.github/actions/run-released-opensearch/action.yml
@@ -0,0 +1,52 @@
+name: Run OpenSearch
+description: Runs a released version of OpenSearch
+inputs:
+  version:
+    description: The version of OpenSearch to run
+    required: true
+  secured:
+    description: Whether to enable the security plugin
+    required: true
+outputs:
+  opensearch_url:
+    description: The URL where the OpenSearch node is accessible
+    value: ${{ steps.opensearch.outputs.opensearch_url }}
+  admin_password:
+    description: The initial admin password
+    value: ${{ steps.opensearch.outputs.admin_password }}
+runs:
+  using: composite
+  steps:
+    - name: Restore cached OpenSearch distro
+      id: cache-restore
+      uses: actions/cache/restore@v3
+      with:
+        path: opensearch-*
+        key: opensearch-${{ inputs.version }}-${{ runner.os }}
+
+    - name: Download OpenSearch
+      if: steps.cache-restore.outputs.cache-hit != 'true'
+      shell: bash -eo pipefail {0}
+      run: |
+        if [[ "$RUNNER_OS" != "Windows" ]]; then
+          curl -sSLO https://artifacts.opensearch.org/releases/bundle/opensearch/${{ inputs.version }}/opensearch-${{ inputs.version }}-linux-x64.tar.gz
+          tar -xzf opensearch-*.tar.gz
+          rm -f opensearch-*.tar.gz
+        else
+          curl -sSLO https://artifacts.opensearch.org/releases/bundle/opensearch/${{ inputs.version }}/opensearch-${{ inputs.version }}-windows-x64.zip
+          unzip opensearch-*.zip
+          rm -f opensearch-*.zip
+        fi
+
+    - name: Save cached OpenSearch distro
+      if: steps.cache-restore.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v3
+      with:
+        path: opensearch-*
+        key: opensearch-${{ inputs.version }}-${{ runner.os }}
+
+    - name: Start OpenSearch
+      id: opensearch
+      uses: ./client/.github/actions/start-opensearch
+      with:
+        secured: ${{ inputs.secured }}
diff --git a/.github/actions/start-opensearch/action.yml b/.github/actions/start-opensearch/action.yml
@@ -0,0 +1,98 @@
+name: Start OpenSearch
+description: Configures and starts an OpenSearch daemon
+inputs:
+  secured:
+    description: Whether to enable the security plugin
+    default: 'false'
+outputs:
+  opensearch_url:
+    description: The URL where the OpenSearch node is accessible
+    value: ${{ steps.opensearch.outputs.url }}
+  admin_password:
+    description: The initial admin password
+    value: ${{ steps.opensearch.outputs.password }}
+runs:
+  using: composite
+  steps:
+    - name: Install Java
+      uses: actions/setup-java@v3
+      with:
+        distribution: zulu
+        java-version: 11
+
+    - name: Start OpenSearch
+      id: opensearch
+      shell: bash -eo pipefail {0}
+      run: |
+        if [[ "$RUNNER_OS" == "macOS" ]]; then
+          brew install -q coreutils
+        fi
+        OPENSEARCH_HOME=$(realpath ./opensearch-[1-9]*)
+        CONFIG_DIR=$OPENSEARCH_HOME/config
+        CONFIG_FILE=$CONFIG_DIR/opensearch.yml
+        SECURITY_DIR=$OPENSEARCH_HOME/plugins/opensearch-security
+        OPENSEARCH_JAVA_OPTS="-Djava.net.preferIPv4Stack=true"
+
+        URL="http://localhost:9200"
+        cp ./client/.ci/opensearch/opensearch.yml $CONFIG_FILE
+
+        bash ./client/.ci/generate-certs.sh
+        
+        export OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin
+
+        if [[ -d "$SECURITY_DIR" ]]; then
+          if [[ "$SECURED" == "true" ]]; then
+            SECURITY_VERSION=$(cat $SECURITY_DIR/plugin-descriptor.properties | grep '^version=' | cut -d'=' -f 2)
+            SECURITY_VERSION_COMPONENTS=(${SECURITY_VERSION//./ })
+            SECURITY_MAJOR="${SECURITY_VERSION_COMPONENTS[0]}"
+            SECURITY_MINOR="${SECURITY_VERSION_COMPONENTS[1]}"
+        
+            if (( $SECURITY_MAJOR > 2 || ( $SECURITY_MAJOR == 2 && $SECURITY_MINOR >= 12 ) )); then
+              export OPENSEARCH_INITIAL_ADMIN_PASSWORD=$(LC_ALL=C tr -dc A-Za-z0-9 </dev/urandom | head -c 16)
+            fi
+        
+            bash "$SECURITY_DIR/tools/install_demo_configuration.sh" -y -i -s
+        
+            sed -i.bak -e 's/plugins.security.audit.type:.*/plugins.security.audit.type: log4j/' $CONFIG_FILE
+            cp ./client/.ci/opensearch/*.pem $CONFIG_DIR/
+            URL="https://localhost:9200"
+          else
+            printf "\nplugins.security.disabled: true" >> $CONFIG_FILE
+          fi
+        fi
+
+        if [[ "$RUNNER_OS" == "macOS" ]]; then
+          sed -i.bak -e 's/bootstrap.memory_lock:.*/bootstrap.memory_lock: false/' $CONFIG_FILE
+        fi
+
+        {
+          echo "url=$URL"
+          echo "password=$OPENSEARCH_INITIAL_ADMIN_PASSWORD"
+        } | tee -a $GITHUB_OUTPUT
+        
+        if [[ "$RUNNER_OS" == "Linux" ]]; then
+          sudo swapoff -a
+          sudo sysctl -w vm.swappiness=1
+          sudo sysctl -w fs.file-max=262144
+          sudo sysctl -w vm.max_map_count=262144
+          sudo prlimit --pid $$ --memlock=unlimited:unlimited
+        fi
+
+        if [[ "$RUNNER_OS" != "Windows" ]]; then
+          $OPENSEARCH_HOME/bin/opensearch &
+        else
+          $OPENSEARCH_HOME/bin/opensearch.bat -d &
+        fi
+
+        for attempt in {1..20}; do
+          sleep 5
+          if curl -k -sS --cacert ./client/.ci/certs/root-ca.crt -u admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD} $URL; then
+            echo '=====> ready'
+            exit 0
+          fi
+          echo '=====> waiting...'
+        done
+        exit 1
+      env:
+        SECURED: ${{ inputs.secured }}
+        RUNNER_OS: ${{ runner.os }}
diff --git a/.github/workflows/integration-yaml-tests.yml b/.github/workflows/integration-yaml-tests.yml
@@ -0,0 +1,152 @@
+name: YAML Tests
+
+on:
+  push:
+    branches-ignore:
+      - 'dependabot/**'
+  pull_request: {}
+
+jobs:
+  test-yaml:
+    name: YAML Tests (Released OpenSearch)
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - 2.11.1
+          - 2.10.0
+          - 2.8.0
+          - 2.6.0
+          - 2.4.1
+          - 2.2.1
+          - 2.0.1
+          - 1.3.14
+          - 1.2.4
+          - 1.1.0
+    steps:
+      - name: Checkout Client
+        uses: actions/checkout@v3
+        with:
+          path: client
+
+      - uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: |
+            5.0.x
+            6.0.x
+
+      - uses: actions/cache@v3
+        with:
+          path: ~/.nuget/packages
+          key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.?sproj') }}
+          restore-keys: |
+            ${{ runner.os }}-nuget-
+
+      - name: Run OpenSearch
+        id: opensearch
+        uses: ./client/.github/actions/run-released-opensearch
+        with:
+          version: ${{ matrix.version }}
+          secured: true
+
+      - name: Run YAML tests
+        working-directory: client
+        run: |
+          dotnet run \
+            --project ./tests/Tests.YamlRunner/Tests.YamlRunner.fsproj \
+            -- \
+            --endpoint $OPENSEARCH_URL \
+            --auth-cert ./.ci/certs/kirk.p12 \
+            --auth-cert-pass kirk \
+            --junit-output-file ./test-results.xml
+        env:
+          OPENSEARCH_URL: ${{ steps.opensearch.outputs.opensearch_url }}
+
+      - name: Save OpenSearch logs
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: opensearch-logs-${{ matrix.version }}
+          path: |
+            opensearch-*/logs/*
+
+      - name: Upload test report
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: report-yaml-${{ matrix.version }}
+          path: client/test-results.xml
+
+  test-yaml-unreleased:
+    name: YAML Tests (Unreleased OpenSearch)
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        opensearch_ref: ['1.x', '2.x', 'main']
+    steps:
+      - name: Checkout Client
+        uses: actions/checkout@v3
+        with:
+          path: client
+
+      - uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: |
+            5.0.x
+            6.0.x
+
+      - uses: actions/cache@v3
+        with:
+          path: ~/.nuget/packages
+          key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.?sproj') }}
+          restore-keys: |
+            ${{ runner.os }}-nuget-
+
+      - name: Restore or Build OpenSearch
+        id: opensearch_build
+        uses: ./client/.github/actions/build-opensearch
+        with:
+          ref: ${{ matrix.opensearch_ref }}
+          security_plugin: true
+
+      - name: Unpack OpenSearch
+        run: |
+          tar -xzf ${{ steps.opensearch_build.outputs.distribution }} \
+          && ./opensearch-*/bin/opensearch-plugin install --batch file://$(realpath ./opensearch-security/build/distributions/opensearch-security-*-SNAPSHOT.zip)
+
+      - name: Start OpenSearch
+        id: opensearch
+        uses: ./client/.github/actions/start-opensearch
+        with:
+          secured: true
+
+      - name: Run YAML tests
+        working-directory: client
+        run: |
+          dotnet run \
+            --project ./tests/Tests.YamlRunner/Tests.YamlRunner.fsproj \
+            -- \
+            --endpoint $OPENSEARCH_URL \
+            --auth-cert ./.ci/certs/kirk.p12 \
+            --auth-cert-pass kirk \
+            --junit-output-file ./test-results.xml
+        env:
+          OPENSEARCH_URL: ${{ steps.opensearch.outputs.opensearch_url }}
+          ADMIN_PASS: ${{ steps.opensearch.outputs.admin_password }}
+
+      - name: Save OpenSearch logs
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: opensearch-logs-${{ matrix.opensearch_ref }}
+          path: |
+            opensearch-*/logs/*
+
+      - name: Upload test report
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: report-yaml-unreleased-${{ matrix.opensearch_ref }}
+          path: client/test-results.xml
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -17,17 +17,14 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - 2.9.0
+          - 2.11.1
+          - 2.10.0
           - 2.8.0
-          - 2.7.0
           - 2.6.0
-          - 2.5.0
           - 2.4.1
-          - 2.3.0
           - 2.2.1
-          - 2.1.0
           - 2.0.1
-          - 1.3.11
+          - 1.3.14
           - 1.2.4
           - 1.1.0
 
@@ -62,7 +59,6 @@ jobs:
           path: client/build/output/*
 
   integration-opensearch-unreleased:
-    if: false # TODO: Temporarily disabled due to failures building & running OpenSearch from source, pending investigation & fixes (https://github.com/opensearch-project/opensearch-net/issues/268)
     name: Integration OpenSearch Unreleased
     runs-on: ubuntu-latest
     strategy:

diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs
@@ -34,6 +34,7 @@
 using System.Security.Cryptography;
 using System.Text;
 using OpenSearch.OpenSearch.Managed;
+using OpenSearch.OpenSearch.Managed.Configuration;
 using OpenSearch.Stack.ArtifactsApi;
 
 namespace OpenSearch.OpenSearch.Ephemeral
@@ -59,6 +60,13 @@ protected EphemeralCluster(TConfiguration clusterConfiguration) : base(clusterCo
 
 		protected EphemeralClusterComposer<TConfiguration> Composer { get; }
 
+		protected override void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port)
+		{
+			base.ModifyNodeConfiguration(nodeConfiguration, port);
+
+			if (!ClusterConfiguration.EnableSsl) nodeConfiguration.Add("plugins.security.disabled", "true");
+		}
+
 		public virtual ICollection<Uri> NodesUris(string hostName = null)
 		{
 			hostName = hostName ?? (ClusterConfiguration.HttpFiddlerAware && Process.GetProcessesByName("fiddler").Any()

diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs
@@ -172,22 +172,35 @@ protected static void WriteFileIfNotExist(string fileLocation, string contents)
 
 		protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
 			string binary, string description, params string[] arguments) =>
-			ExecuteBinaryInternal(config, writer, binary, description, arguments);
+			ExecuteBinaryInternal(config, writer, binary, description, null, arguments);
+
+		protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
+			string binary, string description, IDictionary<string, string> environmentVariables,
+			params string[] arguments) =>
+			ExecuteBinaryInternal(config, writer, binary, description, environmentVariables, arguments);
 
 		private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
-			string binary, string description, params string[] arguments)
+			string binary, string description, IDictionary<string, string> environmentVariables, params string[] arguments)
 		{
 			var command = $"{{{binary}}} {{{string.Join(" ", arguments)}}}";
 			writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} starting process [{description}] {command}");
 
+			var environment = new Dictionary<string, string>
+			{
+				{config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath},
+				{"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome}
+			};
+
+			if (environmentVariables != null)
+			{
+				foreach (var kvp in environmentVariables)
+					environment[kvp.Key] = kvp.Value;
+			}
+
 			var timeout = TimeSpan.FromSeconds(420);
 			var processStartArguments = new StartArguments(binary, arguments)
 			{
-				Environment = new Dictionary<string, string>
-				{
-					{config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath},
-					{"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome},
-				}
+				Environment = environment
 			};
 
 			var result = Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter());