Updates references to security demo configuration

.github/actions/run-released-opensearch/action.yml

@@ -11,6 +11,9 @@ outputs:
   opensearch_url:
     description: The URL where the OpenSearch node is accessible
     value: ${{ steps.opensearch.outputs.opensearch_url }}
+  admin_password:
+    description: The initial admin password
+    value: ${{ steps.opensearch.outputs.admin_password }}
 runs:
   using: composite
   steps:

.github/actions/start-opensearch/action.yml

@@ -8,6 +8,9 @@ outputs:
   opensearch_url:
     description: The URL where the OpenSearch node is accessible
     value: ${{ steps.opensearch.outputs.url }}
+  admin_password:
+    description: The initial admin password
+    value: ${{ steps.opensearch.outputs.password }}
 runs:
   using: composite
   steps:
@@ -24,30 +27,49 @@ runs:
         if [[ "$RUNNER_OS" == "macOS" ]]; then
           brew install -q coreutils
         fi
-        OPENSEARCH_HOME=$(realpath ./opensearch-*)
+        OPENSEARCH_HOME=$(realpath ./opensearch-[1-9]*)
+        CONFIG_DIR=$OPENSEARCH_HOME/config
+        CONFIG_FILE=$CONFIG_DIR/opensearch.yml
+        SECURITY_DIR=$OPENSEARCH_HOME/plugins/opensearch-security
         OPENSEARCH_JAVA_OPTS="-Djava.net.preferIPv4Stack=true"
 
-        url="http://localhost:9200"
-        cp ./client/.ci/opensearch/opensearch.yml $OPENSEARCH_HOME/config/
+        URL="http://localhost:9200"
+        cp ./client/.ci/opensearch/opensearch.yml $CONFIG_FILE
 
         bash ./client/.ci/generate-certs.sh
+
+        export OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin
 
-        if [[ -d "$OPENSEARCH_HOME/plugins/opensearch-security" ]]; then
+        if [[ -d "$SECURITY_DIR" ]]; then
           if [[ "$SECURED" == "true" ]]; then
-            bash $OPENSEARCH_HOME/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s
-            sed -i.bak -e 's/plugins.security.audit.type:.*/plugins.security.audit.type: log4j/' $OPENSEARCH_HOME/config/opensearch.yml
-            cp ./client/.ci/opensearch/*.pem $OPENSEARCH_HOME/config/
-            url="https://localhost:9200"
+            SECURITY_VERSION=$(cat $SECURITY_DIR/plugin-descriptor.properties | grep '^version=' | cut -d'=' -f 2)
+            SECURITY_VERSION_COMPONENTS=(${SECURITY_VERSION//./ })
+            SECURITY_MAJOR="${SECURITY_VERSION_COMPONENTS[0]}"
+            SECURITY_MINOR="${SECURITY_VERSION_COMPONENTS[1]}"
+
+            if (( $SECURITY_MAJOR > 2 || ( $SECURITY_MAJOR == 2 && $SECURITY_MINOR >= 12 ) )); then
+              export OPENSEARCH_INITIAL_ADMIN_PASSWORD=$(LC_ALL=C tr -dc A-Za-z0-9 </dev/urandom | head -c 16)
+            fi
+
+            bash "$SECURITY_DIR/tools/install_demo_configuration.sh" -y -i -s
+
+            sed -i.bak -e 's/plugins.security.audit.type:.*/plugins.security.audit.type: log4j/' $CONFIG_FILE
+            cp ./client/.ci/opensearch/*.pem $CONFIG_DIR/
+            URL="https://localhost:9200"
           else
-            printf "\nplugins.security.disabled: true" >> $OPENSEARCH_HOME/config/opensearch.yml
+            printf "\nplugins.security.disabled: true" >> $CONFIG_FILE
           fi
         fi
 
         if [[ "$RUNNER_OS" == "macOS" ]]; then
-          sed -i.bak -e 's/bootstrap.memory_lock:.*/bootstrap.memory_lock: false/' $OPENSEARCH_HOME/config/opensearch.yml
+          sed -i.bak -e 's/bootstrap.memory_lock:.*/bootstrap.memory_lock: false/' $CONFIG_FILE
         fi
 
-        echo "url=$url" >> $GITHUB_OUTPUT
+        {
+          echo "url=$URL"
+          echo "password=$OPENSEARCH_INITIAL_ADMIN_PASSWORD"
+        } | tee -a $GITHUB_OUTPUT
+
         if [[ "$RUNNER_OS" == "Linux" ]]; then
           sudo swapoff -a
           sudo sysctl -w vm.swappiness=1
@@ -64,7 +86,7 @@ runs:
 
         for attempt in {1..20}; do
           sleep 5
-          if curl -k -sS --cacert ./client/.ci/certs/root-ca.crt -u admin:admin $url; then
+          if curl -k -sS --cacert ./client/.ci/certs/root-ca.crt -u admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD} $URL; then
             echo '=====> ready'
             exit 0
           fi

.github/workflows/integration-yaml-tests.yml

@@ -13,7 +13,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: [ '1.2.4', '1.3.11', '2.2.0', '2.4.0', '2.6.0', '2.8.0', '2.9.0' ]
+        version:
+          - 2.11.1
+          - 2.10.0
+          - 2.8.0
+          - 2.6.0
+          - 2.4.1
+          - 2.2.1
+          - 2.0.1
+          - 1.3.14
+          - 1.2.4
+          - 1.1.0
     steps:
       - name: Checkout Client
         uses: actions/checkout@v3
@@ -99,13 +109,18 @@ jobs:
         uses: ./client/.github/actions/build-opensearch
         with:
           ref: ${{ matrix.opensearch_ref }}
+          security_plugin: true
 
       - name: Unpack OpenSearch
-        run: tar -xzf ${{ steps.opensearch_build.outputs.distribution }}
+        run: |
+          tar -xzf ${{ steps.opensearch_build.outputs.distribution }} \
+          && ./opensearch-*/bin/opensearch-plugin install --batch file://$(realpath ./opensearch-security/build/distributions/opensearch-security-*-SNAPSHOT.zip)
 
       - name: Start OpenSearch
         id: opensearch
         uses: ./client/.github/actions/start-opensearch
+        with:
+          secured: true
 
       - name: Run YAML tests
         working-directory: client
@@ -114,9 +129,12 @@ jobs:
             --project ./tests/Tests.YamlRunner/Tests.YamlRunner.fsproj \
             -- \
             --endpoint $OPENSEARCH_URL \
+            --auth-cert ./.ci/certs/kirk.p12 \
+            --auth-cert-pass kirk \
             --junit-output-file ./test-results.xml
         env:
           OPENSEARCH_URL: ${{ steps.opensearch.outputs.opensearch_url }}
+          ADMIN_PASS: ${{ steps.opensearch.outputs.admin_password }}
 
       - name: Save OpenSearch logs
         if: failure()

.github/workflows/integration.yml

@@ -17,17 +17,14 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - 2.9.0
+          - 2.11.1
+          - 2.10.0
           - 2.8.0
-          - 2.7.0
           - 2.6.0
-          - 2.5.0
           - 2.4.1
-          - 2.3.0
           - 2.2.1
-          - 2.1.0
           - 2.0.1
-          - 1.3.11
+          - 1.3.14
           - 1.2.4
           - 1.1.0
 
@@ -62,7 +59,6 @@ jobs:
           path: client/build/output/*
 
   integration-opensearch-unreleased:
-    if: false # TODO: Temporarily disabled due to failures building & running OpenSearch from source, pending investigation & fixes (https://github.com/opensearch-project/opensearch-net/issues/268)
     name: Integration OpenSearch Unreleased
     runs-on: ubuntu-latest
     strategy:

abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs

@@ -34,6 +34,7 @@
 using System.Security.Cryptography;
 using System.Text;
 using OpenSearch.OpenSearch.Managed;
+using OpenSearch.OpenSearch.Managed.Configuration;
 using OpenSearch.Stack.ArtifactsApi;
 
 namespace OpenSearch.OpenSearch.Ephemeral
@@ -59,6 +60,13 @@ protected EphemeralCluster(TConfiguration clusterConfiguration) : base(clusterCo
 
 		protected EphemeralClusterComposer<TConfiguration> Composer { get; }
 
+		protected override void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port)
+		{
+			base.ModifyNodeConfiguration(nodeConfiguration, port);
+
+			if (!ClusterConfiguration.EnableSsl) nodeConfiguration.Add("plugins.security.disabled", "true");
+		}
+
 		public virtual ICollection<Uri> NodesUris(string hostName = null)
 		{
 			hostName = hostName ?? (ClusterConfiguration.HttpFiddlerAware && Process.GetProcessesByName("fiddler").Any()

abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs

@@ -172,22 +172,35 @@ protected static void WriteFileIfNotExist(string fileLocation, string contents)
 
 		protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
 			string binary, string description, params string[] arguments) =>
-			ExecuteBinaryInternal(config, writer, binary, description, arguments);
+			ExecuteBinaryInternal(config, writer, binary, description, null, arguments);
+
+		protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
+			string binary, string description, IDictionary<string, string> environmentVariables,
+			params string[] arguments) =>
+			ExecuteBinaryInternal(config, writer, binary, description, environmentVariables, arguments);
 
 		private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
-			string binary, string description, params string[] arguments)
+			string binary, string description, IDictionary<string, string> environmentVariables, params string[] arguments)
 		{
 			var command = $"{{{binary}}} {{{string.Join(" ", arguments)}}}";
 			writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} starting process [{description}] {command}");
 
+			var environment = new Dictionary<string, string>
+			{
+				{config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath},
+				{"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome}
+			};
+
+			if (environmentVariables != null)
+			{
+				foreach (var kvp in environmentVariables)
+					environment[kvp.Key] = kvp.Value;
+			}
+
 			var timeout = TimeSpan.FromSeconds(420);
 			var processStartArguments = new StartArguments(binary, arguments)
 			{
-				Environment = new Dictionary<string, string>
-				{
-					{config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath},
-					{"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome},
-				}
+				Environment = environment
 			};
 
 			var result = Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter());

abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs

@@ -26,10 +26,12 @@
 *  under the License.
 */
 
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using OpenSearch.OpenSearch.Managed.ConsoleWriters;
-using OpenSearch.Stack.ArtifactsApi;
+using OpenSearch.Stack.ArtifactsApi.Products;
+using SemanticVersioning;
 
 namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks
 {
@@ -38,30 +40,39 @@ public class InitialConfiguration : ClusterComposeTask
 		public override void Run(IEphemeralCluster<EphemeralClusterConfiguration> cluster)
 		{
 			var fs = cluster.FileSystem;
-			var configFile = Path.Combine(fs.OpenSearchHome, "config", "opensearch.yml");
 
-			if (File.Exists(configFile) && File.ReadLines(configFile).Any(l => !string.IsNullOrWhiteSpace(l) && !l.StartsWith("#")))
-			{
-				cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} opensearch.yml already exists, skipping initial configuration");
+			var installConfigDir = Path.Combine(fs.OpenSearchHome, "config");
+			var installConfigFile = Path.Combine(installConfigDir, "opensearch.yml");
+			var pluginSecurity = Path.Combine(fs.OpenSearchHome, "plugins/opensearch-security");
+
+			if (!Directory.Exists(pluginSecurity))
 				return;
-			}
 
-			var securityInstallDemoConfigSubPath = "plugins/opensearch-security/tools/install_demo_configuration.sh";
-			var securityInstallDemoConfig = Path.Combine(fs.OpenSearchHome, securityInstallDemoConfigSubPath);
+			var isNewDemoScript = cluster.ClusterConfiguration.Version.BaseVersion() >= new Version(2, 12, 0);
+
+			const string securityInstallDemoConfigSubPath = "tools/install_demo_configuration.sh";
+			var securityInstallDemoConfig = Path.Combine(pluginSecurity, securityInstallDemoConfigSubPath);
 
 			cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} going to run [{securityInstallDemoConfigSubPath}]");
 
+			if (File.Exists(installConfigFile) && File.ReadLines(installConfigFile).Any(l => l.Contains("plugins.security"))) return;
+
+			var env = new Dictionary<string, string>();
+			var args = new List<string> { securityInstallDemoConfig, "-y", "-i" };
+
+			if (isNewDemoScript)
+			{
+				env.Add("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "admin");
+				args.Add("-t");
+			}
+
 			ExecuteBinary(
 				cluster.ClusterConfiguration,
 				cluster.Writer,
 				"/bin/bash",
 				"install security plugin demo configuration",
-				securityInstallDemoConfig,
-				"-y", "-i", "-s");
-
-			if (cluster.ClusterConfiguration.EnableSsl) return;
-
-			File.AppendAllText(configFile, "plugins.security.disabled: true");
+				env,
+				args.ToArray());
 		}
 	}
 }

abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs

@@ -27,6 +27,7 @@
 */
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
@@ -93,12 +94,21 @@ public override void Run(IEphemeralCluster<EphemeralClusterConfiguration> cluste
 				cluster.Writer?.WriteDiagnostic(
 					$"{{{nameof(InstallPlugins)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}: {{{plugin.IsValid(v)}}}");
 
-				if (!Directory.Exists(fs.ConfigPath)) Directory.CreateDirectory(fs.ConfigPath);
+				var homeConfigPath = Path.Combine(fs.OpenSearchHome, "config");
+
+				if (!Directory.Exists(homeConfigPath)) Directory.CreateDirectory(homeConfigPath);
+
+				var env = new Dictionary<string, string>
+				{
+					{ fs.ConfigEnvironmentVariableName, homeConfigPath }
+				};
+
 				ExecuteBinary(
 					cluster.ClusterConfiguration,
 					cluster.Writer,
 					fs.PluginBinary,
 					$"install opensearch plugin: {plugin.SubProductName}",
+					env,
 					"install", "--batch", GetPluginLocation(plugin, v));
 
 				CopyConfigDirectoryToHomeCacheConfigDirectory(cluster, plugin);

guides/document-lifecycle.md

@@ -7,7 +7,7 @@ Assuming you have OpenSearch running locally on port 9200, you can create a clie
 var node = new Uri("https://localhost:9200");
 var config = new ConnectionSettings(node)
     .ServerCertificateValidationCallback(CertificateValidations.AllowAll)
-    .BasicAuthentication("admin", "admin")
+    .BasicAuthentication("admin", <admin-password>)
     .DisableDirectStreaming();
 var client = new OpenSearchClient(config);
 

guides/index-template.md

@@ -12,7 +12,7 @@ using OpenSearch.Net;
 var node = new Uri("https://localhost:9200");
 var config = new ConnectionSettings(node)
     .ServerCertificateValidationCallback(CertificateValidations.AllowAll)
-    .BasicAuthentication("admin", "admin");
+    .BasicAuthentication("admin", <admin-password>);
 
 var client = new OpenSearchClient(config);;
 ```

guides/search.md

@@ -12,7 +12,7 @@ var node = new Uri("https://localhost:9200");
 var config = new ConnectionSettings(node)
     .ThrowExceptions()
     .ServerCertificateValidationCallback(CertificateValidations.AllowAll)
-    .BasicAuthentication("admin", "admin");
+    .BasicAuthentication("admin", <admin-password>);
 var client = new OpenSearchClient(config);
 
 class Movie

tests/Tests.YamlRunner/Models.fs

@@ -61,9 +61,9 @@ let (|IsDoCatch|_|) (s:string) =
     | "conflict" -> Some Conflict 
     | "unavailable" -> Some Unavailable
     | "param" -> Some UnknownParameter
-    | "request" -> Some OtherBadResponse 
-    | s -> Some <| CatchRegex (s.Trim('/'))
-    
+    | "request" -> Some OtherBadResponse
+    | s -> Some <| CatchRegex (Regex.Replace(s.Trim('/'), @"(?<!\\)\\_", "_"))
+
 type NodeSelector =
     | NodeVersionSelector of string
     | NodeAttributeSelector of string * string

tests/Tests/Search/SearchTemplate/RenderSearchTemplate/RenderSearchTemplateApiTests.cs

@@ -40,6 +40,7 @@
 
 namespace Tests.Search.SearchTemplate.RenderSearchTemplate
 {
+	[SkipVersion("2.10.*,2.11.*", "Broken by security plugin https://github.com/opensearch-project/security/issues/3672")]
 	public class RenderSearchTemplateApiTests
 		: ApiIntegrationTestBase<ReadOnlyCluster, RenderSearchTemplateResponse, IRenderSearchTemplateRequest, RenderSearchTemplateDescriptor,
 			RenderSearchTemplateRequest>