diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 06b8e4c078..068000e68c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -17,17 +17,14 @@ jobs: fail-fast: false matrix: version: - - 2.9.0 + - 2.11.1 + - 2.10.0 - 2.8.0 - - 2.7.0 - 2.6.0 - - 2.5.0 - 2.4.1 - - 2.3.0 - 2.2.1 - - 2.1.0 - 2.0.1 - - 1.3.11 + - 1.3.14 - 1.2.4 - 1.1.0 @@ -62,7 +59,6 @@ jobs: path: client/build/output/* integration-opensearch-unreleased: - if: false # TODO: Temporarily disabled due to failures building & running OpenSearch from source, pending investigation & fixes (https://github.com/opensearch-project/opensearch-net/issues/268) name: Integration OpenSearch Unreleased runs-on: ubuntu-latest strategy: diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs index bba2455f5f..f0e3769c96 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs @@ -34,6 +34,7 @@ using System.Security.Cryptography; using System.Text; using OpenSearch.OpenSearch.Managed; +using OpenSearch.OpenSearch.Managed.Configuration; using OpenSearch.Stack.ArtifactsApi; namespace OpenSearch.OpenSearch.Ephemeral @@ -59,6 +60,13 @@ protected EphemeralCluster(TConfiguration clusterConfiguration) : base(clusterCo protected EphemeralClusterComposer Composer { get; } + protected override void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port) + { + base.ModifyNodeConfiguration(nodeConfiguration, port); + + if (!ClusterConfiguration.EnableSsl) nodeConfiguration.Add("plugins.security.disabled", "true"); + } + public virtual ICollection NodesUris(string hostName = null) { hostName = hostName ?? (ClusterConfiguration.HttpFiddlerAware && Process.GetProcessesByName("fiddler").Any() diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs index 40b200f28a..74441951ef 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs @@ -172,22 +172,35 @@ protected static void WriteFileIfNotExist(string fileLocation, string contents) protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer, string binary, string description, params string[] arguments) => - ExecuteBinaryInternal(config, writer, binary, description, arguments); + ExecuteBinaryInternal(config, writer, binary, description, null, arguments); + + protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer, + string binary, string description, IDictionary environmentVariables, + params string[] arguments) => + ExecuteBinaryInternal(config, writer, binary, description, environmentVariables, arguments); private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config, IConsoleLineHandler writer, - string binary, string description, params string[] arguments) + string binary, string description, IDictionary environmentVariables, params string[] arguments) { var command = $"{{{binary}}} {{{string.Join(" ", arguments)}}}"; writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} starting process [{description}] {command}"); + var environment = new Dictionary + { + {config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath}, + {"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome} + }; + + if (environmentVariables != null) + { + foreach (var kvp in environmentVariables) + environment[kvp.Key] = kvp.Value; + } + var timeout = TimeSpan.FromSeconds(420); var processStartArguments = new StartArguments(binary, arguments) { - Environment = new Dictionary - { - {config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath}, - {"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome}, - } + Environment = environment }; var result = Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter()); diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs index b938890102..9540c07edb 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs @@ -26,10 +26,12 @@ * under the License. */ +using System.Collections.Generic; using System.IO; using System.Linq; using OpenSearch.OpenSearch.Managed.ConsoleWriters; -using OpenSearch.Stack.ArtifactsApi; +using OpenSearch.Stack.ArtifactsApi.Products; +using SemanticVersioning; namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks { @@ -38,30 +40,39 @@ public class InitialConfiguration : ClusterComposeTask public override void Run(IEphemeralCluster cluster) { var fs = cluster.FileSystem; - var configFile = Path.Combine(fs.OpenSearchHome, "config", "opensearch.yml"); - if (File.Exists(configFile) && File.ReadLines(configFile).Any(l => !string.IsNullOrWhiteSpace(l) && !l.StartsWith("#"))) - { - cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} opensearch.yml already exists, skipping initial configuration"); + var installConfigDir = Path.Combine(fs.OpenSearchHome, "config"); + var installConfigFile = Path.Combine(installConfigDir, "opensearch.yml"); + var pluginSecurity = Path.Combine(fs.OpenSearchHome, "plugins/opensearch-security"); + + if (!Directory.Exists(pluginSecurity)) return; - } - var securityInstallDemoConfigSubPath = "plugins/opensearch-security/tools/install_demo_configuration.sh"; - var securityInstallDemoConfig = Path.Combine(fs.OpenSearchHome, securityInstallDemoConfigSubPath); + var isNewDemoScript = cluster.ClusterConfiguration.Version.BaseVersion() >= new Version(2, 12, 0); + + const string securityInstallDemoConfigSubPath = "tools/install_demo_configuration.sh"; + var securityInstallDemoConfig = Path.Combine(pluginSecurity, securityInstallDemoConfigSubPath); cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} going to run [{securityInstallDemoConfigSubPath}]"); + if (File.Exists(installConfigFile) && File.ReadLines(installConfigFile).Any(l => l.Contains("plugins.security"))) return; + + var env = new Dictionary(); + var args = new List { securityInstallDemoConfig, "-y", "-i" }; + + if (isNewDemoScript) + { + env.Add("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "admin"); + args.Add("-t"); + } + ExecuteBinary( cluster.ClusterConfiguration, cluster.Writer, "/bin/bash", "install security plugin demo configuration", - securityInstallDemoConfig, - "-y", "-i", "-s"); - - if (cluster.ClusterConfiguration.EnableSsl) return; - - File.AppendAllText(configFile, "plugins.security.disabled: true"); + env, + args.ToArray()); } } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs index c9509b7d03..3225476353 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs @@ -27,6 +27,7 @@ */ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; @@ -93,12 +94,21 @@ public override void Run(IEphemeralCluster cluste cluster.Writer?.WriteDiagnostic( $"{{{nameof(InstallPlugins)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}: {{{plugin.IsValid(v)}}}"); - if (!Directory.Exists(fs.ConfigPath)) Directory.CreateDirectory(fs.ConfigPath); + var homeConfigPath = Path.Combine(fs.OpenSearchHome, "config"); + + if (!Directory.Exists(homeConfigPath)) Directory.CreateDirectory(homeConfigPath); + + var env = new Dictionary + { + { fs.ConfigEnvironmentVariableName, homeConfigPath } + }; + ExecuteBinary( cluster.ClusterConfiguration, cluster.Writer, fs.PluginBinary, $"install opensearch plugin: {plugin.SubProductName}", + env, "install", "--batch", GetPluginLocation(plugin, v)); CopyConfigDirectoryToHomeCacheConfigDirectory(cluster, plugin); diff --git a/guides/document-lifecycle.md b/guides/document-lifecycle.md index e0746f7e7c..67cb37588d 100644 --- a/guides/document-lifecycle.md +++ b/guides/document-lifecycle.md @@ -7,7 +7,7 @@ Assuming you have OpenSearch running locally on port 9200, you can create a clie var node = new Uri("https://localhost:9200"); var config = new ConnectionSettings(node) .ServerCertificateValidationCallback(CertificateValidations.AllowAll) - .BasicAuthentication("admin", "admin") + .BasicAuthentication("admin", ) .DisableDirectStreaming(); var client = new OpenSearchClient(config); diff --git a/guides/index-template.md b/guides/index-template.md index 495d873b39..81049f128b 100644 --- a/guides/index-template.md +++ b/guides/index-template.md @@ -12,7 +12,7 @@ using OpenSearch.Net; var node = new Uri("https://localhost:9200"); var config = new ConnectionSettings(node) .ServerCertificateValidationCallback(CertificateValidations.AllowAll) - .BasicAuthentication("admin", "admin"); + .BasicAuthentication("admin", ); var client = new OpenSearchClient(config);; ``` diff --git a/guides/search.md b/guides/search.md index ed8d022a97..e7f00e4bec 100644 --- a/guides/search.md +++ b/guides/search.md @@ -12,7 +12,7 @@ var node = new Uri("https://localhost:9200"); var config = new ConnectionSettings(node) .ThrowExceptions() .ServerCertificateValidationCallback(CertificateValidations.AllowAll) - .BasicAuthentication("admin", "admin"); + .BasicAuthentication("admin", ); var client = new OpenSearchClient(config); class Movie diff --git a/tests/Tests.YamlRunner/Models.fs b/tests/Tests.YamlRunner/Models.fs index 761cdf0269..5d4f372df5 100644 --- a/tests/Tests.YamlRunner/Models.fs +++ b/tests/Tests.YamlRunner/Models.fs @@ -61,9 +61,9 @@ let (|IsDoCatch|_|) (s:string) = | "conflict" -> Some Conflict | "unavailable" -> Some Unavailable | "param" -> Some UnknownParameter - | "request" -> Some OtherBadResponse - | s -> Some <| CatchRegex (s.Trim('/')) - + | "request" -> Some OtherBadResponse + | s -> Some <| CatchRegex (Regex.Replace(s.Trim('/'), @"(?