diff --git a/.github/actions/run-released-opensearch/action.yml b/.github/actions/run-released-opensearch/action.yml index f230940ffe..c87f63e341 100644 --- a/.github/actions/run-released-opensearch/action.yml +++ b/.github/actions/run-released-opensearch/action.yml @@ -11,6 +11,9 @@ outputs: opensearch_url: description: The URL where the OpenSearch node is accessible value: ${{ steps.opensearch.outputs.opensearch_url }} + admin_password: + description: The initial admin password + value: ${{ steps.opensearch.outputs.admin_password }} runs: using: composite steps: diff --git a/.github/actions/start-opensearch/action.yml b/.github/actions/start-opensearch/action.yml index a9223013d7..8e4eb154e5 100644 --- a/.github/actions/start-opensearch/action.yml +++ b/.github/actions/start-opensearch/action.yml @@ -8,6 +8,9 @@ outputs: opensearch_url: description: The URL where the OpenSearch node is accessible value: ${{ steps.opensearch.outputs.url }} + admin_password: + description: The initial admin password + value: ${{ steps.opensearch.outputs.password }} runs: using: composite steps: @@ -24,30 +27,49 @@ runs: if [[ "$RUNNER_OS" == "macOS" ]]; then brew install -q coreutils fi - OPENSEARCH_HOME=$(realpath ./opensearch-*) + OPENSEARCH_HOME=$(realpath ./opensearch-[1-9]*) + CONFIG_DIR=$OPENSEARCH_HOME/config + CONFIG_FILE=$CONFIG_DIR/opensearch.yml + SECURITY_DIR=$OPENSEARCH_HOME/plugins/opensearch-security OPENSEARCH_JAVA_OPTS="-Djava.net.preferIPv4Stack=true" - url="http://localhost:9200" - cp ./client/.ci/opensearch/opensearch.yml $OPENSEARCH_HOME/config/ + URL="http://localhost:9200" + cp ./client/.ci/opensearch/opensearch.yml $CONFIG_FILE bash ./client/.ci/generate-certs.sh + + export OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin - if [[ -d "$OPENSEARCH_HOME/plugins/opensearch-security" ]]; then + if [[ -d "$SECURITY_DIR" ]]; then if [[ "$SECURED" == "true" ]]; then - bash $OPENSEARCH_HOME/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s - sed -i.bak -e 's/plugins.security.audit.type:.*/plugins.security.audit.type: log4j/' $OPENSEARCH_HOME/config/opensearch.yml - cp ./client/.ci/opensearch/*.pem $OPENSEARCH_HOME/config/ - url="https://localhost:9200" + SECURITY_VERSION=$(cat $SECURITY_DIR/plugin-descriptor.properties | grep '^version=' | cut -d'=' -f 2) + SECURITY_VERSION_COMPONENTS=(${SECURITY_VERSION//./ }) + SECURITY_MAJOR="${SECURITY_VERSION_COMPONENTS[0]}" + SECURITY_MINOR="${SECURITY_VERSION_COMPONENTS[1]}" + + if (( $SECURITY_MAJOR > 2 || ( $SECURITY_MAJOR == 2 && $SECURITY_MINOR >= 12 ) )); then + export OPENSEARCH_INITIAL_ADMIN_PASSWORD=$(LC_ALL=C tr -dc A-Za-z0-9 > $OPENSEARCH_HOME/config/opensearch.yml + printf "\nplugins.security.disabled: true" >> $CONFIG_FILE fi fi if [[ "$RUNNER_OS" == "macOS" ]]; then - sed -i.bak -e 's/bootstrap.memory_lock:.*/bootstrap.memory_lock: false/' $OPENSEARCH_HOME/config/opensearch.yml + sed -i.bak -e 's/bootstrap.memory_lock:.*/bootstrap.memory_lock: false/' $CONFIG_FILE fi - echo "url=$url" >> $GITHUB_OUTPUT + { + echo "url=$URL" + echo "password=$OPENSEARCH_INITIAL_ADMIN_PASSWORD" + } | tee -a $GITHUB_OUTPUT + if [[ "$RUNNER_OS" == "Linux" ]]; then sudo swapoff -a sudo sysctl -w vm.swappiness=1 @@ -64,7 +86,7 @@ runs: for attempt in {1..20}; do sleep 5 - if curl -k -sS --cacert ./client/.ci/certs/root-ca.crt -u admin:admin $url; then + if curl -k -sS --cacert ./client/.ci/certs/root-ca.crt -u admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD} $URL; then echo '=====> ready' exit 0 fi diff --git a/.github/workflows/integration-yaml-tests.yml b/.github/workflows/integration-yaml-tests.yml index 9a44c4bb25..b8799a62fe 100644 --- a/.github/workflows/integration-yaml-tests.yml +++ b/.github/workflows/integration-yaml-tests.yml @@ -13,7 +13,17 @@ jobs: strategy: fail-fast: false matrix: - version: [ '1.2.4', '1.3.11', '2.2.0', '2.4.0', '2.6.0', '2.8.0', '2.9.0' ] + version: + - 2.11.1 + - 2.10.0 + - 2.8.0 + - 2.6.0 + - 2.4.1 + - 2.2.1 + - 2.0.1 + - 1.3.14 + - 1.2.4 + - 1.1.0 steps: - name: Checkout Client uses: actions/checkout@v3 @@ -99,13 +109,18 @@ jobs: uses: ./client/.github/actions/build-opensearch with: ref: ${{ matrix.opensearch_ref }} + security_plugin: true - name: Unpack OpenSearch - run: tar -xzf ${{ steps.opensearch_build.outputs.distribution }} + run: | + tar -xzf ${{ steps.opensearch_build.outputs.distribution }} \ + && ./opensearch-*/bin/opensearch-plugin install --batch file://$(realpath ./opensearch-security/build/distributions/opensearch-security-*-SNAPSHOT.zip) - name: Start OpenSearch id: opensearch uses: ./client/.github/actions/start-opensearch + with: + secured: true - name: Run YAML tests working-directory: client @@ -114,9 +129,12 @@ jobs: --project ./tests/Tests.YamlRunner/Tests.YamlRunner.fsproj \ -- \ --endpoint $OPENSEARCH_URL \ + --auth-cert ./.ci/certs/kirk.p12 \ + --auth-cert-pass kirk \ --junit-output-file ./test-results.xml env: OPENSEARCH_URL: ${{ steps.opensearch.outputs.opensearch_url }} + ADMIN_PASS: ${{ steps.opensearch.outputs.admin_password }} - name: Save OpenSearch logs if: failure() diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f102c1535a..3b5c888500 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -17,17 +17,14 @@ jobs: fail-fast: false matrix: version: - - 2.9.0 + - 2.11.1 + - 2.10.0 - 2.8.0 - - 2.7.0 - 2.6.0 - - 2.5.0 - 2.4.1 - - 2.3.0 - 2.2.1 - - 2.1.0 - 2.0.1 - - 1.3.11 + - 1.3.14 - 1.2.4 - 1.1.0 @@ -62,7 +59,6 @@ jobs: path: client/build/output/* integration-opensearch-unreleased: - if: false # TODO: Temporarily disabled due to failures building & running OpenSearch from source, pending investigation & fixes (https://github.com/opensearch-project/opensearch-net/issues/268) name: Integration OpenSearch Unreleased runs-on: ubuntu-latest strategy: diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs index bba2455f5f..f0e3769c96 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs @@ -34,6 +34,7 @@ using System.Security.Cryptography; using System.Text; using OpenSearch.OpenSearch.Managed; +using OpenSearch.OpenSearch.Managed.Configuration; using OpenSearch.Stack.ArtifactsApi; namespace OpenSearch.OpenSearch.Ephemeral @@ -59,6 +60,13 @@ protected EphemeralCluster(TConfiguration clusterConfiguration) : base(clusterCo protected EphemeralClusterComposer Composer { get; } + protected override void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port) + { + base.ModifyNodeConfiguration(nodeConfiguration, port); + + if (!ClusterConfiguration.EnableSsl) nodeConfiguration.Add("plugins.security.disabled", "true"); + } + public virtual ICollection NodesUris(string hostName = null) { hostName = hostName ?? (ClusterConfiguration.HttpFiddlerAware && Process.GetProcessesByName("fiddler").Any() diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs index 40b200f28a..74441951ef 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs @@ -172,22 +172,35 @@ protected static void WriteFileIfNotExist(string fileLocation, string contents) protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer, string binary, string description, params string[] arguments) => - ExecuteBinaryInternal(config, writer, binary, description, arguments); + ExecuteBinaryInternal(config, writer, binary, description, null, arguments); + + protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer, + string binary, string description, IDictionary environmentVariables, + params string[] arguments) => + ExecuteBinaryInternal(config, writer, binary, description, environmentVariables, arguments); private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config, IConsoleLineHandler writer, - string binary, string description, params string[] arguments) + string binary, string description, IDictionary environmentVariables, params string[] arguments) { var command = $"{{{binary}}} {{{string.Join(" ", arguments)}}}"; writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} starting process [{description}] {command}"); + var environment = new Dictionary + { + {config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath}, + {"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome} + }; + + if (environmentVariables != null) + { + foreach (var kvp in environmentVariables) + environment[kvp.Key] = kvp.Value; + } + var timeout = TimeSpan.FromSeconds(420); var processStartArguments = new StartArguments(binary, arguments) { - Environment = new Dictionary - { - {config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath}, - {"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome}, - } + Environment = environment }; var result = Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter()); diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs index b938890102..9540c07edb 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs @@ -26,10 +26,12 @@ * under the License. */ +using System.Collections.Generic; using System.IO; using System.Linq; using OpenSearch.OpenSearch.Managed.ConsoleWriters; -using OpenSearch.Stack.ArtifactsApi; +using OpenSearch.Stack.ArtifactsApi.Products; +using SemanticVersioning; namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks { @@ -38,30 +40,39 @@ public class InitialConfiguration : ClusterComposeTask public override void Run(IEphemeralCluster cluster) { var fs = cluster.FileSystem; - var configFile = Path.Combine(fs.OpenSearchHome, "config", "opensearch.yml"); - if (File.Exists(configFile) && File.ReadLines(configFile).Any(l => !string.IsNullOrWhiteSpace(l) && !l.StartsWith("#"))) - { - cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} opensearch.yml already exists, skipping initial configuration"); + var installConfigDir = Path.Combine(fs.OpenSearchHome, "config"); + var installConfigFile = Path.Combine(installConfigDir, "opensearch.yml"); + var pluginSecurity = Path.Combine(fs.OpenSearchHome, "plugins/opensearch-security"); + + if (!Directory.Exists(pluginSecurity)) return; - } - var securityInstallDemoConfigSubPath = "plugins/opensearch-security/tools/install_demo_configuration.sh"; - var securityInstallDemoConfig = Path.Combine(fs.OpenSearchHome, securityInstallDemoConfigSubPath); + var isNewDemoScript = cluster.ClusterConfiguration.Version.BaseVersion() >= new Version(2, 12, 0); + + const string securityInstallDemoConfigSubPath = "tools/install_demo_configuration.sh"; + var securityInstallDemoConfig = Path.Combine(pluginSecurity, securityInstallDemoConfigSubPath); cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} going to run [{securityInstallDemoConfigSubPath}]"); + if (File.Exists(installConfigFile) && File.ReadLines(installConfigFile).Any(l => l.Contains("plugins.security"))) return; + + var env = new Dictionary(); + var args = new List { securityInstallDemoConfig, "-y", "-i" }; + + if (isNewDemoScript) + { + env.Add("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "admin"); + args.Add("-t"); + } + ExecuteBinary( cluster.ClusterConfiguration, cluster.Writer, "/bin/bash", "install security plugin demo configuration", - securityInstallDemoConfig, - "-y", "-i", "-s"); - - if (cluster.ClusterConfiguration.EnableSsl) return; - - File.AppendAllText(configFile, "plugins.security.disabled: true"); + env, + args.ToArray()); } } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs index c9509b7d03..3225476353 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs @@ -27,6 +27,7 @@ */ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; @@ -93,12 +94,21 @@ public override void Run(IEphemeralCluster cluste cluster.Writer?.WriteDiagnostic( $"{{{nameof(InstallPlugins)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}: {{{plugin.IsValid(v)}}}"); - if (!Directory.Exists(fs.ConfigPath)) Directory.CreateDirectory(fs.ConfigPath); + var homeConfigPath = Path.Combine(fs.OpenSearchHome, "config"); + + if (!Directory.Exists(homeConfigPath)) Directory.CreateDirectory(homeConfigPath); + + var env = new Dictionary + { + { fs.ConfigEnvironmentVariableName, homeConfigPath } + }; + ExecuteBinary( cluster.ClusterConfiguration, cluster.Writer, fs.PluginBinary, $"install opensearch plugin: {plugin.SubProductName}", + env, "install", "--batch", GetPluginLocation(plugin, v)); CopyConfigDirectoryToHomeCacheConfigDirectory(cluster, plugin); diff --git a/guides/document-lifecycle.md b/guides/document-lifecycle.md index e0746f7e7c..67cb37588d 100644 --- a/guides/document-lifecycle.md +++ b/guides/document-lifecycle.md @@ -7,7 +7,7 @@ Assuming you have OpenSearch running locally on port 9200, you can create a clie var node = new Uri("https://localhost:9200"); var config = new ConnectionSettings(node) .ServerCertificateValidationCallback(CertificateValidations.AllowAll) - .BasicAuthentication("admin", "admin") + .BasicAuthentication("admin", ) .DisableDirectStreaming(); var client = new OpenSearchClient(config); diff --git a/guides/index-template.md b/guides/index-template.md index 495d873b39..81049f128b 100644 --- a/guides/index-template.md +++ b/guides/index-template.md @@ -12,7 +12,7 @@ using OpenSearch.Net; var node = new Uri("https://localhost:9200"); var config = new ConnectionSettings(node) .ServerCertificateValidationCallback(CertificateValidations.AllowAll) - .BasicAuthentication("admin", "admin"); + .BasicAuthentication("admin", ); var client = new OpenSearchClient(config);; ``` diff --git a/guides/search.md b/guides/search.md index ed8d022a97..e7f00e4bec 100644 --- a/guides/search.md +++ b/guides/search.md @@ -12,7 +12,7 @@ var node = new Uri("https://localhost:9200"); var config = new ConnectionSettings(node) .ThrowExceptions() .ServerCertificateValidationCallback(CertificateValidations.AllowAll) - .BasicAuthentication("admin", "admin"); + .BasicAuthentication("admin", ); var client = new OpenSearchClient(config); class Movie diff --git a/tests/Tests.YamlRunner/Models.fs b/tests/Tests.YamlRunner/Models.fs index 761cdf0269..5d4f372df5 100644 --- a/tests/Tests.YamlRunner/Models.fs +++ b/tests/Tests.YamlRunner/Models.fs @@ -61,9 +61,9 @@ let (|IsDoCatch|_|) (s:string) = | "conflict" -> Some Conflict | "unavailable" -> Some Unavailable | "param" -> Some UnknownParameter - | "request" -> Some OtherBadResponse - | s -> Some <| CatchRegex (s.Trim('/')) - + | "request" -> Some OtherBadResponse + | s -> Some <| CatchRegex (Regex.Replace(s.Trim('/'), @"(?