diff --git a/.ci/certs/openssl.conf b/.ci/certs/openssl.conf new file mode 100644 index 0000000000..023ed99385 --- /dev/null +++ b/.ci/certs/openssl.conf @@ -0,0 +1,33 @@ +[req] +distinguished_name = req_distinguished_name + +[req_distinguished_name] + +[root-ca] +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always +basicConstraints = critical, CA:TRUE +keyUsage = critical, digitalSignature, keyCertSign, cRLSign + +[esnode] +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always, issuer:always +basicConstraints = critical, CA:FALSE +keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment +extendedKeyUsage = critical, serverAuth, clientAuth +subjectAltName = @esnode-san + +[esnode-san] +DNS.1 = localhost +DNS.2 = instance +DNS.3 = instance1 +DNS.4 = instance2 +IP.1 = 127.0.0.1 +IP.2 = 0:0:0:0:0:0:0:1 + +[kirk] +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always, issuer:always +basicConstraints = critical, CA:FALSE +keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment +extendedKeyUsage = critical, clientAuth diff --git a/.ci/generate-certs.sh b/.ci/generate-certs.sh new file mode 100644 index 0000000000..ce19a54637 --- /dev/null +++ b/.ci/generate-certs.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +set -eo pipefail + +script_path=$(dirname $(realpath -s $0)) +certs_dir="$script_path/certs" +opensearch_dir="$script_path/opensearch" + +openssl_conf="$certs_dir/openssl.conf" + +root_ca_key="$certs_dir/root-ca.key" +root_ca_crt="$certs_dir/root-ca.crt" +root_ca_pem="$opensearch_dir/root-ca.pem" + +esnode_key="$certs_dir/esnode.key" +esnode_key_pem="$opensearch_dir/esnode-key.pem" +esnode_csr="$certs_dir/esnode.csr" +esnode_crt="$certs_dir/esnode.crt" +esnode_pem="$opensearch_dir/esnode.pem" + +kirk_key="$certs_dir/kirk.key" +kirk_csr="$certs_dir/kirk.csr" +kirk_crt="$certs_dir/kirk.crt" +kirk_p12="$certs_dir/kirk.p12" + +common_crt_args="-extfile $openssl_conf -days 36500 -CA $root_ca_crt -CAkey $root_ca_key -CAcreateserial" +common_csr_args="-config $openssl_conf -days 36500" + +# Stop Git Bash / MSYS / Cygwin from mangling the cert subjects +subj_prefix="" +if [[ "$(uname)" == MINGW* ]]; then + subj_prefix="/" +fi + +if [[ ! -f $root_ca_key ]]; then + rm -f $root_ca_crt + openssl genrsa -out $root_ca_key +fi + +if [[ ! -f $root_ca_crt ]]; then + rm -f *.crt $root_ca_pem + openssl req -new -x509 \ + -key $root_ca_key \ + -subj "$subj_prefix/DC=com/DC=example/O=Example Com Inc./OU=Example Com Inc. Root CA/CN=Example Com Inc. Root CA" \ + $common_csr_args -extensions root-ca \ + -out $root_ca_crt +fi + +if [[ ! -f $root_ca_pem ]]; then + cp $root_ca_crt $root_ca_pem +fi + +if [[ ! -f $esnode_key ]]; then + rm -f $esnode_csr $esnode_key_pem + openssl genrsa -out $esnode_key +fi + +if [[ ! -f $esnode_key_pem ]]; then + openssl pkcs8 -topk8 -in $esnode_key -nocrypt -out $esnode_key_pem +fi + +if [[ ! -f $esnode_csr ]]; then + rm -f $esnode_crt + openssl req -new \ + $common_csr_args \ + -key $esnode_key \ + -subj "$subj_prefix/DC=de/L=test/O=node/OU=node/CN=node-0.example.com" \ + -out $esnode_csr +fi + +if [[ ! -f $esnode_crt ]]; then + rm -f $esnode_pem + openssl x509 -req -in $esnode_csr $common_crt_args -extensions esnode -out $esnode_crt +fi + +if [[ ! -f $esnode_pem ]]; then + cp $esnode_crt $esnode_pem +fi + +if [[ ! -f $kirk_key ]]; then + rm -f $kirk_csr + openssl genrsa -out $kirk_key +fi + +if [[ ! -f $kirk_csr ]]; then + rm -f $kirk_crt + openssl req -new \ + $common_csr_args \ + -subj "$subj_prefix/C=de/L=test/O=client/OU=client/CN=kirk" \ + -key $kirk_key \ + -out $kirk_csr +fi + +if [[ ! -f $kirk_crt ]]; then + rm -f $kirk_p12 + openssl x509 -req -in $kirk_csr $common_crt_args -extensions kirk -out $kirk_crt +fi + +if [[ ! -f $kirk_p12 ]]; then + openssl pkcs12 -export \ + -in $kirk_crt \ + -inkey $kirk_key \ + -descert \ + -passout pass:kirk \ + -out $kirk_p12 +fi diff --git a/.ci/opensearch/opensearch.yml b/.ci/opensearch/opensearch.yml new file mode 100644 index 0000000000..626ef4816c --- /dev/null +++ b/.ci/opensearch/opensearch.yml @@ -0,0 +1,11 @@ +network.host: 0.0.0.0 +node.name: instance +cluster.name: search-rest-test +cluster.initial_master_nodes: instance +discovery.seed_hosts: instance +cluster.routing.allocation.disk.threshold_enabled: false +bootstrap.memory_lock: true +node.attr.testattr: test +path.repo: /tmp +repositories.url.allowed_urls: http://snapshot.test* +action.destructive_requires_name: false diff --git a/.github/actions/build-opensearch/action.yml b/.github/actions/build-opensearch/action.yml new file mode 100644 index 0000000000..8ad36ee266 --- /dev/null +++ b/.github/actions/build-opensearch/action.yml @@ -0,0 +1,103 @@ +name: Restore or Build OpenSearch +description: Restore or Build OpenSearch from source +inputs: + ref: + description: The git ref to clone + build_snapshot: + description: Whether to build a snapshot version + default: "true" + security_plugin: + description: Whether to build the security plugin + default: "false" + knn_plugin: + description: Whether to build the k-nn plugin + default: "false" + plugins_output_directory: + description: The directory to output the plugins to + default: "" +outputs: + distribution: + description: The path to the OpenSearch distribution + value: ${{ steps.determine.outputs.distribution }} + version: + description: The version of OpenSearch + value: ${{ steps.determine.outputs.version }} +runs: + using: composite + steps: + - name: Restore or Build OpenSearch + uses: ./client/.github/actions/cached-git-build + with: + repository: opensearch-project/OpenSearch + ref: ${{ inputs.ref }} + path: opensearch + cache_key_suffix: ${{ inputs.build_snapshot == 'true' && '-snapshot' || '' }} + cached_paths: | + ./opensearch/distribution/archives/linux-tar/build/distributions/opensearch-*.tar.gz + ./opensearch/plugins/*/build/distributions/*.zip + build_script: | + ./gradlew :distribution:archives:linux-tar:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }} + + PluginList=("analysis-icu" "analysis-kuromoji" "analysis-nori" "analysis-phonetic" "ingest-attachment" "mapper-murmur3") + for plugin in ${PluginList[*]}; do + ./gradlew :plugins:$plugin:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }} + done + + - name: Determine OpenSearch distribution path and version + id: determine + shell: bash -eo pipefail {0} + run: | + distribution=`ls -1 $PWD/opensearch/distribution/archives/linux-tar/build/distributions/opensearch-*.tar.gz | head -1` + version=`basename $distribution | cut -d'-' -f3,${{ inputs.build_snapshot == 'true' && 4 || 3 }}` + echo "distribution=$distribution" | tee -a $GITHUB_OUTPUT + echo "version=$version" | tee -a $GITHUB_OUTPUT + + - name: Restore or Build OpenSearch Security + uses: ./client/.github/actions/cached-git-build + if: inputs.security_plugin == 'true' + with: + repository: opensearch-project/security + ref: ${{ inputs.ref }} + path: opensearch-security + cache_key_suffix: ${{ inputs.build_snapshot == 'true' && '-snapshot' || '' }} + cached_paths: | + ./opensearch-security/build/distributions/opensearch-security-*.zip + build_script: ./gradlew assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }} + + - name: Restore or Build OpenSearch k-NN + uses: ./client/.github/actions/cached-git-build + if: inputs.knn_plugin == 'true' + with: + repository: opensearch-project/k-NN + ref: ${{ inputs.ref }} + path: opensearch-knn + cache_key_suffix: ${{ inputs.build_snapshot == 'true' && '-snapshot' || '' }} + cached_paths: | + ./opensearch-knn/build/distributions/opensearch-knn-*.zip + build_script: | + sudo apt-get install -y libopenblas-dev libomp-dev + ./gradlew buildJniLib assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }} + distributions=./build/distributions + lib_dir=$distributions/lib + mkdir $lib_dir + cp -v $(ldconfig -p | grep libgomp | cut -d ' ' -f 4) $lib_dir + cp -v ./jni/release/libopensearchknn_* $lib_dir + ls -l $lib_dir + cd $distributions + zip -ur opensearch-knn-*.zip lib + + - name: Copy OpenSearch plugins + shell: bash -eo pipefail {0} + if: inputs.plugins_output_directory != '' + run: | + mkdir -p ${{ inputs.plugins_output_directory }} + cp -v ./opensearch/plugins/*/build/distributions/*.zip ${{ inputs.plugins_output_directory }}/ + + plugins=("opensearch-knn" "opensearch-security") + for plugin in ${plugins[*]}; do + if [[ -d "./$plugin" ]]; then + cp -v ./$plugin/build/distributions/$plugin-*.zip ${{ inputs.plugins_output_directory }}/ + fi + done + + ls -l ${{ inputs.plugins_output_directory }} diff --git a/.github/actions/cached-git-build/action.yml b/.github/actions/cached-git-build/action.yml index b5e427d811..f9df08043d 100644 --- a/.github/actions/cached-git-build/action.yml +++ b/.github/actions/cached-git-build/action.yml @@ -11,6 +11,9 @@ inputs: description: A list of paths to cache build_script: description: The script to run to build the repository + cache_key_suffix: + description: A suffix to append to the cache key + default: '' runs: using: composite steps: @@ -32,7 +35,7 @@ runs: uses: actions/cache/restore@v3 with: path: ${{ inputs.cached_paths }} - key: ${{ inputs.repository }}-${{ steps.get-sha.outputs.sha }} + key: ${{ inputs.repository }}-${{ steps.get-sha.outputs.sha }}${{ inputs.cache_key_suffix }} - name: Build if: steps.restore.outputs.cache-hit != 'true' @@ -45,4 +48,4 @@ runs: uses: actions/cache/save@v3 with: path: ${{ inputs.cached_paths }} - key: ${{ inputs.repository }}-${{ steps.get-sha.outputs.sha }} + key: ${{ inputs.repository }}-${{ steps.get-sha.outputs.sha }}${{ inputs.cache_key_suffix }} diff --git a/.github/actions/run-released-opensearch/action.yml b/.github/actions/run-released-opensearch/action.yml new file mode 100644 index 0000000000..c87f63e341 --- /dev/null +++ b/.github/actions/run-released-opensearch/action.yml @@ -0,0 +1,52 @@ +name: Run OpenSearch +description: Runs a released version of OpenSearch +inputs: + version: + description: The version of OpenSearch to run + required: true + secured: + description: Whether to enable the security plugin + required: true +outputs: + opensearch_url: + description: The URL where the OpenSearch node is accessible + value: ${{ steps.opensearch.outputs.opensearch_url }} + admin_password: + description: The initial admin password + value: ${{ steps.opensearch.outputs.admin_password }} +runs: + using: composite + steps: + - name: Restore cached OpenSearch distro + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: opensearch-* + key: opensearch-${{ inputs.version }}-${{ runner.os }} + + - name: Download OpenSearch + if: steps.cache-restore.outputs.cache-hit != 'true' + shell: bash -eo pipefail {0} + run: | + if [[ "$RUNNER_OS" != "Windows" ]]; then + curl -sSLO https://artifacts.opensearch.org/releases/bundle/opensearch/${{ inputs.version }}/opensearch-${{ inputs.version }}-linux-x64.tar.gz + tar -xzf opensearch-*.tar.gz + rm -f opensearch-*.tar.gz + else + curl -sSLO https://artifacts.opensearch.org/releases/bundle/opensearch/${{ inputs.version }}/opensearch-${{ inputs.version }}-windows-x64.zip + unzip opensearch-*.zip + rm -f opensearch-*.zip + fi + + - name: Save cached OpenSearch distro + if: steps.cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: opensearch-* + key: opensearch-${{ inputs.version }}-${{ runner.os }} + + - name: Start OpenSearch + id: opensearch + uses: ./client/.github/actions/start-opensearch + with: + secured: ${{ inputs.secured }} diff --git a/.github/actions/start-opensearch/action.yml b/.github/actions/start-opensearch/action.yml new file mode 100644 index 0000000000..8e4eb154e5 --- /dev/null +++ b/.github/actions/start-opensearch/action.yml @@ -0,0 +1,98 @@ +name: Start OpenSearch +description: Configures and starts an OpenSearch daemon +inputs: + secured: + description: Whether to enable the security plugin + default: 'false' +outputs: + opensearch_url: + description: The URL where the OpenSearch node is accessible + value: ${{ steps.opensearch.outputs.url }} + admin_password: + description: The initial admin password + value: ${{ steps.opensearch.outputs.password }} +runs: + using: composite + steps: + - name: Install Java + uses: actions/setup-java@v3 + with: + distribution: zulu + java-version: 11 + + - name: Start OpenSearch + id: opensearch + shell: bash -eo pipefail {0} + run: | + if [[ "$RUNNER_OS" == "macOS" ]]; then + brew install -q coreutils + fi + OPENSEARCH_HOME=$(realpath ./opensearch-[1-9]*) + CONFIG_DIR=$OPENSEARCH_HOME/config + CONFIG_FILE=$CONFIG_DIR/opensearch.yml + SECURITY_DIR=$OPENSEARCH_HOME/plugins/opensearch-security + OPENSEARCH_JAVA_OPTS="-Djava.net.preferIPv4Stack=true" + + URL="http://localhost:9200" + cp ./client/.ci/opensearch/opensearch.yml $CONFIG_FILE + + bash ./client/.ci/generate-certs.sh + + export OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin + + if [[ -d "$SECURITY_DIR" ]]; then + if [[ "$SECURED" == "true" ]]; then + SECURITY_VERSION=$(cat $SECURITY_DIR/plugin-descriptor.properties | grep '^version=' | cut -d'=' -f 2) + SECURITY_VERSION_COMPONENTS=(${SECURITY_VERSION//./ }) + SECURITY_MAJOR="${SECURITY_VERSION_COMPONENTS[0]}" + SECURITY_MINOR="${SECURITY_VERSION_COMPONENTS[1]}" + + if (( $SECURITY_MAJOR > 2 || ( $SECURITY_MAJOR == 2 && $SECURITY_MINOR >= 12 ) )); then + export OPENSEARCH_INITIAL_ADMIN_PASSWORD=$(LC_ALL=C tr -dc A-Za-z0-9 > $CONFIG_FILE + fi + fi + + if [[ "$RUNNER_OS" == "macOS" ]]; then + sed -i.bak -e 's/bootstrap.memory_lock:.*/bootstrap.memory_lock: false/' $CONFIG_FILE + fi + + { + echo "url=$URL" + echo "password=$OPENSEARCH_INITIAL_ADMIN_PASSWORD" + } | tee -a $GITHUB_OUTPUT + + if [[ "$RUNNER_OS" == "Linux" ]]; then + sudo swapoff -a + sudo sysctl -w vm.swappiness=1 + sudo sysctl -w fs.file-max=262144 + sudo sysctl -w vm.max_map_count=262144 + sudo prlimit --pid $$ --memlock=unlimited:unlimited + fi + + if [[ "$RUNNER_OS" != "Windows" ]]; then + $OPENSEARCH_HOME/bin/opensearch & + else + $OPENSEARCH_HOME/bin/opensearch.bat -d & + fi + + for attempt in {1..20}; do + sleep 5 + if curl -k -sS --cacert ./client/.ci/certs/root-ca.crt -u admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD} $URL; then + echo '=====> ready' + exit 0 + fi + echo '=====> waiting...' + done + exit 1 + env: + SECURED: ${{ inputs.secured }} + RUNNER_OS: ${{ runner.os }} diff --git a/.github/workflows/integration-yaml-tests.yml b/.github/workflows/integration-yaml-tests.yml new file mode 100644 index 0000000000..b8799a62fe --- /dev/null +++ b/.github/workflows/integration-yaml-tests.yml @@ -0,0 +1,152 @@ +name: YAML Tests + +on: + push: + branches-ignore: + - 'dependabot/**' + pull_request: {} + +jobs: + test-yaml: + name: YAML Tests (Released OpenSearch) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + version: + - 2.11.1 + - 2.10.0 + - 2.8.0 + - 2.6.0 + - 2.4.1 + - 2.2.1 + - 2.0.1 + - 1.3.14 + - 1.2.4 + - 1.1.0 + steps: + - name: Checkout Client + uses: actions/checkout@v3 + with: + path: client + + - uses: actions/setup-dotnet@v3 + with: + dotnet-version: | + 5.0.x + 6.0.x + + - uses: actions/cache@v3 + with: + path: ~/.nuget/packages + key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.?sproj') }} + restore-keys: | + ${{ runner.os }}-nuget- + + - name: Run OpenSearch + id: opensearch + uses: ./client/.github/actions/run-released-opensearch + with: + version: ${{ matrix.version }} + secured: true + + - name: Run YAML tests + working-directory: client + run: | + dotnet run \ + --project ./tests/Tests.YamlRunner/Tests.YamlRunner.fsproj \ + -- \ + --endpoint $OPENSEARCH_URL \ + --auth-cert ./.ci/certs/kirk.p12 \ + --auth-cert-pass kirk \ + --junit-output-file ./test-results.xml + env: + OPENSEARCH_URL: ${{ steps.opensearch.outputs.opensearch_url }} + + - name: Save OpenSearch logs + if: failure() + uses: actions/upload-artifact@v3 + with: + name: opensearch-logs-${{ matrix.version }} + path: | + opensearch-*/logs/* + + - name: Upload test report + if: failure() + uses: actions/upload-artifact@v3 + with: + name: report-yaml-${{ matrix.version }} + path: client/test-results.xml + + test-yaml-unreleased: + name: YAML Tests (Unreleased OpenSearch) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + opensearch_ref: ['1.x', '2.x', 'main'] + steps: + - name: Checkout Client + uses: actions/checkout@v3 + with: + path: client + + - uses: actions/setup-dotnet@v3 + with: + dotnet-version: | + 5.0.x + 6.0.x + + - uses: actions/cache@v3 + with: + path: ~/.nuget/packages + key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.?sproj') }} + restore-keys: | + ${{ runner.os }}-nuget- + + - name: Restore or Build OpenSearch + id: opensearch_build + uses: ./client/.github/actions/build-opensearch + with: + ref: ${{ matrix.opensearch_ref }} + security_plugin: true + + - name: Unpack OpenSearch + run: | + tar -xzf ${{ steps.opensearch_build.outputs.distribution }} \ + && ./opensearch-*/bin/opensearch-plugin install --batch file://$(realpath ./opensearch-security/build/distributions/opensearch-security-*-SNAPSHOT.zip) + + - name: Start OpenSearch + id: opensearch + uses: ./client/.github/actions/start-opensearch + with: + secured: true + + - name: Run YAML tests + working-directory: client + run: | + dotnet run \ + --project ./tests/Tests.YamlRunner/Tests.YamlRunner.fsproj \ + -- \ + --endpoint $OPENSEARCH_URL \ + --auth-cert ./.ci/certs/kirk.p12 \ + --auth-cert-pass kirk \ + --junit-output-file ./test-results.xml + env: + OPENSEARCH_URL: ${{ steps.opensearch.outputs.opensearch_url }} + ADMIN_PASS: ${{ steps.opensearch.outputs.admin_password }} + + - name: Save OpenSearch logs + if: failure() + uses: actions/upload-artifact@v3 + with: + name: opensearch-logs-${{ matrix.opensearch_ref }} + path: | + opensearch-*/logs/* + + - name: Upload test report + if: failure() + uses: actions/upload-artifact@v3 + with: + name: report-yaml-unreleased-${{ matrix.opensearch_ref }} + path: client/test-results.xml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 06b8e4c078..3b5c888500 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -17,17 +17,14 @@ jobs: fail-fast: false matrix: version: - - 2.9.0 + - 2.11.1 + - 2.10.0 - 2.8.0 - - 2.7.0 - 2.6.0 - - 2.5.0 - 2.4.1 - - 2.3.0 - 2.2.1 - - 2.1.0 - 2.0.1 - - 1.3.11 + - 1.3.14 - 1.2.4 - 1.1.0 @@ -62,7 +59,6 @@ jobs: path: client/build/output/* integration-opensearch-unreleased: - if: false # TODO: Temporarily disabled due to failures building & running OpenSearch from source, pending investigation & fixes (https://github.com/opensearch-project/opensearch-net/issues/268) name: Integration OpenSearch Unreleased runs-on: ubuntu-latest strategy: @@ -91,6 +87,7 @@ jobs: key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.?sproj') }} restore-keys: | ${{ runner.os }}-nuget- + - name: Restore or Build OpenSearch id: opensearch uses: ./client/.github/actions/build-opensearch diff --git a/.github/workflows/test-jobs.yml b/.github/workflows/test-jobs.yml index 53a53ec24a..7304409562 100644 --- a/.github/workflows/test-jobs.yml +++ b/.github/workflows/test-jobs.yml @@ -39,17 +39,16 @@ jobs: name: Test - name: Test Results if: always() - uses: mikepenz/action-junit-report@v2 + uses: mikepenz/action-junit-report@v3 with: report_paths: 'build/output/junit-*.xml' github_token: ${{ secrets.GITHUB_TOKEN }} fail_on_failure: true require_tests: true check_name: Unit Test Results - # This is the only way to get test reports out from GitHub Actions - # TODO remove this before release - - uses: actions/upload-artifact@v2 - if: ${{ always() }} + - name: Upload test report + if: failure() + uses: actions/upload-artifact@v3 with: name: unit-test-report path: build/output/* @@ -78,18 +77,16 @@ jobs: name: Test - name: Test Results if: always() - uses: mikepenz/action-junit-report@v2 + uses: mikepenz/action-junit-report@v3 with: report_paths: 'build/output/junit-*.xml' github_token: ${{ secrets.GITHUB_TOKEN }} fail_on_failure: true require_tests: true check_name: Canary Test Results - # This is the only way to get test reports out from GitHub Actions - # TODO remove this before release - - uses: actions/upload-artifact@v2 - if: ${{ always() }} + - name: Upload test report + if: failure() + uses: actions/upload-artifact@v3 with: name: canary-test-report path: build/output/* - diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs index bba2455f5f..f0e3769c96 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs @@ -34,6 +34,7 @@ using System.Security.Cryptography; using System.Text; using OpenSearch.OpenSearch.Managed; +using OpenSearch.OpenSearch.Managed.Configuration; using OpenSearch.Stack.ArtifactsApi; namespace OpenSearch.OpenSearch.Ephemeral @@ -59,6 +60,13 @@ protected EphemeralCluster(TConfiguration clusterConfiguration) : base(clusterCo protected EphemeralClusterComposer Composer { get; } + protected override void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port) + { + base.ModifyNodeConfiguration(nodeConfiguration, port); + + if (!ClusterConfiguration.EnableSsl) nodeConfiguration.Add("plugins.security.disabled", "true"); + } + public virtual ICollection NodesUris(string hostName = null) { hostName = hostName ?? (ClusterConfiguration.HttpFiddlerAware && Process.GetProcessesByName("fiddler").Any() diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs index 40b200f28a..74441951ef 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs @@ -172,22 +172,35 @@ protected static void WriteFileIfNotExist(string fileLocation, string contents) protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer, string binary, string description, params string[] arguments) => - ExecuteBinaryInternal(config, writer, binary, description, arguments); + ExecuteBinaryInternal(config, writer, binary, description, null, arguments); + + protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer, + string binary, string description, IDictionary environmentVariables, + params string[] arguments) => + ExecuteBinaryInternal(config, writer, binary, description, environmentVariables, arguments); private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config, IConsoleLineHandler writer, - string binary, string description, params string[] arguments) + string binary, string description, IDictionary environmentVariables, params string[] arguments) { var command = $"{{{binary}}} {{{string.Join(" ", arguments)}}}"; writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} starting process [{description}] {command}"); + var environment = new Dictionary + { + {config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath}, + {"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome} + }; + + if (environmentVariables != null) + { + foreach (var kvp in environmentVariables) + environment[kvp.Key] = kvp.Value; + } + var timeout = TimeSpan.FromSeconds(420); var processStartArguments = new StartArguments(binary, arguments) { - Environment = new Dictionary - { - {config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath}, - {"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome}, - } + Environment = environment }; var result = Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter()); diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs index b938890102..9540c07edb 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs @@ -26,10 +26,12 @@ * under the License. */ +using System.Collections.Generic; using System.IO; using System.Linq; using OpenSearch.OpenSearch.Managed.ConsoleWriters; -using OpenSearch.Stack.ArtifactsApi; +using OpenSearch.Stack.ArtifactsApi.Products; +using SemanticVersioning; namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks { @@ -38,30 +40,39 @@ public class InitialConfiguration : ClusterComposeTask public override void Run(IEphemeralCluster cluster) { var fs = cluster.FileSystem; - var configFile = Path.Combine(fs.OpenSearchHome, "config", "opensearch.yml"); - if (File.Exists(configFile) && File.ReadLines(configFile).Any(l => !string.IsNullOrWhiteSpace(l) && !l.StartsWith("#"))) - { - cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} opensearch.yml already exists, skipping initial configuration"); + var installConfigDir = Path.Combine(fs.OpenSearchHome, "config"); + var installConfigFile = Path.Combine(installConfigDir, "opensearch.yml"); + var pluginSecurity = Path.Combine(fs.OpenSearchHome, "plugins/opensearch-security"); + + if (!Directory.Exists(pluginSecurity)) return; - } - var securityInstallDemoConfigSubPath = "plugins/opensearch-security/tools/install_demo_configuration.sh"; - var securityInstallDemoConfig = Path.Combine(fs.OpenSearchHome, securityInstallDemoConfigSubPath); + var isNewDemoScript = cluster.ClusterConfiguration.Version.BaseVersion() >= new Version(2, 12, 0); + + const string securityInstallDemoConfigSubPath = "tools/install_demo_configuration.sh"; + var securityInstallDemoConfig = Path.Combine(pluginSecurity, securityInstallDemoConfigSubPath); cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} going to run [{securityInstallDemoConfigSubPath}]"); + if (File.Exists(installConfigFile) && File.ReadLines(installConfigFile).Any(l => l.Contains("plugins.security"))) return; + + var env = new Dictionary(); + var args = new List { securityInstallDemoConfig, "-y", "-i" }; + + if (isNewDemoScript) + { + env.Add("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "admin"); + args.Add("-t"); + } + ExecuteBinary( cluster.ClusterConfiguration, cluster.Writer, "/bin/bash", "install security plugin demo configuration", - securityInstallDemoConfig, - "-y", "-i", "-s"); - - if (cluster.ClusterConfiguration.EnableSsl) return; - - File.AppendAllText(configFile, "plugins.security.disabled: true"); + env, + args.ToArray()); } } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs index c9509b7d03..3225476353 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs @@ -27,6 +27,7 @@ */ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; @@ -93,12 +94,21 @@ public override void Run(IEphemeralCluster cluste cluster.Writer?.WriteDiagnostic( $"{{{nameof(InstallPlugins)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}: {{{plugin.IsValid(v)}}}"); - if (!Directory.Exists(fs.ConfigPath)) Directory.CreateDirectory(fs.ConfigPath); + var homeConfigPath = Path.Combine(fs.OpenSearchHome, "config"); + + if (!Directory.Exists(homeConfigPath)) Directory.CreateDirectory(homeConfigPath); + + var env = new Dictionary + { + { fs.ConfigEnvironmentVariableName, homeConfigPath } + }; + ExecuteBinary( cluster.ClusterConfiguration, cluster.Writer, fs.PluginBinary, $"install opensearch plugin: {plugin.SubProductName}", + env, "install", "--batch", GetPluginLocation(plugin, v)); CopyConfigDirectoryToHomeCacheConfigDirectory(cluster, plugin); diff --git a/guides/document-lifecycle.md b/guides/document-lifecycle.md index e0746f7e7c..67cb37588d 100644 --- a/guides/document-lifecycle.md +++ b/guides/document-lifecycle.md @@ -7,7 +7,7 @@ Assuming you have OpenSearch running locally on port 9200, you can create a clie var node = new Uri("https://localhost:9200"); var config = new ConnectionSettings(node) .ServerCertificateValidationCallback(CertificateValidations.AllowAll) - .BasicAuthentication("admin", "admin") + .BasicAuthentication("admin", ) .DisableDirectStreaming(); var client = new OpenSearchClient(config); diff --git a/guides/index-template.md b/guides/index-template.md index 495d873b39..81049f128b 100644 --- a/guides/index-template.md +++ b/guides/index-template.md @@ -12,7 +12,7 @@ using OpenSearch.Net; var node = new Uri("https://localhost:9200"); var config = new ConnectionSettings(node) .ServerCertificateValidationCallback(CertificateValidations.AllowAll) - .BasicAuthentication("admin", "admin"); + .BasicAuthentication("admin", ); var client = new OpenSearchClient(config);; ``` diff --git a/guides/search.md b/guides/search.md index ed8d022a97..e7f00e4bec 100644 --- a/guides/search.md +++ b/guides/search.md @@ -12,7 +12,7 @@ var node = new Uri("https://localhost:9200"); var config = new ConnectionSettings(node) .ThrowExceptions() .ServerCertificateValidationCallback(CertificateValidations.AllowAll) - .BasicAuthentication("admin", "admin"); + .BasicAuthentication("admin", ); var client = new OpenSearchClient(config); class Movie diff --git a/tests/Tests.YamlRunner/Commands.fs b/tests/Tests.YamlRunner/Commands.fs index 97019d37b4..1122a7dab2 100644 --- a/tests/Tests.YamlRunner/Commands.fs +++ b/tests/Tests.YamlRunner/Commands.fs @@ -28,6 +28,8 @@ module Tests.YamlRunner.Commands open System +open System.IO.Compression +open FSharp.Data open ShellProgressBar open Tests.YamlRunner.AsyncExtensions open Tests.YamlRunner.TestsLocator @@ -51,23 +53,48 @@ let private subBarOptions = ) let LocateTests namedSuite revision directoryFilter fileFilter = async { - let! folders = TestsLocator.ListFolders namedSuite revision directoryFilter - if folders.Length = 0 then - raise <| Exception("No folders found trying to list the yaml specs") - - let l = folders.Length - use progress = new ProgressBar(l, sprintf "Listing %i folders" l, barOptions) - progress.WriteLine <| sprintf "Listing %i folders" l - let folderDownloads = - folders - |> Seq.map(fun folder -> TestsLocator.DownloadTestsInFolder folder fileFilter namedSuite revision progress subBarOptions) - let! completed = Async.ForEachAsync 4 folderDownloads - return completed + let! response = Http.AsyncRequestStream( + $"https://api.github.com/repos/opensearch-project/OpenSearch/zipball/%s{revision}", + headers=[ + "User-Agent", "OpenSearch .NET YAML Tests" + ] + ) + use zip = new ZipArchive(response.ResponseStream, ZipArchiveMode.Read) + + let folders = + let testDir = "rest-api-spec/src/main/resources/rest-api-spec/test/" + let trimParent (p:string) = + p.Substring(p.IndexOf('/')+1) + + zip.Entries + |> Seq.map (fun e -> e, trimParent e.FullName) + |> Seq.filter (fun (_, p) -> p.StartsWith(testDir) && p.EndsWith(".yml")) + |> Seq.map (fun (e, p) -> e, p.Substring(testDir.Length)) + |> Seq.map (fun (e, p) -> + let parts = p.Split('/') + parts[0], parts[1], e + ) + |> Seq.groupBy (fun (folder, _, _) -> folder) + |> Seq.filter (fun (folder, _) -> match directoryFilter with | Some d -> folder.StartsWith(d, StringComparison.OrdinalIgnoreCase) | None -> true) + |> Seq.map (fun (folder, entries) -> + ExtractTestsInFolder folder (entries |> Seq.map (fun (_, f, e) -> f, e)) fileFilter namedSuite revision + ) + + let! completed = Async.ForEachAsync 1 folders + + return completed } let ReadTests (tests:LocateResults list) = - - let readPaths paths = paths |> List.map TestsReader.ReadYamlFile + let safeRead yamlInfo = + try + ReadYamlFile yamlInfo |> Some + with + | ex -> + printfn "%s" ex.Message + None + + let readPaths paths = paths |> List.map safeRead |> List.filter Option.isSome |> List.map Option.get tests |> List.map (fun t -> { Folder= t.Folder; Files = readPaths t.Paths}) diff --git a/tests/Tests.YamlRunner/Models.fs b/tests/Tests.YamlRunner/Models.fs index 761cdf0269..5d4f372df5 100644 --- a/tests/Tests.YamlRunner/Models.fs +++ b/tests/Tests.YamlRunner/Models.fs @@ -61,9 +61,9 @@ let (|IsDoCatch|_|) (s:string) = | "conflict" -> Some Conflict | "unavailable" -> Some Unavailable | "param" -> Some UnknownParameter - | "request" -> Some OtherBadResponse - | s -> Some <| CatchRegex (s.Trim('/')) - + | "request" -> Some OtherBadResponse + | s -> Some <| CatchRegex (Regex.Replace(s.Trim('/'), @"(? stashes.Response().Dictionary.ToDictionary() :> Object match assertValue with - | Value o -> OperationExecutor.JTokenDeepEquals op o value + | Value o -> + let resolvedData = + match o with + | :? YamlMap as m -> stashes.Resolve progress m :> Object + | o -> o + OperationExecutor.JTokenDeepEquals op resolvedData value | Id id -> let found, expected = stashes.TryGetValue id match found with diff --git a/tests/Tests.YamlRunner/Program.fs b/tests/Tests.YamlRunner/Program.fs index 251f23dbbc..629f476e29 100644 --- a/tests/Tests.YamlRunner/Program.fs +++ b/tests/Tests.YamlRunner/Program.fs @@ -30,31 +30,38 @@ module Tests.YamlRunner.Main open System open System.Linq open System.Diagnostics +open System.Security.Cryptography.X509Certificates open Argu open Tests.YamlRunner open Tests.YamlRunner.Models open OpenSearch.Net type Arguments = - | [] NamedSuite of string - | []Folder of string - | []TestFile of string - | []TestSection of string - | []Endpoint of string - | []Revision of string - | []JUnitOutputFile of string - | []Profile of bool + | [] Named_Suite of string + | Folder of string + | Test_File of string + | Test_Section of string + | Endpoint of string + | Auth_Basic of string + | Auth_Cert of string + | Auth_Cert_Pass of string + | Revision of string + | JUnit_Output_File of string + | Profile of bool with interface IArgParserTemplate with member s.Usage = match s with - | NamedSuite _ -> "specify a known yaml test suite. defaults to `opensource`." + | Named_Suite _ -> "specify a known yaml test suite. defaults to `opensource`." | Revision _ -> "The git revision to reference (commit/branch/tag). defaults to `main`" | Folder _ -> "Only run tests in this folder" - | TestFile _ -> "Only run tests starting with this filename" - | TestSection _ -> "Only run test with this name (best used in conjuction with -t)" + | Test_File _ -> "Only run tests starting with this filename" + | Test_Section _ -> "Only run test with this name (best used in conjunction with --test-file)" | Endpoint _ -> "The opensearch endpoint to run tests against" - | JUnitOutputFile _ -> "The path and file name to use for the junit xml output, defaults to a random tmp filename" + | Auth_Basic _ -> "The username and password to use for client authentication in the form of `username:password`" + | Auth_Cert _ -> "The certificate to use for client authentication" + | Auth_Cert_Pass _ -> "The password to use for the auth certificate" + | JUnit_Output_File _ -> "The path and file name to use for the junit xml output, defaults to a random tmp filename" | Profile _ -> "Print out process id and wait for confirmation to kick off the tests" let private runningMitmProxy = Process.GetProcessesByName("mitmproxy").Length > 0 @@ -67,37 +74,34 @@ let private defaultEndpoint namedSuite = let https = "s" // "" sprintf "http%s://%s:9200" https host; -let private createClient endpoint namedSuite = - let uri, userInfo = - let e = Uri(endpoint) - let sanitized = UriBuilder(e) - sanitized.UserName <- null - sanitized.Password <- null - let uri = sanitized.Uri - let tokens = e.UserInfo.Split(':') |> Seq.toList - match (tokens, namedSuite) with - | ([username; password], _) -> uri, Some (username, password) - | _ -> uri, None - let settings = new ConnectionConfiguration(uri) +let private createClient endpoint (authBasic: string option) (authCert: string option * string option) namedSuite = + let mutable settings = new ConnectionConfiguration(Uri(endpoint)) + settings <- + settings.DisableDirectStreaming(true) // proxy - let proxySettings = - match (runningMitmProxy, namedSuite) with - | (true, _) -> settings.Proxy(Uri("http://ipv4.fiddler:8080"), String(null), String(null)) + settings <- + match runningMitmProxy with + | true -> settings.Proxy(Uri("http://ipv4.fiddler:8080"), String(null), String(null)) | _ -> settings // auth - let authSettings = - match userInfo with - | Some(username, password) -> proxySettings.BasicAuthentication(username, password) - | _ -> proxySettings + settings <- + match (authCert, authBasic) with + | (Some(certPath), None), _ -> + settings.ClientCertificate(new X509Certificate2(certPath)) + | (Some(certPath), Some(certPass)), _ -> + settings.ClientCertificate(new X509Certificate2(certPath, certPass)) + | _, Some(userPass) -> + match userPass.Split(':') with + | [| username; password |] -> settings.BasicAuthentication(username, password) + | _ -> settings + | _ -> settings // certs - let certSettings = - match namedSuite with -// authSettings.ServerCertificateValidationCallback(fun _ _ _ _ -> true) - | _ -> authSettings - OpenSearchLowLevelClient(certSettings) + settings <- + settings.ServerCertificateValidationCallback(fun _ _ _ _ -> true) + OpenSearchLowLevelClient(settings) -let validateRevisionParams endpoint _passedRevision namedSuite = - let client = createClient endpoint namedSuite +let validateRevisionParams endpoint authBasic authCert _passedRevision namedSuite = + let client = createClient endpoint authBasic authCert namedSuite let node = client.Settings.ConnectionPool.Nodes.First() let auth = @@ -108,9 +112,7 @@ let validateRevisionParams endpoint _passedRevision namedSuite = printfn "Running opensearch %O %s" (node.Uri) auth let r = - let config = RequestConfiguration(DisableDirectStreaming=Nullable(true)) - let p = RootNodeInfoRequestParameters(RequestConfiguration = config) - client.RootNodeInfo(p) + client.RootNodeInfo() printfn "%s" r.DebugInformation if not r.Success then @@ -127,22 +129,25 @@ let validateRevisionParams endpoint _passedRevision namedSuite = (client, revision, version) let runMain (parsed:ParseResults) = async { - let namedSuite = parsed.TryGetResult NamedSuite |> Option.defaultValue "_" + let namedSuite = parsed.TryGetResult Named_Suite |> Option.defaultValue "_" let directory = parsed.TryGetResult Folder //|> Option.defaultValue "indices.create" |> Some - let file = parsed.TryGetResult TestFile //|> Option.defaultValue "10_basic.yml" |> Some - let section = parsed.TryGetResult TestSection //|> Option.defaultValue "10_basic.yml" |> Some + let file = parsed.TryGetResult Test_File //|> Option.defaultValue "10_basic.yml" |> Some + let section = parsed.TryGetResult Test_Section //|> Option.defaultValue "10_basic.yml" |> Some let endpoint = parsed.TryGetResult Endpoint |> Option.defaultValue (defaultEndpoint namedSuite) + let authBasic = parsed.TryGetResult Auth_Basic + let authCert = parsed.TryGetResult Auth_Cert + let authCertPass = parsed.TryGetResult Auth_Cert_Pass let profile = parsed.TryGetResult Profile |> Option.defaultValue false let passedRevision = parsed.TryGetResult Revision let outputFile = - parsed.TryGetResult JUnitOutputFile + parsed.TryGetResult JUnit_Output_File |> Option.defaultValue (System.IO.Path.GetTempFileName()) - let (client, revision, version) = validateRevisionParams endpoint passedRevision namedSuite + let client, revision, version = validateRevisionParams endpoint authBasic (authCert, authCertPass) passedRevision namedSuite printfn "Found version %s downloading specs from: %s" version revision - let! locateResults = Commands.LocateTests namedSuite revision directory file + let! locateResults = Commands.LocateTests namedSuite revision directory file let readResults = Commands.ReadTests locateResults if profile then printf "Waiting for profiler to attach to pid: %O" <| Process.GetCurrentProcess().Id diff --git a/tests/Tests.YamlRunner/SkipList.fs b/tests/Tests.YamlRunner/SkipList.fs index 5e18dad3f7..a68a0c2ac9 100644 --- a/tests/Tests.YamlRunner/SkipList.fs +++ b/tests/Tests.YamlRunner/SkipList.fs @@ -31,64 +31,25 @@ type SkipSection = All | Section of string | Sections of string list type SkipFile = SkipFile of string -let SkipList = dict [ - // funny looking dispatch /_security/privilege/app?name - SkipFile "privileges/10_basic.yml", All +let SkipList = dict [ + // Incorrectly being run due to OpenSearch 1.x/2.x being numerically <7.2.0, but feature-wise >7.10 + SkipFile "cat.indices/10_basic.yml", Section "Test cat indices output for closed index (pre 7.2.0)" + SkipFile "cluster.health/10_basic.yml", Section "cluster health with closed index (pre 7.2.0)" - // We skip the generation of this API till one of the later minors - SkipFile "indices.upgrade/10_basic.yml", All - - // - Failed: Assert operation NumericAssert Length invalidated_api_keys "Long" Reason: Expected 2.000000 = 3.000000 - SkipFile "api_key/11_invalidation.yml", Section "Test invalidate api key by realm name" - - // Uses variables in strings e.g Bearer ${token} we can not due variable substitution in string yet - SkipFile "token/10_basic.yml", All - - SkipFile "change_password/11_token.yml", Section "Test user changing their password authenticating with token not allowed" - - SkipFile "change_password/10_basic.yml", Sections [ - // Changing password locks out tests - "Test user changing their own password" - // Uses variables in strings e.g Bearer ${token} we can not due variable substitution in string yet - "Test user changing their password authenticating with token not allowed" - ] - - // TEMPORARY: Missing 'body: { indices: "test_index" }' payload, TODO: PR - SkipFile "snapshot/10_basic.yml", Section "Create a source only snapshot and then restore it" - // illegal_argument_exception: Provided password hash uses [NOOP] but the configured hashing algorithm is [BCRYPT] - SkipFile "users/10_basic.yml", Section "Test put user with password hash" - // Slash in index name is not escaped (BUG) - SkipFile "security/authz/13_index_datemath.yml", Section "Test indexing documents with datemath, when permitted" - // Possibly a cluster health color mismatch... - SkipFile "security/authz/14_cat_indices.yml", All - - // Snapshot testing requires local filesystem access - SkipFile "snapshot.create/10_basic.yml", All - SkipFile "snapshot.get/10_basic.yml", All - SkipFile "snapshot.get_repository/10_basic.yml", All - SkipFile "snapshot.restore/10_basic.yml", All - SkipFile "snapshot.status/10_basic.yml", All - - // uses $stashed id in match with object - SkipFile "cluster.reroute/11_explain.yml", Sections [ - "Explain API for non-existent node & shard" - ] + // .NET method arg typings make this not possible, index is a required parameter + SkipFile "indices.put_mapping/all_path_options_with_types.yml", Section "put mapping with blank index" - //These are ignored because they were flagged on a big PR. - - //additional enters in regex - SkipFile "cat.templates/10_basic.yml", Sections [ "Multiple template"; "Sort templates"; "No templates" ] + // The client doesn't support the indices.upgrade API + SkipFile "indices.upgrade/10_basic.yml", All - //Replace stashed value in body that is passed as string json - SkipFile "api_key/10_basic.yml", Section "Test get api key" - - //new API TODO remove when we regenerate - SkipFile "cluster.voting_config_exclusions/10_basic.yml", All + // TODO: Add support for point-in-time APIs + SkipFile "pit/10_basic.yml", All + // TODO: Add support for search pipeline APIs + SkipFile "search_pipeline/10_basic.yml", All - //TODO has dates without strings which trips up our yaml parser - SkipFile "runtime_fields/40_date.yml", All - - SkipFile "nodes.info/10_basic.yml", Section "node_info role test" + // TODO: Better support parsing and asserting unsigned longs (hitting long vs double precision issues) + SkipFile "search.aggregation/20_terms.yml", Section "Unsigned Long test" + SkipFile "search.aggregation/230_composite_unsigned.yml", All + SkipFile "search.aggregation/370_multi_terms.yml", Section "Unsigned Long test" + SkipFile "search/90_search_after.yml", Section "unsigned long" ] - - diff --git a/tests/Tests.YamlRunner/TestSuiteBootstrap.fs b/tests/Tests.YamlRunner/TestSuiteBootstrap.fs index 98700fbd91..cd99079be0 100644 --- a/tests/Tests.YamlRunner/TestSuiteBootstrap.fs +++ b/tests/Tests.YamlRunner/TestSuiteBootstrap.fs @@ -28,53 +28,48 @@ module Tests.YamlRunner.TestSuiteBootstrap open System -open System.Linq open OpenSearch.Net open OpenSearch.Net.Specification.CatApi -open OpenSearch.Net.Specification.ClusterApi open OpenSearch.Net.Specification.IndicesApi open Tests.YamlRunner.Models -let DefaultSetup : Operation list = [Actions("Setup", fun (client, suite) -> - let firstFailure (responses:DynamicResponse seq) = - responses - |> Seq.filter (fun r -> not r.Success && r.HttpStatusCode <> Nullable.op_Implicit 404) - |> Seq.tryHead +let private deleteAllIndices (client: IOpenSearchLowLevelClient) = + [ + client.Indices.Delete("*", DeleteIndexRequestParameters(ExpandWildcards=ExpandWildcards.All)) + ] - let deleteAll () = - let dp = DeleteIndexRequestParameters() - dp.SetQueryString("expand_wildcards", "open,closed,hidden") - client.Indices.Delete("*", dp) - let templates () = - client.Cat.Templates("*", CatTemplatesRequestParameters(Headers=["name";"order"].ToArray())) - .Body.Split("\n") - |> Seq.map(fun line -> line.Split(" ", StringSplitOptions.RemoveEmptyEntries)) - |> Seq.filter(fun line -> line.Length = 2) - |> Seq.map(fun tokens -> tokens.[0], Int32.Parse(tokens.[1])) - //assume templates with order 100 or higher are defaults - |> Seq.filter(fun (_, order) -> order < 100) - |> Seq.filter(fun (name, _) -> not(String.IsNullOrWhiteSpace(name)) && not(name.StartsWith(".")) && name <> "security-audit-log") - //TODO template does not accept comma separated list but is documented as such - |> Seq.map(fun (template, _) -> - let result = client.Indices.DeleteTemplateForAll(template) - match result.Success with - | true -> result - | false -> client.Indices.DeleteComposableTemplateForAll(template) - ) - |> Seq.toList +let private deleteAllTemplates (client: IOpenSearchLowLevelClient) = + [ + client.Indices.DeleteTemplateForAll("*") + client.Indices.DeleteComposableTemplateForAll("*") + client.Cluster.DeleteComponentTemplate("*") + ] - let snapshots = - client.Cat.Snapshots(CatSnapshotsRequestParameters(Headers=["id,repository"].ToArray())) +let private deleteAllSnapshotsAndRepositories (client: IOpenSearchLowLevelClient) = + let snapshotResps = + client.Cat.Repositories(CatRepositoriesRequestParameters(Headers=[| "id" |])) .Body.Split("\n") - |> Seq.map(fun line -> line.Split " ") - |> Seq.filter(fun tokens -> tokens.Length = 2) - |> Seq.map(fun tokens -> (tokens.[0].Trim(), tokens.[1].Trim())) - |> Seq.filter(fun (id, repos) -> not(String.IsNullOrWhiteSpace(id)) && not(String.IsNullOrWhiteSpace(repos))) - //TODO template does not accept comma separated list but is documented as such - |> Seq.map(fun (id, repos) -> client.Snapshot.Delete(repos, id)) - |> Seq.toList + |> Seq.filter (fun line -> not (String.IsNullOrWhiteSpace(line))) + |> Seq.map (fun repo -> client.Snapshot.Delete(repo, "*")) + |> Seq.toList + snapshotResps @ [ + client.Snapshot.DeleteRepository("*") + ] + +let private resetSettings (client: IOpenSearchLowLevelClient) = + [ + client.Cluster.PutSettings(PostData.String("{ \"persistent\": { \"*\": null }, \"transient\": { \"*\": null } }")) + ] - let deleteRepositories = client.Snapshot.DeleteRepository("*") - firstFailure <| [deleteAll()] @ templates() @ snapshots @ [deleteRepositories] +let DefaultSetup : Operation list = [Actions("Setup", fun (client, _) -> + seq { + deleteAllIndices; + deleteAllTemplates; + deleteAllSnapshotsAndRepositories; + resetSettings + } + |> Seq.collect (fun f -> f client) + |> Seq.filter (fun r -> not r.Success) + |> Seq.tryHead )] diff --git a/tests/Tests.YamlRunner/Tests.YamlRunner.fsproj b/tests/Tests.YamlRunner/Tests.YamlRunner.fsproj index 78fd12a347..76ff1f8ab3 100644 --- a/tests/Tests.YamlRunner/Tests.YamlRunner.fsproj +++ b/tests/Tests.YamlRunner/Tests.YamlRunner.fsproj @@ -18,7 +18,6 @@ - diff --git a/tests/Tests.YamlRunner/TestsDownloader.fs b/tests/Tests.YamlRunner/TestsDownloader.fs deleted file mode 100644 index 000af03aa9..0000000000 --- a/tests/Tests.YamlRunner/TestsDownloader.fs +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// -// The OpenSearch Contributors require contributions made to -// this file be licensed under the Apache-2.0 license or a -// compatible open source license. -// -// Modifications Copyright OpenSearch Contributors. See -// GitHub history for details. -// -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// - -module Tests.YamlRunner.TestsDownloader - -open System -open System.IO -open FSharp.Data -open Tests.YamlRunner.Models - - -let private rootListingUrl = "https://github.com/opensearch-project/opensearch-net" -let private rootRawUrl = "https://raw.githubusercontent.com/opensearch-project/opensearch-net" - -let private openSourceResourcePath = "rest-api-spec/src/main/resources" - -let private path namedSuite revision = - let path = openSourceResourcePath - sprintf "%s/%s/rest-api-spec/test" revision path - -let TestGithubRootUrl namedSuite revision = sprintf "%s/tree/%s" rootListingUrl <| path namedSuite revision - -let FolderListUrl namedSuite revision folder = - let root = TestGithubRootUrl namedSuite revision - sprintf "%s/%s" root folder - -let TestRawUrl namedSuite revision folder file = - let path = path namedSuite revision - sprintf "%s/%s/%s/%s" rootRawUrl path folder file - -let private randomTime = Random() - -let TemporaryPath revision suite = lazy(Path.Combine(Path.GetTempPath(), "opensearch", sprintf "tests-%s-%s" suite revision)) - -let private download url = async { - let! _wait = Async.Sleep (randomTime.Next(500, 900)) - let! yaml = Http.AsyncRequestString url - return yaml -} -let CachedOrDownload namedSuite revision folder file url = async { - let parent = (TemporaryPath revision "_").Force() - let directory = Path.Combine(parent, folder) - let file = Path.Combine(directory, file) - let fileExists = File.Exists file - let directoryExists = Directory.Exists directory - let! result = async { - match (fileExists, directoryExists) with - | (true, _) -> - let! text = Async.AwaitTask <| File.ReadAllTextAsync file - return text - | (_, d) -> - if (not d) then Directory.CreateDirectory(directory) |> ignore - let! contents = download url - let write = File.WriteAllTextAsync(file, contents) - do! Async.AwaitTask write - return contents - } - return (file, result) -} - diff --git a/tests/Tests.YamlRunner/TestsLocator.fs b/tests/Tests.YamlRunner/TestsLocator.fs index b0f39c609a..d3b3a98096 100644 --- a/tests/Tests.YamlRunner/TestsLocator.fs +++ b/tests/Tests.YamlRunner/TestsLocator.fs @@ -28,39 +28,9 @@ module Tests.YamlRunner.TestsLocator open System -open System.Threading -open FSharp.Data +open System.IO +open System.IO.Compression open Tests.YamlRunner.AsyncExtensions -open ShellProgressBar -open Tests.YamlRunner - - -let ListFolders namedSuite revision directory = async { - let url = TestsDownloader.TestGithubRootUrl namedSuite revision - let! (_, html) = TestsDownloader.CachedOrDownload namedSuite revision "_root_" "index.html" url - let doc = HtmlDocument.Parse(html) - - return - doc.CssSelect("div.js-details-container a.js-navigation-open") - |> List.map (fun a -> a.InnerText()) - |> List.filter (fun f -> match directory with | Some s -> f.StartsWith(s, StringComparison.OrdinalIgnoreCase) | None -> true) - |> List.filter(fun f -> f.Replace(" ", "") <> "..") - |> List.filter (fun f -> not <| f.EndsWith(".asciidoc")) -} - -let ListFolderFiles namedSuite revision folder fileFilter = async { - let url = TestsDownloader.FolderListUrl namedSuite revision folder - let! (_, html) = TestsDownloader.CachedOrDownload namedSuite revision folder "index.html" url - let doc = HtmlDocument.Parse(html) - let yamlFiles = - let fileUrl file = (file, TestsDownloader.TestRawUrl namedSuite revision folder file) - doc.CssSelect("div.js-details-container a.js-navigation-open") - |> List.map(fun a -> a.InnerText()) - |> List.filter(fun f -> f.EndsWith(".yml")) - |> List.filter (fun f -> match fileFilter with | Some s -> f.StartsWith(s, StringComparison.OrdinalIgnoreCase) | None -> true) - |> List.map fileUrl - return yamlFiles -} type YamlFileInfo = { File: string; Yaml: string } @@ -68,44 +38,51 @@ let TestLocalFile file = let yaml = System.IO.File.ReadAllText file { File = file; Yaml = yaml } -let private downloadTestsInFolder (yamlFiles:list) folder namedSuite revision (progress: IProgressBar) subBarOptions = async { - let mutable seenFiles = 0; - use filesProgress = progress.Spawn(yamlFiles.Length, sprintf "Downloading [0/%i] files in %s" yamlFiles.Length folder, subBarOptions) - let actions = - yamlFiles - |> Seq.map (fun (file, url) -> async { - let! (localFile, yaml) = TestsDownloader.CachedOrDownload namedSuite revision folder file url - let i = Interlocked.Increment (&seenFiles) - let message = sprintf "Downloaded [%i/%i] files in %s" i yamlFiles.Length folder - filesProgress.Tick(message) - match String.IsNullOrWhiteSpace yaml with - | true -> - progress.WriteLine(sprintf "Skipped %s since it returned no data" url) - return None - | _ -> - return Some {File = localFile; Yaml = yaml} - }) - |> Seq.toList - - let! completed = Async.ForEachAsync 4 actions - let files = completed |> List.choose id; - return files -} - type LocateResults = { Folder: string; Paths: YamlFileInfo list } -let DownloadTestsInFolder folder fileFilter namedSuite revision (progress: IProgressBar) subBarOptions = async { - let! token = Async.StartChild <| ListFolderFiles namedSuite revision folder fileFilter - let! yamlFiles = token +let TemporaryPath revision suite = lazy(Path.Combine(Path.GetTempPath(), "opensearch", $"tests-%s{suite}-%s{revision}")) + +let ExtractTestsInFolder folder (entries:seq) fileFilter namedSuite revision = async { + let parent = (TemporaryPath revision "_").Force() + let directory = Path.Combine(parent, folder) + + let cachedOrExtract file (entry: ZipArchiveEntry) = async { + let file = Path.Combine(directory, file) + let fileExists = File.Exists file + let directoryExists = Directory.Exists directory + let! result = async { + match (fileExists, directoryExists) with + | true, _ -> + let! text = Async.AwaitTask <| File.ReadAllTextAsync file + return text + | _, d -> + if (not d) then Directory.CreateDirectory(directory) |> ignore + use zipStream = new StreamReader(entry.Open()) + let! contents = zipStream.ReadToEndAsync() |> Async.AwaitTask + do! File.WriteAllTextAsync(file, contents) |> Async.AwaitTask + return contents + } + return (file, result) + } + let! localFiles = async { - match yamlFiles.Length with - | 0 -> - progress.WriteLine(sprintf "%s folder yielded no tests (fileFilter: %O)" folder fileFilter) - return List.empty - | _ -> - let! result = downloadTestsInFolder yamlFiles folder namedSuite revision progress subBarOptions - return result + let actions = + entries + |> Seq.filter (fun (file, _) -> match fileFilter with | Some f -> file.StartsWith(f, StringComparison.OrdinalIgnoreCase) | None -> true) + |> Seq.map (fun (file, entry) -> async { + let! localFile, yaml = cachedOrExtract file entry + match String.IsNullOrWhiteSpace yaml with + | true -> + return None + | _ -> + return Some {File = localFile; Yaml = yaml} + }) + |> Seq.toList + + let! completed = Async.ForEachAsync 1 actions + let files = completed |> List.choose id; + return files } - progress.Tick() + return { Folder = folder; Paths = localFiles } } diff --git a/tests/Tests/Search/SearchTemplate/RenderSearchTemplate/RenderSearchTemplateApiTests.cs b/tests/Tests/Search/SearchTemplate/RenderSearchTemplate/RenderSearchTemplateApiTests.cs index 82775a3be9..d01c1cd955 100644 --- a/tests/Tests/Search/SearchTemplate/RenderSearchTemplate/RenderSearchTemplateApiTests.cs +++ b/tests/Tests/Search/SearchTemplate/RenderSearchTemplate/RenderSearchTemplateApiTests.cs @@ -40,6 +40,7 @@ namespace Tests.Search.SearchTemplate.RenderSearchTemplate { + [SkipVersion("2.10.*,2.11.*", "Broken by security plugin https://github.com/opensearch-project/security/issues/3672")] public class RenderSearchTemplateApiTests : ApiIntegrationTestBase