Skip to content

Commit

Permalink
Update CI check for integ-test-with-security to run all integ tests w…
Browse files Browse the repository at this point in the history
…ith security (opensearch-project#1243)

* Run all integ tests with security

Signed-off-by: Craig Perkins <[email protected]>

* Use adminClient when deleting ISM index in RestStopRollupActionIT

Signed-off-by: Craig Perkins <[email protected]>

* Update RolloverActionIT

Signed-off-by: Craig Perkins <[email protected]>

* Fix RestStopSnapshotManagementIT

Signed-off-by: Craig Perkins <[email protected]>

* Fix RestChangePolicyActionIT

Signed-off-by: Craig Perkins <[email protected]>

* Fix more tests

Signed-off-by: Craig Perkins <[email protected]>

* Fix more tests

Signed-off-by: Craig Perkins <[email protected]>

* Fix more tests

Signed-off-by: Craig Perkins <[email protected]>

* Fix RestExplainTransformActionIT

Signed-off-by: Craig Perkins <[email protected]>

* Fix more tests

Signed-off-by: Craig Perkins <[email protected]>

* Make sure read from system index also uses adminClient

Signed-off-by: Craig Perkins <[email protected]>

---------

Signed-off-by: Craig Perkins <[email protected]>
  • Loading branch information
cwperks authored Sep 11, 2024
1 parent c0942eb commit 395317d
Show file tree
Hide file tree
Showing 17 changed files with 55 additions and 24 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/security-test-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ jobs:
- name: Run integration tests
run: |
chown -R 1000:1000 `pwd`
su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*SecurityBehaviorIT'"
su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*IT'"
- name: Upload failed logs
uses: actions/upload-artifact@v2
if: failure()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,7 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase()
}
""".trimIndent()
val res =
client().makeRequest(
adminClient().makeRequest(
"PUT", "$index/_settings", emptyMap(),
StringEntity(body, ContentType.APPLICATION_JSON),
)
Expand Down Expand Up @@ -483,7 +483,7 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase()

protected fun updateManagedIndexConfigPolicySeqNo(update: ManagedIndexConfig) {
val response =
client().makeRequest(
adminClient().makeRequest(
"POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.id}",
StringEntity(
"{\"doc\":{\"managed_index\":{\"policy_seq_no\":\"${update.policySeqNo}\"}}}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -784,7 +784,7 @@ class RolloverActionIT : IndexStateManagementRestTestCase() {
}
// Manually produce transaction failure
val response =
client().makeRequest(
adminClient().makeRequest(
"POST", "$INDEX_MANAGEMENT_INDEX/_update/${managedIndexConfig.id}%23metadata",
StringEntity(
"{\n" +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package org.opensearch.indexmanagement.indexstatemanagement.resthandler

import org.junit.Before
import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.common.settings.Settings
import org.opensearch.core.rest.RestStatus
Expand Down Expand Up @@ -35,7 +36,7 @@ import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaDa
import org.opensearch.indexmanagement.waitFor
import org.opensearch.rest.RestRequest
import java.time.Instant
import java.util.Locale
import java.util.*

class RestChangePolicyActionIT : IndexStateManagementRestTestCase() {
private val testIndexName = javaClass.simpleName.lowercase(Locale.ROOT)
Expand Down Expand Up @@ -89,7 +90,10 @@ class RestChangePolicyActionIT : IndexStateManagementRestTestCase() {
}

fun `test nonexistent ism config index`() {
if (indexExists(INDEX_MANAGEMENT_INDEX)) deleteIndex(INDEX_MANAGEMENT_INDEX)
if (indexExists(INDEX_MANAGEMENT_INDEX)) {
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
}
try {
val changePolicy = ChangePolicy("some_id", null, emptyList(), false)
client().makeRequest(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@

package org.opensearch.indexmanagement.rollup.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.common.settings.Settings
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementIndices
import org.opensearch.indexmanagement.IndexManagementPlugin
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ROLLUP_JOBS_BASE_URI
import org.opensearch.indexmanagement.common.model.dimension.DateHistogram
import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_HIDDEN
Expand All @@ -22,7 +24,7 @@ import org.opensearch.indexmanagement.waitFor
import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule
import java.time.Instant
import java.time.temporal.ChronoUnit
import java.util.Locale
import java.util.*

class RestStartRollupActionIT : RollupRestAPITestCase() {
private val testName = javaClass.simpleName.lowercase(Locale.ROOT)
Expand Down Expand Up @@ -200,7 +202,8 @@ class RestStartRollupActionIT : RollupRestAPITestCase() {

fun `test start rollup when multiple shards configured for IM config index`() {
// setup ism-config index with multiple primary shards
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
val mapping = IndexManagementIndices.indexManagementMappings.trim().trimStart('{').trimEnd('}')
val settings =
Settings.builder()
Expand Down Expand Up @@ -253,7 +256,7 @@ class RestStartRollupActionIT : RollupRestAPITestCase() {

// clearing the config index to prevent other tests using this multi shard index
Thread.sleep(2000L)
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
adminClient().performRequest(deleteISMIndexRequest)
Thread.sleep(2000L)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@

package org.opensearch.indexmanagement.rollup.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.common.settings.Settings
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementIndices
import org.opensearch.indexmanagement.IndexManagementPlugin
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ROLLUP_JOBS_BASE_URI
import org.opensearch.indexmanagement.common.model.dimension.DateHistogram
import org.opensearch.indexmanagement.common.model.dimension.Terms
Expand Down Expand Up @@ -256,7 +258,8 @@ class RestStopRollupActionIT : RollupRestAPITestCase() {

fun `test stop rollup when multiple shards configured for IM config index`() {
// setup ism-config index with multiple primary shards
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
val mapping = IndexManagementIndices.indexManagementMappings.trim().trimStart('{').trimEnd('}')
val settings =
Settings.builder()
Expand Down Expand Up @@ -313,7 +316,7 @@ class RestStopRollupActionIT : RollupRestAPITestCase() {

// clearing the config index to prevent other tests using this multi shard index
Thread.sleep(2000L)
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
adminClient().performRequest(deleteISMIndexRequest)
Thread.sleep(2000L)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1466,7 +1466,7 @@ class RollupRunnerIT : RollupRestTestCase() {
// - Source index with pattern mapping to some closed indices

private fun deleteRollupMetadata(metadataId: String) {
val response = client().makeRequest("DELETE", "${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX}/_doc/$metadataId")
val response = adminClient().makeRequest("DELETE", "${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX}/_doc/$metadataId")
assertEquals("Unable to delete rollup metadata $metadataId", RestStatus.OK, response.restStatus())
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ abstract class SnapshotManagementRestTestCase : IndexManagementRestTestCase() {
val startTimeMillis = desiredStartTimeMillis ?: (now().toEpochMilli() - millis)
val waitForActiveShards = if (isMultiNode) "all" else "1"
val response =
client().makeRequest(
adminClient().makeRequest(
"POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.id}?wait_for_active_shards=$waitForActiveShards",
StringEntity(
"{\"doc\":{\"sm_policy\":{\"schedule\":{\"interval\":{\"start_time\":\"$startTimeMillis\"}}}}}",
Expand Down Expand Up @@ -171,7 +171,7 @@ abstract class SnapshotManagementRestTestCase : IndexManagementRestTestCase() {
val startTimeMillis = desiredStartTimeMillis ?: (now().toEpochMilli() - millis)
val waitForActiveShards = if (isMultiNode) "all" else "1"
val response =
client().makeRequest(
adminClient().makeRequest(
"POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.metadataID}?wait_for_active_shards=$waitForActiveShards",
StringEntity(
"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package org.opensearch.indexmanagement.snapshotmanagement.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementPlugin
Expand Down Expand Up @@ -38,7 +39,8 @@ class RestDeleteSnapshotManagementIT : SnapshotManagementRestTestCase() {

fun `test deleting a snapshot management policy that doesn't exist and config index doesnt exist`() {
try {
deleteIndex(INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
client().makeRequest("DELETE", "${IndexManagementPlugin.SM_POLICIES_URI}/nonexistent_policy")
fail("expected 404 ResponseException")
} catch (e: ResponseException) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package org.opensearch.indexmanagement.snapshotmanagement.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.common.xcontent.XContentType
import org.opensearch.core.rest.RestStatus
Expand Down Expand Up @@ -141,7 +142,8 @@ class RestExplainSnapshotManagementIT : SnapshotManagementRestTestCase() {

fun `test explain sm policy when config index doesn't exist`() {
try {
deleteIndex(INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
explainSMPolicy(randomAlphaOfLength(10))
fail("expected response exception")
} catch (e: ResponseException) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,11 @@ package org.opensearch.indexmanagement.snapshotmanagement.resthandler

import org.apache.hc.core5.http.HttpHeaders
import org.apache.hc.core5.http.message.BasicHeader
import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementPlugin
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
import org.opensearch.indexmanagement.makeRequest
import org.opensearch.indexmanagement.opensearchapi.convertToMap
import org.opensearch.indexmanagement.snapshotmanagement.SnapshotManagementRestTestCase
Expand Down Expand Up @@ -48,7 +50,8 @@ class RestGetSnapshotManagementIT : SnapshotManagementRestTestCase() {
@Throws(Exception::class)
fun `test getting a snapshot management policy that doesn't exist and config index doesnt exist`() {
try {
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
getSMPolicy(randomAlphaOfLength(20))
fail("expected response exception")
} catch (e: ResponseException) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package org.opensearch.indexmanagement.snapshotmanagement.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.common.xcontent.XContentType
import org.opensearch.core.rest.RestStatus
Expand Down Expand Up @@ -130,7 +131,8 @@ class RestIndexSnapshotManagementIT : SnapshotManagementRestTestCase() {
@Throws(Exception::class)
@Suppress("UNCHECKED_CAST")
fun `test mappings after sm policy creation`() {
deleteIndex(INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
createSMPolicy(randomSMPolicy())

val response = client().makeRequest("GET", "/$INDEX_MANAGEMENT_INDEX/_mapping")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@

package org.opensearch.indexmanagement.snapshotmanagement.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementPlugin
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
import org.opensearch.indexmanagement.makeRequest
import org.opensearch.indexmanagement.snapshotmanagement.SnapshotManagementRestTestCase
import org.opensearch.indexmanagement.snapshotmanagement.randomSMPolicy
Expand Down Expand Up @@ -59,7 +61,8 @@ class RestStartSnapshotManagementIT : SnapshotManagementRestTestCase() {

fun `test starting a snapshot management policy with no config index fails`() {
try {
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
val request = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(request)
client().makeRequest("POST", "${IndexManagementPlugin.SM_POLICIES_URI}/nonexistent_foo/_start")
fail("expected response exception")
} catch (e: ResponseException) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@

package org.opensearch.indexmanagement.snapshotmanagement.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementPlugin
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
import org.opensearch.indexmanagement.makeRequest
import org.opensearch.indexmanagement.snapshotmanagement.SnapshotManagementRestTestCase
import org.opensearch.indexmanagement.snapshotmanagement.randomSMPolicy
Expand Down Expand Up @@ -59,7 +61,8 @@ class RestStopSnapshotManagementIT : SnapshotManagementRestTestCase() {

fun `test stopping a snapshot management policy with no config index fails`() {
try {
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
client().makeRequest("POST", "${IndexManagementPlugin.SM_POLICIES_URI}/nonexistent_foo/_stop")
fail("expected response exception")
} catch (e: ResponseException) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ abstract class TransformRestTestCase : IndexManagementRestTestCase() {

protected fun getTransformMetadata(metadataId: String): TransformMetadata {
val response =
client().makeRequest(
adminClient().makeRequest(
"GET", "$INDEX_MANAGEMENT_INDEX/_doc/$metadataId", null, BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"),
)
assertEquals("Unable to get transform metadata $metadataId", RestStatus.OK, response.restStatus())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package org.opensearch.indexmanagement.transform.resthandler

import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
Expand Down Expand Up @@ -76,7 +77,10 @@ class RestDeleteTransformActionIT : TransformRestTestCase() {
@Throws(Exception::class)
fun `test deleting a transform that doesn't exist and config index doesn't exist`() {
try {
if (indexExists(INDEX_MANAGEMENT_INDEX)) deleteIndex(INDEX_MANAGEMENT_INDEX)
if (indexExists(INDEX_MANAGEMENT_INDEX)) {
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
}
val res = client().makeRequest("DELETE", "$TRANSFORM_BASE_URI/foobarbaz")
fail("expected 404 ResponseException: ${res.asMap()}")
} catch (e: ResponseException) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@
package org.opensearch.indexmanagement.transform.resthandler

import org.junit.Assert
import org.opensearch.client.Request
import org.opensearch.client.ResponseException
import org.opensearch.core.rest.RestStatus
import org.opensearch.indexmanagement.IndexManagementPlugin
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.TRANSFORM_BASE_URI
import org.opensearch.indexmanagement.makeRequest
import org.opensearch.indexmanagement.transform.TransformRestTestCase
Expand Down Expand Up @@ -160,7 +161,8 @@ class RestExplainTransformActionIT : TransformRestTestCase() {

@Throws(Exception::class)
fun `test explain transform when config doesnt exist`() {
deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
adminClient().performRequest(deleteISMIndexRequest)
val responseExplicit = client().makeRequest("GET", "$TRANSFORM_BASE_URI/no_config_some_transform/_explain")
val expectedResponse = mapOf("no_config_some_transform" to "Failed to search transform metadata")
assertEquals("Non-existent transform didn't return null", expectedResponse, responseExplicit.asMap())
Expand Down

0 comments on commit 395317d

Please sign in to comment.