Add automated migration for Destinations to Notification configs (ope…

…nsearch-project#379) * Add Destination migration classes Signed-off-by: Mohammad Qureshi <[email protected]> * Rename ElasticExtensions to OpenSearchExtensions Signed-off-by: Mohammad Qureshi <[email protected]> * Refactor blocking calls to use coroutines Signed-off-by: Mohammad Qureshi <[email protected]> * Add Notifications and Notifications core plugins to test clusters Signed-off-by: Mohammad Qureshi <[email protected]> * Replace deprecated toUpperCase usage with uppercase Signed-off-by: Mohammad Qureshi <[email protected]>
riysaxen-amzn · Apr 11, 2022 · bbde68c · bbde68c
1 parent dcf7acf
commit bbde68c
Show file tree

Hide file tree

Showing 41 changed files with 949 additions and 54 deletions.
diff --git a/alerting/build.gradle b/alerting/build.gradle
@@ -111,6 +111,20 @@ testClusters.integTest {
             debugPort += 1
         }
     }
+
+    plugin(provider({
+        new RegularFile() {
+            @Override
+            File getAsFile() { fileTree("src/test/resources/notifications-core").getSingleFile() }
+        }
+    }))
+
+    plugin(provider({
+        new RegularFile() {
+            @Override
+            File getAsFile() { fileTree("src/test/resources/notifications").getSingleFile() }
+        }
+    }))
 }
 
 testClusters.integTest.nodes.each { node ->
@@ -187,7 +201,19 @@ task prepareBwcTests {
     dependsOn bundle
     doLast {
         plugins = [
-                project.getObjects().fileProperty().value(bundle.getArchiveFile())
+                project.getObjects().fileProperty().value(bundle.getArchiveFile()),
+                provider({
+                    new RegularFile() {
+                        @Override
+                        File getAsFile() { fileTree("src/test/resources/notifications-core").getSingleFile() }
+                    }
+                }),
+                provider({
+                    new RegularFile() {
+                        @Override
+                        File getAsFile() { fileTree("src/test/resources/notifications").getSingleFile() }
+                    }
+                })
         ]
     }
 }

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt
@@ -17,9 +17,6 @@ import org.opensearch.action.search.SearchRequest
 import org.opensearch.action.search.SearchResponse
 import org.opensearch.alerting.alerts.AlertError
 import org.opensearch.alerting.alerts.AlertIndices
-import org.opensearch.alerting.elasticapi.firstFailureOrNull
-import org.opensearch.alerting.elasticapi.retry
-import org.opensearch.alerting.elasticapi.suspendUntil
 import org.opensearch.alerting.model.ActionExecutionResult
 import org.opensearch.alerting.model.ActionRunResult
 import org.opensearch.alerting.model.AggregationResultBucket
@@ -29,6 +26,9 @@ import org.opensearch.alerting.model.Monitor
 import org.opensearch.alerting.model.QueryLevelTriggerRunResult
 import org.opensearch.alerting.model.Trigger
 import org.opensearch.alerting.model.action.AlertCategory
+import org.opensearch.alerting.opensearchapi.firstFailureOrNull
+import org.opensearch.alerting.opensearchapi.retry
+import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.alerting.script.QueryLevelTriggerExecutionContext
 import org.opensearch.alerting.util.IndexUtils
 import org.opensearch.alerting.util.getBucketKeysHash

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt
@@ -82,6 +82,7 @@ import org.opensearch.alerting.transport.TransportIndexMonitorAction
 import org.opensearch.alerting.transport.TransportSearchEmailAccountAction
 import org.opensearch.alerting.transport.TransportSearchEmailGroupAction
 import org.opensearch.alerting.transport.TransportSearchMonitorAction
+import org.opensearch.alerting.util.destinationmigration.DestinationMigrationCoordinator
 import org.opensearch.client.Client
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver
 import org.opensearch.cluster.node.DiscoveryNodes
@@ -149,6 +150,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
     lateinit var threadPool: ThreadPool
     lateinit var alertIndices: AlertIndices
     lateinit var clusterService: ClusterService
+    lateinit var destinationMigrationCoordinator: DestinationMigrationCoordinator
 
     override fun getRestHandlers(
         settings: Settings,
@@ -248,9 +250,10 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
         scheduledJobIndices = ScheduledJobIndices(client.admin(), clusterService)
         scheduler = JobScheduler(threadPool, runner)
         sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES)
+        destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool)
         this.threadPool = threadPool
         this.clusterService = clusterService
-        return listOf(sweeper, scheduler, runner, scheduledJobIndices)
+        return listOf(sweeper, scheduler, runner, scheduledJobIndices, destinationMigrationCoordinator)
     }
 
     override fun getSettings(): List<Setting<*>> {

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt
@@ -10,11 +10,11 @@ import org.opensearch.action.search.SearchRequest
 import org.opensearch.action.search.SearchResponse
 import org.opensearch.alerting.core.model.ClusterMetricsInput
 import org.opensearch.alerting.core.model.SearchInput
-import org.opensearch.alerting.elasticapi.convertToMap
-import org.opensearch.alerting.elasticapi.suspendUntil
 import org.opensearch.alerting.model.InputRunResults
 import org.opensearch.alerting.model.Monitor
 import org.opensearch.alerting.model.TriggerAfterKey
+import org.opensearch.alerting.opensearchapi.convertToMap
+import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.alerting.util.AggregationQueryRewriter
 import org.opensearch.alerting.util.addUserBackendRolesFilter
 import org.opensearch.alerting.util.executeTransportAction

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt
@@ -17,9 +17,6 @@ import org.opensearch.alerting.alerts.AlertIndices
 import org.opensearch.alerting.alerts.moveAlerts
 import org.opensearch.alerting.core.JobRunner
 import org.opensearch.alerting.core.model.ScheduledJob
-import org.opensearch.alerting.elasticapi.InjectorContextElement
-import org.opensearch.alerting.elasticapi.retry
-import org.opensearch.alerting.elasticapi.withClosableContext
 import org.opensearch.alerting.model.ActionRunResult
 import org.opensearch.alerting.model.Alert
 import org.opensearch.alerting.model.AlertingConfigAccessor
@@ -39,6 +36,9 @@ import org.opensearch.alerting.model.action.AlertCategory
 import org.opensearch.alerting.model.action.PerAlertActionScope
 import org.opensearch.alerting.model.action.PerExecutionActionScope
 import org.opensearch.alerting.model.destination.DestinationContextFactory
+import org.opensearch.alerting.opensearchapi.InjectorContextElement
+import org.opensearch.alerting.opensearchapi.retry
+import org.opensearch.alerting.opensearchapi.withClosableContext
 import org.opensearch.alerting.script.BucketLevelTriggerExecutionContext
 import org.opensearch.alerting.script.QueryLevelTriggerExecutionContext
 import org.opensearch.alerting.script.TriggerExecutionContext

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alerts/AlertError.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alerts/AlertError.kt
@@ -5,8 +5,8 @@
 
 package org.opensearch.alerting.alerts
 
-import org.opensearch.alerting.elasticapi.instant
-import org.opensearch.alerting.elasticapi.optionalTimeField
+import org.opensearch.alerting.opensearchapi.instant
+import org.opensearch.alerting.opensearchapi.optionalTimeField
 import org.opensearch.common.io.stream.StreamInput
 import org.opensearch.common.io.stream.StreamOutput
 import org.opensearch.common.io.stream.Writeable

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alerts/AlertIndices.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alerts/AlertIndices.kt
@@ -23,7 +23,7 @@ import org.opensearch.action.support.IndicesOptions
 import org.opensearch.action.support.master.AcknowledgedResponse
 import org.opensearch.alerting.alerts.AlertIndices.Companion.ALERT_INDEX
 import org.opensearch.alerting.alerts.AlertIndices.Companion.HISTORY_WRITE_INDEX
-import org.opensearch.alerting.elasticapi.suspendUntil
+import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_HISTORY_ENABLED
 import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_HISTORY_INDEX_MAX_AGE

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alerts/AlertMover.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alerts/AlertMover.kt
@@ -13,9 +13,9 @@ import org.opensearch.action.search.SearchRequest
 import org.opensearch.action.search.SearchResponse
 import org.opensearch.alerting.alerts.AlertIndices.Companion.ALERT_INDEX
 import org.opensearch.alerting.alerts.AlertIndices.Companion.HISTORY_WRITE_INDEX
-import org.opensearch.alerting.elasticapi.suspendUntil
 import org.opensearch.alerting.model.Alert
 import org.opensearch.alerting.model.Monitor
+import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.client.Client
 import org.opensearch.common.bytes.BytesReference
 import org.opensearch.common.xcontent.LoggingDeprecationHandler

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/ActionExecutionResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/ActionExecutionResult.kt
@@ -5,8 +5,8 @@
 
 package org.opensearch.alerting.model
 
-import org.opensearch.alerting.elasticapi.instant
-import org.opensearch.alerting.elasticapi.optionalTimeField
+import org.opensearch.alerting.opensearchapi.instant
+import org.opensearch.alerting.opensearchapi.optionalTimeField
 import org.opensearch.common.io.stream.StreamInput
 import org.opensearch.common.io.stream.StreamOutput
 import org.opensearch.common.io.stream.Writeable

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/Alert.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/Alert.kt
@@ -6,9 +6,9 @@
 package org.opensearch.alerting.model
 
 import org.opensearch.alerting.alerts.AlertError
-import org.opensearch.alerting.elasticapi.instant
-import org.opensearch.alerting.elasticapi.optionalTimeField
-import org.opensearch.alerting.elasticapi.optionalUserField
+import org.opensearch.alerting.opensearchapi.instant
+import org.opensearch.alerting.opensearchapi.optionalTimeField
+import org.opensearch.alerting.opensearchapi.optionalUserField
 import org.opensearch.alerting.util.IndexUtils.Companion.NO_SCHEMA_VERSION
 import org.opensearch.common.io.stream.StreamInput
 import org.opensearch.common.io.stream.StreamOutput

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/AlertingConfigAccessor.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/AlertingConfigAccessor.kt
@@ -10,10 +10,10 @@ import kotlinx.coroutines.withContext
 import org.opensearch.action.get.GetRequest
 import org.opensearch.action.get.GetResponse
 import org.opensearch.alerting.core.model.ScheduledJob
-import org.opensearch.alerting.elasticapi.suspendUntil
 import org.opensearch.alerting.model.destination.Destination
 import org.opensearch.alerting.model.destination.email.EmailAccount
 import org.opensearch.alerting.model.destination.email.EmailGroup
+import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.client.Client
 import org.opensearch.common.bytes.BytesReference
 import org.opensearch.common.xcontent.LoggingDeprecationHandler

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/Monitor.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/Monitor.kt
@@ -11,9 +11,9 @@ import org.opensearch.alerting.core.model.Input
 import org.opensearch.alerting.core.model.Schedule
 import org.opensearch.alerting.core.model.ScheduledJob
 import org.opensearch.alerting.core.model.SearchInput
-import org.opensearch.alerting.elasticapi.instant
-import org.opensearch.alerting.elasticapi.optionalTimeField
-import org.opensearch.alerting.elasticapi.optionalUserField
+import org.opensearch.alerting.opensearchapi.instant
+import org.opensearch.alerting.opensearchapi.optionalTimeField
+import org.opensearch.alerting.opensearchapi.optionalUserField
 import org.opensearch.alerting.settings.AlertingSettings.Companion.MONITOR_MAX_INPUTS
 import org.opensearch.alerting.settings.AlertingSettings.Companion.MONITOR_MAX_TRIGGERS
 import org.opensearch.alerting.settings.SupportedClusterMetricsSettings
@@ -289,7 +289,7 @@ data class Monitor(
                 requireNotNull(schedule) { "Monitor schedule is null" },
                 lastUpdateTime ?: Instant.now(),
                 enabledTime,
-                MonitorType.valueOf(monitorType.toUpperCase(Locale.ROOT)),
+                MonitorType.valueOf(monitorType.uppercase(Locale.ROOT)),
                 user,
                 schemaVersion,
                 inputs.toList(),

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/MonitorRunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/MonitorRunResult.kt
@@ -8,7 +8,7 @@ package org.opensearch.alerting.model
 import org.apache.logging.log4j.LogManager
 import org.opensearch.OpenSearchException
 import org.opensearch.alerting.alerts.AlertError
-import org.opensearch.alerting.elasticapi.optionalTimeField
+import org.opensearch.alerting.opensearchapi.optionalTimeField
 import org.opensearch.common.io.stream.StreamInput
 import org.opensearch.common.io.stream.StreamOutput
 import org.opensearch.common.io.stream.Writeable

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/action/Throttle.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/action/Throttle.kt
@@ -58,7 +58,7 @@ data class Throttle(
                 xcp.nextToken()
                 when (fieldName) {
                     UNIT_FIELD -> {
-                        val unitString = xcp.text().toUpperCase(Locale.ROOT)
+                        val unitString = xcp.text().uppercase(Locale.ROOT)
                         require(StringUtils.equals(unitString, ChronoUnit.MINUTES.name), { "Only support MINUTES throttle unit currently" })
                         unit = ChronoUnit.valueOf(unitString)
                     }

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/destination/Chime.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/destination/Chime.kt
@@ -5,7 +5,7 @@
 
 package org.opensearch.alerting.model.destination
 
-import org.opensearch.alerting.elasticapi.string
+import org.opensearch.alerting.opensearchapi.string
 import org.opensearch.common.Strings
 import org.opensearch.common.io.stream.StreamInput
 import org.opensearch.common.io.stream.StreamOutput

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/destination/Destination.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/destination/Destination.kt
@@ -12,11 +12,11 @@ import org.opensearch.alerting.destination.message.ChimeMessage
 import org.opensearch.alerting.destination.message.CustomWebhookMessage
 import org.opensearch.alerting.destination.message.EmailMessage
 import org.opensearch.alerting.destination.message.SlackMessage
-import org.opensearch.alerting.elasticapi.convertToMap
-import org.opensearch.alerting.elasticapi.instant
-import org.opensearch.alerting.elasticapi.optionalTimeField
-import org.opensearch.alerting.elasticapi.optionalUserField
 import org.opensearch.alerting.model.destination.email.Email
+import org.opensearch.alerting.opensearchapi.convertToMap
+import org.opensearch.alerting.opensearchapi.instant
+import org.opensearch.alerting.opensearchapi.optionalTimeField
+import org.opensearch.alerting.opensearchapi.optionalUserField
 import org.opensearch.alerting.util.DestinationType
 import org.opensearch.alerting.util.IndexUtils.Companion.NO_SCHEMA_VERSION
 import org.opensearch.alerting.util.isHostInDenylist
@@ -193,7 +193,7 @@ data class Destination(
                 schemaVersion,
                 seqNo,
                 primaryTerm,
-                DestinationType.valueOf(type.toUpperCase(Locale.ROOT)),
+                DestinationType.valueOf(type.uppercase(Locale.ROOT)),
                 requireNotNull(name) { "Destination name is null" },
                 user,
                 lastUpdateTime ?: Instant.now(),

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/destination/Slack.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/destination/Slack.kt
@@ -5,7 +5,7 @@
 
 package org.opensearch.alerting.model.destination
 
-import org.opensearch.alerting.elasticapi.string
+import org.opensearch.alerting.opensearchapi.string
 import org.opensearch.common.Strings
 import org.opensearch.common.io.stream.StreamInput
 import org.opensearch.common.io.stream.StreamOutput

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/destination/email/Email.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/destination/email/Email.kt
@@ -173,7 +173,7 @@ data class Recipient(
             }
 
             return Recipient(
-                RecipientType.valueOf(type.toUpperCase(Locale.ROOT)),
+                RecipientType.valueOf(type.uppercase(Locale.ROOT)),
                 emailGroupID,
                 email
             )

diff --git a/...ting/src/main/kotlin/org/opensearch/alerting/transport/TransportAcknowledgeAlertAction.kt b/...ting/src/main/kotlin/org/opensearch/alerting/transport/TransportAcknowledgeAlertAction.kt
@@ -18,8 +18,8 @@ import org.opensearch.alerting.action.AcknowledgeAlertAction
 import org.opensearch.alerting.action.AcknowledgeAlertRequest
 import org.opensearch.alerting.action.AcknowledgeAlertResponse
 import org.opensearch.alerting.alerts.AlertIndices
-import org.opensearch.alerting.elasticapi.optionalTimeField
 import org.opensearch.alerting.model.Alert
+import org.opensearch.alerting.opensearchapi.optionalTimeField
 import org.opensearch.alerting.util.AlertingException
 import org.opensearch.client.Client
 import org.opensearch.common.inject.Inject

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsAction.kt
@@ -15,8 +15,8 @@ import org.opensearch.alerting.action.GetAlertsAction
 import org.opensearch.alerting.action.GetAlertsRequest
 import org.opensearch.alerting.action.GetAlertsResponse
 import org.opensearch.alerting.alerts.AlertIndices
-import org.opensearch.alerting.elasticapi.addFilter
 import org.opensearch.alerting.model.Alert
+import org.opensearch.alerting.opensearchapi.addFilter
 import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.alerting.util.AlertingException
 import org.opensearch.client.Client

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetDestinationsAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetDestinationsAction.kt
@@ -15,8 +15,8 @@ import org.opensearch.alerting.action.GetDestinationsAction
 import org.opensearch.alerting.action.GetDestinationsRequest
 import org.opensearch.alerting.action.GetDestinationsResponse
 import org.opensearch.alerting.core.model.ScheduledJob
-import org.opensearch.alerting.elasticapi.addFilter
 import org.opensearch.alerting.model.destination.Destination
+import org.opensearch.alerting.opensearchapi.addFilter
 import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.alerting.util.AlertingException
 import org.opensearch.client.Client

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt
@@ -13,7 +13,7 @@ import org.opensearch.action.support.ActionFilters
 import org.opensearch.action.support.HandledTransportAction
 import org.opensearch.alerting.action.SearchMonitorAction
 import org.opensearch.alerting.action.SearchMonitorRequest
-import org.opensearch.alerting.elasticapi.addFilter
+import org.opensearch.alerting.opensearchapi.addFilter
 import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.alerting.util.AlertingException
 import org.opensearch.client.Client

diff --git a/...src/main/kotlin/org/opensearch/alerting/util/SupportedClusterMetricsSettingsExtensions.kt b/...src/main/kotlin/org/opensearch/alerting/util/SupportedClusterMetricsSettingsExtensions.kt
@@ -24,7 +24,7 @@ import org.opensearch.action.admin.indices.recovery.RecoveryRequest
 import org.opensearch.action.admin.indices.recovery.RecoveryResponse
 import org.opensearch.alerting.core.model.ClusterMetricsInput
 import org.opensearch.alerting.core.model.ClusterMetricsInput.ClusterMetricType
-import org.opensearch.alerting.elasticapi.convertToMap
+import org.opensearch.alerting.opensearchapi.convertToMap
 import org.opensearch.alerting.settings.SupportedClusterMetricsSettings
 import org.opensearch.alerting.settings.SupportedClusterMetricsSettings.Companion.resolveToActionRequest
 import org.opensearch.client.Client