diff --git a/plugins/events-correlation-engine/src/internalClusterTest/java/org/opensearch/plugin/correlation/EventsCorrelationPluginTransportIT.java b/plugins/events-correlation-engine/src/internalClusterTest/java/org/opensearch/plugin/correlation/EventsCorrelationPluginTransportIT.java index 028848a91213e..237855cd6d511 100644 --- a/plugins/events-correlation-engine/src/internalClusterTest/java/org/opensearch/plugin/correlation/EventsCorrelationPluginTransportIT.java +++ b/plugins/events-correlation-engine/src/internalClusterTest/java/org/opensearch/plugin/correlation/EventsCorrelationPluginTransportIT.java @@ -13,11 +13,24 @@ import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest; import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse; import org.opensearch.action.admin.cluster.node.info.PluginsAndModules; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.rest.RestStatus; import org.opensearch.index.query.NestedQueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.plugin.correlation.events.action.IndexCorrelationAction; +import org.opensearch.plugin.correlation.events.action.IndexCorrelationRequest; +import org.opensearch.plugin.correlation.events.action.IndexCorrelationResponse; +import org.opensearch.plugin.correlation.events.action.SearchCorrelatedEventsAction; +import org.opensearch.plugin.correlation.events.action.SearchCorrelatedEventsRequest; +import org.opensearch.plugin.correlation.events.action.SearchCorrelatedEventsResponse; +import org.opensearch.plugin.correlation.events.model.Correlation; import org.opensearch.plugin.correlation.rules.action.IndexCorrelationRuleAction; import org.opensearch.plugin.correlation.rules.action.IndexCorrelationRuleRequest; import org.opensearch.plugin.correlation.rules.action.IndexCorrelationRuleResponse; @@ -30,10 +43,13 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.Assert; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -174,4 +190,739 @@ public void testCreatingACorrelationRuleWithNoTimestampField() throws Exception .get("timestampField") ); } + + public void testEventOnIndexWithNoRules() throws ExecutionException, InterruptedException { + String windowsIndex = "windows"; + CreateIndexRequest windowsRequest = new CreateIndexRequest(windowsIndex).mapping(windowsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(windowsRequest).get(); + + String appLogsIndex = "app_logs"; + CreateIndexRequest appLogsRequest = new CreateIndexRequest(appLogsIndex).mapping(appLogsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(appLogsRequest).get(); + + List correlationQueries = List.of( + new CorrelationQuery("app_logs", "endpoint:\\/customer_records.txt", "timestamp", List.of()) + ); + CorrelationRule correlationRule = new CorrelationRule("windows to app logs", correlationQueries); + IndexCorrelationRuleRequest request = new IndexCorrelationRuleRequest(correlationRule, RestRequest.Method.POST); + client().execute(IndexCorrelationRuleAction.INSTANCE, request).get(); + + IndexRequest indexRequestWindows = new IndexRequest("windows").source(sampleWindowsEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse response = client().index(indexRequestWindows).get(); + String eventId = response.getId(); + + IndexCorrelationRequest correlationRequest = new IndexCorrelationRequest("windows", eventId, false); + IndexCorrelationResponse correlationResponse = client().execute(IndexCorrelationAction.INSTANCE, correlationRequest).get(); + + Assert.assertEquals(200, correlationResponse.getStatus().getStatus()); + Assert.assertTrue(correlationResponse.getOrphan()); + Assert.assertEquals(0, correlationResponse.getNeighborEvents().size()); + } + + public void testEventOnIndexWithNoMatchingRules() throws ExecutionException, InterruptedException { + String windowsIndex = "windows"; + CreateIndexRequest windowsRequest = new CreateIndexRequest(windowsIndex).mapping(windowsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(windowsRequest).get(); + + String appLogsIndex = "app_logs"; + CreateIndexRequest appLogsRequest = new CreateIndexRequest(appLogsIndex).mapping(appLogsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(appLogsRequest).get(); + + List correlationQueries = Arrays.asList( + new CorrelationQuery("windows", "host.hostname:EC2BMAZ*", "winlog.timestamp", List.of()) + ); + CorrelationRule correlationRule = new CorrelationRule("windows to app logs", correlationQueries); + IndexCorrelationRuleRequest request = new IndexCorrelationRuleRequest(correlationRule, RestRequest.Method.POST); + client().execute(IndexCorrelationRuleAction.INSTANCE, request).get(); + + IndexRequest indexRequestWindows = new IndexRequest("windows").source(sampleWindowsEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse response = client().index(indexRequestWindows).get(); + String eventId = response.getId(); + + IndexCorrelationRequest correlationRequest = new IndexCorrelationRequest("windows", eventId, false); + IndexCorrelationResponse correlationResponse = client().execute(IndexCorrelationAction.INSTANCE, correlationRequest).get(); + + Assert.assertEquals(200, correlationResponse.getStatus().getStatus()); + Assert.assertTrue(correlationResponse.getOrphan()); + Assert.assertEquals(0, correlationResponse.getNeighborEvents().size()); + } + + public void testCorrelationWithSingleRule() throws ExecutionException, InterruptedException { + String windowsIndex = "windows"; + CreateIndexRequest windowsRequest = new CreateIndexRequest(windowsIndex).mapping(windowsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(windowsRequest).get(); + + String appLogsIndex = "app_logs"; + CreateIndexRequest appLogsRequest = new CreateIndexRequest(appLogsIndex).mapping(appLogsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(appLogsRequest).get(); + + List correlationQueries = Arrays.asList( + new CorrelationQuery("windows", "host.hostname:EC2AMAZ*", "winlog.timestamp", List.of()), + new CorrelationQuery("app_logs", "endpoint:\\/customer_records.txt", "timestamp", List.of()) + ); + CorrelationRule correlationRule = new CorrelationRule("windows to app logs", correlationQueries); + IndexCorrelationRuleRequest request = new IndexCorrelationRuleRequest(correlationRule, RestRequest.Method.POST); + client().execute(IndexCorrelationRuleAction.INSTANCE, request).get(); + + IndexRequest indexRequestWindows = new IndexRequest("windows").source(sampleWindowsEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + client().index(indexRequestWindows).get(); + + IndexRequest indexRequestAppLogs = new IndexRequest("app_logs").source(sampleAppLogsEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse response = client().index(indexRequestAppLogs).get(); + String eventId = response.getId(); + + IndexCorrelationRequest correlationRequest = new IndexCorrelationRequest("app_logs", eventId, false); + IndexCorrelationResponse correlationResponse = client().execute(IndexCorrelationAction.INSTANCE, correlationRequest).get(); + + Assert.assertEquals(200, correlationResponse.getStatus().getStatus()); + Assert.assertEquals(1, correlationResponse.getNeighborEvents().size()); + Assert.assertFalse(correlationResponse.getOrphan()); + } + + public void testSearchCorrelationWithSingleRule() throws ExecutionException, InterruptedException, IOException { + String windowsIndex = "windows"; + CreateIndexRequest windowsRequest = new CreateIndexRequest(windowsIndex).mapping(windowsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(windowsRequest).get(); + + String appLogsIndex = "app_logs"; + CreateIndexRequest appLogsRequest = new CreateIndexRequest(appLogsIndex).mapping(appLogsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(appLogsRequest).get(); + + List correlationQueries = Arrays.asList( + new CorrelationQuery("windows", "host.hostname:EC2AMAZ*", "winlog.timestamp", List.of()), + new CorrelationQuery("app_logs", "endpoint:\\/customer_records.txt", "timestamp", List.of()) + ); + CorrelationRule correlationRule = new CorrelationRule("windows to app logs", correlationQueries); + IndexCorrelationRuleRequest request = new IndexCorrelationRuleRequest(correlationRule, RestRequest.Method.POST); + client().execute(IndexCorrelationRuleAction.INSTANCE, request).get(); + + IndexRequest indexRequestWindows = new IndexRequest("windows").source(sampleWindowsEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse response = client().index(indexRequestWindows).get(); + String windowsEventId = response.getId(); + + IndexRequest indexRequestAppLogs = new IndexRequest("app_logs").source(sampleAppLogsEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + response = client().index(indexRequestAppLogs).get(); + String eventId = response.getId(); + + IndexCorrelationRequest correlationRequest = new IndexCorrelationRequest("app_logs", eventId, true); + IndexCorrelationResponse correlationResponse = client().execute(IndexCorrelationAction.INSTANCE, correlationRequest).get(); + + Assert.assertEquals(200, correlationResponse.getStatus().getStatus()); + Assert.assertEquals(1, correlationResponse.getNeighborEvents().size()); + Assert.assertFalse(correlationResponse.getOrphan()); + + correlationRequest = new IndexCorrelationRequest("windows", windowsEventId, true); + client().execute(IndexCorrelationAction.INSTANCE, correlationRequest).get(); + + SearchCorrelatedEventsRequest searchCorrelatedEventsRequest = new SearchCorrelatedEventsRequest( + "windows", + windowsEventId, + "winlog.timestamp", + 300000L, + 5 + ); + SearchCorrelatedEventsResponse correlatedEventsResponse = client().execute( + SearchCorrelatedEventsAction.INSTANCE, + searchCorrelatedEventsRequest + ).get(); + Assert.assertEquals(1, correlatedEventsResponse.getEvents().size()); + } + + public void testCorrelationWithMultipleRule() throws ExecutionException, InterruptedException { + String windowsIndex = "windows"; + CreateIndexRequest windowsRequest = new CreateIndexRequest(windowsIndex).mapping(windowsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(windowsRequest).get(); + + String appLogsIndex = "app_logs"; + CreateIndexRequest appLogsRequest = new CreateIndexRequest(appLogsIndex).mapping(appLogsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(appLogsRequest).get(); + + List correlationQueries1 = Arrays.asList( + new CorrelationQuery("windows", "host.hostname:EC2AMAZ*", "winlog.timestamp", List.of()), + new CorrelationQuery("app_logs", "endpoint:\\/customer_records.txt", "timestamp", List.of()) + ); + CorrelationRule correlationRule1 = new CorrelationRule("windows to app logs", correlationQueries1); + IndexCorrelationRuleRequest request1 = new IndexCorrelationRuleRequest(correlationRule1, RestRequest.Method.POST); + client().execute(IndexCorrelationRuleAction.INSTANCE, request1).get(); + + List correlationQueries2 = Arrays.asList( + new CorrelationQuery("windows", "host.hostname:EC2BMAZ*", "winlog.timestamp", List.of()), + new CorrelationQuery("app_logs", "endpoint:\\/customer_records1.txt", "timestamp", List.of()) + ); + CorrelationRule correlationRule2 = new CorrelationRule("windows to app logs", correlationQueries2); + IndexCorrelationRuleRequest request2 = new IndexCorrelationRuleRequest(correlationRule2, RestRequest.Method.POST); + client().execute(IndexCorrelationRuleAction.INSTANCE, request2).get(); + + IndexRequest indexRequestWindows = new IndexRequest("windows").source(sampleWindowsEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + client().index(indexRequestWindows).get(); + + IndexRequest indexRequestAppLogs = new IndexRequest("app_logs").source(sampleAppLogsEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse response = client().index(indexRequestAppLogs).get(); + String eventId = response.getId(); + + IndexCorrelationRequest correlationRequest = new IndexCorrelationRequest("app_logs", eventId, false); + IndexCorrelationResponse correlationResponse = client().execute(IndexCorrelationAction.INSTANCE, correlationRequest).get(); + + Assert.assertEquals(200, correlationResponse.getStatus().getStatus()); + Assert.assertFalse(correlationResponse.getOrphan()); + Assert.assertEquals(1, correlationResponse.getNeighborEvents().size()); + } + + public void testStoringCorrelationWithMultipleRule() throws ExecutionException, InterruptedException { + String networkIndex = "vpc_flow"; + CreateIndexRequest networkRequest = new CreateIndexRequest(networkIndex).mapping(networkMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(networkRequest).get(); + + String adLdapIndex = "ad_logs"; + CreateIndexRequest adLdapRequest = new CreateIndexRequest(adLdapIndex).mapping(adLdapMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(adLdapRequest).get(); + + String windowsIndex = "windows"; + CreateIndexRequest windowsRequest = new CreateIndexRequest(windowsIndex).mapping(windowsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(windowsRequest).get(); + + String appLogsIndex = "app_logs"; + CreateIndexRequest appLogsRequest = new CreateIndexRequest(appLogsIndex).mapping(appLogsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(appLogsRequest).get(); + + String s3AccessLogsIndex = "s3_access_logs"; + CreateIndexRequest s3AccessLogsRequest = new CreateIndexRequest(s3AccessLogsIndex).mapping(s3AccessLogsMapping()) + .settings(Settings.EMPTY); + + client().admin().indices().create(s3AccessLogsRequest).get(); + + List windowsAppLogsQuery = Arrays.asList( + new CorrelationQuery("windows", "host.hostname:EC2AMAZ*", "winlog.timestamp", List.of()), + new CorrelationQuery("app_logs", "endpoint:\\/customer_records.txt", "timestamp", List.of()) + ); + CorrelationRule windowsAppLogsRule = new CorrelationRule("windows to app logs", windowsAppLogsQuery); + IndexCorrelationRuleRequest windowsAppLogsRequest = new IndexCorrelationRuleRequest(windowsAppLogsRule, RestRequest.Method.POST); + client().execute(IndexCorrelationRuleAction.INSTANCE, windowsAppLogsRequest).get(); + + List networkWindowsAdLdapQuery = Arrays.asList( + new CorrelationQuery("vpc_flow", "dstaddr:4.5.6.7 or dstaddr:4.5.6.6", "timestamp", List.of()), + new CorrelationQuery("windows", "winlog.event_data.SubjectDomainName:NTAUTHORI*", "winlog.timestamp", List.of()), + new CorrelationQuery("ad_logs", "ResultType:50126", "timestamp", List.of()) + ); + CorrelationRule networkWindowsAdLdapRule = new CorrelationRule("netowrk to windows to ad/ldap", networkWindowsAdLdapQuery); + IndexCorrelationRuleRequest networkWindowsAdLdapRequest = new IndexCorrelationRuleRequest( + networkWindowsAdLdapRule, + RestRequest.Method.POST + ); + client().execute(IndexCorrelationRuleAction.INSTANCE, networkWindowsAdLdapRequest).get(); + + List s3AppLogsQuery = Arrays.asList( + new CorrelationQuery("s3_access_logs", "aws.cloudtrail.eventName:ReplicateObject", "timestamp", List.of()), + new CorrelationQuery("app_logs", "keywords:PermissionDenied", "timestamp", List.of()) + ); + CorrelationRule s3AppLogsRule = new CorrelationRule("s3 to app logs", s3AppLogsQuery); + IndexCorrelationRuleRequest s3AppLogsRequest = new IndexCorrelationRuleRequest(s3AppLogsRule, RestRequest.Method.POST); + client().execute(IndexCorrelationRuleAction.INSTANCE, s3AppLogsRequest).get(); + + IndexRequest indexRequestNetwork = new IndexRequest("vpc_flow").source(sampleNetworkEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponseNetwork = client().index(indexRequestNetwork).get(); + + String networkEventId = indexResponseNetwork.getId(); + IndexCorrelationRequest networkCorrelationRequest = new IndexCorrelationRequest("vpc_flow", networkEventId, true); + IndexCorrelationResponse networkCorrelationResponse = client().execute(IndexCorrelationAction.INSTANCE, networkCorrelationRequest) + .get(); + Assert.assertEquals(200, networkCorrelationResponse.getStatus().getStatus()); + Assert.assertEquals(true, networkCorrelationResponse.getOrphan()); + + IndexRequest indexRequestAdLdap = new IndexRequest("ad_logs").source(sampleAdLdapEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponseAdLdap = client().index(indexRequestAdLdap).get(); + + String adLdapEventId = indexResponseAdLdap.getId(); + IndexCorrelationRequest adLdapCorrelationRequest = new IndexCorrelationRequest("ad_logs", adLdapEventId, true); + IndexCorrelationResponse adLdapCorrelationResponse = client().execute(IndexCorrelationAction.INSTANCE, adLdapCorrelationRequest) + .get(); + Assert.assertEquals(200, adLdapCorrelationResponse.getStatus().getStatus()); + Assert.assertEquals(false, adLdapCorrelationResponse.getOrphan()); + Assert.assertEquals(1, adLdapCorrelationResponse.getNeighborEvents().size()); + + IndexRequest indexRequestWindows = new IndexRequest("windows").source(sampleWindowsEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponseWindows = client().index(indexRequestWindows).get(); + + String windowsEventId = indexResponseWindows.getId(); + IndexCorrelationRequest windowsCorrelationRequest = new IndexCorrelationRequest("windows", windowsEventId, true); + IndexCorrelationResponse windowsCorrelationResponse = client().execute(IndexCorrelationAction.INSTANCE, windowsCorrelationRequest) + .get(); + Assert.assertEquals(200, windowsCorrelationResponse.getStatus().getStatus()); + Assert.assertEquals(false, windowsCorrelationResponse.getOrphan()); + Assert.assertEquals(2, windowsCorrelationResponse.getNeighborEvents().size()); + + IndexRequest indexRequestAppLogs = new IndexRequest("app_logs").source(sampleAppLogsEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponseAppLogs = client().index(indexRequestAppLogs).get(); + + String appLogsEventId = indexResponseAppLogs.getId(); + IndexCorrelationRequest appLogsCorrelationRequest = new IndexCorrelationRequest("app_logs", appLogsEventId, true); + IndexCorrelationResponse appLogsCorrelationResponse = client().execute(IndexCorrelationAction.INSTANCE, appLogsCorrelationRequest) + .get(); + Assert.assertEquals(200, appLogsCorrelationResponse.getStatus().getStatus()); + Assert.assertEquals(false, appLogsCorrelationResponse.getOrphan()); + Assert.assertEquals(1, appLogsCorrelationResponse.getNeighborEvents().size()); + + IndexRequest indexRequestS3Logs = new IndexRequest("s3_access_logs").source(sampleS3AccessEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponseS3 = client().index(indexRequestS3Logs).get(); + + String s3EventId = indexResponseS3.getId(); + IndexCorrelationRequest s3CorrelationRequest = new IndexCorrelationRequest("s3_access_logs", s3EventId, true); + IndexCorrelationResponse s3CorrelationResponse = client().execute(IndexCorrelationAction.INSTANCE, s3CorrelationRequest).get(); + Assert.assertEquals(200, s3CorrelationResponse.getStatus().getStatus()); + Assert.assertEquals(false, s3CorrelationResponse.getOrphan()); + Assert.assertEquals(1, s3CorrelationResponse.getNeighborEvents().size()); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(100); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(Correlation.CORRELATION_HISTORY_INDEX); + searchRequest.source(searchSourceBuilder); + + SearchResponse searchResponse = client().search(searchRequest).get(); + Assert.assertEquals(12L, searchResponse.getHits().getTotalHits().value); + } + + public void testStoringCorrelationWithMultipleLevels() throws ExecutionException, InterruptedException { + String networkIndex = "vpc_flow"; + CreateIndexRequest networkRequest = new CreateIndexRequest(networkIndex).mapping(networkMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(networkRequest).get(); + + String adLdapIndex = "ad_logs"; + CreateIndexRequest adLdapRequest = new CreateIndexRequest(adLdapIndex).mapping(adLdapMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(adLdapRequest).get(); + + String windowsIndex = "windows"; + CreateIndexRequest windowsRequest = new CreateIndexRequest(windowsIndex).mapping(windowsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(windowsRequest).get(); + + String appLogsIndex = "app_logs"; + CreateIndexRequest appLogsRequest = new CreateIndexRequest(appLogsIndex).mapping(appLogsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(appLogsRequest).get(); + + List networkWindowsAdLdapQuery = Arrays.asList( + new CorrelationQuery("vpc_flow", "dstaddr:4.5.6.7 or dstaddr:4.5.6.6", "timestamp", List.of()), + new CorrelationQuery("windows", "winlog.event_data.SubjectDomainName:NTAUTHORI*", "winlog.timestamp", List.of()), + new CorrelationQuery("ad_logs", "ResultType:50126", "timestamp", List.of()) + ); + CorrelationRule networkWindowsAdLdapRule = new CorrelationRule("netowrk to windows to ad/ldap", networkWindowsAdLdapQuery); + IndexCorrelationRuleRequest networkWindowsAdLdapRequest = new IndexCorrelationRuleRequest( + networkWindowsAdLdapRule, + RestRequest.Method.POST + ); + client().execute(IndexCorrelationRuleAction.INSTANCE, networkWindowsAdLdapRequest).get(); + + IndexRequest indexRequestNetwork = new IndexRequest("vpc_flow").source(sampleNetworkEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponseNetwork = client().index(indexRequestNetwork).get(); + + String networkEventId = indexResponseNetwork.getId(); + IndexCorrelationRequest networkCorrelationRequest = new IndexCorrelationRequest("vpc_flow", networkEventId, true); + IndexCorrelationResponse networkCorrelationResponse = client().execute(IndexCorrelationAction.INSTANCE, networkCorrelationRequest) + .get(); + Assert.assertEquals(200, networkCorrelationResponse.getStatus().getStatus()); + Assert.assertEquals(true, networkCorrelationResponse.getOrphan()); + + IndexRequest indexRequestAdLdap = new IndexRequest("ad_logs").source(sampleAdLdapEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponseAdLdap = client().index(indexRequestAdLdap).get(); + + String adLdapEventId = indexResponseAdLdap.getId(); + IndexCorrelationRequest adLdapCorrelationRequest = new IndexCorrelationRequest("ad_logs", adLdapEventId, true); + IndexCorrelationResponse adLdapCorrelationResponse = client().execute(IndexCorrelationAction.INSTANCE, adLdapCorrelationRequest) + .get(); + Assert.assertEquals(200, adLdapCorrelationResponse.getStatus().getStatus()); + Assert.assertEquals(false, adLdapCorrelationResponse.getOrphan()); + Assert.assertEquals(1, adLdapCorrelationResponse.getNeighborEvents().size()); + + IndexRequest indexRequestWindows = new IndexRequest("windows").source(sampleWindowsEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponseWindows = client().index(indexRequestWindows).get(); + + String windowsEventId = indexResponseWindows.getId(); + IndexCorrelationRequest windowsCorrelationRequest = new IndexCorrelationRequest("windows", windowsEventId, true); + IndexCorrelationResponse windowsCorrelationResponse = client().execute(IndexCorrelationAction.INSTANCE, windowsCorrelationRequest) + .get(); + Assert.assertEquals(200, windowsCorrelationResponse.getStatus().getStatus()); + Assert.assertEquals(false, windowsCorrelationResponse.getOrphan()); + Assert.assertEquals(2, windowsCorrelationResponse.getNeighborEvents().size()); + + IndexRequest indexRequestAppLogs = new IndexRequest("app_logs").source(sampleAppLogsEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponseAppLogs = client().index(indexRequestAppLogs).get(); + + String appLogsEventId = indexResponseAppLogs.getId(); + IndexCorrelationRequest appLogsCorrelationRequest = new IndexCorrelationRequest("app_logs", appLogsEventId, true); + IndexCorrelationResponse appLogsCorrelationResponse = client().execute(IndexCorrelationAction.INSTANCE, appLogsCorrelationRequest) + .get(); + Assert.assertEquals(200, appLogsCorrelationResponse.getStatus().getStatus()); + Assert.assertEquals(true, appLogsCorrelationResponse.getOrphan()); + Assert.assertEquals(0, appLogsCorrelationResponse.getNeighborEvents().size()); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.matchQuery("index1", "app_logs")); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(100); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(Correlation.CORRELATION_HISTORY_INDEX); + searchRequest.source(searchSourceBuilder); + + SearchResponse searchResponse = client().search(searchRequest).get(); + Assert.assertEquals(1L, searchResponse.getHits().getTotalHits().value); + Assert.assertEquals(100, Objects.requireNonNull(searchResponse.getHits().getHits()[0].getSourceAsMap()).get("level")); + } + + public void testStoringCorrelationWithMultipleLevelsWithSeparateGroups() throws ExecutionException, InterruptedException { + String networkIndex = "vpc_flow"; + CreateIndexRequest networkRequest = new CreateIndexRequest(networkIndex).mapping(networkMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(networkRequest).get(); + + String adLdapIndex = "ad_logs"; + CreateIndexRequest adLdapRequest = new CreateIndexRequest(adLdapIndex).mapping(adLdapMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(adLdapRequest).get(); + + String windowsIndex = "windows"; + CreateIndexRequest windowsRequest = new CreateIndexRequest(windowsIndex).mapping(windowsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(windowsRequest).get(); + + String appLogsIndex = "app_logs"; + CreateIndexRequest appLogsRequest = new CreateIndexRequest(appLogsIndex).mapping(appLogsMappings()).settings(Settings.EMPTY); + + client().admin().indices().create(appLogsRequest).get(); + + List networkWindowsAdLdapQuery = Arrays.asList( + new CorrelationQuery("vpc_flow", "dstaddr:4.5.6.7 or dstaddr:4.5.6.6", "timestamp", List.of()), + new CorrelationQuery("windows", "winlog.event_data.SubjectDomainName:NTAUTHORI*", "winlog.timestamp", List.of()), + new CorrelationQuery("ad_logs", "ResultType:50126", "timestamp", List.of()) + ); + CorrelationRule networkWindowsAdLdapRule = new CorrelationRule("netowrk to windows to ad/ldap", networkWindowsAdLdapQuery); + IndexCorrelationRuleRequest networkWindowsAdLdapRequest = new IndexCorrelationRuleRequest( + networkWindowsAdLdapRule, + RestRequest.Method.POST + ); + client().execute(IndexCorrelationRuleAction.INSTANCE, networkWindowsAdLdapRequest).get(); + + List s3AppLogsQuery = Arrays.asList( + new CorrelationQuery("s3_access_logs", "aws.cloudtrail.eventName:ReplicateObject", "timestamp", List.of()), + new CorrelationQuery("app_logs", "keywords:PermissionDenied", "timestamp", List.of()) + ); + CorrelationRule s3AppLogsRule = new CorrelationRule("s3 to app logs", s3AppLogsQuery); + IndexCorrelationRuleRequest s3AppLogsRequest = new IndexCorrelationRuleRequest(s3AppLogsRule, RestRequest.Method.POST); + client().execute(IndexCorrelationRuleAction.INSTANCE, s3AppLogsRequest).get(); + + IndexRequest indexRequestNetwork = new IndexRequest("vpc_flow").source(sampleNetworkEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponseNetwork = client().index(indexRequestNetwork).get(); + + String networkEventId = indexResponseNetwork.getId(); + IndexCorrelationRequest networkCorrelationRequest = new IndexCorrelationRequest("vpc_flow", networkEventId, true); + IndexCorrelationResponse networkCorrelationResponse = client().execute(IndexCorrelationAction.INSTANCE, networkCorrelationRequest) + .get(); + Assert.assertEquals(200, networkCorrelationResponse.getStatus().getStatus()); + Assert.assertEquals(true, networkCorrelationResponse.getOrphan()); + + IndexRequest indexRequestAdLdap = new IndexRequest("ad_logs").source(sampleAdLdapEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponseAdLdap = client().index(indexRequestAdLdap).get(); + + String adLdapEventId = indexResponseAdLdap.getId(); + IndexCorrelationRequest adLdapCorrelationRequest = new IndexCorrelationRequest("ad_logs", adLdapEventId, true); + IndexCorrelationResponse adLdapCorrelationResponse = client().execute(IndexCorrelationAction.INSTANCE, adLdapCorrelationRequest) + .get(); + Assert.assertEquals(200, adLdapCorrelationResponse.getStatus().getStatus()); + Assert.assertEquals(false, adLdapCorrelationResponse.getOrphan()); + Assert.assertEquals(1, adLdapCorrelationResponse.getNeighborEvents().size()); + + IndexRequest indexRequestWindows = new IndexRequest("windows").source(sampleWindowsEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponseWindows = client().index(indexRequestWindows).get(); + + String windowsEventId = indexResponseWindows.getId(); + IndexCorrelationRequest windowsCorrelationRequest = new IndexCorrelationRequest("windows", windowsEventId, true); + IndexCorrelationResponse windowsCorrelationResponse = client().execute(IndexCorrelationAction.INSTANCE, windowsCorrelationRequest) + .get(); + Assert.assertEquals(200, windowsCorrelationResponse.getStatus().getStatus()); + Assert.assertEquals(false, windowsCorrelationResponse.getOrphan()); + Assert.assertEquals(2, windowsCorrelationResponse.getNeighborEvents().size()); + + IndexRequest indexRequestAppLogs = new IndexRequest("app_logs").source(sampleAppLogsEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponseAppLogs = client().index(indexRequestAppLogs).get(); + + String appLogsEventId = indexResponseAppLogs.getId(); + IndexCorrelationRequest appLogsCorrelationRequest = new IndexCorrelationRequest("app_logs", appLogsEventId, true); + IndexCorrelationResponse appLogsCorrelationResponse = client().execute(IndexCorrelationAction.INSTANCE, appLogsCorrelationRequest) + .get(); + Assert.assertEquals(200, appLogsCorrelationResponse.getStatus().getStatus()); + Assert.assertEquals(true, appLogsCorrelationResponse.getOrphan()); + + IndexRequest indexRequestS3Logs = new IndexRequest("s3_access_logs").source(sampleS3AccessEvent(), XContentType.JSON) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexResponse indexResponseS3 = client().index(indexRequestS3Logs).get(); + + String s3EventId = indexResponseS3.getId(); + IndexCorrelationRequest s3CorrelationRequest = new IndexCorrelationRequest("s3_access_logs", s3EventId, true); + IndexCorrelationResponse s3CorrelationResponse = client().execute(IndexCorrelationAction.INSTANCE, s3CorrelationRequest).get(); + Assert.assertEquals(200, s3CorrelationResponse.getStatus().getStatus()); + Assert.assertEquals(false, s3CorrelationResponse.getOrphan()); + Assert.assertEquals(1, s3CorrelationResponse.getNeighborEvents().size()); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query( + QueryBuilders.boolQuery() + .must(QueryBuilders.matchQuery("index2", "app_logs")) + .must(QueryBuilders.matchQuery("index1", "s3_access_logs")) + ); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(100); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(Correlation.CORRELATION_HISTORY_INDEX); + searchRequest.source(searchSourceBuilder); + + SearchResponse searchResponse = client().search(searchRequest).get(); + Assert.assertEquals(1L, searchResponse.getHits().getTotalHits().value); + Assert.assertEquals(75, Objects.requireNonNull(searchResponse.getHits().getHits()[0].getSourceAsMap()).get("level")); + } + + private String networkMappings() { + return "\"properties\": {\n" + + " \"version\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"account-id\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"interface-id\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"srcaddr\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"dstaddr\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"srcport\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"dstport\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"severity_id\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"class_name\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"timestamp\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " }"; + } + + private String sampleNetworkEvent() { + return "{\n" + + " \"version\": 1,\n" + + " \"account-id\": \"A12345\",\n" + + " \"interface-id\": \"I12345\",\n" + + " \"srcaddr\": \"1.2.3.4\",\n" + + " \"dstaddr\": \"4.5.6.7\",\n" + + " \"srcport\": 9000,\n" + + " \"dstport\": 8000,\n" + + " \"severity_id\": \"-1\",\n" + + " \"class_name\": \"Network Activity\",\n" + + " \"timestamp\": " + + System.currentTimeMillis() + + "\n" + + "}"; + } + + private String adLdapMappings() { + return "\"properties\": {\n" + + " \"ResultType\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"ResultDescription\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"winlog.event_data.TargetUserName\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"timestamp\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " }"; + } + + private String sampleAdLdapEvent() { + return "{\n" + + " \"ResultType\": 50126,\n" + + " \"ResultDescription\": \"Invalid username or password or Invalid on-premises username or password.\",\n" + + " \"winlog.event_data.TargetUserName\": \"DEYSUBHO\",\n" + + " \"timestamp\": " + + System.currentTimeMillis() + + "\n" + + "}"; + } + + private String windowsMappings() { + return " \"properties\": {\n" + + " \"server.user.hash\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"winlog.event_id\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"host.hostname\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"windows.message\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"winlog.provider_name\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"winlog.event_data.ServiceName\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"winlog.timestamp\": {\n" + + " \"type\": \"long\"\n" + + " }\n" + + " }\n"; + } + + private String sampleWindowsEvent() { + return "{\n" + + " \"EventTime\": \"2020-02-04T14:59:39.343541+00:00\",\n" + + " \"host.hostname\": \"EC2AMAZ-EPO7HKA\",\n" + + " \"Keywords\": \"9223372036854775808\",\n" + + " \"SeverityValue\": 2,\n" + + " \"Severity\": \"INFO\",\n" + + " \"winlog.event_id\": 22,\n" + + " \"SourceName\": \"Microsoft-Windows-Sysmon\",\n" + + " \"ProviderGuid\": \"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + " \"Version\": 5,\n" + + " \"TaskValue\": 22,\n" + + " \"OpcodeValue\": 0,\n" + + " \"RecordNumber\": 9532,\n" + + " \"ExecutionProcessID\": 1996,\n" + + " \"ExecutionThreadID\": 2616,\n" + + " \"Channel\": \"Microsoft-Windows-Sysmon/Operational\",\n" + + " \"winlog.event_data.SubjectDomainName\": \"NTAUTHORITY\",\n" + + " \"AccountName\": \"SYSTEM\",\n" + + " \"UserID\": \"S-1-5-18\",\n" + + " \"AccountType\": \"User\",\n" + + " \"windows.message\": \"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + " \"Category\": \"Dns query (rule: DnsQuery)\",\n" + + " \"Opcode\": \"Info\",\n" + + " \"UtcTime\": \"2020-02-04 14:59:38.349\",\n" + + " \"ProcessGuid\": \"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + " \"ProcessId\": \"1904\",\n" + + " \"QueryName\": \"EC2AMAZ-EPO7HKA\",\n" + + " \"QueryStatus\": \"0\",\n" + + " \"QueryResults\": \"172.31.46.38;\",\n" + + " \"Image\": \"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + " \"EventReceivedTime\": \"2020-02-04T14:59:40.780905+00:00\",\n" + + " \"SourceModuleName\": \"in\",\n" + + " \"SourceModuleType\": \"im_msvistalog\",\n" + + " \"CommandLine\": \"eachtest\",\n" + + " \"Initiated\": \"true\",\n" + + " \"winlog.timestamp\": " + + System.currentTimeMillis() + + "\n" + + "}"; + } + + private String appLogsMappings() { + return " \"properties\": {\n" + + " \"http_method\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"endpoint\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"keywords\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"timestamp\": {\n" + + " \"type\": \"long\"\n" + + " }\n" + + " }\n"; + } + + private String sampleAppLogsEvent() { + return "{\n" + + " \"endpoint\": \"/customer_records.txt\",\n" + + " \"http_method\": \"POST\",\n" + + " \"keywords\": \"PermissionDenied\",\n" + + " \"timestamp\": " + + System.currentTimeMillis() + + "\n" + + "}"; + } + + private String s3AccessLogsMapping() { + return "\"properties\": {\n" + + " \"aws.cloudtrail.eventSource\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"aws.cloudtrail.eventName\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"aws.cloudtrail.eventTime\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"timestamp\": {\n" + + " \"type\": \"long\"\n" + + " }\n" + + " }"; + } + + private String sampleS3AccessEvent() { + return "{\n" + + " \"aws.cloudtrail.eventSource\": \"s3.amazonaws.com\",\n" + + " \"aws.cloudtrail.eventName\": \"ReplicateObject\",\n" + + " \"aws.cloudtrail.eventTime\": 1,\n" + + " \"timestamp\": " + + System.currentTimeMillis() + + "\n" + + "}"; + } } diff --git a/plugins/events-correlation-engine/src/javaRestTest/java/org/opensearch/plugin/correlation/EventsCorrelationPluginRestIT.java b/plugins/events-correlation-engine/src/javaRestTest/java/org/opensearch/plugin/correlation/EventsCorrelationPluginRestIT.java index 3791a5cdf5db0..e042ea119989c 100644 --- a/plugins/events-correlation-engine/src/javaRestTest/java/org/opensearch/plugin/correlation/EventsCorrelationPluginRestIT.java +++ b/plugins/events-correlation-engine/src/javaRestTest/java/org/opensearch/plugin/correlation/EventsCorrelationPluginRestIT.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.List; +import java.util.Locale; import java.util.Map; /** @@ -102,6 +103,67 @@ public void testCreatingACorrelationRuleWithNoTimestampField() throws IOExceptio ); } + @SuppressWarnings("unchecked") + public void testCorrelationWithSingleRule() throws IOException { + String windowsIndex = "windows"; + Request request = new Request("PUT", "/" + windowsIndex); + request.setJsonEntity(windowsMappings()); + client().performRequest(request); + + String appLogsIndex = "app_logs"; + request = new Request("PUT", "/" + appLogsIndex); + request.setJsonEntity(appLogMappings()); + client().performRequest(request); + + String correlationRule = windowsToAppLogsCorrelationRule(); + request = new Request("POST", "/_correlation/rules"); + request.setJsonEntity(correlationRule); + client().performRequest(request); + + request = new Request("POST", String.format(Locale.ROOT, "/%s/_doc?refresh", windowsIndex)); + request.setJsonEntity(sampleWindowsEvent()); + client().performRequest(request); + + request = new Request("POST", String.format(Locale.ROOT, "/%s/_doc?refresh", appLogsIndex)); + request.setJsonEntity(sampleAppLogsEvent()); + Response response = client().performRequest(request); + String appLogsId = responseAsMap(response).get("_id").toString(); + + request = new Request("POST", "/_correlation/events"); + request.setJsonEntity(prepareCorrelateEventRequest(appLogsIndex, appLogsId)); + response = client().performRequest(request); + Map responseAsMap = responseAsMap(response); + Assert.assertEquals(1, ((Map) responseAsMap.get("neighbor_events")).size()); + } + + private String prepareCorrelateEventRequest(String index, String event) { + return "{\n" + " \"index\": \"" + index + "\",\n" + " \"event\": \"" + event + "\",\n" + " \"store\": false\n" + "}"; + } + + private String windowsToAppLogsCorrelationRule() { + return "{\n" + + " \"name\": \"windows to app logs\",\n" + + " \"correlate\": [\n" + + " {\n" + + " \"index\": \"windows\",\n" + + " \"query\": \"host.hostname:EC2AMAZ*\",\n" + + " \"timestampField\": \"winlog.timestamp\",\n" + + " \"tags\": [\n" + + " \"windows\"\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"index\": \"app_logs\",\n" + + " \"query\": \"endpoint:\\\\/customer_records.txt\",\n" + + " \"timestampField\": \"timestamp\",\n" + + " \"tags\": [\n" + + " \"others_application\"\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"; + } + private String sampleCorrelationRule() { return "{\n" + " \"name\": \"s3 to app logs\",\n" @@ -151,4 +213,115 @@ private String sampleCorrelationRuleWithNoTimestamp() { private String matchIdQuery(String id) { return "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + id + "\"\n" + " }\n" + " }\n" + "}"; } + + private String windowsMappings() { + return "{" + + " \"settings\": {" + + " \"number_of_shards\": 1" + + " }," + + " \"mappings\": {" + + " \"properties\": {\n" + + " \"server.user.hash\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"winlog.event_id\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"host.hostname\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"windows.message\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"winlog.provider_name\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"winlog.event_data.ServiceName\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"winlog.timestamp\": {\n" + + " \"type\": \"long\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + } + + private String appLogMappings() { + return "{" + + " \"settings\": {" + + " \"number_of_shards\": 1" + + " }," + + " \"mappings\": {" + + " \"properties\": {\n" + + " \"http_method\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"endpoint\": {\n" + + " \"type\": \"text\",\n" + + " \"analyzer\": \"whitespace\"" + + " },\n" + + " \"keywords\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"timestamp\": {\n" + + " \"type\": \"long\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + } + + private String sampleWindowsEvent() { + return "{\n" + + " \"EventTime\": \"2020-02-04T14:59:39.343541+00:00\",\n" + + " \"host.hostname\": \"EC2AMAZEPO7HKA\",\n" + + " \"Keywords\": \"9223372036854775808\",\n" + + " \"SeverityValue\": 2,\n" + + " \"Severity\": \"INFO\",\n" + + " \"winlog.event_id\": 22,\n" + + " \"SourceName\": \"Microsoft-Windows-Sysmon\",\n" + + " \"ProviderGuid\": \"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + " \"Version\": 5,\n" + + " \"TaskValue\": 22,\n" + + " \"OpcodeValue\": 0,\n" + + " \"RecordNumber\": 9532,\n" + + " \"ExecutionProcessID\": 1996,\n" + + " \"ExecutionThreadID\": 2616,\n" + + " \"Channel\": \"Microsoft-Windows-Sysmon/Operational\",\n" + + " \"winlog.event_data.SubjectDomainName\": \"NTAUTHORITY\",\n" + + " \"AccountName\": \"SYSTEM\",\n" + + " \"UserID\": \"S-1-5-18\",\n" + + " \"AccountType\": \"User\",\n" + + " \"windows.message\": \"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + " \"Category\": \"Dns query (rule: DnsQuery)\",\n" + + " \"Opcode\": \"Info\",\n" + + " \"UtcTime\": \"2020-02-04 14:59:38.349\",\n" + + " \"ProcessGuid\": \"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + " \"ProcessId\": \"1904\",\n" + + " \"QueryName\": \"EC2AMAZ-EPO7HKA\",\n" + + " \"QueryStatus\": \"0\",\n" + + " \"QueryResults\": \"172.31.46.38;\",\n" + + " \"Image\": \"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + " \"EventReceivedTime\": \"2020-02-04T14:59:40.780905+00:00\",\n" + + " \"SourceModuleName\": \"in\",\n" + + " \"SourceModuleType\": \"im_msvistalog\",\n" + + " \"CommandLine\": \"eachtest\",\n" + + " \"Initiated\": \"true\",\n" + + " \"winlog.timestamp\": " + + System.currentTimeMillis() + + "\n" + + "}"; + } + + private String sampleAppLogsEvent() { + return "{\n" + + " \"endpoint\": \"/customer_records.txt\",\n" + + " \"http_method\": \"POST\",\n" + + " \"keywords\": \"PermissionDenied\",\n" + + " \"timestamp\": " + + System.currentTimeMillis() + + "\n" + + "}"; + } } diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/EventsCorrelationPlugin.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/EventsCorrelationPlugin.java index 9637042974d03..40664e6605456 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/EventsCorrelationPlugin.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/EventsCorrelationPlugin.java @@ -30,10 +30,19 @@ import org.opensearch.plugin.correlation.core.index.mapper.CorrelationVectorFieldMapper; import org.opensearch.plugin.correlation.core.index.mapper.VectorFieldMapper; import org.opensearch.plugin.correlation.core.index.query.CorrelationQueryBuilder; +import org.opensearch.plugin.correlation.events.action.IndexCorrelationAction; +import org.opensearch.plugin.correlation.events.action.SearchCorrelatedEventsAction; +import org.opensearch.plugin.correlation.events.action.StoreCorrelationAction; +import org.opensearch.plugin.correlation.events.resthandler.RestIndexCorrelationAction; +import org.opensearch.plugin.correlation.events.resthandler.RestSearchCorrelatedEventsAction; +import org.opensearch.plugin.correlation.events.transport.TransportIndexCorrelationAction; +import org.opensearch.plugin.correlation.events.transport.TransportSearchCorrelatedEventsAction; +import org.opensearch.plugin.correlation.events.transport.TransportStoreCorrelationAction; import org.opensearch.plugin.correlation.rules.action.IndexCorrelationRuleAction; import org.opensearch.plugin.correlation.rules.resthandler.RestIndexCorrelationRuleAction; import org.opensearch.plugin.correlation.rules.transport.TransportIndexCorrelationRuleAction; import org.opensearch.plugin.correlation.settings.EventsCorrelationSettings; +import org.opensearch.plugin.correlation.utils.CorrelationIndices; import org.opensearch.plugin.correlation.utils.CorrelationRuleIndices; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.EnginePlugin; @@ -67,9 +76,12 @@ public class EventsCorrelationPlugin extends Plugin implements ActionPlugin, Map * events-correlation-engine rules uri */ public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/rules"; + public static final String CORRELATION_EVENTS_BASE_URI = PLUGINS_BASE_URI + "/events"; private CorrelationRuleIndices correlationRuleIndices; + private CorrelationIndices correlationIndices; + /** * Default constructor */ @@ -90,7 +102,8 @@ public Collection createComponents( Supplier repositoriesServiceSupplier ) { correlationRuleIndices = new CorrelationRuleIndices(client, clusterService); - return List.of(correlationRuleIndices); + correlationIndices = new CorrelationIndices(client, clusterService, clusterService.getSettings()); + return List.of(correlationRuleIndices, correlationIndices); } @Override @@ -103,7 +116,7 @@ public List getRestHandlers( IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster ) { - return List.of(new RestIndexCorrelationRuleAction()); + return List.of(new RestIndexCorrelationRuleAction(), new RestSearchCorrelatedEventsAction(), new RestIndexCorrelationAction()); } @Override @@ -132,11 +145,20 @@ public List> getQueries() { @Override public List> getActions() { - return List.of(new ActionPlugin.ActionHandler<>(IndexCorrelationRuleAction.INSTANCE, TransportIndexCorrelationRuleAction.class)); + return List.of( + new ActionPlugin.ActionHandler<>(IndexCorrelationRuleAction.INSTANCE, TransportIndexCorrelationRuleAction.class), + new ActionPlugin.ActionHandler<>(IndexCorrelationAction.INSTANCE, TransportIndexCorrelationAction.class), + new ActionPlugin.ActionHandler<>(StoreCorrelationAction.INSTANCE, TransportStoreCorrelationAction.class), + new ActionPlugin.ActionHandler<>(SearchCorrelatedEventsAction.INSTANCE, TransportSearchCorrelatedEventsAction.class) + ); } @Override public List> getSettings() { - return List.of(EventsCorrelationSettings.IS_CORRELATION_INDEX_SETTING, EventsCorrelationSettings.CORRELATION_TIME_WINDOW); + return List.of( + EventsCorrelationSettings.IS_CORRELATION_INDEX_SETTING, + EventsCorrelationSettings.CORRELATION_HISTORY_INDEX_SHARDS, + EventsCorrelationSettings.CORRELATION_TIME_WINDOW + ); } } diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationAction.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationAction.java new file mode 100644 index 0000000000000..8b0654eddc4e8 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationAction.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.action.ActionType; + +/** + * Transport Action for indexing correlations + * + * @opensearch.internal + */ +public class IndexCorrelationAction extends ActionType { + + /** + * Instance of IndexCorrelationAction + */ + public static final IndexCorrelationAction INSTANCE = new IndexCorrelationAction(); + /** + * Name of IndexCorrelationAction + */ + public static final String NAME = "cluster:admin/index/correlation/events"; + + private IndexCorrelationAction() { + super(NAME, IndexCorrelationResponse::new); + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationRequest.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationRequest.java new file mode 100644 index 0000000000000..f7c8fdf65da9d --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationRequest.java @@ -0,0 +1,140 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; + +/** + * A request to index correlations + * + * @opensearch.api + */ +public class IndexCorrelationRequest extends ActionRequest { + + private static final ParseField INDEX_FIELD = new ParseField("index"); + private static final ParseField EVENT_FIELD = new ParseField("event"); + private static final ParseField STORE_FIELD = new ParseField("store"); + private static final ObjectParser PARSER = new ObjectParser<>( + "IndexCorrelationRequest", + IndexCorrelationRequest::new + ); + + static { + PARSER.declareString(IndexCorrelationRequest::setIndex, INDEX_FIELD); + PARSER.declareString(IndexCorrelationRequest::setEvent, EVENT_FIELD); + PARSER.declareBoolean(IndexCorrelationRequest::setStore, STORE_FIELD); + } + + private String index; + + private String event; + + private Boolean store; + + private IndexCorrelationRequest() {} + + /** + * Parameterized ctor for IndexCorrelationRequest + * @param index index for correlation + * @param event event from index which needs to be correlated + * @param store an optional param which is used to decide whether to store correlations in vectordb or not. + */ + public IndexCorrelationRequest(String index, String event, Boolean store) { + super(); + this.index = index; + this.event = event; + this.store = store; + } + + /** + * StreamInput ctor of IndexCorrelationRequest + * @param sin StreamInput + * @throws IOException IOException + */ + public IndexCorrelationRequest(StreamInput sin) throws IOException { + this(sin.readString(), sin.readString(), sin.readBoolean()); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(index); + out.writeString(event); + out.writeBoolean(store); + } + + /** + * Parse into IndexCorrelationRequest + * @param xcp XContentParser + * @return IndexCorrelationRequest + */ + public static IndexCorrelationRequest parse(XContentParser xcp) { + return PARSER.apply(xcp, null); + } + + /** + * set index for correlation + * @param index index for correlation + */ + public void setIndex(String index) { + this.index = index; + } + + /** + * get index for correlation + * @return index for correlation + */ + public String getIndex() { + return index; + } + + /** + * set event to be correlated + * @param event event to be correlated + */ + public void setEvent(String event) { + this.event = event; + } + + /** + * get event to be correlated + * @return event to be correlated + */ + public String getEvent() { + return event; + } + + /** + * set boolean param to check if correlations stored + * @param store boolean param to check if correlations stored + */ + public void setStore(Boolean store) { + this.store = store; + } + + /** + * boolean param to check if correlations stored + * @return boolean param to check if correlations stored + */ + public Boolean getStore() { + return store; + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationResponse.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationResponse.java new file mode 100644 index 0000000000000..c6c339e5bac0b --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationResponse.java @@ -0,0 +1,99 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.core.ParseField; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Transport Response for indexing correlations + * + * @opensearch.internal + */ +public class IndexCorrelationResponse extends ActionResponse implements ToXContentObject { + + private static final ParseField IS_ORPHAN_FIELD = new ParseField("is_orphan"); + private static final ParseField NEIGHBOR_EVENTS_FIELD = new ParseField("neighbor_events"); + + private Boolean isOrphan; + + private Map> neighborEvents; + + private RestStatus status; + + /** + * Parameterized ctor for IndexCorrelationResponse + * @param isOrphan boolean param if event is orphan + * @param neighborEvents map of neighboring events with each entry storing index to events pair. + * @param status REST status of the request + */ + public IndexCorrelationResponse(Boolean isOrphan, Map> neighborEvents, RestStatus status) { + super(); + this.isOrphan = isOrphan; + this.neighborEvents = neighborEvents; + this.status = status; + } + + /** + * StreamInput ctor of IndexCorrelationResponse + * @param sin StreamInput + * @throws IOException IOException + */ + public IndexCorrelationResponse(StreamInput sin) throws IOException { + this(sin.readBoolean(), sin.readMap(StreamInput::readString, StreamInput::readStringList), sin.readEnum(RestStatus.class)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(IS_ORPHAN_FIELD.getPreferredName(), isOrphan) + .field(NEIGHBOR_EVENTS_FIELD.getPreferredName(), neighborEvents); + return builder.endObject(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(isOrphan); + out.writeMap(neighborEvents, StreamOutput::writeString, StreamOutput::writeStringCollection); + out.writeEnum(status); + } + + /** + * get REST status of the request + * @return REST status of the request + */ + public RestStatus getStatus() { + return status; + } + + /** + * get if event is orphan + * @return boolean param to check if event is orphan + */ + public Boolean getOrphan() { + return isOrphan; + } + + /** + * get neighboring events for the input event generated by correlation engine + * @return neighboring events for the input event + */ + public Map> getNeighborEvents() { + return neighborEvents; + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsAction.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsAction.java new file mode 100644 index 0000000000000..05aa0c759b29a --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsAction.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.action.ActionType; + +/** + * Transport Action for searching correlated events + * + * @opensearch.internal + */ +public class SearchCorrelatedEventsAction extends ActionType { + + /** + * Instance of SearchCorrelatedEventsAction + */ + public static final SearchCorrelatedEventsAction INSTANCE = new SearchCorrelatedEventsAction(); + /** + * Name of SearchCorrelatedEventsAction + */ + public static final String NAME = "cluster:admin/search/correlation/events"; + + private SearchCorrelatedEventsAction() { + super(NAME, SearchCorrelatedEventsResponse::new); + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsRequest.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsRequest.java new file mode 100644 index 0000000000000..b7c6c5c59a5a5 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsRequest.java @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; + +/** + * Transport request to search correlated events + * + * @opensearch.internal + */ +public class SearchCorrelatedEventsRequest extends ActionRequest { + + private String index; + + private String event; + + private String timestampField; + + private Long timeWindow; + + private Integer nearbyEvents; + + /** + * Parameterized ctor of SearchCorrelatedEventsRequest + * @param index index of the event for which correlations are searched + * @param event event for which correlations are searched + * @param timestampField timestamp field in the index + * @param timeWindow time window dimension of correlation + * @param nearbyEvents number of nearby correlated events + */ + public SearchCorrelatedEventsRequest(String index, String event, String timestampField, Long timeWindow, Integer nearbyEvents) { + super(); + this.index = index; + this.event = event; + this.timestampField = timestampField; + this.timeWindow = timeWindow; + this.nearbyEvents = nearbyEvents; + } + + /** + * StreamInput ctor of SearchCorrelatedEventsRequest + * @param sin StreamInput + * @throws IOException IOException + */ + public SearchCorrelatedEventsRequest(StreamInput sin) throws IOException { + this(sin.readString(), sin.readString(), sin.readString(), sin.readLong(), sin.readInt()); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(index); + out.writeString(event); + out.writeString(timestampField); + out.writeLong(timeWindow); + out.writeInt(nearbyEvents); + } + + /** + * get index of the event for which correlations are searched + * @return index of the event for which correlations are searched + */ + public String getIndex() { + return index; + } + + /** + * get event for which correlations are searched + * @return event for which correlations are searched + */ + public String getEvent() { + return event; + } + + /** + * get timestamp field of the index whose event correlations are searched + * @return timestamp field of the index whose event correlations are searched + */ + public String getTimestampField() { + return timestampField; + } + + /** + * get time window dimension of correlation + * @return time window dimension of correlation + */ + public Long getTimeWindow() { + return timeWindow; + } + + /** + * get number of nearby correlated events + * @return number of nearby correlated events + */ + public Integer getNearbyEvents() { + return nearbyEvents; + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsResponse.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsResponse.java new file mode 100644 index 0000000000000..ed0e8214e4343 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsResponse.java @@ -0,0 +1,83 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.plugin.correlation.events.model.EventWithScore; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +/** + * Transport response for searching correlated events + * + * @opensearch.internal + */ +public class SearchCorrelatedEventsResponse extends ActionResponse implements ToXContentObject { + + private List events; + + private RestStatus status; + + private static final String EVENTS = "events"; + + /** + * Parameterized ctor of SearchCorrelatedEventsResponse + * @param events list of neighboring events with scores + * @param status REST status of the request + */ + public SearchCorrelatedEventsResponse(List events, RestStatus status) { + super(); + this.events = events; + this.status = status; + } + + /** + * StreamInput ctor of SearchCorrelatedEventsResponse + * @param sin StreamInput + * @throws IOException IOException + */ + public SearchCorrelatedEventsResponse(StreamInput sin) throws IOException { + this(Collections.unmodifiableList(sin.readList(EventWithScore::new)), sin.readEnum(RestStatus.class)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject().field(EVENTS, events).endObject(); + return builder.endObject(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(events); + out.writeEnum(status); + } + + /** + * get correlated events + * @return correlated events + */ + public List getEvents() { + return events; + } + + /** + * get REST status + * @return REST status + */ + public RestStatus getStatus() { + return status; + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationAction.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationAction.java new file mode 100644 index 0000000000000..40357897983ff --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationAction.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.action.ActionType; + +/** + * Transport Action for storing correlations + * + * @opensearch.internal + */ +public class StoreCorrelationAction extends ActionType { + + /** + * Instance of StoreCorrelationAction + */ + public static final StoreCorrelationAction INSTANCE = new StoreCorrelationAction(); + /** + * Name of StoreCorrelationAction + */ + public static final String NAME = "cluster:admin/store/correlation/events"; + + private StoreCorrelationAction() { + super(NAME, StoreCorrelationResponse::new); + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationRequest.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationRequest.java new file mode 100644 index 0000000000000..4e00de7c1a656 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationRequest.java @@ -0,0 +1,128 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Transport request to store correlations + * + * @opensearch.internal + */ +public class StoreCorrelationRequest extends ActionRequest { + + private String index; + + private String event; + + private Long timestamp; + + private Map> eventsAdjacencyList; + + private List tags; + + /** + * Parameterized ctor of StoreCorrelationRequest + * @param index index of the event which is being correlated. + * @param event event which is being correlated. + * @param timestamp timestamp of the correlated event + * @param eventsAdjacencyList the adjacency list of events which are correlated with the input event + * @param tags optional tags for event to be correlated. + */ + public StoreCorrelationRequest( + String index, + String event, + Long timestamp, + Map> eventsAdjacencyList, + List tags + ) { + super(); + this.index = index; + this.event = event; + this.timestamp = timestamp; + this.eventsAdjacencyList = eventsAdjacencyList; + this.tags = tags; + } + + /** + * StreamInput ctor of StoreCorrelationRequest + * @param sin StreamInput + * @throws IOException IOException + */ + public StoreCorrelationRequest(StreamInput sin) throws IOException { + this( + sin.readString(), + sin.readString(), + sin.readLong(), + sin.readMap(StreamInput::readString, StreamInput::readStringList), + sin.readStringList() + ); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(index); + out.writeString(event); + out.writeLong(timestamp); + out.writeMap(eventsAdjacencyList, StreamOutput::writeString, StreamOutput::writeStringCollection); + out.writeStringCollection(tags); + } + + /** + * get input event which is being correlated + * @return input event which is being correlated + */ + public String getEvent() { + return event; + } + + /** + * get index of event which is being correlated + * @return index of event which is being correlated + */ + public String getIndex() { + return index; + } + + /** + * get timestamp of correlated event + * @return timestamp of correlated event + */ + public Long getTimestamp() { + return timestamp; + } + + /** + * get optional tags of correlated event + * @return optional tags of correlated event + */ + public List getTags() { + return tags; + } + + /** + * get adjacency list of correlated events for the input event + * @return adjacency list of correlated events for the input event + */ + public Map> getEventsAdjacencyList() { + return eventsAdjacencyList; + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationResponse.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationResponse.java new file mode 100644 index 0000000000000..345b7c2691762 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationResponse.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; + +import java.io.IOException; + +/** + * Transport Response to store correlations + * + * @opensearch.internal + */ +public class StoreCorrelationResponse extends ActionResponse { + + private RestStatus status; + + /** + * Parameterized ctor of StoreCorrelationResponse + * @param status REST status of the request + */ + public StoreCorrelationResponse(RestStatus status) { + super(); + this.status = status; + } + + /** + * StreamInput ctor of StoreCorrelationResponse + * @param sin StreamInput + * @throws IOException IOException + */ + public StoreCorrelationResponse(StreamInput sin) throws IOException { + this(sin.readEnum(RestStatus.class)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeEnum(status); + } + + /** + * get REST status of request + * @return REST status of request + */ + public RestStatus getStatus() { + return status; + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/package-info.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/package-info.java new file mode 100644 index 0000000000000..0acc7824e0c94 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Transport Actions, Requests and Responses for correlation events + */ +package org.opensearch.plugin.correlation.events.action; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/model/Correlation.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/model/Correlation.java new file mode 100644 index 0000000000000..8006608d00431 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/model/Correlation.java @@ -0,0 +1,366 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * Correlation Object to store. A correlation is defined as an edge joining two events. + * + * @opensearch.internal + */ +public class Correlation implements Writeable, ToXContentObject { + + private static final Logger log = LogManager.getLogger(Correlation.class); + + /** + * Correlation history index + */ + public static final String CORRELATION_HISTORY_INDEX = ".opensearch-correlation-history"; + + private static final ParseField ID_FIELD = new ParseField("id"); + private static final ParseField VERSION_FIELD = new ParseField("version"); + private static final ParseField ROOT_FIELD = new ParseField("root"); + private static final ParseField LEVEL_FIELD = new ParseField("level"); + private static final ParseField EVENT1_FIELD = new ParseField("event1"); + private static final ParseField EVENT2_FIELD = new ParseField("event2"); + private static final ParseField CORRELATION_VECTOR_FIELD = new ParseField("corr_vector"); + private static final ParseField TIMESTAMP_FIELD = new ParseField("timestamp"); + private static final ParseField INDEX1_FIELD = new ParseField("index1"); + private static final ParseField INDEX2_FIELD = new ParseField("index2"); + private static final ParseField TAGS_FIELD = new ParseField("tags"); + private static final ParseField SCORE_TIMESTAMP_FIELD = new ParseField("score_timestamp"); + private static final ObjectParser PARSER = new ObjectParser<>("Correlation", Correlation::new); + + static { + PARSER.declareBoolean(Correlation::setRoot, ROOT_FIELD); + PARSER.declareString(Correlation::setId, ID_FIELD); + PARSER.declareLong(Correlation::setVersion, VERSION_FIELD); + PARSER.declareLong(Correlation::setLevel, LEVEL_FIELD); + PARSER.declareString(Correlation::setEvent1, EVENT1_FIELD); + PARSER.declareString(Correlation::setEvent2, EVENT2_FIELD); + PARSER.declareFloatArray(Correlation::setCorrelationVector, CORRELATION_VECTOR_FIELD); + PARSER.declareLong(Correlation::setTimestamp, TIMESTAMP_FIELD); + PARSER.declareLong(Correlation::setScoreTimestamp, SCORE_TIMESTAMP_FIELD); + PARSER.declareString(Correlation::setIndex1, INDEX1_FIELD); + PARSER.declareString(Correlation::setIndex2, INDEX2_FIELD); + PARSER.declareStringArray(Correlation::setTags, TAGS_FIELD); + } + + private String id; + + private Long version; + + private Boolean isRoot; + + private Long level; + + private String event1; + + private String event2; + + private float[] correlationVector; + + private Long timestamp; + + private String index1; + + private String index2; + + private List tags; + + private Long scoreTimestamp; + + private Correlation() {} + + /** + * Parameterized ctor for Correlation + * @param id id of correlation object + * @param version version of correlation + * @param isRoot is it root correlation record. The root record is used as the base record which store timestamp related metadata. + * @param level level at which correlations are stored which help distinguish one group of correlations from another. + * @param event1 first event which is correlated + * @param event2 second event which is correlated + * @param correlationVector the vector representation of the correlation object + * @param timestamp timestamp of correlation + * @param index1 index of the first event + * @param index2 index of the second event + * @param tags list of tags for the correlation object + * @param scoreTimestamp score timestamp of correlation used to bypass the problem of Lucene vectors can only be of type byte array or floats. + */ + public Correlation( + String id, + Long version, + Boolean isRoot, + Long level, + String event1, + String event2, + float[] correlationVector, + Long timestamp, + String index1, + String index2, + List tags, + Long scoreTimestamp + ) { + this.id = id; + this.version = version; + this.isRoot = isRoot; + this.level = level; + this.event1 = event1; + this.event2 = event2; + this.correlationVector = correlationVector; + this.timestamp = timestamp; + this.index1 = index1; + this.index2 = index2; + this.tags = tags; + this.scoreTimestamp = scoreTimestamp; + } + + /** + * Parameterized ctor of Correlation object + * @param isRoot is it root correlation record. The root record is used as the base record which store timestamp related metadata. + * @param level level at which correlations are stored which help distinguish one group of correlations from another. + * @param event1 first event which is correlated + * @param event2 second event which is correlated + * @param correlationVector the vector representation of the correlation object + * @param timestamp timestamp of correlation + * @param index1 index of the first event + * @param index2 index of the second event + * @param tags list of tags for the correlation object + * @param scoreTimestamp score timestamp of correlation used to bypass the problem of Lucene vectors can only be of type byte array or floats. + */ + public Correlation( + Boolean isRoot, + Long level, + String event1, + String event2, + float[] correlationVector, + Long timestamp, + String index1, + String index2, + List tags, + Long scoreTimestamp + ) { + this("", 1L, isRoot, level, event1, event2, correlationVector, timestamp, index1, index2, tags, scoreTimestamp); + } + + /** + * StreamInput ctor of Correlation object + * @param sin StreamInput + * @throws IOException IOException + */ + public Correlation(StreamInput sin) throws IOException { + this( + sin.readString(), + sin.readLong(), + sin.readBoolean(), + sin.readLong(), + sin.readString(), + sin.readString(), + sin.readFloatArray(), + sin.readLong(), + sin.readString(), + sin.readString(), + sin.readStringList(), + sin.readLong() + ); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(ROOT_FIELD.getPreferredName(), isRoot); + builder.field(LEVEL_FIELD.getPreferredName(), level); + builder.field(EVENT1_FIELD.getPreferredName(), event1); + builder.field(EVENT2_FIELD.getPreferredName(), event2); + builder.field(CORRELATION_VECTOR_FIELD.getPreferredName(), correlationVector); + builder.field(TIMESTAMP_FIELD.getPreferredName(), timestamp); + builder.field(INDEX1_FIELD.getPreferredName(), index1); + builder.field(INDEX2_FIELD.getPreferredName(), index2); + builder.field(TAGS_FIELD.getPreferredName(), tags); + builder.field(SCORE_TIMESTAMP_FIELD.getPreferredName(), scoreTimestamp); + return builder.endObject(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeBoolean(isRoot); + out.writeLong(level); + out.writeString(event1); + out.writeString(event2); + out.writeFloatArray(correlationVector); + out.writeLong(timestamp); + out.writeString(index1); + out.writeString(index2); + out.writeStringCollection(tags); + out.writeLong(scoreTimestamp); + } + + /** + * Parse into Correlation + * @param xcp XContentParser + * @return Correlation + * @throws IOException IOException + */ + public static Correlation parse(XContentParser xcp) throws IOException { + return PARSER.apply(xcp, null); + } + + /** + * convert StreamInput to Correlation + * @param sin StreamInput + * @return Correlation + * @throws IOException IOException + */ + public static Correlation readFrom(StreamInput sin) throws IOException { + return new Correlation(sin); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Correlation that = (Correlation) o; + return id.equals(that.id) + && version.equals(that.version) + && isRoot.equals(that.isRoot) + && level.equals(that.level) + && event1.equals(that.event1) + && event2.equals(that.event2) + && Arrays.equals(correlationVector, that.correlationVector) + && timestamp.equals(that.timestamp) + && index1.equals(that.index1) + && index2.equals(that.index2) + && tags.equals(that.tags) + && scoreTimestamp.equals(that.scoreTimestamp); + } + + @Override + public int hashCode() { + int result = Objects.hash(id, version, isRoot, level, event1, event2, timestamp, index1, index2, tags, scoreTimestamp); + result = 31 * result + Arrays.hashCode(correlationVector); + return result; + } + + /** + * set if it is root correlation record + * @param root set if it is root correlation record + */ + public void setRoot(Boolean root) { + isRoot = root; + } + + /** + * set id of correlation object + * @param id id of correlation object + */ + public void setId(String id) { + this.id = id; + } + + /** + * set version of correlation object + * @param version version of correlation object + */ + public void setVersion(Long version) { + this.version = version; + } + + /** + * set level of correlation object + * @param level level of correlation object + */ + public void setLevel(Long level) { + this.level = level; + } + + /** + * set first event of correlation object + * @param event1 first event of correlation object + */ + public void setEvent1(String event1) { + this.event1 = event1; + } + + /** + * set second event of correlation object + * @param event2 second event of correlation object + */ + public void setEvent2(String event2) { + this.event2 = event2; + } + + /** + * set the vector representation of the correlation object + * @param correlationVector the vector representation of the correlation object + */ + public void setCorrelationVector(List correlationVector) { + int size = correlationVector.size(); + this.correlationVector = new float[size]; + for (int i = 0; i < size; ++i) { + this.correlationVector[i] = correlationVector.get(i); + } + } + + /** + * set timestamp of correlation + * @param timestamp timestamp of correlation + */ + public void setTimestamp(Long timestamp) { + this.timestamp = timestamp; + } + + /** + * set score timestamp of correlation object + * @param scoreTimestamp score timestamp of correlation object + */ + public void setScoreTimestamp(Long scoreTimestamp) { + this.scoreTimestamp = scoreTimestamp; + } + + /** + * set index of first event + * @param index1 index of first event + */ + public void setIndex1(String index1) { + this.index1 = index1; + } + + /** + * set index of second event + * @param index2 index of second event + */ + public void setIndex2(String index2) { + this.index2 = index2; + } + + /** + * set tags for correlation object + * @param tags tags for correlation object + */ + public void setTags(List tags) { + this.tags = tags; + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/model/EventWithScore.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/model/EventWithScore.java new file mode 100644 index 0000000000000..f21a4caf390de --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/model/EventWithScore.java @@ -0,0 +1,201 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.model; + +import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * Event with score output + * { + * "index": "app_logs", + * "event": "EYT06YgBpmAY3ZcggCts", + * "score": 1, + * "tags": [] + * } + * + * @opensearch.api + * @opensearch.experimental + */ +public class EventWithScore implements Writeable, ToXContentObject { + + private static final ParseField INDEX_FIELD = new ParseField("index"); + private static final ParseField EVENT_FIELD = new ParseField("event"); + private static final ParseField SCORE_FIELD = new ParseField("score"); + private static final ParseField TAGS_FIELD = new ParseField("tags"); + private static final ObjectParser PARSER = new ObjectParser( + "EventWithScore", + EventWithScore::new + ); + + private String index; + + private String event; + + private Double score; + + private List tags; + + static { + PARSER.declareString(EventWithScore::setIndex, INDEX_FIELD); + PARSER.declareString(EventWithScore::setEvent, EVENT_FIELD); + PARSER.declareDouble(EventWithScore::setScore, SCORE_FIELD); + PARSER.declareStringArray(EventWithScore::setTags, TAGS_FIELD); + } + + private EventWithScore() {} + + /** + * Parameterized ctor of Event with score object + * @param index index of correlated event + * @param event correlated event + * @param score score of correlation + * @param tags tags of correlated event + */ + public EventWithScore(String index, String event, Double score, List tags) { + this.index = index; + this.event = event; + this.score = score; + this.tags = tags; + } + + /** + * StreamInput ctor of Event with score object + * @param sin StreamInput + * @throws IOException IOException + */ + public EventWithScore(StreamInput sin) throws IOException { + this(sin.readString(), sin.readString(), sin.readDouble(), sin.readStringList()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(index); + out.writeString(event); + out.writeDouble(score); + out.writeStringCollection(tags); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(INDEX_FIELD.getPreferredName(), index) + .field(EVENT_FIELD.getPreferredName(), event) + .field(SCORE_FIELD.getPreferredName(), score) + .field(TAGS_FIELD.getPreferredName(), tags); + return builder.endObject(); + } + + /** + * Parse into EventWithScore + * @param xcp XContentParser + * @return EventWithScore + * @throws IOException IOException + */ + public static EventWithScore parse(XContentParser xcp) throws IOException { + return PARSER.apply(xcp, null); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EventWithScore that = (EventWithScore) o; + return index.equals(that.index) && event.equals(that.event) && score.equals(that.score) && tags.equals(that.tags); + } + + @Override + public int hashCode() { + return Objects.hash(index, event, score, tags); + } + + /** + * convert StreamInput to EventWithScore + * @param sin StreamInput + * @return EventWithScore + * @throws IOException IOException + */ + public static EventWithScore readFrom(StreamInput sin) throws IOException { + return new EventWithScore(sin); + } + + /** + * set index of correlated event + * @param index index of correlated event + */ + public void setIndex(String index) { + this.index = index; + } + + /** + * get index of correlated event + * @return index of correlated event + */ + public String getIndex() { + return index; + } + + /** + * set tags of correlated event + * @param tags tags of correlated event + */ + public void setTags(List tags) { + this.tags = tags; + } + + /** + * get tags of correlated event + * @return tags of correlated event + */ + public List getTags() { + return tags; + } + + /** + * set correlated event + * @param event correlated event + */ + public void setEvent(String event) { + this.event = event; + } + + /** + * get correlated event + * @return correlated event + */ + public String getEvent() { + return event; + } + + /** + * set score of correlation + * @param score score of correlation + */ + public void setScore(Double score) { + this.score = score; + } + + /** + * get score of correlation + * @return score of correlation + */ + public Double getScore() { + return score; + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/model/package-info.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/model/package-info.java new file mode 100644 index 0000000000000..a387fc983cc27 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/model/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * data models for correlation events + */ +package org.opensearch.plugin.correlation.events.model; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/resthandler/RestIndexCorrelationAction.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/resthandler/RestIndexCorrelationAction.java new file mode 100644 index 0000000000000..2455b4238f211 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/resthandler/RestIndexCorrelationAction.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.plugin.correlation.EventsCorrelationPlugin; +import org.opensearch.plugin.correlation.events.action.IndexCorrelationAction; +import org.opensearch.plugin.correlation.events.action.IndexCorrelationRequest; +import org.opensearch.plugin.correlation.events.action.IndexCorrelationResponse; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +/** + * Rest action for indexing an event and its correlations + */ +public class RestIndexCorrelationAction extends BaseRestHandler { + + private static final Logger log = LogManager.getLogger(RestIndexCorrelationAction.class); + + /** + * Default constructor + */ + public RestIndexCorrelationAction() {} + + @Override + public String getName() { + return "index_correlation_action"; + } + + @Override + public List routes() { + return List.of(new Route(RestRequest.Method.POST, EventsCorrelationPlugin.CORRELATION_EVENTS_BASE_URI)); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.ROOT, "%s %s", request.method(), EventsCorrelationPlugin.CORRELATION_EVENTS_BASE_URI)); + + XContentParser xcp = request.contentParser(); + IndexCorrelationRequest correlationRequest = IndexCorrelationRequest.parse(xcp); + + return channel -> client.doExecute(IndexCorrelationAction.INSTANCE, correlationRequest, indexCorrelationResponse(channel)); + } + + private RestResponseListener indexCorrelationResponse(RestChannel channel) { + return new RestResponseListener<>(channel) { + @Override + public RestResponse buildResponse(IndexCorrelationResponse indexCorrelationResponse) throws Exception { + return new BytesRestResponse( + indexCorrelationResponse.getStatus(), + indexCorrelationResponse.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS) + ); + } + }; + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/resthandler/RestSearchCorrelatedEventsAction.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/resthandler/RestSearchCorrelatedEventsAction.java new file mode 100644 index 0000000000000..70c6fc5b227d0 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/resthandler/RestSearchCorrelatedEventsAction.java @@ -0,0 +1,89 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.plugin.correlation.EventsCorrelationPlugin; +import org.opensearch.plugin.correlation.events.action.SearchCorrelatedEventsAction; +import org.opensearch.plugin.correlation.events.action.SearchCorrelatedEventsRequest; +import org.opensearch.plugin.correlation.events.action.SearchCorrelatedEventsResponse; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +/** + * Rest action for searching correlated events + * + * @opensearch.api + */ +public class RestSearchCorrelatedEventsAction extends BaseRestHandler { + + private static final Logger log = LogManager.getLogger(RestSearchCorrelatedEventsAction.class); + + /** + * Default constructor + */ + public RestSearchCorrelatedEventsAction() {} + + @Override + public String getName() { + return "search_correlated_events_action"; + } + + @Override + public List routes() { + return List.of(new Route(RestRequest.Method.GET, EventsCorrelationPlugin.CORRELATION_EVENTS_BASE_URI)); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.ROOT, "%s %s", request.method(), EventsCorrelationPlugin.CORRELATION_EVENTS_BASE_URI)); + + String index = request.param("index"); + String event = request.param("event"); + String timestampField = request.param("timestamp_field"); + Long timeWindow = request.paramAsLong("time_window", 300000L); + int noOfNearbyEvents = request.paramAsInt("nearby_events", 5); + + SearchCorrelatedEventsRequest correlatedEventsRequest = new SearchCorrelatedEventsRequest( + index, + event, + timestampField, + timeWindow, + noOfNearbyEvents + ); + return channel -> client.doExecute( + SearchCorrelatedEventsAction.INSTANCE, + correlatedEventsRequest, + searchCorrelatedEventsResponse(channel) + ); + } + + private RestResponseListener searchCorrelatedEventsResponse(RestChannel channel) { + return new RestResponseListener<>(channel) { + @Override + public RestResponse buildResponse(SearchCorrelatedEventsResponse searchCorrelatedEventsResponse) throws Exception { + return new BytesRestResponse( + searchCorrelatedEventsResponse.getStatus(), + searchCorrelatedEventsResponse.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS) + ); + } + }; + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/resthandler/package-info.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/resthandler/package-info.java new file mode 100644 index 0000000000000..4e5317f9b368a --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/resthandler/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Rest Handlers for correlation events + */ +package org.opensearch.plugin.correlation.events.resthandler; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/transport/TransportIndexCorrelationAction.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/transport/TransportIndexCorrelationAction.java new file mode 100644 index 0000000000000..867752f52ac39 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/transport/TransportIndexCorrelationAction.java @@ -0,0 +1,498 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.search.join.ScoreMode; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.search.MultiSearchRequest; +import org.opensearch.action.search.MultiSearchResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.NestedQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.plugin.correlation.events.action.IndexCorrelationAction; +import org.opensearch.plugin.correlation.events.action.IndexCorrelationRequest; +import org.opensearch.plugin.correlation.events.action.IndexCorrelationResponse; +import org.opensearch.plugin.correlation.events.action.StoreCorrelationAction; +import org.opensearch.plugin.correlation.events.action.StoreCorrelationRequest; +import org.opensearch.plugin.correlation.events.action.StoreCorrelationResponse; +import org.opensearch.plugin.correlation.rules.model.CorrelationQuery; +import org.opensearch.plugin.correlation.rules.model.CorrelationRule; +import org.opensearch.plugin.correlation.settings.EventsCorrelationSettings; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Transport Action for indexing correlations for a particular event + * + * @opensearch.internal + */ +public class TransportIndexCorrelationAction extends HandledTransportAction { + + private static final Logger log = LogManager.getLogger(TransportIndexCorrelationAction.class); + + private final Client client; + + private final NamedXContentRegistry xContentRegistry; + + private final Settings settings; + + private final ClusterService clusterService; + + private volatile long correlationTimeWindow; + + /** + * Parameterized ctor for Transport Action + * @param transportService TransportService + * @param client OS client + * @param xContentRegistry XContentRegistry + * @param settings Settings + * @param actionFilters ActionFilters + * @param clusterService ClusterService + */ + @Inject + public TransportIndexCorrelationAction( + TransportService transportService, + Client client, + NamedXContentRegistry xContentRegistry, + Settings settings, + ActionFilters actionFilters, + ClusterService clusterService + ) { + super(IndexCorrelationAction.NAME, transportService, actionFilters, IndexCorrelationRequest::new); + this.client = client; + this.xContentRegistry = xContentRegistry; + this.settings = settings; + this.clusterService = clusterService; + this.correlationTimeWindow = EventsCorrelationSettings.CORRELATION_TIME_WINDOW.get(this.settings).getMillis(); + + this.clusterService.getClusterSettings() + .addSettingsUpdateConsumer(EventsCorrelationSettings.CORRELATION_TIME_WINDOW, it -> correlationTimeWindow = it.getMillis()); + } + + @Override + protected void doExecute(Task task, IndexCorrelationRequest request, ActionListener listener) { + AsyncIndexCorrelationAction asyncAction = new AsyncIndexCorrelationAction(request, listener); + asyncAction.start(); + } + + class AsyncIndexCorrelationAction { + private final IndexCorrelationRequest request; + + private final ActionListener listener; + + AsyncIndexCorrelationAction(IndexCorrelationRequest request, ActionListener listener) { + this.request = request; + this.listener = listener; + } + + void start() { + String inputIndex = request.getIndex(); + String event = request.getEvent(); + + NestedQueryBuilder queryBuilder = QueryBuilders.nestedQuery( + "correlate", + QueryBuilders.matchQuery("correlate.index", inputIndex), + ScoreMode.None + ); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(true); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(CorrelationRule.CORRELATION_RULE_INDEX); + searchRequest.source(searchSourceBuilder); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT)); + } + + Iterator hits = response.getHits().iterator(); + List correlationRules = new ArrayList<>(); + while (hits.hasNext()) { + try { + SearchHit hit = hits.next(); + + XContentParser xcp = XContentType.JSON.xContent() + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString()); + + CorrelationRule rule = CorrelationRule.parse(xcp); + correlationRules.add(rule); + } catch (IOException e) { + onFailures(e); + } + } + + prepRulesForCorrelatedEventsGeneration(inputIndex, event, correlationRules); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + private void prepRulesForCorrelatedEventsGeneration(String index, String event, List correlationRules) { + MultiSearchRequest mSearchRequest = new MultiSearchRequest(); + SearchRequest eventSearchRequest = null; + + for (CorrelationRule rule : correlationRules) { + // assuming no index duplication in a rule. + Optional query = rule.getCorrelationQueries() + .stream() + .filter(correlationQuery -> correlationQuery.getIndex().equals(index)) + .findFirst(); + + if (query.isPresent()) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .must(QueryBuilders.matchQuery("_id", event)) + .must(QueryBuilders.queryStringQuery(query.get().getQuery())); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(false); + + // assuming all queries belonging to an index use the same timestamp field. + searchSourceBuilder.fetchField(query.get().getTimestampField()); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(index); + searchRequest.source(searchSourceBuilder); + mSearchRequest.add(searchRequest); + + if (eventSearchRequest == null) { + SearchSourceBuilder eventSearchSourceBuilder = new SearchSourceBuilder(); + eventSearchSourceBuilder.query(QueryBuilders.matchQuery("_id", event)); + eventSearchSourceBuilder.fetchSource(false); + + // assuming all queries belonging to an index use the same timestamp field. + eventSearchSourceBuilder.fetchField(query.get().getTimestampField()); + + eventSearchRequest = new SearchRequest(); + eventSearchRequest.indices(index); + eventSearchRequest.source(eventSearchSourceBuilder); + } + } + } + + if (!mSearchRequest.requests().isEmpty()) { + SearchRequest finalEventSearchRequest = eventSearchRequest; + client.multiSearch(mSearchRequest, new ActionListener<>() { + @Override + public void onResponse(MultiSearchResponse items) { + MultiSearchResponse.Item[] responses = items.getResponses(); + Map> indexQueriesMap = new HashMap<>(); + Long timestamp = null; + + int idx = 0; + for (MultiSearchResponse.Item response : responses) { + if (response.isFailure()) { + log.error("error:", response.getFailure()); + // suppress exception + continue; + } + + SearchHits searchHits = response.getResponse().getHits(); + if (searchHits.getTotalHits().value == 1) { + for (CorrelationQuery query : correlationRules.get(idx).getCorrelationQueries()) { + List queries; + if (indexQueriesMap.containsKey(query.getIndex())) { + queries = indexQueriesMap.get(query.getIndex()); + } else { + queries = new ArrayList<>(); + } + queries.add(query); + indexQueriesMap.put(query.getIndex(), queries); + + if (query.getIndex().equals(index)) { + // assuming all queries belonging to an index use the same timestamp field. + timestamp = searchHits.getAt(0).getFields().get(query.getTimestampField()).getValue(); + } + } + } + ++idx; + } + + if (timestamp == null) { + client.search(finalEventSearchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT)); + } + SearchHits searchHits = response.getHits(); + if (searchHits.getTotalHits().value == 1) { + Optional timestampField = searchHits.getAt(0).getFields().keySet().stream().findFirst(); + timestampField.ifPresent( + s -> generateCorrelatedEvents( + index, + event, + searchHits.getAt(0).getFields().get(s).getValue(), + indexQueriesMap + ) + ); + } else { + onFailures( + new OpenSearchStatusException( + "failed at generate correlated events", + RestStatus.INTERNAL_SERVER_ERROR + ) + ); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else { + generateCorrelatedEvents(index, event, timestamp, indexQueriesMap); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else { + // orphan event + if (request.getStore()) { + StoreCorrelationRequest storeCorrelationRequest = new StoreCorrelationRequest( + index, + event, + // as there are no rules there is no timestamp field, assuming event is inserted now. + System.currentTimeMillis(), + Map.of(), + List.of() + ); + client.execute(StoreCorrelationAction.INSTANCE, storeCorrelationRequest, new ActionListener<>() { + @Override + public void onResponse(StoreCorrelationResponse response) { + if (response.getStatus().equals(RestStatus.OK)) { + onOperation(true, new HashMap<>()); + } else { + onFailures(new OpenSearchStatusException("Failed to store correlations", RestStatus.INTERNAL_SERVER_ERROR)); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else { + onOperation(true, new HashMap<>()); + } + } + } + + private void generateCorrelatedEvents( + String inputIndex, + String event, + Long timestamp, + Map> indexQueriesMap + ) { + MultiSearchRequest mSearchRequest = new MultiSearchRequest(); + + for (Map.Entry> indexQueriesEntry : indexQueriesMap.entrySet()) { + String index = indexQueriesEntry.getKey(); + List correlationQueries = indexQueriesEntry.getValue(); + + // assuming all queries belonging to an index use the same timestamp field. + String timestampField = correlationQueries.get(0).getTimestampField(); + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .filter( + QueryBuilders.rangeQuery(timestampField) + .gte(timestamp - correlationTimeWindow) + .lte(timestamp + correlationTimeWindow) + ); + + if (index.equals(inputIndex)) { + queryBuilder = queryBuilder.mustNot(QueryBuilders.matchQuery("_id", event)); + } + + for (CorrelationQuery query : correlationQueries) { + queryBuilder = queryBuilder.should(QueryBuilders.queryStringQuery(query.getQuery())); + } + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(false); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(index); + searchRequest.source(searchSourceBuilder); + + mSearchRequest.add(searchRequest); + } + + if (!mSearchRequest.requests().isEmpty()) { + client.multiSearch(mSearchRequest, new ActionListener<>() { + @Override + public void onResponse(MultiSearchResponse items) { + MultiSearchResponse.Item[] responses = items.getResponses(); + Map> eventsAdjacencyList = new HashMap<>(); + + for (MultiSearchResponse.Item response : responses) { + if (response.isFailure()) { + // suppress exception + continue; + } + + Iterator searchHits = response.getResponse().getHits().iterator(); + + while (searchHits.hasNext()) { + SearchHit hit = searchHits.next(); + + String index = hit.getIndex(); + String id = hit.getId(); + + Set neighborEvents; + if (eventsAdjacencyList.containsKey(index)) { + neighborEvents = eventsAdjacencyList.get(index); + } else { + neighborEvents = new HashSet<>(); + } + neighborEvents.add(id); + eventsAdjacencyList.put(index, neighborEvents); + } + } + + Map> neighborEvents = new HashMap<>(); + for (Map.Entry> neighborEvent : eventsAdjacencyList.entrySet()) { + neighborEvents.put(neighborEvent.getKey(), new ArrayList<>(neighborEvent.getValue())); + } + + if (request.getStore()) { + StoreCorrelationRequest storeCorrelationRequest = new StoreCorrelationRequest( + inputIndex, + event, + timestamp, + neighborEvents, + List.of() + ); + client.execute(StoreCorrelationAction.INSTANCE, storeCorrelationRequest, new ActionListener<>() { + @Override + public void onResponse(StoreCorrelationResponse response) { + if (response.getStatus().equals(RestStatus.OK)) { + if (neighborEvents.isEmpty()) { + onOperation(true, neighborEvents); + } else { + onOperation(false, neighborEvents); + } + } else { + onFailures( + new OpenSearchStatusException("Failed to store correlations", RestStatus.INTERNAL_SERVER_ERROR) + ); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else { + if (neighborEvents.isEmpty()) { + onOperation(true, neighborEvents); + } else { + onOperation(false, neighborEvents); + } + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else { + // orphan event + if (request.getStore()) { + StoreCorrelationRequest storeCorrelationRequest = new StoreCorrelationRequest( + inputIndex, + event, + timestamp, + Map.of(), + List.of() + ); + client.execute(StoreCorrelationAction.INSTANCE, storeCorrelationRequest, new ActionListener<>() { + @Override + public void onResponse(StoreCorrelationResponse response) { + if (response.getStatus().equals(RestStatus.OK)) { + onOperation(true, new HashMap<>()); + } else { + onFailures(new OpenSearchStatusException("Failed to store correlations", RestStatus.INTERNAL_SERVER_ERROR)); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else { + onOperation(true, new HashMap<>()); + } + } + } + + private void onOperation(Boolean isOrphan, Map> neighborEvents) { + finishHim(isOrphan, neighborEvents, null); + } + + private void onFailures(Exception t) { + log.error("error:", t); + finishHim(null, null, t); + } + + private void finishHim(Boolean isOrphan, Map> neighborEvents, Exception t) { + if (t != null) { + listener.onFailure(t); + } else { + listener.onResponse(new IndexCorrelationResponse(isOrphan, neighborEvents, RestStatus.OK)); + } + } + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/transport/TransportSearchCorrelatedEventsAction.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/transport/TransportSearchCorrelatedEventsAction.java new file mode 100644 index 0000000000000..c4a0746c3bf18 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/transport/TransportSearchCorrelatedEventsAction.java @@ -0,0 +1,300 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.transport; + +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.plugin.correlation.core.index.query.CorrelationQueryBuilder; +import org.opensearch.plugin.correlation.events.action.SearchCorrelatedEventsAction; +import org.opensearch.plugin.correlation.events.action.SearchCorrelatedEventsRequest; +import org.opensearch.plugin.correlation.events.action.SearchCorrelatedEventsResponse; +import org.opensearch.plugin.correlation.events.model.Correlation; +import org.opensearch.plugin.correlation.events.model.EventWithScore; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Transport Action for searching correlated events of a particular event + * + * @opensearch.internal + */ +public class TransportSearchCorrelatedEventsAction extends HandledTransportAction< + SearchCorrelatedEventsRequest, + SearchCorrelatedEventsResponse> { + + private final Client client; + + /** + * Parameterized ctor for Transport Action + * @param transportService TransportService + * @param client OS client + * @param actionFilters ActionFilters + */ + @Inject + public TransportSearchCorrelatedEventsAction(TransportService transportService, Client client, ActionFilters actionFilters) { + super(SearchCorrelatedEventsAction.NAME, transportService, actionFilters, SearchCorrelatedEventsRequest::new); + this.client = client; + } + + @Override + protected void doExecute(Task task, SearchCorrelatedEventsRequest request, ActionListener listener) { + AsyncSearchCorrelatedEventsAction asyncAction = new AsyncSearchCorrelatedEventsAction(request, listener); + asyncAction.start(); + } + + class AsyncSearchCorrelatedEventsAction { + + private SearchCorrelatedEventsRequest request; + private ActionListener listener; + + AsyncSearchCorrelatedEventsAction(SearchCorrelatedEventsRequest request, ActionListener listener) { + this.request = request; + this.listener = listener; + } + + void start() { + String index = request.getIndex(); + String event = request.getEvent(); + String timestampField = request.getTimestampField(); + Long timeWindow = request.getTimeWindow(); + Integer nearbyEvents = request.getNearbyEvents(); + + MatchQueryBuilder queryBuilder = QueryBuilders.matchQuery("_id", event); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(false); + searchSourceBuilder.fetchField(timestampField); + searchSourceBuilder.size(1); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(index); + searchRequest.source(searchSourceBuilder); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT)); + } + if (response.getHits().getTotalHits().value != 1) { + onFailures(new OpenSearchStatusException("Event not found", RestStatus.INTERNAL_SERVER_ERROR)); + } + + SearchHit hit = response.getHits().getAt(0); + long eventTimestamp = hit.getFields().get(timestampField).getValue(); + + BoolQueryBuilder scoreQueryBuilder = QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("score_timestamp", 0L)); + SearchSourceBuilder scoreSearchSourceBuilder = new SearchSourceBuilder(); + scoreSearchSourceBuilder.query(scoreQueryBuilder); + scoreSearchSourceBuilder.fetchSource(true); + scoreSearchSourceBuilder.size(1); + + SearchRequest scoreSearchRequest = new SearchRequest(); + scoreSearchRequest.indices(Correlation.CORRELATION_HISTORY_INDEX); + scoreSearchRequest.source(scoreSearchSourceBuilder); + + client.search(scoreSearchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT)); + } + if (response.getHits().getTotalHits().value != 1) { + onFailures(new OpenSearchStatusException("Score Root Record not found", RestStatus.INTERNAL_SERVER_ERROR)); + } + + Map source = response.getHits().getHits()[0].getSourceAsMap(); + assert source != null; + + long scoreTimestamp; + if (source.get("score_timestamp") instanceof Integer) { + scoreTimestamp = ((Integer) source.get("score_timestamp")).longValue(); + } else { + scoreTimestamp = (long) source.get("score_timestamp"); + } + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .must(QueryBuilders.matchQuery("event1", event)) + .must(QueryBuilders.matchQuery("event2", "")); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(false); + searchSourceBuilder.fetchField("level"); + searchSourceBuilder.size(1); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(Correlation.CORRELATION_HISTORY_INDEX); + searchRequest.source(searchSourceBuilder); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT)); + } + if (response.getHits().getTotalHits().value != 1) { + onFailures( + new OpenSearchStatusException( + "Event not found in Correlation Index", + RestStatus.INTERNAL_SERVER_ERROR + ) + ); + } + + SearchHit hit = response.getHits().getHits()[0]; + long level = hit.getFields().get("level").getValue(); + float[] query = new float[3]; + for (int i = 0; i < 2; ++i) { + query[i] = (2.0f * ((float) level) - 50.0f) / 2.0f; + } + query[2] = Long.valueOf((eventTimestamp - scoreTimestamp) / 1000L).floatValue(); + + CorrelationQueryBuilder correlationQueryBuilder = new CorrelationQueryBuilder( + "corr_vector", + query, + nearbyEvents, + QueryBuilders.boolQuery() + .mustNot(QueryBuilders.matchQuery("event1", "")) + .mustNot(QueryBuilders.matchQuery("event2", "")) + .filter( + QueryBuilders.rangeQuery("timestamp") + .gte(eventTimestamp - timeWindow) + .lte(eventTimestamp + timeWindow) + ) + ); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(correlationQueryBuilder); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(nearbyEvents); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(Correlation.CORRELATION_HISTORY_INDEX); + searchRequest.source(searchSourceBuilder); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT)); + } + SearchHit[] hits = response.getHits().getHits(); + Map correlatedFindings = new HashMap<>(); + + for (SearchHit hit : hits) { + Map source = hit.getSourceAsMap(); + assert source != null; + if (!source.get("event1").toString().equals(event)) { + String eventKey1 = source.get("event1").toString() + + " " + + source.get("index1").toString(); + + if (correlatedFindings.containsKey(eventKey1)) { + correlatedFindings.put( + eventKey1, + Math.max(correlatedFindings.get(eventKey1), hit.getScore()) + ); + } else { + correlatedFindings.put(eventKey1, (double) hit.getScore()); + } + } + if (!source.get("event2").toString().equals(event)) { + String eventKey2 = source.get("event2").toString() + + " " + + source.get("index2").toString(); + + if (correlatedFindings.containsKey(eventKey2)) { + correlatedFindings.put( + eventKey2, + Math.max(correlatedFindings.get(eventKey2), hit.getScore()) + ); + } else { + correlatedFindings.put(eventKey2, (double) hit.getScore()); + } + } + } + + List eventWithScores = new ArrayList<>(); + for (Map.Entry correlatedFinding : correlatedFindings.entrySet()) { + String[] eventIndexSplit = correlatedFinding.getKey().split(" "); + eventWithScores.add( + new EventWithScore( + eventIndexSplit[1], + eventIndexSplit[0], + correlatedFinding.getValue(), + List.of() + ) + ); + } + onOperation(eventWithScores); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + private void onOperation(List events) { + finishHim(events, null); + } + + private void onFailures(Exception t) { + finishHim(null, t); + } + + private void finishHim(List events, Exception t) { + if (t != null) { + listener.onFailure(t); + } else { + listener.onResponse(new SearchCorrelatedEventsResponse(events, RestStatus.OK)); + } + } + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/transport/TransportStoreCorrelationAction.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/transport/TransportStoreCorrelationAction.java new file mode 100644 index 0000000000000..3ac2b0a7dc26a --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/transport/TransportStoreCorrelationAction.java @@ -0,0 +1,752 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.search.MultiSearchRequest; +import org.opensearch.action.search.MultiSearchResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.plugin.correlation.core.index.query.CorrelationQueryBuilder; +import org.opensearch.plugin.correlation.events.action.StoreCorrelationAction; +import org.opensearch.plugin.correlation.events.action.StoreCorrelationRequest; +import org.opensearch.plugin.correlation.events.action.StoreCorrelationResponse; +import org.opensearch.plugin.correlation.events.model.Correlation; +import org.opensearch.plugin.correlation.settings.EventsCorrelationSettings; +import org.opensearch.plugin.correlation.utils.CorrelationIndices; +import org.opensearch.plugin.correlation.utils.IndexUtils; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Transport Action for converting events to vectors and then storing them on disk. + * + * @opensearch.internal + */ +public class TransportStoreCorrelationAction extends HandledTransportAction { + + private static final Logger log = LogManager.getLogger(TransportStoreCorrelationAction.class); + + private final Client client; + + private final Settings settings; + + private final CorrelationIndices correlationIndices; + + private final ClusterService clusterService; + + private final long setupTimestamp; + + private volatile long correlationTimeWindow; + + /** + * Parameterized ctor for Transport Action + * @param transportService TransportService + * @param client OS client + * @param settings Settings + * @param actionFilters ActionFilters + * @param correlationIndices CorrelationIndices which manages lifecycle of correlation history index. + * @param clusterService ClusterService + */ + @Inject + public TransportStoreCorrelationAction( + TransportService transportService, + Client client, + Settings settings, + ActionFilters actionFilters, + CorrelationIndices correlationIndices, + ClusterService clusterService + ) { + super(StoreCorrelationAction.NAME, transportService, actionFilters, StoreCorrelationRequest::new); + this.client = client; + this.settings = settings; + this.correlationIndices = correlationIndices; + this.clusterService = clusterService; + this.setupTimestamp = System.currentTimeMillis(); + this.correlationTimeWindow = EventsCorrelationSettings.CORRELATION_TIME_WINDOW.get(this.settings).getMillis(); + + this.clusterService.getClusterSettings() + .addSettingsUpdateConsumer(EventsCorrelationSettings.CORRELATION_TIME_WINDOW, it -> correlationTimeWindow = it.getMillis()); + } + + @Override + protected void doExecute(Task task, StoreCorrelationRequest request, ActionListener listener) { + AsyncStoreCorrelationAction asyncAction = new AsyncStoreCorrelationAction(request, listener); + + if (!this.correlationIndices.correlationIndexExists()) { + try { + this.correlationIndices.initCorrelationIndex(new ActionListener<>() { + @Override + public void onResponse(CreateIndexResponse response) { + if (response.isAcknowledged()) { + IndexUtils.correlationHistoryIndexUpdated(); + try { + correlationIndices.setupCorrelationIndex(setupTimestamp, new ActionListener<>() { + @Override + public void onResponse(BulkResponse response) { + if (!response.hasFailures()) { + asyncAction.start(); + } else { + asyncAction.onFailures( + new OpenSearchStatusException(response.toString(), RestStatus.INTERNAL_SERVER_ERROR) + ); + } + } + + @Override + public void onFailure(Exception e) { + asyncAction.onFailures(e); + } + }); + } catch (IOException e) { + asyncAction.onFailures(e); + } + } + } + + @Override + public void onFailure(Exception e) { + asyncAction.onFailures(e); + } + }); + } catch (IOException e) { + asyncAction.onFailures(e); + } + } else { + asyncAction.start(); + } + } + + class AsyncStoreCorrelationAction { + private final StoreCorrelationRequest request; + + private final ActionListener listener; + + AsyncStoreCorrelationAction(StoreCorrelationRequest request, ActionListener listener) { + this.request = request; + this.listener = listener; + } + + void start() { + prepareCorrelationHistoryIndex(); + } + + private void prepareCorrelationHistoryIndex() { + try { + if (!IndexUtils.correlationHistoryIndexUpdated) { + IndexUtils.updateIndexMapping( + Correlation.CORRELATION_HISTORY_INDEX, + CorrelationIndices.correlationMappings(), + clusterService.state(), + client.admin().indices(), + new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse response) { + if (response.isAcknowledged()) { + IndexUtils.correlationHistoryIndexUpdated(); + generateTimestampFeature(); + } else { + onFailures( + new OpenSearchStatusException( + "Failed to create correlation Index", + RestStatus.INTERNAL_SERVER_ERROR + ) + ); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + } + ); + } else { + generateTimestampFeature(); + } + } catch (IOException ex) { + onFailures(ex); + } + } + + private void generateTimestampFeature() { + Long eventTimestamp = request.getTimestamp(); + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("score_timestamp", 0L)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(1); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(Correlation.CORRELATION_HISTORY_INDEX); + searchRequest.source(searchSourceBuilder); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT)); + } + if (response.getHits().getTotalHits().value != 1) { + onFailures(new OpenSearchStatusException("Score Root Record not found", RestStatus.INTERNAL_SERVER_ERROR)); + } + + SearchHit hit = response.getHits().getHits()[0]; + String id = hit.getId(); + Map source = hit.getSourceAsMap(); + + long scoreTimestamp = 0L; + if (source != null) { + if (source.get("score_timestamp") instanceof Integer) { + scoreTimestamp = ((Integer) source.get("score_timestamp")).longValue(); + } else { + scoreTimestamp = (long) source.get("score_timestamp"); + } + } + if (eventTimestamp - CorrelationIndices.FIXED_HISTORICAL_INTERVAL > scoreTimestamp) { + try { + Correlation scoreRootRecord = new Correlation( + id, + 1L, + false, + 0L, + "", + "", + new float[] {}, + 0L, + "", + "", + List.of(), + eventTimestamp - CorrelationIndices.FIXED_HISTORICAL_INTERVAL + ); + + IndexRequest scoreIndexRequest = new IndexRequest(Correlation.CORRELATION_HISTORY_INDEX).id(id) + .source(scoreRootRecord.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .timeout(TimeValue.timeValueSeconds(60)) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + client.index(scoreIndexRequest, new ActionListener<>() { + @Override + public void onResponse(IndexResponse response) { + if (response.status().equals(RestStatus.OK)) { + if (request.getEventsAdjacencyList() == null || request.getEventsAdjacencyList().isEmpty()) { + insertOrphanEvents( + Long.valueOf(CorrelationIndices.FIXED_HISTORICAL_INTERVAL / 1000L).floatValue() + ); + } else { + insertCorrelatedEvents( + Long.valueOf(CorrelationIndices.FIXED_HISTORICAL_INTERVAL / 1000L).floatValue() + ); + } + } else { + onFailures( + new OpenSearchStatusException( + "Failed to update Score Root record", + RestStatus.INTERNAL_SERVER_ERROR + ) + ); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } catch (IOException ex) { + onFailures(ex); + } + } else { + float timestampFeature = Long.valueOf((eventTimestamp - scoreTimestamp) / 1000L).floatValue(); + if (request.getEventsAdjacencyList() == null || request.getEventsAdjacencyList().isEmpty()) { + insertOrphanEvents(timestampFeature); + } else { + insertCorrelatedEvents(timestampFeature); + } + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + private void insertCorrelatedEvents(float timestampFeature) { + long eventTimestamp = request.getTimestamp(); + Map> neighborEvents = request.getEventsAdjacencyList(); + + MatchQueryBuilder queryBuilder = QueryBuilders.matchQuery("root", true); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(1); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(Correlation.CORRELATION_HISTORY_INDEX); + searchRequest.source(searchSourceBuilder); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT)); + } + if (response.getHits().getTotalHits().value != 1) { + onFailures(new OpenSearchStatusException("Root Record not found", RestStatus.INTERNAL_SERVER_ERROR)); + } + + SearchHit hit = response.getHits().getHits()[0]; + Map source = hit.getSourceAsMap(); + + assert source != null; + long level = Long.parseLong(source.get("level").toString()); + + MultiSearchRequest mSearchRequest = new MultiSearchRequest(); + for (Map.Entry> neighborEvent : neighborEvents.entrySet()) { + for (String event : neighborEvent.getValue()) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .must(QueryBuilders.matchQuery("event1", event)) + .must(QueryBuilders.matchQuery("event2", "")); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(1); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(Correlation.CORRELATION_HISTORY_INDEX); + searchRequest.source(searchSourceBuilder); + mSearchRequest.add(searchRequest); + } + } + + client.multiSearch(mSearchRequest, new ActionListener<>() { + @Override + public void onResponse(MultiSearchResponse items) { + MultiSearchResponse.Item[] responses = items.getResponses(); + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + long prevLevel = -1L; + long totalNeighbors = 0L; + for (MultiSearchResponse.Item response : responses) { + if (response.isFailure()) { + log.info(response.getFailureMessage()); + continue; + } + + if (response.getResponse().getHits().getTotalHits().value == 1) { + ++totalNeighbors; + + SearchHit hit = response.getResponse().getHits().getHits()[0]; + Map source = hit.getSourceAsMap(); + + assert source != null; + long neighborLevel = Long.parseLong(source.get("level").toString()); + String correlatedEvent = source.get("event1").toString(); + String correlatedIndex = source.get("index1").toString(); + + try { + float[] corrVector = new float[3]; + if (level != prevLevel) { + for (int i = 0; i < 2; ++i) { + corrVector[i] = ((float) level) - 50.0f; + } + corrVector[0] = (float) level; + corrVector[2] = timestampFeature; + + Correlation event = new Correlation( + false, + level, + request.getEvent(), + "", + corrVector, + eventTimestamp, + request.getIndex(), + "", + request.getTags(), + 0L + ); + + IndexRequest indexRequest = new IndexRequest(Correlation.CORRELATION_HISTORY_INDEX).source( + event.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS) + ).timeout(TimeValue.timeValueSeconds(60)); + bulkRequest.add(indexRequest); + } + + corrVector = new float[3]; + for (int i = 0; i < 2; ++i) { + corrVector[i] = ((float) level) - 50.0f; + } + corrVector[0] = (2.0f * ((float) level) - 50.0f) / 2.0f; + corrVector[1] = (2.0f * ((float) neighborLevel) - 50.0f) / 2.0f; + corrVector[2] = timestampFeature; + + Correlation event = new Correlation( + false, + (long) ((2.0f * ((float) level) - 50.0f) / 2.0f), + request.getEvent(), + correlatedEvent, + corrVector, + eventTimestamp, + request.getIndex(), + correlatedIndex, + request.getTags(), + 0L + ); + + IndexRequest indexRequest = new IndexRequest(Correlation.CORRELATION_HISTORY_INDEX).source( + event.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS) + ).timeout(TimeValue.timeValueSeconds(60)); + bulkRequest.add(indexRequest); + } catch (IOException ex) { + onFailures(ex); + } + prevLevel = level; + } + } + + if (totalNeighbors > 0L) { + client.bulk(bulkRequest, new ActionListener<>() { + @Override + public void onResponse(BulkResponse response) { + if (response.hasFailures()) { + onFailures( + new OpenSearchStatusException( + "Correlation of finding failed", + RestStatus.INTERNAL_SERVER_ERROR + ) + ); + } + onOperation(); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else { + insertOrphanEvents(timestampFeature); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + private void insertOrphanEvents(float timestampFeature) { + long eventTimestamp = request.getTimestamp(); + + MatchQueryBuilder queryBuilder = QueryBuilders.matchQuery("root", true); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(1); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(Correlation.CORRELATION_HISTORY_INDEX); + searchRequest.source(searchSourceBuilder); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT)); + } + if (response.getHits().getTotalHits().value != 1) { + onFailures(new OpenSearchStatusException("Root Record not found", RestStatus.INTERNAL_SERVER_ERROR)); + } + + SearchHit hit = response.getHits().getHits()[0]; + Map source = hit.getSourceAsMap(); + String id = hit.getId(); + + assert source != null; + long level = Long.parseLong(source.get("level").toString()); + long timestamp = Long.parseLong(source.get("timestamp").toString()); + + try { + if (level == 0L) { + updateRootRecord(id, 50L, new ActionListener<>() { + @Override + public void onResponse(IndexResponse response) { + if (response.status().equals(RestStatus.OK)) { + try { + float[] corrVector = new float[3]; + corrVector[0] = 50.0f; + corrVector[2] = timestampFeature; + + storeEvents(50L, corrVector); + } catch (IOException ex) { + onFailures(ex); + } + } else { + onFailures( + new OpenSearchStatusException("Root Record not updated", RestStatus.INTERNAL_SERVER_ERROR) + ); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else { + if (eventTimestamp - timestamp > correlationTimeWindow) { + updateRootRecord(id, 50L, new ActionListener<>() { + @Override + public void onResponse(IndexResponse response) { + if (response.status().equals(RestStatus.OK)) { + try { + float[] corrVector = new float[3]; + corrVector[0] = 50.0f; + corrVector[2] = timestampFeature; + + storeEvents(50L, corrVector); + } catch (IOException ex) { + onFailures(ex); + } + } else { + onFailures( + new OpenSearchStatusException("Root Record not updated", RestStatus.INTERNAL_SERVER_ERROR) + ); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else { + float[] query = new float[3]; + for (int i = 0; i < 2; ++i) { + query[i] = (2.0f * ((float) level) - 50.0f) / 2.0f; + } + query[2] = timestampFeature; + + CorrelationQueryBuilder correlationQueryBuilder = new CorrelationQueryBuilder( + "corr_vector", + query, + 3, + QueryBuilders.boolQuery() + .mustNot(QueryBuilders.matchQuery("event1", "")) + .mustNot(QueryBuilders.matchQuery("event2", "")) + .filter( + QueryBuilders.rangeQuery("timestamp") + .gte(eventTimestamp - correlationTimeWindow) + .lte(eventTimestamp + correlationTimeWindow) + ) + ); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(correlationQueryBuilder); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(1); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(Correlation.CORRELATION_HISTORY_INDEX); + searchRequest.source(searchSourceBuilder); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT)); + } + + long totalHits = response.getHits().getTotalHits().value; + SearchHit hit = totalHits > 0 ? response.getHits().getHits()[0] : null; + long existLevel = 0L; + + if (hit != null) { + Map source = hit.getSourceAsMap(); + assert source != null; + existLevel = Long.parseLong(source.get("level").toString()); + } + + try { + if (totalHits == 0L || existLevel != ((long) (2.0f * ((float) level) - 50.0f) / 2.0f)) { + float[] corrVector = new float[3]; + for (int i = 0; i < 2; ++i) { + corrVector[i] = ((float) level) - 50.0f; + } + corrVector[0] = (float) level; + corrVector[2] = timestampFeature; + + storeEvents(level, corrVector); + } else { + updateRootRecord(id, level + 50L, new ActionListener<>() { + @Override + public void onResponse(IndexResponse response) { + if (response.status().equals(RestStatus.OK)) { + try { + float[] corrVector = new float[3]; + for (int i = 0; i < 2; ++i) { + corrVector[i] = (float) level; + } + corrVector[0] = level + 50.0f; + corrVector[2] = timestampFeature; + + storeEvents(level + 50L, corrVector); + } catch (IOException ex) { + onFailures(ex); + } + } + } + + @Override + public void onFailure(Exception e) { + + } + }); + } + } catch (IOException ex) { + onFailures(ex); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + } + } catch (IOException ex) { + onFailures(ex); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + private void updateRootRecord(String id, Long level, ActionListener listener) throws IOException { + Correlation rootRecord = new Correlation( + id, + 1L, + true, + level, + "", + "", + new float[] { 0.0f, 0.0f, 0.0f }, + request.getTimestamp(), + "", + "", + List.of(), + 0L + ); + + IndexRequest indexRequest = new IndexRequest(Correlation.CORRELATION_HISTORY_INDEX).id(id) + .source(rootRecord.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .timeout(TimeValue.timeValueSeconds(60)) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + client.index(indexRequest, listener); + } + + private void storeEvents(Long level, float[] correlationVector) throws IOException { + Correlation event = new Correlation( + false, + level, + request.getEvent(), + "", + correlationVector, + request.getTimestamp(), + request.getIndex(), + "", + request.getTags(), + 0L + ); + + IndexRequest indexRequest = new IndexRequest(Correlation.CORRELATION_HISTORY_INDEX).source( + event.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS) + ).timeout(TimeValue.timeValueSeconds(60)).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(IndexResponse response) { + if (response.status().equals(RestStatus.CREATED)) { + onOperation(); + } else { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.INTERNAL_SERVER_ERROR)); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + private void onOperation() { + finishHim(null); + } + + private void onFailures(Exception t) { + finishHim(t); + } + + private void finishHim(Exception t) { + if (t != null) { + listener.onFailure(t); + } else { + listener.onResponse(new StoreCorrelationResponse(RestStatus.OK)); + } + } + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/transport/package-info.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/transport/package-info.java new file mode 100644 index 0000000000000..741a51cc68838 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/transport/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Transport Actions for Correlation Events + */ +package org.opensearch.plugin.correlation.events.transport; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/model/CorrelationRule.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/model/CorrelationRule.java index 6978d7248e199..e1a3519abf68d 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/model/CorrelationRule.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/model/CorrelationRule.java @@ -144,11 +144,9 @@ public void writeTo(StreamOutput out) throws IOException { /** * parse into CorrelationRule * @param xcp XContentParser - * @param id id of rule - * @param version version of rule * @return CorrelationRule */ - public static CorrelationRule parse(XContentParser xcp, String id, Long version) { + public static CorrelationRule parse(XContentParser xcp) { return PARSER.apply(xcp, null); } diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/resthandler/RestIndexCorrelationRuleAction.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/resthandler/RestIndexCorrelationRuleAction.java index 3b2b7eb02ae5f..b43341c1960b1 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/resthandler/RestIndexCorrelationRuleAction.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/rules/resthandler/RestIndexCorrelationRuleAction.java @@ -68,7 +68,10 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli XContentParser xcp = request.contentParser(); - CorrelationRule correlationRule = CorrelationRule.parse(xcp, id, 1L); + CorrelationRule correlationRule = CorrelationRule.parse(xcp); + correlationRule.setId(id); + correlationRule.setVersion(1L); + IndexCorrelationRuleRequest indexCorrelationRuleRequest = new IndexCorrelationRuleRequest(id, correlationRule, request.method()); return channel -> client.execute( IndexCorrelationRuleAction.INSTANCE, diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/settings/EventsCorrelationSettings.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/settings/EventsCorrelationSettings.java index 2e2dbbffbeaa2..c1a0c4bd4e4e2 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/settings/EventsCorrelationSettings.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/settings/EventsCorrelationSettings.java @@ -30,6 +30,12 @@ public class EventsCorrelationSettings { * Boolean setting to check if an OS index is a correlation index. */ public static final Setting IS_CORRELATION_INDEX_SETTING = Setting.boolSetting(CORRELATION_INDEX, false, IndexScope); + public static final Setting CORRELATION_HISTORY_INDEX_SHARDS = Setting.intSetting( + "plugins.events_correlation.correlation_history_index_shards", + 1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); /** * Global time window setting for Correlations */ diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/utils/CorrelationIndices.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/utils/CorrelationIndices.java new file mode 100644 index 0000000000000..d91e3bb5a8291 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/utils/CorrelationIndices.java @@ -0,0 +1,153 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.utils; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.plugin.correlation.events.model.Correlation; +import org.opensearch.plugin.correlation.settings.EventsCorrelationSettings; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Objects; + +/** + * Correlation History Index manager + * + * @opensearch.internal + */ +public class CorrelationIndices { + + private static final Logger log = LogManager.getLogger(CorrelationIndices.class); + public static final long FIXED_HISTORICAL_INTERVAL = 24L * 60L * 60L * 20L * 1000L; + + private final Client client; + + private final ClusterService clusterService; + + private final Settings settings; + + private volatile int noOfShards; + + /** + * Parameterized ctor for CorrelationIndices + * @param client OS client + * @param clusterService ClusterService + * @param settings Settings + */ + public CorrelationIndices(Client client, ClusterService clusterService, Settings settings) { + this.client = client; + this.clusterService = clusterService; + this.settings = settings; + this.noOfShards = EventsCorrelationSettings.CORRELATION_HISTORY_INDEX_SHARDS.get(this.settings); + this.clusterService.getClusterSettings() + .addSettingsUpdateConsumer(EventsCorrelationSettings.CORRELATION_HISTORY_INDEX_SHARDS, it -> noOfShards = it); + } + + /** + * get correlation history index mappings + * @return correlation history index mappings + * @throws IOException IOException + */ + public static String correlationMappings() throws IOException { + return new String( + Objects.requireNonNull(CorrelationIndices.class.getClassLoader().getResourceAsStream("mappings/correlation.json")) + .readAllBytes(), + Charset.defaultCharset() + ); + } + + /** + * init the correlation history index + * @param actionListener listener + * @throws IOException IOException + */ + public void initCorrelationIndex(ActionListener actionListener) throws IOException { + if (!correlationIndexExists()) { + CreateIndexRequest indexRequest = new CreateIndexRequest(Correlation.CORRELATION_HISTORY_INDEX).mapping(correlationMappings()) + .settings( + Settings.builder().put("index.hidden", true).put("number_of_shards", noOfShards).put("index.correlation", true).build() + ); + client.admin().indices().create(indexRequest, actionListener); + } + } + + /** + * check if the correlation history index exists + * @return boolean + */ + public boolean correlationIndexExists() { + ClusterState clusterState = clusterService.state(); + return clusterState.getRoutingTable().hasIndex(Correlation.CORRELATION_HISTORY_INDEX); + } + + /** + * setup correlation history index with default metadata after initializing it + * @param setupTimestamp initial timestamp when security analytics starts + * @param listener listener + * @throws IOException IOException + */ + public void setupCorrelationIndex(Long setupTimestamp, ActionListener listener) throws IOException { + long currentTimestamp = System.currentTimeMillis(); + + Correlation rootRecord = new Correlation( + true, + 0L, + "", + "", + new float[] { 0.0f, 0.0f, 0.0f }, + currentTimestamp, + "", + "", + List.of(), + 0L + ); + IndexRequest indexRequest = new IndexRequest(Correlation.CORRELATION_HISTORY_INDEX).source( + rootRecord.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS) + ).timeout(TimeValue.timeValueSeconds(60)); + + Correlation scoreRootRecord = new Correlation( + false, + 0L, + "", + "", + new float[] { 0.0f, 0.0f, 0.0f }, + 0L, + "", + "", + List.of(), + setupTimestamp + ); + IndexRequest scoreIndexRequest = new IndexRequest(Correlation.CORRELATION_HISTORY_INDEX).source( + scoreRootRecord.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS) + ).timeout(TimeValue.timeValueSeconds(60)); + + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(indexRequest); + bulkRequest.add(scoreIndexRequest); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + client.bulk(bulkRequest, listener); + } +} diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/utils/IndexUtils.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/utils/IndexUtils.java index 362be3d2932e3..668e95ba81bc0 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/utils/IndexUtils.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/utils/IndexUtils.java @@ -40,6 +40,7 @@ public class IndexUtils { * manages the mappings lifecycle for correlation rule index */ public static Boolean correlationRuleIndexUpdated = false; + public static Boolean correlationHistoryIndexUpdated = false; private IndexUtils() {} @@ -50,6 +51,10 @@ public static void correlationRuleIndexUpdated() { correlationRuleIndexUpdated = true; } + public static void correlationHistoryIndexUpdated() { + correlationHistoryIndexUpdated = true; + } + /** * util method which decides based on schema version whether to update an index. * @param index IndexMetadata @@ -134,6 +139,9 @@ public static void updateIndexMapping( } else { actionListener.onResponse(new AcknowledgedResponse(true)); } + } else { + // this is an impossible scenario. only added to pass unit tests. + actionListener.onResponse(new AcknowledgedResponse(true)); } } } diff --git a/plugins/events-correlation-engine/src/main/resources/mappings/correlation.json b/plugins/events-correlation-engine/src/main/resources/mappings/correlation.json new file mode 100644 index 0000000000000..008366149d0d7 --- /dev/null +++ b/plugins/events-correlation-engine/src/main/resources/mappings/correlation.json @@ -0,0 +1,50 @@ +{ + "_meta" : { + "schema_version": 1 + }, + "properties": { + "root": { + "type": "boolean" + }, + "level":{ + "type": "long" + }, + "event1":{ + "type": "keyword" + }, + "event2":{ + "type": "keyword" + }, + "corr_vector": { + "type": "correlation_vector", + "dimension": 3, + "correlation_ctx": { + "similarityFunction": "EUCLIDEAN", + "parameters": { + "m": 16, + "ef_construction": 128 + } + } + }, + "timestamp":{ + "type": "long" + }, + "index1": { + "type": "keyword" + }, + "index2": { + "type": "keyword" + }, + "tags": { + "type": "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "score_timestamp": { + "type": "long" + } + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/CorrelationCodecTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/CorrelationCodecTests.java index 7223b450a136c..a7b5c1c7d2510 100644 --- a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/CorrelationCodecTests.java +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/CorrelationCodecTests.java @@ -34,6 +34,7 @@ import static org.opensearch.plugin.correlation.core.index.codec.BasePerFieldCorrelationVectorsFormat.METHOD_PARAMETER_M; import static org.opensearch.plugin.correlation.core.index.codec.CorrelationCodecVersion.V_9_9_0; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; @@ -106,7 +107,7 @@ private void testCorrelationVectorIndex( IndexReader reader = writer.getReader(); writer.close(); - verify(perFieldCorrelationVectorsFormatSpy).getKnnVectorsFormatForField(eq(FIELD_NAME_ONE)); + verify(perFieldCorrelationVectorsFormatSpy, atLeastOnce()).getKnnVectorsFormatForField(eq(FIELD_NAME_ONE)); IndexSearcher searcher = new IndexSearcher(reader); Query query = CorrelationQueryFactory.create( diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationActionTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationActionTests.java new file mode 100644 index 0000000000000..e58e3f1edc714 --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationActionTests.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; + +public class IndexCorrelationActionTests extends OpenSearchTestCase { + + public void testIndexCorrelationActionName() { + Assert.assertNotNull(IndexCorrelationAction.INSTANCE.name()); + Assert.assertEquals(IndexCorrelationAction.INSTANCE.name(), IndexCorrelationAction.NAME); + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationRequestTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationRequestTests.java new file mode 100644 index 0000000000000..4ddcdea1f631a --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationRequestTests.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; + +import java.io.IOException; + +public class IndexCorrelationRequestTests extends OpenSearchTestCase { + + public void testIndexCorrelationRequest() throws IOException { + IndexCorrelationRequest request = new IndexCorrelationRequest("index1", "event1", true); + Assert.assertNotNull(request); + + BytesStreamOutput out = new BytesStreamOutput(); + request.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IndexCorrelationRequest newRequest = new IndexCorrelationRequest(sin); + + Assert.assertEquals("index1", newRequest.getIndex()); + Assert.assertEquals("event1", newRequest.getEvent()); + Assert.assertEquals(true, newRequest.getStore()); + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationResponseTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationResponseTests.java new file mode 100644 index 0000000000000..ad31f16cece60 --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationResponseTests.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; + +import java.io.IOException; +import java.util.Map; + +public class IndexCorrelationResponseTests extends OpenSearchTestCase { + + public void testIndexCorrelationResponse() throws IOException { + IndexCorrelationResponse response = new IndexCorrelationResponse(true, Map.of(), RestStatus.CREATED); + Assert.assertNotNull(response); + + BytesStreamOutput out = new BytesStreamOutput(); + response.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + + IndexCorrelationResponse newResponse = new IndexCorrelationResponse(sin); + Assert.assertEquals(true, newResponse.getOrphan()); + Assert.assertEquals(Map.of(), newResponse.getNeighborEvents()); + Assert.assertEquals(RestStatus.CREATED, newResponse.getStatus()); + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsActionTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsActionTests.java new file mode 100644 index 0000000000000..756e92c197790 --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsActionTests.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; + +public class SearchCorrelatedEventsActionTests extends OpenSearchTestCase { + + public void testSearchCorrelatedEventsActionName() { + Assert.assertNotNull(SearchCorrelatedEventsAction.INSTANCE.name()); + Assert.assertEquals(SearchCorrelatedEventsAction.INSTANCE.name(), SearchCorrelatedEventsAction.NAME); + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsRequestTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsRequestTests.java new file mode 100644 index 0000000000000..0718181d76522 --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsRequestTests.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; + +import java.io.IOException; + +public class SearchCorrelatedEventsRequestTests extends OpenSearchTestCase { + + public void testSearchCorrelatedEventsRequest() throws IOException { + SearchCorrelatedEventsRequest request = new SearchCorrelatedEventsRequest("index1", "event1", "timestamp1", 100000L, 5); + Assert.assertNotNull(request); + + BytesStreamOutput out = new BytesStreamOutput(); + request.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SearchCorrelatedEventsRequest newRequest = new SearchCorrelatedEventsRequest(sin); + + Assert.assertEquals("index1", newRequest.getIndex()); + Assert.assertEquals("event1", newRequest.getEvent()); + Assert.assertEquals("timestamp1", newRequest.getTimestampField()); + Assert.assertEquals(100000L, (long) newRequest.getTimeWindow()); + Assert.assertEquals(5, (int) newRequest.getNearbyEvents()); + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsResponseTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsResponseTests.java new file mode 100644 index 0000000000000..60ffcdde6b007 --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/SearchCorrelatedEventsResponseTests.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.plugin.correlation.events.model.EventWithScore; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; + +import java.io.IOException; +import java.util.List; + +public class SearchCorrelatedEventsResponseTests extends OpenSearchTestCase { + + public void testSearchCorrelatedEventsResponse() throws IOException { + EventWithScore event = new EventWithScore("index1", "event1", 0.0, List.of()); + SearchCorrelatedEventsResponse response = new SearchCorrelatedEventsResponse(List.of(event), RestStatus.OK); + + BytesStreamOutput out = new BytesStreamOutput(); + response.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SearchCorrelatedEventsResponse newResponse = new SearchCorrelatedEventsResponse(sin); + + Assert.assertNotNull(newResponse.getEvents()); + Assert.assertEquals(1, newResponse.getEvents().size()); + Assert.assertEquals(event, newResponse.getEvents().get(0)); + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationActionTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationActionTests.java new file mode 100644 index 0000000000000..c8b7dfc8a5d0d --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationActionTests.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; + +public class StoreCorrelationActionTests extends OpenSearchTestCase { + + public void testStoreCorrelationActionName() { + Assert.assertNotNull(StoreCorrelationAction.INSTANCE.name()); + Assert.assertEquals(StoreCorrelationAction.INSTANCE.name(), StoreCorrelationAction.NAME); + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationRequestTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationRequestTests.java new file mode 100644 index 0000000000000..0f8de1e6e408a --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationRequestTests.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class StoreCorrelationRequestTests extends OpenSearchTestCase { + + public void testStoreCorrelationRequest() throws IOException { + StoreCorrelationRequest request = new StoreCorrelationRequest("index1", "event1", 100000L, Map.of(), List.of()); + Assert.assertNotNull(request); + + BytesStreamOutput out = new BytesStreamOutput(); + request.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + StoreCorrelationRequest newRequest = new StoreCorrelationRequest(sin); + + Assert.assertEquals("index1", newRequest.getIndex()); + Assert.assertEquals("event1", newRequest.getEvent()); + Assert.assertEquals(100000L, (long) newRequest.getTimestamp()); + Assert.assertEquals(Map.of(), newRequest.getEventsAdjacencyList()); + Assert.assertEquals(List.of(), newRequest.getTags()); + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationResponseTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationResponseTests.java new file mode 100644 index 0000000000000..50036773eb1b1 --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/action/StoreCorrelationResponseTests.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.action; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; + +import java.io.IOException; + +public class StoreCorrelationResponseTests extends OpenSearchTestCase { + + public void testStoreCorrelationResponse() throws IOException { + StoreCorrelationResponse response = new StoreCorrelationResponse(RestStatus.OK); + BytesStreamOutput out = new BytesStreamOutput(); + response.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + StoreCorrelationResponse newResponse = new StoreCorrelationResponse(sin); + + Assert.assertEquals(RestStatus.OK, newResponse.getStatus()); + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/model/CorrelationTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/model/CorrelationTests.java new file mode 100644 index 0000000000000..f33c79ac2f43a --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/model/CorrelationTests.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.model; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +import org.opensearch.common.Randomness; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.plugin.correlation.utils.TestHelpers; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; + +import java.io.IOException; +import java.util.List; + +public class CorrelationTests extends OpenSearchTestCase { + + public void testCorrelationAsStream() throws IOException { + Correlation correlation = randomCorrelation(null, null, null, null, new float[] {}, null, null, null, null, null); + BytesStreamOutput out = new BytesStreamOutput(); + correlation.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + Correlation newCorrelation = Correlation.readFrom(sin); + Assert.assertEquals("Round tripping Correlation doesn't work", correlation, newCorrelation); + } + + public void testCorrelationParsing() throws IOException { + Correlation correlation = randomCorrelation(null, null, null, null, new float[] {}, null, null, null, null, null); + String correlationStr = TestHelpers.toJsonString(correlation); + + Correlation parsedCorrelation = Correlation.parse(TestHelpers.parse(correlationStr)); + parsedCorrelation.setId(""); + parsedCorrelation.setVersion(1L); + + Assert.assertEquals(correlation, parsedCorrelation); + } + + private Correlation randomCorrelation( + Boolean isRootParam, + Long levelParam, + String event1Param, + String event2Param, + float[] correlationVectorParam, + Long timestampParam, + String index1Param, + String index2Param, + List tagsParam, + Long scoreTimestampParam + ) { + Boolean isRoot = isRootParam != null ? isRootParam : false; + Long level = levelParam != null ? levelParam : 1L; + String event1 = event1Param != null ? event1Param : RandomStrings.randomAsciiLettersOfLength(Randomness.get(), 10); + String event2 = event2Param != null ? event2Param : RandomStrings.randomAsciiLettersOfLength(Randomness.get(), 10); + float[] correlationVector = correlationVectorParam.length > 0 ? correlationVectorParam : new float[] { 1.0f, 1.0f, 1.0f }; + Long timestamp = timestampParam != null ? timestampParam : System.currentTimeMillis(); + String index1 = index1Param != null ? index1Param : RandomStrings.randomAsciiLettersOfLength(Randomness.get(), 10); + String index2 = index2Param != null ? index2Param : RandomStrings.randomAsciiLettersOfLength(Randomness.get(), 10); + List tags = tagsParam != null ? tagsParam : List.of(RandomStrings.randomAsciiLettersOfLength(Randomness.get(), 10)); + Long scoreTimestamp = scoreTimestampParam != null ? scoreTimestampParam : System.currentTimeMillis(); + return new Correlation("", 1L, isRoot, level, event1, event2, correlationVector, timestamp, index1, index2, tags, scoreTimestamp); + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/model/EventWithScoreTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/model/EventWithScoreTests.java new file mode 100644 index 0000000000000..b24f1421bf1f8 --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/model/EventWithScoreTests.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.model; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +import org.opensearch.common.Randomness; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.plugin.correlation.utils.TestHelpers; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; + +import java.io.IOException; +import java.util.List; + +public class EventWithScoreTests extends OpenSearchTestCase { + + public void testEventWithScoreAsStream() throws IOException { + EventWithScore eventWithScore = randomEventWithScore(null, null, null, null); + BytesStreamOutput out = new BytesStreamOutput(); + eventWithScore.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + EventWithScore newEventWithScore = EventWithScore.readFrom(sin); + Assert.assertEquals("Round tripping EventWithScore doesn't work", eventWithScore, newEventWithScore); + } + + public void testEventWithScoreParsing() throws IOException { + EventWithScore eventWithScore = randomEventWithScore(null, null, null, null); + String eventWithScoreStr = TestHelpers.toJsonString(eventWithScore); + + EventWithScore parsedEventWithScore = EventWithScore.parse(TestHelpers.parse(eventWithScoreStr)); + Assert.assertEquals(eventWithScore, parsedEventWithScore); + } + + private EventWithScore randomEventWithScore(String indexParam, String eventParam, Double scoreParam, List tagsParam) { + String index = indexParam != null ? indexParam : RandomStrings.randomAsciiLettersOfLength(Randomness.get(), 10); + String event = eventParam != null ? eventParam : RandomStrings.randomAsciiLettersOfLength(Randomness.get(), 10); + Double score = scoreParam != null ? scoreParam : 0.0; + List tags = tagsParam != null ? tagsParam : List.of(); + return new EventWithScore(index, event, score, tags); + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/resthandler/RestIndexCorrelationActionTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/resthandler/RestIndexCorrelationActionTests.java new file mode 100644 index 0000000000000..96464bb68114a --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/resthandler/RestIndexCorrelationActionTests.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.resthandler; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionType; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.SetOnce; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.plugin.correlation.utils.TestHelpers; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.client.NoOpNodeClient; +import org.opensearch.test.rest.FakeRestChannel; +import org.opensearch.test.rest.FakeRestRequest; +import org.junit.Assert; + +public class RestIndexCorrelationActionTests extends OpenSearchTestCase { + + public void testPrepareRequest() throws Exception { + SetOnce transportActionCalled = new SetOnce<>(); + try (NodeClient client = new NoOpNodeClient(this.getTestName()) { + @Override + public void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + transportActionCalled.set(true); + super.doExecute(action, request, listener); + } + }) { + RestIndexCorrelationAction action = new RestIndexCorrelationAction(); + RestRequest request = new FakeRestRequest.Builder(TestHelpers.xContentRegistry()).withContent( + new BytesArray("{\"index\":\"app_logs\",\"event\":\"l2UZD4kBSz-dGVL1EZFJ\",\"store\":false}"), + XContentType.JSON + ).build(); + FakeRestChannel channel = new FakeRestChannel(request, false, 0); + action.handleRequest(request, channel, client); + Assert.assertEquals(true, transportActionCalled.get()); + } + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/resthandler/RestSearchCorrelatedEventsActionTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/resthandler/RestSearchCorrelatedEventsActionTests.java new file mode 100644 index 0000000000000..726e207d105ca --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/resthandler/RestSearchCorrelatedEventsActionTests.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.resthandler; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionType; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.SetOnce; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.plugin.correlation.utils.TestHelpers; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.client.NoOpNodeClient; +import org.opensearch.test.rest.FakeRestChannel; +import org.opensearch.test.rest.FakeRestRequest; +import org.junit.Assert; + +import java.util.HashMap; +import java.util.Map; + +public class RestSearchCorrelatedEventsActionTests extends OpenSearchTestCase { + + public void testPrepareRequest() throws Exception { + SetOnce transportActionCalled = new SetOnce<>(); + try (NodeClient client = new NoOpNodeClient(this.getTestName()) { + @Override + public void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + transportActionCalled.set(true); + super.doExecute(action, request, listener); + } + }) { + RestSearchCorrelatedEventsAction action = new RestSearchCorrelatedEventsAction(); + Map params = new HashMap<>(); + params.put("index", "app_logs"); + params.put("event", "l2UZD4kBSz-dGVL1EZFJ"); + params.put("timestamp_field", "@timestamp"); + + RestRequest request = new FakeRestRequest.Builder(TestHelpers.xContentRegistry()).withParams(params).build(); + FakeRestChannel channel = new FakeRestChannel(request, false, 0); + action.handleRequest(request, channel, client); + Assert.assertEquals(true, transportActionCalled.get()); + } + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/transport/TransportIndexCorrelationActionTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/transport/TransportIndexCorrelationActionTests.java new file mode 100644 index 0000000000000..3ed920a9810d0 --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/transport/TransportIndexCorrelationActionTests.java @@ -0,0 +1,231 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.transport; + +import org.apache.lucene.search.TotalHits; +import org.opensearch.Version; +import org.opensearch.action.search.MultiSearchRequest; +import org.opensearch.action.search.MultiSearchResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.SearchResponseSections; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.document.DocumentField; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.plugin.correlation.events.action.IndexCorrelationRequest; +import org.opensearch.plugin.correlation.events.action.IndexCorrelationResponse; +import org.opensearch.plugin.correlation.settings.EventsCorrelationSettings; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.telemetry.tracing.noop.NoopTracer; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.VersionUtils; +import org.opensearch.test.transport.CapturingTransport; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static java.util.Collections.emptyMap; +import static org.opensearch.test.ClusterServiceUtils.createClusterService; + +public class TransportIndexCorrelationActionTests extends OpenSearchTestCase { + + private static ThreadPool threadPool; + private Settings settings; + private ClusterService clusterService; + + @BeforeClass + public static void beforeClass() { + threadPool = new TestThreadPool("TransportIndexCorrelationActionTests"); + } + + @AfterClass + public static void afterClass() { + ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); + threadPool = null; + } + + @Before + public void setUp() throws Exception { + super.setUp(); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + VersionUtils.randomCompatibleVersion(random(), Version.CURRENT) + ); + settings = Settings.builder() + .put(EventsCorrelationSettings.CORRELATION_TIME_WINDOW.getKey(), new TimeValue(5, TimeUnit.MINUTES)) + .build(); + Set> settingSet = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + settingSet.add(EventsCorrelationSettings.CORRELATION_TIME_WINDOW); + ClusterSettings clusterSettings = new ClusterSettings(settings, settingSet); + clusterService = createClusterService(threadPool, discoveryNode, clusterSettings); + } + + public void tearDown() throws Exception { + super.tearDown(); + clusterService.close(); + } + + private TransportIndexCorrelationAction createAction() { + CapturingTransport capturingTransport = new CapturingTransport(); + TransportService transportService = capturingTransport.createTransportService( + clusterService.getSettings(), + threadPool, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, + boundAddress -> clusterService.localNode(), + null, + Collections.emptySet(), + NoopTracer.INSTANCE + ); + transportService.start(); + transportService.acceptIncomingRequests(); + ActionFilters actionFilters = new ActionFilters(new HashSet<>()); + NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(List.of()); + + NodeClient client = new NodeClient(Settings.EMPTY, threadPool) { + + @Override + public void multiSearch(MultiSearchRequest request, ActionListener listener) { + SearchHit hit = new SearchHit( + 1, + "1", + Map.of( + "@timestamp", + new DocumentField("@timestamp", List.of(200000L)), + "level", + new DocumentField("level", List.of(1L)) + ), + null + ); + SearchResponse response = new SearchResponse( + new SearchResponseSections( + new SearchHits(new SearchHit[] { hit }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), + null, + null, + false, + false, + null, + 0 + ), + null, + 1, + 1, + 0, + 20L, + null, + null + ); + MultiSearchResponse mSearchResponse = new MultiSearchResponse( + new MultiSearchResponse.Item[] { new MultiSearchResponse.Item(response, null) }, + 1L + ); + listener.onResponse(mSearchResponse); + } + + @Override + public void search(SearchRequest request, ActionListener listener) { + SearchHit hit = new SearchHit(1).sourceRef( + new BytesArray( + "{\n" + + " \"name\": \"s3 to app logs\",\n" + + " \"correlate\": [\n" + + " {\n" + + " \"index\": \"test-index\",\n" + + " \"query\": \"aws.cloudtrail.eventName:ReplicateObject\",\n" + + " \"timestampField\": \"@timestamp\",\n" + + " \"tags\": [\n" + + " \"s3\"\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"index\": \"app_logs\",\n" + + " \"query\": \"keywords:PermissionDenied\",\n" + + " \"timestampField\": \"@timestamp\",\n" + + " \"tags\": [\n" + + " \"others_application\"\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}" + ) + ); + listener.onResponse( + new SearchResponse( + new SearchResponseSections( + new SearchHits(new SearchHit[] { hit }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), + null, + null, + false, + false, + null, + 0 + ), + null, + 1, + 1, + 0, + 20L, + null, + null + ) + ); + } + }; + return new TransportIndexCorrelationAction( + transportService, + client, + xContentRegistry, + Settings.EMPTY, + actionFilters, + clusterService + ); + } + + public void testDoExecute() { + TransportIndexCorrelationAction action = createAction(); + + IndexCorrelationRequest request = new IndexCorrelationRequest("test-index", "test-event", false); + action.doExecute(null, request, new ActionListener<>() { + @Override + public void onResponse(IndexCorrelationResponse response) { + Assert.assertEquals(RestStatus.OK, response.getStatus()); + } + + @Override + public void onFailure(Exception e) { + // ignore + } + }); + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/transport/TransportSearchCorrelatedEventsActionTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/transport/TransportSearchCorrelatedEventsActionTests.java new file mode 100644 index 0000000000000..ba5c183a8699d --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/transport/TransportSearchCorrelatedEventsActionTests.java @@ -0,0 +1,176 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.transport; + +import org.apache.lucene.search.TotalHits; +import org.opensearch.Version; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.SearchResponseSections; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.document.DocumentField; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.plugin.correlation.events.action.SearchCorrelatedEventsRequest; +import org.opensearch.plugin.correlation.events.action.SearchCorrelatedEventsResponse; +import org.opensearch.plugin.correlation.events.model.EventWithScore; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.telemetry.tracing.noop.NoopTracer; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.VersionUtils; +import org.opensearch.test.transport.CapturingTransport; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static java.util.Collections.emptyMap; +import static org.opensearch.test.ClusterServiceUtils.createClusterService; + +public class TransportSearchCorrelatedEventsActionTests extends OpenSearchTestCase { + + private static ThreadPool threadPool; + private ClusterService clusterService; + + @BeforeClass + public static void beforeClass() { + threadPool = new TestThreadPool("TransportSearchCorrelatedEventsActionTests"); + } + + @AfterClass + public static void afterClass() { + ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); + threadPool = null; + } + + @Before + public void setUp() throws Exception { + super.setUp(); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + VersionUtils.randomCompatibleVersion(random(), Version.CURRENT) + ); + clusterService = createClusterService(threadPool, discoveryNode); + } + + public void tearDown() throws Exception { + super.tearDown(); + clusterService.close(); + } + + private TransportSearchCorrelatedEventsAction createAction() { + CapturingTransport capturingTransport = new CapturingTransport(); + TransportService transportService = capturingTransport.createTransportService( + clusterService.getSettings(), + threadPool, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, + boundAddress -> clusterService.localNode(), + null, + Collections.emptySet(), + NoopTracer.INSTANCE + ); + transportService.start(); + transportService.acceptIncomingRequests(); + ActionFilters actionFilters = new ActionFilters(new HashSet<>()); + + NodeClient client = new NodeClient(Settings.EMPTY, threadPool) { + @Override + public void search(SearchRequest request, ActionListener listener) { + SearchHit hit = new SearchHit( + 1, + "1", + Map.of( + "@timestamp", + new DocumentField("@timestamp", List.of(200000L)), + "level", + new DocumentField("level", List.of(1L)) + ), + null + ).sourceRef( + new BytesArray( + "{\n" + + " \"score_timestamp\": 200000,\n" + + " \"event1\": \"correlated-event\",\n" + + " \"index1\": \"correlated-index\",\n" + + " \"event2\": \"correlated-event\",\n" + + " \"index2\": \"correlated-index\"\n" + + "}" + ) + ); + hit.score(1.0f); + listener.onResponse( + new SearchResponse( + new SearchResponseSections( + new SearchHits(new SearchHit[] { hit }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), + null, + null, + false, + false, + null, + 0 + ), + null, + 1, + 1, + 0, + 20L, + null, + null + ) + ); + } + }; + + return new TransportSearchCorrelatedEventsAction(transportService, client, actionFilters); + } + + public void testDoExecute() { + TransportSearchCorrelatedEventsAction action = createAction(); + + SearchCorrelatedEventsRequest request = new SearchCorrelatedEventsRequest("test-index", "test-event", "@timestamp", 300000L, 5); + action.doExecute(null, request, new ActionListener<>() { + @Override + public void onResponse(SearchCorrelatedEventsResponse response) { + List events = response.getEvents(); + for (EventWithScore event : events) { + Assert.assertEquals("correlated-index", event.getIndex()); + Assert.assertEquals("correlated-event", event.getEvent()); + Assert.assertEquals(1.0, event.getScore(), 0.0); + Assert.assertTrue(event.getTags().isEmpty()); + } + + Assert.assertEquals(RestStatus.OK, response.getStatus()); + } + + @Override + public void onFailure(Exception e) { + // ignore + } + }); + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/transport/TransportStoreCorrelationActionTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/transport/TransportStoreCorrelationActionTests.java new file mode 100644 index 0000000000000..1b1840498c944 --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/events/transport/TransportStoreCorrelationActionTests.java @@ -0,0 +1,456 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.events.transport; + +import org.apache.lucene.search.TotalHits; +import org.opensearch.Version; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.bulk.BulkItemResponse; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.search.MultiSearchRequest; +import org.opensearch.action.search.MultiSearchResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.SearchResponseSections; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.replication.ReplicationResponse; +import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.document.DocumentField; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.plugin.correlation.events.action.StoreCorrelationRequest; +import org.opensearch.plugin.correlation.events.action.StoreCorrelationResponse; +import org.opensearch.plugin.correlation.events.model.Correlation; +import org.opensearch.plugin.correlation.settings.EventsCorrelationSettings; +import org.opensearch.plugin.correlation.utils.CorrelationIndices; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.telemetry.tracing.noop.NoopTracer; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.VersionUtils; +import org.opensearch.test.transport.CapturingTransport; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import static java.util.Collections.emptyMap; +import static org.opensearch.test.ClusterServiceUtils.createClusterService; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransportStoreCorrelationActionTests extends OpenSearchTestCase { + + private static ThreadPool threadPool; + private Settings settings; + private ClusterService clusterService; + private CorrelationIndices correlationIndices; + + @BeforeClass + public static void beforeClass() { + threadPool = new TestThreadPool("TransportStoreCorrelationActionTests"); + } + + @AfterClass + public static void afterClass() { + ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); + threadPool = null; + } + + @SuppressWarnings("unchecked") + @Before + public void setUp() throws Exception { + super.setUp(); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + VersionUtils.randomCompatibleVersion(random(), Version.CURRENT) + ); + + settings = Settings.builder() + .put(EventsCorrelationSettings.CORRELATION_TIME_WINDOW.getKey(), new TimeValue(5, TimeUnit.MINUTES)) + .build(); + Set> settingSet = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + settingSet.add(EventsCorrelationSettings.CORRELATION_TIME_WINDOW); + ClusterSettings clusterSettings = new ClusterSettings(settings, settingSet); + clusterService = createClusterService(threadPool, discoveryNode, clusterSettings); + } + + public void tearDown() throws Exception { + super.tearDown(); + clusterService.close(); + } + + @SuppressWarnings("unchecked") + private void setupCorrelationIndices(boolean flag) throws IOException { + correlationIndices = mock(CorrelationIndices.class); + when(correlationIndices.correlationIndexExists()).thenReturn(flag); + doAnswer((Answer) invocation -> { + ((ActionListener) invocation.getArgument(0)).onResponse( + new CreateIndexResponse(true, true, Correlation.CORRELATION_HISTORY_INDEX) + ); + return null; + }).when(correlationIndices).initCorrelationIndex(any()); + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + ((ActionListener) invocation.getArgument(1)).onResponse(new BulkResponse(new BulkItemResponse[] {}, 3, 4)); + return null; + } + }).when(correlationIndices).setupCorrelationIndex(anyLong(), any()); + } + + private TransportStoreCorrelationAction createActionForOrphanEvents(boolean indexListenerStatus) { + CapturingTransport capturingTransport = new CapturingTransport(); + TransportService transportService = capturingTransport.createTransportService( + clusterService.getSettings(), + threadPool, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, + boundAddress -> clusterService.localNode(), + null, + Collections.emptySet(), + NoopTracer.INSTANCE + ); + transportService.start(); + transportService.acceptIncomingRequests(); + ActionFilters actionFilters = new ActionFilters(new HashSet<>()); + + NodeClient client = new NodeClient(Settings.EMPTY, threadPool) { + @Override + public void search(SearchRequest request, ActionListener listener) { + SearchHit hit = new SearchHit( + 1, + "1", + Map.of( + "@timestamp", + new DocumentField("@timestamp", List.of(200000L)), + "level", + new DocumentField("level", List.of(1L)) + ), + null + ).sourceRef( + new BytesArray( + "{\n" + + " \"score_timestamp\": 200000,\n" + + " \"event1\": \"correlated-event\",\n" + + " \"index1\": \"correlated-index\",\n" + + " \"event2\": \"correlated-event\",\n" + + " \"index2\": \"correlated-index\",\n" + + " \"level\": \"1\",\n" + + " \"timestamp\": \"10000\"\n" + + "}" + ) + ); + hit.score(1.0f); + listener.onResponse( + new SearchResponse( + new SearchResponseSections( + new SearchHits(new SearchHit[] { hit }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), + null, + null, + false, + false, + null, + 0 + ), + null, + 1, + 1, + 0, + 20L, + null, + null + ) + ); + } + + @Override + public void multiSearch(MultiSearchRequest request, ActionListener listener) { + listener.onResponse(new MultiSearchResponse(new MultiSearchResponse.Item[] {}, 10000L)); + } + + @Override + public void index(IndexRequest request, ActionListener listener) { + IndexResponse response = new IndexResponse( + new ShardId(new Index("test-index", ""), 0), + "0", + 0L, + 0L, + 0L, + indexListenerStatus + ); + response.setShardInfo(new ReplicationResponse.ShardInfo()); + listener.onResponse(response); + } + }; + + return new TransportStoreCorrelationAction(transportService, client, settings, actionFilters, correlationIndices, clusterService); + } + + private TransportStoreCorrelationAction createActionForCorrelatedEvents(boolean indexListenerStatus) { + CapturingTransport capturingTransport = new CapturingTransport(); + TransportService transportService = capturingTransport.createTransportService( + clusterService.getSettings(), + threadPool, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, + boundAddress -> clusterService.localNode(), + null, + Collections.emptySet(), + NoopTracer.INSTANCE + ); + transportService.start(); + transportService.acceptIncomingRequests(); + ActionFilters actionFilters = new ActionFilters(new HashSet<>()); + + NodeClient client = new NodeClient(Settings.EMPTY, threadPool) { + @Override + public void search(SearchRequest request, ActionListener listener) { + SearchHit hit = new SearchHit( + 1, + "1", + Map.of( + "@timestamp", + new DocumentField("@timestamp", List.of(200000L)), + "level", + new DocumentField("level", List.of(1L)) + ), + null + ).sourceRef( + new BytesArray( + "{\n" + + " \"score_timestamp\": 200000,\n" + + " \"event1\": \"correlated-event\",\n" + + " \"index1\": \"correlated-index\",\n" + + " \"event2\": \"correlated-event\",\n" + + " \"index2\": \"correlated-index\",\n" + + " \"level\": \"1\",\n" + + " \"timestamp\": \"10000\"\n" + + "}" + ) + ); + hit.score(1.0f); + listener.onResponse( + new SearchResponse( + new SearchResponseSections( + new SearchHits(new SearchHit[] { hit }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), + null, + null, + false, + false, + null, + 0 + ), + null, + 1, + 1, + 0, + 20L, + null, + null + ) + ); + } + + @Override + public void multiSearch(MultiSearchRequest request, ActionListener listener) { + SearchHit hit = new SearchHit( + 1, + "1", + Map.of( + "@timestamp", + new DocumentField("@timestamp", List.of(200000L)), + "level", + new DocumentField("level", List.of(1L)) + ), + null + ).sourceRef( + new BytesArray( + "{\n" + + " \"score_timestamp\": 200000,\n" + + " \"event1\": \"correlated-event\",\n" + + " \"index1\": \"correlated-index\",\n" + + " \"event2\": \"correlated-event\",\n" + + " \"index2\": \"correlated-index\",\n" + + " \"level\": \"1\",\n" + + " \"timestamp\": \"10000\"\n" + + "}" + ) + ); + MultiSearchResponse.Item[] items = new MultiSearchResponse.Item[] { + new MultiSearchResponse.Item( + new SearchResponse( + new SearchResponseSections( + new SearchHits(new SearchHit[] { hit }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), + null, + null, + false, + false, + null, + 0 + ), + null, + 1, + 1, + 0, + 20L, + null, + null + ), + null + ) }; + listener.onResponse(new MultiSearchResponse(items, 10000L)); + } + + @Override + public void index(IndexRequest request, ActionListener listener) { + IndexResponse response = new IndexResponse( + new ShardId(new Index("test-index", ""), 0), + "0", + 0L, + 0L, + 0L, + indexListenerStatus + ); + response.setShardInfo(new ReplicationResponse.ShardInfo()); + listener.onResponse(response); + } + + @Override + public void bulk(BulkRequest request, ActionListener listener) { + listener.onResponse(new BulkResponse(new BulkItemResponse[] {}, 60000)); + } + }; + + return new TransportStoreCorrelationAction(transportService, client, settings, actionFilters, correlationIndices, clusterService); + } + + public void testDoExecuteOrphanEvent() throws IOException { + setupCorrelationIndices(true); + TransportStoreCorrelationAction action = createActionForOrphanEvents(true); + StoreCorrelationRequest request = new StoreCorrelationRequest( + "test-index", + "test-event", + 300000L, + Map.of("network", List.of("correlated-event1", "correlated-event2")), + List.of() + ); + action.doExecute(null, request, new ActionListener<>() { + @Override + public void onResponse(StoreCorrelationResponse storeCorrelationResponse) { + Assert.assertEquals(RestStatus.OK, storeCorrelationResponse.getStatus()); + } + + @Override + public void onFailure(Exception e) { + // ignore + } + }); + } + + public void testDoExecuteOrphanEventWithSettingUpCorrelationIndex() throws IOException { + setupCorrelationIndices(false); + TransportStoreCorrelationAction action = createActionForOrphanEvents(true); + StoreCorrelationRequest request = new StoreCorrelationRequest( + "test-index", + "test-event", + 300000L, + Map.of("network", List.of("correlated-event1", "correlated-event2")), + List.of() + ); + action.doExecute(null, request, new ActionListener<>() { + @Override + public void onResponse(StoreCorrelationResponse storeCorrelationResponse) { + Assert.assertEquals(RestStatus.OK, storeCorrelationResponse.getStatus()); + } + + @Override + public void onFailure(Exception e) { + // ignore + } + }); + } + + public void testDoExecuteOrphanEventWithScoreTimestampReset() throws IOException { + setupCorrelationIndices(true); + TransportStoreCorrelationAction action = createActionForOrphanEvents(false); + StoreCorrelationRequest request = new StoreCorrelationRequest( + "test-index", + "test-event", + 1728300000L, + Map.of("network", List.of("correlated-event1", "correlated-event2")), + List.of() + ); + action.doExecute(null, request, new ActionListener<>() { + @Override + public void onResponse(StoreCorrelationResponse storeCorrelationResponse) { + Assert.assertEquals(RestStatus.OK, storeCorrelationResponse.getStatus()); + } + + @Override + public void onFailure(Exception e) { + // ignore + } + }); + } + + public void testExecuteCorrelatedEvents() throws IOException { + setupCorrelationIndices(true); + TransportStoreCorrelationAction action = createActionForCorrelatedEvents(false); + StoreCorrelationRequest request = new StoreCorrelationRequest( + "test-index", + "test-event", + 1728300000L, + Map.of("network", List.of("correlated-event1", "correlated-event2")), + List.of() + ); + action.doExecute(null, request, new ActionListener<>() { + @Override + public void onResponse(StoreCorrelationResponse storeCorrelationResponse) { + Assert.assertEquals(RestStatus.OK, storeCorrelationResponse.getStatus()); + } + + @Override + public void onFailure(Exception e) { + // ignore + } + }); + } +} diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/utils/TestHelpers.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/utils/TestHelpers.java new file mode 100644 index 0000000000000..a43f22ef9797f --- /dev/null +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/utils/TestHelpers.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.correlation.utils; + +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.plugin.correlation.events.model.Correlation; +import org.opensearch.plugin.correlation.events.model.EventWithScore; + +import java.io.IOException; +import java.util.List; + +public class TestHelpers { + + public static String toJsonString(EventWithScore eventWithScore) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = eventWithScore.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + + public static String toJsonString(Correlation correlation) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = correlation.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + + public static XContentParser parse(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + } + + public static NamedXContentRegistry xContentRegistry() { + return new NamedXContentRegistry(List.of()); + } +}