From 727fda6c664eb2070eb232bd6ead08d63626022f Mon Sep 17 00:00:00 2001 From: qchea Date: Thu, 1 Apr 2021 17:47:15 -0500 Subject: [PATCH 01/18] REF: integrationTest.gradle --- data-prepper-core/integrationTest.gradle | 98 +++++++++++------------- 1 file changed, 46 insertions(+), 52 deletions(-) diff --git a/data-prepper-core/integrationTest.gradle b/data-prepper-core/integrationTest.gradle index bad36c7e1..d319ef8ae 100644 --- a/data-prepper-core/integrationTest.gradle +++ b/data-prepper-core/integrationTest.gradle @@ -71,54 +71,39 @@ task buildDataPrepperDockerImage(type: DockerBuildImage) { images.add("integ-test-pipeline-image") } -task createDataPrepper1DockerContainer(type: DockerCreateContainer) { - dependsOn buildDataPrepperDockerImage - dependsOn createDataPrepperNetwork - containerName = "data-prepper1" - hostConfig.portBindings = ['21890:21890', '4900:4900'] - hostConfig.network = createDataPrepperNetwork.getNetworkName() - targetImageId buildDataPrepperDockerImage.getImageId() -} - -task createDataPrepper2DockerContainer(type: DockerCreateContainer) { - dependsOn buildDataPrepperDockerImage - dependsOn createDataPrepperNetwork - containerName = "data-prepper2" - hostConfig.portBindings = ['21891:21890', '4901:4900'] - hostConfig.network = createDataPrepperNetwork.getNetworkName() - targetImageId buildDataPrepperDockerImage.getImageId() -} - -task startDataPrepper1DockerContainer(type: DockerStartContainer) { - dependsOn createDataPrepper1DockerContainer - targetContainerId createDataPrepper1DockerContainer.getContainerId() - doLast { - sleep(10*1000) +def createDataPrepperDockerContainer(final String taskBaseName, final String dataPrepperName, final int grpcPort, + final int serverPort, final String pipelineConfigYAML) { + return tasks.create("create${taskBaseName}", DockerCreateContainer) { + dependsOn buildDataPrepperDockerImage + dependsOn createDataPrepperNetwork + containerName = dataPrepperName + hostConfig.portBindings = ['%d:21890'.formatted(grpcPort), '%d:4900'.formatted(serverPort)] + hostConfig.network = createDataPrepperNetwork.getNetworkName() + cmd = ["java", "-jar", "data-prepper.jar", pipelineConfigYAML, "/app/data_prepper.yml"] + targetImageId buildDataPrepperDockerImage.getImageId() } } -task startDataPrepper2DockerContainer(type: DockerStartContainer) { - dependsOn createDataPrepper2DockerContainer - targetContainerId createDataPrepper2DockerContainer.getContainerId() - doLast { - sleep(10*1000) +def startDataPrepperDockerContainer(final DockerCreateContainer createDataPrepperDockerContainerTask) { + return tasks.create("start${createDataPrepperDockerContainerTask.getName()}", DockerStartContainer) { + dependsOn createDataPrepperDockerContainerTask + targetContainerId createDataPrepperDockerContainerTask.getContainerId() + doLast { + sleep(10*1000) + } } } -task stopDataPrepper1DockerContainer(type: DockerStopContainer) { - targetContainerId createDataPrepper1DockerContainer.getContainerId() -} - -task stopDataPrepper2DockerContainer(type: DockerStopContainer) { - targetContainerId createDataPrepper2DockerContainer.getContainerId() -} - -task removeDataPrepper1DockerContainer(type: DockerRemoveContainer) { - targetContainerId stopDataPrepper1DockerContainer.getContainerId() +def stopDataPrepperDockerContainer(final DockerStartContainer startDataPrepperDockerContainerTask) { + return tasks.create("stop${startDataPrepperDockerContainerTask.getName()}", DockerStopContainer) { + targetContainerId startDataPrepperDockerContainerTask.getContainerId() + } } -task removeDataPrepper2DockerContainer(type: DockerRemoveContainer) { - targetContainerId stopDataPrepper2DockerContainer.getContainerId() +def removeDataPrepperDockerContainer(final DockerStopContainer stopDataPrepperDockerContainerTask) { + return tasks.create("remove${stopDataPrepperDockerContainerTask.getName()}", DockerRemoveContainer) { + targetContainerId stopDataPrepperDockerContainerTask.getContainerId() + } } /** @@ -159,8 +144,10 @@ task stopOdfeDockerContainer(type: DockerStopContainer) { task rawSpanEndToEndTest(type: Test) { dependsOn build dependsOn startOdfeDockerContainer - dependsOn startDataPrepper1DockerContainer - startDataPrepper1DockerContainer.mustRunAfter 'startOdfeDockerContainer' + def createDataPrepperTask = createDataPrepperDockerContainer("rawSpanDataPrepper1", "data-prepper1", 21890, 4900, "/app/pipeline.yml") + def startDataPrepperTask = startDataPrepperDockerContainer(createDataPrepperTask as DockerCreateContainer) + dependsOn startDataPrepperTask + startDataPrepperTask.mustRunAfter 'startOdfeDockerContainer' description = 'Runs the raw span integration tests.' group = 'verification' @@ -172,18 +159,23 @@ task rawSpanEndToEndTest(type: Test) { } finalizedBy stopOdfeDockerContainer - finalizedBy stopDataPrepper1DockerContainer - finalizedBy removeDataPrepper1DockerContainer + def stopDataPrepperTask = stopDataPrepperDockerContainer(startDataPrepperTask as DockerStartContainer) + finalizedBy stopDataPrepperTask + finalizedBy removeDataPrepperDockerContainer(stopDataPrepperTask as DockerStopContainer) finalizedBy removeDataPrepperNetwork } task serviceMapEndToEndTest(type: Test) { dependsOn build dependsOn startOdfeDockerContainer - dependsOn startDataPrepper1DockerContainer - dependsOn startDataPrepper2DockerContainer - startDataPrepper1DockerContainer.mustRunAfter 'startOdfeDockerContainer' - startDataPrepper2DockerContainer.mustRunAfter 'startOdfeDockerContainer' + def createDataPrepper1Task = createDataPrepperDockerContainer("serviceMapDataPrepper1", "data-prepper1", 21890, 4900, "/app/pipeline.yml") + def createDataPrepper2Task = createDataPrepperDockerContainer("serviceMapDataPrepper2", "data-prepper2", 21891, 4901, "/app/pipeline.yml") + def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) + def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) + dependsOn startDataPrepper1Task + dependsOn startDataPrepper2Task + startDataPrepper1Task.mustRunAfter 'startOdfeDockerContainer' + startDataPrepper2Task.mustRunAfter 'startOdfeDockerContainer' description = 'Runs the raw span integration tests.' group = 'verification' @@ -195,9 +187,11 @@ task serviceMapEndToEndTest(type: Test) { } finalizedBy stopOdfeDockerContainer - finalizedBy stopDataPrepper1DockerContainer - finalizedBy stopDataPrepper2DockerContainer - finalizedBy removeDataPrepper1DockerContainer - finalizedBy removeDataPrepper2DockerContainer + def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer) + def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer) + finalizedBy stopDataPrepper1Task + finalizedBy stopDataPrepper2Task + finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer) + finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer) finalizedBy removeDataPrepperNetwork } From 2f85c424e384e09acb05744420593a01d2aeb7bb Mon Sep 17 00:00:00 2001 From: qchea Date: Fri, 2 Apr 2021 10:01:23 -0500 Subject: [PATCH 02/18] ENH: separate pipeline configs --- data-prepper-core/integrationTest.gradle | 17 +++++--- .../resources/raw-span-e2e-pipeline.yml | 40 +++++++++++++++++++ ...eline.yml => service-map-e2e-pipeline.yml} | 0 3 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml rename data-prepper-core/src/integrationTest/resources/{pipeline.yml => service-map-e2e-pipeline.yml} (100%) diff --git a/data-prepper-core/integrationTest.gradle b/data-prepper-core/integrationTest.gradle index d319ef8ae..47b806b96 100644 --- a/data-prepper-core/integrationTest.gradle +++ b/data-prepper-core/integrationTest.gradle @@ -48,6 +48,9 @@ task removeDataPrepperNetwork(type: DockerRemoveNetwork) { networkId = createDataPrepperNetwork.getNetworkId() } +def RAW_SPAN_PIPELINE_YAML = "raw-span-e2e-pipeline.yml" +def SERVICE_MAP_PIPELINE_YAML = "service-map-e2e-pipeline.yml" + /** * DataPrepper Docker tasks */ @@ -59,9 +62,10 @@ task createDataPrepperDockerFile(type: Dockerfile) { exposePort(4900) workingDir("/app") copyFile("build/libs/${jar.archiveName}", "/app/data-prepper.jar") - copyFile("src/integrationTest/resources/pipeline.yml", "/app/pipeline.yml") + copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_YAML}", "/app/${RAW_SPAN_PIPELINE_YAML}") + copyFile("src/integrationTest/resources/${SERVICE_MAP_PIPELINE_YAML}", "/app/${SERVICE_MAP_PIPELINE_YAML}") copyFile("src/integrationTest/resources/data_prepper.yml", "/app/data_prepper.yml") - defaultCommand("java", "-jar", "data-prepper.jar", "/app/pipeline.yml", "/app/data_prepper.yml") + defaultCommand("java", "-jar", "data-prepper.jar", "/app/${RAW_SPAN_PIPELINE_YAML}", "/app/data_prepper.yml") } task buildDataPrepperDockerImage(type: DockerBuildImage) { @@ -144,7 +148,8 @@ task stopOdfeDockerContainer(type: DockerStopContainer) { task rawSpanEndToEndTest(type: Test) { dependsOn build dependsOn startOdfeDockerContainer - def createDataPrepperTask = createDataPrepperDockerContainer("rawSpanDataPrepper1", "data-prepper1", 21890, 4900, "/app/pipeline.yml") + def createDataPrepperTask = createDataPrepperDockerContainer( + "rawSpanDataPrepper1", "data-prepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}") def startDataPrepperTask = startDataPrepperDockerContainer(createDataPrepperTask as DockerCreateContainer) dependsOn startDataPrepperTask startDataPrepperTask.mustRunAfter 'startOdfeDockerContainer' @@ -168,8 +173,10 @@ task rawSpanEndToEndTest(type: Test) { task serviceMapEndToEndTest(type: Test) { dependsOn build dependsOn startOdfeDockerContainer - def createDataPrepper1Task = createDataPrepperDockerContainer("serviceMapDataPrepper1", "data-prepper1", 21890, 4900, "/app/pipeline.yml") - def createDataPrepper2Task = createDataPrepperDockerContainer("serviceMapDataPrepper2", "data-prepper2", 21891, 4901, "/app/pipeline.yml") + def createDataPrepper1Task = createDataPrepperDockerContainer( + "serviceMapDataPrepper1", "data-prepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}") + def createDataPrepper2Task = createDataPrepperDockerContainer( + "serviceMapDataPrepper2", "data-prepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_YAML}") def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) dependsOn startDataPrepper1Task diff --git a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml new file mode 100644 index 000000000..9ee53599f --- /dev/null +++ b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml @@ -0,0 +1,40 @@ +entry-pipeline: + source: + otel_trace_source: + ssl: false + sink: + - pipeline: + name: "raw-pipeline" + - pipeline: + name: "service-map-pipeline" +raw-pipeline: + source: + pipeline: + name: "entry-pipeline" + prepper: + - otel_trace_raw_prepper: + root_span_flush_delay: 1 + trace_flush_interval: 3 + - otel_trace_group_prepper: + hosts: [ "https://node-0.example.com:9200" ] + username: "admin" + password: "admin" + sink: + - elasticsearch: + hosts: [ "https://node-0.example.com:9200" ] + username: "admin" + password: "admin" + trace_analytics_raw: true +service-map-pipeline: + source: + pipeline: + name: "entry-pipeline" + prepper: + - service_map_stateful: + window_duration: 3 + sink: + - elasticsearch: + hosts: ["https://node-0.example.com:9200"] + username: "admin" + password: "admin" + trace_analytics_service_map: true \ No newline at end of file diff --git a/data-prepper-core/src/integrationTest/resources/pipeline.yml b/data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml similarity index 100% rename from data-prepper-core/src/integrationTest/resources/pipeline.yml rename to data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml From fc5ba2b207a26ecbec7af011fcc27e865beb608b Mon Sep 17 00:00:00 2001 From: qchea Date: Fri, 2 Apr 2021 13:40:21 -0500 Subject: [PATCH 03/18] MAINT: replace random test data with designed data --- .../integration/EndToEndRawSpanTest.java | 104 +++++++++--------- 1 file changed, 55 insertions(+), 49 deletions(-) diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java index 070e6e163..c7351031f 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java @@ -25,7 +25,6 @@ import org.junit.Test; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -33,34 +32,52 @@ import java.util.List; import java.util.Map; import java.util.Random; -import java.util.UUID; import java.util.concurrent.TimeUnit; import static org.awaitility.Awaitility.await; public class EndToEndRawSpanTest { - - private static final Random RANDOM = new Random(); - private static final List SPAN_KINDS = - Arrays.asList(Span.SpanKind.SPAN_KIND_CLIENT, Span.SpanKind.SPAN_KIND_CONSUMER, Span.SpanKind.SPAN_KIND_INTERNAL, Span.SpanKind.SPAN_KIND_PRODUCER, Span.SpanKind.SPAN_KIND_SERVER); + private static final int DATA_PREPPER_PORT_1 = 21890; + private static final int DATA_PREPPER_PORT_2 = 21891; + + private static final String TEST_TRACEID_1 = "ABC"; + private static final String TEST_TRACEID_2 = "CBA"; + private static final List testDataSet11 = Arrays.asList( + ServiceMapTestData.DATA_100, ServiceMapTestData.DATA_200, ServiceMapTestData.DATA_500, ServiceMapTestData.DATA_600, + ServiceMapTestData.DATA_700, ServiceMapTestData.DATA_1000); + private static final List testDataSet12 = Arrays.asList( + ServiceMapTestData.DATA_300, ServiceMapTestData.DATA_400, ServiceMapTestData.DATA_800, + ServiceMapTestData.DATA_900, ServiceMapTestData.DATA_1100); + private static final List testDataSet21 = Arrays.asList( + ServiceMapTestData.DATA_101, ServiceMapTestData.DATA_201, ServiceMapTestData.DATA_401, ServiceMapTestData.DATA_501); + private static final List testDataSet22 = Collections.singletonList(ServiceMapTestData.DATA_301); private static final String INDEX_NAME = "otel-v1-apm-span-000001"; @Test public void testPipelineEndToEnd() throws IOException, InterruptedException { //Send data to otel trace source - final ExportTraceServiceRequest exportTraceServiceRequest1 = getExportTraceServiceRequest( - getRandomResourceSpans(10) + final ExportTraceServiceRequest exportTraceServiceRequest11 = getExportTraceServiceRequest( + getResourceSpansBatch(TEST_TRACEID_1, testDataSet11) ); - - final ExportTraceServiceRequest exportTraceServiceRequest2 = getExportTraceServiceRequest( - getRandomResourceSpans(10) + final ExportTraceServiceRequest exportTraceServiceRequest12 = getExportTraceServiceRequest( + getResourceSpansBatch(TEST_TRACEID_1, testDataSet12) + ); + final ExportTraceServiceRequest exportTraceServiceRequest21 = getExportTraceServiceRequest( + getResourceSpansBatch(TEST_TRACEID_2, testDataSet21) + ); + final ExportTraceServiceRequest exportTraceServiceRequest22 = getExportTraceServiceRequest( + getResourceSpansBatch(TEST_TRACEID_2, testDataSet22) ); - sendExportTraceServiceRequestToSource(exportTraceServiceRequest1); - sendExportTraceServiceRequestToSource(exportTraceServiceRequest2); + sendExportTraceServiceRequestToSource(exportTraceServiceRequest11); + sendExportTraceServiceRequestToSource(exportTraceServiceRequest12); + sendExportTraceServiceRequestToSource(exportTraceServiceRequest21); + sendExportTraceServiceRequestToSource(exportTraceServiceRequest22); //Verify data in elasticsearch sink - final List> expectedDocuments = getExpectedDocuments(exportTraceServiceRequest1, exportTraceServiceRequest2); + final List> expectedDocuments = getExpectedDocuments( + exportTraceServiceRequest11, exportTraceServiceRequest12, + exportTraceServiceRequest21, exportTraceServiceRequest22); final ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder( Collections.singletonList("https://127.0.0.1:9200")); builder.withUsername("admin"); @@ -88,7 +105,7 @@ public void testPipelineEndToEnd() throws IOException, InterruptedException { */ expectedDocuments.forEach(expectedDoc -> { Assert.assertTrue(foundSources.stream() - .filter(i -> i.get("traceId").equals(expectedDoc.get("traceId"))) + .filter(i -> i.get("spanId").equals(expectedDoc.get("spanId"))) .findFirst().get() .entrySet().containsAll(expectedDoc.entrySet())); }); @@ -113,15 +130,9 @@ private List> getSourcesFromSearchHits(final SearchHits sear return sources; } - public static ResourceSpans getResourceSpans(final String serviceName, final String spanName, final byte[] - spanId, final byte[] parentId, final byte[] traceId, final Span.SpanKind spanKind, final int statusCode, final String statusMessage) { + spanId, final byte[] parentId, final byte[] traceId, final Span.SpanKind spanKind) { final ByteString parentSpanId = parentId != null ? ByteString.copyFrom(parentId) : ByteString.EMPTY; - final Status.Builder statusBuilder = Status.newBuilder().setCodeValue(statusCode); - if (statusMessage != null) { - statusBuilder.setMessage(statusMessage); - } - final Status status = statusBuilder.build(); return ResourceSpans.newBuilder() .setResource( Resource.newBuilder() @@ -139,7 +150,6 @@ public static ResourceSpans getResourceSpans(final String serviceName, final Str .setKind(spanKind) .setSpanId(ByteString.copyFrom(spanId)) .setParentSpanId(parentSpanId) - .setStatus(status) .setTraceId(ByteString.copyFrom(traceId)) .build() ) @@ -154,6 +164,28 @@ public static ExportTraceServiceRequest getExportTraceServiceRequest(List getResourceSpansBatch(final String traceId, final List data) { + final ArrayList spansList = new ArrayList<>(); + for(int i=0; i < data.size(); i++) { + final ServiceMapTestData currData = data.get(i); + final String parentId = currData.parentId; + final String spanId = currData.spanId; + final String serviceName = currData.serviceName; + final String spanName = currData.name; + final Span.SpanKind spanKind = currData.spanKind; + final ResourceSpans rs = getResourceSpans( + serviceName, + spanName, + spanId.getBytes(), + parentId != null ? parentId.getBytes() : null, + traceId.getBytes(), + spanKind + ); + spansList.add(rs); + } + return spansList; + } + private List> getExpectedDocuments(ExportTraceServiceRequest...exportTraceServiceRequests) { final List> expectedDocuments = new ArrayList<>(); for(int i=0; i getExpectedEsDocumentSource(final Span span, final S esDocSource.put("name", span.getName()); esDocSource.put("kind", span.getKind().name()); esDocSource.put("status.code", span.getStatus().getCodeValue()); - esDocSource.put("status.message", span.getStatus().getMessage()); esDocSource.put("serviceName", serviceName); if (span.getParentSpanId().isEmpty()) { esDocSource.put("traceGroup", span.getName()); @@ -185,31 +216,6 @@ private Map getExpectedEsDocumentSource(final Span span, final S return esDocSource; } - private byte[] getRandomBytes(int len) { - final byte[] bytes = new byte[len]; - RANDOM.nextBytes(bytes); - return bytes; - } - - private List getRandomResourceSpans(int len) throws UnsupportedEncodingException { - final ArrayList spansList = new ArrayList<>(); - for(int i=0; i kv.getKey().equals("service.name")) .findFirst().get().getValue().getStringValue(); From 41042448e50290c6fedcbf298bcb29184f3529c6 Mon Sep 17 00:00:00 2001 From: qchea Date: Fri, 2 Apr 2021 14:03:15 -0500 Subject: [PATCH 04/18] E2E: set up a second instance --- data-prepper-core/integrationTest.gradle | 24 ++++++++++++------- .../integration/EndToEndRawSpanTest.java | 13 ++++++---- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/data-prepper-core/integrationTest.gradle b/data-prepper-core/integrationTest.gradle index 47b806b96..1547a4499 100644 --- a/data-prepper-core/integrationTest.gradle +++ b/data-prepper-core/integrationTest.gradle @@ -148,11 +148,16 @@ task stopOdfeDockerContainer(type: DockerStopContainer) { task rawSpanEndToEndTest(type: Test) { dependsOn build dependsOn startOdfeDockerContainer - def createDataPrepperTask = createDataPrepperDockerContainer( + def createDataPrepper1Task = createDataPrepperDockerContainer( "rawSpanDataPrepper1", "data-prepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}") - def startDataPrepperTask = startDataPrepperDockerContainer(createDataPrepperTask as DockerCreateContainer) - dependsOn startDataPrepperTask - startDataPrepperTask.mustRunAfter 'startOdfeDockerContainer' + def createDataPrepper2Task = createDataPrepperDockerContainer( + "rawSpanDataPrepper2", "data-prepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_YAML}") + def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) + def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) + dependsOn startDataPrepper1Task + dependsOn startDataPrepper2Task + startDataPrepper1Task.mustRunAfter 'startOdfeDockerContainer' + startDataPrepper2Task.mustRunAfter 'startOdfeDockerContainer' description = 'Runs the raw span integration tests.' group = 'verification' @@ -164,9 +169,12 @@ task rawSpanEndToEndTest(type: Test) { } finalizedBy stopOdfeDockerContainer - def stopDataPrepperTask = stopDataPrepperDockerContainer(startDataPrepperTask as DockerStartContainer) - finalizedBy stopDataPrepperTask - finalizedBy removeDataPrepperDockerContainer(stopDataPrepperTask as DockerStopContainer) + def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer) + def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer) + finalizedBy stopDataPrepper1Task + finalizedBy stopDataPrepper2Task + finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer) + finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer) finalizedBy removeDataPrepperNetwork } @@ -184,7 +192,7 @@ task serviceMapEndToEndTest(type: Test) { startDataPrepper1Task.mustRunAfter 'startOdfeDockerContainer' startDataPrepper2Task.mustRunAfter 'startOdfeDockerContainer' - description = 'Runs the raw span integration tests.' + description = 'Runs the service-map integration tests.' group = 'verification' testClassesDirs = sourceSets.integrationTest.output.classesDirs classpath = sourceSets.integrationTest.runtimeClasspath diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java index c7351031f..194fea298 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java @@ -69,10 +69,10 @@ public void testPipelineEndToEnd() throws IOException, InterruptedException { getResourceSpansBatch(TEST_TRACEID_2, testDataSet22) ); - sendExportTraceServiceRequestToSource(exportTraceServiceRequest11); - sendExportTraceServiceRequestToSource(exportTraceServiceRequest12); - sendExportTraceServiceRequestToSource(exportTraceServiceRequest21); - sendExportTraceServiceRequestToSource(exportTraceServiceRequest22); + sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest11); + sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest21); + sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest12); + sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest22); //Verify data in elasticsearch sink final List> expectedDocuments = getExpectedDocuments( @@ -124,6 +124,11 @@ private void sendExportTraceServiceRequestToSource(ExportTraceServiceRequest req client.export(request); } + private void sendExportTraceServiceRequestToSource(final int port, final ExportTraceServiceRequest request) { + Clients.newClient(String.format("gproto+http://127.0.0.1:%d/", port), + TraceServiceGrpc.TraceServiceBlockingStub.class).export(request); + } + private List> getSourcesFromSearchHits(final SearchHits searchHits) { final List> sources = new ArrayList<>(); searchHits.forEach(hit -> sources.add(hit.getSourceAsMap())); From d70cd4b002b5cff946b8b0a2b0d9b6dbbd0382b0 Mon Sep 17 00:00:00 2001 From: qchea Date: Fri, 2 Apr 2021 15:06:23 -0500 Subject: [PATCH 05/18] ADD: traceGroup inclusion in expectedDocuments --- .../integration/EndToEndRawSpanTest.java | 39 +++++++++---------- .../resources/raw-span-e2e-pipeline.yml | 2 +- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java index 194fea298..db53e408b 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java @@ -13,7 +13,6 @@ import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; import io.opentelemetry.proto.trace.v1.Span; -import io.opentelemetry.proto.trace.v1.Status; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -31,7 +30,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.concurrent.TimeUnit; import static org.awaitility.Awaitility.await; @@ -42,15 +40,20 @@ public class EndToEndRawSpanTest { private static final String TEST_TRACEID_1 = "ABC"; private static final String TEST_TRACEID_2 = "CBA"; + private static final Map TEST_TRACEID_TO_TRACE_GROUP = new HashMap() {{ + put(Hex.toHexString(TEST_TRACEID_1.getBytes()), ServiceMapTestData.DATA_100.name); + put(Hex.toHexString(TEST_TRACEID_2.getBytes()), ServiceMapTestData.DATA_101.name); + }}; private static final List testDataSet11 = Arrays.asList( - ServiceMapTestData.DATA_100, ServiceMapTestData.DATA_200, ServiceMapTestData.DATA_500, ServiceMapTestData.DATA_600, - ServiceMapTestData.DATA_700, ServiceMapTestData.DATA_1000); + ServiceMapTestData.DATA_100, ServiceMapTestData.DATA_200, ServiceMapTestData.DATA_300, + ServiceMapTestData.DATA_400, ServiceMapTestData.DATA_500, ServiceMapTestData.DATA_600); private static final List testDataSet12 = Arrays.asList( - ServiceMapTestData.DATA_300, ServiceMapTestData.DATA_400, ServiceMapTestData.DATA_800, - ServiceMapTestData.DATA_900, ServiceMapTestData.DATA_1100); + ServiceMapTestData.DATA_700, ServiceMapTestData.DATA_800, ServiceMapTestData.DATA_900, + ServiceMapTestData.DATA_1000, ServiceMapTestData.DATA_1100); private static final List testDataSet21 = Arrays.asList( - ServiceMapTestData.DATA_101, ServiceMapTestData.DATA_201, ServiceMapTestData.DATA_401, ServiceMapTestData.DATA_501); - private static final List testDataSet22 = Collections.singletonList(ServiceMapTestData.DATA_301); + ServiceMapTestData.DATA_101, ServiceMapTestData.DATA_201, ServiceMapTestData.DATA_301); + private static final List testDataSet22 = Arrays.asList( + ServiceMapTestData.DATA_401, ServiceMapTestData.DATA_501); private static final String INDEX_NAME = "otel-v1-apm-span-000001"; @Test @@ -69,10 +72,10 @@ public void testPipelineEndToEnd() throws IOException, InterruptedException { getResourceSpansBatch(TEST_TRACEID_2, testDataSet22) ); - sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest11); + sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequest11); + sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequest22); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest21); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest12); - sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest22); //Verify data in elasticsearch sink final List> expectedDocuments = getExpectedDocuments( @@ -84,7 +87,7 @@ public void testPipelineEndToEnd() throws IOException, InterruptedException { builder.withPassword("admin"); final RestHighLevelClient restHighLevelClient = builder.build().createClient(); // Wait for otel-trace-raw-prepper by trace_flush_interval - Thread.sleep(3000); + Thread.sleep(5000); // Wait for data to flow through pipeline and be indexed by ES await().atMost(10, TimeUnit.SECONDS).untilAsserted( () -> { @@ -118,12 +121,6 @@ private void refreshIndices(final RestHighLevelClient restHighLevelClient) throw restHighLevelClient.indices().refresh(requestAll, RequestOptions.DEFAULT); } - private void sendExportTraceServiceRequestToSource(ExportTraceServiceRequest request) { - final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.newClient( - "gproto+http://127.0.0.1:21890/", TraceServiceGrpc.TraceServiceBlockingStub.class); - client.export(request); - } - private void sendExportTraceServiceRequestToSource(final int port, final ExportTraceServiceRequest request) { Clients.newClient(String.format("gproto+http://127.0.0.1:%d/", port), TraceServiceGrpc.TraceServiceBlockingStub.class).export(request); @@ -208,16 +205,16 @@ private List> getExpectedDocuments(ExportTraceServiceRequest private Map getExpectedEsDocumentSource(final Span span, final String serviceName) { final Map esDocSource = new HashMap<>(); - esDocSource.put("traceId", Hex.toHexString(span.getTraceId().toByteArray())); + final String traceId = Hex.toHexString(span.getTraceId().toByteArray()); + esDocSource.put("traceId", traceId); esDocSource.put("spanId", Hex.toHexString(span.getSpanId().toByteArray())); esDocSource.put("parentSpanId", Hex.toHexString(span.getParentSpanId().toByteArray())); esDocSource.put("name", span.getName()); esDocSource.put("kind", span.getKind().name()); esDocSource.put("status.code", span.getStatus().getCodeValue()); esDocSource.put("serviceName", serviceName); - if (span.getParentSpanId().isEmpty()) { - esDocSource.put("traceGroup", span.getName()); - } + final String traceGroup = TEST_TRACEID_TO_TRACE_GROUP.get(traceId); + esDocSource.put("traceGroup", traceGroup); return esDocSource; } diff --git a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml index 9ee53599f..0f24b6bd2 100644 --- a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml +++ b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml @@ -14,7 +14,7 @@ raw-pipeline: prepper: - otel_trace_raw_prepper: root_span_flush_delay: 1 - trace_flush_interval: 3 + trace_flush_interval: 5 - otel_trace_group_prepper: hosts: [ "https://node-0.example.com:9200" ] username: "admin" From c5d2117542ee3d94351124b3097cdbcd66c2f57b Mon Sep 17 00:00:00 2001 From: qchea Date: Fri, 2 Apr 2021 15:11:52 -0500 Subject: [PATCH 06/18] MAINT: simplify raw-span-e2e-pipeline.yml --- .../resources/raw-span-e2e-pipeline.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml index 0f24b6bd2..6b93c3039 100644 --- a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml +++ b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml @@ -5,8 +5,6 @@ entry-pipeline: sink: - pipeline: name: "raw-pipeline" - - pipeline: - name: "service-map-pipeline" raw-pipeline: source: pipeline: @@ -24,17 +22,4 @@ raw-pipeline: hosts: [ "https://node-0.example.com:9200" ] username: "admin" password: "admin" - trace_analytics_raw: true -service-map-pipeline: - source: - pipeline: - name: "entry-pipeline" - prepper: - - service_map_stateful: - window_duration: 3 - sink: - - elasticsearch: - hosts: ["https://node-0.example.com:9200"] - username: "admin" - password: "admin" - trace_analytics_service_map: true \ No newline at end of file + trace_analytics_raw: true \ No newline at end of file From b37918977a40498e2f60a7202ccc15d9fb033637 Mon Sep 17 00:00:00 2001 From: qchea Date: Fri, 2 Apr 2021 15:16:39 -0500 Subject: [PATCH 07/18] MAINT: simplify service-map-e2e-pipeline.yml --- .../resources/service-map-e2e-pipeline.yml | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml b/data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml index 04d339899..9850b20a8 100644 --- a/data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml +++ b/data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml @@ -8,28 +8,8 @@ entry-pipeline: static_endpoints: ["data-prepper1", "data-prepper2"] ssl: false sink: - - pipeline: - name: "raw-pipeline" - pipeline: name: "service-map-pipeline" -raw-pipeline: - source: - pipeline: - name: "entry-pipeline" - prepper: - - otel_trace_raw_prepper: - root_span_flush_delay: 1 - trace_flush_interval: 3 - - otel_trace_group_prepper: - hosts: [ "https://node-0.example.com:9200" ] - username: "admin" - password: "admin" - sink: - - elasticsearch: - hosts: [ "https://node-0.example.com:9200" ] - username: "admin" - password: "admin" - trace_analytics_raw: true service-map-pipeline: source: pipeline: From 41d8ac8d6277a6c3a90a9bf57611f39944671cbd Mon Sep 17 00:00:00 2001 From: qchea Date: Fri, 2 Apr 2021 15:24:15 -0500 Subject: [PATCH 08/18] MAINT: ServiceMapTestData -> EndToEndTestData --- .../integration/EndToEndRawSpanTest.java | 28 +++++------ .../integration/EndToEndServiceMapTest.java | 48 +++++++++---------- ...MapTestData.java => EndToEndTestData.java} | 4 +- 3 files changed, 40 insertions(+), 40 deletions(-) rename data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/{ServiceMapTestData.java => EndToEndTestData.java} (92%) diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java index db53e408b..39ff025fa 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java @@ -41,19 +41,19 @@ public class EndToEndRawSpanTest { private static final String TEST_TRACEID_1 = "ABC"; private static final String TEST_TRACEID_2 = "CBA"; private static final Map TEST_TRACEID_TO_TRACE_GROUP = new HashMap() {{ - put(Hex.toHexString(TEST_TRACEID_1.getBytes()), ServiceMapTestData.DATA_100.name); - put(Hex.toHexString(TEST_TRACEID_2.getBytes()), ServiceMapTestData.DATA_101.name); + put(Hex.toHexString(TEST_TRACEID_1.getBytes()), EndToEndTestData.DATA_100.name); + put(Hex.toHexString(TEST_TRACEID_2.getBytes()), EndToEndTestData.DATA_101.name); }}; - private static final List testDataSet11 = Arrays.asList( - ServiceMapTestData.DATA_100, ServiceMapTestData.DATA_200, ServiceMapTestData.DATA_300, - ServiceMapTestData.DATA_400, ServiceMapTestData.DATA_500, ServiceMapTestData.DATA_600); - private static final List testDataSet12 = Arrays.asList( - ServiceMapTestData.DATA_700, ServiceMapTestData.DATA_800, ServiceMapTestData.DATA_900, - ServiceMapTestData.DATA_1000, ServiceMapTestData.DATA_1100); - private static final List testDataSet21 = Arrays.asList( - ServiceMapTestData.DATA_101, ServiceMapTestData.DATA_201, ServiceMapTestData.DATA_301); - private static final List testDataSet22 = Arrays.asList( - ServiceMapTestData.DATA_401, ServiceMapTestData.DATA_501); + private static final List testDataSet11 = Arrays.asList( + EndToEndTestData.DATA_100, EndToEndTestData.DATA_200, EndToEndTestData.DATA_300, + EndToEndTestData.DATA_400, EndToEndTestData.DATA_500, EndToEndTestData.DATA_600); + private static final List testDataSet12 = Arrays.asList( + EndToEndTestData.DATA_700, EndToEndTestData.DATA_800, EndToEndTestData.DATA_900, + EndToEndTestData.DATA_1000, EndToEndTestData.DATA_1100); + private static final List testDataSet21 = Arrays.asList( + EndToEndTestData.DATA_101, EndToEndTestData.DATA_201, EndToEndTestData.DATA_301); + private static final List testDataSet22 = Arrays.asList( + EndToEndTestData.DATA_401, EndToEndTestData.DATA_501); private static final String INDEX_NAME = "otel-v1-apm-span-000001"; @Test @@ -166,10 +166,10 @@ public static ExportTraceServiceRequest getExportTraceServiceRequest(List getResourceSpansBatch(final String traceId, final List data) { + private List getResourceSpansBatch(final String traceId, final List data) { final ArrayList spansList = new ArrayList<>(); for(int i=0; i < data.size(); i++) { - final ServiceMapTestData currData = data.get(i); + final EndToEndTestData currData = data.get(i); final String parentId = currData.parentId; final String spanId = currData.spanId; final String serviceName = currData.serviceName; diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java index 78d656a81..5475365ed 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java @@ -41,15 +41,15 @@ public class EndToEndServiceMapTest { private static final String TEST_TRACEID_2 = "CBA"; private static final int DATA_PREPPER_PORT_1 = 21890; private static final int DATA_PREPPER_PORT_2 = 21891; - private static final List testDataSet11 = Arrays.asList( - ServiceMapTestData.DATA_100, ServiceMapTestData.DATA_200, ServiceMapTestData.DATA_500, ServiceMapTestData.DATA_600, - ServiceMapTestData.DATA_700, ServiceMapTestData.DATA_1000); - private static final List testDataSet12 = Arrays.asList( - ServiceMapTestData.DATA_300, ServiceMapTestData.DATA_400, ServiceMapTestData.DATA_800, - ServiceMapTestData.DATA_900, ServiceMapTestData.DATA_1100); - private static final List testDataSet21 = Arrays.asList( - ServiceMapTestData.DATA_101, ServiceMapTestData.DATA_201, ServiceMapTestData.DATA_401, ServiceMapTestData.DATA_501); - private static final List testDataSet22 = Collections.singletonList(ServiceMapTestData.DATA_301); + private static final List testDataSet11 = Arrays.asList( + EndToEndTestData.DATA_100, EndToEndTestData.DATA_200, EndToEndTestData.DATA_500, EndToEndTestData.DATA_600, + EndToEndTestData.DATA_700, EndToEndTestData.DATA_1000); + private static final List testDataSet12 = Arrays.asList( + EndToEndTestData.DATA_300, EndToEndTestData.DATA_400, EndToEndTestData.DATA_800, + EndToEndTestData.DATA_900, EndToEndTestData.DATA_1100); + private static final List testDataSet21 = Arrays.asList( + EndToEndTestData.DATA_101, EndToEndTestData.DATA_201, EndToEndTestData.DATA_401, EndToEndTestData.DATA_501); + private static final List testDataSet22 = Collections.singletonList(EndToEndTestData.DATA_301); private static final String SERVICE_MAP_INDEX_NAME = "otel-v1-apm-service-map"; @Test @@ -66,7 +66,7 @@ public void testPipelineEndToEnd() throws IOException, InterruptedException { sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest12); //Verify data in elasticsearch sink - final List testDataSet1 = Stream.of(testDataSet11, testDataSet12) + final List testDataSet1 = Stream.of(testDataSet11, testDataSet12) .flatMap(Collection::stream).collect(Collectors.toList()); final List> possibleEdges = getPossibleEdges(TEST_TRACEID_1, testDataSet1); final ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder( @@ -100,7 +100,7 @@ public void testPipelineEndToEnd() throws IOException, InterruptedException { sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequest21); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest22); - final List testDataSet2 = Stream.of(testDataSet21, testDataSet22) + final List testDataSet2 = Stream.of(testDataSet21, testDataSet22) .flatMap(Collection::stream).collect(Collectors.toList()); possibleEdges.addAll(getPossibleEdges(TEST_TRACEID_2, testDataSet2)); // Wait for service map prepper by 2 * window_duration @@ -176,10 +176,10 @@ public static ExportTraceServiceRequest getExportTraceServiceRequest(List getResourceSpansBatch(final String traceId, final List data) { + private List getResourceSpansBatch(final String traceId, final List data) { final ArrayList spansList = new ArrayList<>(); for(int i=0; i < data.size(); i++) { - final ServiceMapTestData currData = data.get(i); + final EndToEndTestData currData = data.get(i); final String parentId = currData.parentId; final String spanId = currData.spanId; final String serviceName = currData.serviceName; @@ -198,14 +198,14 @@ private List getResourceSpansBatch(final String traceId, final Li return spansList; } - private List> getPossibleEdges(final String traceId, final List data) { - final Map spanIdToServiceMapTestData = data.stream() - .collect(Collectors.toMap(serviceMapTestData -> serviceMapTestData.spanId, serviceMapTestData -> serviceMapTestData)); + private List> getPossibleEdges(final String traceId, final List data) { + final Map spanIdToServiceMapTestData = data.stream() + .collect(Collectors.toMap(endToEndTestData -> endToEndTestData.spanId, endToEndTestData -> endToEndTestData)); final List> possibleEdges = new ArrayList<>(); - for (final ServiceMapTestData currData : data) { + for (final EndToEndTestData currData : data) { final String parentId = currData.parentId; if (parentId != null) { - final ServiceMapTestData parentData = spanIdToServiceMapTestData.get(parentId); + final EndToEndTestData parentData = spanIdToServiceMapTestData.get(parentId); if (parentData != null && !parentData.serviceName.equals(currData.serviceName)) { String rootSpanName = getRootSpanName(parentId, spanIdToServiceMapTestData); possibleEdges.addAll(getEdgeMaps(rootSpanName, currData, parentData)); @@ -216,16 +216,16 @@ private List> getPossibleEdges(final String traceId, final L return possibleEdges; } - private String getRootSpanName(String spanId, final Map spanIdToServiceMapTestData) { - ServiceMapTestData rootServiceMapTestData = spanIdToServiceMapTestData.get(spanId); - while (rootServiceMapTestData.parentId != null) { - rootServiceMapTestData = spanIdToServiceMapTestData.get(rootServiceMapTestData.parentId); + private String getRootSpanName(String spanId, final Map spanIdToServiceMapTestData) { + EndToEndTestData rootEndToEndTestData = spanIdToServiceMapTestData.get(spanId); + while (rootEndToEndTestData.parentId != null) { + rootEndToEndTestData = spanIdToServiceMapTestData.get(rootEndToEndTestData.parentId); } - return rootServiceMapTestData.name; + return rootEndToEndTestData.name; } private List> getEdgeMaps( - final String rootSpanName, final ServiceMapTestData currData, final ServiceMapTestData parentData) { + final String rootSpanName, final EndToEndTestData currData, final EndToEndTestData parentData) { final List> edges = new ArrayList<>(); Map destination = new HashMap<>(); diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/ServiceMapTestData.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestData.java similarity index 92% rename from data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/ServiceMapTestData.java rename to data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestData.java index ad849f620..07680ef53 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/ServiceMapTestData.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestData.java @@ -2,7 +2,7 @@ import io.opentelemetry.proto.trace.v1.Span; -public enum ServiceMapTestData { +public enum EndToEndTestData { DATA_100("100", "ServiceA", null, "FRUITS", Span.SpanKind.SPAN_KIND_INTERNAL), DATA_200("200", "ServiceA", "100", "CALL_SERVICE_B_APPLE", Span.SpanKind.SPAN_KIND_CLIENT), @@ -28,7 +28,7 @@ public enum ServiceMapTestData { public final String name; public final Span.SpanKind spanKind; - ServiceMapTestData(String spanId, String serviceName, String parentId, String name, Span.SpanKind kind) { + EndToEndTestData(String spanId, String serviceName, String parentId, String name, Span.SpanKind kind) { this.spanId = spanId; this.serviceName = serviceName; this.parentId = parentId; From 1e6bce5397ddbbb4b33899172b5eda92e5e3982c Mon Sep 17 00:00:00 2001 From: qchea Date: Fri, 2 Apr 2021 15:36:59 -0500 Subject: [PATCH 09/18] MAINT: capitalize static final variables --- .../integration/EndToEndRawSpanTest.java | 18 ++++++++--------- .../integration/EndToEndServiceMapTest.java | 20 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java index 39ff025fa..830d08948 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java @@ -44,32 +44,32 @@ public class EndToEndRawSpanTest { put(Hex.toHexString(TEST_TRACEID_1.getBytes()), EndToEndTestData.DATA_100.name); put(Hex.toHexString(TEST_TRACEID_2.getBytes()), EndToEndTestData.DATA_101.name); }}; - private static final List testDataSet11 = Arrays.asList( + private static final List TEST_DATA_SET_1_WITH_ROOT_SPAN = Arrays.asList( EndToEndTestData.DATA_100, EndToEndTestData.DATA_200, EndToEndTestData.DATA_300, EndToEndTestData.DATA_400, EndToEndTestData.DATA_500, EndToEndTestData.DATA_600); - private static final List testDataSet12 = Arrays.asList( + private static final List TEST_DATA_SET_1_WITHOUT_ROOT_SPAN = Arrays.asList( EndToEndTestData.DATA_700, EndToEndTestData.DATA_800, EndToEndTestData.DATA_900, EndToEndTestData.DATA_1000, EndToEndTestData.DATA_1100); - private static final List testDataSet21 = Arrays.asList( + private static final List TEST_DATA_SET_2_WITH_ROOT_SPAN = Arrays.asList( EndToEndTestData.DATA_101, EndToEndTestData.DATA_201, EndToEndTestData.DATA_301); - private static final List testDataSet22 = Arrays.asList( + private static final List TEST_DATA_SET_2_WITHOUT_ROOT_SPAN = Arrays.asList( EndToEndTestData.DATA_401, EndToEndTestData.DATA_501); private static final String INDEX_NAME = "otel-v1-apm-span-000001"; @Test - public void testPipelineEndToEnd() throws IOException, InterruptedException { + public void testPipelineEndToEnd() throws InterruptedException { //Send data to otel trace source final ExportTraceServiceRequest exportTraceServiceRequest11 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_1, testDataSet11) + getResourceSpansBatch(TEST_TRACEID_1, TEST_DATA_SET_1_WITH_ROOT_SPAN) ); final ExportTraceServiceRequest exportTraceServiceRequest12 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_1, testDataSet12) + getResourceSpansBatch(TEST_TRACEID_1, TEST_DATA_SET_1_WITHOUT_ROOT_SPAN) ); final ExportTraceServiceRequest exportTraceServiceRequest21 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_2, testDataSet21) + getResourceSpansBatch(TEST_TRACEID_2, TEST_DATA_SET_2_WITH_ROOT_SPAN) ); final ExportTraceServiceRequest exportTraceServiceRequest22 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_2, testDataSet22) + getResourceSpansBatch(TEST_TRACEID_2, TEST_DATA_SET_2_WITHOUT_ROOT_SPAN) ); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequest11); diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java index 5475365ed..c7d43b65d 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java @@ -41,32 +41,32 @@ public class EndToEndServiceMapTest { private static final String TEST_TRACEID_2 = "CBA"; private static final int DATA_PREPPER_PORT_1 = 21890; private static final int DATA_PREPPER_PORT_2 = 21891; - private static final List testDataSet11 = Arrays.asList( + private static final List TEST_DATA_SET_11 = Arrays.asList( EndToEndTestData.DATA_100, EndToEndTestData.DATA_200, EndToEndTestData.DATA_500, EndToEndTestData.DATA_600, EndToEndTestData.DATA_700, EndToEndTestData.DATA_1000); - private static final List testDataSet12 = Arrays.asList( + private static final List TEST_DATA_SET_12 = Arrays.asList( EndToEndTestData.DATA_300, EndToEndTestData.DATA_400, EndToEndTestData.DATA_800, EndToEndTestData.DATA_900, EndToEndTestData.DATA_1100); - private static final List testDataSet21 = Arrays.asList( + private static final List TEST_DATA_SET_21 = Arrays.asList( EndToEndTestData.DATA_101, EndToEndTestData.DATA_201, EndToEndTestData.DATA_401, EndToEndTestData.DATA_501); - private static final List testDataSet22 = Collections.singletonList(EndToEndTestData.DATA_301); + private static final List TEST_DATA_SET_22 = Collections.singletonList(EndToEndTestData.DATA_301); private static final String SERVICE_MAP_INDEX_NAME = "otel-v1-apm-service-map"; @Test public void testPipelineEndToEnd() throws IOException, InterruptedException { // Send test trace group 1 final ExportTraceServiceRequest exportTraceServiceRequest11 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_1, testDataSet11) + getResourceSpansBatch(TEST_TRACEID_1, TEST_DATA_SET_11) ); final ExportTraceServiceRequest exportTraceServiceRequest12 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_1, testDataSet12) + getResourceSpansBatch(TEST_TRACEID_1, TEST_DATA_SET_12) ); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequest11); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest12); //Verify data in elasticsearch sink - final List testDataSet1 = Stream.of(testDataSet11, testDataSet12) + final List testDataSet1 = Stream.of(TEST_DATA_SET_11, TEST_DATA_SET_12) .flatMap(Collection::stream).collect(Collectors.toList()); final List> possibleEdges = getPossibleEdges(TEST_TRACEID_1, testDataSet1); final ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder( @@ -91,16 +91,16 @@ public void testPipelineEndToEnd() throws IOException, InterruptedException { sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest12); // Send test trace group 2 final ExportTraceServiceRequest exportTraceServiceRequest21 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_2, testDataSet21) + getResourceSpansBatch(TEST_TRACEID_2, TEST_DATA_SET_21) ); final ExportTraceServiceRequest exportTraceServiceRequest22 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_2, testDataSet22) + getResourceSpansBatch(TEST_TRACEID_2, TEST_DATA_SET_22) ); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequest21); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest22); - final List testDataSet2 = Stream.of(testDataSet21, testDataSet22) + final List testDataSet2 = Stream.of(TEST_DATA_SET_21, TEST_DATA_SET_22) .flatMap(Collection::stream).collect(Collectors.toList()); possibleEdges.addAll(getPossibleEdges(TEST_TRACEID_2, testDataSet2)); // Wait for service map prepper by 2 * window_duration From 99defb4c1f8cd657ae201cc842669fcfa70c4c3a Mon Sep 17 00:00:00 2001 From: qchea Date: Mon, 5 Apr 2021 17:14:45 -0500 Subject: [PATCH 10/18] REF: sleep after start data-prepper --- data-prepper-core/integrationTest.gradle | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/data-prepper-core/integrationTest.gradle b/data-prepper-core/integrationTest.gradle index 1547a4499..f10c0b3e2 100644 --- a/data-prepper-core/integrationTest.gradle +++ b/data-prepper-core/integrationTest.gradle @@ -92,9 +92,6 @@ def startDataPrepperDockerContainer(final DockerCreateContainer createDataPreppe return tasks.create("start${createDataPrepperDockerContainerTask.getName()}", DockerStartContainer) { dependsOn createDataPrepperDockerContainerTask targetContainerId createDataPrepperDockerContainerTask.getContainerId() - doLast { - sleep(10*1000) - } } } @@ -158,6 +155,10 @@ task rawSpanEndToEndTest(type: Test) { dependsOn startDataPrepper2Task startDataPrepper1Task.mustRunAfter 'startOdfeDockerContainer' startDataPrepper2Task.mustRunAfter 'startOdfeDockerContainer' + // wait for data-preppers to be ready + doFirst { + sleep(10*1000) + } description = 'Runs the raw span integration tests.' group = 'verification' @@ -191,6 +192,10 @@ task serviceMapEndToEndTest(type: Test) { dependsOn startDataPrepper2Task startDataPrepper1Task.mustRunAfter 'startOdfeDockerContainer' startDataPrepper2Task.mustRunAfter 'startOdfeDockerContainer' + // wait for data-preppers to be ready + doFirst { + sleep(10*1000) + } description = 'Runs the service-map integration tests.' group = 'verification' From bb31dd805fc03ea4d20817218b38365b7d498ed8 Mon Sep 17 00:00:00 2001 From: qchea Date: Mon, 5 Apr 2021 17:36:45 -0500 Subject: [PATCH 11/18] MAINT: increase wait time --- .../amazon/dataprepper/integration/EndToEndRawSpanTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java index 830d08948..dc0b95a34 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java @@ -86,8 +86,8 @@ public void testPipelineEndToEnd() throws InterruptedException { builder.withUsername("admin"); builder.withPassword("admin"); final RestHighLevelClient restHighLevelClient = builder.build().createClient(); - // Wait for otel-trace-raw-prepper by trace_flush_interval - Thread.sleep(5000); + // Wait for otel-trace-raw-prepper by at least trace_flush_interval + Thread.sleep(6000); // Wait for data to flow through pipeline and be indexed by ES await().atMost(10, TimeUnit.SECONDS).untilAsserted( () -> { From 38b4b99a71a32b01757464a81244c888945dbf23 Mon Sep 17 00:00:00 2001 From: qchea Date: Mon, 5 Apr 2021 17:41:54 -0500 Subject: [PATCH 12/18] MAINT: some nits --- .../integration/EndToEndRawSpanTest.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java index dc0b95a34..17422e97d 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java @@ -166,15 +166,14 @@ public static ExportTraceServiceRequest getExportTraceServiceRequest(List getResourceSpansBatch(final String traceId, final List data) { + private List getResourceSpansBatch(final String traceId, final List dataList) { final ArrayList spansList = new ArrayList<>(); - for(int i=0; i < data.size(); i++) { - final EndToEndTestData currData = data.get(i); - final String parentId = currData.parentId; - final String spanId = currData.spanId; - final String serviceName = currData.serviceName; - final String spanName = currData.name; - final Span.SpanKind spanKind = currData.spanKind; + for(final EndToEndTestData data : dataList) { + final String parentId = data.parentId; + final String spanId = data.spanId; + final String serviceName = data.serviceName; + final String spanName = data.name; + final Span.SpanKind spanKind = data.spanKind; final ResourceSpans rs = getResourceSpans( serviceName, spanName, From 2e00605e9137579b3465018c5e161efc3074e532 Mon Sep 17 00:00:00 2001 From: qchea Date: Mon, 5 Apr 2021 18:16:40 -0500 Subject: [PATCH 13/18] MAINT: digest traceID into test data --- .../integration/EndToEndRawSpanTest.java | 17 +++++---- .../integration/EndToEndTestData.java | 36 ++++++++++--------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java index 17422e97d..81c414594 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java @@ -38,11 +38,9 @@ public class EndToEndRawSpanTest { private static final int DATA_PREPPER_PORT_1 = 21890; private static final int DATA_PREPPER_PORT_2 = 21891; - private static final String TEST_TRACEID_1 = "ABC"; - private static final String TEST_TRACEID_2 = "CBA"; private static final Map TEST_TRACEID_TO_TRACE_GROUP = new HashMap() {{ - put(Hex.toHexString(TEST_TRACEID_1.getBytes()), EndToEndTestData.DATA_100.name); - put(Hex.toHexString(TEST_TRACEID_2.getBytes()), EndToEndTestData.DATA_101.name); + put(Hex.toHexString(EndToEndTestData.DATA_100.traceId.getBytes()), EndToEndTestData.DATA_100.name); + put(Hex.toHexString(EndToEndTestData.DATA_101.traceId.getBytes()), EndToEndTestData.DATA_101.name); }}; private static final List TEST_DATA_SET_1_WITH_ROOT_SPAN = Arrays.asList( EndToEndTestData.DATA_100, EndToEndTestData.DATA_200, EndToEndTestData.DATA_300, @@ -60,16 +58,16 @@ public class EndToEndRawSpanTest { public void testPipelineEndToEnd() throws InterruptedException { //Send data to otel trace source final ExportTraceServiceRequest exportTraceServiceRequest11 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_1, TEST_DATA_SET_1_WITH_ROOT_SPAN) + getResourceSpansBatch(TEST_DATA_SET_1_WITH_ROOT_SPAN) ); final ExportTraceServiceRequest exportTraceServiceRequest12 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_1, TEST_DATA_SET_1_WITHOUT_ROOT_SPAN) + getResourceSpansBatch(TEST_DATA_SET_1_WITHOUT_ROOT_SPAN) ); final ExportTraceServiceRequest exportTraceServiceRequest21 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_2, TEST_DATA_SET_2_WITH_ROOT_SPAN) + getResourceSpansBatch(TEST_DATA_SET_2_WITH_ROOT_SPAN) ); final ExportTraceServiceRequest exportTraceServiceRequest22 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_2, TEST_DATA_SET_2_WITHOUT_ROOT_SPAN) + getResourceSpansBatch(TEST_DATA_SET_2_WITHOUT_ROOT_SPAN) ); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequest11); @@ -166,9 +164,10 @@ public static ExportTraceServiceRequest getExportTraceServiceRequest(List getResourceSpansBatch(final String traceId, final List dataList) { + private List getResourceSpansBatch(final List dataList) { final ArrayList spansList = new ArrayList<>(); for(final EndToEndTestData data : dataList) { + final String traceId = data.traceId; final String parentId = data.parentId; final String spanId = data.spanId; final String serviceName = data.serviceName; diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestData.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestData.java index 07680ef53..d5d139f19 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestData.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestData.java @@ -4,31 +4,33 @@ public enum EndToEndTestData { - DATA_100("100", "ServiceA", null, "FRUITS", Span.SpanKind.SPAN_KIND_INTERNAL), - DATA_200("200", "ServiceA", "100", "CALL_SERVICE_B_APPLE", Span.SpanKind.SPAN_KIND_CLIENT), - DATA_300("300", "ServiceB", "200", "/APPLE", Span.SpanKind.SPAN_KIND_SERVER), - DATA_400("400", "ServiceB", "300", "CALL_SERVICE_C_ORANGE", Span.SpanKind.SPAN_KIND_CLIENT), - DATA_500("500", "ServiceC", "400", "/ORANGE", Span.SpanKind.SPAN_KIND_SERVER), - DATA_600("600", "ServiceC", "500", "SOME_INTERNAL", Span.SpanKind.SPAN_KIND_INTERNAL), - DATA_700("700", "ServiceA", "100", "CALL_SERVICE_B_JACKFRUIT", Span.SpanKind.SPAN_KIND_CLIENT), - DATA_800("800", "ServiceB", "700", "/JACKFRUIT", Span.SpanKind.SPAN_KIND_SERVER), - DATA_900("900", "ServiceB", "800", "SOME_INTERNAL", Span.SpanKind.SPAN_KIND_INTERNAL), - DATA_1000("1000", "ServiceA", "100", "CALL_SERVICE_D_PEAR", Span.SpanKind.SPAN_KIND_CLIENT), - DATA_1100("1100", "ServiceD", "1000", "/PEAR", Span.SpanKind.SPAN_KIND_SERVER), + DATA_100("TRACE_1", "100", "ServiceA", null, "FRUITS", Span.SpanKind.SPAN_KIND_INTERNAL), + DATA_200("TRACE_1", "200", "ServiceA", "100", "CALL_SERVICE_B_APPLE", Span.SpanKind.SPAN_KIND_CLIENT), + DATA_300("TRACE_1", "300", "ServiceB", "200", "/APPLE", Span.SpanKind.SPAN_KIND_SERVER), + DATA_400("TRACE_1", "400", "ServiceB", "300", "CALL_SERVICE_C_ORANGE", Span.SpanKind.SPAN_KIND_CLIENT), + DATA_500("TRACE_1", "500", "ServiceC", "400", "/ORANGE", Span.SpanKind.SPAN_KIND_SERVER), + DATA_600("TRACE_1", "600", "ServiceC", "500", "SOME_INTERNAL", Span.SpanKind.SPAN_KIND_INTERNAL), + DATA_700("TRACE_1", "700", "ServiceA", "100", "CALL_SERVICE_B_JACKFRUIT", Span.SpanKind.SPAN_KIND_CLIENT), + DATA_800("TRACE_1", "800", "ServiceB", "700", "/JACKFRUIT", Span.SpanKind.SPAN_KIND_SERVER), + DATA_900("TRACE_1", "900", "ServiceB", "800", "SOME_INTERNAL", Span.SpanKind.SPAN_KIND_INTERNAL), + DATA_1000("TRACE_1", "1000", "ServiceA", "100", "CALL_SERVICE_D_PEAR", Span.SpanKind.SPAN_KIND_CLIENT), + DATA_1100("TRACE_1", "1100", "ServiceD", "1000", "/PEAR", Span.SpanKind.SPAN_KIND_SERVER), - DATA_101("101", "ServiceA", null, "VEGGIES", Span.SpanKind.SPAN_KIND_INTERNAL), - DATA_201("201", "ServiceA", "101", "CALL_SERVICE_B_ONION", Span.SpanKind.SPAN_KIND_CLIENT), - DATA_301("301", "ServiceB", "201", "/ONION", Span.SpanKind.SPAN_KIND_SERVER), - DATA_401("401", "ServiceA", "101", "CALL_SERVICE_E_POTATO", Span.SpanKind.SPAN_KIND_CLIENT), - DATA_501("501", "ServiceE", "401", "/POTATO", Span.SpanKind.SPAN_KIND_SERVER); + DATA_101("TRACE_2", "101", "ServiceA", null, "VEGGIES", Span.SpanKind.SPAN_KIND_INTERNAL), + DATA_201("TRACE_2", "201", "ServiceA", "101", "CALL_SERVICE_B_ONION", Span.SpanKind.SPAN_KIND_CLIENT), + DATA_301("TRACE_2", "301", "ServiceB", "201", "/ONION", Span.SpanKind.SPAN_KIND_SERVER), + DATA_401("TRACE_2", "401", "ServiceA", "101", "CALL_SERVICE_E_POTATO", Span.SpanKind.SPAN_KIND_CLIENT), + DATA_501("TRACE_2", "501", "ServiceE", "401", "/POTATO", Span.SpanKind.SPAN_KIND_SERVER); + public final String traceId; public final String spanId; public final String serviceName; public final String parentId; public final String name; public final Span.SpanKind spanKind; - EndToEndTestData(String spanId, String serviceName, String parentId, String name, Span.SpanKind kind) { + EndToEndTestData(String traceId, String spanId, String serviceName, String parentId, String name, Span.SpanKind kind) { + this.traceId = traceId; this.spanId = spanId; this.serviceName = serviceName; this.parentId = parentId; From 81034030014b59c92c3835fcca8525fb4e0069fb Mon Sep 17 00:00:00 2001 From: qchea Date: Tue, 6 Apr 2021 09:34:39 -0500 Subject: [PATCH 14/18] RNM: variables --- .../integration/EndToEndRawSpanTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java index 81c414594..e7e142d87 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java @@ -57,28 +57,28 @@ public class EndToEndRawSpanTest { @Test public void testPipelineEndToEnd() throws InterruptedException { //Send data to otel trace source - final ExportTraceServiceRequest exportTraceServiceRequest11 = getExportTraceServiceRequest( + final ExportTraceServiceRequest exportTraceServiceRequestTrace1BatchWithRoot = getExportTraceServiceRequest( getResourceSpansBatch(TEST_DATA_SET_1_WITH_ROOT_SPAN) ); - final ExportTraceServiceRequest exportTraceServiceRequest12 = getExportTraceServiceRequest( + final ExportTraceServiceRequest exportTraceServiceRequestTrace1BatchNoRoot = getExportTraceServiceRequest( getResourceSpansBatch(TEST_DATA_SET_1_WITHOUT_ROOT_SPAN) ); - final ExportTraceServiceRequest exportTraceServiceRequest21 = getExportTraceServiceRequest( + final ExportTraceServiceRequest exportTraceServiceRequestTrace2BatchWithRoot = getExportTraceServiceRequest( getResourceSpansBatch(TEST_DATA_SET_2_WITH_ROOT_SPAN) ); - final ExportTraceServiceRequest exportTraceServiceRequest22 = getExportTraceServiceRequest( + final ExportTraceServiceRequest exportTraceServiceRequestTrace2BatchNoRoot = getExportTraceServiceRequest( getResourceSpansBatch(TEST_DATA_SET_2_WITHOUT_ROOT_SPAN) ); - sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequest11); - sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequest22); - sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest21); - sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest12); + sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequestTrace1BatchWithRoot); + sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequestTrace2BatchNoRoot); + sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequestTrace2BatchWithRoot); + sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequestTrace1BatchNoRoot); //Verify data in elasticsearch sink final List> expectedDocuments = getExpectedDocuments( - exportTraceServiceRequest11, exportTraceServiceRequest12, - exportTraceServiceRequest21, exportTraceServiceRequest22); + exportTraceServiceRequestTrace1BatchWithRoot, exportTraceServiceRequestTrace1BatchNoRoot, + exportTraceServiceRequestTrace2BatchWithRoot, exportTraceServiceRequestTrace2BatchNoRoot); final ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder( Collections.singletonList("https://127.0.0.1:9200")); builder.withUsername("admin"); From c32a2086493e53a7398600469b7ab70a565c9188 Mon Sep 17 00:00:00 2001 From: qchea Date: Tue, 6 Apr 2021 09:42:10 -0500 Subject: [PATCH 15/18] RNM: EndToEndTestData -> EndToEndTestSpan --- .../integration/EndToEndRawSpanTest.java | 28 +++++------ .../integration/EndToEndServiceMapTest.java | 48 +++++++++---------- ...EndTestData.java => EndToEndTestSpan.java} | 4 +- 3 files changed, 40 insertions(+), 40 deletions(-) rename data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/{EndToEndTestData.java => EndToEndTestSpan.java} (95%) diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java index e7e142d87..64f776772 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java @@ -39,19 +39,19 @@ public class EndToEndRawSpanTest { private static final int DATA_PREPPER_PORT_2 = 21891; private static final Map TEST_TRACEID_TO_TRACE_GROUP = new HashMap() {{ - put(Hex.toHexString(EndToEndTestData.DATA_100.traceId.getBytes()), EndToEndTestData.DATA_100.name); - put(Hex.toHexString(EndToEndTestData.DATA_101.traceId.getBytes()), EndToEndTestData.DATA_101.name); + put(Hex.toHexString(EndToEndTestSpan.DATA_100.traceId.getBytes()), EndToEndTestSpan.DATA_100.name); + put(Hex.toHexString(EndToEndTestSpan.DATA_101.traceId.getBytes()), EndToEndTestSpan.DATA_101.name); }}; - private static final List TEST_DATA_SET_1_WITH_ROOT_SPAN = Arrays.asList( - EndToEndTestData.DATA_100, EndToEndTestData.DATA_200, EndToEndTestData.DATA_300, - EndToEndTestData.DATA_400, EndToEndTestData.DATA_500, EndToEndTestData.DATA_600); - private static final List TEST_DATA_SET_1_WITHOUT_ROOT_SPAN = Arrays.asList( - EndToEndTestData.DATA_700, EndToEndTestData.DATA_800, EndToEndTestData.DATA_900, - EndToEndTestData.DATA_1000, EndToEndTestData.DATA_1100); - private static final List TEST_DATA_SET_2_WITH_ROOT_SPAN = Arrays.asList( - EndToEndTestData.DATA_101, EndToEndTestData.DATA_201, EndToEndTestData.DATA_301); - private static final List TEST_DATA_SET_2_WITHOUT_ROOT_SPAN = Arrays.asList( - EndToEndTestData.DATA_401, EndToEndTestData.DATA_501); + private static final List TEST_DATA_SET_1_WITH_ROOT_SPAN = Arrays.asList( + EndToEndTestSpan.DATA_100, EndToEndTestSpan.DATA_200, EndToEndTestSpan.DATA_300, + EndToEndTestSpan.DATA_400, EndToEndTestSpan.DATA_500, EndToEndTestSpan.DATA_600); + private static final List TEST_DATA_SET_1_WITHOUT_ROOT_SPAN = Arrays.asList( + EndToEndTestSpan.DATA_700, EndToEndTestSpan.DATA_800, EndToEndTestSpan.DATA_900, + EndToEndTestSpan.DATA_1000, EndToEndTestSpan.DATA_1100); + private static final List TEST_DATA_SET_2_WITH_ROOT_SPAN = Arrays.asList( + EndToEndTestSpan.DATA_101, EndToEndTestSpan.DATA_201, EndToEndTestSpan.DATA_301); + private static final List TEST_DATA_SET_2_WITHOUT_ROOT_SPAN = Arrays.asList( + EndToEndTestSpan.DATA_401, EndToEndTestSpan.DATA_501); private static final String INDEX_NAME = "otel-v1-apm-span-000001"; @Test @@ -164,9 +164,9 @@ public static ExportTraceServiceRequest getExportTraceServiceRequest(List getResourceSpansBatch(final List dataList) { + private List getResourceSpansBatch(final List dataList) { final ArrayList spansList = new ArrayList<>(); - for(final EndToEndTestData data : dataList) { + for(final EndToEndTestSpan data : dataList) { final String traceId = data.traceId; final String parentId = data.parentId; final String spanId = data.spanId; diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java index c7d43b65d..51861c5c8 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java @@ -41,15 +41,15 @@ public class EndToEndServiceMapTest { private static final String TEST_TRACEID_2 = "CBA"; private static final int DATA_PREPPER_PORT_1 = 21890; private static final int DATA_PREPPER_PORT_2 = 21891; - private static final List TEST_DATA_SET_11 = Arrays.asList( - EndToEndTestData.DATA_100, EndToEndTestData.DATA_200, EndToEndTestData.DATA_500, EndToEndTestData.DATA_600, - EndToEndTestData.DATA_700, EndToEndTestData.DATA_1000); - private static final List TEST_DATA_SET_12 = Arrays.asList( - EndToEndTestData.DATA_300, EndToEndTestData.DATA_400, EndToEndTestData.DATA_800, - EndToEndTestData.DATA_900, EndToEndTestData.DATA_1100); - private static final List TEST_DATA_SET_21 = Arrays.asList( - EndToEndTestData.DATA_101, EndToEndTestData.DATA_201, EndToEndTestData.DATA_401, EndToEndTestData.DATA_501); - private static final List TEST_DATA_SET_22 = Collections.singletonList(EndToEndTestData.DATA_301); + private static final List TEST_DATA_SET_11 = Arrays.asList( + EndToEndTestSpan.DATA_100, EndToEndTestSpan.DATA_200, EndToEndTestSpan.DATA_500, EndToEndTestSpan.DATA_600, + EndToEndTestSpan.DATA_700, EndToEndTestSpan.DATA_1000); + private static final List TEST_DATA_SET_12 = Arrays.asList( + EndToEndTestSpan.DATA_300, EndToEndTestSpan.DATA_400, EndToEndTestSpan.DATA_800, + EndToEndTestSpan.DATA_900, EndToEndTestSpan.DATA_1100); + private static final List TEST_DATA_SET_21 = Arrays.asList( + EndToEndTestSpan.DATA_101, EndToEndTestSpan.DATA_201, EndToEndTestSpan.DATA_401, EndToEndTestSpan.DATA_501); + private static final List TEST_DATA_SET_22 = Collections.singletonList(EndToEndTestSpan.DATA_301); private static final String SERVICE_MAP_INDEX_NAME = "otel-v1-apm-service-map"; @Test @@ -66,7 +66,7 @@ public void testPipelineEndToEnd() throws IOException, InterruptedException { sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest12); //Verify data in elasticsearch sink - final List testDataSet1 = Stream.of(TEST_DATA_SET_11, TEST_DATA_SET_12) + final List testDataSet1 = Stream.of(TEST_DATA_SET_11, TEST_DATA_SET_12) .flatMap(Collection::stream).collect(Collectors.toList()); final List> possibleEdges = getPossibleEdges(TEST_TRACEID_1, testDataSet1); final ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder( @@ -100,7 +100,7 @@ public void testPipelineEndToEnd() throws IOException, InterruptedException { sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequest21); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest22); - final List testDataSet2 = Stream.of(TEST_DATA_SET_21, TEST_DATA_SET_22) + final List testDataSet2 = Stream.of(TEST_DATA_SET_21, TEST_DATA_SET_22) .flatMap(Collection::stream).collect(Collectors.toList()); possibleEdges.addAll(getPossibleEdges(TEST_TRACEID_2, testDataSet2)); // Wait for service map prepper by 2 * window_duration @@ -176,10 +176,10 @@ public static ExportTraceServiceRequest getExportTraceServiceRequest(List getResourceSpansBatch(final String traceId, final List data) { + private List getResourceSpansBatch(final String traceId, final List data) { final ArrayList spansList = new ArrayList<>(); for(int i=0; i < data.size(); i++) { - final EndToEndTestData currData = data.get(i); + final EndToEndTestSpan currData = data.get(i); final String parentId = currData.parentId; final String spanId = currData.spanId; final String serviceName = currData.serviceName; @@ -198,14 +198,14 @@ private List getResourceSpansBatch(final String traceId, final Li return spansList; } - private List> getPossibleEdges(final String traceId, final List data) { - final Map spanIdToServiceMapTestData = data.stream() - .collect(Collectors.toMap(endToEndTestData -> endToEndTestData.spanId, endToEndTestData -> endToEndTestData)); + private List> getPossibleEdges(final String traceId, final List data) { + final Map spanIdToServiceMapTestData = data.stream() + .collect(Collectors.toMap(endToEndTestSpan -> endToEndTestSpan.spanId, endToEndTestSpan -> endToEndTestSpan)); final List> possibleEdges = new ArrayList<>(); - for (final EndToEndTestData currData : data) { + for (final EndToEndTestSpan currData : data) { final String parentId = currData.parentId; if (parentId != null) { - final EndToEndTestData parentData = spanIdToServiceMapTestData.get(parentId); + final EndToEndTestSpan parentData = spanIdToServiceMapTestData.get(parentId); if (parentData != null && !parentData.serviceName.equals(currData.serviceName)) { String rootSpanName = getRootSpanName(parentId, spanIdToServiceMapTestData); possibleEdges.addAll(getEdgeMaps(rootSpanName, currData, parentData)); @@ -216,16 +216,16 @@ private List> getPossibleEdges(final String traceId, final L return possibleEdges; } - private String getRootSpanName(String spanId, final Map spanIdToServiceMapTestData) { - EndToEndTestData rootEndToEndTestData = spanIdToServiceMapTestData.get(spanId); - while (rootEndToEndTestData.parentId != null) { - rootEndToEndTestData = spanIdToServiceMapTestData.get(rootEndToEndTestData.parentId); + private String getRootSpanName(String spanId, final Map spanIdToServiceMapTestData) { + EndToEndTestSpan rootEndToEndTestSpan = spanIdToServiceMapTestData.get(spanId); + while (rootEndToEndTestSpan.parentId != null) { + rootEndToEndTestSpan = spanIdToServiceMapTestData.get(rootEndToEndTestSpan.parentId); } - return rootEndToEndTestData.name; + return rootEndToEndTestSpan.name; } private List> getEdgeMaps( - final String rootSpanName, final EndToEndTestData currData, final EndToEndTestData parentData) { + final String rootSpanName, final EndToEndTestSpan currData, final EndToEndTestSpan parentData) { final List> edges = new ArrayList<>(); Map destination = new HashMap<>(); diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestData.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestSpan.java similarity index 95% rename from data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestData.java rename to data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestSpan.java index d5d139f19..0f6d0270b 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestData.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestSpan.java @@ -2,7 +2,7 @@ import io.opentelemetry.proto.trace.v1.Span; -public enum EndToEndTestData { +public enum EndToEndTestSpan { DATA_100("TRACE_1", "100", "ServiceA", null, "FRUITS", Span.SpanKind.SPAN_KIND_INTERNAL), DATA_200("TRACE_1", "200", "ServiceA", "100", "CALL_SERVICE_B_APPLE", Span.SpanKind.SPAN_KIND_CLIENT), @@ -29,7 +29,7 @@ public enum EndToEndTestData { public final String name; public final Span.SpanKind spanKind; - EndToEndTestData(String traceId, String spanId, String serviceName, String parentId, String name, Span.SpanKind kind) { + EndToEndTestSpan(String traceId, String spanId, String serviceName, String parentId, String name, Span.SpanKind kind) { this.traceId = traceId; this.spanId = spanId; this.serviceName = serviceName; From c8f8b2c97241e38639f4f381689d532b570c1f3b Mon Sep 17 00:00:00 2001 From: qchea Date: Tue, 6 Apr 2021 10:00:16 -0500 Subject: [PATCH 16/18] MAINT: test data names --- .../integration/EndToEndRawSpanTest.java | 16 +++++----- .../integration/EndToEndServiceMapTest.java | 14 ++++---- .../integration/EndToEndTestSpan.java | 32 +++++++++---------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java index 64f776772..ead2c1040 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java @@ -39,19 +39,19 @@ public class EndToEndRawSpanTest { private static final int DATA_PREPPER_PORT_2 = 21891; private static final Map TEST_TRACEID_TO_TRACE_GROUP = new HashMap() {{ - put(Hex.toHexString(EndToEndTestSpan.DATA_100.traceId.getBytes()), EndToEndTestSpan.DATA_100.name); - put(Hex.toHexString(EndToEndTestSpan.DATA_101.traceId.getBytes()), EndToEndTestSpan.DATA_101.name); + put(Hex.toHexString(EndToEndTestSpan.TRACE_1_ROOT_SPAN.traceId.getBytes()), EndToEndTestSpan.TRACE_1_ROOT_SPAN.name); + put(Hex.toHexString(EndToEndTestSpan.TRACE_2_ROOT_SPAN.traceId.getBytes()), EndToEndTestSpan.TRACE_2_ROOT_SPAN.name); }}; private static final List TEST_DATA_SET_1_WITH_ROOT_SPAN = Arrays.asList( - EndToEndTestSpan.DATA_100, EndToEndTestSpan.DATA_200, EndToEndTestSpan.DATA_300, - EndToEndTestSpan.DATA_400, EndToEndTestSpan.DATA_500, EndToEndTestSpan.DATA_600); + EndToEndTestSpan.TRACE_1_ROOT_SPAN, EndToEndTestSpan.TRACE_1_SPAN_2, EndToEndTestSpan.TRACE_1_SPAN_3, + EndToEndTestSpan.TRACE_1_SPAN_4, EndToEndTestSpan.TRACE_1_SPAN_5, EndToEndTestSpan.TRACE_1_SPAN_6); private static final List TEST_DATA_SET_1_WITHOUT_ROOT_SPAN = Arrays.asList( - EndToEndTestSpan.DATA_700, EndToEndTestSpan.DATA_800, EndToEndTestSpan.DATA_900, - EndToEndTestSpan.DATA_1000, EndToEndTestSpan.DATA_1100); + EndToEndTestSpan.TRACE_1_SPAN_7, EndToEndTestSpan.TRACE_1_SPAN_8, EndToEndTestSpan.TRACE_1_SPAN_9, + EndToEndTestSpan.TRACE_1_SPAN_10, EndToEndTestSpan.TRACE_1_SPAN_11); private static final List TEST_DATA_SET_2_WITH_ROOT_SPAN = Arrays.asList( - EndToEndTestSpan.DATA_101, EndToEndTestSpan.DATA_201, EndToEndTestSpan.DATA_301); + EndToEndTestSpan.TRACE_2_ROOT_SPAN, EndToEndTestSpan.TRACE_2_SPAN_2, EndToEndTestSpan.TRACE_2_SPAN_3); private static final List TEST_DATA_SET_2_WITHOUT_ROOT_SPAN = Arrays.asList( - EndToEndTestSpan.DATA_401, EndToEndTestSpan.DATA_501); + EndToEndTestSpan.TRACE_2_SPAN_4, EndToEndTestSpan.TRACE_2_SPAN_5); private static final String INDEX_NAME = "otel-v1-apm-span-000001"; @Test diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java index 51861c5c8..1c3240b90 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java @@ -42,14 +42,14 @@ public class EndToEndServiceMapTest { private static final int DATA_PREPPER_PORT_1 = 21890; private static final int DATA_PREPPER_PORT_2 = 21891; private static final List TEST_DATA_SET_11 = Arrays.asList( - EndToEndTestSpan.DATA_100, EndToEndTestSpan.DATA_200, EndToEndTestSpan.DATA_500, EndToEndTestSpan.DATA_600, - EndToEndTestSpan.DATA_700, EndToEndTestSpan.DATA_1000); + EndToEndTestSpan.TRACE_1_ROOT_SPAN, EndToEndTestSpan.TRACE_1_SPAN_2, EndToEndTestSpan.TRACE_1_SPAN_5, + EndToEndTestSpan.TRACE_1_SPAN_6, EndToEndTestSpan.TRACE_1_SPAN_7, EndToEndTestSpan.TRACE_1_SPAN_10); private static final List TEST_DATA_SET_12 = Arrays.asList( - EndToEndTestSpan.DATA_300, EndToEndTestSpan.DATA_400, EndToEndTestSpan.DATA_800, - EndToEndTestSpan.DATA_900, EndToEndTestSpan.DATA_1100); - private static final List TEST_DATA_SET_21 = Arrays.asList( - EndToEndTestSpan.DATA_101, EndToEndTestSpan.DATA_201, EndToEndTestSpan.DATA_401, EndToEndTestSpan.DATA_501); - private static final List TEST_DATA_SET_22 = Collections.singletonList(EndToEndTestSpan.DATA_301); + EndToEndTestSpan.TRACE_1_SPAN_3, EndToEndTestSpan.TRACE_1_SPAN_4, EndToEndTestSpan.TRACE_1_SPAN_8, + EndToEndTestSpan.TRACE_1_SPAN_9, EndToEndTestSpan.TRACE_1_SPAN_11); + private static final List TEST_DATA_SET_21 = Arrays.asList(EndToEndTestSpan.TRACE_2_ROOT_SPAN, + EndToEndTestSpan.TRACE_2_SPAN_2, EndToEndTestSpan.TRACE_2_SPAN_4, EndToEndTestSpan.TRACE_2_SPAN_5); + private static final List TEST_DATA_SET_22 = Collections.singletonList(EndToEndTestSpan.TRACE_2_SPAN_3); private static final String SERVICE_MAP_INDEX_NAME = "otel-v1-apm-service-map"; @Test diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestSpan.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestSpan.java index 0f6d0270b..58abea1ca 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestSpan.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestSpan.java @@ -4,23 +4,23 @@ public enum EndToEndTestSpan { - DATA_100("TRACE_1", "100", "ServiceA", null, "FRUITS", Span.SpanKind.SPAN_KIND_INTERNAL), - DATA_200("TRACE_1", "200", "ServiceA", "100", "CALL_SERVICE_B_APPLE", Span.SpanKind.SPAN_KIND_CLIENT), - DATA_300("TRACE_1", "300", "ServiceB", "200", "/APPLE", Span.SpanKind.SPAN_KIND_SERVER), - DATA_400("TRACE_1", "400", "ServiceB", "300", "CALL_SERVICE_C_ORANGE", Span.SpanKind.SPAN_KIND_CLIENT), - DATA_500("TRACE_1", "500", "ServiceC", "400", "/ORANGE", Span.SpanKind.SPAN_KIND_SERVER), - DATA_600("TRACE_1", "600", "ServiceC", "500", "SOME_INTERNAL", Span.SpanKind.SPAN_KIND_INTERNAL), - DATA_700("TRACE_1", "700", "ServiceA", "100", "CALL_SERVICE_B_JACKFRUIT", Span.SpanKind.SPAN_KIND_CLIENT), - DATA_800("TRACE_1", "800", "ServiceB", "700", "/JACKFRUIT", Span.SpanKind.SPAN_KIND_SERVER), - DATA_900("TRACE_1", "900", "ServiceB", "800", "SOME_INTERNAL", Span.SpanKind.SPAN_KIND_INTERNAL), - DATA_1000("TRACE_1", "1000", "ServiceA", "100", "CALL_SERVICE_D_PEAR", Span.SpanKind.SPAN_KIND_CLIENT), - DATA_1100("TRACE_1", "1100", "ServiceD", "1000", "/PEAR", Span.SpanKind.SPAN_KIND_SERVER), + TRACE_1_ROOT_SPAN("TRACE_1", "100", "ServiceA", null, "FRUITS", Span.SpanKind.SPAN_KIND_INTERNAL), + TRACE_1_SPAN_2("TRACE_1", "200", "ServiceA", "100", "CALL_SERVICE_B_APPLE", Span.SpanKind.SPAN_KIND_CLIENT), + TRACE_1_SPAN_3("TRACE_1", "300", "ServiceB", "200", "/APPLE", Span.SpanKind.SPAN_KIND_SERVER), + TRACE_1_SPAN_4("TRACE_1", "400", "ServiceB", "300", "CALL_SERVICE_C_ORANGE", Span.SpanKind.SPAN_KIND_CLIENT), + TRACE_1_SPAN_5("TRACE_1", "500", "ServiceC", "400", "/ORANGE", Span.SpanKind.SPAN_KIND_SERVER), + TRACE_1_SPAN_6("TRACE_1", "600", "ServiceC", "500", "SOME_INTERNAL", Span.SpanKind.SPAN_KIND_INTERNAL), + TRACE_1_SPAN_7("TRACE_1", "700", "ServiceA", "100", "CALL_SERVICE_B_JACKFRUIT", Span.SpanKind.SPAN_KIND_CLIENT), + TRACE_1_SPAN_8("TRACE_1", "800", "ServiceB", "700", "/JACKFRUIT", Span.SpanKind.SPAN_KIND_SERVER), + TRACE_1_SPAN_9("TRACE_1", "900", "ServiceB", "800", "SOME_INTERNAL", Span.SpanKind.SPAN_KIND_INTERNAL), + TRACE_1_SPAN_10("TRACE_1", "1000", "ServiceA", "100", "CALL_SERVICE_D_PEAR", Span.SpanKind.SPAN_KIND_CLIENT), + TRACE_1_SPAN_11("TRACE_1", "1100", "ServiceD", "1000", "/PEAR", Span.SpanKind.SPAN_KIND_SERVER), - DATA_101("TRACE_2", "101", "ServiceA", null, "VEGGIES", Span.SpanKind.SPAN_KIND_INTERNAL), - DATA_201("TRACE_2", "201", "ServiceA", "101", "CALL_SERVICE_B_ONION", Span.SpanKind.SPAN_KIND_CLIENT), - DATA_301("TRACE_2", "301", "ServiceB", "201", "/ONION", Span.SpanKind.SPAN_KIND_SERVER), - DATA_401("TRACE_2", "401", "ServiceA", "101", "CALL_SERVICE_E_POTATO", Span.SpanKind.SPAN_KIND_CLIENT), - DATA_501("TRACE_2", "501", "ServiceE", "401", "/POTATO", Span.SpanKind.SPAN_KIND_SERVER); + TRACE_2_ROOT_SPAN("TRACE_2", "101", "ServiceA", null, "VEGGIES", Span.SpanKind.SPAN_KIND_INTERNAL), + TRACE_2_SPAN_2("TRACE_2", "201", "ServiceA", "101", "CALL_SERVICE_B_ONION", Span.SpanKind.SPAN_KIND_CLIENT), + TRACE_2_SPAN_3("TRACE_2", "301", "ServiceB", "201", "/ONION", Span.SpanKind.SPAN_KIND_SERVER), + TRACE_2_SPAN_4("TRACE_2", "401", "ServiceA", "101", "CALL_SERVICE_E_POTATO", Span.SpanKind.SPAN_KIND_CLIENT), + TRACE_2_SPAN_5("TRACE_2", "501", "ServiceE", "401", "/POTATO", Span.SpanKind.SPAN_KIND_SERVER); public final String traceId; public final String spanId; From c4015c3d9cf863a6d646e61c568fedc0927caef3 Mon Sep 17 00:00:00 2001 From: qchea Date: Tue, 6 Apr 2021 10:12:28 -0500 Subject: [PATCH 17/18] Variable rename --- .../integration/EndToEndRawSpanTest.java | 16 +++++++-------- .../integration/EndToEndServiceMapTest.java | 20 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java index ead2c1040..008b4b3ae 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java @@ -42,15 +42,15 @@ public class EndToEndRawSpanTest { put(Hex.toHexString(EndToEndTestSpan.TRACE_1_ROOT_SPAN.traceId.getBytes()), EndToEndTestSpan.TRACE_1_ROOT_SPAN.name); put(Hex.toHexString(EndToEndTestSpan.TRACE_2_ROOT_SPAN.traceId.getBytes()), EndToEndTestSpan.TRACE_2_ROOT_SPAN.name); }}; - private static final List TEST_DATA_SET_1_WITH_ROOT_SPAN = Arrays.asList( + private static final List TEST_SPAN_SET_1_WITH_ROOT_SPAN = Arrays.asList( EndToEndTestSpan.TRACE_1_ROOT_SPAN, EndToEndTestSpan.TRACE_1_SPAN_2, EndToEndTestSpan.TRACE_1_SPAN_3, EndToEndTestSpan.TRACE_1_SPAN_4, EndToEndTestSpan.TRACE_1_SPAN_5, EndToEndTestSpan.TRACE_1_SPAN_6); - private static final List TEST_DATA_SET_1_WITHOUT_ROOT_SPAN = Arrays.asList( + private static final List TEST_SPAN_SET_1_WITHOUT_ROOT_SPAN = Arrays.asList( EndToEndTestSpan.TRACE_1_SPAN_7, EndToEndTestSpan.TRACE_1_SPAN_8, EndToEndTestSpan.TRACE_1_SPAN_9, EndToEndTestSpan.TRACE_1_SPAN_10, EndToEndTestSpan.TRACE_1_SPAN_11); - private static final List TEST_DATA_SET_2_WITH_ROOT_SPAN = Arrays.asList( + private static final List TEST_SPAN_SET_2_WITH_ROOT_SPAN = Arrays.asList( EndToEndTestSpan.TRACE_2_ROOT_SPAN, EndToEndTestSpan.TRACE_2_SPAN_2, EndToEndTestSpan.TRACE_2_SPAN_3); - private static final List TEST_DATA_SET_2_WITHOUT_ROOT_SPAN = Arrays.asList( + private static final List TEST_SPAN_SET_2_WITHOUT_ROOT_SPAN = Arrays.asList( EndToEndTestSpan.TRACE_2_SPAN_4, EndToEndTestSpan.TRACE_2_SPAN_5); private static final String INDEX_NAME = "otel-v1-apm-span-000001"; @@ -58,16 +58,16 @@ public class EndToEndRawSpanTest { public void testPipelineEndToEnd() throws InterruptedException { //Send data to otel trace source final ExportTraceServiceRequest exportTraceServiceRequestTrace1BatchWithRoot = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_DATA_SET_1_WITH_ROOT_SPAN) + getResourceSpansBatch(TEST_SPAN_SET_1_WITH_ROOT_SPAN) ); final ExportTraceServiceRequest exportTraceServiceRequestTrace1BatchNoRoot = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_DATA_SET_1_WITHOUT_ROOT_SPAN) + getResourceSpansBatch(TEST_SPAN_SET_1_WITHOUT_ROOT_SPAN) ); final ExportTraceServiceRequest exportTraceServiceRequestTrace2BatchWithRoot = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_DATA_SET_2_WITH_ROOT_SPAN) + getResourceSpansBatch(TEST_SPAN_SET_2_WITH_ROOT_SPAN) ); final ExportTraceServiceRequest exportTraceServiceRequestTrace2BatchNoRoot = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_DATA_SET_2_WITHOUT_ROOT_SPAN) + getResourceSpansBatch(TEST_SPAN_SET_2_WITHOUT_ROOT_SPAN) ); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequestTrace1BatchWithRoot); diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java index 1c3240b90..d2dbb3f55 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java @@ -41,32 +41,32 @@ public class EndToEndServiceMapTest { private static final String TEST_TRACEID_2 = "CBA"; private static final int DATA_PREPPER_PORT_1 = 21890; private static final int DATA_PREPPER_PORT_2 = 21891; - private static final List TEST_DATA_SET_11 = Arrays.asList( + private static final List TEST_TRACE_1_BATCH_1 = Arrays.asList( EndToEndTestSpan.TRACE_1_ROOT_SPAN, EndToEndTestSpan.TRACE_1_SPAN_2, EndToEndTestSpan.TRACE_1_SPAN_5, EndToEndTestSpan.TRACE_1_SPAN_6, EndToEndTestSpan.TRACE_1_SPAN_7, EndToEndTestSpan.TRACE_1_SPAN_10); - private static final List TEST_DATA_SET_12 = Arrays.asList( + private static final List TEST_TRACE_1_BATCH_2 = Arrays.asList( EndToEndTestSpan.TRACE_1_SPAN_3, EndToEndTestSpan.TRACE_1_SPAN_4, EndToEndTestSpan.TRACE_1_SPAN_8, EndToEndTestSpan.TRACE_1_SPAN_9, EndToEndTestSpan.TRACE_1_SPAN_11); - private static final List TEST_DATA_SET_21 = Arrays.asList(EndToEndTestSpan.TRACE_2_ROOT_SPAN, + private static final List TEST_TRACE_2_BATCH_1 = Arrays.asList(EndToEndTestSpan.TRACE_2_ROOT_SPAN, EndToEndTestSpan.TRACE_2_SPAN_2, EndToEndTestSpan.TRACE_2_SPAN_4, EndToEndTestSpan.TRACE_2_SPAN_5); - private static final List TEST_DATA_SET_22 = Collections.singletonList(EndToEndTestSpan.TRACE_2_SPAN_3); + private static final List TEST_TRACE_2_BATCH_2 = Collections.singletonList(EndToEndTestSpan.TRACE_2_SPAN_3); private static final String SERVICE_MAP_INDEX_NAME = "otel-v1-apm-service-map"; @Test public void testPipelineEndToEnd() throws IOException, InterruptedException { // Send test trace group 1 final ExportTraceServiceRequest exportTraceServiceRequest11 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_1, TEST_DATA_SET_11) + getResourceSpansBatch(TEST_TRACEID_1, TEST_TRACE_1_BATCH_1) ); final ExportTraceServiceRequest exportTraceServiceRequest12 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_1, TEST_DATA_SET_12) + getResourceSpansBatch(TEST_TRACEID_1, TEST_TRACE_1_BATCH_2) ); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequest11); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest12); //Verify data in elasticsearch sink - final List testDataSet1 = Stream.of(TEST_DATA_SET_11, TEST_DATA_SET_12) + final List testDataSet1 = Stream.of(TEST_TRACE_1_BATCH_1, TEST_TRACE_1_BATCH_2) .flatMap(Collection::stream).collect(Collectors.toList()); final List> possibleEdges = getPossibleEdges(TEST_TRACEID_1, testDataSet1); final ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder( @@ -91,16 +91,16 @@ public void testPipelineEndToEnd() throws IOException, InterruptedException { sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest12); // Send test trace group 2 final ExportTraceServiceRequest exportTraceServiceRequest21 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_2, TEST_DATA_SET_21) + getResourceSpansBatch(TEST_TRACEID_2, TEST_TRACE_2_BATCH_1) ); final ExportTraceServiceRequest exportTraceServiceRequest22 = getExportTraceServiceRequest( - getResourceSpansBatch(TEST_TRACEID_2, TEST_DATA_SET_22) + getResourceSpansBatch(TEST_TRACEID_2, TEST_TRACE_2_BATCH_2) ); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_1, exportTraceServiceRequest21); sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequest22); - final List testDataSet2 = Stream.of(TEST_DATA_SET_21, TEST_DATA_SET_22) + final List testDataSet2 = Stream.of(TEST_TRACE_2_BATCH_1, TEST_TRACE_2_BATCH_2) .flatMap(Collection::stream).collect(Collectors.toList()); possibleEdges.addAll(getPossibleEdges(TEST_TRACEID_2, testDataSet2)); // Wait for service map prepper by 2 * window_duration From 0806774d38fa15aff849e3aa32f98e4cabf7f148 Mon Sep 17 00:00:00 2001 From: qchea Date: Thu, 8 Apr 2021 14:15:13 -0500 Subject: [PATCH 18/18] MAINT: arg name --- .../integration/EndToEndRawSpanTest.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java index 008b4b3ae..08a8911bd 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java +++ b/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java @@ -164,15 +164,15 @@ public static ExportTraceServiceRequest getExportTraceServiceRequest(List getResourceSpansBatch(final List dataList) { + private List getResourceSpansBatch(final List testSpanList) { final ArrayList spansList = new ArrayList<>(); - for(final EndToEndTestSpan data : dataList) { - final String traceId = data.traceId; - final String parentId = data.parentId; - final String spanId = data.spanId; - final String serviceName = data.serviceName; - final String spanName = data.name; - final Span.SpanKind spanKind = data.spanKind; + for(final EndToEndTestSpan testSpan : testSpanList) { + final String traceId = testSpan.traceId; + final String parentId = testSpan.parentId; + final String spanId = testSpan.spanId; + final String serviceName = testSpan.serviceName; + final String spanName = testSpan.name; + final Span.SpanKind spanKind = testSpan.spanKind; final ResourceSpans rs = getResourceSpans( serviceName, spanName,