TST: e2e with trace group propagation

data-prepper-core/integrationTest.gradle

@@ -48,6 +48,9 @@ task removeDataPrepperNetwork(type: DockerRemoveNetwork) {
     networkId = createDataPrepperNetwork.getNetworkId()
 }
 
+def RAW_SPAN_PIPELINE_YAML = "raw-span-e2e-pipeline.yml"
+def SERVICE_MAP_PIPELINE_YAML = "service-map-e2e-pipeline.yml"
+
 /**
  * DataPrepper Docker tasks
  */
@@ -59,9 +62,10 @@ task createDataPrepperDockerFile(type: Dockerfile) {
     exposePort(4900)
     workingDir("/app")
     copyFile("build/libs/${jar.archiveName}", "/app/data-prepper.jar")
-    copyFile("src/integrationTest/resources/pipeline.yml", "/app/pipeline.yml")
+    copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_YAML}", "/app/${RAW_SPAN_PIPELINE_YAML}")
+    copyFile("src/integrationTest/resources/${SERVICE_MAP_PIPELINE_YAML}", "/app/${SERVICE_MAP_PIPELINE_YAML}")
     copyFile("src/integrationTest/resources/data_prepper.yml", "/app/data_prepper.yml")
-    defaultCommand("java", "-jar", "data-prepper.jar", "/app/pipeline.yml", "/app/data_prepper.yml")
+    defaultCommand("java", "-jar", "data-prepper.jar", "/app/${RAW_SPAN_PIPELINE_YAML}", "/app/data_prepper.yml")
 }
 
 task buildDataPrepperDockerImage(type: DockerBuildImage) {
@@ -71,54 +75,36 @@ task buildDataPrepperDockerImage(type: DockerBuildImage) {
     images.add("integ-test-pipeline-image")
 }
 
-task createDataPrepper1DockerContainer(type: DockerCreateContainer) {
-    dependsOn buildDataPrepperDockerImage
-    dependsOn createDataPrepperNetwork
-    containerName = "data-prepper1"
-    hostConfig.portBindings = ['21890:21890', '4900:4900']
-    hostConfig.network = createDataPrepperNetwork.getNetworkName()
-    targetImageId buildDataPrepperDockerImage.getImageId()
-}
-
-task createDataPrepper2DockerContainer(type: DockerCreateContainer) {
-    dependsOn buildDataPrepperDockerImage
-    dependsOn createDataPrepperNetwork
-    containerName = "data-prepper2"
-    hostConfig.portBindings = ['21891:21890', '4901:4900']
-    hostConfig.network = createDataPrepperNetwork.getNetworkName()
-    targetImageId buildDataPrepperDockerImage.getImageId()
-}
-
-task startDataPrepper1DockerContainer(type: DockerStartContainer) {
-    dependsOn createDataPrepper1DockerContainer
-    targetContainerId createDataPrepper1DockerContainer.getContainerId()
-    doLast {
-        sleep(10*1000)
+def createDataPrepperDockerContainer(final String taskBaseName, final String dataPrepperName, final int grpcPort,
+                                     final int serverPort, final String pipelineConfigYAML) {
+    return tasks.create("create${taskBaseName}", DockerCreateContainer) {
+        dependsOn buildDataPrepperDockerImage
+        dependsOn createDataPrepperNetwork
+        containerName = dataPrepperName
+        hostConfig.portBindings = ['%d:21890'.formatted(grpcPort), '%d:4900'.formatted(serverPort)]
+        hostConfig.network = createDataPrepperNetwork.getNetworkName()
+        cmd = ["java", "-jar", "data-prepper.jar", pipelineConfigYAML, "/app/data_prepper.yml"]
+        targetImageId buildDataPrepperDockerImage.getImageId()
     }
 }
 
-task startDataPrepper2DockerContainer(type: DockerStartContainer) {
-    dependsOn createDataPrepper2DockerContainer
-    targetContainerId createDataPrepper2DockerContainer.getContainerId()
-    doLast {
-        sleep(10*1000)
+def startDataPrepperDockerContainer(final DockerCreateContainer createDataPrepperDockerContainerTask) {
+    return tasks.create("start${createDataPrepperDockerContainerTask.getName()}", DockerStartContainer) {
+        dependsOn createDataPrepperDockerContainerTask
+        targetContainerId createDataPrepperDockerContainerTask.getContainerId()
     }
 }
 
-task stopDataPrepper1DockerContainer(type: DockerStopContainer) {
-    targetContainerId createDataPrepper1DockerContainer.getContainerId()
-}
-
-task stopDataPrepper2DockerContainer(type: DockerStopContainer) {
-    targetContainerId createDataPrepper2DockerContainer.getContainerId()
-}
-
-task removeDataPrepper1DockerContainer(type: DockerRemoveContainer) {
-    targetContainerId stopDataPrepper1DockerContainer.getContainerId()
+def stopDataPrepperDockerContainer(final DockerStartContainer startDataPrepperDockerContainerTask) {
+    return tasks.create("stop${startDataPrepperDockerContainerTask.getName()}", DockerStopContainer) {
+        targetContainerId startDataPrepperDockerContainerTask.getContainerId()
+    }
 }
 
-task removeDataPrepper2DockerContainer(type: DockerRemoveContainer) {
-    targetContainerId stopDataPrepper2DockerContainer.getContainerId()
+def removeDataPrepperDockerContainer(final DockerStopContainer stopDataPrepperDockerContainerTask) {
+    return tasks.create("remove${stopDataPrepperDockerContainerTask.getName()}", DockerRemoveContainer) {
+        targetContainerId stopDataPrepperDockerContainerTask.getContainerId()
+    }
 }
 
 /**
@@ -159,8 +145,20 @@ task stopOdfeDockerContainer(type: DockerStopContainer) {
 task rawSpanEndToEndTest(type: Test) {
     dependsOn build
     dependsOn startOdfeDockerContainer
-    dependsOn startDataPrepper1DockerContainer
-    startDataPrepper1DockerContainer.mustRunAfter 'startOdfeDockerContainer'
+    def createDataPrepper1Task = createDataPrepperDockerContainer(
+            "rawSpanDataPrepper1", "data-prepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}")
+    def createDataPrepper2Task = createDataPrepperDockerContainer(
+            "rawSpanDataPrepper2", "data-prepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_YAML}")
+    def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer)
+    def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer)
+    dependsOn startDataPrepper1Task
+    dependsOn startDataPrepper2Task
+    startDataPrepper1Task.mustRunAfter 'startOdfeDockerContainer'
+    startDataPrepper2Task.mustRunAfter 'startOdfeDockerContainer'
+    // wait for data-preppers to be ready
+    doFirst {
+        sleep(10*1000)
+    }
 
     description = 'Runs the raw span integration tests.'
     group = 'verification'
@@ -172,20 +170,34 @@ task rawSpanEndToEndTest(type: Test) {
     }
 
     finalizedBy stopOdfeDockerContainer
-    finalizedBy stopDataPrepper1DockerContainer
-    finalizedBy removeDataPrepper1DockerContainer
+    def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer)
+    def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer)
+    finalizedBy stopDataPrepper1Task
+    finalizedBy stopDataPrepper2Task
+    finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer)
+    finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer)
     finalizedBy removeDataPrepperNetwork
 }
 
 task serviceMapEndToEndTest(type: Test) {
     dependsOn build
     dependsOn startOdfeDockerContainer
-    dependsOn startDataPrepper1DockerContainer
-    dependsOn startDataPrepper2DockerContainer
-    startDataPrepper1DockerContainer.mustRunAfter 'startOdfeDockerContainer'
-    startDataPrepper2DockerContainer.mustRunAfter 'startOdfeDockerContainer'
+    def createDataPrepper1Task = createDataPrepperDockerContainer(
+            "serviceMapDataPrepper1", "data-prepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}")
+    def createDataPrepper2Task = createDataPrepperDockerContainer(
+            "serviceMapDataPrepper2", "data-prepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_YAML}")
+    def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer)
+    def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer)
+    dependsOn startDataPrepper1Task
+    dependsOn startDataPrepper2Task
+    startDataPrepper1Task.mustRunAfter 'startOdfeDockerContainer'
+    startDataPrepper2Task.mustRunAfter 'startOdfeDockerContainer'
+    // wait for data-preppers to be ready
+    doFirst {
+        sleep(10*1000)
+    }
 
-    description = 'Runs the raw span integration tests.'
+    description = 'Runs the service-map integration tests.'
     group = 'verification'
     testClassesDirs = sourceSets.integrationTest.output.classesDirs
     classpath = sourceSets.integrationTest.runtimeClasspath
@@ -195,9 +207,11 @@ task serviceMapEndToEndTest(type: Test) {
     }
 
     finalizedBy stopOdfeDockerContainer
-    finalizedBy stopDataPrepper1DockerContainer
-    finalizedBy stopDataPrepper2DockerContainer
-    finalizedBy removeDataPrepper1DockerContainer
-    finalizedBy removeDataPrepper2DockerContainer
+    def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer)
+    def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer)
+    finalizedBy stopDataPrepper1Task
+    finalizedBy stopDataPrepper2Task
+    finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer)
+    finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer)
     finalizedBy removeDataPrepperNetwork
 }