Skip to content
This repository has been archived by the owner on Feb 15, 2022. It is now read-only.

TST: e2e with trace group propagation #480

Merged
Show file tree
Hide file tree
Changes from 18 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 69 additions & 55 deletions data-prepper-core/integrationTest.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ task removeDataPrepperNetwork(type: DockerRemoveNetwork) {
networkId = createDataPrepperNetwork.getNetworkId()
}

def RAW_SPAN_PIPELINE_YAML = "raw-span-e2e-pipeline.yml"
def SERVICE_MAP_PIPELINE_YAML = "service-map-e2e-pipeline.yml"

/**
* DataPrepper Docker tasks
*/
Expand All @@ -59,9 +62,10 @@ task createDataPrepperDockerFile(type: Dockerfile) {
exposePort(4900)
workingDir("/app")
copyFile("build/libs/${jar.archiveName}", "/app/data-prepper.jar")
copyFile("src/integrationTest/resources/pipeline.yml", "/app/pipeline.yml")
copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_YAML}", "/app/${RAW_SPAN_PIPELINE_YAML}")
copyFile("src/integrationTest/resources/${SERVICE_MAP_PIPELINE_YAML}", "/app/${SERVICE_MAP_PIPELINE_YAML}")
copyFile("src/integrationTest/resources/data_prepper.yml", "/app/data_prepper.yml")
defaultCommand("java", "-jar", "data-prepper.jar", "/app/pipeline.yml", "/app/data_prepper.yml")
defaultCommand("java", "-jar", "data-prepper.jar", "/app/${RAW_SPAN_PIPELINE_YAML}", "/app/data_prepper.yml")
}

task buildDataPrepperDockerImage(type: DockerBuildImage) {
Expand All @@ -71,54 +75,36 @@ task buildDataPrepperDockerImage(type: DockerBuildImage) {
images.add("integ-test-pipeline-image")
}

task createDataPrepper1DockerContainer(type: DockerCreateContainer) {
dependsOn buildDataPrepperDockerImage
dependsOn createDataPrepperNetwork
containerName = "data-prepper1"
hostConfig.portBindings = ['21890:21890', '4900:4900']
hostConfig.network = createDataPrepperNetwork.getNetworkName()
targetImageId buildDataPrepperDockerImage.getImageId()
}

task createDataPrepper2DockerContainer(type: DockerCreateContainer) {
dependsOn buildDataPrepperDockerImage
dependsOn createDataPrepperNetwork
containerName = "data-prepper2"
hostConfig.portBindings = ['21891:21890', '4901:4900']
hostConfig.network = createDataPrepperNetwork.getNetworkName()
targetImageId buildDataPrepperDockerImage.getImageId()
}

task startDataPrepper1DockerContainer(type: DockerStartContainer) {
dependsOn createDataPrepper1DockerContainer
targetContainerId createDataPrepper1DockerContainer.getContainerId()
doLast {
sleep(10*1000)
def createDataPrepperDockerContainer(final String taskBaseName, final String dataPrepperName, final int grpcPort,
final int serverPort, final String pipelineConfigYAML) {
wrijeff marked this conversation as resolved.
Show resolved Hide resolved
return tasks.create("create${taskBaseName}", DockerCreateContainer) {
dependsOn buildDataPrepperDockerImage
dependsOn createDataPrepperNetwork
containerName = dataPrepperName
hostConfig.portBindings = ['%d:21890'.formatted(grpcPort), '%d:4900'.formatted(serverPort)]
hostConfig.network = createDataPrepperNetwork.getNetworkName()
cmd = ["java", "-jar", "data-prepper.jar", pipelineConfigYAML, "/app/data_prepper.yml"]
targetImageId buildDataPrepperDockerImage.getImageId()
}
}

task startDataPrepper2DockerContainer(type: DockerStartContainer) {
dependsOn createDataPrepper2DockerContainer
targetContainerId createDataPrepper2DockerContainer.getContainerId()
doLast {
sleep(10*1000)
def startDataPrepperDockerContainer(final DockerCreateContainer createDataPrepperDockerContainerTask) {
return tasks.create("start${createDataPrepperDockerContainerTask.getName()}", DockerStartContainer) {
dependsOn createDataPrepperDockerContainerTask
targetContainerId createDataPrepperDockerContainerTask.getContainerId()
}
}

task stopDataPrepper1DockerContainer(type: DockerStopContainer) {
targetContainerId createDataPrepper1DockerContainer.getContainerId()
}

task stopDataPrepper2DockerContainer(type: DockerStopContainer) {
targetContainerId createDataPrepper2DockerContainer.getContainerId()
}

task removeDataPrepper1DockerContainer(type: DockerRemoveContainer) {
targetContainerId stopDataPrepper1DockerContainer.getContainerId()
def stopDataPrepperDockerContainer(final DockerStartContainer startDataPrepperDockerContainerTask) {
return tasks.create("stop${startDataPrepperDockerContainerTask.getName()}", DockerStopContainer) {
targetContainerId startDataPrepperDockerContainerTask.getContainerId()
}
}

task removeDataPrepper2DockerContainer(type: DockerRemoveContainer) {
targetContainerId stopDataPrepper2DockerContainer.getContainerId()
def removeDataPrepperDockerContainer(final DockerStopContainer stopDataPrepperDockerContainerTask) {
return tasks.create("remove${stopDataPrepperDockerContainerTask.getName()}", DockerRemoveContainer) {
targetContainerId stopDataPrepperDockerContainerTask.getContainerId()
}
}

/**
Expand Down Expand Up @@ -159,8 +145,20 @@ task stopOdfeDockerContainer(type: DockerStopContainer) {
task rawSpanEndToEndTest(type: Test) {
dependsOn build
dependsOn startOdfeDockerContainer
dependsOn startDataPrepper1DockerContainer
startDataPrepper1DockerContainer.mustRunAfter 'startOdfeDockerContainer'
def createDataPrepper1Task = createDataPrepperDockerContainer(
"rawSpanDataPrepper1", "data-prepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}")
def createDataPrepper2Task = createDataPrepperDockerContainer(
"rawSpanDataPrepper2", "data-prepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_YAML}")
def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer)
def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer)
dependsOn startDataPrepper1Task
dependsOn startDataPrepper2Task
startDataPrepper1Task.mustRunAfter 'startOdfeDockerContainer'
startDataPrepper2Task.mustRunAfter 'startOdfeDockerContainer'
// wait for data-preppers to be ready
doFirst {
sleep(10*1000)
}

description = 'Runs the raw span integration tests.'
group = 'verification'
Expand All @@ -172,20 +170,34 @@ task rawSpanEndToEndTest(type: Test) {
}

finalizedBy stopOdfeDockerContainer
finalizedBy stopDataPrepper1DockerContainer
finalizedBy removeDataPrepper1DockerContainer
def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer)
def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer)
finalizedBy stopDataPrepper1Task
finalizedBy stopDataPrepper2Task
finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer)
finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer)
finalizedBy removeDataPrepperNetwork
}

task serviceMapEndToEndTest(type: Test) {
dependsOn build
dependsOn startOdfeDockerContainer
dependsOn startDataPrepper1DockerContainer
dependsOn startDataPrepper2DockerContainer
startDataPrepper1DockerContainer.mustRunAfter 'startOdfeDockerContainer'
startDataPrepper2DockerContainer.mustRunAfter 'startOdfeDockerContainer'
def createDataPrepper1Task = createDataPrepperDockerContainer(
"serviceMapDataPrepper1", "data-prepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}")
def createDataPrepper2Task = createDataPrepperDockerContainer(
"serviceMapDataPrepper2", "data-prepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_YAML}")
def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer)
def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer)
dependsOn startDataPrepper1Task
dependsOn startDataPrepper2Task
startDataPrepper1Task.mustRunAfter 'startOdfeDockerContainer'
startDataPrepper2Task.mustRunAfter 'startOdfeDockerContainer'
// wait for data-preppers to be ready
doFirst {
sleep(10*1000)
}

description = 'Runs the raw span integration tests.'
description = 'Runs the service-map integration tests.'
group = 'verification'
testClassesDirs = sourceSets.integrationTest.output.classesDirs
classpath = sourceSets.integrationTest.runtimeClasspath
Expand All @@ -195,9 +207,11 @@ task serviceMapEndToEndTest(type: Test) {
}

finalizedBy stopOdfeDockerContainer
finalizedBy stopDataPrepper1DockerContainer
finalizedBy stopDataPrepper2DockerContainer
finalizedBy removeDataPrepper1DockerContainer
finalizedBy removeDataPrepper2DockerContainer
def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer)
def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer)
finalizedBy stopDataPrepper1Task
finalizedBy stopDataPrepper2Task
finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer)
finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer)
finalizedBy removeDataPrepperNetwork
}
Loading