Skip to content

Commit

Permalink
fix: eclipse-edc#86 sample 04 terminating on startup (eclipse-edc#128)
Browse files Browse the repository at this point in the history
* fix: working sample 04

* test: added test for docker compose sample

* style: cosmetic changes

* test: temporarily disable progress checking of transfer

* docs: added explanation for commented line

* test: download OpenTelemetry Jar before test execution

* gradle: download OpenTelemetry before build step

* removing container name from prometheus as it's not supported by testcontainers

* fixed typo in libs.versions.toml

* copy opentelemetry jar from gradle cache

* fixed default sample tests for sample 04

* fixed build process

* asserting Jaeger state OK in telemetry test

* fix: fixing wrong testcontainer reference in .toml file

* fix: running gradle build before test

* fix: removing gradle build before test for github action

---------

Co-authored-by: Matthias De Geyter <[email protected]>
  • Loading branch information
2 people authored and majadlymhmd committed Nov 15, 2023
1 parent db3387c commit 2ea2768
Show file tree
Hide file tree
Showing 12 changed files with 212 additions and 67 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/verify.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ jobs:
- name: End to End Integration Tests
uses: ./.github/actions/run-tests
with:
command: ./gradlew test -DincludeTags="EndToEndTest"
command:
- ./gradlew test -DincludeTags="EndToEndTest"

Upload-Test-Report:
needs:
Expand Down
10 changes: 6 additions & 4 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,8 @@ okhttp-mockwebserver = "5.0.0-alpha.11"
openTelemetry = "1.18.0"
restAssured = "5.3.2"
rsApi = "3.1.0"
testcontainers = "1.19.1"
kafkaClients = "3.6.0"
testContainers = "1.19.1"


[libraries]
assertj = { module = "org.assertj:assertj-core", version.ref = "assertj" }
Expand Down Expand Up @@ -69,9 +68,12 @@ junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "jun
okhttp-mockwebserver = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "okhttp-mockwebserver" }
opentelemetry-annotations = { module = "io.opentelemetry:opentelemetry-extension-annotations", version.ref = "openTelemetry" }
restAssured = { module = "io.rest-assured:rest-assured", version.ref = "restAssured" }
testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" }
testcontainers-junit-jupiter = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" }
kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "kafkaClients" }
testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testContainers" }
testcontainers-junit = { module = "org.testcontainers:junit-jupiter", version.ref = "testContainers" }
testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testcontainers" }
testcontainers-junit = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" }
opentelemetry = "io.opentelemetry.javaagent:opentelemetry-javaagent:1.30.0"

[plugins]
shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" }
3 changes: 3 additions & 0 deletions system-tests/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ dependencies {
testImplementation(libs.awaitility)
testImplementation(libs.okhttp.mockwebserver)
testImplementation(libs.restAssured)
testImplementation(libs.testcontainers)
testImplementation(libs.testcontainers.junit)
testImplementation(libs.testcontainers.kafka)
testImplementation(libs.kafka.clients)
Expand All @@ -36,6 +37,8 @@ dependencies {
testCompileOnly(project(":transfer:transfer-01-file-transfer:file-transfer-provider"))
testCompileOnly(project(":transfer:transfer-02-file-transfer-listener:file-transfer-listener-consumer"))
testCompileOnly(project(":transfer:transfer-03-modify-transferprocess:modify-transferprocess-consumer"))
testCompileOnly(project(":transfer:transfer-04-open-telemetry:open-telemetry-consumer"))
testCompileOnly(project(":transfer:transfer-04-open-telemetry:open-telemetry-provider"))
testCompileOnly(project(":transfer:streaming:streaming-01-http-to-http:streaming-01-runtime"))
testCompileOnly(project(":transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime"))
}
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public class FileTransferSampleTestCommon {
final String sampleAssetFilePath;
final File sampleAssetFile;
final File destinationFile;
Duration timeout = Duration.ofSeconds(30);
Duration timeout = Duration.ofSeconds(15);
Duration pollInterval = Duration.ofMillis(500);

String contractNegotiationId;
Expand Down Expand Up @@ -133,10 +133,15 @@ void assertTransferProcessStatusConsumerSide(String transferProcessId) {
* This method corresponds to the command in the sample: {@code curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/contractoffer.json "http://localhost:9192/management/v2/contractnegotiations"}
*/
void initiateContractNegotiation() {
initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH);
}


void initiateContractNegotiation(String contractOfferFilePath) {
contractNegotiationId = given()
.headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE)
.contentType(ContentType.JSON)
.body(new File(TestUtils.findBuildRoot(), CONTRACT_OFFER_FILE_PATH))
.body(new File(TestUtils.findBuildRoot(), contractOfferFilePath))
.when()
.post(MANAGEMENT_API_URL + "/v2/contractnegotiations")
.then()
Expand Down Expand Up @@ -181,8 +186,8 @@ void lookUpContractAgreementId() {
*
* @throws IOException Thrown if there was an error accessing the transfer request file defined in {@link FileTransferSampleTestCommon#TRANSFER_FILE_PATH}.
*/
String requestTransferFile() throws IOException {
var transferJsonFile = getFileFromRelativePath(TRANSFER_FILE_PATH);
String requestTransferFile(String transferFilePath) throws IOException {
var transferJsonFile = getFileFromRelativePath(transferFilePath);
var requestBody = readAndUpdateDataRequestFromJsonFile(transferJsonFile, contractAgreementId);

var jsonPath = given()
Expand All @@ -205,6 +210,10 @@ String requestTransferFile() throws IOException {
return transferProcessId;
}

String requestTransferFile() throws IOException {
return requestTransferFile(TRANSFER_FILE_PATH);
}

/**
* Reads a transfer request file with changed value for contract agreement ID and file destination path.
*
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
* Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0
*
* SPDX-License-Identifier: Apache-2.0
*
* Contributors:
* Mercedes-Benz Tech Innovation GmbH - Sample workflow test
*
*/

package org.eclipse.edc.samples.transfer;

import org.apache.http.HttpStatus;
import org.eclipse.edc.junit.annotations.EndToEndTest;
import org.junit.ClassRule;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.DockerComposeContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.junit.jupiter.Testcontainers;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.fail;

@EndToEndTest
@Testcontainers
public class Transfer04openTelemetryTest {

private static final String SAMPLE_FOLDER = "transfer/transfer-04-open-telemetry";
private static final String DOCKER_COMPOSE_YAML = "/docker-compose.yaml";
private static final String SAMPLE_ASSET_FILE_PATH = SAMPLE_FOLDER + "/README.md";
private static final String DESTINATION_FILE_PATH = SAMPLE_FOLDER + "/README_transferred.md";
private static final String CONTRACT_OFFER_FILE_PATH = SAMPLE_FOLDER + "/contractoffer.json";
private static final String FILE_TRANSFER_FILE_PATH = SAMPLE_FOLDER + "/filetransfer.json";
private static final String JAEGER_URL = "http://localhost:16686";

private final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH);

@ClassRule
public static DockerComposeContainer environment =
new DockerComposeContainer(FileTransferSampleTestCommon.getFileFromRelativePath(SAMPLE_FOLDER + DOCKER_COMPOSE_YAML))
.withLocalCompose(true)
.waitingFor("consumer", Wait.forLogMessage(".*ready.*", 1));

@BeforeAll
static void setUp() {
environment.start();
}

@Test
void runSampleSteps() throws Exception {
testUtils.assertTestPrerequisites();
testUtils.initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH);
testUtils.lookUpContractAgreementId();
var transferProcessId = testUtils.requestTransferFile(FILE_TRANSFER_FILE_PATH);
testUtils.assertDestinationFileContent();
testUtils.assertTransferProcessStatusConsumerSide(transferProcessId);
assertJaegerState();
}

private void assertJaegerState() {
try {
var url = new URL(JAEGER_URL);
var huc = (HttpURLConnection) url.openConnection();
assertThat(huc.getResponseCode()).isEqualTo(HttpStatus.SC_OK);
} catch (IOException e) {
fail("Unable to assert Jaeger state", e);
}
}

@AfterEach
protected void tearDown() {
testUtils.cleanTemporaryTestFiles();
}
}
7 changes: 1 addition & 6 deletions transfer/transfer-04-open-telemetry/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,7 @@ several popular [libraries and frameworks](https://github.com/open-telemetry/ope

In order to visualize and analyze the traces and metrics, we use
[OpenTelemetry exporters](https://opentelemetry.io/docs/instrumentation/js/exporters/) to export data into the Jaeger
tracing backend and a Prometheus endpoint.

## Prerequisites

Download the [opentelemetry-javaagent.jar](https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.12.0/opentelemetry-javaagent.jar)
and place it in the root folder of this sample.
tracing backend and a Prometheus endpoint.

## Run the sample

Expand Down
47 changes: 14 additions & 33 deletions transfer/transfer-04-open-telemetry/contractoffer.json
Original file line number Diff line number Diff line change
@@ -1,43 +1,24 @@
{
"@context": {
"edc": "https://w3id.org/edc/v0.0.1/ns/",
"odrl": "http://www.w3.org/ns/odrl/2/"
},
"@type": "NegotiationInitiateRequestDto",
"connectorId": "provider",
"consumerId": "consumer",
"providerId": "provider",
"connectorAddress": "http://provider:8282/protocol",
"protocol": "dataspace-protocol-http",
"offer": {
"offerId": "1:3a75736e-001d-4364-8bd4-9888490edb58",
"offerId": "1:test-document:3a75736e-001d-4364-8bd4-9888490edb58",
"assetId": "test-document",
"policy": {
"uid": "956e172f-2de1-4501-8881-057a57fd0e69",
"permissions": [
{
"edctype": "dataspaceconnector:permission",
"uid": null,
"target": "test-document",
"action": {
"type": "USE",
"includedIn": null,
"constraint": null
},
"assignee": null,
"assigner": null,
"constraints": [],
"duties": []
}
],
"prohibitions": [],
"obligations": [],
"extensibleProperties": {},
"inheritsFrom": null,
"assigner": null,
"assignee": null,
"target": null,
"@type": {
"@policytype": "set"
}
},
"asset": {
"properties": {
"asset:prop:id": "test-document"
}
"@id": "1:test-document:13dce0f1-52ed-4554-a194-e83e92733ee5",
"@type": "set",
"odrl:permission": [],
"odrl:prohibition": [],
"odrl:obligation": [],
"odrl:target": "test-document"
}
}
}
17 changes: 12 additions & 5 deletions transfer/transfer-04-open-telemetry/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ services:
consumer:
image: openjdk:17-jdk-slim-buster
environment:
EDC_HOSTNAME: consumer
OTEL_SERVICE_NAME: consumer
OTEL_TRACES_EXPORTER: jaeger
OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250
Expand All @@ -13,9 +14,12 @@ services:
WEB_HTTP_PATH: /api
WEB_HTTP_MANAGEMENT_PORT: 9192
WEB_HTTP_MANAGEMENT_PATH: /management
WEB_HTTP_DSP_PORT: 9292
WEB_HTTP_PROTOCOL_PORT: 9292
WEB_HTTP_PROTOCOL_PATH: /protocol
EDC_DSP_CALLBACK_ADDRESS: http://consumer:9292/protocol
EDC_PARTICIPANT_ID: consumer
WEB_HTTP_DSP_PORT: 9393
WEB_HTTP_DSP_PATH: /protocol
DSP_WEBHOOK_ADDRESS: http://consumer:9292
EDC_API_AUTH_KEY: password
volumes:
- ../:/samples
Expand All @@ -30,15 +34,19 @@ services:
provider:
image: openjdk:17-jdk-slim-buster
environment:
EDC_HOSTNAME: provider
OTEL_SERVICE_NAME: provider
OTEL_TRACES_EXPORTER: jaeger
OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250
WEB_HTTP_PORT: 8181
WEB_HTTP_PATH: /api
WEB_HTTP_MANAGEMENT_PORT: 8182
WEB_HTTP_MANAGEMENT_PATH: /management
DSP_WEBHOOK_ADDRESS: http://provider:8282
EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /samples/transfer-04-open-telemetry/input-file.txt
WEB_HTTP_PROTOCOL_PORT: 8282
WEB_HTTP_PROTOCOL_PATH: /protocol
EDC_DSP_CALLBACK_ADDRESS: http://provider:8282/protocol
EDC_PARTICIPANT_ID: provider
EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /samples/transfer-04-open-telemetry/README.md
volumes:
- ../:/samples
ports:
Expand All @@ -56,7 +64,6 @@ services:

prometheus:
image: prom/prometheus:v2.30.3
container_name: prometheus
volumes:
- ./prometheus/:/etc/prometheus/
ports:
Expand Down
21 changes: 11 additions & 10 deletions transfer/transfer-04-open-telemetry/filetransfer.json
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
{
"@context": {
"edc": "https://w3id.org/edc/v0.0.1/ns/"
},
"@type": "TransferRequestDto",
"dataDestination": {
"type": "File",
"path": "/samples/transfer-04-open-telemetry/README_transferred.md",
"keyName": "keyName"
},
"protocol": "dataspace-protocol-http",
"assetId": "test-document",
"contractId": "{agreement ID}",
"dataDestination": {
"path": "/samples/output-file.txt",
"keyName": "keyName",
"type": "File"
},
"transferType": {
"contentType": "application/octet-stream",
"isFinite": true
},
"connectorId": "provider",
"connectorAddress": "http://provider:8282/protocol",
"connectorId": "consumer"
"privateProperties": {}
}
1 change: 0 additions & 1 deletion transfer/transfer-04-open-telemetry/input-file.txt

This file was deleted.

Loading

0 comments on commit 2ea2768

Please sign in to comment.