From c5cb4969b4265b6e2f70413fea2a8b3dd784f0e7 Mon Sep 17 00:00:00 2001 From: Matthias De Geyter Date: Tue, 25 Jul 2023 16:48:08 +0200 Subject: [PATCH 01/16] fix: working sample 04 --- .../contractoffer.json | 49 ++++++------------- .../docker-compose.yaml | 12 +++-- .../filetransfer.json | 23 ++++----- .../open-telemetry-consumer/build.gradle.kts | 1 + .../open-telemetry-provider/build.gradle.kts | 1 + 5 files changed, 36 insertions(+), 50 deletions(-) diff --git a/transfer/transfer-04-open-telemetry/contractoffer.json b/transfer/transfer-04-open-telemetry/contractoffer.json index 64ae5c1e..5baecd25 100644 --- a/transfer/transfer-04-open-telemetry/contractoffer.json +++ b/transfer/transfer-04-open-telemetry/contractoffer.json @@ -1,45 +1,24 @@ { + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@type": "NegotiationInitiateRequestDto", "connectorId": "provider", + "consumerId": "consumer", + "providerId": "provider", "connectorAddress": "http://provider:8282/protocol", "protocol": "dataspace-protocol-http", "offer": { - "offerId": "1:3a75736e-001d-4364-8bd4-9888490edb58", + "offerId": "1:test-document:3a75736e-001d-4364-8bd4-9888490edb58", "assetId": "test-document", "policy": { - "uid": "956e172f-2de1-4501-8881-057a57fd0e69", - "permissions": [ - { - "edctype": "dataspaceconnector:permission", - "uid": null, - "target": "test-document", - "action": { - "type": "USE", - "includedIn": null, - "constraint": null - }, - "assignee": null, - "assigner": null, - "constraints": [], - "duties": [] - } - ], - "prohibitions": [], - "obligations": [], - "extensibleProperties": {}, - "inheritsFrom": null, - "assigner": null, - "assignee": null, - "target": null, - "@type": { - "@policytype": "set" - } - }, - "asset": { - "properties": { - "ids:byteSize": null, - "asset:prop:id": "test-document", - "ids:fileName": null - } + "@id": "1:test-document:13dce0f1-52ed-4554-a194-e83e92733ee5", + "@type": "set", + "odrl:permission": [], + "odrl:prohibition": [], + "odrl:obligation": [], + "odrl:target": "test-document" } } } diff --git a/transfer/transfer-04-open-telemetry/docker-compose.yaml b/transfer/transfer-04-open-telemetry/docker-compose.yaml index b0660041..d5d15cc4 100644 --- a/transfer/transfer-04-open-telemetry/docker-compose.yaml +++ b/transfer/transfer-04-open-telemetry/docker-compose.yaml @@ -13,9 +13,10 @@ services: WEB_HTTP_PATH: /api WEB_HTTP_MANAGEMENT_PORT: 9192 WEB_HTTP_MANAGEMENT_PATH: /management - WEB_HTTP_IDS_PORT: 9292 - WEB_HTTP_IDS_PATH: /protocol - IDS_WEBHOOK_ADDRESS: http://consumer:9292 + WEB_HTTP_PROTOCOL_PORT: 9292 + WEB_HTTP_PROTOCOL_PATH: /protocol + EDC_DSP_CALLBACK_ADDRESS: http://consumer:9292/protocol + EDC_PARTICIPANT_ID: consumer EDC_API_AUTH_KEY: password volumes: - ../:/samples @@ -37,7 +38,10 @@ services: WEB_HTTP_PATH: /api WEB_HTTP_MANAGEMENT_PORT: 8182 WEB_HTTP_MANAGEMENT_PATH: /management - IDS_WEBHOOK_ADDRESS: http://provider:8282 + WEB_HTTP_PROTOCOL_PORT: 8282 + WEB_HTTP_PROTOCOL_PATH: /protocol + EDC_DSP_CALLBACK_ADDRESS: http://provider:8282/protocol + EDC_PARTICIPANT_ID: provider EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /samples/transfer-04-open-telemetry/input-file.txt volumes: - ../:/samples diff --git a/transfer/transfer-04-open-telemetry/filetransfer.json b/transfer/transfer-04-open-telemetry/filetransfer.json index 777e3781..9bddb28f 100644 --- a/transfer/transfer-04-open-telemetry/filetransfer.json +++ b/transfer/transfer-04-open-telemetry/filetransfer.json @@ -1,17 +1,18 @@ { - "protocol": "dataspace-protocol-http", - "assetId": "test-document", - "contractId": "{agreement ID}", + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", "dataDestination": { + "type": "File", "path": "/samples/output-file.txt", - "keyName": "keyName", - "type": "File" + "keyName": "keyName" }, - "transferType": { - "contentType": "application/octet-stream", - "isFinite": true - }, - "managedResources": false, + "protocol": "dataspace-protocol-http", + "assetId": "test-document", + "contractId": "{agreement ID}", + "connectorId": "provider", "connectorAddress": "http://provider:8282/protocol", - "connectorId": "consumer" + "privateProperties": {}, + "managedResources": false } diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts index bd4cc160..f53f76f0 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts @@ -21,6 +21,7 @@ plugins { dependencies { implementation(libs.edc.control.plane.core) + implementation(libs.edc.data.plane.selector.core) implementation(libs.edc.micrometer.core) implementation(libs.edc.configuration.filesystem) diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts index 9e52967f..79a685e9 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts @@ -20,6 +20,7 @@ plugins { dependencies { implementation(libs.edc.control.plane.core) + implementation(libs.edc.data.plane.selector.core) implementation(libs.edc.api.observability) From 720ebcb5bd2a610c284c645702f468cbe9144e3d Mon Sep 17 00:00:00 2001 From: Matthias De Geyter Date: Wed, 2 Aug 2023 14:50:54 +0200 Subject: [PATCH 02/16] test: added test for docker compose sample --- gradle/libs.versions.toml | 3 + system-tests/build.gradle.kts | 4 ++ .../FileTransferSampleTestCommon.java | 14 +++- .../transfer/Transfer04openTelemetryTest.java | 66 +++++++++++++++++++ .../contractoffer.json | 51 +++++--------- .../docker-compose.yaml | 16 +++-- .../filetransfer.json | 23 +++---- .../open-telemetry-consumer/build.gradle.kts | 1 + .../open-telemetry-provider/build.gradle.kts | 2 +- 9 files changed, 123 insertions(+), 57 deletions(-) create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 20ef9079..4528c0fb 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,6 +11,7 @@ jupiter = "5.10.0" openTelemetry = "1.18.0" restAssured = "5.3.1" rsApi = "3.1.0" +testcontainers = "1.18.3" [libraries] @@ -61,6 +62,8 @@ junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", vers junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "junit-pioneer" } opentelemetry-annotations = { module = "io.opentelemetry:opentelemetry-extension-annotations", version.ref = "openTelemetry" } restAssured = { module = "io.rest-assured:rest-assured", version.ref = "restAssured" } +testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } +testcontainers-junit-jupiter = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } [plugins] shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } diff --git a/system-tests/build.gradle.kts b/system-tests/build.gradle.kts index 62aec820..2b63449f 100644 --- a/system-tests/build.gradle.kts +++ b/system-tests/build.gradle.kts @@ -22,6 +22,8 @@ dependencies { testImplementation(libs.awaitility) testImplementation(libs.restAssured) + testImplementation(libs.testcontainers) + testImplementation(libs.testcontainers.junit.jupiter) // runtimes testCompileOnly(project(":basic:basic-01-basic-connector")) @@ -31,4 +33,6 @@ dependencies { testCompileOnly(project(":transfer:transfer-01-file-transfer:file-transfer-provider")) testCompileOnly(project(":transfer:transfer-02-file-transfer-listener:file-transfer-listener-consumer")) testCompileOnly(project(":transfer:transfer-03-modify-transferprocess:modify-transferprocess-consumer")) + testCompileOnly(project(":transfer:transfer-04-open-telemetry:open-telemetry-consumer")) + testCompileOnly(project(":transfer:transfer-04-open-telemetry:open-telemetry-provider")) } diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java index 41ca7f18..d15f7502 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java @@ -119,10 +119,14 @@ void assertTransferProcessStatusConsumerSide(String transferProcessId) { * This method corresponds to the command in the sample: {@code curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/contractoffer.json "http://localhost:9192/management/v2/contractnegotiations"} */ void initiateContractNegotiation() { + initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); + } + + void initiateContractNegotiation(String contractOfferFilePath) { contractNegotiationId = given() .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) .contentType(ContentType.JSON) - .body(new File(TestUtils.findBuildRoot(), CONTRACT_OFFER_FILE_PATH)) + .body(new File(TestUtils.findBuildRoot(), contractOfferFilePath)) .when() .post(MANAGEMENT_API_URL + "/v2/contractnegotiations") .then() @@ -167,8 +171,8 @@ void lookUpContractAgreementId() { * * @throws IOException Thrown if there was an error accessing the transfer request file defined in {@link FileTransferSampleTestCommon#TRANSFER_FILE_PATH}. */ - String requestTransferFile() throws IOException { - var transferJsonFile = getFileFromRelativePath(TRANSFER_FILE_PATH); + String requestTransferFile(String transferFilePath) throws IOException { + var transferJsonFile = getFileFromRelativePath(transferFilePath); var requestBody = readAndUpdateDataRequestFromJsonFile(transferJsonFile, contractAgreementId); var jsonPath = given() @@ -191,6 +195,10 @@ String requestTransferFile() throws IOException { return transferProcessId; } + String requestTransferFile() throws IOException { + return requestTransferFile(TRANSFER_FILE_PATH); + } + /** * Reads a transfer request file with changed value for contract agreement ID and file destination path. * diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java new file mode 100644 index 00000000..dd9a403b --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023 imec + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * imec + * + */ + +package org.eclipse.edc.samples.transfer; + +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.junit.ClassRule; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Testcontainers; + + + +@EndToEndTest +@Testcontainers +public class Transfer04openTelemetryTest { + + private static final String SAMPLE_FOLDER = "transfer/transfer-04-open-telemetry"; + static final String CONTRACT_OFFER_FILE_PATH = SAMPLE_FOLDER + "/contractoffer.json"; + + static final String TRANSFER_FILE_PATH = SAMPLE_FOLDER + "/filetransfer.json"; + + static final String SAMPLE_ASSET_FILE_PATH = SAMPLE_FOLDER + "/README.md"; + static final String DESTINATION_FILE_PATH = "transfer/requested.test.txt"; + final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); + + @ClassRule + public static DockerComposeContainer environment = + new DockerComposeContainer(FileTransferSampleTestCommon.getFileFromRelativePath("transfer/transfer-04-open-telemetry/docker-compose.yaml")).withLocalCompose(true) + .waitingFor("consumer", Wait.forLogMessage(".*ready.*", 1)); + + @BeforeAll + static void setUp() { + environment.start(); + } + + @Test + void runSampleSteps() throws Exception { + testUtils.assertTestPrerequisites(); + testUtils.initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); + testUtils.lookUpContractAgreementId(); + var transferProcessId = testUtils.requestTransferFile(TRANSFER_FILE_PATH); + testUtils.assertDestinationFileContent(); + + testUtils.assertTransferProcessStatusConsumerSide(transferProcessId); + } + + @AfterEach + protected void tearDown() { + testUtils.cleanTemporaryTestFiles(); + } +} diff --git a/transfer/transfer-04-open-telemetry/contractoffer.json b/transfer/transfer-04-open-telemetry/contractoffer.json index 64ae5c1e..5d1637e3 100644 --- a/transfer/transfer-04-open-telemetry/contractoffer.json +++ b/transfer/transfer-04-open-telemetry/contractoffer.json @@ -1,45 +1,24 @@ { + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@type": "NegotiationInitiateRequestDto", "connectorId": "provider", + "consumerId": "consumer", + "providerId": "provider", "connectorAddress": "http://provider:8282/protocol", "protocol": "dataspace-protocol-http", "offer": { - "offerId": "1:3a75736e-001d-4364-8bd4-9888490edb58", + "offerId": "1:test-document:3a75736e-001d-4364-8bd4-9888490edb58", "assetId": "test-document", "policy": { - "uid": "956e172f-2de1-4501-8881-057a57fd0e69", - "permissions": [ - { - "edctype": "dataspaceconnector:permission", - "uid": null, - "target": "test-document", - "action": { - "type": "USE", - "includedIn": null, - "constraint": null - }, - "assignee": null, - "assigner": null, - "constraints": [], - "duties": [] - } - ], - "prohibitions": [], - "obligations": [], - "extensibleProperties": {}, - "inheritsFrom": null, - "assigner": null, - "assignee": null, - "target": null, - "@type": { - "@policytype": "set" - } - }, - "asset": { - "properties": { - "ids:byteSize": null, - "asset:prop:id": "test-document", - "ids:fileName": null - } + "@id": "1:test-document:13dce0f1-52ed-4554-a194-e83e92733ee5", + "@type": "set", + "odrl:permission": [], + "odrl:prohibition": [], + "odrl:obligation": [], + "odrl:target": "test-document" } } -} +} \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/docker-compose.yaml b/transfer/transfer-04-open-telemetry/docker-compose.yaml index b0660041..fdada711 100644 --- a/transfer/transfer-04-open-telemetry/docker-compose.yaml +++ b/transfer/transfer-04-open-telemetry/docker-compose.yaml @@ -13,9 +13,10 @@ services: WEB_HTTP_PATH: /api WEB_HTTP_MANAGEMENT_PORT: 9192 WEB_HTTP_MANAGEMENT_PATH: /management - WEB_HTTP_IDS_PORT: 9292 - WEB_HTTP_IDS_PATH: /protocol - IDS_WEBHOOK_ADDRESS: http://consumer:9292 + WEB_HTTP_PROTOCOL_PORT: 9292 + WEB_HTTP_PROTOCOL_PATH: /protocol + EDC_DSP_CALLBACK_ADDRESS: http://consumer:9292/protocol + EDC_PARTICIPANT_ID: consumer EDC_API_AUTH_KEY: password volumes: - ../:/samples @@ -37,8 +38,11 @@ services: WEB_HTTP_PATH: /api WEB_HTTP_MANAGEMENT_PORT: 8182 WEB_HTTP_MANAGEMENT_PATH: /management - IDS_WEBHOOK_ADDRESS: http://provider:8282 - EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /samples/transfer-04-open-telemetry/input-file.txt + WEB_HTTP_PROTOCOL_PORT: 8282 + WEB_HTTP_PROTOCOL_PATH: /protocol + EDC_DSP_CALLBACK_ADDRESS: http://provider:8282/protocol + EDC_PARTICIPANT_ID: provider + EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /samples/transfer-04-open-telemetry/README.md volumes: - ../:/samples ports: @@ -56,7 +60,7 @@ services: prometheus: image: prom/prometheus:v2.30.3 - container_name: prometheus +# container_name: prometheus volumes: - ./prometheus/:/etc/prometheus/ ports: diff --git a/transfer/transfer-04-open-telemetry/filetransfer.json b/transfer/transfer-04-open-telemetry/filetransfer.json index 777e3781..6f31c36f 100644 --- a/transfer/transfer-04-open-telemetry/filetransfer.json +++ b/transfer/transfer-04-open-telemetry/filetransfer.json @@ -1,17 +1,18 @@ { + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "dataDestination": { + "type": "File", + "path": "/samples/requested.test.txt", + "keyName": "keyName" + }, "protocol": "dataspace-protocol-http", "assetId": "test-document", "contractId": "{agreement ID}", - "dataDestination": { - "path": "/samples/output-file.txt", - "keyName": "keyName", - "type": "File" - }, - "transferType": { - "contentType": "application/octet-stream", - "isFinite": true - }, - "managedResources": false, + "connectorId": "provider", "connectorAddress": "http://provider:8282/protocol", - "connectorId": "consumer" + "privateProperties": {}, + "managedResources": false } diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts index bd4cc160..f53f76f0 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts @@ -21,6 +21,7 @@ plugins { dependencies { implementation(libs.edc.control.plane.core) + implementation(libs.edc.data.plane.selector.core) implementation(libs.edc.micrometer.core) implementation(libs.edc.configuration.filesystem) diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts index 9e52967f..11ebfe26 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts @@ -20,7 +20,7 @@ plugins { dependencies { implementation(libs.edc.control.plane.core) - + implementation(libs.edc.data.plane.selector.core) implementation(libs.edc.api.observability) implementation(libs.edc.configuration.filesystem) From 7d1d81825c21351172b67149bf911e27242fd7e3 Mon Sep 17 00:00:00 2001 From: Matthias De Geyter Date: Wed, 2 Aug 2023 15:10:30 +0200 Subject: [PATCH 03/16] style: cosmetic changes --- transfer/transfer-04-open-telemetry/contractoffer.json | 2 +- transfer/transfer-04-open-telemetry/docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/transfer/transfer-04-open-telemetry/contractoffer.json b/transfer/transfer-04-open-telemetry/contractoffer.json index 5d1637e3..5baecd25 100644 --- a/transfer/transfer-04-open-telemetry/contractoffer.json +++ b/transfer/transfer-04-open-telemetry/contractoffer.json @@ -21,4 +21,4 @@ "odrl:target": "test-document" } } -} \ No newline at end of file +} diff --git a/transfer/transfer-04-open-telemetry/docker-compose.yaml b/transfer/transfer-04-open-telemetry/docker-compose.yaml index fdada711..80e94d27 100644 --- a/transfer/transfer-04-open-telemetry/docker-compose.yaml +++ b/transfer/transfer-04-open-telemetry/docker-compose.yaml @@ -60,7 +60,7 @@ services: prometheus: image: prom/prometheus:v2.30.3 -# container_name: prometheus + # container_name: prometheus # does not work with testcontainers volumes: - ./prometheus/:/etc/prometheus/ ports: From f893ec42f7432887ed243c727814bb815a35afae Mon Sep 17 00:00:00 2001 From: Matthias De Geyter Date: Wed, 2 Aug 2023 15:11:10 +0200 Subject: [PATCH 04/16] test: temporarily disable progress checking of transfer --- .../edc/samples/transfer/Transfer04openTelemetryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java index dd9a403b..6ff8eea1 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java @@ -56,7 +56,7 @@ void runSampleSteps() throws Exception { var transferProcessId = testUtils.requestTransferFile(TRANSFER_FILE_PATH); testUtils.assertDestinationFileContent(); - testUtils.assertTransferProcessStatusConsumerSide(transferProcessId); + // testUtils.assertTransferProcessStatusConsumerSide(transferProcessId); } @AfterEach From d5f4f684c2852902d5041bbb36d790b69d416208 Mon Sep 17 00:00:00 2001 From: Matthias De Geyter Date: Wed, 2 Aug 2023 15:40:57 +0200 Subject: [PATCH 05/16] docs: added explanation for commented line --- .../edc/samples/transfer/Transfer04openTelemetryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java index 6ff8eea1..e8027e16 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java @@ -55,7 +55,7 @@ void runSampleSteps() throws Exception { testUtils.lookUpContractAgreementId(); var transferProcessId = testUtils.requestTransferFile(TRANSFER_FILE_PATH); testUtils.assertDestinationFileContent(); - + // next one fails with "No checker found for process error (related to https://github.com/eclipse-edc/Connector/issues/3334 ?) // testUtils.assertTransferProcessStatusConsumerSide(transferProcessId); } From de3f57be79d41c4c42c2f62a5de796e1ccef6152 Mon Sep 17 00:00:00 2001 From: hamidonos Date: Thu, 28 Sep 2023 14:48:49 +0200 Subject: [PATCH 06/16] test: download OpenTelemetry Jar before test execution --- .../transfer/Transfer04openTelemetryTest.java | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java index e8027e16..b435c2c6 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java @@ -15,6 +15,7 @@ package org.eclipse.edc.samples.transfer; import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.spi.EdcException; import org.junit.ClassRule; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -23,19 +24,23 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Testcontainers; +import java.io.BufferedInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URL; @EndToEndTest @Testcontainers public class Transfer04openTelemetryTest { + private static final String OPEN_TELEMETRY_JAR_URL = "https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.27.0/opentelemetry-javaagent.jar"; private static final String SAMPLE_FOLDER = "transfer/transfer-04-open-telemetry"; - static final String CONTRACT_OFFER_FILE_PATH = SAMPLE_FOLDER + "/contractoffer.json"; - - static final String TRANSFER_FILE_PATH = SAMPLE_FOLDER + "/filetransfer.json"; - - static final String SAMPLE_ASSET_FILE_PATH = SAMPLE_FOLDER + "/README.md"; - static final String DESTINATION_FILE_PATH = "transfer/requested.test.txt"; + private static final String CONTRACT_OFFER_FILE_PATH = SAMPLE_FOLDER + "/contractoffer.json"; + private static final String OPEN_TELEMETRY_JAR_PATH = SAMPLE_FOLDER + "/opentelemetry-javaagent.jar"; + private static final String TRANSFER_FILE_PATH = SAMPLE_FOLDER + "/filetransfer.json"; + private static final String SAMPLE_ASSET_FILE_PATH = SAMPLE_FOLDER + "/README.md"; + private static final String DESTINATION_FILE_PATH = "transfer/requested.test.txt"; final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); @ClassRule @@ -45,6 +50,7 @@ public class Transfer04openTelemetryTest { @BeforeAll static void setUp() { + downloadOpenTelemetryJar(); environment.start(); } @@ -63,4 +69,17 @@ void runSampleSteps() throws Exception { protected void tearDown() { testUtils.cleanTemporaryTestFiles(); } + + private static void downloadOpenTelemetryJar() { + try (var in = new BufferedInputStream(new URL(OPEN_TELEMETRY_JAR_URL).openStream()); + var out = new FileOutputStream("../" + OPEN_TELEMETRY_JAR_PATH)) { + var dataBuffer = new byte[1024]; + int bytesRead; + while ((bytesRead = in.read(dataBuffer, 0, 1024)) != -1) { + out.write(dataBuffer, 0, bytesRead); + } + } catch (IOException e) { + throw new EdcException(e); + } + } } From 398a290c0c29ba29164512e1563ad4c8292c8028 Mon Sep 17 00:00:00 2001 From: hamidonos Date: Fri, 29 Sep 2023 10:08:07 +0200 Subject: [PATCH 07/16] gradle: download OpenTelemetry before build step --- .../transfer/Transfer04openTelemetryTest.java | 45 +++++-------------- transfer/transfer-04-open-telemetry/README.md | 7 +-- .../open-telemetry-consumer/build.gradle.kts | 18 ++++++++ .../open-telemetry-provider/build.gradle.kts | 18 ++++++++ 4 files changed, 49 insertions(+), 39 deletions(-) diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java index b435c2c6..15d006fe 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 imec + * Copyright (c) 2023 - Mercedes-Benz Tech Innovation GmbH * * This program and the accompanying materials are made available under the * terms of the Apache License, Version 2.0 which is available at @@ -8,14 +8,13 @@ * SPDX-License-Identifier: Apache-2.0 * * Contributors: - * imec + * Mercedes-Benz Tech Innovation GmbH - Sample workflow test * */ package org.eclipse.edc.samples.transfer; import org.eclipse.edc.junit.annotations.EndToEndTest; -import org.eclipse.edc.spi.EdcException; import org.junit.ClassRule; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -24,62 +23,42 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Testcontainers; -import java.io.BufferedInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.net.URL; - +import java.time.Duration; @EndToEndTest @Testcontainers public class Transfer04openTelemetryTest { - private static final String OPEN_TELEMETRY_JAR_URL = "https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.27.0/opentelemetry-javaagent.jar"; private static final String SAMPLE_FOLDER = "transfer/transfer-04-open-telemetry"; - private static final String CONTRACT_OFFER_FILE_PATH = SAMPLE_FOLDER + "/contractoffer.json"; - private static final String OPEN_TELEMETRY_JAR_PATH = SAMPLE_FOLDER + "/opentelemetry-javaagent.jar"; - private static final String TRANSFER_FILE_PATH = SAMPLE_FOLDER + "/filetransfer.json"; + private static final String DOCKER_COMPOSE_YAML = "/docker-compose.yaml"; private static final String SAMPLE_ASSET_FILE_PATH = SAMPLE_FOLDER + "/README.md"; private static final String DESTINATION_FILE_PATH = "transfer/requested.test.txt"; - final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); + private final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); @ClassRule public static DockerComposeContainer environment = - new DockerComposeContainer(FileTransferSampleTestCommon.getFileFromRelativePath("transfer/transfer-04-open-telemetry/docker-compose.yaml")).withLocalCompose(true) - .waitingFor("consumer", Wait.forLogMessage(".*ready.*", 1)); + new DockerComposeContainer(FileTransferSampleTestCommon.getFileFromRelativePath(SAMPLE_FOLDER + DOCKER_COMPOSE_YAML)) + .withLocalCompose(true) + .waitingFor("consumer", Wait.forLogMessage(".*ready.*", 1)) + .withStartupTimeout(Duration.ofSeconds(10)); @BeforeAll static void setUp() { - downloadOpenTelemetryJar(); environment.start(); } @Test void runSampleSteps() throws Exception { testUtils.assertTestPrerequisites(); - testUtils.initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); + testUtils.initiateContractNegotiation(); testUtils.lookUpContractAgreementId(); - var transferProcessId = testUtils.requestTransferFile(TRANSFER_FILE_PATH); + var transferProcessId = testUtils.requestTransferFile(); testUtils.assertDestinationFileContent(); - // next one fails with "No checker found for process error (related to https://github.com/eclipse-edc/Connector/issues/3334 ?) - // testUtils.assertTransferProcessStatusConsumerSide(transferProcessId); + testUtils.assertTransferProcessStatusConsumerSide(transferProcessId); } @AfterEach protected void tearDown() { testUtils.cleanTemporaryTestFiles(); } - - private static void downloadOpenTelemetryJar() { - try (var in = new BufferedInputStream(new URL(OPEN_TELEMETRY_JAR_URL).openStream()); - var out = new FileOutputStream("../" + OPEN_TELEMETRY_JAR_PATH)) { - var dataBuffer = new byte[1024]; - int bytesRead; - while ((bytesRead = in.read(dataBuffer, 0, 1024)) != -1) { - out.write(dataBuffer, 0, bytesRead); - } - } catch (IOException e) { - throw new EdcException(e); - } - } } diff --git a/transfer/transfer-04-open-telemetry/README.md b/transfer/transfer-04-open-telemetry/README.md index fd8dfce6..ef088d8c 100644 --- a/transfer/transfer-04-open-telemetry/README.md +++ b/transfer/transfer-04-open-telemetry/README.md @@ -12,12 +12,7 @@ several popular [libraries and frameworks](https://github.com/open-telemetry/ope In order to visualize and analyze the traces and metrics, we use [OpenTelemetry exporters](https://opentelemetry.io/docs/instrumentation/js/exporters/) to export data into the Jaeger -tracing backend and a Prometheus endpoint. - -## Prerequisites - -Download the [opentelemetry-javaagent.jar](https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.12.0/opentelemetry-javaagent.jar) -and place it in the root folder of this sample. +tracing backend and a Prometheus endpoint. ## Run the sample diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts index f53f76f0..74eb8434 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts @@ -45,3 +45,21 @@ tasks.withType { mergeServiceFiles() archiveFileName.set("consumer.jar") } + +tasks.register("downloadOpenTelemetryJar"){ + val filePath = "../opentelemetry-javaagent.jar" + val file = File(filePath) + if (!file.isFile) { + val sourceUrl = "https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.27.0/opentelemetry-javaagent.jar" + download(sourceUrl,filePath) + } +} + +tasks.build { + dependsOn("downloadOpenTelemetryJar") +} + +fun download(url : String, path : String){ + val destFile = File(path) + ant.invokeMethod("get", mapOf("src" to url, "dest" to destFile)) +} \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts index 79a685e9..a3938430 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts @@ -45,3 +45,21 @@ tasks.withType { mergeServiceFiles() archiveFileName.set("provider.jar") } + +tasks.register("downloadOpenTelemetryJar"){ + val filePath = "../opentelemetry-javaagent.jar" + val file = File(filePath) + if (!file.isFile) { + val sourceUrl = "https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.27.0/opentelemetry-javaagent.jar" + download(sourceUrl,filePath) + } +} + +tasks.build { + dependsOn("downloadOpenTelemetryJar") +} + +fun download(url : String, path : String){ + val destFile = File(path) + ant.invokeMethod("get", mapOf("src" to url, "dest" to destFile)) +} \ No newline at end of file From 61abb6555ab5ecdb4563542c474aa5d8d9665417 Mon Sep 17 00:00:00 2001 From: hamidonos Date: Fri, 29 Sep 2023 10:27:25 +0200 Subject: [PATCH 08/16] removing container name from prometheus as it's not supported by testcontainers --- transfer/transfer-04-open-telemetry/docker-compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/transfer/transfer-04-open-telemetry/docker-compose.yaml b/transfer/transfer-04-open-telemetry/docker-compose.yaml index 80e94d27..6cd135d5 100644 --- a/transfer/transfer-04-open-telemetry/docker-compose.yaml +++ b/transfer/transfer-04-open-telemetry/docker-compose.yaml @@ -60,7 +60,6 @@ services: prometheus: image: prom/prometheus:v2.30.3 - # container_name: prometheus # does not work with testcontainers volumes: - ./prometheus/:/etc/prometheus/ ports: From af4c24df18c73835fc396a0b4e69e8fcbd4297d1 Mon Sep 17 00:00:00 2001 From: hamidonos Date: Fri, 29 Sep 2023 10:29:24 +0200 Subject: [PATCH 09/16] fixed typo in libs.versions.toml --- gradle/libs.versions.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 3196c908..a7085ab9 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -13,7 +13,7 @@ openTelemetry = "1.18.0" restAssured = "5.3.2" rsApi = "3.1.0" kafkaClients = "3.5.1" -testContainers = "1.19.0" +testcontainers = "1.19.0" [libraries] assertj = { module = "org.assertj:assertj-core", version.ref = "assertj" } @@ -71,8 +71,8 @@ restAssured = { module = "io.rest-assured:rest-assured", version.ref = "restAssu testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } testcontainers-junit-jupiter = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "kafkaClients" } -testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testContainers" } -testcontainers-junit = { module = "org.testcontainers:junit-jupiter", version.ref = "testContainers" } +testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testcontainers" } +testcontainers-junit = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } [plugins] shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } From 4a5da68b391f847b34f7b64549fdc84b1ba93954 Mon Sep 17 00:00:00 2001 From: hamidonos Date: Fri, 29 Sep 2023 19:48:58 +0200 Subject: [PATCH 10/16] copy opentelemetry jar from gradle cache --- gradle/libs.versions.toml | 1 + .../FileTransferSampleTestCommon.java | 3 +- .../transfer/Transfer04openTelemetryTest.java | 6 ++-- .../contractoffer.json | 32 ------------------- .../docker-compose.yaml | 7 ++-- .../filetransfer.json | 18 +++-------- .../transfer-04-open-telemetry/input-file.txt | 1 - .../open-telemetry-consumer/build.gradle.kts | 31 +++++++++++------- .../open-telemetry-provider/build.gradle.kts | 31 +++++++++++------- 9 files changed, 52 insertions(+), 78 deletions(-) delete mode 100644 transfer/transfer-04-open-telemetry/input-file.txt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a7085ab9..7fd9683f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -73,6 +73,7 @@ testcontainers-junit-jupiter = { module = "org.testcontainers:junit-jupiter", ve kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "kafkaClients" } testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testcontainers" } testcontainers-junit = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } +opentelemetry = "io.opentelemetry.javaagent:opentelemetry-javaagent:1.30.0" [plugins] shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java index 3884fa13..78e100bb 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java @@ -52,7 +52,7 @@ public class FileTransferSampleTestCommon { final String sampleAssetFilePath; final File sampleAssetFile; final File destinationFile; - Duration timeout = Duration.ofSeconds(30); + Duration timeout = Duration.ofSeconds(15); Duration pollInterval = Duration.ofMillis(500); String contractNegotiationId; @@ -136,6 +136,7 @@ void initiateContractNegotiation() { initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); } + void initiateContractNegotiation(String contractOfferFilePath) { contractNegotiationId = given() .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java index 15d006fe..2b8cec46 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java @@ -32,7 +32,9 @@ public class Transfer04openTelemetryTest { private static final String SAMPLE_FOLDER = "transfer/transfer-04-open-telemetry"; private static final String DOCKER_COMPOSE_YAML = "/docker-compose.yaml"; private static final String SAMPLE_ASSET_FILE_PATH = SAMPLE_FOLDER + "/README.md"; - private static final String DESTINATION_FILE_PATH = "transfer/requested.test.txt"; + private static final String DESTINATION_FILE_PATH = "transfer/README_transferred.md"; + private static final String CONTRACT_OFFER_FILE_PATH = SAMPLE_FOLDER + "/contractoffer.json"; + private final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); @ClassRule @@ -50,7 +52,7 @@ static void setUp() { @Test void runSampleSteps() throws Exception { testUtils.assertTestPrerequisites(); - testUtils.initiateContractNegotiation(); + testUtils.initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); testUtils.lookUpContractAgreementId(); var transferProcessId = testUtils.requestTransferFile(); testUtils.assertDestinationFileContent(); diff --git a/transfer/transfer-04-open-telemetry/contractoffer.json b/transfer/transfer-04-open-telemetry/contractoffer.json index 67d8dc2b..5baecd25 100644 --- a/transfer/transfer-04-open-telemetry/contractoffer.json +++ b/transfer/transfer-04-open-telemetry/contractoffer.json @@ -19,38 +19,6 @@ "odrl:prohibition": [], "odrl:obligation": [], "odrl:target": "test-document" - "uid": "956e172f-2de1-4501-8881-057a57fd0e69", - "permissions": [ - { - "edctype": "dataspaceconnector:permission", - "uid": null, - "target": "test-document", - "action": { - "type": "USE", - "includedIn": null, - "constraint": null - }, - "assignee": null, - "assigner": null, - "constraints": [], - "duties": [] - } - ], - "prohibitions": [], - "obligations": [], - "extensibleProperties": {}, - "inheritsFrom": null, - "assigner": null, - "assignee": null, - "target": null, - "@type": { - "@policytype": "set" - } - }, - "asset": { - "properties": { - "asset:prop:id": "test-document" - } } } } diff --git a/transfer/transfer-04-open-telemetry/docker-compose.yaml b/transfer/transfer-04-open-telemetry/docker-compose.yaml index 7d34b8a5..cd7be7fa 100644 --- a/transfer/transfer-04-open-telemetry/docker-compose.yaml +++ b/transfer/transfer-04-open-telemetry/docker-compose.yaml @@ -5,6 +5,7 @@ services: consumer: image: openjdk:17-jdk-slim-buster environment: + EDC_HOSTNAME: consumer OTEL_SERVICE_NAME: consumer OTEL_TRACES_EXPORTER: jaeger OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 @@ -17,9 +18,8 @@ services: WEB_HTTP_PROTOCOL_PATH: /protocol EDC_DSP_CALLBACK_ADDRESS: http://consumer:9292/protocol EDC_PARTICIPANT_ID: consumer - WEB_HTTP_DSP_PORT: 9292 + WEB_HTTP_DSP_PORT: 9393 WEB_HTTP_DSP_PATH: /protocol - DSP_WEBHOOK_ADDRESS: http://consumer:9292 EDC_API_AUTH_KEY: password volumes: - ../:/samples @@ -34,6 +34,7 @@ services: provider: image: openjdk:17-jdk-slim-buster environment: + EDC_HOSTNAME: provider OTEL_SERVICE_NAME: provider OTEL_TRACES_EXPORTER: jaeger OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 @@ -46,8 +47,6 @@ services: EDC_DSP_CALLBACK_ADDRESS: http://provider:8282/protocol EDC_PARTICIPANT_ID: provider EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /samples/transfer-04-open-telemetry/README.md - DSP_WEBHOOK_ADDRESS: http://provider:8282 - EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /samples/transfer-04-open-telemetry/input-file.txt volumes: - ../:/samples ports: diff --git a/transfer/transfer-04-open-telemetry/filetransfer.json b/transfer/transfer-04-open-telemetry/filetransfer.json index edd07d56..7d08ce21 100644 --- a/transfer/transfer-04-open-telemetry/filetransfer.json +++ b/transfer/transfer-04-open-telemetry/filetransfer.json @@ -5,23 +5,13 @@ "@type": "TransferRequestDto", "dataDestination": { "type": "File", - "path": "/samples/requested.test.txt", + "path": "/samples/README_transferred.md", "keyName": "keyName" }, "protocol": "dataspace-protocol-http", "assetId": "test-document", - "contractId": "{agreement ID}", + "contractId": "afa67850-6b8f-4417-b5f0-60630d55be28", "connectorId": "provider", - "dataDestination": { - "path": "/samples/output-file.txt", - "keyName": "keyName", - "type": "File" - }, - "transferType": { - "contentType": "application/octet-stream", - "isFinite": true - }, - "connectorAddress": "http://provider:8282/protocol", - "privateProperties": {}, - "managedResources": false + "connectorAddress": "http://localhost:8282/protocol", + "privateProperties": {} } diff --git a/transfer/transfer-04-open-telemetry/input-file.txt b/transfer/transfer-04-open-telemetry/input-file.txt deleted file mode 100644 index b4f8b365..00000000 --- a/transfer/transfer-04-open-telemetry/input-file.txt +++ /dev/null @@ -1 +0,0 @@ -This is a sample file \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts index 1ba4a3d9..b1aa0fce 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts @@ -1,3 +1,7 @@ +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.StandardCopyOption + /* * Copyright (c) 2022 Microsoft Corporation * @@ -20,6 +24,8 @@ plugins { } dependencies { + implementation(libs.opentelemetry) + implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.core) implementation(libs.edc.data.plane.selector.core) implementation(libs.edc.micrometer.core) @@ -45,20 +51,21 @@ tasks.withType { archiveFileName.set("consumer.jar") } -tasks.register("downloadOpenTelemetryJar"){ - val filePath = "../opentelemetry-javaagent.jar" - val file = File(filePath) - if (!file.isFile) { - val sourceUrl = "https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.27.0/opentelemetry-javaagent.jar" - download(sourceUrl,filePath) +tasks.register("copyOpenTelemetryJar") { + doLast { + sourceSets["main"] + .runtimeClasspath + .files + .find { file -> file.name.contains("opentelemetry-javaagent") } + ?.path + ?.let { + val sourcePath = Paths.get(it) + val targetPath = Paths.get("transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar") + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) + } } } tasks.build { - dependsOn("downloadOpenTelemetryJar") + dependsOn("copyOpenTelemetryJar") } - -fun download(url : String, path : String){ - val destFile = File(path) - ant.invokeMethod("get", mapOf("src" to url, "dest" to destFile)) -} \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts index d62183de..5e954d15 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts @@ -1,3 +1,7 @@ +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.StandardCopyOption + /* * Copyright (c) 2022 Microsoft Corporation * @@ -19,6 +23,8 @@ plugins { } dependencies { + implementation(libs.opentelemetry) + implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.core) implementation(libs.edc.data.plane.selector.core) @@ -45,20 +51,21 @@ tasks.withType { archiveFileName.set("provider.jar") } -tasks.register("downloadOpenTelemetryJar"){ - val filePath = "../opentelemetry-javaagent.jar" - val file = File(filePath) - if (!file.isFile) { - val sourceUrl = "https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.27.0/opentelemetry-javaagent.jar" - download(sourceUrl,filePath) +tasks.register("copyOpenTelemetryJar") { + doLast { + sourceSets["main"] + .runtimeClasspath + .files + .find { file -> file.name.contains("opentelemetry-javaagent") } + ?.path + ?.let { + val sourcePath = Paths.get(it) + val targetPath = Paths.get("transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar") + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) + } } } tasks.build { - dependsOn("downloadOpenTelemetryJar") -} - -fun download(url : String, path : String){ - val destFile = File(path) - ant.invokeMethod("get", mapOf("src" to url, "dest" to destFile)) + dependsOn("copyOpenTelemetryJar") } \ No newline at end of file From 78040af4f24c83fa4185c5110d7c0a0e7986305a Mon Sep 17 00:00:00 2001 From: hamidonos Date: Mon, 2 Oct 2023 12:17:07 +0200 Subject: [PATCH 11/16] fixed default sample tests for sample 04 --- .../edc/samples/transfer/Transfer04openTelemetryTest.java | 5 +++-- transfer/transfer-04-open-telemetry/filetransfer.json | 6 +++--- .../open-telemetry-consumer/build.gradle.kts | 7 ++++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java index 2b8cec46..a8d3ec3d 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java @@ -32,8 +32,9 @@ public class Transfer04openTelemetryTest { private static final String SAMPLE_FOLDER = "transfer/transfer-04-open-telemetry"; private static final String DOCKER_COMPOSE_YAML = "/docker-compose.yaml"; private static final String SAMPLE_ASSET_FILE_PATH = SAMPLE_FOLDER + "/README.md"; - private static final String DESTINATION_FILE_PATH = "transfer/README_transferred.md"; + private static final String DESTINATION_FILE_PATH = SAMPLE_FOLDER + "/README_transferred.md"; private static final String CONTRACT_OFFER_FILE_PATH = SAMPLE_FOLDER + "/contractoffer.json"; + private static final String FILE_TRANSFER_FILE_PATH = SAMPLE_FOLDER + "/filetransfer.json"; private final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); @@ -54,7 +55,7 @@ void runSampleSteps() throws Exception { testUtils.assertTestPrerequisites(); testUtils.initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); testUtils.lookUpContractAgreementId(); - var transferProcessId = testUtils.requestTransferFile(); + var transferProcessId = testUtils.requestTransferFile(FILE_TRANSFER_FILE_PATH); testUtils.assertDestinationFileContent(); testUtils.assertTransferProcessStatusConsumerSide(transferProcessId); } diff --git a/transfer/transfer-04-open-telemetry/filetransfer.json b/transfer/transfer-04-open-telemetry/filetransfer.json index 7d08ce21..d0f591e3 100644 --- a/transfer/transfer-04-open-telemetry/filetransfer.json +++ b/transfer/transfer-04-open-telemetry/filetransfer.json @@ -5,13 +5,13 @@ "@type": "TransferRequestDto", "dataDestination": { "type": "File", - "path": "/samples/README_transferred.md", + "path": "/samples/transfer-04-open-telemetry/README_transferred.md", "keyName": "keyName" }, "protocol": "dataspace-protocol-http", "assetId": "test-document", - "contractId": "afa67850-6b8f-4417-b5f0-60630d55be28", + "contractId": "{agreement ID}", "connectorId": "provider", - "connectorAddress": "http://localhost:8282/protocol", + "connectorAddress": "http://provider:8282/protocol", "privateProperties": {} } diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts index b1aa0fce..a9acfd3c 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts @@ -29,14 +29,15 @@ dependencies { implementation(libs.edc.control.plane.core) implementation(libs.edc.data.plane.selector.core) implementation(libs.edc.micrometer.core) - + implementation(libs.edc.api.observability) implementation(libs.edc.configuration.filesystem) implementation(libs.edc.iam.mock) - implementation(libs.edc.auth.tokenbased) implementation(libs.edc.management.api) - implementation(libs.edc.dsp) + + implementation(project(":transfer:transfer-01-file-transfer:status-checker")) + runtimeOnly(libs.edc.jersey.micrometer) runtimeOnly(libs.edc.jetty.micrometer) runtimeOnly(libs.edc.monitor.jdk.logger) From 465927f47f3ddd25da2bbf07e2deaf05f4ea407d Mon Sep 17 00:00:00 2001 From: hamidonos Date: Mon, 2 Oct 2023 14:10:28 +0200 Subject: [PATCH 12/16] fixed build process --- .../transfer/Transfer04openTelemetryTest.java | 7 ++--- .../open-telemetry-consumer/build.gradle.kts | 31 +++++++++++-------- .../open-telemetry-provider/build.gradle.kts | 29 ++++++++++------- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java index a8d3ec3d..fe370a9c 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 - Mercedes-Benz Tech Innovation GmbH + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH * * This program and the accompanying materials are made available under the * terms of the Apache License, Version 2.0 which is available at @@ -23,8 +23,6 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Testcontainers; -import java.time.Duration; - @EndToEndTest @Testcontainers public class Transfer04openTelemetryTest { @@ -42,8 +40,7 @@ public class Transfer04openTelemetryTest { public static DockerComposeContainer environment = new DockerComposeContainer(FileTransferSampleTestCommon.getFileFromRelativePath(SAMPLE_FOLDER + DOCKER_COMPOSE_YAML)) .withLocalCompose(true) - .waitingFor("consumer", Wait.forLogMessage(".*ready.*", 1)) - .withStartupTimeout(Duration.ofSeconds(10)); + .waitingFor("consumer", Wait.forLogMessage(".*ready.*", 1)); @BeforeAll static void setUp() { diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts index a9acfd3c..b5c53191 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts @@ -54,19 +54,24 @@ tasks.withType { tasks.register("copyOpenTelemetryJar") { doLast { - sourceSets["main"] - .runtimeClasspath - .files - .find { file -> file.name.contains("opentelemetry-javaagent") } - ?.path - ?.let { - val sourcePath = Paths.get(it) - val targetPath = Paths.get("transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar") - Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) - } + val filePath = "transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar" + val file = File(filePath) + + if (!file.exists()) { + sourceSets["main"] + .runtimeClasspath + .files + .find { it.name.contains("opentelemetry-javaagent") } + ?.path + ?.let { + val sourcePath = Paths.get(it) + val targetPath = Paths.get(filePath) + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) + } + } } } -tasks.build { - dependsOn("copyOpenTelemetryJar") -} +tasks.compileJava { + finalizedBy("copyOpenTelemetryJar") +} \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts index 5e954d15..e9864bdd 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts @@ -53,19 +53,24 @@ tasks.withType { tasks.register("copyOpenTelemetryJar") { doLast { - sourceSets["main"] - .runtimeClasspath - .files - .find { file -> file.name.contains("opentelemetry-javaagent") } - ?.path - ?.let { - val sourcePath = Paths.get(it) - val targetPath = Paths.get("transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar") - Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) - } + val filePath = "transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar" + val file = File(filePath) + + if (!file.exists()) { + sourceSets["main"] + .runtimeClasspath + .files + .find { it.name.contains("opentelemetry-javaagent") } + ?.path + ?.let { + val sourcePath = Paths.get(it) + val targetPath = Paths.get(filePath) + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) + } + } } } -tasks.build { - dependsOn("copyOpenTelemetryJar") +tasks.compileJava { + finalizedBy("copyOpenTelemetryJar") } \ No newline at end of file From 0f0426e059422294f58dfa17314e74d750eb1df9 Mon Sep 17 00:00:00 2001 From: hamidonos Date: Mon, 2 Oct 2023 14:28:30 +0200 Subject: [PATCH 13/16] asserting Jaeger state OK in telemetry test --- .../transfer/Transfer04openTelemetryTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java index fe370a9c..223099ff 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java @@ -14,6 +14,7 @@ package org.eclipse.edc.samples.transfer; +import org.apache.http.HttpStatus; import org.eclipse.edc.junit.annotations.EndToEndTest; import org.junit.ClassRule; import org.junit.jupiter.api.AfterEach; @@ -22,6 +23,12 @@ import org.testcontainers.containers.DockerComposeContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Testcontainers; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; @EndToEndTest @Testcontainers @@ -33,6 +40,7 @@ public class Transfer04openTelemetryTest { private static final String DESTINATION_FILE_PATH = SAMPLE_FOLDER + "/README_transferred.md"; private static final String CONTRACT_OFFER_FILE_PATH = SAMPLE_FOLDER + "/contractoffer.json"; private static final String FILE_TRANSFER_FILE_PATH = SAMPLE_FOLDER + "/filetransfer.json"; + private static final String JAEGER_URL = "http://localhost:16686"; private final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); @@ -55,6 +63,17 @@ void runSampleSteps() throws Exception { var transferProcessId = testUtils.requestTransferFile(FILE_TRANSFER_FILE_PATH); testUtils.assertDestinationFileContent(); testUtils.assertTransferProcessStatusConsumerSide(transferProcessId); + assertJaegerState(); + } + + private void assertJaegerState() { + try { + var url = new URL(JAEGER_URL); + var huc = (HttpURLConnection) url.openConnection(); + assertThat(huc.getResponseCode()).isEqualTo(HttpStatus.SC_OK); + } catch (IOException e) { + fail("Unable to assert Jaeger state", e); + } } @AfterEach From 5d6b1fb5609f24dac60728e33aff6bade9bbe1ea Mon Sep 17 00:00:00 2001 From: hamidonos Date: Wed, 4 Oct 2023 12:24:09 +0200 Subject: [PATCH 14/16] fix: fixing wrong testcontainer reference in .toml file --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a0d1904c..0e3fb4cd 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -13,7 +13,7 @@ openTelemetry = "1.18.0" restAssured = "5.3.2" rsApi = "3.1.0" kafkaClients = "3.5.1" -testContainers = "1.19.1" +testcontainers = "1.19.1" [libraries] assertj = { module = "org.assertj:assertj-core", version.ref = "assertj" } From c8d1941873c1e246a7b0d184c03f36b043648fec Mon Sep 17 00:00:00 2001 From: hamidonos Date: Thu, 5 Oct 2023 09:09:23 +0200 Subject: [PATCH 15/16] fix: running gradle build before test --- .github/workflows/verify.yaml | 4 +++- .../edc/samples/transfer/Transfer04openTelemetryTest.java | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index b6ee42e3..1a04acf6 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -46,7 +46,9 @@ jobs: - name: End to End Integration Tests uses: ./.github/actions/run-tests with: - command: ./gradlew test -DincludeTags="EndToEndTest" + command: + - ./gradlew build -x test + - ./gradlew test -DincludeTags="EndToEndTest" Upload-Test-Report: needs: diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java index 223099ff..eb3d7029 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java @@ -23,13 +23,14 @@ import org.testcontainers.containers.DockerComposeContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Testcontainers; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.fail; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + @EndToEndTest @Testcontainers public class Transfer04openTelemetryTest { From 2e106adabb5c5aac750d13b0e1f61cad2bb9f12e Mon Sep 17 00:00:00 2001 From: hamidonos Date: Fri, 6 Oct 2023 08:35:26 +0200 Subject: [PATCH 16/16] fix: removing gradle build before test for github action --- .github/workflows/verify.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index 1a04acf6..27141139 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -47,7 +47,6 @@ jobs: uses: ./.github/actions/run-tests with: command: - - ./gradlew build -x test - ./gradlew test -DincludeTags="EndToEndTest" Upload-Test-Report: