From eb9d24ac5494af33581b7f6b9b53a57a1cc31517 Mon Sep 17 00:00:00 2001 From: etiennehomer Date: Tue, 26 Nov 2024 15:00:00 +0100 Subject: [PATCH] Use DNS for S3 storage + set storage root directory in application.yml (#51) Signed-off-by: Etienne Homer --- .../datasource/S3CaseDataSourceService.java | 4 +-- .../caseserver/service/CaseService.java | 2 ++ .../caseserver/service/FsCaseService.java | 21 +++++++++++++-- .../caseserver/service/S3CaseService.java | 26 ++++++++++++------- src/main/resources/application-local.yml | 10 +++---- src/main/resources/config/application.yaml | 9 ++++--- .../AbstractSupervisionControllerTest.java | 8 ++---- .../FsSupervisionControllerTest.java | 2 +- .../AbstractCaseDataSourceControllerTest.java | 4 --- .../FsCaseDataSourceControllerTest.java | 4 +-- .../service/AbstractCaseControllerTest.java | 10 +++---- .../service/MinioContainerConfig.java | 2 +- 12 files changed, 58 insertions(+), 44 deletions(-) diff --git a/src/main/java/com/powsybl/caseserver/datasource/S3CaseDataSourceService.java b/src/main/java/com/powsybl/caseserver/datasource/S3CaseDataSourceService.java index 87030ad..6becf98 100644 --- a/src/main/java/com/powsybl/caseserver/datasource/S3CaseDataSourceService.java +++ b/src/main/java/com/powsybl/caseserver/datasource/S3CaseDataSourceService.java @@ -54,11 +54,11 @@ public byte[] getInputStream(UUID caseUuid, String fileName) { // For archived cases (.zip, .tar, ...), individual files are gzipped in S3 server. // Here the requested file is decompressed and simply returned. if (S3CaseService.isArchivedCaseFile(caseName)) { - caseFileKey = uuidToKeyWithFileName(caseUuid, fileName + GZIP_EXTENSION); + caseFileKey = s3CaseService.uuidToKeyWithFileName(caseUuid, fileName + GZIP_EXTENSION); return s3CaseService.withS3DownloadedTempPath(caseUuid, caseFileKey, file -> S3CaseService.decompress(Files.readAllBytes(file))); } else { - caseFileKey = uuidToKeyWithFileName(caseUuid, caseName); + caseFileKey = s3CaseService.uuidToKeyWithFileName(caseUuid, caseName); return s3CaseService.withS3DownloadedTempPath(caseUuid, caseFileKey, casePath -> IOUtils.toByteArray(DataSource.fromPath(casePath).newInputStream(fileName))); } diff --git a/src/main/java/com/powsybl/caseserver/service/CaseService.java b/src/main/java/com/powsybl/caseserver/service/CaseService.java index 1cd3ca6..97574fa 100644 --- a/src/main/java/com/powsybl/caseserver/service/CaseService.java +++ b/src/main/java/com/powsybl/caseserver/service/CaseService.java @@ -168,4 +168,6 @@ default List getCasesToReindex() { CaseMetadataRepository getCaseMetadataRepository(); ComputationManager getComputationManager(); + + String getRootDirectory(); } diff --git a/src/main/java/com/powsybl/caseserver/service/FsCaseService.java b/src/main/java/com/powsybl/caseserver/service/FsCaseService.java index 47f74fe..417daf3 100644 --- a/src/main/java/com/powsybl/caseserver/service/FsCaseService.java +++ b/src/main/java/com/powsybl/caseserver/service/FsCaseService.java @@ -15,6 +15,7 @@ import com.powsybl.computation.local.LocalComputationManager; import com.powsybl.iidm.network.Importer; import com.powsybl.iidm.network.Network; +import jakarta.annotation.PostConstruct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -35,6 +36,7 @@ import java.util.stream.Stream; import static com.powsybl.caseserver.CaseException.createDirectoryNotFound; +import static com.powsybl.caseserver.service.S3CaseService.DELIMITER; /** * @author Abdelsalem Hedhili @@ -59,13 +61,28 @@ public class FsCaseService implements CaseService { @Autowired private CaseInfosService caseInfosService; - @Value("${case-store-directory:#{systemProperties['user.home'].concat(\"/cases\")}}") + @Value("${case-home:#{systemProperties['user.home']}}") + private String caseHome; + + @Value("${case-subpath}") + private String caseSubpath; + private String rootDirectory; public FsCaseService(CaseMetadataRepository caseMetadataRepository) { this.caseMetadataRepository = caseMetadataRepository; } + @PostConstruct + private void postConstruct() { + rootDirectory = caseHome + DELIMITER + caseSubpath; + } + + @Override + public String getRootDirectory() { + return rootDirectory; + } + @Override public String getFormat(UUID caseUuid) { Path file = getCaseFile(caseUuid); @@ -300,7 +317,7 @@ public void deleteAllCases() { } public Path getStorageRootDir() { - return fileSystem.getPath(rootDirectory); + return fileSystem.getPath(getRootDirectory()); } private boolean isStorageCreated() { diff --git a/src/main/java/com/powsybl/caseserver/service/S3CaseService.java b/src/main/java/com/powsybl/caseserver/service/S3CaseService.java index 65dbbce..2ab037c 100644 --- a/src/main/java/com/powsybl/caseserver/service/S3CaseService.java +++ b/src/main/java/com/powsybl/caseserver/service/S3CaseService.java @@ -80,7 +80,8 @@ public class S3CaseService implements CaseService { @Value("${spring.cloud.aws.bucket}") private String bucketName; - private static final String CASES_PREFIX = "gsi-cases/"; + @Value("${case-subpath}") + private String rootDirectory; public static final String NOT_FOUND = " not found"; @@ -91,6 +92,11 @@ public S3CaseService(CaseMetadataRepository caseMetadataRepository) { this.caseMetadataRepository = caseMetadataRepository; } + @Override + public String getRootDirectory() { + return rootDirectory; + } + String getFormat(Path caseFile) { Importer importer = getImporterOrThrowsException(caseFile); return importer.getFormat(); @@ -172,7 +178,7 @@ public String getOriginalFilename(UUID caseUuid) { return getCaseMetaDataEntity(caseUuid).getOriginalFilename(); } - // key format is "gsi-cases/UUID/path/to/file" + // key format is "/UUID/path/to/file" private UUID parseUuidFromKey(String key) { int firstSlash = key.indexOf(DELIMITER); int secondSlash = key.indexOf(DELIMITER, firstSlash + 1); @@ -185,11 +191,11 @@ private String parseFilenameFromKey(String key) { return key.substring(secondSlash + 1); } - public static String uuidToKeyPrefix(UUID uuid) { - return CASES_PREFIX + uuid.toString() + DELIMITER; + public String uuidToKeyPrefix(UUID uuid) { + return rootDirectory + DELIMITER + uuid.toString() + DELIMITER; } - public static String uuidToKeyWithFileName(UUID uuid, String filename) { + public String uuidToKeyWithFileName(UUID uuid, String filename) { return uuidToKeyPrefix(uuid) + filename; } @@ -257,7 +263,7 @@ public Optional getCaseBytes(UUID caseUuid) { public List getCases() { List caseInfosList = new ArrayList<>(); CaseInfos caseInfos; - for (S3Object o : getCaseS3Objects(CASES_PREFIX)) { + for (S3Object o : getCaseS3Objects(rootDirectory + DELIMITER)) { caseInfos = getCaseInfos(parseUuidFromKey(o.key())); if (Objects.nonNull(caseInfos)) { caseInfosList.add(caseInfos); @@ -322,7 +328,7 @@ public Set listName(UUID caseUuid, String regex) { filenames = List.of(removeExtension(originalFilename, "." + getCompressionFormat(caseUuid))); } else { List s3Objects = getCaseS3Objects(caseUuid); - filenames = s3Objects.stream().map(obj -> Paths.get(obj.key()).toString().replace(CASES_PREFIX + caseUuid.toString() + DELIMITER, "")).toList(); + filenames = s3Objects.stream().map(obj -> Paths.get(obj.key()).toString().replace(rootDirectory + DELIMITER + caseUuid.toString() + DELIMITER, "")).toList(); // For archived cases : if (isArchivedCaseFile(originalFilename)) { filenames = filenames.stream() @@ -414,9 +420,9 @@ private void copyEntry(UUID sourcecaseUuid, UUID caseUuid, String fileName) { // To optimize copy, files to copy are not downloaded on the case-server. They are directly copied on the S3 server. CopyObjectRequest copyObjectRequest = CopyObjectRequest.builder() .sourceBucket(bucketName) - .sourceKey(CASES_PREFIX + sourcecaseUuid + DELIMITER + fileName) + .sourceKey(rootDirectory + DELIMITER + sourcecaseUuid + DELIMITER + fileName) .destinationBucket(bucketName) - .destinationKey(CASES_PREFIX + caseUuid + DELIMITER + fileName) + .destinationKey(rootDirectory + DELIMITER + caseUuid + DELIMITER + fileName) .build(); try { s3Client.copyObject(copyObjectRequest); @@ -529,7 +535,7 @@ public void deleteCase(UUID caseUuid) { public void deleteAllCases() { ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder() .bucket(bucketName) - .prefix(CASES_PREFIX) + .prefix(rootDirectory + DELIMITER) .build(); ListObjectsV2Response listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index c73be09..869d277 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -11,14 +11,12 @@ spring: ## to fill if authentication is needed # user: # password: + cloud: + aws: + endpoint: http://localhost:19000 powsybl-ws: database: host: localhost -cleaning-cases-cron: 0 * 2 * * ? - -cloud: - aws: - s3: - endpoint: 172.17.0.1 +cleaning-cases-cron: 0 * 2 * * ? \ No newline at end of file diff --git a/src/main/resources/config/application.yaml b/src/main/resources/config/application.yaml index 2a27af7..d1315ff 100644 --- a/src/main/resources/config/application.yaml +++ b/src/main/resources/config/application.yaml @@ -11,12 +11,13 @@ spring: aws: s3: path-style-access-enabled: true - endpoint: http://172.17.0.1:19000 + # classic minio port, useful default for exploring + endpoint: http://s3-storage:9000 region: profile: name: default static: test - bucket: bucket-gridsuite + bucket: my-bucket credentials: access-key: minioadmin secret-key: minioadmin @@ -28,4 +29,6 @@ powsybl-ws: cleaning-cases-cron: 0 * 2 * * ? storage: - type: FS # FS or S3 \ No newline at end of file + type: FS # FS or S3 + +case-subpath: cases diff --git a/src/test/java/com/powsybl/caseserver/AbstractSupervisionControllerTest.java b/src/test/java/com/powsybl/caseserver/AbstractSupervisionControllerTest.java index 5adb997..0bea637 100644 --- a/src/test/java/com/powsybl/caseserver/AbstractSupervisionControllerTest.java +++ b/src/test/java/com/powsybl/caseserver/AbstractSupervisionControllerTest.java @@ -12,7 +12,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.MediaType; @@ -33,7 +32,7 @@ * @author Jamal KHEYYAD */ @AutoConfigureMockMvc -@SpringBootTest(classes = {CaseApplication.class}, properties = {"case-store-directory=/cases"}) +@SpringBootTest(classes = {CaseApplication.class}) abstract class AbstractSupervisionControllerTest { @Autowired private SupervisionService supervisionService; @@ -43,9 +42,6 @@ abstract class AbstractSupervisionControllerTest { @Autowired protected MockMvc mockMvc; - @Value("${case-store-directory:#{systemProperties['user.home'].concat(\"/cases\")}}") - String rootDirectory; - private static final String TEST_CASE = "testCase.xiidm"; FileSystem fileSystem; @@ -112,7 +108,7 @@ void tearDown() throws Exception { } private void createStorageDir() throws IOException { - Path path = fileSystem.getPath(rootDirectory); + Path path = fileSystem.getPath(caseService.getRootDirectory()); if (!Files.exists(path)) { Files.createDirectories(path); } diff --git a/src/test/java/com/powsybl/caseserver/FsSupervisionControllerTest.java b/src/test/java/com/powsybl/caseserver/FsSupervisionControllerTest.java index 14e3b65..84ced64 100644 --- a/src/test/java/com/powsybl/caseserver/FsSupervisionControllerTest.java +++ b/src/test/java/com/powsybl/caseserver/FsSupervisionControllerTest.java @@ -21,7 +21,7 @@ * @author Jamal KHEYYAD */ @AutoConfigureMockMvc -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK, properties = {"case-store-directory=/cases"}) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK) @TestPropertySource(properties = {"storage.type=FS"}) class FsSupervisionControllerTest extends AbstractSupervisionControllerTest { diff --git a/src/test/java/com/powsybl/caseserver/datasource/AbstractCaseDataSourceControllerTest.java b/src/test/java/com/powsybl/caseserver/datasource/AbstractCaseDataSourceControllerTest.java index 45f91bc..f0fafce 100644 --- a/src/test/java/com/powsybl/caseserver/datasource/AbstractCaseDataSourceControllerTest.java +++ b/src/test/java/com/powsybl/caseserver/datasource/AbstractCaseDataSourceControllerTest.java @@ -12,7 +12,6 @@ import com.powsybl.commons.datasource.DataSource; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.cloud.stream.function.StreamBridge; import org.springframework.mock.web.MockMultipartFile; @@ -48,9 +47,6 @@ public abstract class AbstractCaseDataSourceControllerTest { protected static CaseService caseService; - @Value("${case-store-directory:#{systemProperties['user.home'].concat(\"/cases\")}}") - protected String rootDirectory; - static final String CGMES_ZIP_NAME = "CGMES_v2415_MicroGridTestConfiguration_BC_BE_v2.zip"; static final String CGMES_FILE_NAME = "CGMES_v2415_MicroGridTestConfiguration_BC_BE_v2/MicroGridTestConfiguration_BC_BE_DL_V2.xml"; diff --git a/src/test/java/com/powsybl/caseserver/datasource/FsCaseDataSourceControllerTest.java b/src/test/java/com/powsybl/caseserver/datasource/FsCaseDataSourceControllerTest.java index 4b89b22..257262b 100644 --- a/src/test/java/com/powsybl/caseserver/datasource/FsCaseDataSourceControllerTest.java +++ b/src/test/java/com/powsybl/caseserver/datasource/FsCaseDataSourceControllerTest.java @@ -40,11 +40,11 @@ class FsCaseDataSourceControllerTest extends AbstractCaseDataSourceControllerTes void setUp() throws URISyntaxException, IOException { caseService = fsCaseService; cgmesCaseUuid = UUID.randomUUID(); - Path path = fileSystem.getPath(rootDirectory); + Path path = fileSystem.getPath(caseService.getRootDirectory()); if (!Files.exists(path)) { Files.createDirectories(path); } - Path cgmesCaseDirectory = fileSystem.getPath(rootDirectory).resolve(cgmesCaseUuid.toString()); + Path cgmesCaseDirectory = fileSystem.getPath(caseService.getRootDirectory()).resolve(cgmesCaseUuid.toString()); if (!Files.exists(cgmesCaseDirectory)) { Files.createDirectories(cgmesCaseDirectory); } diff --git a/src/test/java/com/powsybl/caseserver/service/AbstractCaseControllerTest.java b/src/test/java/com/powsybl/caseserver/service/AbstractCaseControllerTest.java index 06e4f16..15e50e5 100644 --- a/src/test/java/com/powsybl/caseserver/service/AbstractCaseControllerTest.java +++ b/src/test/java/com/powsybl/caseserver/service/AbstractCaseControllerTest.java @@ -18,7 +18,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.cloud.stream.binder.test.OutputDestination; @@ -55,7 +54,7 @@ * @author Franck Lecuyer */ @AutoConfigureMockMvc -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK, properties = {"case-store-directory=/cases"}) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK) @ContextConfigurationWithTestChannel abstract class AbstractCaseControllerTest { private static final String TEST_CASE = "testCase.xiidm"; @@ -81,9 +80,6 @@ abstract class AbstractCaseControllerTest { @Autowired private ObjectMapper mapper; - @Value("${case-store-directory:#{systemProperties['user.home'].concat(\"/cases\")}}") - String rootDirectory; - FileSystem fileSystem; private final String caseImportDestination = "case.import.destination"; @@ -96,7 +92,7 @@ public void tearDown() throws Exception { } private void createStorageDir() throws IOException { - Path path = fileSystem.getPath(rootDirectory); + Path path = fileSystem.getPath(caseService.getRootDirectory()); if (!Files.exists(path)) { Files.createDirectories(path); } @@ -726,7 +722,7 @@ private static String getDateSearchTerm(String entsoeFormatDate) { @Test void invalidFileInCaseDirectoryShouldBeIgnored() throws Exception { createStorageDir(); - Path filePath = fileSystem.getPath(rootDirectory).resolve("randomFile.txt"); + Path filePath = fileSystem.getPath(caseService.getRootDirectory()).resolve("randomFile.txt"); Files.createFile(filePath); importCase(TEST_CASE, false); diff --git a/src/test/java/com/powsybl/caseserver/service/MinioContainerConfig.java b/src/test/java/com/powsybl/caseserver/service/MinioContainerConfig.java index 50300d2..6fa8eae 100644 --- a/src/test/java/com/powsybl/caseserver/service/MinioContainerConfig.java +++ b/src/test/java/com/powsybl/caseserver/service/MinioContainerConfig.java @@ -19,7 +19,7 @@ */ public interface MinioContainerConfig { String MINIO_DOCKER_IMAGE_NAME = "minio/minio"; - String BUCKET_NAME = "bucket-gridsuite"; + String BUCKET_NAME = "my-bucket"; // Just a fixed version, latest at the time of writing this String MINIO_DOCKER_IMAGE_VERSION = "RELEASE.2023-09-27T15-22-50Z"; int MINIO_PORT = 9000;