From 4e569084cc811d385c448e6400919a446be43312 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Thu, 5 Dec 2024 08:47:13 +0100 Subject: [PATCH 01/13] Add raw data saving --- src/main/java/fr/insee/genesis/Constants.java | 1 + .../rest/responses/ResponseController.java | 117 +++++++++++++++++- .../rawdata/LunaticXmlDataModel.java | 16 +++ .../domain/ports/api/RawDataApiPort.java | 8 ++ .../ports/spi/LunaticXmlPersistancePort.java | 9 ++ .../rawdata/LunaticXmlRawDataService.java | 34 +++++ .../adapter/LunaticXmlMongoAdapter.java | 35 ++++++ .../rawdata/xml/LunaticXmlDataDocument.java | 26 ++++ .../document/schedule/ScheduleDocument.java | 3 +- .../surveyunit/SurveyUnitDocument.java | 3 +- .../mappers/LunaticXmlDocumentMapper.java | 25 ++++ .../LunaticXmlMongoDBRepository.java | 9 ++ .../responses/ResponseControllerTest.java | 39 ++++++ .../stubs/LunaticXmlPersistanceStub.java | 25 ++++ 14 files changed, 343 insertions(+), 7 deletions(-) create mode 100644 src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java create mode 100644 src/main/java/fr/insee/genesis/domain/ports/api/RawDataApiPort.java create mode 100644 src/main/java/fr/insee/genesis/domain/ports/spi/LunaticXmlPersistancePort.java create mode 100644 src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticXmlRawDataService.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/document/rawdata/xml/LunaticXmlDataDocument.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticXmlDocumentMapper.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/repository/LunaticXmlMongoDBRepository.java create mode 100644 src/test/java/fr/insee/genesis/stubs/LunaticXmlPersistanceStub.java diff --git a/src/main/java/fr/insee/genesis/Constants.java b/src/main/java/fr/insee/genesis/Constants.java index f1711c2e..307c1ecf 100644 --- a/src/main/java/fr/insee/genesis/Constants.java +++ b/src/main/java/fr/insee/genesis/Constants.java @@ -15,6 +15,7 @@ public class Constants { public static final String MONGODB_SCHEDULE_COLLECTION_NAME = "schedules"; public static final String LOOP_NAME_PREFIX = "BOUCLE"; public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses"; + public static final String MONGODB_LUNATIC_XML_DATA_COLLECTION_NAME = "lunaticxmldata"; public static final String VOLUMETRY_FOLDER_NAME = "genesis_volumetries"; public static final String VOLUMETRY_FILE_SUFFIX = "_VOLUMETRY"; public static final String VOLUMETRY_FILE_DATE_FORMAT = "yyyy_MM_dd"; diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 98f8ba93..86051fc2 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -14,10 +14,12 @@ import fr.insee.genesis.controller.sources.xml.LunaticXmlDataSequentialParser; import fr.insee.genesis.controller.sources.xml.LunaticXmlSurveyUnit; import fr.insee.genesis.controller.utils.ControllerUtils; +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel; import fr.insee.genesis.domain.model.surveyunit.CollectedVariable; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.Variable; +import fr.insee.genesis.domain.ports.api.RawDataApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.exceptions.GenesisError; @@ -62,15 +64,23 @@ public class ResponseController { private static final String DDI_REGEX = "ddi[\\w,\\s-]+\\.xml"; public static final String S_S = "%s/%s"; + private static final String CAMPAIGN_ERROR = "Error for campaign {} : {}"; + private static final String SUCCESS_MESSAGE = "Data saved"; private final SurveyUnitApiPort surveyUnitService; private final SurveyUnitQualityService surveyUnitQualityService; + private final RawDataApiPort rawDataApiPort; private final FileUtils fileUtils; private final ControllerUtils controllerUtils; @Autowired - public ResponseController(SurveyUnitApiPort surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, FileUtils fileUtils, ControllerUtils controllerUtils) { + public ResponseController(SurveyUnitApiPort surveyUnitService, + SurveyUnitQualityService surveyUnitQualityService, + RawDataApiPort rawDataApiPort, + FileUtils fileUtils, + ControllerUtils controllerUtils) { this.surveyUnitService = surveyUnitService; this.surveyUnitQualityService = surveyUnitQualityService; + this.rawDataApiPort = rawDataApiPort; this.fileUtils = fileUtils; this.controllerUtils = controllerUtils; } @@ -131,18 +141,58 @@ public ResponseEntity saveResponsesFromXmlCampaignFolder(@RequestParam(" //Don't stop if NoDataError thrown log.warn(nde.getMessage()); }catch (Exception e){ - log.error("Error for campaign {} : {}", campaignName, e.toString()); + log.error(CAMPAIGN_ERROR, campaignName, e.toString()); return ResponseEntity.status(500).body(e.getMessage()); } } if (errors.isEmpty()){ - return ResponseEntity.ok(isAnyDataSaved ? "Data saved" : "No data has been saved"); + return ResponseEntity.ok(isAnyDataSaved ? SUCCESS_MESSAGE : "No data has been saved"); }else{ return ResponseEntity.internalServerError().body(errors.getFirst().getMessage()); } } + @Operation(summary = "Save one file of raw responses to Genesis Database, passing its path as a parameter") + @PutMapping(path = "/lunatic-xml/raw/save-one") + public ResponseEntity saveRawResponsesFromXmlFile(@RequestParam("pathLunaticXml") String xmlFile, + @RequestParam(value = "pathSpecFile") String metadataFilePath, + @RequestParam(value = "mode") Mode modeSpecified + )throws Exception { + log.info(String.format("Try to read Xml file : %s", xmlFile)); + Path filepath = Paths.get(xmlFile); + + return treatRawXmlFile(filepath, modeSpecified); + } + + @Operation(summary = "Save multiple raw files to Genesis Database from the campaign root folder") + @PutMapping(path = "/lunatic-xml/raw/save-folder") + public ResponseEntity saveRawResponsesFromXmlCampaignFolder(@RequestParam("campaignName") String campaignName, + @RequestParam(value = "mode", required = false) Mode modeSpecified + )throws Exception { + boolean isAnyDataSaved = false; + + log.info("Try to import raw lunatic XML data for campaign : {}", campaignName); + + List modesList = controllerUtils.getModesList(campaignName, modeSpecified); + for (Mode currentMode : modesList) { + try { + treatRawCampaignWithMode(campaignName, currentMode, null); + isAnyDataSaved = true; + }catch (NoDataException nde){ + //Don't stop if NoDataError thrown + log.warn(nde.getMessage()); + }catch (Exception e){ + log.error(CAMPAIGN_ERROR, campaignName, e.toString()); + return ResponseEntity.status(500).body(e.getMessage()); + } + } + + return ResponseEntity.ok(isAnyDataSaved ? SUCCESS_MESSAGE : "No data has been saved"); + } + + + //SAVE ALL @Operation(summary = "Save all files to Genesis Database (differential data folder only), regardless of the campaign") @PutMapping(path = "/lunatic-xml/save-all-campaigns") @@ -167,12 +217,12 @@ public ResponseEntity saveResponsesFromAllCampaignFolders(){ log.warn(nde.getMessage()); } catch (Exception e) { - log.warn("Error for campaign {} : {}", campaignName, e.toString()); + log.warn(CAMPAIGN_ERROR, campaignName, e.toString()); errors.add(new GenesisError(e.getMessage())); } } if (errors.isEmpty()) { - return ResponseEntity.ok("Data saved"); + return ResponseEntity.ok(SUCCESS_MESSAGE); } else { return ResponseEntity.status(209).body("Data saved with " + errors.size() + " errors"); } @@ -373,6 +423,43 @@ private void treatCampaignWithMode(String campaignName, Mode mode, List dataFiles = fileUtils.listFiles(dataFolder); + log.info("Numbers of files to load in folder {} : {}", dataFolder, dataFiles.size()); + if (dataFiles.isEmpty()) { + throw new NoDataException("No data file found in folder %s".formatted(dataFolder)); + } + + //For each XML data file + for (String fileName : dataFiles.stream().filter(s -> s.endsWith(".xml")).toList()) { + String filepathString = String.format(S_S, dataFolder, fileName); + Path filepath = Paths.get(filepathString); + //Check if file not in done folder, delete if true + if(isDataFileInDoneFolder(filepath, campaignName, mode.getFolder())){ + log.warn("File {} already exists in DONE folder ! Deleting...", fileName); + Files.deleteIfExists(filepath); + }else{ + //Read file + log.info("Try to read Xml file : {}", fileName); + ResponseEntity response = treatRawXmlFile(filepath, mode); + + log.debug("File {} saved", fileName); + if(response.getStatusCode() == HttpStatus.OK){ + fileUtils.moveDataFile(campaignName, mode.getFolder(), filepath); + }else{ + log.error("Error on file {}", fileName); + } + } + } + } + private VariablesMap readMetadatas(String campaignName, Mode mode, List errors, boolean withDDI) { VariablesMap variablesMap; if(withDDI){ @@ -431,6 +518,26 @@ private ResponseEntity treatXmlFileWithMemory(Path filepath, Mode modeSp return ResponseEntity.ok().build(); } + private ResponseEntity treatRawXmlFile(Path filepath, Mode modeSpecified) throws IOException, + ParserConfigurationException, SAXException { + LunaticXmlCampaign campaign; + // DOM method + LunaticXmlDataParser parser = new LunaticXmlDataParser(); + try { + campaign = parser.parseDataFile(filepath); + } catch (GenesisException e) { + log.error(e.toString()); + return ResponseEntity.status(e.getStatus()).body(e.getMessage()); + } + + log.debug("Begin saving raw xml data file {}", filepath); + rawDataApiPort.saveData(campaign, modeSpecified); + log.debug(SUCCESS_MESSAGE); + + log.info("File {} treated" , filepath.getFileName()); + return ResponseEntity.ok().build(); + } + private ResponseEntity treatXmlFileSequentially(Path filepath, Mode modeSpecified, VariablesMap variablesMap) throws IOException, XMLStreamException { LunaticXmlCampaign campaign; //Sequential method diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java new file mode 100644 index 00000000..59319574 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java @@ -0,0 +1,16 @@ +package fr.insee.genesis.domain.model.surveyunit.rawdata; + +import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import lombok.Builder; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@Builder +public class LunaticXmlDataModel{ + private Mode mode; + private LunaticXmlCampaign data; + private LocalDateTime recordDate; +} diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/RawDataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/RawDataApiPort.java new file mode 100644 index 00000000..bd15e4f1 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/api/RawDataApiPort.java @@ -0,0 +1,8 @@ +package fr.insee.genesis.domain.ports.api; + +import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign; +import fr.insee.genesis.domain.model.surveyunit.Mode; + +public interface RawDataApiPort { + void saveData(LunaticXmlCampaign data, Mode mode); +} diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticXmlPersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticXmlPersistancePort.java new file mode 100644 index 00000000..1fc6a11c --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticXmlPersistancePort.java @@ -0,0 +1,9 @@ +package fr.insee.genesis.domain.ports.spi; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel; + +import java.util.List; + +public interface LunaticXmlPersistancePort { + void save(LunaticXmlDataModel lunaticXmlDataModels); +} diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticXmlRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticXmlRawDataService.java new file mode 100644 index 00000000..125e335e --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticXmlRawDataService.java @@ -0,0 +1,34 @@ +package fr.insee.genesis.domain.service.rawdata; + +import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel; +import fr.insee.genesis.domain.ports.api.RawDataApiPort; +import fr.insee.genesis.domain.ports.spi.LunaticXmlPersistancePort; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; + +@Service +public class LunaticXmlRawDataService implements RawDataApiPort { + @Qualifier("lunaticXmlMongoAdapter") + private final LunaticXmlPersistancePort lunaticXmlPersistancePort; + + @Autowired + public LunaticXmlRawDataService(LunaticXmlPersistancePort lunaticXmlPersistancePort) { + this.lunaticXmlPersistancePort = lunaticXmlPersistancePort; + } + + @Override + public void saveData(LunaticXmlCampaign campaign, Mode mode) { + LunaticXmlDataModel lunaticXmlDataModel = LunaticXmlDataModel.builder() + .mode(mode) + .data(campaign) + .recordDate(LocalDateTime.now()) + .build(); + + lunaticXmlPersistancePort.save(lunaticXmlDataModel); + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java new file mode 100644 index 00000000..1f63fda4 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java @@ -0,0 +1,35 @@ +package fr.insee.genesis.infrastructure.adapter; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel; +import fr.insee.genesis.domain.ports.spi.LunaticXmlPersistancePort; +import fr.insee.genesis.infrastructure.document.rawdata.xml.LunaticXmlDataDocument; +import fr.insee.genesis.infrastructure.mappers.LunaticXmlDocumentMapper; +import fr.insee.genesis.infrastructure.mappers.SurveyUnitDocumentMapper; +import fr.insee.genesis.infrastructure.repository.LunaticXmlMongoDBRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Slf4j +@Service +@Qualifier("lunaticXmlMongoAdapter") +public class LunaticXmlMongoAdapter implements LunaticXmlPersistancePort { + + private final LunaticXmlMongoDBRepository mongoRepository; + + @Autowired + public LunaticXmlMongoAdapter(LunaticXmlMongoDBRepository mongoRepository, MongoTemplate mongoTemplate) { + this.mongoRepository = mongoRepository; + } + + @Override + public void save(LunaticXmlDataModel lunaticXmlDataModel) { + LunaticXmlDataDocument document = LunaticXmlDocumentMapper.INSTANCE + .modelToDocument(lunaticXmlDataModel); + mongoRepository.insert(document); + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/xml/LunaticXmlDataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/xml/LunaticXmlDataDocument.java new file mode 100644 index 00000000..436cb57a --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/xml/LunaticXmlDataDocument.java @@ -0,0 +1,26 @@ +package fr.insee.genesis.infrastructure.document.rawdata.xml; + +import fr.insee.genesis.Constants; +import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.bson.types.ObjectId; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; + +import java.time.LocalDateTime; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Document(collection = Constants.MONGODB_LUNATIC_XML_DATA_COLLECTION_NAME) +public class LunaticXmlDataDocument { + @Id + private ObjectId id; + private LunaticXmlCampaign lunaticXmlData; + private Mode mode; + private LocalDateTime recordDate; +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/schedule/ScheduleDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/schedule/ScheduleDocument.java index e93a7d6e..4843f679 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/schedule/ScheduleDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/schedule/ScheduleDocument.java @@ -1,5 +1,6 @@ package fr.insee.genesis.infrastructure.document.schedule; +import fr.insee.genesis.Constants; import fr.insee.genesis.domain.model.schedule.KraftwerkExecutionSchedule; import lombok.AllArgsConstructor; import lombok.Data; @@ -14,7 +15,7 @@ @Data @NoArgsConstructor @AllArgsConstructor -@Document(collection = "schedules") +@Document(collection = Constants.MONGODB_SCHEDULE_COLLECTION_NAME) public class ScheduleDocument { public ScheduleDocument(String surveyName, List kraftwerkExecutionScheduleList) { diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java index 49e8a7f1..1589c5cf 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java @@ -1,5 +1,6 @@ package fr.insee.genesis.infrastructure.document.surveyunit; +import fr.insee.genesis.Constants; import lombok.Data; import org.springframework.data.mongodb.core.mapping.Document; @@ -7,7 +8,7 @@ import java.util.List; @Data -@Document(collection = "responses") +@Document(collection = Constants.MONGODB_RESPONSE_COLLECTION_NAME) public class SurveyUnitDocument { private String idCampaign; private String idUE; diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticXmlDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticXmlDocumentMapper.java new file mode 100644 index 00000000..f529e937 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticXmlDocumentMapper.java @@ -0,0 +1,25 @@ +package fr.insee.genesis.infrastructure.mappers; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel; +import fr.insee.genesis.infrastructure.document.rawdata.xml.LunaticXmlDataDocument; +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; +import org.mapstruct.factory.Mappers; + +import java.util.List; + +@Mapper +public interface LunaticXmlDocumentMapper { + LunaticXmlDocumentMapper INSTANCE = Mappers.getMapper(LunaticXmlDocumentMapper.class); + + @Mapping(source = "lunaticXmlData", target = "data") + LunaticXmlDataModel documentToModel(LunaticXmlDataDocument lunaticXmlDataDocument); + + @Mapping(source = "data", target = "lunaticXmlData") + LunaticXmlDataDocument modelToDocument(LunaticXmlDataModel lunaticXmlDataModel); + + List listDocumentToListModel(List lunaticXmlDataDocuments); + + List listModelToListDocument(List lunaticXmlDataModels); + +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticXmlMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticXmlMongoDBRepository.java new file mode 100644 index 00000000..0bc32df8 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticXmlMongoDBRepository.java @@ -0,0 +1,9 @@ +package fr.insee.genesis.infrastructure.repository; + +import fr.insee.genesis.infrastructure.document.rawdata.xml.LunaticXmlDataDocument; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface LunaticXmlMongoDBRepository extends MongoRepository { +} diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index 1da8d11a..55d78820 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -9,11 +9,14 @@ import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.ports.api.RawDataApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; +import fr.insee.genesis.domain.service.rawdata.LunaticXmlRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; +import fr.insee.genesis.stubs.LunaticXmlPersistanceStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -35,6 +38,7 @@ class ResponseControllerTest { //Given static ResponseController responseControllerStatic; static SurveyUnitPersistencePortStub surveyUnitPersistencePortStub; + static LunaticXmlPersistanceStub lunaticXmlPersistanceStub; static List surveyUnitIdList; //Constants @@ -46,10 +50,14 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService(surveyUnitPersistencePortStub); + lunaticXmlPersistanceStub = new LunaticXmlPersistanceStub(); + RawDataApiPort rawDataApiPort = new LunaticXmlRawDataService(lunaticXmlPersistanceStub); + FileUtils fileUtils = new FileUtils(new ConfigStub()); responseControllerStatic = new ResponseController( surveyUnitApiPort , new SurveyUnitQualityService() + , rawDataApiPort , fileUtils , new ControllerUtils(fileUtils) ); @@ -65,6 +73,8 @@ void reset() throws IOException { //When + Then + + //Survey units @Test void saveResponseFromXMLFileTest() throws Exception { responseControllerStatic.saveResponsesFromXmlFile( @@ -112,6 +122,34 @@ void saveResponsesFromXmlCampaignFolderTest_noData() throws Exception { Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).isEmpty(); } + //Raw data + @Test + void saveXmlRawDataFromFileTest() throws Exception { + lunaticXmlPersistanceStub.getMongoStub().clear(); + + responseControllerStatic.saveRawResponsesFromXmlFile( + Path.of(TestConstants.TEST_RESOURCES_DIRECTORY, "IN/WEB/SAMPLETEST-PARADATA-v1/reponse-platine/data.complete.validated.STPDv1.20231122164209.xml").toString() + , Path.of(TestConstants.TEST_RESOURCES_DIRECTORY, "specs/SAMPLETEST-PARADATA-v1/ddi-SAMPLETEST-PARADATA-v1.xml").toString() + , Mode.WEB + ); + + Assertions.assertThat(lunaticXmlPersistanceStub.getMongoStub()).isNotEmpty(); + Assertions.assertThat(lunaticXmlPersistanceStub.getMongoStub().getFirst().getRecordDate()).isNotNull(); + Assertions.assertThat(lunaticXmlPersistanceStub.getMongoStub().getFirst().getLunaticXmlData()).isNotNull(); + } + + @Test + void saveXmlRawDataFromFolderTest() throws Exception { + lunaticXmlPersistanceStub.getMongoStub().clear(); + + responseControllerStatic.saveRawResponsesFromXmlCampaignFolder( + "SAMPLETEST-PARADATA-v1" + , Mode.WEB + ); + + Assertions.assertThat(lunaticXmlPersistanceStub.getMongoStub()).isNotEmpty(); + } + @Test void saveResponsesFromAllCampaignFoldersTests(){ surveyUnitPersistencePortStub.getMongoStub().clear(); @@ -120,6 +158,7 @@ void saveResponsesFromAllCampaignFoldersTests(){ Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).isNotEmpty(); } + //Gets @Test void findResponsesByUEAndQuestionnaireTest() { ResponseEntity> response = responseControllerStatic.findResponsesByUEAndQuestionnaire(defaultIdUE, defaultIdQuest); diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticXmlPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticXmlPersistanceStub.java new file mode 100644 index 00000000..6e33e27d --- /dev/null +++ b/src/test/java/fr/insee/genesis/stubs/LunaticXmlPersistanceStub.java @@ -0,0 +1,25 @@ +package fr.insee.genesis.stubs; + +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel; +import fr.insee.genesis.domain.ports.spi.LunaticXmlPersistancePort; +import fr.insee.genesis.domain.ports.spi.SurveyUnitPersistencePort; +import fr.insee.genesis.infrastructure.document.rawdata.xml.LunaticXmlDataDocument; +import fr.insee.genesis.infrastructure.mappers.LunaticXmlDocumentMapper; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +@Getter +public class LunaticXmlPersistanceStub implements LunaticXmlPersistancePort { + List mongoStub = new ArrayList<>(); + + @Override + public void save(LunaticXmlDataModel lunaticXmlDataModel) { + mongoStub.add(LunaticXmlDocumentMapper.INSTANCE.modelToDocument(lunaticXmlDataModel)); + } +} From a1da64dfb342b24286c44cbe16503e50c4fbe6ad Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Thu, 5 Dec 2024 08:50:53 +0100 Subject: [PATCH 02/13] Some refactors --- .../rest/responses/ResponseController.java | 1 - .../responses/ResponseControllerTest.java | 48 +++++++++---------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 86051fc2..9d6388f2 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -14,7 +14,6 @@ import fr.insee.genesis.controller.sources.xml.LunaticXmlDataSequentialParser; import fr.insee.genesis.controller.sources.xml.LunaticXmlSurveyUnit; import fr.insee.genesis.controller.utils.ControllerUtils; -import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel; import fr.insee.genesis.domain.model.surveyunit.CollectedVariable; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index 55d78820..88e77aa0 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -42,8 +42,8 @@ class ResponseControllerTest { static List surveyUnitIdList; //Constants - static final String defaultIdUE = "TESTIDUE"; - static final String defaultIdQuest = "TESTIDQUESTIONNAIRE"; + static final String DEFAULT_ID_UE = "TESTIDUE"; + static final String DEFAULT_ID_QUEST = "TESTIDQUESTIONNAIRE"; @BeforeAll static void init() { @@ -63,7 +63,7 @@ static void init() { ); surveyUnitIdList = new ArrayList<>(); - surveyUnitIdList.add(new SurveyUnitId(defaultIdUE)); + surveyUnitIdList.add(new SurveyUnitId(DEFAULT_ID_UE)); } @BeforeEach @@ -161,28 +161,28 @@ void saveResponsesFromAllCampaignFoldersTests(){ //Gets @Test void findResponsesByUEAndQuestionnaireTest() { - ResponseEntity> response = responseControllerStatic.findResponsesByUEAndQuestionnaire(defaultIdUE, defaultIdQuest); + ResponseEntity> response = responseControllerStatic.findResponsesByUEAndQuestionnaire(DEFAULT_ID_UE, DEFAULT_ID_QUEST); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); - Assertions.assertThat(response.getBody().getFirst().getIdUE()).isEqualTo(defaultIdUE); - Assertions.assertThat(response.getBody().getFirst().getIdQuest()).isEqualTo(defaultIdQuest); + Assertions.assertThat(response.getBody().getFirst().getIdUE()).isEqualTo(DEFAULT_ID_UE); + Assertions.assertThat(response.getBody().getFirst().getIdQuest()).isEqualTo(DEFAULT_ID_QUEST); } @Test void findAllResponsesByQuestionnaireTest() { - Path path = Path.of(TestConstants.TEST_RESOURCES_DIRECTORY, "OUT", defaultIdQuest); + Path path = Path.of(TestConstants.TEST_RESOURCES_DIRECTORY, "OUT", DEFAULT_ID_QUEST); File dir = new File(String.valueOf(path)); FileSystemUtils.deleteRecursively(dir); - ResponseEntity response = responseControllerStatic.findAllResponsesByQuestionnaire(defaultIdQuest); + ResponseEntity response = responseControllerStatic.findAllResponsesByQuestionnaire(DEFAULT_ID_QUEST); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull(); Assertions.assertThat(Files.exists(path)).isTrue(); - File[] dir_contents = dir.listFiles(); - Assertions.assertThat(dir_contents).hasSize(1); - Assertions.assertThat(dir_contents[0].length()).isPositive().isNotNull(); + File[] dircontents = dir.listFiles(); + Assertions.assertThat(dircontents).hasSize(1); + Assertions.assertThat(dircontents[0].length()).isPositive().isNotNull(); FileSystemUtils.deleteRecursively(dir); dir.deleteOnExit(); } @@ -193,14 +193,14 @@ void getAllResponsesByQuestionnaireTestSequential() throws IOException { surveyUnitPersistencePortStub.getMongoStub().clear(); for (int i = 0; i < Constants.BATCH_SIZE + 2; i++) { - Utils.addAdditionalDtoToMongoStub("TESTIDCAMPAIGN", defaultIdUE + i, + Utils.addAdditionalDtoToMongoStub("TESTIDCAMPAIGN", DEFAULT_ID_UE + i, LocalDateTime.of(2023, 1, 1, 0, 0, 0), LocalDateTime.of(2024, 1, 1, 0, 0, 0), surveyUnitPersistencePortStub); } //When - ResponseEntity response = responseControllerStatic.findAllResponsesByQuestionnaire(defaultIdQuest); + ResponseEntity response = responseControllerStatic.findAllResponsesByQuestionnaire(DEFAULT_ID_QUEST); //Then Assertions.assertThat(response).isNotNull(); @@ -217,32 +217,32 @@ void getAllResponsesByQuestionnaireTestSequential() throws IOException { void getLatestByUETest() { Utils.addAdditionalDtoToMongoStub(surveyUnitPersistencePortStub); - ResponseEntity> response = responseControllerStatic.getLatestByUE(defaultIdUE, defaultIdQuest); + ResponseEntity> response = responseControllerStatic.getLatestByUE(DEFAULT_ID_UE, DEFAULT_ID_QUEST); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); - Assertions.assertThat(response.getBody().getFirst().getIdUE()).isEqualTo(defaultIdUE); - Assertions.assertThat(response.getBody().getFirst().getIdQuest()).isEqualTo(defaultIdQuest); + Assertions.assertThat(response.getBody().getFirst().getIdUE()).isEqualTo(DEFAULT_ID_UE); + Assertions.assertThat(response.getBody().getFirst().getIdQuest()).isEqualTo(DEFAULT_ID_QUEST); Assertions.assertThat(response.getBody().getFirst().getFileDate()).hasMonth(Month.FEBRUARY); } @Test void getLatestByUEOneObjectTest() { - ResponseEntity response = responseControllerStatic.getLatestByUEOneObject(defaultIdUE, defaultIdQuest, Mode.WEB); + ResponseEntity response = responseControllerStatic.getLatestByUEOneObject(DEFAULT_ID_UE, DEFAULT_ID_QUEST, Mode.WEB); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull(); - Assertions.assertThat(response.getBody().getIdUE()).isEqualTo(defaultIdUE); - Assertions.assertThat(response.getBody().getIdQuest()).isEqualTo(defaultIdQuest); + Assertions.assertThat(response.getBody().getIdUE()).isEqualTo(DEFAULT_ID_UE); + Assertions.assertThat(response.getBody().getIdQuest()).isEqualTo(DEFAULT_ID_QUEST); } @Test void getLatestForUEListTest() { - ResponseEntity> response = responseControllerStatic.getLatestForUEList(defaultIdQuest, surveyUnitIdList); + ResponseEntity> response = responseControllerStatic.getLatestForUEList(DEFAULT_ID_QUEST, surveyUnitIdList); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); - Assertions.assertThat(response.getBody().getFirst().getIdUE()).isEqualTo(defaultIdUE); + Assertions.assertThat(response.getBody().getFirst().getIdUE()).isEqualTo(DEFAULT_ID_UE); } // Perret tests @@ -280,8 +280,8 @@ void getLatestByStatesSurveyDataTest(){ //WHEN ResponseEntity response = responseControllerStatic.findResponsesByUEAndQuestionnaireLatestStates( - defaultIdUE, - defaultIdQuest + DEFAULT_ID_UE, + DEFAULT_ID_QUEST ); @@ -289,7 +289,7 @@ void getLatestByStatesSurveyDataTest(){ SurveyUnitDto surveyUnitDto = response.getBody(); Assertions.assertThat(surveyUnitDto).isNotNull(); - Assertions.assertThat(surveyUnitDto.getSurveyUnitId()).isEqualTo(defaultIdUE); + Assertions.assertThat(surveyUnitDto.getSurveyUnitId()).isEqualTo(DEFAULT_ID_UE); Assertions.assertThat(surveyUnitDto.getCollectedVariables().getFirst().getVariableName()) .isEqualTo("TESTIDVAR"); From 9bf21f066efd33c8d475c79fe555601d0283fc0c Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Thu, 5 Dec 2024 08:54:58 +0100 Subject: [PATCH 03/13] We don't use specs to save raw data --- .../genesis/controller/rest/responses/ResponseController.java | 1 - .../controller/rest/responses/ResponseControllerTest.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 9d6388f2..8906ee02 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -155,7 +155,6 @@ public ResponseEntity saveResponsesFromXmlCampaignFolder(@RequestParam(" @Operation(summary = "Save one file of raw responses to Genesis Database, passing its path as a parameter") @PutMapping(path = "/lunatic-xml/raw/save-one") public ResponseEntity saveRawResponsesFromXmlFile(@RequestParam("pathLunaticXml") String xmlFile, - @RequestParam(value = "pathSpecFile") String metadataFilePath, @RequestParam(value = "mode") Mode modeSpecified )throws Exception { log.info(String.format("Try to read Xml file : %s", xmlFile)); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index 88e77aa0..9cc46762 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -129,7 +129,6 @@ void saveXmlRawDataFromFileTest() throws Exception { responseControllerStatic.saveRawResponsesFromXmlFile( Path.of(TestConstants.TEST_RESOURCES_DIRECTORY, "IN/WEB/SAMPLETEST-PARADATA-v1/reponse-platine/data.complete.validated.STPDv1.20231122164209.xml").toString() - , Path.of(TestConstants.TEST_RESOURCES_DIRECTORY, "specs/SAMPLETEST-PARADATA-v1/ddi-SAMPLETEST-PARADATA-v1.xml").toString() , Mode.WEB ); From a79d4d524f7f7ef1e855bed8d168ec8d24ce1d98 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Thu, 5 Dec 2024 09:05:06 +0100 Subject: [PATCH 04/13] add id --- .../domain/model/surveyunit/rawdata/LunaticXmlDataModel.java | 2 ++ .../document/rawdata/xml/LunaticXmlDataDocument.java | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java index 59319574..ed7d43ef 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java @@ -4,12 +4,14 @@ import fr.insee.genesis.domain.model.surveyunit.Mode; import lombok.Builder; import lombok.Data; +import org.bson.types.ObjectId; import java.time.LocalDateTime; @Data @Builder public class LunaticXmlDataModel{ + private ObjectId id; private Mode mode; private LunaticXmlCampaign data; private LocalDateTime recordDate; diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/xml/LunaticXmlDataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/xml/LunaticXmlDataDocument.java index 436cb57a..d5599f1f 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/xml/LunaticXmlDataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/xml/LunaticXmlDataDocument.java @@ -3,7 +3,6 @@ import fr.insee.genesis.Constants; import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign; import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; From 3fc623147116bb3108dbc1a5c27e09a3cce4748b Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Thu, 5 Dec 2024 12:27:44 +0100 Subject: [PATCH 05/13] Add lunatic json raw import --- src/main/java/fr/insee/genesis/Constants.java | 1 + .../rest/responses/ResponseController.java | 33 ++++++++++-- .../sources/json/LunaticJsonDataResponse.java | 2 +- ...java => LunaticJsonExternalVariables.java} | 2 +- .../rawdata/LunaticJsonDataModel.java | 19 +++++++ .../ports/api/LunaticJsonRawDataApiPort.java | 8 +++ ...ort.java => LunaticXmlRawDataApiPort.java} | 2 +- .../ports/spi/LunaticJsonPersistancePort.java | 7 +++ .../rawdata/LunaticJsonRawDataService.java | 34 ++++++++++++ .../rawdata/LunaticXmlRawDataService.java | 4 +- .../adapter/LunaticJsonMongoAdapter.java | 31 +++++++++++ .../adapter/LunaticXmlMongoAdapter.java | 7 +-- .../rawdata/LunaticJsonDataDocument.java | 27 ++++++++++ .../{xml => }/LunaticXmlDataDocument.java | 2 +- .../mappers/LunaticJsonDocumentMapper.java | 52 +++++++++++++++++++ .../mappers/LunaticXmlDocumentMapper.java | 2 +- .../LunaticJsonMongoDBRepository.java | 9 ++++ .../LunaticXmlMongoDBRepository.java | 2 +- .../responses/ResponseControllerTest.java | 38 ++++++++++++-- .../stubs/LunaticJsonPersistanceStub.java | 20 +++++++ .../stubs/LunaticXmlPersistanceStub.java | 7 +-- 21 files changed, 282 insertions(+), 27 deletions(-) rename src/main/java/fr/insee/genesis/controller/sources/json/{LunaticJSonExternalVariables.java => LunaticJsonExternalVariables.java} (89%) create mode 100644 src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java create mode 100644 src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java rename src/main/java/fr/insee/genesis/domain/ports/api/{RawDataApiPort.java => LunaticXmlRawDataApiPort.java} (83%) create mode 100644 src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonPersistancePort.java create mode 100644 src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java rename src/main/java/fr/insee/genesis/infrastructure/document/rawdata/{xml => }/LunaticXmlDataDocument.java (92%) create mode 100644 src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticJsonDocumentMapper.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java create mode 100644 src/test/java/fr/insee/genesis/stubs/LunaticJsonPersistanceStub.java diff --git a/src/main/java/fr/insee/genesis/Constants.java b/src/main/java/fr/insee/genesis/Constants.java index 307c1ecf..013d5da7 100644 --- a/src/main/java/fr/insee/genesis/Constants.java +++ b/src/main/java/fr/insee/genesis/Constants.java @@ -16,6 +16,7 @@ public class Constants { public static final String LOOP_NAME_PREFIX = "BOUCLE"; public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses"; public static final String MONGODB_LUNATIC_XML_DATA_COLLECTION_NAME = "lunaticxmldata"; + public static final String MONGODB_LUNATIC_JSON_DATA_COLLECTION_NAME = "lunaticjsondata"; public static final String VOLUMETRY_FOLDER_NAME = "genesis_volumetries"; public static final String VOLUMETRY_FILE_SUFFIX = "_VOLUMETRY"; public static final String VOLUMETRY_FILE_DATE_FORMAT = "yyyy_MM_dd"; diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 8906ee02..5630f783 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -1,5 +1,6 @@ package fr.insee.genesis.controller.rest.responses; +import com.fasterxml.jackson.core.JsonProcessingException; import fr.insee.bpm.exceptions.MetadataParserException; import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.bpm.metadata.reader.ddi.DDIReader; @@ -18,7 +19,8 @@ import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.Variable; -import fr.insee.genesis.domain.ports.api.RawDataApiPort; +import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.domain.ports.api.LunaticXmlRawDataApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.exceptions.GenesisError; @@ -67,19 +69,22 @@ public class ResponseController { private static final String SUCCESS_MESSAGE = "Data saved"; private final SurveyUnitApiPort surveyUnitService; private final SurveyUnitQualityService surveyUnitQualityService; - private final RawDataApiPort rawDataApiPort; + private final LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort; + private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort; private final FileUtils fileUtils; private final ControllerUtils controllerUtils; @Autowired public ResponseController(SurveyUnitApiPort surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, - RawDataApiPort rawDataApiPort, + LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort, + LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort, FileUtils fileUtils, ControllerUtils controllerUtils) { this.surveyUnitService = surveyUnitService; this.surveyUnitQualityService = surveyUnitQualityService; - this.rawDataApiPort = rawDataApiPort; + this.lunaticXmlRawDataApiPort = lunaticXmlRawDataApiPort; + this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort; this.fileUtils = fileUtils; this.controllerUtils = controllerUtils; } @@ -189,6 +194,24 @@ public ResponseEntity saveRawResponsesFromXmlCampaignFolder(@RequestPara return ResponseEntity.ok(isAnyDataSaved ? SUCCESS_MESSAGE : "No data has been saved"); } + //JSON + @Operation(summary = "Save lunatic json data to Genesis Database from the campaign root folder") + @PutMapping(path = "/lunatic-json/raw/save") + public ResponseEntity saveRawResponsesFromJsonBody( + @RequestParam("campaignName") String campaignName, + @RequestParam(value = "mode", required = false) Mode modeSpecified, + @RequestBody String dataJson + ) { + log.info("Try to import raw lunatic JSON data for campaign : {}", campaignName); + try { + lunaticJsonRawDataApiPort.saveData(campaignName, dataJson, modeSpecified); + }catch (JsonProcessingException jpe){ + log.error(jpe.toString()); + return ResponseEntity.badRequest().body("Invalid JSON synthax"); + } + log.info("Data saved for {}", campaignName); + return ResponseEntity.ok(SUCCESS_MESSAGE); + } //SAVE ALL @@ -529,7 +552,7 @@ private ResponseEntity treatRawXmlFile(Path filepath, Mode modeSpecified } log.debug("Begin saving raw xml data file {}", filepath); - rawDataApiPort.saveData(campaign, modeSpecified); + lunaticXmlRawDataApiPort.saveData(campaign, modeSpecified); log.debug(SUCCESS_MESSAGE); log.info("File {} treated" , filepath.getFileName()); diff --git a/src/main/java/fr/insee/genesis/controller/sources/json/LunaticJsonDataResponse.java b/src/main/java/fr/insee/genesis/controller/sources/json/LunaticJsonDataResponse.java index d45568ad..bf066240 100644 --- a/src/main/java/fr/insee/genesis/controller/sources/json/LunaticJsonDataResponse.java +++ b/src/main/java/fr/insee/genesis/controller/sources/json/LunaticJsonDataResponse.java @@ -9,7 +9,7 @@ public class LunaticJsonDataResponse { @JsonProperty("EXTERNAL") - private LunaticJSonExternalVariables externalVariables; + private LunaticJsonExternalVariables externalVariables; @JsonProperty("COLLECTED") private LunaticJsonCollectedVariables collectedVariables; diff --git a/src/main/java/fr/insee/genesis/controller/sources/json/LunaticJSonExternalVariables.java b/src/main/java/fr/insee/genesis/controller/sources/json/LunaticJsonExternalVariables.java similarity index 89% rename from src/main/java/fr/insee/genesis/controller/sources/json/LunaticJSonExternalVariables.java rename to src/main/java/fr/insee/genesis/controller/sources/json/LunaticJsonExternalVariables.java index b803ba1f..26c2123d 100644 --- a/src/main/java/fr/insee/genesis/controller/sources/json/LunaticJSonExternalVariables.java +++ b/src/main/java/fr/insee/genesis/controller/sources/json/LunaticJsonExternalVariables.java @@ -7,7 +7,7 @@ import java.util.Map; @Data -public class LunaticJSonExternalVariables { +public class LunaticJsonExternalVariables { private Map externalMap = new HashMap<>(); @JsonAnySetter diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java new file mode 100644 index 00000000..51231255 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java @@ -0,0 +1,19 @@ +package fr.insee.genesis.domain.model.surveyunit.rawdata; + +import fr.insee.genesis.domain.model.surveyunit.Mode; +import lombok.Builder; +import lombok.Data; +import org.bson.types.ObjectId; + +import java.time.LocalDateTime; + +@Data +@Builder +public class LunaticJsonDataModel { + private ObjectId id; + private String campaignId; + private Mode mode; + private String dataJson; + private LocalDateTime recordDate; + private LocalDateTime processDate; +} diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java new file mode 100644 index 00000000..1180e34f --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java @@ -0,0 +1,8 @@ +package fr.insee.genesis.domain.ports.api; + +import com.fasterxml.jackson.core.JsonProcessingException; +import fr.insee.genesis.domain.model.surveyunit.Mode; + +public interface LunaticJsonRawDataApiPort { + void saveData(String campaignName, String dataJson, Mode mode) throws JsonProcessingException; +} diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/RawDataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticXmlRawDataApiPort.java similarity index 83% rename from src/main/java/fr/insee/genesis/domain/ports/api/RawDataApiPort.java rename to src/main/java/fr/insee/genesis/domain/ports/api/LunaticXmlRawDataApiPort.java index bd15e4f1..95677a6f 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/RawDataApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticXmlRawDataApiPort.java @@ -3,6 +3,6 @@ import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign; import fr.insee.genesis.domain.model.surveyunit.Mode; -public interface RawDataApiPort { +public interface LunaticXmlRawDataApiPort { void saveData(LunaticXmlCampaign data, Mode mode); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonPersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonPersistancePort.java new file mode 100644 index 00000000..5885ff34 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonPersistancePort.java @@ -0,0 +1,7 @@ +package fr.insee.genesis.domain.ports.spi; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel; + +public interface LunaticJsonPersistancePort { + void save(LunaticJsonDataModel lunaticJsonDataModel); +} diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java new file mode 100644 index 00000000..cf9fe17c --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -0,0 +1,34 @@ +package fr.insee.genesis.domain.service.rawdata; + +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel; +import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.domain.ports.spi.LunaticJsonPersistancePort; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; + +@Service +public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort { + @Qualifier("lunaticJsonMongoAdapter") + private final LunaticJsonPersistancePort lunaticJsonPersistancePort; + + @Autowired + public LunaticJsonRawDataService(LunaticJsonPersistancePort lunaticJsonPersistancePort) { + this.lunaticJsonPersistancePort = lunaticJsonPersistancePort; + } + + @Override + public void saveData(String campaignName, String dataJson, Mode mode){ + LunaticJsonDataModel lunaticJsonDataModel = LunaticJsonDataModel.builder() + .campaignId(campaignName) + .mode(mode) + .dataJson(dataJson) + .recordDate(LocalDateTime.now()) + .build(); + + lunaticJsonPersistancePort.save(lunaticJsonDataModel); + } +} diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticXmlRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticXmlRawDataService.java index 125e335e..a1696737 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticXmlRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticXmlRawDataService.java @@ -3,7 +3,7 @@ import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel; -import fr.insee.genesis.domain.ports.api.RawDataApiPort; +import fr.insee.genesis.domain.ports.api.LunaticXmlRawDataApiPort; import fr.insee.genesis.domain.ports.spi.LunaticXmlPersistancePort; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -12,7 +12,7 @@ import java.time.LocalDateTime; @Service -public class LunaticXmlRawDataService implements RawDataApiPort { +public class LunaticXmlRawDataService implements LunaticXmlRawDataApiPort { @Qualifier("lunaticXmlMongoAdapter") private final LunaticXmlPersistancePort lunaticXmlPersistancePort; diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java new file mode 100644 index 00000000..29e1f893 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java @@ -0,0 +1,31 @@ +package fr.insee.genesis.infrastructure.adapter; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel; +import fr.insee.genesis.domain.ports.spi.LunaticJsonPersistancePort; +import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument; +import fr.insee.genesis.infrastructure.mappers.LunaticJsonDocumentMapper; +import fr.insee.genesis.infrastructure.repository.LunaticJsonMongoDBRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@Qualifier("lunaticJsonMongoAdapter") +public class LunaticJsonMongoAdapter implements LunaticJsonPersistancePort { + + private final LunaticJsonMongoDBRepository lunaticJsonMongoDBRepository; + + @Autowired + public LunaticJsonMongoAdapter(LunaticJsonMongoDBRepository lunaticJsonMongoDBRepository) { + this.lunaticJsonMongoDBRepository = lunaticJsonMongoDBRepository; + } + + @Override + public void save(LunaticJsonDataModel lunaticJsonDataModel) { + LunaticJsonDataDocument document = LunaticJsonDocumentMapper.INSTANCE + .modelToDocument(lunaticJsonDataModel); + lunaticJsonMongoDBRepository.insert(document); + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java index 1f63fda4..14c02073 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java @@ -2,9 +2,8 @@ import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel; import fr.insee.genesis.domain.ports.spi.LunaticXmlPersistancePort; -import fr.insee.genesis.infrastructure.document.rawdata.xml.LunaticXmlDataDocument; +import fr.insee.genesis.infrastructure.document.rawdata.LunaticXmlDataDocument; import fr.insee.genesis.infrastructure.mappers.LunaticXmlDocumentMapper; -import fr.insee.genesis.infrastructure.mappers.SurveyUnitDocumentMapper; import fr.insee.genesis.infrastructure.repository.LunaticXmlMongoDBRepository; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -12,8 +11,6 @@ import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.stereotype.Service; -import java.util.List; - @Slf4j @Service @Qualifier("lunaticXmlMongoAdapter") @@ -22,7 +19,7 @@ public class LunaticXmlMongoAdapter implements LunaticXmlPersistancePort { private final LunaticXmlMongoDBRepository mongoRepository; @Autowired - public LunaticXmlMongoAdapter(LunaticXmlMongoDBRepository mongoRepository, MongoTemplate mongoTemplate) { + public LunaticXmlMongoAdapter(LunaticXmlMongoDBRepository mongoRepository) { this.mongoRepository = mongoRepository; } diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java new file mode 100644 index 00000000..94d68dce --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java @@ -0,0 +1,27 @@ +package fr.insee.genesis.infrastructure.document.rawdata; + +import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.bson.types.ObjectId; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; + +import java.time.LocalDateTime; +import java.util.Map; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Document(collection = Constants.MONGODB_LUNATIC_JSON_DATA_COLLECTION_NAME) +public class LunaticJsonDataDocument { + @Id + private ObjectId id; + private String campaignId; + private Mode mode; + private Map data; + private LocalDateTime recordDate; + private LocalDateTime processDate; +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/xml/LunaticXmlDataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticXmlDataDocument.java similarity index 92% rename from src/main/java/fr/insee/genesis/infrastructure/document/rawdata/xml/LunaticXmlDataDocument.java rename to src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticXmlDataDocument.java index d5599f1f..2b4c8ea7 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/xml/LunaticXmlDataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticXmlDataDocument.java @@ -1,4 +1,4 @@ -package fr.insee.genesis.infrastructure.document.rawdata.xml; +package fr.insee.genesis.infrastructure.document.rawdata; import fr.insee.genesis.Constants; import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign; diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticJsonDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticJsonDocumentMapper.java new file mode 100644 index 00000000..0d083a67 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticJsonDocumentMapper.java @@ -0,0 +1,52 @@ +package fr.insee.genesis.infrastructure.mappers; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel; +import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument; +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; +import org.mapstruct.Named; +import org.mapstruct.factory.Mappers; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +@Mapper(componentModel = "spring") +public interface LunaticJsonDocumentMapper { + LunaticJsonDocumentMapper INSTANCE = Mappers.getMapper(LunaticJsonDocumentMapper.class); + + @Mapping(source = "data", target = "dataJson", qualifiedByName = "fromMapToJson") + LunaticJsonDataModel documentToModel(LunaticJsonDataDocument lunaticJsonDataDocument); + + @Mapping(source = "dataJson", target = "data", qualifiedByName = "fromJsonToMap") + LunaticJsonDataDocument modelToDocument(LunaticJsonDataModel lunaticJsonDataModel); + + List listDocumentToListModel(List lunaticJsonDataDocuments); + + List listModelToListDocument(List lunaticJsonDataModels); + + + @Named(value = "fromJsonToMap") + default Map fromJsonToMap(String dataJson) throws IOException { + if (Objects.nonNull(dataJson)) { + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.readValue(dataJson, new TypeReference<>() { + }); + } + return Collections.emptyMap(); + } + + @Named(value = "fromMapToJson") + default String fromMapToJson(Map dataMap) throws JsonProcessingException { + if (Objects.nonNull(dataMap)) { + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.writeValueAsString(dataMap); + } + return null; + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticXmlDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticXmlDocumentMapper.java index f529e937..77972382 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticXmlDocumentMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticXmlDocumentMapper.java @@ -1,7 +1,7 @@ package fr.insee.genesis.infrastructure.mappers; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel; -import fr.insee.genesis.infrastructure.document.rawdata.xml.LunaticXmlDataDocument; +import fr.insee.genesis.infrastructure.document.rawdata.LunaticXmlDataDocument; import org.mapstruct.Mapper; import org.mapstruct.Mapping; import org.mapstruct.factory.Mappers; diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java new file mode 100644 index 00000000..a668ad0c --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java @@ -0,0 +1,9 @@ +package fr.insee.genesis.infrastructure.repository; + +import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface LunaticJsonMongoDBRepository extends MongoRepository { +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticXmlMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticXmlMongoDBRepository.java index 0bc32df8..ecdcc4c5 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticXmlMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticXmlMongoDBRepository.java @@ -1,6 +1,6 @@ package fr.insee.genesis.infrastructure.repository; -import fr.insee.genesis.infrastructure.document.rawdata.xml.LunaticXmlDataDocument; +import fr.insee.genesis.infrastructure.document.rawdata.LunaticXmlDataDocument; import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.stereotype.Repository; diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index 9cc46762..5e231e18 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -9,13 +9,16 @@ import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; -import fr.insee.genesis.domain.ports.api.RawDataApiPort; +import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.domain.ports.api.LunaticXmlRawDataApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; +import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; import fr.insee.genesis.domain.service.rawdata.LunaticXmlRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; +import fr.insee.genesis.stubs.LunaticJsonPersistanceStub; import fr.insee.genesis.stubs.LunaticXmlPersistanceStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; @@ -39,6 +42,7 @@ class ResponseControllerTest { static ResponseController responseControllerStatic; static SurveyUnitPersistencePortStub surveyUnitPersistencePortStub; static LunaticXmlPersistanceStub lunaticXmlPersistanceStub; + static LunaticJsonPersistanceStub lunaticJsonPersistanceStub; static List surveyUnitIdList; //Constants @@ -51,13 +55,17 @@ static void init() { SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService(surveyUnitPersistencePortStub); lunaticXmlPersistanceStub = new LunaticXmlPersistanceStub(); - RawDataApiPort rawDataApiPort = new LunaticXmlRawDataService(lunaticXmlPersistanceStub); + LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort = new LunaticXmlRawDataService(lunaticXmlPersistanceStub); + + lunaticJsonPersistanceStub = new LunaticJsonPersistanceStub(); + LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService(lunaticJsonPersistanceStub); FileUtils fileUtils = new FileUtils(new ConfigStub()); responseControllerStatic = new ResponseController( surveyUnitApiPort , new SurveyUnitQualityService() - , rawDataApiPort + , lunaticXmlRawDataApiPort + , lunaticJsonRawDataApiPort , fileUtils , new ControllerUtils(fileUtils) ); @@ -123,6 +131,7 @@ void saveResponsesFromXmlCampaignFolderTest_noData() throws Exception { } //Raw data + //xml @Test void saveXmlRawDataFromFileTest() throws Exception { lunaticXmlPersistanceStub.getMongoStub().clear(); @@ -156,6 +165,29 @@ void saveResponsesFromAllCampaignFoldersTests(){ Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).isNotEmpty(); } + //json + @Test + void saveJsonRawDataFromStringTest() throws Exception { + lunaticJsonPersistanceStub.getMongoStub().clear(); + String campaignId = "SAMPLETEST-PARADATA-v1"; + + responseControllerStatic.saveRawResponsesFromJsonBody( + campaignId + , Mode.WEB + , "{\"testdata\": \"test\"}" + ); + + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub()).isNotEmpty(); + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getCampaignId()).isNotNull().isEqualTo(campaignId); + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getMode()).isEqualTo(Mode.WEB); + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getData().get("testdata")).isNotNull(); + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getData().get("testdata").asText()).isNotNull().isEqualTo("test"); + + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getRecordDate()).isNotNull(); + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getProcessDate()).isNull(); + + } + //Gets @Test diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonPersistanceStub.java new file mode 100644 index 00000000..56c32eed --- /dev/null +++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonPersistanceStub.java @@ -0,0 +1,20 @@ +package fr.insee.genesis.stubs; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel; +import fr.insee.genesis.domain.ports.spi.LunaticJsonPersistancePort; +import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument; +import fr.insee.genesis.infrastructure.mappers.LunaticJsonDocumentMapper; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +public class LunaticJsonPersistanceStub implements LunaticJsonPersistancePort { + List mongoStub = new ArrayList<>(); + + @Override + public void save(LunaticJsonDataModel lunaticJsonDataModel) { + mongoStub.add(LunaticJsonDocumentMapper.INSTANCE.modelToDocument(lunaticJsonDataModel)); + } +} diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticXmlPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticXmlPersistanceStub.java index 6e33e27d..15b1296d 100644 --- a/src/test/java/fr/insee/genesis/stubs/LunaticXmlPersistanceStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LunaticXmlPersistanceStub.java @@ -1,18 +1,13 @@ package fr.insee.genesis.stubs; -import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel; import fr.insee.genesis.domain.ports.spi.LunaticXmlPersistancePort; -import fr.insee.genesis.domain.ports.spi.SurveyUnitPersistencePort; -import fr.insee.genesis.infrastructure.document.rawdata.xml.LunaticXmlDataDocument; +import fr.insee.genesis.infrastructure.document.rawdata.LunaticXmlDataDocument; import fr.insee.genesis.infrastructure.mappers.LunaticXmlDocumentMapper; import lombok.Getter; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; -import java.util.stream.Stream; @Getter public class LunaticXmlPersistanceStub implements LunaticXmlPersistancePort { From 2965b52c3c50d439b9cd521481ffcf7d4277c339 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Fri, 6 Dec 2024 09:41:44 +0100 Subject: [PATCH 06/13] Test fixes + refactors --- src/main/java/fr/insee/genesis/Constants.java | 2 - .../rest/responses/ResponseController.java | 148 +++++++++--------- .../rawdata/LunaticJsonDataModel.java | 19 ++- .../rawdata/LunaticXmlDataModel.java | 14 +- .../rawdata/LunaticJsonDataDocument.java | 3 +- .../rawdata/LunaticXmlDataDocument.java | 3 +- .../infrastructure/utils/FileUtils.java | 14 +- .../responses/ResponseControllerTest.java | 2 +- .../infrastructure/utils/FileUtilsTest.java | 2 +- 9 files changed, 97 insertions(+), 110 deletions(-) diff --git a/src/main/java/fr/insee/genesis/Constants.java b/src/main/java/fr/insee/genesis/Constants.java index 013d5da7..f1711c2e 100644 --- a/src/main/java/fr/insee/genesis/Constants.java +++ b/src/main/java/fr/insee/genesis/Constants.java @@ -15,8 +15,6 @@ public class Constants { public static final String MONGODB_SCHEDULE_COLLECTION_NAME = "schedules"; public static final String LOOP_NAME_PREFIX = "BOUCLE"; public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses"; - public static final String MONGODB_LUNATIC_XML_DATA_COLLECTION_NAME = "lunaticxmldata"; - public static final String MONGODB_LUNATIC_JSON_DATA_COLLECTION_NAME = "lunaticjsondata"; public static final String VOLUMETRY_FOLDER_NAME = "genesis_volumetries"; public static final String VOLUMETRY_FILE_SUFFIX = "_VOLUMETRY"; public static final String VOLUMETRY_FILE_DATE_FORMAT = "yyyy_MM_dd"; diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 5630f783..278246d8 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -65,8 +65,9 @@ public class ResponseController { private static final String DDI_REGEX = "ddi[\\w,\\s-]+\\.xml"; public static final String S_S = "%s/%s"; - private static final String CAMPAIGN_ERROR = "Error for campaign {} : {}"; + private static final String CAMPAIGN_ERROR = "Error for campaign {}: {}"; private static final String SUCCESS_MESSAGE = "Data saved"; + private static final String SUCCESS_NO_DATA_MESSAGE = "No data has been saved"; private final SurveyUnitApiPort surveyUnitService; private final SurveyUnitQualityService surveyUnitQualityService; private final LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort; @@ -119,10 +120,9 @@ public ResponseEntity saveResponsesFromXmlFile(@RequestParam("pathLunati Path filepath = Paths.get(xmlFile); if (filepath.toFile().length() / 1024 / 1024 <= Constants.MAX_FILE_SIZE_UNTIL_SEQUENTIAL) { - return treatXmlFileWithMemory(filepath, modeSpecified, variablesMap); - } else { - return treatXmlFileSequentially(filepath, modeSpecified, variablesMap); + return processXmlFileWithMemory(filepath, modeSpecified, variablesMap); } + return processXmlFileSequentially(filepath, modeSpecified, variablesMap); } @Operation(summary = "Save multiple files to Genesis Database from the campaign root folder") @@ -134,12 +134,12 @@ public ResponseEntity saveResponsesFromXmlCampaignFolder(@RequestParam(" List errors = new ArrayList<>(); boolean isAnyDataSaved = false; - log.info("Try to import XML data for campaign : {}", campaignName); + log.info("Try to import XML data for campaign: {}", campaignName); List modesList = controllerUtils.getModesList(campaignName, modeSpecified); for (Mode currentMode : modesList) { try { - treatCampaignWithMode(campaignName, currentMode, errors, null, withDDI); + processCampaignWithMode(campaignName, currentMode, errors, null, withDDI); isAnyDataSaved = true; }catch (NoDataException nde){ //Don't stop if NoDataError thrown @@ -151,10 +151,9 @@ public ResponseEntity saveResponsesFromXmlCampaignFolder(@RequestParam(" } if (errors.isEmpty()){ - return ResponseEntity.ok(isAnyDataSaved ? SUCCESS_MESSAGE : "No data has been saved"); - }else{ - return ResponseEntity.internalServerError().body(errors.getFirst().getMessage()); + return ResponseEntity.ok(getSuccessMessage(isAnyDataSaved)); } + return ResponseEntity.internalServerError().body(errors.getFirst().getMessage()); } @Operation(summary = "Save one file of raw responses to Genesis Database, passing its path as a parameter") @@ -165,7 +164,7 @@ public ResponseEntity saveRawResponsesFromXmlFile(@RequestParam("pathLun log.info(String.format("Try to read Xml file : %s", xmlFile)); Path filepath = Paths.get(xmlFile); - return treatRawXmlFile(filepath, modeSpecified); + return processRawXmlFile(filepath, modeSpecified); } @Operation(summary = "Save multiple raw files to Genesis Database from the campaign root folder") @@ -175,12 +174,12 @@ public ResponseEntity saveRawResponsesFromXmlCampaignFolder(@RequestPara )throws Exception { boolean isAnyDataSaved = false; - log.info("Try to import raw lunatic XML data for campaign : {}", campaignName); + log.info("Try to import raw lunatic XML data for campaign: {}", campaignName); List modesList = controllerUtils.getModesList(campaignName, modeSpecified); for (Mode currentMode : modesList) { try { - treatRawCampaignWithMode(campaignName, currentMode, null); + processRawCampaignWithMode(campaignName, currentMode, null); isAnyDataSaved = true; }catch (NoDataException nde){ //Don't stop if NoDataError thrown @@ -191,7 +190,7 @@ public ResponseEntity saveRawResponsesFromXmlCampaignFolder(@RequestPara } } - return ResponseEntity.ok(isAnyDataSaved ? SUCCESS_MESSAGE : "No data has been saved"); + return ResponseEntity.ok(getSuccessMessage(isAnyDataSaved)); } //JSON @@ -202,7 +201,7 @@ public ResponseEntity saveRawResponsesFromJsonBody( @RequestParam(value = "mode", required = false) Mode modeSpecified, @RequestBody String dataJson ) { - log.info("Try to import raw lunatic JSON data for campaign : {}", campaignName); + log.info("Try to import raw lunatic JSON data for campaign: {}", campaignName); try { lunaticJsonRawDataApiPort.saveData(campaignName, dataJson, modeSpecified); }catch (JsonProcessingException jpe){ @@ -227,12 +226,12 @@ public ResponseEntity saveResponsesFromAllCampaignFolders(){ for (File campaignFolder: campaignFolders) { String campaignName = campaignFolder.getName(); - log.info("Try to import data for campaign : {}", campaignName); + log.info("Try to import data for campaign: {}", campaignName); try { List modesList = controllerUtils.getModesList(campaignName, null); //modeSpecified null = all modes for (Mode currentMode : modesList) { - treatCampaignWithMode(campaignName, currentMode, errors, Constants.DIFFRENTIAL_DATA_FOLDER_NAME); + processCampaignWithMode(campaignName, currentMode, errors, Constants.DIFFRENTIAL_DATA_FOLDER_NAME); } }catch (NoDataException nde){ log.warn(nde.getMessage()); @@ -244,9 +243,8 @@ public ResponseEntity saveResponsesFromAllCampaignFolders(){ } if (errors.isEmpty()) { return ResponseEntity.ok(SUCCESS_MESSAGE); - } else { - return ResponseEntity.status(209).body("Data saved with " + errors.size() + " errors"); } + return ResponseEntity.status(209).body("Data saved with " + errors.size() + " errors"); } @@ -297,7 +295,7 @@ public ResponseEntity findAllResponsesByQuestionnaire(@RequestParam("idQue log.error("Error while writing file", e); return ResponseEntity.internalServerError().body(filepath); } - log.info("End of extraction, responses extracted : {}", idUEsResponses.size()); + log.info("End of extraction, responses extracted: {}", idUEsResponses.size()); return ResponseEntity.ok(filepath); } @@ -366,23 +364,23 @@ public ResponseEntity> getLatestForUEList(@RequestPar //Utilities /** - * Checks if DDI is present for a campaign and mode or not and treats it accordingly + * Checks if DDI is present for a campaign and mode or not and processes it accordingly * @param campaignName name of campaign * @param currentMode mode of collected data * @param errors error list to fill */ - private void treatCampaignWithMode(String campaignName, Mode currentMode, List errors, String rootDataFolder) throws GenesisException, + private void processCampaignWithMode(String campaignName, Mode currentMode, List errors, String rootDataFolder) throws GenesisException, SAXException, XMLStreamException, NoDataException { try { fileUtils.findFile(String.format(S_S, fileUtils.getSpecFolder(campaignName),currentMode), DDI_REGEX); //DDI if DDI file found - treatCampaignWithMode(campaignName, currentMode, errors, rootDataFolder, true); + processCampaignWithMode(campaignName, currentMode, errors, rootDataFolder, true); }catch (RuntimeException re){ //Lunatic if no DDI log.info("No DDI File found for {}, {} mode. Will try to use Lunatic...", campaignName, currentMode.getModeName()); try{ - treatCampaignWithMode(campaignName, currentMode, errors, rootDataFolder, false); + processCampaignWithMode(campaignName, currentMode, errors, rootDataFolder, false); } catch (IOException | ParserConfigurationException e) { throw new GenesisException(500, e.toString()); } @@ -394,17 +392,17 @@ private void treatCampaignWithMode(String campaignName, Mode currentMode, List errors, String rootDataFolder, boolean withDDI) + private void processCampaignWithMode(String campaignName, Mode mode, List errors, String rootDataFolder, boolean withDDI) throws IOException, ParserConfigurationException, SAXException, XMLStreamException, NoDataException { log.info("Try to import data for mode : {}", mode.getModeName()); String dataFolder = rootDataFolder == null ? - fileUtils.getDataFolder(campaignName, mode.getFolder()) + fileUtils.getDataFolder(campaignName, mode.getFolder(), null) : fileUtils.getDataFolder(campaignName, mode.getFolder(), rootDataFolder); List dataFiles = fileUtils.listFiles(dataFolder); log.info("Numbers of files to load in folder {} : {}", dataFolder, dataFiles.size()); @@ -425,32 +423,32 @@ private void treatCampaignWithMode(String campaignName, Mode mode, List response; - if (filepath.toFile().length() / 1024 / 1024 <= Constants.MAX_FILE_SIZE_UNTIL_SEQUENTIAL) { - response = treatXmlFileWithMemory(filepath, mode, variablesMap); - } else { - response = treatXmlFileSequentially(filepath, mode, variablesMap); - } - log.debug("File {} saved", fileName); - if(response.getStatusCode() == HttpStatus.OK){ - fileUtils.moveDataFile(campaignName, mode.getFolder(), filepath); - }else{ - log.error("Error on file {}", fileName); - } + return; + } + //Read file + log.info("Try to read Xml file : {}", fileName); + ResponseEntity response; + if (filepath.toFile().length() / 1024 / 1024 <= Constants.MAX_FILE_SIZE_UNTIL_SEQUENTIAL) { + response = processXmlFileWithMemory(filepath, mode, variablesMap); + } else { + response = processXmlFileSequentially(filepath, mode, variablesMap); + } + log.debug("File {} saved", fileName); + if(response.getStatusCode() == HttpStatus.OK){ + fileUtils.moveDataFile(campaignName, mode.getFolder(), filepath); + return; } + log.error("Error on file {}", fileName); } } - private void treatRawCampaignWithMode(String campaignName, - Mode mode, - String rootDataFolder + private void processRawCampaignWithMode(String campaignName, + Mode mode, + String rootDataFolder ) throws IOException, ParserConfigurationException, SAXException, NoDataException { log.info("Try to import data for mode : {}", mode.getModeName()); String dataFolder = rootDataFolder == null ? - fileUtils.getDataFolder(campaignName, mode.getFolder()) + fileUtils.getDataFolder(campaignName, mode.getFolder(), null) : fileUtils.getDataFolder(campaignName, mode.getFolder(), rootDataFolder); List dataFiles = fileUtils.listFiles(dataFolder); log.info("Numbers of files to load in folder {} : {}", dataFolder, dataFiles.size()); @@ -466,18 +464,18 @@ private void treatRawCampaignWithMode(String campaignName, if(isDataFileInDoneFolder(filepath, campaignName, mode.getFolder())){ log.warn("File {} already exists in DONE folder ! Deleting...", fileName); Files.deleteIfExists(filepath); - }else{ - //Read file - log.info("Try to read Xml file : {}", fileName); - ResponseEntity response = treatRawXmlFile(filepath, mode); - - log.debug("File {} saved", fileName); - if(response.getStatusCode() == HttpStatus.OK){ - fileUtils.moveDataFile(campaignName, mode.getFolder(), filepath); - }else{ - log.error("Error on file {}", fileName); - } + return; + } + //Read file + log.info("Try to read Xml file : {}", fileName); + ResponseEntity response = processRawXmlFile(filepath, mode); + + log.debug("File {} saved", fileName); + if(response.getStatusCode() == HttpStatus.OK){ + fileUtils.moveDataFile(campaignName, mode.getFolder(), filepath); + return; } + log.error("Error on file {}", fileName); } } @@ -495,17 +493,17 @@ private VariablesMap readMetadatas(String campaignName, Mode mode, List treatXmlFileWithMemory(Path filepath, Mode modeSpecified, VariablesMap variablesMap) throws IOException, ParserConfigurationException, SAXException { + private ResponseEntity processXmlFileWithMemory(Path filepath, Mode modeSpecified, VariablesMap variablesMap) throws IOException, ParserConfigurationException, SAXException { LunaticXmlCampaign campaign; // DOM method LunaticXmlDataParser parser = new LunaticXmlDataParser(); @@ -535,11 +533,11 @@ private ResponseEntity treatXmlFileWithMemory(Path filepath, Mode modeSp surveyUnitService.saveSurveyUnits(suDtos); log.debug("Survey units updates saved"); - log.info("File {} treated with {} survey units", filepath.getFileName(), suDtos.size()); + log.info("File {} processed with {} survey units", filepath.getFileName(), suDtos.size()); return ResponseEntity.ok().build(); } - private ResponseEntity treatRawXmlFile(Path filepath, Mode modeSpecified) throws IOException, + private ResponseEntity processRawXmlFile(Path filepath, Mode modeSpecified) throws IOException, ParserConfigurationException, SAXException { LunaticXmlCampaign campaign; // DOM method @@ -555,11 +553,11 @@ private ResponseEntity treatRawXmlFile(Path filepath, Mode modeSpecified lunaticXmlRawDataApiPort.saveData(campaign, modeSpecified); log.debug(SUCCESS_MESSAGE); - log.info("File {} treated" , filepath.getFileName()); + log.info("File {} processed" , filepath.getFileName()); return ResponseEntity.ok().build(); } - private ResponseEntity treatXmlFileSequentially(Path filepath, Mode modeSpecified, VariablesMap variablesMap) throws IOException, XMLStreamException { + private ResponseEntity processXmlFileSequentially(Path filepath, Mode modeSpecified, VariablesMap variablesMap) throws IOException, XMLStreamException { LunaticXmlCampaign campaign; //Sequential method log.warn("File size > " + Constants.MAX_FILE_SIZE_UNTIL_SEQUENTIAL + "MB! Parsing XML file using sequential method..."); @@ -583,7 +581,11 @@ private ResponseEntity treatXmlFileSequentially(Path filepath, Mode mode log.info("Saved {} survey units updates", suCount); } - log.info("File {} treated", filepath.getFileName()); + log.info("File {} processed", filepath.getFileName()); return ResponseEntity.ok().build(); } + + private static String getSuccessMessage(boolean isAnyDataSaved) { + return isAnyDataSaved ? SUCCESS_MESSAGE : SUCCESS_NO_DATA_MESSAGE; + } } diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java index 51231255..d7faa038 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java @@ -2,18 +2,17 @@ import fr.insee.genesis.domain.model.surveyunit.Mode; import lombok.Builder; -import lombok.Data; import org.bson.types.ObjectId; import java.time.LocalDateTime; -@Data + @Builder -public class LunaticJsonDataModel { - private ObjectId id; - private String campaignId; - private Mode mode; - private String dataJson; - private LocalDateTime recordDate; - private LocalDateTime processDate; -} +public record LunaticJsonDataModel( + ObjectId id, + String campaignId, + Mode mode, + String dataJson, + LocalDateTime recordDate, + LocalDateTime processDate +){} diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java index ed7d43ef..24339cca 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java @@ -3,16 +3,14 @@ import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign; import fr.insee.genesis.domain.model.surveyunit.Mode; import lombok.Builder; -import lombok.Data; import org.bson.types.ObjectId; import java.time.LocalDateTime; -@Data @Builder -public class LunaticXmlDataModel{ - private ObjectId id; - private Mode mode; - private LunaticXmlCampaign data; - private LocalDateTime recordDate; -} +public record LunaticXmlDataModel( + ObjectId id, + Mode mode, + LunaticXmlCampaign data, + LocalDateTime recordDate +){}; diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java index 94d68dce..e063fbb4 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java @@ -1,6 +1,5 @@ package fr.insee.genesis.infrastructure.document.rawdata; -import fr.insee.genesis.Constants; import fr.insee.genesis.domain.model.surveyunit.Mode; import lombok.AllArgsConstructor; import lombok.Data; @@ -15,7 +14,7 @@ @Data @NoArgsConstructor @AllArgsConstructor -@Document(collection = Constants.MONGODB_LUNATIC_JSON_DATA_COLLECTION_NAME) +@Document(collection = "lunaticjsondata") public class LunaticJsonDataDocument { @Id private ObjectId id; diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticXmlDataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticXmlDataDocument.java index 2b4c8ea7..7ef9b080 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticXmlDataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticXmlDataDocument.java @@ -1,6 +1,5 @@ package fr.insee.genesis.infrastructure.document.rawdata; -import fr.insee.genesis.Constants; import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign; import fr.insee.genesis.domain.model.surveyunit.Mode; import lombok.AllArgsConstructor; @@ -15,7 +14,7 @@ @Data @NoArgsConstructor @AllArgsConstructor -@Document(collection = Constants.MONGODB_LUNATIC_XML_DATA_COLLECTION_NAME) +@Document(collection = "lunaticxmldata") public class LunaticXmlDataDocument { @Id private ObjectId id; diff --git a/src/main/java/fr/insee/genesis/infrastructure/utils/FileUtils.java b/src/main/java/fr/insee/genesis/infrastructure/utils/FileUtils.java index 155acc35..010181f4 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/utils/FileUtils.java +++ b/src/main/java/fr/insee/genesis/infrastructure/utils/FileUtils.java @@ -133,16 +133,6 @@ public Path findFile(String directory, String regex) throws IOException { } } - /** - * Get the path of the folder where the data files are stored - * @param campaign name of campaign - * @param dataSource folder of the mode (ENQ, WEB...) - * @return Path of the data folder - */ - public String getDataFolder(String campaign, String dataSource) { - return String.format("%s/%s/%s/%s", dataFolderSource, "IN", dataSource, campaign); - } - /** * Get the path of the folder where the data files are stored * @param campaign name of campaign @@ -151,7 +141,9 @@ public String getDataFolder(String campaign, String dataSource) { * @return Path of the data folder */ public String getDataFolder(String campaign, String dataSource, String rootDataFolder) { - return String.format("%s/%s/%s/%s/%s", dataFolderSource, "IN", dataSource, campaign, rootDataFolder); + return rootDataFolder == null ? + String.format("%s/%s/%s/%s", dataFolderSource, "IN", dataSource, campaign) + : String.format("%s/%s/%s/%s/%s", dataFolderSource, "IN", dataSource, campaign, rootDataFolder); } diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index 5e231e18..cad484f9 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -181,7 +181,7 @@ void saveJsonRawDataFromStringTest() throws Exception { Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getCampaignId()).isNotNull().isEqualTo(campaignId); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getData().get("testdata")).isNotNull(); - Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getData().get("testdata").asText()).isNotNull().isEqualTo("test"); + Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getData().get("testdata")).isNotNull().isEqualTo("test"); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getRecordDate()).isNotNull(); Assertions.assertThat(lunaticJsonPersistanceStub.getMongoStub().getFirst().getProcessDate()).isNull(); diff --git a/src/test/java/fr/insee/genesis/infrastructure/utils/FileUtilsTest.java b/src/test/java/fr/insee/genesis/infrastructure/utils/FileUtilsTest.java index ae43e05d..e8392bf2 100644 --- a/src/test/java/fr/insee/genesis/infrastructure/utils/FileUtilsTest.java +++ b/src/test/java/fr/insee/genesis/infrastructure/utils/FileUtilsTest.java @@ -22,7 +22,7 @@ class FileUtilsTest { @Test void getDataFolderTest() { assertEquals(TestConstants.TEST_RESOURCES_DIRECTORY + "/IN/WEB/TEST", - fileUtils.getDataFolder("TEST", "WEB")); + fileUtils.getDataFolder("TEST", "WEB", null)); } @Test From 1fbdcc550d1dc216a9a79ad39f0ecbe0b0905bc6 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Fri, 6 Dec 2024 09:59:19 +0100 Subject: [PATCH 07/13] Removed some useless autowired --- .../controller/rest/responses/ResponseController.java | 4 +--- .../infrastructure/adapter/LunaticJsonMongoAdapter.java | 2 -- .../infrastructure/adapter/LunaticXmlMongoAdapter.java | 3 --- .../genesis/infrastructure/adapter/ScheduleMongoAdapter.java | 2 -- .../infrastructure/adapter/SurveyUnitMongoAdapter.java | 5 ++--- 5 files changed, 3 insertions(+), 13 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 278246d8..f3cfd539 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -30,7 +30,6 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; @@ -74,8 +73,7 @@ public class ResponseController { private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort; private final FileUtils fileUtils; private final ControllerUtils controllerUtils; - - @Autowired + public ResponseController(SurveyUnitApiPort surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort, diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java index 29e1f893..bc002dc5 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java @@ -6,7 +6,6 @@ import fr.insee.genesis.infrastructure.mappers.LunaticJsonDocumentMapper; import fr.insee.genesis.infrastructure.repository.LunaticJsonMongoDBRepository; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; @@ -17,7 +16,6 @@ public class LunaticJsonMongoAdapter implements LunaticJsonPersistancePort { private final LunaticJsonMongoDBRepository lunaticJsonMongoDBRepository; - @Autowired public LunaticJsonMongoAdapter(LunaticJsonMongoDBRepository lunaticJsonMongoDBRepository) { this.lunaticJsonMongoDBRepository = lunaticJsonMongoDBRepository; } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java index 14c02073..44d59397 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java @@ -6,9 +6,7 @@ import fr.insee.genesis.infrastructure.mappers.LunaticXmlDocumentMapper; import fr.insee.genesis.infrastructure.repository.LunaticXmlMongoDBRepository; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.stereotype.Service; @Slf4j @@ -18,7 +16,6 @@ public class LunaticXmlMongoAdapter implements LunaticXmlPersistancePort { private final LunaticXmlMongoDBRepository mongoRepository; - @Autowired public LunaticXmlMongoAdapter(LunaticXmlMongoDBRepository mongoRepository) { this.mongoRepository = mongoRepository; } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/ScheduleMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/ScheduleMongoAdapter.java index c0e77a66..b1cb467c 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/ScheduleMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/ScheduleMongoAdapter.java @@ -26,8 +26,6 @@ public class ScheduleMongoAdapter implements SchedulePersistencePort { private final ScheduleMongoDBRepository scheduleMongoDBRepository; private final MongoTemplate mongoTemplate; - - @Autowired public ScheduleMongoAdapter(ScheduleMongoDBRepository scheduleMongoDBRepository, MongoTemplate mongoTemplate) { this.scheduleMongoDBRepository = scheduleMongoDBRepository; this.mongoTemplate = mongoTemplate; diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java index d8a64d10..053bd363 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java @@ -27,10 +27,9 @@ @Qualifier("surveyUnitMongoAdapter") public class SurveyUnitMongoAdapter implements SurveyUnitPersistencePort { - private SurveyUnitMongoDBRepository mongoRepository; - private MongoTemplate mongoTemplate; + private final SurveyUnitMongoDBRepository mongoRepository; + private final MongoTemplate mongoTemplate; - @Autowired public SurveyUnitMongoAdapter(SurveyUnitMongoDBRepository mongoRepository, MongoTemplate mongoTemplate) { this.mongoRepository = mongoRepository; this.mongoTemplate = mongoTemplate; From ed8f777ddec5e86023c943a28800b2ae73c251f2 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Fri, 6 Dec 2024 10:03:07 +0100 Subject: [PATCH 08/13] Revert "Removed some useless autowired" This reverts commit 1fbdcc550d1dc216a9a79ad39f0ecbe0b0905bc6. --- .../controller/rest/responses/ResponseController.java | 4 +++- .../infrastructure/adapter/LunaticJsonMongoAdapter.java | 2 ++ .../infrastructure/adapter/LunaticXmlMongoAdapter.java | 3 +++ .../genesis/infrastructure/adapter/ScheduleMongoAdapter.java | 2 ++ .../infrastructure/adapter/SurveyUnitMongoAdapter.java | 5 +++-- 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index f3cfd539..278246d8 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -30,6 +30,7 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; @@ -73,7 +74,8 @@ public class ResponseController { private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort; private final FileUtils fileUtils; private final ControllerUtils controllerUtils; - + + @Autowired public ResponseController(SurveyUnitApiPort surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort, diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java index bc002dc5..29e1f893 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonMongoAdapter.java @@ -6,6 +6,7 @@ import fr.insee.genesis.infrastructure.mappers.LunaticJsonDocumentMapper; import fr.insee.genesis.infrastructure.repository.LunaticJsonMongoDBRepository; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; @@ -16,6 +17,7 @@ public class LunaticJsonMongoAdapter implements LunaticJsonPersistancePort { private final LunaticJsonMongoDBRepository lunaticJsonMongoDBRepository; + @Autowired public LunaticJsonMongoAdapter(LunaticJsonMongoDBRepository lunaticJsonMongoDBRepository) { this.lunaticJsonMongoDBRepository = lunaticJsonMongoDBRepository; } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java index 44d59397..14c02073 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticXmlMongoAdapter.java @@ -6,7 +6,9 @@ import fr.insee.genesis.infrastructure.mappers.LunaticXmlDocumentMapper; import fr.insee.genesis.infrastructure.repository.LunaticXmlMongoDBRepository; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.stereotype.Service; @Slf4j @@ -16,6 +18,7 @@ public class LunaticXmlMongoAdapter implements LunaticXmlPersistancePort { private final LunaticXmlMongoDBRepository mongoRepository; + @Autowired public LunaticXmlMongoAdapter(LunaticXmlMongoDBRepository mongoRepository) { this.mongoRepository = mongoRepository; } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/ScheduleMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/ScheduleMongoAdapter.java index b1cb467c..c0e77a66 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/ScheduleMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/ScheduleMongoAdapter.java @@ -26,6 +26,8 @@ public class ScheduleMongoAdapter implements SchedulePersistencePort { private final ScheduleMongoDBRepository scheduleMongoDBRepository; private final MongoTemplate mongoTemplate; + + @Autowired public ScheduleMongoAdapter(ScheduleMongoDBRepository scheduleMongoDBRepository, MongoTemplate mongoTemplate) { this.scheduleMongoDBRepository = scheduleMongoDBRepository; this.mongoTemplate = mongoTemplate; diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java index 053bd363..d8a64d10 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java @@ -27,9 +27,10 @@ @Qualifier("surveyUnitMongoAdapter") public class SurveyUnitMongoAdapter implements SurveyUnitPersistencePort { - private final SurveyUnitMongoDBRepository mongoRepository; - private final MongoTemplate mongoTemplate; + private SurveyUnitMongoDBRepository mongoRepository; + private MongoTemplate mongoTemplate; + @Autowired public SurveyUnitMongoAdapter(SurveyUnitMongoDBRepository mongoRepository, MongoTemplate mongoTemplate) { this.mongoRepository = mongoRepository; this.mongoTemplate = mongoTemplate; From 83ee8b557a4f82fe524e329ff52fd4cecaf2ec1d Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Tue, 10 Dec 2024 10:06:32 +0100 Subject: [PATCH 09/13] removed useless autowired in controllers --- .../fr/insee/genesis/controller/rest/HealthCheckController.java | 2 +- .../fr/insee/genesis/controller/rest/ScheduleController.java | 1 - .../java/fr/insee/genesis/controller/rest/UtilsController.java | 2 +- .../genesis/controller/rest/responses/CampaignController.java | 2 +- .../insee/genesis/controller/rest/responses/IdUEController.java | 2 +- .../insee/genesis/controller/rest/responses/ModeController.java | 2 +- .../controller/rest/responses/QuestionnaireController.java | 2 +- .../genesis/controller/rest/responses/ResponseController.java | 2 +- 8 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/HealthCheckController.java b/src/main/java/fr/insee/genesis/controller/rest/HealthCheckController.java index 98903910..dfee7481 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/HealthCheckController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/HealthCheckController.java @@ -18,7 +18,7 @@ public class HealthCheckController { @Value("${fr.insee.genesis.version}") private String projectVersion; - @Autowired + public HealthCheckController(SurveyUnitApiPort surveyUnitApiPort, ScheduleApiPort scheduleApiPort) { this.surveyUnitApiPort = surveyUnitApiPort; this.scheduleApiPort = scheduleApiPort; diff --git a/src/main/java/fr/insee/genesis/controller/rest/ScheduleController.java b/src/main/java/fr/insee/genesis/controller/rest/ScheduleController.java index d36effc2..9e48504b 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/ScheduleController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/ScheduleController.java @@ -42,7 +42,6 @@ public class ScheduleController { private final ScheduleApiPort scheduleApiPort; private final FileUtils fileUtils; - @Autowired public ScheduleController(ScheduleApiPort scheduleApiPort, FileUtils fileUtils) { this.scheduleApiPort = scheduleApiPort; this.fileUtils = fileUtils; diff --git a/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java b/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java index c7d8683f..3b088e5d 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java @@ -25,7 +25,7 @@ public class UtilsController { private final SurveyUnitApiPort surveyUnitService; - @Autowired + public UtilsController(SurveyUnitApiPort surveyUnitService,VolumetryLogService volumetryLogService) { this.surveyUnitService = surveyUnitService; this.volumetryLogService = volumetryLogService; diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java index 08998118..1de9f7ac 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java @@ -20,7 +20,7 @@ public class CampaignController { private final SurveyUnitApiPort surveyUnitService; - @Autowired + public CampaignController(SurveyUnitApiPort surveyUnitService) { this.surveyUnitService = surveyUnitService; } diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/IdUEController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/IdUEController.java index fb8be06a..ab20b6ae 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/IdUEController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/IdUEController.java @@ -20,7 +20,7 @@ public class IdUEController { private final SurveyUnitApiPort surveyUnitService; - @Autowired + public IdUEController(SurveyUnitApiPort surveyUnitService) { this.surveyUnitService = surveyUnitService; } diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java index ca183e0f..307d0466 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java @@ -20,7 +20,7 @@ public class ModeController { private final SurveyUnitApiPort surveyUnitService; - @Autowired + public ModeController(SurveyUnitApiPort surveyUnitService) { this.surveyUnitService = surveyUnitService; diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java index d7a60619..cced2a44 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java @@ -21,7 +21,7 @@ public class QuestionnaireController { private final SurveyUnitApiPort surveyUnitService; - @Autowired + public QuestionnaireController(SurveyUnitApiPort surveyUnitService) { this.surveyUnitService = surveyUnitService; } diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 278246d8..08b4b914 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -75,7 +75,7 @@ public class ResponseController { private final FileUtils fileUtils; private final ControllerUtils controllerUtils; - @Autowired + public ResponseController(SurveyUnitApiPort surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort, From 4a3ca92f27e087be15af27b685ad3a22acc0e514 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Tue, 10 Dec 2024 11:23:02 +0100 Subject: [PATCH 10/13] removed unused imports --- .../fr/insee/genesis/controller/rest/HealthCheckController.java | 1 - .../fr/insee/genesis/controller/rest/ScheduleController.java | 1 - .../java/fr/insee/genesis/controller/rest/UtilsController.java | 1 - .../genesis/controller/rest/responses/CampaignController.java | 1 - .../insee/genesis/controller/rest/responses/IdUEController.java | 1 - .../insee/genesis/controller/rest/responses/ModeController.java | 1 - .../controller/rest/responses/QuestionnaireController.java | 1 - .../genesis/controller/rest/responses/ResponseController.java | 1 - 8 files changed, 8 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/HealthCheckController.java b/src/main/java/fr/insee/genesis/controller/rest/HealthCheckController.java index dfee7481..8723e819 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/HealthCheckController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/HealthCheckController.java @@ -2,7 +2,6 @@ import fr.insee.genesis.domain.ports.api.ScheduleApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.ResponseEntity; import org.springframework.security.core.context.SecurityContextHolder; diff --git a/src/main/java/fr/insee/genesis/controller/rest/ScheduleController.java b/src/main/java/fr/insee/genesis/controller/rest/ScheduleController.java index 9e48504b..a8d596f7 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/ScheduleController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/ScheduleController.java @@ -14,7 +14,6 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.DeleteMapping; diff --git a/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java b/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java index 3b088e5d..e238ba4e 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java @@ -6,7 +6,6 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.PutMapping; diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java index 1de9f7ac..1e3135c4 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java @@ -4,7 +4,6 @@ import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import io.swagger.v3.oas.annotations.Operation; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/IdUEController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/IdUEController.java index ab20b6ae..b47df612 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/IdUEController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/IdUEController.java @@ -4,7 +4,6 @@ import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import io.swagger.v3.oas.annotations.Operation; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java index 307d0466..320af308 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java @@ -4,7 +4,6 @@ import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import io.swagger.v3.oas.annotations.Operation; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java index cced2a44..ad8303c4 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java @@ -4,7 +4,6 @@ import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import io.swagger.v3.oas.annotations.Operation; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 08b4b914..baac0b59 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -30,7 +30,6 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; From 3928b9bbc092f16014dee32235b3a32b64403025 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Tue, 10 Dec 2024 11:45:36 +0100 Subject: [PATCH 11/13] Fix reset method --- .../controller/rest/responses/Utils.java | 207 ++++++++---------- 1 file changed, 95 insertions(+), 112 deletions(-) diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java b/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java index 4ea51bd1..424bb4b0 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java @@ -46,140 +46,123 @@ static void reset(SurveyUnitPersistencePortStub surveyUnitPersistencePortStub) t //Recreate data files //SAMPLETEST-PARADATA-v1 //Root - if (!testResourcesPath + //Partial + Path source = testResourcesPath .resolve("IN") .resolve("WEB") .resolve("SAMPLETEST-PARADATA-v1") - .resolve("data.complete.validated.STPDv1.20231122164209.xml") - .toFile().exists() - ){ - Files.copy( - testResourcesPath - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-PARADATA-v1") - .resolve("reponse-platine") - .resolve("data.complete.partial.STPDv1.20231122164209.xml") - , testResourcesPath - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-PARADATA-v1") - .resolve("data.complete.partial.STPDv1.20231122164209.xml") - ); - Files.copy( - testResourcesPath - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-PARADATA-v1") - .resolve("reponse-platine") - .resolve("data.complete.validated.STPDv1.20231122164209.xml") - , testResourcesPath - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-PARADATA-v1") - .resolve("data.complete.validated.STPDv1.20231122164209.xml") - ); + .resolve("reponse-platine") + .resolve("data.complete.partial.STPDv1.20231122164209.xml"); + Path dest = testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("data.complete.partial.STPDv1.20231122164209.xml"); + if (!dest.toFile().exists()) { + Files.copy(source, dest); + } + + //Validated + source = testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("reponse-platine") + .resolve("data.complete.validated.STPDv1.20231122164209.xml"); + dest = testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("data.complete.validated.STPDv1.20231122164209.xml"); + if (!dest.toFile().exists()) { + Files.copy(source, dest); } + //Differential data - if (!testResourcesPath + //Partial + source = testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("reponse-platine") + .resolve("data.complete.partial.STPDv1.20231122164209.xml"); + dest = testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("differential") + .resolve("data") + .resolve("data.complete.partial.STPDv1.20231122164209.xml"); + if (!dest.toFile().exists()) { + Files.copy(source, dest); + } + + source = testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("reponse-platine") + .resolve("data.complete.validated.STPDv1.20231122164209.xml"); + dest = testResourcesPath .resolve("IN") .resolve("WEB") .resolve("SAMPLETEST-PARADATA-v1") .resolve("differential") .resolve("data") - .resolve("data.complete.validated.STPDv1.20231122164209.xml") - .toFile().exists() - ) { - Files.copy( - testResourcesPath - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-PARADATA-v1") - .resolve("reponse-platine") - .resolve("data.complete.partial.STPDv1.20231122164209.xml") - , testResourcesPath - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-PARADATA-v1") - .resolve("differential") - .resolve("data") - .resolve("data.complete.partial.STPDv1.20231122164209.xml") - ); - Files.copy( - testResourcesPath - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-PARADATA-v1") - .resolve("reponse-platine") - .resolve("data.complete.validated.STPDv1.20231122164209.xml") - , testResourcesPath - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-PARADATA-v1") - .resolve("differential") - .resolve("data") - .resolve("data.complete.validated.STPDv1.20231122164209.xml") - ); + .resolve("data.complete.validated.STPDv1.20231122164209.xml"); + if (!dest.toFile().exists()) { + Files.copy(source, dest); } + //SAMPLETEST-PARADATA-v2 - if (!testResourcesPath + //Partial + source = testResourcesPath .resolve("IN") .resolve("WEB") .resolve("SAMPLETEST-PARADATA-v2") - .resolve("data.complete.validated.STPDv2.20231122164209.xml") - .toFile().exists() - ){ - Files.copy( - testResourcesPath - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-PARADATA-v2") - .resolve("reponse-platine") - .resolve("data.complete.partial.STPDv2.20231122164209.xml") - , testResourcesPath - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-PARADATA-v2") - .resolve("data.complete.partial.STPDv2.20231122164209.xml") - ); - Files.copy( - testResourcesPath - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-PARADATA-v2") - .resolve("reponse-platine") - .resolve("data.complete.validated.STPDv2.20231122164209.xml") - , testResourcesPath - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-PARADATA-v2") - .resolve("data.complete.validated.STPDv2.20231122164209.xml") - ); + .resolve("reponse-platine") + .resolve("data.complete.partial.STPDv2.20231122164209.xml"); + dest = testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v2") + .resolve("data.complete.partial.STPDv2.20231122164209.xml"); + if (!dest.toFile().exists()) { + Files.copy(source, dest); } + + //Validated + source = testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v2") + .resolve("reponse-platine") + .resolve("data.complete.validated.STPDv2.20231122164209.xml"); + dest = testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v2") + .resolve("data.complete.validated.STPDv2.20231122164209.xml"); + if (!dest.toFile().exists()) { + Files.copy(source, dest); + } + //SAMPLETEST-NO-COLLECTED - if (!testResourcesPath + source = testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-NO-COLLECTED") + .resolve("data_diff_no_collected.xml"); + + dest = testResourcesPath .resolve("IN") .resolve("WEB") .resolve("SAMPLETEST-NO-COLLECTED") .resolve("differential") .resolve("data") - .resolve("data_diff_no_collected.xml") - .toFile().exists() - ){ - Files.copy( - testResourcesPath - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-NO-COLLECTED") - .resolve("data_diff_no_collected.xml") - , testResourcesPath - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-NO-COLLECTED") - .resolve("differential") - .resolve("data") - .resolve("data_diff_no_collected.xml") - ); + .resolve("data_diff_no_collected.xml"); + if (!dest.toFile().exists()) { + Files.copy(source, dest); } } From 9bdb6fa6b932bc13eb3fd8b5e2133bbbb38363ea Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Tue, 10 Dec 2024 14:06:10 +0100 Subject: [PATCH 12/13] Add processDate to xml raw data --- .../rawdata/LunaticXmlDataModel.java | 5 +++-- .../rawdata/LunaticXmlDataDocument.java | 1 + .../responses/ResponseControllerTest.java | 22 ++++++++++++------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java index 24339cca..12819049 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java @@ -12,5 +12,6 @@ public record LunaticXmlDataModel( ObjectId id, Mode mode, LunaticXmlCampaign data, - LocalDateTime recordDate -){}; + LocalDateTime recordDate, + LocalDateTime processDate +){} diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticXmlDataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticXmlDataDocument.java index 7ef9b080..7ea6b29c 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticXmlDataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticXmlDataDocument.java @@ -21,4 +21,5 @@ public class LunaticXmlDataDocument { private LunaticXmlCampaign lunaticXmlData; private Mode mode; private LocalDateTime recordDate; + private LocalDateTime processDate; } diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index cad484f9..77dbdeac 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -143,7 +143,11 @@ void saveXmlRawDataFromFileTest() throws Exception { Assertions.assertThat(lunaticXmlPersistanceStub.getMongoStub()).isNotEmpty(); Assertions.assertThat(lunaticXmlPersistanceStub.getMongoStub().getFirst().getRecordDate()).isNotNull(); + Assertions.assertThat(lunaticXmlPersistanceStub.getMongoStub().getFirst().getProcessDate()).isNull(); Assertions.assertThat(lunaticXmlPersistanceStub.getMongoStub().getFirst().getLunaticXmlData()).isNotNull(); + Assertions.assertThat(lunaticXmlPersistanceStub.getMongoStub().getFirst().getLunaticXmlData().getSurveyUnits()).isNotNull().isNotEmpty(); + + } @Test @@ -158,16 +162,9 @@ void saveXmlRawDataFromFolderTest() throws Exception { Assertions.assertThat(lunaticXmlPersistanceStub.getMongoStub()).isNotEmpty(); } - @Test - void saveResponsesFromAllCampaignFoldersTests(){ - surveyUnitPersistencePortStub.getMongoStub().clear(); - responseControllerStatic.saveResponsesFromAllCampaignFolders(); - - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).isNotEmpty(); - } //json @Test - void saveJsonRawDataFromStringTest() throws Exception { + void saveJsonRawDataFromStringTest(){ lunaticJsonPersistanceStub.getMongoStub().clear(); String campaignId = "SAMPLETEST-PARADATA-v1"; @@ -188,6 +185,15 @@ void saveJsonRawDataFromStringTest() throws Exception { } + //All data + @Test + void saveResponsesFromAllCampaignFoldersTests(){ + surveyUnitPersistencePortStub.getMongoStub().clear(); + responseControllerStatic.saveResponsesFromAllCampaignFolders(); + + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).isNotEmpty(); + } + //Gets @Test From 93585c7255370cfadb2dc21da55290629c0a97e1 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Thu, 12 Dec 2024 14:35:06 +0100 Subject: [PATCH 13/13] Fixed error 400 when invalid json request + add test --- .../rawdata/LunaticJsonRawDataService.java | 21 ++++++++++++++- .../mappers/LunaticJsonDocumentMapper.java | 16 +++--------- .../LunaticJsonRawDataServiceTest.java | 26 +++++++++++++++++++ 3 files changed, 49 insertions(+), 14 deletions(-) create mode 100644 src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index cf9fe17c..febdf9ce 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -1,5 +1,9 @@ package fr.insee.genesis.domain.service.rawdata; +import com.fasterxml.jackson.core.JacksonException; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; @@ -21,7 +25,10 @@ public LunaticJsonRawDataService(LunaticJsonPersistancePort lunaticJsonPersistan } @Override - public void saveData(String campaignName, String dataJson, Mode mode){ + public void saveData(String campaignName, String dataJson, Mode mode) throws JsonParseException { + if(!isJsonValid(dataJson)){ + throw new JsonParseException("Invalid JSON synthax"); + } LunaticJsonDataModel lunaticJsonDataModel = LunaticJsonDataModel.builder() .campaignId(campaignName) .mode(mode) @@ -31,4 +38,16 @@ public void saveData(String campaignName, String dataJson, Mode mode){ lunaticJsonPersistancePort.save(lunaticJsonDataModel); } + + + private boolean isJsonValid(String json) { + ObjectMapper mapper = new ObjectMapper() + .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); + try { + mapper.readTree(json); + } catch (JacksonException e) { + return false; + } + return true; + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticJsonDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticJsonDocumentMapper.java index 0d083a67..1600c013 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticJsonDocumentMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticJsonDocumentMapper.java @@ -10,11 +10,8 @@ import org.mapstruct.Named; import org.mapstruct.factory.Mappers; -import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Objects; @Mapper(componentModel = "spring") public interface LunaticJsonDocumentMapper { @@ -32,21 +29,14 @@ public interface LunaticJsonDocumentMapper { @Named(value = "fromJsonToMap") - default Map fromJsonToMap(String dataJson) throws IOException { - if (Objects.nonNull(dataJson)) { - ObjectMapper objectMapper = new ObjectMapper(); - return objectMapper.readValue(dataJson, new TypeReference<>() { - }); - } - return Collections.emptyMap(); + default Map fromJsonToMap(String dataJson) throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.readValue(dataJson, new TypeReference<>(){}); } @Named(value = "fromMapToJson") default String fromMapToJson(Map dataMap) throws JsonProcessingException { - if (Objects.nonNull(dataMap)) { ObjectMapper objectMapper = new ObjectMapper(); return objectMapper.writeValueAsString(dataMap); - } - return null; } } diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java new file mode 100644 index 00000000..742f4515 --- /dev/null +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -0,0 +1,26 @@ +package fr.insee.genesis.domain.service.rawdata; + +import com.fasterxml.jackson.core.JsonParseException; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.stubs.LunaticJsonPersistanceStub; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +class LunaticJsonRawDataServiceTest { + LunaticJsonPersistanceStub lunaticJsonPersistanceStub = new LunaticJsonPersistanceStub(); + LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService(lunaticJsonPersistanceStub); + + @Test + void saveDataTest_Invalid(){ + lunaticJsonPersistanceStub.getMongoStub().clear(); + String campaignId = "SAMPLETEST-PARADATA-v1"; + + Assertions.assertThatThrownBy(() -> { + lunaticJsonRawDataService.saveData( + campaignId + ,"{\"testdata\": \"ERROR" + , Mode.WEB + ); + }).isInstanceOf(JsonParseException.class); + } +} \ No newline at end of file