Skip to content

Commit

Permalink
Fix save folder saving only 1 file at a time + add cucumber test
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexisszmundy
alexisszmundy committed Jan 8, 2025
1 parent 99b21c1 commit 4708d1c
Show file tree
Hide file tree
Showing 9 changed files with 289,752 additions and 12 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,4 @@ hs_err_pid*
src/test/resources/OUT/*
/src/test/resources/DONE/*
/src/test/resources/genesis_deleted_schedules/*
/src/test/resources/IN/WEB/SAMPLETEST/
12 changes: 7 additions & 5 deletions src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -422,7 +422,7 @@ private void processCampaignWithMode(String campaignName, Mode mode, List<Genesi
if(isDataFileInDoneFolder(filepath, campaignName, mode.getFolder())){
log.warn("File {} already exists in DONE folder ! Deleting...", fileName);
Files.deleteIfExists(filepath);
return;
continue; //Go to next file
}
//Read file
log.info("Try to read Xml file : {}", fileName);
Expand All @@ -433,11 +433,13 @@ private void processCampaignWithMode(String campaignName, Mode mode, List<Genesi
response = processXmlFileSequentially(filepath, mode, variablesMap);
}
log.debug("File {} saved", fileName);
if(response.getStatusCode() == HttpStatus.OK){
fileUtils.moveDataFile(campaignName, mode.getFolder(), filepath);
return;
if (response.getStatusCode() == HttpStatus.OK) {
fileUtils.moveDataFile(campaignName, mode.getFolder(),
filepath);
//Sonar is annoying when there is more than 1 continue
} else {
log.error("Error {} on file {} : {}", response.getStatusCode(), fileName, response.getBody());
}
log.error("Error on file {}", fileName);
}
}

Expand Down
80 changes: 73 additions & 7 deletions src/test/java/cucumber/functional_tests/MainDefinitions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,30 @@
import fr.insee.bpm.exceptions.MetadataParserException;
import fr.insee.bpm.metadata.model.VariablesMap;
import fr.insee.bpm.metadata.reader.ddi.DDIReader;
import fr.insee.genesis.configuration.Config;
import fr.insee.genesis.controller.adapter.LunaticXmlAdapter;
import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
import fr.insee.genesis.controller.rest.responses.ResponseController;
import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign;
import fr.insee.genesis.controller.sources.xml.LunaticXmlDataParser;
import fr.insee.genesis.controller.sources.xml.LunaticXmlSurveyUnit;
import fr.insee.genesis.controller.utils.ControllerUtils;
import fr.insee.genesis.domain.model.surveyunit.CollectedVariable;
import fr.insee.genesis.domain.model.surveyunit.DataState;
import fr.insee.genesis.domain.model.surveyunit.Mode;
import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
import fr.insee.genesis.domain.model.surveyunit.Variable;
import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService;
import fr.insee.genesis.domain.service.rawdata.LunaticXmlRawDataService;
import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService;
import fr.insee.genesis.exceptions.GenesisException;
import fr.insee.genesis.infrastructure.utils.FileUtils;
import fr.insee.genesis.stubs.ConfigStub;
import fr.insee.genesis.stubs.LunaticJsonPersistanceStub;
import fr.insee.genesis.stubs.LunaticXmlPersistanceStub;
import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
import io.cucumber.java.After;
import io.cucumber.java.Before;
import io.cucumber.java.en.Given;
import io.cucumber.java.en.Then;
import io.cucumber.java.en.When;
Expand All @@ -24,8 +37,10 @@
import javax.xml.parsers.ParserConfigurationException;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
Expand All @@ -36,19 +51,37 @@ public class MainDefinitions {
final Path ddiDirectory = Paths.get(TestConstants.FUNCTIONAL_TESTS_DDI_DIRECTORY);

SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService();
SurveyUnitPersistencePortStub surveyUnitPersistence = new SurveyUnitPersistencePortStub();
Config config = new ConfigStub();

ResponseController responseController = new ResponseController(
new SurveyUnitService(surveyUnitPersistence),
surveyUnitQualityService,
new LunaticXmlRawDataService(new LunaticXmlPersistanceStub()),
new LunaticJsonRawDataService(new LunaticJsonPersistanceStub()),
new FileUtils(config),
new ControllerUtils(new FileUtils(config))
);

List<SurveyUnitModel> surveyUnitModels;

@Before
public void init() {
this.surveyUnitPersistence.getMongoStub().clear();
}

@Given("We have data in directory {string}")
public void init(String directory){
public void init_folder(String directory) throws IOException {
this.directory = directory;
inDirectory = inDirectory.resolve(directory);
Files.createDirectories(inDirectory);
}

@When("We create DTOs from file {string} with DDI {string}")
public void get_dtos(String fileName, String DDIName) throws IOException, ParserConfigurationException, SAXException, GenesisException, MetadataParserException {
public void get_dtos(String fileName, String ddiName) throws IOException, ParserConfigurationException,
SAXException, GenesisException, MetadataParserException {
Path filePath = inDirectory.resolve(fileName);
Path ddiFilePath = ddiDirectory.resolve(directory).resolve(DDIName);
Path ddiFilePath = ddiDirectory.resolve(directory).resolve(ddiName);

if(fileName.endsWith(".xml")){
LunaticXmlDataParser parser = new LunaticXmlDataParser();
Expand All @@ -67,6 +100,32 @@ public void get_dtos(String fileName, String DDIName) throws IOException, Parser
}
}

@When("We copy data file {string} to that directory")
public void copy_data_file(String dataFile) throws IOException {
Path dataFilePath = Paths.get(TestConstants.TEST_RESOURCES_DIRECTORY).resolve(dataFile);
Files.copy(dataFilePath, this.inDirectory.resolve(dataFilePath.getFileName().toString()),
StandardCopyOption.REPLACE_EXISTING);
}

@When("We save data from that directory")
public void get_dtos_from_folder() throws Exception {
responseController.saveResponsesFromXmlCampaignFolder(this.inDirectory.getFileName().toString(), null, true);
}

@When("We delete that directory")
public void delete_directory() throws IOException {
org.springframework.util.FileSystemUtils.deleteRecursively(inDirectory);
}

@Then("There should be {int} {string} SurveyUnit in database")
public void check_surveyunits_by_state(int expectedCount, String expectedStatus) {
Assertions.assertThat(this.surveyUnitPersistence.getMongoStub().stream().filter(
surveyUnitModel -> surveyUnitModel.getState().name().equals(expectedStatus)
).toList()).hasSize(expectedCount);
}



@Then("For SurveyUnit {string} there should be at least one {string} SurveyUnit DTO")
public void check_expected_datastate_dto(String surveyUnitId, String expectedDataState) {
Assertions.assertThat(this.surveyUnitModels).filteredOn(surveyUnitDto ->
Expand All @@ -76,9 +135,9 @@ public void check_expected_datastate_dto(String surveyUnitId, String expectedDat
}

@Then("For SurveyUnit {string} there shouldn't be a {string} SurveyUnit DTO")
public void check_unexpected_datastate_dto(String surveyUnitId, String UnexpectedDataState) {
public void check_unexpected_datastate_dto(String surveyUnitId, String unexpectedDataState) {
Assertions.assertThat(this.surveyUnitModels).filteredOn(surveyUnitDto ->
surveyUnitDto.getState().toString().equals(UnexpectedDataState)
surveyUnitDto.getState().toString().equals(unexpectedDataState)
&& surveyUnitDto.getIdUE().equals(surveyUnitId)
).isEmpty();
}
Expand Down Expand Up @@ -185,4 +244,11 @@ public void external_variable_content_check(String externalVariableName, String
Assertions.assertThat(concernedVariable.getValues()).hasSizeGreaterThan(expectedValueIndex);
Assertions.assertThat(concernedVariable.getValues().get(expectedValueIndex)).isEqualTo(expectedValue);
}

@After
public void clean() throws IOException {
//Clean DONE test folder
org.springframework.util.FileSystemUtils.deleteRecursively(Path.of(TestConstants.TEST_RESOURCES_DIRECTORY,
"DONE"));
}
}
6 changes: 6 additions & 0 deletions src/test/resources/data_backup/data.complete.partial.STPDv1.20231122164209.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<Campaign>
<Id>SAMPLETEST-PARADATA-v1</Id>
<Label />
<SurveyUnits />
</Campaign>
Loading

0 comments on commit 4708d1c

Please sign in to comment.