Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix save folder saving only 1 file at a time #148

Merged
merged 1 commit into from
Jan 8, 2025
Merged

Fix save folder saving only 1 file at a time #148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,4 @@ hs_err_pid*
src/test/resources/OUT/*
/src/test/resources/DONE/*
/src/test/resources/genesis_deleted_schedules/*
/src/test/resources/IN/WEB/SAMPLETEST/
12 changes: 7 additions & 5 deletions src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -422,7 +422,7 @@ private void processCampaignWithMode(String campaignName, Mode mode, List<Genesi
if(isDataFileInDoneFolder(filepath, campaignName, mode.getFolder())){
log.warn("File {} already exists in DONE folder ! Deleting...", fileName);
Files.deleteIfExists(filepath);
return;
continue; //Go to next file
}
//Read file
log.info("Try to read Xml file : {}", fileName);
Expand All @@ -433,11 +433,13 @@ private void processCampaignWithMode(String campaignName, Mode mode, List<Genesi
response = processXmlFileSequentially(filepath, mode, variablesMap);
}
log.debug("File {} saved", fileName);
if(response.getStatusCode() == HttpStatus.OK){
fileUtils.moveDataFile(campaignName, mode.getFolder(), filepath);
return;
if (response.getStatusCode() == HttpStatus.OK) {
fileUtils.moveDataFile(campaignName, mode.getFolder(),
filepath);
//Sonar is annoying when there is more than 1 continue
} else {
log.error("Error {} on file {} : {}", response.getStatusCode(), fileName, response.getBody());
}
log.error("Error on file {}", fileName);
}
}

Expand Down
80 changes: 73 additions & 7 deletions src/test/java/cucumber/functional_tests/MainDefinitions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,30 @@
import fr.insee.bpm.exceptions.MetadataParserException;
import fr.insee.bpm.metadata.model.VariablesMap;
import fr.insee.bpm.metadata.reader.ddi.DDIReader;
import fr.insee.genesis.configuration.Config;
import fr.insee.genesis.controller.adapter.LunaticXmlAdapter;
import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
import fr.insee.genesis.controller.rest.responses.ResponseController;
import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign;
import fr.insee.genesis.controller.sources.xml.LunaticXmlDataParser;
import fr.insee.genesis.controller.sources.xml.LunaticXmlSurveyUnit;
import fr.insee.genesis.controller.utils.ControllerUtils;
import fr.insee.genesis.domain.model.surveyunit.CollectedVariable;
import fr.insee.genesis.domain.model.surveyunit.DataState;
import fr.insee.genesis.domain.model.surveyunit.Mode;
import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
import fr.insee.genesis.domain.model.surveyunit.Variable;
import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService;
import fr.insee.genesis.domain.service.rawdata.LunaticXmlRawDataService;
import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService;
import fr.insee.genesis.exceptions.GenesisException;
import fr.insee.genesis.infrastructure.utils.FileUtils;
import fr.insee.genesis.stubs.ConfigStub;
import fr.insee.genesis.stubs.LunaticJsonPersistanceStub;
import fr.insee.genesis.stubs.LunaticXmlPersistanceStub;
import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
import io.cucumber.java.After;
import io.cucumber.java.Before;
import io.cucumber.java.en.Given;
import io.cucumber.java.en.Then;
import io.cucumber.java.en.When;
Expand All @@ -24,8 +37,10 @@
import javax.xml.parsers.ParserConfigurationException;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
Expand All @@ -36,19 +51,37 @@ public class MainDefinitions {
final Path ddiDirectory = Paths.get(TestConstants.FUNCTIONAL_TESTS_DDI_DIRECTORY);

SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService();
SurveyUnitPersistencePortStub surveyUnitPersistence = new SurveyUnitPersistencePortStub();
Config config = new ConfigStub();

ResponseController responseController = new ResponseController(
new SurveyUnitService(surveyUnitPersistence),
surveyUnitQualityService,
new LunaticXmlRawDataService(new LunaticXmlPersistanceStub()),
new LunaticJsonRawDataService(new LunaticJsonPersistanceStub()),
new FileUtils(config),
new ControllerUtils(new FileUtils(config))
);

List<SurveyUnitModel> surveyUnitModels;

@Before
public void init() {
this.surveyUnitPersistence.getMongoStub().clear();
}

@Given("We have data in directory {string}")
public void init(String directory){
public void init_folder(String directory) throws IOException {
this.directory = directory;
inDirectory = inDirectory.resolve(directory);
Files.createDirectories(inDirectory);
}

@When("We create DTOs from file {string} with DDI {string}")
public void get_dtos(String fileName, String DDIName) throws IOException, ParserConfigurationException, SAXException, GenesisException, MetadataParserException {
public void get_dtos(String fileName, String ddiName) throws IOException, ParserConfigurationException,
SAXException, GenesisException, MetadataParserException {
Path filePath = inDirectory.resolve(fileName);
Path ddiFilePath = ddiDirectory.resolve(directory).resolve(DDIName);
Path ddiFilePath = ddiDirectory.resolve(directory).resolve(ddiName);

if(fileName.endsWith(".xml")){
LunaticXmlDataParser parser = new LunaticXmlDataParser();
Expand All @@ -67,6 +100,32 @@ public void get_dtos(String fileName, String DDIName) throws IOException, Parser
}
}

@When("We copy data file {string} to that directory")
public void copy_data_file(String dataFile) throws IOException {
Path dataFilePath = Paths.get(TestConstants.TEST_RESOURCES_DIRECTORY).resolve(dataFile);
Files.copy(dataFilePath, this.inDirectory.resolve(dataFilePath.getFileName().toString()),
StandardCopyOption.REPLACE_EXISTING);
}

@When("We save data from that directory")
public void get_dtos_from_folder() throws Exception {
responseController.saveResponsesFromXmlCampaignFolder(this.inDirectory.getFileName().toString(), null, true);
}

@When("We delete that directory")
public void delete_directory() throws IOException {
org.springframework.util.FileSystemUtils.deleteRecursively(inDirectory);
}

@Then("There should be {int} {string} SurveyUnit in database")
public void check_surveyunits_by_state(int expectedCount, String expectedStatus) {
Assertions.assertThat(this.surveyUnitPersistence.getMongoStub().stream().filter(
surveyUnitModel -> surveyUnitModel.getState().name().equals(expectedStatus)
).toList()).hasSize(expectedCount);
}



@Then("For SurveyUnit {string} there should be at least one {string} SurveyUnit DTO")
public void check_expected_datastate_dto(String surveyUnitId, String expectedDataState) {
Assertions.assertThat(this.surveyUnitModels).filteredOn(surveyUnitDto ->
Expand All @@ -76,9 +135,9 @@ public void check_expected_datastate_dto(String surveyUnitId, String expectedDat
}

@Then("For SurveyUnit {string} there shouldn't be a {string} SurveyUnit DTO")
public void check_unexpected_datastate_dto(String surveyUnitId, String UnexpectedDataState) {
public void check_unexpected_datastate_dto(String surveyUnitId, String unexpectedDataState) {
Assertions.assertThat(this.surveyUnitModels).filteredOn(surveyUnitDto ->
surveyUnitDto.getState().toString().equals(UnexpectedDataState)
surveyUnitDto.getState().toString().equals(unexpectedDataState)
&& surveyUnitDto.getIdUE().equals(surveyUnitId)
).isEmpty();
}
Expand Down Expand Up @@ -185,4 +244,11 @@ public void external_variable_content_check(String externalVariableName, String
Assertions.assertThat(concernedVariable.getValues()).hasSizeGreaterThan(expectedValueIndex);
Assertions.assertThat(concernedVariable.getValues().get(expectedValueIndex)).isEqualTo(expectedValue);
}

@After
public void clean() throws IOException {
//Clean DONE test folder
org.springframework.util.FileSystemUtils.deleteRecursively(Path.of(TestConstants.TEST_RESOURCES_DIRECTORY,
"DONE"));
}
}
6 changes: 6 additions & 0 deletions src/test/resources/data_backup/data.complete.partial.STPDv1.20231122164209.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<Campaign>
<Id>SAMPLETEST-PARADATA-v1</Id>
<Label />
<SurveyUnits />
</Campaign>
Loading
Loading