diff --git a/pom.xml b/pom.xml index 93293644..cfc47a3d 100644 --- a/pom.xml +++ b/pom.xml @@ -35,8 +35,8 @@ UTF-8 17 - 3.3.1 - 1.19.8 + 3.3.2 + 1.20.0 more-project diff --git a/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/PolarVerityObservation.java b/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/PolarVerityObservation.java index 1e43c399..ea263056 100644 --- a/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/PolarVerityObservation.java +++ b/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/PolarVerityObservation.java @@ -44,9 +44,9 @@ private enum DataViewInfoType implements DataViewInfo { DataViewInfoType(String i18nKey, DataView.ChartType chartType, ViewConfig viewConfig) { this( - "data.charts.polarVerity.%s.label".formatted(i18nKey), - "data.charts.polarVerity.%s.title".formatted(i18nKey), - "data.charts.polarVerity.%s.description".formatted(i18nKey), + "monitoring.charts.polarVerity.%s.label".formatted(i18nKey), + "monitoring.charts.polarVerity.%s.title".formatted(i18nKey), + "monitoring.charts.polarVerity.%s.description".formatted(i18nKey), chartType, viewConfig ); diff --git a/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/QuestionObservation.java b/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/QuestionObservation.java index d4af1c84..ec9816b9 100644 --- a/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/QuestionObservation.java +++ b/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/QuestionObservation.java @@ -65,9 +65,9 @@ private enum DataViewInfoType implements DataViewInfo { DataViewInfoType(String i18nKey, DataView.ChartType chartType, ViewConfig viewConfig) { this( - "data.charts.simpleQuestion.%s.label".formatted(i18nKey), - "data.charts.simpleQuestion.%s.title".formatted(i18nKey), - "data.charts.simpleQuestion.%s.description".formatted(i18nKey), + "monitoring.charts.simpleQuestion.%s.label".formatted(i18nKey), + "monitoring.charts.simpleQuestion.%s.title".formatted(i18nKey), + "monitoring.charts.simpleQuestion.%s.description".formatted(i18nKey), chartType, viewConfig ); diff --git a/studymanager/pom.xml b/studymanager/pom.xml index 7ef08e52..8554120b 100644 --- a/studymanager/pom.xml +++ b/studymanager/pom.xml @@ -260,6 +260,7 @@ Instant=java.time.Instant LocalTime=java.time.LocalTime + DataExport=org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody Instant=java.time.Instant diff --git a/studymanager/src/main/java/io/redlink/more/studymanager/controller/studymanager/ImportExportApiV1Controller.java b/studymanager/src/main/java/io/redlink/more/studymanager/controller/studymanager/ImportExportApiV1Controller.java index af8a46dc..c5978075 100644 --- a/studymanager/src/main/java/io/redlink/more/studymanager/controller/studymanager/ImportExportApiV1Controller.java +++ b/studymanager/src/main/java/io/redlink/more/studymanager/controller/studymanager/ImportExportApiV1Controller.java @@ -21,20 +21,29 @@ import io.redlink.more.studymanager.service.ImportExportService; import io.redlink.more.studymanager.service.OAuth2AuthenticationService; import io.redlink.more.studymanager.utils.MapperUtils; -import jakarta.servlet.http.HttpServletResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.core.io.Resource; +import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; +import org.springframework.web.servlet.support.ServletUriComponentsBuilder; import java.io.IOException; +import java.time.Instant; +import java.util.List; import java.util.Optional; @RestController @RequestMapping(value = "/api/v1") public class ImportExportApiV1Controller implements ImportExportApi { + private static final Logger LOGGER = LoggerFactory.getLogger(StudyApiV1Controller.class); private final ImportExportService service; @@ -84,28 +93,38 @@ public ResponseEntity exportStudy(Long studyId) { } @Override - @RequiresStudyRole({StudyRole.STUDY_ADMIN, StudyRole.STUDY_OPERATOR}) - public ResponseEntity generateDownloadToken(Long studyId) { - return ResponseEntity.ok(new GenerateDownloadToken200ResponseDTO().token( - tokenRepository.createToken(studyId).getToken() - )); - } - - @RequestMapping( - method = RequestMethod.GET, - value = "/studies/{studyId}/export/studydata/{token}", - produces = { "application/json" } - ) - public void exportStudyData(@PathVariable Long studyId, @PathVariable("token") String token, HttpServletResponse response) throws IOException { + public ResponseEntity exportStudyData(Long studyId, String token, List studyGroupId, List participantId, List observationId, Instant from, Instant to) { Optional dt = tokenRepository.getToken(token).filter(t -> t.getStudyId().equals(studyId)); - if(dt.isPresent()) { - response.setHeader("Content-Disposition", "attachment;filename=" + dt.get().getFilename()); - service.exportStudyData(response.getOutputStream(), studyId); + + if (dt.isPresent()) { + HttpHeaders responseHeaders = new HttpHeaders(); + responseHeaders.set("Content-Disposition", "attachment;filename=" + dt.get().getFilename()); + + return ResponseEntity + .ok() + .headers(responseHeaders) + .contentType(MediaType.APPLICATION_JSON) + .body(outputStream -> { + try { + service.exportStudyData(outputStream, studyId, studyGroupId, participantId, observationId, from, to); + } catch (Exception e) { + LOGGER.warn("Error exporting study data for study_{}: {}", studyId, e.getMessage(), e); + } + }); } else { - response.setStatus(403); + return ResponseEntity.notFound().build(); } } + @Override + @RequiresStudyRole({StudyRole.STUDY_ADMIN, StudyRole.STUDY_OPERATOR}) + public ResponseEntity generateDownloadToken(Long studyId, List studyGroupId, List participantId, List observationId, Instant from, Instant to) { + var token = tokenRepository.createToken(studyId).getToken(); + var uri = ServletUriComponentsBuilder.fromCurrentRequest().pathSegment(token).build(true).toUri(); + + return ResponseEntity.created(uri).body(new GenerateDownloadToken200ResponseDTO().token(token)); + } + @Override public ResponseEntity importStudy(MultipartFile file) { try { diff --git a/studymanager/src/main/java/io/redlink/more/studymanager/service/ElasticService.java b/studymanager/src/main/java/io/redlink/more/studymanager/service/ElasticService.java index ebd725a1..a83e5b62 100644 --- a/studymanager/src/main/java/io/redlink/more/studymanager/service/ElasticService.java +++ b/studymanager/src/main/java/io/redlink/more/studymanager/service/ElasticService.java @@ -14,7 +14,9 @@ import co.elastic.clients.elasticsearch._types.FieldValue; import co.elastic.clients.elasticsearch._types.SortOrder; import co.elastic.clients.elasticsearch._types.aggregations.StringTermsBucket; +import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField; import co.elastic.clients.elasticsearch.core.DeleteByQueryRequest; import co.elastic.clients.elasticsearch.core.SearchRequest; import co.elastic.clients.elasticsearch.core.SearchResponse; @@ -22,6 +24,7 @@ import co.elastic.clients.elasticsearch.indices.CloseIndexRequest; import co.elastic.clients.elasticsearch.indices.DeleteIndexRequest; import co.elastic.clients.elasticsearch.indices.ExistsRequest; +import co.elastic.clients.json.JsonData; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Iterables; import io.redlink.more.studymanager.core.io.TimeRange; @@ -31,22 +34,19 @@ import io.redlink.more.studymanager.model.data.SimpleDataPoint; import io.redlink.more.studymanager.properties.ElasticProperties; import io.redlink.more.studymanager.utils.MapperUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.stereotype.Service; + import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.stereotype.Service; @Service @EnableConfigurationProperties({ElasticProperties.class}) @@ -328,23 +328,23 @@ public List getParticipationData(Long studyId){ } } - public void exportData(Long studyId, OutputStream outputStream) throws IOException { + public void exportData(OutputStream outputStream, Long studyId, List studyGroupId, List participantId, List observationId, Instant from, Instant to) throws IOException { String index = getStudyIdString(studyId); if(!client.indices().exists(e -> e.index(index)).value()) { return; } - SearchRequest request = getQuery(index, null); + SearchRequest request = getQuery(index, studyGroupId, participantId, observationId, from, to, null); SearchResponse rsp = client.search(request, JsonNode.class); while (rsp.hits().hits().size() > 0) { writeHits(rsp.hits().hits(), outputStream); outputStream.flush(); List searchAfterSort = Iterables.getLast(rsp.hits().hits()).sort(); - request = getQuery(index, searchAfterSort); + request = getQuery(index, studyGroupId, participantId, observationId, from, to, searchAfterSort); rsp = client.search(request, JsonNode.class); - if(rsp.hits().hits().size() > 0) { + if (rsp.hits().hits().size() > 0) { outputStream.write(",".getBytes(StandardCharsets.UTF_8)); } } @@ -358,21 +358,46 @@ private void writeHits(List> hits, OutputStream outputStream) thro outputStream.write(datapoints.getBytes(StandardCharsets.UTF_8)); } - private SearchRequest getQuery(String index, List searchAfterSort) { + private SearchRequest getQuery(String index, List studyGroupIds, List participantIds, List observationIds, Instant from, Instant to, List searchAfterSort) { SearchRequest.Builder builder = new SearchRequest.Builder(); + BoolQuery.Builder boolQueryBuilder = new BoolQuery.Builder(); + + if (studyGroupIds != null && !studyGroupIds.isEmpty()) { + List studyGroupIdsStrings = studyGroupIds.stream() + .map(ElasticService::getStudyGroupIdString) + .toList(); + boolQueryBuilder.filter(f -> f.terms(t -> t.field("study_group_id").terms(TermsQueryField.of(tf -> tf.value(studyGroupIdsStrings.stream().map(FieldValue::of).collect(Collectors.toList())))))); + } + + if (participantIds != null && !participantIds.isEmpty()) { + List participantIdStrings = participantIds.stream() + .map(ElasticService::getParticipantIdString) + .toList(); + boolQueryBuilder.filter(f -> f.terms(t -> t.field("participant_id").terms(TermsQueryField.of(tf -> tf.value(participantIdStrings.stream().map(FieldValue::of).collect(Collectors.toList())))))); + } + + if (observationIds != null && !observationIds.isEmpty()) { + boolQueryBuilder.filter(f -> f.terms(t -> t.field("observation_id").terms(v -> v.value(observationIds.stream().map(FieldValue::of).collect(Collectors.toList()))))); + } + + if (from != null && to != null) { + boolQueryBuilder.filter(f -> f.range(r -> r.field("effective_time_frame").gte(JsonData.of(from)).lte(JsonData.of(to)))); + } + builder.index(index) - .query(q -> q.matchAll(m -> m)) - //.pit(p -> p.id(pitId).keepAlive(k -> k.time("1m"))) + .query(q -> q.bool(boolQueryBuilder.build())) .sort(s -> s.field(f -> f.field("effective_time_frame").order(SortOrder.Asc))) + .sort(s -> s.field(f -> f.field("datapoint_id.keyword").order(SortOrder.Asc))) .size(BATCH_SIZE_FOR_EXPORT_REQUESTS); - if(searchAfterSort != null) { + if (searchAfterSort != null) { builder.searchAfter(searchAfterSort); } return builder.build(); } + public List listDataPoints( Long studyId, Integer participantId, Integer observationId, String isoDate, int size) throws IOException { SearchRequest.Builder builder = new SearchRequest.Builder(); diff --git a/studymanager/src/main/java/io/redlink/more/studymanager/service/ImportExportService.java b/studymanager/src/main/java/io/redlink/more/studymanager/service/ImportExportService.java index a4273d75..d4889342 100644 --- a/studymanager/src/main/java/io/redlink/more/studymanager/service/ImportExportService.java +++ b/studymanager/src/main/java/io/redlink/more/studymanager/service/ImportExportService.java @@ -10,7 +10,6 @@ import io.redlink.more.studymanager.exception.NotFoundException; import io.redlink.more.studymanager.model.*; -import jakarta.servlet.ServletOutputStream; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,7 +21,9 @@ import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.*; @Service @@ -151,19 +152,19 @@ public Study importStudy(StudyImportExport studyImport, AuthenticatedUser user) return newStudy; } - public void exportStudyData(ServletOutputStream outputStream, Long studyId) { + public void exportStudyData(OutputStream outputStream, Long studyId, List studyGroupId, List participantId, List observationId, Instant from, Instant to) { if (studyService.existsStudy(studyId).orElse(false)) { - exportStudyDataAsync(outputStream, studyId); + exportStudyDataAsync(outputStream, studyId, studyGroupId, participantId, observationId, from, to); } else { throw NotFoundException.Study(studyId); } } @Async - public void exportStudyDataAsync(ServletOutputStream outputStream, Long studyId) { + public void exportStudyDataAsync(OutputStream outputStream, Long studyId, List studyGroupId, List participantId, List observationId, Instant from, Instant to) { try (outputStream) { outputStream.write("[".getBytes(StandardCharsets.UTF_8)); - elasticService.exportData(studyId, outputStream); + elasticService.exportData(outputStream, studyId, studyGroupId, participantId, observationId, from, to); outputStream.write("]".getBytes(StandardCharsets.UTF_8)); } catch (IOException e) { LOGGER.error("Cannot export study data for {}", studyId, e); diff --git a/studymanager/src/main/resources/application.yaml b/studymanager/src/main/resources/application.yaml index c945047c..8a5cd547 100644 --- a/studymanager/src/main/resources/application.yaml +++ b/studymanager/src/main/resources/application.yaml @@ -50,6 +50,9 @@ spring: flyway: out-of-order: true + mvc: + async: + request-timeout: -1 server: forward-headers-strategy: framework error: diff --git a/studymanager/src/main/resources/openapi/StudyManagerAPI.yaml b/studymanager/src/main/resources/openapi/StudyManagerAPI.yaml index 7b751336..36d05e43 100644 --- a/studymanager/src/main/resources/openapi/StudyManagerAPI.yaml +++ b/studymanager/src/main/resources/openapi/StudyManagerAPI.yaml @@ -1073,7 +1073,7 @@ paths: operationId: exportStudy responses: '200': - description: foo + description: Operation successful content: application/json: schema: @@ -1088,10 +1088,31 @@ paths: tags: - importExport operationId: generateDownloadToken - description: generate a download token that can be used once for download and is valid for 10 minutes + description: Generate a download token that can be used once for download and is valid for 10 minutes + parameters: + - name: studyGroupId + in: query + schema: + type: array + items: + type: integer + - name: participantId + in: query + schema: + type: array + items: + type: integer + - name: observationId + in: query + schema: + type: array + items: + type: integer + - $ref: '#/components/parameters/From' + - $ref: '#/components/parameters/To' responses: '200': - description: foo + description: Operation successful content: application/json: schema: @@ -1102,6 +1123,50 @@ paths: '404': description: study not found + /studies/{studyId}/export/studydata/{token}: + parameters: + - $ref: '#/components/parameters/StudyId' + - name: token + in: path + required: true + schema: + type: string + get: + tags: + - importExport + description: Download Study data + operationId: exportStudyData + parameters: + - name: studyGroupId + in: query + schema: + type: array + items: + type: integer + - name: participantId + in: query + schema: + type: array + items: + type: integer + - name: observationId + in: query + schema: + type: array + items: + type: integer + - $ref: '#/components/parameters/From' + - $ref: '#/components/parameters/To' + responses: + '200': + description: Operation successful + content: + application/json: + schema: + $ref: '#/components/schemas/DataExport' + '404': + description: study not found + /studies/{studyId}/import/participants: parameters: - $ref: '#/components/parameters/StudyId' @@ -1135,7 +1200,7 @@ paths: operationId: exportParticipants responses: '200': - description: foo + description: Operation successful content: text/csv: schema: @@ -2111,6 +2176,31 @@ components: required: - version - date + DataExport: + type: array + items: + type: object + properties: + datapoint_id: + type: string + participant_id: + type: string + study_id: + type: string + study_group_id: + type: string + nullable: true + observation_id: + type: string + observation_type: + type: string + data_type: + type: string + storage_date: + type: string + effective_time_frame: + type: string + additionalProperties: true parameters: ComponentType: diff --git a/studymanager/src/test/java/io/redlink/more/studymanager/service/ElasticSearchServiceTest.java b/studymanager/src/test/java/io/redlink/more/studymanager/service/ElasticSearchServiceTest.java index 47efea77..7b119c98 100644 --- a/studymanager/src/test/java/io/redlink/more/studymanager/service/ElasticSearchServiceTest.java +++ b/studymanager/src/test/java/io/redlink/more/studymanager/service/ElasticSearchServiceTest.java @@ -14,18 +14,11 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.io.Resources; import io.redlink.more.studymanager.configuration.ElasticConfiguration; -import io.redlink.more.studymanager.model.data.ElasticActionDataPoint; -import io.redlink.more.studymanager.model.Study; import io.redlink.more.studymanager.core.io.Timeframe; - -import java.io.ByteArrayOutputStream; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.UUID; - +import io.redlink.more.studymanager.model.Study; +import io.redlink.more.studymanager.model.data.ElasticActionDataPoint; import io.redlink.more.studymanager.model.data.ElasticObservationDataPoint; import io.redlink.more.studymanager.utils.MapperUtils; -import org.junit.Assert; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.jackson.JacksonAutoConfiguration; @@ -38,12 +31,17 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.List; +import java.util.Map; +import java.util.UUID; -import static org.assertj.core.api.Assertions.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE) @Testcontainers @@ -109,14 +107,14 @@ void testRecordAction() { @Test void testExportData() throws JsonProcessingException, InterruptedException { for (int i = 0; i < 1200; i++) { - setDataPoint(1L, 2, i); + setDataPoint(1L, "2", "1", "1", i); } //wait for auto commit Thread.sleep(2000); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try(outputStream) { outputStream.write("[".getBytes(StandardCharsets.UTF_8)); - elasticService.exportData(1L, outputStream); + elasticService.exportData(outputStream, 1L, List.of(2), List.of(1), List.of(1), null, null); outputStream.write("]".getBytes(StandardCharsets.UTF_8)); } catch (IOException e) { //do nothing than close @@ -126,13 +124,13 @@ void testExportData() throws JsonProcessingException, InterruptedException { assertThat(MapperUtils.MAPPER.readValue(result, List.class)).hasSize(1200); } - private void setDataPoint(Long studyId, int participantId, int i) { + private void setDataPoint(Long studyId, String studyGroupId, String participantId, String observationId, int i) { elasticService.setDataPoint(studyId, new ElasticObservationDataPoint( "DP_" + studyId + "_" + participantId + "_" + i, "participant_"+ participantId, "study_" + studyId, - null, - "2", + "study_group_" + studyGroupId, + observationId, "acc-mobile-observation", "acc-mobile-observation", Instant.now(),