From fba283716c47a69213a0c139025f03a2623d6c30 Mon Sep 17 00:00:00 2001 From: bnitsch Date: Wed, 17 Jul 2024 13:47:04 +0200 Subject: [PATCH] #317: Data download improvements * It is now possible to filter the data you want to download. * Remove the default timeout for requests (30 seconds) to avoid aborting the download. * Change translation key, because we now differ between monitoring and data tab. --- .../observation/PolarVerityObservation.java | 6 +- .../observation/QuestionObservation.java | 6 +- studymanager/pom.xml | 1 + .../ImportExportApiV1Controller.java | 54 ++++++--- .../studymanager/service/ElasticService.java | 60 ++++++--- .../service/ImportExportService.java | 11 +- .../src/main/resources/application.yaml | 3 + .../resources/openapi/StudyManagerAPI.yaml | 114 +++++++++++++++++- .../service/ElasticSearchServiceTest.java | 28 ++--- 9 files changed, 217 insertions(+), 66 deletions(-) diff --git a/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/PolarVerityObservation.java b/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/PolarVerityObservation.java index 1e43c399..ea263056 100644 --- a/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/PolarVerityObservation.java +++ b/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/PolarVerityObservation.java @@ -44,9 +44,9 @@ private enum DataViewInfoType implements DataViewInfo { DataViewInfoType(String i18nKey, DataView.ChartType chartType, ViewConfig viewConfig) { this( - "data.charts.polarVerity.%s.label".formatted(i18nKey), - "data.charts.polarVerity.%s.title".formatted(i18nKey), - "data.charts.polarVerity.%s.description".formatted(i18nKey), + "monitoring.charts.polarVerity.%s.label".formatted(i18nKey), + "monitoring.charts.polarVerity.%s.title".formatted(i18nKey), + "monitoring.charts.polarVerity.%s.description".formatted(i18nKey), chartType, viewConfig ); diff --git a/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/QuestionObservation.java b/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/QuestionObservation.java index d4af1c84..ec9816b9 100644 --- a/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/QuestionObservation.java +++ b/studymanager-observation/src/main/java/io/redlink/more/studymanager/component/observation/QuestionObservation.java @@ -65,9 +65,9 @@ private enum DataViewInfoType implements DataViewInfo { DataViewInfoType(String i18nKey, DataView.ChartType chartType, ViewConfig viewConfig) { this( - "data.charts.simpleQuestion.%s.label".formatted(i18nKey), - "data.charts.simpleQuestion.%s.title".formatted(i18nKey), - "data.charts.simpleQuestion.%s.description".formatted(i18nKey), + "monitoring.charts.simpleQuestion.%s.label".formatted(i18nKey), + "monitoring.charts.simpleQuestion.%s.title".formatted(i18nKey), + "monitoring.charts.simpleQuestion.%s.description".formatted(i18nKey), chartType, viewConfig ); diff --git a/studymanager/pom.xml b/studymanager/pom.xml index 7ef08e52..8554120b 100644 --- a/studymanager/pom.xml +++ b/studymanager/pom.xml @@ -260,6 +260,7 @@ Instant=java.time.Instant LocalTime=java.time.LocalTime + DataExport=org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody Instant=java.time.Instant diff --git a/studymanager/src/main/java/io/redlink/more/studymanager/controller/studymanager/ImportExportApiV1Controller.java b/studymanager/src/main/java/io/redlink/more/studymanager/controller/studymanager/ImportExportApiV1Controller.java index af8a46dc..734f9067 100644 --- a/studymanager/src/main/java/io/redlink/more/studymanager/controller/studymanager/ImportExportApiV1Controller.java +++ b/studymanager/src/main/java/io/redlink/more/studymanager/controller/studymanager/ImportExportApiV1Controller.java @@ -21,14 +21,26 @@ import io.redlink.more.studymanager.service.ImportExportService; import io.redlink.more.studymanager.service.OAuth2AuthenticationService; import io.redlink.more.studymanager.utils.MapperUtils; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.Cookie; import jakarta.servlet.http.HttpServletResponse; import org.springframework.core.io.Resource; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; +import org.springframework.http.*; +import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; +import org.springframework.web.servlet.support.ServletUriComponentsBuilder; import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.time.Instant; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.util.Collection; +import java.util.List; +import java.util.Locale; import java.util.Optional; @@ -84,28 +96,32 @@ public ResponseEntity exportStudy(Long studyId) { } @Override - @RequiresStudyRole({StudyRole.STUDY_ADMIN, StudyRole.STUDY_OPERATOR}) - public ResponseEntity generateDownloadToken(Long studyId) { - return ResponseEntity.ok(new GenerateDownloadToken200ResponseDTO().token( - tokenRepository.createToken(studyId).getToken() - )); - } - - @RequestMapping( - method = RequestMethod.GET, - value = "/studies/{studyId}/export/studydata/{token}", - produces = { "application/json" } - ) - public void exportStudyData(@PathVariable Long studyId, @PathVariable("token") String token, HttpServletResponse response) throws IOException { + public ResponseEntity exportStudyData(Long studyId, String token, List studyGroupId, List participantId, List observationId, LocalDate from, LocalDate to) { Optional dt = tokenRepository.getToken(token).filter(t -> t.getStudyId().equals(studyId)); - if(dt.isPresent()) { - response.setHeader("Content-Disposition", "attachment;filename=" + dt.get().getFilename()); - service.exportStudyData(response.getOutputStream(), studyId); + + if (dt.isPresent()) { + HttpHeaders responseHeaders = new HttpHeaders(); + responseHeaders.set("Content-Disposition", "attachment;filename=" + dt.get().getFilename()); + + return ResponseEntity + .ok() + .headers(responseHeaders) + .contentType(MediaType.APPLICATION_JSON) + .body(outputStream -> service.exportStudyData(outputStream, studyId, studyGroupId, participantId, observationId, from, to)); } else { - response.setStatus(403); + return ResponseEntity.notFound().build(); } } + @Override + @RequiresStudyRole({StudyRole.STUDY_ADMIN, StudyRole.STUDY_OPERATOR}) + public ResponseEntity generateDownloadToken(Long studyId, List studyGroupId, List participantId, List observationId, LocalDate from, LocalDate to) { + var token = tokenRepository.createToken(studyId).getToken(); + var uri = ServletUriComponentsBuilder.fromCurrentRequest().pathSegment(token).build(true).toUri(); + + return ResponseEntity.created(uri).body(new GenerateDownloadToken200ResponseDTO().token(token)); + } + @Override public ResponseEntity importStudy(MultipartFile file) { try { diff --git a/studymanager/src/main/java/io/redlink/more/studymanager/service/ElasticService.java b/studymanager/src/main/java/io/redlink/more/studymanager/service/ElasticService.java index ebd725a1..eefadee7 100644 --- a/studymanager/src/main/java/io/redlink/more/studymanager/service/ElasticService.java +++ b/studymanager/src/main/java/io/redlink/more/studymanager/service/ElasticService.java @@ -14,7 +14,9 @@ import co.elastic.clients.elasticsearch._types.FieldValue; import co.elastic.clients.elasticsearch._types.SortOrder; import co.elastic.clients.elasticsearch._types.aggregations.StringTermsBucket; +import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField; import co.elastic.clients.elasticsearch.core.DeleteByQueryRequest; import co.elastic.clients.elasticsearch.core.SearchRequest; import co.elastic.clients.elasticsearch.core.SearchResponse; @@ -22,6 +24,7 @@ import co.elastic.clients.elasticsearch.indices.CloseIndexRequest; import co.elastic.clients.elasticsearch.indices.DeleteIndexRequest; import co.elastic.clients.elasticsearch.indices.ExistsRequest; +import co.elastic.clients.json.JsonData; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Iterables; import io.redlink.more.studymanager.core.io.TimeRange; @@ -31,22 +34,20 @@ import io.redlink.more.studymanager.model.data.SimpleDataPoint; import io.redlink.more.studymanager.properties.ElasticProperties; import io.redlink.more.studymanager.utils.MapperUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.stereotype.Service; + import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.time.LocalDate; +import java.util.*; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.stereotype.Service; @Service @EnableConfigurationProperties({ElasticProperties.class}) @@ -328,23 +329,23 @@ public List getParticipationData(Long studyId){ } } - public void exportData(Long studyId, OutputStream outputStream) throws IOException { + public void exportData(OutputStream outputStream, Long studyId, List studyGroupId, List participantId, List observationId, LocalDate from, LocalDate to) throws IOException { String index = getStudyIdString(studyId); if(!client.indices().exists(e -> e.index(index)).value()) { return; } - SearchRequest request = getQuery(index, null); + SearchRequest request = getQuery(index, studyGroupId, participantId, observationId, from, to, null); SearchResponse rsp = client.search(request, JsonNode.class); while (rsp.hits().hits().size() > 0) { writeHits(rsp.hits().hits(), outputStream); outputStream.flush(); List searchAfterSort = Iterables.getLast(rsp.hits().hits()).sort(); - request = getQuery(index, searchAfterSort); + request = getQuery(index, studyGroupId, participantId, observationId, from, to, searchAfterSort); rsp = client.search(request, JsonNode.class); - if(rsp.hits().hits().size() > 0) { + if (rsp.hits().hits().size() > 0) { outputStream.write(",".getBytes(StandardCharsets.UTF_8)); } } @@ -358,21 +359,46 @@ private void writeHits(List> hits, OutputStream outputStream) thro outputStream.write(datapoints.getBytes(StandardCharsets.UTF_8)); } - private SearchRequest getQuery(String index, List searchAfterSort) { + private SearchRequest getQuery(String index, List studyGroupIds, List participantIds, List observationIds, LocalDate from, LocalDate to, List searchAfterSort) { SearchRequest.Builder builder = new SearchRequest.Builder(); + BoolQuery.Builder boolQueryBuilder = new BoolQuery.Builder(); + + if (studyGroupIds != null && !studyGroupIds.isEmpty()) { + List studyGroupIdsStrings = studyGroupIds.stream() + .map(ElasticService::getStudyGroupIdString) + .toList(); + boolQueryBuilder.filter(f -> f.terms(t -> t.field("study_group_id").terms(TermsQueryField.of(tf -> tf.value(studyGroupIdsStrings.stream().map(FieldValue::of).collect(Collectors.toList())))))); + } + + if (participantIds != null && !participantIds.isEmpty()) { + List participantIdStrings = participantIds.stream() + .map(ElasticService::getParticipantIdString) + .toList(); + boolQueryBuilder.filter(f -> f.terms(t -> t.field("participant_id").terms(TermsQueryField.of(tf -> tf.value(participantIdStrings.stream().map(FieldValue::of).collect(Collectors.toList())))))); + } + + if (observationIds != null && !observationIds.isEmpty()) { + boolQueryBuilder.filter(f -> f.terms(t -> t.field("observation_id").terms(v -> v.value(observationIds.stream().map(FieldValue::of).collect(Collectors.toList()))))); + } + + if (from != null && to != null) { + boolQueryBuilder.filter(f -> f.range(r -> r.field("effective_time_frame").gte(JsonData.of(from)).lte(JsonData.of(to)))); + } + builder.index(index) - .query(q -> q.matchAll(m -> m)) - //.pit(p -> p.id(pitId).keepAlive(k -> k.time("1m"))) + .query(q -> q.bool(boolQueryBuilder.build())) .sort(s -> s.field(f -> f.field("effective_time_frame").order(SortOrder.Asc))) + .sort(s -> s.field(f -> f.field("datapoint_id.keyword").order(SortOrder.Asc))) .size(BATCH_SIZE_FOR_EXPORT_REQUESTS); - if(searchAfterSort != null) { + if (searchAfterSort != null) { builder.searchAfter(searchAfterSort); } return builder.build(); } + public List listDataPoints( Long studyId, Integer participantId, Integer observationId, String isoDate, int size) throws IOException { SearchRequest.Builder builder = new SearchRequest.Builder(); diff --git a/studymanager/src/main/java/io/redlink/more/studymanager/service/ImportExportService.java b/studymanager/src/main/java/io/redlink/more/studymanager/service/ImportExportService.java index a4273d75..6527e039 100644 --- a/studymanager/src/main/java/io/redlink/more/studymanager/service/ImportExportService.java +++ b/studymanager/src/main/java/io/redlink/more/studymanager/service/ImportExportService.java @@ -10,7 +10,6 @@ import io.redlink.more.studymanager.exception.NotFoundException; import io.redlink.more.studymanager.model.*; -import jakarta.servlet.ServletOutputStream; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,7 +21,9 @@ import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; +import java.time.LocalDate; import java.util.*; @Service @@ -151,19 +152,19 @@ public Study importStudy(StudyImportExport studyImport, AuthenticatedUser user) return newStudy; } - public void exportStudyData(ServletOutputStream outputStream, Long studyId) { + public void exportStudyData(OutputStream outputStream, Long studyId, List studyGroupId, List participantId, List observationId, LocalDate from, LocalDate to) { if (studyService.existsStudy(studyId).orElse(false)) { - exportStudyDataAsync(outputStream, studyId); + exportStudyDataAsync(outputStream, studyId, studyGroupId, participantId, observationId, from, to); } else { throw NotFoundException.Study(studyId); } } @Async - public void exportStudyDataAsync(ServletOutputStream outputStream, Long studyId) { + public void exportStudyDataAsync(OutputStream outputStream, Long studyId, List studyGroupId, List participantId, List observationId, LocalDate from, LocalDate to) { try (outputStream) { outputStream.write("[".getBytes(StandardCharsets.UTF_8)); - elasticService.exportData(studyId, outputStream); + elasticService.exportData(outputStream, studyId, studyGroupId, participantId, observationId, from, to); outputStream.write("]".getBytes(StandardCharsets.UTF_8)); } catch (IOException e) { LOGGER.error("Cannot export study data for {}", studyId, e); diff --git a/studymanager/src/main/resources/application.yaml b/studymanager/src/main/resources/application.yaml index c945047c..8a5cd547 100644 --- a/studymanager/src/main/resources/application.yaml +++ b/studymanager/src/main/resources/application.yaml @@ -50,6 +50,9 @@ spring: flyway: out-of-order: true + mvc: + async: + request-timeout: -1 server: forward-headers-strategy: framework error: diff --git a/studymanager/src/main/resources/openapi/StudyManagerAPI.yaml b/studymanager/src/main/resources/openapi/StudyManagerAPI.yaml index 96b07046..7149f1f4 100644 --- a/studymanager/src/main/resources/openapi/StudyManagerAPI.yaml +++ b/studymanager/src/main/resources/openapi/StudyManagerAPI.yaml @@ -1095,7 +1095,7 @@ paths: operationId: exportStudy responses: '200': - description: foo + description: Operation successful content: application/json: schema: @@ -1110,10 +1110,39 @@ paths: tags: - importExport operationId: generateDownloadToken - description: generate a download token that can be used once for download and is valid for 10 minutes + description: Generate a download token that can be used once for download and is valid for 10 minutes + parameters: + - name: studyGroupId + in: query + schema: + type: array + items: + type: integer + - name: participantId + in: query + schema: + type: array + items: + type: integer + - name: observationId + in: query + schema: + type: array + items: + type: integer + - name: from + in: query + schema: + type: string + format: date + - name: to + in: query + schema: + type: string + format: date responses: '200': - description: foo + description: Operation successful content: application/json: schema: @@ -1124,6 +1153,58 @@ paths: '404': description: study not found + /studies/{studyId}/export/studydata/{token}: + parameters: + - $ref: '#/components/parameters/StudyId' + - name: token + in: path + required: true + schema: + type: string + get: + tags: + - importExport + description: Download Study data + operationId: exportStudyData + parameters: + - name: studyGroupId + in: query + schema: + type: array + items: + type: integer + - name: participantId + in: query + schema: + type: array + items: + type: integer + - name: observationId + in: query + schema: + type: array + items: + type: integer + - name: from + in: query + schema: + type: string + format: date + - name: to + in: query + schema: + type: string + format: date + responses: + '200': + description: Operation successful + content: + application/json: + schema: + $ref: '#/components/schemas/DataExport' + '404': + description: study not found + /studies/{studyId}/import/participants: parameters: - $ref: '#/components/parameters/StudyId' @@ -1157,7 +1238,7 @@ paths: operationId: exportParticipants responses: '200': - description: foo + description: Operation successful content: text/csv: schema: @@ -2134,6 +2215,31 @@ components: required: - version - date + DataExport: + type: array + items: + type: object + properties: + datapoint_id: + type: string + participant_id: + type: string + study_id: + type: string + study_group_id: + type: string + nullable: true + observation_id: + type: string + observation_type: + type: string + data_type: + type: string + storage_date: + type: string + effective_time_frame: + type: string + additionalProperties: true parameters: StudyId: diff --git a/studymanager/src/test/java/io/redlink/more/studymanager/service/ElasticSearchServiceTest.java b/studymanager/src/test/java/io/redlink/more/studymanager/service/ElasticSearchServiceTest.java index 47efea77..7b119c98 100644 --- a/studymanager/src/test/java/io/redlink/more/studymanager/service/ElasticSearchServiceTest.java +++ b/studymanager/src/test/java/io/redlink/more/studymanager/service/ElasticSearchServiceTest.java @@ -14,18 +14,11 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.io.Resources; import io.redlink.more.studymanager.configuration.ElasticConfiguration; -import io.redlink.more.studymanager.model.data.ElasticActionDataPoint; -import io.redlink.more.studymanager.model.Study; import io.redlink.more.studymanager.core.io.Timeframe; - -import java.io.ByteArrayOutputStream; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.UUID; - +import io.redlink.more.studymanager.model.Study; +import io.redlink.more.studymanager.model.data.ElasticActionDataPoint; import io.redlink.more.studymanager.model.data.ElasticObservationDataPoint; import io.redlink.more.studymanager.utils.MapperUtils; -import org.junit.Assert; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.jackson.JacksonAutoConfiguration; @@ -38,12 +31,17 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.List; +import java.util.Map; +import java.util.UUID; -import static org.assertj.core.api.Assertions.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE) @Testcontainers @@ -109,14 +107,14 @@ void testRecordAction() { @Test void testExportData() throws JsonProcessingException, InterruptedException { for (int i = 0; i < 1200; i++) { - setDataPoint(1L, 2, i); + setDataPoint(1L, "2", "1", "1", i); } //wait for auto commit Thread.sleep(2000); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try(outputStream) { outputStream.write("[".getBytes(StandardCharsets.UTF_8)); - elasticService.exportData(1L, outputStream); + elasticService.exportData(outputStream, 1L, List.of(2), List.of(1), List.of(1), null, null); outputStream.write("]".getBytes(StandardCharsets.UTF_8)); } catch (IOException e) { //do nothing than close @@ -126,13 +124,13 @@ void testExportData() throws JsonProcessingException, InterruptedException { assertThat(MapperUtils.MAPPER.readValue(result, List.class)).hasSize(1200); } - private void setDataPoint(Long studyId, int participantId, int i) { + private void setDataPoint(Long studyId, String studyGroupId, String participantId, String observationId, int i) { elasticService.setDataPoint(studyId, new ElasticObservationDataPoint( "DP_" + studyId + "_" + participantId + "_" + i, "participant_"+ participantId, "study_" + studyId, - null, - "2", + "study_group_" + studyGroupId, + observationId, "acc-mobile-observation", "acc-mobile-observation", Instant.now(),