Skip to content

Commit

Permalink
#317: Data download improvements
Browse files Browse the repository at this point in the history 
* It is now possible to filter the data you want to download.
* Remove the default timeout for requests (30 seconds) to avoid aborting the download.
* Change translation key, because we now differ between monitoring and data tab.
  • Loading branch information
@benitsch
benitsch committed Jul 17, 2024
1 parent 23deea4 commit fba2837
Show file tree
Hide file tree
Showing 9 changed files with 217 additions and 66 deletions.
6 changes: 3 additions & 3 deletions .../main/java/io/redlink/more/studymanager/component/observation/PolarVerityObservation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ private enum DataViewInfoType implements DataViewInfo {

DataViewInfoType(String i18nKey, DataView.ChartType chartType, ViewConfig viewConfig) {
this(
"data.charts.polarVerity.%s.label".formatted(i18nKey),
"data.charts.polarVerity.%s.title".formatted(i18nKey),
"data.charts.polarVerity.%s.description".formatted(i18nKey),
"monitoring.charts.polarVerity.%s.label".formatted(i18nKey),
"monitoring.charts.polarVerity.%s.title".formatted(i18nKey),
"monitoring.charts.polarVerity.%s.description".formatted(i18nKey),
chartType,
viewConfig
);
Expand Down
6 changes: 3 additions & 3 deletions ...src/main/java/io/redlink/more/studymanager/component/observation/QuestionObservation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,9 @@ private enum DataViewInfoType implements DataViewInfo {

DataViewInfoType(String i18nKey, DataView.ChartType chartType, ViewConfig viewConfig) {
this(
"data.charts.simpleQuestion.%s.label".formatted(i18nKey),
"data.charts.simpleQuestion.%s.title".formatted(i18nKey),
"data.charts.simpleQuestion.%s.description".formatted(i18nKey),
"monitoring.charts.simpleQuestion.%s.label".formatted(i18nKey),
"monitoring.charts.simpleQuestion.%s.title".formatted(i18nKey),
"monitoring.charts.simpleQuestion.%s.description".formatted(i18nKey),
chartType,
viewConfig
);
Expand Down
1 change: 1 addition & 0 deletions studymanager/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,7 @@
<schemaMappings>
<schemaMapping>Instant=java.time.Instant</schemaMapping>
<schemaMapping>LocalTime=java.time.LocalTime</schemaMapping>
<schemaMapping>DataExport=org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody</schemaMapping>
</schemaMappings>
<importMappings>
<importMapping>Instant=java.time.Instant</importMapping>
Expand Down
54 changes: 35 additions & 19 deletions ...ava/io/redlink/more/studymanager/controller/studymanager/ImportExportApiV1Controller.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,26 @@
import io.redlink.more.studymanager.service.ImportExportService;
import io.redlink.more.studymanager.service.OAuth2AuthenticationService;
import io.redlink.more.studymanager.utils.MapperUtils;
import jakarta.servlet.ServletOutputStream;
import jakarta.servlet.http.Cookie;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.http.*;
import org.springframework.util.MultiValueMap;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import org.springframework.web.servlet.support.ServletUriComponentsBuilder;

import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.time.Instant;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
import java.util.Optional;


Expand Down Expand Up @@ -84,28 +96,32 @@ public ResponseEntity<StudyImportExportDTO> exportStudy(Long studyId) {
}

@Override
@RequiresStudyRole({StudyRole.STUDY_ADMIN, StudyRole.STUDY_OPERATOR})
public ResponseEntity<GenerateDownloadToken200ResponseDTO> generateDownloadToken(Long studyId) {
return ResponseEntity.ok(new GenerateDownloadToken200ResponseDTO().token(
tokenRepository.createToken(studyId).getToken()
));
}

@RequestMapping(
method = RequestMethod.GET,
value = "/studies/{studyId}/export/studydata/{token}",
produces = { "application/json" }
)
public void exportStudyData(@PathVariable Long studyId, @PathVariable("token") String token, HttpServletResponse response) throws IOException {
public ResponseEntity<StreamingResponseBody> exportStudyData(Long studyId, String token, List<Integer> studyGroupId, List<Integer> participantId, List<Integer> observationId, LocalDate from, LocalDate to) {
Optional<DownloadToken> dt = tokenRepository.getToken(token).filter(t -> t.getStudyId().equals(studyId));
if(dt.isPresent()) {
response.setHeader("Content-Disposition", "attachment;filename=" + dt.get().getFilename());
service.exportStudyData(response.getOutputStream(), studyId);

if (dt.isPresent()) {
HttpHeaders responseHeaders = new HttpHeaders();
responseHeaders.set("Content-Disposition", "attachment;filename=" + dt.get().getFilename());

return ResponseEntity
.ok()
.headers(responseHeaders)
.contentType(MediaType.APPLICATION_JSON)
.body(outputStream -> service.exportStudyData(outputStream, studyId, studyGroupId, participantId, observationId, from, to));
} else {
response.setStatus(403);
return ResponseEntity.notFound().build();
}
}

@Override
@RequiresStudyRole({StudyRole.STUDY_ADMIN, StudyRole.STUDY_OPERATOR})
public ResponseEntity<GenerateDownloadToken200ResponseDTO> generateDownloadToken(Long studyId, List<Integer> studyGroupId, List<Integer> participantId, List<Integer> observationId, LocalDate from, LocalDate to) {
var token = tokenRepository.createToken(studyId).getToken();
var uri = ServletUriComponentsBuilder.fromCurrentRequest().pathSegment(token).build(true).toUri();

return ResponseEntity.created(uri).body(new GenerateDownloadToken200ResponseDTO().token(token));
}

@Override
public ResponseEntity<StudyDTO> importStudy(MultipartFile file) {
try {
Expand Down
60 changes: 43 additions & 17 deletions studymanager/src/main/java/io/redlink/more/studymanager/service/ElasticService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,17 @@
import co.elastic.clients.elasticsearch._types.FieldValue;
import co.elastic.clients.elasticsearch._types.SortOrder;
import co.elastic.clients.elasticsearch._types.aggregations.StringTermsBucket;
import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField;
import co.elastic.clients.elasticsearch.core.DeleteByQueryRequest;
import co.elastic.clients.elasticsearch.core.SearchRequest;
import co.elastic.clients.elasticsearch.core.SearchResponse;
import co.elastic.clients.elasticsearch.core.search.Hit;
import co.elastic.clients.elasticsearch.indices.CloseIndexRequest;
import co.elastic.clients.elasticsearch.indices.DeleteIndexRequest;
import co.elastic.clients.elasticsearch.indices.ExistsRequest;
import co.elastic.clients.json.JsonData;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.Iterables;
import io.redlink.more.studymanager.core.io.TimeRange;
Expand All @@ -31,22 +34,20 @@
import io.redlink.more.studymanager.model.data.SimpleDataPoint;
import io.redlink.more.studymanager.properties.ElasticProperties;
import io.redlink.more.studymanager.utils.MapperUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.time.LocalDate;
import java.util.*;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.stereotype.Service;

@Service
@EnableConfigurationProperties({ElasticProperties.class})
Expand Down Expand Up @@ -328,23 +329,23 @@ public List<ParticipationData> getParticipationData(Long studyId){
}
}

public void exportData(Long studyId, OutputStream outputStream) throws IOException {
public void exportData(OutputStream outputStream, Long studyId, List<Integer> studyGroupId, List<Integer> participantId, List<Integer> observationId, LocalDate from, LocalDate to) throws IOException {
String index = getStudyIdString(studyId);

if(!client.indices().exists(e -> e.index(index)).value()) {
return;
}

SearchRequest request = getQuery(index, null);
SearchRequest request = getQuery(index, studyGroupId, participantId, observationId, from, to, null);
SearchResponse<JsonNode> rsp = client.search(request, JsonNode.class);

while (rsp.hits().hits().size() > 0) {
writeHits(rsp.hits().hits(), outputStream);
outputStream.flush();
List<FieldValue> searchAfterSort = Iterables.getLast(rsp.hits().hits()).sort();
request = getQuery(index, searchAfterSort);
request = getQuery(index, studyGroupId, participantId, observationId, from, to, searchAfterSort);
rsp = client.search(request, JsonNode.class);
if(rsp.hits().hits().size() > 0) {
if (rsp.hits().hits().size() > 0) {
outputStream.write(",".getBytes(StandardCharsets.UTF_8));
}
}
Expand All @@ -358,21 +359,46 @@ private void writeHits(List<Hit<JsonNode>> hits, OutputStream outputStream) thro
outputStream.write(datapoints.getBytes(StandardCharsets.UTF_8));
}

private SearchRequest getQuery(String index, List<FieldValue> searchAfterSort) {
private SearchRequest getQuery(String index, List<Integer> studyGroupIds, List<Integer> participantIds, List<Integer> observationIds, LocalDate from, LocalDate to, List<FieldValue> searchAfterSort) {
SearchRequest.Builder builder = new SearchRequest.Builder();
BoolQuery.Builder boolQueryBuilder = new BoolQuery.Builder();

if (studyGroupIds != null && !studyGroupIds.isEmpty()) {
List<String> studyGroupIdsStrings = studyGroupIds.stream()
.map(ElasticService::getStudyGroupIdString)
.toList();
boolQueryBuilder.filter(f -> f.terms(t -> t.field("study_group_id").terms(TermsQueryField.of(tf -> tf.value(studyGroupIdsStrings.stream().map(FieldValue::of).collect(Collectors.toList()))))));
}

if (participantIds != null && !participantIds.isEmpty()) {
List<String> participantIdStrings = participantIds.stream()
.map(ElasticService::getParticipantIdString)
.toList();
boolQueryBuilder.filter(f -> f.terms(t -> t.field("participant_id").terms(TermsQueryField.of(tf -> tf.value(participantIdStrings.stream().map(FieldValue::of).collect(Collectors.toList()))))));
}

if (observationIds != null && !observationIds.isEmpty()) {
boolQueryBuilder.filter(f -> f.terms(t -> t.field("observation_id").terms(v -> v.value(observationIds.stream().map(FieldValue::of).collect(Collectors.toList())))));
}

if (from != null && to != null) {
boolQueryBuilder.filter(f -> f.range(r -> r.field("effective_time_frame").gte(JsonData.of(from)).lte(JsonData.of(to))));
}

builder.index(index)
.query(q -> q.matchAll(m -> m))
//.pit(p -> p.id(pitId).keepAlive(k -> k.time("1m")))
.query(q -> q.bool(boolQueryBuilder.build()))
.sort(s -> s.field(f -> f.field("effective_time_frame").order(SortOrder.Asc)))
.sort(s -> s.field(f -> f.field("datapoint_id.keyword").order(SortOrder.Asc)))
.size(BATCH_SIZE_FOR_EXPORT_REQUESTS);

if(searchAfterSort != null) {
if (searchAfterSort != null) {
builder.searchAfter(searchAfterSort);
}

return builder.build();
}


public List<SimpleDataPoint> listDataPoints(
Long studyId, Integer participantId, Integer observationId, String isoDate, int size) throws IOException {
SearchRequest.Builder builder = new SearchRequest.Builder();
Expand Down
11 changes: 6 additions & 5 deletions studymanager/src/main/java/io/redlink/more/studymanager/service/ImportExportService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@

import io.redlink.more.studymanager.exception.NotFoundException;
import io.redlink.more.studymanager.model.*;
import jakarta.servlet.ServletOutputStream;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -22,7 +21,9 @@

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.util.*;

@Service
Expand Down Expand Up @@ -151,19 +152,19 @@ public Study importStudy(StudyImportExport studyImport, AuthenticatedUser user)
return newStudy;
}

public void exportStudyData(ServletOutputStream outputStream, Long studyId) {
public void exportStudyData(OutputStream outputStream, Long studyId, List<Integer> studyGroupId, List<Integer> participantId, List<Integer> observationId, LocalDate from, LocalDate to) {
if (studyService.existsStudy(studyId).orElse(false)) {
exportStudyDataAsync(outputStream, studyId);
exportStudyDataAsync(outputStream, studyId, studyGroupId, participantId, observationId, from, to);
} else {
throw NotFoundException.Study(studyId);
}
}

@Async
public void exportStudyDataAsync(ServletOutputStream outputStream, Long studyId) {
public void exportStudyDataAsync(OutputStream outputStream, Long studyId, List<Integer> studyGroupId, List<Integer> participantId, List<Integer> observationId, LocalDate from, LocalDate to) {
try (outputStream) {
outputStream.write("[".getBytes(StandardCharsets.UTF_8));
elasticService.exportData(studyId, outputStream);
elasticService.exportData(outputStream, studyId, studyGroupId, participantId, observationId, from, to);
outputStream.write("]".getBytes(StandardCharsets.UTF_8));
} catch (IOException e) {
LOGGER.error("Cannot export study data for {}", studyId, e);
Expand Down
3 changes: 3 additions & 0 deletions studymanager/src/main/resources/application.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ spring:
flyway:
out-of-order: true

mvc:
async:
request-timeout: -1
server:
forward-headers-strategy: framework
error:
Expand Down
Loading

0 comments on commit fba2837

Please sign in to comment.