Persist dump file content for dynamic simulation (#116)

Signed-off-by: Thang PHAM <[email protected]>

src/main/java/org/gridsuite/ds/server/DynamicSimulationException.java

@@ -25,6 +25,7 @@ public enum Type {
         CREATE_TIME_SERIES_ERROR,
         DELETE_TIME_SERIES_ERROR,
         MAPPING_NOT_LAST_RULE_WITH_EMPTY_FILTER_ERROR,
+        DUMP_FILE_ERROR,
     }
 
     private final Type type;

src/main/java/org/gridsuite/ds/server/RestResponseEntityExceptionHandler.java

@@ -37,7 +37,8 @@ protected ResponseEntity<Object> handleDynamicSimulationException(DynamicSimulat
                     GET_DYNAMIC_MAPPING_ERROR,
                     EXPORT_PARAMETERS_ERROR,
                     CREATE_TIME_SERIES_ERROR,
-                    DELETE_TIME_SERIES_ERROR
+                    DELETE_TIME_SERIES_ERROR,
+                    DUMP_FILE_ERROR
                     -> ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(exception.getMessage());
             case MAPPING_NOT_PROVIDED,
                     MAPPING_NOT_LAST_RULE_WITH_EMPTY_FILTER_ERROR

src/main/java/org/gridsuite/ds/server/model/ResultEntity.java

@@ -25,11 +25,24 @@
 @Entity
 public class ResultEntity implements Serializable {
 
-    public ResultEntity(UUID id, UUID timeSeriesId, UUID timeLineId, DynamicSimulationStatus status) {
+    public interface WithoutOutputState {
+        DynamicSimulationStatus getStatus();
+
+        UUID getTimeSeriesId();
+
+        UUID getTimeLineId();
+    }
+
+    public interface OutputStateOnly {
+        byte[] getOutputState();
+    }
+
+    public ResultEntity(UUID id, UUID timeSeriesId, UUID timeLineId, DynamicSimulationStatus status, byte[] outputState) {
         this.id = id;
         this.timeSeriesId = timeSeriesId;
         this.timeLineId = timeLineId;
         this.status = status;
+        this.outputState = outputState;
     }
 
     @Id
@@ -46,4 +59,7 @@ public ResultEntity(UUID id, UUID timeSeriesId, UUID timeLineId, DynamicSimulati
     @Enumerated(EnumType.STRING)
     private DynamicSimulationStatus status;
 
+    @Column(name = "outputState")
+    private byte[] outputState;
+
 }

src/main/java/org/gridsuite/ds/server/repository/ResultRepository.java

@@ -6,15 +6,34 @@
  */
 package org.gridsuite.ds.server.repository;
 
+import org.gridsuite.ds.server.dto.DynamicSimulationStatus;
 import org.gridsuite.ds.server.model.ResultEntity;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.query.Param;
 import org.springframework.stereotype.Repository;
 
+import java.util.List;
+import java.util.Optional;
 import java.util.UUID;
 
 /**
  * @author Abdelsalem Hedhili <abdelsalem.hedhili at rte-france.com>
  */
 @Repository
 public interface ResultRepository extends JpaRepository<ResultEntity, UUID> {
+    <T> Optional<T> findById(UUID id, Class<T> type);
+
+    <T> List<T> findBy(Class<T> type);
+
+    @Modifying
+    @Query("UPDATE ResultEntity r SET r.status = :status WHERE r.id IN :resultUuids")
+    int updateStatus(@Param("resultUuids") List<UUID> resultUuids, @Param("status") DynamicSimulationStatus status);
+
+    @Modifying
+    @Query("UPDATE ResultEntity r SET r.status = :status, r.timeSeriesId = :timeSeriesId, r.timeLineId = :timeLineId, r.outputState = :outputState" +
+           " WHERE r.id = :resultUuid")
+    int updateResult(@Param("resultUuid") UUID resultUuid, @Param("timeSeriesId") UUID timeSeriesId, @Param("timeLineId") UUID timeLineId,
+                      @Param("status") DynamicSimulationStatus status, @Param("outputState") byte[] outputState);
 }

src/main/java/org/gridsuite/ds/server/service/DynamicSimulationResultService.java

@@ -1,8 +1,8 @@
 package org.gridsuite.ds.server.service;
 
 import com.powsybl.timeseries.TimeSeries;
-import org.gridsuite.ds.server.DynamicSimulationException;
 import com.powsybl.ws.commons.computation.service.AbstractComputationResultService;
+import org.gridsuite.ds.server.DynamicSimulationException;
 import org.gridsuite.ds.server.dto.DynamicSimulationStatus;
 import org.gridsuite.ds.server.dto.timeseries.TimeSeriesGroupInfos;
 import org.gridsuite.ds.server.model.ResultEntity;
@@ -13,10 +13,7 @@
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.UUID;
+import java.util.*;
 
 import static org.gridsuite.ds.server.DynamicSimulationException.Type.RESULT_UUID_NOT_FOUND;
 
@@ -36,30 +33,32 @@ public DynamicSimulationResultService(ResultRepository resultRepository, TimeSer
 
     public UUID getTimeSeriesId(UUID resultUuid) {
         Objects.requireNonNull(resultUuid);
-        return resultRepository.findById(resultUuid)
+        return resultRepository.findById(resultUuid, ResultEntity.WithoutOutputState.class)
                 .orElseThrow(() -> new DynamicSimulationException(RESULT_UUID_NOT_FOUND, MSG_RESULT_UUID_NOT_FOUND + resultUuid))
                 .getTimeSeriesId();
     }
 
     public UUID getTimeLineId(UUID resultUuid) {
         Objects.requireNonNull(resultUuid);
-        return resultRepository.findById(resultUuid)
+        return resultRepository.findById(resultUuid, ResultEntity.WithoutOutputState.class)
                 .orElseThrow(() -> new DynamicSimulationException(RESULT_UUID_NOT_FOUND, MSG_RESULT_UUID_NOT_FOUND + resultUuid))
                 .getTimeLineId();
     }
 
+    public byte[] getOutputState(UUID resultUuid) {
+        Objects.requireNonNull(resultUuid);
+        return resultRepository.findById(resultUuid, ResultEntity.OutputStateOnly.class)
+                .orElseThrow(() -> new DynamicSimulationException(RESULT_UUID_NOT_FOUND, MSG_RESULT_UUID_NOT_FOUND + resultUuid))
+                .getOutputState();
+    }
+
     @Transactional
     public List<UUID> updateStatus(List<UUID> resultUuids, DynamicSimulationStatus status) {
-        // find result entities
-        List<ResultEntity> resultEntities = resultRepository.findAllById(resultUuids);
-        // set entity with new values
-        resultEntities.forEach(resultEntity -> resultEntity.setStatus(status));
-        // save entities into database
-        return resultRepository.saveAllAndFlush(resultEntities).stream().map(ResultEntity::getId).toList();
+        return resultRepository.updateStatus(resultUuids, status) > 0 ? resultUuids : Collections.emptyList();
     }
 
     @Transactional
-    public void updateResult(UUID resultUuid, List<TimeSeries<?, ?>> timeSeries, List<TimeSeries<?, ?>> timeLineSeries, DynamicSimulationStatus status) {
+    public void updateResult(UUID resultUuid, List<TimeSeries<?, ?>> timeSeries, List<TimeSeries<?, ?>> timeLineSeries, DynamicSimulationStatus status, byte[] outputState) {
 
         // send time-series/timeline to time-series-server
         UUID timeSeriesUuid = Optional.ofNullable(timeSeriesClient.sendTimeSeries(timeSeries))
@@ -69,29 +68,26 @@ public void updateResult(UUID resultUuid, List<TimeSeries<?, ?>> timeSeries, Lis
                 .map(TimeSeriesGroupInfos::getId)
                 .orElse(null);
 
-        LOGGER.info("Update dynamic simulation [resultUuid={}, timeSeriesUuid={}, timeLineUuid={}, status={}",
+        LOGGER.debug("Update dynamic simulation [resultUuid={}, timeSeriesUuid={}, timeLineUuid={}, status={}",
                 resultUuid, timeSeriesUuid, timeLineUuid, status);
 
-        // update time-series/timeline uuids and result status to the db
-        ResultEntity resultEntity = resultRepository.findById(resultUuid).orElseThrow();
-        resultEntity.setTimeSeriesId(timeSeriesUuid);
-        resultEntity.setTimeLineId(timeLineUuid);
-        resultEntity.setStatus(status);
+        // update time-series/timeline uuids, status and outputState to the db
+        resultRepository.updateResult(resultUuid, timeSeriesUuid, timeLineUuid, status, outputState);
     }
 
     @Override
     @Transactional
     public void insertStatus(List<UUID> resultUuids, DynamicSimulationStatus status) {
         Objects.requireNonNull(resultUuids);
         resultRepository.saveAll(resultUuids.stream()
-                .map(uuid -> new ResultEntity(uuid, null, null, status)).toList());
+                .map(uuid -> new ResultEntity(uuid, null, null, status, null)).toList());
     }
 
     @Override
     @Transactional
     public void delete(UUID resultUuid) {
         Objects.requireNonNull(resultUuid);
-        ResultEntity resultEntity = resultRepository.findById(resultUuid).orElse(null);
+        ResultEntity.WithoutOutputState resultEntity = resultRepository.findById(resultUuid, ResultEntity.WithoutOutputState.class).orElse(null);
         if (resultEntity == null) {
             return;
         }
@@ -106,22 +102,22 @@ public void delete(UUID resultUuid) {
     @Override
     @Transactional
     public void deleteAll() {
-        List<ResultEntity> resultEntities = resultRepository.findAll();
+        List<ResultEntity.WithoutOutputState> resultEntities = resultRepository.findBy(ResultEntity.WithoutOutputState.class);
 
         // call time series client to delete time-series and timeline
-        for (ResultEntity resultEntity : resultEntities) {
+        for (ResultEntity.WithoutOutputState resultEntity : resultEntities) {
             timeSeriesClient.deleteTimeSeriesGroup(resultEntity.getTimeSeriesId());
             timeSeriesClient.deleteTimeSeriesGroup(resultEntity.getTimeLineId());
         }
 
         // then delete all results in local db
-        resultRepository.deleteAllById(resultEntities.stream().map(ResultEntity::getId).toList());
+        resultRepository.deleteAll();
     }
 
     @Override
     public DynamicSimulationStatus findStatus(UUID resultUuid) {
         Objects.requireNonNull(resultUuid);
-        return resultRepository.findById(resultUuid)
+        return resultRepository.findById(resultUuid, ResultEntity.WithoutOutputState.class)
                 .orElseThrow(() -> new DynamicSimulationException(RESULT_UUID_NOT_FOUND, MSG_RESULT_UUID_NOT_FOUND + resultUuid))
                 .getStatus();
     }

src/main/java/org/gridsuite/ds/server/service/DynamicSimulationWorkerService.java

@@ -9,11 +9,14 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.powsybl.commons.PowsyblException;
+import com.powsybl.commons.io.FileUtil;
 import com.powsybl.computation.ComputationManager;
 import com.powsybl.dynamicsimulation.*;
 import com.powsybl.dynamicsimulation.groovy.GroovyExtension;
 import com.powsybl.dynamicsimulation.groovy.GroovyOutputVariablesSupplier;
 import com.powsybl.dynamicsimulation.groovy.OutputVariableGroovyExtension;
+import com.powsybl.dynawo.DumpFileParameters;
+import com.powsybl.dynawo.DynawoSimulationParameters;
 import com.powsybl.dynawo.DynawoSimulationProvider;
 import com.powsybl.dynawo.suppliers.dynamicmodels.DynamicModelConfig;
 import com.powsybl.dynawo.suppliers.dynamicmodels.DynawoModelsSupplier;
@@ -26,6 +29,7 @@
 import com.powsybl.timeseries.TimeSeries;
 import com.powsybl.ws.commons.computation.service.*;
 import org.apache.commons.collections4.CollectionUtils;
+import org.gridsuite.ds.server.DynamicSimulationException;
 import org.gridsuite.ds.server.dto.DynamicSimulationParametersInfos;
 import org.gridsuite.ds.server.dto.DynamicSimulationStatus;
 import org.gridsuite.ds.server.dto.dynamicmapping.InputMapping;
@@ -34,22 +38,26 @@
 import org.gridsuite.ds.server.service.contexts.DynamicSimulationResultContext;
 import org.gridsuite.ds.server.service.contexts.DynamicSimulationRunContext;
 import org.gridsuite.ds.server.service.parameters.ParametersService;
+import org.gridsuite.ds.server.utils.Utils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.messaging.Message;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.Nullable;
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.UUID;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.Consumer;
+import java.util.stream.Stream;
 
+import static org.gridsuite.ds.server.DynamicSimulationException.Type.DUMP_FILE_ERROR;
 import static org.gridsuite.ds.server.service.DynamicSimulationService.COMPUTATION_TYPE;
 
 /**
@@ -92,7 +100,7 @@ protected DynamicSimulationResultContext fromMessage(Message<String> message) {
         return DynamicSimulationResultContext.fromMessage(message, objectMapper);
     }
 
-    public void updateResult(UUID resultUuid, DynamicSimulationResult result) {
+    public void updateResult(UUID resultUuid, DynamicSimulationResult result, byte[] outputState) {
         Objects.requireNonNull(resultUuid);
         List<TimeSeries<?, ?>> timeSeries = new ArrayList<>(result.getCurves().values());
         List<TimeSeries<?, ?>> timeLineSeries = new ArrayList<>();
@@ -105,7 +113,7 @@ public void updateResult(UUID resultUuid, DynamicSimulationResult result) {
                 try {
                     return objectMapper.writeValueAsString(event);
                 } catch (JsonProcessingException e) {
-                    throw new PowsyblException("Error while serializing time line event: " + event.toString(), e);
+                    throw new PowsyblException("Error occurred while serializing time line event: " + event.toString(), e);
                 }
             }).toArray(String[]::new);
             timeLineSeries.add(TimeSeries.createString("timeLine", new IrregularTimeSeriesIndex(timeLineIndexes), timeLineValues));
@@ -115,12 +123,19 @@ public void updateResult(UUID resultUuid, DynamicSimulationResult result) {
                 DynamicSimulationStatus.CONVERGED :
                 DynamicSimulationStatus.DIVERGED;
 
-        resultService.updateResult(resultUuid, timeSeries, timeLineSeries, status);
+        resultService.updateResult(resultUuid, timeSeries, timeLineSeries, status, outputState);
     }
 
     @Override
     protected void saveResult(Network network, AbstractResultContext<DynamicSimulationRunContext> resultContext, DynamicSimulationResult result) {
-        updateResult(resultContext.getResultUuid(), result);
+        // read dump file
+        Path dumpDir = getDumpDir(resultContext.getRunContext().getDynamicSimulationParameters());
+        byte[] outputState = null;
+        if (dumpDir != null) {
+            outputState = zipDumpFile(dumpDir);
+        }
+
+        updateResult(resultContext.getResultUuid(), result, outputState);
     }
 
     @Override
@@ -159,6 +174,14 @@ public void preRun(DynamicSimulationRunContext runContext) {
         runContext.setDynamicModelContent(dynamicModel);
         runContext.setEventModelContent(eventModel);
         runContext.setCurveContent(curveModel);
+
+        // create a working folder for this run
+        Path workDir;
+        workDir = createWorkingDirectory();
+        runContext.setWorkDir(workDir);
+
+        // enrich dump parameters
+        setupDumpParameters(workDir, parameters);
     }
 
     @Override
@@ -198,4 +221,71 @@ public Consumer<Message<String>> consumeRun() {
     public Consumer<Message<String>> consumeCancel() {
         return super.consumeCancel();
     }
+
+    @Override
+    protected void clean(AbstractResultContext<DynamicSimulationRunContext> resultContext) {
+        super.clean(resultContext);
+        // clean working directory
+        Path workDir = resultContext.getRunContext().getWorkDir();
+        removeWorkingDirectory(workDir);
+    }
+
+    // --- Dump file related methods --- //
+
+    private void setupDumpParameters(Path workDir, DynamicSimulationParameters parameters) {
+        Path dumpDir = workDir.resolve("dump");
+        FileUtil.createDirectory(dumpDir);
+        DynawoSimulationParameters dynawoSimulationParameters = parameters.getExtension(DynawoSimulationParameters.class);
+        dynawoSimulationParameters.setDumpFileParameters(DumpFileParameters.createExportDumpFileParameters(dumpDir));
+    }
+
+    @Nullable
+    private Path getDumpDir(DynamicSimulationParameters dynamicSimulationParameters) {
+        return Optional.ofNullable(dynamicSimulationParameters)
+                .map(parameters -> parameters.getExtension(DynawoSimulationParameters.class))
+                .map(dynawoSimulationParameters -> ((DynawoSimulationParameters) dynawoSimulationParameters).getDumpFileParameters().dumpFileFolder())
+                .orElse(null);
+    }
+
+    @Nullable
+    private byte[] zipDumpFile(Path dumpDir) {
+        byte[] outputState = null;
+        try (Stream<Path> files = Files.list(dumpDir)) {
+            // dynawo export only one dump file
+            Path dumpFile = files.findFirst().orElse(null);
+            if (dumpFile != null) {
+                // ZIP output state
+                outputState = Utils.zip(dumpFile);
+            }
+
+        } catch (IOException e) {
+            throw new DynamicSimulationException(DUMP_FILE_ERROR, String.format("Error occurred while reading the dump file in the directory %s",
+                    dumpDir.toAbsolutePath()));
+        }
+        return outputState;
+    }
+
+    private Path createWorkingDirectory() {
+        Path workDir;
+        Path localDir = getComputationManager().getLocalDir();
+        try {
+            workDir = Files.createTempDirectory(localDir, "dynamic_simulation_");
+        } catch (IOException e) {
+            throw new DynamicSimulationException(DUMP_FILE_ERROR, String.format("Error occurred while creating a working directory inside the local directory %s",
+                    localDir.toAbsolutePath()));
+        }
+        return workDir;
+    }
+
+    private void removeWorkingDirectory(Path workDir) {
+        if (workDir != null) {
+            try {
+                FileUtil.removeDir(workDir);
+            } catch (IOException e) {
+                LOGGER.error(String.format("%s: Error occurred while cleaning working directory at %s", getComputationType(), workDir.toAbsolutePath()), e);
+            }
+        } else {
+            LOGGER.info("{}: No working directory to clean", getComputationType());
+        }
+    }
 }