From 6165f6c45567fa7749513d0650a99cfeb67eee27 Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Fri, 23 Apr 2021 11:49:47 +0200 Subject: [PATCH 01/59] Init timeseries server-based storage Signed-off-by: bhorvilleur --- afs-timeseries-server/pom.xml | 73 ++++++ .../TimeSeriesServerAppStorage.java | 220 ++++++++++++++++++ .../TimeSeriesSorageDelegate.java | 166 +++++++++++++ 3 files changed, 459 insertions(+) create mode 100644 afs-timeseries-server/pom.xml create mode 100644 afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java create mode 100644 afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java diff --git a/afs-timeseries-server/pom.xml b/afs-timeseries-server/pom.xml new file mode 100644 index 00000000..1c875cd3 --- /dev/null +++ b/afs-timeseries-server/pom.xml @@ -0,0 +1,73 @@ + + + + 4.0.0 + + + com.powsybl + powsybl-afs + 3.6.0-SNAPSHOT + + + powsybl-afs-timeseries-server + AFS TimeSeries Server impl + AFS TimeSeries Server implementation + + + + + ${project.groupId} + powsybl-afs-core + ${project.version} + + + ${project.groupId} + powsybl-afs-core + ${project.version} + + + ${project.groupId} + powsybl-time-series-server-interfaces + 0.0.1-SNAPSHOT + + + javax + javaee-api + + + + + junit + junit + test + + + org.assertj + assertj-core + test + + + org.slf4j + slf4j-simple + test + + + + ${project.groupId} + powsybl-afs-storage-api + ${project.version} + test-jar + test + + + + diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java new file mode 100644 index 00000000..4c9fd829 --- /dev/null +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java @@ -0,0 +1,220 @@ +package com.powsybl.afs.timeseriesserver; + +import com.powsybl.afs.storage.*; +import com.powsybl.timeseries.DoubleDataChunk; +import com.powsybl.timeseries.StringDataChunk; +import com.powsybl.timeseries.TimeSeriesMetadata; +import org.apache.commons.lang3.NotImplementedException; + +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +public class TimeSeriesServerAppStorage extends AbstractAppStorage { + + /** + * This storage is used for all non-timeseries-related operations + */ + AppStorage generalDelegate; + + /** + * This storage handles all the timeseries-related operations + */ + TimeSeriesSorageDelegate timeSeriesDelegate; + + + public TimeSeriesServerAppStorage(AppStorage generalDelegate) { + this.generalDelegate = generalDelegate; + timeSeriesDelegate = new TimeSeriesSorageDelegate(); + timeSeriesDelegate.createAFSAppIfNotExists(); + } + + @Override + public String getFileSystemName() { + return generalDelegate.getFileSystemName(); + } + + @Override + public boolean isRemote() { + return generalDelegate.isRemote(); + } + + @Override + public NodeInfo createRootNodeIfNotExists(String name, String nodePseudoClass) { + return generalDelegate.createRootNodeIfNotExists(name, nodePseudoClass); + } + + @Override + public NodeInfo createNode(String parentNodeId, String name, String nodePseudoClass, String description, int version, NodeGenericMetadata genericMetadata) { + return generalDelegate.createNode(parentNodeId, name, nodePseudoClass, description, version, genericMetadata); + } + + @Override + public boolean isWritable(String nodeId) { + return generalDelegate.isWritable(nodeId); + } + + @Override + public NodeInfo getNodeInfo(String nodeId) { + return generalDelegate.getNodeInfo(nodeId); + } + + @Override + public void setDescription(String nodeId, String description) { + generalDelegate.setDescription(nodeId, description); + } + + @Override + public void updateModificationTime(String nodeId) { + generalDelegate.updateModificationTime(nodeId); + } + + @Override + public List getChildNodes(String nodeId) { + return generalDelegate.getChildNodes(nodeId); + } + + @Override + public Optional getChildNode(String nodeId, String name) { + return generalDelegate.getChildNode(nodeId, name); + } + + @Override + public Optional getParentNode(String nodeId) { + return generalDelegate.getParentNode(nodeId); + } + + @Override + public void setParentNode(String nodeId, String newParentNodeId) { + generalDelegate.setParentNode(nodeId, newParentNodeId); + } + + @Override + public String deleteNode(String nodeId) { + return generalDelegate.deleteNode(nodeId); + } + + @Override + public Optional readBinaryData(String nodeId, String name) { + return generalDelegate.readBinaryData(nodeId, name); + } + + @Override + public OutputStream writeBinaryData(String nodeId, String name) { + return generalDelegate.writeBinaryData(nodeId, name); + } + + @Override + public boolean dataExists(String nodeId, String name) { + return generalDelegate.dataExists(nodeId, name); + } + + @Override + public Set getDataNames(String nodeId) { + return generalDelegate.getDataNames(nodeId); + } + + @Override + public boolean removeData(String nodeId, String name) { + return generalDelegate.removeData(nodeId, name); + } + + @Override + public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { + timeSeriesDelegate.createTimeSeries(nodeId, metadata); + } + + @Override + public Set getTimeSeriesNames(String nodeId) { + return timeSeriesDelegate.getTimeSeriesNames(nodeId); + } + + @Override + public boolean timeSeriesExists(String nodeId, String timeSeriesName) { + return timeSeriesDelegate.timeSeriesExists(nodeId, timeSeriesName); + } + + @Override + public List getTimeSeriesMetadata(String nodeId, Set timeSeriesNames) { + return timeSeriesDelegate.getTimeSeriesMetadata(nodeId, timeSeriesNames); + } + + @Override + public Set getTimeSeriesDataVersions(String nodeId) { + return timeSeriesDelegate.getTimeSeriesDataVersions(nodeId, null); + } + + @Override + public Set getTimeSeriesDataVersions(String nodeId, String timeSeriesName) { + return timeSeriesDelegate.getTimeSeriesDataVersions(nodeId, timeSeriesName); + } + + @Override + public Map> getDoubleTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { + //TODO + return null; + } + + @Override + public void addDoubleTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { + //TODO + } + + @Override + public Map> getStringTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { + throw new NotImplementedException("Not implemented in V1"); + } + + @Override + public void addStringTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { + throw new NotImplementedException("Not implemented in V1"); + } + + @Override + public void clearTimeSeries(String nodeId) { + //TODO + } + + @Override + public void addDependency(String nodeId, String name, String toNodeId) { + generalDelegate.addDependency(nodeId, name, toNodeId); + } + + @Override + public Set getDependencies(String nodeId, String name) { + return generalDelegate.getDependencies(nodeId, name); + } + + @Override + public Set getDependencies(String nodeId) { + return generalDelegate.getDependencies(nodeId); + } + + @Override + public Set getBackwardDependencies(String nodeId) { + return generalDelegate.getBackwardDependencies(nodeId); + } + + @Override + public void removeDependency(String nodeId, String name, String toNodeId) { + generalDelegate.removeDependency(nodeId, name, toNodeId); + } + + @Override + public void flush() { + generalDelegate.flush(); + } + + @Override + public boolean isClosed() { + return generalDelegate.isClosed(); + } + + @Override + public void close() { + generalDelegate.close(); + } +} diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java new file mode 100644 index 00000000..36b15a15 --- /dev/null +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -0,0 +1,166 @@ +package com.powsybl.afs.timeseriesserver; + +import com.powsybl.timeseries.RegularTimeSeriesIndex; +import com.powsybl.timeseries.TimeSeriesDataType; +import com.powsybl.timeseries.TimeSeriesIndex; +import com.powsybl.timeseries.TimeSeriesMetadata; +import com.powsybl.timeseries.storer.query.create.CreateQuery; +import com.powsybl.timeseries.storer.query.search.SearchQuery; +import com.powsybl.timeseries.storer.query.search.SearchQueryResults; +import org.apache.commons.lang3.NotImplementedException; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; +import java.net.URI; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +public class TimeSeriesSorageDelegate { + + private static final String AFS_APP = "AFS"; + + private URI timeSeriesServerURI; + + + private WebTarget buildBaseRequest(Client client) { + return client.target(timeSeriesServerURI) + .path("v1") + .path("timeseries") + .path("apps"); + } + + public void createAFSAppIfNotExists() { + Client client = ClientBuilder.newClient(); + try { + Response response = buildBaseRequest(client).request().get(); + + Collection apps = response.readEntity(Collection.class); + if (apps.contains(AFS_APP)) { + return; + } + + buildBaseRequest(client).request().post(Entity.json(AFS_APP)); + + } finally { + client.close(); + } + + } + + public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { + if (!(metadata.getIndex() instanceof RegularTimeSeriesIndex)) { + throw new NotImplementedException("TimeSeriesServer only handles regular time series for now."); + } + RegularTimeSeriesIndex index = (RegularTimeSeriesIndex) metadata.getIndex(); + + CreateQuery createQuery = new CreateQuery(); + createQuery.setMatrix(nodeId); + createQuery.setName(metadata.getName()); + createQuery.setTags(metadata.getTags()); + createQuery.setTimeStepCount(index.getPointCount()); + createQuery.setTimeStepDuration(index.getSpacing()); + LocalDateTime startDate = Instant.ofEpochMilli(index.getStartTime()).atZone(ZoneId.systemDefault()).toLocalDateTime(); + createQuery.setStartDate(startDate); + + Client client = ClientBuilder.newClient(); + try { + buildBaseRequest(client) + .path(AFS_APP) + .path("series") + .request().post(Entity.json(createQuery)); + } finally { + client.close(); + } + } + + public SearchQueryResults performSearch(SearchQuery query) { + + + SearchQueryResults results = null; + + Client client = ClientBuilder.newClient(); + try { + Response response = buildBaseRequest(client) + .path(AFS_APP) + .path("series") + .path("_search") + .request().post(Entity.json(query)); + results = response.readEntity(SearchQueryResults.class); + } finally { + client.close(); + } + + return results; + } + + public Set getTimeSeriesNames(String nodeId) { + + SearchQuery searchQuery = new SearchQuery(); + searchQuery.setMatrix(nodeId); + + SearchQueryResults results = performSearch(searchQuery); + if (results != null) { + return results.getTimeSeriesInformations() + .stream().map(t -> t.getName()).collect(Collectors.toSet()); + } + return null; + } + + public boolean timeSeriesExists(String nodeId, String name) { + + SearchQuery searchQuery = new SearchQuery(); + searchQuery.setMatrix(nodeId); + searchQuery.setNames(Collections.singleton(name)); + SearchQueryResults results = performSearch(searchQuery); + if (results != null) { + return results.getTimeSeriesInformations().size() > 0; + } + return false; + } + + public List getTimeSeriesMetadata(String nodeId, Set timeSeriesNames) { + + SearchQuery searchQuery = new SearchQuery(); + searchQuery.setMatrix(nodeId); + searchQuery.setNames(timeSeriesNames); + SearchQueryResults results = performSearch(searchQuery); + if (results != null) { + return results.getTimeSeriesInformations() + .stream().map(t -> { + long startTime = t.getStartDate().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + long spacing = t.getTimeStepDuration(); + long endTime = startTime + spacing * t.getTimeStepCount(); + TimeSeriesIndex index = new RegularTimeSeriesIndex(startTime, endTime, spacing); + return new TimeSeriesMetadata(t.getName(), TimeSeriesDataType.DOUBLE, t.getTags(), index); + }) + .collect(Collectors.toList()); + } + return null; + } + + public Set getTimeSeriesDataVersions(String nodeId, String timeSeriesName) { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.setMatrix(nodeId); + if(timeSeriesName != null) + { + searchQuery.setNames(Collections.singleton(timeSeriesName)); + } + SearchQueryResults results = performSearch(searchQuery); + if (results != null) { + return results.getTimeSeriesInformations() + .stream().flatMap(t -> t.getVersions().keySet().stream()) + .map(t->Integer.parseInt(t)) + .collect(Collectors.toSet()); + } + return null; + } +} From 07feb0648ac1d101b7eccd83fbdab58a19b8b957 Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Thu, 6 May 2021 10:30:29 +0200 Subject: [PATCH 02/59] Create AFS storage using TimeSeriesServer (WIP) Signed-off-by: bhorvilleur --- .../cassandra/CassandraAppStorageTest.java | 2 +- .../powsybl/afs/server/StorageServerTest.java | 2 +- afs-timeseries-server/pom.xml | 57 +++++++++++++++++-- .../TimeSeriesServerAppStorage.java | 18 ++++-- .../TimeSeriesSorageDelegate.java | 23 +++++--- .../TimeSeriesServerAppStorageTest.java | 25 ++++++++ 6 files changed, 108 insertions(+), 19 deletions(-) create mode 100644 afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java diff --git a/afs-cassandra/src/test/java/com/powsybl/afs/cassandra/CassandraAppStorageTest.java b/afs-cassandra/src/test/java/com/powsybl/afs/cassandra/CassandraAppStorageTest.java index e2452d98..748c7ce0 100644 --- a/afs-cassandra/src/test/java/com/powsybl/afs/cassandra/CassandraAppStorageTest.java +++ b/afs-cassandra/src/test/java/com/powsybl/afs/cassandra/CassandraAppStorageTest.java @@ -22,7 +22,7 @@ public class CassandraAppStorageTest extends AbstractAppStorageTest { public CassandraCQLUnit cassandraCQLUnit = new CassandraCQLUnit(new ClassPathCQLDataSet("afs.cql", CassandraConstants.AFS_KEYSPACE), null, 20000L); @Override - protected AppStorage createStorage() { + public AppStorage createStorage() { return new CassandraAppStorage("test", () -> new CassandraTestContext(cassandraCQLUnit), new CassandraAppStorageConfig(), new InMemoryEventsBus()); } diff --git a/afs-spring-server/src/test/java/com/powsybl/afs/server/StorageServerTest.java b/afs-spring-server/src/test/java/com/powsybl/afs/server/StorageServerTest.java index 5552b84b..f03c6de0 100644 --- a/afs-spring-server/src/test/java/com/powsybl/afs/server/StorageServerTest.java +++ b/afs-spring-server/src/test/java/com/powsybl/afs/server/StorageServerTest.java @@ -79,7 +79,7 @@ private URI getRestUri() { } @Override - protected AppStorage createStorage() { + public AppStorage createStorage() { URI restUri = getRestUri(); return new RemoteAppStorage(FS_TEST_NAME, restUri, ""); } diff --git a/afs-timeseries-server/pom.xml b/afs-timeseries-server/pom.xml index 1c875cd3..16b3fd67 100644 --- a/afs-timeseries-server/pom.xml +++ b/afs-timeseries-server/pom.xml @@ -29,16 +29,21 @@ powsybl-afs-core ${project.version} - - ${project.groupId} - powsybl-afs-core - ${project.version} - ${project.groupId} powsybl-time-series-server-interfaces 0.0.1-SNAPSHOT + + org.jboss.resteasy + resteasy-client + provided + + + org.jboss.resteasy + resteasy-jackson-provider + 3.0.19.Final + javax javaee-api @@ -60,6 +65,41 @@ slf4j-simple test + + org.cassandraunit + cassandra-unit + test + + + ${project.groupId} + powsybl-afs-cassandra + ${project.version} + test-jar + test + + + ${project.groupId} + powsybl-afs-local + ${project.version} + test-jar + test + + + ${project.groupId} + powsybl-afs-local + ${project.version} + test + + + org.mockito + mockito-core + test + + + com.google.jimfs + jimfs + test + ${project.groupId} @@ -68,6 +108,13 @@ test-jar test + + ${project.groupId} + powsybl-afs-ext-base + ${project.version} + test + test-jar + diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java index 4c9fd829..23b24518 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java @@ -8,12 +8,13 @@ import java.io.InputStream; import java.io.OutputStream; +import java.net.URI; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -public class TimeSeriesServerAppStorage extends AbstractAppStorage { +public class TimeSeriesServerAppStorage implements AppStorage { /** * This storage is used for all non-timeseries-related operations @@ -25,10 +26,9 @@ public class TimeSeriesServerAppStorage extends AbstractAppStorage { */ TimeSeriesSorageDelegate timeSeriesDelegate; - - public TimeSeriesServerAppStorage(AppStorage generalDelegate) { + public TimeSeriesServerAppStorage(AppStorage generalDelegate, URI timeSeriesServerURI) { this.generalDelegate = generalDelegate; - timeSeriesDelegate = new TimeSeriesSorageDelegate(); + timeSeriesDelegate = new TimeSeriesSorageDelegate(timeSeriesServerURI); timeSeriesDelegate.createAFSAppIfNotExists(); } @@ -203,6 +203,11 @@ public void removeDependency(String nodeId, String name, String toNodeId) { generalDelegate.removeDependency(nodeId, name, toNodeId); } + @Override + public EventsBus getEventsBus() { + return generalDelegate.getEventsBus(); + } + @Override public void flush() { generalDelegate.flush(); @@ -217,4 +222,9 @@ public boolean isClosed() { public void close() { generalDelegate.close(); } + + @Override + public boolean isConsistent(String nodeId) { + return generalDelegate.isConsistent(nodeId); + } } diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index 36b15a15..8d8be8b2 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -8,9 +8,9 @@ import com.powsybl.timeseries.storer.query.search.SearchQuery; import com.powsybl.timeseries.storer.query.search.SearchQueryResults; import org.apache.commons.lang3.NotImplementedException; +import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; import javax.ws.rs.client.Client; -import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response; @@ -30,6 +30,15 @@ public class TimeSeriesSorageDelegate { private URI timeSeriesServerURI; + public TimeSeriesSorageDelegate(URI timeSeriesServerURI) { + this.timeSeriesServerURI = timeSeriesServerURI; + } + + public static Client createClient() { + return new ResteasyClientBuilder() + .connectionPoolSize(50) + .build(); + } private WebTarget buildBaseRequest(Client client) { return client.target(timeSeriesServerURI) @@ -39,7 +48,7 @@ private WebTarget buildBaseRequest(Client client) { } public void createAFSAppIfNotExists() { - Client client = ClientBuilder.newClient(); + Client client = createClient(); try { Response response = buildBaseRequest(client).request().get(); @@ -71,7 +80,7 @@ public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { LocalDateTime startDate = Instant.ofEpochMilli(index.getStartTime()).atZone(ZoneId.systemDefault()).toLocalDateTime(); createQuery.setStartDate(startDate); - Client client = ClientBuilder.newClient(); + Client client = createClient(); try { buildBaseRequest(client) .path(AFS_APP) @@ -84,10 +93,9 @@ public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { public SearchQueryResults performSearch(SearchQuery query) { - SearchQueryResults results = null; - Client client = ClientBuilder.newClient(); + Client client = createClient(); try { Response response = buildBaseRequest(client) .path(AFS_APP) @@ -150,15 +158,14 @@ public List getTimeSeriesMetadata(String nodeId, Set public Set getTimeSeriesDataVersions(String nodeId, String timeSeriesName) { SearchQuery searchQuery = new SearchQuery(); searchQuery.setMatrix(nodeId); - if(timeSeriesName != null) - { + if (timeSeriesName != null) { searchQuery.setNames(Collections.singleton(timeSeriesName)); } SearchQueryResults results = performSearch(searchQuery); if (results != null) { return results.getTimeSeriesInformations() .stream().flatMap(t -> t.getVersions().keySet().stream()) - .map(t->Integer.parseInt(t)) + .map(t -> Integer.parseInt(t)) .collect(Collectors.toSet()); } return null; diff --git a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java new file mode 100644 index 00000000..096e88e3 --- /dev/null +++ b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java @@ -0,0 +1,25 @@ +package com.powsybl.afs.timeseriesserver; + +import com.powsybl.afs.storage.AbstractAppStorageTest; +import com.powsybl.afs.storage.AppStorage; +import org.junit.Before; + +import java.net.URI; + +public class TimeSeriesServerAppStorageTest extends AbstractAppStorageTest { + + private URI timeSeriesServerURI; + + @Override + @Before + public void setUp() throws Exception { + timeSeriesServerURI = new URI("http://localhost:9000/"); + super.setUp(); + } + + @Override + protected AppStorage createStorage() { + return null; //TODO + } + +} From 8fd48e4ca5358c130052f89e8e4f31c45616d94d Mon Sep 17 00:00:00 2001 From: amichaut Date: Thu, 6 May 2021 17:27:24 +0200 Subject: [PATCH 03/59] Add time series server project to master pom --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 68ac3230..c2b9ca20 100644 --- a/pom.xml +++ b/pom.xml @@ -64,6 +64,7 @@ afs-storage-api afs-ws afs-spring-server + afs-timeseries-server From 6afa4f8847db04d3e07437bf14edf22c2c01c28f Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Mon, 10 May 2021 14:21:24 +0200 Subject: [PATCH 04/59] Test TimeSeriesServerAppStorage with a MapDB implementation for general delegate Signed-off-by: bhorvilleur --- afs-timeseries-server/pom.xml | 3 +- .../TimeSeriesServerAppStorage.java | 46 +++++++++++++++---- .../TimeSeriesServerAppStorageTest.java | 4 +- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/afs-timeseries-server/pom.xml b/afs-timeseries-server/pom.xml index 16b3fd67..cc228b9b 100644 --- a/afs-timeseries-server/pom.xml +++ b/afs-timeseries-server/pom.xml @@ -110,10 +110,9 @@ ${project.groupId} - powsybl-afs-ext-base + powsybl-afs-mapdb-storage ${project.version} test - test-jar diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java index 23b24518..2dbab3e9 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java @@ -1,6 +1,8 @@ package com.powsybl.afs.timeseriesserver; import com.powsybl.afs.storage.*; +import com.powsybl.afs.storage.events.AppStorageListener; +import com.powsybl.afs.storage.events.TimeSeriesCreated; import com.powsybl.timeseries.DoubleDataChunk; import com.powsybl.timeseries.StringDataChunk; import com.powsybl.timeseries.TimeSeriesMetadata; @@ -14,20 +16,30 @@ import java.util.Optional; import java.util.Set; -public class TimeSeriesServerAppStorage implements AppStorage { +public class TimeSeriesServerAppStorage extends AbstractAppStorage { /** * This storage is used for all non-timeseries-related operations */ - AppStorage generalDelegate; + private AbstractAppStorage generalDelegate; /** * This storage handles all the timeseries-related operations */ - TimeSeriesSorageDelegate timeSeriesDelegate; + private TimeSeriesSorageDelegate timeSeriesDelegate; - public TimeSeriesServerAppStorage(AppStorage generalDelegate, URI timeSeriesServerURI) { + + /** + * A listener that copies all event from the general delegate event bus to this class event bus. + * This has to be a field because the listeners of an event bus are stored in a WeakReferenceList + */ + private AppStorageListener notifyGeneralDelegateEventListener; + + public TimeSeriesServerAppStorage(AbstractAppStorage generalDelegate, URI timeSeriesServerURI) { this.generalDelegate = generalDelegate; + eventsBus = new InMemoryEventsBus(); + notifyGeneralDelegateEventListener = t -> t.getEvents().forEach(e -> pushEvent(e, t.getTopic())); + generalDelegate.getEventsBus().addListener(notifyGeneralDelegateEventListener); timeSeriesDelegate = new TimeSeriesSorageDelegate(timeSeriesServerURI); timeSeriesDelegate.createAFSAppIfNotExists(); } @@ -125,6 +137,7 @@ public boolean removeData(String nodeId, String name) { @Override public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { timeSeriesDelegate.createTimeSeries(nodeId, metadata); + pushEvent(new TimeSeriesCreated(nodeId, metadata.getName()), APPSTORAGE_TIMESERIES_TOPIC); } @Override @@ -203,14 +216,11 @@ public void removeDependency(String nodeId, String name, String toNodeId) { generalDelegate.removeDependency(nodeId, name, toNodeId); } - @Override - public EventsBus getEventsBus() { - return generalDelegate.getEventsBus(); - } @Override public void flush() { generalDelegate.flush(); + eventsBus.flush(); } @Override @@ -227,4 +237,24 @@ public void close() { public boolean isConsistent(String nodeId) { return generalDelegate.isConsistent(nodeId); } + + @Override + public void setMetadata(String nodeId, NodeGenericMetadata genericMetadata) { + generalDelegate.setMetadata(nodeId, genericMetadata); + } + + @Override + public void setConsistent(String nodeId) { + generalDelegate.setConsistent(nodeId); + } + + @Override + public List getInconsistentNodes() { + return generalDelegate.getInconsistentNodes(); + } + + @Override + public void renameNode(String nodeId, String name) { + generalDelegate.renameNode(nodeId, name); + } } diff --git a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java index 096e88e3..db22d3a6 100644 --- a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java +++ b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java @@ -1,7 +1,9 @@ package com.powsybl.afs.timeseriesserver; +import com.powsybl.afs.mapdb.storage.MapDbAppStorage; import com.powsybl.afs.storage.AbstractAppStorageTest; import com.powsybl.afs.storage.AppStorage; +import com.powsybl.afs.storage.InMemoryEventsBus; import org.junit.Before; import java.net.URI; @@ -19,7 +21,7 @@ public void setUp() throws Exception { @Override protected AppStorage createStorage() { - return null; //TODO + return new TimeSeriesServerAppStorage(MapDbAppStorage.createMem("mem", new InMemoryEventsBus()), timeSeriesServerURI); } } From b99902ebc0e07f4a7b5c27d8b6d38dae8da2ed7e Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Tue, 11 May 2021 10:05:55 +0200 Subject: [PATCH 05/59] Bugfixes on create and search time series methods: - Errors on date serialization / deserialization - Bugfix on computation of end date : must be inclusive (last time step of the series) Signed-off-by: bhorvilleur --- .../TimeSeriesSorageDelegate.java | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index 8d8be8b2..16e6feb6 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -1,5 +1,8 @@ package com.powsybl.afs.timeseriesserver; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.powsybl.afs.storage.AfsStorageException; import com.powsybl.timeseries.RegularTimeSeriesIndex; import com.powsybl.timeseries.TimeSeriesDataType; import com.powsybl.timeseries.TimeSeriesIndex; @@ -9,11 +12,14 @@ import com.powsybl.timeseries.storer.query.search.SearchQueryResults; import org.apache.commons.lang3.NotImplementedException; import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.ws.rs.client.Client; import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response; +import java.io.IOException; import java.net.URI; import java.time.Instant; import java.time.LocalDateTime; @@ -27,6 +33,8 @@ public class TimeSeriesSorageDelegate { private static final String AFS_APP = "AFS"; + private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesSorageDelegate.class); + private URI timeSeriesServerURI; @@ -57,7 +65,11 @@ public void createAFSAppIfNotExists() { return; } - buildBaseRequest(client).request().post(Entity.json(AFS_APP)); + response = buildBaseRequest(client).path(AFS_APP).request().post(Entity.json("")); + if(response.getStatus() != 200) + { + throw new AfsStorageException("Error while initializing AFS timeseries app storage"); + } } finally { client.close(); @@ -85,7 +97,9 @@ public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { buildBaseRequest(client) .path(AFS_APP) .path("series") - .request().post(Entity.json(createQuery)); + .request().post(Entity.json(new ObjectMapper().writeValueAsString(createQuery))); + } catch (JsonProcessingException e) { + LOGGER.error(e.getMessage(), e); } finally { client.close(); } @@ -101,8 +115,11 @@ public SearchQueryResults performSearch(SearchQuery query) { .path(AFS_APP) .path("series") .path("_search") - .request().post(Entity.json(query)); - results = response.readEntity(SearchQueryResults.class); + .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); + String json = response.readEntity(String.class); + results = new ObjectMapper().readValue(json, SearchQueryResults.class); + } catch (IOException e) { + LOGGER.error(e.getMessage(), e); } finally { client.close(); } @@ -146,7 +163,7 @@ public List getTimeSeriesMetadata(String nodeId, Set .stream().map(t -> { long startTime = t.getStartDate().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); long spacing = t.getTimeStepDuration(); - long endTime = startTime + spacing * t.getTimeStepCount(); + long endTime = startTime + spacing * (t.getTimeStepCount() - 1); TimeSeriesIndex index = new RegularTimeSeriesIndex(startTime, endTime, spacing); return new TimeSeriesMetadata(t.getName(), TimeSeriesDataType.DOUBLE, t.getTags(), index); }) From c355abc8f0562dd3c72379ff219c31f769de4ee7 Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Tue, 11 May 2021 18:00:21 +0200 Subject: [PATCH 06/59] TimeSeriesServerAppStorage : add publish timeseries method Signed-off-by: bhorvilleur --- .../TimeSeriesServerAppStorage.java | 4 +- .../TimeSeriesSorageDelegate.java | 39 +++++++++++++++++-- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java index 2dbab3e9..4e333c76 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java @@ -3,6 +3,7 @@ import com.powsybl.afs.storage.*; import com.powsybl.afs.storage.events.AppStorageListener; import com.powsybl.afs.storage.events.TimeSeriesCreated; +import com.powsybl.afs.storage.events.TimeSeriesDataUpdated; import com.powsybl.timeseries.DoubleDataChunk; import com.powsybl.timeseries.StringDataChunk; import com.powsybl.timeseries.TimeSeriesMetadata; @@ -173,7 +174,8 @@ public Map> getDoubleTimeSeriesData(String nodeId, @Override public void addDoubleTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { - //TODO + timeSeriesDelegate.addDoubleTimeSeriesData(nodeId, version, timeSeriesName, chunks); + pushEvent(new TimeSeriesDataUpdated(nodeId, timeSeriesName), APPSTORAGE_TIMESERIES_TOPIC); } @Override diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index 16e6feb6..6381c1e7 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -3,13 +3,12 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.powsybl.afs.storage.AfsStorageException; -import com.powsybl.timeseries.RegularTimeSeriesIndex; -import com.powsybl.timeseries.TimeSeriesDataType; -import com.powsybl.timeseries.TimeSeriesIndex; -import com.powsybl.timeseries.TimeSeriesMetadata; +import com.powsybl.timeseries.*; import com.powsybl.timeseries.storer.query.create.CreateQuery; +import com.powsybl.timeseries.storer.query.publish.PublishQuery; import com.powsybl.timeseries.storer.query.search.SearchQuery; import com.powsybl.timeseries.storer.query.search.SearchQueryResults; +import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.NotImplementedException; import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; import org.slf4j.Logger; @@ -187,4 +186,36 @@ public Set getTimeSeriesDataVersions(String nodeId, String timeSeriesNa } return null; } + + public void addDoubleTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { + + //First step : retrieve metadata and reconstitute the time series + TimeSeriesMetadata metadata = getTimeSeriesMetadata(nodeId, Collections.singleton(timeSeriesName)).get(0); + StoredDoubleTimeSeries ts = new StoredDoubleTimeSeries(metadata, chunks); + + //Second step : perform a publish request + PublishQuery publishQuery = new PublishQuery<>(); + publishQuery.setMatrix(nodeId); + publishQuery.setTimeSeriesName(timeSeriesName); + publishQuery.setVersionName(String.valueOf(version)); + publishQuery.setData(ArrayUtils.toObject(ts.toArray())); + + Client client = createClient(); + try { + Response response = buildBaseRequest(client) + .path(AFS_APP) + .path("series") + .path("_search") + .request().post(Entity.json(new ObjectMapper().writeValueAsString(publishQuery))); + if(response.getStatus() != 200) + { + throw new AfsStorageException("Error while publishing data to time series server"); + } + } catch (IOException e) { + LOGGER.error(e.getMessage(), e); + } finally { + client.close(); + } + + } } From 554d15719193e6918199ca0eacb1d99386222994 Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Tue, 11 May 2021 18:20:45 +0200 Subject: [PATCH 07/59] Bugfix call publish timeseries method Signed-off-by: bhorvilleur --- .../powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index 6381c1e7..66d9f15d 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -205,8 +205,7 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie Response response = buildBaseRequest(client) .path(AFS_APP) .path("series") - .path("_search") - .request().post(Entity.json(new ObjectMapper().writeValueAsString(publishQuery))); + .request().put(Entity.json(new ObjectMapper().writeValueAsString(publishQuery))); if(response.getStatus() != 200) { throw new AfsStorageException("Error while publishing data to time series server"); From b05fe4cb4dd1b62adf819df2e095be1df955e1c3 Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Tue, 11 May 2021 20:40:48 +0200 Subject: [PATCH 08/59] TimeSeriesServerAppStorage : add fetch timeseries method Signed-off-by: bhorvilleur --- .../TimeSeriesServerAppStorage.java | 2 +- .../TimeSeriesSorageDelegate.java | 60 ++++++++++++++++--- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java index 4e333c76..4f7ac372 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java @@ -169,7 +169,7 @@ public Set getTimeSeriesDataVersions(String nodeId, String timeSeriesNa @Override public Map> getDoubleTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { //TODO - return null; + return timeSeriesDelegate.getDoubleTimeSeriesData(nodeId, timeSeriesNames, version); } @Override diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index 66d9f15d..57132b4b 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -5,6 +5,8 @@ import com.powsybl.afs.storage.AfsStorageException; import com.powsybl.timeseries.*; import com.powsybl.timeseries.storer.query.create.CreateQuery; +import com.powsybl.timeseries.storer.query.fetch.FetchQuery; +import com.powsybl.timeseries.storer.query.fetch.FetchQueryResult; import com.powsybl.timeseries.storer.query.publish.PublishQuery; import com.powsybl.timeseries.storer.query.search.SearchQuery; import com.powsybl.timeseries.storer.query.search.SearchQueryResults; @@ -23,10 +25,7 @@ import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; public class TimeSeriesSorageDelegate { @@ -65,8 +64,7 @@ public void createAFSAppIfNotExists() { } response = buildBaseRequest(client).path(AFS_APP).request().post(Entity.json("")); - if(response.getStatus() != 200) - { + if (response.getStatus() != 200) { throw new AfsStorageException("Error while initializing AFS timeseries app storage"); } @@ -206,8 +204,7 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie .path(AFS_APP) .path("series") .request().put(Entity.json(new ObjectMapper().writeValueAsString(publishQuery))); - if(response.getStatus() != 200) - { + if (response.getStatus() != 200) { throw new AfsStorageException("Error while publishing data to time series server"); } } catch (IOException e) { @@ -217,4 +214,51 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie } } + + public Map> getDoubleTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { + + String versionString = String.valueOf(version); + SearchQuery searchQuery = new SearchQuery(); + searchQuery.setMatrix(nodeId); + searchQuery.setNames(timeSeriesNames); + SearchQueryResults results = performSearch(searchQuery); + List versionIDs = results.getTimeSeriesInformations() + .stream() + .map(t->t.getVersions().get(versionString)) + .collect(Collectors.toList()); + + Map versionIDToTSName = results.getTimeSeriesInformations() + .stream() + .collect(Collectors.toMap(t->t.getVersions().get(versionString), t->t.getName())); + + FetchQuery query = new FetchQuery(versionIDs, null, null); + Client client = createClient(); + try { + Response response = buildBaseRequest(client) + .path(AFS_APP) + .path("series") + .path("_fetch") + .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); + if (response.getStatus() != 200) { + throw new AfsStorageException("Error while fetching data from time series server"); + } + String json = response.readEntity(String.class); + FetchQueryResult fetchResults = new ObjectMapper().readValue(json, FetchQueryResult.class); + + Map> toReturn = new HashMap<>(); + for(int i=0; i Date: Tue, 18 May 2021 17:21:49 +0200 Subject: [PATCH 09/59] Adapt AbstractAppStorageTest to storages which do not handle chunk conservation and string timeseries, such as TimeSeriesServerAppStorage Signed-off-by: bhorvilleur --- .../afs/storage/AbstractAppStorageTest.java | 119 +++++++++++------- .../TimeSeriesServerAppStorageTest.java | 4 + 2 files changed, 78 insertions(+), 45 deletions(-) diff --git a/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java b/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java index bab2dc05..cdc95d49 100644 --- a/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java +++ b/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java @@ -43,10 +43,23 @@ public abstract class AbstractAppStorageTest { protected AppStorage storage; + protected boolean conservesChunks; + + protected boolean handlesStringTimeSeries; + protected BlockingQueue eventStack; protected AppStorageListener l = eventList -> eventStack.addAll(eventList.getEvents()); + public AbstractAppStorageTest() { + this(true, true); + } + + public AbstractAppStorageTest(boolean conservesChunks, boolean handlesStringTimeSeries) { + this.conservesChunks = conservesChunks; + this.handlesStringTimeSeries = handlesStringTimeSeries; + } + protected abstract AppStorage createStorage(); @Before @@ -385,8 +398,16 @@ public void test() throws IOException, InterruptedException { assertTrue(storage.getTimeSeriesMetadata(testData3Info.getId(), Sets.newHashSet("ts1")).isEmpty()); // 14) add data to double time series - storage.addDoubleTimeSeriesData(testData2Info.getId(), 0, "ts1", Arrays.asList(new UncompressedDoubleDataChunk(2, new double[] {1d, 2d}), - new UncompressedDoubleDataChunk(5, new double[] {3d}))); + if(conservesChunks) + { + storage.addDoubleTimeSeriesData(testData2Info.getId(), 0, "ts1", Arrays.asList(new UncompressedDoubleDataChunk(2, new double[] {1d, 2d}), + new UncompressedDoubleDataChunk(5, new double[] {3d}))); + } + else + { + storage.addDoubleTimeSeriesData(testData2Info.getId(), 0, "ts1", Arrays.asList(new UncompressedDoubleDataChunk(0, new double[] {1d, 2d, 3d}))); + } + storage.flush(); // check event @@ -399,52 +420,60 @@ public void test() throws IOException, InterruptedException { // check double time series data query Map> doubleTimeSeriesData = storage.getDoubleTimeSeriesData(testData2Info.getId(), Sets.newHashSet("ts1"), 0); assertEquals(1, doubleTimeSeriesData.size()); - assertEquals(Arrays.asList(new UncompressedDoubleDataChunk(2, new double[] {1d, 2d}), - new UncompressedDoubleDataChunk(5, new double[] {3d})), - doubleTimeSeriesData.get("ts1")); + if (conservesChunks) { + assertEquals(Arrays.asList(new UncompressedDoubleDataChunk(2, new double[] {1d, 2d}), + new UncompressedDoubleDataChunk(5, new double[] {3d})), + doubleTimeSeriesData.get("ts1")); + } + else { + assertEquals(Arrays.asList(new UncompressedDoubleDataChunk(0, new double[] {1d, 2d, 3d})), + doubleTimeSeriesData.get("ts1")); + } assertTrue(storage.getDoubleTimeSeriesData(testData3Info.getId(), Sets.newHashSet("ts1"), 0).isEmpty()); // 15) create a second string time series - TimeSeriesMetadata metadata2 = new TimeSeriesMetadata("ts2", - TimeSeriesDataType.STRING, - ImmutableMap.of(), - RegularTimeSeriesIndex.create(Interval.parse("2015-01-01T00:00:00Z/2015-01-01T01:15:00Z"), - Duration.ofMinutes(15))); - storage.createTimeSeries(testData2Info.getId(), metadata2); - storage.flush(); - - // check event - assertEventStack(new TimeSeriesCreated(testData2Info.getId(), "ts2")); - - // check string time series query - assertEquals(Sets.newHashSet("ts1", "ts2"), storage.getTimeSeriesNames(testData2Info.getId())); - metadataList = storage.getTimeSeriesMetadata(testData2Info.getId(), Sets.newHashSet("ts1")); - assertEquals(1, metadataList.size()); - - // 16) add data to double time series - storage.addStringTimeSeriesData(testData2Info.getId(), 0, "ts2", Arrays.asList(new UncompressedStringDataChunk(2, new String[] {"a", "b"}), - new UncompressedStringDataChunk(5, new String[] {"c"}))); - storage.flush(); - - // check event - assertEventStack(new TimeSeriesDataUpdated(testData2Info.getId(), "ts2")); - - // check string time series data query - Map> stringTimeSeriesData = storage.getStringTimeSeriesData(testData2Info.getId(), Sets.newHashSet("ts2"), 0); - assertEquals(1, stringTimeSeriesData.size()); - assertEquals(Arrays.asList(new UncompressedStringDataChunk(2, new String[] {"a", "b"}), - new UncompressedStringDataChunk(5, new String[] {"c"})), - stringTimeSeriesData.get("ts2")); - - // 17) clear time series - storage.clearTimeSeries(testData2Info.getId()); - storage.flush(); - - // check event - assertEventStack(new TimeSeriesCleared(testData2Info.getId())); - - // check there is no more time series - assertTrue(storage.getTimeSeriesNames(testData2Info.getId()).isEmpty()); + if(handlesStringTimeSeries) { + TimeSeriesMetadata metadata2 = new TimeSeriesMetadata("ts2", + TimeSeriesDataType.STRING, + ImmutableMap.of(), + RegularTimeSeriesIndex.create(Interval.parse("2015-01-01T00:00:00Z/2015-01-01T01:15:00Z"), + Duration.ofMinutes(15))); + storage.createTimeSeries(testData2Info.getId(), metadata2); + storage.flush(); + + // check event + assertEventStack(new TimeSeriesCreated(testData2Info.getId(), "ts2")); + + // check string time series query + assertEquals(Sets.newHashSet("ts1", "ts2"), storage.getTimeSeriesNames(testData2Info.getId())); + metadataList = storage.getTimeSeriesMetadata(testData2Info.getId(), Sets.newHashSet("ts1")); + assertEquals(1, metadataList.size()); + + // 16) add data to double time series + storage.addStringTimeSeriesData(testData2Info.getId(), 0, "ts2", Arrays.asList(new UncompressedStringDataChunk(2, new String[] {"a", "b"}), + new UncompressedStringDataChunk(5, new String[] {"c"}))); + storage.flush(); + + // check event + assertEventStack(new TimeSeriesDataUpdated(testData2Info.getId(), "ts2")); + + // check string time series data query + Map> stringTimeSeriesData = storage.getStringTimeSeriesData(testData2Info.getId(), Sets.newHashSet("ts2"), 0); + assertEquals(1, stringTimeSeriesData.size()); + assertEquals(Arrays.asList(new UncompressedStringDataChunk(2, new String[] {"a", "b"}), + new UncompressedStringDataChunk(5, new String[] {"c"})), + stringTimeSeriesData.get("ts2")); + + // 17) clear time series + storage.clearTimeSeries(testData2Info.getId()); + storage.flush(); + + // check event + assertEventStack(new TimeSeriesCleared(testData2Info.getId())); + + // check there is no more time series + assertTrue(storage.getTimeSeriesNames(testData2Info.getId()).isEmpty()); + } // 18) change parent test NodeInfo folder1Info = storage.createNode(rootFolderInfo.getId(), "test1", FOLDER_PSEUDO_CLASS, "", 0, new NodeGenericMetadata()); diff --git a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java index db22d3a6..a8e17c33 100644 --- a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java +++ b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java @@ -12,6 +12,10 @@ public class TimeSeriesServerAppStorageTest extends AbstractAppStorageTest { private URI timeSeriesServerURI; + public TimeSeriesServerAppStorageTest() { + super(false, false); + } + @Override @Before public void setUp() throws Exception { From 8028d8395cddc8382be85056f3600aa3ec25bd18 Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Tue, 18 May 2021 17:24:29 +0200 Subject: [PATCH 10/59] Specialize TimeSeriesServer in Double time series String time series will have to be handled with different methods and end points. Signed-off-by: bhorvilleur --- .../afs/timeseriesserver/TimeSeriesSorageDelegate.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index 57132b4b..e7576549 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -192,7 +192,7 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie StoredDoubleTimeSeries ts = new StoredDoubleTimeSeries(metadata, chunks); //Second step : perform a publish request - PublishQuery publishQuery = new PublishQuery<>(); + PublishQuery publishQuery = new PublishQuery(); publishQuery.setMatrix(nodeId); publishQuery.setTimeSeriesName(timeSeriesName); publishQuery.setVersionName(String.valueOf(version)); @@ -243,7 +243,7 @@ public Map> getDoubleTimeSeriesData(String nodeId, throw new AfsStorageException("Error while fetching data from time series server"); } String json = response.readEntity(String.class); - FetchQueryResult fetchResults = new ObjectMapper().readValue(json, FetchQueryResult.class); + FetchQueryResult fetchResults = new ObjectMapper().readValue(json, FetchQueryResult.class); Map> toReturn = new HashMap<>(); for(int i=0; i> getDoubleTimeSeriesData(String nodeId, UncompressedDoubleDataChunk chunk = new UncompressedDoubleDataChunk(0, values); toReturn.put(versionIDToTSName.get(versionIDs.get(i)), Arrays.asList(chunk)); } + return toReturn; } catch (IOException e) { LOGGER.error(e.getMessage(), e); From 0bde97f4b7579f4cab0582e0f6bc977da5ec7e6c Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Wed, 26 May 2021 14:56:31 +0200 Subject: [PATCH 11/59] Checkstyle fix Signed-off-by: bhorvilleur --- .../afs/timeseriesserver/TimeSeriesServerAppStorage.java | 1 - .../afs/timeseriesserver/TimeSeriesSorageDelegate.java | 8 +++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java index 4f7ac372..6d3a741c 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java @@ -218,7 +218,6 @@ public void removeDependency(String nodeId, String name, String toNodeId) { generalDelegate.removeDependency(nodeId, name, toNodeId); } - @Override public void flush() { generalDelegate.flush(); diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index e7576549..03d7562e 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -33,7 +33,6 @@ public class TimeSeriesSorageDelegate { private static final String AFS_APP = "AFS"; private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesSorageDelegate.class); - private URI timeSeriesServerURI; public TimeSeriesSorageDelegate(URI timeSeriesServerURI) { @@ -224,12 +223,12 @@ public Map> getDoubleTimeSeriesData(String nodeId, SearchQueryResults results = performSearch(searchQuery); List versionIDs = results.getTimeSeriesInformations() .stream() - .map(t->t.getVersions().get(versionString)) + .map(t -> t.getVersions().get(versionString)) .collect(Collectors.toList()); Map versionIDToTSName = results.getTimeSeriesInformations() .stream() - .collect(Collectors.toMap(t->t.getVersions().get(versionString), t->t.getName())); + .collect(Collectors.toMap(t -> t.getVersions().get(versionString), t -> t.getName())); FetchQuery query = new FetchQuery(versionIDs, null, null); Client client = createClient(); @@ -246,8 +245,7 @@ public Map> getDoubleTimeSeriesData(String nodeId, FetchQueryResult fetchResults = new ObjectMapper().readValue(json, FetchQueryResult.class); Map> toReturn = new HashMap<>(); - for(int i=0; i Date: Wed, 26 May 2021 15:04:22 +0200 Subject: [PATCH 12/59] AFS time series server : customize the "AFS" app Signed-off-by: bhorvilleur --- .../TimeSeriesSorageDelegate.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index 03d7562e..730efb0a 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -30,13 +30,26 @@ public class TimeSeriesSorageDelegate { - private static final String AFS_APP = "AFS"; + private static final String AFS_DEFAULT_APP = "AFS"; private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesSorageDelegate.class); + /** + * The name of the app in TimeSeriesServer, in which AFS will store time series + */ + private final String app; + + /** + * The address of the TimeSeriesServer + */ private URI timeSeriesServerURI; public TimeSeriesSorageDelegate(URI timeSeriesServerURI) { + this(timeSeriesServerURI, AFS_DEFAULT_APP); + } + + public TimeSeriesSorageDelegate(URI timeSeriesServerURI, String app) { this.timeSeriesServerURI = timeSeriesServerURI; + this.app = app; } public static Client createClient() { @@ -58,11 +71,11 @@ public void createAFSAppIfNotExists() { Response response = buildBaseRequest(client).request().get(); Collection apps = response.readEntity(Collection.class); - if (apps.contains(AFS_APP)) { + if (apps.contains(app)) { return; } - response = buildBaseRequest(client).path(AFS_APP).request().post(Entity.json("")); + response = buildBaseRequest(client).path(app).request().post(Entity.json("")); if (response.getStatus() != 200) { throw new AfsStorageException("Error while initializing AFS timeseries app storage"); } @@ -91,7 +104,7 @@ public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { Client client = createClient(); try { buildBaseRequest(client) - .path(AFS_APP) + .path(app) .path("series") .request().post(Entity.json(new ObjectMapper().writeValueAsString(createQuery))); } catch (JsonProcessingException e) { @@ -108,7 +121,7 @@ public SearchQueryResults performSearch(SearchQuery query) { Client client = createClient(); try { Response response = buildBaseRequest(client) - .path(AFS_APP) + .path(app) .path("series") .path("_search") .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); @@ -200,7 +213,7 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie Client client = createClient(); try { Response response = buildBaseRequest(client) - .path(AFS_APP) + .path(app) .path("series") .request().put(Entity.json(new ObjectMapper().writeValueAsString(publishQuery))); if (response.getStatus() != 200) { @@ -234,7 +247,7 @@ public Map> getDoubleTimeSeriesData(String nodeId, Client client = createClient(); try { Response response = buildBaseRequest(client) - .path(AFS_APP) + .path(app) .path("series") .path("_fetch") .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); From 99d825f1c6a7416d110c62d1a1b9e503b554dd05 Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 09:57:20 +0200 Subject: [PATCH 13/59] Code style fixes --- .../powsybl/afs/storage/AbstractAppStorageTest.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java b/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java index cdc95d49..f6d5b1e5 100644 --- a/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java +++ b/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java @@ -398,13 +398,10 @@ public void test() throws IOException, InterruptedException { assertTrue(storage.getTimeSeriesMetadata(testData3Info.getId(), Sets.newHashSet("ts1")).isEmpty()); // 14) add data to double time series - if(conservesChunks) - { + if (conservesChunks) { storage.addDoubleTimeSeriesData(testData2Info.getId(), 0, "ts1", Arrays.asList(new UncompressedDoubleDataChunk(2, new double[] {1d, 2d}), new UncompressedDoubleDataChunk(5, new double[] {3d}))); - } - else - { + } else { storage.addDoubleTimeSeriesData(testData2Info.getId(), 0, "ts1", Arrays.asList(new UncompressedDoubleDataChunk(0, new double[] {1d, 2d, 3d}))); } @@ -424,10 +421,8 @@ public void test() throws IOException, InterruptedException { assertEquals(Arrays.asList(new UncompressedDoubleDataChunk(2, new double[] {1d, 2d}), new UncompressedDoubleDataChunk(5, new double[] {3d})), doubleTimeSeriesData.get("ts1")); - } - else { - assertEquals(Arrays.asList(new UncompressedDoubleDataChunk(0, new double[] {1d, 2d, 3d})), - doubleTimeSeriesData.get("ts1")); + } else { + assertEquals(Arrays.asList(new UncompressedDoubleDataChunk(0, new double[]{1d, 2d, 3d})), doubleTimeSeriesData.get("ts1")); } assertTrue(storage.getDoubleTimeSeriesData(testData3Info.getId(), Sets.newHashSet("ts1"), 0).isEmpty()); From 59828bb4418dcef8b934d97c6a931901a769faad Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 18:24:11 +0200 Subject: [PATCH 14/59] Create a special module for timeseries config services --- afs-timeseries-server-storage/pom.xml | 87 +++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 afs-timeseries-server-storage/pom.xml diff --git a/afs-timeseries-server-storage/pom.xml b/afs-timeseries-server-storage/pom.xml new file mode 100644 index 00000000..fe2942cf --- /dev/null +++ b/afs-timeseries-server-storage/pom.xml @@ -0,0 +1,87 @@ + + + + 4.0.0 + + 3.6.0-SNAPSHOT + 2.12.1 + 0.0.1-SNAPSHOT + + + + powsybl-afs + com.powsybl + 3.6.0-SNAPSHOT + + + powsybl-afs-timeseries-server-storage + AFS time series server filesystem storage implementations + An AFS time series provider based on time series server + + + + com.powsybl + powsybl-afs-storage-api + ${project.version} + + + com.powsybl + powsybl-time-series-server-interfaces + ${time-series-server.version} + + + + org.slf4j + slf4j-simple + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + org.apache.commons + commons-lang3 + 3.11 + + + org.jboss.resteasy + resteasy-client + provided + + + + + com.powsybl + powsybl-afs-mapdb-storage + ${project.version} + test + + + junit + junit + test + + + com.powsybl + powsybl-afs-storage-api + 3.6.0-SNAPSHOT + test-jar + test + + + org.mock-server + mockserver-netty + 5.11.2 + + + \ No newline at end of file From 4c08ce7f14b8a0a5dec67f17f7ce4915e8131bc7 Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 18:24:57 +0200 Subject: [PATCH 15/59] Initialization for TS server app file system configuration components --- .../TSServerAppFileSystem.java | 14 +++ .../TSServerAppFileSystemConfig.java | 96 +++++++++++++++++++ .../TSServerAppFileSystemProvider.java | 70 ++++++++++++++ 3 files changed, 180 insertions(+) create mode 100644 afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystem.java create mode 100644 afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfig.java create mode 100644 afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystem.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystem.java new file mode 100644 index 00000000..b5ef4f22 --- /dev/null +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystem.java @@ -0,0 +1,14 @@ +package com.powsybl.afs.timeseriesserver; + +import com.powsybl.afs.AppFileSystem; +import com.powsybl.afs.storage.AppStorage; + +/** + * @author amichaut@artelys.com + */ +public class TSServerAppFileSystem extends AppFileSystem { + + public TSServerAppFileSystem(final String name, final boolean remotelyAccessible, final AppStorage storage) { + super(name, remotelyAccessible, storage); + } +} diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfig.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfig.java new file mode 100644 index 00000000..67c9c8a6 --- /dev/null +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfig.java @@ -0,0 +1,96 @@ +package com.powsybl.afs.timeseriesserver; + +import com.powsybl.afs.mapdb.MapDbAppFileSystemConfig; +import com.powsybl.afs.storage.AbstractAppFileSystemConfig; +import com.powsybl.commons.config.PlatformConfig; +import lombok.Getter; +import lombok.Setter; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; + +/** + * App file system configuration for time series server app storage + * + * @author amichaut@artelys.com + */ +public class TSServerAppFileSystemConfig extends AbstractAppFileSystemConfig { + + private static final String TSSERVER_APP_FILE_SYSTEM = "tsserver-app-file-system"; + private static final String DRIVE_NAME = "drive-name"; + private static final String REMOTELY_ACCESSIBLE = "remotely-accessible"; + private static final boolean DEFAULT_REMOTELY_ACCESSIBLE = false; + private static final String HOST = "host"; + private static final String PORT = "port"; + private static final String SCHEME = "scheme"; + private static final String APP = "app"; + private static final String DEFAULT_APP = "AFS"; + public static final String DELEGATE_MAPDB_APP_FILE_SYSTEM = "delegate-mapdb-app-file-system"; + + @Setter + private String host; + + @Setter + private int port; + + @Setter + private String scheme; + + @Getter + @Setter + private String app = DEFAULT_APP; + + @Getter + @Setter + private MapDbAppFileSystemConfig delegateConfig; + + public TSServerAppFileSystemConfig(final String driveName, final boolean remotelyAccessible) { + super(driveName, remotelyAccessible); + } + + public static TSServerAppFileSystemConfig load() { + return load(PlatformConfig.defaultConfig()); + } + + public static TSServerAppFileSystemConfig load(final PlatformConfig platformConfig) { + return platformConfig.getOptionalModuleConfig(TSSERVER_APP_FILE_SYSTEM) + .map(moduleConfig -> { + final String driveName; + if (moduleConfig.hasProperty(DRIVE_NAME)) { + driveName = moduleConfig.getStringProperty(DRIVE_NAME); + } else { + throw new IllegalArgumentException("Please provide a drive name for timeseries server app file system configuration"); + } + final boolean remotelyAccessible = moduleConfig.getBooleanProperty(REMOTELY_ACCESSIBLE, DEFAULT_REMOTELY_ACCESSIBLE); + final TSServerAppFileSystemConfig config = new TSServerAppFileSystemConfig(driveName, remotelyAccessible); + if (moduleConfig.hasProperty(HOST)) { + config.setHost(moduleConfig.getStringProperty(HOST)); + } + if (moduleConfig.hasProperty(PORT)) { + config.setPort(moduleConfig.getIntProperty(PORT)); + } + if (moduleConfig.hasProperty(SCHEME)) { + config.setScheme(moduleConfig.getStringProperty(SCHEME)); + } + config.setApp(moduleConfig.getStringProperty(APP, DEFAULT_APP)); + // Load delegate config + final List delegateConfig = MapDbAppFileSystemConfig.load(platformConfig.getModuleConfig(DELEGATE_MAPDB_APP_FILE_SYSTEM)); + if (delegateConfig.size() != 1) { + throw new IllegalArgumentException("A single mapdb delegate configuration is necessary"); + } + config.setDelegateConfig(delegateConfig.get(0)); + return config; + }) + .orElse(null); + } + + /** + * @return target URI for timeseries service + * @throws URISyntaxException if information does not allow to build a proper URI + */ + public URI getURI() throws URISyntaxException { + return new URI(scheme, null, host, port, null, null, null); + } + +} diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java new file mode 100644 index 00000000..be33f2d2 --- /dev/null +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java @@ -0,0 +1,70 @@ +package com.powsybl.afs.timeseriesserver; + +import com.google.auto.service.AutoService; +import com.powsybl.afs.AppFileSystem; +import com.powsybl.afs.AppFileSystemProvider; +import com.powsybl.afs.AppFileSystemProviderContext; +import com.powsybl.afs.mapdb.storage.MapDbAppStorage; +import com.powsybl.afs.storage.AbstractAppStorage; +import com.powsybl.afs.storage.EventsBus; +import com.powsybl.afs.timeseriesserver.storage.TimeSeriesServerAppStorage; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; + +/** + * @author amichaut@artelys.com + */ +@AutoService(AppFileSystemProvider.class) +public class TSServerAppFileSystemProvider implements AppFileSystemProvider { + + private final TSServerAppFileSystemConfig configuration; + + private final TimeSeriesServerAppStorage.TimeSeriesServerAppStorageProvider storageProvider; + + private final MapDbAppStorage.MapDbAppStorageProvider delegateStorageProvider; + + public TSServerAppFileSystemProvider() { + this( + TSServerAppFileSystemConfig.load(), + TimeSeriesServerAppStorage::new, + (name, path, eventsStore) -> MapDbAppStorage.createMmapFile(name, path.toFile(), eventsStore) + ); + } + + public TSServerAppFileSystemProvider(final TSServerAppFileSystemConfig configuration, + final TimeSeriesServerAppStorage.TimeSeriesServerAppStorageProvider provider, + final MapDbAppStorage.MapDbAppStorageProvider delegateStorageProvider) { + this.configuration = configuration; + this.storageProvider = provider; + this.delegateStorageProvider = delegateStorageProvider; + } + + @Override + public List getFileSystems(final AppFileSystemProviderContext context) { + return Collections.singletonList(getFileSystem(context)); + } + + /** + * @param context a context holding necessary event bus + * @return a single time series server app file system, built using internal configuration and provided context + */ + private TSServerAppFileSystem getFileSystem(final AppFileSystemProviderContext context) { + // Make an URI form configuration + URI uri; + try { + uri = configuration.getURI(); + } catch (URISyntaxException e) { + throw new IllegalStateException("Could not build a proper target URI for time series server"); + } + MapDbAppStorage delegateAppStorage = delegateStorageProvider.apply(configuration.getDriveName(), configuration.getDelegateConfig().getDbFile(), context.getEventsBus()); + return new TSServerAppFileSystem( + configuration.getDriveName(), + configuration.isRemotelyAccessible(), + storageProvider.apply(uri, configuration.getApp(), delegateAppStorage) + ); + } +} From 0908732384129bc48f9b011f6eceaecbdc2ff622 Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 18:25:56 +0200 Subject: [PATCH 16/59] Add unit tests to TS server config components --- .../TSServerAppFileSystemConfigTest.java | 84 +++++++++++++ .../TSServerAppFileSystemProviderTest.java | 118 ++++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfigTest.java create mode 100644 afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java diff --git a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfigTest.java b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfigTest.java new file mode 100644 index 00000000..fbc18f69 --- /dev/null +++ b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfigTest.java @@ -0,0 +1,84 @@ +package com.powsybl.afs.timeseriesserver; + +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import com.powsybl.afs.mapdb.MapDbAppFileSystemConfig; +import com.powsybl.commons.config.InMemoryPlatformConfig; +import com.powsybl.commons.config.MapModuleConfig; +import com.powsybl.commons.config.PlatformConfig; +import org.junit.Before; +import org.junit.Test; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileSystem; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class TSServerAppFileSystemConfigTest { + + private static final String APP = "test-app"; + private static final String PORT = "8765"; + private static final String HOST = "localhost"; + private static final String SCHEME = "http"; + private static final String REMOTELY_ACCESSIBLE = "false"; + private static final String DRIVE_NAME = "test-drive"; + private static final String DB_FILE = "/db/test.db"; + + private FileSystem fileSystem; + private PlatformConfig platformConfig; + + @Before + public void setup() { + fileSystem = Jimfs.newFileSystem(Configuration.unix()); + InMemoryPlatformConfig conf = new InMemoryPlatformConfig(fileSystem); + final MapModuleConfig tsServerConfig = conf.createModuleConfig("tsserver-app-file-system"); + tsServerConfig.setStringProperty("drive-name", DRIVE_NAME); + tsServerConfig.setStringProperty("remotely-accessible", REMOTELY_ACCESSIBLE); + tsServerConfig.setStringProperty("scheme", SCHEME); + tsServerConfig.setStringProperty("host", HOST); + tsServerConfig.setStringProperty("port", PORT); + tsServerConfig.setStringProperty("app", APP); + final MapModuleConfig mapdbConfig = conf.createModuleConfig("delegate-mapdb-app-file-system"); + mapdbConfig.setStringProperty("drive-name", DRIVE_NAME); + mapdbConfig.setPathProperty("db-file", fileSystem.getPath(DB_FILE)); + mapdbConfig.setStringProperty("remotely-accessible", REMOTELY_ACCESSIBLE); + platformConfig = conf; + } + + @Test + public void loadTest() throws URISyntaxException { + final TSServerAppFileSystemConfig conf = TSServerAppFileSystemConfig.load(platformConfig); + assertEquals(conf.getApp(), APP); + assertEquals(conf.getDriveName(), DRIVE_NAME); + final URI uri = conf.getURI(); + assertEquals(uri.getScheme(), SCHEME); + assertEquals(uri.getPort(), Integer.parseInt(PORT)); + assertEquals(uri.getHost(), HOST); + assertEquals(conf.isRemotelyAccessible(), Boolean.valueOf(REMOTELY_ACCESSIBLE)); + + final MapDbAppFileSystemConfig delegateConf = conf.getDelegateConfig(); + assertEquals(delegateConf.getDriveName(), DRIVE_NAME); + assertEquals(delegateConf.isRemotelyAccessible(), Boolean.valueOf(REMOTELY_ACCESSIBLE)); + assertEquals(fileSystem.getPath(DB_FILE), delegateConf.getDbFile()); + } + + /** + * Refer to config.yaml file in test resources + */ + @Test + public void emptyLoadTest() throws URISyntaxException { + final TSServerAppFileSystemConfig load = TSServerAppFileSystemConfig.load(); + assertEquals(load.getDriveName(), "test-fs"); + assertFalse(load.isRemotelyAccessible()); + assertEquals(load.getApp(), "AFS"); + final URI uri = load.getURI(); + assertEquals(uri.getScheme(), "http"); + assertEquals(uri.getHost(), "localhost"); + assertEquals(uri.getPort(), 8080); + final MapDbAppFileSystemConfig delegateConfig = load.getDelegateConfig(); + assertEquals(delegateConfig.getDriveName(), "test-fs"); + assertFalse(delegateConfig.isRemotelyAccessible()); + } +} diff --git a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java new file mode 100644 index 00000000..190a916b --- /dev/null +++ b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java @@ -0,0 +1,118 @@ +package com.powsybl.afs.timeseriesserver; + +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import com.powsybl.afs.AppFileSystem; +import com.powsybl.afs.AppFileSystemProviderContext; +import com.powsybl.afs.mapdb.MapDbAppFileSystemConfig; +import com.powsybl.afs.mapdb.storage.MapDbAppStorage; +import com.powsybl.afs.storage.AbstractAppStorage; +import com.powsybl.afs.storage.EventsBus; +import com.powsybl.afs.storage.InMemoryEventsBus; +import com.powsybl.afs.timeseriesserver.storage.TimeSeriesServerAppStorage; +import com.powsybl.computation.ComputationManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockserver.integration.ClientAndServer; +import org.mockserver.model.MediaType; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; +import static org.junit.Assert.assertEquals; + +public class TSServerAppFileSystemProviderTest { + + private static final int srvPort = 9876; + public static final String DRIVE = "drive"; + + private TSServerAppFileSystemConfig config; + private MapDbAppFileSystemConfig delegateConfig; + + /** + * Mock server to simulate time series provider + */ + private ClientAndServer mockServer; + + /** + * In-memory fs for mapDB + */ + private FileSystem fileSystem; + + /** + * mapDB file + */ + private Path dbFile; + + @Before + public void setUp() { + mockServer = ClientAndServer.startClientAndServer(srvPort); + setupMockServer(); + + // Setup connection parameters + config = new TSServerAppFileSystemConfig(DRIVE, true); + config.setScheme("http"); + config.setHost("localhost"); + config.setPort(mockServer.getPort()); + + // Setup delegate config + fileSystem = Jimfs.newFileSystem(Configuration.unix()); + dbFile = fileSystem.getPath("/db"); + delegateConfig = new MapDbAppFileSystemConfig(DRIVE, true, dbFile); + config.setDelegateConfig(delegateConfig); + } + + /** + * Setup mock server for tests + */ + private void setupMockServer() { + mockServer.when(request() + .withMethod("GET") + .withPath("/v1/timeseries/apps") + ).respond(response() + .withStatusCode(200) + .withContentType(MediaType.APPLICATION_JSON) + .withBody("[]") + ); + mockServer.when(request() + .withMethod("POST") + .withPath("/v1/timeseries/apps/.*") + ).respond(response() + .withStatusCode(200) + .withContentType(MediaType.APPLICATION_JSON) + ); + } + + @After + public void tearDown() throws IOException { + mockServer.stop(); + fileSystem.close(); + } + + @Test + public void provideTest() { + // Build a mock computation context + ComputationManager computationManager = Mockito.mock(ComputationManager.class); + final AppFileSystemProviderContext context = new AppFileSystemProviderContext(computationManager, null, new InMemoryEventsBus()); + // Build a new provider + final MapDbAppStorage.MapDbAppStorageProvider delegateAppStorageProvider = + (name, path, eventsStore) -> MapDbAppStorage.createMem(name, eventsStore); + final TimeSeriesServerAppStorage.TimeSeriesServerAppStorageProvider appStorageProvider = TimeSeriesServerAppStorage::new; + TSServerAppFileSystemProvider provider = new TSServerAppFileSystemProvider(config, appStorageProvider, delegateAppStorageProvider); + // Check that FS is correct + final List fileSystems = provider.getFileSystems(context); + assertEquals(fileSystems.size(), 1); + assertTrue(fileSystems.get(0) instanceof TSServerAppFileSystem); + final TSServerAppFileSystem fs = (TSServerAppFileSystem) fileSystems.get(0); + assertEquals(fs.getName(), DRIVE); + assertTrue(fs.isRemotelyAccessible()); + } +} From d7bd572adf781ddb96adee98ac7aa12d152e46db Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 18:26:18 +0200 Subject: [PATCH 17/59] Setup a conf.yaml file for tests --- afs-timeseries-server/src/test/resources/conf.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 afs-timeseries-server/src/test/resources/conf.yaml diff --git a/afs-timeseries-server/src/test/resources/conf.yaml b/afs-timeseries-server/src/test/resources/conf.yaml new file mode 100644 index 00000000..5b85e3bb --- /dev/null +++ b/afs-timeseries-server/src/test/resources/conf.yaml @@ -0,0 +1,12 @@ +tsserver-app-file-system: + drive-name: "test-fs" + remotely-accessible: false + app: AFS + scheme: http + host: localhost + port: 8080 +delegate-mapdb-app-file-system: + drive-name: "test-fs" + remotely-accessible: false + db-file: "/db/test.db" + From d0c65b798d1df0b4b3deb4abbc16c8f340f2e0f4 Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 18:29:39 +0200 Subject: [PATCH 18/59] Rework ts server storage classes to setup the modules separation (storage / config) --- .../storage}/TimeSeriesServerAppStorage.java | 22 +++++++++++++------ .../storage/TimeSeriesStorageDelegate.java | 13 ++++------- .../TimeSeriesServerAppStorageTest.java | 6 ++++- 3 files changed, 24 insertions(+), 17 deletions(-) rename {afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver => afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage}/TimeSeriesServerAppStorage.java (91%) rename afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java => afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java (96%) rename {afs-timeseries-server => afs-timeseries-server-storage}/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java (69%) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java similarity index 91% rename from afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java rename to afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java index 6d3a741c..9607a168 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java @@ -1,4 +1,4 @@ -package com.powsybl.afs.timeseriesserver; +package com.powsybl.afs.timeseriesserver.storage; import com.powsybl.afs.storage.*; import com.powsybl.afs.storage.events.AppStorageListener; @@ -19,6 +19,11 @@ public class TimeSeriesServerAppStorage extends AbstractAppStorage { + @FunctionalInterface + public interface TimeSeriesServerAppStorageProvider { + R apply(F first, S second, T third); + } + /** * This storage is used for all non-timeseries-related operations */ @@ -27,8 +32,7 @@ public class TimeSeriesServerAppStorage extends AbstractAppStorage { /** * This storage handles all the timeseries-related operations */ - private TimeSeriesSorageDelegate timeSeriesDelegate; - + private TimeSeriesStorageDelegate timeSeriesDelegate; /** * A listener that copies all event from the general delegate event bus to this class event bus. @@ -36,12 +40,16 @@ public class TimeSeriesServerAppStorage extends AbstractAppStorage { */ private AppStorageListener notifyGeneralDelegateEventListener; - public TimeSeriesServerAppStorage(AbstractAppStorage generalDelegate, URI timeSeriesServerURI) { + public TimeSeriesServerAppStorage(final URI targetURI, final String app, final AbstractAppStorage generalDelegate) { + this(generalDelegate, targetURI, app); + } + + public TimeSeriesServerAppStorage(final AbstractAppStorage generalDelegate, final URI timeSeriesServerURI, final String app) { this.generalDelegate = generalDelegate; eventsBus = new InMemoryEventsBus(); notifyGeneralDelegateEventListener = t -> t.getEvents().forEach(e -> pushEvent(e, t.getTopic())); generalDelegate.getEventsBus().addListener(notifyGeneralDelegateEventListener); - timeSeriesDelegate = new TimeSeriesSorageDelegate(timeSeriesServerURI); + timeSeriesDelegate = new TimeSeriesStorageDelegate(timeSeriesServerURI, app); timeSeriesDelegate.createAFSAppIfNotExists(); } @@ -138,7 +146,7 @@ public boolean removeData(String nodeId, String name) { @Override public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { timeSeriesDelegate.createTimeSeries(nodeId, metadata); - pushEvent(new TimeSeriesCreated(nodeId, metadata.getName()), APPSTORAGE_TIMESERIES_TOPIC); + pushEvent(new TimeSeriesCreated(nodeId, metadata.getName()), AbstractAppStorage.APPSTORAGE_TIMESERIES_TOPIC); } @Override @@ -175,7 +183,7 @@ public Map> getDoubleTimeSeriesData(String nodeId, @Override public void addDoubleTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { timeSeriesDelegate.addDoubleTimeSeriesData(nodeId, version, timeSeriesName, chunks); - pushEvent(new TimeSeriesDataUpdated(nodeId, timeSeriesName), APPSTORAGE_TIMESERIES_TOPIC); + pushEvent(new TimeSeriesDataUpdated(nodeId, timeSeriesName), AbstractAppStorage.APPSTORAGE_TIMESERIES_TOPIC); } @Override diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java similarity index 96% rename from afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java rename to afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java index 730efb0a..f0ce795e 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java @@ -1,4 +1,4 @@ -package com.powsybl.afs.timeseriesserver; +package com.powsybl.afs.timeseriesserver.storage; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -28,10 +28,9 @@ import java.util.*; import java.util.stream.Collectors; -public class TimeSeriesSorageDelegate { +public class TimeSeriesStorageDelegate { - private static final String AFS_DEFAULT_APP = "AFS"; - private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesSorageDelegate.class); + private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesStorageDelegate.class); /** * The name of the app in TimeSeriesServer, in which AFS will store time series @@ -43,11 +42,7 @@ public class TimeSeriesSorageDelegate { */ private URI timeSeriesServerURI; - public TimeSeriesSorageDelegate(URI timeSeriesServerURI) { - this(timeSeriesServerURI, AFS_DEFAULT_APP); - } - - public TimeSeriesSorageDelegate(URI timeSeriesServerURI, String app) { + public TimeSeriesStorageDelegate(URI timeSeriesServerURI, String app) { this.timeSeriesServerURI = timeSeriesServerURI; this.app = app; } diff --git a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java b/afs-timeseries-server-storage/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java similarity index 69% rename from afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java rename to afs-timeseries-server-storage/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java index a8e17c33..6ff39091 100644 --- a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java +++ b/afs-timeseries-server-storage/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java @@ -4,6 +4,7 @@ import com.powsybl.afs.storage.AbstractAppStorageTest; import com.powsybl.afs.storage.AppStorage; import com.powsybl.afs.storage.InMemoryEventsBus; +import com.powsybl.afs.timeseriesserver.storage.TimeSeriesServerAppStorage; import org.junit.Before; import java.net.URI; @@ -12,6 +13,8 @@ public class TimeSeriesServerAppStorageTest extends AbstractAppStorageTest { private URI timeSeriesServerURI; + private static final String AFS_APP = "AFS"; + public TimeSeriesServerAppStorageTest() { super(false, false); } @@ -25,7 +28,8 @@ public void setUp() throws Exception { @Override protected AppStorage createStorage() { - return new TimeSeriesServerAppStorage(MapDbAppStorage.createMem("mem", new InMemoryEventsBus()), timeSeriesServerURI); + final MapDbAppStorage storage = MapDbAppStorage.createMem("mem", new InMemoryEventsBus()); + return new TimeSeriesServerAppStorage(storage, timeSeriesServerURI, AFS_APP); } } From b819782582c3b24cb9fbb5f597c3015a509f2961 Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 18:30:07 +0200 Subject: [PATCH 19/59] Add new module in parent pom --- afs-timeseries-server/pom.xml | 29 +++++++++++++++++++---------- pom.xml | 1 + 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/afs-timeseries-server/pom.xml b/afs-timeseries-server/pom.xml index cc228b9b..eb9cf184 100644 --- a/afs-timeseries-server/pom.xml +++ b/afs-timeseries-server/pom.xml @@ -29,24 +29,29 @@ powsybl-afs-core ${project.version} + + com.powsybl + powsybl-afs-timeseries-server-storage + ${project.version} + ${project.groupId} powsybl-time-series-server-interfaces 0.0.1-SNAPSHOT + + org.jboss.spec.javax.ws.rs + jboss-jaxrs-api_2.0_spec + 1.0.1.Beta1 + org.jboss.resteasy resteasy-client - provided org.jboss.resteasy resteasy-jackson-provider - 3.0.19.Final - - - javax - javaee-api + 3.1.4.Final @@ -74,7 +79,6 @@ ${project.groupId} powsybl-afs-cassandra ${project.version} - test-jar test @@ -109,10 +113,15 @@ test - ${project.groupId} - powsybl-afs-mapdb-storage + org.projectlombok + lombok + 1.18.20 + + + com.powsybl + powsybl-afs-mapdb ${project.version} - test + compile diff --git a/pom.xml b/pom.xml index c2b9ca20..63b88996 100644 --- a/pom.xml +++ b/pom.xml @@ -65,6 +65,7 @@ afs-ws afs-spring-server afs-timeseries-server + afs-timeseries-server-storage From e99a63e7d642ce7607aad3fdc952ba649293e44c Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 18:31:35 +0200 Subject: [PATCH 20/59] Rework mapdb app file system configuration to allow delegate provider config (ts server app storage) --- .../afs/mapdb/MapDbAppFileSystemConfig.java | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/afs-mapdb/src/main/java/com/powsybl/afs/mapdb/MapDbAppFileSystemConfig.java b/afs-mapdb/src/main/java/com/powsybl/afs/mapdb/MapDbAppFileSystemConfig.java index 8df06574..4df922c4 100644 --- a/afs-mapdb/src/main/java/com/powsybl/afs/mapdb/MapDbAppFileSystemConfig.java +++ b/afs-mapdb/src/main/java/com/powsybl/afs/mapdb/MapDbAppFileSystemConfig.java @@ -8,6 +8,7 @@ import com.powsybl.afs.AfsException; import com.powsybl.afs.storage.AbstractAppFileSystemConfig; +import com.powsybl.commons.config.ModuleConfig; import com.powsybl.commons.config.PlatformConfig; import java.nio.file.Files; @@ -31,30 +32,32 @@ public static List load() { public static List load(PlatformConfig platformConfig) { return platformConfig.getOptionalModuleConfig("mapdb-app-file-system") - .map(moduleConfig -> { - List configs = new ArrayList<>(); - if (moduleConfig.hasProperty("drive-name") - && moduleConfig.hasProperty("db-file")) { - String driveName = moduleConfig.getStringProperty("drive-name"); - boolean remotelyAccessible = moduleConfig.getBooleanProperty("remotely-accessible", DEFAULT_REMOTELY_ACCESSIBLE); - Path rootDir = moduleConfig.getPathProperty("db-file"); - configs.add(new MapDbAppFileSystemConfig(driveName, remotelyAccessible, rootDir)); - } - int maxAdditionalDriveCount = moduleConfig.getIntProperty("max-additional-drive-count", 0); - for (int i = 0; i < maxAdditionalDriveCount; i++) { - if (moduleConfig.hasProperty("drive-name-" + i) - && moduleConfig.hasProperty("db-file-" + i)) { - String driveName = moduleConfig.getStringProperty("drive-name-" + i); - boolean remotelyAccessible = moduleConfig.getBooleanProperty("remotely-accessible-" + i, DEFAULT_REMOTELY_ACCESSIBLE); - Path rootDir = moduleConfig.getPathProperty("db-file-" + i); - configs.add(new MapDbAppFileSystemConfig(driveName, remotelyAccessible, rootDir)); - } - } - return configs; - }) + .map(MapDbAppFileSystemConfig::load) .orElse(Collections.emptyList()); } + public static List load(ModuleConfig moduleConfig) { + List configs = new ArrayList<>(); + if (moduleConfig.hasProperty("drive-name") + && moduleConfig.hasProperty("db-file")) { + String driveName = moduleConfig.getStringProperty("drive-name"); + boolean remotelyAccessible = moduleConfig.getBooleanProperty("remotely-accessible", DEFAULT_REMOTELY_ACCESSIBLE); + Path rootDir = moduleConfig.getPathProperty("db-file"); + configs.add(new MapDbAppFileSystemConfig(driveName, remotelyAccessible, rootDir)); + } + int maxAdditionalDriveCount = moduleConfig.getIntProperty("max-additional-drive-count", 0); + for (int i = 0; i < maxAdditionalDriveCount; i++) { + if (moduleConfig.hasProperty("drive-name-" + i) + && moduleConfig.hasProperty("db-file-" + i)) { + String driveName = moduleConfig.getStringProperty("drive-name-" + i); + boolean remotelyAccessible = moduleConfig.getBooleanProperty("remotely-accessible-" + i, DEFAULT_REMOTELY_ACCESSIBLE); + Path rootDir = moduleConfig.getPathProperty("db-file-" + i); + configs.add(new MapDbAppFileSystemConfig(driveName, remotelyAccessible, rootDir)); + } + } + return configs; + } + private static Path checkDbFile(Path dbFile) { Objects.requireNonNull(dbFile); if (Files.isDirectory(dbFile)) { From c0895f540ba8b89bcd7069b9c476539fca1ee7bd Mon Sep 17 00:00:00 2001 From: amichaut Date: Thu, 17 Jun 2021 18:42:56 +0200 Subject: [PATCH 21/59] Add support for String time series in time series server AppStorage --- .../storage/TimeSeriesServerAppStorage.java | 6 +- .../storage/TimeSeriesStorageDelegate.java | 255 ++++++++++++------ 2 files changed, 183 insertions(+), 78 deletions(-) diff --git a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java index 9607a168..83bb0357 100644 --- a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java @@ -176,7 +176,6 @@ public Set getTimeSeriesDataVersions(String nodeId, String timeSeriesNa @Override public Map> getDoubleTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { - //TODO return timeSeriesDelegate.getDoubleTimeSeriesData(nodeId, timeSeriesNames, version); } @@ -188,12 +187,13 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie @Override public Map> getStringTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { - throw new NotImplementedException("Not implemented in V1"); + return timeSeriesDelegate.getStringTimeSeriesData(nodeId, timeSeriesNames, version); } @Override public void addStringTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { - throw new NotImplementedException("Not implemented in V1"); + timeSeriesDelegate.addStringTimeSeriesData(nodeId, version, timeSeriesName, chunks); + pushEvent(new TimeSeriesDataUpdated(nodeId, timeSeriesName), AbstractAppStorage.APPSTORAGE_TIMESERIES_TOPIC); } @Override diff --git a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java index f0ce795e..9b5ab910 100644 --- a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java @@ -4,10 +4,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.powsybl.afs.storage.AfsStorageException; import com.powsybl.timeseries.*; +import com.powsybl.timeseries.storer.data.dto.TimeSeriesInfoDto; import com.powsybl.timeseries.storer.query.create.CreateQuery; import com.powsybl.timeseries.storer.query.fetch.FetchQuery; -import com.powsybl.timeseries.storer.query.fetch.FetchQueryResult; +import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryDoubleResult; +import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryResult; +import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryStringResult; +import com.powsybl.timeseries.storer.query.publish.PublishDoubleQuery; import com.powsybl.timeseries.storer.query.publish.PublishQuery; +import com.powsybl.timeseries.storer.query.publish.PublishStringQuery; import com.powsybl.timeseries.storer.query.search.SearchQuery; import com.powsybl.timeseries.storer.query.search.SearchQueryResults; import org.apache.commons.lang3.ArrayUtils; @@ -49,15 +54,15 @@ public TimeSeriesStorageDelegate(URI timeSeriesServerURI, String app) { public static Client createClient() { return new ResteasyClientBuilder() - .connectionPoolSize(50) - .build(); + .connectionPoolSize(50) + .build(); } private WebTarget buildBaseRequest(Client client) { return client.target(timeSeriesServerURI) - .path("v1") - .path("timeseries") - .path("apps"); + .path("v1") + .path("timeseries") + .path("apps"); } public void createAFSAppIfNotExists() { @@ -99,9 +104,9 @@ public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { Client client = createClient(); try { buildBaseRequest(client) - .path(app) - .path("series") - .request().post(Entity.json(new ObjectMapper().writeValueAsString(createQuery))); + .path(app) + .path("series") + .request().post(Entity.json(new ObjectMapper().writeValueAsString(createQuery))); } catch (JsonProcessingException e) { LOGGER.error(e.getMessage(), e); } finally { @@ -116,10 +121,10 @@ public SearchQueryResults performSearch(SearchQuery query) { Client client = createClient(); try { Response response = buildBaseRequest(client) - .path(app) - .path("series") - .path("_search") - .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); + .path(app) + .path("series") + .path("_search") + .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); String json = response.readEntity(String.class); results = new ObjectMapper().readValue(json, SearchQueryResults.class); } catch (IOException e) { @@ -139,17 +144,14 @@ public Set getTimeSeriesNames(String nodeId) { SearchQueryResults results = performSearch(searchQuery); if (results != null) { return results.getTimeSeriesInformations() - .stream().map(t -> t.getName()).collect(Collectors.toSet()); + .stream().map(t -> t.getName()).collect(Collectors.toSet()); } return null; } public boolean timeSeriesExists(String nodeId, String name) { - SearchQuery searchQuery = new SearchQuery(); - searchQuery.setMatrix(nodeId); - searchQuery.setNames(Collections.singleton(name)); - SearchQueryResults results = performSearch(searchQuery); + SearchQueryResults results = doSearch(nodeId, Collections.singleton(name)); if (results != null) { return results.getTimeSeriesInformations().size() > 0; } @@ -158,20 +160,17 @@ public boolean timeSeriesExists(String nodeId, String name) { public List getTimeSeriesMetadata(String nodeId, Set timeSeriesNames) { - SearchQuery searchQuery = new SearchQuery(); - searchQuery.setMatrix(nodeId); - searchQuery.setNames(timeSeriesNames); - SearchQueryResults results = performSearch(searchQuery); + SearchQueryResults results = doSearch(nodeId, timeSeriesNames); if (results != null) { return results.getTimeSeriesInformations() - .stream().map(t -> { - long startTime = t.getStartDate().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); - long spacing = t.getTimeStepDuration(); - long endTime = startTime + spacing * (t.getTimeStepCount() - 1); - TimeSeriesIndex index = new RegularTimeSeriesIndex(startTime, endTime, spacing); - return new TimeSeriesMetadata(t.getName(), TimeSeriesDataType.DOUBLE, t.getTags(), index); - }) - .collect(Collectors.toList()); + .stream().map(t -> { + long startTime = t.getStartDate().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + long spacing = t.getTimeStepDuration(); + long endTime = startTime + spacing * (t.getTimeStepCount() - 1); + TimeSeriesIndex index = new RegularTimeSeriesIndex(startTime, endTime, spacing); + return new TimeSeriesMetadata(t.getName(), TimeSeriesDataType.DOUBLE, t.getTags(), index); + }) + .collect(Collectors.toList()); } return null; } @@ -185,13 +184,45 @@ public Set getTimeSeriesDataVersions(String nodeId, String timeSeriesNa SearchQueryResults results = performSearch(searchQuery); if (results != null) { return results.getTimeSeriesInformations() - .stream().flatMap(t -> t.getVersions().keySet().stream()) - .map(t -> Integer.parseInt(t)) - .collect(Collectors.toSet()); + .stream().flatMap(t -> t.getVersions().keySet().stream()) + .map(t -> Integer.parseInt(t)) + .collect(Collectors.toSet()); } return null; } + /** + * Add a new string time series + * + * @param nodeId identifier for the node + * @param version version to publish to + * @param timeSeriesName name of the time series + * @param chunks actual (string) data to publish + */ + public void addStringTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { + // Retrieve metadata and build time series + TimeSeriesMetadata metadata = getTimeSeriesMetadata(nodeId, Collections.singleton(timeSeriesName)).get(0); + StringTimeSeries ts = new StringTimeSeries(metadata, chunks); + + // Prepare publish request + PublishQuery publishQuery = new PublishStringQuery(); + publishQuery.setMatrix(nodeId); + publishQuery.setTimeSeriesName(timeSeriesName); + publishQuery.setVersionName(String.valueOf(version)); + publishQuery.setData(ts.toArray()); + + // Do run query + doPublish(publishQuery); + } + + /** + * Add a new double time series + * + * @param nodeId identifier for the node + * @param version version to publish to + * @param timeSeriesName name of the time series + * @param chunks actual (double) data to publish + */ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { //First step : retrieve metadata and reconstitute the time series @@ -199,73 +230,147 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie StoredDoubleTimeSeries ts = new StoredDoubleTimeSeries(metadata, chunks); //Second step : perform a publish request - PublishQuery publishQuery = new PublishQuery(); + PublishQuery publishQuery = new PublishDoubleQuery(); publishQuery.setMatrix(nodeId); publishQuery.setTimeSeriesName(timeSeriesName); publishQuery.setVersionName(String.valueOf(version)); publishQuery.setData(ArrayUtils.toObject(ts.toArray())); - Client client = createClient(); + // Do run query + doPublish(publishQuery); + } + + /** + * Retrieve time series String data from server + * Perform a search query with provided criteria, followed by a fetch query (using search results) + * @param nodeId identifier for the node to retrieve data for + * @param timeSeriesNames names of time series to retrieve + * @param version version of the data to fetch + * @return a map contaning all queried time series, indexed by their name + */ + public Map> getStringTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { + String versionString = String.valueOf(version); + // First perform a search query + final SearchQueryResults searchResults = doSearch(nodeId, timeSeriesNames); + List versionIDs = searchResults.getVersionIds(versionString); + Map versionToName = searchResults.getVersionToNameMap(versionString); + // Then perform the fetch query + final Response response = doFetch(versionToName); + // Extract fetch result + String json = response.readEntity(String.class); + FetchQueryStringResult fetchResults; try { - Response response = buildBaseRequest(client) - .path(app) - .path("series") - .request().put(Entity.json(new ObjectMapper().writeValueAsString(publishQuery))); - if (response.getStatus() != 200) { - throw new AfsStorageException("Error while publishing data to time series server"); - } - } catch (IOException e) { - LOGGER.error(e.getMessage(), e); - } finally { - client.close(); + fetchResults = new ObjectMapper().readValue(json, FetchQueryStringResult.class); + } catch (JsonProcessingException e) { + throw new AfsStorageException("Could not process fetch query result into a String query result"); } - + Map> result = new HashMap<>(); + for (int i = 0; i < versionToName.keySet().size(); i++) { + String[] values = fetchResults.getData().get(i).toArray(new String[0]); + StringDataChunk chunk = new UncompressedStringDataChunk(0, values); + result.put(versionToName.get(versionIDs.get(i)), Collections.singletonList(chunk)); + } + return result; } + /** + * Perform search then fetch queries to retrieve double data against time series server + * + * @param nodeId identifier for the node to retrieve time series for + * @param timeSeriesNames names of time series to search for + * @param version version of the data to fetch + * @return time series names, mapped to their respective results (as chunks) + */ public Map> getDoubleTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { - String versionString = String.valueOf(version); + // First perform a search query + final SearchQueryResults searchResults = doSearch(nodeId, timeSeriesNames); + List versionIDs = searchResults.getVersionIds(versionString); + Map versionToName = searchResults.getVersionToNameMap(versionString); + // Then perform the fetch query + final Response response = doFetch(versionToName); + // Extract fetch result + String json = response.readEntity(String.class); + FetchQueryDoubleResult fetchResults; + try { + fetchResults = new ObjectMapper().readValue(json, FetchQueryDoubleResult.class); + } catch (JsonProcessingException e) { + throw new AfsStorageException("Could not process fetch query result into a String query result"); + } + // Extract data from results + Map> toReturn = new HashMap<>(); + for (int i = 0; i < versionIDs.size(); i++) { + double[] values = fetchResults.getData().get(i).stream().mapToDouble(Double::doubleValue).toArray(); + UncompressedDoubleDataChunk chunk = new UncompressedDoubleDataChunk(0, values); + toReturn.put(versionToName.get(versionIDs.get(i)), Collections.singletonList(chunk)); + } + return toReturn; + } + + /** + * Perform search request against time series server + * + * @param nodeId identifier of the node to search for + * @param timeSeriesNames names of time series to search for + * @return a SearchQueryResults object representing the TS server response + */ + private SearchQueryResults doSearch(final String nodeId, final Set timeSeriesNames) { + // Prepare search query SearchQuery searchQuery = new SearchQuery(); searchQuery.setMatrix(nodeId); searchQuery.setNames(timeSeriesNames); - SearchQueryResults results = performSearch(searchQuery); - List versionIDs = results.getTimeSeriesInformations() - .stream() - .map(t -> t.getVersions().get(versionString)) - .collect(Collectors.toList()); - - Map versionIDToTSName = results.getTimeSeriesInformations() - .stream() - .collect(Collectors.toMap(t -> t.getVersions().get(versionString), t -> t.getName())); + // Run search query + return performSearch(searchQuery); + } - FetchQuery query = new FetchQuery(versionIDs, null, null); + /** + * Perform a search query against distant API + * + * @param versionToName a map of TS version -> TS name + * @return fetch HTTP response + */ + private Response doFetch(final Map versionToName) { + // Prepare fetch query + FetchQuery query = new FetchQuery(versionToName.keySet(), null, null); Client client = createClient(); try { Response response = buildBaseRequest(client) - .path(app) - .path("series") - .path("_fetch") - .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); + .path(app) + .path("series") + .path("_fetch") + .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); if (response.getStatus() != 200) { throw new AfsStorageException("Error while fetching data from time series server"); } - String json = response.readEntity(String.class); - FetchQueryResult fetchResults = new ObjectMapper().readValue(json, FetchQueryResult.class); + return response; + } catch (JsonProcessingException e) { + throw new AfsStorageException("Error while fetching data from time series server"); + } finally { + client.close(); + } + } - Map> toReturn = new HashMap<>(); - for (int i = 0; i < versionIDs.size(); i++) { - double[] values = fetchResults.getData().get(i).stream().mapToDouble(Double::doubleValue).toArray(); - UncompressedDoubleDataChunk chunk = new UncompressedDoubleDataChunk(0, values); - toReturn.put(versionIDToTSName.get(versionIDs.get(i)), Arrays.asList(chunk)); + /** + * Perform a publish query + * + * @param publishQuery query to issue + */ + private void doPublish(final PublishQuery publishQuery) { + // Run request + Client client = createClient(); + try { + Response response = buildBaseRequest(client) + .path(app) + .path("series") + .request() + .put(Entity.json(new ObjectMapper().writeValueAsString(publishQuery))); + if (response.getStatus() != 200) { + throw new AfsStorageException("Error while publishing data to time series server"); } - return toReturn; - - } catch (IOException e) { - LOGGER.error(e.getMessage(), e); + } catch (IOException ioe) { + LOGGER.error("Could not publish String time series", ioe); } finally { client.close(); } - - return null; } } From 4c55a33996304035bc88635e5949153c0052310a Mon Sep 17 00:00:00 2001 From: amichaut Date: Fri, 18 Jun 2021 09:55:16 +0200 Subject: [PATCH 22/59] Fix code to match powsybl standards --- .../storage/TimeSeriesStorageDelegate.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java index 9b5ab910..be80ff09 100644 --- a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java @@ -4,14 +4,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.powsybl.afs.storage.AfsStorageException; import com.powsybl.timeseries.*; -import com.powsybl.timeseries.storer.data.dto.TimeSeriesInfoDto; import com.powsybl.timeseries.storer.query.create.CreateQuery; import com.powsybl.timeseries.storer.query.fetch.FetchQuery; import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryDoubleResult; -import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryResult; import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryStringResult; import com.powsybl.timeseries.storer.query.publish.PublishDoubleQuery; -import com.powsybl.timeseries.storer.query.publish.PublishQuery; +import com.powsybl.timeseries.storer.query.publish.AbstractPublishQuery; import com.powsybl.timeseries.storer.query.publish.PublishStringQuery; import com.powsybl.timeseries.storer.query.search.SearchQuery; import com.powsybl.timeseries.storer.query.search.SearchQueryResults; @@ -205,7 +203,7 @@ public void addStringTimeSeriesData(String nodeId, int version, String timeSerie StringTimeSeries ts = new StringTimeSeries(metadata, chunks); // Prepare publish request - PublishQuery publishQuery = new PublishStringQuery(); + AbstractPublishQuery publishQuery = new PublishStringQuery(); publishQuery.setMatrix(nodeId); publishQuery.setTimeSeriesName(timeSeriesName); publishQuery.setVersionName(String.valueOf(version)); @@ -230,7 +228,7 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie StoredDoubleTimeSeries ts = new StoredDoubleTimeSeries(metadata, chunks); //Second step : perform a publish request - PublishQuery publishQuery = new PublishDoubleQuery(); + AbstractPublishQuery publishQuery = new PublishDoubleQuery(); publishQuery.setMatrix(nodeId); publishQuery.setTimeSeriesName(timeSeriesName); publishQuery.setVersionName(String.valueOf(version)); @@ -355,7 +353,7 @@ private Response doFetch(final Map versionToName) { * * @param publishQuery query to issue */ - private void doPublish(final PublishQuery publishQuery) { + private void doPublish(final AbstractPublishQuery publishQuery) { // Run request Client client = createClient(); try { From 3c876f08dfe28650492ce5ebe4b6beea3d1a53be Mon Sep 17 00:00:00 2001 From: Paul Bui-Quang Date: Wed, 7 Jul 2021 14:54:57 +0200 Subject: [PATCH 23/59] Fix checkstyle + remove missing dependency Signed-off-by: Paul Bui-Quang --- .../afs/storage/AbstractAppStorageTest.java | 2 +- .../storage/TimeSeriesServerAppStorage.java | 1 - .../storage/TimeSeriesStorageDelegate.java | 15 +++++++-------- afs-timeseries-server/pom.xml | 7 ------- .../TSServerAppFileSystemProvider.java | 5 ++--- .../TSServerAppFileSystemProviderTest.java | 7 +++---- 6 files changed, 13 insertions(+), 24 deletions(-) diff --git a/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java b/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java index f6d5b1e5..facbc3cd 100644 --- a/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java +++ b/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java @@ -427,7 +427,7 @@ public void test() throws IOException, InterruptedException { assertTrue(storage.getDoubleTimeSeriesData(testData3Info.getId(), Sets.newHashSet("ts1"), 0).isEmpty()); // 15) create a second string time series - if(handlesStringTimeSeries) { + if (handlesStringTimeSeries) { TimeSeriesMetadata metadata2 = new TimeSeriesMetadata("ts2", TimeSeriesDataType.STRING, ImmutableMap.of(), diff --git a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java index 83bb0357..208ab4e5 100644 --- a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java @@ -7,7 +7,6 @@ import com.powsybl.timeseries.DoubleDataChunk; import com.powsybl.timeseries.StringDataChunk; import com.powsybl.timeseries.TimeSeriesMetadata; -import org.apache.commons.lang3.NotImplementedException; import java.io.InputStream; import java.io.OutputStream; diff --git a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java index be80ff09..c69d400e 100644 --- a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java @@ -161,14 +161,13 @@ public List getTimeSeriesMetadata(String nodeId, Set SearchQueryResults results = doSearch(nodeId, timeSeriesNames); if (results != null) { return results.getTimeSeriesInformations() - .stream().map(t -> { - long startTime = t.getStartDate().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); - long spacing = t.getTimeStepDuration(); - long endTime = startTime + spacing * (t.getTimeStepCount() - 1); - TimeSeriesIndex index = new RegularTimeSeriesIndex(startTime, endTime, spacing); - return new TimeSeriesMetadata(t.getName(), TimeSeriesDataType.DOUBLE, t.getTags(), index); - }) - .collect(Collectors.toList()); + .stream().map(t -> { + long startTime = t.getStartDate().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + long spacing = t.getTimeStepDuration(); + long endTime = startTime + spacing * (t.getTimeStepCount() - 1); + TimeSeriesIndex index = new RegularTimeSeriesIndex(startTime, endTime, spacing); + return new TimeSeriesMetadata(t.getName(), TimeSeriesDataType.DOUBLE, t.getTags(), index); + }).collect(Collectors.toList()); } return null; } diff --git a/afs-timeseries-server/pom.xml b/afs-timeseries-server/pom.xml index eb9cf184..55095157 100644 --- a/afs-timeseries-server/pom.xml +++ b/afs-timeseries-server/pom.xml @@ -81,13 +81,6 @@ ${project.version} test - - ${project.groupId} - powsybl-afs-local - ${project.version} - test-jar - test - ${project.groupId} powsybl-afs-local diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java index be33f2d2..d0c1d706 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java @@ -29,9 +29,8 @@ public class TSServerAppFileSystemProvider implements AppFileSystemProvider { public TSServerAppFileSystemProvider() { this( - TSServerAppFileSystemConfig.load(), - TimeSeriesServerAppStorage::new, - (name, path, eventsStore) -> MapDbAppStorage.createMmapFile(name, path.toFile(), eventsStore) + TSServerAppFileSystemConfig.load(), + TimeSeriesServerAppStorage::new, (name, path, eventsStore) -> MapDbAppStorage.createMmapFile(name, path.toFile(), eventsStore) ); } diff --git a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java index 190a916b..402fef5f 100644 --- a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java +++ b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java @@ -31,7 +31,7 @@ public class TSServerAppFileSystemProviderTest { - private static final int srvPort = 9876; + private static final int SRV_PORT = 9876; public static final String DRIVE = "drive"; private TSServerAppFileSystemConfig config; @@ -54,7 +54,7 @@ public class TSServerAppFileSystemProviderTest { @Before public void setUp() { - mockServer = ClientAndServer.startClientAndServer(srvPort); + mockServer = ClientAndServer.startClientAndServer(SRV_PORT); setupMockServer(); // Setup connection parameters @@ -103,8 +103,7 @@ public void provideTest() { ComputationManager computationManager = Mockito.mock(ComputationManager.class); final AppFileSystemProviderContext context = new AppFileSystemProviderContext(computationManager, null, new InMemoryEventsBus()); // Build a new provider - final MapDbAppStorage.MapDbAppStorageProvider delegateAppStorageProvider = - (name, path, eventsStore) -> MapDbAppStorage.createMem(name, eventsStore); + final MapDbAppStorage.MapDbAppStorageProvider delegateAppStorageProvider = (name, path, eventsStore) -> MapDbAppStorage.createMem(name, eventsStore); final TimeSeriesServerAppStorage.TimeSeriesServerAppStorageProvider appStorageProvider = TimeSeriesServerAppStorage::new; TSServerAppFileSystemProvider provider = new TSServerAppFileSystemProvider(config, appStorageProvider, delegateAppStorageProvider); // Check that FS is correct From 3439ca736dd8386dc427d626cdddc488b65b4fe7 Mon Sep 17 00:00:00 2001 From: Mathieu BAGUE Date: Mon, 16 Nov 2020 09:31:09 +0100 Subject: [PATCH 24/59] Prepare release v3.4.0 Signed-off-by: Mathieu BAGUE Signed-off-by: Arthur Michaut --- afs-action-dsl/pom.xml | 2 +- afs-cassandra/pom.xml | 2 +- afs-contingency/pom.xml | 2 +- afs-core/pom.xml | 2 +- afs-distribution/pom.xml | 2 +- afs-ext-base/pom.xml | 2 +- afs-local/pom.xml | 2 +- afs-mapdb-storage/pom.xml | 2 +- afs-mapdb/pom.xml | 2 +- afs-network/afs-network-client/pom.xml | 2 +- afs-network/afs-network-server/pom.xml | 2 +- afs-network/pom.xml | 2 +- afs-scripting/pom.xml | 2 +- afs-security-analysis-local/pom.xml | 2 +- afs-security-analysis/pom.xml | 2 +- afs-storage-api/pom.xml | 2 +- afs-ws/afs-ws-client-utils/pom.xml | 2 +- afs-ws/afs-ws-client/pom.xml | 2 +- afs-ws/afs-ws-server-utils/pom.xml | 2 +- afs-ws/afs-ws-server/pom.xml | 2 +- afs-ws/afs-ws-storage/pom.xml | 2 +- afs-ws/afs-ws-utils/pom.xml | 2 +- afs-ws/pom.xml | 2 +- pom.xml | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/afs-action-dsl/pom.xml b/afs-action-dsl/pom.xml index 7e3f8c1e..9d1db08c 100644 --- a/afs-action-dsl/pom.xml +++ b/afs-action-dsl/pom.xml @@ -13,7 +13,7 @@ com.powsybl powsybl-afs - 3.4.0-SNAPSHOT + 3.4.0 4.0.0 diff --git a/afs-cassandra/pom.xml b/afs-cassandra/pom.xml index 9a4b2cf3..ff750514 100644 --- a/afs-cassandra/pom.xml +++ b/afs-cassandra/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-cassandra diff --git a/afs-contingency/pom.xml b/afs-contingency/pom.xml index c7d9500a..09462167 100644 --- a/afs-contingency/pom.xml +++ b/afs-contingency/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-contingency diff --git a/afs-core/pom.xml b/afs-core/pom.xml index 14ba316a..10101320 100644 --- a/afs-core/pom.xml +++ b/afs-core/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-core diff --git a/afs-distribution/pom.xml b/afs-distribution/pom.xml index 98b7fcd2..df365e87 100644 --- a/afs-distribution/pom.xml +++ b/afs-distribution/pom.xml @@ -13,7 +13,7 @@ powsybl-afs com.powsybl - 3.4.0-SNAPSHOT + 3.4.0 4.0.0 diff --git a/afs-ext-base/pom.xml b/afs-ext-base/pom.xml index 1e133dc0..a8cd37db 100644 --- a/afs-ext-base/pom.xml +++ b/afs-ext-base/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-ext-base diff --git a/afs-local/pom.xml b/afs-local/pom.xml index 3cd8c8b8..7913f2f1 100644 --- a/afs-local/pom.xml +++ b/afs-local/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-local diff --git a/afs-mapdb-storage/pom.xml b/afs-mapdb-storage/pom.xml index e544d44b..4a25d98a 100644 --- a/afs-mapdb-storage/pom.xml +++ b/afs-mapdb-storage/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-mapdb-storage diff --git a/afs-mapdb/pom.xml b/afs-mapdb/pom.xml index e59b7dd4..3a1c2b95 100644 --- a/afs-mapdb/pom.xml +++ b/afs-mapdb/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-mapdb diff --git a/afs-network/afs-network-client/pom.xml b/afs-network/afs-network-client/pom.xml index e5a5211e..86ef62f6 100644 --- a/afs-network/afs-network-client/pom.xml +++ b/afs-network/afs-network-client/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-network - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-network-client diff --git a/afs-network/afs-network-server/pom.xml b/afs-network/afs-network-server/pom.xml index d39807be..f52d6d12 100644 --- a/afs-network/afs-network-server/pom.xml +++ b/afs-network/afs-network-server/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-network - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-network-server diff --git a/afs-network/pom.xml b/afs-network/pom.xml index 2e3809a5..d46508ae 100644 --- a/afs-network/pom.xml +++ b/afs-network/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-network diff --git a/afs-scripting/pom.xml b/afs-scripting/pom.xml index 651ccfd3..414ba47e 100644 --- a/afs-scripting/pom.xml +++ b/afs-scripting/pom.xml @@ -13,7 +13,7 @@ powsybl-afs com.powsybl - 3.4.0-SNAPSHOT + 3.4.0 4.0.0 diff --git a/afs-security-analysis-local/pom.xml b/afs-security-analysis-local/pom.xml index ae8705d0..047a5b75 100644 --- a/afs-security-analysis-local/pom.xml +++ b/afs-security-analysis-local/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-security-analysis-local diff --git a/afs-security-analysis/pom.xml b/afs-security-analysis/pom.xml index 5ca165f0..9ee4086b 100644 --- a/afs-security-analysis/pom.xml +++ b/afs-security-analysis/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-security-analysis diff --git a/afs-storage-api/pom.xml b/afs-storage-api/pom.xml index 79746ee5..2b02c2ba 100644 --- a/afs-storage-api/pom.xml +++ b/afs-storage-api/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-storage-api diff --git a/afs-ws/afs-ws-client-utils/pom.xml b/afs-ws/afs-ws-client-utils/pom.xml index 18d23d30..11c5f8e8 100644 --- a/afs-ws/afs-ws-client-utils/pom.xml +++ b/afs-ws/afs-ws-client-utils/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-ws - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-ws-client-utils diff --git a/afs-ws/afs-ws-client/pom.xml b/afs-ws/afs-ws-client/pom.xml index 64036143..a5e10ec2 100644 --- a/afs-ws/afs-ws-client/pom.xml +++ b/afs-ws/afs-ws-client/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-ws-client diff --git a/afs-ws/afs-ws-server-utils/pom.xml b/afs-ws/afs-ws-server-utils/pom.xml index 6cc2f57f..6bccccb0 100644 --- a/afs-ws/afs-ws-server-utils/pom.xml +++ b/afs-ws/afs-ws-server-utils/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-ws - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-ws-server-utils diff --git a/afs-ws/afs-ws-server/pom.xml b/afs-ws/afs-ws-server/pom.xml index 4ab78852..56a2b25a 100644 --- a/afs-ws/afs-ws-server/pom.xml +++ b/afs-ws/afs-ws-server/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-ws-server diff --git a/afs-ws/afs-ws-storage/pom.xml b/afs-ws/afs-ws-storage/pom.xml index da9b19c5..b889fb19 100644 --- a/afs-ws/afs-ws-storage/pom.xml +++ b/afs-ws/afs-ws-storage/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-ws-storage diff --git a/afs-ws/afs-ws-utils/pom.xml b/afs-ws/afs-ws-utils/pom.xml index 06a554ae..97d5edee 100644 --- a/afs-ws/afs-ws-utils/pom.xml +++ b/afs-ws/afs-ws-utils/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-ws-utils diff --git a/afs-ws/pom.xml b/afs-ws/pom.xml index 60effef3..23b1c26b 100644 --- a/afs-ws/pom.xml +++ b/afs-ws/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.4.0-SNAPSHOT + 3.4.0 powsybl-afs-ws diff --git a/pom.xml b/pom.xml index 99dea164..836cd53e 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ powsybl-afs - 3.4.0-SNAPSHOT + 3.4.0 pom powsybl-afs From a8a7bbec794fd3cb4704f2c1bfa118f6c84c30c5 Mon Sep 17 00:00:00 2001 From: Mathieu BAGUE Date: Mon, 16 Nov 2020 09:31:48 +0100 Subject: [PATCH 25/59] Prepare next release v3.5.0 Signed-off-by: Mathieu BAGUE Signed-off-by: Arthur Michaut --- afs-action-dsl/pom.xml | 2 +- afs-cassandra/pom.xml | 2 +- afs-contingency/pom.xml | 2 +- afs-core/pom.xml | 2 +- afs-distribution/pom.xml | 2 +- afs-ext-base/pom.xml | 2 +- afs-local/pom.xml | 2 +- afs-mapdb-storage/pom.xml | 2 +- afs-mapdb/pom.xml | 2 +- afs-network/afs-network-client/pom.xml | 2 +- afs-network/afs-network-server/pom.xml | 2 +- afs-network/pom.xml | 2 +- afs-scripting/pom.xml | 2 +- afs-security-analysis-local/pom.xml | 2 +- afs-security-analysis/pom.xml | 2 +- afs-storage-api/pom.xml | 2 +- afs-ws/afs-ws-client-utils/pom.xml | 2 +- afs-ws/afs-ws-client/pom.xml | 2 +- afs-ws/afs-ws-server-utils/pom.xml | 2 +- afs-ws/afs-ws-server/pom.xml | 2 +- afs-ws/afs-ws-storage/pom.xml | 2 +- afs-ws/afs-ws-utils/pom.xml | 2 +- afs-ws/pom.xml | 2 +- pom.xml | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/afs-action-dsl/pom.xml b/afs-action-dsl/pom.xml index 9d1db08c..aab7c609 100644 --- a/afs-action-dsl/pom.xml +++ b/afs-action-dsl/pom.xml @@ -13,7 +13,7 @@ com.powsybl powsybl-afs - 3.4.0 + 3.5.0-SNAPSHOT 4.0.0 diff --git a/afs-cassandra/pom.xml b/afs-cassandra/pom.xml index ff750514..38b36aec 100644 --- a/afs-cassandra/pom.xml +++ b/afs-cassandra/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-cassandra diff --git a/afs-contingency/pom.xml b/afs-contingency/pom.xml index 09462167..de047635 100644 --- a/afs-contingency/pom.xml +++ b/afs-contingency/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-contingency diff --git a/afs-core/pom.xml b/afs-core/pom.xml index 10101320..7fc3237f 100644 --- a/afs-core/pom.xml +++ b/afs-core/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-core diff --git a/afs-distribution/pom.xml b/afs-distribution/pom.xml index df365e87..f03ef560 100644 --- a/afs-distribution/pom.xml +++ b/afs-distribution/pom.xml @@ -13,7 +13,7 @@ powsybl-afs com.powsybl - 3.4.0 + 3.5.0-SNAPSHOT 4.0.0 diff --git a/afs-ext-base/pom.xml b/afs-ext-base/pom.xml index a8cd37db..1313a1cc 100644 --- a/afs-ext-base/pom.xml +++ b/afs-ext-base/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-ext-base diff --git a/afs-local/pom.xml b/afs-local/pom.xml index 7913f2f1..5bbecc6f 100644 --- a/afs-local/pom.xml +++ b/afs-local/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-local diff --git a/afs-mapdb-storage/pom.xml b/afs-mapdb-storage/pom.xml index 4a25d98a..206c06b0 100644 --- a/afs-mapdb-storage/pom.xml +++ b/afs-mapdb-storage/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-mapdb-storage diff --git a/afs-mapdb/pom.xml b/afs-mapdb/pom.xml index 3a1c2b95..5d8307bf 100644 --- a/afs-mapdb/pom.xml +++ b/afs-mapdb/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-mapdb diff --git a/afs-network/afs-network-client/pom.xml b/afs-network/afs-network-client/pom.xml index 86ef62f6..fcba33b7 100644 --- a/afs-network/afs-network-client/pom.xml +++ b/afs-network/afs-network-client/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-network - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-network-client diff --git a/afs-network/afs-network-server/pom.xml b/afs-network/afs-network-server/pom.xml index f52d6d12..882bd734 100644 --- a/afs-network/afs-network-server/pom.xml +++ b/afs-network/afs-network-server/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-network - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-network-server diff --git a/afs-network/pom.xml b/afs-network/pom.xml index d46508ae..8b1a0325 100644 --- a/afs-network/pom.xml +++ b/afs-network/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-network diff --git a/afs-scripting/pom.xml b/afs-scripting/pom.xml index 414ba47e..1c543aba 100644 --- a/afs-scripting/pom.xml +++ b/afs-scripting/pom.xml @@ -13,7 +13,7 @@ powsybl-afs com.powsybl - 3.4.0 + 3.5.0-SNAPSHOT 4.0.0 diff --git a/afs-security-analysis-local/pom.xml b/afs-security-analysis-local/pom.xml index 047a5b75..d2cfbc77 100644 --- a/afs-security-analysis-local/pom.xml +++ b/afs-security-analysis-local/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-security-analysis-local diff --git a/afs-security-analysis/pom.xml b/afs-security-analysis/pom.xml index 9ee4086b..974717b1 100644 --- a/afs-security-analysis/pom.xml +++ b/afs-security-analysis/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-security-analysis diff --git a/afs-storage-api/pom.xml b/afs-storage-api/pom.xml index 2b02c2ba..a9ebe0c0 100644 --- a/afs-storage-api/pom.xml +++ b/afs-storage-api/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-storage-api diff --git a/afs-ws/afs-ws-client-utils/pom.xml b/afs-ws/afs-ws-client-utils/pom.xml index 11c5f8e8..ac5eef0f 100644 --- a/afs-ws/afs-ws-client-utils/pom.xml +++ b/afs-ws/afs-ws-client-utils/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-ws - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-ws-client-utils diff --git a/afs-ws/afs-ws-client/pom.xml b/afs-ws/afs-ws-client/pom.xml index a5e10ec2..f53bc5cc 100644 --- a/afs-ws/afs-ws-client/pom.xml +++ b/afs-ws/afs-ws-client/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-ws-client diff --git a/afs-ws/afs-ws-server-utils/pom.xml b/afs-ws/afs-ws-server-utils/pom.xml index 6bccccb0..fa6af326 100644 --- a/afs-ws/afs-ws-server-utils/pom.xml +++ b/afs-ws/afs-ws-server-utils/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-ws - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-ws-server-utils diff --git a/afs-ws/afs-ws-server/pom.xml b/afs-ws/afs-ws-server/pom.xml index 56a2b25a..35c19869 100644 --- a/afs-ws/afs-ws-server/pom.xml +++ b/afs-ws/afs-ws-server/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-ws-server diff --git a/afs-ws/afs-ws-storage/pom.xml b/afs-ws/afs-ws-storage/pom.xml index b889fb19..413815d8 100644 --- a/afs-ws/afs-ws-storage/pom.xml +++ b/afs-ws/afs-ws-storage/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-ws-storage diff --git a/afs-ws/afs-ws-utils/pom.xml b/afs-ws/afs-ws-utils/pom.xml index 97d5edee..c47d5288 100644 --- a/afs-ws/afs-ws-utils/pom.xml +++ b/afs-ws/afs-ws-utils/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-ws-utils diff --git a/afs-ws/pom.xml b/afs-ws/pom.xml index 23b1c26b..aa841fa2 100644 --- a/afs-ws/pom.xml +++ b/afs-ws/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.4.0 + 3.5.0-SNAPSHOT powsybl-afs-ws diff --git a/pom.xml b/pom.xml index 836cd53e..dd92bf74 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ powsybl-afs - 3.4.0 + 3.5.0-SNAPSHOT pom powsybl-afs From fe303a17bf963b36121b569ca5e4e78c79da4e55 Mon Sep 17 00:00:00 2001 From: Paul Bui-Quang Date: Fri, 15 Jan 2021 11:06:05 +0100 Subject: [PATCH 26/59] Remove ifnotexist when inserting node_data_name (#65) Now that the node_data_names are removed before any writing, the insert must always happen. Also the ifNotExists clause happen to malfunction (after the delete, the ifNotExists still picks up the data_name). Signed-off-by: Paul Bui-Quang Signed-off-by: Arthur Michaut --- .../main/java/com/powsybl/afs/cassandra/CassandraAppStorage.java | 1 - 1 file changed, 1 deletion(-) diff --git a/afs-cassandra/src/main/java/com/powsybl/afs/cassandra/CassandraAppStorage.java b/afs-cassandra/src/main/java/com/powsybl/afs/cassandra/CassandraAppStorage.java index eb5367e5..ca3fdb0c 100644 --- a/afs-cassandra/src/main/java/com/powsybl/afs/cassandra/CassandraAppStorage.java +++ b/afs-cassandra/src/main/java/com/powsybl/afs/cassandra/CassandraAppStorage.java @@ -1027,7 +1027,6 @@ public void close() { // update data names getSession().execute(insertInto(NODE_DATA_NAMES) - .ifNotExists() .value(ID, nodeUuid) .value(NAME, name)); } From 150b2df146465d0e1583fc9539a563f7370f3da9 Mon Sep 17 00:00:00 2001 From: Paul Bui-Quang Date: Mon, 8 Feb 2021 14:51:53 +0100 Subject: [PATCH 27/59] Use java 11 for sonar analysis (#70) Use java 11 for sonar analysis Signed-off-by: Paul Bui-Quang Signed-off-by: Arthur Michaut --- .github/workflows/maven.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 3cfb3bdd..dc9ed965 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -20,12 +20,22 @@ jobs: uses: actions/checkout@v1 - name: Build with Maven - run: mvn --batch-mode package + if: matrix.os == 'ubuntu-latest' + run: mvn --batch-mode -Pjacoco install + + - name: Build with Maven + if: matrix.os != 'ubuntu-latest' + run: mvn --batch-mode install + + - name: Set up JDK 11 for sonar analysis + uses: actions/setup-java@v1 + with: + java-version: 11 - name: Run SonarCloud analysis if: matrix.os == 'ubuntu-latest' run: > - mvn --batch-mode -Pjacoco verify sonar:sonar + mvn --batch-mode -DskipTests sonar:sonar -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=powsybl-ci-github -Dsonar.projectKey=com.powsybl:powsybl-afs From 97a8ffdcd1fb7bbe8df24dd93be0797c73c094eb Mon Sep 17 00:00:00 2001 From: Sylvain Leclerc Date: Thu, 18 Feb 2021 17:51:43 +0100 Subject: [PATCH 28/59] Springboot based implementation of AFS backend (#63) Adding a spring based implementation of AFS backend Co-authored-by: THIYAGARASA Pratheep Ext Signed-off-by: Sylvain Leclerc Signed-off-by: THIYAGARASA Pratheep Ext Signed-off-by: Arthur Michaut --- afs-distribution/pom.xml | 5 + afs-spring-server/pom.xml | 91 +++ .../powsybl/afs/server/AppDataWrapper.java | 55 ++ .../com/powsybl/afs/server/StorageServer.java | 611 ++++++++++++++++++ .../afs/server/StorageSwaggerConfig.java | 47 ++ .../afs/server/events/NodeEventForwarder.java | 49 ++ .../afs/server/events/NodeEventHandler.java | 93 +++ .../afs/server/events/SessionAttributes.java | 55 ++ .../afs/server/events/TaskEventHandler.java | 88 +++ .../afs/server/events/WebSocketConstants.java | 28 + .../afs/server/events/WebSocketContext.java | 48 ++ .../afs/server/events/WebSocketServer.java | 71 ++ .../afs/server/events/WebSocketUtils.java | 31 + .../com/powsybl/afs/server/io/GzipFilter.java | 103 +++ .../afs/server/io/GzipFilterRegistration.java | 29 + .../afs/server/io/GzippedRequestWrapper.java | 82 +++ .../afs/server/io/GzippedResponseWrapper.java | 127 ++++ .../powsybl/afs/server/io/JsonProvider.java | 14 + .../afs/server/AppDataWrapperTest.java | 42 ++ .../powsybl/afs/server/StorageServerTest.java | 122 ++++ .../server/events/TaskEventHandlerTest.java | 96 +++ pom.xml | 14 +- 22 files changed, 1898 insertions(+), 3 deletions(-) create mode 100644 afs-spring-server/pom.xml create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/AppDataWrapper.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/StorageServer.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/StorageSwaggerConfig.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/events/NodeEventForwarder.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/events/NodeEventHandler.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/events/SessionAttributes.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/events/TaskEventHandler.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketConstants.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketContext.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketServer.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketUtils.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzipFilter.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzipFilterRegistration.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzippedRequestWrapper.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzippedResponseWrapper.java create mode 100644 afs-spring-server/src/main/java/com/powsybl/afs/server/io/JsonProvider.java create mode 100644 afs-spring-server/src/test/java/com/powsybl/afs/server/AppDataWrapperTest.java create mode 100644 afs-spring-server/src/test/java/com/powsybl/afs/server/StorageServerTest.java create mode 100644 afs-spring-server/src/test/java/com/powsybl/afs/server/events/TaskEventHandlerTest.java diff --git a/afs-distribution/pom.xml b/afs-distribution/pom.xml index f03ef560..88e7a33a 100644 --- a/afs-distribution/pom.xml +++ b/afs-distribution/pom.xml @@ -88,6 +88,11 @@ powsybl-afs-security-analysis-local ${project.version} + + ${project.groupId} + powsybl-afs-spring-server + ${project.version} + ${project.groupId} powsybl-afs-storage-api diff --git a/afs-spring-server/pom.xml b/afs-spring-server/pom.xml new file mode 100644 index 00000000..0cd821f9 --- /dev/null +++ b/afs-spring-server/pom.xml @@ -0,0 +1,91 @@ + + + 4.0.0 + + + com.powsybl + powsybl-afs + 3.5.0-SNAPSHOT + + + powsybl-afs-spring-server + powsybl-afs-spring-server + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 1.4.1 + + + + + + + + + + + + org.springframework.boot + spring-boot-dependencies + ${springboot.version} + pom + import + + + + + + + + com.powsybl + powsybl-afs-core + ${project.version} + + + + io.springfox + springfox-swagger2 + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-websocket + + + + + com.powsybl + powsybl-afs-ws-client + ${project.version} + test + + + com.powsybl + powsybl-afs-storage-api + ${project.version} + test-jar + test + + + org.springframework.boot + spring-boot-starter-test + test + + + com.powsybl + powsybl-afs-mapdb + ${project.version} + test + + + + diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/AppDataWrapper.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/AppDataWrapper.java new file mode 100644 index 00000000..81c3ee2f --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/AppDataWrapper.java @@ -0,0 +1,55 @@ +/** + * Copyright (c) 2018, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server; + +import com.powsybl.afs.AppData; +import com.powsybl.afs.AppFileSystem; +import com.powsybl.afs.storage.AppStorage; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; +import org.springframework.web.server.ResponseStatusException; + +import java.util.Objects; + +/** + * Wrapper around {@link AppData} which provides additional checks around access to storage and filesystems. + * + * @author Geoffroy Jamgotchian + */ +@Component +public class AppDataWrapper { + + private final AppData appData; + + @Autowired + public AppDataWrapper(AppData appData) { + this.appData = appData; + } + + public AppData getAppData() { + return appData; + } + + public AppStorage getStorage(String fileSystemName) { + AppStorage storage = appData.getRemotelyAccessibleStorage(fileSystemName); + if (storage == null) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "App file system '" + fileSystemName + "' not found"); + } + return storage; + } + + public AppFileSystem getFileSystem(String name) { + Objects.requireNonNull(appData); + Objects.requireNonNull(name); + AppFileSystem fileSystem = appData.getFileSystem(name); + if (fileSystem == null) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "App file system '" + name + "' not found"); + } + return fileSystem; + } +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/StorageServer.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/StorageServer.java new file mode 100644 index 00000000..39ee0760 --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/StorageServer.java @@ -0,0 +1,611 @@ +/** + * Copyright (c) 2019, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server; + +import com.powsybl.afs.*; +import com.powsybl.afs.storage.*; +import com.powsybl.afs.storage.buffer.*; +import com.powsybl.timeseries.DoubleDataChunk; +import com.powsybl.timeseries.StringDataChunk; +import com.powsybl.timeseries.TimeSeriesMetadata; +import io.swagger.annotations.*; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.*; + +@RestController +@RequestMapping(value = "/rest/afs/" + StorageServer.API_VERSION) +@Api(value = "/afs", tags = "afs") +@ComponentScan(basePackageClasses = {AppDataWrapper.class, StorageServer.class}) +public class StorageServer { + + private static final Logger LOGGER = LoggerFactory.getLogger(StorageServer.class); + public static final String API_VERSION = "v1"; + + private final AppDataWrapper appDataWrapper; + + @Autowired + public StorageServer(AppDataWrapper appDataWrapper) { + this.appDataWrapper = appDataWrapper; + } + + @GetMapping(value = "fileSystems") + @ApiOperation(value = "Get file system list", response = List.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = "The list of available file systems"), @ApiResponse(code = 404, message = "There is no file system available.")}) + public List getFileSystemNames() { + return appDataWrapper.getAppData().getRemotelyAccessibleFileSystemNames(); + } + + @PutMapping(value = "fileSystems/{fileSystemName}/rootNode", produces = "application/json") + @ApiOperation(value = "Get file system root node and create it if not exist", response = NodeInfo.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = "The root node"), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity createRootNodeIfNotExists(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Root node name") @RequestParam("nodeName") String nodeName, + @ApiParam(value = "Root node pseudo class") @RequestParam("nodePseudoClass") String nodePseudoClass) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + NodeInfo rootNodeInfo = storage.createRootNodeIfNotExists(nodeName, nodePseudoClass); + return ok(rootNodeInfo); + } + + @PostMapping(value = "fileSystems/{fileSystemName}/flush", consumes = "application/json") + @ApiOperation(value = "") + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity flush(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Storage Change Set") @RequestBody StorageChangeSet changeSet) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + + for (StorageChange change : changeSet.getChanges()) { + switch (change.getType()) { + case TIME_SERIES_CREATION: + TimeSeriesCreation creation = (TimeSeriesCreation) change; + storage.createTimeSeries(creation.getNodeId(), creation.getMetadata()); + break; + case DOUBLE_TIME_SERIES_CHUNKS_ADDITION: + DoubleTimeSeriesChunksAddition doubleAddition = (DoubleTimeSeriesChunksAddition) change; + storage.addDoubleTimeSeriesData(doubleAddition.getNodeId(), doubleAddition.getVersion(), + doubleAddition.getTimeSeriesName(), doubleAddition.getChunks()); + break; + case STRING_TIME_SERIES_CHUNKS_ADDITION: + StringTimeSeriesChunksAddition stringAddition = (StringTimeSeriesChunksAddition) change; + storage.addStringTimeSeriesData(stringAddition.getNodeId(), stringAddition.getVersion(), + stringAddition.getTimeSeriesName(), stringAddition.getChunks()); + break; + default: + throw new AssertionError("Unknown change type " + change.getType()); + } + } + // propagate flush to underlying storage + storage.flush(); + return ok(); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/writable", produces = MediaType.TEXT_PLAIN_VALUE) + @ApiOperation(value = "", response = Boolean.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity isWritable(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + boolean writable = storage.isWritable(nodeId); + return ok(Boolean.toString(writable)); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/consistent", produces = MediaType.TEXT_PLAIN_VALUE) + @ApiOperation(value = "", response = Boolean.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity isConsistent(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + boolean consistent = storage.isConsistent(nodeId); + return ok(Boolean.toString(consistent)); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/parent", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "Get Parent Node", response = NodeInfo.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = "Returns the parent node"), @ApiResponse(code = 404, message = "No parent node for nodeId"), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity getParentNode(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + return okIfPresent(storage.getParentNode(nodeId)); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "", response = InputStream.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity getNodeInfo(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + NodeInfo nodeInfo = storage.getNodeInfo(nodeId); + return ok(nodeInfo); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/children", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "Get child nodes", response = List.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = "The list of chid nodes"), @ApiResponse(code = 404, message = "Thera are no child nodes"), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity> getChildNodes(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + List childNodes = storage.getChildNodes(nodeId); + return ok(childNodes); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/inconsistentChildNodes", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "Get inconsistent child nodes", response = List.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = "The list of inconsistent chid nodes"), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity> getInconsistentNodes(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + List childNodes = storage.getInconsistentNodes(); + return ok(childNodes); + } + + @PostMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/children/{childName}", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "Create Node", response = NodeInfo.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = "The node is created"), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity createNode(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Child Name") @PathVariable("childName") String childName, + @ApiParam(value = "Description") @RequestParam("description") String description, + @ApiParam(value = "Node Pseudo Class") @RequestParam("nodePseudoClass") String nodePseudoClass, + @ApiParam(value = "Version") @RequestParam("version") int version, + @ApiParam(value = "Node Meta Data") @RequestBody NodeGenericMetadata nodeMetadata) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + try { + NodeInfo newNodeInfo = storage.createNode(nodeId, childName, nodePseudoClass, description, version, nodeMetadata); + return ok(newNodeInfo); + } catch (AfsStorageException e) { + return ResponseEntity.status(HttpStatus.FORBIDDEN).body(null); + } + } + + @PutMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/name") + @ApiOperation(value = "Rename Node") + @ApiResponses(value = {@ApiResponse(code = 200, message = "The node is renamed"), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity renameNode(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "New node's name") @RequestBody String name) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + storage.renameNode(nodeId, name); + return ok(); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/children/{childName}", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "Get Child Node", response = NodeInfo.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = "Returns the child node"), @ApiResponse(code = 404, message = "No child node for nodeId"), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity getChildNode(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Child Name") @PathVariable("childName") String childName) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + return storage.getChildNode(nodeId, childName) + .map(StorageServer::ok) + .orElseGet(StorageServer::noContent); + } + + @PutMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/description", consumes = MediaType.TEXT_PLAIN_VALUE) + @ApiOperation(value = "") + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity setDescription(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Description") @RequestBody String description) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + storage.setDescription(nodeId, description); + return ok(); + } + + @PutMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/consistent", consumes = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "") + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity consistent(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + storage.setConsistent(nodeId); + return ok(); + } + + @PutMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/modificationTime") + @ApiOperation(value = "") + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity updateModificationTime(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + storage.updateModificationTime(nodeId); + return ok(); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/dependencies", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "", response = Set.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity> getDependencies(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + Set dependencies = storage.getDependencies(nodeId); + return ok(dependencies); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/backwardDependencies", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "", response = Set.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity> getBackwardDependencies(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + Set backwardDependencyNodes = storage.getBackwardDependencies(nodeId); + return ok(backwardDependencyNodes); + } + + @PutMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/data/{name}", consumes = MediaType.APPLICATION_OCTET_STREAM_VALUE) + @ApiOperation(value = "") + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity writeBinaryData(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Name") @PathVariable("name") String name, + @ApiParam(value = "Binary Data") InputStream is) throws IOException { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + try (OutputStream os = storage.writeBinaryData(nodeId, name)) { + if (os != null) { + IOUtils.copy(is, os); + } + return ok(); + } + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/data", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "", response = Set.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity> getDataNames(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + Set dataNames = storage.getDataNames(nodeId); + return ok(dataNames); + } + + @PutMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/dependencies/{name}/{toNodeId}", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "Add dependency to Node") + @ApiResponses(value = {@ApiResponse(code = 200, message = "Dependency is added"), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity addDependency(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Name") @PathVariable("name") String name, + @ApiParam(value = "To Node ID") @PathVariable("toNodeId") String toNodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + storage.addDependency(nodeId, name, toNodeId); + return ok(); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/dependencies/{name}", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "", response = Set.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity> getDependencies(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Name") @PathVariable("name") String name) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + Set dependencies = storage.getDependencies(nodeId, name); + return ok(dependencies); + } + + @DeleteMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/dependencies/{name}/{toNodeId}") + @ApiOperation(value = "") + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity removeDependency(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Name") @PathVariable("name") String name, + @ApiParam(value = "To Node ID") @PathVariable("toNodeId") String toNodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + storage.removeDependency(nodeId, name, toNodeId); + return ok(); + } + + @DeleteMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}", produces = MediaType.TEXT_PLAIN_VALUE) + @ApiOperation(value = "") + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity deleteNode(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + String parentNodeId = storage.deleteNode(nodeId); + return ok(parentNodeId); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/data/{name}", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) + @ApiOperation (value = "", response = InputStream.class) + @ApiResponses (value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity readBinaryAttribute(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Name") @PathVariable("name") String name) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + return okIfPresent(storage.readBinaryData(nodeId, name).map(StorageServer::copyToBodyAndClose)); + } + + private static StreamingResponseBody copyToBodyAndClose(InputStream inputStream) { + return outputStream -> { + try (InputStream toClose = inputStream) { + IOUtils.copy(toClose, outputStream); + } + }; + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/data/{name}", produces = MediaType.TEXT_PLAIN_VALUE) + @ApiOperation(value = "", response = String.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity dataExists(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Name") @PathVariable("name") String name) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + boolean exists = storage.dataExists(nodeId, name); + return ok(Boolean.toString(exists)); + } + + @DeleteMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/data/{name}", produces = MediaType.TEXT_PLAIN_VALUE) + @ApiOperation(value = "", response = Boolean.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity removeData(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Data name") @PathVariable("name") String name) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + boolean removed = storage.removeData(nodeId, name); + return ok(Boolean.toString(removed)); + } + + @PostMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/timeSeries", consumes = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "") + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity createTimeSeries(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Time Series Meta Data") TimeSeriesMetadata metadata) { + + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + storage.createTimeSeries(nodeId, metadata); + return ok(); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/timeSeries/name", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "", response = Set.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity> getTimeSeriesNames(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + Set timeSeriesNames = storage.getTimeSeriesNames(nodeId); + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_ENCODING, "gzip") + .body(timeSeriesNames); + + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/timeSeries/{timeSeriesName}", produces = MediaType.TEXT_PLAIN_VALUE) + @ApiOperation(value = "", response = Boolean.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity timeSeriesExists(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Time series name") @PathVariable("timeSeriesName") String timeSeriesName) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + boolean exists = storage.timeSeriesExists(nodeId, timeSeriesName); + return ok(Boolean.toString(exists)); + } + + @PostMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/timeSeries/metadata", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "", response = List.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity> getTimeSeriesMetadata(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Time series names") @RequestBody Set timeSeriesNames) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + List metadataList = storage.getTimeSeriesMetadata(nodeId, timeSeriesNames); + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_ENCODING, "gzip") + .body(metadataList); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/timeSeries/versions", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> getTimeSeriesDataVersions(@PathVariable("fileSystemName") String fileSystemName, + @PathVariable("nodeId") String nodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + Set versions = storage.getTimeSeriesDataVersions(nodeId); + return ok(versions); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/timeSeries/{timeSeriesName}/versions", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "", response = Set.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity> getTimeSeriesDataVersions(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Time series name") @PathVariable("timeSeriesName") String timeSeriesName) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + Set versions = storage.getTimeSeriesDataVersions(nodeId, timeSeriesName); + return ok(versions); + } + + @PostMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/timeSeries/double/{version}", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "", response = List.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity>> getDoubleTimeSeriesData(@PathVariable("fileSystemName") String fileSystemName, + @PathVariable("nodeId") String nodeId, + @PathVariable("version") int version, + @RequestBody Set timeSeriesNames) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + Map> timeSeriesData = storage.getDoubleTimeSeriesData(nodeId, timeSeriesNames, version); + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_ENCODING, "gzip") + .body(timeSeriesData); + } + + @PostMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/timeSeries/string/{version}", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "", response = List.class) + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity>> getStringTimeSeriesData(@PathVariable("fileSystemName") String fileSystemName, + @PathVariable("nodeId") String nodeId, + @PathVariable("version") int version, + @RequestBody Set timeSeriesNames) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + Map> timeSeriesData = storage.getStringTimeSeriesData(nodeId, timeSeriesNames, version); + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_ENCODING, "gzip") + .body(timeSeriesData); + } + + @DeleteMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/timeSeries") + @ApiOperation(value = "") + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity clearTimeSeries(@PathVariable("fileSystemName") String fileSystemName, + @PathVariable("nodeId") String nodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + storage.clearTimeSeries(nodeId); + return ok(); + } + + @PutMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/parent", consumes = MediaType.TEXT_PLAIN_VALUE) + @ApiOperation(value = "") + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity setParentNode(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "New Parent Node ID") @RequestBody String newParentNodeId) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + storage.setParentNode(nodeId, newParentNodeId); + return ok(); + } + + @GetMapping(value = "fileSystems/{fileSystemName}/tasks", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity takeSnapshot(@PathVariable("fileSystemName") String fileSystemName, + @RequestParam("projectId") String projectId) { + AppFileSystem fileSystem = appDataWrapper.getFileSystem(fileSystemName); + TaskMonitor.Snapshot snapshot = fileSystem.getTaskMonitor().takeSnapshot(projectId); + return ok(snapshot); + } + + @PutMapping(value = "fileSystems/{fileSystemName}/tasks", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "Start a task") + @ApiResponses(value = {@ApiResponse(code = 200, message = "", response = TaskMonitor.Task.class), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity startTask(@PathVariable("fileSystemName") String fileSystemName, + @RequestParam(value = "projectFileId", required = false) String projectFileId, + @RequestParam(value = "projectId", required = false) String projectId, + @RequestParam(value = "name", required = false) String name) { + logInfo("Starting task {} for node {} ({})", name, projectFileId, projectId); + AppFileSystem fileSystem = appDataWrapper.getFileSystem(fileSystemName); + TaskMonitor.Task task; + if (projectFileId != null) { + ProjectFile projectFile = fileSystem.findProjectFile(projectFileId, ProjectFile.class); + if (projectFile == null) { + throw new AfsException("Project file '" + projectFileId + "' not found in file system '" + fileSystemName + "'"); + } + task = fileSystem.getTaskMonitor().startTask(projectFile); + } else if (projectId != null && name != null) { + Project project = fileSystem + .findProject(projectId) + .orElseThrow(() -> new AfsException("Project '" + projectId + "' not found in file system '" + fileSystemName + "'")); + task = fileSystem.getTaskMonitor().startTask(name, project); + } else { + throw new AfsException("Missing arguments"); + } + return ResponseEntity.ok(task); + } + + @PostMapping(value = "fileSystems/{fileSystemName}/tasks/{taskId}", consumes = MediaType.TEXT_PLAIN_VALUE) + public ResponseEntity updateTaskMessage(@PathVariable("fileSystemName") String fileSystemName, + @PathVariable("taskId") UUID taskId, + @RequestBody String message) { + AppFileSystem fileSystem = appDataWrapper.getFileSystem(fileSystemName); + fileSystem.getTaskMonitor().updateTaskMessage(taskId, message); + return ok(); + } + + @DeleteMapping(value = "fileSystems/{fileSystemName}/tasks/{taskId}") + public ResponseEntity stopTask(@PathVariable("fileSystemName") String fileSystemName, + @PathVariable("taskId") UUID taskId) { + AppFileSystem fileSystem = appDataWrapper.getFileSystem(fileSystemName); + fileSystem.getTaskMonitor().stopTask(taskId); + return ok(); + } + + @PutMapping(value = "fileSystems/{fileSystemName}/tasks/{taskId}/_cancel", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "Cancel a task tracked process") + @ApiResponses(value = {@ApiResponse(code = 200, message = "", response = Boolean.class), @ApiResponse(code = 404, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity cancel(@PathVariable("fileSystemName") String fileSystemName, + @PathVariable("taskId") String taskId) { + logInfo("Canceling task {}", taskId); + AppFileSystem fileSystem = appDataWrapper.getFileSystem(fileSystemName); + boolean success = fileSystem.getTaskMonitor().cancelTaskComputation(UUID.fromString(taskId)); + return ResponseEntity.ok(success); + } + + @PostMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/timeSeries/double/{version}/{timeSeriesName}", consumes = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "") + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity addDoubleTimeSeriesData(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Version") @PathVariable("version") int version, + @ApiParam(value = "Time series name") @PathVariable("timeSeriesName") String timeSeriesName, + @ApiParam(value = "List double array chunk") @RequestBody List chunks) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + storage.addDoubleTimeSeriesData(nodeId, version, timeSeriesName, chunks); + return ok(); + } + + @PostMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/timeSeries/string/{version}/{timeSeriesName}", consumes = MediaType.APPLICATION_OCTET_STREAM_VALUE) + @ApiOperation(value = "") + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity addStringTimeSeriesData(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Version") @PathVariable("version") int version, + @ApiParam(value = "Time Series Name") @PathVariable("timeSeriesName") String timeSeriesName, + @ApiParam(value = "List string array chunkFile system name") @RequestBody List chunks) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + storage.addStringTimeSeriesData(nodeId, version, timeSeriesName, chunks); + return ok(); + } + + @PutMapping(value = "fileSystems/{fileSystemName}/nodes/{nodeId}/metadata", produces = MediaType.APPLICATION_JSON_VALUE) + @ApiOperation(value = "Update node's metadata") + @ApiResponses(value = {@ApiResponse(code = 200, message = ""), @ApiResponse(code = 500, message = "Error")}) + public ResponseEntity setMetadata(@ApiParam(value = "File system name") @PathVariable("fileSystemName") String fileSystemName, + @ApiParam(value = "Node ID") @PathVariable("nodeId") String nodeId, + @ApiParam(value = "Node Meta Data") @RequestBody NodeGenericMetadata nodeMetadata) { + AppStorage storage = appDataWrapper.getStorage(fileSystemName); + storage.setMetadata(nodeId, nodeMetadata); + return ok(); + } + + private static ResponseEntity ok() { + return ResponseEntity.ok().build(); + } + + private static ResponseEntity noContent() { + return ResponseEntity.noContent().build(); + } + + private static ResponseEntity ok(T body) { + return ResponseEntity.ok(body); + } + + private static ResponseEntity okIfPresent(Optional body) { + return body + .map(StorageServer::ok) + .orElseGet(StorageServer::noContent); + } + + private static void logInfo(String message, Object... params) { + if (LOGGER.isInfoEnabled()) { + Object[] objects = Arrays.stream(params) + .map(StorageServer::encode) + .toArray(); + LOGGER.info(message, objects); + } + } + + private static Object encode(Object input) { + if (input instanceof String) { + return ((String) input).replaceAll("[\n\r\t]", "_"); + } + return input; + } +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/StorageSwaggerConfig.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/StorageSwaggerConfig.java new file mode 100644 index 00000000..158f9a5b --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/StorageSwaggerConfig.java @@ -0,0 +1,47 @@ +/** + * Copyright (c) 2019, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import springfox.documentation.builders.ApiInfoBuilder; +import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.service.ApiInfo; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spring.web.plugins.Docket; +import springfox.documentation.swagger2.annotations.EnableSwagger2WebMvc; + +import java.util.function.Predicate; + +@Configuration +@EnableSwagger2WebMvc +public class StorageSwaggerConfig { + @Bean + public Docket produceApi() { + return new Docket(DocumentationType.SWAGGER_2) + .apiInfo(apiInfo()) + .select() + .apis(RequestHandlerSelectors.basePackage(StorageServer.class.getPackage().getName())) + .paths(paths()) + .build(); + } + + // Describe your apis + private ApiInfo apiInfo() { + return new ApiInfoBuilder() + .title("AFS storage API") + .description("This is the documentation of AFS storage REST API") + .version(StorageServer.API_VERSION) + .build(); + } + + // Only select apis that matches the given Predicates. + private Predicate paths() { + // Match all paths except /error + return input -> input.matches("/rest/afs/" + StorageServer.API_VERSION + ".*") && !input.matches("/error.*"); + } +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/events/NodeEventForwarder.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/NodeEventForwarder.java new file mode 100644 index 00000000..77ec1157 --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/NodeEventForwarder.java @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2020, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server.events; + +import com.fasterxml.jackson.databind.ObjectWriter; +import com.powsybl.afs.storage.events.AppStorageListener; +import com.powsybl.afs.storage.events.NodeEventList; +import com.powsybl.commons.json.JsonUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.socket.TextMessage; +import org.springframework.web.socket.WebSocketSession; + +/** + * A listener which forwards events to a websocket client. + * + * @author Sylvain Leclerc + */ +public class NodeEventForwarder implements AppStorageListener { + + private static final Logger LOGGER = LoggerFactory.getLogger(NodeEventForwarder.class); + + private static final ObjectWriter NODE_EVENT_WRITER = JsonUtil.createObjectMapper().writerFor(NodeEventList.class); + + private final WebSocketSession session; + + NodeEventForwarder(WebSocketSession session) { + this.session = WebSocketUtils.concurrent(session); + } + + @Override + public void onEvents(NodeEventList eventList) { + if (session.isOpen()) { + try { + String eventListEncode = NODE_EVENT_WRITER.writeValueAsString(eventList); + session.sendMessage(new TextMessage(eventListEncode)); + } catch (Exception e) { + LOGGER.error("Failed to send events {} to {}:", eventList, session.getRemoteAddress(), e); + } + } else { + LOGGER.error("Could not send events {} to client {}: session is closed.", + eventList, session.getRemoteAddress()); + } + } +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/events/NodeEventHandler.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/NodeEventHandler.java new file mode 100644 index 00000000..5eab7af3 --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/NodeEventHandler.java @@ -0,0 +1,93 @@ +/** + * Copyright (c) 2019, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server.events; + +import com.fasterxml.jackson.databind.ObjectReader; +import com.powsybl.afs.server.AppDataWrapper; +import com.powsybl.afs.storage.AppStorage; +import com.powsybl.afs.storage.events.AppStorageListener; +import com.powsybl.afs.storage.events.NodeEventContainer; +import com.powsybl.commons.json.JsonUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.socket.CloseStatus; +import org.springframework.web.socket.TextMessage; +import org.springframework.web.socket.WebSocketSession; +import org.springframework.web.socket.handler.TextWebSocketHandler; + +import java.io.IOException; +import java.io.UncheckedIOException; + +/** + * Web socket handler for node events : + *
    + *
  • On connection, registers a listener to forward events to the client
    • + *
  • On receiving events from the client, forwards it to underlying event bus
    • + *
+ * + * @author Sylvain Leclerc + */ +public class NodeEventHandler extends TextWebSocketHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(NodeEventHandler.class); + + private static final ObjectReader NODE_EVENT_READER = JsonUtil.createObjectMapper() + .readerFor(NodeEventContainer.class); + + private final AppDataWrapper appDataWrapper; + private final WebSocketContext webSocketContext; + + public NodeEventHandler(AppDataWrapper appDataWrapper, WebSocketContext webSocketContext) { + this.appDataWrapper = appDataWrapper; + this.webSocketContext = webSocketContext; + } + + @Override + public void afterConnectionEstablished(WebSocketSession session) { + String fileSystemName = SessionAttributes.of(session).getFilesystem(); + LOGGER.debug("WebSocket session '{}' opened for file system '{}'", session.getId(), fileSystemName); + + registerListener(session); + } + + private void registerListener(WebSocketSession session) { + AppStorageListener eventForwarder = new NodeEventForwarder(session); + appDataWrapper.getAppData().getEventsBus().addListener(eventForwarder); + SessionAttributes.of(session).setListener(eventForwarder); + webSocketContext.addSession(session); + } + + @Override + protected void handleTextMessage(WebSocketSession session, TextMessage message) { + LOGGER.trace("Node event websocket session '' of type '{}' id: ", + session.getId()); + + forwardEventsToBus(message); + } + + private void forwardEventsToBus(TextMessage message) { + try { + NodeEventContainer container = NODE_EVENT_READER.readValue(message.getPayload()); + AppStorage storage = appDataWrapper.getStorage(container.getFileSystemName()); + storage.getEventsBus().pushEvent(container.getNodeEvent(), container.getTopic()); + storage.getEventsBus().flush(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public void afterConnectionClosed(WebSocketSession session, CloseStatus status) { + removeSession(session); + } + + private void removeSession(WebSocketSession session) { + AppStorageListener listener = SessionAttributes.of(session).getListener(); + appDataWrapper.getAppData().getEventsBus().removeListener(listener); + webSocketContext.removeSession(session); + } +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/events/SessionAttributes.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/SessionAttributes.java new file mode 100644 index 00000000..56ca98b8 --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/SessionAttributes.java @@ -0,0 +1,55 @@ +/** + * Copyright (c) 2020, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server.events; + +import com.powsybl.afs.TaskListener; +import com.powsybl.afs.storage.events.AppStorageListener; +import org.springframework.web.socket.WebSocketSession; + +import java.util.Map; + +/** + * Utility to get/set session attributes. + * + * @author Sylvain Leclerc + */ +public class SessionAttributes { + + private static final String LISTENER_ATTRIBUTE = "listener"; + private static final String TASK_LISTENER_ATTRIBUTE = "taskListener"; + private static final String FILESYSTEM_ATTRIBUTE = "fileSystemName"; + + private final Map attributes; + + public SessionAttributes(WebSocketSession session) { + this.attributes = session.getAttributes(); + } + + public static SessionAttributes of(WebSocketSession session) { + return new SessionAttributes(session); + } + + public void setListener(AppStorageListener listener) { + attributes.put(LISTENER_ATTRIBUTE, listener); + } + + public AppStorageListener getListener() { + return (AppStorageListener) attributes.get(LISTENER_ATTRIBUTE); + } + + public void setTaskListener(TaskListener listener) { + attributes.put(TASK_LISTENER_ATTRIBUTE, listener); + } + + public TaskListener getTaskListener() { + return (TaskListener) attributes.get(TASK_LISTENER_ATTRIBUTE); + } + + public String getFilesystem() { + return (String) attributes.get(FILESYSTEM_ATTRIBUTE); + } +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/events/TaskEventHandler.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/TaskEventHandler.java new file mode 100644 index 00000000..b51dbd98 --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/TaskEventHandler.java @@ -0,0 +1,88 @@ +/** + * Copyright (c) 2019, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server.events; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.powsybl.afs.AppFileSystem; +import com.powsybl.afs.TaskEvent; +import com.powsybl.afs.TaskListener; +import com.powsybl.afs.server.AppDataWrapper; +import com.powsybl.commons.json.JsonUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.socket.CloseStatus; +import org.springframework.web.socket.TextMessage; +import org.springframework.web.socket.WebSocketSession; +import org.springframework.web.socket.handler.TextWebSocketHandler; + +import java.io.IOException; +import java.io.UncheckedIOException; + +public class TaskEventHandler extends TextWebSocketHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(TaskEventHandler.class); + + private final AppDataWrapper appDataWrapper; + private final WebSocketContext webSocketContext; + + private final ObjectMapper objectMapper = JsonUtil.createObjectMapper(); + + public TaskEventHandler(AppDataWrapper appDataWrapper, WebSocketContext webSocketContext) { + this.appDataWrapper = appDataWrapper; + this.webSocketContext = webSocketContext; + } + + @Override + public void afterConnectionEstablished(WebSocketSession session) throws Exception { + + String fileSystemName = SessionAttributes.of(session).getFilesystem(); + AppFileSystem fileSystem = appDataWrapper.getFileSystem(fileSystemName); + String projectId = session.getAttributes().get("projectId").toString(); + + LOGGER.debug("WebSocket session '{}' opened for file system '{}'", session.getId(), fileSystemName); + + TaskListener listener = new TaskListener() { + + private final WebSocketSession internalSession = WebSocketUtils.concurrent(session); + + @Override + public String getProjectId() { + return projectId; + } + + @Override + public void onEvent(TaskEvent event) { + if (internalSession.isOpen()) { + try { + String taskEventEncode = objectMapper.writeValueAsString(event); + internalSession.sendMessage(new TextMessage(taskEventEncode)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + }; + + SessionAttributes.of(session).setTaskListener(listener); + fileSystem.getTaskMonitor().addListener(listener); + + webSocketContext.addSession(session); + } + + @Override + public void afterConnectionClosed(WebSocketSession session, CloseStatus status) { + removeSession(session); + } + + private void removeSession(WebSocketSession session) { + SessionAttributes attrs = SessionAttributes.of(session); + AppFileSystem fileSystem = appDataWrapper.getFileSystem(attrs.getFilesystem()); + + fileSystem.getTaskMonitor().removeListener(attrs.getTaskListener()); + webSocketContext.removeSession(session); + } +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketConstants.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketConstants.java new file mode 100644 index 00000000..a8d2543f --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketConstants.java @@ -0,0 +1,28 @@ +/** + * Copyright (c) 2020, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server.events; + +/** + * Constants for web sockets. + * + * @author Sylvain Leclerc + */ +public final class WebSocketConstants { + + private WebSocketConstants() { + } + + /** + * Sends will timeout after 10s + */ + public static final int SEND_TIMEOUT_MILLIS = 10 * 1000; + + /** + * 1M buffer size + */ + public static final int SEND_BUFFER_SIZE = 1024 * 1024; +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketContext.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketContext.java new file mode 100644 index 00000000..29d8c040 --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketContext.java @@ -0,0 +1,48 @@ +/** + * Copyright (c) 2018, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server.events; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.CloseStatus; +import org.springframework.web.socket.WebSocketSession; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author Geoffroy Jamgotchian + */ +@Component +public class WebSocketContext implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(WebSocketContext.class); + + private final Set sessions = new HashSet<>(); + + public synchronized void addSession(WebSocketSession session) { + sessions.add(session); + } + + public synchronized void removeSession(WebSocketSession session) { + sessions.remove(session); + } + + @Override + public synchronized void close() { + for (WebSocketSession session : sessions) { + try { + session.close(CloseStatus.SERVER_ERROR); + } catch (Exception e) { + LOGGER.error("An error occurred while closing web socket sessions", e); + } + } + sessions.clear(); + } + +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketServer.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketServer.java new file mode 100644 index 00000000..02eac871 --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketServer.java @@ -0,0 +1,71 @@ +/** + * Copyright (c) 2019, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server.events; + +import com.powsybl.afs.server.AppDataWrapper; +import com.powsybl.afs.server.StorageServer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.server.ServerHttpRequest; +import org.springframework.http.server.ServerHttpResponse; +import org.springframework.http.server.ServletServerHttpRequest; +import org.springframework.web.servlet.HandlerMapping; +import org.springframework.web.socket.WebSocketHandler; +import org.springframework.web.socket.config.annotation.EnableWebSocket; +import org.springframework.web.socket.config.annotation.WebSocketConfigurer; +import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; +import org.springframework.web.socket.server.HandshakeInterceptor; + +import javax.servlet.http.HttpServletRequest; +import java.util.Map; +import java.util.Objects; + +@Configuration +@EnableWebSocket +public class WebSocketServer implements WebSocketConfigurer { + + private static final Logger LOGGER = LoggerFactory.getLogger(WebSocketServer.class); + + private final AppDataWrapper appDataWrapper; + private final WebSocketContext webSocketContext; + + @Autowired + public WebSocketServer(AppDataWrapper appDataWrapper, + WebSocketContext webSocketContext) { + this.appDataWrapper = Objects.requireNonNull(appDataWrapper); + this.webSocketContext = Objects.requireNonNull(webSocketContext); + } + + @Override + public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { + registry + .addHandler(new NodeEventHandler(appDataWrapper, webSocketContext), "/messages/afs/" + StorageServer.API_VERSION + "/node_events/{fileSystemName}") + .addHandler(new TaskEventHandler(appDataWrapper, webSocketContext), "/messages/afs/" + StorageServer.API_VERSION + "/task_events/{fileSystemName}/{projectId}") + .setAllowedOrigins("*") + .addInterceptors(new UriTemplateHandshakeInterceptor()); + } + + private static class UriTemplateHandshakeInterceptor implements HandshakeInterceptor { + @Override + public boolean beforeHandshake(ServerHttpRequest request, ServerHttpResponse response, WebSocketHandler wsHandler, Map attributes) throws Exception { + HttpServletRequest origRequest = ((ServletServerHttpRequest) request).getServletRequest(); + /* Retrieve template variables */ + Map uriTemplateVars = (Map) origRequest.getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE); + /* Put template variables into WebSocket session attributes */ + if (uriTemplateVars != null) { + attributes.putAll(uriTemplateVars); + } + return true; + } + + @Override + public void afterHandshake(ServerHttpRequest request, ServerHttpResponse response, WebSocketHandler wsHandler, Exception exception) { + } + } +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketUtils.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketUtils.java new file mode 100644 index 00000000..418f5340 --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/events/WebSocketUtils.java @@ -0,0 +1,31 @@ +/** + * Copyright (c) 2020, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server.events; + +import org.springframework.web.socket.WebSocketSession; +import org.springframework.web.socket.handler.ConcurrentWebSocketSessionDecorator; + +/** + * Utilities for web sockets. + * + * @author Sylvain Leclerc + */ +public final class WebSocketUtils { + + private WebSocketUtils() { + } + + /** + * Wraps the provided session into a thread safe session. + */ + public static WebSocketSession concurrent(WebSocketSession session) { + return new ConcurrentWebSocketSessionDecorator(session, + WebSocketConstants.SEND_TIMEOUT_MILLIS, + WebSocketConstants.SEND_BUFFER_SIZE, + ConcurrentWebSocketSessionDecorator.OverflowStrategy.DROP); + } +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzipFilter.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzipFilter.java new file mode 100644 index 00000000..3b4cd2b7 --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzipFilter.java @@ -0,0 +1,103 @@ +/** + * Copyright (c) 2019, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server.io; + +import org.springframework.http.HttpHeaders; + +import javax.servlet.*; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * An http servlet filter which tries to handle automatic gzip compression and decompression: + *
    + *
  • for POST request with gzip content-encoding, wrap the request to gunzip its content
    • + *
  • Wraps responses : responses with gzip content-encoding will get their output stream wrapped in a gzip output stream
    • + *
+ * + * The wrapping of response does not work very well: + * the gzip output stream needs to be closed, but apparently spring never closes it. + * Therefore, this filter closes the stream after the filter chain, but it will break + * the streaming of responses ({@link org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody}. + * This filter must not be activated on such endpoints. + * + * @author Sylvain Leclerc + */ +public class GzipFilter implements Filter { + + @Override + public final void doFilter(final ServletRequest servletRequest, + final ServletResponse servletResponse, + final FilterChain chain) throws IOException, ServletException { + + HttpServletRequest request = (HttpServletRequest) servletRequest; + HttpServletResponse response = (HttpServletResponse) servletResponse; + + String requestedContentEncoding = request.getHeader(HttpHeaders.CONTENT_ENCODING); + boolean isGzipped = requestedContentEncoding != null && requestedContentEncoding.contains("gzip"); + + boolean requestTypeSupported = "POST".equals(request.getMethod()) || "PUT".equals(request.getMethod()); + if (isGzipped && !requestTypeSupported) { + throw new IllegalStateException(request.getMethod() + + " is not supports gzipped body of parameters." + + " Only POST requests are currently supported."); + } + if (isGzipped) { + request = new GzippedRequestWrapper((HttpServletRequest) servletRequest); + } + GzippedResponseWrapper gzipResponse = new GzippedResponseWrapper(response); + chain.doFilter(request, gzipResponse); + + //This seems necessary because otherwise spring does not close the output stream, + //and therefore the gzip data is invalid. + applyAfterCompletion(request, gzipResponse::finish); + } + + @FunctionalInterface + private interface IORunnable { + void run() throws IOException; + } + + /** + * Apply a method after the completion of the request, be it synchronous or asynchronous. + */ + private void applyAfterCompletion(HttpServletRequest request, IORunnable runnable) throws IOException { + if (request.isAsyncStarted()) { + request.getAsyncContext().addListener(new AsyncListener() { + @Override + public void onComplete(AsyncEvent asyncEvent) throws IOException { + runnable.run(); + } + + @Override + public void onTimeout(AsyncEvent asyncEvent) throws IOException { + runnable.run(); + } + + @Override + public void onError(AsyncEvent asyncEvent) throws IOException { + runnable.run(); + } + + @Override + public void onStartAsync(AsyncEvent asyncEvent) throws IOException { + } + }); + } else { + runnable.run(); + } + } + + @Override + public void init(FilterConfig filterConfig) { + } + + @Override + public void destroy() { + } +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzipFilterRegistration.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzipFilterRegistration.java new file mode 100644 index 00000000..638d206d --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzipFilterRegistration.java @@ -0,0 +1,29 @@ +/** + * Copyright (c) 2020, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server.io; + +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Registers the {@link GzipFilter} only for AFS endpoints. + * + * @author Sylvain Leclerc + */ +@Configuration +public class GzipFilterRegistration { + + @Bean + public FilterRegistrationBean registerGzipFilter() { + FilterRegistrationBean registration = new FilterRegistrationBean<>(); + registration.setFilter(new GzipFilter()); + registration.addUrlPatterns("/rest/afs/v1/*"); + registration.setAsyncSupported(true); + return registration; + } +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzippedRequestWrapper.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzippedRequestWrapper.java new file mode 100644 index 00000000..0600c449 --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzippedRequestWrapper.java @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2019, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server.io; + +import javax.servlet.ReadListener; +import javax.servlet.ServletInputStream; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; +import java.io.IOException; +import java.util.zip.GZIPInputStream; + +/** + * Wraps a request to gunzip its content. + * + * @author Sylvain Leclerc + */ +public class GzippedRequestWrapper extends HttpServletRequestWrapper { + + private ServletInputStream in; + + public GzippedRequestWrapper(final HttpServletRequest request) { + super(request); + } + + @Override + public ServletInputStream getInputStream() throws IOException { + if (in == null) { + in = new GzippedServletInputStream(getRequest().getInputStream()); + } + return in; + } + + private static final class GzippedServletInputStream extends ServletInputStream { + + private final ServletInputStream delegate; + private final GZIPInputStream gzipIn; + + private GzippedServletInputStream(ServletInputStream delegate) throws IOException { + this.delegate = delegate; + this.gzipIn = new GZIPInputStream(delegate); + } + + @Override + public int read() throws IOException { + return gzipIn.read(); + } + + @Override + public int read(byte[] b) throws IOException { + return gzipIn.read(b); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return gzipIn.read(b, off, len); + } + + @Override + public void close() throws IOException { + gzipIn.close(); + } + + @Override + public boolean isFinished() { + return delegate.isFinished(); + } + + @Override + public boolean isReady() { + return delegate.isReady(); + } + + @Override + public void setReadListener(ReadListener listener) { + delegate.setReadListener(listener); + } + } +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzippedResponseWrapper.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzippedResponseWrapper.java new file mode 100644 index 00000000..88eb8ded --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/io/GzippedResponseWrapper.java @@ -0,0 +1,127 @@ +/** + * Copyright (c) 2019, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server.io; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.util.Objects; +import java.util.zip.GZIPOutputStream; + +import javax.servlet.ServletOutputStream; +import javax.servlet.WriteListener; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServletResponseWrapper; + +import org.springframework.http.HttpHeaders; + +/** + * Wraps a response to gzip its content if its content-encoding is set to "gzip". + * + * @author Sylvain Leclerc + */ +public class GzippedResponseWrapper extends HttpServletResponseWrapper { + + private ServletOutputStream servletOuput; + private PrintWriter printerWriter; + + public GzippedResponseWrapper(HttpServletResponse response) { + super(response); + } + + /** + * Closes the underlying output streams / writers. + * This is necessary to get a valid gzipped body. + */ + public void finish() throws IOException { + if (printerWriter != null) { + printerWriter.close(); + } + if (servletOuput != null) { + servletOuput.close(); + } + } + + @Override + public void flushBuffer() throws IOException { + if (printerWriter != null) { + printerWriter.flush(); + } + if (servletOuput != null) { + servletOuput.flush(); + } + super.flushBuffer(); + } + + @Override + public ServletOutputStream getOutputStream() throws IOException { + if (servletOuput == null) { + if (super.getHeader(HttpHeaders.CONTENT_ENCODING) != null && super.getHeader(HttpHeaders.CONTENT_ENCODING).equals("gzip")) { + servletOuput = new GzipResponseStream(getResponse().getOutputStream()); + } else { + servletOuput = getResponse().getOutputStream(); + } + } + return servletOuput; + } + + @Override + public PrintWriter getWriter() throws IOException { + if (printerWriter == null) { + printerWriter = new PrintWriter(new OutputStreamWriter(getOutputStream())); + } + return printerWriter; + } + + static class GzipResponseStream extends ServletOutputStream { + + private final ServletOutputStream delegate; + private final GZIPOutputStream gzOut; + + public GzipResponseStream(ServletOutputStream delegate) throws IOException { + this.delegate = Objects.requireNonNull(delegate); + this.gzOut = new GZIPOutputStream(delegate); + } + + @Override + public boolean isReady() { + return delegate.isReady(); + } + + @Override + public void setWriteListener(WriteListener listener) { + delegate.setWriteListener(listener); + } + + @Override + public void flush() throws IOException { + gzOut.flush(); + delegate.flush(); + } + + @Override + public void close() throws IOException { + gzOut.close(); + delegate.close(); + } + + @Override + public void write(int b) throws IOException { + gzOut.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + gzOut.write(b, off, len); + } + + @Override + public void write(byte[] b) throws IOException { + gzOut.write(b); + } + } +} diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/io/JsonProvider.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/io/JsonProvider.java new file mode 100644 index 00000000..15818b10 --- /dev/null +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/io/JsonProvider.java @@ -0,0 +1,14 @@ +/** + * Copyright (c) 2019, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.afs.server.io; + +import com.powsybl.afs.storage.json.AppStorageJsonModule; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class JsonProvider extends AppStorageJsonModule { +} diff --git a/afs-spring-server/src/test/java/com/powsybl/afs/server/AppDataWrapperTest.java b/afs-spring-server/src/test/java/com/powsybl/afs/server/AppDataWrapperTest.java new file mode 100644 index 00000000..901a3c15 --- /dev/null +++ b/afs-spring-server/src/test/java/com/powsybl/afs/server/AppDataWrapperTest.java @@ -0,0 +1,42 @@ +package com.powsybl.afs.server; + +import com.powsybl.afs.AppData; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.web.server.ResponseStatusException; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * + * @author THIYAGARASA Pratheep Ext + */ +@RunWith(MockitoJUnitRunner.class) +public class AppDataWrapperTest { + + @InjectMocks + private AppDataWrapper appDataWrapper; + + @Mock + private AppData appData; + + @Test + public void failedToGetStorage() { + Mockito.when(appData.getRemotelyAccessibleStorage("fileSystem")).thenReturn(null); + assertThatThrownBy(() -> appDataWrapper.getStorage("fileSystem")) + .isInstanceOf(ResponseStatusException.class) + .hasMessage("404 NOT_FOUND \"App file system 'fileSystem' not found\""); + } + + @Test + public void failedToGetFileSystem() { + Mockito.when(appData.getFileSystem("fileSystem")).thenReturn(null); + assertThatThrownBy(() -> appDataWrapper.getFileSystem("fileSystem")) + .isInstanceOf(ResponseStatusException.class) + .hasMessage("404 NOT_FOUND \"App file system 'fileSystem' not found\""); + } +} diff --git a/afs-spring-server/src/test/java/com/powsybl/afs/server/StorageServerTest.java b/afs-spring-server/src/test/java/com/powsybl/afs/server/StorageServerTest.java new file mode 100644 index 00000000..5552b84b --- /dev/null +++ b/afs-spring-server/src/test/java/com/powsybl/afs/server/StorageServerTest.java @@ -0,0 +1,122 @@ +/** + * Copyright (c) 2019, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package com.powsybl.afs.server; + +import com.google.common.collect.ImmutableList; +import com.powsybl.afs.*; +import com.powsybl.afs.mapdb.storage.MapDbAppStorage; +import com.powsybl.afs.storage.*; +import com.powsybl.afs.ws.storage.RemoteAppStorage; +import com.powsybl.afs.ws.storage.RemoteTaskMonitor; +import com.powsybl.commons.exceptions.UncheckedUriSyntaxException; +import com.powsybl.computation.ComputationManager; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; + +import javax.servlet.ServletContext; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.mockito.Mockito.when; + +/** + * @author THIYAGARASA Pratheep Ext + */ +@RunWith(SpringRunner.class) +@SpringBootTest(classes = StorageServer.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@EnableAutoConfiguration +@ActiveProfiles("test") +public class StorageServerTest extends AbstractAppStorageTest { + + private static final String FS_TEST_NAME = "test"; + @LocalServerPort + private int port; + + @Autowired + private ServletContext servletContext; + + @Autowired + private AppDataWrapper appDataWrapper; + + @TestConfiguration + public static class AppDataConfig { + @Bean + public AppData getAppData() { + EventsBus eventBus = new InMemoryEventsBus(); + AppStorage storage = MapDbAppStorage.createMem("mem", eventBus); + AppFileSystem fs = new AppFileSystem(FS_TEST_NAME, true, storage, new LocalTaskMonitor()); + ComputationManager cm = Mockito.mock(ComputationManager.class); + + List fsProviders = ImmutableList.of(m -> ImmutableList.of(fs)); + return new AppData(cm, cm, fsProviders, eventBus); + } + } + + private URI getRestUri() { + try { + String sheme = "http"; + return new URI(sheme + "://localhost:" + port + servletContext.getContextPath()); + } catch (URISyntaxException e) { + throw new UncheckedUriSyntaxException(e); + } + } + + @Override + protected AppStorage createStorage() { + URI restUri = getRestUri(); + return new RemoteAppStorage(FS_TEST_NAME, restUri, ""); + } + + @Override + protected void nextDependentTests() throws InterruptedException { + super.nextDependentTests(); + RemoteTaskMonitor taskMonitor = new RemoteTaskMonitor(FS_TEST_NAME, getRestUri(), null); + NodeInfo root = storage.createRootNodeIfNotExists(storage.getFileSystemName(), Folder.PSEUDO_CLASS); + NodeInfo projectNode = storage.createNode(root.getId(), "project", Project.PSEUDO_CLASS, "test project", 0, new NodeGenericMetadata()); + + Project project = Mockito.mock(Project.class); + when(project.getId()).thenReturn(projectNode.getId()); + TaskMonitor.Task task = taskMonitor.startTask("task_test", project); + assertThat(task).isNotNull(); + TaskMonitor.Snapshot snapshot = taskMonitor.takeSnapshot(project.getId()); + assertThat(snapshot.getTasks().stream().anyMatch(t -> t.getId().equals(task.getId()))).isTrue(); + + taskMonitor.updateTaskMessage(task.getId(), "new Message"); + TaskMonitor.Snapshot snapshotAfterUpdate = taskMonitor.takeSnapshot(project.getId()); + TaskMonitor.Task taskAfterUpdate = snapshotAfterUpdate.getTasks().stream().filter(t -> t.getId().equals(task.getId())).findFirst().get(); + assertThat(taskAfterUpdate.getId()).isEqualTo(task.getId()); + assertThat(taskAfterUpdate.getMessage()).isEqualTo("new Message"); + + taskMonitor.stopTask(task.getId()); + TaskMonitor.Snapshot snapshotAfterStop = taskMonitor.takeSnapshot(project.getId()); + assertThat(snapshotAfterStop.getTasks().stream().anyMatch(t -> t.getId().equals(task.getId()))).isFalse(); + + assertThatCode(() -> taskMonitor.updateTaskFuture(task.getId(), CompletableFuture.runAsync(() -> { + }))).isInstanceOf(TaskMonitor.NotACancellableTaskMonitor.class); + assertThat(taskMonitor.cancelTaskComputation(task.getId())).isFalse(); + + // cleanup + storage.deleteNode(projectNode.getId()); + + // clear events + eventStack.take(); + eventStack.take(); + } +} diff --git a/afs-spring-server/src/test/java/com/powsybl/afs/server/events/TaskEventHandlerTest.java b/afs-spring-server/src/test/java/com/powsybl/afs/server/events/TaskEventHandlerTest.java new file mode 100644 index 00000000..f174f4cc --- /dev/null +++ b/afs-spring-server/src/test/java/com/powsybl/afs/server/events/TaskEventHandlerTest.java @@ -0,0 +1,96 @@ +/** + * Copyright (c) 2019, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package com.powsybl.afs.server.events; + +import com.powsybl.afs.AppFileSystem; +import com.powsybl.afs.TaskEvent; +import com.powsybl.afs.TaskListener; +import com.powsybl.afs.TaskMonitor; +import com.powsybl.afs.server.AppDataWrapper; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.web.socket.CloseStatus; +import org.springframework.web.socket.WebSocketSession; + +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Java6Assertions.assertThat; + +/** + * + * @author THIYAGARASA Pratheep Ext + */ +@RunWith(MockitoJUnitRunner.class) +public class TaskEventHandlerTest { + + @Mock + private AppDataWrapper appDataWrapper; + + @Mock + private WebSocketContext webSocketContext; + + @Mock + private WebSocketSession socketSession; + + @Mock + private AppFileSystem appFileSystem; + + @Mock + private TaskMonitor taskMonitor; + + @Captor + private ArgumentCaptor taskListenerArgumentCaptor; + + @Test + public void afterConnection() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("fileSystemName", "fs"); + attributes.put("projectId", "id"); + TaskEventHandler taskEventHandler = new TaskEventHandler(appDataWrapper, webSocketContext); + Mockito.when(socketSession.getAttributes()).thenReturn(attributes); + Mockito.when(appDataWrapper.getFileSystem("fs")).thenReturn(appFileSystem); + Mockito.when(appFileSystem.getTaskMonitor()).thenReturn(taskMonitor); + taskEventHandler.afterConnectionEstablished(socketSession); + + Mockito.verify(webSocketContext).addSession(socketSession); + Mockito.verify(taskMonitor).addListener(taskListenerArgumentCaptor.capture()); + assertThat(taskListenerArgumentCaptor.getValue().getProjectId()).isEqualTo("id"); + } + + @Test + public void afterConnectionClose() throws Exception { + TaskListener listener = new TaskListener() { + @Override + public String getProjectId() { + return "id"; + } + + @Override + public void onEvent(TaskEvent event) { + } + }; + + Map attributes = new HashMap<>(); + attributes.put("fileSystemName", "fs"); + attributes.put("taskListener", listener); + TaskEventHandler taskEventHandler = new TaskEventHandler(appDataWrapper, webSocketContext); + Mockito.when(socketSession.getAttributes()).thenReturn(attributes); + Mockito.when(appDataWrapper.getFileSystem("fs")).thenReturn(appFileSystem); + Mockito.when(appFileSystem.getTaskMonitor()).thenReturn(taskMonitor); + taskEventHandler.afterConnectionClosed(socketSession, CloseStatus.NORMAL); + + Mockito.verify(webSocketContext).removeSession(socketSession); + Mockito.verify(taskMonitor).removeListener(Mockito.eq(listener)); + } +} diff --git a/pom.xml b/pom.xml index dd92bf74..6d3c6b1b 100644 --- a/pom.xml +++ b/pom.xml @@ -63,6 +63,7 @@ afs-security-analysis-local afs-storage-api afs-ws + afs-spring-server @@ -78,7 +79,7 @@ 3.1.3.2 2.5.8 20.0 - 2.8.11 + 2.8.11 7.0 1.1 0.9.0 @@ -89,6 +90,8 @@ 3.1.4.Final 2.2.6 1.7.22 + 2.10.5 + 2.2.13.RELEASE 1.5.16 1.13.1 11.0.0.Final @@ -135,7 +138,7 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - ${jackson.jaxrs.version} + ${jackson.version} com.google.auto.service @@ -264,6 +267,12 @@ slf4j-api ${slf4j.version} + + io.springfox + springfox-swagger2 + ${springfox.version} + + @@ -335,4 +344,3 @@ - From 6f73aa1cd19c1bfc251d3f272a7cb874e10fabdc Mon Sep 17 00:00:00 2001 From: Sylvain Leclerc Date: Fri, 19 Feb 2021 11:55:24 +0100 Subject: [PATCH 29/59] Back to springfox v2.9.2, v2.10.5 is broken (#76) see https://github.com/springfox/springfox/issues/3336 Signed-off-by: Sylvain Leclerc Signed-off-by: Arthur Michaut --- .../com/powsybl/afs/server/StorageSwaggerConfig.java | 12 ++++++++---- pom.xml | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/afs-spring-server/src/main/java/com/powsybl/afs/server/StorageSwaggerConfig.java b/afs-spring-server/src/main/java/com/powsybl/afs/server/StorageSwaggerConfig.java index 158f9a5b..38718a65 100644 --- a/afs-spring-server/src/main/java/com/powsybl/afs/server/StorageSwaggerConfig.java +++ b/afs-spring-server/src/main/java/com/powsybl/afs/server/StorageSwaggerConfig.java @@ -6,6 +6,7 @@ */ package com.powsybl.afs.server; +import com.google.common.base.Predicate; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import springfox.documentation.builders.ApiInfoBuilder; @@ -13,12 +14,14 @@ import springfox.documentation.service.ApiInfo; import springfox.documentation.spi.DocumentationType; import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2WebMvc; +import springfox.documentation.swagger2.annotations.EnableSwagger2; -import java.util.function.Predicate; +import static com.google.common.base.Predicates.and; +import static com.google.common.base.Predicates.not; +import static springfox.documentation.builders.PathSelectors.regex; @Configuration -@EnableSwagger2WebMvc +@EnableSwagger2 public class StorageSwaggerConfig { @Bean public Docket produceApi() { @@ -42,6 +45,7 @@ private ApiInfo apiInfo() { // Only select apis that matches the given Predicates. private Predicate paths() { // Match all paths except /error - return input -> input.matches("/rest/afs/" + StorageServer.API_VERSION + ".*") && !input.matches("/error.*"); + return and(regex("/rest/afs/" + StorageServer.API_VERSION + ".*"), + not(regex("/error.*"))); } } diff --git a/pom.xml b/pom.xml index 6d3c6b1b..d514aad3 100644 --- a/pom.xml +++ b/pom.xml @@ -90,7 +90,7 @@ 3.1.4.Final 2.2.6 1.7.22 - 2.10.5 + 2.9.2 2.2.13.RELEASE 1.5.16 1.13.1 From 789b69a5150e034443d6894510618e41ef083c88 Mon Sep 17 00:00:00 2001 From: Sylvain Leclerc Date: Tue, 2 Mar 2021 15:42:34 +0100 Subject: [PATCH 30/59] Release v3.5.0 Signed-off-by: Sylvain Leclerc Signed-off-by: Arthur Michaut --- afs-action-dsl/pom.xml | 2 +- afs-cassandra/pom.xml | 2 +- afs-contingency/pom.xml | 2 +- afs-core/pom.xml | 2 +- afs-distribution/pom.xml | 2 +- afs-ext-base/pom.xml | 2 +- afs-local/pom.xml | 2 +- afs-mapdb-storage/pom.xml | 2 +- afs-mapdb/pom.xml | 2 +- afs-network/afs-network-client/pom.xml | 2 +- afs-network/afs-network-server/pom.xml | 2 +- afs-network/pom.xml | 2 +- afs-scripting/pom.xml | 2 +- afs-security-analysis-local/pom.xml | 2 +- afs-security-analysis/pom.xml | 2 +- afs-spring-server/pom.xml | 2 +- afs-storage-api/pom.xml | 2 +- afs-ws/afs-ws-client-utils/pom.xml | 2 +- afs-ws/afs-ws-client/pom.xml | 2 +- afs-ws/afs-ws-server-utils/pom.xml | 2 +- afs-ws/afs-ws-server/pom.xml | 2 +- afs-ws/afs-ws-storage/pom.xml | 2 +- afs-ws/afs-ws-utils/pom.xml | 2 +- afs-ws/pom.xml | 2 +- pom.xml | 2 +- 25 files changed, 25 insertions(+), 25 deletions(-) diff --git a/afs-action-dsl/pom.xml b/afs-action-dsl/pom.xml index aab7c609..e2e2c9b6 100644 --- a/afs-action-dsl/pom.xml +++ b/afs-action-dsl/pom.xml @@ -13,7 +13,7 @@ com.powsybl powsybl-afs - 3.5.0-SNAPSHOT + 3.5.0 4.0.0 diff --git a/afs-cassandra/pom.xml b/afs-cassandra/pom.xml index 38b36aec..53098587 100644 --- a/afs-cassandra/pom.xml +++ b/afs-cassandra/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-cassandra diff --git a/afs-contingency/pom.xml b/afs-contingency/pom.xml index de047635..07bd8dd5 100644 --- a/afs-contingency/pom.xml +++ b/afs-contingency/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-contingency diff --git a/afs-core/pom.xml b/afs-core/pom.xml index 7fc3237f..cf42990d 100644 --- a/afs-core/pom.xml +++ b/afs-core/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-core diff --git a/afs-distribution/pom.xml b/afs-distribution/pom.xml index 88e7a33a..c574eafc 100644 --- a/afs-distribution/pom.xml +++ b/afs-distribution/pom.xml @@ -13,7 +13,7 @@ powsybl-afs com.powsybl - 3.5.0-SNAPSHOT + 3.5.0 4.0.0 diff --git a/afs-ext-base/pom.xml b/afs-ext-base/pom.xml index 1313a1cc..44663cc8 100644 --- a/afs-ext-base/pom.xml +++ b/afs-ext-base/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-ext-base diff --git a/afs-local/pom.xml b/afs-local/pom.xml index 5bbecc6f..fd82ee9b 100644 --- a/afs-local/pom.xml +++ b/afs-local/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-local diff --git a/afs-mapdb-storage/pom.xml b/afs-mapdb-storage/pom.xml index 206c06b0..17622b9b 100644 --- a/afs-mapdb-storage/pom.xml +++ b/afs-mapdb-storage/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-mapdb-storage diff --git a/afs-mapdb/pom.xml b/afs-mapdb/pom.xml index 5d8307bf..c7aa9c17 100644 --- a/afs-mapdb/pom.xml +++ b/afs-mapdb/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-mapdb diff --git a/afs-network/afs-network-client/pom.xml b/afs-network/afs-network-client/pom.xml index fcba33b7..9ed3fcd4 100644 --- a/afs-network/afs-network-client/pom.xml +++ b/afs-network/afs-network-client/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-network - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-network-client diff --git a/afs-network/afs-network-server/pom.xml b/afs-network/afs-network-server/pom.xml index 882bd734..f72bf047 100644 --- a/afs-network/afs-network-server/pom.xml +++ b/afs-network/afs-network-server/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-network - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-network-server diff --git a/afs-network/pom.xml b/afs-network/pom.xml index 8b1a0325..10f3d68b 100644 --- a/afs-network/pom.xml +++ b/afs-network/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-network diff --git a/afs-scripting/pom.xml b/afs-scripting/pom.xml index 1c543aba..7d95d8ed 100644 --- a/afs-scripting/pom.xml +++ b/afs-scripting/pom.xml @@ -13,7 +13,7 @@ powsybl-afs com.powsybl - 3.5.0-SNAPSHOT + 3.5.0 4.0.0 diff --git a/afs-security-analysis-local/pom.xml b/afs-security-analysis-local/pom.xml index d2cfbc77..a199ebd3 100644 --- a/afs-security-analysis-local/pom.xml +++ b/afs-security-analysis-local/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-security-analysis-local diff --git a/afs-security-analysis/pom.xml b/afs-security-analysis/pom.xml index 974717b1..547a0419 100644 --- a/afs-security-analysis/pom.xml +++ b/afs-security-analysis/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-security-analysis diff --git a/afs-spring-server/pom.xml b/afs-spring-server/pom.xml index 0cd821f9..3e7dbe1a 100644 --- a/afs-spring-server/pom.xml +++ b/afs-spring-server/pom.xml @@ -7,7 +7,7 @@ com.powsybl powsybl-afs - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-spring-server diff --git a/afs-storage-api/pom.xml b/afs-storage-api/pom.xml index a9ebe0c0..d87cc19c 100644 --- a/afs-storage-api/pom.xml +++ b/afs-storage-api/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-storage-api diff --git a/afs-ws/afs-ws-client-utils/pom.xml b/afs-ws/afs-ws-client-utils/pom.xml index ac5eef0f..8a7a40eb 100644 --- a/afs-ws/afs-ws-client-utils/pom.xml +++ b/afs-ws/afs-ws-client-utils/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-ws - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-ws-client-utils diff --git a/afs-ws/afs-ws-client/pom.xml b/afs-ws/afs-ws-client/pom.xml index f53bc5cc..b22dde38 100644 --- a/afs-ws/afs-ws-client/pom.xml +++ b/afs-ws/afs-ws-client/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-ws-client diff --git a/afs-ws/afs-ws-server-utils/pom.xml b/afs-ws/afs-ws-server-utils/pom.xml index fa6af326..5294df7a 100644 --- a/afs-ws/afs-ws-server-utils/pom.xml +++ b/afs-ws/afs-ws-server-utils/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-ws - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-ws-server-utils diff --git a/afs-ws/afs-ws-server/pom.xml b/afs-ws/afs-ws-server/pom.xml index 35c19869..8dd146cd 100644 --- a/afs-ws/afs-ws-server/pom.xml +++ b/afs-ws/afs-ws-server/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-ws-server diff --git a/afs-ws/afs-ws-storage/pom.xml b/afs-ws/afs-ws-storage/pom.xml index 413815d8..ffeaf0dd 100644 --- a/afs-ws/afs-ws-storage/pom.xml +++ b/afs-ws/afs-ws-storage/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-ws-storage diff --git a/afs-ws/afs-ws-utils/pom.xml b/afs-ws/afs-ws-utils/pom.xml index c47d5288..e99e65ad 100644 --- a/afs-ws/afs-ws-utils/pom.xml +++ b/afs-ws/afs-ws-utils/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-ws-utils diff --git a/afs-ws/pom.xml b/afs-ws/pom.xml index aa841fa2..d9ac77c1 100644 --- a/afs-ws/pom.xml +++ b/afs-ws/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.5.0-SNAPSHOT + 3.5.0 powsybl-afs-ws diff --git a/pom.xml b/pom.xml index d514aad3..ce0e237a 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ powsybl-afs - 3.5.0-SNAPSHOT + 3.5.0 pom powsybl-afs From 540d5346468d4cf2d25c803dc52e060e927086ea Mon Sep 17 00:00:00 2001 From: Sylvain Leclerc Date: Tue, 2 Mar 2021 15:48:25 +0100 Subject: [PATCH 31/59] Prepare next release v3.6.0 Signed-off-by: Sylvain Leclerc Signed-off-by: Arthur Michaut --- afs-action-dsl/pom.xml | 2 +- afs-cassandra/pom.xml | 2 +- afs-contingency/pom.xml | 2 +- afs-core/pom.xml | 2 +- afs-distribution/pom.xml | 2 +- afs-ext-base/pom.xml | 2 +- afs-local/pom.xml | 2 +- afs-mapdb-storage/pom.xml | 2 +- afs-mapdb/pom.xml | 2 +- afs-network/afs-network-client/pom.xml | 2 +- afs-network/afs-network-server/pom.xml | 2 +- afs-network/pom.xml | 2 +- afs-scripting/pom.xml | 2 +- afs-security-analysis-local/pom.xml | 2 +- afs-security-analysis/pom.xml | 2 +- afs-spring-server/pom.xml | 2 +- afs-storage-api/pom.xml | 2 +- afs-ws/afs-ws-client-utils/pom.xml | 2 +- afs-ws/afs-ws-client/pom.xml | 2 +- afs-ws/afs-ws-server-utils/pom.xml | 2 +- afs-ws/afs-ws-server/pom.xml | 2 +- afs-ws/afs-ws-storage/pom.xml | 2 +- afs-ws/afs-ws-utils/pom.xml | 2 +- afs-ws/pom.xml | 2 +- pom.xml | 2 +- 25 files changed, 25 insertions(+), 25 deletions(-) diff --git a/afs-action-dsl/pom.xml b/afs-action-dsl/pom.xml index e2e2c9b6..fed81fd9 100644 --- a/afs-action-dsl/pom.xml +++ b/afs-action-dsl/pom.xml @@ -13,7 +13,7 @@ com.powsybl powsybl-afs - 3.5.0 + 3.6.0-SNAPSHOT 4.0.0 diff --git a/afs-cassandra/pom.xml b/afs-cassandra/pom.xml index 53098587..03159620 100644 --- a/afs-cassandra/pom.xml +++ b/afs-cassandra/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-cassandra diff --git a/afs-contingency/pom.xml b/afs-contingency/pom.xml index 07bd8dd5..7fb16ab8 100644 --- a/afs-contingency/pom.xml +++ b/afs-contingency/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-contingency diff --git a/afs-core/pom.xml b/afs-core/pom.xml index cf42990d..d0c49978 100644 --- a/afs-core/pom.xml +++ b/afs-core/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-core diff --git a/afs-distribution/pom.xml b/afs-distribution/pom.xml index c574eafc..206e32c2 100644 --- a/afs-distribution/pom.xml +++ b/afs-distribution/pom.xml @@ -13,7 +13,7 @@ powsybl-afs com.powsybl - 3.5.0 + 3.6.0-SNAPSHOT 4.0.0 diff --git a/afs-ext-base/pom.xml b/afs-ext-base/pom.xml index 44663cc8..67620696 100644 --- a/afs-ext-base/pom.xml +++ b/afs-ext-base/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-ext-base diff --git a/afs-local/pom.xml b/afs-local/pom.xml index fd82ee9b..c15bd0cb 100644 --- a/afs-local/pom.xml +++ b/afs-local/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-local diff --git a/afs-mapdb-storage/pom.xml b/afs-mapdb-storage/pom.xml index 17622b9b..7cd4b413 100644 --- a/afs-mapdb-storage/pom.xml +++ b/afs-mapdb-storage/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-mapdb-storage diff --git a/afs-mapdb/pom.xml b/afs-mapdb/pom.xml index c7aa9c17..c853d6a7 100644 --- a/afs-mapdb/pom.xml +++ b/afs-mapdb/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-mapdb diff --git a/afs-network/afs-network-client/pom.xml b/afs-network/afs-network-client/pom.xml index 9ed3fcd4..4e51da15 100644 --- a/afs-network/afs-network-client/pom.xml +++ b/afs-network/afs-network-client/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-network - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-network-client diff --git a/afs-network/afs-network-server/pom.xml b/afs-network/afs-network-server/pom.xml index f72bf047..a4d54756 100644 --- a/afs-network/afs-network-server/pom.xml +++ b/afs-network/afs-network-server/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-network - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-network-server diff --git a/afs-network/pom.xml b/afs-network/pom.xml index 10f3d68b..6b1d102e 100644 --- a/afs-network/pom.xml +++ b/afs-network/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-network diff --git a/afs-scripting/pom.xml b/afs-scripting/pom.xml index 7d95d8ed..a55bea96 100644 --- a/afs-scripting/pom.xml +++ b/afs-scripting/pom.xml @@ -13,7 +13,7 @@ powsybl-afs com.powsybl - 3.5.0 + 3.6.0-SNAPSHOT 4.0.0 diff --git a/afs-security-analysis-local/pom.xml b/afs-security-analysis-local/pom.xml index a199ebd3..f0d1b784 100644 --- a/afs-security-analysis-local/pom.xml +++ b/afs-security-analysis-local/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-security-analysis-local diff --git a/afs-security-analysis/pom.xml b/afs-security-analysis/pom.xml index 547a0419..5745686f 100644 --- a/afs-security-analysis/pom.xml +++ b/afs-security-analysis/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-security-analysis diff --git a/afs-spring-server/pom.xml b/afs-spring-server/pom.xml index 3e7dbe1a..e4dc52c5 100644 --- a/afs-spring-server/pom.xml +++ b/afs-spring-server/pom.xml @@ -7,7 +7,7 @@ com.powsybl powsybl-afs - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-spring-server diff --git a/afs-storage-api/pom.xml b/afs-storage-api/pom.xml index d87cc19c..f65ba9eb 100644 --- a/afs-storage-api/pom.xml +++ b/afs-storage-api/pom.xml @@ -15,7 +15,7 @@ powsybl-afs com.powsybl - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-storage-api diff --git a/afs-ws/afs-ws-client-utils/pom.xml b/afs-ws/afs-ws-client-utils/pom.xml index 8a7a40eb..cec61fb6 100644 --- a/afs-ws/afs-ws-client-utils/pom.xml +++ b/afs-ws/afs-ws-client-utils/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-ws - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-ws-client-utils diff --git a/afs-ws/afs-ws-client/pom.xml b/afs-ws/afs-ws-client/pom.xml index b22dde38..d7130296 100644 --- a/afs-ws/afs-ws-client/pom.xml +++ b/afs-ws/afs-ws-client/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-ws-client diff --git a/afs-ws/afs-ws-server-utils/pom.xml b/afs-ws/afs-ws-server-utils/pom.xml index 5294df7a..fe18055c 100644 --- a/afs-ws/afs-ws-server-utils/pom.xml +++ b/afs-ws/afs-ws-server-utils/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs-ws - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-ws-server-utils diff --git a/afs-ws/afs-ws-server/pom.xml b/afs-ws/afs-ws-server/pom.xml index 8dd146cd..ea0b056e 100644 --- a/afs-ws/afs-ws-server/pom.xml +++ b/afs-ws/afs-ws-server/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-ws-server diff --git a/afs-ws/afs-ws-storage/pom.xml b/afs-ws/afs-ws-storage/pom.xml index ffeaf0dd..a5394440 100644 --- a/afs-ws/afs-ws-storage/pom.xml +++ b/afs-ws/afs-ws-storage/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-ws-storage diff --git a/afs-ws/afs-ws-utils/pom.xml b/afs-ws/afs-ws-utils/pom.xml index e99e65ad..6d47afbe 100644 --- a/afs-ws/afs-ws-utils/pom.xml +++ b/afs-ws/afs-ws-utils/pom.xml @@ -14,7 +14,7 @@ com.powsybl powsybl-afs-ws - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-ws-utils diff --git a/afs-ws/pom.xml b/afs-ws/pom.xml index d9ac77c1..428baf2a 100644 --- a/afs-ws/pom.xml +++ b/afs-ws/pom.xml @@ -15,7 +15,7 @@ com.powsybl powsybl-afs - 3.5.0 + 3.6.0-SNAPSHOT powsybl-afs-ws diff --git a/pom.xml b/pom.xml index ce0e237a..68ac3230 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ powsybl-afs - 3.5.0 + 3.6.0-SNAPSHOT pom powsybl-afs From 5847cbcd06bd4c62bc38a8789ea50c251b1b73d7 Mon Sep 17 00:00:00 2001 From: marifunf Date: Fri, 16 Apr 2021 11:54:45 +0200 Subject: [PATCH 32/59] [Iissue-78] Add NetworkListener on virtual case (#79) * [Iissue-78] Add NetworkListener on virtual case Signed-off-by: funfrockmar * [issue-78] Add NetworkListener on virtual case - review Signed-off-by: funfrockmar * [issue-78] Add NetworkListener on virtual case - sonar Signed-off-by: funfrockmar * [issue-78] Add NetworkListener on virtual case - javadoc Signed-off-by: funfrockmar Signed-off-by: Arthur Michaut --- afs-action-dsl/pom.xml | 10 +++++++ afs-contingency/pom.xml | 10 +++++++ afs-core/pom.xml | 10 +++++++ .../powsybl/afs/AbstractProjectFileTest.java | 9 ++++--- .../powsybl/afs/ext/base/ImportedCase.java | 7 +++++ .../ext/base/LocalNetworkCacheService.java | 26 +++++++++++++------ .../afs/ext/base/NetworkCacheService.java | 5 ++++ .../com/powsybl/afs/ext/base/ProjectCase.java | 12 +++++++++ .../com/powsybl/afs/ext/base/VirtualCase.java | 7 +++++ .../afs/ext/base/ImportedCaseTest.java | 12 +++++++++ .../powsybl/afs/ext/base/VirtualCaseTest.java | 23 ++++++++++++++++ .../client/RemoteNetworkCacheService.java | 10 +++++++ afs-security-analysis-local/pom.xml | 10 +++++++ 13 files changed, 139 insertions(+), 12 deletions(-) diff --git a/afs-action-dsl/pom.xml b/afs-action-dsl/pom.xml index fed81fd9..d5bff901 100644 --- a/afs-action-dsl/pom.xml +++ b/afs-action-dsl/pom.xml @@ -81,5 +81,15 @@ ${project.version} test + + com.powsybl + powsybl-config-test + test + + + com.powsybl + powsybl-iidm-impl + test + diff --git a/afs-contingency/pom.xml b/afs-contingency/pom.xml index 7fb16ab8..ab3a5ad4 100644 --- a/afs-contingency/pom.xml +++ b/afs-contingency/pom.xml @@ -78,5 +78,15 @@ ${project.version} test + + com.powsybl + powsybl-config-test + test + + + com.powsybl + powsybl-iidm-impl + test + diff --git a/afs-core/pom.xml b/afs-core/pom.xml index d0c49978..e50a64d5 100644 --- a/afs-core/pom.xml +++ b/afs-core/pom.xml @@ -115,6 +115,16 @@ test-jar test + + com.powsybl + powsybl-config-test + test + + + com.powsybl + powsybl-iidm-impl + test + diff --git a/afs-core/src/test/java/com/powsybl/afs/AbstractProjectFileTest.java b/afs-core/src/test/java/com/powsybl/afs/AbstractProjectFileTest.java index b5f32b36..1833269f 100644 --- a/afs-core/src/test/java/com/powsybl/afs/AbstractProjectFileTest.java +++ b/afs-core/src/test/java/com/powsybl/afs/AbstractProjectFileTest.java @@ -47,10 +47,11 @@ protected List getServiceExtensions() { @Before public void setup() throws IOException { - network = Mockito.mock(Network.class); - Substation s = Mockito.mock(Substation.class); - Mockito.when(s.getId()).thenReturn("s1"); - Mockito.when(network.getSubstations()).thenReturn(Collections.singletonList(s)); + network = Network.create("test", "test"); + Substation s = network.newSubstation() + .setId("s1") + .setTso("TSO") + .add(); ComputationManager computationManager = Mockito.mock(ComputationManager.class); storage = createStorage(); afs = new AppFileSystem("mem", false, storage); diff --git a/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/ImportedCase.java b/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/ImportedCase.java index f6efeaad..68d5e180 100644 --- a/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/ImportedCase.java +++ b/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/ImportedCase.java @@ -14,12 +14,14 @@ import com.powsybl.iidm.import_.Importer; import com.powsybl.iidm.import_.ImportersLoader; import com.powsybl.iidm.network.Network; +import com.powsybl.iidm.network.NetworkListener; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; +import java.util.List; import java.util.Objects; import java.util.Properties; @@ -79,6 +81,11 @@ public Network getNetwork() { return findService(NetworkCacheService.class).getNetwork(this); } + @Override + public Network getNetwork(List listeners) { + return findService(NetworkCacheService.class).getNetwork(this, listeners); + } + @Override public void invalidateNetworkCache() { findService(NetworkCacheService.class).invalidateCache(this); diff --git a/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/LocalNetworkCacheService.java b/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/LocalNetworkCacheService.java index 3e712252..bd1f1f87 100644 --- a/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/LocalNetworkCacheService.java +++ b/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/LocalNetworkCacheService.java @@ -11,13 +11,12 @@ import com.powsybl.commons.datasource.ReadOnlyDataSource; import com.powsybl.iidm.import_.Importer; import com.powsybl.iidm.network.Network; +import com.powsybl.iidm.network.NetworkListener; import groovy.json.JsonOutput; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Objects; -import java.util.Properties; -import java.util.UUID; +import java.util.*; /** * @author Geoffroy Jamgotchian @@ -44,13 +43,14 @@ public LocalNetworkCacheService() { }); } - private static ScriptResult loadNetworkFromImportedCase(ImportedCase importedCase) { + private static ScriptResult loadNetworkFromImportedCase(ImportedCase importedCase, List listeners) { LOGGER.info("Loading network of project case {}", importedCase.getId()); Importer importer = importedCase.getImporter(); ReadOnlyDataSource dataSource = importedCase.getDataSource(); Properties parameters = importedCase.getParameters(); Network network = importer.importData(dataSource, parameters); + listeners.forEach(network::addListener); return ScriptResult.of(network); } @@ -64,11 +64,11 @@ private static ScriptResult applyScript(Network network, String previou } } - private static ScriptResult loadNetworkFromVirtualCase(VirtualCase virtualCase) { + private static ScriptResult loadNetworkFromVirtualCase(VirtualCase virtualCase, List listeners) { ProjectCase baseCase = (ProjectCase) virtualCase.getCase() .orElseThrow(() -> new AfsException("Case link is dead")); - ScriptResult network = loadNetworkFromProjectCase(baseCase); + ScriptResult network = loadNetworkFromProjectCase(baseCase, listeners); if (network.getError() != null) { return network; @@ -83,10 +83,14 @@ private static ScriptResult loadNetworkFromVirtualCase(VirtualCase virt } private static ScriptResult loadNetworkFromProjectCase(ProjectCase projectCase) { + return loadNetworkFromProjectCase(projectCase, Collections.emptyList()); + } + + private static ScriptResult loadNetworkFromProjectCase(ProjectCase projectCase, List listeners) { if (projectCase instanceof ImportedCase) { - return loadNetworkFromImportedCase((ImportedCase) projectCase); + return loadNetworkFromImportedCase((ImportedCase) projectCase, listeners); } else if (projectCase instanceof VirtualCase) { - return loadNetworkFromVirtualCase((VirtualCase) projectCase); + return loadNetworkFromVirtualCase((VirtualCase) projectCase, listeners); } else { throw new AssertionError("ProjectCase implementation " + projectCase.getClass().getName() + " not supported"); } @@ -112,6 +116,12 @@ public Network getNetwork(T projectCase) { return cache.get(projectCase).getValueOrThrowIfError(projectCase); } + @Override + public Network getNetwork(T projectCase, List listeners) { + ScriptResult network = loadNetworkFromProjectCase(projectCase, listeners); + return network.getValueOrThrowIfError(projectCase); + } + @Override public void invalidateCache(T projectCase) { cache.invalidate(projectCase); diff --git a/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/NetworkCacheService.java b/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/NetworkCacheService.java index 95afe1d8..aada5e8d 100644 --- a/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/NetworkCacheService.java +++ b/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/NetworkCacheService.java @@ -8,6 +8,9 @@ import com.powsybl.afs.ProjectFile; import com.powsybl.iidm.network.Network; +import com.powsybl.iidm.network.NetworkListener; + +import java.util.List; /** * Provides caching capabilities for loaded {@code Network} objects. @@ -18,6 +21,8 @@ public interface NetworkCacheService { Network getNetwork(T projectCase); + Network getNetwork(T projectCase, List listeners); + String queryNetwork(T projectCase, ScriptType scriptType, String scriptContent); void invalidateCache(T projectCase); diff --git a/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/ProjectCase.java b/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/ProjectCase.java index acdbcce2..93dc3114 100644 --- a/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/ProjectCase.java +++ b/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/ProjectCase.java @@ -7,6 +7,9 @@ package com.powsybl.afs.ext.base; import com.powsybl.iidm.network.Network; +import com.powsybl.iidm.network.NetworkListener; + +import java.util.List; /** * Common interface for project files able to provide a Network. @@ -19,6 +22,15 @@ public interface ProjectCase { Network getNetwork(); + /** + * Get the network and add a listeners on it in order to listen changes due to virtual case script application. + * The listeners will not be removed from the network at the end of the network loading, + * so the user of this method must make sure to handle it on its own. + * The contract of being notified may not be honored by all implementations (see remote service cache : listeners will not be added to the network). + * The user must check with the cache implementation he will use. + */ + Network getNetwork(List listeners); + void invalidateNetworkCache(); void addListener(ProjectCaseListener l); diff --git a/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/VirtualCase.java b/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/VirtualCase.java index 4b431fd1..88d05e57 100644 --- a/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/VirtualCase.java +++ b/afs-ext-base/src/main/java/com/powsybl/afs/ext/base/VirtualCase.java @@ -8,8 +8,10 @@ import com.powsybl.afs.*; import com.powsybl.iidm.network.Network; +import com.powsybl.iidm.network.NetworkListener; import java.util.Collections; +import java.util.List; import java.util.Objects; import java.util.Optional; @@ -69,6 +71,11 @@ public Network getNetwork() { return findService(NetworkCacheService.class).getNetwork(this); } + @Override + public Network getNetwork(List listeners) { + return findService(NetworkCacheService.class).getNetwork(this, listeners); + } + @Override public void invalidateNetworkCache() { findService(NetworkCacheService.class).invalidateCache(this); diff --git a/afs-ext-base/src/test/java/com/powsybl/afs/ext/base/ImportedCaseTest.java b/afs-ext-base/src/test/java/com/powsybl/afs/ext/base/ImportedCaseTest.java index c3dcdd71..c4be72bd 100644 --- a/afs-ext-base/src/test/java/com/powsybl/afs/ext/base/ImportedCaseTest.java +++ b/afs-ext-base/src/test/java/com/powsybl/afs/ext/base/ImportedCaseTest.java @@ -21,7 +21,9 @@ import com.powsybl.iidm.import_.ImportConfig; import com.powsybl.iidm.import_.ImportersLoader; import com.powsybl.iidm.import_.ImportersLoaderList; +import com.powsybl.iidm.network.DefaultNetworkListener; import com.powsybl.iidm.network.Network; +import com.powsybl.iidm.network.NetworkListener; import com.powsybl.iidm.xml.XMLExporter; import com.powsybl.iidm.xml.XMLImporter; import org.junit.After; @@ -36,6 +38,9 @@ import java.util.List; import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; /** * @author Geoffroy Jamgotchian @@ -131,6 +136,13 @@ public void test() { assertNotNull(importedCase.getNetwork()); assertTrue(importedCase.getDependencies().isEmpty()); + // test network listener + NetworkListener mockedListener = mock(DefaultNetworkListener.class); + assertNotNull(importedCase.getNetwork(Collections.singletonList(mockedListener))); + network.getSubstation("s1").setTso("tso_new"); + verify(mockedListener, times(1)) + .onUpdate(network.getSubstation("s1"), "tso", "TSO", "tso_new"); + // test network query assertEquals("[\"s1\"]", importedCase.queryNetwork(ScriptType.GROOVY, "network.substations.collect { it.id }")); diff --git a/afs-ext-base/src/test/java/com/powsybl/afs/ext/base/VirtualCaseTest.java b/afs-ext-base/src/test/java/com/powsybl/afs/ext/base/VirtualCaseTest.java index 8ff0b787..92d51fa6 100644 --- a/afs-ext-base/src/test/java/com/powsybl/afs/ext/base/VirtualCaseTest.java +++ b/afs-ext-base/src/test/java/com/powsybl/afs/ext/base/VirtualCaseTest.java @@ -16,14 +16,20 @@ import com.powsybl.iidm.import_.ImportConfig; import com.powsybl.iidm.import_.ImportersLoader; import com.powsybl.iidm.import_.ImportersLoaderList; +import com.powsybl.iidm.network.DefaultNetworkListener; +import com.powsybl.iidm.network.NetworkListener; import org.junit.Before; import org.junit.Test; import java.io.IOException; +import java.util.Collections; import java.util.List; import static org.assertj.core.api.Assertions.assertThatCode; import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; /** * @author Geoffroy Jamgotchian @@ -193,5 +199,22 @@ public void test() { assertEquals(importedCase3.getName(), virtualCase3.getCase().map(ProjectFile::getName).orElse(null)); assertThatCode(() -> virtualCase3.setCase(virtualCase3)).isInstanceOf(AfsCircularDependencyException.class); + + // test network listener + ModificationScript scriptModif = folder.fileBuilder(ModificationScriptBuilder.class) + .withName("scriptModif") + .withType(ScriptType.GROOVY) + .withContent("network.getSubstation('s1').setTso('tso_new')") + .build(); + VirtualCase virtualCase4 = folder.fileBuilder(VirtualCaseBuilder.class) + .withName("network4") + .withCase(importedCase3) + .withScript(scriptModif) + .build(); + + NetworkListener mockedListener = mock(DefaultNetworkListener.class); + virtualCase4.getNetwork(Collections.singletonList(mockedListener)); + verify(mockedListener, times(1)) + .onUpdate(network.getSubstation("s1"), "tso", "TSO", "tso_new"); } } diff --git a/afs-network/afs-network-client/src/main/java/com/powsybl/afs/network/client/RemoteNetworkCacheService.java b/afs-network/afs-network-client/src/main/java/com/powsybl/afs/network/client/RemoteNetworkCacheService.java index 80863c89..37da630e 100644 --- a/afs-network/afs-network-client/src/main/java/com/powsybl/afs/network/client/RemoteNetworkCacheService.java +++ b/afs-network/afs-network-client/src/main/java/com/powsybl/afs/network/client/RemoteNetworkCacheService.java @@ -16,6 +16,7 @@ import com.powsybl.afs.ws.client.utils.ClientUtils; import com.powsybl.afs.ws.client.utils.RemoteServiceConfig; import com.powsybl.iidm.network.Network; +import com.powsybl.iidm.network.NetworkListener; import com.powsybl.iidm.xml.NetworkXml; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +31,7 @@ import java.io.InputStream; import java.io.UncheckedIOException; import java.net.URI; +import java.util.List; import java.util.Objects; import java.util.Optional; @@ -65,6 +67,14 @@ private RemoteServiceConfig getConfig() { return Objects.requireNonNull(configSupplier.get()).orElseThrow(() -> new AfsException("Remote service config is missing")); } + /** + * Listeners will not be added to the network + */ + @Override + public Network getNetwork(T projectCase, List listeners) { + return getNetwork(projectCase); + } + @Override public Network getNetwork(T projectCase) { Objects.requireNonNull(projectCase); diff --git a/afs-security-analysis-local/pom.xml b/afs-security-analysis-local/pom.xml index f0d1b784..c43e6c9b 100644 --- a/afs-security-analysis-local/pom.xml +++ b/afs-security-analysis-local/pom.xml @@ -82,6 +82,16 @@ test-jar test + + com.powsybl + powsybl-config-test + test + + + com.powsybl + powsybl-iidm-impl + test + From 8a603ccbb22673d5a50d7a1cfec204f88a589756 Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Fri, 23 Apr 2021 11:49:47 +0200 Subject: [PATCH 33/59] Init timeseries server-based storage Signed-off-by: bhorvilleur Signed-off-by: Arthur Michaut --- afs-timeseries-server/pom.xml | 73 ++++++ .../TimeSeriesServerAppStorage.java | 220 ++++++++++++++++++ .../TimeSeriesSorageDelegate.java | 166 +++++++++++++ 3 files changed, 459 insertions(+) create mode 100644 afs-timeseries-server/pom.xml create mode 100644 afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java create mode 100644 afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java diff --git a/afs-timeseries-server/pom.xml b/afs-timeseries-server/pom.xml new file mode 100644 index 00000000..1c875cd3 --- /dev/null +++ b/afs-timeseries-server/pom.xml @@ -0,0 +1,73 @@ + + + + 4.0.0 + + + com.powsybl + powsybl-afs + 3.6.0-SNAPSHOT + + + powsybl-afs-timeseries-server + AFS TimeSeries Server impl + AFS TimeSeries Server implementation + + + + + ${project.groupId} + powsybl-afs-core + ${project.version} + + + ${project.groupId} + powsybl-afs-core + ${project.version} + + + ${project.groupId} + powsybl-time-series-server-interfaces + 0.0.1-SNAPSHOT + + + javax + javaee-api + + + + + junit + junit + test + + + org.assertj + assertj-core + test + + + org.slf4j + slf4j-simple + test + + + + ${project.groupId} + powsybl-afs-storage-api + ${project.version} + test-jar + test + + + + diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java new file mode 100644 index 00000000..4c9fd829 --- /dev/null +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java @@ -0,0 +1,220 @@ +package com.powsybl.afs.timeseriesserver; + +import com.powsybl.afs.storage.*; +import com.powsybl.timeseries.DoubleDataChunk; +import com.powsybl.timeseries.StringDataChunk; +import com.powsybl.timeseries.TimeSeriesMetadata; +import org.apache.commons.lang3.NotImplementedException; + +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +public class TimeSeriesServerAppStorage extends AbstractAppStorage { + + /** + * This storage is used for all non-timeseries-related operations + */ + AppStorage generalDelegate; + + /** + * This storage handles all the timeseries-related operations + */ + TimeSeriesSorageDelegate timeSeriesDelegate; + + + public TimeSeriesServerAppStorage(AppStorage generalDelegate) { + this.generalDelegate = generalDelegate; + timeSeriesDelegate = new TimeSeriesSorageDelegate(); + timeSeriesDelegate.createAFSAppIfNotExists(); + } + + @Override + public String getFileSystemName() { + return generalDelegate.getFileSystemName(); + } + + @Override + public boolean isRemote() { + return generalDelegate.isRemote(); + } + + @Override + public NodeInfo createRootNodeIfNotExists(String name, String nodePseudoClass) { + return generalDelegate.createRootNodeIfNotExists(name, nodePseudoClass); + } + + @Override + public NodeInfo createNode(String parentNodeId, String name, String nodePseudoClass, String description, int version, NodeGenericMetadata genericMetadata) { + return generalDelegate.createNode(parentNodeId, name, nodePseudoClass, description, version, genericMetadata); + } + + @Override + public boolean isWritable(String nodeId) { + return generalDelegate.isWritable(nodeId); + } + + @Override + public NodeInfo getNodeInfo(String nodeId) { + return generalDelegate.getNodeInfo(nodeId); + } + + @Override + public void setDescription(String nodeId, String description) { + generalDelegate.setDescription(nodeId, description); + } + + @Override + public void updateModificationTime(String nodeId) { + generalDelegate.updateModificationTime(nodeId); + } + + @Override + public List getChildNodes(String nodeId) { + return generalDelegate.getChildNodes(nodeId); + } + + @Override + public Optional getChildNode(String nodeId, String name) { + return generalDelegate.getChildNode(nodeId, name); + } + + @Override + public Optional getParentNode(String nodeId) { + return generalDelegate.getParentNode(nodeId); + } + + @Override + public void setParentNode(String nodeId, String newParentNodeId) { + generalDelegate.setParentNode(nodeId, newParentNodeId); + } + + @Override + public String deleteNode(String nodeId) { + return generalDelegate.deleteNode(nodeId); + } + + @Override + public Optional readBinaryData(String nodeId, String name) { + return generalDelegate.readBinaryData(nodeId, name); + } + + @Override + public OutputStream writeBinaryData(String nodeId, String name) { + return generalDelegate.writeBinaryData(nodeId, name); + } + + @Override + public boolean dataExists(String nodeId, String name) { + return generalDelegate.dataExists(nodeId, name); + } + + @Override + public Set getDataNames(String nodeId) { + return generalDelegate.getDataNames(nodeId); + } + + @Override + public boolean removeData(String nodeId, String name) { + return generalDelegate.removeData(nodeId, name); + } + + @Override + public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { + timeSeriesDelegate.createTimeSeries(nodeId, metadata); + } + + @Override + public Set getTimeSeriesNames(String nodeId) { + return timeSeriesDelegate.getTimeSeriesNames(nodeId); + } + + @Override + public boolean timeSeriesExists(String nodeId, String timeSeriesName) { + return timeSeriesDelegate.timeSeriesExists(nodeId, timeSeriesName); + } + + @Override + public List getTimeSeriesMetadata(String nodeId, Set timeSeriesNames) { + return timeSeriesDelegate.getTimeSeriesMetadata(nodeId, timeSeriesNames); + } + + @Override + public Set getTimeSeriesDataVersions(String nodeId) { + return timeSeriesDelegate.getTimeSeriesDataVersions(nodeId, null); + } + + @Override + public Set getTimeSeriesDataVersions(String nodeId, String timeSeriesName) { + return timeSeriesDelegate.getTimeSeriesDataVersions(nodeId, timeSeriesName); + } + + @Override + public Map> getDoubleTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { + //TODO + return null; + } + + @Override + public void addDoubleTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { + //TODO + } + + @Override + public Map> getStringTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { + throw new NotImplementedException("Not implemented in V1"); + } + + @Override + public void addStringTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { + throw new NotImplementedException("Not implemented in V1"); + } + + @Override + public void clearTimeSeries(String nodeId) { + //TODO + } + + @Override + public void addDependency(String nodeId, String name, String toNodeId) { + generalDelegate.addDependency(nodeId, name, toNodeId); + } + + @Override + public Set getDependencies(String nodeId, String name) { + return generalDelegate.getDependencies(nodeId, name); + } + + @Override + public Set getDependencies(String nodeId) { + return generalDelegate.getDependencies(nodeId); + } + + @Override + public Set getBackwardDependencies(String nodeId) { + return generalDelegate.getBackwardDependencies(nodeId); + } + + @Override + public void removeDependency(String nodeId, String name, String toNodeId) { + generalDelegate.removeDependency(nodeId, name, toNodeId); + } + + @Override + public void flush() { + generalDelegate.flush(); + } + + @Override + public boolean isClosed() { + return generalDelegate.isClosed(); + } + + @Override + public void close() { + generalDelegate.close(); + } +} diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java new file mode 100644 index 00000000..36b15a15 --- /dev/null +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -0,0 +1,166 @@ +package com.powsybl.afs.timeseriesserver; + +import com.powsybl.timeseries.RegularTimeSeriesIndex; +import com.powsybl.timeseries.TimeSeriesDataType; +import com.powsybl.timeseries.TimeSeriesIndex; +import com.powsybl.timeseries.TimeSeriesMetadata; +import com.powsybl.timeseries.storer.query.create.CreateQuery; +import com.powsybl.timeseries.storer.query.search.SearchQuery; +import com.powsybl.timeseries.storer.query.search.SearchQueryResults; +import org.apache.commons.lang3.NotImplementedException; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; +import java.net.URI; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +public class TimeSeriesSorageDelegate { + + private static final String AFS_APP = "AFS"; + + private URI timeSeriesServerURI; + + + private WebTarget buildBaseRequest(Client client) { + return client.target(timeSeriesServerURI) + .path("v1") + .path("timeseries") + .path("apps"); + } + + public void createAFSAppIfNotExists() { + Client client = ClientBuilder.newClient(); + try { + Response response = buildBaseRequest(client).request().get(); + + Collection apps = response.readEntity(Collection.class); + if (apps.contains(AFS_APP)) { + return; + } + + buildBaseRequest(client).request().post(Entity.json(AFS_APP)); + + } finally { + client.close(); + } + + } + + public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { + if (!(metadata.getIndex() instanceof RegularTimeSeriesIndex)) { + throw new NotImplementedException("TimeSeriesServer only handles regular time series for now."); + } + RegularTimeSeriesIndex index = (RegularTimeSeriesIndex) metadata.getIndex(); + + CreateQuery createQuery = new CreateQuery(); + createQuery.setMatrix(nodeId); + createQuery.setName(metadata.getName()); + createQuery.setTags(metadata.getTags()); + createQuery.setTimeStepCount(index.getPointCount()); + createQuery.setTimeStepDuration(index.getSpacing()); + LocalDateTime startDate = Instant.ofEpochMilli(index.getStartTime()).atZone(ZoneId.systemDefault()).toLocalDateTime(); + createQuery.setStartDate(startDate); + + Client client = ClientBuilder.newClient(); + try { + buildBaseRequest(client) + .path(AFS_APP) + .path("series") + .request().post(Entity.json(createQuery)); + } finally { + client.close(); + } + } + + public SearchQueryResults performSearch(SearchQuery query) { + + + SearchQueryResults results = null; + + Client client = ClientBuilder.newClient(); + try { + Response response = buildBaseRequest(client) + .path(AFS_APP) + .path("series") + .path("_search") + .request().post(Entity.json(query)); + results = response.readEntity(SearchQueryResults.class); + } finally { + client.close(); + } + + return results; + } + + public Set getTimeSeriesNames(String nodeId) { + + SearchQuery searchQuery = new SearchQuery(); + searchQuery.setMatrix(nodeId); + + SearchQueryResults results = performSearch(searchQuery); + if (results != null) { + return results.getTimeSeriesInformations() + .stream().map(t -> t.getName()).collect(Collectors.toSet()); + } + return null; + } + + public boolean timeSeriesExists(String nodeId, String name) { + + SearchQuery searchQuery = new SearchQuery(); + searchQuery.setMatrix(nodeId); + searchQuery.setNames(Collections.singleton(name)); + SearchQueryResults results = performSearch(searchQuery); + if (results != null) { + return results.getTimeSeriesInformations().size() > 0; + } + return false; + } + + public List getTimeSeriesMetadata(String nodeId, Set timeSeriesNames) { + + SearchQuery searchQuery = new SearchQuery(); + searchQuery.setMatrix(nodeId); + searchQuery.setNames(timeSeriesNames); + SearchQueryResults results = performSearch(searchQuery); + if (results != null) { + return results.getTimeSeriesInformations() + .stream().map(t -> { + long startTime = t.getStartDate().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + long spacing = t.getTimeStepDuration(); + long endTime = startTime + spacing * t.getTimeStepCount(); + TimeSeriesIndex index = new RegularTimeSeriesIndex(startTime, endTime, spacing); + return new TimeSeriesMetadata(t.getName(), TimeSeriesDataType.DOUBLE, t.getTags(), index); + }) + .collect(Collectors.toList()); + } + return null; + } + + public Set getTimeSeriesDataVersions(String nodeId, String timeSeriesName) { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.setMatrix(nodeId); + if(timeSeriesName != null) + { + searchQuery.setNames(Collections.singleton(timeSeriesName)); + } + SearchQueryResults results = performSearch(searchQuery); + if (results != null) { + return results.getTimeSeriesInformations() + .stream().flatMap(t -> t.getVersions().keySet().stream()) + .map(t->Integer.parseInt(t)) + .collect(Collectors.toSet()); + } + return null; + } +} From e3e2453f95f40615de02860cb1066bd4a8ba7f24 Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Thu, 6 May 2021 10:30:29 +0200 Subject: [PATCH 34/59] Create AFS storage using TimeSeriesServer (WIP) Signed-off-by: bhorvilleur Signed-off-by: Arthur Michaut --- .../cassandra/CassandraAppStorageTest.java | 2 +- .../powsybl/afs/server/StorageServerTest.java | 2 +- afs-timeseries-server/pom.xml | 57 +++++++++++++++++-- .../TimeSeriesServerAppStorage.java | 18 ++++-- .../TimeSeriesSorageDelegate.java | 23 +++++--- .../TimeSeriesServerAppStorageTest.java | 25 ++++++++ 6 files changed, 108 insertions(+), 19 deletions(-) create mode 100644 afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java diff --git a/afs-cassandra/src/test/java/com/powsybl/afs/cassandra/CassandraAppStorageTest.java b/afs-cassandra/src/test/java/com/powsybl/afs/cassandra/CassandraAppStorageTest.java index e2452d98..748c7ce0 100644 --- a/afs-cassandra/src/test/java/com/powsybl/afs/cassandra/CassandraAppStorageTest.java +++ b/afs-cassandra/src/test/java/com/powsybl/afs/cassandra/CassandraAppStorageTest.java @@ -22,7 +22,7 @@ public class CassandraAppStorageTest extends AbstractAppStorageTest { public CassandraCQLUnit cassandraCQLUnit = new CassandraCQLUnit(new ClassPathCQLDataSet("afs.cql", CassandraConstants.AFS_KEYSPACE), null, 20000L); @Override - protected AppStorage createStorage() { + public AppStorage createStorage() { return new CassandraAppStorage("test", () -> new CassandraTestContext(cassandraCQLUnit), new CassandraAppStorageConfig(), new InMemoryEventsBus()); } diff --git a/afs-spring-server/src/test/java/com/powsybl/afs/server/StorageServerTest.java b/afs-spring-server/src/test/java/com/powsybl/afs/server/StorageServerTest.java index 5552b84b..f03c6de0 100644 --- a/afs-spring-server/src/test/java/com/powsybl/afs/server/StorageServerTest.java +++ b/afs-spring-server/src/test/java/com/powsybl/afs/server/StorageServerTest.java @@ -79,7 +79,7 @@ private URI getRestUri() { } @Override - protected AppStorage createStorage() { + public AppStorage createStorage() { URI restUri = getRestUri(); return new RemoteAppStorage(FS_TEST_NAME, restUri, ""); } diff --git a/afs-timeseries-server/pom.xml b/afs-timeseries-server/pom.xml index 1c875cd3..16b3fd67 100644 --- a/afs-timeseries-server/pom.xml +++ b/afs-timeseries-server/pom.xml @@ -29,16 +29,21 @@ powsybl-afs-core ${project.version} - - ${project.groupId} - powsybl-afs-core - ${project.version} - ${project.groupId} powsybl-time-series-server-interfaces 0.0.1-SNAPSHOT + + org.jboss.resteasy + resteasy-client + provided + + + org.jboss.resteasy + resteasy-jackson-provider + 3.0.19.Final + javax javaee-api @@ -60,6 +65,41 @@ slf4j-simple test + + org.cassandraunit + cassandra-unit + test + + + ${project.groupId} + powsybl-afs-cassandra + ${project.version} + test-jar + test + + + ${project.groupId} + powsybl-afs-local + ${project.version} + test-jar + test + + + ${project.groupId} + powsybl-afs-local + ${project.version} + test + + + org.mockito + mockito-core + test + + + com.google.jimfs + jimfs + test + ${project.groupId} @@ -68,6 +108,13 @@ test-jar test + + ${project.groupId} + powsybl-afs-ext-base + ${project.version} + test + test-jar + diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java index 4c9fd829..23b24518 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java @@ -8,12 +8,13 @@ import java.io.InputStream; import java.io.OutputStream; +import java.net.URI; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -public class TimeSeriesServerAppStorage extends AbstractAppStorage { +public class TimeSeriesServerAppStorage implements AppStorage { /** * This storage is used for all non-timeseries-related operations @@ -25,10 +26,9 @@ public class TimeSeriesServerAppStorage extends AbstractAppStorage { */ TimeSeriesSorageDelegate timeSeriesDelegate; - - public TimeSeriesServerAppStorage(AppStorage generalDelegate) { + public TimeSeriesServerAppStorage(AppStorage generalDelegate, URI timeSeriesServerURI) { this.generalDelegate = generalDelegate; - timeSeriesDelegate = new TimeSeriesSorageDelegate(); + timeSeriesDelegate = new TimeSeriesSorageDelegate(timeSeriesServerURI); timeSeriesDelegate.createAFSAppIfNotExists(); } @@ -203,6 +203,11 @@ public void removeDependency(String nodeId, String name, String toNodeId) { generalDelegate.removeDependency(nodeId, name, toNodeId); } + @Override + public EventsBus getEventsBus() { + return generalDelegate.getEventsBus(); + } + @Override public void flush() { generalDelegate.flush(); @@ -217,4 +222,9 @@ public boolean isClosed() { public void close() { generalDelegate.close(); } + + @Override + public boolean isConsistent(String nodeId) { + return generalDelegate.isConsistent(nodeId); + } } diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index 36b15a15..8d8be8b2 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -8,9 +8,9 @@ import com.powsybl.timeseries.storer.query.search.SearchQuery; import com.powsybl.timeseries.storer.query.search.SearchQueryResults; import org.apache.commons.lang3.NotImplementedException; +import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; import javax.ws.rs.client.Client; -import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response; @@ -30,6 +30,15 @@ public class TimeSeriesSorageDelegate { private URI timeSeriesServerURI; + public TimeSeriesSorageDelegate(URI timeSeriesServerURI) { + this.timeSeriesServerURI = timeSeriesServerURI; + } + + public static Client createClient() { + return new ResteasyClientBuilder() + .connectionPoolSize(50) + .build(); + } private WebTarget buildBaseRequest(Client client) { return client.target(timeSeriesServerURI) @@ -39,7 +48,7 @@ private WebTarget buildBaseRequest(Client client) { } public void createAFSAppIfNotExists() { - Client client = ClientBuilder.newClient(); + Client client = createClient(); try { Response response = buildBaseRequest(client).request().get(); @@ -71,7 +80,7 @@ public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { LocalDateTime startDate = Instant.ofEpochMilli(index.getStartTime()).atZone(ZoneId.systemDefault()).toLocalDateTime(); createQuery.setStartDate(startDate); - Client client = ClientBuilder.newClient(); + Client client = createClient(); try { buildBaseRequest(client) .path(AFS_APP) @@ -84,10 +93,9 @@ public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { public SearchQueryResults performSearch(SearchQuery query) { - SearchQueryResults results = null; - Client client = ClientBuilder.newClient(); + Client client = createClient(); try { Response response = buildBaseRequest(client) .path(AFS_APP) @@ -150,15 +158,14 @@ public List getTimeSeriesMetadata(String nodeId, Set public Set getTimeSeriesDataVersions(String nodeId, String timeSeriesName) { SearchQuery searchQuery = new SearchQuery(); searchQuery.setMatrix(nodeId); - if(timeSeriesName != null) - { + if (timeSeriesName != null) { searchQuery.setNames(Collections.singleton(timeSeriesName)); } SearchQueryResults results = performSearch(searchQuery); if (results != null) { return results.getTimeSeriesInformations() .stream().flatMap(t -> t.getVersions().keySet().stream()) - .map(t->Integer.parseInt(t)) + .map(t -> Integer.parseInt(t)) .collect(Collectors.toSet()); } return null; diff --git a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java new file mode 100644 index 00000000..096e88e3 --- /dev/null +++ b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java @@ -0,0 +1,25 @@ +package com.powsybl.afs.timeseriesserver; + +import com.powsybl.afs.storage.AbstractAppStorageTest; +import com.powsybl.afs.storage.AppStorage; +import org.junit.Before; + +import java.net.URI; + +public class TimeSeriesServerAppStorageTest extends AbstractAppStorageTest { + + private URI timeSeriesServerURI; + + @Override + @Before + public void setUp() throws Exception { + timeSeriesServerURI = new URI("http://localhost:9000/"); + super.setUp(); + } + + @Override + protected AppStorage createStorage() { + return null; //TODO + } + +} From 328c1c46802efbd65bf20b6b7033059f9b5dd43e Mon Sep 17 00:00:00 2001 From: amichaut Date: Thu, 6 May 2021 17:27:24 +0200 Subject: [PATCH 35/59] Add time series server project to master pom Signed-off-by: Arthur Michaut --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 68ac3230..c2b9ca20 100644 --- a/pom.xml +++ b/pom.xml @@ -64,6 +64,7 @@ afs-storage-api afs-ws afs-spring-server + afs-timeseries-server From 3fa9d97678fe8d913e79d6bc4d0df9a478edd83e Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Mon, 10 May 2021 14:21:24 +0200 Subject: [PATCH 36/59] Test TimeSeriesServerAppStorage with a MapDB implementation for general delegate Signed-off-by: bhorvilleur Signed-off-by: Arthur Michaut --- afs-timeseries-server/pom.xml | 3 +- .../TimeSeriesServerAppStorage.java | 46 +++++++++++++++---- .../TimeSeriesServerAppStorageTest.java | 4 +- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/afs-timeseries-server/pom.xml b/afs-timeseries-server/pom.xml index 16b3fd67..cc228b9b 100644 --- a/afs-timeseries-server/pom.xml +++ b/afs-timeseries-server/pom.xml @@ -110,10 +110,9 @@ ${project.groupId} - powsybl-afs-ext-base + powsybl-afs-mapdb-storage ${project.version} test - test-jar diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java index 23b24518..2dbab3e9 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java @@ -1,6 +1,8 @@ package com.powsybl.afs.timeseriesserver; import com.powsybl.afs.storage.*; +import com.powsybl.afs.storage.events.AppStorageListener; +import com.powsybl.afs.storage.events.TimeSeriesCreated; import com.powsybl.timeseries.DoubleDataChunk; import com.powsybl.timeseries.StringDataChunk; import com.powsybl.timeseries.TimeSeriesMetadata; @@ -14,20 +16,30 @@ import java.util.Optional; import java.util.Set; -public class TimeSeriesServerAppStorage implements AppStorage { +public class TimeSeriesServerAppStorage extends AbstractAppStorage { /** * This storage is used for all non-timeseries-related operations */ - AppStorage generalDelegate; + private AbstractAppStorage generalDelegate; /** * This storage handles all the timeseries-related operations */ - TimeSeriesSorageDelegate timeSeriesDelegate; + private TimeSeriesSorageDelegate timeSeriesDelegate; - public TimeSeriesServerAppStorage(AppStorage generalDelegate, URI timeSeriesServerURI) { + + /** + * A listener that copies all event from the general delegate event bus to this class event bus. + * This has to be a field because the listeners of an event bus are stored in a WeakReferenceList + */ + private AppStorageListener notifyGeneralDelegateEventListener; + + public TimeSeriesServerAppStorage(AbstractAppStorage generalDelegate, URI timeSeriesServerURI) { this.generalDelegate = generalDelegate; + eventsBus = new InMemoryEventsBus(); + notifyGeneralDelegateEventListener = t -> t.getEvents().forEach(e -> pushEvent(e, t.getTopic())); + generalDelegate.getEventsBus().addListener(notifyGeneralDelegateEventListener); timeSeriesDelegate = new TimeSeriesSorageDelegate(timeSeriesServerURI); timeSeriesDelegate.createAFSAppIfNotExists(); } @@ -125,6 +137,7 @@ public boolean removeData(String nodeId, String name) { @Override public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { timeSeriesDelegate.createTimeSeries(nodeId, metadata); + pushEvent(new TimeSeriesCreated(nodeId, metadata.getName()), APPSTORAGE_TIMESERIES_TOPIC); } @Override @@ -203,14 +216,11 @@ public void removeDependency(String nodeId, String name, String toNodeId) { generalDelegate.removeDependency(nodeId, name, toNodeId); } - @Override - public EventsBus getEventsBus() { - return generalDelegate.getEventsBus(); - } @Override public void flush() { generalDelegate.flush(); + eventsBus.flush(); } @Override @@ -227,4 +237,24 @@ public void close() { public boolean isConsistent(String nodeId) { return generalDelegate.isConsistent(nodeId); } + + @Override + public void setMetadata(String nodeId, NodeGenericMetadata genericMetadata) { + generalDelegate.setMetadata(nodeId, genericMetadata); + } + + @Override + public void setConsistent(String nodeId) { + generalDelegate.setConsistent(nodeId); + } + + @Override + public List getInconsistentNodes() { + return generalDelegate.getInconsistentNodes(); + } + + @Override + public void renameNode(String nodeId, String name) { + generalDelegate.renameNode(nodeId, name); + } } diff --git a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java index 096e88e3..db22d3a6 100644 --- a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java +++ b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java @@ -1,7 +1,9 @@ package com.powsybl.afs.timeseriesserver; +import com.powsybl.afs.mapdb.storage.MapDbAppStorage; import com.powsybl.afs.storage.AbstractAppStorageTest; import com.powsybl.afs.storage.AppStorage; +import com.powsybl.afs.storage.InMemoryEventsBus; import org.junit.Before; import java.net.URI; @@ -19,7 +21,7 @@ public void setUp() throws Exception { @Override protected AppStorage createStorage() { - return null; //TODO + return new TimeSeriesServerAppStorage(MapDbAppStorage.createMem("mem", new InMemoryEventsBus()), timeSeriesServerURI); } } From fd0c955718597464b8a6b32d21cb1a009cace03d Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Tue, 11 May 2021 10:05:55 +0200 Subject: [PATCH 37/59] Bugfixes on create and search time series methods: - Errors on date serialization / deserialization - Bugfix on computation of end date : must be inclusive (last time step of the series) Signed-off-by: bhorvilleur Signed-off-by: Arthur Michaut --- .../TimeSeriesSorageDelegate.java | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index 8d8be8b2..16e6feb6 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -1,5 +1,8 @@ package com.powsybl.afs.timeseriesserver; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.powsybl.afs.storage.AfsStorageException; import com.powsybl.timeseries.RegularTimeSeriesIndex; import com.powsybl.timeseries.TimeSeriesDataType; import com.powsybl.timeseries.TimeSeriesIndex; @@ -9,11 +12,14 @@ import com.powsybl.timeseries.storer.query.search.SearchQueryResults; import org.apache.commons.lang3.NotImplementedException; import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.ws.rs.client.Client; import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response; +import java.io.IOException; import java.net.URI; import java.time.Instant; import java.time.LocalDateTime; @@ -27,6 +33,8 @@ public class TimeSeriesSorageDelegate { private static final String AFS_APP = "AFS"; + private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesSorageDelegate.class); + private URI timeSeriesServerURI; @@ -57,7 +65,11 @@ public void createAFSAppIfNotExists() { return; } - buildBaseRequest(client).request().post(Entity.json(AFS_APP)); + response = buildBaseRequest(client).path(AFS_APP).request().post(Entity.json("")); + if(response.getStatus() != 200) + { + throw new AfsStorageException("Error while initializing AFS timeseries app storage"); + } } finally { client.close(); @@ -85,7 +97,9 @@ public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { buildBaseRequest(client) .path(AFS_APP) .path("series") - .request().post(Entity.json(createQuery)); + .request().post(Entity.json(new ObjectMapper().writeValueAsString(createQuery))); + } catch (JsonProcessingException e) { + LOGGER.error(e.getMessage(), e); } finally { client.close(); } @@ -101,8 +115,11 @@ public SearchQueryResults performSearch(SearchQuery query) { .path(AFS_APP) .path("series") .path("_search") - .request().post(Entity.json(query)); - results = response.readEntity(SearchQueryResults.class); + .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); + String json = response.readEntity(String.class); + results = new ObjectMapper().readValue(json, SearchQueryResults.class); + } catch (IOException e) { + LOGGER.error(e.getMessage(), e); } finally { client.close(); } @@ -146,7 +163,7 @@ public List getTimeSeriesMetadata(String nodeId, Set .stream().map(t -> { long startTime = t.getStartDate().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); long spacing = t.getTimeStepDuration(); - long endTime = startTime + spacing * t.getTimeStepCount(); + long endTime = startTime + spacing * (t.getTimeStepCount() - 1); TimeSeriesIndex index = new RegularTimeSeriesIndex(startTime, endTime, spacing); return new TimeSeriesMetadata(t.getName(), TimeSeriesDataType.DOUBLE, t.getTags(), index); }) From 95ce4ba0a14089afe675b74dd5648e9929df3d2d Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Tue, 11 May 2021 18:00:21 +0200 Subject: [PATCH 38/59] TimeSeriesServerAppStorage : add publish timeseries method Signed-off-by: bhorvilleur Signed-off-by: Arthur Michaut --- .../TimeSeriesServerAppStorage.java | 4 +- .../TimeSeriesSorageDelegate.java | 39 +++++++++++++++++-- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java index 2dbab3e9..4e333c76 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java @@ -3,6 +3,7 @@ import com.powsybl.afs.storage.*; import com.powsybl.afs.storage.events.AppStorageListener; import com.powsybl.afs.storage.events.TimeSeriesCreated; +import com.powsybl.afs.storage.events.TimeSeriesDataUpdated; import com.powsybl.timeseries.DoubleDataChunk; import com.powsybl.timeseries.StringDataChunk; import com.powsybl.timeseries.TimeSeriesMetadata; @@ -173,7 +174,8 @@ public Map> getDoubleTimeSeriesData(String nodeId, @Override public void addDoubleTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { - //TODO + timeSeriesDelegate.addDoubleTimeSeriesData(nodeId, version, timeSeriesName, chunks); + pushEvent(new TimeSeriesDataUpdated(nodeId, timeSeriesName), APPSTORAGE_TIMESERIES_TOPIC); } @Override diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index 16e6feb6..6381c1e7 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -3,13 +3,12 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.powsybl.afs.storage.AfsStorageException; -import com.powsybl.timeseries.RegularTimeSeriesIndex; -import com.powsybl.timeseries.TimeSeriesDataType; -import com.powsybl.timeseries.TimeSeriesIndex; -import com.powsybl.timeseries.TimeSeriesMetadata; +import com.powsybl.timeseries.*; import com.powsybl.timeseries.storer.query.create.CreateQuery; +import com.powsybl.timeseries.storer.query.publish.PublishQuery; import com.powsybl.timeseries.storer.query.search.SearchQuery; import com.powsybl.timeseries.storer.query.search.SearchQueryResults; +import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.NotImplementedException; import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; import org.slf4j.Logger; @@ -187,4 +186,36 @@ public Set getTimeSeriesDataVersions(String nodeId, String timeSeriesNa } return null; } + + public void addDoubleTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { + + //First step : retrieve metadata and reconstitute the time series + TimeSeriesMetadata metadata = getTimeSeriesMetadata(nodeId, Collections.singleton(timeSeriesName)).get(0); + StoredDoubleTimeSeries ts = new StoredDoubleTimeSeries(metadata, chunks); + + //Second step : perform a publish request + PublishQuery publishQuery = new PublishQuery<>(); + publishQuery.setMatrix(nodeId); + publishQuery.setTimeSeriesName(timeSeriesName); + publishQuery.setVersionName(String.valueOf(version)); + publishQuery.setData(ArrayUtils.toObject(ts.toArray())); + + Client client = createClient(); + try { + Response response = buildBaseRequest(client) + .path(AFS_APP) + .path("series") + .path("_search") + .request().post(Entity.json(new ObjectMapper().writeValueAsString(publishQuery))); + if(response.getStatus() != 200) + { + throw new AfsStorageException("Error while publishing data to time series server"); + } + } catch (IOException e) { + LOGGER.error(e.getMessage(), e); + } finally { + client.close(); + } + + } } From e47ffbae9b7cc8344f757b268f0a4c1bf09a86e8 Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Tue, 11 May 2021 18:20:45 +0200 Subject: [PATCH 39/59] Bugfix call publish timeseries method Signed-off-by: bhorvilleur Signed-off-by: Arthur Michaut --- .../powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index 6381c1e7..66d9f15d 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -205,8 +205,7 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie Response response = buildBaseRequest(client) .path(AFS_APP) .path("series") - .path("_search") - .request().post(Entity.json(new ObjectMapper().writeValueAsString(publishQuery))); + .request().put(Entity.json(new ObjectMapper().writeValueAsString(publishQuery))); if(response.getStatus() != 200) { throw new AfsStorageException("Error while publishing data to time series server"); From 715ba5f903736342ebcf15efa18f675ef8443495 Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Tue, 11 May 2021 20:40:48 +0200 Subject: [PATCH 40/59] TimeSeriesServerAppStorage : add fetch timeseries method Signed-off-by: bhorvilleur Signed-off-by: Arthur Michaut --- .../TimeSeriesServerAppStorage.java | 2 +- .../TimeSeriesSorageDelegate.java | 60 ++++++++++++++++--- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java index 4e333c76..4f7ac372 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java @@ -169,7 +169,7 @@ public Set getTimeSeriesDataVersions(String nodeId, String timeSeriesNa @Override public Map> getDoubleTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { //TODO - return null; + return timeSeriesDelegate.getDoubleTimeSeriesData(nodeId, timeSeriesNames, version); } @Override diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index 66d9f15d..57132b4b 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -5,6 +5,8 @@ import com.powsybl.afs.storage.AfsStorageException; import com.powsybl.timeseries.*; import com.powsybl.timeseries.storer.query.create.CreateQuery; +import com.powsybl.timeseries.storer.query.fetch.FetchQuery; +import com.powsybl.timeseries.storer.query.fetch.FetchQueryResult; import com.powsybl.timeseries.storer.query.publish.PublishQuery; import com.powsybl.timeseries.storer.query.search.SearchQuery; import com.powsybl.timeseries.storer.query.search.SearchQueryResults; @@ -23,10 +25,7 @@ import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; public class TimeSeriesSorageDelegate { @@ -65,8 +64,7 @@ public void createAFSAppIfNotExists() { } response = buildBaseRequest(client).path(AFS_APP).request().post(Entity.json("")); - if(response.getStatus() != 200) - { + if (response.getStatus() != 200) { throw new AfsStorageException("Error while initializing AFS timeseries app storage"); } @@ -206,8 +204,7 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie .path(AFS_APP) .path("series") .request().put(Entity.json(new ObjectMapper().writeValueAsString(publishQuery))); - if(response.getStatus() != 200) - { + if (response.getStatus() != 200) { throw new AfsStorageException("Error while publishing data to time series server"); } } catch (IOException e) { @@ -217,4 +214,51 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie } } + + public Map> getDoubleTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { + + String versionString = String.valueOf(version); + SearchQuery searchQuery = new SearchQuery(); + searchQuery.setMatrix(nodeId); + searchQuery.setNames(timeSeriesNames); + SearchQueryResults results = performSearch(searchQuery); + List versionIDs = results.getTimeSeriesInformations() + .stream() + .map(t->t.getVersions().get(versionString)) + .collect(Collectors.toList()); + + Map versionIDToTSName = results.getTimeSeriesInformations() + .stream() + .collect(Collectors.toMap(t->t.getVersions().get(versionString), t->t.getName())); + + FetchQuery query = new FetchQuery(versionIDs, null, null); + Client client = createClient(); + try { + Response response = buildBaseRequest(client) + .path(AFS_APP) + .path("series") + .path("_fetch") + .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); + if (response.getStatus() != 200) { + throw new AfsStorageException("Error while fetching data from time series server"); + } + String json = response.readEntity(String.class); + FetchQueryResult fetchResults = new ObjectMapper().readValue(json, FetchQueryResult.class); + + Map> toReturn = new HashMap<>(); + for(int i=0; i Date: Tue, 18 May 2021 17:21:49 +0200 Subject: [PATCH 41/59] Adapt AbstractAppStorageTest to storages which do not handle chunk conservation and string timeseries, such as TimeSeriesServerAppStorage Signed-off-by: bhorvilleur Signed-off-by: Arthur Michaut --- .../afs/storage/AbstractAppStorageTest.java | 119 +++++++++++------- .../TimeSeriesServerAppStorageTest.java | 4 + 2 files changed, 78 insertions(+), 45 deletions(-) diff --git a/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java b/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java index bab2dc05..cdc95d49 100644 --- a/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java +++ b/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java @@ -43,10 +43,23 @@ public abstract class AbstractAppStorageTest { protected AppStorage storage; + protected boolean conservesChunks; + + protected boolean handlesStringTimeSeries; + protected BlockingQueue eventStack; protected AppStorageListener l = eventList -> eventStack.addAll(eventList.getEvents()); + public AbstractAppStorageTest() { + this(true, true); + } + + public AbstractAppStorageTest(boolean conservesChunks, boolean handlesStringTimeSeries) { + this.conservesChunks = conservesChunks; + this.handlesStringTimeSeries = handlesStringTimeSeries; + } + protected abstract AppStorage createStorage(); @Before @@ -385,8 +398,16 @@ public void test() throws IOException, InterruptedException { assertTrue(storage.getTimeSeriesMetadata(testData3Info.getId(), Sets.newHashSet("ts1")).isEmpty()); // 14) add data to double time series - storage.addDoubleTimeSeriesData(testData2Info.getId(), 0, "ts1", Arrays.asList(new UncompressedDoubleDataChunk(2, new double[] {1d, 2d}), - new UncompressedDoubleDataChunk(5, new double[] {3d}))); + if(conservesChunks) + { + storage.addDoubleTimeSeriesData(testData2Info.getId(), 0, "ts1", Arrays.asList(new UncompressedDoubleDataChunk(2, new double[] {1d, 2d}), + new UncompressedDoubleDataChunk(5, new double[] {3d}))); + } + else + { + storage.addDoubleTimeSeriesData(testData2Info.getId(), 0, "ts1", Arrays.asList(new UncompressedDoubleDataChunk(0, new double[] {1d, 2d, 3d}))); + } + storage.flush(); // check event @@ -399,52 +420,60 @@ public void test() throws IOException, InterruptedException { // check double time series data query Map> doubleTimeSeriesData = storage.getDoubleTimeSeriesData(testData2Info.getId(), Sets.newHashSet("ts1"), 0); assertEquals(1, doubleTimeSeriesData.size()); - assertEquals(Arrays.asList(new UncompressedDoubleDataChunk(2, new double[] {1d, 2d}), - new UncompressedDoubleDataChunk(5, new double[] {3d})), - doubleTimeSeriesData.get("ts1")); + if (conservesChunks) { + assertEquals(Arrays.asList(new UncompressedDoubleDataChunk(2, new double[] {1d, 2d}), + new UncompressedDoubleDataChunk(5, new double[] {3d})), + doubleTimeSeriesData.get("ts1")); + } + else { + assertEquals(Arrays.asList(new UncompressedDoubleDataChunk(0, new double[] {1d, 2d, 3d})), + doubleTimeSeriesData.get("ts1")); + } assertTrue(storage.getDoubleTimeSeriesData(testData3Info.getId(), Sets.newHashSet("ts1"), 0).isEmpty()); // 15) create a second string time series - TimeSeriesMetadata metadata2 = new TimeSeriesMetadata("ts2", - TimeSeriesDataType.STRING, - ImmutableMap.of(), - RegularTimeSeriesIndex.create(Interval.parse("2015-01-01T00:00:00Z/2015-01-01T01:15:00Z"), - Duration.ofMinutes(15))); - storage.createTimeSeries(testData2Info.getId(), metadata2); - storage.flush(); - - // check event - assertEventStack(new TimeSeriesCreated(testData2Info.getId(), "ts2")); - - // check string time series query - assertEquals(Sets.newHashSet("ts1", "ts2"), storage.getTimeSeriesNames(testData2Info.getId())); - metadataList = storage.getTimeSeriesMetadata(testData2Info.getId(), Sets.newHashSet("ts1")); - assertEquals(1, metadataList.size()); - - // 16) add data to double time series - storage.addStringTimeSeriesData(testData2Info.getId(), 0, "ts2", Arrays.asList(new UncompressedStringDataChunk(2, new String[] {"a", "b"}), - new UncompressedStringDataChunk(5, new String[] {"c"}))); - storage.flush(); - - // check event - assertEventStack(new TimeSeriesDataUpdated(testData2Info.getId(), "ts2")); - - // check string time series data query - Map> stringTimeSeriesData = storage.getStringTimeSeriesData(testData2Info.getId(), Sets.newHashSet("ts2"), 0); - assertEquals(1, stringTimeSeriesData.size()); - assertEquals(Arrays.asList(new UncompressedStringDataChunk(2, new String[] {"a", "b"}), - new UncompressedStringDataChunk(5, new String[] {"c"})), - stringTimeSeriesData.get("ts2")); - - // 17) clear time series - storage.clearTimeSeries(testData2Info.getId()); - storage.flush(); - - // check event - assertEventStack(new TimeSeriesCleared(testData2Info.getId())); - - // check there is no more time series - assertTrue(storage.getTimeSeriesNames(testData2Info.getId()).isEmpty()); + if(handlesStringTimeSeries) { + TimeSeriesMetadata metadata2 = new TimeSeriesMetadata("ts2", + TimeSeriesDataType.STRING, + ImmutableMap.of(), + RegularTimeSeriesIndex.create(Interval.parse("2015-01-01T00:00:00Z/2015-01-01T01:15:00Z"), + Duration.ofMinutes(15))); + storage.createTimeSeries(testData2Info.getId(), metadata2); + storage.flush(); + + // check event + assertEventStack(new TimeSeriesCreated(testData2Info.getId(), "ts2")); + + // check string time series query + assertEquals(Sets.newHashSet("ts1", "ts2"), storage.getTimeSeriesNames(testData2Info.getId())); + metadataList = storage.getTimeSeriesMetadata(testData2Info.getId(), Sets.newHashSet("ts1")); + assertEquals(1, metadataList.size()); + + // 16) add data to double time series + storage.addStringTimeSeriesData(testData2Info.getId(), 0, "ts2", Arrays.asList(new UncompressedStringDataChunk(2, new String[] {"a", "b"}), + new UncompressedStringDataChunk(5, new String[] {"c"}))); + storage.flush(); + + // check event + assertEventStack(new TimeSeriesDataUpdated(testData2Info.getId(), "ts2")); + + // check string time series data query + Map> stringTimeSeriesData = storage.getStringTimeSeriesData(testData2Info.getId(), Sets.newHashSet("ts2"), 0); + assertEquals(1, stringTimeSeriesData.size()); + assertEquals(Arrays.asList(new UncompressedStringDataChunk(2, new String[] {"a", "b"}), + new UncompressedStringDataChunk(5, new String[] {"c"})), + stringTimeSeriesData.get("ts2")); + + // 17) clear time series + storage.clearTimeSeries(testData2Info.getId()); + storage.flush(); + + // check event + assertEventStack(new TimeSeriesCleared(testData2Info.getId())); + + // check there is no more time series + assertTrue(storage.getTimeSeriesNames(testData2Info.getId()).isEmpty()); + } // 18) change parent test NodeInfo folder1Info = storage.createNode(rootFolderInfo.getId(), "test1", FOLDER_PSEUDO_CLASS, "", 0, new NodeGenericMetadata()); diff --git a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java index db22d3a6..a8e17c33 100644 --- a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java +++ b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java @@ -12,6 +12,10 @@ public class TimeSeriesServerAppStorageTest extends AbstractAppStorageTest { private URI timeSeriesServerURI; + public TimeSeriesServerAppStorageTest() { + super(false, false); + } + @Override @Before public void setUp() throws Exception { From 30f47ee26df637e8cbb5420a1739a34d5a4b6fbc Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Tue, 18 May 2021 17:24:29 +0200 Subject: [PATCH 42/59] Specialize TimeSeriesServer in Double time series String time series will have to be handled with different methods and end points. Signed-off-by: bhorvilleur Signed-off-by: Arthur Michaut --- .../afs/timeseriesserver/TimeSeriesSorageDelegate.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index 57132b4b..e7576549 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -192,7 +192,7 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie StoredDoubleTimeSeries ts = new StoredDoubleTimeSeries(metadata, chunks); //Second step : perform a publish request - PublishQuery publishQuery = new PublishQuery<>(); + PublishQuery publishQuery = new PublishQuery(); publishQuery.setMatrix(nodeId); publishQuery.setTimeSeriesName(timeSeriesName); publishQuery.setVersionName(String.valueOf(version)); @@ -243,7 +243,7 @@ public Map> getDoubleTimeSeriesData(String nodeId, throw new AfsStorageException("Error while fetching data from time series server"); } String json = response.readEntity(String.class); - FetchQueryResult fetchResults = new ObjectMapper().readValue(json, FetchQueryResult.class); + FetchQueryResult fetchResults = new ObjectMapper().readValue(json, FetchQueryResult.class); Map> toReturn = new HashMap<>(); for(int i=0; i> getDoubleTimeSeriesData(String nodeId, UncompressedDoubleDataChunk chunk = new UncompressedDoubleDataChunk(0, values); toReturn.put(versionIDToTSName.get(versionIDs.get(i)), Arrays.asList(chunk)); } + return toReturn; } catch (IOException e) { LOGGER.error(e.getMessage(), e); From f09a560f89d2b82aff85153e2fd03ebef80ec561 Mon Sep 17 00:00:00 2001 From: bhorvilleur Date: Wed, 26 May 2021 14:56:31 +0200 Subject: [PATCH 43/59] Checkstyle fix Signed-off-by: bhorvilleur Signed-off-by: Arthur Michaut --- .../afs/timeseriesserver/TimeSeriesServerAppStorage.java | 1 - .../afs/timeseriesserver/TimeSeriesSorageDelegate.java | 8 +++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java index 4f7ac372..6d3a741c 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java @@ -218,7 +218,6 @@ public void removeDependency(String nodeId, String name, String toNodeId) { generalDelegate.removeDependency(nodeId, name, toNodeId); } - @Override public void flush() { generalDelegate.flush(); diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index e7576549..03d7562e 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -33,7 +33,6 @@ public class TimeSeriesSorageDelegate { private static final String AFS_APP = "AFS"; private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesSorageDelegate.class); - private URI timeSeriesServerURI; public TimeSeriesSorageDelegate(URI timeSeriesServerURI) { @@ -224,12 +223,12 @@ public Map> getDoubleTimeSeriesData(String nodeId, SearchQueryResults results = performSearch(searchQuery); List versionIDs = results.getTimeSeriesInformations() .stream() - .map(t->t.getVersions().get(versionString)) + .map(t -> t.getVersions().get(versionString)) .collect(Collectors.toList()); Map versionIDToTSName = results.getTimeSeriesInformations() .stream() - .collect(Collectors.toMap(t->t.getVersions().get(versionString), t->t.getName())); + .collect(Collectors.toMap(t -> t.getVersions().get(versionString), t -> t.getName())); FetchQuery query = new FetchQuery(versionIDs, null, null); Client client = createClient(); @@ -246,8 +245,7 @@ public Map> getDoubleTimeSeriesData(String nodeId, FetchQueryResult fetchResults = new ObjectMapper().readValue(json, FetchQueryResult.class); Map> toReturn = new HashMap<>(); - for(int i=0; i Date: Wed, 26 May 2021 15:04:22 +0200 Subject: [PATCH 44/59] AFS time series server : customize the "AFS" app Signed-off-by: bhorvilleur Signed-off-by: Arthur Michaut --- .../TimeSeriesSorageDelegate.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java index 03d7562e..730efb0a 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java @@ -30,13 +30,26 @@ public class TimeSeriesSorageDelegate { - private static final String AFS_APP = "AFS"; + private static final String AFS_DEFAULT_APP = "AFS"; private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesSorageDelegate.class); + /** + * The name of the app in TimeSeriesServer, in which AFS will store time series + */ + private final String app; + + /** + * The address of the TimeSeriesServer + */ private URI timeSeriesServerURI; public TimeSeriesSorageDelegate(URI timeSeriesServerURI) { + this(timeSeriesServerURI, AFS_DEFAULT_APP); + } + + public TimeSeriesSorageDelegate(URI timeSeriesServerURI, String app) { this.timeSeriesServerURI = timeSeriesServerURI; + this.app = app; } public static Client createClient() { @@ -58,11 +71,11 @@ public void createAFSAppIfNotExists() { Response response = buildBaseRequest(client).request().get(); Collection apps = response.readEntity(Collection.class); - if (apps.contains(AFS_APP)) { + if (apps.contains(app)) { return; } - response = buildBaseRequest(client).path(AFS_APP).request().post(Entity.json("")); + response = buildBaseRequest(client).path(app).request().post(Entity.json("")); if (response.getStatus() != 200) { throw new AfsStorageException("Error while initializing AFS timeseries app storage"); } @@ -91,7 +104,7 @@ public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { Client client = createClient(); try { buildBaseRequest(client) - .path(AFS_APP) + .path(app) .path("series") .request().post(Entity.json(new ObjectMapper().writeValueAsString(createQuery))); } catch (JsonProcessingException e) { @@ -108,7 +121,7 @@ public SearchQueryResults performSearch(SearchQuery query) { Client client = createClient(); try { Response response = buildBaseRequest(client) - .path(AFS_APP) + .path(app) .path("series") .path("_search") .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); @@ -200,7 +213,7 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie Client client = createClient(); try { Response response = buildBaseRequest(client) - .path(AFS_APP) + .path(app) .path("series") .request().put(Entity.json(new ObjectMapper().writeValueAsString(publishQuery))); if (response.getStatus() != 200) { @@ -234,7 +247,7 @@ public Map> getDoubleTimeSeriesData(String nodeId, Client client = createClient(); try { Response response = buildBaseRequest(client) - .path(AFS_APP) + .path(app) .path("series") .path("_fetch") .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); From d3cd8202ca4cd50d06028955e6af7cb223651eac Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 09:57:20 +0200 Subject: [PATCH 45/59] Code style fixes Signed-off-by: Arthur Michaut --- .../powsybl/afs/storage/AbstractAppStorageTest.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java b/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java index cdc95d49..f6d5b1e5 100644 --- a/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java +++ b/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java @@ -398,13 +398,10 @@ public void test() throws IOException, InterruptedException { assertTrue(storage.getTimeSeriesMetadata(testData3Info.getId(), Sets.newHashSet("ts1")).isEmpty()); // 14) add data to double time series - if(conservesChunks) - { + if (conservesChunks) { storage.addDoubleTimeSeriesData(testData2Info.getId(), 0, "ts1", Arrays.asList(new UncompressedDoubleDataChunk(2, new double[] {1d, 2d}), new UncompressedDoubleDataChunk(5, new double[] {3d}))); - } - else - { + } else { storage.addDoubleTimeSeriesData(testData2Info.getId(), 0, "ts1", Arrays.asList(new UncompressedDoubleDataChunk(0, new double[] {1d, 2d, 3d}))); } @@ -424,10 +421,8 @@ public void test() throws IOException, InterruptedException { assertEquals(Arrays.asList(new UncompressedDoubleDataChunk(2, new double[] {1d, 2d}), new UncompressedDoubleDataChunk(5, new double[] {3d})), doubleTimeSeriesData.get("ts1")); - } - else { - assertEquals(Arrays.asList(new UncompressedDoubleDataChunk(0, new double[] {1d, 2d, 3d})), - doubleTimeSeriesData.get("ts1")); + } else { + assertEquals(Arrays.asList(new UncompressedDoubleDataChunk(0, new double[]{1d, 2d, 3d})), doubleTimeSeriesData.get("ts1")); } assertTrue(storage.getDoubleTimeSeriesData(testData3Info.getId(), Sets.newHashSet("ts1"), 0).isEmpty()); From 955ead9a2d429164fd7fb623fe720554f78135c9 Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 18:24:11 +0200 Subject: [PATCH 46/59] Create a special module for timeseries config services Signed-off-by: Arthur Michaut --- afs-timeseries-server-storage/pom.xml | 87 +++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 afs-timeseries-server-storage/pom.xml diff --git a/afs-timeseries-server-storage/pom.xml b/afs-timeseries-server-storage/pom.xml new file mode 100644 index 00000000..fe2942cf --- /dev/null +++ b/afs-timeseries-server-storage/pom.xml @@ -0,0 +1,87 @@ + + + + 4.0.0 + + 3.6.0-SNAPSHOT + 2.12.1 + 0.0.1-SNAPSHOT + + + + powsybl-afs + com.powsybl + 3.6.0-SNAPSHOT + + + powsybl-afs-timeseries-server-storage + AFS time series server filesystem storage implementations + An AFS time series provider based on time series server + + + + com.powsybl + powsybl-afs-storage-api + ${project.version} + + + com.powsybl + powsybl-time-series-server-interfaces + ${time-series-server.version} + + + + org.slf4j + slf4j-simple + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + org.apache.commons + commons-lang3 + 3.11 + + + org.jboss.resteasy + resteasy-client + provided + + + + + com.powsybl + powsybl-afs-mapdb-storage + ${project.version} + test + + + junit + junit + test + + + com.powsybl + powsybl-afs-storage-api + 3.6.0-SNAPSHOT + test-jar + test + + + org.mock-server + mockserver-netty + 5.11.2 + + + \ No newline at end of file From 5b63954952570515f8cdef84136fd4a0ab7e9ea9 Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 18:24:57 +0200 Subject: [PATCH 47/59] Initialization for TS server app file system configuration components Signed-off-by: Arthur Michaut --- .../TSServerAppFileSystem.java | 14 +++ .../TSServerAppFileSystemConfig.java | 96 +++++++++++++++++++ .../TSServerAppFileSystemProvider.java | 70 ++++++++++++++ 3 files changed, 180 insertions(+) create mode 100644 afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystem.java create mode 100644 afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfig.java create mode 100644 afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystem.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystem.java new file mode 100644 index 00000000..b5ef4f22 --- /dev/null +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystem.java @@ -0,0 +1,14 @@ +package com.powsybl.afs.timeseriesserver; + +import com.powsybl.afs.AppFileSystem; +import com.powsybl.afs.storage.AppStorage; + +/** + * @author amichaut@artelys.com + */ +public class TSServerAppFileSystem extends AppFileSystem { + + public TSServerAppFileSystem(final String name, final boolean remotelyAccessible, final AppStorage storage) { + super(name, remotelyAccessible, storage); + } +} diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfig.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfig.java new file mode 100644 index 00000000..67c9c8a6 --- /dev/null +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfig.java @@ -0,0 +1,96 @@ +package com.powsybl.afs.timeseriesserver; + +import com.powsybl.afs.mapdb.MapDbAppFileSystemConfig; +import com.powsybl.afs.storage.AbstractAppFileSystemConfig; +import com.powsybl.commons.config.PlatformConfig; +import lombok.Getter; +import lombok.Setter; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; + +/** + * App file system configuration for time series server app storage + * + * @author amichaut@artelys.com + */ +public class TSServerAppFileSystemConfig extends AbstractAppFileSystemConfig { + + private static final String TSSERVER_APP_FILE_SYSTEM = "tsserver-app-file-system"; + private static final String DRIVE_NAME = "drive-name"; + private static final String REMOTELY_ACCESSIBLE = "remotely-accessible"; + private static final boolean DEFAULT_REMOTELY_ACCESSIBLE = false; + private static final String HOST = "host"; + private static final String PORT = "port"; + private static final String SCHEME = "scheme"; + private static final String APP = "app"; + private static final String DEFAULT_APP = "AFS"; + public static final String DELEGATE_MAPDB_APP_FILE_SYSTEM = "delegate-mapdb-app-file-system"; + + @Setter + private String host; + + @Setter + private int port; + + @Setter + private String scheme; + + @Getter + @Setter + private String app = DEFAULT_APP; + + @Getter + @Setter + private MapDbAppFileSystemConfig delegateConfig; + + public TSServerAppFileSystemConfig(final String driveName, final boolean remotelyAccessible) { + super(driveName, remotelyAccessible); + } + + public static TSServerAppFileSystemConfig load() { + return load(PlatformConfig.defaultConfig()); + } + + public static TSServerAppFileSystemConfig load(final PlatformConfig platformConfig) { + return platformConfig.getOptionalModuleConfig(TSSERVER_APP_FILE_SYSTEM) + .map(moduleConfig -> { + final String driveName; + if (moduleConfig.hasProperty(DRIVE_NAME)) { + driveName = moduleConfig.getStringProperty(DRIVE_NAME); + } else { + throw new IllegalArgumentException("Please provide a drive name for timeseries server app file system configuration"); + } + final boolean remotelyAccessible = moduleConfig.getBooleanProperty(REMOTELY_ACCESSIBLE, DEFAULT_REMOTELY_ACCESSIBLE); + final TSServerAppFileSystemConfig config = new TSServerAppFileSystemConfig(driveName, remotelyAccessible); + if (moduleConfig.hasProperty(HOST)) { + config.setHost(moduleConfig.getStringProperty(HOST)); + } + if (moduleConfig.hasProperty(PORT)) { + config.setPort(moduleConfig.getIntProperty(PORT)); + } + if (moduleConfig.hasProperty(SCHEME)) { + config.setScheme(moduleConfig.getStringProperty(SCHEME)); + } + config.setApp(moduleConfig.getStringProperty(APP, DEFAULT_APP)); + // Load delegate config + final List delegateConfig = MapDbAppFileSystemConfig.load(platformConfig.getModuleConfig(DELEGATE_MAPDB_APP_FILE_SYSTEM)); + if (delegateConfig.size() != 1) { + throw new IllegalArgumentException("A single mapdb delegate configuration is necessary"); + } + config.setDelegateConfig(delegateConfig.get(0)); + return config; + }) + .orElse(null); + } + + /** + * @return target URI for timeseries service + * @throws URISyntaxException if information does not allow to build a proper URI + */ + public URI getURI() throws URISyntaxException { + return new URI(scheme, null, host, port, null, null, null); + } + +} diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java new file mode 100644 index 00000000..be33f2d2 --- /dev/null +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java @@ -0,0 +1,70 @@ +package com.powsybl.afs.timeseriesserver; + +import com.google.auto.service.AutoService; +import com.powsybl.afs.AppFileSystem; +import com.powsybl.afs.AppFileSystemProvider; +import com.powsybl.afs.AppFileSystemProviderContext; +import com.powsybl.afs.mapdb.storage.MapDbAppStorage; +import com.powsybl.afs.storage.AbstractAppStorage; +import com.powsybl.afs.storage.EventsBus; +import com.powsybl.afs.timeseriesserver.storage.TimeSeriesServerAppStorage; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; + +/** + * @author amichaut@artelys.com + */ +@AutoService(AppFileSystemProvider.class) +public class TSServerAppFileSystemProvider implements AppFileSystemProvider { + + private final TSServerAppFileSystemConfig configuration; + + private final TimeSeriesServerAppStorage.TimeSeriesServerAppStorageProvider storageProvider; + + private final MapDbAppStorage.MapDbAppStorageProvider delegateStorageProvider; + + public TSServerAppFileSystemProvider() { + this( + TSServerAppFileSystemConfig.load(), + TimeSeriesServerAppStorage::new, + (name, path, eventsStore) -> MapDbAppStorage.createMmapFile(name, path.toFile(), eventsStore) + ); + } + + public TSServerAppFileSystemProvider(final TSServerAppFileSystemConfig configuration, + final TimeSeriesServerAppStorage.TimeSeriesServerAppStorageProvider provider, + final MapDbAppStorage.MapDbAppStorageProvider delegateStorageProvider) { + this.configuration = configuration; + this.storageProvider = provider; + this.delegateStorageProvider = delegateStorageProvider; + } + + @Override + public List getFileSystems(final AppFileSystemProviderContext context) { + return Collections.singletonList(getFileSystem(context)); + } + + /** + * @param context a context holding necessary event bus + * @return a single time series server app file system, built using internal configuration and provided context + */ + private TSServerAppFileSystem getFileSystem(final AppFileSystemProviderContext context) { + // Make an URI form configuration + URI uri; + try { + uri = configuration.getURI(); + } catch (URISyntaxException e) { + throw new IllegalStateException("Could not build a proper target URI for time series server"); + } + MapDbAppStorage delegateAppStorage = delegateStorageProvider.apply(configuration.getDriveName(), configuration.getDelegateConfig().getDbFile(), context.getEventsBus()); + return new TSServerAppFileSystem( + configuration.getDriveName(), + configuration.isRemotelyAccessible(), + storageProvider.apply(uri, configuration.getApp(), delegateAppStorage) + ); + } +} From 99c615251eb25ee798b7fa2a5dd605617195f4d7 Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 18:25:56 +0200 Subject: [PATCH 48/59] Add unit tests to TS server config components Signed-off-by: Arthur Michaut --- .../TSServerAppFileSystemConfigTest.java | 84 +++++++++++++ .../TSServerAppFileSystemProviderTest.java | 118 ++++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfigTest.java create mode 100644 afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java diff --git a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfigTest.java b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfigTest.java new file mode 100644 index 00000000..fbc18f69 --- /dev/null +++ b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemConfigTest.java @@ -0,0 +1,84 @@ +package com.powsybl.afs.timeseriesserver; + +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import com.powsybl.afs.mapdb.MapDbAppFileSystemConfig; +import com.powsybl.commons.config.InMemoryPlatformConfig; +import com.powsybl.commons.config.MapModuleConfig; +import com.powsybl.commons.config.PlatformConfig; +import org.junit.Before; +import org.junit.Test; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileSystem; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class TSServerAppFileSystemConfigTest { + + private static final String APP = "test-app"; + private static final String PORT = "8765"; + private static final String HOST = "localhost"; + private static final String SCHEME = "http"; + private static final String REMOTELY_ACCESSIBLE = "false"; + private static final String DRIVE_NAME = "test-drive"; + private static final String DB_FILE = "/db/test.db"; + + private FileSystem fileSystem; + private PlatformConfig platformConfig; + + @Before + public void setup() { + fileSystem = Jimfs.newFileSystem(Configuration.unix()); + InMemoryPlatformConfig conf = new InMemoryPlatformConfig(fileSystem); + final MapModuleConfig tsServerConfig = conf.createModuleConfig("tsserver-app-file-system"); + tsServerConfig.setStringProperty("drive-name", DRIVE_NAME); + tsServerConfig.setStringProperty("remotely-accessible", REMOTELY_ACCESSIBLE); + tsServerConfig.setStringProperty("scheme", SCHEME); + tsServerConfig.setStringProperty("host", HOST); + tsServerConfig.setStringProperty("port", PORT); + tsServerConfig.setStringProperty("app", APP); + final MapModuleConfig mapdbConfig = conf.createModuleConfig("delegate-mapdb-app-file-system"); + mapdbConfig.setStringProperty("drive-name", DRIVE_NAME); + mapdbConfig.setPathProperty("db-file", fileSystem.getPath(DB_FILE)); + mapdbConfig.setStringProperty("remotely-accessible", REMOTELY_ACCESSIBLE); + platformConfig = conf; + } + + @Test + public void loadTest() throws URISyntaxException { + final TSServerAppFileSystemConfig conf = TSServerAppFileSystemConfig.load(platformConfig); + assertEquals(conf.getApp(), APP); + assertEquals(conf.getDriveName(), DRIVE_NAME); + final URI uri = conf.getURI(); + assertEquals(uri.getScheme(), SCHEME); + assertEquals(uri.getPort(), Integer.parseInt(PORT)); + assertEquals(uri.getHost(), HOST); + assertEquals(conf.isRemotelyAccessible(), Boolean.valueOf(REMOTELY_ACCESSIBLE)); + + final MapDbAppFileSystemConfig delegateConf = conf.getDelegateConfig(); + assertEquals(delegateConf.getDriveName(), DRIVE_NAME); + assertEquals(delegateConf.isRemotelyAccessible(), Boolean.valueOf(REMOTELY_ACCESSIBLE)); + assertEquals(fileSystem.getPath(DB_FILE), delegateConf.getDbFile()); + } + + /** + * Refer to config.yaml file in test resources + */ + @Test + public void emptyLoadTest() throws URISyntaxException { + final TSServerAppFileSystemConfig load = TSServerAppFileSystemConfig.load(); + assertEquals(load.getDriveName(), "test-fs"); + assertFalse(load.isRemotelyAccessible()); + assertEquals(load.getApp(), "AFS"); + final URI uri = load.getURI(); + assertEquals(uri.getScheme(), "http"); + assertEquals(uri.getHost(), "localhost"); + assertEquals(uri.getPort(), 8080); + final MapDbAppFileSystemConfig delegateConfig = load.getDelegateConfig(); + assertEquals(delegateConfig.getDriveName(), "test-fs"); + assertFalse(delegateConfig.isRemotelyAccessible()); + } +} diff --git a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java new file mode 100644 index 00000000..190a916b --- /dev/null +++ b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java @@ -0,0 +1,118 @@ +package com.powsybl.afs.timeseriesserver; + +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import com.powsybl.afs.AppFileSystem; +import com.powsybl.afs.AppFileSystemProviderContext; +import com.powsybl.afs.mapdb.MapDbAppFileSystemConfig; +import com.powsybl.afs.mapdb.storage.MapDbAppStorage; +import com.powsybl.afs.storage.AbstractAppStorage; +import com.powsybl.afs.storage.EventsBus; +import com.powsybl.afs.storage.InMemoryEventsBus; +import com.powsybl.afs.timeseriesserver.storage.TimeSeriesServerAppStorage; +import com.powsybl.computation.ComputationManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockserver.integration.ClientAndServer; +import org.mockserver.model.MediaType; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; +import static org.junit.Assert.assertEquals; + +public class TSServerAppFileSystemProviderTest { + + private static final int srvPort = 9876; + public static final String DRIVE = "drive"; + + private TSServerAppFileSystemConfig config; + private MapDbAppFileSystemConfig delegateConfig; + + /** + * Mock server to simulate time series provider + */ + private ClientAndServer mockServer; + + /** + * In-memory fs for mapDB + */ + private FileSystem fileSystem; + + /** + * mapDB file + */ + private Path dbFile; + + @Before + public void setUp() { + mockServer = ClientAndServer.startClientAndServer(srvPort); + setupMockServer(); + + // Setup connection parameters + config = new TSServerAppFileSystemConfig(DRIVE, true); + config.setScheme("http"); + config.setHost("localhost"); + config.setPort(mockServer.getPort()); + + // Setup delegate config + fileSystem = Jimfs.newFileSystem(Configuration.unix()); + dbFile = fileSystem.getPath("/db"); + delegateConfig = new MapDbAppFileSystemConfig(DRIVE, true, dbFile); + config.setDelegateConfig(delegateConfig); + } + + /** + * Setup mock server for tests + */ + private void setupMockServer() { + mockServer.when(request() + .withMethod("GET") + .withPath("/v1/timeseries/apps") + ).respond(response() + .withStatusCode(200) + .withContentType(MediaType.APPLICATION_JSON) + .withBody("[]") + ); + mockServer.when(request() + .withMethod("POST") + .withPath("/v1/timeseries/apps/.*") + ).respond(response() + .withStatusCode(200) + .withContentType(MediaType.APPLICATION_JSON) + ); + } + + @After + public void tearDown() throws IOException { + mockServer.stop(); + fileSystem.close(); + } + + @Test + public void provideTest() { + // Build a mock computation context + ComputationManager computationManager = Mockito.mock(ComputationManager.class); + final AppFileSystemProviderContext context = new AppFileSystemProviderContext(computationManager, null, new InMemoryEventsBus()); + // Build a new provider + final MapDbAppStorage.MapDbAppStorageProvider delegateAppStorageProvider = + (name, path, eventsStore) -> MapDbAppStorage.createMem(name, eventsStore); + final TimeSeriesServerAppStorage.TimeSeriesServerAppStorageProvider appStorageProvider = TimeSeriesServerAppStorage::new; + TSServerAppFileSystemProvider provider = new TSServerAppFileSystemProvider(config, appStorageProvider, delegateAppStorageProvider); + // Check that FS is correct + final List fileSystems = provider.getFileSystems(context); + assertEquals(fileSystems.size(), 1); + assertTrue(fileSystems.get(0) instanceof TSServerAppFileSystem); + final TSServerAppFileSystem fs = (TSServerAppFileSystem) fileSystems.get(0); + assertEquals(fs.getName(), DRIVE); + assertTrue(fs.isRemotelyAccessible()); + } +} From ff059476f198378d4c74cfc964de6a00935f08bb Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 18:26:18 +0200 Subject: [PATCH 49/59] Setup a conf.yaml file for tests Signed-off-by: Arthur Michaut --- afs-timeseries-server/src/test/resources/conf.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 afs-timeseries-server/src/test/resources/conf.yaml diff --git a/afs-timeseries-server/src/test/resources/conf.yaml b/afs-timeseries-server/src/test/resources/conf.yaml new file mode 100644 index 00000000..5b85e3bb --- /dev/null +++ b/afs-timeseries-server/src/test/resources/conf.yaml @@ -0,0 +1,12 @@ +tsserver-app-file-system: + drive-name: "test-fs" + remotely-accessible: false + app: AFS + scheme: http + host: localhost + port: 8080 +delegate-mapdb-app-file-system: + drive-name: "test-fs" + remotely-accessible: false + db-file: "/db/test.db" + From 8c0ebb3fd3f3bdf573cf4bf4bb8f07c1970c1889 Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 18:29:39 +0200 Subject: [PATCH 50/59] Rework ts server storage classes to setup the modules separation (storage / config) Signed-off-by: Arthur Michaut --- .../storage}/TimeSeriesServerAppStorage.java | 22 +++++++++++++------ .../storage/TimeSeriesStorageDelegate.java | 13 ++++------- .../TimeSeriesServerAppStorageTest.java | 6 ++++- 3 files changed, 24 insertions(+), 17 deletions(-) rename {afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver => afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage}/TimeSeriesServerAppStorage.java (91%) rename afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java => afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java (96%) rename {afs-timeseries-server => afs-timeseries-server-storage}/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java (69%) diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java similarity index 91% rename from afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java rename to afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java index 6d3a741c..9607a168 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java @@ -1,4 +1,4 @@ -package com.powsybl.afs.timeseriesserver; +package com.powsybl.afs.timeseriesserver.storage; import com.powsybl.afs.storage.*; import com.powsybl.afs.storage.events.AppStorageListener; @@ -19,6 +19,11 @@ public class TimeSeriesServerAppStorage extends AbstractAppStorage { + @FunctionalInterface + public interface TimeSeriesServerAppStorageProvider { + R apply(F first, S second, T third); + } + /** * This storage is used for all non-timeseries-related operations */ @@ -27,8 +32,7 @@ public class TimeSeriesServerAppStorage extends AbstractAppStorage { /** * This storage handles all the timeseries-related operations */ - private TimeSeriesSorageDelegate timeSeriesDelegate; - + private TimeSeriesStorageDelegate timeSeriesDelegate; /** * A listener that copies all event from the general delegate event bus to this class event bus. @@ -36,12 +40,16 @@ public class TimeSeriesServerAppStorage extends AbstractAppStorage { */ private AppStorageListener notifyGeneralDelegateEventListener; - public TimeSeriesServerAppStorage(AbstractAppStorage generalDelegate, URI timeSeriesServerURI) { + public TimeSeriesServerAppStorage(final URI targetURI, final String app, final AbstractAppStorage generalDelegate) { + this(generalDelegate, targetURI, app); + } + + public TimeSeriesServerAppStorage(final AbstractAppStorage generalDelegate, final URI timeSeriesServerURI, final String app) { this.generalDelegate = generalDelegate; eventsBus = new InMemoryEventsBus(); notifyGeneralDelegateEventListener = t -> t.getEvents().forEach(e -> pushEvent(e, t.getTopic())); generalDelegate.getEventsBus().addListener(notifyGeneralDelegateEventListener); - timeSeriesDelegate = new TimeSeriesSorageDelegate(timeSeriesServerURI); + timeSeriesDelegate = new TimeSeriesStorageDelegate(timeSeriesServerURI, app); timeSeriesDelegate.createAFSAppIfNotExists(); } @@ -138,7 +146,7 @@ public boolean removeData(String nodeId, String name) { @Override public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { timeSeriesDelegate.createTimeSeries(nodeId, metadata); - pushEvent(new TimeSeriesCreated(nodeId, metadata.getName()), APPSTORAGE_TIMESERIES_TOPIC); + pushEvent(new TimeSeriesCreated(nodeId, metadata.getName()), AbstractAppStorage.APPSTORAGE_TIMESERIES_TOPIC); } @Override @@ -175,7 +183,7 @@ public Map> getDoubleTimeSeriesData(String nodeId, @Override public void addDoubleTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { timeSeriesDelegate.addDoubleTimeSeriesData(nodeId, version, timeSeriesName, chunks); - pushEvent(new TimeSeriesDataUpdated(nodeId, timeSeriesName), APPSTORAGE_TIMESERIES_TOPIC); + pushEvent(new TimeSeriesDataUpdated(nodeId, timeSeriesName), AbstractAppStorage.APPSTORAGE_TIMESERIES_TOPIC); } @Override diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java similarity index 96% rename from afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java rename to afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java index 730efb0a..f0ce795e 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TimeSeriesSorageDelegate.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java @@ -1,4 +1,4 @@ -package com.powsybl.afs.timeseriesserver; +package com.powsybl.afs.timeseriesserver.storage; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -28,10 +28,9 @@ import java.util.*; import java.util.stream.Collectors; -public class TimeSeriesSorageDelegate { +public class TimeSeriesStorageDelegate { - private static final String AFS_DEFAULT_APP = "AFS"; - private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesSorageDelegate.class); + private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesStorageDelegate.class); /** * The name of the app in TimeSeriesServer, in which AFS will store time series @@ -43,11 +42,7 @@ public class TimeSeriesSorageDelegate { */ private URI timeSeriesServerURI; - public TimeSeriesSorageDelegate(URI timeSeriesServerURI) { - this(timeSeriesServerURI, AFS_DEFAULT_APP); - } - - public TimeSeriesSorageDelegate(URI timeSeriesServerURI, String app) { + public TimeSeriesStorageDelegate(URI timeSeriesServerURI, String app) { this.timeSeriesServerURI = timeSeriesServerURI; this.app = app; } diff --git a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java b/afs-timeseries-server-storage/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java similarity index 69% rename from afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java rename to afs-timeseries-server-storage/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java index a8e17c33..6ff39091 100644 --- a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java +++ b/afs-timeseries-server-storage/src/test/java/com/powsybl/afs/timeseriesserver/TimeSeriesServerAppStorageTest.java @@ -4,6 +4,7 @@ import com.powsybl.afs.storage.AbstractAppStorageTest; import com.powsybl.afs.storage.AppStorage; import com.powsybl.afs.storage.InMemoryEventsBus; +import com.powsybl.afs.timeseriesserver.storage.TimeSeriesServerAppStorage; import org.junit.Before; import java.net.URI; @@ -12,6 +13,8 @@ public class TimeSeriesServerAppStorageTest extends AbstractAppStorageTest { private URI timeSeriesServerURI; + private static final String AFS_APP = "AFS"; + public TimeSeriesServerAppStorageTest() { super(false, false); } @@ -25,7 +28,8 @@ public void setUp() throws Exception { @Override protected AppStorage createStorage() { - return new TimeSeriesServerAppStorage(MapDbAppStorage.createMem("mem", new InMemoryEventsBus()), timeSeriesServerURI); + final MapDbAppStorage storage = MapDbAppStorage.createMem("mem", new InMemoryEventsBus()); + return new TimeSeriesServerAppStorage(storage, timeSeriesServerURI, AFS_APP); } } From 63aca9aaeec9053a0b7b82e6d5ae22d0bd76fd25 Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 18:30:07 +0200 Subject: [PATCH 51/59] Add new module in parent pom Signed-off-by: Arthur Michaut --- afs-timeseries-server/pom.xml | 29 +++++++++++++++++++---------- pom.xml | 1 + 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/afs-timeseries-server/pom.xml b/afs-timeseries-server/pom.xml index cc228b9b..eb9cf184 100644 --- a/afs-timeseries-server/pom.xml +++ b/afs-timeseries-server/pom.xml @@ -29,24 +29,29 @@ powsybl-afs-core ${project.version} + + com.powsybl + powsybl-afs-timeseries-server-storage + ${project.version} + ${project.groupId} powsybl-time-series-server-interfaces 0.0.1-SNAPSHOT + + org.jboss.spec.javax.ws.rs + jboss-jaxrs-api_2.0_spec + 1.0.1.Beta1 + org.jboss.resteasy resteasy-client - provided org.jboss.resteasy resteasy-jackson-provider - 3.0.19.Final - - - javax - javaee-api + 3.1.4.Final @@ -74,7 +79,6 @@ ${project.groupId} powsybl-afs-cassandra ${project.version} - test-jar test @@ -109,10 +113,15 @@ test - ${project.groupId} - powsybl-afs-mapdb-storage + org.projectlombok + lombok + 1.18.20 + + + com.powsybl + powsybl-afs-mapdb ${project.version} - test + compile diff --git a/pom.xml b/pom.xml index c2b9ca20..63b88996 100644 --- a/pom.xml +++ b/pom.xml @@ -65,6 +65,7 @@ afs-ws afs-spring-server afs-timeseries-server + afs-timeseries-server-storage From cc362354ac9171e90b0e9343f9c1d4fdf310efec Mon Sep 17 00:00:00 2001 From: amichaut Date: Tue, 1 Jun 2021 18:31:35 +0200 Subject: [PATCH 52/59] Rework mapdb app file system configuration to allow delegate provider config (ts server app storage) Signed-off-by: Arthur Michaut --- .../afs/mapdb/MapDbAppFileSystemConfig.java | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/afs-mapdb/src/main/java/com/powsybl/afs/mapdb/MapDbAppFileSystemConfig.java b/afs-mapdb/src/main/java/com/powsybl/afs/mapdb/MapDbAppFileSystemConfig.java index 8df06574..4df922c4 100644 --- a/afs-mapdb/src/main/java/com/powsybl/afs/mapdb/MapDbAppFileSystemConfig.java +++ b/afs-mapdb/src/main/java/com/powsybl/afs/mapdb/MapDbAppFileSystemConfig.java @@ -8,6 +8,7 @@ import com.powsybl.afs.AfsException; import com.powsybl.afs.storage.AbstractAppFileSystemConfig; +import com.powsybl.commons.config.ModuleConfig; import com.powsybl.commons.config.PlatformConfig; import java.nio.file.Files; @@ -31,30 +32,32 @@ public static List load() { public static List load(PlatformConfig platformConfig) { return platformConfig.getOptionalModuleConfig("mapdb-app-file-system") - .map(moduleConfig -> { - List configs = new ArrayList<>(); - if (moduleConfig.hasProperty("drive-name") - && moduleConfig.hasProperty("db-file")) { - String driveName = moduleConfig.getStringProperty("drive-name"); - boolean remotelyAccessible = moduleConfig.getBooleanProperty("remotely-accessible", DEFAULT_REMOTELY_ACCESSIBLE); - Path rootDir = moduleConfig.getPathProperty("db-file"); - configs.add(new MapDbAppFileSystemConfig(driveName, remotelyAccessible, rootDir)); - } - int maxAdditionalDriveCount = moduleConfig.getIntProperty("max-additional-drive-count", 0); - for (int i = 0; i < maxAdditionalDriveCount; i++) { - if (moduleConfig.hasProperty("drive-name-" + i) - && moduleConfig.hasProperty("db-file-" + i)) { - String driveName = moduleConfig.getStringProperty("drive-name-" + i); - boolean remotelyAccessible = moduleConfig.getBooleanProperty("remotely-accessible-" + i, DEFAULT_REMOTELY_ACCESSIBLE); - Path rootDir = moduleConfig.getPathProperty("db-file-" + i); - configs.add(new MapDbAppFileSystemConfig(driveName, remotelyAccessible, rootDir)); - } - } - return configs; - }) + .map(MapDbAppFileSystemConfig::load) .orElse(Collections.emptyList()); } + public static List load(ModuleConfig moduleConfig) { + List configs = new ArrayList<>(); + if (moduleConfig.hasProperty("drive-name") + && moduleConfig.hasProperty("db-file")) { + String driveName = moduleConfig.getStringProperty("drive-name"); + boolean remotelyAccessible = moduleConfig.getBooleanProperty("remotely-accessible", DEFAULT_REMOTELY_ACCESSIBLE); + Path rootDir = moduleConfig.getPathProperty("db-file"); + configs.add(new MapDbAppFileSystemConfig(driveName, remotelyAccessible, rootDir)); + } + int maxAdditionalDriveCount = moduleConfig.getIntProperty("max-additional-drive-count", 0); + for (int i = 0; i < maxAdditionalDriveCount; i++) { + if (moduleConfig.hasProperty("drive-name-" + i) + && moduleConfig.hasProperty("db-file-" + i)) { + String driveName = moduleConfig.getStringProperty("drive-name-" + i); + boolean remotelyAccessible = moduleConfig.getBooleanProperty("remotely-accessible-" + i, DEFAULT_REMOTELY_ACCESSIBLE); + Path rootDir = moduleConfig.getPathProperty("db-file-" + i); + configs.add(new MapDbAppFileSystemConfig(driveName, remotelyAccessible, rootDir)); + } + } + return configs; + } + private static Path checkDbFile(Path dbFile) { Objects.requireNonNull(dbFile); if (Files.isDirectory(dbFile)) { From 831783231e1646bcd5b0a7b187498219eaa72c43 Mon Sep 17 00:00:00 2001 From: amichaut Date: Thu, 17 Jun 2021 18:42:56 +0200 Subject: [PATCH 53/59] Add support for String time series in time series server AppStorage Signed-off-by: Arthur Michaut --- .../storage/TimeSeriesServerAppStorage.java | 6 +- .../storage/TimeSeriesStorageDelegate.java | 255 ++++++++++++------ 2 files changed, 183 insertions(+), 78 deletions(-) diff --git a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java index 9607a168..83bb0357 100644 --- a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java @@ -176,7 +176,6 @@ public Set getTimeSeriesDataVersions(String nodeId, String timeSeriesNa @Override public Map> getDoubleTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { - //TODO return timeSeriesDelegate.getDoubleTimeSeriesData(nodeId, timeSeriesNames, version); } @@ -188,12 +187,13 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie @Override public Map> getStringTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { - throw new NotImplementedException("Not implemented in V1"); + return timeSeriesDelegate.getStringTimeSeriesData(nodeId, timeSeriesNames, version); } @Override public void addStringTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { - throw new NotImplementedException("Not implemented in V1"); + timeSeriesDelegate.addStringTimeSeriesData(nodeId, version, timeSeriesName, chunks); + pushEvent(new TimeSeriesDataUpdated(nodeId, timeSeriesName), AbstractAppStorage.APPSTORAGE_TIMESERIES_TOPIC); } @Override diff --git a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java index f0ce795e..9b5ab910 100644 --- a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java @@ -4,10 +4,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.powsybl.afs.storage.AfsStorageException; import com.powsybl.timeseries.*; +import com.powsybl.timeseries.storer.data.dto.TimeSeriesInfoDto; import com.powsybl.timeseries.storer.query.create.CreateQuery; import com.powsybl.timeseries.storer.query.fetch.FetchQuery; -import com.powsybl.timeseries.storer.query.fetch.FetchQueryResult; +import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryDoubleResult; +import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryResult; +import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryStringResult; +import com.powsybl.timeseries.storer.query.publish.PublishDoubleQuery; import com.powsybl.timeseries.storer.query.publish.PublishQuery; +import com.powsybl.timeseries.storer.query.publish.PublishStringQuery; import com.powsybl.timeseries.storer.query.search.SearchQuery; import com.powsybl.timeseries.storer.query.search.SearchQueryResults; import org.apache.commons.lang3.ArrayUtils; @@ -49,15 +54,15 @@ public TimeSeriesStorageDelegate(URI timeSeriesServerURI, String app) { public static Client createClient() { return new ResteasyClientBuilder() - .connectionPoolSize(50) - .build(); + .connectionPoolSize(50) + .build(); } private WebTarget buildBaseRequest(Client client) { return client.target(timeSeriesServerURI) - .path("v1") - .path("timeseries") - .path("apps"); + .path("v1") + .path("timeseries") + .path("apps"); } public void createAFSAppIfNotExists() { @@ -99,9 +104,9 @@ public void createTimeSeries(String nodeId, TimeSeriesMetadata metadata) { Client client = createClient(); try { buildBaseRequest(client) - .path(app) - .path("series") - .request().post(Entity.json(new ObjectMapper().writeValueAsString(createQuery))); + .path(app) + .path("series") + .request().post(Entity.json(new ObjectMapper().writeValueAsString(createQuery))); } catch (JsonProcessingException e) { LOGGER.error(e.getMessage(), e); } finally { @@ -116,10 +121,10 @@ public SearchQueryResults performSearch(SearchQuery query) { Client client = createClient(); try { Response response = buildBaseRequest(client) - .path(app) - .path("series") - .path("_search") - .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); + .path(app) + .path("series") + .path("_search") + .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); String json = response.readEntity(String.class); results = new ObjectMapper().readValue(json, SearchQueryResults.class); } catch (IOException e) { @@ -139,17 +144,14 @@ public Set getTimeSeriesNames(String nodeId) { SearchQueryResults results = performSearch(searchQuery); if (results != null) { return results.getTimeSeriesInformations() - .stream().map(t -> t.getName()).collect(Collectors.toSet()); + .stream().map(t -> t.getName()).collect(Collectors.toSet()); } return null; } public boolean timeSeriesExists(String nodeId, String name) { - SearchQuery searchQuery = new SearchQuery(); - searchQuery.setMatrix(nodeId); - searchQuery.setNames(Collections.singleton(name)); - SearchQueryResults results = performSearch(searchQuery); + SearchQueryResults results = doSearch(nodeId, Collections.singleton(name)); if (results != null) { return results.getTimeSeriesInformations().size() > 0; } @@ -158,20 +160,17 @@ public boolean timeSeriesExists(String nodeId, String name) { public List getTimeSeriesMetadata(String nodeId, Set timeSeriesNames) { - SearchQuery searchQuery = new SearchQuery(); - searchQuery.setMatrix(nodeId); - searchQuery.setNames(timeSeriesNames); - SearchQueryResults results = performSearch(searchQuery); + SearchQueryResults results = doSearch(nodeId, timeSeriesNames); if (results != null) { return results.getTimeSeriesInformations() - .stream().map(t -> { - long startTime = t.getStartDate().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); - long spacing = t.getTimeStepDuration(); - long endTime = startTime + spacing * (t.getTimeStepCount() - 1); - TimeSeriesIndex index = new RegularTimeSeriesIndex(startTime, endTime, spacing); - return new TimeSeriesMetadata(t.getName(), TimeSeriesDataType.DOUBLE, t.getTags(), index); - }) - .collect(Collectors.toList()); + .stream().map(t -> { + long startTime = t.getStartDate().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + long spacing = t.getTimeStepDuration(); + long endTime = startTime + spacing * (t.getTimeStepCount() - 1); + TimeSeriesIndex index = new RegularTimeSeriesIndex(startTime, endTime, spacing); + return new TimeSeriesMetadata(t.getName(), TimeSeriesDataType.DOUBLE, t.getTags(), index); + }) + .collect(Collectors.toList()); } return null; } @@ -185,13 +184,45 @@ public Set getTimeSeriesDataVersions(String nodeId, String timeSeriesNa SearchQueryResults results = performSearch(searchQuery); if (results != null) { return results.getTimeSeriesInformations() - .stream().flatMap(t -> t.getVersions().keySet().stream()) - .map(t -> Integer.parseInt(t)) - .collect(Collectors.toSet()); + .stream().flatMap(t -> t.getVersions().keySet().stream()) + .map(t -> Integer.parseInt(t)) + .collect(Collectors.toSet()); } return null; } + /** + * Add a new string time series + * + * @param nodeId identifier for the node + * @param version version to publish to + * @param timeSeriesName name of the time series + * @param chunks actual (string) data to publish + */ + public void addStringTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { + // Retrieve metadata and build time series + TimeSeriesMetadata metadata = getTimeSeriesMetadata(nodeId, Collections.singleton(timeSeriesName)).get(0); + StringTimeSeries ts = new StringTimeSeries(metadata, chunks); + + // Prepare publish request + PublishQuery publishQuery = new PublishStringQuery(); + publishQuery.setMatrix(nodeId); + publishQuery.setTimeSeriesName(timeSeriesName); + publishQuery.setVersionName(String.valueOf(version)); + publishQuery.setData(ts.toArray()); + + // Do run query + doPublish(publishQuery); + } + + /** + * Add a new double time series + * + * @param nodeId identifier for the node + * @param version version to publish to + * @param timeSeriesName name of the time series + * @param chunks actual (double) data to publish + */ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSeriesName, List chunks) { //First step : retrieve metadata and reconstitute the time series @@ -199,73 +230,147 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie StoredDoubleTimeSeries ts = new StoredDoubleTimeSeries(metadata, chunks); //Second step : perform a publish request - PublishQuery publishQuery = new PublishQuery(); + PublishQuery publishQuery = new PublishDoubleQuery(); publishQuery.setMatrix(nodeId); publishQuery.setTimeSeriesName(timeSeriesName); publishQuery.setVersionName(String.valueOf(version)); publishQuery.setData(ArrayUtils.toObject(ts.toArray())); - Client client = createClient(); + // Do run query + doPublish(publishQuery); + } + + /** + * Retrieve time series String data from server + * Perform a search query with provided criteria, followed by a fetch query (using search results) + * @param nodeId identifier for the node to retrieve data for + * @param timeSeriesNames names of time series to retrieve + * @param version version of the data to fetch + * @return a map contaning all queried time series, indexed by their name + */ + public Map> getStringTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { + String versionString = String.valueOf(version); + // First perform a search query + final SearchQueryResults searchResults = doSearch(nodeId, timeSeriesNames); + List versionIDs = searchResults.getVersionIds(versionString); + Map versionToName = searchResults.getVersionToNameMap(versionString); + // Then perform the fetch query + final Response response = doFetch(versionToName); + // Extract fetch result + String json = response.readEntity(String.class); + FetchQueryStringResult fetchResults; try { - Response response = buildBaseRequest(client) - .path(app) - .path("series") - .request().put(Entity.json(new ObjectMapper().writeValueAsString(publishQuery))); - if (response.getStatus() != 200) { - throw new AfsStorageException("Error while publishing data to time series server"); - } - } catch (IOException e) { - LOGGER.error(e.getMessage(), e); - } finally { - client.close(); + fetchResults = new ObjectMapper().readValue(json, FetchQueryStringResult.class); + } catch (JsonProcessingException e) { + throw new AfsStorageException("Could not process fetch query result into a String query result"); } - + Map> result = new HashMap<>(); + for (int i = 0; i < versionToName.keySet().size(); i++) { + String[] values = fetchResults.getData().get(i).toArray(new String[0]); + StringDataChunk chunk = new UncompressedStringDataChunk(0, values); + result.put(versionToName.get(versionIDs.get(i)), Collections.singletonList(chunk)); + } + return result; } + /** + * Perform search then fetch queries to retrieve double data against time series server + * + * @param nodeId identifier for the node to retrieve time series for + * @param timeSeriesNames names of time series to search for + * @param version version of the data to fetch + * @return time series names, mapped to their respective results (as chunks) + */ public Map> getDoubleTimeSeriesData(String nodeId, Set timeSeriesNames, int version) { - String versionString = String.valueOf(version); + // First perform a search query + final SearchQueryResults searchResults = doSearch(nodeId, timeSeriesNames); + List versionIDs = searchResults.getVersionIds(versionString); + Map versionToName = searchResults.getVersionToNameMap(versionString); + // Then perform the fetch query + final Response response = doFetch(versionToName); + // Extract fetch result + String json = response.readEntity(String.class); + FetchQueryDoubleResult fetchResults; + try { + fetchResults = new ObjectMapper().readValue(json, FetchQueryDoubleResult.class); + } catch (JsonProcessingException e) { + throw new AfsStorageException("Could not process fetch query result into a String query result"); + } + // Extract data from results + Map> toReturn = new HashMap<>(); + for (int i = 0; i < versionIDs.size(); i++) { + double[] values = fetchResults.getData().get(i).stream().mapToDouble(Double::doubleValue).toArray(); + UncompressedDoubleDataChunk chunk = new UncompressedDoubleDataChunk(0, values); + toReturn.put(versionToName.get(versionIDs.get(i)), Collections.singletonList(chunk)); + } + return toReturn; + } + + /** + * Perform search request against time series server + * + * @param nodeId identifier of the node to search for + * @param timeSeriesNames names of time series to search for + * @return a SearchQueryResults object representing the TS server response + */ + private SearchQueryResults doSearch(final String nodeId, final Set timeSeriesNames) { + // Prepare search query SearchQuery searchQuery = new SearchQuery(); searchQuery.setMatrix(nodeId); searchQuery.setNames(timeSeriesNames); - SearchQueryResults results = performSearch(searchQuery); - List versionIDs = results.getTimeSeriesInformations() - .stream() - .map(t -> t.getVersions().get(versionString)) - .collect(Collectors.toList()); - - Map versionIDToTSName = results.getTimeSeriesInformations() - .stream() - .collect(Collectors.toMap(t -> t.getVersions().get(versionString), t -> t.getName())); + // Run search query + return performSearch(searchQuery); + } - FetchQuery query = new FetchQuery(versionIDs, null, null); + /** + * Perform a search query against distant API + * + * @param versionToName a map of TS version -> TS name + * @return fetch HTTP response + */ + private Response doFetch(final Map versionToName) { + // Prepare fetch query + FetchQuery query = new FetchQuery(versionToName.keySet(), null, null); Client client = createClient(); try { Response response = buildBaseRequest(client) - .path(app) - .path("series") - .path("_fetch") - .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); + .path(app) + .path("series") + .path("_fetch") + .request().post(Entity.json(new ObjectMapper().writeValueAsString(query))); if (response.getStatus() != 200) { throw new AfsStorageException("Error while fetching data from time series server"); } - String json = response.readEntity(String.class); - FetchQueryResult fetchResults = new ObjectMapper().readValue(json, FetchQueryResult.class); + return response; + } catch (JsonProcessingException e) { + throw new AfsStorageException("Error while fetching data from time series server"); + } finally { + client.close(); + } + } - Map> toReturn = new HashMap<>(); - for (int i = 0; i < versionIDs.size(); i++) { - double[] values = fetchResults.getData().get(i).stream().mapToDouble(Double::doubleValue).toArray(); - UncompressedDoubleDataChunk chunk = new UncompressedDoubleDataChunk(0, values); - toReturn.put(versionIDToTSName.get(versionIDs.get(i)), Arrays.asList(chunk)); + /** + * Perform a publish query + * + * @param publishQuery query to issue + */ + private void doPublish(final PublishQuery publishQuery) { + // Run request + Client client = createClient(); + try { + Response response = buildBaseRequest(client) + .path(app) + .path("series") + .request() + .put(Entity.json(new ObjectMapper().writeValueAsString(publishQuery))); + if (response.getStatus() != 200) { + throw new AfsStorageException("Error while publishing data to time series server"); } - return toReturn; - - } catch (IOException e) { - LOGGER.error(e.getMessage(), e); + } catch (IOException ioe) { + LOGGER.error("Could not publish String time series", ioe); } finally { client.close(); } - - return null; } } From 291b4ef5d3e9400c8dc9de7cb3493f0172d9ff39 Mon Sep 17 00:00:00 2001 From: amichaut Date: Fri, 18 Jun 2021 09:55:16 +0200 Subject: [PATCH 54/59] Fix code to match powsybl standards Signed-off-by: Arthur Michaut --- .../storage/TimeSeriesStorageDelegate.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java index 9b5ab910..be80ff09 100644 --- a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java @@ -4,14 +4,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.powsybl.afs.storage.AfsStorageException; import com.powsybl.timeseries.*; -import com.powsybl.timeseries.storer.data.dto.TimeSeriesInfoDto; import com.powsybl.timeseries.storer.query.create.CreateQuery; import com.powsybl.timeseries.storer.query.fetch.FetchQuery; import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryDoubleResult; -import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryResult; import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryStringResult; import com.powsybl.timeseries.storer.query.publish.PublishDoubleQuery; -import com.powsybl.timeseries.storer.query.publish.PublishQuery; +import com.powsybl.timeseries.storer.query.publish.AbstractPublishQuery; import com.powsybl.timeseries.storer.query.publish.PublishStringQuery; import com.powsybl.timeseries.storer.query.search.SearchQuery; import com.powsybl.timeseries.storer.query.search.SearchQueryResults; @@ -205,7 +203,7 @@ public void addStringTimeSeriesData(String nodeId, int version, String timeSerie StringTimeSeries ts = new StringTimeSeries(metadata, chunks); // Prepare publish request - PublishQuery publishQuery = new PublishStringQuery(); + AbstractPublishQuery publishQuery = new PublishStringQuery(); publishQuery.setMatrix(nodeId); publishQuery.setTimeSeriesName(timeSeriesName); publishQuery.setVersionName(String.valueOf(version)); @@ -230,7 +228,7 @@ public void addDoubleTimeSeriesData(String nodeId, int version, String timeSerie StoredDoubleTimeSeries ts = new StoredDoubleTimeSeries(metadata, chunks); //Second step : perform a publish request - PublishQuery publishQuery = new PublishDoubleQuery(); + AbstractPublishQuery publishQuery = new PublishDoubleQuery(); publishQuery.setMatrix(nodeId); publishQuery.setTimeSeriesName(timeSeriesName); publishQuery.setVersionName(String.valueOf(version)); @@ -355,7 +353,7 @@ private Response doFetch(final Map versionToName) { * * @param publishQuery query to issue */ - private void doPublish(final PublishQuery publishQuery) { + private void doPublish(final AbstractPublishQuery publishQuery) { // Run request Client client = createClient(); try { From 9216dad2170522ff1bc402484a358ccd3703624f Mon Sep 17 00:00:00 2001 From: Paul Bui-Quang Date: Wed, 7 Jul 2021 14:54:57 +0200 Subject: [PATCH 55/59] Fix checkstyle + remove missing dependency Signed-off-by: Paul Bui-Quang Signed-off-by: Arthur Michaut --- .../afs/storage/AbstractAppStorageTest.java | 2 +- .../storage/TimeSeriesServerAppStorage.java | 1 - .../storage/TimeSeriesStorageDelegate.java | 15 +++++++-------- afs-timeseries-server/pom.xml | 7 ------- .../TSServerAppFileSystemProvider.java | 5 ++--- .../TSServerAppFileSystemProviderTest.java | 7 +++---- 6 files changed, 13 insertions(+), 24 deletions(-) diff --git a/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java b/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java index f6d5b1e5..facbc3cd 100644 --- a/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java +++ b/afs-storage-api/src/test/java/com/powsybl/afs/storage/AbstractAppStorageTest.java @@ -427,7 +427,7 @@ public void test() throws IOException, InterruptedException { assertTrue(storage.getDoubleTimeSeriesData(testData3Info.getId(), Sets.newHashSet("ts1"), 0).isEmpty()); // 15) create a second string time series - if(handlesStringTimeSeries) { + if (handlesStringTimeSeries) { TimeSeriesMetadata metadata2 = new TimeSeriesMetadata("ts2", TimeSeriesDataType.STRING, ImmutableMap.of(), diff --git a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java index 83bb0357..208ab4e5 100644 --- a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesServerAppStorage.java @@ -7,7 +7,6 @@ import com.powsybl.timeseries.DoubleDataChunk; import com.powsybl.timeseries.StringDataChunk; import com.powsybl.timeseries.TimeSeriesMetadata; -import org.apache.commons.lang3.NotImplementedException; import java.io.InputStream; import java.io.OutputStream; diff --git a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java index be80ff09..c69d400e 100644 --- a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java @@ -161,14 +161,13 @@ public List getTimeSeriesMetadata(String nodeId, Set SearchQueryResults results = doSearch(nodeId, timeSeriesNames); if (results != null) { return results.getTimeSeriesInformations() - .stream().map(t -> { - long startTime = t.getStartDate().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); - long spacing = t.getTimeStepDuration(); - long endTime = startTime + spacing * (t.getTimeStepCount() - 1); - TimeSeriesIndex index = new RegularTimeSeriesIndex(startTime, endTime, spacing); - return new TimeSeriesMetadata(t.getName(), TimeSeriesDataType.DOUBLE, t.getTags(), index); - }) - .collect(Collectors.toList()); + .stream().map(t -> { + long startTime = t.getStartDate().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + long spacing = t.getTimeStepDuration(); + long endTime = startTime + spacing * (t.getTimeStepCount() - 1); + TimeSeriesIndex index = new RegularTimeSeriesIndex(startTime, endTime, spacing); + return new TimeSeriesMetadata(t.getName(), TimeSeriesDataType.DOUBLE, t.getTags(), index); + }).collect(Collectors.toList()); } return null; } diff --git a/afs-timeseries-server/pom.xml b/afs-timeseries-server/pom.xml index eb9cf184..55095157 100644 --- a/afs-timeseries-server/pom.xml +++ b/afs-timeseries-server/pom.xml @@ -81,13 +81,6 @@ ${project.version} test - - ${project.groupId} - powsybl-afs-local - ${project.version} - test-jar - test - ${project.groupId} powsybl-afs-local diff --git a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java index be33f2d2..d0c1d706 100644 --- a/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java +++ b/afs-timeseries-server/src/main/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProvider.java @@ -29,9 +29,8 @@ public class TSServerAppFileSystemProvider implements AppFileSystemProvider { public TSServerAppFileSystemProvider() { this( - TSServerAppFileSystemConfig.load(), - TimeSeriesServerAppStorage::new, - (name, path, eventsStore) -> MapDbAppStorage.createMmapFile(name, path.toFile(), eventsStore) + TSServerAppFileSystemConfig.load(), + TimeSeriesServerAppStorage::new, (name, path, eventsStore) -> MapDbAppStorage.createMmapFile(name, path.toFile(), eventsStore) ); } diff --git a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java index 190a916b..402fef5f 100644 --- a/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java +++ b/afs-timeseries-server/src/test/java/com/powsybl/afs/timeseriesserver/TSServerAppFileSystemProviderTest.java @@ -31,7 +31,7 @@ public class TSServerAppFileSystemProviderTest { - private static final int srvPort = 9876; + private static final int SRV_PORT = 9876; public static final String DRIVE = "drive"; private TSServerAppFileSystemConfig config; @@ -54,7 +54,7 @@ public class TSServerAppFileSystemProviderTest { @Before public void setUp() { - mockServer = ClientAndServer.startClientAndServer(srvPort); + mockServer = ClientAndServer.startClientAndServer(SRV_PORT); setupMockServer(); // Setup connection parameters @@ -103,8 +103,7 @@ public void provideTest() { ComputationManager computationManager = Mockito.mock(ComputationManager.class); final AppFileSystemProviderContext context = new AppFileSystemProviderContext(computationManager, null, new InMemoryEventsBus()); // Build a new provider - final MapDbAppStorage.MapDbAppStorageProvider delegateAppStorageProvider = - (name, path, eventsStore) -> MapDbAppStorage.createMem(name, eventsStore); + final MapDbAppStorage.MapDbAppStorageProvider delegateAppStorageProvider = (name, path, eventsStore) -> MapDbAppStorage.createMem(name, eventsStore); final TimeSeriesServerAppStorage.TimeSeriesServerAppStorageProvider appStorageProvider = TimeSeriesServerAppStorage::new; TSServerAppFileSystemProvider provider = new TSServerAppFileSystemProvider(config, appStorageProvider, delegateAppStorageProvider); // Check that FS is correct From 74e501b33de96a575c67e6376a53d4a1a7a868bb Mon Sep 17 00:00:00 2001 From: amichaut Date: Fri, 9 Jul 2021 14:18:31 +0200 Subject: [PATCH 56/59] Add missing jackson provider dependency Signed-off-by: Arthur Michaut --- afs-timeseries-server-storage/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/afs-timeseries-server-storage/pom.xml b/afs-timeseries-server-storage/pom.xml index fe2942cf..bfb7a84f 100644 --- a/afs-timeseries-server-storage/pom.xml +++ b/afs-timeseries-server-storage/pom.xml @@ -48,6 +48,11 @@ jackson-core ${jackson.version} + + org.jboss.resteasy + resteasy-jackson-provider + ${resteasy.version} + org.apache.commons commons-lang3 From 4d29d364f6fb884bdfd6470447556f81cea900a3 Mon Sep 17 00:00:00 2001 From: amichaut Date: Fri, 9 Jul 2021 15:25:34 +0200 Subject: [PATCH 57/59] Fix fetch results subtypes handling in TS storage delegate : retrieve mixed results and extract the expected values types explicitly Signed-off-by: Arthur Michaut --- .../storage/TimeSeriesStorageDelegate.java | 43 ++++++++----------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java index c69d400e..d499ec95 100644 --- a/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java +++ b/afs-timeseries-server-storage/src/main/java/com/powsybl/afs/timeseriesserver/storage/TimeSeriesStorageDelegate.java @@ -6,10 +6,10 @@ import com.powsybl.timeseries.*; import com.powsybl.timeseries.storer.query.create.CreateQuery; import com.powsybl.timeseries.storer.query.fetch.FetchQuery; -import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryDoubleResult; -import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryStringResult; -import com.powsybl.timeseries.storer.query.publish.PublishDoubleQuery; +import com.powsybl.timeseries.storer.query.fetch.result.AbstractFetchQueryResult; +import com.powsybl.timeseries.storer.query.fetch.result.FetchQueryMixedResult; import com.powsybl.timeseries.storer.query.publish.AbstractPublishQuery; +import com.powsybl.timeseries.storer.query.publish.PublishDoubleQuery; import com.powsybl.timeseries.storer.query.publish.PublishStringQuery; import com.powsybl.timeseries.storer.query.search.SearchQuery; import com.powsybl.timeseries.storer.query.search.SearchQueryResults; @@ -252,18 +252,13 @@ public Map> getStringTimeSeriesData(String nodeId, List versionIDs = searchResults.getVersionIds(versionString); Map versionToName = searchResults.getVersionToNameMap(versionString); // Then perform the fetch query - final Response response = doFetch(versionToName); - // Extract fetch result - String json = response.readEntity(String.class); - FetchQueryStringResult fetchResults; - try { - fetchResults = new ObjectMapper().readValue(json, FetchQueryStringResult.class); - } catch (JsonProcessingException e) { - throw new AfsStorageException("Could not process fetch query result into a String query result"); - } + FetchQueryMixedResult fetchResults = (FetchQueryMixedResult) doFetch(versionToName); Map> result = new HashMap<>(); for (int i = 0; i < versionToName.keySet().size(); i++) { - String[] values = fetchResults.getData().get(i).toArray(new String[0]); + String[] values = fetchResults.getData().get(i) + .stream() + .map(v -> (String) v) + .toArray(String[]::new); StringDataChunk chunk = new UncompressedStringDataChunk(0, values); result.put(versionToName.get(versionIDs.get(i)), Collections.singletonList(chunk)); } @@ -285,19 +280,15 @@ public Map> getDoubleTimeSeriesData(String nodeId, List versionIDs = searchResults.getVersionIds(versionString); Map versionToName = searchResults.getVersionToNameMap(versionString); // Then perform the fetch query - final Response response = doFetch(versionToName); - // Extract fetch result - String json = response.readEntity(String.class); - FetchQueryDoubleResult fetchResults; - try { - fetchResults = new ObjectMapper().readValue(json, FetchQueryDoubleResult.class); - } catch (JsonProcessingException e) { - throw new AfsStorageException("Could not process fetch query result into a String query result"); - } + FetchQueryMixedResult fetchResults = (FetchQueryMixedResult) doFetch(versionToName); // Extract data from results Map> toReturn = new HashMap<>(); for (int i = 0; i < versionIDs.size(); i++) { - double[] values = fetchResults.getData().get(i).stream().mapToDouble(Double::doubleValue).toArray(); + double[] values = fetchResults.getData() + .get(i) + .stream() + .mapToDouble(val -> ((Number)val).doubleValue()) + .toArray(); UncompressedDoubleDataChunk chunk = new UncompressedDoubleDataChunk(0, values); toReturn.put(versionToName.get(versionIDs.get(i)), Collections.singletonList(chunk)); } @@ -326,7 +317,7 @@ private SearchQueryResults doSearch(final String nodeId, final Set timeS * @param versionToName a map of TS version -> TS name * @return fetch HTTP response */ - private Response doFetch(final Map versionToName) { + private AbstractFetchQueryResult doFetch(final Map versionToName) { // Prepare fetch query FetchQuery query = new FetchQuery(versionToName.keySet(), null, null); Client client = createClient(); @@ -339,7 +330,9 @@ private Response doFetch(final Map versionToName) { if (response.getStatus() != 200) { throw new AfsStorageException("Error while fetching data from time series server"); } - return response; + + String json = response.readEntity(String.class); + return new ObjectMapper().readValue(json, AbstractFetchQueryResult.class); } catch (JsonProcessingException e) { throw new AfsStorageException("Error while fetching data from time series server"); } finally { From 2c449b0ed686125beb85055449ca7f79176bac39 Mon Sep 17 00:00:00 2001 From: amichaut Date: Fri, 9 Jul 2021 15:45:35 +0200 Subject: [PATCH 58/59] Add missing test dependency : assertj-core Signed-off-by: Arthur Michaut --- afs-timeseries-server-storage/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/afs-timeseries-server-storage/pom.xml b/afs-timeseries-server-storage/pom.xml index bfb7a84f..685e8299 100644 --- a/afs-timeseries-server-storage/pom.xml +++ b/afs-timeseries-server-storage/pom.xml @@ -87,6 +87,11 @@ org.mock-server mockserver-netty 5.11.2 + test + + + org.assertj + assertj-core \ No newline at end of file From 91d6495ab15ae3d6b1b11c1683f5130e2eb3c095 Mon Sep 17 00:00:00 2001 From: amichaut Date: Thu, 29 Jul 2021 17:04:32 +0200 Subject: [PATCH 59/59] TimeSeriesDAO now trigger IOException when failing to publish data Signed-off-by: amichaut --- afs-timeseries-server/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/afs-timeseries-server/pom.xml b/afs-timeseries-server/pom.xml index 55095157..cbceaae3 100644 --- a/afs-timeseries-server/pom.xml +++ b/afs-timeseries-server/pom.xml @@ -92,6 +92,11 @@ mockito-core test + + org.mock-server + mockserver-netty + 5.11.2 + com.google.jimfs jimfs