Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] AFS data store implementation #52

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

[WIP] AFS data store implementation #52

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
518812d
AppStorageDataStore implementation
quinarygio Jul 1, 2020
2548a68
Add test and small fixes
quinarygio Jul 7, 2020
a6931c6
Simplify implementation
quinarygio Sep 15, 2020
0edb5a3
Build ImportedCase from DataPack
quinarygio Sep 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 15 additions & 8 deletions afs-ext-base/src/main/java/com/powsybl/afs/ext/base/ImportedCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@
import com.powsybl.afs.ProjectFile;
import com.powsybl.afs.ProjectFileCreationContext;
import com.powsybl.afs.storage.AppStorageDataSource;
import com.powsybl.afs.storage.AppStorageDataStore;
import com.powsybl.commons.datasource.ReadOnlyDataSource;
import com.powsybl.commons.datastore.ReadOnlyDataStore;
import com.powsybl.iidm.import_.Importer;
import com.powsybl.iidm.import_.ImportersLoader;
import com.powsybl.iidm.network.Network;
Expand All @@ -24,10 +26,11 @@
import java.util.Properties;

/**
* A type of {@code ProjectFile} which represents a {@link Network} imported to the project,
* and provides methods to get the {@code Network} object or query it with a script.
* A type of {@code ProjectFile} which represents a {@link Network} imported to
* the project, and provides methods to get the {@code Network} object or query
* it with a script.
*
* @author Geoffroy Jamgotchian <geoffroy.jamgotchian at rte-france.com>
* @author Geoffroy Jamgotchian <geoffroy.jamgotchian at rte-france.com>
*/
public class ImportedCase extends ProjectFile implements ProjectCase {

Expand All @@ -48,9 +51,15 @@ public ReadOnlyDataSource getDataSource() {
return new AppStorageDataSource(storage, info.getId(), info.getName());
}

public ReadOnlyDataStore getDataStore() {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually here, I think it would be better to return a DataPack, since an imported case is already supposed to be a consistent network representation (and with an identified format).
If we only return a DataStore, it leaves the user with the work of doing again the resolution work, which he shouldn't need to do.

return new AppStorageDataStore(storage, info.getId());
}

public Properties getParameters() {
Properties parameters = new Properties();
try (Reader reader = new InputStreamReader(storage.readBinaryData(info.getId(), PARAMETERS).orElseThrow(AssertionError::new), StandardCharsets.UTF_8)) {
try (Reader reader = new InputStreamReader(
storage.readBinaryData(info.getId(), PARAMETERS).orElseThrow(AssertionError::new),
StandardCharsets.UTF_8)) {
parameters.load(reader);
} catch (IOException e) {
throw new UncheckedIOException(e);
Expand All @@ -60,11 +69,9 @@ public Properties getParameters() {

public Importer getImporter() {
String format = info.getGenericMetadata().getString(FORMAT);
return importersLoader.loadImporters()
.stream()
return importersLoader.loadImporters().stream()
.filter(importer -> importer.getFormat().equals(format))
.findFirst()
.orElseThrow(() -> new AfsException("Importer not found for format " + format));
.findFirst().orElseThrow(() -> new AfsException("Importer not found for format " + format));
}

@Override
Expand Down
53 changes: 44 additions & 9 deletions afs-ext-base/src/main/java/com/powsybl/afs/ext/base/ImportedCaseBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@
import com.powsybl.commons.datasource.DataSourceUtil;
import com.powsybl.commons.datasource.MemDataSource;
import com.powsybl.commons.datasource.ReadOnlyDataSource;
import com.powsybl.commons.datastore.DataFormat;
import com.powsybl.commons.datastore.DataPack;
import com.powsybl.commons.datastore.NonUniqueResultException;
import com.powsybl.commons.datastore.ReadOnlyDataStore;
import com.powsybl.iidm.export.Exporters;
import com.powsybl.iidm.export.ExportersLoader;
import com.powsybl.iidm.export.ExportersServiceLoader;
Expand All @@ -33,6 +37,7 @@
import java.nio.file.Path;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;

/**
Expand All @@ -58,6 +63,8 @@ public class ImportedCaseBuilder implements ProjectFileBuilder<ImportedCase> {

private final Properties parameters = new Properties();

private ReadOnlyDataStore dataStore;

public ImportedCaseBuilder(ProjectFileBuildContext context, ImportersLoader importersLoader, ImportConfig importConfig) {
this(context, new ExportersServiceLoader(), importersLoader, importConfig);
}
Expand Down Expand Up @@ -102,6 +109,11 @@ public ImportedCaseBuilder withDatasource(ReadOnlyDataSource dataSource) {
return this;
}

public ImportedCaseBuilder withDatastore(ReadOnlyDataStore dataStore) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should instead provide directly a DataPack here, because what we need is a consistent set of files.

Possibly we can also have an additional method to take directly a ReadOnlyDataStore, but then it would need a name to search for in the data store.

this.dataStore = Objects.requireNonNull(dataStore);
return this;
}

public ImportedCaseBuilder withNetwork(Network network) {
Objects.requireNonNull(network);
if (name == null) {
Expand All @@ -124,7 +136,7 @@ public ImportedCaseBuilder withParameters(Map<String, String> parameters) {

@Override
public ImportedCase build() {
if (dataSource == null) {
if (dataSource == null && dataStore == null) {
throw new AfsException("Case or data source is not set");
}
if (name == null) {
Expand All @@ -134,14 +146,37 @@ public ImportedCase build() {
if (context.getStorage().getChildNode(context.getFolderInfo().getId(), name).isPresent()) {
throw new AfsException("Parent folder already contains a '" + name + "' node");
}

// create project file
NodeInfo info = context.getStorage().createNode(context.getFolderInfo().getId(), name, ImportedCase.PSEUDO_CLASS, "", ImportedCase.VERSION,
new NodeGenericMetadata().setString(ImportedCase.FORMAT, importer.getFormat()));

// store case data
importer.copy(dataSource, new AppStorageDataSource(context.getStorage(), info.getId(), info.getName()));

NodeInfo info = null;
if (dataSource != null) {

// create project file
info = context.getStorage().createNode(context.getFolderInfo().getId(), name, ImportedCase.PSEUDO_CLASS, "", ImportedCase.VERSION,
new NodeGenericMetadata().setString(ImportedCase.FORMAT, importer.getFormat()));
// store case data
importer.copy(dataSource, new AppStorageDataSource(context.getStorage(), info.getId(), info.getName()));
} else {
importer = Importers.findImporter(dataStore, name, importersLoader, context.getProject().getFileSystem().getData().getShortTimeExecutionComputationManager(), importConfig);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Related to the comment above about using a data pack instead of data store. Here the name is actually the future name of the ImportedCase in AFS, not the name of the network file in the data store, so it will probably not work.

With the data pack, it should be simpler, because you should be able to simply do something like

dataPack.copyTo(new AppStorageDataStore(context.getStorage(), info.getId()));

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, I should have read the part below before :)

But the comment is still right: with a data pack as input, you will not even need to check for an importer. Just copy the data and get the format ID from the pack.

if (importer == null) {
throw new AfsException("No importer found for this data source");
}

// create project file
info = context.getStorage().createNode(context.getFolderInfo().getId(), name, ImportedCase.PSEUDO_CLASS, "", ImportedCase.VERSION,
new NodeGenericMetadata().setString(ImportedCase.FORMAT, importer.getFormat()));
DataFormat df = importer.getDataFormat();
try {
Optional<DataPack> dp = df.newDataResolver().resolve(dataStore, this.name, this.parameters);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should not be needed anymore, but better use optional syntax for the sake of readability:

DataPack dp = df.newDataResolver().resolve(dataStore, this.name, this.parameters)
                                                          .orElseThrow(() -> new AfsException("DataPack not resolved"));
dp.copyTo(...);

if (dp.isPresent()) {
dp.get().copyTo(new AppStorageDataStore(context.getStorage(), info.getId()));
} else {
throw new AfsException("DataPack not resolved");
}
} catch (IOException e) {
throw new UncheckedIOException(e);
} catch (NonUniqueResultException e) {
throw new AfsException("DataPack not unique");
}
}
// store parameters
try (Writer writer = new OutputStreamWriter(context.getStorage().writeBinaryData(info.getId(), ImportedCase.PARAMETERS), StandardCharsets.UTF_8)) {
parameters.store(writer, "");
Expand Down
9 changes: 8 additions & 1 deletion afs-ext-base/src/main/java/com/powsybl/afs/ext/base/LocalNetworkCacheService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.powsybl.afs.AfsException;
import com.powsybl.afs.ProjectFile;
import com.powsybl.commons.datasource.ReadOnlyDataSource;
import com.powsybl.commons.datastore.ReadOnlyDataStore;
import com.powsybl.iidm.import_.Importer;
import com.powsybl.iidm.network.Network;
import groovy.json.JsonOutput;
Expand Down Expand Up @@ -50,7 +51,13 @@ private static ScriptResult<Network> loadNetworkFromImportedCase(ImportedCase im
Importer importer = importedCase.getImporter();
ReadOnlyDataSource dataSource = importedCase.getDataSource();
Properties parameters = importedCase.getParameters();
Network network = importer.importData(dataSource, parameters);
Network network;
if (dataSource != null) {
network = importer.importData(dataSource, parameters);
} else {
ReadOnlyDataStore dataStore = importedCase.getDataStore();
network = importer.importDataStore(dataStore, importedCase.getName(), parameters);
}
return ScriptResult.of(network);
}

Expand Down
22 changes: 22 additions & 0 deletions afs-ext-base/src/test/java/com/powsybl/afs/ext/base/ImportedCaseTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import com.powsybl.afs.storage.InMemoryEventsBus;
import com.powsybl.afs.storage.NodeGenericMetadata;
import com.powsybl.afs.storage.NodeInfo;
import com.powsybl.commons.datastore.DataStores;
import com.powsybl.iidm.export.ExportersLoader;
import com.powsybl.iidm.export.ExportersLoaderList;
import com.powsybl.iidm.import_.ImportConfig;
Expand Down Expand Up @@ -207,4 +208,25 @@ public void testNetwork() {
assertNotNull(importedCase2);
assertEquals("NetworkID", importedCase2.getName());
}

@Test
public void testDataStore() throws IOException {
Folder root = afs.getRootFolder();

// create project
Project project = root.createProject("project");
assertNotNull(project);

// create project folder
ProjectFolder folder = project.getRootFolder().createFolder("folder");
assertTrue(folder.getChildren().isEmpty());

ImportedCase importedCase = folder.fileBuilder(ImportedCaseBuilder.class)
.withDatastore(DataStores.createDataStore(fileSystem.getPath("/work")))
.withName("network.tst")
.build();
assertNotNull(importedCase);
assertEquals("network.tst", importedCase.getName());
assertNotNull(importedCase.getNetwork());
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should add additional test for the "reading" part, ie the getDataStore method.

We can check that it contains the single entry "network.tst".

}
}
77 changes: 77 additions & 0 deletions afs-ext-base/src/test/java/com/powsybl/afs/ext/base/TestDataFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/**
* Copyright (c) 2020, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package com.powsybl.afs.ext.base;

import java.io.IOException;
import java.util.List;
import java.util.Optional;
import java.util.Properties;

import com.google.common.collect.ImmutableList;
import com.google.common.io.Files;
import com.powsybl.commons.datastore.DataEntry;
import com.powsybl.commons.datastore.DataFormat;
import com.powsybl.commons.datastore.DataPack;
import com.powsybl.commons.datastore.DataResolver;
import com.powsybl.commons.datastore.NonUniqueResultException;
import com.powsybl.commons.datastore.ReadOnlyDataStore;

/**
* @author Giovanni Ferrari <giovanni.ferrari at techrain.eu>
*/
public class TestDataFormat implements DataFormat {

private final String format;
private static final List<String> EXTENSIONS = ImmutableList.of("tst");

public TestDataFormat(String format) {
this.format = format;
}

@Override
public String getId() {
return format;
}

@Override
public String getDescription() {
return "Dummy data format";
}

@Override
public DataResolver newDataResolver() {
return new DataResolver() {

@Override
public Optional<DataPack> resolve(ReadOnlyDataStore store, String mainFileName, Properties parameters)
throws IOException, NonUniqueResultException {
DataPack dp = null;
if (checkFileExtension(mainFileName) && store.exists(mainFileName)) {
dp = new DataPack(store, getId());
DataEntry d = new DataEntry(mainFileName, DataPack.MAIN_ENTRY_TAG);
dp.addEntry(d);
}
return Optional.ofNullable(dp);
}

@Override
public boolean validate(DataPack pack, Properties parameters) {
return pack.getMainEntry().isPresent();
}
};
}

private static boolean checkFileExtension(String filename) {
return EXTENSIONS.contains(Files.getFileExtension(filename));
}

@Override
public List<String> getExtensions() {
return EXTENSIONS;
}

}
27 changes: 27 additions & 0 deletions afs-ext-base/src/test/java/com/powsybl/afs/ext/base/TestImporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@
import com.google.common.collect.ImmutableList;
import com.powsybl.commons.datasource.DataSource;
import com.powsybl.commons.datasource.ReadOnlyDataSource;
import com.powsybl.commons.datastore.DataFormat;
import com.powsybl.commons.datastore.DataPack;
import com.powsybl.commons.datastore.NonUniqueResultException;
import com.powsybl.commons.datastore.ReadOnlyDataStore;
import com.powsybl.iidm.import_.Importer;
import com.powsybl.iidm.network.Network;
import com.powsybl.iidm.parameters.Parameter;
Expand All @@ -18,6 +22,7 @@
import java.io.UncheckedIOException;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;

/**
Expand Down Expand Up @@ -67,4 +72,26 @@ public Network importData(ReadOnlyDataSource dataSource, Properties parameters)
@Override
public void copy(ReadOnlyDataSource fromDataSource, DataSource toDataSource) {
}

@Override
public boolean exists(ReadOnlyDataStore dataStore, String fileName) {
TestDataFormat df = new TestDataFormat(getFormat());
try {
Optional<DataPack> dp = df.newDataResolver().resolve(dataStore, fileName, null);
return dp.isPresent();
} catch (IOException | NonUniqueResultException e) {
return false;
}
}

@Override
public Network importDataStore(ReadOnlyDataStore dataStore, String fileName, Properties parameters) {
return network;
}

@Override
public DataFormat getDataFormat() {
return new TestDataFormat("tst");
}

}
Loading