#810 Postpone all files generation to optionally include them in a zi…

…p (work in progress)

Signed-off-by: Dimitri Baron <[email protected]>

etc/Dictionaries/DFLError_en_GB.dic

@@ -34,6 +34,7 @@ MissingICInWarmStartingPointMode =  no initial condition (p, q, v or theta) was
 //------------------ Outputs -----------------------
 MissingGeneratorHvdcParameterInSettings = parameter %1% not found in settings file for generator or hvdc %2%
 NoSVCInFlatStartingPointMode =        simulation of secondary voltage controls is not possible with 'FLAT' starting point mode
+FileCreationFailed           =        failed to create %1%
 //------------------ Main ---------------------------
 EnvVariableMissing           =     cannot find environnement variable %1% please check runtime environment
 NetworkFileNotFound          =     network file %1% does not exist

sources/Context.cpp

@@ -179,34 +179,48 @@ void Context::exportOutputs() {
   dydOutput.append(basename_ + ".dyd");
   outputs::Dyd dydWriter(outputs::Dyd::DydDefinition(basename_, dydOutput.generic_string(), generators_, loads_, slackNode_, hvdcLineDefinitions_,
                                                      networkManager_.getBusRegulationMap(), dynamicDataBaseManager_, dynamicModels_, staticVarCompensators_));
-  dydWriter.write();
+  std::ostringstream dydStream = dydWriter.write();
+  mapData_[dydOutput.generic_string()] = dydStream.str();
 
   // create Network.par
   file::path networkOutput(config_.outputDir());
   networkOutput.append("Network.par");
   outputs::Network networkWriter(outputs::Network::NetworkDefinition(networkOutput, config_.getStartingPointMode()));
-  networkWriter.write();
+  std::ostringstream networkStream = networkWriter.write();
+  mapData_[networkOutput.generic_string()] = networkStream.str();
 
   // create specific par
   file::path parOutput(config_.outputDir());
   parOutput.append(basename_ + ".par");
   outputs::Par parWriter(outputs::Par::ParDefinition(basename_, config_, parOutput, generators_, hvdcLineDefinitions_, networkManager_.getBusRegulationMap(),
                                                      dynamicDataBaseManager_, counters_, dynamicModels_, linesById_, tfosById_, staticVarCompensators_,
                                                      loads_));
-  parWriter.write();
+  std::ostringstream parStream = parWriter.write();
+  mapData_[parOutput.generic_string()] = parStream.str();
 
   // Diagram
   file::path diagramDirectory(config_.outputDir());
   diagramDirectory.append(basename_ + common::constants::diagramDirectorySuffix);
   outputs::Diagram diagramWriter(outputs::Diagram::DiagramDefinition(basename_, diagramDirectory.generic_string(), generators_, hvdcLineDefinitions_));
-  diagramWriter.write();
+  diagramWriter.write(mapData_);
 
   outputs::Solver solverWriter{dfl::outputs::Solver::SolverDefinition(config_)};
-  solverWriter.write();
+  solverWriter.write(mapData_);
 
   if (def_.simulationKind == dfl::inputs::Configuration::SimulationKind::SECURITY_ANALYSIS) {
     exportOutputsContingencies();
   }
+
+  for (auto outputFile : mapData_) {
+    std::string filepath = createAbsolutePath(outputFile.first, config_.outputDir().generic_string());
+    std::ofstream file;
+    file.open(filepath.c_str(), std::ios::binary);
+    if (!file.is_open()) {
+      throw Error(FileCreationFailed, filepath.c_str());
+    }
+    file << outputFile.second;
+    file.close();
+  }
 }
 
 void Context::exportOutputJob() {
@@ -218,19 +232,24 @@ void Context::exportOutputJob() {
   if (!mpiContext.isRootProc())
     return;
 
+  std::stringstream jobStream;
+
   switch (def_.simulationKind) {
   case dfl::inputs::Configuration::SimulationKind::SECURITY_ANALYSIS: {
     // For security analysis always export the main jobs file, as dynawo-algorithms will need it
-    outputs::Job::exportJob(jobEntry_, absolute(def_.networkFilepath), config_);
+    outputs::Job::exportJob(jobStream, jobEntry_, absolute(def_.networkFilepath), config_);
     break;
   }
   default:
     // For the rest of calculations, only export the jobs file when in DEBUG mode
 #if _DEBUG_
-    outputs::Job::exportJob(jobEntry_, absolute(def_.networkFilepath), config_);
+    outputs::Job::exportJob(jobStream, jobEntry_, absolute(def_.networkFilepath), config_);
 #endif
     break;
   }
+
+  std::string jobFileName = jobEntry_->getName() + ".jobs";
+  mapData_[jobFileName] = jobStream.str();
 }
 
 void Context::exportOutputsContingencies() {
@@ -266,7 +285,10 @@ void Context::exportOutputsContingency(const inputs::Contingency &contingency, c
   outputs::Job jobEventWriter(outputs::Job::JobDefinition(basenameEvent, def_.dynawoLogLevel, config_, contingency.id, basename_));
   boost::shared_ptr<job::JobEntry> jobEvent = jobEventWriter.write();
   jobsEvents_.emplace_back(jobEvent);
-  outputs::Job::exportJob(jobEvent, absolute(def_.networkFilepath), config_);
+  std::stringstream contingencyJobStream;
+  outputs::Job::exportJob(contingencyJobStream, jobEvent, absolute(def_.networkFilepath), config_);
+  std::string jobEventFileName = jobEntry_->getName() + ".jobs";
+  mapData_[jobEventFileName] = contingencyJobStream.str();
 #endif
 }
 

sources/Context.h

@@ -205,5 +205,7 @@ class Context {
 
   boost::shared_ptr<job::JobEntry> jobEntry_;                 ///< Dynawo job entry
   std::vector<boost::shared_ptr<job::JobEntry>> jobsEvents_;  ///< Dynawo job entries for contingencies
+
+  std::map<std::string, std::string> mapData_;
 };
 }  // namespace dfl

sources/Outputs/include/Diagram.h

@@ -66,7 +66,7 @@ class Diagram {
   /**
    * @brief Write the Diagram file
    */
-  void write() const;
+  void write(std::map<std::string, std::string>& mapData) const;
 
  private:
   /// @brief Different tables in the diagram, qmin or qmax
@@ -104,23 +104,26 @@ class Diagram {
   static void writeTable(const T& element, std::stringstream& buffer, Tables table);
 
   /// @brief Write generator diagrams
-  void writeGenerators() const;
+  void writeGenerators(std::map<std::string, std::string>& mapData) const;
   /// @brief Write VSC converters diagrams
-  void writeConverters() const;
+  void writeConverters(std::map<std::string, std::string>& mapData) const;
 
   /**
    * @brief Write VSC converter diagram
    * @param vscDefinition the VSC definition to use
    */
-  void writeVSC(const dfl::algo::VSCDefinition& vscDefinition) const;
+  void writeVSC(const dfl::algo::VSCDefinition& vscDefinition, std::map<std::string, std::string>& mapData) const;
 
   /**
    * @brief Write LCC converter diagram
    * @param converterId the id of the LCC converter
    * @param powerFactor the power factor of the LCC
    * @param pMax the maximum p of the HVDC line which owns the LCC converter
    */
-  void writeLCC(const algo::HVDCDefinition::ConverterId& converterId, double powerFactor, double pMax) const;
+  void writeLCC(const algo::HVDCDefinition::ConverterId& converterId,
+                double powerFactor,
+                double pMax,
+                std::map<std::string, std::string>& mapData) const;
 
  private:
   DiagramDefinition def_;  ///< Diagram file information

sources/Outputs/include/Dyd.h

@@ -101,7 +101,7 @@ class Dyd {
   /**
    * @brief Write the dyd file
    */
-  void write() const;
+  std::ostringstream write() const;
 
  private:
   DydDefinition def_;  ///< Dyd file information

sources/Outputs/include/Job.h

@@ -75,8 +75,10 @@ class Job {
    * @param networkFileEntry path to the input network file
    * @param config configuration
    */
-  static void exportJob(const boost::shared_ptr<job::JobEntry> &jobEntry, const boost::filesystem::path &networkFileEntry,
-                        const dfl::inputs::Configuration &config);
+  static void exportJob(std::stringstream& ss,
+                        const boost::shared_ptr<job::JobEntry>& jobEntry,
+                        const boost::filesystem::path& networkFileEntry,
+                        const dfl::inputs::Configuration& config);
   /**
    * @brief Constructor
    *

sources/Outputs/include/Network.h

@@ -56,7 +56,7 @@ class Network {
   /**
    * @brief Export Network model parameters file
    */
-  void write() const;
+  std::ostringstream write() const;
 
  private:
   /**

sources/Outputs/include/Par.h

@@ -103,7 +103,7 @@ class Par {
   /**
    * @brief Export PAR file
    */
-  void write() const;
+  std::ostringstream write() const;
 
  private:
   ParDefinition def_;  ///< PAR file definition

sources/Outputs/include/Solver.h

@@ -54,7 +54,7 @@ class Solver {
   /**
    * @brief Export solver par file
    */
-  void write() const;
+  void write(std::map<std::string, std::string>& mapData) const;
 
  private:
   /**

sources/Outputs/src/Diagram.cpp

@@ -43,12 +43,12 @@ Diagram::Diagram(DiagramDefinition &&def) : def_{std::move(def)} {
   }
 }
 
-void Diagram::write() const {
-  writeGenerators();
-  writeConverters();
+void Diagram::write(std::map<std::string, std::string>& mapData) const {
+  writeGenerators(mapData);
+  writeConverters(mapData);
 }
 
-void Diagram::writeGenerators() const {
+void Diagram::writeGenerators(std::map<std::string, std::string>& mapData) const {
   for (const auto &generator : def_.generators) {
     if (!generator.isUsingDiagram() || generator.isUsingRectangularDiagram())
       continue;
@@ -64,13 +64,11 @@ void Diagram::writeGenerators() const {
     writeTable(generator, buffer, Tables::TABLE_QMAX);
     boost::filesystem::path dir(def_.directoryPath);
     std::string filename = dir.append(outputs::constants::diagramFilename(generator.id)).generic_string();
-    std::ofstream ofs(filename, std::ios::binary);
-    ofs << buffer.str();
-    ofs.close();
+    mapData[filename] = buffer.str();
   }
 }
 
-void Diagram::writeVSC(const algo::VSCDefinition &vscDefinition) const {
+void Diagram::writeVSC(const algo::VSCDefinition &vscDefinition, std::map<std::string, std::string>& mapData) const {
   if (!boost::filesystem::exists(def_.directoryPath)) {
     boost::filesystem::create_directories(def_.directoryPath);
   }
@@ -82,12 +80,13 @@ void Diagram::writeVSC(const algo::VSCDefinition &vscDefinition) const {
   writeTable(vscDefinition, buffer, Tables::TABLE_QMAX);
   boost::filesystem::path dir(def_.directoryPath);
   std::string filename = dir.append(outputs::constants::diagramFilename(vscDefinition.id)).generic_string();
-  std::ofstream ofs(filename, std::ios::binary);
-  ofs << buffer.str();
-  ofs.close();
+  mapData[filename] = buffer.str();
 }
 
-void Diagram::writeLCC(const algo::HVDCDefinition::ConverterId &converterId, double powerFactor, double pMax) const {
+void Diagram::writeLCC(const algo::HVDCDefinition::ConverterId &converterId,
+                        double powerFactor,
+                        double pMax,
+                        std::map<std::string, std::string>& mapData) const {
   if (!boost::filesystem::exists(def_.directoryPath)) {
     boost::filesystem::create_directories(def_.directoryPath);
   }
@@ -102,12 +101,10 @@ void Diagram::writeLCC(const algo::HVDCDefinition::ConverterId &converterId, dou
   writeTable(lccDefinition, buffer, Tables::TABLE_QMAX);
   boost::filesystem::path dir(def_.directoryPath);
   std::string filename = dir.append(outputs::constants::diagramFilename(converterId)).generic_string();
-  std::ofstream ofs(filename, std::ios::binary);
-  ofs << buffer.str();
-  ofs.close();
+  mapData[filename] = buffer.str();
 }
 
-void Diagram::writeConverters() const {
+void Diagram::writeConverters(std::map<std::string, std::string>& mapData) const {
   for (const auto &hvdcDefPair : def_.hvdcDefinitions.hvdcLines) {
     const auto &hvdcDef = hvdcDefPair.second;
     if (!hvdcDef.hasDiagramModel()) {
@@ -119,30 +116,30 @@ void Diagram::writeConverters() const {
     switch (hvdcDef.position) {
     case algo::HVDCDefinition::Position::FIRST_IN_MAIN_COMPONENT: {
       if (hvdcDef.vscDefinition1) {
-        writeVSC(*hvdcDef.vscDefinition1);
+        writeVSC(*hvdcDef.vscDefinition1, mapData);
       } else {
-        writeLCC(hvdcDef.converter1Id, hvdcDef.powerFactors.at(0), hvdcDef.pMax);
+        writeLCC(hvdcDef.converter1Id, hvdcDef.powerFactors.at(0), hvdcDef.pMax, mapData);
       }
       break;
     }
     case algo::HVDCDefinition::Position::SECOND_IN_MAIN_COMPONENT: {
       if (hvdcDef.vscDefinition2) {
-        writeVSC(*hvdcDef.vscDefinition2);
+        writeVSC(*hvdcDef.vscDefinition2, mapData);
       } else {
-        writeLCC(hvdcDef.converter2Id, hvdcDef.powerFactors.at(1), hvdcDef.pMax);
+        writeLCC(hvdcDef.converter2Id, hvdcDef.powerFactors.at(1), hvdcDef.pMax, mapData);
       }
       break;
     }
     case algo::HVDCDefinition::Position::BOTH_IN_MAIN_COMPONENT: {
       if (hvdcDef.vscDefinition1) {
-        writeVSC(*hvdcDef.vscDefinition1);
+        writeVSC(*hvdcDef.vscDefinition1, mapData);
       } else {
-        writeLCC(hvdcDef.converter1Id, hvdcDef.powerFactors.at(0), hvdcDef.pMax);
+        writeLCC(hvdcDef.converter1Id, hvdcDef.powerFactors.at(0), hvdcDef.pMax, mapData);
       }
       if (hvdcDef.vscDefinition2) {
-        writeVSC(*hvdcDef.vscDefinition2);
+        writeVSC(*hvdcDef.vscDefinition2, mapData);
       } else {
-        writeLCC(hvdcDef.converter2Id, hvdcDef.powerFactors.at(1), hvdcDef.pMax);
+        writeLCC(hvdcDef.converter2Id, hvdcDef.powerFactors.at(1), hvdcDef.pMax, mapData);
       }
       break;
     }

sources/Outputs/src/Dyd.cpp

@@ -37,7 +37,7 @@ namespace outputs {
 
 Dyd::Dyd(DydDefinition&& def) : def_{std::move(def)} {}
 
-void
+std::ostringstream
 Dyd::write() const {
   dynamicdata::XmlExporter exporter;
   auto dynamicModelsToConnect = dynamicdata::DynamicModelsCollectionFactory::newCollection();
@@ -49,7 +49,10 @@ Dyd::write() const {
   def_.dydSVarC_->write(dynamicModelsToConnect, def_.basename_);
   def_.dydVRRemote_->writeVRRemotes(dynamicModelsToConnect, def_.basename_);
 
-  exporter.exportToFile(dynamicModelsToConnect, def_.filename_, constants::xmlEncoding);
+  std::ostringstream dydStream;
+  exporter.exportToStream(dynamicModelsToConnect, dydStream, constants::xmlEncoding);
+
+  return dydStream;
 }
 
 }  // namespace outputs

sources/Outputs/src/Job.cpp

@@ -164,18 +164,11 @@ boost::shared_ptr<job::OutputsEntry> Job::writeOutputs() const {
   return output;
 }
 
-void Job::exportJob(const boost::shared_ptr<job::JobEntry> &jobEntry, const boost::filesystem::path &networkFileEntry,
-                    const dfl::inputs::Configuration &config) {
-  boost::filesystem::path path(config.outputDir());
-
-  if (!boost::filesystem::is_directory(path)) {
-    boost::filesystem::create_directories(path);
-  }
-
-  path.append(jobEntry->getName() + ".jobs");
-  std::ofstream os(path.c_str(), std::ios::binary);
-
-  auto formatter = xml::sax::formatter::Formatter::createFormatter(os);
+void Job::exportJob(std::stringstream& ss,
+                    const boost::shared_ptr<job::JobEntry>& jobEntry,
+                    const boost::filesystem::path& networkFileEntry,
+                    const dfl::inputs::Configuration& config) {
+  auto formatter = xml::sax::formatter::Formatter::createFormatter(ss);
   formatter->addNamespace("dyn", "http://www.rte-france.com/dynawo");
 
   formatter->startDocument();

sources/Outputs/src/Network.cpp

@@ -29,13 +29,17 @@ namespace outputs {
 
 Network::Network(NetworkDefinition&& def) : def_{std::move(def)} {}
 
-void
+std::ostringstream
 Network::write() const {
   parameters::XmlExporter exporter;
   boost::shared_ptr<parameters::ParametersSetCollection> paramSetCollection = parameters::ParametersSetCollectionFactory::newCollection();
   paramSetCollection->addParametersSet(writeNetworkSet());
   boost::filesystem::path networkFileName(def_.filepath_);
-  exporter.exportToFile(paramSetCollection, networkFileName.generic_string(), constants::xmlEncoding);
+
+  std::ostringstream networkStream;
+  exporter.exportToStream(paramSetCollection, networkStream, constants::xmlEncoding);
+
+  return networkStream;
 }
 
 std::shared_ptr<parameters::ParametersSet>

sources/Outputs/src/Par.cpp

@@ -27,7 +27,8 @@ namespace outputs {
 
 Par::Par(ParDefinition&& def) : def_{std::move(def)} {}
 
-void Par::write() const {
+std::ostringstream
+Par::write() const {
   parameters::XmlExporter exporter;
 
   auto paramSetCollection = parameters::ParametersSetCollectionFactory::newCollection();
@@ -40,7 +41,10 @@ void Par::write() const {
   def_.parDynModel_->write(paramSetCollection, def_.dynamicDataBaseManager_, def_.shuntCounters_, def_.linesByIdDefinitions_, def_.tfosByIdDefinitions_);
   def_.parVRRemote_->writeVRRemotes(paramSetCollection);
 
-  exporter.exportToFile(paramSetCollection, def_.filepath_.generic_string(), constants::xmlEncoding);
+  std::ostringstream parStream;
+  exporter.exportToStream(paramSetCollection, parStream, constants::xmlEncoding);
+
+  return parStream;
 }
 
 }  // namespace outputs