Merge pull request #1669 from GRIDAPPSD/releases/2022.08.0

Release of version 2022.08.0

Dockerfile

@@ -62,9 +62,22 @@ RUN mkdir ${TEMP_DIR} \
   && rm .git -rf \ 
   && cp -r * /gridappsd/services/gridappsd-topology-processor \
   && cp /gridappsd/services/gridappsd-topology-processor/gridappsd-topology-service.config /gridappsd/services/ \
+  && cp /gridappsd/services/gridappsd-topology-processor/gridappsd-topology-daemon.config /gridappsd/services/ \
   && cd \
   && rm -rf ${TEMP_DIR}
 
+# Get the gridappsd-toolbox from the proper repository
+RUN mkdir ${TEMP_DIR} \
+  && cd ${TEMP_DIR} \
+  && git clone https://github.com/GRIDAPPSD/gridappsd-toolbox -b main \
+  && cd gridappsd-toolbox \
+  && mkdir -p /gridappsd/services/gridappsd-toolbox \
+  && rm .git -rf \ 
+  && cp -r * /gridappsd/services/gridappsd-toolbox \
+  && cp /gridappsd/services/gridappsd-toolbox/static-ybus/gridappsd-static-ybus-service.config /gridappsd/services/ \
+  && cp /gridappsd/services/gridappsd-toolbox/dynamic-ybus/gridappsd-dynamic-ybus-service.config /gridappsd/services/ \
+  && cd \
+  && rm -rf ${TEMP_DIR}
 
 # Copy initial applications and services into the container.
 # 

gov.pnnl.goss.gridappsd/bnd.bnd

@@ -29,7 +29,7 @@
 	osgi.enroute.base.api,\
 	org.mockito.mockito-all,\
 	httpcore,\
-	cimhub.lib;version=0.2.4,\
+	cimhub.lib;version=1.0.1,\
 	httpclient,\
 	com.bigdata.rdf,\
 	org.glassfish.jersey.core.jersey-client;version=2.26,\
@@ -38,8 +38,9 @@
 	proven-client;version=0.2.5,\
 	proven-message;version=0.5,\
 	com.nimbusds.nimbus-jose-jwt-dependencies,\
-
-
+	org.apache.servicemix.bundles.poi;version=3.17,\
+	org.apache.commons.commons-collections4;version=4.4
+
 -plugin org.apache.felix.dm.annotation.plugin.bnd.AnnotationPlugin;log=debug
 
 -sub: *.bnd

gov.pnnl.goss.gridappsd/run.bnd.bndrun

@@ -76,7 +76,7 @@
 	httpcore,\
 	httpclient,\
 	xml-apis,\
-	cimhub.lib;version=0.2.4,\
+	cimhub.lib;version=1.0.1,\
 	org.eclipse.jetty.aggregate.jetty-all-server;version=7.6.9,\
 	com.bigdata.rdf,\
 	proven-message;version=0.5,\
@@ -107,9 +107,9 @@
 	org.apache.felix.http.servlet-api,\
 	com.fasterxml.jackson.jaxrs.jackson-jaxrs-base,\
 	javax.ws.rs.jsr311-api,\
-	javax.ws.rs-api
-
-	
+	org.apache.servicemix.bundles.poi;version=3.17,\
+	org.apache.commons.commons-collections4;version=4.4
+
 
 # Add broker name to the properties defined in shared.runprops
 -runproperties:  ${shared.runprops},\

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/configuration/GLDAllConfigurationHandler.java

@@ -40,11 +40,16 @@
 package gov.pnnl.goss.gridappsd.configuration;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.PrintWriter;
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
@@ -53,6 +58,13 @@
 import org.apache.felix.dm.annotation.api.Start;
 import org.apache.jena.query.QuerySolution;
 import org.apache.jena.query.ResultSet;
+
+import org.apache.poi.hssf.usermodel.HSSFWorkbook;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -119,6 +131,7 @@ public class GLDAllConfigurationHandler extends BaseConfigurationHandler impleme
 	public static final String ENDTIME_FILTER = "endTime";
 	public static final String MODEL_STATE = "model_state";
 	public static final String SIMULATOR = "simulator";
+	public static final String SEPARATED_LOADS_FILE = "separated_loads_file";
 	public static final int TIMEFILTER_YEAR = 2013;
 
 //	public static final String CONFIGTARGET = "glm";
@@ -164,6 +177,7 @@ public void start(){
 	public void generateConfig(Properties parameters, PrintWriter out, String processId, String username) throws Exception {
 		boolean bWantZip = true;
 		boolean bWantSched = false;
+		List<String> separateLoads = new ArrayList<String>();
 
 		logManager.info(ProcessStatus.RUNNING,processId,"Generating all GridLAB-D configuration files using parameters: "+parameters);
 
@@ -244,11 +258,21 @@ public void generateConfig(Properties parameters, PrintWriter out, String proces
 		boolean useClimate = true;//GridAppsDConstants.getBooleanProperty(parameters, USECLIMATE, false);
 
 		boolean bHaveEventGen = true;
+
+		String separatedLoadsFile = GridAppsDConstants.getStringProperty(parameters, SEPARATED_LOADS_FILE, null);
+		String simulator = GridAppsDConstants.getStringProperty(parameters, SIMULATOR, null);
+		//TODO parse xlsx spreadsheet specified in separatedLoadsFile
+		if(separatedLoadsFile!=null && simulator.equalsIgnoreCase("ochre")) {
+			separateLoads = getSeparatedLoadNames(separatedLoadsFile);
+		} else if(separatedLoadsFile==null && simulator.equalsIgnoreCase("ochre")) {
+			logManager.error(ProcessStatus.ERROR,processId,"No "+SEPARATED_LOADS_FILE+" parameter provided");
+			throw new Exception("Missing parameter "+SEPARATED_LOADS_FILE);
+		}
 
 		//cimhub utility uses
 		CIMImporter cimImporter = new CIMImporter();
 		CIMQuerySetter qs = new CIMQuerySetter();
-		cimImporter.start(queryHandler, qs, CONFIGTARGET, fRoot, scheduleName, loadScale, bWantSched, bWantZip, bWantRandomFractions, useHouses, zFraction, iFraction, pFraction, bHaveEventGen, modelState, false);
+		cimImporter.start(queryHandler, qs, CONFIGTARGET, fRoot, scheduleName, loadScale, bWantSched, bWantZip, bWantRandomFractions, useHouses, zFraction, iFraction, pFraction, -1, bHaveEventGen, modelState, false, separateLoads);
 		String tempDataPath = dir.getAbsolutePath();
 
 		//If use climate, then generate gridlabd weather data file
@@ -512,5 +536,39 @@ protected void generateStartupFile(Properties parameters, String tempDataPath, P
 
 	}
 
+	private List<String> getSeparatedLoadNames(String fileName) {
+
+		List<String> loadNames = new ArrayList<String>();
+		boolean isHeader = true;
+
+		try {
+			FileInputStream fis = new FileInputStream(fileName);
+			Workbook workbook = null;
+			if(fileName.toLowerCase().endsWith("xlsx")){
+				workbook = new XSSFWorkbook(fis);
+			}else if(fileName.toLowerCase().endsWith("xls")){
+				workbook = new HSSFWorkbook(fis);
+			}
+
+			Sheet sheet = workbook.getSheetAt(0);
+			Iterator<Row> rowIterator = sheet.iterator();
+			while (rowIterator.hasNext()) 
+	        {
+
+				Row row = rowIterator.next();
+				if(!isHeader){
+					loadNames.add(row.getCell(5).getStringCellValue());
+					System.out.println(row.getCell(5).getStringCellValue());
+				}
+				isHeader=false;
+	        }
+			fis.close();
+
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+
+		return loadNames;
+	}
 
 }

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/configuration/GLDSimulationOutputConfigurationHandler.java

@@ -341,7 +341,7 @@ void parseMeasurement(Map<String, JsonArray> measurements, JsonObject measuremen
 			} else {
 				throw new JsonParseException(String.format("CimMeasurementsToGldPubs::parseMeasurement: The value of measurementType is not a valid type.\nValid types for LinearShuntCompensators are VA, Pos, and PNV.\nmeasurementType = %s.",measurementType));
 			}
-		} else if (conductingEquipmentType.contains("PowerTransformer")) {
+		} else if (conductingEquipmentType.contains("PowerTransformer") || conductingEquipmentType.contains("TransformerTank")) {
 			if(measurementType.equals("VA")) {
 				objectName = conductingEquipmentName;
 				propertyName = "power_in_" + phases;
@@ -352,7 +352,7 @@ void parseMeasurement(Map<String, JsonArray> measurements, JsonObject measuremen
 				objectName = conductingEquipmentName;
 				propertyName = "current_in_" + phases;
 			} else {
-				throw new JsonParseException(String.format("CimMeasurementsToGldPubs::parseMeasurement: The value of measurementType is not a valid type.\nValid types for PowerTransformers are VA, PNV, and A.\nmeasurementType = %s.",measurementType));
+				throw new JsonParseException(String.format("CimMeasurementsToGldPubs::parseMeasurement: The value of measurementType is not a valid type.\nValid types for PowerTransformers and TransformerTanks are VA, PNV, and A.\nmeasurementType = %s.",measurementType));
 			}
 		} else if (conductingEquipmentType.contains("RatioTapChanger")) {
 			if(measurementType.equals("VA")) {
@@ -462,7 +462,7 @@ void parseMeasurement(Map<String, JsonArray> measurements, JsonObject measuremen
 				throw new JsonParseException(String.format("CimMeasurementsToGldPubs::parseMeasurement: The value of measurementType is not a valid type.\nValid types for SynchronousMachine are VA, A, and PNV.\nmeasurementType = %s.",measurementType));
 			}
 		} else {
-			throw new JsonParseException(String.format("CimMeasurementsToGldPubs::parseMeasurement: The value of ConductingEquipment_type is not a recognized object type.\nValid types are ACLineSegment, LinearShuntCompesator, RatioTapChanger, LoadBreakSwitch, EnergyConsumer, PowerElectronicsConnection, and PowerTransformer.\nConductingEquipment_type = %s.",conductingEquipmentType));
+			throw new JsonParseException(String.format("CimMeasurementsToGldPubs::parseMeasurement: The value of ConductingEquipment_type is not a recognized object type.\nValid types are ACLineSegment, LinearShuntCompesator, RatioTapChanger, LoadBreakSwitch, EnergyConsumer, PowerElectronicsConnection, TransformerTank, and PowerTransformer.\nConductingEquipment_type = %s.",conductingEquipmentType));
 
 		}
 		if(measurements.containsKey(objectName)) {

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/configuration/OchreAllConfigurationHandler.java

@@ -147,6 +147,14 @@ public void generateConfig(Properties parameters, PrintWriter out, String proces
 			throw new Exception("Missing parameter "+MODEL_ID);
 		}
 
+		String separated_loads_file = GridAppsDConstants.getStringProperty(parameters, GLDAllConfigurationHandler.SEPARATED_LOADS_FILE, null);
+		if(separated_loads_file==null || separated_loads_file.trim().length()==0){
+			logManager.error(ProcessStatus.ERROR,processId,"No "+GLDAllConfigurationHandler.SEPARATED_LOADS_FILE+" parameter provided");
+			throw new Exception("Missing parameter "+GLDAllConfigurationHandler.SEPARATED_LOADS_FILE);
+		}
+
+
+
 		try{
 			File tmpDir = new File(tempDataPath);
 			RunCommandLine.runCommand("cp -r /gridappsd/services/gridappsd-ochre/inputs/ "+tempDataPath);
@@ -160,14 +168,16 @@ public void generateConfig(Properties parameters, PrintWriter out, String proces
             							CONFIG_FILENAME+" "+
             							simulationBrokerPort+" "+
             							processId+" "+
-            							model_id);
+            							model_id+ " "+
+            							separated_loads_file);
             logManager.info(ProcessStatus.RUNNING, processId, "python /gridappsd/services/gridappsd-ochre/bin/make_config_file.py "+
 					simulationBrokerHost+" "+ 
 					tempDataPath+" "+
 					CONFIG_FILENAME+" "+
 					simulationBrokerPort+" "+
 					processId+" "+
-					model_id);
+					model_id+" "+
+					separated_loads_file);
 
 		}catch(Exception e){
             log.warn("Could not create OCHRE HELICS config file");

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/data/ProvenTimeSeriesDataManagerImpl.java

@@ -217,9 +217,11 @@ public void onMessage(Serializable message) {
         				JsonElement data = parser.parse(event.getData().toString());
         				if (data.isJsonObject()) {
         					JsonObject dataObj = data.getAsJsonObject();
-        					String datatype = dataObj.get("datatype").getAsString();
-        					if(datatype!=null)
-        						provenWriteProducer.sendBulkMessage(event.getData().toString(), datatype, instanceId, simulationId, new Date().getTime());
+        					if(dataObj.get("datatype")!=null){
+        						String datatype = dataObj.get("datatype").getAsString();
+        						if(datatype!=null)
+        							provenWriteProducer.sendBulkMessage(event.getData().toString(), datatype, instanceId, simulationId, new Date().getTime());
+        					}
         				}
             		}
             		else{

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/distributed/FieldBusManagerImpl.java

@@ -164,69 +164,71 @@ public void getFieldMeasurementIds(String modelId) {
 		request.modelId = modelId;
 		request.requestType = PowergridModelDataRequest.RequestType.QUERY_OBJECT_MEASUREMENTS.toString();
 		Response response = null;
-		List<String> measurementList = new ArrayList<String>();
 
 		try {
 
 			// Get Feeder level measurement ids
+			List<String> feederMeasurementList = new ArrayList<String>();
 			for (String equipmentId : topology.root.feeders.addressable_equipment) {
 				request.objectId = equipmentId;
 				response = dataManager.processDataRequest(request, "powergridmodel", null, null,
 						securityConfig.getManagerUser());
 				if (response != null && (response instanceof DataResponse)) {
 					String str = ((DataResponse) response).getData().toString();
 					JSONArray array = new JSONArray(str);
-					measurementList.clear();
 					for (int i = 0; i < array.length(); i++) {
 						JSONObject object = array.getJSONObject(i);
 						String measid = object.getString("measid");
 						measId_messageBus_map.put(measid, topology.root.feeders.message_bus_id);
-						measurementList.add(measid);
+						feederMeasurementList.add(measid);
 					}
-					messageBus_measIds_map.put(topology.root.feeders.message_bus_id, measurementList);
+
 				}
-
 			}
-
+			messageBus_measIds_map.put(topology.root.feeders.message_bus_id, feederMeasurementList);
+
 			// Get switch level measurement ids
 			for (SwitchArea switchArea : topology.root.feeders.switch_areas) {
+				List<String> switchAreaMeasurementList = new ArrayList<String>();
 				for (String equipmentId : switchArea.addressable_equipment) {
 					request.objectId = equipmentId;
 					response = dataManager.processDataRequest(request, "powergridmodel", null, null,
 							securityConfig.getManagerUser());
 					if (response != null && (response instanceof DataResponse)) {
 						String str = ((DataResponse) response).getData().toString();
 						JSONArray array = new JSONArray(str);
-						measurementList.clear();
 						for (int i = 0; i < array.length(); i++) {
 							JSONObject object = array.getJSONObject(i);
 							String measid = object.getString("measid");
 							measId_messageBus_map.put(measid, switchArea.message_bus_id);
-							measurementList.add(measid);
+							switchAreaMeasurementList.add(measid);
 						}
-						messageBus_measIds_map.put(switchArea.message_bus_id, measurementList);
+
 					}
+				}
+
+				messageBus_measIds_map.put(switchArea.message_bus_id, switchAreaMeasurementList);
 
-					// Get Secondary level measurement ids
-					for (SecondaryArea secondaryArea : switchArea.secondary_areas) {
-						for (String equipmentid : secondaryArea.addressable_equipment) {
-							request.objectId = equipmentid;
-							response = dataManager.processDataRequest(request, "powergridmodel", null, null,
-									securityConfig.getManagerUser());
-							if (response != null && (response instanceof DataResponse)) {
-								String str = ((DataResponse) response).getData().toString();
-								JSONArray array = new JSONArray(str);
-								measurementList.clear();
-								for (int i = 0; i < array.length(); i++) {
-									JSONObject object = array.getJSONObject(i);
-									String measid = object.getString("measid");
-									measId_messageBus_map.put(measid, secondaryArea.message_bus_id);
-									measurementList.add(measid);
-								}
-								messageBus_measIds_map.put(secondaryArea.message_bus_id, measurementList);
+				// Get Secondary level measurement ids
+				for (SecondaryArea secondaryArea : switchArea.secondary_areas) {
+					List<String> secondaryAreaMeasurementList = new ArrayList<String>();
+					for (String equipmentId : secondaryArea.addressable_equipment) {
+						request.objectId = equipmentId;
+						response = dataManager.processDataRequest(request, "powergridmodel", null, null,
+								securityConfig.getManagerUser());
+						if (response != null && (response instanceof DataResponse)) {
+							String str = ((DataResponse) response).getData().toString();
+							JSONArray array = new JSONArray(str);
+							for (int i = 0; i < array.length(); i++) {
+								JSONObject object = array.getJSONObject(i);
+								String measid = object.getString("measid");
+								measId_messageBus_map.put(measid, secondaryArea.message_bus_id);
+								secondaryAreaMeasurementList.add(measid);
 							}
+
 						}
 					}
+					messageBus_measIds_map.put(secondaryArea.message_bus_id, secondaryAreaMeasurementList);
 				}
 			}
 		} catch (Exception e) {
@@ -373,13 +375,13 @@ public void run() {
 			int switch_area_index = 0;
 			for (SwitchArea switchArea : root.feeders.switch_areas) {
 				switchArea.message_bus_id = root.feeders.feeder_id + "." + switch_area_index;
-				switch_area_index++;
 				int secondary_area_index = 0;
 				for (SecondaryArea secondaryArea : switchArea.secondary_areas) {
 					secondaryArea.message_bus_id = root.feeders.feeder_id + "." + switch_area_index + "."
 							+ secondary_area_index;
 					secondary_area_index++;
 				}
+				switch_area_index++;
 			}
 
 			this.getFieldMeasurementIds(fieldModelMrid);
@@ -467,4 +469,4 @@ public void getFieldMeasurementIds(String fieldModelMrid) {
 
 	}
 
-}
+}

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/dto/ModelCreationConfig.java

@@ -64,6 +64,7 @@ public class ModelCreationConfig implements Serializable{
 	public boolean randomize_zipload_fractions = false; // should randomize the zipload fraction values (eg. z, i, p_fractions)
 	public boolean use_houses = false;  
 	public ModelState model_state;
+	public String separated_loads_file; // option xslx file containing loads names that will be modeled separate from the main powerflow simulator.
 
 
 
@@ -159,6 +160,12 @@ public ModelState getModel_state() {
 	public void setModel_state(ModelState model_state) {
 		this.model_state = model_state;
 	}
+	public String getSeparateLoadsFile() {
+		return separated_loads_file;
+	}
+	public void setSeparateLoadsFile(String fileName) {
+		this.separated_loads_file = fileName;
+	}
 
 	public static ModelCreationConfig parse(String jsonString){
 		Gson  gson = new Gson();