Merge pull request #882 from GRIDAPPSD/releases/2019.03.0

2019.03.0 release

.travis.yml

@@ -17,21 +17,5 @@ before_install:
   - './.travis/main.sh'
 
 script:
-  # - './gradlew check'
   - './gradlew export'
-  - './.travis/build-docker.sh -b'
-
-# only execute the following instructions in
-# the case of a success (failing at this point
-# won't mark the build as a failure).
-# To have `DOCKER_USERNAME` and `DOCKER_PASSWORD`
-# filled you need to either use `travis`' cli 
-# and then `travis set ..` or go to the travis
-# page of your repository and then change the 
-# environment in the settings pannel.
-after_success:
-  - if [ -n "$DOCKER_USERNAME" -a -n "$DOCKER_PASSWORD" ]; then
-      docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD ;
-      ./.travis/build-docker.sh -p ;
-    fi
-
+  - './.travis/build-docker.sh'

.travis/build-docker.sh

@@ -1,12 +1,6 @@
 #!/bin/bash
 
-usage () {
-  /bin/echo "Usage:  $0 -b Build the docker image"
-  /bin/echo "           -p Push image to dockerhub"
-  exit 2
-}
-
-TAG="$TRAVIS_BRANCH"
+TAG="${TRAVIS_BRANCH//\//_}"
 
 ORG=`echo $DOCKER_PROJECT | tr '[:upper:]' '[:lower:]'`
 ORG="${ORG:+${ORG}/}"
@@ -17,26 +11,59 @@ GITHASH=`git log -1 --pretty=format:"%h"`
 BUILD_VERSION="${TIMESTAMP}_${GITHASH}${TRAVIS_BRANCH:+:$TRAVIS_BRANCH}"
 echo "BUILD_VERSION $BUILD_VERSION"
 
-# parse options
-while getopts bp option ; do
-  case $option in
-    b) # Pass gridappsd tag to docker-compose
-      # Docker file on travis relative from root.
-      docker build --build-arg TIMESTAMP="${BUILD_VERSION}" -t ${IMAGE}:${TIMESTAMP}_${GITHASH} .
-      ;;
-    p) # Pass gridappsd tag to docker-compose
-      if [ -n "$TAG" -a -n "$ORG" ]; then
-        echo "docker push ${IMAGE}:${TIMESTAMP}_${GITHASH}"
-        docker push ${IMAGE}:${TIMESTAMP}_${GITHASH}
-        docker tag ${IMAGE}:${TIMESTAMP}_${GITHASH} ${IMAGE}:$TAG
-        echo "docker push ${IMAGE}:$TAG"
-        docker push ${IMAGE}:$TAG
-      fi
-      ;;
-    *) # Print Usage
-      usage
-      ;;
-  esac
-done
-shift `expr $OPTIND - 1`
+# Pass gridappsd tag to docker-compose
+docker build --build-arg TIMESTAMP="${BUILD_VERSION}" -t ${IMAGE}:${TIMESTAMP}_${GITHASH} .
+status=$?
+if [ $status -ne 0 ]; then
+  echo "Error: status $status"
+  exit 1
+fi
+
+# To have `DOCKER_USERNAME` and `DOCKER_PASSWORD`
+# filled you need to either use `travis`' cli
+# (https://github.com/travis-ci/travis.rb)
+# and then `travis set ..` or go to the travis
+# page of your repository and then change the
+# environment in the settings pannel.
+
+if [ -n "$DOCKER_USERNAME" -a -n "$DOCKER_PASSWORD" ]; then
+
+  echo " "
+  echo "Connecting to docker"
+
+  echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin
+  status=$?
+  if [ $status -ne 0 ]; then
+    echo "Error: status $status"
+    exit 1
+  fi
+
+  if [ -n "$TAG" -a -n "$ORG" ]; then
+    # Get the built container name
+    CONTAINER=`docker images --format "{{.Repository}}:{{.Tag}}" ${IMAGE}`
+
+    echo "docker push ${CONTAINER}"
+    docker push "${CONTAINER}"
+    status=$?
+    if [ $status -ne 0 ]; then
+      echo "Error: status $status"
+      exit 1
+    fi
+
+    echo "docker tag ${CONTAINER} ${IMAGE}:$TAG"
+    docker tag ${CONTAINER} ${IMAGE}:$TAG
+    status=$?
+    if [ $status -ne 0 ]; then
+      echo "Error: status $status"
+      exit 1
+    fi
 
+    echo "docker push ${IMAGE}:$TAG"
+    docker push ${IMAGE}:$TAG
+    status=$?
+    if [ $status -ne 0 ]; then
+      echo "Error: status $status"
+      exit 1
+    fi
+  fi
+fi

gov.pnnl.goss.gridappsd/bnd.bnd

@@ -32,8 +32,8 @@
 	blazegraph.cim2glm;version=8.0,\
 	httpclient,\
 	com.bigdata.rdf,\
-        proven-client;version=0.2.1,\
-        proven-message;version=0.2
+    proven-client;version=0.2.1,\
+    proven-message;version=0.3
 
 -plugin org.apache.felix.dm.annotation.plugin.bnd.AnnotationPlugin;log=debug
 

gov.pnnl.goss.gridappsd/gridappsd.api.bnd

@@ -1,4 +1,5 @@
 Bundle-Version: 1.0.0.${tstamp}
 Export-Package: \
 	gov.pnnl.goss.gridappsd.api,\
-	gov.pnnl.goss.gridappsd.dto
+	gov.pnnl.goss.gridappsd.dto
+Private-Package: gov.pnnl.goss.gridappsd.data.conversion

gov.pnnl.goss.gridappsd/run.bnd.bndrun

@@ -78,7 +78,7 @@
 	org.eclipse.jetty.aggregate.jetty-all-server;version=7.6.9,\
 	javax.servlet-api,\
 	com.bigdata.rdf,\
-	proven-message;version=0.2,\
+	proven-message;version=0.3,\
 	proven-client;version=0.2.1,\
 	javax.ws.rs-api,\
 	org.apache.commons.codec,\

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/configuration/GLDAllConfigurationHandler.java

@@ -58,6 +58,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.gson.JsonSyntaxException;
+
 import gov.pnnl.goss.cim2glm.CIMImporter;
 import gov.pnnl.goss.cim2glm.queryhandler.QueryHandler;
 import gov.pnnl.goss.gridappsd.api.ConfigurationHandler;
@@ -69,6 +71,7 @@
 import gov.pnnl.goss.gridappsd.data.conversion.ProvenWeatherToGridlabdWeatherConverter;
 import gov.pnnl.goss.gridappsd.data.handlers.BlazegraphQueryHandler;
 import gov.pnnl.goss.gridappsd.dto.RequestTimeseriesData;
+import gov.pnnl.goss.gridappsd.dto.LogMessage.LogLevel;
 import gov.pnnl.goss.gridappsd.dto.RequestTimeseriesData.RequestType;
 import gov.pnnl.goss.gridappsd.utils.GridAppsDConstants;
 import pnnl.goss.core.Client;
@@ -236,30 +239,45 @@ public void generateConfig(Properties parameters, PrintWriter out, String proces
 		String tempDataPath = dir.getAbsolutePath();
 
 		//If use climate, then generate gridlabd weather data file
-		if(useClimate){
-			RequestTimeseriesData weatherRequest = new RequestTimeseriesData();
-			weatherRequest.setQueryMeasurement(RequestType.weather);
-			weatherRequest.setResponseFormat(ProvenWeatherToGridlabdWeatherConverter.OUTPUT_FORMAT);
-			Map<String, String> queryFilter = new HashMap<String, String>();
-
-			Calendar c = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
-			//For both the start and end time, set the year to the one that currently has data in the database
-			//TODO either we need more weather data in the database, or make this more flexible wehre we only have to search by month/day
-			c.setTime(new Date(simulationStartTime*1000));
-			c.set(Calendar.YEAR, TIMEFILTER_YEAR);
-			//Convert to UTC time until the input time is correct
-			////TODO this will be changed in the future
-			c.add(Calendar.HOUR, 6);
-			queryFilter.put(STARTTIME_FILTER, ""+c.getTimeInMillis()+"000");
-		        c.add(Calendar.SECOND, new Long(simulationDuration).intValue());	
-			queryFilter.put(ENDTIME_FILTER, ""+c.getTimeInMillis()+"000");
-			weatherRequest.setQueryFilter(queryFilter);
-			DataResponse resp = (DataResponse)dataManager.processDataRequest(weatherRequest, ProvenTimeSeriesDataManagerImpl.DATA_MANAGER_TYPE, simId, tempDataPath, username);
-			File weatherFile = new File(directory+File.separator+WEATHER_FILENAME);
-			FileOutputStream fout = new FileOutputStream(weatherFile);
-			fout.write(resp.getData().toString().getBytes());
-			fout.flush();
-			fout.close();
+		try {
+			if(useClimate){
+				RequestTimeseriesData weatherRequest = new RequestTimeseriesData();
+				weatherRequest.setQueryMeasurement(RequestType.weather);
+				weatherRequest.setResponseFormat(ProvenWeatherToGridlabdWeatherConverter.OUTPUT_FORMAT);
+				Map<String, String> queryFilter = new HashMap<String, String>();
+
+				Calendar c = Calendar.getInstance();
+				//For both the start and end time, set the year to the one that currently has data in the database
+				//TODO either we need more weather data in the database, or make this more flexible where we only have to search by month/day
+				c.setTime(new Date(simulationStartTime*1000));
+				c.set(Calendar.YEAR, TIMEFILTER_YEAR);
+				//Convert to UTC time until the input time is correct
+				////TODO this will be changed in the future
+				//c.add(Calendar.HOUR, 6);
+				queryFilter.put(STARTTIME_FILTER, ""+c.getTimeInMillis()+"000");
+			    c.add(Calendar.SECOND, new Long(simulationDuration).intValue());	
+				queryFilter.put(ENDTIME_FILTER, ""+c.getTimeInMillis()+"000");
+				weatherRequest.setQueryFilter(queryFilter);
+				DataResponse resp = (DataResponse)dataManager.processDataRequest(weatherRequest, ProvenTimeSeriesDataManagerImpl.DATA_MANAGER_TYPE, simId, tempDataPath, username);
+				if(resp.getData()==null){
+					useClimate = false;
+					throw new Exception("No weather data in time series data store. Setting useClimate = false.");
+				}
+				else{
+					File weatherFile = new File(directory+File.separator+WEATHER_FILENAME);
+					FileOutputStream fout = new FileOutputStream(weatherFile);
+					fout.write(resp.getData().toString().getBytes());
+					fout.flush();
+					fout.close();
+				}
+			}
+		} catch (JsonSyntaxException e) {
+			logRunning("No weather data was found in proven. Running Simulation without weather data.",
+					processId, username, logManager, LogLevel.WARN);
+			useClimate = false;
+		}catch (Exception e) {
+			logRunning(e.getMessage(),
+					processId, username, logManager, LogLevel.WARN);
 		}
 
 		//Generate startup file

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/configuration/GLDSimulationOutputConfigurationHandler.java

@@ -379,7 +379,11 @@ void parseMeasurement(Map<String, JsonArray> measurements, JsonObject measuremen
 		} else if (conductingEquipmentType.contains("PowerElectronicsConnection")) {
 			if(measurementType.equals("VA")) {
 				objectName = conductingEquipmentName;
-				propertyName = "measured_power_" + phases;
+				if(phases.equals("1") || phases.equals("2")) {
+					propertyName = "indiv_measured_power_" + phases;
+				} else {
+					propertyName = "measured_power_" + phases;
+				}
 			} else if (measurementType.equals("PNV")) {
 				objectName = conductingEquipmentName;
 				propertyName = "voltage_" + phases;
@@ -394,7 +398,10 @@ void parseMeasurement(Map<String, JsonArray> measurements, JsonObject measuremen
 
 		}
 		if(measurements.containsKey(objectName)) {
-			measurements.get(objectName).add(new JsonPrimitive(propertyName));
+			JsonPrimitive p = new JsonPrimitive(propertyName);
+			if(!measurements.get(objectName).contains(p)) {
+				measurements.get(objectName).add(new JsonPrimitive(propertyName));
+			}
 		} else {
 			JsonArray newMeasurements = new JsonArray();
 			newMeasurements.add(new JsonPrimitive(propertyName));

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/data/ProvenTimeSeriesDataManagerImpl.java

@@ -24,6 +24,7 @@
 import gov.pnnl.goss.gridappsd.dto.LogMessage.LogLevel;
 import gov.pnnl.goss.gridappsd.dto.LogMessage.ProcessStatus;
 import gov.pnnl.goss.gridappsd.dto.RequestTimeseriesData;
+import gov.pnnl.goss.gridappsd.dto.TimeSeriesResult;
 import gov.pnnl.goss.gridappsd.utils.GridAppsDConstants;
 import gov.pnnl.proven.api.producer.ProvenProducer;
 import gov.pnnl.proven.api.producer.ProvenResponse;
@@ -126,7 +127,9 @@ public Serializable query(RequestTimeseriesData requestTimeseriesData) throws Ex
 		provenProducer.restProducer(provenQueryUri, null, null);
 		provenProducer.setMessageInfo("GridAPPSD", "QUERY", this.getClass().getSimpleName(), keywords);
 		ProvenResponse response = provenProducer.sendMessage(requestTimeseriesData.toString(), requestId);
-
+		TimeSeriesResult result = TimeSeriesResult.parse(response.data.toString());
+		if(result.getMeasurements().get(0).getPoints().size()==0)
+			return null;
 		String origFormat = "PROVEN_"+requestTimeseriesData.getQueryMeasurement().toString();
 		String responseFormat = requestTimeseriesData.getResponseFormat();
 		DataFormatConverter converter = dataManager.getConverter(origFormat, responseFormat);

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/data/conversion/ProvenWeatherToGridlabdWeatherConverter.java

@@ -4,6 +4,7 @@
 import java.io.PrintWriter;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
@@ -40,7 +41,7 @@ public class ProvenWeatherToGridlabdWeatherConverter implements DataFormatConver
 	public static String HUMIDITY = "TowerRH";
 	public static String LONGITUDE = "long";
 	public static String LATITUDE = "lat";
-	public static String MST = "MST";
+	public static String MST = "UTC";
 	public static String TEMPERATURE = "TowerDryBulbTemp";
 	public static String DATE = "DATE";
 	public static String TIME = "time";
@@ -53,10 +54,10 @@ public class ProvenWeatherToGridlabdWeatherConverter implements DataFormatConver
 	@ServiceDependency 
 	private volatile LogManager logManager;
 
-	static{
-	    sdfIn.setTimeZone(TimeZone.getTimeZone("MST"));
+	/*static{
+	    sdfIn.setTimeZone(TimeZone.getTimeZone("UTC"));
 		sdfOut.setTimeZone(TimeZone.getTimeZone("UTC"));
-	}
+	}*/
 
 	public ProvenWeatherToGridlabdWeatherConverter(){}
 	public ProvenWeatherToGridlabdWeatherConverter(LogManager logManager, DataManager dataManager) {
@@ -87,7 +88,6 @@ public void start(){
 	@Override
 	public void convert(String inputContent, PrintWriter outputContent) throws Exception {
 		boolean headerPrinted = false;
-
 		TimeSeriesResult resultObj = TimeSeriesResult.parse(inputContent);
 		for(TimeSeriesMeasurementResult record: resultObj.getMeasurements()){
 			if(!headerPrinted){
@@ -111,7 +111,6 @@ public void convert(InputStream inputContent, PrintWriter outputContent)  throws
 			}
 			convertRecord(record, outputContent);
 		}
-
 	}
 
 	protected void printGLDHeader(TimeSeriesMeasurementResult record, PrintWriter outputContent){
@@ -144,12 +143,19 @@ protected void convertRecord(TimeSeriesMeasurementResult record, PrintWriter out
 		for(TimeSeriesRowResult result: record.getPoints()){
 			Map<String, String> map = result.getRow().getEntryMap();
 
-			String dateStr = map.get(DATE);
-			String timeStr = map.get(MST);
+			//String dateStr = map.get(DATE);
+			//String timeStr = map.get(MST);
 			try {
-				Date datetime = sdfIn.parse(dateStr+" "+timeStr);
-				outputContent.print(sdfOut.format(datetime)+",");
-			} catch (ParseException e) {
+
+
+				Calendar c = Calendar.getInstance();
+				//For both the start and end time, set the year to the one that currently has data in the database
+				//TODO either we need more weather data in the database, or make this more flexible where we only have to search by month/day
+				c.setTime(new Date(Long.parseLong(map.get("time"))*1000));
+				c.set(Calendar.YEAR, 2013);
+				//Date datetime = sdfIn.parse(dateStr+" "+timeStr);
+				outputContent.print(sdfOut.format(c.getTime())+",");
+			} catch (NumberFormatException e) {
 				e.printStackTrace();
 				//todo throw exception
 			}

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/dto/RequestTimeseriesData.java

@@ -56,8 +56,8 @@ public static RequestTimeseriesData parse(String jsonString){
 		Gson  gson = new Gson();
 		RequestTimeseriesData obj = gson.fromJson(jsonString, RequestTimeseriesData.class);
 		if(obj.queryMeasurement==RequestType.PROVEN_MEASUREMENT)
-			if(obj.queryFilter==null || !obj.queryFilter.containsKey("hasSimulationId"))
-				throw new JsonSyntaxException("Expected filter hasSimulationId not found.");
+			if(obj.queryFilter==null || !obj.queryFilter.containsKey("simulation_id"))
+				throw new JsonSyntaxException("Expected filter simulation_id not found.");
 		return obj;
 	}