From 568fd269080871512f508426e9077566d21e91ca Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Sat, 22 Sep 2018 10:26:58 -0600 Subject: [PATCH 1/5] Added more filter options to the StationIO to capture most of the parameters represented in the SOAP parameter list --- climata/snotel/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/climata/snotel/__init__.py b/climata/snotel/__init__.py index 27b879e..18eb2ed 100644 --- a/climata/snotel/__init__.py +++ b/climata/snotel/__init__.py @@ -83,10 +83,13 @@ class StationIO(SnotelIO): county = FilterOpt(url_param='countyNames', multi=True) basin = FilterOpt(url_param='hucs', multi=True) parameter = FilterOpt(url_param='elementCds', multi=True) + network = FilterOpt(url_param='networkCds', multi=True) # Additional options min_latitude = FilterOpt(url_param='minLatitude') max_latitude = FilterOpt(url_param='maxLatitude') + min_longitude = FilterOpt(url_param='minLongitude') + max_longitude = FilterOpt(url_param='maxLongitude') min_elevation = FilterOpt(url_param='minElevation') max_elevation = FilterOpt(url_param='maxElevation') ordinals = FilterOpt(url_param='ordinals') From 396b4fb980c52445b8950322617365df94b03004 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Sun, 23 Sep 2018 10:02:24 -0600 Subject: [PATCH 2/5] Added serialize_params to snotel.py to not join the values if passed a list as this is extremely inefficient use of the web service --- climata/base.py | 8 +++++++- climata/snotel/__init__.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/climata/base.py b/climata/base.py index dcf3bcc..81c7f74 100644 --- a/climata/base.py +++ b/climata/base.py @@ -223,7 +223,13 @@ def serialize_params(self, params, complex=False): """ if complex: # See climata.acis for an example implementation - raise NotImplementedError("Cannot serialize %s!" % params) +# raise NotImplementedError("Cannot serialize %s!" % params) + nparams = {} + for key, val in list(params.items()): + url_param = self.get_url_param(key) + if len(val) == 1 and isinstance(val[0], str): + val = val[0] + nparams[url_param] = val else: # Simpler queries can use traditional URL parameters return { diff --git a/climata/snotel/__init__.py b/climata/snotel/__init__.py index 18eb2ed..e78cb72 100644 --- a/climata/snotel/__init__.py +++ b/climata/snotel/__init__.py @@ -53,6 +53,25 @@ def load(self): else: parse = str self.data = [parse(row) for row in self.data] + + def serialize_params(self, params, complex_type): + """ + The AWDB NRCS webservice allows for multiple parameters, need + to have the ability to query multiple stations at a time for + effeciency. If one of the parameters is a list, the client will + be able to make multiple types for the list. + + Therefore, overwrite the `serialize_params` to not join vals if + it's a list + """ + + if complex_type: + raise NotImplementedError("Cannot serialize %s!" % params) + else: + return { + self.get_url_param(key): val + for key, val in params.items() + } # Some records may have additional fields; loop through entire # array to ensure all field names are accounted for. (Otherwise BaseIO From f6981686599ba520d23fc45a993c170e687f1678 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 24 Sep 2018 07:26:49 -0600 Subject: [PATCH 3/5] For Snotel, call 'getStationMetadataMultiple' which removes individual calls to 'getStationMetadata' --- climata/snotel/__init__.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/climata/snotel/__init__.py b/climata/snotel/__init__.py index e78cb72..830d688 100644 --- a/climata/snotel/__init__.py +++ b/climata/snotel/__init__.py @@ -127,12 +127,20 @@ class StationIO(SnotelIO): def load(self): super(StationIO, self).load() - self.data = [ - StationMetaIO(station=station, debug=self.debug).data[0] - for station in self.data - ] + if len(self.data) > 0: + self.data = StationMetaMultipleIO(stations=self.data, debug=self.debug).data +class StationMetaMultipleIO(SnotelIO): + """ + Wrapper for getStationMetadataMultiple() - used internally by StationIO. + """ + + data_function = 'getStationMetadataMultiple' + + stations = FilterOpt(required=True, url_param='stationTriplets', multi=True) + + class StationMetaIO(SnotelIO): """ Wrapper for getStationMetadata() - used internally by StationIO. From 0d8fe7f5dac5e1de8449caa6e81b7d7161f16302 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 24 Sep 2018 13:55:23 -0600 Subject: [PATCH 4/5] parsing date time and created new StationHourlyDataElementIO to query just one element at a time without adding an extra element query each time --- climata/base.py | 23 +++++++++++++++++++++-- climata/snotel/__init__.py | 31 ++++++++++++++++++++++--------- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/climata/base.py b/climata/base.py index 81c7f74..eb77323 100644 --- a/climata/base.py +++ b/climata/base.py @@ -5,6 +5,7 @@ parse_date = make_date_mapper('%Y-%m-%d') +parse_datetime = make_date_mapper('%Y-%m-%d %H:%M:%S') class FilterOpt(object): @@ -48,6 +49,24 @@ def parse(self, value): return [value] return value +class DateTimeOpt(FilterOpt): + date_only = True + + def parse_datetime(self, value): + return parse_datetime(value) + + def parse(self, value): + """ + Parse date + """ + value = super(DateTimeOpt, self).parse(value) + if value is None: + return None + if isinstance(value, str): + value = self.parse_datetime(value) + if isinstance(value, datetime) and self.date_only: + value = value.date() + return value class DateOpt(FilterOpt): date_only = True @@ -64,8 +83,8 @@ def parse(self, value): return None if isinstance(value, str): value = self.parse_date(value) - if isinstance(value, datetime) and self.date_only: - value = value.date() + if isinstance(value, datetime): + value = value.strftime("%Y-%m-%d %H:%M:%S") return value diff --git a/climata/snotel/__init__.py b/climata/snotel/__init__.py index 830d688..42eb88c 100644 --- a/climata/snotel/__init__.py +++ b/climata/snotel/__init__.py @@ -7,7 +7,7 @@ from suds.client import Client from suds.sudsobject import asdict, Object as SudsObject -from climata.base import WebserviceLoader, FilterOpt, DateOpt +from climata.base import WebserviceLoader, FilterOpt, DateOpt, DateTimeOpt from climata.base import fill_date_range, as_list url = 'https://wcc.sc.egov.usda.gov/awdbWebService/services?WSDL' @@ -381,7 +381,7 @@ class HourlyDataIO(TimeSeriesMapper, SnotelIO): ] # Applicable WebserviceLoader default options - station = FilterOpt(required=True, url_param='stationTriplets') + station = FilterOpt(required=True, url_param='stationTriplets', multi=True) parameter = FilterOpt(required=True, url_param='elementCd') start_date = DateOpt(required=True, url_param='beginDate') end_date = DateOpt(required=True, url_param='endDate') @@ -398,14 +398,27 @@ class HourlyDataIO(TimeSeriesMapper, SnotelIO): def load(self): super(HourlyDataIO, self).load() - if self.data and 'values' in self.data[0]: - self.data = [ - asdict(row) - for row in as_list(self.data[0]['values']) - ] - else: - raise NoData + d = {} # since mutiple triplets can be used, then go through each station + for data in self.data: + stationTriplet = data['stationTriplet'] + if data and 'values' in data: + data = [ + asdict(row) + for row in as_list(data['values']) + ] + else: + data = [] + d[stationTriplet] = data + self.data = d +class StationHourlyDataElementIO(HourlyDataIO): + """ + Requests hourly data for the specified stations, just request the + data without asking for the elements. + + """ + inner_io_class = HourlyDataIO + duration = "HOURLY" class StationHourlyDataIO(StationDataIO): """ From 0d03e56721d9a05b945e5bf03acbb99d39d2a8ff Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Tue, 25 Sep 2018 09:27:22 -0600 Subject: [PATCH 5/5] removed the serialize params in base to make it not implemented --- climata/base.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/climata/base.py b/climata/base.py index eb77323..fa3e37b 100644 --- a/climata/base.py +++ b/climata/base.py @@ -242,13 +242,7 @@ def serialize_params(self, params, complex=False): """ if complex: # See climata.acis for an example implementation -# raise NotImplementedError("Cannot serialize %s!" % params) - nparams = {} - for key, val in list(params.items()): - url_param = self.get_url_param(key) - if len(val) == 1 and isinstance(val[0], str): - val = val[0] - nparams[url_param] = val + raise NotImplementedError("Cannot serialize %s!" % params) else: # Simpler queries can use traditional URL parameters return {