ADD: Adding new example for working with ARM QC and Xarray Transforma… (

ARM-DOE#734)

* ADD: Adding new example for working with ARM QC and Xarray Transformations.  Updating one example to follow ds convention, and fixing warn message for armfiles

* ENH: PEP8 update

* DOC: test updates

act/io/armfiles.py

@@ -105,7 +105,7 @@ def read_netcdf(
 
     """
 
-    message = 'act.io.armfiles.read_netcdf will be replaced in version 2.0.0 by act.io.arm.read_netcdf()'
+    message = 'act.io.armfiles.read_netcdf will be replaced in version 2.0.0 by act.io.arm.read_arm_netcdf()'
     warnings.warn(message, DeprecationWarning, 2)
 
 

examples/qc/plot_arm_qc.py

@@ -43,11 +43,11 @@
 # the cleanup_qc keyword. This will convert the quality control variable from the ARM stanard
 # to Climate and Forecast standard used internally for all the quality control calls.
 keep_vars = [variable, qc_variable, 'lat', 'lon']
-obj = act.io.armfiles.read_netcdf(results, keep_variables=keep_vars, cleanup_qc=True)
-print(obj)
+ds = act.io.armfiles.read_netcdf(results, keep_variables=keep_vars, cleanup_qc=True)
+print(ds)
 
 # Create a plotting display object with 2 plots
-display = act.plotting.TimeSeriesDisplay(obj, figsize=(15, 10), subplot_shape=(2,))
+display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(2,))
 
 # Plot up the diffuse variable in the first plot
 display.plot(variable, subplot_index=(0,), day_night_background=True)
@@ -62,10 +62,10 @@
 # By default the ancillary quality control variable is removed after appying the test
 # results, but we are going to use the del_qc_var to keep in Dataset so it
 # can be used with additional tests later.
-obj.qcfilter.datafilter(variable, rm_tests=[2, 3], del_qc_var=False)
+ds.qcfilter.datafilter(variable, rm_tests=[2, 3], del_qc_var=False)
 
 # Create a plotting display object with 2 plots
-display = act.plotting.TimeSeriesDisplay(obj, figsize=(15, 10), subplot_shape=(2,))
+display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(2,))
 
 # Plot up the diffuse variable in the first plot
 display.plot(variable, subplot_index=(0,), day_night_background=True)
@@ -84,10 +84,10 @@
 
 # Query the ARM DQR Webservice and update the ancillary quality control variable to
 # contain a new test using information from the DQR.
-obj = act.qc.arm.add_dqr_to_qc(obj, variable=variable)
+ds = act.qc.arm.add_dqr_to_qc(ds, variable=variable)
 
 # Create a plotting display object with 2 plots
-display = act.plotting.TimeSeriesDisplay(obj, figsize=(15, 10), subplot_shape=(2,))
+display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(2,))
 
 # Plot up the diffuse variable in the first plot
 display.plot(variable, subplot_index=(0,), day_night_background=True)
@@ -102,13 +102,13 @@
 # going to filter the data based on this new test and plot up the results.
 
 # Add a new maximum tests
-obj.qcfilter.add_greater_test(variable, 0.4, test_meaning='New maximum tests limit')
+ds.qcfilter.add_greater_test(variable, 0.4, test_meaning='New maximum tests limit')
 
 # Filter that test out
-obj.qcfilter.datafilter(variable, rm_tests=5, del_qc_var=False)
+ds.qcfilter.datafilter(variable, rm_tests=5, del_qc_var=False)
 
 # Create a plotting display object with 2 plots
-display = act.plotting.TimeSeriesDisplay(obj, figsize=(15, 10), subplot_shape=(2,))
+display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(2,))
 
 # Plot up the diffuse variable in the first plot
 display.plot(variable, subplot_index=(0,), day_night_background=True)
@@ -123,10 +123,10 @@
 # it is applied in a moving window style approach.
 
 # Apply test
-obj = act.qc.fft_shading_test(obj, variable=variable)
+ds = act.qc.fft_shading_test(ds, variable=variable)
 
 # Create a plotting display object with 2 plots
-display = act.plotting.TimeSeriesDisplay(obj, figsize=(15, 10), subplot_shape=(2,))
+display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(2,))
 
 # Plot up the diffuse variable in the first plot
 display.plot(variable, subplot_index=(0,), day_night_background=True)
@@ -146,16 +146,16 @@
 # incorrect or suspect values that can be read and applied to the Dataset.
 from act.qc.add_supplemental_qc import apply_supplemental_qc
 
-apply_supplemental_qc(obj, 'sgpmfrsr7nchE11.b1.yaml')
+apply_supplemental_qc(ds, 'sgpmfrsr7nchE11.b1.yaml')
 
 # We can apply or reapply the data filter on the variable in the Dataset to change
 # the data values failing tests to NaN by passing a list of test numbers we want
 # to use. In this case we are not going to apply the DQR test (number 4) so we leave
 # that number out of the list.
-obj.qcfilter.datafilter(variable, rm_tests=[2, 3, 5, 6, 7, 8], del_qc_var=False)
+ds.qcfilter.datafilter(variable, rm_tests=[2, 3, 5, 6, 7, 8], del_qc_var=False)
 
 # Create a plotting display object with 2 plots
-display = act.plotting.TimeSeriesDisplay(obj, figsize=(15, 10), subplot_shape=(2,))
+display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(2,))
 
 # Plot up the diffuse variable in the first plot
 display.plot(variable, subplot_index=(0,), day_night_background=True)

examples/workflows/plot_qc_transforms.py

@@ -0,0 +1,48 @@
+"""
+Transformations and QC
+----------------------
+
+Built-in transformations using xarray are not
+quality-control aware.  This example shows how
+a user should apply QC prior to performing transformations.
+
+"""
+
+import act
+import xarray as xr
+import matplotlib.pyplot as plt
+
+# Read in some sample MFRSR data and clean up the QC
+ds = act.io.armfiles.read_netcdf(act.tests.sample_files.EXAMPLE_MFRSR, cleanup_qc=True)
+
+# Let's resample the data to 5 minutes and take the mean
+ds_5min = ds.resample(time='5min').mean()
+
+variable = 'diffuse_hemisp_narrowband_filter4'
+
+# Let's look at a before and after of one of the qc variables
+print('With no QC applied before transformation')
+print('Before (10 1-minute samples): ', ds['qc_' + variable].values[0:10])
+print('After: (2 5-minute averages)', ds_5min['qc_' + variable].values[0:2])
+
+# That new QC variable does not make sense at all and should be an int
+# What needs to happen is that we apply QC as the user see's fit to all
+# variables before the transformations take place.
+print('\nAverage of ', variable, ' before and after applying QC')
+print('Note the change in the second value')
+print('Before (2 5 - minute averages): ', ds[variable].values[0:2])
+
+ds.qcfilter.datafilter(rm_assessments=['Bad', 'Indeterminate'])
+ds_5minb = ds.resample(time='5min').mean()
+
+# Print out the corresponding variable values
+print('After: (2 5 - minute averages)', ds_5minb[variable].values[0:2])
+
+## Plot up the variable and qc block plot
+display = act.plotting.TimeSeriesDisplay({'Original': ds, 'Average': ds_5min, 'Average_QCd': ds_5minb},
+                                         figsize=(15, 10), subplot_shape=(2,))
+display.plot(variable, dsname='Original', subplot_index=(0,), day_night_background=True)
+display.plot(variable, dsname='Average', subplot_index=(1,), day_night_background=True, label='No QC')
+display.plot(variable, dsname='Average_QCd', subplot_index=(1,), day_night_background=True, label='QC')
+plt.legend()
+plt.show()