IO Changes for V2.0.0 (#739)

* ENH: Initial commit for io v2 * ENH: Updating IO module for V2.0.0 standards and all the subsequent changes. * FIX: Fix for old syntax. * FIX: Update test_plotting.py for new syntax. * FIX: Fix for example needing to be updated. --------- Co-authored-by: Zach Sherman <[email protected]> Co-authored-by: zssherman <[email protected]>
ARM-DOE · Nov 30, 2023 · c985def · c985def
1 parent 14a9757
commit c985def
Show file tree

Hide file tree

Showing 67 changed files with 290 additions and 302 deletions.
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -198,7 +198,7 @@ An example:
 
 .. code-block:: python
 
-    def read_netcdf(filenames, variables=None):
+    def read_arm_netcdf(filenames, variables=None):
 
         """
         Returns `xarray.Dataset` with stored data and metadata from a
@@ -226,7 +226,7 @@ An example:
 
             import act
 
-            the_ds, the_flag = act.io.armfiles.read_netcdf(
+            the_ds, the_flag = act.io.arm.read_arm_netcdf(
                 act.tests.sample_files.EXAMPLE_SONDE_WILDCARD)
             print(the_ds.act.datastream)
         """

diff --git a/act/io/__init__.py b/act/io/__init__.py
@@ -8,17 +8,17 @@
 __getattr__, __dir__, __all__ = lazy.attach(
     __name__,
 
-    submodules=['armfiles', 'csvfiles', 'icartt', 'mpl', 'neon', 'noaagml', 'noaapsl', 'pysp2'],
+    submodules=['arm', 'csv', 'icartt', 'mpl', 'neon', 'noaagml', 'noaapsl', 'pysp2'],
     submod_attrs={
-        'armfiles': [
+        'arm': [
             'WriteDataset',
             'check_arm_standards',
             'create_ds_from_arm_dod',
-            'read_netcdf',
+            'read_arm_netcdf',
             'check_if_tar_gz_file',
-            'read_mmcr',
+            'read_arm_mmcr',
         ],
-        'csvfiles': ['read_csv'],
+        'csv': ['read_csv'],
         'icartt': ['read_icartt'],
         'mpl': ['proc_sigma_mplv5_read', 'read_sigma_mplv5'],
         'neon': ['read_neon_csv'],

diff --git a/act/io/armfiles.py → act/io/arm.py b/act/io/armfiles.py → act/io/arm.py
@@ -27,7 +27,7 @@
 from act.utils.io_utils import cleanup_files, is_gunzip_file, unpack_gzip, unpack_tar
 
 
-def read_netcdf(
+def read_arm_netcdf(
     filenames,
     concat_dim=None,
     return_None=False,
@@ -99,14 +99,11 @@ def read_netcdf(
     .. code-block :: python
 
         import act
-        ds = act.io.armfiles.read_netcdf(act.tests.sample_files.EXAMPLE_SONDE_WILDCARD)
+        ds = act.io.arm.read_arm_netcdf(act.tests.sample_files.EXAMPLE_SONDE_WILDCARD)
         print(ds)
 
     """
 
-    message = 'act.io.armfiles.read_netcdf will be replaced in version 2.0.0 by act.io.arm.read_arm_netcdf()'
-    warnings.warn(message, DeprecationWarning, 2)
-
     ds = None
     filenames, cleanup_temp_directory = check_if_tar_gz_file(filenames)
 
@@ -315,7 +312,7 @@ def keep_variables_to_drop_variables(filenames, keep_variables, drop_variables=N
 
         import act
         filename = '/data/datastream/hou/houkasacrcfrM1.a1/houkasacrcfrM1.a1.20220404.*.nc'
-        drop_vars = act.io.armfiles.keep_variables_to_drop_variables(
+        drop_vars = act.io.arm.keep_variables_to_drop_variables(
             filename, ['lat','lon','alt','crosspolar_differential_phase'],
             drop_variables='variable_name_that_only_exists_in_last_file_of_the_day')
 
@@ -440,7 +437,7 @@ def create_ds_from_arm_dod(
     .. code-block :: python
 
         dims = {'time': 1440, 'drop_diameter': 50}
-        ds = act.io.armfiles.create_ds_from_arm_dod(
+        ds = act.io.arm.create_ds_from_arm_dod(
             'vdis.b1', dims, version='1.2', scalar_fill_dim='time')
 
     """
@@ -811,7 +808,7 @@ def check_if_tar_gz_file(filenames):
     return filenames, cleanup
 
 
-def read_mmcr(filenames):
+def read_arm_mmcr(filenames):
     """
 
     Reads in ARM MMCR files and splits up the variables into specific

diff --git a/act/io/csvfiles.py → act/io/csv.py b/act/io/csvfiles.py → act/io/csv.py
@@ -4,10 +4,8 @@
 """
 
 import pathlib
-
 import pandas as pd
-
-from .armfiles import check_arm_standards
+from act.io.arm import check_arm_standards
 
 
 def read_csv(filename, sep=',', engine='python', column_names=None, skipfooter=0, ignore_index=True, **kwargs):
@@ -48,7 +46,7 @@ def read_csv(filename, sep=',', engine='python', column_names=None, skipfooter=0
 
         import act
 
-        ds = act.io.csvfiles.read(act.tests.sample_files.EXAMPLE_CSV_WILDCARD)
+        ds = act.io.csv.read(act.tests.sample_files.EXAMPLE_CSV_WILDCARD)
 
     """
 

diff --git a/act/io/mpl.py b/act/io/mpl.py
@@ -7,11 +7,9 @@
 import shutil
 import subprocess
 import tempfile
-
 import dask
 import xarray as xr
-
-from act.io.armfiles import check_arm_standards
+from act.io.arm import check_arm_standards
 
 if shutil.which('mpl2nc') is not None:
     MPLIMPORT = True

diff --git a/act/io/neon.py b/act/io/neon.py
@@ -6,7 +6,7 @@
 import xarray as xr
 import datetime as dt
 import numpy as np
-from act.io.csvfiles import read_csv
+from act.io.csv import read_csv
 
 
 def read_neon_csv(files, variable_files=None, position_files=None):

diff --git a/act/io/noaagml.py b/act/io/noaagml.py
@@ -241,7 +241,7 @@ def read_gml_halo(filename, **kwargs):
                 break
             header += 1
 
-    ds = act.io.csvfiles.read_csv(
+    ds = act.io.csv.read_csv(
         filename, sep=r'\s+', header=header,
         na_values=['Nan', 'NaN', 'nan', 'NAN'], **kwargs)
     var_names = list(ds.data_vars)
@@ -418,7 +418,7 @@ def read_gml_co2(filename=None, convert_missing=True, **kwargs):
     with open(test_filename) as fc:
         skiprows = int(fc.readline().strip().split()[-1]) - 1
 
-    ds = act.io.csvfiles.read_csv(
+    ds = act.io.csv.read_csv(
         filename, sep=r'\s+', skiprows=skiprows, **kwargs)
 
     timestamp = np.full(ds['year'].size, np.nan, dtype="datetime64[ns]")
@@ -538,7 +538,7 @@ def read_gml_ozone(filename=None, **kwargs):
                 pass
             skiprows += 1
 
-    ds = act.io.csvfiles.read_csv(
+    ds = act.io.csv.read_csv(
         filename, sep=r'\s+', skiprows=skiprows, **kwargs)
     ds.attrs['station'] = str(ds['STN'].values[0]).lower()
 
@@ -772,7 +772,7 @@ def read_gml_radiation(filename=None, convert_missing=True,
         names.insert(ii + num, 'qc_' + name)
         num += 1
 
-    ds = act.io.csvfiles.read_csv(filename, sep=r'\s+', header=None, skiprows=2, column_names=names, **kwargs)
+    ds = act.io.csv.read_csv(filename, sep=r'\s+', header=None, skiprows=2, column_names=names, **kwargs)
 
     if isinstance(filename, (list, tuple)):
         filename = filename[0]
@@ -994,7 +994,7 @@ def read_gml_met(filename=None, convert_missing=True, **kwargs):
         minutes = False
         del column_names['minute']
 
-    ds = act.io.csvfiles.read_csv(
+    ds = act.io.csv.read_csv(
         filename, sep=r'\s+', header=None,
         column_names=column_names.keys(), **kwargs)
 

diff --git a/act/io/noaapsl.py b/act/io/noaapsl.py
@@ -13,7 +13,7 @@
 import xarray as xr
 import datetime as dt
 
-from act.io.csvfiles import read_csv
+from act.io.csv import read_csv
 
 
 def read_psl_wind_profiler(filepath, transpose=True):

diff --git a/act/plotting/distributiondisplay.py b/act/plotting/distributiondisplay.py
@@ -20,7 +20,7 @@ class DistributionDisplay(Display):
 
     .. code-block:: python
 
-        ds = act.read_netcdf(the_file)
+        ds = act.io.read_arm_netcdf(the_file)
         disp = act.plotting.DistsributionDisplay(ds, subplot_shape=(3,), figsize=(15, 5))
 
     The DistributionDisplay constructor takes in the same keyword arguments as

diff --git a/act/plotting/histogramdisplay.py b/act/plotting/histogramdisplay.py
@@ -20,7 +20,7 @@ class HistogramDisplay(Display):
 
     .. code-block:: python
 
-        ds = act.read_netcdf(the_file)
+        ds = act.io.read_arm_netcdf(the_file)
         disp = act.plotting.HistogramDisplay(ds, subplot_shape=(3,), figsize=(15, 5))
 
     The HistogramDisplay constructor takes in the same keyword arguments as

diff --git a/act/plotting/skewtdisplay.py b/act/plotting/skewtdisplay.py
@@ -43,7 +43,7 @@ class and has therefore has the same attributes as that class.
 
     .. code-block :: python
 
-        sonde_ds = act.io.armfiles.read_netcdf(
+        sonde_ds = act.io.arm.read_arm_netcdf(
            act.tests.sample_files.EXAMPLE_SONDE1)
 
         skewt = act.plotting.SkewTDisplay(sonde_ds)

diff --git a/act/plotting/timeseriesdisplay.py b/act/plotting/timeseriesdisplay.py
@@ -39,7 +39,7 @@ class TimeSeriesDisplay(Display):
 
     .. code-block:: python
 
-        ds = act.read_netcdf(the_file)
+        ds = act.io.read_arm_netcdf(the_file)
         disp = act.plotting.TimeSeriesDisplay(ds, subplot_shape=(3,), figsize=(15, 5))
 
     The TimeSeriesDisplay constructor takes in the same keyword arguments as
@@ -825,7 +825,7 @@ def plot_barbs_from_spd_dir(
         --------
         ..code-block :: python
 
-            sonde_ds = act.io.armfiles.read_netcdf(
+            sonde_ds = act.io.arm.read_arm_netcdf(
                 act.tests.sample_files.EXAMPLE_TWP_SONDE_WILDCARD)
             BarbDisplay = act.plotting.TimeSeriesDisplay(
                 {'sonde_darwin': sonde_ds}, figsize=(10,5))

diff --git a/act/plotting/windrosedisplay.py b/act/plotting/windrosedisplay.py
@@ -30,7 +30,7 @@ class and has therefore has the same attributes as that class.
 
     .. code-block :: python
 
-        sonde_ds = act.io.armfiles.read_netcdf('sonde_data.nc')
+        sonde_ds = act.io.arm.read_arm_netcdf('sonde_data.nc')
         WindDisplay = act.plotting.WindRoseDisplay(sonde_ds, figsize=(8,10))
 
     """

diff --git a/act/qc/add_supplemental_qc.py b/act/qc/add_supplemental_qc.py
@@ -113,9 +113,9 @@ def read_yaml_supplemental_qc(
     .. code-block:: python
 
         from act.tests import EXAPLE_MET_YAML, EXAMPLE_MET1
-        from act.io.armfiles import read_netcdf
+        from act.io.arm import read_arm_netcdf
         from act.qc.add_supplemental_qc import read_yaml_supplemental_qc
-        ds = read_netcdf(EXAMPLE_MET1, cleanup_qc=True)
+        ds = read_arm_netcdf(EXAMPLE_MET1, cleanup_qc=True)
         result = read_yaml_supplemental_qc(ds, EXAPLE_MET_YAML,
                                      variables=['rh_mean'], assessments='Bad')
         print(result)
@@ -263,9 +263,9 @@ def apply_supplemental_qc(
     .. code-block:: python
 
         from act.tests import EXAPLE_MET_YAML, EXAMPLE_MET1
-        from act.io.armfiles import read_netcdf
+        from act.io.arm import read_arm_netcdf
         from act.qc.add_supplemental_qc import apply_supplemental_qc
-        ds = read_netcdf(EXAMPLE_MET1, cleanup_qc=True)
+        ds = read_arm_netcdf(EXAMPLE_MET1, cleanup_qc=True)
         apply_supplemental_qc(ds, EXAPLE_MET_YAML, apply_all=False)
         print(ds['qc_temp_mean'].attrs['flag_meanings'])
 

diff --git a/act/qc/bsrn_tests.py b/act/qc/bsrn_tests.py
@@ -174,7 +174,7 @@ def bsrn_limits_test(
         --------
             .. code-block:: python
 
-                ds = act.io.armfiles.read_netcdf(act.tests.EXAMPLE_BRS, cleanup_qc=True)
+                ds = act.io.arm.read_arm_netcdf(act.tests.EXAMPLE_BRS, cleanup_qc=True)
                 ds.qcfilter.bsrn_limits_test(
                     gbl_SW_dn_name='down_short_hemisp',
                     glb_diffuse_SW_dn_name='down_short_diffuse_hemisp',
@@ -404,7 +404,7 @@ def bsrn_comparison_tests(
         --------
             .. code-block:: python
 
-                ds = act.io.armfiles.read_netcdf(act.tests.EXAMPLE_BRS, cleanup_qc=True)
+                ds = act.io.arm.read_arm_netcdf(act.tests.EXAMPLE_BRS, cleanup_qc=True)
                 ds.qcfilter.bsrn_comparison_tests(
                     gbl_SW_dn_name='down_short_hemisp',
                     glb_diffuse_SW_dn_name='down_short_diffuse_hemisp',

diff --git a/act/qc/clean.py b/act/qc/clean.py
@@ -127,7 +127,7 @@ def cleanup(
             .. code-block:: python
 
                 files = act.tests.sample_files.EXAMPLE_MET1
-                ds = act.io.armfiles.read_netcdf(files)
+                ds = act.io.arm.read_arm_netcdf(files)
                 ds.clean.cleanup()
 
         """
@@ -779,12 +779,12 @@ def normalize_assessment(
         --------
             .. code-block:: python
 
-                ds = act.io.armfiles.read_netcdf(files)
+                ds = act.io.arm.read_arm_netcdf(files)
                 ds.clean.normalize_assessment(variables='temp_mean')
 
             .. code-block:: python
 
-                ds = act.io.armfiles.read_netcdf(files, cleanup_qc=True)
+                ds = act.io.arm.read_arm_netcdf(files, cleanup_qc=True)
                 ds.clean.normalize_assessment(qc_lookup={'Bad': 'Incorrect', 'Indeterminate': 'Suspect'})
 
         """
@@ -847,12 +847,12 @@ def clean_cf_qc(self, variables=None, sep='__', **kwargs):
         --------
             .. code-block:: python
 
-                ds = act.io.armfiles.read_netcdf(files)
+                ds = act.io.arm.read_arm_netcdf(files)
                 ds.clean.clean_cf_qc(variables='temp_mean')
 
             .. code-block:: python
 
-                ds = act.io.armfiles.read_netcdf(files, cleanup_qc=True)
+                ds = act.io.arm.read_arm_netcdf(files, cleanup_qc=True)
 
         """