Added in the storage tests

Finished migration from YANK to OpenMMTools for storage module
Added in the quantity from string utility as a special math_eval
Updated readme
Updated __init__ import statements
bumped the version

README.md

@@ -53,3 +53,13 @@ The module `openmmtools.cache` implements a shared LRU cache for `Context` objec
 If differences in energies in excess of `ENERGY_TOLERANCE` (default: 0.06 kcal/mol) are detected, these systems will be serialized to XML for further debugging.
 
 This is installed onto the command line when the repository is installed.
+
+## Storage
+
+This module `openmmtools.storage` provides a user-friendly storage IO interface to store data to disk. The primary 
+function of this module is to remove the need for the user to define how to format and configure Python variables and 
+OpenMM Quantities to and from the disk. The module is extensible to any type of data a user wants. Currently supports 
+NetCDF storage types for now.
+- `StorageIODriver`: Abstract extendable class to write format-specific IO drivers such as the `NetCDFIODriver`
+- `NetCDFIODriver`: User-configurable IO driver for NetCDF files. Handles built in Python types and `simtk.unit.Quantity`'s
+- `StorageInterface`: A layer which runs on top of a provided `StorageIODriver` to create an way for users to interface with the disk with as minimal effort as possible. 

openmmtools/__init__.py

@@ -6,8 +6,12 @@
 """
 
 # Define global version.
-from openmmtools import version
+from . import version
 __version__ = version.version
 
 # Import modules.
-from openmmtools import testsystems, integrators
+from . import testsystems
+from . import integrators
+from . import storage
+from . import cache
+from . import states

openmmtools/storage/iodrivers.py

@@ -1373,9 +1373,9 @@ def _bind_read(self):
         # Handle variable size objects
         # This line will not happen unless target is real, so output_mode will return the correct value
         if self._output_mode is 'a':
-             self._save_shape = self._bound_target.shape[1:]
+            self._save_shape = self._bound_target.shape[1:]
         else:
-             self._save_shape = self._bound_target.shape
+            self._save_shape = self._bound_target.shape
         self._unit = self._bound_target.getncattr('IODriver_Unit')
         self._set_codifiers(self._bound_target.getncattr('type'))
 
@@ -1455,9 +1455,6 @@ def read(self):
         self._output_mode
         data = self._decoder(self._bound_target)
         unit_name = self._bound_target.getncattr('IODriver_Unit')
-        # Do some things to handle the way quantity_from_string parses units that only have a denominator (e.g. Hz)
-        if unit_name[0] == '/':
-            unit_name = "(" + unit_name[1:] + ")**(-1)"
         cast_unit = quantity_from_string(unit_name)
         if isinstance(cast_unit, unit.Quantity):
             cast_unit = cast_unit.unit
@@ -1651,8 +1648,6 @@ def _decode_dict(self):
             # If Quantity, assign unit.
             if 'units' in output_ncvar.ncattrs():
                 output_unit_name = output_ncvar.getncattr('units')
-                if output_unit_name[0] == '/':
-                    output_unit_name = '(' + output_unit_name[1:] + ')**(-1)'
                 output_unit = quantity_from_string(output_unit_name)
                 output_value = output_value * output_unit
             # Store output.

openmmtools/tests/test_storage_interface.py

@@ -0,0 +1,181 @@
+#!/usr/local/bin/env python
+
+"""
+Test storageinterface.py facility.
+
+The tests are written around the netcdf storage handler for its asserts (its default)
+Testing the storage handlers themselves should be left to the test_storage_iodrivers.py file
+
+"""
+
+# =============================================================================================
+# GLOBAL IMPORTS
+# =============================================================================================
+
+import numpy as np
+from simtk import unit
+import contextlib
+import tempfile
+import shutil
+
+from nose import tools
+
+from openmmtools.storage import StorageInterface, NetCDFIODriver
+
+
+# =============================================================================================
+# TEST HELPER FUNCTIONS
+# =============================================================================================
+
+def spawn_driver(path):
+    """Create a driver that is used to test the StorageInterface class at path location"""
+    return NetCDFIODriver(path)
+
+
+@contextlib.contextmanager
+def temporary_directory():
+    """Context for safe creation of temporary directories."""
+    tmp_dir = tempfile.mkdtemp()
+    try:
+        yield tmp_dir
+    finally:
+        shutil.rmtree(tmp_dir)
+
+# =============================================================================================
+# STORAGE INTERFACE TESTING FUNCTIONS
+# =============================================================================================
+
+
+def test_storage_interface_creation():
+    """Test that the storage interface can create a top level file and read from it"""
+    with temporary_directory() as tmp_dir:
+        test_store = tmp_dir + '/teststore.nc'
+        driver = spawn_driver(test_store)
+        si = StorageInterface(driver)
+        si.add_metadata('name', 'data')
+        assert si.storage_system.ncfile.getncattr('name') == 'data'
+
+
+@tools.raises(Exception)
+def test_read_trap():
+    """Test that attempting to read a non-existent file fails"""
+    with temporary_directory() as tmp_dir:
+        test_store = tmp_dir + '/teststore.nc'
+        driver = spawn_driver(test_store)
+        si = StorageInterface(driver)
+        si.var1.read()
+
+
+def test_variable_write_read():
+    """Test that a variable can be create and written to file"""
+    with temporary_directory() as tmp_dir:
+        test_store = tmp_dir + '/teststore.nc'
+        driver = spawn_driver(test_store)
+        si = StorageInterface(driver)
+        input_data = 4
+        si.four.write(input_data)
+        output_data = si.four.read()
+        assert output_data == input_data
+
+
+def test_variable_append_read():
+    """Test that a variable can be create and written to file"""
+    with temporary_directory() as tmp_dir:
+        test_store = tmp_dir + '/teststore.nc'
+        driver = spawn_driver(test_store)
+        si = StorageInterface(driver)
+        input_data = np.eye(3) * 4.0
+        si.four.append(input_data)
+        si.four.append(input_data)
+        output_data = si.four.read()
+        assert np.all(output_data[0] == input_data)
+        assert np.all(output_data[1] == input_data)
+
+
+@tools.raises(Exception)
+def test_write_protect():
+    """Test that writing twice without removing protection raises an error"""
+    with temporary_directory() as tmp_dir:
+        test_store = tmp_dir + '/teststore.nc'
+        driver = spawn_driver(test_store)
+        si = StorageInterface(driver)
+        input_data = 4
+        si.four.write(input_data)
+        si.four.write(input_data)
+
+
+def test_unbound_read():
+    """Test that a variable can read from the file without previous binding"""
+    with temporary_directory() as tmp_dir:
+        test_store = tmp_dir + '/teststore.nc'
+        driver = spawn_driver(test_store)
+        si = StorageInterface(driver)
+        input_data = 4*unit.kelvin
+        si.four.write(input_data)
+        si.storage_system.close_down()
+        del si
+        driver = spawn_driver(test_store)
+        si = StorageInterface(driver)
+        output_data = si.four.read()
+        assert input_data == output_data
+
+
+def test_directory_creation():
+    """Test that automatic directory-like objects are created on the fly"""
+    with temporary_directory() as tmp_dir:
+        test_store = tmp_dir + '/teststore.nc'
+        driver = spawn_driver(test_store)
+        si = StorageInterface(driver)
+        input_data = 'four'
+        si.dir0.dir1.dir2.var.write(input_data)
+        ncfile = si.storage_system.ncfile
+        target = ncfile
+        for i in range(3):
+            my_dir = 'dir{}'.format(i)
+            assert my_dir in target.groups
+            target = target.groups[my_dir]
+        si.storage_system.close_down()
+        del si
+        driver = spawn_driver(test_store)
+        si = StorageInterface(driver)
+        target = si
+        for i in range(3):
+            my_dir = 'dir{}'.format(i)
+            target = getattr(target, my_dir)
+        assert target.var.read() == input_data
+
+
+def test_multi_variable_creation():
+    """Test that multiple variables can be created in a single directory structure"""
+    with temporary_directory() as tmp_dir:
+        test_store = tmp_dir + '/teststore.nc'
+        driver = spawn_driver(test_store)
+        si = StorageInterface(driver)
+        input_data = [4.0, 4.0, 4.0]
+        si.dir0.var0.write(input_data)
+        si.dir0.var1.append(input_data)
+        si.dir0.var1.append(input_data)
+        si.storage_system.close_down()
+        del si, driver
+        driver = spawn_driver(test_store)
+        si = StorageInterface(driver)
+        assert si.dir0.var0.read() == input_data
+        app_data = si.dir0.var1.read()
+        assert app_data[0] == input_data
+        assert app_data[1] == input_data
+
+
+def test_metadata_creation():
+    """Test that metadata can be added to variables and directories"""
+    with temporary_directory() as tmp_dir:
+        test_store = tmp_dir + '/teststore.nc'
+        driver = spawn_driver(test_store)
+        si = StorageInterface(driver)
+        input_data = 4
+        si.dir0.var1.write(input_data)
+        si.dir0.add_metadata('AmIAGroup', 'yes')
+        si.dir0.var1.add_metadata('AmIAGroup', 'no')
+        dir0 = si.storage_system.ncfile.groups['dir0']
+        var1 = dir0.variables['var1']
+        assert dir0.getncattr('AmIAGroup') == 'yes'
+        assert var1.getncattr('AmIAGroup') == 'no'