Merge pull request #242 from texadactyl/master

Clean up console messages during file writing

VERSION-HISTORY.md

@@ -3,6 +3,7 @@ This file is a version history of blimpy amendments, beginning with version 2.0.
 <br>
 |    Date    | Version | Contents |
 | :--: | :--: | :-- |
+| 2021-08-18 | 2.0.29 | Clean up messages when writing files (Issue #241).  |
 | 2021-08-18 | 2.0.28 | Fix utility stax difficulties with the time-axis (Issue #238).  |
 | | | More fixes to the regression tests. |
 | 2021-08-18 | 2.0.27.1 | Fix problems with stix and stax.  |

blimpy/fil2h5.py

@@ -9,6 +9,7 @@
 
 import sys
 import os
+import time
 from argparse import ArgumentParser
 
 # Logging set up
@@ -49,7 +50,9 @@ def make_h5_file(filename, out_dir='./', new_filename=None, t_start=None, t_stop
     if '.h5' not in new_filename:
         new_filename = new_filename+'.h5'
 
+    t0 = time.time()
     write_to_h5(wf, new_filename)
+    wf.logger.info('Conversion time: %2.2fsec' % (time.time()- t0))
 
 
 def cmd_tool(args=None):

blimpy/io/fil_writer.py

@@ -6,12 +6,14 @@
 from .sigproc import generate_sigproc_header
 
 
-def write_to_fil(wf, filename_out, *args, **kwargs):
+def write_to_fil(wf, filename_out):
     """ Write data to .fil file.
-        It check the file size then decides how to write the file.
+        It checks the file size then decides how to write the file.
 
     Args:
-        filename_out (str): Name of output file
+        wf : Waterfall object
+        filename_out : str
+            Name of output file
     """
 
     # For timing how long it takes to write a file.
@@ -33,7 +35,9 @@ def __write_to_fil_heavy(wf, filename_out):
     """ Write data to .fil file.
 
     Args:
-        filename_out (str): Name of output file
+        wf : Waterfall object
+        filename_out : str
+            Name of output file
     """
 
     # Note that a chunk is not a blob!!
@@ -44,7 +48,9 @@ def __write_to_fil_heavy(wf, filename_out):
     # Calculate number of bytes per data element
     n_bytes = wf.header['nbits'] / 8
 
-    wf.logger.info('Using %i n_blobs to write the data.' % n_blobs)
+    wf.logger.info("__write_to_fil_heavy: For {}, chunk_dim={}, blob_dim={}, n_blobs={}"
+                   .format(filename_out, chunk_dim, blob_dim, n_blobs))
+
     with open(filename_out, "wb") as fileh:
 
         # Write header of .fil file
@@ -53,7 +59,7 @@ def __write_to_fil_heavy(wf, filename_out):
         # For each blob
         for ii in range(0, n_blobs):
 
-            wf.logger.info('Reading %i of %i' % (ii + 1, n_blobs))
+            wf.logger.info('__write_to_fil_heavy: Processing %i of %i' % (ii + 1, n_blobs))
             bob = wf.container.read_blob(blob_dim, n_blob=ii)
 
             # Write data of .fil file.
@@ -69,9 +75,13 @@ def __write_to_fil_light(wf, filename_out):
     """ Write data to .fil file.
 
     Args:
-        filename_out (str): Name of output file
+        wf : Waterfall object
+        filename_out : str
+            Name of output file
     """
 
+    wf.logger.info("__write_to_fil_light: Writing the spectra matrix for {} in one go."
+                   .format(filename_out))
     n_bytes = wf.header['nbits'] / 8
     with open(filename_out, "wb") as fileh:
         fileh.write(generate_sigproc_header(wf))

blimpy/io/hdf_writer.py

@@ -48,6 +48,8 @@ def __write_to_hdf5_heavy(wf, filename_out, f_scrunch=None, *args, **kwargs):
     chunk_dim = tuple(chunk_list)
     blob_dim  = wf._get_blob_dimensions(chunk_dim)
     n_blobs   = wf.container.calc_n_blobs(blob_dim)
+    wf.logger.info("__write_to_hdf5_heavy: For {}, chunk_dim={}, blob_dim={}, n_blobs={}"
+                   .format(filename_out, chunk_dim, blob_dim, n_blobs))
 
     # ===============================
     # Attempt to write the HDF5 file.
@@ -103,9 +105,8 @@ def __write_to_hdf5_heavy(wf, filename_out, f_scrunch=None, *args, **kwargs):
 
             if blob_dim[wf.freq_axis] < wf.selection_shape[wf.freq_axis]:
 
-                wf.logger.info('Using %i n_blobs to write the data.'% n_blobs)
                 for ii in range(0, n_blobs):
-                    wf.logger.info('Reading %i of %i' % (ii + 1, n_blobs))
+                    wf.logger.info('__write_to_hdf5_heavy: Processing blob %i of %i' % (ii + 1, n_blobs))
 
                     bob = wf.container.read_blob(blob_dim, n_blob=ii)
 
@@ -129,9 +130,8 @@ def __write_to_hdf5_heavy(wf, filename_out, f_scrunch=None, *args, **kwargs):
 
             else:
 
-                wf.logger.info('Using %i n_blobs to write the data.'% n_blobs)
                 for ii in range(0, n_blobs):
-                    wf.logger.info('Reading %i of %i' % (ii + 1, n_blobs))
+                    wf.logger.info('__write_to_hdf5_heavy: Processing blob %i of %i' % (ii + 1, n_blobs))
 
                     bob = wf.container.read_blob(blob_dim, n_blob=ii)
                     t_start = wf.container.t_start + ii * blob_dim[wf.time_axis]
@@ -152,18 +152,18 @@ def __write_to_hdf5_heavy(wf, filename_out, f_scrunch=None, *args, **kwargs):
         # =================================
 
     except Exception as ex1: # Something went wrong!
-        wf.logger.error("blimpy __write_to_hdf5_heavy: Writing the output HDF5 file {} failed!"
+        wf.logger.error("__write_to_hdf5_heavy: Writing the output HDF5 file {} failed!"
                        .format(filename_out))
         print(repr(ex1))
         try:
             if os.path.exists(filename_out):
                 os.remove(filename_out) # scrap a potentially corrupted HDF5 file.
                 # Removal succeeded.  Exit to the O/S with a nonzero exit code.
-                wf.logger.info("blimpy __write_to_hdf5_heavy: Removal of partial HDF5 file {} succeeded."
+                wf.logger.info("__write_to_hdf5_heavy: Removal of partial HDF5 file {} succeeded."
                                .format(filename_out))
             sys.exit(86)
         except Exception as ex2:
-            wf.logger.error("blimpy __write_to_hdf5_heavy: Removal of partial HDF5 file {} failed!"
+            wf.logger.error("__write_to_hdf5_heavy: Removal of partial HDF5 file {} failed!"
                            .format(filename_out))
             print(repr(ex2))
             sys.exit(86)
@@ -177,6 +177,8 @@ def __write_to_hdf5_light(wf, filename_out, f_scrunch=None, *args, **kwargs):
         f_scrunch (int or None): Average (scrunch) N channels together
     """
 
+    wf.logger.info("__write_to_hdf5_light: Writing the spectra matrix for {} in one go."
+                   .format(filename_out))
     try:
         os.remove(filename_out)  # Try to pre-remove output .h5 file.
     except:

setup.py

@@ -3,7 +3,7 @@
 """
 from setuptools import setup, find_packages
 
-__version__ = '2.0.28'
+__version__ = '2.0.29'
 
 with open("README.md", "r") as fh:
     long_description = fh.read()