From f22c75d4a40fc922fd06548daf65b0d94a5305da Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 16 Nov 2022 16:01:07 -0600 Subject: [PATCH 1/9] Bump version to 2.1.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8def913..a2fe14d 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ """ from setuptools import setup, find_packages -__version__ = '2.1.3' +__version__ = '2.1.4' with open("README.md", "r") as fh: long_description = fh.read() From 9f68b5e6834ae03fd9b9a9634d3c1d1c0282a542 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 16 Nov 2022 16:02:57 -0600 Subject: [PATCH 2/9] Note issue #276 --- VERSION-HISTORY.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/VERSION-HISTORY.md b/VERSION-HISTORY.md index f41de6f..6a53375 100644 --- a/VERSION-HISTORY.md +++ b/VERSION-HISTORY.md @@ -3,7 +3,8 @@ This file is a version history of blimpy amendments, beginning with version 2.0.
| Date | Version | Contents | | :--: | :--: | :-- | -| 2022-08-02 | 2.1.2 | Write .fil files as well as .h5 files (issue #272). | +| 2022-11-16 | 2.1.4 | Make bl_scrunch actually work (issue #276). | +| 2022-08-02 | 2.1.3 | Write .fil files as well as .h5 files (issue #272). | | 2022-07-22 | 2.1.2 | More container fields needed (issue #270). | | 2022-07-21 | 2.1.1 | New Waterfall class option, an alternative to file loading (issue #264). | | 2022-07-08 | 2.1.0 | New utility: dsamp (issue #267). | From 20b38f4ff0b6554a15329c57d4b29f2f73160bbd Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 16 Nov 2022 16:04:03 -0600 Subject: [PATCH 3/9] Fix issue #276 --- blimpy/bl_scrunch.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/blimpy/bl_scrunch.py b/blimpy/bl_scrunch.py index 67d01bc..551bc4b 100755 --- a/blimpy/bl_scrunch.py +++ b/blimpy/bl_scrunch.py @@ -49,9 +49,9 @@ def cmd_tool(args=None): r""" Command line utility for scrunching an input HDF5 file or Filterbank file. """ - p = ArgumentParser(description='Command line utility for converting HDF5 (.h5) to Sigproc filterbank (.fil) format \n >>h52fil [options]') - p.add_argument('filepath', type=str, help='Name of file path to open (.h5 or .fil).') - p.add_argument('-f', '--fscrunch', dest='f_scrunch', type=int, + p = ArgumentParser(description='Command line utility for scrunching an HDF5 file (.h5) or a Sigproc Filterbank file (.fil) to an output HDF5 file.') + p.add_argument('filepath', type=str, help='Input file path to open (.h5 or .fil).') + p.add_argument('-f', '--fscrunch', dest='f_scrunch', type=int, required=True, help='Number of frequency channels to average (scrunch) together.') p.add_argument('-o', '--out_dir', dest='out_dir', type=str, default='./', help='Location for output files. Default: current directory.') @@ -78,3 +78,4 @@ def cmd_tool(args=None): if __name__ == "__main__": cmd_tool() + From 11641272bbb446e83a80ca5a99fc98b9cacba0dd Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 16 Nov 2022 16:04:45 -0600 Subject: [PATCH 4/9] Fix issue #276 --- blimpy/io/hdf_writer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/blimpy/io/hdf_writer.py b/blimpy/io/hdf_writer.py index c5c130d..0291056 100644 --- a/blimpy/io/hdf_writer.py +++ b/blimpy/io/hdf_writer.py @@ -78,6 +78,7 @@ def __write_to_hdf5_heavy(wf, filename_out, f_scrunch=None, *args, **kwargs): dout_shape[-1] //= f_scrunch dout_chunk_dim[-1] //= f_scrunch wf.header['foff'] *= f_scrunch + wf.header['nchans'] //= f_scrunch dset = h5.create_dataset('data', shape=tuple(dout_shape), @@ -220,6 +221,7 @@ def __write_to_hdf5_light(wf, filename_out, f_scrunch=None, *args, **kwargs): wf.logger.info('Frequency scrunching by %i' % f_scrunch) data_out = utils.rebin(wf.data, n_z=f_scrunch) wf.header['foff'] *= f_scrunch + wf.header['nchans'] //= f_scrunch dset = h5.create_dataset('data', data=data_out, From f5831ddf5608fa871b3ab3d2ae1440441e7faeed Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 16 Nov 2022 16:49:12 -0600 Subject: [PATCH 5/9] Obsolete: match_fils.py --- blimpy/match_fils.py | 159 ------------------------------------------- 1 file changed, 159 deletions(-) delete mode 100755 blimpy/match_fils.py diff --git a/blimpy/match_fils.py b/blimpy/match_fils.py deleted file mode 100755 index 4f492f0..0000000 --- a/blimpy/match_fils.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python - -from optparse import OptionParser -import socket -import subprocess -import sys -import os -try: - from .waterfall import Waterfall -except: - from blimpy import Waterfall - -#import pdb #pdb.set_trace() - -local_host = socket.gethostname() - -def reset_outs(): - """ Returns None, None, used as a helper """ - - return None,None - -def make_batch_script(): - """ This creates a batch script for the heavy lifting""" - - script_text = '#! /bin/bash\n# This script calculates the md5sum for the last number of bits (nbits).# Usage :\n# ./tail_sum file bnits\n#\n\nfile_name=$1\nbit_num=$2\n\ntail -c $bit_num $file_name | md5sum\n' - - with open('tail_sum.sh', 'w') as batch_script: - batch_script.write(script_text) - - os.chmod('tail_sum.sh', 0o775) - -def find_header_size(filename): - """Script to find the header size of a filterbank file - - Args: - filename (str): Name of file - - Returns size of header in Bytes (int) - """ - - # open datafile - filfile=open(filename,'rb') - # go to the start of the file - filfile.seek(0) - #read some region larger than the header. - round1 = filfile.read(1000) - headersize = round1.find(b'HEADER_END')+len(b'HEADER_END') - - return headersize - -def cmd_tool(args=None): - """ Command line tool to make a MD5sum comparison of two .fil files. - - Supplies a command line: - matchfils - which will compare the two files to confirm the files have matching MD5sums - - Notes: - This command is specific to BL @ Green Bank - """ - - if 'bl' in local_host: - header_loc = '/usr/local/sigproc/bin/header' #Current location of header command in GBT. - else: - raise IOError('Script only able to run in BL systems.') - - p = OptionParser() - p.set_usage('matchfils ') - opts, args = p.parse_args(sys.argv[1:]) - - file1 = args[0] - file2 = args[1] - - #------------------------------------ - #Create batch script - - make_batch_script() - - #------------------------------------ - #First checksum - - headersize1 = find_header_size(file1) - file_size1 = os.path.getsize(file1) - - #Strip header from file, and calculate the md5sum of the rest. - #command=['tail','-c',str(file_size1-headersize1),file1,'|','md5sum'] - command=['./tail_sum.sh',file1,str(file_size1-headersize1)] - print('[matchfils] '+' '.join(command)) - - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (out, err) = proc.communicate() - - check_sum1 = out.split()[0] - print('[matchfils] Checksum is:', check_sum1) - - if err: - raise Error('There is an error.') - - #--- - out,err = reset_outs() - - command=[header_loc,file1] - print('[matchfils] Header information:') - - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (out, err) = proc.communicate() - - header1 = out - print(header1) - - #------------------------------------ - #Second checksum - - out,err = reset_outs() - - headersize2 = find_header_size(file2) - file_size2 = os.path.getsize(file2) - - #Strip header from file, and calculate the md5sum of the rest. - command=['./tail_sum.sh',file2,str(file_size2-headersize2)] - print('[matchfils] '+' '.join(command)) - - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (out, err) = proc.communicate() - - check_sum2 = out.split()[0] - print('[matchfils] Checksum is:', check_sum2) - - if err: - raise Error('There is an error.') - - #--- - out,err = reset_outs() - - command=[header_loc,file2] - print('[matchfils] Header information:') - - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (out, err) = proc.communicate() - - header2 = out - print(header2) - - #------------------------------------ - #check the checksums - - if check_sum1 != check_sum2: - print('[matchfils] Booo! Checksum does not match between files.') - else: - print('[matchfils] Hooray! Checksum matches between files.') - - #------------------------------------ - #Remove batch script - - os.remove('tail_sum.sh') - -if __name__ == "__main__": - cmd_tool() - From 0cf94b9538b598585cfd43328660cfc59af85783 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 16 Nov 2022 16:49:35 -0600 Subject: [PATCH 6/9] Obsolete: match_fils.py --- blimpy/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/blimpy/__init__.py b/blimpy/__init__.py index ef7a4e9..f3de3cb 100644 --- a/blimpy/__init__.py +++ b/blimpy/__init__.py @@ -13,7 +13,6 @@ from . import rawhdr from . import stax from . import stix - from . import match_fils from . import dsamp from blimpy.io import file_wrapper except: From 28d31601feedc090a26638ff2c0182ab2e6b454d Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 16 Nov 2022 16:49:56 -0600 Subject: [PATCH 7/9] Obsolete: match_fils.py --- tests/test_match_fils.py | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 tests/test_match_fils.py diff --git a/tests/test_match_fils.py b/tests/test_match_fils.py deleted file mode 100644 index 82cf086..0000000 --- a/tests/test_match_fils.py +++ /dev/null @@ -1,28 +0,0 @@ -import pytest -import blimpy as bl -from tests.data import voyager_fil - -def test_reset_works(): - """ - The test is trivial, but coverage must be - increased at all costs. - """ - a, b = bl.match_fils.reset_outs() - assert a is None - assert b is None - -def test_batcher(): - """ - This test may be obsolete once/if test_cmd - can be patched up. - """ - bl.match_fils.make_batch_script() - -def test_find_header_size(): - assert bl.match_fils.find_header_size(voyager_fil) > 0 - -def test_cmdline(): - args = [voyager_fil, voyager_fil] - with pytest.raises(OSError): - bl.match_fils.cmd_tool(args) - From c0bb42fde8598e1b107292c5006ddf9d95191966 Mon Sep 17 00:00:00 2001 From: texadactyl Date: Wed, 16 Nov 2022 15:02:00 -0800 Subject: [PATCH 8/9] Don't create obsolete matchfils utility --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index a2fe14d..a1e2f66 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,6 @@ 'fil2h5 = blimpy.fil2h5:cmd_tool', 'h52fil = blimpy.h52fil:cmd_tool', 'h5diag = blimpy.h5diag:cmd_tool', - 'matchfils = blimpy.match_fils:cmd_tool', 'peek = blimpy.peek:cmd_tool', 'rawhdr = blimpy.rawhdr:cmd_tool', 'rawutil = blimpy.guppi:cmd_tool', From 3b3f197eeb6c35cafb96d33739dc9e56a386a2e8 Mon Sep 17 00:00:00 2001 From: Richard Elkins Date: Wed, 16 Nov 2022 17:19:08 -0600 Subject: [PATCH 9/9] Validate the -f parameter --- blimpy/bl_scrunch.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/blimpy/bl_scrunch.py b/blimpy/bl_scrunch.py index 551bc4b..f36b32b 100755 --- a/blimpy/bl_scrunch.py +++ b/blimpy/bl_scrunch.py @@ -3,7 +3,7 @@ producing a new HDF5 file of Filterbank file. """ -import os +import os, sys from argparse import ArgumentParser from blimpy.waterfall import Waterfall from .utils import change_the_ext @@ -40,6 +40,10 @@ def bl_scrunch(in_path, out_dir='./', new_filename='', max_load=None, f_scrunch= else: out_path = out_dir + new_filename + if f_scrunch < 2 or f_scrunch >= wf.header["nchans"] : + print("\n*** Number of frequency channels to average together must be > 1 and < the input file header nchans value!!\n") + sys.exit(1) + print("bl_scrunch: Output path: {}".format(out_path)) wf.write_to_hdf5(out_path, f_scrunch=f_scrunch) print("bl_scrunch: End") @@ -66,7 +70,7 @@ def cmd_tool(args=None): args = p.parse_args() else: args = p.parse_args(args) - + bl_scrunch(args.filepath, out_dir=args.out_dir, new_filename=args.new_filename, max_load=args.max_load, f_scrunch=args.f_scrunch)