From 684baa8fa9f1500f6ee4da11d157977d50f3707f Mon Sep 17 00:00:00 2001 From: Sasha Rahlin Date: Tue, 23 Jan 2024 21:59:57 -0600 Subject: [PATCH 1/2] Skip empty files in G3Reader Avoid throwing an IO error when an empty G3 file is included in the input filenames by silently moving on to the next file in the list until reaching a non-empty file or the end of the list. Fixes #134. --- core/src/G3Reader.cxx | 2 +- core/tests/fileio.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/core/src/G3Reader.cxx b/core/src/G3Reader.cxx index db0fceec..718c0d08 100644 --- a/core/src/G3Reader.cxx +++ b/core/src/G3Reader.cxx @@ -80,7 +80,7 @@ void G3Reader::Process(G3FramePtr frame, std::deque &out) if (Py_IsInitialized()) _save = PyEval_SaveThread(); - if (stream_.peek() == EOF) { + while (stream_.peek() == EOF) { if (filename_.size() > 0) { StartFile(filename_.front()); filename_.pop_front(); diff --git a/core/tests/fileio.py b/core/tests/fileio.py index 0b36de3f..c8b073dc 100755 --- a/core/tests/fileio.py +++ b/core/tests/fileio.py @@ -49,6 +49,17 @@ def checkinfo(fr): assert n == 10, 'Wrong number of frames read (%d should be %d)' % (n, 10) +# Skip empty files +wr = core.G3Writer("empty.g3") +del wr +n = 0 +pipe = core.G3Pipeline() +pipe.Add(core.G3Reader, filename=["empty.g3", "test.g3", "empty.g3"], track_filename=True) +pipe.Add(checkinfo) +pipe.Run() + +assert n == 10, 'Wrong number of frames read (%d should be %d)' % (n, 10) + # Indexing class CachingReader: def __init__(self, filename='test.g3'): From d7bf61620e0473829996b41dba29235e00b93190 Mon Sep 17 00:00:00 2001 From: Sasha Rahlin Date: Wed, 24 Jan 2024 11:42:51 -0600 Subject: [PATCH 2/2] log an error message for empty files --- core/include/core/G3Reader.h | 1 + core/src/G3Reader.cxx | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/core/include/core/G3Reader.h b/core/include/core/G3Reader.h index 57288829..df554be6 100644 --- a/core/include/core/G3Reader.h +++ b/core/include/core/G3Reader.h @@ -26,6 +26,7 @@ class G3Reader : public G3Module { boost::iostreams::filtering_istream stream_; int n_frames_to_read_; int n_frames_read_; + int n_frames_cur_; float timeout_; bool track_filename_; diff --git a/core/src/G3Reader.cxx b/core/src/G3Reader.cxx index 718c0d08..77f81754 100644 --- a/core/src/G3Reader.cxx +++ b/core/src/G3Reader.cxx @@ -7,7 +7,8 @@ G3Reader::G3Reader(std::string filename, int n_frames_to_read, float timeout, bool track_filename) : prefix_file_(false), n_frames_to_read_(n_frames_to_read), - n_frames_read_(0), timeout_(timeout), track_filename_(track_filename) + n_frames_read_(0), n_frames_cur_(0), timeout_(timeout), + track_filename_(track_filename) { boost::filesystem::path fpath(filename); if (filename.find("://") == std::string::npos && @@ -20,7 +21,8 @@ G3Reader::G3Reader(std::string filename, int n_frames_to_read, G3Reader::G3Reader(std::vector filename, int n_frames_to_read, float timeout, bool track_filename) : prefix_file_(false), n_frames_to_read_(n_frames_to_read), - n_frames_read_(0), timeout_(timeout), track_filename_(track_filename) + n_frames_read_(0), n_frames_cur_(0), timeout_(timeout), + track_filename_(track_filename) { if (filename.size() == 0) log_fatal("Empty file list provided to G3Reader"); @@ -41,6 +43,7 @@ void G3Reader::StartFile(std::string path) { log_info("Starting file %s\n", path.c_str()); cur_file_ = path; + n_frames_cur_ = 0; (void) g3_istream_from_path(stream_, path, timeout_); } @@ -81,6 +84,8 @@ void G3Reader::Process(G3FramePtr frame, std::deque &out) _save = PyEval_SaveThread(); while (stream_.peek() == EOF) { + if (n_frames_cur_ == 0) + log_error("Empty file %s", cur_file_.c_str()); if (filename_.size() > 0) { StartFile(filename_.front()); filename_.pop_front(); @@ -109,6 +114,7 @@ void G3Reader::Process(G3FramePtr frame, std::deque &out) out.push_back(frame); n_frames_read_++; + n_frames_cur_++; } off_t G3Reader::Seek(off_t offset) {