Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Skip empty files in G3Reader #137

Merged
merged 2 commits into from
Jan 24, 2024
Merged

Skip empty files in G3Reader #137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
684baa8
Skip empty files in G3Reader
arahlin Jan 24, 2024
d7bf616
log an error message for empty files
arahlin Jan 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions core/include/core/G3Reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ class G3Reader : public G3Module {
boost::iostreams::filtering_istream stream_;
int n_frames_to_read_;
int n_frames_read_;
int n_frames_cur_;
float timeout_;
bool track_filename_;

Expand Down
12 changes: 9 additions & 3 deletions core/src/G3Reader.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
G3Reader::G3Reader(std::string filename, int n_frames_to_read,
float timeout, bool track_filename) :
prefix_file_(false), n_frames_to_read_(n_frames_to_read),
n_frames_read_(0), timeout_(timeout), track_filename_(track_filename)
n_frames_read_(0), n_frames_cur_(0), timeout_(timeout),
track_filename_(track_filename)
{
boost::filesystem::path fpath(filename);
if (filename.find("://") == std::string::npos &&
Expand All @@ -20,7 +21,8 @@ G3Reader::G3Reader(std::string filename, int n_frames_to_read,
G3Reader::G3Reader(std::vector<std::string> filename, int n_frames_to_read,
float timeout, bool track_filename) :
prefix_file_(false), n_frames_to_read_(n_frames_to_read),
n_frames_read_(0), timeout_(timeout), track_filename_(track_filename)
n_frames_read_(0), n_frames_cur_(0), timeout_(timeout),
track_filename_(track_filename)
{
if (filename.size() == 0)
log_fatal("Empty file list provided to G3Reader");
Expand All @@ -41,6 +43,7 @@ void G3Reader::StartFile(std::string path)
{
log_info("Starting file %s\n", path.c_str());
cur_file_ = path;
n_frames_cur_ = 0;
(void) g3_istream_from_path(stream_, path, timeout_);
}

Expand Down Expand Up @@ -80,7 +83,9 @@ void G3Reader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
if (Py_IsInitialized())
_save = PyEval_SaveThread();

if (stream_.peek() == EOF) {
while (stream_.peek() == EOF) {
if (n_frames_cur_ == 0)
log_error("Empty file %s", cur_file_.c_str());
if (filename_.size() > 0) {
StartFile(filename_.front());
filename_.pop_front();
Expand Down Expand Up @@ -109,6 +114,7 @@ void G3Reader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
out.push_back(frame);

n_frames_read_++;
n_frames_cur_++;
}

off_t G3Reader::Seek(off_t offset) {
Expand Down
11 changes: 11 additions & 0 deletions core/tests/fileio.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,17 @@ def checkinfo(fr):

assert n == 10, 'Wrong number of frames read (%d should be %d)' % (n, 10)

# Skip empty files
wr = core.G3Writer("empty.g3")
del wr
n = 0
pipe = core.G3Pipeline()
pipe.Add(core.G3Reader, filename=["empty.g3", "test.g3", "empty.g3"], track_filename=True)
pipe.Add(checkinfo)
pipe.Run()

assert n == 10, 'Wrong number of frames read (%d should be %d)' % (n, 10)

# Indexing
class CachingReader:
def __init__(self, filename='test.g3'):
Expand Down
Loading