Track source filename for G3Frame objects that are read in using G3Reader

core/include/core/G3Frame.h

@@ -60,6 +60,9 @@ class G3Frame {
 
 	FrameType type;
 
+	// Source filename (if output from G3Reader)
+	std::string _filename;
+
 	// Add and remove data from the frame. Note that retrieved data is
 	// const. Get<> is like operator [], except that it does a dynamic
 	// cast to a type for you.

core/include/core/G3Reader.h

@@ -10,9 +10,9 @@
 class G3Reader : public G3Module {
 public:
 	G3Reader(std::string filename, int n_frames_to_read = -1,
-                 float timeout = -1.);
+                 float timeout = -1., bool track_filename = false);
 	G3Reader(std::vector<std::string> filenames, int n_frames_to_read = -1,
-                 float timeout = -1.);
+                 float timeout = -1., bool track_filename = false);
 
 	void Process(G3FramePtr frame, std::deque<G3FramePtr> &out);
 	off_t Seek(off_t offset);
@@ -27,6 +27,7 @@ class G3Reader : public G3Module {
 	int n_frames_to_read_;
 	int n_frames_read_;
 	float timeout_;
+	bool track_filename_;
 
 	SET_LOGGER("G3Reader");
 };

core/src/G3Reader.cxx

@@ -5,9 +5,9 @@
 #include <boost/filesystem.hpp>
 
 G3Reader::G3Reader(std::string filename, int n_frames_to_read,
-    float timeout) :
+    float timeout, bool track_filename) :
     prefix_file_(false), n_frames_to_read_(n_frames_to_read),
-    n_frames_read_(0), timeout_(timeout)
+    n_frames_read_(0), timeout_(timeout), track_filename_(track_filename)
 {
 	boost::filesystem::path fpath(filename);
 	if (filename.find("://") == std::string::npos &&
@@ -18,9 +18,9 @@ G3Reader::G3Reader(std::string filename, int n_frames_to_read,
 }
 
 G3Reader::G3Reader(std::vector<std::string> filename, int n_frames_to_read,
-    float timeout) :
+    float timeout, bool track_filename) :
     prefix_file_(false), n_frames_to_read_(n_frames_to_read),
-    n_frames_read_(0), timeout_(timeout)
+    n_frames_read_(0), timeout_(timeout), track_filename_(track_filename)
 {
 	if (filename.size() == 0)
 		log_fatal("Empty file list provided to G3Reader");
@@ -104,7 +104,10 @@ void G3Reader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 	if (_save != nullptr)
 		PyEval_RestoreThread(_save);
 
+	if (track_filename_)
+		frame->_filename = cur_file_;
 	out.push_back(frame);
+
 	n_frames_read_++;
 }
 
@@ -130,11 +133,13 @@ PYBINDINGS("core") {
 	      "streams, resulting in EOF behavior on expiry; unfortunately this "
 	      "cannot be used for polling, you have to close the connection. "
 	      "Use the `tell` and `seek` methods to record the position of and "
-	      "seek to the beginning of a particular frame in the file.",
-	init<std::string, int, float>((arg("filename"),
-	    arg("n_frames_to_read")=0,arg("timeout")=-1.)))
-	.def(init<std::vector<std::string>, int, float>((arg("filename"),
-	    arg("n_frames_to_read")=0, arg("timeout")=-1.)))
+	      "seek to the beginning of a particular frame in the file.  Set "
+	      "track_filename to True to record the filename for each frame in "
+	      "the ._filename attribute (fragile).",
+	init<std::string, int, float, bool>((arg("filename"),
+	    arg("n_frames_to_read")=0,arg("timeout")=-1.,arg("track_filename")=false)))
+	.def(init<std::vector<std::string>, int, float, bool>((arg("filename"),
+	    arg("n_frames_to_read")=0, arg("timeout")=-1.,arg("track_filename")=false)))
 	.def("tell", &G3Reader::Tell)
 	.def("seek", &G3Reader::Seek)
 	.def_readonly("__g3module__", true)

core/src/python.cxx

@@ -459,6 +459,8 @@ BOOST_PYTHON_MODULE(core)
 	    .def("__init__", bp::make_constructor(g3frame_char_constructor, bp::default_call_policies(), bp::args("adhoctypecode")), "Create a frame with an ad-hoc (non-standard) type code. Use sparingly and with care.")
 	    .def_readwrite("type", &G3Frame::type, "Type code for frame. "
 	      "See general G3Frame docstring.")
+	    .def_readonly("_filename", &G3Frame::_filename, "Source filename for frame, "
+	      "if read in using G3Reader. This attribute is fragile, use at your own risk.")
 	    .def("__setitem__", &g3frame_python_put)
 	    .def("__getitem__", &g3frame_python_get)
 	    .def("keys", &g3frame_keys, "Returns a list of keys in the frame.")

core/tests/fileio.py

@@ -33,7 +33,7 @@ def addinfo(fr):
 # And back from disk
 print('Reading')
 pipe = core.G3Pipeline()
-pipe.Add(core.G3Reader, filename='test.g3')
+pipe.Add(core.G3Reader, filename='test.g3', track_filename=True)
 pipe.Add(core.Dump)
 n = 0
 def checkinfo(fr):
@@ -42,6 +42,7 @@ def checkinfo(fr):
         return
     assert 'time' in fr, 'No time key in frame'
     assert fr['count'] == n, 'Out of order frame'
+    assert fr._filename == 'test.g3', 'Wrong filename'
     n += 1
 pipe.Add(checkinfo)
 pipe.Run()

doc/frames.rst

@@ -8,6 +8,8 @@ Frames have fast serialization to and from disk (see the G3Reader and G3Writer m
 
 Each frame has a type defined under the ``core.G3FrameType`` namespace. These are meant to indicate different types of data (calibration vs. scan data, for example) and, in general, many types of frames will be interleaved in the same data stream. A good general rule for whether data should be in a different frame type is to consider the rates at which the data change: data that change at the same speed (e.g. bolometer data and pointing) should share a frame, while data that change at different speeds (e.g. calibration constants and bolometer data) should be separated.
 
+Each frame also has a ``._filename`` attribute, which is set when frames are read in using the ``core.G3Reader`` pipeline module with the ``track_filename`` option set to ``True``. This attribute is not serialized with the frame, so will be an empty string when used with pickling or multiprocessing.  This attribute is fragile and should not be relied upon for production code.
+
 A brief description of the intention of each frame type follows along with a table containing a representative minimal set of data likely to be contained by a frame of a given type. The list is neither exhaustive nor truly minimal: other data can and will be present and some of the data listed here may have been removed or renamed. Neither is this an exhaustive list of frame types: any single character code can be [ab]used as a frame type for special purpose tools.
 
 .. contents:: Contents

gcp/src/ARCFileReader.cxx

@@ -82,9 +82,9 @@ enum {
 class ARCFileReader : public G3Module {
 public:
 	ARCFileReader(const std::string &path,
-	    Experiment experiment=Experiment::SPT);
+	    Experiment experiment=Experiment::SPT, bool track_filename=false);
 	ARCFileReader(const std::vector<std::string> & filename,
-	    Experiment experiment=Experiment::SPT);
+	    Experiment experiment=Experiment::SPT, bool track_filename=false);
 	virtual ~ARCFileReader() {}
 
 	void Process(G3FramePtr frame, std::deque<G3FramePtr> &out);
@@ -124,12 +124,15 @@ class ARCFileReader : public G3Module {
 	Experiment experiment;
 	G3TimePtr GCPToTime(uint8_t *buffer, off_t offset);
 
+	bool track_filename_;
+
 	SET_LOGGER("ARCFileReader");
 };
 
 
 ARCFileReader::ARCFileReader(const std::string &path,
-    Experiment experiment) : experiment(experiment)
+    Experiment experiment, bool track_filename) :
+    experiment(experiment), track_filename_(track_filename)
 {
 	if (experiment == Experiment::SPT || experiment == Experiment::BK) {
 		ms_jiffie_base_ = G3Units::ms;
@@ -149,7 +152,8 @@ ARCFileReader::ARCFileReader(const std::string &path,
 
 
 ARCFileReader::ARCFileReader(const std::vector<std::string> &filename,
-    Experiment experiment) : experiment(experiment)
+    Experiment experiment, bool track_filename) :
+    experiment(experiment), track_filename_(track_filename)
 {
 	if (experiment == Experiment::SPT || experiment == Experiment::BK) {
 		ms_jiffie_base_ = G3Units::ms;
@@ -859,6 +863,8 @@ void ARCFileReader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 		outframe->Put(temp->first, templ);
 	}
 
+	if (track_filename_)
+		outframe->_filename = cur_file_;
 	out.push_back(outframe);
 
 	delete [] buffer;
@@ -870,13 +876,14 @@ PYBINDINGS("gcp") {
 	// Instead of EXPORT_G3MODULE since there are two constructors
 	class_<ARCFileReader, bases<G3Module>, boost::shared_ptr<ARCFileReader>,
 	    boost::noncopyable>("ARCFileReader",
-	    "Read GCP archive file (or files if you pass an iterable of paths)."
-	    " For non-SPT ARC file formats, please set Experiment to the "
-	    " appropriate value.",
-		init<std::string, optional<Experiment> >(
-		        args("filename", "experiment")))
-		.def(init<std::vector<std::string>, optional<Experiment> >(
-		        args("filename", "experiment")))
+	    "Read GCP archive file (or files if you pass an iterable of paths). "
+	    "For non-SPT ARC file formats, please set Experiment to the "
+	    "appropriate value.  Set track_filename to True to record the "
+	    "filename for each frame in the ._filename attribute (fragile).",
+		init<std::string, Experiment, bool>((arg("filename"),
+			arg("experiment")=Experiment::SPT, arg("track_filename")=false)))
+		.def(init<std::vector<std::string>, Experiment, bool>((arg("filename"),
+			arg("experiment")=Experiment::SPT, arg("track_filename")=false)))
 		.def_readonly("__g3module__", true)
 	;
 }