store correct repr for G3FrameObjects, improve tests and docs

core/include/core/G3PipelineInfo.h

@@ -2,27 +2,34 @@
 #define _G3_PIPELINEINFO_H
 
 #include <G3Frame.h>
-#include <G3Map.h>
 #include <vector>
+#include <map>
 
-class G3ModulePythonArg : public G3FrameObject {
+class G3ModuleArg : public G3FrameObject {
 public:
-	std::string value;
-
-	G3ModulePythonArg(const std::string &v) : value(v) {}
-	G3ModulePythonArg() {}
+	G3ModuleArg(const std::string &repr) : repr_(repr) {}
+	G3ModuleArg(const std::string &repr, G3FrameObjectPtr obj) :
+	    repr_(repr), obj_(obj) {}
+	G3ModuleArg() {}
 
 	template <class A> void serialize(A &ar, unsigned v);
-	std::string Summary() const { return value; };
-	bool operator ==(const G3ModulePythonArg &other) const { return other.value == value; };
+	std::string repr() const { return repr_; };
+	G3FrameObjectPtr object() const { return obj_; };
+
+	std::string Summary() const { return repr_; };
+	bool operator ==(const G3ModuleArg &other) const { return other.repr_ == repr_; };
+
+private:
+	std::string repr_;
+	G3FrameObjectPtr obj_;
 };
 
 class G3ModuleConfig : public G3FrameObject {
 public:
 	std::string modname;
 	std::string instancename;
 
-	G3MapFrameObject config;
+	std::map<std::string, G3ModuleArg> config;
 
         template <class A> void load(A &ar, unsigned v);
         template <class A> void save(A &ar, unsigned v) const;
@@ -60,15 +67,15 @@ class G3PipelineInfo : public G3FrameObject {
 	SET_LOGGER("G3PipelineInfo");
 };
 
-G3_POINTERS(G3ModulePythonArg);
+G3_POINTERS(G3ModuleArg);
 G3_POINTERS(G3ModuleConfig);
 G3_POINTERS(G3PipelineInfo);
 
 namespace cereal {
         template <class A> struct specialize<A, G3ModuleConfig, cereal::specialization::member_load_save> {};
 }
 
-G3_SERIALIZABLE(G3ModulePythonArg, 1);
+G3_SERIALIZABLE(G3ModuleArg, 1);
 G3_SERIALIZABLE(G3ModuleConfig, 2);
 G3_SERIALIZABLE(G3PipelineInfo, 2);
 

core/src/G3PipelineInfo.cxx

@@ -2,13 +2,18 @@
 #include <serialization.h>
 #include <G3PipelineInfo.h>
 
-template <class A> void G3ModulePythonArg::serialize(A &ar, unsigned v)
+#include <cereal/types/map.hpp>
+
+namespace bp = boost::python;
+
+template <class A> void G3ModuleArg::serialize(A &ar, unsigned v)
 {
 	G3_CHECK_VERSION(v);
 
 	ar & cereal::make_nvp("G3FrameObject",
 	    cereal::base_class<G3FrameObject>(this));
-	ar & cereal::make_nvp("value", value);
+	ar & cereal::make_nvp("repr", repr_);
+	ar & cereal::make_nvp("obj", obj_);
 }
 
 template <class A> void G3ModuleConfig::save(A &ar, unsigned v) const
@@ -48,11 +53,14 @@ template <class A> void G3ModuleConfig::load(A &ar, unsigned v)
 		if (is_frameobject) {
 			G3FrameObjectPtr fo;
 			ar >> cereal::make_nvp("value", fo);
-			config[key] = fo;
+			// Some frame objects (e.g. G3Vectors) have repr only
+			// defined properly via the python interface.
+			std::string repr = bp::extract<std::string>(bp::object(fo).attr("__repr__")());
+			config[key] = G3ModuleArg(repr, fo);
 		} else {
 			std::string repr;
 			ar >> cereal::make_nvp("value", repr);
-			config[key] = boost::make_shared<G3ModulePythonArg>(repr);
+			config[key] = G3ModuleArg(repr);
 		}
 	}
 }
@@ -62,7 +70,7 @@ G3ModuleConfig::Summary() const
 {
 	std::string rv = "pipe.Add(" + modname;
 	for (auto i : config) {
-		rv += ", " + i.first + "=" + i.second->Summary();
+		rv += ", " + i.first + "=" + i.second.Summary();
 	}
 
 	if (instancename.size() != 0 && instancename != modname)
@@ -148,70 +156,82 @@ G3PipelineInfo_repr(const G3PipelineInfo &pi)
 	return rv;
 }
 
-namespace bp = boost::python;
+static void
+G3PipelineInfo_run(const G3PipelineInfo &pi)
+{
+	bp::object main = bp::import("__main__");
+	bp::dict global = bp::dict(main.attr("__dict__"));
+	global["__main__"] = main;
+
+	std::string pipe = G3PipelineInfo_repr(pi);
+	pipe += "pipe.Run()\n";
+
+	bp::exec(bp::str(pipe), global, global);
+}
 
 static bp::object
-G3ModuleConfig_get(G3ModuleConfigConstPtr mc, std::string key)
+G3ModuleConfig_get(const G3ModuleConfig &mc, std::string key)
 {
-	auto item = mc->config.find(key);
-	if (item == mc->config.end()) {
+	auto item = mc.config.find(key);
+	if (item == mc.config.end()) {
 		PyErr_SetString(PyExc_KeyError, key.c_str());
 		bp::throw_error_already_set();
 	}
 
-	G3ModulePythonArgConstPtr arg =
-	    boost::dynamic_pointer_cast<const G3ModulePythonArg>(item->second);
-	if (!arg)
-		return bp::object(item->second);
+	auto arg = item->second;
+	if (arg.object())
+		return bp::object(arg.object());
 
 	bp::object main = bp::import("__main__");
-	bp::object global = main.attr("__dict__");
+	bp::dict global = bp::dict(main.attr("__dict__"));
+	global["__main__"] = main;
 
 	try {
-		return bp::eval(bp::str(arg->value), global, global);
+		return bp::eval(bp::str(arg.repr()), global, global);
 	} catch (const bp::error_already_set& e) {
 		PyErr_Clear();
-		return bp::object(arg->value);
+		return bp::object(arg.repr());
 	}
 }
 
 static void
-G3ModuleConfig_set(G3ModuleConfigPtr mc, std::string key, bp::object obj)
+G3ModuleConfig_set(G3ModuleConfig &mc, std::string key, bp::object obj)
 {
-	if (bp::extract<G3FrameObjectPtr>(obj).check()) {
-		mc->config[key] = bp::extract<G3FrameObjectPtr>(obj)();
+	std::string repr = bp::extract<std::string>(obj.attr("__repr__")());
+
+	if (!bp::extract<G3FrameObjectPtr>(obj).check()) {
+		mc.config[key] = G3ModuleArg(repr);
 		return;
 	}
 
-	std::string repr = bp::extract<std::string>(obj.attr("__repr__")());
-	mc->config[key] = boost::make_shared<G3ModulePythonArg>(repr);
+	mc.config[key] = G3ModuleArg(repr, bp::extract<G3FrameObjectPtr>(obj)());
 }
 
 static bp::list
-G3ModuleConfig_keys(G3ModuleConfigConstPtr mc)
+G3ModuleConfig_keys(const G3ModuleConfig &mc)
 {
 	bp::list keys;
 
-	for (auto i: mc->config)
+	for (auto i: mc.config)
 		keys.append(i.first);
 
 	return keys;
 }
 
 static bp::list
-G3ModuleConfig_values(G3ModuleConfigConstPtr mc)
+G3ModuleConfig_values(const G3ModuleConfig &mc)
 {
 	bp::list values;
 
-	for (auto i: mc->config)
+	for (auto i: mc.config)
 		values.append(G3ModuleConfig_get(mc, i.first));
 
 	return values;
 }
 
 
 
-G3_SERIALIZABLE_CODE(G3ModulePythonArg);
+G3_SERIALIZABLE_CODE(G3ModuleArg);
 G3_SPLIT_SERIALIZABLE_CODE(G3ModuleConfig);
 G3_SERIALIZABLE_CODE(G3PipelineInfo);
 
@@ -240,6 +260,7 @@ PYBINDINGS("core") {
 	    .def_readwrite("user", &G3PipelineInfo::user)
 	    .def_readwrite("modules", &G3PipelineInfo::modules)
 	    .def("__repr__", &G3PipelineInfo_repr)
+	    .def("Run", &G3PipelineInfo_run)
 	;
 	register_pointer_conversions<G3PipelineInfo>();
 }

core/tests/pipelineinfo.py

@@ -4,9 +4,14 @@
 from spt3g import core
 
 args = {"n": 10, "type": core.G3FrameType.Timepoint}
+arg = core.G3VectorDouble([1, 2, 3])
 
 p = core.G3Pipeline()
 p.Add(core.G3InfiniteSource, **args)
+def twiddle(fr, arg=None):
+	if fr.type == core.G3FrameType.Timepoint:
+		fr['arg'] = arg
+p.Add(twiddle, arg=arg)
 p.Add(core.Dump)
 p.Add(core.G3Writer, filename='testpi.g3')
 p.Run()
@@ -20,13 +25,16 @@
 
 os.remove('testpi.g3')
 print(repr(pi))
-exec(repr(pi))
-pipe.Run()
+pi.Run()
 
 # Check that module arguments survive round-trip to/from storage
 mod_args = dict(pi.modules[0])
 print(mod_args)
 assert(mod_args == args)
+obj_arg = pi.modules[1]['arg']
+print(obj_arg)
+assert(isinstance(obj_arg, type(arg)))
+assert((obj_arg == arg).all())
 
 assert(len(list(core.G3File('testpi.g3'))) == 11)
 

doc/modules.rst

@@ -341,9 +341,11 @@ This information is added immediately following the first added module or segmen
 
 Within some limits imposed by Python (related to lambda functions, most notably), calling ``repr()`` on a G3PipelineInfo object (or a G3Pipeline object) will yield an executable Python script reflecting the exact modules and configuration used to produce the data. To within the mentioned limitations, this script can be rerun to exactly reproduce stored data; it can also be inspected to learn the configuration of the data's source pipeline[s] and thus the processing that produced it.
 
+The G3PipelineInfo ``.Run()`` method provides a convenient way of rerunning the pipeline configuration within it, and the ``.modules`` attribute is a list of G3ModuleConfig objects with dict-like access to the arguments provided to each pipeline module.
+
 Limitations:
 
 - The content of functions defined inline in a script (either by ``def`` or ``lambda``), as opposed to functions defined in an imported Python module, will not appear in the output, though options will. Inline functions defined by ``def`` will at least give the name of the function.
 - Options passed to pre-instantiated modules will not be stored. Only options passed in ``pipe.Add()`` will be recorded. For example, ``pipe.Add(core.G3Reader, filename="test.g3")`` will fully record its arguments, but ``pipe.Add(core.G3Reader(filename="test.g3"))`` will not. Prefer the syntax that records options unless you have a compelling reason to do something else.
 - A G3Pipeline created in C++ will not record configuration; only G3Pipelines created in Python will.
-
+- If the code used to run the pipeline initially has changed, rerunning the configuration from the G3PipelineInfo object will produce different results.  Use the version control information stored in the object to ensure you are running the correct version of the software.