Add (optional at compile-time) G3Frame JSON output

.github/workflows/cmake.yml

@@ -22,7 +22,7 @@ jobs:
       run: |
         if [ `echo ${{matrix.os}} | cut -d - -f 1` = "ubuntu" ]; then
           [ `echo ${{matrix.os}} | cut -d . -f 1` = "ubuntu-20" ] && sudo apt-get install python-is-python3
-          sudo apt-get install libboost-all-dev libflac-dev libnetcdf-dev libfftw3-dev libgsl0-dev python python3 python3-pip
+          sudo apt-get install libboost-all-dev libflac-dev libnetcdf-dev libfftw3-dev libgsl0-dev python python3 python3-pip python3-setuptools
         elif [ `echo ${{matrix.os}} | cut -d - -f 1` = "macOS" ]; then
           brew install [email protected]
           brew link --overwrite [email protected]

CMakeLists.txt

@@ -83,6 +83,15 @@ execute_process(COMMAND mkdir -p ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 execute_process(COMMAND mkdir -p ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
 execute_process(COMMAND ln -fsn ${CMAKE_SOURCE_DIR}/cmake/init.py ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/__init__.py)
 
+# build options
+set(ENABLE_JSON_OUTPUT "FALSE" CACHE BOOL "Enable JSON output of frames (adds to compile time and binary size)")
+
+if (${ENABLE_JSON_OUTPUT})
+	# Note, newer cmake suggests using add_compile_definitions, but I think that requires 3.12
+	add_definitions(-DSPT3G_ENABLE_JSON_OUTPUT)
+endif()
+
+
 set(BUILD_PROJECTS "${BUILD_PROJECTS}" CACHE STRING "The subset of available projects to actually build")
 if(NOT "${BUILD_PROJECTS}" STREQUAL "")
 	set(USE_PROJECT_LIST "TRUE" CACHE BOOL "Whether to build only the subset of projects sepcified by BUILD_PROJECTS")
@@ -268,4 +277,4 @@ configure_file(
 	"${CMAKE_SOURCE_DIR}/cmake/cmake_uninstall.cmake.in"
 	"${CMAKE_BINARY_DIR}/cmake/cmake_uninstall_cmake"
 	IMMEDIATE @ONLY)
-add_custom_target(uninstall "${CMAKE_COMMAND}" -P "${CMAKE_BINARY_DIR}/cmake/cmake_uninstall_cmake")
+add_custom_target(uninstall "${CMAKE_COMMAND}" -P "${CMAKE_BINARY_DIR}/cmake/cmake_uninstall_cmake")

core/CMakeLists.txt

@@ -38,6 +38,7 @@ endif()
 link_python_dir()
 
 add_spt3g_program(bin/spt3g-dump)
+add_spt3g_program(bin/spt3g-jsonify)
 add_spt3g_program(bin/spt3g-verify)
 add_spt3g_program(bin/spt3g-inspect)
 add_spt3g_program(bin/gen-analysis-doc)

core/bin/spt3g-jsonify

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+import sys
+from spt3g import core
+
+
+if len(sys.argv) < 2:
+    print('Usage: %s file.g3 ' % sys.argv[0])
+    print('Dumps the given file to standard output as JSON (not quite indented properly, sorry)')
+    sys.exit(1)
+
+#Crappy hand-crafted json wrapper to make this stream valid json
+print('{') 
+print('    "filename": "%s",' % (sys.argv[1]))
+print('    "frames":')
+print('     [') 
+
+first = True
+indent = 8*' '
+print_indent = 7*' '
+
+for f in core.G3File(sys.argv[1]):
+	if not first: 
+		print (print_indent,',')
+	print(print_indent,indent.join(f.as_json().splitlines(True)))
+	first = False 
+print('    ]')
+print('}') 
+

core/include/core/G3Frame.h

@@ -96,6 +96,11 @@ class G3Frame {
 	template <typename T> void save(T &os) const;
 	template <typename T> void load(T &is);
 
+  // JSON representation of a frame (if JSON output is enabled, otherwise
+  // outputs valid JSON telling you that JSON output isn't enabled)
+  template <typename T> void saveJSON(T &os) const;
+  std::string asJSON() const;
+
 	// Routines for handling the stored serialized copies of data.
 	// These are all const because they only manipulate caches and
 	// so change only performance rather than data contents of the frame.

core/include/core/serialization.h

@@ -7,6 +7,9 @@
 #include <boost/iostreams/filtering_stream.hpp>
 
 #include <cereal/archives/portable_binary.hpp>
+#ifdef SPT3G_ENABLE_JSON_OUTPUT
+#include <cereal/archives/json.hpp>
+#endif
 
 #include <cereal/types/polymorphic.hpp>
 #include <cereal/types/string.hpp>
@@ -17,13 +20,31 @@
 #include <pybindings.h>
 #include <G3Logging.h>
 
+#ifdef SPT3G_ENABLE_JSON_OUTPUT
+
 #define G3_SERIALIZABLE_CODE(x) \
 template void x::serialize(cereal::PortableBinaryOutputArchive &, unsigned); \
+template void x::serialize(cereal::JSONOutputArchive &, unsigned); \
+template void x::serialize(cereal::JSONInputArchive &, unsigned); \
 template void x::serialize(cereal::PortableBinaryInputArchive &, unsigned); \
 
 #define G3_SPLIT_SERIALIZABLE_CODE(x) \
 template void x::save(cereal::PortableBinaryOutputArchive &, unsigned) const; \
+template void x::save(cereal::JSONOutputArchive &, unsigned) const; \
 template void x::load(cereal::PortableBinaryInputArchive &, unsigned); \
+template void x::load(cereal::JSONInputArchive &, unsigned); \
+
+#else
+
+#define G3_SERIALIZABLE_CODE(x) \
+template void x::serialize(cereal::PortableBinaryOutputArchive &, unsigned); \
+template void x::serialize(cereal::PortableBinaryInputArchive &, unsigned); \
+
+#define G3_SPLIT_SERIALIZABLE_CODE(x) \
+template void x::save(cereal::PortableBinaryOutputArchive &, unsigned) const; \
+template void x::load(cereal::PortableBinaryInputArchive &, unsigned); \
+
+#endif
 
 template <class T>
 struct g3frameobject_picklesuite : boost::python::pickle_suite

core/src/G3Frame.cxx

@@ -272,6 +272,37 @@ void G3Frame::save(T &os) const
 	ar << make_nvp("crc", crc);
 }
 
+template<typename T>
+void G3Frame::saveJSON(T & os) const
+{
+#ifdef SPT3G_ENABLE_JSON_OUTPUT
+  using cereal::make_nvp;
+  uint32_t version(1), size(map_.size());
+
+  cereal::JSONOutputArchive ar(os);
+  ar << make_nvp("version", version);
+	ar << make_nvp("size", size);
+	std::string typestr(1,(char) type);
+	ar << make_nvp("type", typestr);
+	for (auto i = map_.begin(); i != map_.end(); i++) {
+    //make sure it's deserialized so we don't just write a blob
+    blob_decode(i->second);
+		ar << make_nvp("name", i->first);
+		ar << make_nvp("val", i->second.frameobject);
+	}
+#else
+  os << " {error: \"spt3g-software compiled without JSON support\"}";
+#endif
+}
+
+std::string G3Frame::asJSON() const
+{
+  std::stringstream str;
+  saveJSON(str);
+  return str.str();
+}
+
+
 template <typename T>
 void G3Frame::load(T &is)
 {
@@ -402,3 +433,7 @@ template void G3Frame::save(boost::iostreams::filtering_ostream &) const;
 template void G3Frame::save(std::ostream &) const;
 template void G3Frame::save(std::ostringstream &) const;
 
+template void G3Frame::saveJSON(std::ostream &) const;
+template void G3Frame::saveJSON(std::ostringstream &) const;
+template void G3Frame::saveJSON(boost::iostreams::filtering_ostream &) const;
+

core/src/G3Timestream.cxx

@@ -36,6 +36,29 @@ static FLAC__StreamEncoderWriteStatus flac_encoder_write_cb(
 	return FLAC__STREAM_ENCODER_WRITE_STATUS_OK;
 }
 
+
+// This is only needed because cereal will generate code for loading json due
+// to the nature of serialize, even though this will never actually be called.
+//
+// I don't know if it works or not, but since it's never called, who cares?  (I
+// guess in the future we can have a fromJSON method in G3Frame as well if we
+// wanted to support round-tripping through JSON for some reason). 
+template <typename A>
+static void loadBinaryFn(A * inbuf, void * buffer, size_t size)
+{
+	inbuf->template loadBinary<1>(buffer,size);
+}
+
+#ifdef SPT3G_ENABLE_JSON_OUTPUT
+template <>
+void loadBinaryFn<cereal::JSONInputArchive>(cereal::JSONInputArchive * inbuf, void * buffer, size_t size)
+{
+	inbuf->loadBinaryValue(buffer,size);
+}
+#endif
+
+
+
 template<typename A>
 static FLAC__StreamDecoderReadStatus flac_decoder_read_cb(
     const FLAC__StreamDecoder *decoder, FLAC__byte buffer[], size_t *bytes,
@@ -51,11 +74,11 @@ static FLAC__StreamDecoderReadStatus flac_decoder_read_cb(
 		return FLAC__STREAM_DECODER_READ_STATUS_END_OF_STREAM;
 	} else if (*bytes >= size_t(bytes_left)) {
 		*bytes = bytes_left;
-		args->inbuf->template loadBinary<1>(buffer, bytes_left);
+ 		loadBinaryFn(args->inbuf, buffer, bytes_left);
 		args->pos += bytes_left;
 		return FLAC__STREAM_DECODER_READ_STATUS_CONTINUE;
 	} else {
-		args->inbuf->template loadBinary<1>(buffer, *bytes);
+		loadBinaryFn(args->inbuf, buffer, *bytes);
 		args->pos += *bytes;
 		return FLAC__STREAM_DECODER_READ_STATUS_CONTINUE;
 	}
@@ -105,7 +128,8 @@ template <class A> void G3Timestream::save(A &ar, unsigned v) const
 	ar & cereal::make_nvp("flac", use_flac_);
 
 #ifdef G3_HAS_FLAC
-	if (use_flac_) {
+	//Don't try to use FLAC when outputing to JSON, even if flac is enabled.
+	if (use_flac_ && typeid(A) == typeid(cereal::PortableBinaryOutputArchive)) {
 		std::vector<int32_t> inbuf;
 		std::vector<uint8_t> outbuf;
 		const int32_t *chanmap[1];
@@ -874,7 +898,7 @@ PYBINDINGS("core") {
 	    .def("SetFLACCompression", &G3Timestream::SetFLACCompression,
 	      "Pass True to turn on FLAC compression when serialized. "
 	      "FLAC compression only works if the timestream is in units of "
-	      "counts.")
+	      "counts. FLAC compression is ignored when outputing to JSON.")
 	    .def_readwrite("units", &G3Timestream::units,
 	      "Units of the data in the timestream, stored as one of the "
 	      "members of core.G3TimestreamUnits.")

core/src/python.cxx

@@ -470,6 +470,7 @@ BOOST_PYTHON_MODULE(core)
 	    .def("generate_blobs", &G3Frame::GenerateBlobs, bp::arg("drop_objects")=false, "Force immediate serialization of all objects. Will save some CPU time later during serialization of the frame in exchange for spending the exact same amount of CPU time right now.")
 	    .def("drop_objects", &G3Frame::DropObjects, "Drop all decoded objects in favor of their serialized copies, where those serialized copies already exist. Saves memory for frames about to be written at the expense of CPU time to re-decode them if they are accessed again later.")
 	    .def_pickle(g3frame_picklesuite())
+      .def("as_json", &G3Frame::asJSON, "JSON representation of frame")
 	;
 	register_vector_of<G3FramePtr>("Frame");
 	register_vector_of<G3FrameObjectPtr>("FrameObject");