Add (optional at compile-time) G3Frame JSON output

core/CMakeLists.txt

@@ -18,6 +18,15 @@ add_spt3g_library(core SHARED
 	src/int_storage.cxx src/pybindings.cxx
 )
 
+
+# json output option
+set(ENABLE_JSON_OUTPUT "FALSE" CACHE BOOL "Enable JSON output of frames (adds to compile time and binary size)")
+
+if (${ENABLE_JSON_OUTPUT})
+	target_compile_definitions(core PUBLIC -DSPT3G_ENABLE_JSON_OUTPUT)
+endif()
+
+
 # Link dependencies
 target_link_libraries(core PUBLIC spt3g)
 
@@ -52,6 +61,7 @@ endif()
 link_python_dir()
 
 add_spt3g_program(bin/spt3g-dump)
+add_spt3g_program(bin/spt3g-jsonify)
 add_spt3g_program(bin/spt3g-verify)
 add_spt3g_program(bin/spt3g-inspect)
 add_spt3g_program(bin/gen-analysis-doc)

core/bin/spt3g-jsonify

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+import sys
+from spt3g import core
+
+if len(sys.argv) < 2:
+    print('Usage: %s file.g3 ' % sys.argv[0])
+    print('Dumps the given file to standard output as JSON (not quite indented properly, sorry)')
+    sys.exit(1)
+
+#Crappy hand-crafted json wrapper to make this stream valid json
+print('{')
+print('    "filename": "%s",' % (sys.argv[1]))
+print('    "frames":')
+print('     [')
+
+first = True
+indent = 8*' '
+print_indent = 7*' '
+
+for f in core.G3File(sys.argv[1]):
+	if not first:
+		print (print_indent,',')
+	print(print_indent,indent.join(f.as_json().splitlines(True)))
+	first = False
+print('    ]')
+print('}')
+

core/include/core/G3Frame.h

@@ -99,6 +99,11 @@ class G3Frame {
 	template <typename T> void save(T &os) const;
 	template <typename T> void load(T &is);
 
+  // JSON representation of a frame (if JSON output is enabled, otherwise
+  // outputs valid JSON telling you that JSON output isn't enabled)
+  template <typename T> void saveJSON(T &os) const;
+  std::string asJSON() const;
+
 	// Routines for handling the stored serialized copies of data.
 	// These are all const because they only manipulate caches and
 	// so change only performance rather than data contents of the frame.

core/include/core/serialization.h

@@ -4,6 +4,9 @@
 #include <dataio.h>
 
 #include <cereal/archives/portable_binary.hpp>
+#ifdef SPT3G_ENABLE_JSON_OUTPUT
+#include <cereal/archives/json.hpp>
+#endif
 
 #include <cereal/types/polymorphic.hpp>
 #include <cereal/types/string.hpp>
@@ -14,13 +17,31 @@
 #include <pybindings.h>
 #include <G3Logging.h>
 
+#ifdef SPT3G_ENABLE_JSON_OUTPUT
+
 #define G3_SERIALIZABLE_CODE(x) \
 template void x::serialize(cereal::PortableBinaryOutputArchive &, unsigned); \
+template void x::serialize(cereal::JSONOutputArchive &, unsigned); \
+template void x::serialize(cereal::JSONInputArchive &, unsigned); \
 template void x::serialize(cereal::PortableBinaryInputArchive &, unsigned); \
 
 #define G3_SPLIT_SERIALIZABLE_CODE(x) \
 template void x::save(cereal::PortableBinaryOutputArchive &, unsigned) const; \
+template void x::save(cereal::JSONOutputArchive &, unsigned) const; \
 template void x::load(cereal::PortableBinaryInputArchive &, unsigned); \
+template void x::load(cereal::JSONInputArchive &, unsigned); \
+
+#else
+
+#define G3_SERIALIZABLE_CODE(x) \
+template void x::serialize(cereal::PortableBinaryOutputArchive &, unsigned); \
+template void x::serialize(cereal::PortableBinaryInputArchive &, unsigned); \
+
+#define G3_SPLIT_SERIALIZABLE_CODE(x) \
+template void x::save(cereal::PortableBinaryOutputArchive &, unsigned) const; \
+template void x::load(cereal::PortableBinaryInputArchive &, unsigned); \
+
+#endif
 
 template <class T>
 struct g3frameobject_picklesuite : boost::python::pickle_suite

core/src/G3Frame.cxx

@@ -277,6 +277,37 @@ void G3Frame::save(T &os) const
 	ar << make_nvp("crc", crc);
 }
 
+template<typename T>
+void G3Frame::saveJSON(T & os) const
+{
+#ifdef SPT3G_ENABLE_JSON_OUTPUT
+	using cereal::make_nvp;
+	uint32_t version(1), size(map_.size());
+
+	cereal::JSONOutputArchive ar(os);
+	ar << make_nvp("version", version);
+	ar << make_nvp("size", size);
+	std::string typestr(1,(char) type);
+	ar << make_nvp("type", typestr);
+	for (auto i = map_.begin(); i != map_.end(); i++) {
+		//make sure it's deserialized so we don't just write a blob
+		blob_decode(i->second);
+		ar << make_nvp("name", i->first);
+		ar << make_nvp("val", i->second.frameobject);
+	}
+#else
+	os << " {error: \"spt3g-software compiled without JSON support\"}" << std::endl;
+#endif
+}
+
+std::string
+G3Frame::asJSON() const
+{
+	std::stringstream str;
+	saveJSON(str);
+	return str.str();
+}
+
 template <typename T>
 void G3Frame::load(T &is)
 {
@@ -406,6 +437,10 @@ template void G3Frame::save(g3_ostream &) const;
 template void G3Frame::save(std::ostream &) const;
 template void G3Frame::save(std::ostringstream &) const;
 
+template void G3Frame::saveJSON(std::ostream &) const;
+template void G3Frame::saveJSON(std::ostringstream &) const;
+template void G3Frame::saveJSON(boost::iostreams::filtering_ostream &) const;
+
 G3FramePtr
 g3frame_char_constructor(std::string max_4_chars)
 {
@@ -630,6 +665,7 @@ PYBINDINGS("core") {
 	      "where those serialized copies already exist. Saves memory for "
 	      "frames about to be written at the expense of CPU time to "
 	      "re-decode them if they are accessed again later.")
+	    .def("as_json", &G3Frame::asJSON, "JSON representation of frame")
 	    .def_pickle(g3frame_picklesuite())
 	;
 	register_vector_of<G3FramePtr>("Frame");

core/src/G3Timestream.cxx

@@ -36,6 +36,34 @@ static FLAC__StreamEncoderWriteStatus flac_encoder_write_cb(
 	return FLAC__STREAM_ENCODER_WRITE_STATUS_OK;
 }
 
+
+// Generic version of loadBinary for all archive types
+template <typename A>
+static void load_binary_fn(A * inbuf, void * buffer, size_t size)
+{
+	inbuf->template loadBinary<1>(buffer,size);
+}
+
+#ifdef SPT3G_ENABLE_JSON_OUTPUT
+
+// SpecializedloadBinary  version for JSONInputArchive
+//
+// This is only needed because cereal will generate code for loading json due
+// to the nature of serialize, even though this will never actually be called.
+//
+// I don't know if it works or not, but since it's never called, who cares?  (I
+// guess in the future we can have a fromJSON method in G3Frame as well if we
+// wanted to support round-tripping through JSON for some reason).
+
+template <>
+void load_binary_fn<cereal::JSONInputArchive>(cereal::JSONInputArchive * inbuf, void * buffer, size_t size)
+{
+	inbuf->loadBinaryValue(buffer,size);
+}
+#endif
+
+
+
 template<typename A>
 static FLAC__StreamDecoderReadStatus flac_decoder_read_cb(
     const FLAC__StreamDecoder *decoder, FLAC__byte buffer[], size_t *bytes,
@@ -51,16 +79,16 @@ static FLAC__StreamDecoderReadStatus flac_decoder_read_cb(
 		return FLAC__STREAM_DECODER_READ_STATUS_END_OF_STREAM;
 	} else if (*bytes >= size_t(bytes_left)) {
 		*bytes = bytes_left;
-		args->inbuf->template loadBinary<1>(buffer, bytes_left);
+ 		load_binary_fn(args->inbuf, buffer, bytes_left);
 		args->pos += bytes_left;
 		return FLAC__STREAM_DECODER_READ_STATUS_CONTINUE;
 	} else {
-		args->inbuf->template loadBinary<1>(buffer, *bytes);
+		load_binary_fn(args->inbuf, buffer, *bytes);
 		args->pos += *bytes;
 		return FLAC__STREAM_DECODER_READ_STATUS_CONTINUE;
 	}
 }
-  
+
 template<typename A>
 static FLAC__StreamDecoderWriteStatus flac_decoder_write_cb(
     const FLAC__StreamDecoder *decoder, const FLAC__Frame *frame,
@@ -75,7 +103,7 @@ static FLAC__StreamDecoderWriteStatus flac_decoder_write_cb(
 		(*args->outbuf)[oldsize + i] = buffer[0][i];
 	return FLAC__STREAM_DECODER_WRITE_STATUS_CONTINUE;
 }
-  
+
 static void flac_decoder_error_cb(const FLAC__StreamDecoder *decoder,
     FLAC__StreamDecoderErrorStatus status, void *client_data)
 {
@@ -112,7 +140,8 @@ template <class A> void G3Timestream::save(A &ar, unsigned v) const
 	ar & cereal::make_nvp("flac", use_flac_);
 
 #ifdef G3_HAS_FLAC
-	if (use_flac_) {
+	//Don't try to use FLAC when outputing to JSON, even if flac is enabled.
+	if (use_flac_ && typeid(A) == typeid(cereal::PortableBinaryOutputArchive)) {
 		std::vector<int32_t> inbuf;
 		std::vector<uint8_t> outbuf;
 		const int32_t *chanmap[1];
@@ -1332,7 +1361,7 @@ PYBINDINGS("core") {
 	    .def("SetFLACCompression", &G3Timestream::SetFLACCompression,
 	      "Pass True to turn on FLAC compression when serialized. "
 	      "FLAC compression only works if the timestream is in units of "
-	      "counts.")
+	      "counts. FLAC compression is ignored when outputing to JSON.")
 	    .def_readwrite("units", &G3Timestream::units,
 	      "Units of the data in the timestream, stored as one of the "
 	      "members of core.G3TimestreamUnits.")

serve/CMakeLists.txt

@@ -0,0 +1,22 @@
+
+function (make_embedded_resource var_name file_name out_file)
+  if (${file_name} IS_NEWER_THAN ${out_file})
+    message(Generating ${out_file} " from " ${file_name})
+    file (WRITE ${out_file} "// automatically generated from ${file_name}\n")
+    file (READ  ${file_name} filedata HEX)
+    string (REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," cfiledata ${filedata})
+    file (APPEND ${out_file} "unsigned char ${var_name}[] = { ${cfiledata} };\n")
+  endif()
+endfunction()
+
+make_embedded_resource(favicon favicon.ico ${CMAKE_BINARY_DIR}/serve/favicon.h)
+
+if(NOT DEFINED ENV{CIBUILDWHEEL})
+  add_spt3g_executable(spt3g-json-serve-files src/spt3g-json-serve-files.cxx)
+
+  target_link_libraries(spt3g-json-serve-files core z)
+  target_include_directories(spt3g-json-serve-files PRIVATE
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+    $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}>
+  )
+endif()

serve/README.rst

@@ -0,0 +1,57 @@
+=====
+serve
+=====
+
+Example web server(s) for .g3 files
+
+The spt3g-json-serve-files program will serves .g3 files within a directory hierarchy in json format, providing
+a crude way to expose a files to a web browser.
+
+
+Running the file server
+=======================
+
+Command line usage::
+
+  Usage: spt3g-json-serve-files [OPTION]
+  	-h,--help		display this message
+  	-P,--Python		initialize python interpreter
+  	-p,--port		port to use (default 2726)
+  	-b,--bind-address	bind address (default 0.0.0.0)
+  	-d,--doc-root		base dir to serve (default is .)
+  	-t,--threads		number of threads to use (default 4)
+  	-v,--verbose		verbose mode
+
+The example server doesn't support authentication and has not rigorously been
+tested for security (though it does try to avoid the most obvious directory
+traversal attacks). If it is to be exposed to the internet, you probably should
+probably at least reverse proxy it behind a basic authentication screendoor.
+
+
+Implemented endpoints
+=====================
+
+*/[?json]
+---------
+
+List a directory within basedir. By default, this uses a clunky HTML browser,
+appending the ``?json`` parameter, the directory listing will be returned
+in a json format more suited to be programtically used by a less clunky browser.
+
+
+*.g3[.gz|.bz2][?N=0&nskip=0]
+----------------------------
+
+List a .g3 file as JSON. Optional parameters ``N`` and ``nskip`` control the
+number of frames (0 means all) and the number of frames from the begining to
+skip, respectively.  Note that this just affects what gets transferred over the
+wire, the server must still deserialize the file up to nskip + N frames each
+time (in the future, caching may be implemented, though the OS file cache probably helps a little bit).
+
+/version
+--------
+List a version number (in JSON format)
+
+/favicon.ico
+------------
+What else, but a favicon?