Switch from json11 to nlohmann/json (#31093)

* Initial commit

* fix release/file_common

* Update common/swaglog.cc

* Swtiched serialize cc

* move to lfs

---------

Co-authored-by: Adeeb Shihadeh <[email protected]>

.gitattributes

@@ -22,3 +22,4 @@ third_party/acados/*/t_renderer filter=lfs diff=lfs merge=lfs -text
 third_party/qt5/larch64/bin/lrelease filter=lfs diff=lfs merge=lfs -text
 third_party/qt5/larch64/bin/lupdate filter=lfs diff=lfs merge=lfs -text
 third_party/catch2/include/catch2/catch.hpp filter=lfs diff=lfs merge=lfs -text
+third_party/nlohmann/json.hpp filter=lfs diff=lfs merge=lfs -text

SConstruct

@@ -205,7 +205,7 @@ env = Environment(
     "#third_party/acados/include/hpipm/include",
     "#third_party/catch2/include",
     "#third_party/libyuv/include",
-    "#third_party/json11",
+    "#third_party/nlohmann",
     "#third_party/linux/include",
     "#third_party/snpe/include",
     "#third_party/mapbox-gl-native-qt/include",
@@ -351,7 +351,7 @@ Export('env', 'qt_env', 'arch', 'real_arch')
 SConscript(['common/SConscript'])
 Import('_common', '_gpucommon')
 
-common = [_common, 'json11']
+common = [_common]
 gpucommon = [_gpucommon]
 
 Export('common', 'gpucommon')

common/SConscript

@@ -12,7 +12,7 @@ common_libs = [
 if arch != "Darwin":
   common_libs.append('gpio.cc')
 
-_common = env.Library('common', common_libs, LIBS="json11")
+_common = env.Library('common', common_libs)
 
 files = [
   'clutil.cc',
@@ -24,10 +24,10 @@ Export('_common', '_gpucommon')
 if GetOption('extras'):
   env.Program('tests/test_common',
               ['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
-              LIBS=[_common, 'json11', 'zmq', 'pthread'])
+              LIBS=[_common, 'zmq', 'pthread'])
 
 # Cython bindings
-params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
+params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq'])
 
 SConscript([
   'transformations/SConscript',

common/swaglog.cc

@@ -11,7 +11,7 @@
 
 #include <zmq.h>
 #include <stdarg.h>
-#include "third_party/json11/json11.hpp"
+#include "nlohmann/json.hpp"
 #include "common/version.h"
 #include "system/hardware/hw.h"
 
@@ -37,7 +37,7 @@ class SwaglogState {
       }
     }
 
-    ctx_j = json11::Json::object{};
+    ctx_j = nlohmann::json();
     if (char* dongle_id = getenv("DONGLE_ID")) {
       ctx_j["dongle_id"] = dongle_id;
     }
@@ -66,17 +66,17 @@ class SwaglogState {
   void* zctx = nullptr;
   void* sock = nullptr;
   int print_level;
-  json11::Json::object ctx_j;
+  nlohmann::json ctx_j;
 };
 
 bool LOG_TIMESTAMPS = getenv("LOG_TIMESTAMPS");
 uint32_t NO_FRAME_ID = std::numeric_limits<uint32_t>::max();
 
 static void cloudlog_common(int levelnum, const char* filename, int lineno, const char* func,
-                            char* msg_buf, const json11::Json::object &msg_j={}) {
+                            char* msg_buf, const nlohmann::json &msg_j={}) {
   static SwaglogState s;
 
-  json11::Json::object log_j = json11::Json::object {
+  nlohmann::json log_j = nlohmann::json {
     {"ctx", s.ctx_j},
     {"levelnum", levelnum},
     {"filename", filename},
@@ -92,7 +92,7 @@ static void cloudlog_common(int levelnum, const char* filename, int lineno, cons
 
   std::string log_s;
   log_s += (char)levelnum;
-  ((json11::Json)log_j).dump(log_s);
+  log_s += log_j.dump();
   s.log(levelnum, filename, lineno, func, msg_buf, log_s);
 
   free(msg_buf);
@@ -115,14 +115,14 @@ void cloudlog_t_common(int levelnum, const char* filename, int lineno, const cha
   char* msg_buf = nullptr;
   int ret = vasprintf(&msg_buf, fmt, args);
   if (ret <= 0 || !msg_buf) return;
-  json11::Json::object tspt_j = json11::Json::object{
+  nlohmann::json tspt_j = nlohmann::json{
     {"event", msg_buf},
     {"time", std::to_string(nanos_since_boot())}
   };
   if (frame_id < NO_FRAME_ID) {
     tspt_j["frame_id"] = std::to_string(frame_id);
   }
-  tspt_j = json11::Json::object{{"timestamp", tspt_j}};
+  tspt_j = nlohmann::json{{"timestamp", tspt_j}};
   cloudlog_common(levelnum, filename, lineno, func, msg_buf, tspt_j);
 }
 

common/tests/test_swaglog.cc

@@ -7,7 +7,7 @@
 #include "common/util.h"
 #include "common/version.h"
 #include "system/hardware/hw.h"
-#include "third_party/json11/json11.hpp"
+#include "nlohmann/json.hpp"
 
 std::string daemon_name = "testy";
 std::string dongle_id = "test_dongle_id";
@@ -39,26 +39,25 @@ void recv_log(int thread_cnt, int thread_msg_cnt) {
 
     REQUIRE(buf[0] == CLOUDLOG_DEBUG);
     std::string err;
-    auto msg = json11::Json::parse(buf + 1, err);
-    REQUIRE(!msg.is_null());
+    nlohmann::json msg = nlohmann::json::parse(buf + 1);
 
-    REQUIRE(msg["levelnum"].int_value() == CLOUDLOG_DEBUG);
-    REQUIRE_THAT(msg["filename"].string_value(), Catch::Contains("test_swaglog.cc"));
-    REQUIRE(msg["funcname"].string_value() == "log_thread");
-    REQUIRE(msg["lineno"].int_value() == LINE_NO);
+    REQUIRE(msg["levelnum"].template get<int>() == CLOUDLOG_DEBUG);
+    REQUIRE_THAT(msg["filename"].template get<std::string>(), Catch::Contains("test_swaglog.cc"));
+    REQUIRE(msg["funcname"].template get<std::string>() == "log_thread");
+    REQUIRE(msg["lineno"].template get<int>() == LINE_NO);
 
     auto ctx = msg["ctx"];
 
-    REQUIRE(ctx["daemon"].string_value() == daemon_name);
-    REQUIRE(ctx["dongle_id"].string_value() == dongle_id);
-    REQUIRE(ctx["dirty"].bool_value() == true);
+    REQUIRE(ctx["daemon"].template get<std::string>() == daemon_name);
+    REQUIRE(ctx["dongle_id"].template get<std::string>() == dongle_id);
+    REQUIRE(ctx["dirty"].template get<bool>() == true);
 
-    REQUIRE(ctx["version"].string_value() == COMMA_VERSION);
+    REQUIRE(ctx["version"].template get<std::string>() == COMMA_VERSION);
 
     std::string device = Hardware::get_name();
-    REQUIRE(ctx["device"].string_value() == device);
+    REQUIRE(ctx["device"].template get<std::string>() == device);
 
-    int thread_id = atoi(msg["msg"].string_value().c_str());
+    int thread_id = atoi(msg["msg"].template get<std::string>().c_str());
     REQUIRE((thread_id >= 0 && thread_id < thread_cnt));
     thread_msgs[thread_id]++;
     total_count++;

release/files_common

@@ -374,8 +374,7 @@ third_party/SConscript
 third_party/linux/**
 third_party/opencl/**
 
-third_party/json11/json11.cpp
-third_party/json11/json11.hpp
+third_party/nlohmann/json.hpp
 
 third_party/qrcode/*.cc
 third_party/qrcode/*.hpp

selfdrive/modeld/thneed/serialize.cc

@@ -1,11 +1,11 @@
 #include <cassert>
 #include <set>
 
-#include "third_party/json11/json11.hpp"
+#include "nlohmann/json.hpp"
 #include "common/util.h"
 #include "common/clutil.h"
 #include "selfdrive/modeld/thneed/thneed.h"
-using namespace json11;
+using namespace nlohmann;
 
 extern map<cl_program, string> g_program_source;
 
@@ -14,25 +14,24 @@ void Thneed::load(const char *filename) {
 
   string buf = util::read_file(filename);
   int jsz = *(int *)buf.data();
-  string jsonerr;
   string jj(buf.data() + sizeof(int), jsz);
-  Json jdat = Json::parse(jj, jsonerr);
+  json jdat = json::parse(jj);
 
   map<cl_mem, cl_mem> real_mem;
   real_mem[NULL] = NULL;
 
   int ptr = sizeof(int)+jsz;
-  for (auto &obj : jdat["objects"].array_items()) {
-    auto mobj = obj.object_items();
-    int sz = mobj["size"].int_value();
+  for (auto &obj : jdat["objects"]) {
+    auto mobj = obj;
+    int sz = mobj["size"].template get<int>();
     cl_mem clbuf = NULL;
 
-    if (mobj["buffer_id"].string_value().size() > 0) {
+    if (mobj["buffer_id"].template get<std::string>().size() > 0) {
       // image buffer must already be allocated
-      clbuf = real_mem[*(cl_mem*)(mobj["buffer_id"].string_value().data())];
-      assert(mobj["needs_load"].bool_value() == false);
+      clbuf = real_mem[*(cl_mem*)(mobj["buffer_id"].template get<std::string>().data())];
+      assert(mobj["needs_load"].template get<bool>() == false);
     } else {
-      if (mobj["needs_load"].bool_value()) {
+      if (mobj["needs_load"].template get<bool>()) {
         clbuf = clCreateBuffer(context, CL_MEM_COPY_HOST_PTR | CL_MEM_READ_WRITE, sz, &buf[ptr], NULL);
         if (debug >= 1) printf("loading %p %d @ 0x%X\n", clbuf, sz, ptr);
         ptr += sz;
@@ -48,9 +47,9 @@ void Thneed::load(const char *filename) {
     if (mobj["arg_type"] == "image2d_t" || mobj["arg_type"] == "image1d_t") {
       cl_image_desc desc = {0};
       desc.image_type = (mobj["arg_type"] == "image2d_t") ? CL_MEM_OBJECT_IMAGE2D : CL_MEM_OBJECT_IMAGE1D_BUFFER;
-      desc.image_width = mobj["width"].int_value();
-      desc.image_height = mobj["height"].int_value();
-      desc.image_row_pitch = mobj["row_pitch"].int_value();
+      desc.image_width = mobj["width"].template get<int>();
+      desc.image_height = mobj["height"].template get<int>();
+      desc.image_row_pitch = mobj["row_pitch"].template get<int>();
       assert(sz == desc.image_height*desc.image_row_pitch);
 #ifdef QCOM2
       desc.buffer = clbuf;
@@ -60,12 +59,12 @@ void Thneed::load(const char *filename) {
 #endif
       cl_image_format format = {0};
       format.image_channel_order = CL_RGBA;
-      format.image_channel_data_type = mobj["float32"].bool_value() ? CL_FLOAT : CL_HALF_FLOAT;
+      format.image_channel_data_type = mobj["float32"].template get<bool>() ? CL_FLOAT : CL_HALF_FLOAT;
 
       cl_int errcode;
 
 #ifndef QCOM2
-      if (mobj["needs_load"].bool_value()) {
+      if (mobj["needs_load"].template get<bool>()) {
         clbuf = clCreateImage(context, CL_MEM_COPY_HOST_PTR | CL_MEM_READ_WRITE, &format, &desc, &buf[ptr-sz], &errcode);
       } else {
         clbuf = clCreateImage(context, CL_MEM_READ_WRITE, &format, &desc, NULL, &errcode);
@@ -80,22 +79,22 @@ void Thneed::load(const char *filename) {
       assert(clbuf != NULL);
     }
 
-    real_mem[*(cl_mem*)(mobj["id"].string_value().data())] = clbuf;
+    real_mem[*(cl_mem*)(mobj["id"].template get<std::string>().data())] = clbuf;
   }
 
   map<string, cl_program> g_programs;
-  for (const auto &[name, source] : jdat["programs"].object_items()) {
-    if (debug >= 1) printf("building %s with size %zu\n", name.c_str(), source.string_value().size());
-    g_programs[name] = cl_program_from_source(context, device_id, source.string_value());
+  for (const auto &[name, source] : jdat["programs"].items()) {
+    if (debug >= 1) printf("building %s with size %zu\n", name.c_str(), source.template get<std::string>().size());
+    g_programs[name] = cl_program_from_source(context, device_id, source.template get<std::string>());
   }
 
-  for (auto &obj : jdat["inputs"].array_items()) {
-    auto mobj = obj.object_items();
-    int sz = mobj["size"].int_value();
-    cl_mem aa = real_mem[*(cl_mem*)(mobj["buffer_id"].string_value().data())];
+  for (auto &obj : jdat["inputs"]) {
+    auto mobj = obj;
+    int sz = mobj["size"].template get<int>();
+    cl_mem aa = real_mem[*(cl_mem*)(mobj["buffer_id"].template get<std::string>().data())];
     input_clmem.push_back(aa);
     input_sizes.push_back(sz);
-    printf("Thneed::load: adding input %s with size %d\n", mobj["name"].string_value().data(), sz);
+    printf("Thneed::load: adding input %s with size %d\n", mobj["name"].template get<std::string>().data(), sz);
 
     cl_int cl_err;
     void *ret = clEnqueueMapBuffer(command_queue, aa, CL_TRUE, CL_MAP_WRITE, 0, sz, 0, NULL, NULL, &cl_err);
@@ -104,39 +103,39 @@ void Thneed::load(const char *filename) {
     inputs.push_back(ret);
   }
 
-  for (auto &obj : jdat["outputs"].array_items()) {
-    auto mobj = obj.object_items();
-    int sz = mobj["size"].int_value();
+  for (auto &obj : jdat["outputs"]) {
+    auto mobj = obj;
+    int sz = mobj["size"].template get<int>();
     printf("Thneed::save: adding output with size %d\n", sz);
     // TODO: support multiple outputs
-    output = real_mem[*(cl_mem*)(mobj["buffer_id"].string_value().data())];
+    output = real_mem[*(cl_mem*)(mobj["buffer_id"].template get<std::string>().data())];
     assert(output != NULL);
   }
 
-  for (auto &obj : jdat["binaries"].array_items()) {
-    string name = obj["name"].string_value();
-    size_t length = obj["length"].int_value();
+  for (auto &obj : jdat["binaries"]) {
+    string name = obj["name"].template get<std::string>();
+    size_t length = obj["length"].template get<int>();
     if (debug >= 1) printf("binary %s with size %zu\n", name.c_str(), length);
     g_programs[name] = cl_program_from_binary(context, device_id, (const uint8_t*)&buf[ptr], length);
     ptr += length;
   }
 
-  for (auto &obj : jdat["kernels"].array_items()) {
+  for (auto &obj : jdat["kernels"]) {
     auto gws = obj["global_work_size"];
     auto lws = obj["local_work_size"];
     auto kk = shared_ptr<CLQueuedKernel>(new CLQueuedKernel(this));
 
-    kk->name = obj["name"].string_value();
+    kk->name = obj["name"].template get<std::string>();
     kk->program = g_programs[kk->name];
-    kk->work_dim = obj["work_dim"].int_value();
+    kk->work_dim = obj["work_dim"].template get<int>();
     for (int i = 0; i < kk->work_dim; i++) {
-      kk->global_work_size[i] = gws[i].int_value();
-      kk->local_work_size[i] = lws[i].int_value();
+      kk->global_work_size[i] = gws[i].template get<int>();
+      kk->local_work_size[i] = lws[i].template get<int>();
     }
-    kk->num_args = obj["num_args"].int_value();
+    kk->num_args = obj["num_args"].template get<int>();
     for (int i = 0; i < kk->num_args; i++) {
-      string arg = obj["args"].array_items()[i].string_value();
-      int arg_size = obj["args_size"].array_items()[i].int_value();
+      string arg = obj["args"][i].template get<std::string>();
+      int arg_size = obj["args_size"][i].template get<int>();
       kk->args_size.push_back(arg_size);
       if (arg_size == 8) {
         cl_mem val = *(cl_mem*)(arg.data());

selfdrive/modeld/thneed/thneed.h

@@ -18,9 +18,6 @@ using namespace std;
 
 cl_int thneed_clSetKernelArg(cl_kernel kernel, cl_uint arg_index, size_t arg_size, const void *arg_value);
 
-namespace json11 {
-  class Json;
-}
 class Thneed;
 
 class GPUMalloc {
@@ -52,7 +49,6 @@ class CLQueuedKernel {
     vector<string> args;
     vector<int> args_size;
     cl_kernel kernel = NULL;
-    json11::Json to_json() const;
 
     cl_uint work_dim;
     size_t global_work_size[3] = {0};

selfdrive/navd/SConscript

@@ -2,7 +2,7 @@ Import('qt_env', 'arch', 'common', 'messaging', 'visionipc', 'cereal', 'transfor
 
 map_env = qt_env.Clone()
 libs = ['qt_widgets', 'qt_util', 'qmapboxgl', common, messaging, cereal, visionipc, transformations,
-        'zmq', 'capnp', 'kj', 'm', 'OpenCL', 'ssl', 'crypto', 'pthread', 'json11'] + map_env["LIBS"]
+        'zmq', 'capnp', 'kj', 'm', 'OpenCL', 'ssl', 'crypto', 'pthread'] + map_env["LIBS"]
 if arch == 'larch64':
   libs.append(':libEGL_mesa.so.0')
 

selfdrive/ui/SConscript

@@ -91,7 +91,7 @@ if GetOption('extras') and arch != "Darwin":
   # setup and factory resetter
   qt_env.Program("qt/setup/reset", ["qt/setup/reset.cc"], LIBS=qt_libs)
   qt_env.Program("qt/setup/setup", ["qt/setup/setup.cc", asset_obj],
-                 LIBS=qt_libs + ['curl', 'common', 'json11'])
+                 LIBS=qt_libs + ['curl', 'common'])
 
   # build updater UI
   qt_env.Program("qt/setup/updater", ["qt/setup/updater.cc", asset_obj], LIBS=qt_libs)
@@ -126,4 +126,4 @@ if GetOption('extras') and arch != "Darwin":
 
 # build watch3
 if arch in ['x86_64', 'aarch64', 'Darwin'] or GetOption('extras'):
-  qt_env.Program("watch3", ["watch3.cc"], LIBS=qt_libs + ['common', 'json11', 'zmq', 'visionipc', 'messaging'])
+  qt_env.Program("watch3", ["watch3.cc"], LIBS=qt_libs + ['common', 'zmq', 'visionipc', 'messaging'])

system/logcatd/SConscript

@@ -1,3 +1,3 @@
 Import('env', 'cereal', 'messaging', 'common')
 
-env.Program('logcatd', 'logcatd_systemd.cc', LIBS=[cereal, messaging, common, 'zmq', 'capnp', 'kj', 'systemd', 'json11'])
+env.Program('logcatd', 'logcatd_systemd.cc', LIBS=[cereal, messaging, common, 'zmq', 'capnp', 'kj', 'systemd'])

system/logcatd/logcatd_systemd.cc

@@ -5,7 +5,7 @@
 #include <map>
 #include <string>
 
-#include "third_party/json11/json11.hpp"
+#include "nlohmann/json.hpp"
 
 #include "cereal/messaging/messaging.h"
 #include "common/timing.h"
@@ -62,7 +62,7 @@ int main(int argc, char *argv[]) {
     // Build message
     auto androidEntry = msg.initEvent().initAndroidLog();
     androidEntry.setTs(timestamp);
-    androidEntry.setMessage(json11::Json(kv).dump());
+    androidEntry.setMessage(nlohmann::json(kv).dump());
     if (kv.count("_PID")) androidEntry.setPid(std::atoi(kv["_PID"].c_str()));
     if (kv.count("PRIORITY")) androidEntry.setPriority(std::atoi(kv["PRIORITY"].c_str()));
     if (kv.count("SYSLOG_IDENTIFIER")) androidEntry.setTag(kv["SYSLOG_IDENTIFIER"]);