Skip to content

Commit

Permalink
Revert "Switch from json11 to nlohmann/json (#31093)"
Browse files Browse the repository at this point in the history
This reverts commit ee03648.
  • Loading branch information
adeebshihadeh committed Jan 23, 2024
1 parent ee03648 commit 88757c1
Show file tree
Hide file tree
Showing 17 changed files with 1,098 additions and 76 deletions.
1 change: 0 additions & 1 deletion .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,3 @@ third_party/acados/*/t_renderer filter=lfs diff=lfs merge=lfs -text
third_party/qt5/larch64/bin/lrelease filter=lfs diff=lfs merge=lfs -text
third_party/qt5/larch64/bin/lupdate filter=lfs diff=lfs merge=lfs -text
third_party/catch2/include/catch2/catch.hpp filter=lfs diff=lfs merge=lfs -text
third_party/nlohmann/json.hpp filter=lfs diff=lfs merge=lfs -text
4 changes: 2 additions & 2 deletions SConstruct
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ env = Environment(
"#third_party/acados/include/hpipm/include",
"#third_party/catch2/include",
"#third_party/libyuv/include",
"#third_party/nlohmann",
"#third_party/json11",
"#third_party/linux/include",
"#third_party/snpe/include",
"#third_party/mapbox-gl-native-qt/include",
Expand Down Expand Up @@ -351,7 +351,7 @@ Export('env', 'qt_env', 'arch', 'real_arch')
SConscript(['common/SConscript'])
Import('_common', '_gpucommon')

common = [_common]
common = [_common, 'json11']
gpucommon = [_gpucommon]

Export('common', 'gpucommon')
Expand Down
6 changes: 3 additions & 3 deletions common/SConscript
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ common_libs = [
if arch != "Darwin":
common_libs.append('gpio.cc')

_common = env.Library('common', common_libs)
_common = env.Library('common', common_libs, LIBS="json11")

files = [
'clutil.cc',
Expand All @@ -24,10 +24,10 @@ Export('_common', '_gpucommon')
if GetOption('extras'):
env.Program('tests/test_common',
['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
LIBS=[_common, 'zmq', 'pthread'])
LIBS=[_common, 'json11', 'zmq', 'pthread'])

# Cython bindings
params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq'])
params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])

SConscript([
'transformations/SConscript',
Expand Down
16 changes: 8 additions & 8 deletions common/swaglog.cc
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

#include <zmq.h>
#include <stdarg.h>
#include "nlohmann/json.hpp"
#include "third_party/json11/json11.hpp"
#include "common/version.h"
#include "system/hardware/hw.h"

Expand All @@ -37,7 +37,7 @@ class SwaglogState {
}
}

ctx_j = nlohmann::json();
ctx_j = json11::Json::object{};
if (char* dongle_id = getenv("DONGLE_ID")) {
ctx_j["dongle_id"] = dongle_id;
}
Expand Down Expand Up @@ -66,17 +66,17 @@ class SwaglogState {
void* zctx = nullptr;
void* sock = nullptr;
int print_level;
nlohmann::json ctx_j;
json11::Json::object ctx_j;
};

bool LOG_TIMESTAMPS = getenv("LOG_TIMESTAMPS");
uint32_t NO_FRAME_ID = std::numeric_limits<uint32_t>::max();

static void cloudlog_common(int levelnum, const char* filename, int lineno, const char* func,
char* msg_buf, const nlohmann::json &msg_j={}) {
char* msg_buf, const json11::Json::object &msg_j={}) {
static SwaglogState s;

nlohmann::json log_j = nlohmann::json {
json11::Json::object log_j = json11::Json::object {
{"ctx", s.ctx_j},
{"levelnum", levelnum},
{"filename", filename},
Expand All @@ -92,7 +92,7 @@ static void cloudlog_common(int levelnum, const char* filename, int lineno, cons

std::string log_s;
log_s += (char)levelnum;
log_s += log_j.dump();
((json11::Json)log_j).dump(log_s);
s.log(levelnum, filename, lineno, func, msg_buf, log_s);

free(msg_buf);
Expand All @@ -115,14 +115,14 @@ void cloudlog_t_common(int levelnum, const char* filename, int lineno, const cha
char* msg_buf = nullptr;
int ret = vasprintf(&msg_buf, fmt, args);
if (ret <= 0 || !msg_buf) return;
nlohmann::json tspt_j = nlohmann::json{
json11::Json::object tspt_j = json11::Json::object{
{"event", msg_buf},
{"time", std::to_string(nanos_since_boot())}
};
if (frame_id < NO_FRAME_ID) {
tspt_j["frame_id"] = std::to_string(frame_id);
}
tspt_j = nlohmann::json{{"timestamp", tspt_j}};
tspt_j = json11::Json::object{{"timestamp", tspt_j}};
cloudlog_common(levelnum, filename, lineno, func, msg_buf, tspt_j);
}

Expand Down
25 changes: 13 additions & 12 deletions common/tests/test_swaglog.cc
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
#include "common/util.h"
#include "common/version.h"
#include "system/hardware/hw.h"
#include "nlohmann/json.hpp"
#include "third_party/json11/json11.hpp"

std::string daemon_name = "testy";
std::string dongle_id = "test_dongle_id";
Expand Down Expand Up @@ -39,25 +39,26 @@ void recv_log(int thread_cnt, int thread_msg_cnt) {

REQUIRE(buf[0] == CLOUDLOG_DEBUG);
std::string err;
nlohmann::json msg = nlohmann::json::parse(buf + 1);
auto msg = json11::Json::parse(buf + 1, err);
REQUIRE(!msg.is_null());

REQUIRE(msg["levelnum"].template get<int>() == CLOUDLOG_DEBUG);
REQUIRE_THAT(msg["filename"].template get<std::string>(), Catch::Contains("test_swaglog.cc"));
REQUIRE(msg["funcname"].template get<std::string>() == "log_thread");
REQUIRE(msg["lineno"].template get<int>() == LINE_NO);
REQUIRE(msg["levelnum"].int_value() == CLOUDLOG_DEBUG);
REQUIRE_THAT(msg["filename"].string_value(), Catch::Contains("test_swaglog.cc"));
REQUIRE(msg["funcname"].string_value() == "log_thread");
REQUIRE(msg["lineno"].int_value() == LINE_NO);

auto ctx = msg["ctx"];

REQUIRE(ctx["daemon"].template get<std::string>() == daemon_name);
REQUIRE(ctx["dongle_id"].template get<std::string>() == dongle_id);
REQUIRE(ctx["dirty"].template get<bool>() == true);
REQUIRE(ctx["daemon"].string_value() == daemon_name);
REQUIRE(ctx["dongle_id"].string_value() == dongle_id);
REQUIRE(ctx["dirty"].bool_value() == true);

REQUIRE(ctx["version"].template get<std::string>() == COMMA_VERSION);
REQUIRE(ctx["version"].string_value() == COMMA_VERSION);

std::string device = Hardware::get_name();
REQUIRE(ctx["device"].template get<std::string>() == device);
REQUIRE(ctx["device"].string_value() == device);

int thread_id = atoi(msg["msg"].template get<std::string>().c_str());
int thread_id = atoi(msg["msg"].string_value().c_str());
REQUIRE((thread_id >= 0 && thread_id < thread_cnt));
thread_msgs[thread_id]++;
total_count++;
Expand Down
3 changes: 2 additions & 1 deletion release/files_common
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,8 @@ third_party/SConscript
third_party/linux/**
third_party/opencl/**

third_party/nlohmann/json.hpp
third_party/json11/json11.cpp
third_party/json11/json11.hpp

third_party/qrcode/*.cc
third_party/qrcode/*.hpp
Expand Down
79 changes: 40 additions & 39 deletions selfdrive/modeld/thneed/serialize.cc
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
#include <cassert>
#include <set>

#include "nlohmann/json.hpp"
#include "third_party/json11/json11.hpp"
#include "common/util.h"
#include "common/clutil.h"
#include "selfdrive/modeld/thneed/thneed.h"
using namespace nlohmann;
using namespace json11;

extern map<cl_program, string> g_program_source;

Expand All @@ -14,24 +14,25 @@ void Thneed::load(const char *filename) {

string buf = util::read_file(filename);
int jsz = *(int *)buf.data();
string jsonerr;
string jj(buf.data() + sizeof(int), jsz);
json jdat = json::parse(jj);
Json jdat = Json::parse(jj, jsonerr);

map<cl_mem, cl_mem> real_mem;
real_mem[NULL] = NULL;

int ptr = sizeof(int)+jsz;
for (auto &obj : jdat["objects"]) {
auto mobj = obj;
int sz = mobj["size"].template get<int>();
for (auto &obj : jdat["objects"].array_items()) {
auto mobj = obj.object_items();
int sz = mobj["size"].int_value();
cl_mem clbuf = NULL;

if (mobj["buffer_id"].template get<std::string>().size() > 0) {
if (mobj["buffer_id"].string_value().size() > 0) {
// image buffer must already be allocated
clbuf = real_mem[*(cl_mem*)(mobj["buffer_id"].template get<std::string>().data())];
assert(mobj["needs_load"].template get<bool>() == false);
clbuf = real_mem[*(cl_mem*)(mobj["buffer_id"].string_value().data())];
assert(mobj["needs_load"].bool_value() == false);
} else {
if (mobj["needs_load"].template get<bool>()) {
if (mobj["needs_load"].bool_value()) {
clbuf = clCreateBuffer(context, CL_MEM_COPY_HOST_PTR | CL_MEM_READ_WRITE, sz, &buf[ptr], NULL);
if (debug >= 1) printf("loading %p %d @ 0x%X\n", clbuf, sz, ptr);
ptr += sz;
Expand All @@ -47,9 +48,9 @@ void Thneed::load(const char *filename) {
if (mobj["arg_type"] == "image2d_t" || mobj["arg_type"] == "image1d_t") {
cl_image_desc desc = {0};
desc.image_type = (mobj["arg_type"] == "image2d_t") ? CL_MEM_OBJECT_IMAGE2D : CL_MEM_OBJECT_IMAGE1D_BUFFER;
desc.image_width = mobj["width"].template get<int>();
desc.image_height = mobj["height"].template get<int>();
desc.image_row_pitch = mobj["row_pitch"].template get<int>();
desc.image_width = mobj["width"].int_value();
desc.image_height = mobj["height"].int_value();
desc.image_row_pitch = mobj["row_pitch"].int_value();
assert(sz == desc.image_height*desc.image_row_pitch);
#ifdef QCOM2
desc.buffer = clbuf;
Expand All @@ -59,12 +60,12 @@ void Thneed::load(const char *filename) {
#endif
cl_image_format format = {0};
format.image_channel_order = CL_RGBA;
format.image_channel_data_type = mobj["float32"].template get<bool>() ? CL_FLOAT : CL_HALF_FLOAT;
format.image_channel_data_type = mobj["float32"].bool_value() ? CL_FLOAT : CL_HALF_FLOAT;

cl_int errcode;

#ifndef QCOM2
if (mobj["needs_load"].template get<bool>()) {
if (mobj["needs_load"].bool_value()) {
clbuf = clCreateImage(context, CL_MEM_COPY_HOST_PTR | CL_MEM_READ_WRITE, &format, &desc, &buf[ptr-sz], &errcode);
} else {
clbuf = clCreateImage(context, CL_MEM_READ_WRITE, &format, &desc, NULL, &errcode);
Expand All @@ -79,22 +80,22 @@ void Thneed::load(const char *filename) {
assert(clbuf != NULL);
}

real_mem[*(cl_mem*)(mobj["id"].template get<std::string>().data())] = clbuf;
real_mem[*(cl_mem*)(mobj["id"].string_value().data())] = clbuf;
}

map<string, cl_program> g_programs;
for (const auto &[name, source] : jdat["programs"].items()) {
if (debug >= 1) printf("building %s with size %zu\n", name.c_str(), source.template get<std::string>().size());
g_programs[name] = cl_program_from_source(context, device_id, source.template get<std::string>());
for (const auto &[name, source] : jdat["programs"].object_items()) {
if (debug >= 1) printf("building %s with size %zu\n", name.c_str(), source.string_value().size());
g_programs[name] = cl_program_from_source(context, device_id, source.string_value());
}

for (auto &obj : jdat["inputs"]) {
auto mobj = obj;
int sz = mobj["size"].template get<int>();
cl_mem aa = real_mem[*(cl_mem*)(mobj["buffer_id"].template get<std::string>().data())];
for (auto &obj : jdat["inputs"].array_items()) {
auto mobj = obj.object_items();
int sz = mobj["size"].int_value();
cl_mem aa = real_mem[*(cl_mem*)(mobj["buffer_id"].string_value().data())];
input_clmem.push_back(aa);
input_sizes.push_back(sz);
printf("Thneed::load: adding input %s with size %d\n", mobj["name"].template get<std::string>().data(), sz);
printf("Thneed::load: adding input %s with size %d\n", mobj["name"].string_value().data(), sz);

cl_int cl_err;
void *ret = clEnqueueMapBuffer(command_queue, aa, CL_TRUE, CL_MAP_WRITE, 0, sz, 0, NULL, NULL, &cl_err);
Expand All @@ -103,39 +104,39 @@ void Thneed::load(const char *filename) {
inputs.push_back(ret);
}

for (auto &obj : jdat["outputs"]) {
auto mobj = obj;
int sz = mobj["size"].template get<int>();
for (auto &obj : jdat["outputs"].array_items()) {
auto mobj = obj.object_items();
int sz = mobj["size"].int_value();
printf("Thneed::save: adding output with size %d\n", sz);
// TODO: support multiple outputs
output = real_mem[*(cl_mem*)(mobj["buffer_id"].template get<std::string>().data())];
output = real_mem[*(cl_mem*)(mobj["buffer_id"].string_value().data())];
assert(output != NULL);
}

for (auto &obj : jdat["binaries"]) {
string name = obj["name"].template get<std::string>();
size_t length = obj["length"].template get<int>();
for (auto &obj : jdat["binaries"].array_items()) {
string name = obj["name"].string_value();
size_t length = obj["length"].int_value();
if (debug >= 1) printf("binary %s with size %zu\n", name.c_str(), length);
g_programs[name] = cl_program_from_binary(context, device_id, (const uint8_t*)&buf[ptr], length);
ptr += length;
}

for (auto &obj : jdat["kernels"]) {
for (auto &obj : jdat["kernels"].array_items()) {
auto gws = obj["global_work_size"];
auto lws = obj["local_work_size"];
auto kk = shared_ptr<CLQueuedKernel>(new CLQueuedKernel(this));

kk->name = obj["name"].template get<std::string>();
kk->name = obj["name"].string_value();
kk->program = g_programs[kk->name];
kk->work_dim = obj["work_dim"].template get<int>();
kk->work_dim = obj["work_dim"].int_value();
for (int i = 0; i < kk->work_dim; i++) {
kk->global_work_size[i] = gws[i].template get<int>();
kk->local_work_size[i] = lws[i].template get<int>();
kk->global_work_size[i] = gws[i].int_value();
kk->local_work_size[i] = lws[i].int_value();
}
kk->num_args = obj["num_args"].template get<int>();
kk->num_args = obj["num_args"].int_value();
for (int i = 0; i < kk->num_args; i++) {
string arg = obj["args"][i].template get<std::string>();
int arg_size = obj["args_size"][i].template get<int>();
string arg = obj["args"].array_items()[i].string_value();
int arg_size = obj["args_size"].array_items()[i].int_value();
kk->args_size.push_back(arg_size);
if (arg_size == 8) {
cl_mem val = *(cl_mem*)(arg.data());
Expand Down
4 changes: 4 additions & 0 deletions selfdrive/modeld/thneed/thneed.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ using namespace std;

cl_int thneed_clSetKernelArg(cl_kernel kernel, cl_uint arg_index, size_t arg_size, const void *arg_value);

namespace json11 {
class Json;
}
class Thneed;

class GPUMalloc {
Expand Down Expand Up @@ -49,6 +52,7 @@ class CLQueuedKernel {
vector<string> args;
vector<int> args_size;
cl_kernel kernel = NULL;
json11::Json to_json() const;

cl_uint work_dim;
size_t global_work_size[3] = {0};
Expand Down
2 changes: 1 addition & 1 deletion selfdrive/navd/SConscript
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Import('qt_env', 'arch', 'common', 'messaging', 'visionipc', 'cereal', 'transfor

map_env = qt_env.Clone()
libs = ['qt_widgets', 'qt_util', 'qmapboxgl', common, messaging, cereal, visionipc, transformations,
'zmq', 'capnp', 'kj', 'm', 'OpenCL', 'ssl', 'crypto', 'pthread'] + map_env["LIBS"]
'zmq', 'capnp', 'kj', 'm', 'OpenCL', 'ssl', 'crypto', 'pthread', 'json11'] + map_env["LIBS"]
if arch == 'larch64':
libs.append(':libEGL_mesa.so.0')

Expand Down
4 changes: 2 additions & 2 deletions selfdrive/ui/SConscript
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ if GetOption('extras') and arch != "Darwin":
# setup and factory resetter
qt_env.Program("qt/setup/reset", ["qt/setup/reset.cc"], LIBS=qt_libs)
qt_env.Program("qt/setup/setup", ["qt/setup/setup.cc", asset_obj],
LIBS=qt_libs + ['curl', 'common'])
LIBS=qt_libs + ['curl', 'common', 'json11'])

# build updater UI
qt_env.Program("qt/setup/updater", ["qt/setup/updater.cc", asset_obj], LIBS=qt_libs)
Expand Down Expand Up @@ -126,4 +126,4 @@ if GetOption('extras') and arch != "Darwin":

# build watch3
if arch in ['x86_64', 'aarch64', 'Darwin'] or GetOption('extras'):
qt_env.Program("watch3", ["watch3.cc"], LIBS=qt_libs + ['common', 'zmq', 'visionipc', 'messaging'])
qt_env.Program("watch3", ["watch3.cc"], LIBS=qt_libs + ['common', 'json11', 'zmq', 'visionipc', 'messaging'])
2 changes: 1 addition & 1 deletion system/logcatd/SConscript
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
Import('env', 'cereal', 'messaging', 'common')

env.Program('logcatd', 'logcatd_systemd.cc', LIBS=[cereal, messaging, common, 'zmq', 'capnp', 'kj', 'systemd'])
env.Program('logcatd', 'logcatd_systemd.cc', LIBS=[cereal, messaging, common, 'zmq', 'capnp', 'kj', 'systemd', 'json11'])
4 changes: 2 additions & 2 deletions system/logcatd/logcatd_systemd.cc
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
#include <map>
#include <string>

#include "nlohmann/json.hpp"
#include "third_party/json11/json11.hpp"

#include "cereal/messaging/messaging.h"
#include "common/timing.h"
Expand Down Expand Up @@ -62,7 +62,7 @@ int main(int argc, char *argv[]) {
// Build message
auto androidEntry = msg.initEvent().initAndroidLog();
androidEntry.setTs(timestamp);
androidEntry.setMessage(nlohmann::json(kv).dump());
androidEntry.setMessage(json11::Json(kv).dump());
if (kv.count("_PID")) androidEntry.setPid(std::atoi(kv["_PID"].c_str()));
if (kv.count("PRIORITY")) androidEntry.setPriority(std::atoi(kv["PRIORITY"].c_str()));
if (kv.count("SYSLOG_IDENTIFIER")) androidEntry.setTag(kv["SYSLOG_IDENTIFIER"]);
Expand Down
Loading

0 comments on commit 88757c1

Please sign in to comment.