Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Switch to nlohmann/json from json11 #31093

Merged
merged 7 commits into from
Jan 23, 2024
Merged

Switch to nlohmann/json from json11 #31093

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
99e8440
Initial commit
mtribiere Jan 20, 2024
1267b66
fix release/file_common
mtribiere Jan 20, 2024
8734833
Update common/swaglog.cc
adeebshihadeh Jan 21, 2024
e793a82
Merge branch 'master' of github.com:commaai/openpilot into json_lib
adeebshihadeh Jan 21, 2024
74f86a1
Swtiched serialize cc
mtribiere Jan 22, 2024
10d52ac
Merge branch 'master' of github.com:commaai/openpilot into json_lib
adeebshihadeh Jan 23, 2024
b8419a9
move to lfs
adeebshihadeh Jan 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions SConstruct
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ env = Environment(
"#third_party/acados/include/hpipm/include",
"#third_party/catch2/include",
"#third_party/libyuv/include",
"#third_party/json11",
"#third_party/nlohmann",
"#third_party/linux/include",
"#third_party/snpe/include",
"#third_party/mapbox-gl-native-qt/include",
Expand Down Expand Up @@ -351,7 +351,7 @@ Export('env', 'qt_env', 'arch', 'real_arch')
SConscript(['common/SConscript'])
Import('_common', '_gpucommon')

common = [_common, 'json11']
common = [_common]
gpucommon = [_gpucommon]

Export('common', 'gpucommon')
Expand Down
6 changes: 3 additions & 3 deletions common/SConscript
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ common_libs = [
if arch != "Darwin":
common_libs.append('gpio.cc')

_common = env.Library('common', common_libs, LIBS="json11")
_common = env.Library('common', common_libs)

files = [
'clutil.cc',
Expand All @@ -24,10 +24,10 @@ Export('_common', '_gpucommon')
if GetOption('extras'):
env.Program('tests/test_common',
['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc', 'tests/test_ratekeeper.cc'],
LIBS=[_common, 'json11', 'zmq', 'pthread'])
LIBS=[_common, 'zmq', 'pthread'])

# Cython bindings
params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq'])

SConscript([
'transformations/SConscript',
Expand Down
16 changes: 8 additions & 8 deletions common/swaglog.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

#include <zmq.h>
#include <stdarg.h>
#include "third_party/json11/json11.hpp"
#include "nlohmann/json.hpp"
#include "common/version.h"
#include "system/hardware/hw.h"

Expand All @@ -37,7 +37,7 @@ class SwaglogState {
}
}

ctx_j = json11::Json::object{};
ctx_j = nlohmann::json();
if (char* dongle_id = getenv("DONGLE_ID")) {
ctx_j["dongle_id"] = dongle_id;
}
Expand Down Expand Up @@ -66,17 +66,17 @@ class SwaglogState {
void* zctx = nullptr;
void* sock = nullptr;
int print_level;
json11::Json::object ctx_j;
nlohmann::json ctx_j;
};

bool LOG_TIMESTAMPS = getenv("LOG_TIMESTAMPS");
uint32_t NO_FRAME_ID = std::numeric_limits<uint32_t>::max();

static void cloudlog_common(int levelnum, const char* filename, int lineno, const char* func,
char* msg_buf, const json11::Json::object &msg_j={}) {
char* msg_buf, const nlohmann::json &msg_j={}) {
static SwaglogState s;

json11::Json::object log_j = json11::Json::object {
nlohmann::json log_j = nlohmann::json {
{"ctx", s.ctx_j},
{"levelnum", levelnum},
{"filename", filename},
Expand All @@ -92,7 +92,7 @@ static void cloudlog_common(int levelnum, const char* filename, int lineno, cons

std::string log_s;
log_s += (char)levelnum;
((json11::Json)log_j).dump(log_s);
log_s = log_j.dump();
adeebshihadeh marked this conversation as resolved.
Show resolved Hide resolved
s.log(levelnum, filename, lineno, func, msg_buf, log_s);

free(msg_buf);
Expand All @@ -115,14 +115,14 @@ void cloudlog_t_common(int levelnum, const char* filename, int lineno, const cha
char* msg_buf = nullptr;
int ret = vasprintf(&msg_buf, fmt, args);
if (ret <= 0 || !msg_buf) return;
json11::Json::object tspt_j = json11::Json::object{
nlohmann::json tspt_j = nlohmann::json{
{"event", msg_buf},
{"time", std::to_string(nanos_since_boot())}
};
if (frame_id < NO_FRAME_ID) {
tspt_j["frame_id"] = std::to_string(frame_id);
}
tspt_j = json11::Json::object{{"timestamp", tspt_j}};
tspt_j = nlohmann::json{{"timestamp", tspt_j}};
cloudlog_common(levelnum, filename, lineno, func, msg_buf, tspt_j);
}

Expand Down
25 changes: 12 additions & 13 deletions common/tests/test_swaglog.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
#include "common/util.h"
#include "common/version.h"
#include "system/hardware/hw.h"
#include "third_party/json11/json11.hpp"
#include "nlohmann/json.hpp"

std::string daemon_name = "testy";
std::string dongle_id = "test_dongle_id";
Expand Down Expand Up @@ -39,26 +39,25 @@ void recv_log(int thread_cnt, int thread_msg_cnt) {

REQUIRE(buf[0] == CLOUDLOG_DEBUG);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test fail, but I'm not sure why it's here in the first place. If you got a JSON as a string, then buf[0] should be '{' right ?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

see

openpilot/common/swaglog.cc

Line 94 in 78e20cf

log_s += (char)levelnum;

std::string err;
auto msg = json11::Json::parse(buf + 1, err);
REQUIRE(!msg.is_null());
nlohmann::json msg = nlohmann::json::parse(buf + 1);

REQUIRE(msg["levelnum"].int_value() == CLOUDLOG_DEBUG);
REQUIRE_THAT(msg["filename"].string_value(), Catch::Contains("test_swaglog.cc"));
REQUIRE(msg["funcname"].string_value() == "log_thread");
REQUIRE(msg["lineno"].int_value() == LINE_NO);
REQUIRE(msg["levelnum"].template get<int>() == CLOUDLOG_DEBUG);
REQUIRE_THAT(msg["filename"].template get<std::string>(), Catch::Contains("test_swaglog.cc"));
REQUIRE(msg["funcname"].template get<std::string>() == "log_thread");
REQUIRE(msg["lineno"].template get<int>() == LINE_NO);

auto ctx = msg["ctx"];

REQUIRE(ctx["daemon"].string_value() == daemon_name);
REQUIRE(ctx["dongle_id"].string_value() == dongle_id);
REQUIRE(ctx["dirty"].bool_value() == true);
REQUIRE(ctx["daemon"].template get<std::string>() == daemon_name);
REQUIRE(ctx["dongle_id"].template get<std::string>() == dongle_id);
REQUIRE(ctx["dirty"].template get<bool>() == true);

REQUIRE(ctx["version"].string_value() == COMMA_VERSION);
REQUIRE(ctx["version"].template get<std::string>() == COMMA_VERSION);

std::string device = Hardware::get_name();
REQUIRE(ctx["device"].string_value() == device);
REQUIRE(ctx["device"].template get<std::string>() == device);

int thread_id = atoi(msg["msg"].string_value().c_str());
int thread_id = atoi(msg["msg"].template get<std::string>().c_str());
REQUIRE((thread_id >= 0 && thread_id < thread_cnt));
thread_msgs[thread_id]++;
total_count++;
Expand Down
3 changes: 1 addition & 2 deletions release/files_common
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -374,8 +374,7 @@ third_party/SConscript
third_party/linux/**
third_party/opencl/**

third_party/json11/json11.cpp
third_party/json11/json11.hpp
third_party/nlohmann/json.hpp

third_party/qrcode/*.cc
third_party/qrcode/*.hpp
Expand Down
2 changes: 1 addition & 1 deletion selfdrive/navd/SConscript
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Import('qt_env', 'arch', 'common', 'messaging', 'visionipc', 'cereal', 'transfor

map_env = qt_env.Clone()
libs = ['qt_widgets', 'qt_util', 'qmapboxgl', common, messaging, cereal, visionipc, transformations,
'zmq', 'capnp', 'kj', 'm', 'OpenCL', 'ssl', 'crypto', 'pthread', 'json11'] + map_env["LIBS"]
'zmq', 'capnp', 'kj', 'm', 'OpenCL', 'ssl', 'crypto', 'pthread'] + map_env["LIBS"]
if arch == 'larch64':
libs.append(':libEGL_mesa.so.0')

Expand Down
4 changes: 2 additions & 2 deletions selfdrive/ui/SConscript
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ if GetOption('extras') and arch != "Darwin":
# setup and factory resetter
qt_env.Program("qt/setup/reset", ["qt/setup/reset.cc"], LIBS=qt_libs)
qt_env.Program("qt/setup/setup", ["qt/setup/setup.cc", asset_obj],
LIBS=qt_libs + ['curl', 'common', 'json11'])
LIBS=qt_libs + ['curl', 'common'])

# build updater UI
qt_env.Program("qt/setup/updater", ["qt/setup/updater.cc", asset_obj], LIBS=qt_libs)
Expand Down Expand Up @@ -126,4 +126,4 @@ if GetOption('extras') and arch != "Darwin":

# build watch3
if arch in ['x86_64', 'aarch64', 'Darwin'] or GetOption('extras'):
qt_env.Program("watch3", ["watch3.cc"], LIBS=qt_libs + ['common', 'json11', 'zmq', 'visionipc', 'messaging'])
qt_env.Program("watch3", ["watch3.cc"], LIBS=qt_libs + ['common', 'zmq', 'visionipc', 'messaging'])
2 changes: 1 addition & 1 deletion system/logcatd/SConscript
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
Import('env', 'cereal', 'messaging', 'common')

env.Program('logcatd', 'logcatd_systemd.cc', LIBS=[cereal, messaging, common, 'zmq', 'capnp', 'kj', 'systemd', 'json11'])
env.Program('logcatd', 'logcatd_systemd.cc', LIBS=[cereal, messaging, common, 'zmq', 'capnp', 'kj', 'systemd'])
4 changes: 2 additions & 2 deletions system/logcatd/logcatd_systemd.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
#include <map>
#include <string>

#include "third_party/json11/json11.hpp"
#include "nlohmann/json.hpp"

#include "cereal/messaging/messaging.h"
#include "common/timing.h"
Expand Down Expand Up @@ -62,7 +62,7 @@ int main(int argc, char *argv[]) {
// Build message
auto androidEntry = msg.initEvent().initAndroidLog();
androidEntry.setTs(timestamp);
androidEntry.setMessage(json11::Json(kv).dump());
androidEntry.setMessage(nlohmann::json(kv).dump());
if (kv.count("_PID")) androidEntry.setPid(std::atoi(kv["_PID"].c_str()));
if (kv.count("PRIORITY")) androidEntry.setPriority(std::atoi(kv["PRIORITY"].c_str()));
if (kv.count("SYSLOG_IDENTIFIER")) androidEntry.setTag(kv["SYSLOG_IDENTIFIER"]);
Expand Down
2 changes: 1 addition & 1 deletion system/proclogd/SConscript
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
Import('env', 'cereal', 'messaging', 'common')
libs = [cereal, messaging, 'pthread', 'zmq', 'capnp', 'kj', 'common', 'zmq', 'json11']
libs = [cereal, messaging, 'pthread', 'zmq', 'capnp', 'kj', 'common', 'zmq']
env.Program('proclogd', ['main.cc', 'proclog.cc'], LIBS=libs)

if GetOption('extras'):
Expand Down
3 changes: 0 additions & 3 deletions third_party/SConscript
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
Import('env')

env.Library('json11', ['json11/json11.cpp'], CCFLAGS=env['CCFLAGS'] + ['-Wno-unqualified-std-cast-call'])
env.Append(CPPPATH=[Dir('json11')])

env.Library('kaitai', ['kaitai/kaitaistream.cpp'], CPPDEFINES=['KS_STR_ENCODING_NONE'])
Loading
Loading