Skip to content

Commit

Permalink
Efficient storage of G3MapInt and G3MapVectorInt objects
Browse files Browse the repository at this point in the history 
Use int64_t for in-memory representation of integer values, but store as 8-, 16-
or 32- bit integers on disk, depending on bit depth of the underlying data.
Backwards compatible with v1 int32 G3Map objects.

Closes #122.
  • Loading branch information
@arahlin
arahlin committed Feb 6, 2024
1 parent cdd6e43 commit c12160a
Show file tree
Hide file tree
Showing 2 changed files with 212 additions and 5 deletions.
17 changes: 14 additions & 3 deletions core/include/core/G3Map.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ class G3Map : public G3FrameObject, public std::map<Key, Value> {
cereal::base_class<std::map<Key, Value> >(this));
}

template <class A> void load(A &ar, unsigned v);
template <class A> void save(A &ar, unsigned v) const;

std::string Summary() const
{
if (this->size() < 5)
Expand Down Expand Up @@ -72,20 +75,28 @@ G3MAP_OF(std::string, double, G3MapDouble);
G3MAP_OF(std::string, G3MapDouble, G3MapMapDouble);
G3MAP_OF(std::string, std::vector<double>, G3MapVectorDouble);
G3MAP_OF(std::string, std::vector<bool>, G3MapVectorBool);
G3MAP_OF(std::string, std::vector<int32_t>, G3MapVectorInt);
G3MAP_OF(std::string, std::vector<std::string>, G3MapVectorString);
G3MAP_OF(std::string, G3VectorVectorString, G3MapVectorVectorString);
G3MAP_OF(std::string, std::vector<std::complex<double> >, G3MapVectorComplexDouble);
G3MAP_OF(std::string, G3VectorTime, G3MapVectorTime);
G3MAP_OF(std::string, int32_t, G3MapInt);
G3MAP_OF(std::string, std::string, G3MapString);
G3MAP_OF(std::string, quat, G3MapQuat);
G3MAP_OF(std::string, G3VectorQuat, G3MapVectorQuat);

#define G3MAP_SPLIT(key, value, name, version) \
typedef G3Map< key, value > name; \
namespace cereal { \
template <class A> struct specialize<A, name, cereal::specialization::member_load_save> {}; \
} \
G3_POINTERS(name); \
G3_SERIALIZABLE(name, version);

G3MAP_SPLIT(std::string, std::vector<int64_t>, G3MapVectorInt, 2);
G3MAP_SPLIT(std::string, int64_t, G3MapInt, 2);

namespace cereal {
template <class A> struct specialize<A, G3MapFrameObject, cereal::specialization::member_load_save> {};
}
G3_POINTERS(G3MapFrameObject);
G3_SERIALIZABLE(G3MapFrameObject, 1);
#endif

200 changes: 198 additions & 2 deletions core/src/G3Map.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,201 @@
#include <container_pybindings.h>
#include <serialization.h>

/* Special load/save for int64_t. */

static
int bit_count(std::map<std::string, int64_t> const &d) {
// Returns the smallest number N such that all ints in the
// vector could be safely expressed as intN_t. Assumes two's
// complement integers. Return value will be between 1 and
// 64.
uint64_t mask = 0;
for (auto c: d) {
if (c.second < 0)
mask |= ~c.second;
else
mask |= c.second;
}
for (int i=1; i<64; i++) {
if (mask == 0)
return i;
mask >>= 1;
}
return 64;
}

template <class A, typename FROM_TYPE, typename TO_TYPE>
void load_as(A &ar, std::map<std::string, TO_TYPE> &dest) {
std::map<std::string, FROM_TYPE> temp;
ar & cereal::make_nvp("map", temp);
dest.insert(temp.begin(), temp.end());
}

template <class A, typename FROM_TYPE, typename TO_TYPE>
void save_as(A &ar, const std::map<std::string, FROM_TYPE> &src) {
std::map<std::string, TO_TYPE> temp(src.begin(), src.end());
ar & cereal::make_nvp("map", temp);
}

template <>
template <class A>
void G3Map<std::string, int64_t>::load(A &ar, const unsigned v)
{
G3_CHECK_VERSION(v);

ar & cereal::make_nvp("G3FrameObject",
cereal::base_class<G3FrameObject>(this));
int store_bits = 32;
if (v >= 2)
ar & cereal::make_nvp("store_bits", store_bits);

switch(store_bits) {
case 64:
ar & cereal::make_nvp("map",
cereal::base_class<std::map<std::string, int64_t> >(this));
break;
case 32:
load_as<A, int32_t, int64_t>(ar, *this);
break;
case 16:
load_as<A, int16_t, int64_t>(ar, *this);
break;
case 8:
load_as<A, int8_t, int64_t>(ar, *this);
break;
}
}

template <>
template <class A>
void G3Map<std::string, int64_t>::save(A &ar, const unsigned v) const
{
// v == 2
ar & cereal::make_nvp("G3FrameObject",
cereal::base_class<G3FrameObject>(this));
// Count the interesting bits, and convert to nearest power of 2.
int sig_bits = bit_count(*this);
int store_bits = 8;
while (store_bits < sig_bits)
store_bits *= 2;
ar & cereal::make_nvp("store_bits", store_bits);
switch(store_bits) {
case 8:
save_as<A, int64_t, int8_t>(ar, *this);
break;
case 16:
save_as<A, int64_t, int16_t>(ar, *this);
break;
case 32:
save_as<A, int64_t, int32_t>(ar, *this);
break;
default:
ar & cereal::make_nvp("map",
cereal::base_class<std::map<std::string, int64_t> >(this));
}
}

static
int bit_count_vector(std::map<std::string, std::vector<int64_t> > const &d) {
// Returns the smallest number N such that all ints in the
// vector could be safely expressed as intN_t. Assumes two's
// complement integers. Return value will be between 1 and
// 64.
uint64_t mask = 0;
for (auto c: d) {
for (auto cc: c.second) {
if (cc < 0)
mask |= ~cc;
else
mask |= cc;
}
}
for (int i=1; i<64; i++) {
if (mask == 0)
return i;
mask >>= 1;
}
return 64;
}

template <class A, typename FROM_TYPE, typename TO_TYPE>
void load_vector_as(A &ar, std::map<std::string, std::vector<TO_TYPE> > &dest) {
std::map<std::string, std::vector<FROM_TYPE> > temp;
ar & cereal::make_nvp("map", temp);
for (auto e: temp) {
std::vector<TO_TYPE> v(e.second.begin(), e.second.end());
dest[e.first] = v;
}
}

template <class A, typename FROM_TYPE, typename TO_TYPE>
void save_vector_as(A &ar, const std::map<std::string, std::vector<FROM_TYPE> > &src) {
std::map<std::string, std::vector<TO_TYPE> > temp;
for (auto e: src) {
std::vector<TO_TYPE> v(e.second.begin(), e.second.end());
temp[e.first] = v;
}
ar & cereal::make_nvp("map", temp);
}

template <>
template <class A>
void G3Map<std::string, std::vector<int64_t> >::load(A &ar, const unsigned v)
{
G3_CHECK_VERSION(v);

ar & cereal::make_nvp("G3FrameObject",
cereal::base_class<G3FrameObject>(this));
int store_bits = 32;
if (v >= 2)
ar & cereal::make_nvp("store_bits", store_bits);

switch(store_bits) {
case 64:
ar & cereal::make_nvp("map",
cereal::base_class<std::map<std::string, std::vector<int64_t> > >(this));
break;
case 32:
load_vector_as<A, int32_t, int64_t>(ar, *this);
break;
case 16:
load_vector_as<A, int16_t, int64_t>(ar, *this);
break;
case 8:
load_vector_as<A, int8_t, int64_t>(ar, *this);
break;
}
}

template <>
template <class A>
void G3Map<std::string, std::vector<int64_t> >::save(A &ar, const unsigned v) const
{
// v == 2
ar & cereal::make_nvp("G3FrameObject",
cereal::base_class<G3FrameObject>(this));
// Count the interesting bits, and convert to nearest power of 2.
int sig_bits = bit_count_vector(*this);
int store_bits = 8;
while (store_bits < sig_bits)
store_bits *= 2;
ar & cereal::make_nvp("store_bits", store_bits);
switch(store_bits) {
case 8:
save_vector_as<A, int64_t, int8_t>(ar, *this);
break;
case 16:
save_vector_as<A, int64_t, int16_t>(ar, *this);
break;
case 32:
save_vector_as<A, int64_t, int32_t>(ar, *this);
break;
default:
ar & cereal::make_nvp("map",
cereal::base_class<std::map<std::string, std::vector<int64_t> > >(this));
}
}

template <class A> void G3MapFrameObject::save(A &ar, const unsigned v) const
{
ar << cereal::make_nvp("G3FrameObject",
Expand Down Expand Up @@ -70,20 +265,21 @@ std::string G3MapFrameObject::Description() const
return s.str();
}

G3_SERIALIZABLE_CODE(G3MapInt);
G3_SERIALIZABLE_CODE(G3MapDouble);
G3_SERIALIZABLE_CODE(G3MapMapDouble);
G3_SERIALIZABLE_CODE(G3MapString);
G3_SERIALIZABLE_CODE(G3MapQuat);
G3_SERIALIZABLE_CODE(G3MapVectorBool);
G3_SERIALIZABLE_CODE(G3MapVectorInt);
G3_SERIALIZABLE_CODE(G3MapVectorDouble);
G3_SERIALIZABLE_CODE(G3MapVectorString);
G3_SERIALIZABLE_CODE(G3MapVectorVectorString);
G3_SERIALIZABLE_CODE(G3MapVectorComplexDouble);
G3_SERIALIZABLE_CODE(G3MapVectorTime);
G3_SERIALIZABLE_CODE(G3MapVectorQuat);

G3_SPLIT_SERIALIZABLE_CODE(G3MapInt);
G3_SPLIT_SERIALIZABLE_CODE(G3MapVectorInt);

G3_SPLIT_SERIALIZABLE_CODE(G3MapFrameObject);

PYBINDINGS("core") {
Expand Down

0 comments on commit c12160a

Please sign in to comment.