Skip to content

Commit

Permalink
Fixed issues with reading/writing JSON floats
Browse files Browse the repository at this point in the history
* Reading/writing floating point was accidentally locale-dependent,
  which caused errors in locales that don't use '.' as the decimal
  point. (Fixes #54)
* Improved round-trip accuracy of Fleece->JSON->Fleece float/double
  conversions, by writing the optimal number of decimal places.
  To do this we brought in a small subcomponent of Swift. (Fixes #55)
  • Loading branch information
snej committed Sep 23, 2019
1 parent a4dc779 commit 8bc68ae
Show file tree
Hide file tree
Showing 9 changed files with 2,683 additions and 10 deletions.
24 changes: 24 additions & 0 deletions Fleece.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,10 @@
27D7218B1F8E8EEA00AA4458 /* FleeceException.hh in Headers */ = {isa = PBXBuildFile; fileRef = 275CED511D3EF7BE001DE46C /* FleeceException.hh */; };
27D7218C1F8E8EEA00AA4458 /* SharedKeys.hh in Headers */ = {isa = PBXBuildFile; fileRef = 27E3DD411DB6A14200F2872D /* SharedKeys.hh */; };
27D721961F8E900400AA4458 /* libFleeceMutableObjC.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 27D721901F8E8EEA00AA4458 /* libFleeceMutableObjC.a */; };
27D965682339595700F4A51C /* NumConversion.hh in Headers */ = {isa = PBXBuildFile; fileRef = 27D965662339595700F4A51C /* NumConversion.hh */; };
27D965692339595700F4A51C /* NumConversion.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27D965672339595700F4A51C /* NumConversion.cc */; };
27D9656D23397EF700F4A51C /* SwiftDtoa.h in Headers */ = {isa = PBXBuildFile; fileRef = 27D9656B23397EF700F4A51C /* SwiftDtoa.h */; };
27D9656E23397EF700F4A51C /* SwiftDtoa.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27D9656C23397EF700F4A51C /* SwiftDtoa.cc */; settings = {COMPILER_FLAGS = "-Wno-implicit-int-conversion -Wno-shadow -Wno-shorten-64-to-32"; }; };
27DE2E922125FA1700123597 /* varint.cc in Sources */ = {isa = PBXBuildFile; fileRef = 270FA2761BF53CEA005DCB13 /* varint.cc */; };
27DE2E962125FA1700123597 /* RefCounted.cc in Sources */ = {isa = PBXBuildFile; fileRef = 274D8254209D1764008BB39F /* RefCounted.cc */; };
27DE2E9D2125FA1700123597 /* slice+CoreFoundation.cc in Sources */ = {isa = PBXBuildFile; fileRef = 272E5A601BF91F6C00848580 /* slice+CoreFoundation.cc */; };
Expand Down Expand Up @@ -424,6 +428,10 @@
27D721241F8C4B7500AA4458 /* MDictIterator.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = MDictIterator.hh; sourceTree = "<group>"; };
27D721501F8D8F3F00AA4458 /* MContext.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = MContext.hh; sourceTree = "<group>"; };
27D721901F8E8EEA00AA4458 /* libFleeceMutableObjC.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libFleeceMutableObjC.a; sourceTree = BUILT_PRODUCTS_DIR; };
27D965662339595700F4A51C /* NumConversion.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = NumConversion.hh; sourceTree = "<group>"; };
27D965672339595700F4A51C /* NumConversion.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = NumConversion.cc; sourceTree = "<group>"; };
27D9656B23397EF700F4A51C /* SwiftDtoa.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SwiftDtoa.h; sourceTree = "<group>"; };
27D9656C23397EF700F4A51C /* SwiftDtoa.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = SwiftDtoa.cc; sourceTree = "<group>"; };
27DE2EDF2125FA1700123597 /* libfleeceBase.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libfleeceBase.a; sourceTree = BUILT_PRODUCTS_DIR; };
27DFAE10219F83AB00DF57EB /* InstanceCounted.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = InstanceCounted.hh; sourceTree = "<group>"; };
27DFAE11219F83AB00DF57EB /* InstanceCounted.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = InstanceCounted.cc; sourceTree = "<group>"; };
Expand Down Expand Up @@ -584,6 +592,7 @@
27298E3E1C00F8A9000CFBA8 /* vendor */ = {
isa = PBXGroup;
children = (
27D9656A23397EBD00F4A51C /* SwiftDtoa */,
27E3DD461DB6B86000F2872D /* catch */,
27298E3F1C00F8A9000CFBA8 /* jsonsl */,
277015321D596436008BADD7 /* libb64 */,
Expand Down Expand Up @@ -739,6 +748,8 @@
276D15451E007D3000543B1B /* JSON5.hh */,
27FE87F11E53E43200C5CF3F /* JSONEncoder.cc */,
27FE87F21E53E43200C5CF3F /* JSONEncoder.hh */,
27D965662339595700F4A51C /* NumConversion.hh */,
27D965672339595700F4A51C /* NumConversion.cc */,
2700BB9B217E8C0C00797537 /* ParseDate.cc */,
2700BB9A217E8C0C00797537 /* ParseDate.hh */,
271507ED2122381E005FE6E8 /* PlatformCompat.hh */,
Expand Down Expand Up @@ -822,6 +833,15 @@
path = ObjC;
sourceTree = "<group>";
};
27D9656A23397EBD00F4A51C /* SwiftDtoa */ = {
isa = PBXGroup;
children = (
27D9656B23397EF700F4A51C /* SwiftDtoa.h */,
27D9656C23397EF700F4A51C /* SwiftDtoa.cc */,
);
path = SwiftDtoa;
sourceTree = "<group>";
};
27DE2EE22125FAC600123597 /* Frameworks */ = {
isa = PBXGroup;
children = (
Expand Down Expand Up @@ -966,6 +986,7 @@
27DE2EBD2125FA1700123597 /* Array.hh in Headers */,
27DE2EBE2125FA1700123597 /* DeepIterator.hh in Headers */,
27DE2EBF2125FA1700123597 /* MArray.hh in Headers */,
27D9656D23397EF700F4A51C /* SwiftDtoa.h in Headers */,
27DE2EC02125FA1700123597 /* Dict.hh in Headers */,
27DE2EC12125FA1700123597 /* StringTable.hh in Headers */,
27DE2EC22125FA1700123597 /* JSONEncoder.hh in Headers */,
Expand Down Expand Up @@ -994,6 +1015,7 @@
27C59BA4212F898F003C8492 /* SmallVector.hh in Headers */,
27DE2ED72125FA1700123597 /* Path.hh in Headers */,
27DE2ED82125FA1700123597 /* HeapDict.hh in Headers */,
27D965682339595700F4A51C /* NumConversion.hh in Headers */,
27DE2ED92125FA1700123597 /* HeapValue.hh in Headers */,
27DE2EDA2125FA1700123597 /* FleeceException.hh in Headers */,
27DE2EDB2125FA1700123597 /* SharedKeys.hh in Headers */,
Expand Down Expand Up @@ -1292,11 +1314,13 @@
27DE2E9E2125FA1700123597 /* slice.cc in Sources */,
27DFAE13219F83AB00DF57EB /* InstanceCounted.cc in Sources */,
27DE2E9D2125FA1700123597 /* slice+CoreFoundation.cc in Sources */,
27D965692339595700F4A51C /* NumConversion.cc in Sources */,
27DE2EEA2125FC5A00123597 /* Writer.cc in Sources */,
27DE2E922125FA1700123597 /* varint.cc in Sources */,
27DE2EE82125FB2100123597 /* cdecode.c in Sources */,
27DE2EE92125FB2100123597 /* cencode.c in Sources */,
2700BB9D217E8C0D00797537 /* ParseDate.cc in Sources */,
27D9656E23397EF700F4A51C /* SwiftDtoa.cc in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down
5 changes: 2 additions & 3 deletions Fleece/Core/JSONConverter.cc
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
//

#include "JSONConverter.hh"
#include "NumConversion.hh"
#include "jsonsl.h"
#include <map>

Expand Down Expand Up @@ -114,9 +115,7 @@ namespace fleece { namespace impl {
unsigned f = state->special_flags;
if (f & JSONSL_SPECIALf_FLOAT) {
char *start = (char*)&_input[state->pos_begin];
char *end;
double n = ::strtod(start, &end);
_encoder.writeDouble(n);
_encoder.writeDouble(ParseDouble(start));
} else if (f & JSONSL_SPECIALf_UNSIGNED) {
_encoder.writeUInt(state->nelem);
} else if (f & JSONSL_SPECIALf_SIGNED) {
Expand Down
18 changes: 13 additions & 5 deletions Fleece/Support/JSONEncoder.hh
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include "Writer.hh"
#include "Value.hh"
#include "FleeceException.hh"
#include "NumConversion.hh"
#include <stdio.h>


Expand Down Expand Up @@ -51,10 +52,10 @@ namespace fleece { namespace impl {
void writeNull() {comma(); _out << slice("null");}
void writeBool(bool b) {comma(); _out.write(b ? "true"_sl : "false"_sl);}

void writeInt(int64_t i) {writef("%lld", i);}
void writeUInt(uint64_t i) {writef("%llu", i);}
void writeFloat(float f) {writef("%.6g", f);}
void writeDouble(double d) {writef("%.16g", d);}
void writeInt(int64_t i) {_writeInt("%lld", i);}
void writeUInt(uint64_t i) {_writeInt("%llu", i);}
void writeFloat(float f) {_writeFloat(f);}
void writeDouble(double d) {_writeFloat(d);}

void writeString(const std::string &s) {writeString(slice(s));}
void writeString(slice s);
Expand Down Expand Up @@ -119,12 +120,19 @@ namespace fleece { namespace impl {
}

template <class T>
void writef(const char *fmt, T t) {
void _writeInt(const char *fmt, T t) {
comma();
char str[32];
_out.write(str, sprintf(str, fmt, t));
}

template <class T>
void _writeFloat(T t) {
comma();
char str[32];
_out.write(str, WriteFloat(t, str, sizeof(str)));
}

Writer _out;
bool _json5 {false};
bool _canonical {false};
Expand Down
47 changes: 47 additions & 0 deletions Fleece/Support/NumConversion.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
//
// NumConversion.cc
//
// Copyright © 2019 Couchbase. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

#include "NumConversion.hh"
#include "SwiftDtoA.h"
#include <stdlib.h>
#include <xlocale.h>

namespace fleece {

double ParseDouble(const char *str) noexcept {
// strtod is locale-aware, so in some locales it will not interpret '.' as a decimal point.
// To work around that, use the C locale explicitly.
#ifdef LC_C_LOCALE
#define kCLocale LC_C_LOCALE
#else
static locale_t kCLocale = newlocale(LC_ALL_MASK, NULL, NULL);
#endif
char *end;
return strtod_l(str, &end, kCLocale);
}


size_t WriteFloat(float n, char *dst, size_t capacity) {
return swift_format_float(n, dst, capacity);
}


size_t WriteFloat(double n, char *dst, size_t capacity) {
return swift_format_double(n, dst, capacity);
}
}
25 changes: 25 additions & 0 deletions Fleece/Support/NumConversion.hh
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
//
// NumConversion.hh
//
// Copyright © 2019 Couchbase. All rights reserved.
//

#pragma once
#include "PlatformCompat.hh"

namespace fleece {

/// Parse `str` as a floating-point number, reading as many digits as possible.
/// (I.e. non-numeric characters after the digits are not treated as an error.)
double ParseDouble(const char *str NONNULL) noexcept;

/// Format a 64-bit-floating point number to a string.
size_t WriteFloat(double n, char *dst, size_t capacity);

/// Format a 32-bit floating-point number to a string.
size_t WriteFloat(float n, char *dst, size_t capacity);

/// Alternative syntax for formatting a 64-bit-floating point number to a string.
static inline size_t WriteDouble(double n, char *dst, size_t c) {return WriteFloat(n, dst, c);}

}
2 changes: 1 addition & 1 deletion Tests/MutableTests.cc
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ namespace fleece {
CHECK(!i);
}

CHECK(ma->asArray()->toJSON() == "[null,false,true,0,-123,2017,123456789,-123456789,\"Hot dog\",3.14159,3.141592653589793]"_sl);
CHECK(ma->asArray()->toJSON() == "[null,false,true,0,-123,2017,123456789,-123456789,\"Hot dog\",3.1415927,3.141592653589793]"_sl);

ma->remove(3, 5);
CHECK(ma->count() == 6);
Expand Down
2 changes: 1 addition & 1 deletion Tests/ObjCTests.mm
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ static void checkIt(id obj, const char* json) {
TEST_CASE("Obj-C Floats", "[Encoder]") {
checkIt(@0.5, "0.5");
checkIt(@-0.5, "-0.5");
checkIt(@((float)M_PI), "3.14159");
checkIt(@((float)M_PI), "3.1415927");
checkIt(@((double)M_PI), "3.141592653589793");
}

Expand Down
Loading

0 comments on commit 8bc68ae

Please sign in to comment.