Skip to content

Commit

Permalink
Fixed issues with reading/writing JSON floats
Browse files Browse the repository at this point in the history
* Reading/writing floating point was accidentally locale-dependent,
  which caused errors in locales that don't use '.' as the decimal
  point. (Fixes #54)
* Improved round-trip accuracy of Fleece->JSON->Fleece float/double
  conversions, by writing the optimal number of decimal places.
  To do this we brought in a small subcomponent of Swift. (Fixes #55)
  • Loading branch information
snej committed Sep 24, 2019
1 parent a4dc779 commit 2d4ff43
Show file tree
Hide file tree
Showing 11 changed files with 2,694 additions and 10 deletions.
3 changes: 3 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ set_base_platform_files(RESULT FLEECE_BASE_PLATFORM_SRC)
set(FLEECE_BASE_SRC Fleece/Support/Backtrace.cc
Fleece/Support/FleeceException.cc
Fleece/Support/InstanceCounted.cc
Fleece/Support/NumConversion.cc
Fleece/Support/ParseDate.cc
Fleece/Support/RefCounted.cc
Fleece/Support/Writer.cc
Expand All @@ -44,6 +45,7 @@ set(FLEECE_BASE_SRC Fleece/Support/Backtrace.cc
Fleece/Support/varint.cc
vendor/libb64/cdecode.c
vendor/libb64/cencode.c
vendor/SwiftDtoa/SwiftDtoa.cc
${FLEECE_BASE_PLATFORM_SRC})

add_library(FleeceBase STATIC ${FLEECE_BASE_SRC})
Expand Down Expand Up @@ -77,5 +79,6 @@ foreach(platform Fleece FleeceStatic FleeceBase fleeceTool FleeceTests)
Experimental
vendor/libb64
vendor/jsonsl
vendor/SwiftDtoa
)
endforeach()
24 changes: 24 additions & 0 deletions Fleece.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,10 @@
27D7218B1F8E8EEA00AA4458 /* FleeceException.hh in Headers */ = {isa = PBXBuildFile; fileRef = 275CED511D3EF7BE001DE46C /* FleeceException.hh */; };
27D7218C1F8E8EEA00AA4458 /* SharedKeys.hh in Headers */ = {isa = PBXBuildFile; fileRef = 27E3DD411DB6A14200F2872D /* SharedKeys.hh */; };
27D721961F8E900400AA4458 /* libFleeceMutableObjC.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 27D721901F8E8EEA00AA4458 /* libFleeceMutableObjC.a */; };
27D965682339595700F4A51C /* NumConversion.hh in Headers */ = {isa = PBXBuildFile; fileRef = 27D965662339595700F4A51C /* NumConversion.hh */; };
27D965692339595700F4A51C /* NumConversion.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27D965672339595700F4A51C /* NumConversion.cc */; };
27D9656D23397EF700F4A51C /* SwiftDtoa.h in Headers */ = {isa = PBXBuildFile; fileRef = 27D9656B23397EF700F4A51C /* SwiftDtoa.h */; };
27D9656E23397EF700F4A51C /* SwiftDtoa.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27D9656C23397EF700F4A51C /* SwiftDtoa.cc */; settings = {COMPILER_FLAGS = "-Wno-implicit-int-conversion -Wno-shadow -Wno-shorten-64-to-32"; }; };
27DE2E922125FA1700123597 /* varint.cc in Sources */ = {isa = PBXBuildFile; fileRef = 270FA2761BF53CEA005DCB13 /* varint.cc */; };
27DE2E962125FA1700123597 /* RefCounted.cc in Sources */ = {isa = PBXBuildFile; fileRef = 274D8254209D1764008BB39F /* RefCounted.cc */; };
27DE2E9D2125FA1700123597 /* slice+CoreFoundation.cc in Sources */ = {isa = PBXBuildFile; fileRef = 272E5A601BF91F6C00848580 /* slice+CoreFoundation.cc */; };
Expand Down Expand Up @@ -424,6 +428,10 @@
27D721241F8C4B7500AA4458 /* MDictIterator.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = MDictIterator.hh; sourceTree = "<group>"; };
27D721501F8D8F3F00AA4458 /* MContext.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = MContext.hh; sourceTree = "<group>"; };
27D721901F8E8EEA00AA4458 /* libFleeceMutableObjC.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libFleeceMutableObjC.a; sourceTree = BUILT_PRODUCTS_DIR; };
27D965662339595700F4A51C /* NumConversion.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = NumConversion.hh; sourceTree = "<group>"; };
27D965672339595700F4A51C /* NumConversion.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = NumConversion.cc; sourceTree = "<group>"; };
27D9656B23397EF700F4A51C /* SwiftDtoa.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SwiftDtoa.h; sourceTree = "<group>"; };
27D9656C23397EF700F4A51C /* SwiftDtoa.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = SwiftDtoa.cc; sourceTree = "<group>"; };
27DE2EDF2125FA1700123597 /* libfleeceBase.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libfleeceBase.a; sourceTree = BUILT_PRODUCTS_DIR; };
27DFAE10219F83AB00DF57EB /* InstanceCounted.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = InstanceCounted.hh; sourceTree = "<group>"; };
27DFAE11219F83AB00DF57EB /* InstanceCounted.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = InstanceCounted.cc; sourceTree = "<group>"; };
Expand Down Expand Up @@ -584,6 +592,7 @@
27298E3E1C00F8A9000CFBA8 /* vendor */ = {
isa = PBXGroup;
children = (
27D9656A23397EBD00F4A51C /* SwiftDtoa */,
27E3DD461DB6B86000F2872D /* catch */,
27298E3F1C00F8A9000CFBA8 /* jsonsl */,
277015321D596436008BADD7 /* libb64 */,
Expand Down Expand Up @@ -739,6 +748,8 @@
276D15451E007D3000543B1B /* JSON5.hh */,
27FE87F11E53E43200C5CF3F /* JSONEncoder.cc */,
27FE87F21E53E43200C5CF3F /* JSONEncoder.hh */,
27D965662339595700F4A51C /* NumConversion.hh */,
27D965672339595700F4A51C /* NumConversion.cc */,
2700BB9B217E8C0C00797537 /* ParseDate.cc */,
2700BB9A217E8C0C00797537 /* ParseDate.hh */,
271507ED2122381E005FE6E8 /* PlatformCompat.hh */,
Expand Down Expand Up @@ -822,6 +833,15 @@
path = ObjC;
sourceTree = "<group>";
};
27D9656A23397EBD00F4A51C /* SwiftDtoa */ = {
isa = PBXGroup;
children = (
27D9656B23397EF700F4A51C /* SwiftDtoa.h */,
27D9656C23397EF700F4A51C /* SwiftDtoa.cc */,
);
path = SwiftDtoa;
sourceTree = "<group>";
};
27DE2EE22125FAC600123597 /* Frameworks */ = {
isa = PBXGroup;
children = (
Expand Down Expand Up @@ -966,6 +986,7 @@
27DE2EBD2125FA1700123597 /* Array.hh in Headers */,
27DE2EBE2125FA1700123597 /* DeepIterator.hh in Headers */,
27DE2EBF2125FA1700123597 /* MArray.hh in Headers */,
27D9656D23397EF700F4A51C /* SwiftDtoa.h in Headers */,
27DE2EC02125FA1700123597 /* Dict.hh in Headers */,
27DE2EC12125FA1700123597 /* StringTable.hh in Headers */,
27DE2EC22125FA1700123597 /* JSONEncoder.hh in Headers */,
Expand Down Expand Up @@ -994,6 +1015,7 @@
27C59BA4212F898F003C8492 /* SmallVector.hh in Headers */,
27DE2ED72125FA1700123597 /* Path.hh in Headers */,
27DE2ED82125FA1700123597 /* HeapDict.hh in Headers */,
27D965682339595700F4A51C /* NumConversion.hh in Headers */,
27DE2ED92125FA1700123597 /* HeapValue.hh in Headers */,
27DE2EDA2125FA1700123597 /* FleeceException.hh in Headers */,
27DE2EDB2125FA1700123597 /* SharedKeys.hh in Headers */,
Expand Down Expand Up @@ -1292,11 +1314,13 @@
27DE2E9E2125FA1700123597 /* slice.cc in Sources */,
27DFAE13219F83AB00DF57EB /* InstanceCounted.cc in Sources */,
27DE2E9D2125FA1700123597 /* slice+CoreFoundation.cc in Sources */,
27D965692339595700F4A51C /* NumConversion.cc in Sources */,
27DE2EEA2125FC5A00123597 /* Writer.cc in Sources */,
27DE2E922125FA1700123597 /* varint.cc in Sources */,
27DE2EE82125FB2100123597 /* cdecode.c in Sources */,
27DE2EE92125FB2100123597 /* cencode.c in Sources */,
2700BB9D217E8C0D00797537 /* ParseDate.cc in Sources */,
27D9656E23397EF700F4A51C /* SwiftDtoa.cc in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down
5 changes: 2 additions & 3 deletions Fleece/Core/JSONConverter.cc
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
//

#include "JSONConverter.hh"
#include "NumConversion.hh"
#include "jsonsl.h"
#include <map>

Expand Down Expand Up @@ -114,9 +115,7 @@ namespace fleece { namespace impl {
unsigned f = state->special_flags;
if (f & JSONSL_SPECIALf_FLOAT) {
char *start = (char*)&_input[state->pos_begin];
char *end;
double n = ::strtod(start, &end);
_encoder.writeDouble(n);
_encoder.writeDouble(ParseDouble(start));
} else if (f & JSONSL_SPECIALf_UNSIGNED) {
_encoder.writeUInt(state->nelem);
} else if (f & JSONSL_SPECIALf_SIGNED) {
Expand Down
18 changes: 13 additions & 5 deletions Fleece/Support/JSONEncoder.hh
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include "Writer.hh"
#include "Value.hh"
#include "FleeceException.hh"
#include "NumConversion.hh"
#include <stdio.h>


Expand Down Expand Up @@ -51,10 +52,10 @@ namespace fleece { namespace impl {
void writeNull() {comma(); _out << slice("null");}
void writeBool(bool b) {comma(); _out.write(b ? "true"_sl : "false"_sl);}

void writeInt(int64_t i) {writef("%lld", i);}
void writeUInt(uint64_t i) {writef("%llu", i);}
void writeFloat(float f) {writef("%.6g", f);}
void writeDouble(double d) {writef("%.16g", d);}
void writeInt(int64_t i) {_writeInt("%lld", i);}
void writeUInt(uint64_t i) {_writeInt("%llu", i);}
void writeFloat(float f) {_writeFloat(f);}
void writeDouble(double d) {_writeFloat(d);}

void writeString(const std::string &s) {writeString(slice(s));}
void writeString(slice s);
Expand Down Expand Up @@ -119,12 +120,19 @@ namespace fleece { namespace impl {
}

template <class T>
void writef(const char *fmt, T t) {
void _writeInt(const char *fmt, T t) {
comma();
char str[32];
_out.write(str, sprintf(str, fmt, t));
}

template <class T>
void _writeFloat(T t) {
comma();
char str[32];
_out.write(str, WriteFloat(t, str, sizeof(str)));
}

Writer _out;
bool _json5 {false};
bool _canonical {false};
Expand Down
52 changes: 52 additions & 0 deletions Fleece/Support/NumConversion.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
//
// NumConversion.cc
//
// Copyright © 2019 Couchbase. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

#include "NumConversion.hh"
#include "SwiftDtoa.h"
#include <locale.h>
#include <stdlib.h>
#ifndef _MSC_VER
#include <xlocale.h>
#endif

namespace fleece {

double ParseDouble(const char *str) noexcept {
// strtod is locale-aware, so in some locales it will not interpret '.' as a decimal point.
// To work around that, use the C locale explicitly.
#ifdef LC_C_LOCALE // Apple & BSD
return strtod_l(str, nullptr, LC_C_LOCALE);
#elif defined(_MSC_VER) // Windows
static _locale_t kCLocale = _create_locale(LC_ALL, "C");
return _strtod_l(str, nullptr, kCLocale);
#else // Linux
static locale_t kCLocale = newlocale(LC_ALL_MASK, NULL, NULL);
return strtod_l(str, nullptr, kCLocale);
#endif
}


size_t WriteFloat(float n, char *dst, size_t capacity) {
return swift_format_float(n, dst, capacity);
}


size_t WriteFloat(double n, char *dst, size_t capacity) {
return swift_format_double(n, dst, capacity);
}
}
26 changes: 26 additions & 0 deletions Fleece/Support/NumConversion.hh
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
//
// NumConversion.hh
//
// Copyright © 2019 Couchbase. All rights reserved.
//

#pragma once
#include "PlatformCompat.hh"
#include <stddef.h>

namespace fleece {

/// Parse `str` as a floating-point number, reading as many digits as possible.
/// (I.e. non-numeric characters after the digits are not treated as an error.)
double ParseDouble(const char *str NONNULL) noexcept;

/// Format a 64-bit-floating point number to a string.
size_t WriteFloat(double n, char *dst, size_t capacity);

/// Format a 32-bit floating-point number to a string.
size_t WriteFloat(float n, char *dst, size_t capacity);

/// Alternative syntax for formatting a 64-bit-floating point number to a string.
static inline size_t WriteDouble(double n, char *dst, size_t c) {return WriteFloat(n, dst, c);}

}
2 changes: 1 addition & 1 deletion Tests/MutableTests.cc
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ namespace fleece {
CHECK(!i);
}

CHECK(ma->asArray()->toJSON() == "[null,false,true,0,-123,2017,123456789,-123456789,\"Hot dog\",3.14159,3.141592653589793]"_sl);
CHECK(ma->asArray()->toJSON() == "[null,false,true,0,-123,2017,123456789,-123456789,\"Hot dog\",3.1415927,3.141592653589793]"_sl);

ma->remove(3, 5);
CHECK(ma->count() == 6);
Expand Down
2 changes: 1 addition & 1 deletion Tests/ObjCTests.mm
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ static void checkIt(id obj, const char* json) {
TEST_CASE("Obj-C Floats", "[Encoder]") {
checkIt(@0.5, "0.5");
checkIt(@-0.5, "-0.5");
checkIt(@((float)M_PI), "3.14159");
checkIt(@((float)M_PI), "3.1415927");
checkIt(@((double)M_PI), "3.141592653589793");
}

Expand Down
2 changes: 2 additions & 0 deletions cmake/platform_base.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ function(set_source_files_base)
Fleece/Support/FileUtils.cc
Fleece/Support/FleeceException.cc
Fleece/Support/InstanceCounted.cc
Fleece/Support/NumConversion.cc
Fleece/Support/JSON5.cc
Fleece/Support/JSONEncoder.cc
Fleece/Support/LibC++Debug.cc
Expand All @@ -47,6 +48,7 @@ function(set_source_files_base)
vendor/jsonsl/jsonsl.c
vendor/libb64/cdecode.c
vendor/libb64/cencode.c
vendor/SwiftDtoa/SwiftDtoa.cc
PARENT_SCOPE
)
endfunction()
Expand Down
Loading

0 comments on commit 2d4ff43

Please sign in to comment.