From 83016d27042d8d2bc96a97c2d8ff328833b18d70 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Wed, 11 Sep 2024 11:02:51 -0700 Subject: [PATCH 1/2] Convert fill values correctly from negative to unsigned integers Positive integers are unaffected by this change anyway. --- src/metadata.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/metadata.jl b/src/metadata.jl index f3dc5df..d6409f0 100644 --- a/src/metadata.jl +++ b/src/metadata.jl @@ -218,3 +218,9 @@ fill_value_decoding(v::Nothing, ::Any) = v fill_value_decoding(v, T) = T(v) fill_value_decoding(v::Number, T::Type{String}) = v == 0 ? "" : T(UInt8[v]) fill_value_decoding(v, ::Type{ASCIIChar}) = v == "" ? nothing : v +# Sometimes when translating between CF (climate and forecast) convention data +# and Zarr groups, fill values are left as "negative integers" to encode unsigned +# integers. So, we have to convert to the signed type with the same number of bytes +# as the unsigned integer, then reinterpret as unsigned. That's how a fill value +# of -1 can have a realistic meaning with an unsigned dtype. +fill_value_decoding(v::Integer, T::Type{<: Unsigned}) = reinterpret(T, signed(T)(v)) From f931176657eb0858338d9c0a1d619a1a619a0663 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 21 Oct 2024 14:09:47 -0700 Subject: [PATCH 2/2] Only reinterpret negative integers when decoding fill values to unsigned --- src/metadata.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/metadata.jl b/src/metadata.jl index d6409f0..cc3e48d 100644 --- a/src/metadata.jl +++ b/src/metadata.jl @@ -216,6 +216,7 @@ Base.eltype(::Metadata{T}) where T = T fill_value_decoding(v::AbstractString, T::Type{<:Number}) = parse(T, v) fill_value_decoding(v::Nothing, ::Any) = v fill_value_decoding(v, T) = T(v) + fill_value_decoding(v::Number, T::Type{String}) = v == 0 ? "" : T(UInt8[v]) fill_value_decoding(v, ::Type{ASCIIChar}) = v == "" ? nothing : v # Sometimes when translating between CF (climate and forecast) convention data @@ -223,4 +224,6 @@ fill_value_decoding(v, ::Type{ASCIIChar}) = v == "" ? nothing : v # integers. So, we have to convert to the signed type with the same number of bytes # as the unsigned integer, then reinterpret as unsigned. That's how a fill value # of -1 can have a realistic meaning with an unsigned dtype. -fill_value_decoding(v::Integer, T::Type{<: Unsigned}) = reinterpret(T, signed(T)(v)) +# However, we have to apply this correction only if the integer is negative. +# If it's positive, then the value might be out of range of the signed integer type. +fill_value_decoding(v::Integer, T::Type{<: Unsigned}) = sign(v) < 0 ? reinterpret(T, signed(T)(v)) : T(v)