From b669834f8d5b015d67bb962c3d4287c49ca10257 Mon Sep 17 00:00:00 2001 From: Andrew Fitzgibbon Date: Thu, 6 Jun 2024 13:54:08 +0100 Subject: [PATCH] Fix for K=64 --- src/gfloat/round.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/gfloat/round.py b/src/gfloat/round.py index 3f5b78b..b31f32c 100644 --- a/src/gfloat/round.py +++ b/src/gfloat/round.py @@ -62,8 +62,6 @@ def round_float( # Extract exponent expval = int(np.floor(np.log2(vpos))) - assert expval > -1024 + p # not yet tested for float64 near-subnormals - # Effective precision, accounting for right shift for subnormal values if fi.has_subnormals: expval = max(expval, 1 - bias) @@ -71,7 +69,8 @@ def round_float( # Lift to "integer * 2^e" expval = expval - p + 1 - fsignificand = vpos * 2.0**-expval + # use ldexp instead of vpos*2**-expval to avoid overflow + fsignificand = math.ldexp(vpos, -expval) # Round isignificand = math.floor(fsignificand)