Support for huge memory units

config/src/main/java/com/typesafe/config/ConfigMemorySize.java

@@ -3,18 +3,21 @@
  */
 package com.typesafe.config;
 
+import java.math.BigInteger;
+
 /**
  * An immutable class representing an amount of memory.  Use
  * static factory methods such as {@link
- * ConfigMemorySize#ofBytes(long)} to create instances.
+ * ConfigMemorySize#ofBytes(BigInteger)} to create instances.
  *
  * @since 1.3.0
  */
 public final class ConfigMemorySize {
-    private final long bytes;
 
-    private ConfigMemorySize(long bytes) {
-        if (bytes < 0)
+    private BigInteger bytes;
+
+    private ConfigMemorySize(BigInteger bytes) {
+        if (bytes.signum() < 0)
             throw new IllegalArgumentException("Attempt to construct ConfigMemorySize with negative number: " + bytes);
         this.bytes = bytes;
     }
@@ -26,16 +29,46 @@ private ConfigMemorySize(long bytes) {
      * @param bytes a number of bytes
      * @return an instance representing the number of bytes
      */
-    public static ConfigMemorySize ofBytes(long bytes) {
+    public static ConfigMemorySize ofBytes(BigInteger bytes) {
         return new ConfigMemorySize(bytes);
     }
 
+    /**
+     * Constructs a ConfigMemorySize representing the given
+     * number of bytes.
+     * @param bytes a number of bytes
+     * @return an instance representing the number of bytes
+     */
+    public static ConfigMemorySize ofBytes(long bytes) {
+        return new ConfigMemorySize(BigInteger.valueOf(bytes));
+    }
+
     /**
      * Gets the size in bytes.
+     *
      * @since 1.3.0
      * @return how many bytes
+     * @exception IllegalArgumentException when memory value
+     * in bytes doesn't fit in a long value. Consider using
+     * {@link #toBytesBigInteger} in this case.
      */
     public long toBytes() {
+        if (bytes.bitLength() < 64)
+            return bytes.longValue();
+        else
+            throw new IllegalArgumentException(
+                "size-in-bytes value is out of range for a 64-bit long: '" + bytes + "'");
+    }
+
+    /**
+     * Gets the size in bytes. The behavior of this method
+     * is the same as that of the {@link #toBytes()} method,
+     * except that the number of bytes returned as a
+     * BigInteger value. Use it when memory value in bytes
+     * doesn't fit in a long value.
+     * @return how many bytes
+     */
+    public BigInteger toBytesBigInteger() {
         return bytes;
     }
 
@@ -47,7 +80,7 @@ public String toString() {
     @Override
     public boolean equals(Object other) {
         if (other instanceof ConfigMemorySize) {
-            return ((ConfigMemorySize)other).bytes == this.bytes;
+            return ((ConfigMemorySize)other).bytes.equals(this.bytes);
         } else {
             return false;
         }
@@ -56,7 +89,7 @@ public boolean equals(Object other) {
     @Override
     public int hashCode() {
         // in Java 8 this can become Long.hashCode(bytes)
-        return Long.valueOf(bytes).hashCode();
+        return bytes.hashCode();
     }
 
 }

config/src/main/java/com/typesafe/config/impl/SimpleConfig.java

@@ -20,6 +20,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigException;
@@ -282,20 +283,54 @@ public Object getAnyRef(String path) {
 
     @Override
     public Long getBytes(String path) {
-        Long size = null;
+        BigInteger bytes = getBytesBigInteger(path);
+        ConfigValue v = find(path, ConfigValueType.STRING);
+        return toLong(bytes, v.origin(), path);
+    }
+
+    private BigInteger getBytesBigInteger(String path) {
+        BigInteger bytes;
+        ConfigValue v = find(path, ConfigValueType.STRING);
         try {
-            size = getLong(path);
+            bytes = BigInteger.valueOf(getLong(path));
         } catch (ConfigException.WrongType e) {
-            ConfigValue v = find(path, ConfigValueType.STRING);
-            size = parseBytes((String) v.unwrapped(),
-                    v.origin(), path);
+            bytes = parseBytes((String) v.unwrapped(),
+                v.origin(), path);
+        }
+        if (bytes.signum() < 0)
+            throw new ConfigException.BadValue(v.origin(), path,
+                "Attempt to construct memory size with negative number: " + bytes);
+        return bytes;
+    }
+
+    private List<BigInteger> getBytesListBigInteger(String path){
+        List<BigInteger> result = new ArrayList<>();
+        List<? extends ConfigValue> list = getList(path);
+
+        for (ConfigValue v : list) {
+            BigInteger bytes;
+            if (v.valueType() == ConfigValueType.NUMBER) {
+                bytes = BigInteger.valueOf(((Number) v.unwrapped()).longValue());
+            } else if (v.valueType() == ConfigValueType.STRING) {
+                String s = (String) v.unwrapped();
+                bytes = parseBytes(s, v.origin(), path);
+            } else {
+                throw new ConfigException.WrongType(v.origin(), path,
+                    "memory size string or number of bytes", v.valueType()
+                    .name());
+            }
+            if (bytes.signum() < 0)
+                throw new ConfigException.BadValue(v.origin(), path,
+                    "Attempt to construct ConfigMemorySize with negative number: " + bytes);
+
+            result.add(bytes);
         }
-        return size;
+        return result;
     }
 
     @Override
     public ConfigMemorySize getMemorySize(String path) {
-        return ConfigMemorySize.ofBytes(getBytes(path));
+        return ConfigMemorySize.ofBytes(getBytesBigInteger(path));
     }
 
     @Deprecated
@@ -482,32 +517,27 @@ public List<? extends Object> getAnyRefList(String path) {
 
     @Override
     public List<Long> getBytesList(String path) {
-        List<Long> l = new ArrayList<Long>();
-        List<? extends ConfigValue> list = getList(path);
-        for (ConfigValue v : list) {
-            if (v.valueType() == ConfigValueType.NUMBER) {
-                l.add(((Number) v.unwrapped()).longValue());
-            } else if (v.valueType() == ConfigValueType.STRING) {
-                String s = (String) v.unwrapped();
-                Long n = parseBytes(s, v.origin(), path);
-                l.add(n);
-            } else {
-                throw new ConfigException.WrongType(v.origin(), path,
-                        "memory size string or number of bytes", v.valueType()
-                                .name());
-            }
+        ConfigValue v = find(path, ConfigValueType.LIST);
+        return getBytesListBigInteger(path).stream()
+            .map(bytes -> toLong(bytes, v.origin(), path))
+            .collect(Collectors.toList());
+    }
+
+    private Long toLong(BigInteger value, ConfigOrigin originForException,
+        String pathForException){
+        if (value.bitLength() < 64) {
+            return value.longValue();
+        } else {
+            throw new ConfigException.BadValue(originForException, pathForException,
+                "size-in-bytes value is out of range for a 64-bit long: '" + value + "'");
         }
-        return l;
     }
 
     @Override
     public List<ConfigMemorySize> getMemorySizeList(String path) {
-        List<Long> list = getBytesList(path);
-        List<ConfigMemorySize> builder = new ArrayList<ConfigMemorySize>();
-        for (Long v : list) {
-            builder.add(ConfigMemorySize.ofBytes(v));
-        }
-        return builder;
+        return getBytesListBigInteger(path).stream()
+            .map(ConfigMemorySize::ofBytes)
+            .collect(Collectors.toList());
     }
 
     @Override
@@ -848,12 +878,12 @@ static MemoryUnit parseUnit(String unit) {
      * @throws ConfigException
      *             if string is invalid
      */
-    public static long parseBytes(String input, ConfigOrigin originForException,
-            String pathForException) {
+    public static BigInteger parseBytes(String input, ConfigOrigin originForException,
+        String pathForException) {
         String s = ConfigImplUtil.unicodeTrim(input);
         String unitString = getUnits(s);
         String numberString = ConfigImplUtil.unicodeTrim(s.substring(0,
-                s.length() - unitString.length()));
+            s.length() - unitString.length()));
 
         // this would be caught later anyway, but the error message
         // is more helpful if we check it here.
@@ -880,11 +910,7 @@ public static long parseBytes(String input, ConfigOrigin originForException,
                 BigDecimal resultDecimal = (new BigDecimal(units.bytes)).multiply(new BigDecimal(numberString));
                 result = resultDecimal.toBigInteger();
             }
-            if (result.bitLength() < 64)
-                return result.longValue();
-            else
-                throw new ConfigException.BadValue(originForException, pathForException,
-                        "size-in-bytes value is out of range for a 64-bit long: '" + input + "'");
+            return result;
         } catch (NumberFormatException e) {
             throw new ConfigException.BadValue(originForException, pathForException,
                     "Could not parse size-in-bytes number '" + numberString + "'");

config/src/test/resources/test01.conf

@@ -78,6 +78,8 @@
         "megsList" : [1M, 1024K, 1048576],
         "megAsNumber" : 1048576,
         "halfMeg" : 0.5M
+        "yottabyte" : 1YB
+        "yottabyteList" : [1YB, 0.5YB]
     },
 
     "system" : {

config/src/test/scala/com/typesafe/config/impl/ConfigMemorySizeTest.scala

@@ -3,6 +3,8 @@
  */
 package com.typesafe.config.impl
 
+import java.math.BigInteger
+
 import org.junit.Assert._
 import org.junit._
 import com.typesafe.config.ConfigMemorySize
@@ -22,4 +24,10 @@ class ConfigMemorySizeTest extends TestUtils {
         val kilobyte = ConfigMemorySize.ofBytes(1024)
         assertEquals(1024, kilobyte.toBytes)
     }
+
+    @Test
+    def testGetBytes() {
+        val yottabyte = ConfigMemorySize.ofBytes(new BigInteger("1000000000000000000000000"))
+        assertEquals(new BigInteger("1000000000000000000000000"), yottabyte.toBytesBigInteger)
+    }
 }

config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala

@@ -3,6 +3,7 @@
  */
 package com.typesafe.config.impl
 
+import java.math.BigInteger
 import java.time.temporal.{ ChronoUnit, TemporalUnit }
 
 import org.junit.Assert._
@@ -831,6 +832,10 @@ class ConfigTest extends TestUtils {
         assertEquals(Seq(1024 * 1024L, 1024 * 1024L, 1024L * 1024L),
             conf.getMemorySizeList("memsizes.megsList").asScala.map(_.toBytes))
         assertEquals(512 * 1024L, conf.getMemorySize("memsizes.halfMeg").toBytes)
+
+        assertEquals(new BigInteger("1000000000000000000000000"), conf.getMemorySize("memsizes.yottabyte").toBytesBigInteger)
+        assertEquals(Seq(new BigInteger("1000000000000000000000000"), new BigInteger("500000000000000000000000")),
+            conf.getMemorySizeList("memsizes.yottabyteList").asScala.map(_.toBytesBigInteger))
     }
 
     @Test

config/src/test/scala/com/typesafe/config/impl/UnitParserTest.scala

@@ -3,6 +3,7 @@
  */
 package com.typesafe.config.impl
 
+import java.math.BigInteger
 import java.time.{ LocalDate, Period }
 import java.time.temporal.ChronoUnit
 
@@ -89,10 +90,10 @@ class UnitParserTest extends TestUtils {
 
     @Test
     def parseMemorySizeInBytes(): Unit = {
-        def parseMem(s: String): Long = SimpleConfig.parseBytes(s, fakeOrigin(), "test")
+        def parseMem(s: String): BigInteger = SimpleConfig.parseBytes(s, fakeOrigin(), "test")
 
-        assertEquals(Long.MaxValue, parseMem(s"${Long.MaxValue} bytes"))
-        assertEquals(Long.MinValue, parseMem(s"${Long.MinValue} bytes"))
+        assertEquals(BigInteger.valueOf(Long.MaxValue), parseMem(s"${Long.MaxValue} bytes"))
+        assertEquals(BigInteger.valueOf(Long.MinValue), parseMem(s"${Long.MinValue} bytes"))
 
         val oneMebis = List("1048576", "1048576b", "1048576bytes", "1048576byte",
             "1048576  b", "1048576  bytes",
@@ -104,7 +105,7 @@ class UnitParserTest extends TestUtils {
 
         for (s <- oneMebis) {
             val result = parseMem(s)
-            assertEquals(1024 * 1024, result)
+            assertEquals(BigInteger.valueOf(1024 * 1024), result)
         }
 
         val oneMegas = List("1000000", "1000000b", "1000000bytes", "1000000byte",
@@ -117,24 +118,24 @@ class UnitParserTest extends TestUtils {
 
         for (s <- oneMegas) {
             val result = parseMem(s)
-            assertEquals(1000 * 1000, result)
+            assertEquals(BigInteger.valueOf(1000 * 1000), result)
         }
 
-        var result = 1024L * 1024 * 1024
-        for (unit <- Seq("tebi", "pebi", "exbi")) {
+        var result = BigInteger.valueOf(1024L * 1024 * 1024)
+        for (unit <- Seq("tebi", "pebi", "exbi", "zebi", "yobi")) {
             val first = unit.substring(0, 1).toUpperCase()
-            result = result * 1024
+            result = result.multiply(BigInteger.valueOf(1024))
             assertEquals(result, parseMem("1" + first))
             assertEquals(result, parseMem("1" + first + "i"))
             assertEquals(result, parseMem("1" + first + "iB"))
             assertEquals(result, parseMem("1" + unit + "byte"))
             assertEquals(result, parseMem("1" + unit + "bytes"))
         }
 
-        result = 1000L * 1000 * 1000
-        for (unit <- Seq("tera", "peta", "exa")) {
+        result = BigInteger.valueOf(1000L * 1000 * 1000)
+        for (unit <- Seq("tera", "peta", "exa", "zetta", "yotta")) {
             val first = unit.substring(0, 1).toUpperCase()
-            result = result * 1000
+            result = result.multiply(BigInteger.valueOf(1000))
             assertEquals(result, parseMem("1" + first + "B"))
             assertEquals(result, parseMem("1" + unit + "byte"))
             assertEquals(result, parseMem("1" + unit + "bytes"))
@@ -156,19 +157,25 @@ class UnitParserTest extends TestUtils {
     // later on we'll want to check this with BigInteger version of getBytes
     @Test
     def parseHugeMemorySizes(): Unit = {
-        def parseMem(s: String): Long = SimpleConfig.parseBytes(s, fakeOrigin(), "test")
+        def parseMem(s: String): Long = ConfigFactory.parseString(s"v = $s").getBytes("v")
         def assertOutOfRange(s: String): Unit = {
             val fail = intercept[ConfigException.BadValue] {
                 parseMem(s)
             }
             assertTrue("number was too big", fail.getMessage.contains("out of range"))
         }
 
+        def assertNegativeNumber(s: String): Unit = {
+            val fail = intercept[ConfigException.BadValue] {
+                parseMem(s)
+            }
+            assertTrue("number was negative", fail.getMessage.contains("negative number"))
+        }
+
         import java.math.BigInteger
         assertOutOfRange(s"${BigInteger.valueOf(Long.MaxValue).add(BigInteger.valueOf(1)).toString} bytes")
-        assertOutOfRange(s"${BigInteger.valueOf(Long.MinValue).subtract(BigInteger.valueOf(1)).toString} bytes")
+        assertNegativeNumber(s"${BigInteger.valueOf(Long.MinValue).subtract(BigInteger.valueOf(1)).toString} bytes")
 
-        var result = 1024L * 1024 * 1024
         for (unit <- Seq("zebi", "yobi")) {
             val first = unit.substring(0, 1).toUpperCase()
             assertOutOfRange("1" + first)
@@ -177,17 +184,16 @@ class UnitParserTest extends TestUtils {
             assertOutOfRange("1" + unit + "byte")
             assertOutOfRange("1" + unit + "bytes")
             assertOutOfRange("1.1" + first)
-            assertOutOfRange("-1" + first)
+            assertNegativeNumber("-1" + first)
         }
 
-        result = 1000L * 1000 * 1000
         for (unit <- Seq("zetta", "yotta")) {
             val first = unit.substring(0, 1).toUpperCase()
             assertOutOfRange("1" + first + "B")
             assertOutOfRange("1" + unit + "byte")
             assertOutOfRange("1" + unit + "bytes")
             assertOutOfRange("1.1" + first + "B")
-            assertOutOfRange("-1" + first + "B")
+            assertNegativeNumber("-1" + first + "B")
         }
 
         assertOutOfRange("1000 exabytes")