Skip to content

Commit

Permalink
Set StreamReadConstraints maxStringLength as early as possible (#355)
Browse files Browse the repository at this point in the history
Leave the override in Backend as well to avoid duplication. Future
enhancements may enable us to construct a ServiceBackend from argv
alone, allowing this to be reverted.

Add test.

Co-authored-by: Christopher Vittal <[email protected]>
  • Loading branch information
jmarshall and chrisvittal authored Dec 15, 2024
1 parent a1a628b commit a8be268
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 0 deletions.
10 changes: 10 additions & 0 deletions hail/python/test/hail/test_ir.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import functools
import random
import re
import unittest
from test.hail.helpers import resource, skip_unless_spark_backend, skip_when_service_backend
Expand Down Expand Up @@ -774,3 +775,12 @@ def test_locus_interval_encoding():
_assert_encoding_roundtrip(start)
_assert_encoding_roundtrip(end)
_assert_encoding_roundtrip(interval)


def test_very_large_ir_deserializes():
# see: https://github.com/hail-is/hail/issues/14580
# https://github.com/hail-is/hail/issues/14650
large_list = [random.getrandbits(63) for _ in range(5_000_000)]
large_lit = hl.literal(large_list, hl.tarray(hl.tint64))
round_trip = hl.eval(large_lit)
assert large_list == round_trip
10 changes: 10 additions & 0 deletions hail/src/main/scala/is/hail/backend/service/Main.scala
Original file line number Diff line number Diff line change
@@ -1,9 +1,19 @@
package is.hail.backend.service

import com.fasterxml.jackson.core.StreamReadConstraints

object Main {
val WORKER = "worker"
val DRIVER = "driver"

/* This constraint should be overridden as early as possible. See:
* - https://github.com/hail-is/hail/issues/14580
* - https://github.com/hail-is/hail/issues/14749
* - The note on this setting in is.hail.backend.Backend */
StreamReadConstraints.overrideDefaultStreamReadConstraints(
StreamReadConstraints.builder().maxStringLength(Integer.MAX_VALUE).build()
)

def main(argv: Array[String]): Unit =
argv(3) match {
case WORKER => Worker.main(argv)
Expand Down

0 comments on commit a8be268

Please sign in to comment.