Skip to content

Commit

Permalink
parser: Accept bytes as input
Browse files Browse the repository at this point in the history
In addition to (Unicode) strings, also accept "bytes" (and corresponding
iterators) as input to the parser. This allows skipping the
decode/encode step when reading raw data from a file or socket, e.g.
with os.read(). This introduces small, but measurable performance
increase for such cases.
  • Loading branch information
spbnick committed Sep 21, 2020
1 parent 1cf3225 commit a10ecdd
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 6 deletions.
14 changes: 9 additions & 5 deletions jq.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,11 @@ cdef class _JSONParser(object):
cdef char* cbytes
cdef ssize_t clen
try:
self._bytes = next(self._text_iter).encode("utf8")
text = next(self._text_iter)
if isinstance(text, bytes):
self._bytes = text
else:
self._bytes = text.encode("utf8")
PyBytes_AsStringAndSize(self._bytes, &cbytes, &clen)
jv_parser_set_buf(self._parser, cbytes, clen, 1)
except StopIteration:
Expand Down Expand Up @@ -419,9 +423,10 @@ def parse_json(text=_NO_VALUE, text_iter=_NO_VALUE):
Either "text" or "text_iter" must be specified.
Args:
text: A string containing the JSON stream to parse.
text_iter: An iterator returning strings - pieces of the JSON stream
to parse.
text: A string or bytes object containing the JSON stream to
parse.
text_iter: An iterator returning strings or bytes - pieces of the
JSON stream to parse.
Returns:
An iterator returning parsed values.
Expand All @@ -442,7 +447,6 @@ def parse_json_file(fp):
Args:
fp: The file-like object to read the JSON stream from.
Must be in text mode.
Returns:
An iterator returning parsed values.
Expand Down
4 changes: 3 additions & 1 deletion tests/jq_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,9 +198,11 @@ def program_string_can_be_retrieved_from_program():
assert_equal(".", program.program_string)

@istest
def parse_json_both_text_and_text_iter_accepted():
def parse_json_all_inputs_accepted():
assert_equal(True, next(jq.parse_json(text="true")))
assert_equal(True, next(jq.parse_json(text_iter=iter(["true"]))))
assert_equal(True, next(jq.parse_json(text=b"true")))
assert_equal(True, next(jq.parse_json(text_iter=iter([b"true"]))))

@istest
def parse_json_file_works():
Expand Down

0 comments on commit a10ecdd

Please sign in to comment.