From ac2c3f6fe2131d4f7bff9c6cda0f0746eb6daa03 Mon Sep 17 00:00:00 2001 From: Hobson Lane Date: Sun, 17 Feb 2013 20:09:58 -0800 Subject: [PATCH 01/11] CHANGES added, LICENSE renamed --- CHANGES.txt | 2 ++ LICENSE => LICENSE.txt | 0 2 files changed, 2 insertions(+) create mode 100644 CHANGES.txt rename LICENSE => LICENSE.txt (100%) diff --git a/CHANGES.txt b/CHANGES.txt new file mode 100644 index 0000000..156affd --- /dev/null +++ b/CHANGES.txt @@ -0,0 +1,2 @@ +0.1, 2013-02-13 -- Initial release. + diff --git a/LICENSE b/LICENSE.txt similarity index 100% rename from LICENSE rename to LICENSE.txt From bf6d2007122ab447c53332194611f8cfc8089b16 Mon Sep 17 00:00:00 2001 From: Hobson Lane Date: Sun, 17 Feb 2013 20:16:11 -0800 Subject: [PATCH 02/11] add manifest and docs --- MANIFEST.in | 6 ++++++ docs/README.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 MANIFEST.in create mode 100644 docs/README.md diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..ca80284 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include *.txt +recursive-include *.txt +include *.md +recursive-include *.md +include *.rst +recursive-include *.rst diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..92d4bf9 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,52 @@ +# flojay + +flojay provides incremental serialization of Python data structures as +JSON documents, and an event-based JSON parser to allow for incremtnal +deserialization. It can be used to create and process JSON documents +that are larger than working memory. + +It is based on [yajl](http://lloyd.github.com/yajl/) version 2, a +copy of which is included in this distribution. + +# example + + import flojay + import sys + + # Produce JSON from a generator, printing partial results + # as they become available + + encoder = flojay.JSONEncoder() + json_generator = encoder.iterencode(xrange(100)) + for hunk in json_generator: + print hunk + + + # Read an array of JSON numbers from stdin input, + # summing the values as they are read + + + class ExampleCallbacks(object): + def __init__(self): + self.sum = 0 + + def handle_start_array(self): + pass + + def handle_end_array(self): + pass + + def handle_number(self, value): + self.sum += value + + callbacks = ExampleCallbacks() + + parser = flojay.JSONEventParser(callbacks) + + while 1: + row = sys.stdin.readline() + if len(row) == 0: + break + parser.parse(row) + print "The current total is: %d" % (callbacks.sum,) + From 1f34709a3a301eb49fcbf5460619df9e98b72689 Mon Sep 17 00:00:00 2001 From: Hobson Lane Date: Sun, 17 Feb 2013 21:08:09 -0800 Subject: [PATCH 03/11] new setup --- README.md | 6 +- flojay/__init__.py | 11 +++ setup.py | 44 +++++++---- tests/marshal_tests.py | 164 --------------------------------------- tests/unmarshal_tests.py | 110 -------------------------- 5 files changed, 47 insertions(+), 288 deletions(-) create mode 100644 flojay/__init__.py delete mode 100644 tests/marshal_tests.py delete mode 100644 tests/unmarshal_tests.py diff --git a/README.md b/README.md index 92d4bf9..c299594 100644 --- a/README.md +++ b/README.md @@ -49,4 +49,8 @@ copy of which is included in this distribution. break parser.parse(row) print "The current total is: %d" % (callbacks.sum,) - + + +# Thank also to + +Lloyd Hilaiel -- creator of yajl -- https://github.com/lloyd diff --git a/flojay/__init__.py b/flojay/__init__.py new file mode 100644 index 0000000..de94d9d --- /dev/null +++ b/flojay/__init__.py @@ -0,0 +1,11 @@ +"""Flojay json streaming module. + +For more information, see: http://github.com/myemma/flojay/ +""" + +# the Canonical way ;) +version_info = (0, 1, 0, 'alpha', 0) + +__version__ = '.'.join(str(i) for i in version_info[0:3]) +version_string = __version__ + diff --git a/setup.py b/setup.py index 6b3a7ad..47a6a91 100644 --- a/setup.py +++ b/setup.py @@ -1,27 +1,45 @@ #!/usr/bin/env python from setuptools import setup, Extension +from distutils.core import setup +import os +from fence import __version__ yajl_sources = ['flojay/lloyd-yajl/src/' + file_ for file_ in \ ('yajl.c', 'yajl_gen.c', 'yajl_alloc.c', 'yajl_lex.c', 'yajl_tree.c', \ 'yajl_encode.c', 'yajl_version.c', 'yajl_buf.c', 'yajl_parser.c')] +major_minor = '.'.split(__version__) + flojay = Extension('flojay', define_macros=[ - ('MAJOR_VERSION', '0'), - ('MINOR_VERSION', '1')], + ('MAJOR_VERSION', major_minor[0]), + ('MINOR_VERSION', major_minor[1])], extra_compile_args=['--std=c99'], include_dirs=['flojay/lloyd-yajl/src'], sources=yajl_sources + ['flojay/flojay.c']) -setup(name='flojay', - version='0.1', - description='Streaming or event-based JSON parser based on yajl', - author='Robert Church', - author_email='rchurch@myemma.com', - url='', - ext_modules=[flojay], - packages=['flojay'], - install_requires=[ - 'nose==1.1.2', - ]) +setup( + name='flojay', + version=__version__, + description='Streaming and event-based JSON parser based on yajl', + author='Robert Church', + author_email='rchurch@myemma.com', + url = "http://github/myemma/flojay/", + ext_modules=[flojay], + packages=['flojay'], + install_requires=[ + 'nose==1.1.2', + keywords = ["json", "stream", "ajax", "webapp", "website", "data", "messaging"], + classifiers = [ + "Programming Language :: Python", + "Development Status :: 2 - Pre-Alpha", + "Environment :: Other Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Internet", + ], + ]) + diff --git a/tests/marshal_tests.py b/tests/marshal_tests.py deleted file mode 100644 index 35932aa..0000000 --- a/tests/marshal_tests.py +++ /dev/null @@ -1,164 +0,0 @@ -import flojay -from unittest import TestCase -from nose.tools import eq_, raises - - -class EventBasedParserTests(TestCase): - - def test_number_callback(self): - callbacks = UnaryCallbacks() - p = flojay.JSONEventParser(callbacks) - long_int = 1 + (2 ** 32) - p.parse("[") - p.parse("123,") - p.parse("1.91") - p.parse("," + str(long_int)) - p.parse(",-1, 1e27]") - eq_(callbacks.buf, [123, 1.91, long_int, -1, 1e+27]) - - def test_null_callback(self): - callbacks = UnaryCallbacks() - p = flojay.JSONEventParser(callbacks) - p.parse("[null]") - eq_(callbacks.buf, [None]) - - def test_partial_nulls(self): - callbacks = UnaryCallbacks() - p = flojay.JSONEventParser(callbacks) - p.parse("[nu") - p.parse("ll]") - eq_(callbacks.buf, [None]) - - def test_boolean_callback(self): - callbacks = UnaryCallbacks() - p = flojay.JSONEventParser(callbacks) - p.parse("[true, false, true]") - eq_(callbacks.buf, [True, False, True]) - - def test_string_callback(self): - callbacks = UnaryCallbacks() - p = flojay.JSONEventParser(callbacks) - p.parse(u'["String", "Ren\xc3e"]') - eq_(callbacks.buf, [u"String", u"Ren\xc3e"]) - - def test_start_end_map(self): - callbacks = MapCallbacks() - p = flojay.JSONEventParser(callbacks) - p.parse(u'{}') - eq_(callbacks.buf, ['{', '}']) - - def test_map_key(self): - callbacks = MapCallbacks() - p = flojay.JSONEventParser(callbacks) - p.parse(u'{"Ren\xc3e": 119}') - eq_(callbacks.buf, ['{', u"Ren\xc3e", 119, '}']) - - def test_array_start_end(self): - callbacks = ArrayCallbacks() - p = flojay.JSONEventParser(callbacks) - p.parse(u'[[], []]') - eq_(callbacks.buf, ['[', '[', ']', '[', ']', ']']) - - @raises(AttributeError) - def test_raises_exception_on_undefined_handlers(self): - p = flojay.JSONEventParser(self) - p.parse('[]') - - @raises(ZeroDivisionError) - def test_exceptions_from_callbacks_are_propogated(self): - flojay.JSONEventParser(CallbacksRaiseAnException()).\ - parse('[]') - - @raises(ValueError) - def test_syntax_error_bad_map(self): - flojay.JSONEventParser(NullCallbacks()).\ - parse('{[]: "a"}') - - @raises(ValueError) - def test_syntax_error_missing_delimiter(self): - flojay.JSONEventParser(NullCallbacks()).\ - parse('["1" "2" "3"]') - - @raises(ValueError) - def test_syntax_error_mismatches(self): - flojay.JSONEventParser(NullCallbacks()).\ - parse('[[}]') - - -class UnaryCallbacks(object): - def __init__(self): - self.buf = [] - - def handle_start_array(self): - pass - - def handle_end_array(self): - pass - - def handle_number(self, number): - self.buf.append(number) - - def handle_null(self): - self.buf.append(None) - - def handle_boolean(self, value): - self.buf.append(value) - - def handle_string(self, string): - self.buf.append(string) - - -class MapCallbacks(object): - def __init__(self): - self.buf = [] - - def handle_number(self, number): - self.buf.append(number) - - def handle_map_key(self, key): - self.buf.append(key) - - def handle_start_map(self): - self.buf.append('{') - - def handle_end_map(self): - self.buf.append('}') - - -class ArrayCallbacks(object): - def __init__(self): - self.buf = [] - - def handle_start_array(self): - self.buf.append('[') - - def handle_end_array(self): - self.buf.append(']') - - -class CallbacksRaiseAnException(object): - def handle_end_array(self): - return 1 / 0 - - def handle_start_array(self): - return 1 / 0 - - -class NullCallbacks(object): - def handle_start_map(self): - pass - - def handle_end_map(self): - pass - - def handle_map_key(self, key): - pass - - def handle_string(self, blah): - pass - - def handle_end_array(self): - pass - - def handle_start_array(self): - pass diff --git a/tests/unmarshal_tests.py b/tests/unmarshal_tests.py deleted file mode 100644 index 9629cbd..0000000 --- a/tests/unmarshal_tests.py +++ /dev/null @@ -1,110 +0,0 @@ -import flojay -from unittest import TestCase -from nose.tools import eq_, raises - - -class UnmarshalTests(TestCase): - def encode(self, it): - return flojay.JSONEncoder().iterencode(it) - - def test_unmarshal(self): - encoder = flojay.JSONEncoder(buffer_size=1) - gen = encoder.iterencode(["a"]) - eq_(gen.next(), '["a"') - eq_(gen.next(), ']') - - def test_unary(self): - eq_(''.join(self.encode('this is just a flat string')), - '"this is just a flat string"') - - eq_(''.join(self.encode(1)), "1") - - def test_unmarshal_nested_array(self): - gen = self.encode(['a', ['b', 1.0, ['d']]]) - eq_(''.join(gen), '["a",["b",1.0,["d"]]]') - eq_(''.join(self.encode([])), '[]') - - def test_dict(self): - eq_(''.join(self.encode({})), '{}') - gen = self.encode({u'key': 3.14}) - eq_(''.join(gen), '{"key":3.14}') - gen = self.encode({'a': 1, 'b': 2}) - eq_(''.join(gen), '{"a":1,"b":2}') - - def test_true_false_null(self): - eq_(''.join(self.encode([True, False, None])), \ - '[true,false,null]') - - def test_longs(self): - result = ''.join(self.encode([100L])) - eq_(result, '[100]') - - def test_unicode(self): - result = ''.join(self.encode([u'test'])) - eq_(result, '["test"]') - eq_(result, str(result)) - - def test_generator(self): - def empty(): - if False is True: - yield "Impossible!" - - def generator(): - yield 1 - yield 2 - yield ['a', 'b', 'c'] - - def generator_with_dict(): - yield {'a': 1, 'b': 2} - - eq_(''.join(self.encode([generator(), 3])), - '[[1,2,["a","b","c"]],3]') - eq_(''.join(self.encode(empty())), '[]') - eq_(''.join(self.encode(generator_with_dict())), '[{"a":1,"b":2}]') - - def test_custom_object_handlerer(self): - import datetime - - def handle_custom_json(obj): - if isinstance(obj, datetime.datetime): - return obj.strftime('@D:%Y-%m-%d') - else: - raise TypeError("Can't encode this " + repr(obj)) - - encoder = flojay.JSONEncoder(default=handle_custom_json) - - eq_(''.join(encoder.iterencode( - ['date', datetime.datetime(2012, 3, 17)])), - '["date","@D:2012-03-17"]') - - @raises(TypeError) - def test_unknown_object_type(self): - ''.join(self.encode({'self': self})) - - def test_string_escape(self): - s = '"A Good Man Is Hard To Find" by Flannery O\'Connor' - - eq_(''.join(self.encode([s])), - '["\\"A Good Man Is Hard To Find\\" by Flannery O\'Connor"]') - - s = 'C:\DOS>' - eq_(''.join(self.encode([s])), - '["C:\\\\DOS>"]') - - s = "I have eaten\nthe plums\nthat were in\n..." - eq_(''.join(self.encode([s])), - '["I have eaten\\nthe plums\\nthat were in\\n..."]') - - def test_utf8(self): - eq_( - ''.join(flojay.JSONEncoder().\ - iterencode([u'Ren\xc3e'])), - '["' + u'Ren\xc3e'.encode('utf8') + '"]' - ) - - def test_beautify(self): - beauty = ''.join(flojay.JSONEncoder(beautify=True, indent_string=' ').\ - iterencode(['a', {'b':2, 'c':3}])) - eq_( - beauty, - '[\n "a",\n {\n "c": 3,\n "b": 2\n }\n]\n') From f92b77a6a023180800efcb30ae524975c12eda48 Mon Sep 17 00:00:00 2001 From: Hobson Lane Date: Sun, 17 Feb 2013 21:08:43 -0800 Subject: [PATCH 04/11] move tests --- flojay/tests/marshal_tests.py | 164 ++++++++++++++++++++++++++++++++ flojay/tests/unmarshal_tests.py | 110 +++++++++++++++++++++ 2 files changed, 274 insertions(+) create mode 100644 flojay/tests/marshal_tests.py create mode 100644 flojay/tests/unmarshal_tests.py diff --git a/flojay/tests/marshal_tests.py b/flojay/tests/marshal_tests.py new file mode 100644 index 0000000..35932aa --- /dev/null +++ b/flojay/tests/marshal_tests.py @@ -0,0 +1,164 @@ +import flojay +from unittest import TestCase +from nose.tools import eq_, raises + + +class EventBasedParserTests(TestCase): + + def test_number_callback(self): + callbacks = UnaryCallbacks() + p = flojay.JSONEventParser(callbacks) + long_int = 1 + (2 ** 32) + p.parse("[") + p.parse("123,") + p.parse("1.91") + p.parse("," + str(long_int)) + p.parse(",-1, 1e27]") + eq_(callbacks.buf, [123, 1.91, long_int, -1, 1e+27]) + + def test_null_callback(self): + callbacks = UnaryCallbacks() + p = flojay.JSONEventParser(callbacks) + p.parse("[null]") + eq_(callbacks.buf, [None]) + + def test_partial_nulls(self): + callbacks = UnaryCallbacks() + p = flojay.JSONEventParser(callbacks) + p.parse("[nu") + p.parse("ll]") + eq_(callbacks.buf, [None]) + + def test_boolean_callback(self): + callbacks = UnaryCallbacks() + p = flojay.JSONEventParser(callbacks) + p.parse("[true, false, true]") + eq_(callbacks.buf, [True, False, True]) + + def test_string_callback(self): + callbacks = UnaryCallbacks() + p = flojay.JSONEventParser(callbacks) + p.parse(u'["String", "Ren\xc3e"]') + eq_(callbacks.buf, [u"String", u"Ren\xc3e"]) + + def test_start_end_map(self): + callbacks = MapCallbacks() + p = flojay.JSONEventParser(callbacks) + p.parse(u'{}') + eq_(callbacks.buf, ['{', '}']) + + def test_map_key(self): + callbacks = MapCallbacks() + p = flojay.JSONEventParser(callbacks) + p.parse(u'{"Ren\xc3e": 119}') + eq_(callbacks.buf, ['{', u"Ren\xc3e", 119, '}']) + + def test_array_start_end(self): + callbacks = ArrayCallbacks() + p = flojay.JSONEventParser(callbacks) + p.parse(u'[[], []]') + eq_(callbacks.buf, ['[', '[', ']', '[', ']', ']']) + + @raises(AttributeError) + def test_raises_exception_on_undefined_handlers(self): + p = flojay.JSONEventParser(self) + p.parse('[]') + + @raises(ZeroDivisionError) + def test_exceptions_from_callbacks_are_propogated(self): + flojay.JSONEventParser(CallbacksRaiseAnException()).\ + parse('[]') + + @raises(ValueError) + def test_syntax_error_bad_map(self): + flojay.JSONEventParser(NullCallbacks()).\ + parse('{[]: "a"}') + + @raises(ValueError) + def test_syntax_error_missing_delimiter(self): + flojay.JSONEventParser(NullCallbacks()).\ + parse('["1" "2" "3"]') + + @raises(ValueError) + def test_syntax_error_mismatches(self): + flojay.JSONEventParser(NullCallbacks()).\ + parse('[[}]') + + +class UnaryCallbacks(object): + def __init__(self): + self.buf = [] + + def handle_start_array(self): + pass + + def handle_end_array(self): + pass + + def handle_number(self, number): + self.buf.append(number) + + def handle_null(self): + self.buf.append(None) + + def handle_boolean(self, value): + self.buf.append(value) + + def handle_string(self, string): + self.buf.append(string) + + +class MapCallbacks(object): + def __init__(self): + self.buf = [] + + def handle_number(self, number): + self.buf.append(number) + + def handle_map_key(self, key): + self.buf.append(key) + + def handle_start_map(self): + self.buf.append('{') + + def handle_end_map(self): + self.buf.append('}') + + +class ArrayCallbacks(object): + def __init__(self): + self.buf = [] + + def handle_start_array(self): + self.buf.append('[') + + def handle_end_array(self): + self.buf.append(']') + + +class CallbacksRaiseAnException(object): + def handle_end_array(self): + return 1 / 0 + + def handle_start_array(self): + return 1 / 0 + + +class NullCallbacks(object): + def handle_start_map(self): + pass + + def handle_end_map(self): + pass + + def handle_map_key(self, key): + pass + + def handle_string(self, blah): + pass + + def handle_end_array(self): + pass + + def handle_start_array(self): + pass diff --git a/flojay/tests/unmarshal_tests.py b/flojay/tests/unmarshal_tests.py new file mode 100644 index 0000000..9629cbd --- /dev/null +++ b/flojay/tests/unmarshal_tests.py @@ -0,0 +1,110 @@ +import flojay +from unittest import TestCase +from nose.tools import eq_, raises + + +class UnmarshalTests(TestCase): + def encode(self, it): + return flojay.JSONEncoder().iterencode(it) + + def test_unmarshal(self): + encoder = flojay.JSONEncoder(buffer_size=1) + gen = encoder.iterencode(["a"]) + eq_(gen.next(), '["a"') + eq_(gen.next(), ']') + + def test_unary(self): + eq_(''.join(self.encode('this is just a flat string')), + '"this is just a flat string"') + + eq_(''.join(self.encode(1)), "1") + + def test_unmarshal_nested_array(self): + gen = self.encode(['a', ['b', 1.0, ['d']]]) + eq_(''.join(gen), '["a",["b",1.0,["d"]]]') + eq_(''.join(self.encode([])), '[]') + + def test_dict(self): + eq_(''.join(self.encode({})), '{}') + gen = self.encode({u'key': 3.14}) + eq_(''.join(gen), '{"key":3.14}') + gen = self.encode({'a': 1, 'b': 2}) + eq_(''.join(gen), '{"a":1,"b":2}') + + def test_true_false_null(self): + eq_(''.join(self.encode([True, False, None])), \ + '[true,false,null]') + + def test_longs(self): + result = ''.join(self.encode([100L])) + eq_(result, '[100]') + + def test_unicode(self): + result = ''.join(self.encode([u'test'])) + eq_(result, '["test"]') + eq_(result, str(result)) + + def test_generator(self): + def empty(): + if False is True: + yield "Impossible!" + + def generator(): + yield 1 + yield 2 + yield ['a', 'b', 'c'] + + def generator_with_dict(): + yield {'a': 1, 'b': 2} + + eq_(''.join(self.encode([generator(), 3])), + '[[1,2,["a","b","c"]],3]') + eq_(''.join(self.encode(empty())), '[]') + eq_(''.join(self.encode(generator_with_dict())), '[{"a":1,"b":2}]') + + def test_custom_object_handlerer(self): + import datetime + + def handle_custom_json(obj): + if isinstance(obj, datetime.datetime): + return obj.strftime('@D:%Y-%m-%d') + else: + raise TypeError("Can't encode this " + repr(obj)) + + encoder = flojay.JSONEncoder(default=handle_custom_json) + + eq_(''.join(encoder.iterencode( + ['date', datetime.datetime(2012, 3, 17)])), + '["date","@D:2012-03-17"]') + + @raises(TypeError) + def test_unknown_object_type(self): + ''.join(self.encode({'self': self})) + + def test_string_escape(self): + s = '"A Good Man Is Hard To Find" by Flannery O\'Connor' + + eq_(''.join(self.encode([s])), + '["\\"A Good Man Is Hard To Find\\" by Flannery O\'Connor"]') + + s = 'C:\DOS>' + eq_(''.join(self.encode([s])), + '["C:\\\\DOS>"]') + + s = "I have eaten\nthe plums\nthat were in\n..." + eq_(''.join(self.encode([s])), + '["I have eaten\\nthe plums\\nthat were in\\n..."]') + + def test_utf8(self): + eq_( + ''.join(flojay.JSONEncoder().\ + iterencode([u'Ren\xc3e'])), + '["' + u'Ren\xc3e'.encode('utf8') + '"]' + ) + + def test_beautify(self): + beauty = ''.join(flojay.JSONEncoder(beautify=True, indent_string=' ').\ + iterencode(['a', {'b':2, 'c':3}])) + eq_( + beauty, + '[\n "a",\n {\n "c": 3,\n "b": 2\n }\n]\n') From aca2753a92348f6e626ecf2c304060c9351dbbc8 Mon Sep 17 00:00:00 2001 From: Hobson Lane Date: Sun, 17 Feb 2013 21:11:41 -0800 Subject: [PATCH 05/11] fix setup --- setup.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 47a6a91..0bb41be 100644 --- a/setup.py +++ b/setup.py @@ -2,13 +2,13 @@ from setuptools import setup, Extension from distutils.core import setup import os -from fence import __version__ +from flojay import __version__ yajl_sources = ['flojay/lloyd-yajl/src/' + file_ for file_ in \ ('yajl.c', 'yajl_gen.c', 'yajl_alloc.c', 'yajl_lex.c', 'yajl_tree.c', \ 'yajl_encode.c', 'yajl_version.c', 'yajl_buf.c', 'yajl_parser.c')] -major_minor = '.'.split(__version__) +major_minor = __version__.split('.') flojay = Extension('flojay', define_macros=[ @@ -28,8 +28,7 @@ url = "http://github/myemma/flojay/", ext_modules=[flojay], packages=['flojay'], - install_requires=[ - 'nose==1.1.2', + install_requires=['nose==1.1.2'], keywords = ["json", "stream", "ajax", "webapp", "website", "data", "messaging"], classifiers = [ "Programming Language :: Python", @@ -41,5 +40,5 @@ "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Internet", ], - ]) + ) From 07d83a26b71c211f4aa1e24b61df9b60b8993449 Mon Sep 17 00:00:00 2001 From: Hobson Lane Date: Wed, 20 Feb 2013 21:31:56 -0800 Subject: [PATCH 06/11] MANIFEST.in --- MANIFEST.in | 7 +++---- README | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 4 ++-- 3 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 README diff --git a/MANIFEST.in b/MANIFEST.in index ca80284..6955fc4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,5 @@ include *.txt -recursive-include *.txt include *.md -recursive-include *.md -include *.rst -recursive-include *.rst +include Makefile +graft flojay +recursive-include flojay *.c *.h *.py *.txt README *.md BUILDING *.win32 ChangeLog *.cmake *.json *.gold *.sh TODO PKG-INFO *.dxy diff --git a/README b/README new file mode 100644 index 0000000..c299594 --- /dev/null +++ b/README @@ -0,0 +1,56 @@ +# flojay + +flojay provides incremental serialization of Python data structures as +JSON documents, and an event-based JSON parser to allow for incremtnal +deserialization. It can be used to create and process JSON documents +that are larger than working memory. + +It is based on [yajl](http://lloyd.github.com/yajl/) version 2, a +copy of which is included in this distribution. + +# example + + import flojay + import sys + + # Produce JSON from a generator, printing partial results + # as they become available + + encoder = flojay.JSONEncoder() + json_generator = encoder.iterencode(xrange(100)) + for hunk in json_generator: + print hunk + + + # Read an array of JSON numbers from stdin input, + # summing the values as they are read + + + class ExampleCallbacks(object): + def __init__(self): + self.sum = 0 + + def handle_start_array(self): + pass + + def handle_end_array(self): + pass + + def handle_number(self, value): + self.sum += value + + callbacks = ExampleCallbacks() + + parser = flojay.JSONEventParser(callbacks) + + while 1: + row = sys.stdin.readline() + if len(row) == 0: + break + parser.parse(row) + print "The current total is: %d" % (callbacks.sum,) + + +# Thank also to + +Lloyd Hilaiel -- creator of yajl -- https://github.com/lloyd diff --git a/setup.py b/setup.py index 0bb41be..7d371c5 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ major_minor = __version__.split('.') -flojay = Extension('flojay', +flojay_extension = Extension('flojay', define_macros=[ ('MAJOR_VERSION', major_minor[0]), ('MINOR_VERSION', major_minor[1])], @@ -26,7 +26,7 @@ author='Robert Church', author_email='rchurch@myemma.com', url = "http://github/myemma/flojay/", - ext_modules=[flojay], + ext_modules=[flojay_extension], packages=['flojay'], install_requires=['nose==1.1.2'], keywords = ["json", "stream", "ajax", "webapp", "website", "data", "messaging"], From 406171f397f7dd20f3c87b854d427d7485f304c0 Mon Sep 17 00:00:00 2001 From: Hobson Lane Date: Wed, 20 Feb 2013 21:32:39 -0800 Subject: [PATCH 07/11] more robust MANIFEST.in --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 6955fc4..748b55f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,4 +2,4 @@ include *.txt include *.md include Makefile graft flojay -recursive-include flojay *.c *.h *.py *.txt README *.md BUILDING *.win32 ChangeLog *.cmake *.json *.gold *.sh TODO PKG-INFO *.dxy + From d8fa54de6f1ba15fc0d170cd044b7b5581a4b9d1 Mon Sep 17 00:00:00 2001 From: Hobson Lane Date: Wed, 20 Feb 2013 22:34:55 -0800 Subject: [PATCH 08/11] recursive include broken --- flojay/__init__.py | 1 + setup.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flojay/__init__.py b/flojay/__init__.py index de94d9d..a21700c 100644 --- a/flojay/__init__.py +++ b/flojay/__init__.py @@ -9,3 +9,4 @@ __version__ = '.'.join(str(i) for i in version_info[0:3]) version_string = __version__ +from flojay_extension import * diff --git a/setup.py b/setup.py index 7d371c5..79dfc26 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ #!/usr/bin/env python from setuptools import setup, Extension -from distutils.core import setup import os from flojay import __version__ @@ -10,7 +9,7 @@ major_minor = __version__.split('.') -flojay_extension = Extension('flojay', +flojay_extension = Extension('flojay_extension', define_macros=[ ('MAJOR_VERSION', major_minor[0]), ('MINOR_VERSION', major_minor[1])], From 250012566fb9add162c981d0c84ebd32a6fd5011 Mon Sep 17 00:00:00 2001 From: Hobson Lane Date: Wed, 20 Feb 2013 22:35:08 -0800 Subject: [PATCH 09/11] blank __init__ --- flojay/__init__.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/flojay/__init__.py b/flojay/__init__.py index a21700c..e69de29 100644 --- a/flojay/__init__.py +++ b/flojay/__init__.py @@ -1,12 +0,0 @@ -"""Flojay json streaming module. - -For more information, see: http://github.com/myemma/flojay/ -""" - -# the Canonical way ;) -version_info = (0, 1, 0, 'alpha', 0) - -__version__ = '.'.join(str(i) for i in version_info[0:3]) -version_string = __version__ - -from flojay_extension import * From 43ed8e81125d65daac28bcb899ca74f8893b9bd9 Mon Sep 17 00:00:00 2001 From: Hobson Lane Date: Wed, 20 Feb 2013 22:37:42 -0800 Subject: [PATCH 10/11] move __init__.__version__ to setup.py --- setup.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 79dfc26..6d2668f 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,19 @@ #!/usr/bin/env python from setuptools import setup, Extension import os -from flojay import __version__ + +"""Flojay json streaming module. + +For more information, see: http://github.com/myemma/flojay/ +""" + +# the Canonical way ;) +version_info = (0, 1, 0, 'alpha', 0) + +__version__ = '.'.join(str(i) for i in version_info[0:3]) +#version_string = __version__ + +#from flojay import __version__ yajl_sources = ['flojay/lloyd-yajl/src/' + file_ for file_ in \ ('yajl.c', 'yajl_gen.c', 'yajl_alloc.c', 'yajl_lex.c', 'yajl_tree.c', \ @@ -9,7 +21,7 @@ major_minor = __version__.split('.') -flojay_extension = Extension('flojay_extension', +flojay_extension = Extension('flojay', define_macros=[ ('MAJOR_VERSION', major_minor[0]), ('MINOR_VERSION', major_minor[1])], From 80136359f7a2e47e48ea598e2d013ec381d9889d Mon Sep 17 00:00:00 2001 From: Hobson Lane Date: Wed, 20 Feb 2013 23:04:49 -0800 Subject: [PATCH 11/11] DELETE __init__.py!!! extensions should never have one --- flojay/__init__.py | 0 setup.py | 23 +++-------------------- 2 files changed, 3 insertions(+), 20 deletions(-) delete mode 100644 flojay/__init__.py diff --git a/flojay/__init__.py b/flojay/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/setup.py b/setup.py index 6d2668f..43f5ce6 100644 --- a/setup.py +++ b/setup.py @@ -1,38 +1,21 @@ #!/usr/bin/env python from setuptools import setup, Extension -import os - -"""Flojay json streaming module. - -For more information, see: http://github.com/myemma/flojay/ -""" - -# the Canonical way ;) -version_info = (0, 1, 0, 'alpha', 0) - -__version__ = '.'.join(str(i) for i in version_info[0:3]) -#version_string = __version__ - -#from flojay import __version__ yajl_sources = ['flojay/lloyd-yajl/src/' + file_ for file_ in \ ('yajl.c', 'yajl_gen.c', 'yajl_alloc.c', 'yajl_lex.c', 'yajl_tree.c', \ 'yajl_encode.c', 'yajl_version.c', 'yajl_buf.c', 'yajl_parser.c')] -major_minor = __version__.split('.') - flojay_extension = Extension('flojay', define_macros=[ - ('MAJOR_VERSION', major_minor[0]), - ('MINOR_VERSION', major_minor[1])], + ('MAJOR_VERSION', '0'), + ('MINOR_VERSION', '1')], extra_compile_args=['--std=c99'], include_dirs=['flojay/lloyd-yajl/src'], sources=yajl_sources + ['flojay/flojay.c']) - setup( name='flojay', - version=__version__, + version='0.1', description='Streaming and event-based JSON parser based on yajl', author='Robert Church', author_email='rchurch@myemma.com',