Skip to content

Commit

Permalink
PyPy-specific performance optimizations. Dropped Python 2.6 support.
Browse files Browse the repository at this point in the history
  • Loading branch information
@vmagamedov
vmagamedov committed Nov 20, 2013
1 parent 57528db commit 3174ae7
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 121 deletions.
4 changes: 2 additions & 2 deletions README.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,8 +261,8 @@ To install `SQLConstruct`, simply::

pip install https://github.com/vmagamedov/sqlconstruct/archive/master.zip

`SQLConstruct` is tested and supported on these Python versions: 2.6, 2.7 and
3.3; PyPy is also supported. Supported `SQLAlchemy` versions includes 0.7, 0.8
`SQLConstruct` is tested and supported on these Python versions: 2.7 and 3.3;
PyPy is also supported. Supported `SQLAlchemy` versions includes 0.7, 0.8
and 0.9.

Examples above are using `SQLAlchemy` >= 0.9, if you are using older versions,
Expand Down
96 changes: 48 additions & 48 deletions sqlconstruct.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,8 @@
:license: BSD, see LICENSE.txt for more details.
"""
import sys
import abc
import inspect
from operator import attrgetter, itemgetter, methodcaller
from operator import attrgetter
from functools import partial
from itertools import chain

Expand Down Expand Up @@ -85,26 +84,23 @@ def __init__(self, name, *exprs, **kw):


class Processable(object):
__metaclass__ = abc.ABCMeta

@abc.abstractmethod
def __columns__(self):
pass
raise NotImplementedError

@abc.abstractmethod
def __process__(self, values_map):
pass
def __processor__(self):
raise NotImplementedError


def _get_value_processor(value):
if isinstance(value, ColumnElement):
return itemgetter(hash(value))
return lambda row_map, row, _hash=hash(value): row[row_map[_hash]]
elif isinstance(value, QueryableAttribute):
return itemgetter(hash(value.__clause_element__()))
return _get_value_processor(value.__clause_element__())
elif isinstance(value, Processable):
return value.__process__
return value.__processor__()
else:
return lambda values_map: value
return lambda row_map, row, _value=value: _value


def _yield_columns(value):
Expand Down Expand Up @@ -142,23 +138,20 @@ class Construct(Bundle):
single_entity = True

def __init__(self, spec):
self._spec_keys, self._spec_values = zip(*(
_iteritems(spec)
)) if spec else [(), ()]
self._columns = tuple(set(chain(*_map(_yield_columns,
self._spec_values))))
self._column_hashes = tuple(_map(hash, self._columns))
self._processors = tuple(_map(_get_value_processor, self._spec_values))
self._keys, self._values = zip(*spec.items()) if spec else [(), ()]
self._columns = tuple(set(chain(*_map(_yield_columns, self._values))))
self._processors = tuple(_map(_get_value_processor, self._values))
self._row_map = {hash(col): i for i, col in enumerate(self._columns)}
self._range = range(len(self._columns))
super(Construct, self).__init__(None, *self._columns)

def _from_row(self, row):
values_map = dict(_zip(self._column_hashes, row))
mc = methodcaller('__call__', values_map)
return Object(_zip(self._spec_keys, _map(mc, self._processors)))
return Object({self._keys[i]: self._processors[i](self._row_map, row)
for i in self._range})

def from_query(self, query):
query = query.with_entities(*self._columns)
return list(_map(self._from_row, query))
return list(self._from_row(row) for row in query)

def create_row_processor(self, query, procs, labels):
def proc(row, result):
Expand All @@ -173,48 +166,55 @@ def __init__(self, condition, then_=None, else_=None):
self._then = then_
self._else = else_

self._cond_proc = _get_value_processor(condition)
self._then_proc = _get_value_processor(then_)
self._else_proc = _get_value_processor(else_)

def __columns__(self):
for obj in (self._cond, self._then, self._else):
for column in _yield_columns(obj):
yield column

def __process__(self, values_map):
if self._cond_proc(values_map):
return self._then_proc(values_map)
else:
return self._else_proc(values_map)
def __processor__(self):
def process(row_map, row,
cond_proc=_get_value_processor(self._cond),
then_proc=_get_value_processor(self._then),
else_proc=_get_value_processor(self._else)):
if cond_proc(row_map, row):
return then_proc(row_map, row)
else:
return else_proc(row_map, row)
return process


class apply_(Processable):

def __init__(self, func, args=None, kwargs=None):
self._func = func
self._args = args or []
self._kw_keys, self._kw_values = zip(*(
_iteritems(kwargs)
)) if kwargs else [(), ()]
self._args_procs = tuple(_map(_get_value_processor, self._args))
self._kwargs_procs = tuple(_map(_get_value_processor, self._kw_values))
self._has_kwargs = bool(kwargs)
self._kwargs = kwargs or {}

def __columns__(self):
for arg in set(self._args).union(self._kw_values):
for arg in set(self._args).union(self._kwargs.values()):
for column in _yield_columns(arg):
yield column

def __process__(self, values_map):
mc = methodcaller('__call__', values_map)
args = _map(mc, self._args_procs)
if self._has_kwargs:
kwargs_values = _map(mc, self._kwargs_procs)
kwargs = dict(_zip(self._kw_keys, kwargs_values))
return self._func(*args, **kwargs)
else:
return self._func(*args)
def __processor__(self):
args = []
eval_dict = {'__func__': self._func}

for i, arg in enumerate(self._args):
args.append('__proc{i}__(row_map, row)'.format(i=i))
eval_dict['__proc{i}__'.format(i=i)] = _get_value_processor(arg)

for key, arg in self._kwargs.items():
args.append('{key}=__{key}_proc__(row_map, row)'.format(key=key))
eval_dict['__{key}_proc__'.format(key=key)] = _get_value_processor(arg)

processor_src = (
'def __processor__(row_map, row):\n'
' return __func__({args})\n'
.format(args=', '.join(args))
)
_exec_in(compile(processor_src, __name__, 'single'),
eval_dict)
return eval_dict['__processor__']


class _arg_helper(object):
Expand Down
87 changes: 38 additions & 49 deletions test_sqlconstruct.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,13 @@ def body(a_id, a_name, b_id, b_name, extra_id, extra_name):
return body, [a.id, a.name, b.id, b.name, extra_id, extra_name]


def proceed(processable, mapping):
keys, row = zip(*mapping.items()) if mapping else [(), ()]
processor = processable.__processor__()
row_map = {hash(key): i for i, key in enumerate(keys)}
return processor(row_map, row)


class TestConstruct(unittest.TestCase):

def setUp(self):
Expand Down Expand Up @@ -135,10 +142,10 @@ def test_basic_construct(self):
'a_id': self.a_cls.id,
'a_name': self.a_cls.name,
})
self.assertEquals(set(struct._columns), set([
self.assertEquals(set(struct._columns), {
self.a_cls.__table__.c.id,
self.a_cls.__table__.c.name,
]))
})
result = {
self.a_cls.__table__.c.id: 1,
self.a_cls.__table__.c.name: 'a1',
Expand All @@ -153,10 +160,10 @@ def test_nested_construct(self):
'a_id': apply_(operator.add, [self.a_cls.id, 5]),
'a_name': apply_(operator.concat, [self.a_cls.name, '-test']),
})
self.assertEquals(set(struct._columns), set([
self.assertEquals(set(struct._columns), {
self.a_cls.__table__.c.id,
self.a_cls.__table__.c.name,
]))
})
result = {
self.a_cls.__table__.c.id: 1,
self.a_cls.__table__.c.name: 'a1',
Expand All @@ -171,23 +178,23 @@ def test_apply(self):

min_pos_apply = apply_(add, [1, 2])
self.assertEqual(set(min_pos_apply.__columns__()), set())
self.assertEqual(min_pos_apply.__process__({}), 1 + 2 + 30 + 400)
self.assertEqual(proceed(min_pos_apply, {}), 1 + 2 + 30 + 400)

min_kw_apply = apply_(add, [], {'a': 1, 'b': 2})
self.assertEqual(set(min_kw_apply.__columns__()), set())
self.assertEqual(min_kw_apply.__process__({}), 1 + 2 + 30 + 400)
self.assertEqual(proceed(min_kw_apply, {}), 1 + 2 + 30 + 400)

max_pos_apply = apply_(add, [1, 2, 33, 444])
self.assertEqual(set(max_pos_apply.__columns__()), set())
self.assertEqual(max_pos_apply.__process__({}), 1 + 2 + 33 + 444)
self.assertEqual(proceed(max_pos_apply, {}), 1 + 2 + 33 + 444)

max_kw_apply = apply_(add, [], {'a': 1, 'b': 2, 'c': 33, 'd': 444})
self.assertEqual(set(max_kw_apply.__columns__()), set())
self.assertEqual(max_kw_apply.__process__({}), 1 + 2 + 33 + 444)
self.assertEqual(proceed(max_kw_apply, {}), 1 + 2 + 33 + 444)

mixed_apply = apply_(add, [1, 2], {'c': 33, 'd': 444})
self.assertEqual(set(mixed_apply.__columns__()), set())
self.assertEqual(mixed_apply.__process__({}), 1 + 2 + 33 + 444)
self.assertEqual(proceed(mixed_apply, {}), 1 + 2 + 33 + 444)

def test_apply_with_columns(self):
f1 = self.a_cls.id
Expand All @@ -200,16 +207,16 @@ def test_apply_with_columns(self):
add = lambda a, b: a + b

apl1 = apply_(add, [f1], {'b': f2})
self.assertEquals(set(apl1.__columns__()), set([c1, c2]))
self.assertEqual(apl1.__process__({hash(c1): 3, hash(c2): 4}), 3 + 4)
self.assertEquals(set(apl1.__columns__()), {c1, c2})
self.assertEqual(proceed(apl1, {c1: 3, c2: 4}), 3 + 4)

apl2 = apply_(add, [c1], {'b': c2})
self.assertEquals(set(apl2.__columns__()), set([c1, c2]))
self.assertEqual(apl1.__process__({hash(c1): 4, hash(c2): 5}), 4 + 5)
self.assertEquals(set(apl2.__columns__()), {c1, c2})
self.assertEqual(proceed(apl1, {c1: 4, c2: 5}), 4 + 5)

apl3 = apply_(add, [fn1], {'b': fn2})
self.assertEquals(set(apl3.__columns__()), set([fn1, fn2]))
self.assertEqual(apl3.__process__({hash(fn1): 5, hash(fn2): 6}), 5 + 6)
self.assertEquals(set(apl3.__columns__()), {fn1, fn2})
self.assertEqual(proceed(apl3, {fn1: 5, fn2: 6}), 5 + 6)

def test_nested_apply(self):
c1 = self.a_cls.__table__.c.id
Expand Down Expand Up @@ -245,9 +252,8 @@ def test_nested_apply(self):
]),
]),
])
self.assertEquals(set(apl.__columns__()), set([c1, c2]))
self.assertEqual(apl.__process__({hash(c1): 4, hash(c2): 5}),
sum(range(10)))
self.assertEquals(set(apl.__columns__()), {c1, c2})
self.assertEqual(proceed(apl, {c1: 4, c2: 5}), sum(range(10)))

def test_if(self):
add = lambda a, b: a + b
Expand All @@ -258,35 +264,21 @@ def test_if(self):

if1 = if_(True, then_=1, else_=2)
self.assertEquals(set(if1.__columns__()), set())
self.assertEqual(if1.__process__({}), 1)
self.assertEqual(proceed(if1, {}), 1)

if2 = if_(False, then_=1, else_=2)
self.assertEquals(set(if2.__columns__()), set())
self.assertEqual(if2.__process__({}), 2)
self.assertEqual(proceed(if2, {}), 2)

if3 = if_(c1, then_=c2, else_=c3)
self.assertEquals(set(if3.__columns__()), set([c1, c2, c3]))
self.assertEqual(
if3.__process__({hash(c1): 0, hash(c2): 3, hash(c3): 6}),
6,
)
self.assertEqual(
if3.__process__({hash(c1): 1, hash(c2): 3, hash(c3): 6}),
3,
)
self.assertEquals(set(if3.__columns__()), {c1, c2, c3})
self.assertEqual(proceed(if3, {c1: 0, c2: 3, c3: 6}), 6)
self.assertEqual(proceed(if3, {c1: 1, c2: 3, c3: 6}), 3)

if4 = if_(c1, then_=apply_(add, [c2, c3]), else_=apply_(add, [c3, c4]))
self.assertEquals(set(if4.__columns__()), set([c1, c2, c3, c4]))
self.assertEqual(
if4.__process__({hash(c1): 0, hash(c2): 2, hash(c3): 3,
hash(c4): 4}),
3 + 4,
)
self.assertEqual(
if4.__process__({hash(c1): 1, hash(c2): 2, hash(c3): 3,
hash(c4): 4}),
2 + 3,
)
self.assertEquals(set(if4.__columns__()), {c1, c2, c3, c4})
self.assertEqual(proceed(if4, {c1: 0, c2: 2, c3: 3, c4: 4}), 3 + 4)
self.assertEqual(proceed(if4, {c1: 1, c2: 2, c3: 3, c4: 4}), 2 + 3)

def test_defined_signatures(self):
obj_spec = inspect.getargspec(defined_func)
Expand Down Expand Up @@ -325,31 +317,28 @@ def test_defined_calls(self):
apl1 = defined_func.defn(self.a_cls, self.b_cls,
extra_id=3, extra_name='baz')
self.assertTrue(isinstance(apl1, apply_), type(apl1))
self.assertEquals(set(apl1.__columns__()), set([c1, c2, c3, c4]))
self.assertEquals(set(apl1.__columns__()), {c1, c2, c3, c4})
self.assertEqual(
apl1.__process__({hash(c1): 1, hash(c2): 'foo', hash(c3): 2,
hash(c4): 'bar'}),
proceed(apl1, {c1: 1, c2: 'foo', c3: 2, c4: 'bar'}),
(1 + 2 + 3, 'foo' + 'bar' + 'baz'),
)

apl2 = defined_func.defn(self.a_cls, self.b_cls,
extra_id=c1, extra_name=c2)
self.assertTrue(isinstance(apl2, apply_), type(apl2))
self.assertEquals(set(apl2.__columns__()), set([c1, c2, c3, c4]))
self.assertEquals(set(apl2.__columns__()), {c1, c2, c3, c4})
self.assertEqual(
apl2.__process__({hash(c1): 1, hash(c2): 'foo', hash(c3): 2,
hash(c4): 'bar'}),
proceed(apl2, {c1: 1, c2: 'foo', c3: 2, c4: 'bar'}),
(1 + 2 + 1, 'foo' + 'bar' + 'foo'),
)

apl3 = defined_func.defn(self.a_cls, self.b_cls,
extra_id=apply_(operator.add, [c1, c3]),
extra_name=apply_(operator.concat, [c2, c4]))
self.assertTrue(isinstance(apl3, apply_), type(apl3))
self.assertEquals(set(apl3.__columns__()), set([c1, c2, c3, c4]))
self.assertEquals(set(apl3.__columns__()), {c1, c2, c3, c4})
self.assertEqual(
apl3.__process__({hash(c1): 1, hash(c2): 'foo', hash(c3): 2,
hash(c4): 'bar'}),
proceed(apl3, {c1: 1, c2: 'foo', c3: 2, c4: 'bar'}),
(1 + 2 + (1 + 2), 'foo' + 'bar' + ('foo' + 'bar')),
)

Expand Down
22 changes: 0 additions & 22 deletions tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,28 +3,6 @@ commands =
nosetests
usedevelop = True

[testenv:py26sqla07]
basepython = python2.6
deps =
nose
unittest2
[email protected]:zzzeek/sqlalchemy.git@rel_0_7#egg=sqlalchemy

[testenv:py26sqla08]
basepython = python2.6
deps =
nose
unittest2
[email protected]:zzzeek/sqlalchemy.git@rel_0_8#egg=sqlalchemy

[testenv:py26sqla0X]
basepython = python2.6
deps =
nose
unittest2
[email protected]:zzzeek/sqlalchemy.git@master#egg=sqlalchemy


[testenv:py27sqla07]
basepython = python2.7
deps =
Expand Down

0 comments on commit 3174ae7

Please sign in to comment.