From f83c8903f4682943ee6404ee9835bbf4d4aa9a4e Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Wed, 25 Jan 2017 16:42:21 -0800 Subject: [PATCH 01/33] add support for "async with cursor_context()" --- aiopg/pool.py | 23 +++++++++++++++++++++-- aiopg/utils.py | 36 +++++++++++++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/aiopg/pool.py b/aiopg/pool.py index e3912834..54d3f7a4 100644 --- a/aiopg/pool.py +++ b/aiopg/pool.py @@ -257,15 +257,34 @@ def release(self, conn): return fut @asyncio.coroutine - def cursor(self, name=None, cursor_factory=None, + def _create_conn_cur(self, name=None, cursor_factory=None, scrollable=None, withhold=False, *, timeout=None): - """XXX""" conn = yield from self.acquire() cur = yield from conn.cursor(name=name, cursor_factory=cursor_factory, scrollable=scrollable, withhold=withhold, timeout=timeout) + + return conn, cur + + @asyncio.coroutine + def cursor(self, name=None, cursor_factory=None, + scrollable=None, withhold=False, *, timeout=None): + """XXX""" + conn, cur = yield from self._create_conn_cur(name=name, + cursor_factory=cursor_factory, + scrollable=scrollable, withhold=withhold, + timeout=timeout) return _PoolCursorContextManager(self, conn, cur) + def cursor_context(self, name=None, cursor_factory=None, + scrollable=None, withhold=False, *, timeout=None): + + conn_cur_co = self._create_conn_cur(name=name, + cursor_factory=cursor_factory, + scrollable=scrollable, withhold=withhold, + timeout=timeout) + return _PoolCursorContextManager(self, None, None, conn_cur_co) + def __enter__(self): raise RuntimeError( '"yield from" should be used as context manager expression') diff --git a/aiopg/utils.py b/aiopg/utils.py index 84a768d2..4416f687 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -125,7 +125,7 @@ class _PoolAcquireContextManager(_ContextManager): __slots__ = ('_coro', '_conn', '_pool') def __init__(self, coro, pool): - self._coro = coro + super().__init__(coro) self._conn = None self._pool = pool @@ -203,14 +203,22 @@ class _PoolCursorContextManager: with pool: + + It also allows the following idiom: + async with pool.cursor_context() as cur: + yield from cur.execute("SELECT 1") """ - __slots__ = ('_pool', '_conn', '_cur') + __slots__ = ('_pool', '_conn', '_cur', '_conn_cur_co') + + def __init__(self, pool, conn, cur, conn_cur_co=None): + if conn_cur_co: + assert conn is None and cur is None - def __init__(self, pool, conn, cur): self._pool = pool self._conn = conn self._cur = cur + self._conn_cur_co = conn_cur_co def __enter__(self): return self._cur @@ -224,6 +232,28 @@ def __exit__(self, *args): self._conn = None self._cur = None + if PY_35: + @asyncio.coroutine + def __aenter__(self): + assert not self._conn and not self._cur + self._conn, self._cur = yield from self._conn_cur_co + + yield from self._conn.__aenter__() + yield from self._cur.__aenter__() + return self + + @asyncio.coroutine + def __aexit__(self, exc_type, exc_val, exc_tb): + conn = self._conn + cur = self._cur + self._pool = None # releases cursor in conn __aexit__ + + try: + yield from cur.__aexit__(exc_type, exc_val, exc_tb) + self._cur = None + finally: + yield from conn.__aexit__(exc_type, exc_val, exc_tb) + self._conn = None if not PY_35: try: From c850e2b4311e1394465d646c42a7583612712619 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Wed, 25 Jan 2017 16:47:03 -0800 Subject: [PATCH 02/33] fix pep warnings --- aiopg/pool.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/aiopg/pool.py b/aiopg/pool.py index 54d3f7a4..33b95aca 100644 --- a/aiopg/pool.py +++ b/aiopg/pool.py @@ -258,7 +258,7 @@ def release(self, conn): @asyncio.coroutine def _create_conn_cur(self, name=None, cursor_factory=None, - scrollable=None, withhold=False, *, timeout=None): + scrollable=None, withhold=False, *, timeout=None): conn = yield from self.acquire() cur = yield from conn.cursor(name=name, cursor_factory=cursor_factory, scrollable=scrollable, withhold=withhold, @@ -270,19 +270,16 @@ def _create_conn_cur(self, name=None, cursor_factory=None, def cursor(self, name=None, cursor_factory=None, scrollable=None, withhold=False, *, timeout=None): """XXX""" - conn, cur = yield from self._create_conn_cur(name=name, - cursor_factory=cursor_factory, - scrollable=scrollable, withhold=withhold, - timeout=timeout) + conn, cur = yield from self._create_conn_cur( + name=name, cursor_factory=cursor_factory, scrollable=scrollable, + withhold=withhold, timeout=timeout) return _PoolCursorContextManager(self, conn, cur) def cursor_context(self, name=None, cursor_factory=None, - scrollable=None, withhold=False, *, timeout=None): - - conn_cur_co = self._create_conn_cur(name=name, - cursor_factory=cursor_factory, - scrollable=scrollable, withhold=withhold, - timeout=timeout) + scrollable=None, withhold=False, *, timeout=None): + conn_cur_co = self._create_conn_cur( + name=name, cursor_factory=cursor_factory, scrollable=scrollable, + withhold=withhold, timeout=timeout) return _PoolCursorContextManager(self, None, None, conn_cur_co) def __enter__(self): From 0ffd3126c4f148190b42bed8ac5a29028b6eb8e6 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Wed, 25 Jan 2017 16:49:22 -0800 Subject: [PATCH 03/33] pep --- aiopg/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aiopg/utils.py b/aiopg/utils.py index 4416f687..90ca6c18 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -255,6 +255,7 @@ def __aexit__(self, exc_type, exc_val, exc_tb): yield from conn.__aexit__(exc_type, exc_val, exc_tb) self._conn = None + if not PY_35: try: from asyncio import coroutines From 157c58007d6dd8a3adb87233a353f20249914f37 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 17 Feb 2017 12:06:36 -0800 Subject: [PATCH 04/33] merge old style with new style --- aiopg/pool.py | 9 --------- aiopg/utils.py | 19 +++++++++++++++---- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/aiopg/pool.py b/aiopg/pool.py index 33b95aca..e970f58b 100644 --- a/aiopg/pool.py +++ b/aiopg/pool.py @@ -266,16 +266,7 @@ def _create_conn_cur(self, name=None, cursor_factory=None, return conn, cur - @asyncio.coroutine def cursor(self, name=None, cursor_factory=None, - scrollable=None, withhold=False, *, timeout=None): - """XXX""" - conn, cur = yield from self._create_conn_cur( - name=name, cursor_factory=cursor_factory, scrollable=scrollable, - withhold=withhold, timeout=timeout) - return _PoolCursorContextManager(self, conn, cur) - - def cursor_context(self, name=None, cursor_factory=None, scrollable=None, withhold=False, *, timeout=None): conn_cur_co = self._create_conn_cur( name=name, cursor_factory=cursor_factory, scrollable=scrollable, diff --git a/aiopg/utils.py b/aiopg/utils.py index 90ca6c18..d22ba9e2 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -1,6 +1,6 @@ import asyncio import sys - +import warnings PY_35 = sys.version_info >= (3, 5) PY_352 = sys.version_info >= (3, 5, 2) @@ -232,14 +232,25 @@ def __exit__(self, *args): self._conn = None self._cur = None - if PY_35: - @asyncio.coroutine - def __aenter__(self): + @asyncio.coroutine + def _acquire_conn_cur(self): assert not self._conn and not self._cur self._conn, self._cur = yield from self._conn_cur_co yield from self._conn.__aenter__() yield from self._cur.__aenter__() + + @asyncio.coroutine + def __await__(self): + if PY_35: + warnings.warn("This usage is deprecated, use 'async with` syntax", + DeprecationWarning) + yield from self._acquire_conn_cur() + + if PY_35: + @asyncio.coroutine + def __aenter__(self): + yield from self._acquire_conn_cur() return self @asyncio.coroutine From 066095619fc411ad26bb76436b1e359d24dd4bf4 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 17 Feb 2017 12:11:56 -0800 Subject: [PATCH 05/33] add new testcase and remove unneeded statements --- tests/pep492/test_async_await.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/tests/pep492/test_async_await.py b/tests/pep492/test_async_await.py index 0e55cdb5..ccf1d6bd 100644 --- a/tests/pep492/test_async_await.py +++ b/tests/pep492/test_async_await.py @@ -1,11 +1,9 @@ -import asyncio import pytest import aiopg import aiopg.sa from aiopg.sa import SAConnection -@asyncio.coroutine async def test_cursor_await(make_connection): conn = await make_connection() @@ -16,7 +14,6 @@ async def test_cursor_await(make_connection): cursor.close() -@asyncio.coroutine async def test_connect_context_manager(loop, pg_params): async with aiopg.connect(loop=loop, **pg_params) as conn: cursor = await conn.cursor() @@ -27,7 +24,15 @@ async def test_connect_context_manager(loop, pg_params): assert conn.closed -@asyncio.coroutine +async def test_cursor_context_manager(loop, pg_params): + async with aiopg.connect(loop=loop, **pg_params) as conn: + async with conn.cursor() as cursor: + await cursor.execute('SELECT 42') + resp = await cursor.fetchone() + assert resp == (42, ) + assert conn.closed + + async def test_connection_context_manager(make_connection): conn = await make_connection() assert not conn.closed @@ -40,7 +45,6 @@ async def test_connection_context_manager(make_connection): assert conn.closed -@asyncio.coroutine async def test_cursor_create_with_context_manager(make_connection): conn = await make_connection() @@ -53,7 +57,6 @@ async def test_cursor_create_with_context_manager(make_connection): assert cursor.closed -@asyncio.coroutine async def test_cursor_with_context_manager(make_connection): conn = await make_connection() cursor = await conn.cursor() @@ -66,7 +69,6 @@ async def test_cursor_with_context_manager(make_connection): assert cursor.closed -@asyncio.coroutine async def test_cursor_lightweight(make_connection): conn = await make_connection() cursor = await conn.cursor() @@ -78,7 +80,6 @@ async def test_cursor_lightweight(make_connection): assert cursor.closed -@asyncio.coroutine async def test_pool_context_manager(pg_params, loop): pool = await aiopg.create_pool(loop=loop, **pg_params) @@ -93,7 +94,6 @@ async def test_pool_context_manager(pg_params, loop): assert pool.closed -@asyncio.coroutine async def test_create_pool_context_manager(pg_params, loop): async with aiopg.create_pool(loop=loop, **pg_params) as pool: async with pool.acquire() as conn: @@ -107,7 +107,6 @@ async def test_create_pool_context_manager(pg_params, loop): assert pool.closed -@asyncio.coroutine async def test_cursor_aiter(make_connection): result = [] conn = await make_connection() @@ -122,7 +121,6 @@ async def test_cursor_aiter(make_connection): assert conn.closed -@asyncio.coroutine async def test_engine_context_manager(pg_params, loop): engine = await aiopg.sa.create_engine(loop=loop, **pg_params) async with engine: @@ -132,7 +130,6 @@ async def test_engine_context_manager(pg_params, loop): assert engine.closed -@asyncio.coroutine async def test_create_engine_context_manager(pg_params, loop): async with aiopg.sa.create_engine(loop=loop, **pg_params) as engine: async with engine.acquire() as conn: @@ -140,7 +137,6 @@ async def test_create_engine_context_manager(pg_params, loop): assert engine.closed -@asyncio.coroutine async def test_result_proxy_aiter(pg_params, loop): sql = 'SELECT generate_series(1, 5);' result = [] @@ -154,7 +150,6 @@ async def test_result_proxy_aiter(pg_params, loop): assert conn.closed -@asyncio.coroutine async def test_transaction_context_manager(pg_params, loop): sql = 'SELECT generate_series(1, 5);' result = [] @@ -181,7 +176,6 @@ async def test_transaction_context_manager(pg_params, loop): assert conn.closed -@asyncio.coroutine async def test_transaction_context_manager_error(pg_params, loop): async with aiopg.sa.create_engine(loop=loop, **pg_params) as engine: async with engine.acquire() as conn: @@ -194,7 +188,6 @@ async def test_transaction_context_manager_error(pg_params, loop): assert conn.closed -@asyncio.coroutine async def test_transaction_context_manager_commit_once(pg_params, loop): async with aiopg.sa.create_engine(loop=loop, **pg_params) as engine: async with engine.acquire() as conn: @@ -214,7 +207,6 @@ async def test_transaction_context_manager_commit_once(pg_params, loop): assert conn.closed -@asyncio.coroutine async def test_transaction_context_manager_nested_commit(pg_params, loop): sql = 'SELECT generate_series(1, 5);' result = [] @@ -244,7 +236,6 @@ async def test_transaction_context_manager_nested_commit(pg_params, loop): assert conn.closed -@asyncio.coroutine async def test_sa_connection_execute(pg_params, loop): sql = 'SELECT generate_series(1, 5);' result = [] From 8af3e6d6e5f525d52ce63ac7f373f9e7d1fa0885 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 17 Feb 2017 12:18:35 -0800 Subject: [PATCH 06/33] pep --- aiopg/pool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiopg/pool.py b/aiopg/pool.py index e970f58b..eda65de0 100644 --- a/aiopg/pool.py +++ b/aiopg/pool.py @@ -266,8 +266,8 @@ def _create_conn_cur(self, name=None, cursor_factory=None, return conn, cur - def cursor(self, name=None, cursor_factory=None, - scrollable=None, withhold=False, *, timeout=None): + def cursor(self, name=None, cursor_factory=None, scrollable=None, + withhold=False, *, timeout=None): conn_cur_co = self._create_conn_cur( name=name, cursor_factory=cursor_factory, scrollable=scrollable, withhold=withhold, timeout=timeout) From fd42c6c3907da6f4928d5520df415a8e6212f151 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 17 Feb 2017 12:50:38 -0800 Subject: [PATCH 07/33] use init trick from asyncpg --- aiopg/pool.py | 2 +- aiopg/utils.py | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/aiopg/pool.py b/aiopg/pool.py index eda65de0..b299746f 100644 --- a/aiopg/pool.py +++ b/aiopg/pool.py @@ -271,7 +271,7 @@ def cursor(self, name=None, cursor_factory=None, scrollable=None, conn_cur_co = self._create_conn_cur( name=name, cursor_factory=cursor_factory, scrollable=scrollable, withhold=withhold, timeout=timeout) - return _PoolCursorContextManager(self, None, None, conn_cur_co) + return _PoolCursorContextManager(self, conn_cur_co) def __enter__(self): raise RuntimeError( diff --git a/aiopg/utils.py b/aiopg/utils.py index d22ba9e2..25bd9749 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -211,13 +211,9 @@ class _PoolCursorContextManager: __slots__ = ('_pool', '_conn', '_cur', '_conn_cur_co') - def __init__(self, pool, conn, cur, conn_cur_co=None): - if conn_cur_co: - assert conn is None and cur is None - + def __init__(self, pool, conn_cur_co=None): self._pool = pool - self._conn = conn - self._cur = cur + self._conn = self._cur = None self._conn_cur_co = conn_cur_co def __enter__(self): @@ -228,29 +224,30 @@ def __exit__(self, *args): self._cur.close() self._pool.release(self._conn) finally: - self._pool = None self._conn = None self._cur = None @asyncio.coroutine - def _acquire_conn_cur(self): - assert not self._conn and not self._cur - self._conn, self._cur = yield from self._conn_cur_co + def _init(self): + assert not self._conn and not self._cur + self._conn, self._cur = yield from self._conn_cur_co - yield from self._conn.__aenter__() - yield from self._cur.__aenter__() + yield from self._conn.__aenter__() + yield from self._cur.__aenter__() @asyncio.coroutine def __await__(self): + # this is using a trick similar to the one here: + # https://magicstack.github.io/asyncpg/current/_modules/asyncpg/pool.html if PY_35: warnings.warn("This usage is deprecated, use 'async with` syntax", DeprecationWarning) - yield from self._acquire_conn_cur() + return self._init.__await__() if PY_35: @asyncio.coroutine def __aenter__(self): - yield from self._acquire_conn_cur() + yield from self._init() return self @asyncio.coroutine From 4949621f955d1f0c065a0f0e8c86a954d506298d Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 17 Feb 2017 13:45:17 -0800 Subject: [PATCH 08/33] bugfix --- aiopg/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/aiopg/utils.py b/aiopg/utils.py index 25bd9749..aef13ad4 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -193,20 +193,19 @@ def __aexit__(self, exc_type, exc_val, exc_tb): class _PoolCursorContextManager: """Context manager. - This enables the following idiom for acquiring and releasing a + This enables the following idioms for acquiring and releasing a cursor around a block: with (yield from pool.cursor()) as cur: yield from cur.execute("SELECT 1") + async with pool.cursor() as cur: + yield from cur.execute("SELECT 1") + while failing loudly when accidentally using: with pool: - - It also allows the following idiom: - async with pool.cursor_context() as cur: - yield from cur.execute("SELECT 1") """ __slots__ = ('_pool', '_conn', '_cur', '_conn_cur_co') @@ -234,6 +233,7 @@ def _init(self): yield from self._conn.__aenter__() yield from self._cur.__aenter__() + return self @asyncio.coroutine def __await__(self): From bebe1b3003dc05f73aa9d6838d79f13e3f53a8e9 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 17 Feb 2017 14:49:31 -0800 Subject: [PATCH 09/33] well that took awhile to figure out --- aiopg/pool.py | 11 ++++++++--- aiopg/utils.py | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/aiopg/pool.py b/aiopg/pool.py index b299746f..4c9f565c 100644 --- a/aiopg/pool.py +++ b/aiopg/pool.py @@ -259,10 +259,15 @@ def release(self, conn): @asyncio.coroutine def _create_conn_cur(self, name=None, cursor_factory=None, scrollable=None, withhold=False, *, timeout=None): + conn = yield from self.acquire() - cur = yield from conn.cursor(name=name, cursor_factory=cursor_factory, - scrollable=scrollable, withhold=withhold, - timeout=timeout) + try: + cur = yield from conn.cursor(name=name, cursor_factory=cursor_factory, + scrollable=scrollable, withhold=withhold, + timeout=timeout) + except: + self.release(conn) + raise return conn, cur diff --git a/aiopg/utils.py b/aiopg/utils.py index aef13ad4..bba418e0 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -235,14 +235,26 @@ def _init(self): yield from self._cur.__aenter__() return self - @asyncio.coroutine + def __iter__(self): + # This will get hit if you use "yield from pool.cursor()" + if PY_35: + warnings.warn("This usage is deprecated, use 'async with` syntax", + DeprecationWarning) + return self._init() + def __await__(self): + # This will get hit directly if you "await pool.cursor()" # this is using a trick similar to the one here: # https://magicstack.github.io/asyncpg/current/_modules/asyncpg/pool.html + # however since `self._init()` is an "asyncio.coroutine" we can't use + # just return self._init().__await__() as that returns a generator + # witout an "__await__" attribute, and we can't return a coroutine from + # here if PY_35: warnings.warn("This usage is deprecated, use 'async with` syntax", DeprecationWarning) - return self._init.__await__() + value = yield from self._init() + return value if PY_35: @asyncio.coroutine From 857abfdd7d1aaa97618341a34aa037a20f7912be Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 17 Feb 2017 14:54:26 -0800 Subject: [PATCH 10/33] pep --- aiopg/pool.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/aiopg/pool.py b/aiopg/pool.py index 4c9f565c..95853704 100644 --- a/aiopg/pool.py +++ b/aiopg/pool.py @@ -262,8 +262,10 @@ def _create_conn_cur(self, name=None, cursor_factory=None, conn = yield from self.acquire() try: - cur = yield from conn.cursor(name=name, cursor_factory=cursor_factory, - scrollable=scrollable, withhold=withhold, + cur = yield from conn.cursor(name=name, + cursor_factory=cursor_factory, + scrollable=scrollable, + withhold=withhold, timeout=timeout) except: self.release(conn) From e8a78b2320a2096355d275aaa29cd901167ed68b Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 17 Feb 2017 15:02:10 -0800 Subject: [PATCH 11/33] fix for py3.4 --- aiopg/utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/aiopg/utils.py b/aiopg/utils.py index bba418e0..78c417db 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -230,9 +230,6 @@ def __exit__(self, *args): def _init(self): assert not self._conn and not self._cur self._conn, self._cur = yield from self._conn_cur_co - - yield from self._conn.__aenter__() - yield from self._cur.__aenter__() return self def __iter__(self): @@ -260,7 +257,9 @@ def __await__(self): @asyncio.coroutine def __aenter__(self): yield from self._init() - return self + yield from self._conn.__aenter__() + yield from self._cur.__aenter__() + return self._cur @asyncio.coroutine def __aexit__(self, exc_type, exc_val, exc_tb): From bfbd60e97ab32c50bd9a93af27c0dbdc6a16edcd Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 17 Feb 2017 15:19:18 -0800 Subject: [PATCH 12/33] revise unittests --- tests/pep492/test_async_await.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/pep492/test_async_await.py b/tests/pep492/test_async_await.py index ccf1d6bd..e3267bd6 100644 --- a/tests/pep492/test_async_await.py +++ b/tests/pep492/test_async_await.py @@ -23,15 +23,23 @@ async def test_connect_context_manager(loop, pg_params): cursor.close() assert conn.closed - -async def test_cursor_context_manager(loop, pg_params): - async with aiopg.connect(loop=loop, **pg_params) as conn: - async with conn.cursor() as cursor: +async def test_pool_cursor_context_manager(loop, pg_params): + async with aiopg.create_pool(loop=loop, **pg_params) as pool: + async with pool.cursor() as cursor: await cursor.execute('SELECT 42') resp = await cursor.fetchone() assert resp == (42, ) - assert conn.closed + assert cursor.closed + assert pool.closed +async def test_pool_cursor_await_context_manager(loop, pg_params): + async with aiopg.create_pool(loop=loop, **pg_params) as pool: + with (await pool.cursor()) as cursor: + await cursor.execute('SELECT 42') + resp = await cursor.fetchone() + assert resp == (42, ) + assert cursor.closed + assert pool.closed async def test_connection_context_manager(make_connection): conn = await make_connection() From 623c3689024f0f82702f4ebbbd53421bc2b3038d Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 17 Feb 2017 15:24:00 -0800 Subject: [PATCH 13/33] arr --- tests/pep492/test_async_await.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/pep492/test_async_await.py b/tests/pep492/test_async_await.py index e3267bd6..a0f05ae7 100644 --- a/tests/pep492/test_async_await.py +++ b/tests/pep492/test_async_await.py @@ -23,6 +23,7 @@ async def test_connect_context_manager(loop, pg_params): cursor.close() assert conn.closed + async def test_pool_cursor_context_manager(loop, pg_params): async with aiopg.create_pool(loop=loop, **pg_params) as pool: async with pool.cursor() as cursor: @@ -32,6 +33,7 @@ async def test_pool_cursor_context_manager(loop, pg_params): assert cursor.closed assert pool.closed + async def test_pool_cursor_await_context_manager(loop, pg_params): async with aiopg.create_pool(loop=loop, **pg_params) as pool: with (await pool.cursor()) as cursor: @@ -41,6 +43,7 @@ async def test_pool_cursor_await_context_manager(loop, pg_params): assert cursor.closed assert pool.closed + async def test_connection_context_manager(make_connection): conn = await make_connection() assert not conn.closed From 4770379a7e41151c07bb0be7dd6b55aea13e1b25 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 17 Feb 2017 16:15:23 -0800 Subject: [PATCH 14/33] refactor to use _PoolConnectionContextManager --- aiopg/pool.py | 8 ++++---- aiopg/utils.py | 55 +++++++++++++++++++++++++++++++------------------- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/aiopg/pool.py b/aiopg/pool.py index 95853704..38330484 100644 --- a/aiopg/pool.py +++ b/aiopg/pool.py @@ -275,10 +275,10 @@ def _create_conn_cur(self, name=None, cursor_factory=None, def cursor(self, name=None, cursor_factory=None, scrollable=None, withhold=False, *, timeout=None): - conn_cur_co = self._create_conn_cur( - name=name, cursor_factory=cursor_factory, scrollable=scrollable, - withhold=withhold, timeout=timeout) - return _PoolCursorContextManager(self, conn_cur_co) + cursor_kwargs = dict(name=name, cursor_factory=cursor_factory, + scrollable=scrollable, withhold=withhold, + timeout=timeout) + return _PoolCursorContextManager(self, cursor_kwargs) def __enter__(self): raise RuntimeError( diff --git a/aiopg/utils.py b/aiopg/utils.py index 78c417db..7174f52c 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -163,6 +163,10 @@ def __init__(self, pool, conn): self._pool = pool self._conn = conn + @property + def conn(self): + return self._conn + def __enter__(self): assert self._conn return self._conn @@ -208,12 +212,12 @@ class _PoolCursorContextManager: """ - __slots__ = ('_pool', '_conn', '_cur', '_conn_cur_co') + __slots__ = ('_pool', '_cursor_kwargs', '_cur') - def __init__(self, pool, conn_cur_co=None): + def __init__(self, pool, cursor_kwargs=None): self._pool = pool - self._conn = self._cur = None - self._conn_cur_co = conn_cur_co + self._cursor_kwargs = cursor_kwargs + self._cur = None def __enter__(self): return self._cur @@ -221,15 +225,26 @@ def __enter__(self): def __exit__(self, *args): try: self._cur.close() - self._pool.release(self._conn) + self._pool.__exit__(*args) finally: - self._conn = None self._cur = None @asyncio.coroutine - def _init(self): - assert not self._conn and not self._cur - self._conn, self._cur = yield from self._conn_cur_co + def _init_pool(self, with_aenter): + assert not self._cur + + if with_aenter: + conn = None + else: + conn = yield from self._pool.acquire() + + # self._pool now morphs into a _PoolConnectionContextManager + self._pool = _PoolConnectionContextManager(self._pool, conn) + + @asyncio.coroutine + def _yield_await_init(self): + yield from self._init_pool(False) + self._cur = yield from self._pool.conn.cursor(**self._cursor_kwargs) return self def __iter__(self): @@ -237,7 +252,7 @@ def __iter__(self): if PY_35: warnings.warn("This usage is deprecated, use 'async with` syntax", DeprecationWarning) - return self._init() + return self._yield_await_init() def __await__(self): # This will get hit directly if you "await pool.cursor()" @@ -250,29 +265,27 @@ def __await__(self): if PY_35: warnings.warn("This usage is deprecated, use 'async with` syntax", DeprecationWarning) - value = yield from self._init() + value = yield from self._yield_await_init() return value if PY_35: @asyncio.coroutine def __aenter__(self): - yield from self._init() - yield from self._conn.__aenter__() - yield from self._cur.__aenter__() + yield from self._init_pool(True) + + # this will create the connection + yield from self._pool.__aenter__() + self._cur = yield from self._pool.conn.cursor(**self._cursor_kwargs) return self._cur @asyncio.coroutine def __aexit__(self, exc_type, exc_val, exc_tb): - conn = self._conn - cur = self._cur - self._pool = None # releases cursor in conn __aexit__ - try: - yield from cur.__aexit__(exc_type, exc_val, exc_tb) + yield from self._cur.__aexit__(exc_type, exc_val, exc_tb) self._cur = None finally: - yield from conn.__aexit__(exc_type, exc_val, exc_tb) - self._conn = None + yield from self._pool.__aexit__(exc_type, exc_val, exc_tb) + self._pool = None if not PY_35: From 452e10b406bc58c1766c42259a04581edce1d6aa Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 17 Feb 2017 16:16:49 -0800 Subject: [PATCH 15/33] remove unused code --- aiopg/pool.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/aiopg/pool.py b/aiopg/pool.py index 38330484..f93e565d 100644 --- a/aiopg/pool.py +++ b/aiopg/pool.py @@ -256,23 +256,6 @@ def release(self, conn): fut = ensure_future(self._wakeup(), loop=self._loop) return fut - @asyncio.coroutine - def _create_conn_cur(self, name=None, cursor_factory=None, - scrollable=None, withhold=False, *, timeout=None): - - conn = yield from self.acquire() - try: - cur = yield from conn.cursor(name=name, - cursor_factory=cursor_factory, - scrollable=scrollable, - withhold=withhold, - timeout=timeout) - except: - self.release(conn) - raise - - return conn, cur - def cursor(self, name=None, cursor_factory=None, scrollable=None, withhold=False, *, timeout=None): cursor_kwargs = dict(name=name, cursor_factory=cursor_factory, From 5da64edc22529c6b7c246e513845d2107c84fde0 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 17 Feb 2017 16:20:15 -0800 Subject: [PATCH 16/33] pep --- aiopg/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aiopg/utils.py b/aiopg/utils.py index 7174f52c..dd81350d 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -275,7 +275,8 @@ def __aenter__(self): # this will create the connection yield from self._pool.__aenter__() - self._cur = yield from self._pool.conn.cursor(**self._cursor_kwargs) + self._cur = yield from self._pool.conn.cursor( + **self._cursor_kwargs) return self._cur @asyncio.coroutine From 375c6bf61f00d0c52002890408ec2b0720f66c99 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 4 Jul 2017 13:20:03 -0700 Subject: [PATCH 17/33] simplify + update deps - simplifies changes to _PoolCursorContextManager - bumps up docker client to work with latest docker for mac - bumps psycopg2 to latest release for testing - fixes issue with dsn comparison --- aiopg/utils.py | 42 ++++++++++++++++++---------------------- requirements.txt | 6 +++--- tests/conftest.py | 25 +++++++++++------------- tests/test_connection.py | 28 ++++++++++++++++++--------- tests/test_sa_engine.py | 13 ++++++++++--- 5 files changed, 62 insertions(+), 52 deletions(-) diff --git a/aiopg/utils.py b/aiopg/utils.py index dd81350d..23a638b1 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -1,6 +1,5 @@ import asyncio import sys -import warnings PY_35 = sys.version_info >= (3, 5) PY_352 = sys.version_info >= (3, 5, 2) @@ -230,7 +229,7 @@ def __exit__(self, *args): self._cur = None @asyncio.coroutine - def _init_pool(self, with_aenter): + def _init_cursor(self, with_aenter): assert not self._cur if with_aenter: @@ -241,18 +240,23 @@ def _init_pool(self, with_aenter): # self._pool now morphs into a _PoolConnectionContextManager self._pool = _PoolConnectionContextManager(self._pool, conn) - @asyncio.coroutine - def _yield_await_init(self): - yield from self._init_pool(False) - self._cur = yield from self._pool.conn.cursor(**self._cursor_kwargs) - return self + if with_aenter: + # this will create the connection + yield from self._pool.__aenter__() + self._cur = yield from self._pool.conn.cursor( + **self._cursor_kwargs) + + return self._cur + else: + self._cur = yield from self._pool.conn.cursor( + **self._cursor_kwargs) + return self + @asyncio.coroutine def __iter__(self): # This will get hit if you use "yield from pool.cursor()" - if PY_35: - warnings.warn("This usage is deprecated, use 'async with` syntax", - DeprecationWarning) - return self._yield_await_init() + result = yield from self._init_cursor(False) + return result def __await__(self): # This will get hit directly if you "await pool.cursor()" @@ -260,24 +264,16 @@ def __await__(self): # https://magicstack.github.io/asyncpg/current/_modules/asyncpg/pool.html # however since `self._init()` is an "asyncio.coroutine" we can't use # just return self._init().__await__() as that returns a generator - # witout an "__await__" attribute, and we can't return a coroutine from + # without an "__await__" attribute and we can't return a coroutine from # here - if PY_35: - warnings.warn("This usage is deprecated, use 'async with` syntax", - DeprecationWarning) - value = yield from self._yield_await_init() + value = yield from self._init_cursor(False) return value if PY_35: @asyncio.coroutine def __aenter__(self): - yield from self._init_pool(True) - - # this will create the connection - yield from self._pool.__aenter__() - self._cur = yield from self._pool.conn.cursor( - **self._cursor_kwargs) - return self._cur + value = yield from self._init_cursor(True) + return value @asyncio.coroutine def __aexit__(self, exc_type, exc_val, exc_tb): diff --git a/requirements.txt b/requirements.txt index 79f2310c..545a6f95 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ coverage==4.4.1 -docker-py==1.10.6 +docker==2.2.1 docutils==0.13.1 ipdb==0.10.3 ipython==6.1.0 @@ -9,10 +9,10 @@ setuptools==36.0.1 -e .[sa] sphinx==1.6.2 tox==2.7.0 -pytest==3.1.2 +pytest==3.1.3 pytest-cov==2.5.1 pytest-sugar==0.8.0 pytest-timeout==1.2.0 sphinxcontrib-asyncio==0.2.0 sqlalchemy==1.1.11 -psycopg2==2.6.2 +psycopg2==2.7.1 diff --git a/tests/conftest.py b/tests/conftest.py index 6be30172..667a55bf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,7 @@ import uuid import warnings -from docker import Client as DockerClient +from docker.client import DockerClient import aiopg from aiopg import sa @@ -126,20 +126,20 @@ def pytest_generate_tests(metafunc): @pytest.yield_fixture(scope='session') def pg_server(unused_port, docker, session_id, pg_tag, request): if not request.config.option.no_pull: - docker.pull('postgres:{}'.format(pg_tag)) - container = docker.create_container( + docker.images.pull('postgres:{}'.format(pg_tag)) + container = docker.containers.create( image='postgres:{}'.format(pg_tag), name='aiopg-test-server-{}-{}'.format(pg_tag, session_id), - ports=[5432], + ports={5432: 5432}, detach=True, ) - docker.start(container=container['Id']) - inspection = docker.inspect_container(container['Id']) - host = inspection['NetworkSettings']['IPAddress'] + container.start() + container.reload() + host = container.attrs['NetworkSettings']['IPAddress'] pg_params = dict(database='postgres', user='postgres', password='mysecretpassword', - host=host, + host='localhost', port=5432) delay = 0.001 for i in range(100): @@ -155,18 +155,15 @@ def pg_server(unused_port, docker, session_id, pg_tag, request): delay *= 2 else: pytest.fail("Cannot start postgres server") - container['host'] = host - container['port'] = 5432 - container['pg_params'] = pg_params + container.attrs['pg_params'] = pg_params yield container - docker.kill(container=container['Id']) - docker.remove_container(container['Id']) + container.remove(force=True) @pytest.fixture def pg_params(pg_server): - return dict(**pg_server['pg_params']) + return dict(**pg_server.attrs['pg_params']) @pytest.yield_fixture() diff --git a/tests/test_connection.py b/tests/test_connection.py index 98b94b88..e9356df5 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -147,10 +147,18 @@ def test_set_session(connect): @asyncio.coroutine def test_dsn(connect, pg_params): conn = yield from connect() + + pg_params = pg_params.copy() pg_params['password'] = 'x' * len(pg_params['password']) - dsn = ('dbname={database} user={user} password={password} ' - 'host={host} port={port}').format_map(pg_params) - assert dsn == conn.dsn + + pg_params['dbname'] = pg_params['database'] + del pg_params['database'] + + pg_params['port'] = str(pg_params['port']) + + # dictionary keys are unsorted so we need this hack + dsn_params = dict([tpl.split('=') for tpl in conn.dsn.split(' ')]) + assert dsn_params == pg_params @asyncio.coroutine @@ -207,11 +215,12 @@ def test_autocommit(connect): def test_isolation_level(connect): conn = yield from connect() - assert 0 == conn.isolation_level + assert psycopg2.extensions.ISOLATION_LEVEL_DEFAULT == conn.isolation_level with pytest.raises(psycopg2.ProgrammingError): - yield from conn.set_isolation_level(1) + yield from conn.set_isolation_level( + psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED) - assert 0 == conn.isolation_level + assert psycopg2.extensions.ISOLATION_LEVEL_DEFAULT == conn.isolation_level @asyncio.coroutine @@ -494,7 +503,7 @@ def test_connect_to_unsupported_port(unused_port, loop, pg_params): pg_params['port'] = port with pytest.raises(psycopg2.OperationalError): - yield from aiopg.connect(loop=loop, **pg_params) + yield from aiopg.connect(loop=loop, timeout=3, **pg_params) @asyncio.coroutine @@ -619,7 +628,7 @@ def test_close_cursor_on_timeout_error(connect): @asyncio.coroutine def test_issue_111_crash_on_connect_error(loop): import aiopg.connection - with pytest.raises(psycopg2.OperationalError): + with pytest.raises(psycopg2.ProgrammingError): yield from aiopg.connection.connect('baddsn:1', loop=loop) @@ -681,7 +690,8 @@ def test_connection_on_server_restart(connect, pg_server, docker): yield from cur.execute('SELECT 1') ret = yield from cur.fetchone() assert (1,) == ret - docker.restart(container=pg_server['Id']) + + pg_server.restart() with pytest.raises(psycopg2.OperationalError): yield from cur.execute('SELECT 1') diff --git a/tests/test_sa_engine.py b/tests/test_sa_engine.py index 2493b813..93647ff0 100644 --- a/tests/test_sa_engine.py +++ b/tests/test_sa_engine.py @@ -39,10 +39,17 @@ def test_driver(engine): def test_dsn(engine, pg_params): + pg_params = pg_params.copy() pg_params['password'] = 'x' * len(pg_params['password']) - dsn = ('dbname={database} user={user} password={password} ' - 'host={host} port={port}').format_map(pg_params) - assert dsn == engine.dsn + + pg_params['dbname'] = pg_params['database'] + del pg_params['database'] + + pg_params['port'] = str(pg_params['port']) + + # dictionary keys are unsorted so we need this hack + dsn_params = dict([tpl.split('=') for tpl in engine.dsn.split(' ')]) + assert dsn_params == pg_params def test_minsize(engine): From dabca4effdf25c3dea4a7e5ed7019f2218713344 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 4 Jul 2017 13:26:17 -0700 Subject: [PATCH 18/33] remove unused var --- tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 667a55bf..bccf7bf9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -135,7 +135,6 @@ def pg_server(unused_port, docker, session_id, pg_tag, request): ) container.start() container.reload() - host = container.attrs['NetworkSettings']['IPAddress'] pg_params = dict(database='postgres', user='postgres', password='mysecretpassword', From cc05d18b2daf90ebaaded711156eaa9fd975c27b Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 4 Jul 2017 13:39:07 -0700 Subject: [PATCH 19/33] attempt switching to the latest docker --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 848f4238..98caaacf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,12 @@ cache: before_cache: - rm -f $HOME/.cache/pip/log/debug.log +before_install: + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + - sudo apt-get update + - sudo apt-get -y install docker-ce + after_script: - codecov From c8b506dfa24d175bba8756de5dc662d555a11d57 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 4 Jul 2017 13:50:53 -0700 Subject: [PATCH 20/33] another attempt --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 98caaacf..443c98ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,10 +34,14 @@ before_cache: - rm -f $HOME/.cache/pip/log/debug.log before_install: + - docker --version # for verification - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - sudo apt-get update - sudo apt-get -y install docker-ce + - docker --version # for verification + - sudo service docker restart + after_script: - codecov From 1e964ffbd2b586670f56019df10268cfbd9a6e02 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 4 Jul 2017 14:03:17 -0700 Subject: [PATCH 21/33] revert docker client upgrade --- .travis.yml | 10 ---------- requirements.txt | 4 ++-- tests/conftest.py | 22 +++++++++++++--------- tests/test_connection.py | 2 +- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 443c98ed..848f4238 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,16 +33,6 @@ cache: before_cache: - rm -f $HOME/.cache/pip/log/debug.log -before_install: - - docker --version # for verification - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - sudo apt-get update - - sudo apt-get -y install docker-ce - - docker --version # for verification - - sudo service docker restart - - after_script: - codecov diff --git a/requirements.txt b/requirements.txt index 545a6f95..3dddaf22 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ coverage==4.4.1 -docker==2.2.1 +docker-py==1.10.6 docutils==0.13.1 ipdb==0.10.3 ipython==6.1.0 @@ -9,7 +9,7 @@ setuptools==36.0.1 -e .[sa] sphinx==1.6.2 tox==2.7.0 -pytest==3.1.3 +pytest==3.1.2 pytest-cov==2.5.1 pytest-sugar==0.8.0 pytest-timeout==1.2.0 diff --git a/tests/conftest.py b/tests/conftest.py index bccf7bf9..8fcffad0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,7 @@ import uuid import warnings -from docker.client import DockerClient +from docker import Client as DockerClient import aiopg from aiopg import sa @@ -126,19 +126,20 @@ def pytest_generate_tests(metafunc): @pytest.yield_fixture(scope='session') def pg_server(unused_port, docker, session_id, pg_tag, request): if not request.config.option.no_pull: - docker.images.pull('postgres:{}'.format(pg_tag)) - container = docker.containers.create( + docker.pull('postgres:{}'.format(pg_tag)) + container = docker.create_container( image='postgres:{}'.format(pg_tag), name='aiopg-test-server-{}-{}'.format(pg_tag, session_id), - ports={5432: 5432}, + ports=[5432], detach=True, ) - container.start() - container.reload() + docker.start(container=container['Id']) + inspection = docker.inspect_container(container['Id']) + host = inspection['NetworkSettings']['IPAddress'] pg_params = dict(database='postgres', user='postgres', password='mysecretpassword', - host='localhost', + host=host, port=5432) delay = 0.001 for i in range(100): @@ -154,10 +155,13 @@ def pg_server(unused_port, docker, session_id, pg_tag, request): delay *= 2 else: pytest.fail("Cannot start postgres server") - container.attrs['pg_params'] = pg_params + container['host'] = host + container['port'] = 5432 + container['pg_params'] = pg_params yield container - container.remove(force=True) + docker.kill(container=container['Id']) + docker.remove_container(container['Id']) @pytest.fixture diff --git a/tests/test_connection.py b/tests/test_connection.py index e9356df5..1886bb84 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -691,7 +691,7 @@ def test_connection_on_server_restart(connect, pg_server, docker): ret = yield from cur.fetchone() assert (1,) == ret - pg_server.restart() + docker.restart(container=pg_server['Id']) with pytest.raises(psycopg2.OperationalError): yield from cur.execute('SELECT 1') From 1ca9b5fb2ebd64e9cccadf689e98354e11fa929c Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 4 Jul 2017 14:04:13 -0700 Subject: [PATCH 22/33] revert docker client upgrade changes --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8fcffad0..6be30172 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -166,7 +166,7 @@ def pg_server(unused_port, docker, session_id, pg_tag, request): @pytest.fixture def pg_params(pg_server): - return dict(**pg_server.attrs['pg_params']) + return dict(**pg_server['pg_params']) @pytest.yield_fixture() From 0a57593499de6bad63f9941da5745ca2cce42331 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 4 Jul 2017 14:25:12 -0700 Subject: [PATCH 23/33] add back missing change --- aiopg/cursor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aiopg/cursor.py b/aiopg/cursor.py index f3db3304..65d549fe 100644 --- a/aiopg/cursor.py +++ b/aiopg/cursor.py @@ -112,6 +112,7 @@ def execute(self, operation, parameters=None, *, timeout=None): try: yield from self._conn._poll(waiter, timeout) except asyncio.TimeoutError: + yield from self._conn.cancel() self._impl.close() raise From a5cec12b639cb41a285adc4847b637c48cc750b7 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 4 Jul 2017 14:27:55 -0700 Subject: [PATCH 24/33] revert as this was later refactored https://github.com/aio-libs/aiopg/commit/2a72b6097e9d0a4c23318b03e57e107 67acbbdd4#diff-3416729d6745fac0ca3dd44b22a69068 --- aiopg/cursor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aiopg/cursor.py b/aiopg/cursor.py index 65d549fe..f3db3304 100644 --- a/aiopg/cursor.py +++ b/aiopg/cursor.py @@ -112,7 +112,6 @@ def execute(self, operation, parameters=None, *, timeout=None): try: yield from self._conn._poll(waiter, timeout) except asyncio.TimeoutError: - yield from self._conn.cancel() self._impl.close() raise From bdc39c3825458d09fc8abe097411a4f71b63cc13 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 12 Sep 2017 12:20:55 -0700 Subject: [PATCH 25/33] add changes --- CHANGES.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 002b82ac..0e2221f2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,10 @@ CHANGES ------- +X.X.X (XXXX-XX-XX) +^^^^^^^^^^^^^^^^^^^ + +* Add `async with` support to cursor context + 0.13.1 (2017-09-10) ^^^^^^^^^^^^^^^^^^^ From 351bcd8119d85e4babb1902693d01f331235916d Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Thu, 15 Mar 2018 14:10:41 -0700 Subject: [PATCH 26/33] fix merge --- tests/test_sa_engine.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_sa_engine.py b/tests/test_sa_engine.py index 2f323f33..07d99bf3 100644 --- a/tests/test_sa_engine.py +++ b/tests/test_sa_engine.py @@ -42,10 +42,11 @@ def test_driver(engine): def test_dsn(engine, pg_params): params = pg_params.copy() params['password'] = 'xxx' - params['dbname'] = params['database'].pop() + params['dbname'] = params.pop('database') params['port'] = str(params['port']) assert parse_dsn(engine.dsn) == params + def test_minsize(engine): assert 1 == engine.minsize From 60e957313534d7189e392a349c24a7b99d4d214f Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Thu, 15 Mar 2018 14:14:29 -0700 Subject: [PATCH 27/33] simplification --- tests/test_connection.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index 8402244e..8e81f90f 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -9,6 +9,7 @@ import time import sys +from psycopg2.extensions import parse_dsn from aiopg.connection import Connection, TIMEOUT from aiopg.cursor import Cursor from aiopg.utils import ensure_future, create_future @@ -150,14 +151,10 @@ def test_dsn(connect, pg_params): pg_params = pg_params.copy() pg_params['password'] = 'x' * len(pg_params['password']) - pg_params['dbname'] = pg_params['database'] - del pg_params['database'] - + pg_params['dbname'] = pg_params.pop('database') pg_params['port'] = str(pg_params['port']) - # dictionary keys are unsorted so we need this hack - dsn_params = dict([tpl.split('=') for tpl in conn.dsn.split(' ')]) - assert dsn_params == pg_params + assert parse_dsn(conn.dsn) == pg_params @asyncio.coroutine From 256f7b8dee69c1cc54baae611cc66be1206ff50b Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Sat, 17 Mar 2018 00:24:53 -0700 Subject: [PATCH 28/33] add missing import --- tests/pep492/test_async_await.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pep492/test_async_await.py b/tests/pep492/test_async_await.py index efa3e95d..c9317f2f 100644 --- a/tests/pep492/test_async_await.py +++ b/tests/pep492/test_async_await.py @@ -1,3 +1,4 @@ +import asyncio import pytest import psycopg2 import aiopg @@ -84,7 +85,7 @@ async def test_pool_context_manager_timeout(pg_params, loop): fut.cancel() cursor_ctx = await pool.cursor() with cursor_ctx as cursor: - resp = await cursor.execute('SELECT 42;') + _ = await cursor.execute('SELECT 42;') resp = await cursor.fetchone() assert resp == (42, ) From 02ca4e1a7fd0644fddc3df8390c3a8eda169656c Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Sat, 17 Mar 2018 00:34:17 -0700 Subject: [PATCH 29/33] pep fix --- tests/pep492/test_async_await.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pep492/test_async_await.py b/tests/pep492/test_async_await.py index c9317f2f..09a3d27f 100644 --- a/tests/pep492/test_async_await.py +++ b/tests/pep492/test_async_await.py @@ -85,7 +85,7 @@ async def test_pool_context_manager_timeout(pg_params, loop): fut.cancel() cursor_ctx = await pool.cursor() with cursor_ctx as cursor: - _ = await cursor.execute('SELECT 42;') + await cursor.execute('SELECT 42;') resp = await cursor.fetchone() assert resp == (42, ) From cbd93c556f6a66d83fff107335274e13fb334d88 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Sat, 17 Mar 2018 00:43:53 -0700 Subject: [PATCH 30/33] integration fix --- tests/test_connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index 8e81f90f..39692712 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -150,7 +150,7 @@ def test_dsn(connect, pg_params): conn = yield from connect() pg_params = pg_params.copy() - pg_params['password'] = 'x' * len(pg_params['password']) + pg_params['password'] = 'xxx' pg_params['dbname'] = pg_params.pop('database') pg_params['port'] = str(pg_params['port']) From 4e52944f7be8d7c6fa4a19f702cbd80154a98d68 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Sat, 17 Mar 2018 00:59:06 -0700 Subject: [PATCH 31/33] more merge fixes --- aiopg/cursor.py | 9 ++++++++- aiopg/utils.py | 7 ------- setup.py | 9 ++++++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/aiopg/cursor.py b/aiopg/cursor.py index 1ca7f8d2..b15eba0c 100644 --- a/aiopg/cursor.py +++ b/aiopg/cursor.py @@ -48,7 +48,14 @@ def description(self): def close(self): """Close the cursor now.""" - self._impl.close() + try: + self._impl.close() + except psycopg2.ProgrammingError: + # seen instances where the cursor fails to close: + # https://github.com/aio-libs/aiopg/issues/364 + # We close it here so we don't return a bad connection to the pool + self._conn.close() + raise @property def closed(self): diff --git a/aiopg/utils.py b/aiopg/utils.py index 15b862fe..4be1793e 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -240,13 +240,6 @@ def __enter__(self): def __exit__(self, *args): try: self._cur.close() - self._pool.__exit__(*args) - except psycopg2.ProgrammingError: - # seen instances where the cursor fails to close: - # https://github.com/aio-libs/aiopg/issues/364 - # We close it here so we don't return a bad connection to the pool - self._conn.close() - raise finally: try: self._pool.__exit__(*args) diff --git a/setup.py b/setup.py index 289a0aed..ad8882ea 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,10 @@ install_requires = ['psycopg2>=2.7.0'] +extras_require = { + 'sa': ['sqlalchemy>=1.1'] +} + PY_VER = sys.version_info if PY_VER < (3, 4): @@ -15,8 +19,6 @@ def read(f): return open(os.path.join(os.path.dirname(__file__), f)).read().strip() -extras_require = {'sa': ['sqlalchemy>=1.1'], } - def read_version(): regexp = re.compile(r"^__version__\W*=\W*'([\d.abrc]+)'") @@ -29,6 +31,7 @@ def read_version(): else: raise RuntimeError('Cannot find version in aiopg/__init__.py') + classifiers = [ 'License :: OSI Approved :: BSD License', 'Intended Audience :: Developers', @@ -48,7 +51,7 @@ def read_version(): setup(name='aiopg', version=read_version(), - description=('Postgres integration with asyncio.'), + description='Postgres integration with asyncio.', long_description='\n\n'.join((read('README.rst'), read('CHANGES.txt'))), classifiers=classifiers, platforms=['POSIX'], From f613762b6e6fa8837b68bfead04e9f630d86f617 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Sat, 17 Mar 2018 01:03:35 -0700 Subject: [PATCH 32/33] PEP fix --- aiopg/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aiopg/utils.py b/aiopg/utils.py index 4be1793e..bae45ac0 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -1,6 +1,5 @@ import asyncio import sys -import psycopg2 PY_35 = sys.version_info >= (3, 5) PY_352 = sys.version_info >= (3, 5, 2) From 1133c6ae3012e0c423ae907238b1132c14e7b8c2 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 16 Oct 2018 14:19:13 -0700 Subject: [PATCH 33/33] fix merge bug --- setup.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup.py b/setup.py index d6940012..fb752e79 100644 --- a/setup.py +++ b/setup.py @@ -7,10 +7,6 @@ install_requires = ['psycopg2>=2.7.0'] extras_require = {'sa': ['sqlalchemy>=1.1']} -extras_require = { - 'sa': ['sqlalchemy>=1.1'] -} - PY_VER = sys.version_info if PY_VER < (3, 4):