From 37ed426fb21582736f078db05889e6f20c6a68bc Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 09:33:28 +0530 Subject: [PATCH 01/21] Changes for Python 3.11 support --- .pylintrc | 4 ++-- benchmark/requirements.txt | 2 +- examples/web_scraper.py | 7 +++++++ proxy/common/types.py | 8 ++++---- proxy/core/base/tcp_server.py | 5 +++++ proxy/core/work/fd/fd.py | 15 +++++++++++++++ proxy/core/work/local.py | 5 +++++ proxy/core/work/remote.py | 5 +++++ proxy/http/server/plugin.py | 2 +- requirements-testing.txt | 2 +- 10 files changed, 46 insertions(+), 9 deletions(-) diff --git a/.pylintrc b/.pylintrc index 25434cea23..18139d0a32 100644 --- a/.pylintrc +++ b/.pylintrc @@ -419,7 +419,7 @@ contextmanager-decorators=contextlib.contextmanager # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. -generated-members= +generated-members=os,io # Tells whether missing members accessed in mixin class should be ignored. A # mixin class is detected if its name ends with "mixin" (case insensitive). @@ -446,7 +446,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis). It # supports qualified module names, as well as Unix pattern matching. -ignored-modules= +ignored-modules=abc # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. diff --git a/benchmark/requirements.txt b/benchmark/requirements.txt index c9fbac1fc8..1b0d1fe2f0 100644 --- a/benchmark/requirements.txt +++ b/benchmark/requirements.txt @@ -1,4 +1,4 @@ -aiohttp==3.8.1 +aiohttp==3.8.2 # Blacksheep depends upon essentials_openapi which is pinned to pyyaml==5.4.1 # and pyyaml>5.3.1 is broken for cython 3 # See https://github.com/yaml/pyyaml/issues/724#issuecomment-1638587228 diff --git a/examples/web_scraper.py b/examples/web_scraper.py index daf31d612f..618b7bf709 100644 --- a/examples/web_scraper.py +++ b/examples/web_scraper.py @@ -9,6 +9,8 @@ :license: BSD, see LICENSE for more details. """ import time +from abc import abstractmethod +from typing import Any from proxy import Proxy from proxy.core.work import Work @@ -52,6 +54,11 @@ async def handle_events( Return True to shutdown work.""" return False + @staticmethod + @abstractmethod + def create(*args: Any) -> TcpClientConnection: + raise NotImplementedError() + if __name__ == '__main__': with Proxy( diff --git a/proxy/common/types.py b/proxy/common/types.py index 984cc3bdd3..78f3bd5040 100644 --- a/proxy/common/types.py +++ b/proxy/common/types.py @@ -14,7 +14,7 @@ import queue import socket import ipaddress -from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union, TypeVar if TYPE_CHECKING: # pragma: no cover @@ -34,8 +34,8 @@ HostPort = Tuple[str, int] if sys.version_info.minor == 6: - RePattern = Any + RePattern = TypeVar('RePattern', bound=Any) elif sys.version_info.minor in (7, 8): - RePattern = re.Pattern # type: ignore + RePattern = TypeVar('RePattern', bound=re.Pattern) # type: ignore else: - RePattern = re.Pattern[Any] # type: ignore + RePattern = TypeVar('RePattern', bound=re.Pattern[Any]) # type: ignore diff --git a/proxy/core/base/tcp_server.py b/proxy/core/base/tcp_server.py index 842e255a7c..4d32949d55 100644 --- a/proxy/core/base/tcp_server.py +++ b/proxy/core/base/tcp_server.py @@ -240,3 +240,8 @@ def _optionally_wrap_socket(self, conn: socket.socket) -> TcpOrTlsSocket: conn = wrap_socket(conn, self.flags.keyfile, self.flags.certfile) self.work._conn = conn return conn + + @staticmethod + @abstractmethod + def create(*args: Any) -> T: + raise NotImplementedError() diff --git a/proxy/core/work/fd/fd.py b/proxy/core/work/fd/fd.py index 730019123c..a13024769c 100644 --- a/proxy/core/work/fd/fd.py +++ b/proxy/core/work/fd/fd.py @@ -9,7 +9,9 @@ :license: BSD, see LICENSE for more details. """ import socket +import asyncio import logging +from abc import abstractmethod from typing import Any, TypeVar, Optional from ...event import eventNames @@ -47,3 +49,16 @@ def work(self, *args: Any) -> None: exc_info=e, ) self._cleanup(fileno) + + @property + @abstractmethod + def loop(self) -> Optional[asyncio.AbstractEventLoop]: + raise NotImplementedError() + + @abstractmethod + def receive_from_work_queue(self) -> bool: + raise NotImplementedError() + + @abstractmethod + def work_queue_fileno(self) -> Optional[int]: + raise NotImplementedError() diff --git a/proxy/core/work/local.py b/proxy/core/work/local.py index 0745e817a7..6b3ae9936d 100644 --- a/proxy/core/work/local.py +++ b/proxy/core/work/local.py @@ -11,6 +11,7 @@ import queue import asyncio import contextlib +from abc import abstractmethod from typing import Any, Optional from .threadless import Threadless @@ -40,3 +41,7 @@ def receive_from_work_queue(self) -> bool: return True self.work(work) return False + + @abstractmethod + def work(self, *args: Any) -> None: + raise NotImplementedError() diff --git a/proxy/core/work/remote.py b/proxy/core/work/remote.py index afac2ebef8..5734438591 100644 --- a/proxy/core/work/remote.py +++ b/proxy/core/work/remote.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for more details. """ import asyncio +from abc import abstractmethod from typing import Any, Optional from multiprocessing import connection @@ -37,3 +38,7 @@ def close_work_queue(self) -> None: def receive_from_work_queue(self) -> bool: self.work(self.work_queue.recv()) return False + + @abstractmethod + def work(self, *args: Any) -> None: + raise NotImplementedError() diff --git a/proxy/http/server/plugin.py b/proxy/http/server/plugin.py index 434fba24b8..48cc5eb2a7 100644 --- a/proxy/http/server/plugin.py +++ b/proxy/http/server/plugin.py @@ -169,7 +169,7 @@ def before_routing(self, request: HttpParser) -> Optional[HttpParser]: def handle_route(self, request: HttpParser, pattern: RePattern) -> Url: """Implement this method if you have configured dynamic routes.""" - pass + raise NotImplementedError() def regexes(self) -> List[str]: """Helper method to return list of route regular expressions.""" diff --git a/requirements-testing.txt b/requirements-testing.txt index 13eba0f9bb..6fc05df56f 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -12,7 +12,7 @@ mypy==0.971 py-spy==0.3.12 tox==3.28.0 mccabe==0.6.1 -pylint==2.13.7 +pylint==3.1.0 rope==1.1.1 # Required by test_http2.py httpx==0.22.0 From 547967be0d8b7bcb8386a7b88e8a0df862d3398b Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 09:34:58 +0530 Subject: [PATCH 02/21] Updated README.md for versioning info --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ab37c7e84e..25cc3a5f14 100644 --- a/README.md +++ b/README.md @@ -2366,7 +2366,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--filtered-client-ips FILTERED_CLIENT_IPS] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] -proxy.py v2.4.4rc5.dev36+g6c9d0315.d20240411 +proxy.py v2.4.4rc6.dev0+g81510a0c.d20240413 options: -h, --help show this help message and exit @@ -2489,8 +2489,8 @@ options: Default: None. Signing certificate to use for signing dynamically generated HTTPS certificates. If used, must also pass --ca-key-file and --ca-signing-key-file - --ca-file CA_FILE Default: /Users/abhinavsingh/Dev/proxy.py/.venv/lib/py - thon3.10/site-packages/certifi/cacert.pem. Provide + --ca-file CA_FILE Default: /Users/abhinavsingh/Dev/proxy.py/.venv3118/li + b/python3.11/site-packages/certifi/cacert.pem. Provide path to custom CA bundle for peer certificate verification --ca-signing-key-file CA_SIGNING_KEY_FILE From dbcff7afe92006c445407c0bffebf1650f570a2d Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 09:40:38 +0530 Subject: [PATCH 03/21] Update `httpx==0.27.0` to avoid `cgi` deprecation warning from pytest on Python 3.11 --- .github/workflows/test-library.yml | 3 ++- README.md | 2 +- requirements-testing.txt | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index a9fcad5160..f523868e8d 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -446,8 +446,9 @@ jobs: # NOTE: The latest and the lowest supported Pythons are prioritized # NOTE: to improve the responsiveness. It's nice to see the most # NOTE: important results first. - - '3.10' + - '3.11' - 3.6 + - '3.10' - 3.9 - 3.8 - 3.7 diff --git a/README.md b/README.md index 25cc3a5f14..57ed8df66d 100644 --- a/README.md +++ b/README.md @@ -2366,7 +2366,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--filtered-client-ips FILTERED_CLIENT_IPS] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] -proxy.py v2.4.4rc6.dev0+g81510a0c.d20240413 +proxy.py v2.4.4rc6.dev2+g547967be.d20240413 options: -h, --help show this help message and exit diff --git a/requirements-testing.txt b/requirements-testing.txt index 6fc05df56f..358cee11f0 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -15,7 +15,7 @@ mccabe==0.6.1 pylint==3.1.0 rope==1.1.1 # Required by test_http2.py -httpx==0.22.0 +httpx==0.27.0 h2==4.1.0 hpack==4.0.0 hyperframe==6.0.1 From 300cb30f38b6b9f415878786268704e4cbd4ff05 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 10:14:36 +0530 Subject: [PATCH 04/21] Make tests work for 3.11 --- .pylintrc | 4 ++-- README.md | 2 +- proxy/plugin/proxy_pool.py | 1 + proxy/testing/test_case.py | 1 + tests/http/proxy/test_http2.py | 4 +--- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.pylintrc b/.pylintrc index 18139d0a32..ccb191138c 100644 --- a/.pylintrc +++ b/.pylintrc @@ -605,5 +605,5 @@ preferred-modules= # Exceptions that will emit a warning when being caught. Defaults to # "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception +overgeneral-exceptions=builtins.BaseException, + builtins.Exception diff --git a/README.md b/README.md index 57ed8df66d..2015ef78cd 100644 --- a/README.md +++ b/README.md @@ -2366,7 +2366,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--filtered-client-ips FILTERED_CLIENT_IPS] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] -proxy.py v2.4.4rc6.dev2+g547967be.d20240413 +proxy.py v2.4.4rc6.dev3+gdbcff7af.d20240413 options: -h, --help show this help message and exit diff --git a/proxy/plugin/proxy_pool.py b/proxy/plugin/proxy_pool.py index 6e04034721..c244adeda1 100644 --- a/proxy/plugin/proxy_pool.py +++ b/proxy/plugin/proxy_pool.py @@ -186,6 +186,7 @@ def handle_upstream_chunk(self, chunk: memoryview) -> Optional[memoryview]: """Will never be called since we didn't establish an upstream connection.""" if not self.upstream: return chunk + # pylint: disable=broad-exception-raised raise Exception("This should have never been called") def on_upstream_connection_close(self) -> None: diff --git a/proxy/testing/test_case.py b/proxy/testing/test_case.py index afd13fe1af..4c2740e0b6 100644 --- a/proxy/testing/test_case.py +++ b/proxy/testing/test_case.py @@ -42,6 +42,7 @@ def setUpClass(cls) -> None: cls.PROXY.flags.plugins[b'HttpProxyBasePlugin'].append( CacheResponsesPlugin, ) + # pylint: disable=unnecessary-dunder-call cls.PROXY.__enter__() assert cls.PROXY.acceptors cls.wait_for_server(cls.PROXY.flags.port) diff --git a/tests/http/proxy/test_http2.py b/tests/http/proxy/test_http2.py index a8f1eeed08..e4a85940f0 100644 --- a/tests/http/proxy/test_http2.py +++ b/tests/http/proxy/test_http2.py @@ -22,9 +22,7 @@ def test_http2_via_proxy(self) -> None: headers={'accept': 'application/json'}, verify=httpx.create_ssl_context(http2=True), timeout=httpx.Timeout(timeout=5.0), - proxies={ - 'all://': 'http://localhost:%d' % self.PROXY.flags.port, - }, + proxy='http://localhost:%d' % self.PROXY.flags.port, ) self.assertEqual(response.status_code, 200) From 0b7cc66eb310863f09f59b805d03d35c8bf19bb4 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 10:17:00 +0530 Subject: [PATCH 05/21] Declare support for 3.11 --- README.md | 2 +- setup.cfg | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2015ef78cd..1ced50de9a 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ [![iOS, iOS Simulator](https://img.shields.io/static/v1?label=tested%20with&message=iOS%20%F0%9F%93%B1%20%7C%20iOS%20Simulator%20%F0%9F%93%B1&color=darkgreen&style=for-the-badge)](https://abhinavsingh.com/proxy-py-a-lightweight-single-file-http-proxy-server-in-python/) [![pypi version](https://img.shields.io/pypi/v/proxy.py?style=flat-square)](https://pypi.org/project/proxy.py/) -[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10&color=blue&style=flat-square)](https://www.python.org/) +[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10%20%7C%203.11&color=blue&style=flat-square)](https://www.python.org/) [![Checked with mypy](https://img.shields.io/static/v1?label=MyPy&message=checked&color=blue&style=flat-square)](http://mypy-lang.org/) [![doc](https://img.shields.io/readthedocs/proxypy/latest?style=flat-square&color=darkgreen)](https://proxypy.readthedocs.io/) diff --git a/setup.cfg b/setup.cfg index 124681dd94..fb14a6515e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -65,6 +65,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Topic :: Internet Topic :: Internet :: Proxy Servers From 2699dac6af0a5268780100f0d42c28f7efc92ff2 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 10:17:37 +0530 Subject: [PATCH 06/21] Use 3.11-alpine for Docker images --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index dcfb3611b1..5277069a5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-alpine as base +FROM python:3.11-alpine as base LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ com.abhinavsingh.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ From ef0f819eb8d60ff73de29557eb0791d27e4b0c9a Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 10:23:35 +0530 Subject: [PATCH 07/21] Preserve pylint version for `python_version <= 3.10` --- requirements-testing.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index 358cee11f0..8b847e3643 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -12,7 +12,8 @@ mypy==0.971 py-spy==0.3.12 tox==3.28.0 mccabe==0.6.1 -pylint==3.1.0 +pylint==2.13.7; python_version <= '3.10' +pylint==3.1.0; python_version > '3.10' rope==1.1.1 # Required by test_http2.py httpx==0.27.0 From 79d0551f1bc250d3dd5ff995f4b13644d785f7c9 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 10:26:18 +0530 Subject: [PATCH 08/21] Preserve httpx version for <= 3.10 --- requirements-testing.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index 8b847e3643..b59dffb459 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -13,10 +13,11 @@ py-spy==0.3.12 tox==3.28.0 mccabe==0.6.1 pylint==2.13.7; python_version <= '3.10' -pylint==3.1.0; python_version > '3.10' +pylint==3.1.0; python_version >= '3.11' rope==1.1.1 # Required by test_http2.py -httpx==0.27.0 +httpx==0.22.0; python_version <= '3.10' +httpx==0.27.0; python_version >= '3.11' h2==4.1.0 hpack==4.0.0 hyperframe==6.0.1 From 11e37a62e361e952bb953ba318b51ce73c84ac79 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 10:47:39 +0530 Subject: [PATCH 09/21] `httpx` usage fix in tests for <=3.10 --- docs/conf.py | 2 ++ tests/http/proxy/test_http2.py | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 40573d5edc..0388bd313e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -321,6 +321,8 @@ (_py_class_role, 'HostPort'), (_py_class_role, 'TcpOrTlsSocket'), (_py_class_role, 're.Pattern'), + (_py_class_role, 'proxy.core.base.tcp_server.T'), + (_py_class_role, 'proxy.common.types.RePattern'), (_py_obj_role, 'proxy.core.work.threadless.T'), (_py_obj_role, 'proxy.core.work.work.T'), (_py_obj_role, 'proxy.core.base.tcp_server.T'), diff --git a/tests/http/proxy/test_http2.py b/tests/http/proxy/test_http2.py index e4a85940f0..6fca13be8c 100644 --- a/tests/http/proxy/test_http2.py +++ b/tests/http/proxy/test_http2.py @@ -8,6 +8,9 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +import sys +from typing import Any, Dict + import httpx from proxy import TestCase @@ -17,12 +20,23 @@ class TestHttp2WithProxy(TestCase): def test_http2_via_proxy(self) -> None: assert self.PROXY + proxy_url = 'http://localhost:%d' % self.PROXY.flags.port + proxies: Dict[str, Any] = ( + {'proxy': proxy_url} + # For Python>=3.11, proxies is deprecated by httpx + if sys.version_info >= (3, 11) + else { + 'proxies': { + 'all://': proxy_url, + }, + } + ) response = httpx.get( 'https://www.google.com', headers={'accept': 'application/json'}, verify=httpx.create_ssl_context(http2=True), timeout=httpx.Timeout(timeout=5.0), - proxy='http://localhost:%d' % self.PROXY.flags.port, + **proxies, ) self.assertEqual(response.status_code, 200) From f4a1163bc6ee14b4462f1a56a555c5de1c0a825d Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 11:05:55 +0530 Subject: [PATCH 10/21] Adjust pylint and pytest for >= 3.11 --- README.md | 2 +- requirements-testing.txt | 21 ++++++++++++++------- tox.ini | 1 - 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1ced50de9a..a6a39b0358 100644 --- a/README.md +++ b/README.md @@ -2366,7 +2366,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--filtered-client-ips FILTERED_CLIENT_IPS] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] -proxy.py v2.4.4rc6.dev3+gdbcff7af.d20240413 +proxy.py v2.4.4rc6.dev9+g11e37a62.d20240413 options: -h, --help show this help message and exit diff --git a/requirements-testing.txt b/requirements-testing.txt index b59dffb459..5ccf9bd99a 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -2,21 +2,28 @@ wheel==0.37.1 python-coveralls==2.9.3 coverage==6.2 flake8==4.0.1 -pytest==7.0.1 -pytest-cov==3.0.0 -pytest-xdist == 2.5.0 -pytest-mock==3.6.1 -pytest-asyncio==0.16.0 +# pytest for Python<3.11 +pytest==7.0.1; python_version < '3.11' +pytest-cov==3.0.0; python_version < '3.11' +pytest-xdist == 2.5.0; python_version < '3.11' +pytest-mock==3.6.1; python_version < '3.11' +pytest-asyncio==0.16.0; python_version < '3.11' +# pytest for Python>=3.11 +pytest==8.1.1; python_version >= '3.11' +pytest-cov==5.0.0; python_version >= '3.11' +pytest-xdist == 3.5.0; python_version >= '3.11' +pytest-mock==3.14.0; python_version >= '3.11' +pytest-asyncio==0.23.6; python_version >= '3.11' autopep8==1.6.0 mypy==0.971 py-spy==0.3.12 tox==3.28.0 mccabe==0.6.1 -pylint==2.13.7; python_version <= '3.10' +pylint==2.13.7; python_version < '3.11' pylint==3.1.0; python_version >= '3.11' rope==1.1.1 # Required by test_http2.py -httpx==0.22.0; python_version <= '3.10' +httpx==0.22.0; python_version < '3.11' httpx==0.27.0; python_version >= '3.11' h2==4.1.0 hpack==4.0.0 diff --git a/tox.ini b/tox.ini index 2a777f9cb5..5e7d706a4b 100644 --- a/tox.ini +++ b/tox.ini @@ -262,7 +262,6 @@ deps = pre-commit pylint >= 2.5.3 pylint-pytest < 1.1.0 - pytest-mock == 3.6.1 -r docs/requirements.in -r requirements-tunnel.txt -r requirements-testing.txt From ac1f05d74676f81b2c0eaed1dc140fb3bcdff99a Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 11:17:05 +0530 Subject: [PATCH 11/21] Use 3.11.8, bad-option-value and httpx proxies fix --- .github/workflows/test-library.yml | 2 +- .pylintrc | 4 ++++ tests/http/proxy/test_http2.py | 8 ++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index f523868e8d..e529f9bf5b 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -446,7 +446,7 @@ jobs: # NOTE: The latest and the lowest supported Pythons are prioritized # NOTE: to improve the responsiveness. It's nice to see the most # NOTE: important results first. - - '3.11' + - '3.11.8' - 3.6 - '3.10' - 3.9 diff --git a/.pylintrc b/.pylintrc index ccb191138c..c705e36297 100644 --- a/.pylintrc +++ b/.pylintrc @@ -131,6 +131,10 @@ disable=raw-checker-failed, useless-return, useless-super-delegation, wrong-import-order, + # Required because to support 3.11 + # we added unnecessary-dunder-call which is not supported for <=3.11 + # see https://github.com/abhinavsingh/proxy.py/actions/runs/8671404475/job/23780537848?pr=1384 + bad-option-value # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/tests/http/proxy/test_http2.py b/tests/http/proxy/test_http2.py index 6fca13be8c..2ef95eaae6 100644 --- a/tests/http/proxy/test_http2.py +++ b/tests/http/proxy/test_http2.py @@ -22,14 +22,14 @@ def test_http2_via_proxy(self) -> None: assert self.PROXY proxy_url = 'http://localhost:%d' % self.PROXY.flags.port proxies: Dict[str, Any] = ( - {'proxy': proxy_url} - # For Python>=3.11, proxies is deprecated by httpx - if sys.version_info >= (3, 11) - else { + { 'proxies': { 'all://': proxy_url, }, } + # For Python>=3.11, proxies keyword is deprecated by httpx + if sys.version_info < (3, 11, 0) + else {'proxy': proxy_url} ) response = httpx.get( 'https://www.google.com', From 38e25a812c0e933f186c3ad5eabbae9ab5564c94 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 12:13:35 +0530 Subject: [PATCH 12/21] tox for 3.11 --- .github/workflows/test-library.yml | 10 +++++++++- Makefile | 2 +- requirements-testing.txt | 3 ++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index e529f9bf5b..b06a48855e 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -501,7 +501,15 @@ jobs: steps.calc-cache-key-py.outputs.py-hash-key }}- ${{ runner.os }}-pip- - - name: Install tox + - name: Install tox for >= 3.11 + if: matrix.python == '3.11.8' + run: >- + python -m + pip install + --user + tox==4.14.2 + - name: Install tox for < 3.11 + if: matrix.python != '3.11.8' run: >- python -m pip install diff --git a/Makefile b/Makefile index f111b2f9e6..e42551765b 100644 --- a/Makefile +++ b/Makefile @@ -120,7 +120,7 @@ lib-mypy: tox -e lint -- mypy --all-files lib-pytest: - $(PYTHON) -m tox -e python -- -v + $(PYTHON) -m tox -e py -- -v lib-test: lib-clean lib-check lib-lint lib-pytest diff --git a/requirements-testing.txt b/requirements-testing.txt index 5ccf9bd99a..3c0da300b0 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -17,7 +17,8 @@ pytest-asyncio==0.23.6; python_version >= '3.11' autopep8==1.6.0 mypy==0.971 py-spy==0.3.12 -tox==3.28.0 +tox==3.28.0; python_version < '3.11' +tox==4.14.2; python_version >= '3.11' mccabe==0.6.1 pylint==2.13.7; python_version < '3.11' pylint==3.1.0; python_version >= '3.11' From a228699acae1a4c563a88f0ea2347610fd2a472f Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 12:16:59 +0530 Subject: [PATCH 13/21] Fix for `TOXENV: py` --- .github/workflows/test-library.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index b06a48855e..85dbbea2a8 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -464,7 +464,7 @@ jobs: env: PY_COLORS: 1 TOX_PARALLEL_NO_SPINNER: 1 - TOXENV: python + TOXENV: py steps: - name: Switch to using Python v${{ matrix.python }} From e73f8f48a0fd0deb82d34cf495942dbf559fe1bb Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 12:23:52 +0530 Subject: [PATCH 14/21] -vv for pytest --- README.md | 2 +- pytest.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a6a39b0358..0ea8544648 100644 --- a/README.md +++ b/README.md @@ -2366,7 +2366,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--filtered-client-ips FILTERED_CLIENT_IPS] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] -proxy.py v2.4.4rc6.dev9+g11e37a62.d20240413 +proxy.py v2.4.4rc6.dev11+gac1f05d7.d20240413 options: -h, --help show this help message and exit diff --git a/pytest.ini b/pytest.ini index 1af93373aa..e90b9393c2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,7 +8,7 @@ addopts = --durations=10 # A bit of verbosity doesn't hurt: - -v + -vv # Report all the things == -rxXs: -ra From 3c74496bfe4bcb78c8d8407c3bcab96fe4566c47 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 12:39:16 +0530 Subject: [PATCH 15/21] Downgrade to `pytest-asyncio==0.21.1` --- pytest.ini | 1 + requirements-testing.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index e90b9393c2..4b4470312d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -87,3 +87,4 @@ norecursedirs = testpaths = tests/ xfail_strict = true +asyncio_mode=strict \ No newline at end of file diff --git a/requirements-testing.txt b/requirements-testing.txt index 3c0da300b0..cad6790ada 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -13,7 +13,7 @@ pytest==8.1.1; python_version >= '3.11' pytest-cov==5.0.0; python_version >= '3.11' pytest-xdist == 3.5.0; python_version >= '3.11' pytest-mock==3.14.0; python_version >= '3.11' -pytest-asyncio==0.23.6; python_version >= '3.11' +pytest-asyncio==0.21.1; python_version >= '3.11' autopep8==1.6.0 mypy==0.971 py-spy==0.3.12 From 2e292e29100049e9ee489467b9eddca7a8cc47a8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 13 Apr 2024 07:10:00 +0000 Subject: [PATCH 16/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 4b4470312d..3a0a471165 100644 --- a/pytest.ini +++ b/pytest.ini @@ -87,4 +87,4 @@ norecursedirs = testpaths = tests/ xfail_strict = true -asyncio_mode=strict \ No newline at end of file +asyncio_mode=strict From 65897d8a623486b0c81bb5d5fc839a30967a4510 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 12:44:36 +0530 Subject: [PATCH 17/21] remove asyncio_mode=strict --- pytest.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 3a0a471165..e90b9393c2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -87,4 +87,3 @@ norecursedirs = testpaths = tests/ xfail_strict = true -asyncio_mode=strict From 82aca7f116941cc734efda678833fb03a64798d9 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 12:58:39 +0530 Subject: [PATCH 18/21] try with `pytest-cov==4.1.0` for 3.11 --- requirements-testing.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index cad6790ada..e59edac6e5 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -5,13 +5,13 @@ flake8==4.0.1 # pytest for Python<3.11 pytest==7.0.1; python_version < '3.11' pytest-cov==3.0.0; python_version < '3.11' -pytest-xdist == 2.5.0; python_version < '3.11' +pytest-xdist==2.5.0; python_version < '3.11' pytest-mock==3.6.1; python_version < '3.11' pytest-asyncio==0.16.0; python_version < '3.11' # pytest for Python>=3.11 pytest==8.1.1; python_version >= '3.11' -pytest-cov==5.0.0; python_version >= '3.11' -pytest-xdist == 3.5.0; python_version >= '3.11' +pytest-cov==4.1.0; python_version >= '3.11' +pytest-xdist==3.5.0; python_version >= '3.11' pytest-mock==3.14.0; python_version >= '3.11' pytest-asyncio==0.21.1; python_version >= '3.11' autopep8==1.6.0 From bcd694bff9a72ed1c5b352a26ce1529319c8bcf3 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 13:03:40 +0530 Subject: [PATCH 19/21] bump coverage for 3.11 --- requirements-testing.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index e59edac6e5..61153d0d62 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -1,6 +1,7 @@ wheel==0.37.1 python-coveralls==2.9.3 -coverage==6.2 +coverage==6.2; python_version < '3.11' +coverage==7.4.4; python_version >= '3.11' flake8==4.0.1 # pytest for Python<3.11 pytest==7.0.1; python_version < '3.11' @@ -10,7 +11,7 @@ pytest-mock==3.6.1; python_version < '3.11' pytest-asyncio==0.16.0; python_version < '3.11' # pytest for Python>=3.11 pytest==8.1.1; python_version >= '3.11' -pytest-cov==4.1.0; python_version >= '3.11' +pytest-cov==5.0.0; python_version >= '3.11' pytest-xdist==3.5.0; python_version >= '3.11' pytest-mock==3.14.0; python_version >= '3.11' pytest-asyncio==0.21.1; python_version >= '3.11' From 5ba71f40348bcbbc2beabca31f6c8ef310aa80af Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 13:09:34 +0530 Subject: [PATCH 20/21] Try `3.11` in GitHub workflow which installs >3.11.8 unavailable via pyenv yet --- .github/workflows/test-library.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 85dbbea2a8..b01f93b0aa 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -446,7 +446,7 @@ jobs: # NOTE: The latest and the lowest supported Pythons are prioritized # NOTE: to improve the responsiveness. It's nice to see the most # NOTE: important results first. - - '3.11.8' + - '3.11' - 3.6 - '3.10' - 3.9 @@ -502,14 +502,14 @@ jobs: }}- ${{ runner.os }}-pip- - name: Install tox for >= 3.11 - if: matrix.python == '3.11.8' + if: matrix.python == '3.11' run: >- python -m pip install --user tox==4.14.2 - name: Install tox for < 3.11 - if: matrix.python != '3.11.8' + if: matrix.python != '3.11' run: >- python -m pip install From 573050c2e455b0e507448e1aa7e94b91bfe5dd68 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 13 Apr 2024 13:13:12 +0530 Subject: [PATCH 21/21] Revert back to `-v` --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index e90b9393c2..1af93373aa 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,7 +8,7 @@ addopts = --durations=10 # A bit of verbosity doesn't hurt: - -vv + -v # Report all the things == -rxXs: -ra