Python 3.11 support (#1384)

* Changes for Python 3.11 support

* Updated README.md for versioning info

* Update `httpx==0.27.0` to avoid `cgi` deprecation warning from pytest on Python 3.11

* Make tests work for 3.11

* Declare support for 3.11

* Use 3.11-alpine for Docker images

* Preserve pylint version for `python_version <= 3.10`

* Preserve httpx version for <= 3.10

* `httpx` usage fix in tests for <=3.10

* Adjust pylint and pytest for >= 3.11

* Use 3.11.8, bad-option-value and httpx proxies fix

* tox for 3.11

* Fix for `TOXENV: py`

* -vv for pytest

* Downgrade to `pytest-asyncio==0.21.1`

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* remove asyncio_mode=strict

* try with `pytest-cov==4.1.0` for 3.11

* bump coverage for 3.11

* Try `3.11` in GitHub workflow which installs >3.11.8 unavailable via pyenv yet

* Revert back to `-v`

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/test-library.yml

@@ -446,8 +446,9 @@ jobs:
         # NOTE: The latest and the lowest supported Pythons are prioritized
         # NOTE: to improve the responsiveness. It's nice to see the most
         # NOTE: important results first.
-        - '3.10'
+        - '3.11'
         - 3.6
+        - '3.10'
         - 3.9
         - 3.8
         - 3.7
@@ -463,7 +464,7 @@ jobs:
     env:
       PY_COLORS: 1
       TOX_PARALLEL_NO_SPINNER: 1
-      TOXENV: python
+      TOXENV: py
 
     steps:
     - name: Switch to using Python v${{ matrix.python }}
@@ -500,7 +501,15 @@ jobs:
             steps.calc-cache-key-py.outputs.py-hash-key
           }}-
           ${{ runner.os }}-pip-
-    - name: Install tox
+    - name: Install tox for >= 3.11
+      if: matrix.python == '3.11'
+      run: >-
+        python -m
+        pip install
+        --user
+        tox==4.14.2
+    - name: Install tox for < 3.11
+      if: matrix.python != '3.11'
       run: >-
         python -m
         pip install

.pylintrc

@@ -131,6 +131,10 @@ disable=raw-checker-failed,
         useless-return,
         useless-super-delegation,
         wrong-import-order,
+        # Required because to support 3.11
+        # we added unnecessary-dunder-call which is not supported for <=3.11
+        # see https://github.com/abhinavsingh/proxy.py/actions/runs/8671404475/job/23780537848?pr=1384
+        bad-option-value
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -419,7 +423,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=
+generated-members=os,io
 
 # Tells whether missing members accessed in mixin class should be ignored. A
 # mixin class is detected if its name ends with "mixin" (case insensitive).
@@ -446,7 +450,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis). It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
+ignored-modules=abc
 
 # Show a hint with possible names when a member name was not found. The aspect
 # of finding the hint is based on edit distance.
@@ -605,5 +609,5 @@ preferred-modules=
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "BaseException, Exception".
-overgeneral-exceptions=BaseException,
-                       Exception
+overgeneral-exceptions=builtins.BaseException,
+                       builtins.Exception

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-alpine as base
+FROM python:3.11-alpine as base
 
 LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \
   com.abhinavsingh.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \

Makefile

@@ -120,7 +120,7 @@ lib-mypy:
 	tox -e lint -- mypy --all-files
 
 lib-pytest:
-	$(PYTHON) -m tox -e python -- -v
+	$(PYTHON) -m tox -e py -- -v
 
 lib-test: lib-clean lib-check lib-lint lib-pytest
 

README.md

@@ -13,7 +13,7 @@
 [![iOS, iOS Simulator](https://img.shields.io/static/v1?label=tested%20with&message=iOS%20%F0%9F%93%B1%20%7C%20iOS%20Simulator%20%F0%9F%93%B1&color=darkgreen&style=for-the-badge)](https://abhinavsingh.com/proxy-py-a-lightweight-single-file-http-proxy-server-in-python/)
 
 [![pypi version](https://img.shields.io/pypi/v/proxy.py?style=flat-square)](https://pypi.org/project/proxy.py/)
-[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10&color=blue&style=flat-square)](https://www.python.org/)
+[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10%20%7C%203.11&color=blue&style=flat-square)](https://www.python.org/)
 [![Checked with mypy](https://img.shields.io/static/v1?label=MyPy&message=checked&color=blue&style=flat-square)](http://mypy-lang.org/)
 
 [![doc](https://img.shields.io/readthedocs/proxypy/latest?style=flat-square&color=darkgreen)](https://proxypy.readthedocs.io/)
@@ -2366,7 +2366,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT]
           [--filtered-client-ips FILTERED_CLIENT_IPS]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
 
-proxy.py v2.4.4rc5.dev36+g6c9d0315.d20240411
+proxy.py v2.4.4rc6.dev11+gac1f05d7.d20240413
 
 options:
   -h, --help            show this help message and exit
@@ -2489,8 +2489,8 @@ options:
                         Default: None. Signing certificate to use for signing
                         dynamically generated HTTPS certificates. If used,
                         must also pass --ca-key-file and --ca-signing-key-file
-  --ca-file CA_FILE     Default: /Users/abhinavsingh/Dev/proxy.py/.venv/lib/py
-                        thon3.10/site-packages/certifi/cacert.pem. Provide
+  --ca-file CA_FILE     Default: /Users/abhinavsingh/Dev/proxy.py/.venv3118/li
+                        b/python3.11/site-packages/certifi/cacert.pem. Provide
                         path to custom CA bundle for peer certificate
                         verification
   --ca-signing-key-file CA_SIGNING_KEY_FILE

benchmark/requirements.txt

@@ -1,4 +1,4 @@
-aiohttp==3.8.1
+aiohttp==3.8.2
 # Blacksheep depends upon essentials_openapi which is pinned to pyyaml==5.4.1
 # and pyyaml>5.3.1 is broken for cython 3
 # See https://github.com/yaml/pyyaml/issues/724#issuecomment-1638587228

docs/conf.py

@@ -321,6 +321,8 @@
     (_py_class_role, 'HostPort'),
     (_py_class_role, 'TcpOrTlsSocket'),
     (_py_class_role, 're.Pattern'),
+    (_py_class_role, 'proxy.core.base.tcp_server.T'),
+    (_py_class_role, 'proxy.common.types.RePattern'),
     (_py_obj_role, 'proxy.core.work.threadless.T'),
     (_py_obj_role, 'proxy.core.work.work.T'),
     (_py_obj_role, 'proxy.core.base.tcp_server.T'),

examples/web_scraper.py

@@ -9,6 +9,8 @@
     :license: BSD, see LICENSE for more details.
 """
 import time
+from abc import abstractmethod
+from typing import Any
 
 from proxy import Proxy
 from proxy.core.work import Work
@@ -52,6 +54,11 @@ async def handle_events(
         Return True to shutdown work."""
         return False
 
+    @staticmethod
+    @abstractmethod
+    def create(*args: Any) -> TcpClientConnection:
+        raise NotImplementedError()
+
 
 if __name__ == '__main__':
     with Proxy(

proxy/common/types.py

@@ -14,7 +14,7 @@
 import queue
 import socket
 import ipaddress
-from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union, TypeVar
 
 
 if TYPE_CHECKING:   # pragma: no cover
@@ -34,8 +34,8 @@
 HostPort = Tuple[str, int]
 
 if sys.version_info.minor == 6:
-    RePattern = Any
+    RePattern = TypeVar('RePattern', bound=Any)
 elif sys.version_info.minor in (7, 8):
-    RePattern = re.Pattern  # type: ignore
+    RePattern = TypeVar('RePattern', bound=re.Pattern)  # type: ignore
 else:
-    RePattern = re.Pattern[Any]  # type: ignore
+    RePattern = TypeVar('RePattern', bound=re.Pattern[Any])  # type: ignore

proxy/core/base/tcp_server.py

@@ -240,3 +240,8 @@ def _optionally_wrap_socket(self, conn: socket.socket) -> TcpOrTlsSocket:
             conn = wrap_socket(conn, self.flags.keyfile, self.flags.certfile)
             self.work._conn = conn
         return conn
+
+    @staticmethod
+    @abstractmethod
+    def create(*args: Any) -> T:
+        raise NotImplementedError()

proxy/core/work/fd/fd.py

@@ -9,7 +9,9 @@
     :license: BSD, see LICENSE for more details.
 """
 import socket
+import asyncio
 import logging
+from abc import abstractmethod
 from typing import Any, TypeVar, Optional
 
 from ...event import eventNames
@@ -47,3 +49,16 @@ def work(self, *args: Any) -> None:
                 exc_info=e,
             )
             self._cleanup(fileno)
+
+    @property
+    @abstractmethod
+    def loop(self) -> Optional[asyncio.AbstractEventLoop]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def receive_from_work_queue(self) -> bool:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def work_queue_fileno(self) -> Optional[int]:
+        raise NotImplementedError()

proxy/core/work/local.py

@@ -11,6 +11,7 @@
 import queue
 import asyncio
 import contextlib
+from abc import abstractmethod
 from typing import Any, Optional
 
 from .threadless import Threadless
@@ -40,3 +41,7 @@ def receive_from_work_queue(self) -> bool:
                 return True
             self.work(work)
         return False
+
+    @abstractmethod
+    def work(self, *args: Any) -> None:
+        raise NotImplementedError()

proxy/core/work/remote.py

@@ -9,6 +9,7 @@
     :license: BSD, see LICENSE for more details.
 """
 import asyncio
+from abc import abstractmethod
 from typing import Any, Optional
 from multiprocessing import connection
 
@@ -37,3 +38,7 @@ def close_work_queue(self) -> None:
     def receive_from_work_queue(self) -> bool:
         self.work(self.work_queue.recv())
         return False
+
+    @abstractmethod
+    def work(self, *args: Any) -> None:
+        raise NotImplementedError()

proxy/http/server/plugin.py

@@ -169,7 +169,7 @@ def before_routing(self, request: HttpParser) -> Optional[HttpParser]:
 
     def handle_route(self, request: HttpParser, pattern: RePattern) -> Url:
         """Implement this method if you have configured dynamic routes."""
-        pass
+        raise NotImplementedError()
 
     def regexes(self) -> List[str]:
         """Helper method to return list of route regular expressions."""

proxy/plugin/proxy_pool.py

@@ -186,6 +186,7 @@ def handle_upstream_chunk(self, chunk: memoryview) -> Optional[memoryview]:
         """Will never be called since we didn't establish an upstream connection."""
         if not self.upstream:
             return chunk
+        # pylint: disable=broad-exception-raised
         raise Exception("This should have never been called")
 
     def on_upstream_connection_close(self) -> None:

proxy/testing/test_case.py

@@ -42,6 +42,7 @@ def setUpClass(cls) -> None:
         cls.PROXY.flags.plugins[b'HttpProxyBasePlugin'].append(
             CacheResponsesPlugin,
         )
+        # pylint: disable=unnecessary-dunder-call
         cls.PROXY.__enter__()
         assert cls.PROXY.acceptors
         cls.wait_for_server(cls.PROXY.flags.port)

requirements-testing.txt

@@ -1,21 +1,32 @@
 wheel==0.37.1
 python-coveralls==2.9.3
-coverage==6.2
+coverage==6.2; python_version < '3.11'
+coverage==7.4.4; python_version >= '3.11'
 flake8==4.0.1
-pytest==7.0.1
-pytest-cov==3.0.0
-pytest-xdist == 2.5.0
-pytest-mock==3.6.1
-pytest-asyncio==0.16.0
+# pytest for Python<3.11
+pytest==7.0.1; python_version < '3.11'
+pytest-cov==3.0.0; python_version < '3.11'
+pytest-xdist==2.5.0; python_version < '3.11'
+pytest-mock==3.6.1; python_version < '3.11'
+pytest-asyncio==0.16.0; python_version < '3.11'
+# pytest for Python>=3.11
+pytest==8.1.1; python_version >= '3.11'
+pytest-cov==5.0.0; python_version >= '3.11'
+pytest-xdist==3.5.0; python_version >= '3.11'
+pytest-mock==3.14.0; python_version >= '3.11'
+pytest-asyncio==0.21.1; python_version >= '3.11'
 autopep8==1.6.0
 mypy==0.971
 py-spy==0.3.12
-tox==3.28.0
+tox==3.28.0; python_version < '3.11'
+tox==4.14.2; python_version >= '3.11'
 mccabe==0.6.1
-pylint==2.13.7
+pylint==2.13.7; python_version < '3.11'
+pylint==3.1.0; python_version >= '3.11'
 rope==1.1.1
 # Required by test_http2.py
-httpx==0.22.0
+httpx==0.22.0; python_version < '3.11'
+httpx==0.27.0; python_version >= '3.11'
 h2==4.1.0
 hpack==4.0.0
 hyperframe==6.0.1

setup.cfg

@@ -65,6 +65,7 @@ classifiers =
   Programming Language :: Python :: 3.8
   Programming Language :: Python :: 3.9
   Programming Language :: Python :: 3.10
+  Programming Language :: Python :: 3.11
 
   Topic :: Internet
   Topic :: Internet :: Proxy Servers

tests/http/proxy/test_http2.py

@@ -8,6 +8,9 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
+import sys
+from typing import Any, Dict
+
 import httpx
 
 from proxy import TestCase
@@ -17,14 +20,23 @@ class TestHttp2WithProxy(TestCase):
 
     def test_http2_via_proxy(self) -> None:
         assert self.PROXY
+        proxy_url = 'http://localhost:%d' % self.PROXY.flags.port
+        proxies: Dict[str, Any] = (
+            {
+                'proxies': {
+                    'all://': proxy_url,
+                },
+            }
+            # For Python>=3.11, proxies keyword is deprecated by httpx
+            if sys.version_info < (3, 11, 0)
+            else {'proxy': proxy_url}
+        )
         response = httpx.get(
             'https://www.google.com',
             headers={'accept': 'application/json'},
             verify=httpx.create_ssl_context(http2=True),
             timeout=httpx.Timeout(timeout=5.0),
-            proxies={
-                'all://': 'http://localhost:%d' % self.PROXY.flags.port,
-            },
+            **proxies,
         )
         self.assertEqual(response.status_code, 200)
 

tox.ini

@@ -262,7 +262,6 @@ deps =
   pre-commit
   pylint >= 2.5.3
   pylint-pytest < 1.1.0
-  pytest-mock == 3.6.1
   -r docs/requirements.in
   -r requirements-tunnel.txt
   -r requirements-testing.txt