diff --git a/.mypy.ini b/.mypy.ini index 04a4760..57e504b 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -24,6 +24,11 @@ warn_unused_ignores = True # See: https://github.com/python/mypy/issues/8754#issuecomment-622376701 implicit_reexport = True +# The code forked from request-unixsocket is not compliant +[mypy-vplan.unixsocket.*] +check_untyped_defs = False +allow_untyped_defs = True + # It's hard to make tests compliant using unittest.mock [mypy-tests.*] check_untyped_defs = False @@ -33,10 +38,6 @@ allow_untyped_defs = True [mypy-pytest] ignore_missing_imports = True -# There is no type hinting for requests_unixsocket -[mypy-requests_unixsocket] -ignore_missing_imports = True - # There is no type hinting for apscheduler [mypy-apscheduler.*] ignore_missing_imports = True diff --git a/.run/tasks/server.sh b/.run/tasks/server.sh index d7ff638..f84e8fe 100644 --- a/.run/tasks/server.sh +++ b/.run/tasks/server.sh @@ -29,6 +29,8 @@ task_server() { run_task rmdb fi + chmod 700 config/local/vplan/server/db + poetry_run uvicorn vplan.engine.server:API \ --port 8080 \ --app-dir src --reload \ diff --git a/Changelog b/Changelog index 577559a..e2b6849 100644 --- a/Changelog +++ b/Changelog @@ -1,6 +1,10 @@ +Version 0.7.0 unreleased + + * Pull in code from requests-unixsocket and patch to support new urllib3. + Version 0.6.5 10 Jun 2023 - * Fix functional problems by pinning urllib<2. + * Fix functional problems by pinning urllib3<2. Version 0.6.4 08 Jun 2023 diff --git a/DEVELOPER.md b/DEVELOPER.md index c4e2605..6375fe0 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -8,7 +8,7 @@ This code runs as a daemon and is intended for use on Linux and UNIX-like platfo The systemd design is heavily based on the excellent [python-systemd-tutorial](https://github.com/torfsen/python-systemd-tutorial). See also the [specifiers](https://www.freedesktop.org/software/systemd/man/systemd.unit.html#Specifiers) documentation (for constructs like `%h`). -Uvicorn is using a user-private UNIX socket rather than opening a port like 8080. For the socket setup, I followed notes [here](https://gist.github.com/kylemanna/d193aaa6b33a89f649524ad27ce47c4b) and [here](https://stackoverflow.com/questions/52507089/running-uvicorn-with-unix-socket). I'm using [requests-unixsocket](https://pypi.org/project/requests-unixsocket/) to make requests to the UNIX socket. +Uvicorn is using a user-private UNIX socket rather than opening a port like 8080. For the socket setup, I followed notes [here](https://gist.github.com/kylemanna/d193aaa6b33a89f649524ad27ce47c4b) and [here](https://stackoverflow.com/questions/52507089/running-uvicorn-with-unix-socket). ## Packaging and Dependencies diff --git a/NOTICE b/NOTICE index 3e3ad73..274a5e9 100644 --- a/NOTICE +++ b/NOTICE @@ -14,3 +14,14 @@ under the following license: The contents of the Apache License can be found in the LICENSE file, or can be downloaded from http://www.apache.org/licenses/LICENSE-2.0 . + +The source code under src/vplan/unixsocket and tests/unixsocket originated at +msabramo/requests-unixsocket on GitHub. The code was pulled from the v0.3.0 +release on PyPI because there is no v0.3.0 tag at GitHub: + + https://github.com/msabramo/requests-unixsocket + https://pypi.org/project/requests-unixsocket/0.3.0/#files + +The requests-unixsocket repository contains a LICENSE file for the Apache +License, Version 2.0, but there is no copyright statement or other explicit +attribution. The PyPI entry and shows the author as . diff --git a/poetry.lock b/poetry.lock index d2e9b90..f2f4068 100644 --- a/poetry.lock +++ b/poetry.lock @@ -409,17 +409,17 @@ test = ["pytest (>=6)"] [[package]] name = "fastapi" -version = "0.96.0" +version = "0.96.1" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = false python-versions = ">=3.7" files = [ - {file = "fastapi-0.96.0-py3-none-any.whl", hash = "sha256:b8e11fe81e81eab4e1504209917338e0b80f783878a42c2b99467e5e1019a1e9"}, - {file = "fastapi-0.96.0.tar.gz", hash = "sha256:71232d47c2787446991c81c41c249f8a16238d52d779c0e6b43927d3773dbe3c"}, + {file = "fastapi-0.96.1-py3-none-any.whl", hash = "sha256:22d773ce95f14f04f8f37a0c8998fc163e67af83b65510d2879de6cbaaa10215"}, + {file = "fastapi-0.96.1.tar.gz", hash = "sha256:5c1d243030e63089ccfc0aec69c2da6d619943917727e8e82ee502358d5119bf"}, ] [package.dependencies] -pydantic = ">=1.6.2,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1.7.3,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0" +pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0" starlette = ">=0.27.0,<0.28.0" [package.extras] @@ -1108,20 +1108,6 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] -[[package]] -name = "requests-unixsocket" -version = "0.3.0" -description = "Use requests to talk HTTP via a UNIX domain socket" -optional = false -python-versions = "*" -files = [ - {file = "requests-unixsocket-0.3.0.tar.gz", hash = "sha256:28304283ea9357d45fff58ad5b11e47708cfbf5806817aa59b2a363228ee971e"}, - {file = "requests_unixsocket-0.3.0-py2.py3-none-any.whl", hash = "sha256:c685c680f0809e1b2955339b1e5afc3c0022b3066f4f7eb343f43a6065fc0e5d"}, -] - -[package.dependencies] -requests = ">=1.1" - [[package]] name = "responses" version = "0.23.1" @@ -1402,6 +1388,17 @@ files = [ {file = "types_urllib3-1.26.25.13-py3-none-any.whl", hash = "sha256:5dbd1d2bef14efee43f5318b5d36d805a489f6600252bb53626d4bfafd95e27c"}, ] +[[package]] +name = "types-waitress" +version = "2.1.4.8" +description = "Typing stubs for waitress" +optional = false +python-versions = "*" +files = [ + {file = "types-waitress-2.1.4.8.tar.gz", hash = "sha256:c6fe82e81726a83dd4ae3c2957ddcb9ac03297316045d850508633a7837595ba"}, + {file = "types_waitress-2.1.4.8-py3-none-any.whl", hash = "sha256:41214933536395fe6d512b8b4585a5b353a16c77d93f3b3d889a6553ee908d68"}, +] + [[package]] name = "typing-extensions" version = "4.6.3" @@ -1444,19 +1441,20 @@ devenv = ["black", "check-manifest", "flake8", "pyroma", "pytest (>=4.3)", "pyte [[package]] name = "urllib3" -version = "1.26.16" +version = "2.0.3" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.7" files = [ - {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, - {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, + {file = "urllib3-2.0.3-py3-none-any.whl", hash = "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1"}, + {file = "urllib3-2.0.3.tar.gz", hash = "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] [[package]] name = "uvicorn" @@ -1496,6 +1494,21 @@ platformdirs = ">=3.2,<4" docs = ["furo (>=2023.3.27)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=22.12)"] test = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.3.1)", "pytest-env (>=0.8.1)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=67.7.1)", "time-machine (>=2.9)"] +[[package]] +name = "waitress" +version = "2.1.2" +description = "Waitress WSGI server" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "waitress-2.1.2-py3-none-any.whl", hash = "sha256:7500c9625927c8ec60f54377d590f67b30c8e70ef4b8894214ac6e4cad233d2a"}, + {file = "waitress-2.1.2.tar.gz", hash = "sha256:780a4082c5fbc0fde6a2fcfe5e26e6efc1e8f425730863c04085769781f51eba"}, +] + +[package.extras] +docs = ["Sphinx (>=1.8.1)", "docutils", "pylons-sphinx-themes (>=1.0.9)"] +testing = ["coverage (>=5.0)", "pytest", "pytest-cover"] + [[package]] name = "wrapt" version = "1.15.0" @@ -1598,4 +1611,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.9,<4" -content-hash = "8e216f1726198be2fcca4c6c5101e3d0cc05d87bb72e5c738beb552ce8be3e35" +content-hash = "45985d331d32d00df23a8c6c5c3311a9097463d57e0dba11c2e2811a1dca325a" diff --git a/pyproject.toml b/pyproject.toml index 4265605..73c9a01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,9 +44,7 @@ uvicorn = "~0, >=0.20.0" pydantic-yaml = "~0, >=0.11.2" semver = "^3.0.0" click = "^8.1.3" -urllib3 = "<2" # compatibility problems with v2.x requests = "^2.31.0" -requests-unixsocket = "~0, >=0.3.0" SQLAlchemy = "^1.4.46" APScheduler = "^3.10.0" python-dotenv = "^1.0.0" @@ -73,6 +71,8 @@ colorama = "~0, >=0.4.5" httpx = "~0, >=0.23.3" responses = "~0, >=0.22.0" types-pyyaml = "^6.0.12.9" +waitress = "^2.1.2" +types-waitress = "^2.1.4.8" [tool.black] line-length = 132 diff --git a/src/vplan/client/client.py b/src/vplan/client/client.py index d2f6590..3273de8 100644 --- a/src/vplan/client/client.py +++ b/src/vplan/client/client.py @@ -10,14 +10,14 @@ import click import requests -import requests_unixsocket from requests import HTTPError, Response +import vplan.unixsocket as requests_unixsocket from vplan.client.config import api_url from vplan.interface import Account, PlanSchema, Status, Version # Add support in requests for http+unix:// URLs to use a UNIX socket -requests_unixsocket.monkeypatch() +requests_unixsocket.monkeypatch() # type: ignore def _url(endpoint: str) -> str: diff --git a/src/vplan/unixsocket/README.md b/src/vplan/unixsocket/README.md new file mode 100644 index 0000000..1d478de --- /dev/null +++ b/src/vplan/unixsocket/README.md @@ -0,0 +1,5 @@ +## Forked Code + +This code was forked from requests-unixsocket, with source taken from [v0.3.0 on PyPI](https://pypi.org/project/requests-unixsocket/0.3.0/#files). The original source code is on GitHub at [msabramo/requests-unixsocket](https://github.com/msabramo/requests-unixsocket), but v0.3.0 doesn't appear in the tags there. This code is licensed under Apache v2, so this is permitted use. + +I forked the code because it's incompatible with [urllib3 v2](https://urllib3.readthedocs.io/en/stable/v2-migration-guide.html), which requests moved to as of [v2.30.0](https://github.com/psf/requests/releases/tag/v2.30.0). We need to be on requests [>= v2.31.0](https://github.com/psf/requests/releases/tag/v2.31.0) due to [CVE-2023-32681](https://nvd.nist.gov/vuln/detail/CVE-2023-32681). The problem with requests-unixsocket is tracked in [issue #70](https://github.com/msabramo/requests-unixsocket/issues/70) and fixed in [PR #69](https://github.com/msabramo/requests-unixsocket/pull/69). However, as of this writing, the requests-unixsocket maintainer hasn't responded to either the issue or the PR. Given how small the code is, it seems safer and simpler to just pull it in rather than waiting for a new package to be released on PyPI. diff --git a/src/vplan/unixsocket/__init__.py b/src/vplan/unixsocket/__init__.py new file mode 100644 index 0000000..e4abf18 --- /dev/null +++ b/src/vplan/unixsocket/__init__.py @@ -0,0 +1,80 @@ +# pylint: disable=super-with-arguments,keyword-arg-before-vararg,invalid-name,redefined-outer-name,unused-argument: + +# This originated at msabramo/requests-unixsocket on GitHub; see README.md for details + +import sys + +import requests + +from .adapters import UnixAdapter + +DEFAULT_SCHEME = "http+unix://" + + +class Session(requests.Session): + def __init__(self, url_scheme=DEFAULT_SCHEME, *args, **kwargs): + super(Session, self).__init__(*args, **kwargs) + self.mount(url_scheme, UnixAdapter()) + + +class monkeypatch: + def __init__(self, url_scheme=DEFAULT_SCHEME): + self.session = Session() + requests = self._get_global_requests_module() + + # Methods to replace + self.methods = ("request", "get", "head", "post", "patch", "put", "delete", "options") + # Store the original methods + self.orig_methods = dict((m, requests.__dict__[m]) for m in self.methods) + # Monkey patch + g = globals() + for m in self.methods: + requests.__dict__[m] = g[m] + + def _get_global_requests_module(self): + return sys.modules["requests"] + + def __enter__(self): + return self + + def __exit__(self, *args): + requests = self._get_global_requests_module() + for m in self.methods: + requests.__dict__[m] = self.orig_methods[m] + + +# These are the same methods defined for the global requests object +def request(method, url, **kwargs): + session = Session() + return session.request(method=method, url=url, **kwargs) + + +def get(url, **kwargs): + kwargs.setdefault("allow_redirects", True) + return request("get", url, **kwargs) + + +def head(url, **kwargs): + kwargs.setdefault("allow_redirects", False) + return request("head", url, **kwargs) + + +def post(url, data=None, json=None, **kwargs): + return request("post", url, data=data, json=json, **kwargs) + + +def patch(url, data=None, **kwargs): + return request("patch", url, data=data, **kwargs) + + +def put(url, data=None, **kwargs): + return request("put", url, data=data, **kwargs) + + +def delete(url, **kwargs): + return request("delete", url, **kwargs) + + +def options(url, **kwargs): + kwargs.setdefault("allow_redirects", True) + return request("options", url, **kwargs) diff --git a/src/vplan/unixsocket/adapters.py b/src/vplan/unixsocket/adapters.py new file mode 100644 index 0000000..7fb1502 --- /dev/null +++ b/src/vplan/unixsocket/adapters.py @@ -0,0 +1,76 @@ +# pylint: disable=super-with-arguments,keyword-arg-before-vararg,ungrouped-imports: + +# This originated at msabramo/requests-unixsocket on GitHub; see README.md for details + +import socket + +import urllib3 +from requests.adapters import HTTPAdapter +from requests.compat import unquote, urlparse + + +# The following was adapted from some code from docker-py +# https://github.com/docker/docker-py/blob/master/docker/transport/unixconn.py +class UnixHTTPConnection(urllib3.connection.HTTPConnection): + def __init__(self, unix_socket_url, timeout=60): + """Create an HTTP connection to a unix domain socket + + :param unix_socket_url: A URL with a scheme of 'http+unix' and the + netloc is a percent-encoded path to a unix domain socket. E.g.: + 'http+unix://%2Ftmp%2Fprofilesvc.sock/status/pid' + """ + super(UnixHTTPConnection, self).__init__("localhost", timeout=timeout) + self.unix_socket_url = unix_socket_url + self.timeout = timeout + self.sock = None + + def __del__(self): # base class does not have d'tor + if self.sock: + self.sock.close() + + def connect(self): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.settimeout(self.timeout) + socket_path = unquote(urlparse(self.unix_socket_url).netloc) + sock.connect(socket_path) + self.sock = sock + + +class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): + def __init__(self, socket_path, timeout=60): + super(UnixHTTPConnectionPool, self).__init__("localhost", timeout=timeout) + self.socket_path = socket_path + self.timeout = timeout + + def _new_conn(self): + return UnixHTTPConnection(self.socket_path, self.timeout) + + +class UnixAdapter(HTTPAdapter): + def __init__(self, timeout=60, pool_connections=25, *args, **kwargs): + super(UnixAdapter, self).__init__(*args, **kwargs) + self.timeout = timeout + self.pools = urllib3._collections.RecentlyUsedContainer(pool_connections, dispose_func=lambda p: p.close()) + + def get_connection(self, url, proxies=None): + proxies = proxies or {} + proxy = proxies.get(urlparse(url.lower()).scheme) + + if proxy: + raise ValueError("%s does not support specifying proxies" % self.__class__.__name__) + + with self.pools.lock: + pool = self.pools.get(url) + if pool: + return pool + + pool = UnixHTTPConnectionPool(url, self.timeout) + self.pools[url] = pool + + return pool + + def request_url(self, request, proxies): + return request.path_url + + def close(self): + self.pools.clear() diff --git a/tests/unixsocket/README.md b/tests/unixsocket/README.md new file mode 100644 index 0000000..1d478de --- /dev/null +++ b/tests/unixsocket/README.md @@ -0,0 +1,5 @@ +## Forked Code + +This code was forked from requests-unixsocket, with source taken from [v0.3.0 on PyPI](https://pypi.org/project/requests-unixsocket/0.3.0/#files). The original source code is on GitHub at [msabramo/requests-unixsocket](https://github.com/msabramo/requests-unixsocket), but v0.3.0 doesn't appear in the tags there. This code is licensed under Apache v2, so this is permitted use. + +I forked the code because it's incompatible with [urllib3 v2](https://urllib3.readthedocs.io/en/stable/v2-migration-guide.html), which requests moved to as of [v2.30.0](https://github.com/psf/requests/releases/tag/v2.30.0). We need to be on requests [>= v2.31.0](https://github.com/psf/requests/releases/tag/v2.31.0) due to [CVE-2023-32681](https://nvd.nist.gov/vuln/detail/CVE-2023-32681). The problem with requests-unixsocket is tracked in [issue #70](https://github.com/msabramo/requests-unixsocket/issues/70) and fixed in [PR #69](https://github.com/msabramo/requests-unixsocket/pull/69). However, as of this writing, the requests-unixsocket maintainer hasn't responded to either the issue or the PR. Given how small the code is, it seems safer and simpler to just pull it in rather than waiting for a new package to be released on PyPI. diff --git a/tests/unixsocket/__init__.py b/tests/unixsocket/__init__.py new file mode 100644 index 0000000..82ee591 --- /dev/null +++ b/tests/unixsocket/__init__.py @@ -0,0 +1 @@ +__all__ = [] # type: ignore diff --git a/tests/unixsocket/test_requests_unixsocket.py b/tests/unixsocket/test_requests_unixsocket.py new file mode 100755 index 0000000..2e28bfb --- /dev/null +++ b/tests/unixsocket/test_requests_unixsocket.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# This originated at msabramo/requests-unixsocket on GitHub; see README.md for details + +"""Tests for requests_unixsocket""" + +import logging + +import pytest +import requests + +import vplan.unixsocket as requests_unixsocket + +from .testutils import UnixSocketServerThread + +logger = logging.getLogger(__name__) + + +def test_unix_domain_adapter_ok(): + with UnixSocketServerThread() as usock_thread: + session = requests_unixsocket.Session("http+unix://") + urlencoded_usock = requests.compat.quote_plus(usock_thread.usock) + url = "http+unix://%s/path/to/page" % urlencoded_usock + + for method in ["get", "post", "head", "patch", "put", "delete", "options"]: + logger.debug("Calling session.%s(%r) ...", method, url) + r = getattr(session, method)(url) + logger.debug("Received response: %r with text: %r and headers: %r", r, r.text, r.headers) + assert r.status_code == 200 + assert r.headers["server"] == "waitress" + assert r.headers["X-Transport"] == "unix domain socket" + assert r.headers["X-Requested-Path"] == "/path/to/page" + assert r.headers["X-Socket-Path"] == usock_thread.usock + assert isinstance(r.connection, requests_unixsocket.UnixAdapter) + assert r.url.lower() == url.lower() + if method == "head": + assert r.text == "" + else: + assert r.text == "Hello world!" + + +def test_unix_domain_adapter_url_with_query_params(): + with UnixSocketServerThread() as usock_thread: + session = requests_unixsocket.Session("http+unix://") + urlencoded_usock = requests.compat.quote_plus(usock_thread.usock) + url = "http+unix://%s" "/containers/nginx/logs?timestamp=true" % urlencoded_usock + + for method in ["get", "post", "head", "patch", "put", "delete", "options"]: + logger.debug("Calling session.%s(%r) ...", method, url) + r = getattr(session, method)(url) + logger.debug("Received response: %r with text: %r and headers: %r", r, r.text, r.headers) + assert r.status_code == 200 + assert r.headers["server"] == "waitress" + assert r.headers["X-Transport"] == "unix domain socket" + assert r.headers["X-Requested-Path"] == "/containers/nginx/logs" + assert r.headers["X-Requested-Query-String"] == "timestamp=true" + assert r.headers["X-Socket-Path"] == usock_thread.usock + assert isinstance(r.connection, requests_unixsocket.UnixAdapter) + assert r.url.lower() == url.lower() + if method == "head": + assert r.text == "" + else: + assert r.text == "Hello world!" + + +def test_unix_domain_adapter_connection_error(): + session = requests_unixsocket.Session("http+unix://") + + for method in ["get", "post", "head", "patch", "put", "delete", "options"]: + with pytest.raises(requests.ConnectionError): + getattr(session, method)("http+unix://socket_does_not_exist/path/to/page") + + +def test_unix_domain_adapter_connection_proxies_error(): + session = requests_unixsocket.Session("http+unix://") + + for method in ["get", "post", "head", "patch", "put", "delete", "options"]: + with pytest.raises(ValueError) as excinfo: + getattr(session, method)( + "http+unix://socket_does_not_exist/path/to/page", proxies={"http+unix": "http://10.10.1.10:1080"} + ) + assert "UnixAdapter does not support specifying proxies" in str(excinfo.value) + + +def test_unix_domain_adapter_monkeypatch(): + # note: the original test does a monkeypatch() in here, but that's not necessary + # because client/client.py does it globally for vplan. + + with UnixSocketServerThread() as usock_thread: + urlencoded_usock = requests.compat.quote_plus(usock_thread.usock) + url = "http+unix://%s/path/to/page" % urlencoded_usock + + for method in ["get", "post", "head", "patch", "put", "delete", "options"]: + logger.debug("Calling session.%s(%r) ...", method, url) + r = getattr(requests, method)(url, timeout=1.0) + logger.debug("Received response: %r with text: %r and headers: %r", r, r.text, r.headers) + assert r.status_code == 200 + assert r.headers["server"] == "waitress" + assert r.headers["X-Transport"] == "unix domain socket" + assert r.headers["X-Requested-Path"] == "/path/to/page" + assert r.headers["X-Socket-Path"] == usock_thread.usock + assert isinstance(r.connection, requests_unixsocket.UnixAdapter) + assert r.url.lower() == url.lower() + if method == "head": + assert r.text == "" + else: + assert r.text == "Hello world!" diff --git a/tests/unixsocket/testutils.py b/tests/unixsocket/testutils.py new file mode 100644 index 0000000..50cde0b --- /dev/null +++ b/tests/unixsocket/testutils.py @@ -0,0 +1,106 @@ +# pylint: disable=super-with-arguments,protected-access,implicit-str-concat,logging-not-lazy + +# This originated at msabramo/requests-unixsocket on GitHub; see README.md for details + +""" +Utilities helpful for writing tests + +Provides a UnixSocketServerThread that creates a running server, listening on a +newly created unix socket. + +Example usage: + +.. code-block:: python + + def test_unix_domain_adapter_monkeypatch(): + with UnixSocketServerThread() as usock_thread: + with requests_unixsocket.monkeypatch('http+unix://'): + urlencoded_usock = quote_plus(usock_process.usock) + url = 'http+unix://%s/path/to/page' % urlencoded_usock + r = requests.get(url) +""" + +import logging +import os +import threading +import time +import uuid + +import waitress + +logger = logging.getLogger(__name__) + + +class KillThread(threading.Thread): + def __init__(self, server, *args, **kwargs): + super(KillThread, self).__init__(*args, **kwargs) + self.server = server + + def run(self): + time.sleep(1) + logger.debug("Sleeping") + self.server._map.clear() + + +class WSGIApp: + server = None + + def __call__(self, environ, start_response): + logger.debug("WSGIApp.__call__: Invoked for %s", environ["PATH_INFO"]) + logger.debug("WSGIApp.__call__: environ = %r", environ) + status_text = "200 OK" + response_headers = [ + ("X-Transport", "unix domain socket"), + ("X-Socket-Path", environ["SERVER_PORT"]), + ("X-Requested-Query-String", environ["QUERY_STRING"]), + ("X-Requested-Path", environ["PATH_INFO"]), + ] + body_bytes = b"Hello world!" + start_response(status_text, response_headers) + logger.debug( + "WSGIApp.__call__: Responding with " "status_text = %r; " "response_headers = %r; " "body_bytes = %r", + status_text, + response_headers, + body_bytes, + ) + return [body_bytes] + + +class UnixSocketServerThread(threading.Thread): + def __init__(self, *args, **kwargs): + super(UnixSocketServerThread, self).__init__(*args, **kwargs) + self.usock = self.get_tempfile_name() + self.server = None + self.server_ready_event = threading.Event() + + def get_tempfile_name(self): + # I'd rather use tempfile.NamedTemporaryFile but IDNA limits + # the hostname to 63 characters and we'll get a "InvalidURL: + # URL has an invalid label" error if we exceed that. + args = (os.stat(__file__).st_ino, os.getpid(), uuid.uuid4().hex[-8:]) + return "/tmp/test_requests.%s_%s_%s" % args + + def run(self): + logger.debug("Call waitress.serve in %r ...", self) + wsgi_app = WSGIApp() + server = waitress.create_server( + wsgi_app, + unix_socket=self.usock, + clear_untrusted_proxy_headers=True, + ) + wsgi_app.server = server + self.server = server + self.server_ready_event.set() + server.run() + + def __enter__(self): + logger.debug("Starting %r ..." % self) + self.start() + logger.debug("Started %r.", self) + self.server_ready_event.wait() + return self + + def __exit__(self, *args): + self.server_ready_event.wait() + if self.server: + KillThread(self.server).start()