From dc5e069ab194c85f449a69448e3c4b135bfa235d Mon Sep 17 00:00:00 2001 From: InvalidInterrupt Date: Sat, 2 Sep 2023 16:31:20 -0700 Subject: [PATCH 1/6] Package twisted.plugins.fd_endpoint again --- setup.cfg | 9 ++++++--- {daphne/twisted => twisted}/plugins/fd_endpoint.py | 0 2 files changed, 6 insertions(+), 3 deletions(-) rename {daphne/twisted => twisted}/plugins/fd_endpoint.py (100%) diff --git a/setup.cfg b/setup.cfg index cdde7036..9c0374ba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,9 +24,7 @@ classifiers = Topic :: Internet :: WWW/HTTP [options] -package_dir = - twisted=daphne/twisted -packages = find: +packages = find_namespace: include_package_data = True install_requires = asgiref>=3.5.2,<4 @@ -48,6 +46,11 @@ tests = pytest pytest-asyncio +[options.packages.find] +include= + daphne.* + twisted.* + [flake8] exclude = venv/*,tox/*,docs/*,testproject/*,js_client/*,.eggs/* extend-ignore = E123, E128, E266, E402, W503, E731, W601 diff --git a/daphne/twisted/plugins/fd_endpoint.py b/twisted/plugins/fd_endpoint.py similarity index 100% rename from daphne/twisted/plugins/fd_endpoint.py rename to twisted/plugins/fd_endpoint.py From 3d9bbebfda33d466fbb6434470517dc788594e11 Mon Sep 17 00:00:00 2001 From: InvalidInterrupt Date: Sat, 2 Sep 2023 16:38:04 -0700 Subject: [PATCH 2/6] Add option to use alternative listeners with DaphneTestingInstance Additionally, allow the caller to disable the default listener on 127.0.0.1. --- daphne/testing.py | 53 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/daphne/testing.py b/daphne/testing.py index 785edf9d..ab5729e2 100644 --- a/daphne/testing.py +++ b/daphne/testing.py @@ -18,11 +18,21 @@ class BaseDaphneTestingInstance: startup_timeout = 2 def __init__( - self, xff=False, http_timeout=None, request_buffer_size=None, *, application + self, + xff=False, + http_timeout=None, + request_buffer_size=None, + *, + application, + host="127.0.0.1", + unix_socket=None, + file_descriptor=None, ): self.xff = xff self.http_timeout = http_timeout - self.host = "127.0.0.1" + self.host = host + self.unix_socket = unix_socket + self.file_descriptor = file_descriptor self.request_buffer_size = request_buffer_size self.application = application @@ -44,6 +54,8 @@ def __enter__(self): # Start up process self.process = DaphneProcess( host=self.host, + unix_socket=self.unix_socket, + file_descriptor=self.file_descriptor, get_application=self.get_application, kwargs=kwargs, setup=self.process_setup, @@ -126,9 +138,20 @@ class DaphneProcess(multiprocessing.Process): port it ends up listening on back to the parent process. """ - def __init__(self, host, get_application, kwargs=None, setup=None, teardown=None): + def __init__( + self, + get_application, + host=None, + file_descriptor=None, + unix_socket=None, + kwargs=None, + setup=None, + teardown=None, + ): super().__init__() self.host = host + self.file_descriptor = file_descriptor + self.unix_socket = unix_socket self.get_application = get_application self.kwargs = kwargs or {} self.setup = setup @@ -153,12 +176,17 @@ def run(self): try: # Create the server class - endpoints = build_endpoint_description_strings(host=self.host, port=0) + endpoints = build_endpoint_description_strings( + host=self.host, + port=0 if self.host else None, + unix_socket=self.unix_socket, + file_descriptor=self.file_descriptor, + ) self.server = Server( application=application, endpoints=endpoints, signal_handlers=False, - **self.kwargs + **self.kwargs, ) # Set up a poller to look for the port reactor.callLater(0.1, self.resolve_port) @@ -177,11 +205,18 @@ def run(self): def resolve_port(self): from twisted.internet import reactor - if self.server.listening_addresses: - self.port.value = self.server.listening_addresses[0][1] - self.ready.set() + if not all(listener.called for listener in self.server.listeners): + pass + elif self.host: + if self.server.listening_addresses: + self.port.value = self.server.listening_addresses[0][1] + self.ready.set() + return else: - reactor.callLater(0.1, self.resolve_port) + self.port.value = -1 + self.ready.set() + return + reactor.callLater(0.1, self.resolve_port) class TestApplication: From f6350940e6c2d3d7e96fb1fba4501b004909d709 Mon Sep 17 00:00:00 2001 From: InvalidInterrupt Date: Sat, 2 Sep 2023 18:30:04 -0700 Subject: [PATCH 3/6] Detect address family for fd endpoints By detecting the address family rather than assuming AF_INET, we allow unix domain sockets to be inherited properly. --- twisted/plugins/fd_endpoint.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/twisted/plugins/fd_endpoint.py b/twisted/plugins/fd_endpoint.py index 313a3154..ddf3a452 100644 --- a/twisted/plugins/fd_endpoint.py +++ b/twisted/plugins/fd_endpoint.py @@ -1,3 +1,4 @@ +import os import socket from twisted.internet import endpoints @@ -10,8 +11,13 @@ class _FDParser: prefix = "fd" - def _parseServer(self, reactor, fileno, domain=socket.AF_INET): + def _parseServer(self, reactor, fileno, domain=None): fileno = int(fileno) + if domain: + domain = getattr(socket, f"AF_{domain}") + else: + with socket.socket(fileno=os.dup(fileno)) as sock: + domain = sock.family return endpoints.AdoptedStreamServerEndpoint(reactor, fileno, domain) def parseStreamServer(self, reactor, *args, **kwargs): From 5517728b19d1a9c688ea6a4147548b415aa87aab Mon Sep 17 00:00:00 2001 From: InvalidInterrupt Date: Sat, 2 Sep 2023 18:48:39 -0700 Subject: [PATCH 4/6] Add test for inherited UNIX sockets --- setup.cfg | 1 + tests/http_base.py | 27 ++++++++++++++++------ tests/test_unixsocket.py | 50 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 tests/test_unixsocket.py diff --git a/setup.cfg b/setup.cfg index 9c0374ba..e8f4d5e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,6 +42,7 @@ console_scripts = [options.extras_require] tests = django + httpunixsocketconnection hypothesis pytest pytest-asyncio diff --git a/tests/http_base.py b/tests/http_base.py index e5a80c21..8d483b8c 100644 --- a/tests/http_base.py +++ b/tests/http_base.py @@ -17,6 +17,20 @@ class DaphneTestCase(unittest.TestCase): to store/retrieve the request/response messages. """ + _instance_endpoint_args = {} + + @staticmethod + def _get_instance_raw_socket_connection(test_app, *, timeout): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.connect((test_app.host, test_app.port)) + return s + + @staticmethod + def _get_instance_http_connection(test_app, *, timeout): + return HTTPConnection(test_app.host, test_app.port, timeout=timeout) + ### Plain HTTP helpers def run_daphne_http( @@ -36,13 +50,15 @@ def run_daphne_http( and response messages. """ with DaphneTestingInstance( - xff=xff, request_buffer_size=request_buffer_size + xff=xff, + request_buffer_size=request_buffer_size, + **self._instance_endpoint_args, ) as test_app: # Add the response messages test_app.add_send_messages(responses) # Send it the request. We have to do this the long way to allow # duplicate headers. - conn = HTTPConnection(test_app.host, test_app.port, timeout=timeout) + conn = self._get_instance_http_connection(test_app, timeout=timeout) if params: path += "?" + parse.urlencode(params, doseq=True) conn.putrequest(method, path, skip_accept_encoding=True, skip_host=True) @@ -74,13 +90,10 @@ def run_daphne_raw(self, data, *, responses=None, timeout=1): Returns what Daphne sends back. """ assert isinstance(data, bytes) - with DaphneTestingInstance() as test_app: + with DaphneTestingInstance(**self._instance_endpoint_args) as test_app: if responses is not None: test_app.add_send_messages(responses) - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(timeout) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - s.connect((test_app.host, test_app.port)) + s = self._get_instance_raw_socket_connection(test_app, timeout=timeout) s.send(data) try: return s.recv(1000000) diff --git a/tests/test_unixsocket.py b/tests/test_unixsocket.py new file mode 100644 index 00000000..821517ad --- /dev/null +++ b/tests/test_unixsocket.py @@ -0,0 +1,50 @@ +import os +import socket +import weakref +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest import skipUnless + +import test_http_response +from http_base import DaphneTestCase +from httpunixsocketconnection import HTTPUnixSocketConnection + +__all__ = ["UnixSocketFDDaphneTestCase", "TestInheritedUnixSocket"] + + +class UnixSocketFDDaphneTestCase(DaphneTestCase): + @property + def _instance_endpoint_args(self): + tmp_dir = TemporaryDirectory() + weakref.finalize(self, tmp_dir.cleanup) + sock_path = str(Path(tmp_dir.name, "test.sock")) + listen_sock = socket.socket(socket.AF_UNIX, type=socket.SOCK_STREAM) + listen_sock.bind(sock_path) + listen_sock.listen() + listen_sock_fileno = os.dup(listen_sock.fileno()) + os.set_inheritable(listen_sock_fileno, True) + listen_sock.close() + return {"host": None, "file_descriptor": listen_sock_fileno} + + @staticmethod + def _get_instance_socket_path(test_app): + with socket.socket(fileno=os.dup(test_app.file_descriptor)) as sock: + return sock.getsockname() + + @classmethod + def _get_instance_raw_socket_connection(cls, test_app, *, timeout): + socket_name = cls._get_instance_socket_path(test_app) + s = socket.socket(socket.AF_UNIX, type=socket.SOCK_STREAM) + s.settimeout(timeout) + s.connect(socket_name) + return s + + @classmethod + def _get_instance_http_connection(cls, test_app, *, timeout): + socket_name = cls._get_instance_socket_path(test_app) + return HTTPUnixSocketConnection(unix_socket=socket_name, timeout=timeout) + + +@skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX support not present.") +class TestInheritedUnixSocket(UnixSocketFDDaphneTestCase): + test_minimal_response = test_http_response.TestHTTPResponse.test_minimal_response From 50829d924209849cd5dfdaaa8782ca0a4497905e Mon Sep 17 00:00:00 2001 From: InvalidInterrupt Date: Sat, 2 Sep 2023 19:04:01 -0700 Subject: [PATCH 5/6] Relax options.packages.find.include to make tox happy Setuptools handled this fine but tox skips installing modules within the top-level package --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index e8f4d5e8..af31319b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,8 +49,8 @@ tests = [options.packages.find] include= - daphne.* - twisted.* + daphne* + twisted* [flake8] exclude = venv/*,tox/*,docs/*,testproject/*,js_client/*,.eggs/* From 34e247547c6178048127df6ecf4b3d951648db9f Mon Sep 17 00:00:00 2001 From: InvalidInterrupt Date: Tue, 5 Sep 2023 00:15:06 -0700 Subject: [PATCH 6/6] fixup! Add test for inherited UNIX sockets Eliminate dependency on httpunixsocketconnection; it does not declare compatibility with Python 3.7. Patching an HTTPConnection object like this is hackish, but may be good enough for a test suite. I think it's pretty unlikely to lead to a false negative. --- setup.cfg | 1 - tests/test_unixsocket.py | 12 +++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index af31319b..c6afb6e4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,6 @@ console_scripts = [options.extras_require] tests = django - httpunixsocketconnection hypothesis pytest pytest-asyncio diff --git a/tests/test_unixsocket.py b/tests/test_unixsocket.py index 821517ad..46ad4f94 100644 --- a/tests/test_unixsocket.py +++ b/tests/test_unixsocket.py @@ -1,13 +1,13 @@ import os import socket import weakref +from http.client import HTTPConnection from pathlib import Path from tempfile import TemporaryDirectory from unittest import skipUnless import test_http_response from http_base import DaphneTestCase -from httpunixsocketconnection import HTTPUnixSocketConnection __all__ = ["UnixSocketFDDaphneTestCase", "TestInheritedUnixSocket"] @@ -41,8 +41,14 @@ def _get_instance_raw_socket_connection(cls, test_app, *, timeout): @classmethod def _get_instance_http_connection(cls, test_app, *, timeout): - socket_name = cls._get_instance_socket_path(test_app) - return HTTPUnixSocketConnection(unix_socket=socket_name, timeout=timeout) + def connect(): + conn.sock = cls._get_instance_raw_socket_connection( + test_app, timeout=timeout + ) + + conn = HTTPConnection("", timeout=timeout) + conn.connect = connect + return conn @skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX support not present.")