From e713752e49f8bc5fe9a3777a54f2f3e74161d773 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Thu, 9 May 2024 10:32:47 +0530 Subject: [PATCH 01/18] Teardown on unhandled exceptions by work (#1406) --- proxy/core/work/threadless.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proxy/core/work/threadless.py b/proxy/core/work/threadless.py index e638940fe4..e8f7339def 100644 --- a/proxy/core/work/threadless.py +++ b/proxy/core/work/threadless.py @@ -370,6 +370,8 @@ async def _run_once(self) -> bool: work_id = task._work_id # type: ignore try: teardown = task.result() + except Exception: + teardown = True finally: if teardown: self._cleanup(work_id) From 367205826df500bf59e1592690f3b0a976a3fe6b Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Fri, 10 May 2024 13:53:52 +0530 Subject: [PATCH 02/18] Grout: ngrok Alternative (#1407) * Grout: An Ngrok Alternative * Consume `grout` entry point within `proxy.py` * Revert `check.py` --- README.md | 161 +++++++++++++++++++++++++++++++++------------- proxy/__init__.py | 5 +- proxy/proxy.py | 110 ++++++++++++++++++++++++++++++- setup.cfg | 1 + 4 files changed, 230 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index f147249d04..b06d7ce81b 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,9 @@ - [End-to-End Encryption](#end-to-end-encryption) - [TLS Interception](#tls-interception) - [TLS Interception With Docker](#tls-interception-with-docker) +- [GROUT (NGROK Alternative)](#grout-ngrok-alternative) + - [How Grout works](#how-grout-works) + - [Self-hosted Grout](#self-hosted-grout) - [Proxy Over SSH Tunnel](#proxy-over-ssh-tunnel) - [Proxy Remote Requests Locally](#proxy-remote-requests-locally) - [Proxy Local Requests Remotely](#proxy-local-requests-remotely) @@ -138,6 +141,7 @@ [//]: # (DO-NOT-REMOVE-docs-badges-END) # Features +- [A drop-in alternative to `ngrok`](#grout-ngrok-alternative) - Fast & Scalable - Scale up by using all available cores on the system @@ -1290,6 +1294,76 @@ with TLS Interception: } ``` +# GROUT (NGROK Alternative) + +`grout` is a drop-in alternative to `ngrok` that comes packaged within `proxy.py` + +```console +❯ grout +NAME: + grout - securely tunnel local files, folders and services to public URLs + +USAGE: + grout route [name] + +DESCRIPTION: + grout exposes local networked services behinds NATs and firewalls to the + public internet over a secure tunnel. Share local folders, directories and websites, + build/test webhook consumers and self-host personal services to public URLs. + +EXAMPLES: + Share Files and Folders: + grout C:\path\to\folder # Share a folder on your system + grout /path/to/folder # Share a folder on your system + grout /path/to/folder --basic-auth user:pass # Add authentication for shared folder + grout /path/to/photo.jpg # Share a specific file on your system + + Expose HTTP, HTTPS and Websockets: + grout http://localhost:9090 # Expose HTTP service running on port 9090 + grout https://localhost:8080 # Expose HTTPS service running on port 8080 + grout https://localhost:8080 --path /worker/ # Expose only certain paths of HTTPS service on port 8080 + grout https://localhost:8080 --basic-auth u:p # Add authentication for exposed HTTPS service on port 8080 + + Expose TCP Services: + grout tcp://:6379 # Expose Redis service running locally on port 6379 + grout tcp://:22 # Expose SSH service running locally on port 22 + + Custom URLs: + grout https://localhost:8080 abhinavsingh # Custom URL for HTTPS service running on port 8080 + grout tcp://:22 abhinavsingh # Custom URL for SSH service running locally on port 22 + + Custom Domains: + grout tcp://:5432 abhinavsingh.domain.tld # Custom URL for Postgres service running locally on port 5432 + + Self-hosted solutions: + grout tcp://:5432 abhinavsingh.my.server # Custom URL for Postgres service running locally on port 5432 + +SUPPORT: + Write to us at support@jaxl.com + + Privacy policy and Terms & conditions + https://jaxl.com/privacy/ + + Created by Jaxl™ + https://jaxl.io +``` + +## How Grout works + +- `grout` infrastructure has 2 components: client and server +- `grout` client has 2 components: a thin and a thick client +- `grout` thin client is part of open source `proxy.py` (BSD 3-Clause License) +- `grout` thick client and servers are hosted at [jaxl.io](https://jaxl.io) + and a copyright of [Jaxl Innovations Private Limited](https://jaxl.com) +- `grout` server has 3 components: a registry server, a reverse proxy server and a tunnel server + +## Self-Hosted `grout` + +- `grout` thick client and servers can also be hosted on your GCP, AWS, Cloud infrastructures +- With a self-hosted version, your traffic flows through the network you control and trust +- `grout` developers at [jaxl.io](https://jaxl.io) provides GCP, AWS, Docker images for self-hosted solutions +- Please drop an email at [support@jaxl.com](mailto:support@jaxl.com) to get started. + # Proxy Over SSH Tunnel **This is a WIP and may not work as documented** @@ -2340,12 +2414,17 @@ To run standalone benchmark for `proxy.py`, use the following command from repo ```console ❯ proxy -h -usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] +usage: -m [-h] [--enable-proxy-protocol] [--threadless] [--threaded] + [--num-workers NUM_WORKERS] [--enable-events] [--enable-conn-pool] + [--key-file KEY_FILE] [--cert-file CERT_FILE] + [--client-recvbuf-size CLIENT_RECVBUF_SIZE] + [--server-recvbuf-size SERVER_RECVBUF_SIZE] + [--max-sendbuf-size MAX_SENDBUF_SIZE] [--timeout TIMEOUT] + [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--tunnel-username TUNNEL_USERNAME] [--tunnel-ssh-key TUNNEL_SSH_KEY] [--tunnel-ssh-key-passphrase TUNNEL_SSH_KEY_PASSPHRASE] - [--tunnel-remote-port TUNNEL_REMOTE_PORT] [--threadless] - [--threaded] [--num-workers NUM_WORKERS] [--enable-events] + [--tunnel-remote-port TUNNEL_REMOTE_PORT] [--local-executor LOCAL_EXECUTOR] [--backlog BACKLOG] [--hostname HOSTNAME] [--hostnames HOSTNAMES [HOSTNAMES ...]] [--port PORT] [--ports PORTS [PORTS ...]] [--port-file PORT_FILE] @@ -2357,10 +2436,6 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--basic-auth BASIC_AUTH] [--enable-ssh-tunnel] [--work-klass WORK_KLASS] [--pid-file PID_FILE] [--openssl OPENSSL] [--data-dir DATA_DIR] [--ssh-listener-klass SSH_LISTENER_KLASS] - [--enable-proxy-protocol] [--enable-conn-pool] [--key-file KEY_FILE] - [--cert-file CERT_FILE] [--client-recvbuf-size CLIENT_RECVBUF_SIZE] - [--server-recvbuf-size SERVER_RECVBUF_SIZE] - [--max-sendbuf-size MAX_SENDBUF_SIZE] [--timeout TIMEOUT] [--disable-http-proxy] [--disable-headers DISABLE_HEADERS] [--ca-key-file CA_KEY_FILE] [--ca-cert-dir CA_CERT_DIR] [--ca-cert-file CA_CERT_FILE] [--ca-file CA_FILE] @@ -2378,10 +2453,45 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--filtered-client-ips FILTERED_CLIENT_IPS] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] -proxy.py v2.4.4rc6.dev172+ge1879403.d20240425 +proxy.py v2.4.4rc6.dev191+gef5a8922 options: -h, --help show this help message and exit + --enable-proxy-protocol + Default: False. If used, will enable proxy protocol. + Only version 1 is currently supported. + --threadless Default: True. Enabled by default on Python 3.8+ (mac, + linux). When disabled a new thread is spawned to + handle each client connection. + --threaded Default: False. Disabled by default on Python < 3.8 + and windows. When enabled a new thread is spawned to + handle each client connection. + --num-workers NUM_WORKERS + Defaults to number of CPU cores. + --enable-events Default: False. Enables core to dispatch lifecycle + events. Plugins can be used to subscribe for core + events. + --enable-conn-pool Default: False. (WIP) Enable upstream connection + pooling. + --key-file KEY_FILE Default: None. Server key file to enable end-to-end + TLS encryption with clients. If used, must also pass + --cert-file. + --cert-file CERT_FILE + Default: None. Server certificate to enable end-to-end + TLS encryption with clients. If used, must also pass + --key-file. + --client-recvbuf-size CLIENT_RECVBUF_SIZE + Default: 128 KB. Maximum amount of data received from + the client in a single recv() operation. + --server-recvbuf-size SERVER_RECVBUF_SIZE + Default: 128 KB. Maximum amount of data received from + the server in a single recv() operation. + --max-sendbuf-size MAX_SENDBUF_SIZE + Default: 64 KB. Maximum amount of data to flush in a + single send() operation. + --timeout TIMEOUT Default: 10.0. Number of seconds after which an + inactive connection must be dropped. Inactivity is + defined by no data sent or received by the client. --tunnel-hostname TUNNEL_HOSTNAME Default: None. Remote hostname or IP address to which SSH tunnel will be established. @@ -2397,17 +2507,6 @@ options: --tunnel-remote-port TUNNEL_REMOTE_PORT Default: 8899. Remote port which will be forwarded locally for proxy. - --threadless Default: True. Enabled by default on Python 3.8+ (mac, - linux). When disabled a new thread is spawned to - handle each client connection. - --threaded Default: False. Disabled by default on Python < 3.8 - and windows. When enabled a new thread is spawned to - handle each client connection. - --num-workers NUM_WORKERS - Defaults to number of CPU cores. - --enable-events Default: False. Enables core to dispatch lifecycle - events. Plugins can be used to subscribe for core - events. --local-executor LOCAL_EXECUTOR Default: 1. Enabled by default. Use 0 to disable. When enabled acceptors will make use of local (same @@ -2463,30 +2562,6 @@ options: --ssh-listener-klass SSH_LISTENER_KLASS Default: proxy.core.ssh.listener.SshTunnelListener. An implementation of BaseSshTunnelListener - --enable-proxy-protocol - Default: False. If used, will enable proxy protocol. - Only version 1 is currently supported. - --enable-conn-pool Default: False. (WIP) Enable upstream connection - pooling. - --key-file KEY_FILE Default: None. Server key file to enable end-to-end - TLS encryption with clients. If used, must also pass - --cert-file. - --cert-file CERT_FILE - Default: None. Server certificate to enable end-to-end - TLS encryption with clients. If used, must also pass - --key-file. - --client-recvbuf-size CLIENT_RECVBUF_SIZE - Default: 128 KB. Maximum amount of data received from - the client in a single recv() operation. - --server-recvbuf-size SERVER_RECVBUF_SIZE - Default: 128 KB. Maximum amount of data received from - the server in a single recv() operation. - --max-sendbuf-size MAX_SENDBUF_SIZE - Default: 64 KB. Maximum amount of data to flush in a - single send() operation. - --timeout TIMEOUT Default: 10.0. Number of seconds after which an - inactive connection must be dropped. Inactivity is - defined by no data sent or received by the client. --disable-http-proxy Default: False. Whether to disable proxy.HttpProxyPlugin. --disable-headers DISABLE_HEADERS diff --git a/proxy/__init__.py b/proxy/__init__.py index 08da6e2aa0..421d39e601 100755 --- a/proxy/__init__.py +++ b/proxy/__init__.py @@ -8,11 +8,14 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from .proxy import Proxy, main, sleep_loop, entry_point +from .proxy import Proxy, main, grout, sleep_loop, entry_point from .testing import TestCase __all__ = [ + # Grout entry point. See + # https://jaxl.io/ + 'grout', # PyPi package entry_point. See # https://github.com/abhinavsingh/proxy.py#from-command-line-when-installed-using-pip 'entry_point', diff --git a/proxy/proxy.py b/proxy/proxy.py index 233893f15f..c607dc29eb 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -10,26 +10,35 @@ """ import os import sys +import gzip +import json import time import pprint import signal +import socket +import getpass import logging import argparse import threading -from typing import TYPE_CHECKING, Any, List, Type, Optional, cast +from typing import TYPE_CHECKING, Any, Dict, List, Type, Tuple, Optional, cast from .core.ssh import SshTunnelListener, SshHttpProtocolHandler from .core.work import ThreadlessPool from .core.event import EventManager +from .http.codes import httpStatusCodes from .common.flag import FlagParser, flags +from .http.client import client from .common.utils import bytes_ from .core.work.fd import RemoteFdExecutor +from .http.methods import httpMethods from .core.acceptor import AcceptorPool from .core.listener import ListenerPool from .core.ssh.base import BaseSshTunnelListener +from .common.plugins import Plugins +from .common.version import __version__ from .common.constants import ( - IS_WINDOWS, DEFAULT_PLUGINS, DEFAULT_VERSION, DEFAULT_LOG_FILE, - DEFAULT_PID_FILE, DEFAULT_LOG_LEVEL, DEFAULT_BASIC_AUTH, + IS_WINDOWS, HTTPS_PROTO, DEFAULT_PLUGINS, DEFAULT_VERSION, + DEFAULT_LOG_FILE, DEFAULT_PID_FILE, DEFAULT_LOG_LEVEL, DEFAULT_BASIC_AUTH, DEFAULT_LOG_FORMAT, DEFAULT_WORK_KLASS, DEFAULT_OPEN_FILE_LIMIT, DEFAULT_ENABLE_DASHBOARD, DEFAULT_ENABLE_SSH_TUNNEL, DEFAULT_SSH_LISTENER_KLASS, @@ -384,3 +393,98 @@ def main(**opts: Any) -> None: def entry_point() -> None: main() + + +def grout() -> None: # noqa: C901 + default_grout_tld = os.environ.get('JAXL_DEFAULT_GROUT_TLD', 'jaxl.io') + + def _clear_line() -> None: + print('\r' + ' ' * 60, end='', flush=True) + + def _env(scheme: bytes, host: bytes, port: int) -> Optional[Dict[str, Any]]: + response = client( + scheme=scheme, + host=host, + port=port, + path=b'/env/', + method=httpMethods.BIND, + body='v={0}&u={1}&h={2}'.format( + __version__, + os.environ.get('USER', getpass.getuser()), + socket.gethostname(), + ).encode(), + ) + if response: + if ( + response.code is not None + and int(response.code) == httpStatusCodes.OK + and response.body is not None + ): + return cast( + Dict[str, Any], + json.loads( + ( + gzip.decompress(response.body).decode() + if response.has_header(b'content-encoding') + and response.header(b'content-encoding') == b'gzip' + else response.body.decode() + ), + ), + ) + if response.code is None: + _clear_line() + print('\r\033[91mUnable to fetch\033[0m', end='', flush=True) + else: + _clear_line() + print( + '\r\033[91mError code {0}\033[0m'.format( + response.code.decode(), + ), + end='', + flush=True, + ) + else: + _clear_line() + print('\r\033[91mUnable to connect\033[0m') + return None + + def _parse() -> Tuple[str, int]: + """Here we deduce registry host/port based upon input parameters.""" + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('route', nargs='?', default=None) + parser.add_argument('name', nargs='?', default=None) + args, _remaining_args = parser.parse_known_args() + grout_tld = default_grout_tld + if args.name is not None and '.' in args.name: + grout_tld = args.name.split('.', maxsplit=1)[1] + grout_tld_parts = grout_tld.split(':') + tld_host = grout_tld_parts[0] + tld_port = 443 + if len(grout_tld_parts) > 1: + tld_port = int(grout_tld_parts[1]) + return tld_host, tld_port + + tld_host, tld_port = _parse() + env = None + attempts = 0 + try: + while True: + env = _env(scheme=HTTPS_PROTO, host=tld_host.encode(), port=int(tld_port)) + attempts += 1 + if env is not None: + print('\rStarting ...' + ' ' * 30 + '\r', end='', flush=True) + break + time.sleep(1) + _clear_line() + print( + '\rWaiting for connection {0}'.format('.' * (attempts % 4)), + end='', + flush=True, + ) + time.sleep(1) + except KeyboardInterrupt: + sys.exit(1) + + assert env is not None + print('\r' + ' ' * 70 + '\r', end='', flush=True) + Plugins.from_bytes(env['m'].encode(), name='client').grout(env=env['e']) # type: ignore[attr-defined] diff --git a/setup.cfg b/setup.cfg index fb14a6515e..4e555abb3e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -111,6 +111,7 @@ install_requires = [options.entry_points] console_scripts = proxy = proxy:entry_point + grout = proxy:grout [options.package_data] proxy = From 32a6bdd47f1522da18602fcdfe010b7d10072df3 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Wed, 15 May 2024 16:36:49 +0530 Subject: [PATCH 03/18] DockerfileBase (#1410) --- .github/workflows/dockerfile-base.yml | 79 +++++++++++++++++++++++++++ DockerfileBase | 34 ++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 .github/workflows/dockerfile-base.yml create mode 100644 DockerfileBase diff --git a/.github/workflows/dockerfile-base.yml b/.github/workflows/dockerfile-base.yml new file mode 100644 index 0000000000..eb83bf9834 --- /dev/null +++ b/.github/workflows/dockerfile-base.yml @@ -0,0 +1,79 @@ +--- +name: lib + +on: # yamllint disable-line rule:truthy + workflow_dispatch: + +concurrency: + group: >- + ${{ + github.workflow + }}-${{ + github.event.pull_request.number || github.sha + }} + cancel-in-progress: true + +jobs: + pre-setup: + name: ⚙️ Pre-set global build settings + runs-on: ubuntu-20.04 + defaults: + run: + shell: bash + outputs: + container-platforms: ${{ steps.container.outputs.platforms }} + steps: + - name: Calculate container attributes + id: container + shell: bash + run: >- + PLATFORMS="linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x"; + echo "::set-output name=platforms::$PLATFORMS" + + ghcr-latest: + runs-on: ubuntu-20.04 + permissions: + packages: write + if: success() + needs: + - pre-setup # transitive, for accessing settings + name: 🐳 ghcr:latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.release-commitish }} + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + # See https://github.com/docker/buildx/issues/850#issuecomment-996408167 + with: + version: v0.7.0 + buildkitd-flags: --debug + config: .github/buildkitd.toml + install: true + - name: Enable Multiarch # This slows down arm build by 4-5x + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: Create builder + run: | + docker buildx create --name proxypybuilder + docker buildx use proxypybuilder + docker buildx inspect + docker buildx ls + - name: Push base to GHCR + run: >- + docker buildx build + --push + --platform ${{ + needs.pre-setup.outputs.container-platforms + }} + -t ghcr.io/abhinavsingh/proxy.py:base + -f DockerfileBase . +... diff --git a/DockerfileBase b/DockerfileBase new file mode 100644 index 0000000000..2440bfef1a --- /dev/null +++ b/DockerfileBase @@ -0,0 +1,34 @@ +FROM python:3.11-alpine + +LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ + com.abhinavsingh.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ + 😈 TLS interception • 🔒 DNS-over-HTTPS • 🔥 Poor Man's VPN • ⏪ Reverse & ⏩ Forward • \ + 👮🏿 \"Proxy Server\" framework • 🌐 \"Web Server\" framework • ➵ ➶ ➷ ➠ \"PubSub\" framework • \ + 👷 \"Work\" acceptor & executor framework" \ + com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \ + com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \ + com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" \ + org.opencontainers.image.source="https://github.com/abhinavsingh/proxy.py" + +ENV PYTHONUNBUFFERED 1 + +# Install paramiko and cryptography to allow +# users to use tunneling features using Docker +RUN apk update && apk --no-cache add \ + --virtual .builddeps \ + gcc \ + musl-dev \ + libffi-dev \ + openssl-dev \ + python3-dev \ + cargo \ + rust \ + make +RUN python -m venv /proxy/venv && \ + /proxy/venv/bin/pip install \ + -U pip wheel && \ + /proxy/venv/bin/pip install \ + paramiko==3.4.0 \ + cryptography==39.0.1 \ + --prefer-binary +RUN apk del .builddeps From 58f884700125573efbfec86b1965a832b8f542a2 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 15 May 2024 16:38:05 +0530 Subject: [PATCH 04/18] `base` workflow --- .github/workflows/dockerfile-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerfile-base.yml b/.github/workflows/dockerfile-base.yml index eb83bf9834..4e7e2e70af 100644 --- a/.github/workflows/dockerfile-base.yml +++ b/.github/workflows/dockerfile-base.yml @@ -1,5 +1,5 @@ --- -name: lib +name: base on: # yamllint disable-line rule:truthy workflow_dispatch: From 0380e8301dba3a89b680720bb1092fc8a3d06605 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 15 May 2024 17:16:38 +0530 Subject: [PATCH 05/18] `--no-cache-dir` to avoid bloating the docker image --- DockerfileBase | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/DockerfileBase b/DockerfileBase index 2440bfef1a..c5c42f8b42 100644 --- a/DockerfileBase +++ b/DockerfileBase @@ -1,14 +1,17 @@ FROM python:3.11-alpine LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ - com.abhinavsingh.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ + org.opencontainers.image.title="proxy.py" \ + org.opencontainers.image.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ 😈 TLS interception • 🔒 DNS-over-HTTPS • 🔥 Poor Man's VPN • ⏪ Reverse & ⏩ Forward • \ 👮🏿 \"Proxy Server\" framework • 🌐 \"Web Server\" framework • ➵ ➶ ➷ ➠ \"PubSub\" framework • \ 👷 \"Work\" acceptor & executor framework" \ - com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \ - com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \ + org.opencontainers.url="https://github.com/abhinavsingh/proxy.py" \ + org.opencontainers.image.source="https://github.com/abhinavsingh/proxy.py" \ com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" \ - org.opencontainers.image.source="https://github.com/abhinavsingh/proxy.py" + org.opencontainers.image.licenses="BSD-3-Clause" \ + org.opencontainers.image.authors="Abhinav Singh " \ + org.opencontainers.image.vendor="Abhinav Singh" ENV PYTHONUNBUFFERED 1 @@ -25,9 +28,9 @@ RUN apk update && apk --no-cache add \ rust \ make RUN python -m venv /proxy/venv && \ - /proxy/venv/bin/pip install \ + /proxy/venv/bin/pip install --no-cache-dir \ -U pip wheel && \ - /proxy/venv/bin/pip install \ + /proxy/venv/bin/pip install --no-cache-dir \ paramiko==3.4.0 \ cryptography==39.0.1 \ --prefer-binary From f19db0ce1be5b53b16439a187227e1f10d893f76 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 15 May 2024 20:24:41 +0530 Subject: [PATCH 06/18] Optimize base docker image size --- DockerfileBase | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/DockerfileBase b/DockerfileBase index c5c42f8b42..99c1ef67ab 100644 --- a/DockerfileBase +++ b/DockerfileBase @@ -14,6 +14,7 @@ LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ org.opencontainers.image.vendor="Abhinav Singh" ENV PYTHONUNBUFFERED 1 +ENV PYTHONDONTWRITEBYTECODE 1 # Install paramiko and cryptography to allow # users to use tunneling features using Docker @@ -28,10 +29,13 @@ RUN apk update && apk --no-cache add \ rust \ make RUN python -m venv /proxy/venv && \ - /proxy/venv/bin/pip install --no-cache-dir \ + /proxy/venv/bin/pip install --no-compile --no-cache-dir \ -U pip wheel && \ - /proxy/venv/bin/pip install --no-cache-dir \ + /proxy/venv/bin/pip install --no-compile --no-cache-dir \ paramiko==3.4.0 \ cryptography==39.0.1 \ - --prefer-binary -RUN apk del .builddeps + --prefer-binary && \ + apk del .builddeps && \ + find . -type d -name '__pycache__' | xargs rm -rf && \ + rm -rf /var/cache/apk/* && \ + rm -rf /root/.cache/ From d124d4ec6ec5047db50ece6e15bdaa4dd538dc71 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 15 May 2024 20:28:58 +0530 Subject: [PATCH 07/18] Use `base` name for base docker image --- .github/workflows/dockerfile-base.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerfile-base.yml b/.github/workflows/dockerfile-base.yml index 4e7e2e70af..f08ddc7e47 100644 --- a/.github/workflows/dockerfile-base.yml +++ b/.github/workflows/dockerfile-base.yml @@ -30,14 +30,14 @@ jobs: PLATFORMS="linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x"; echo "::set-output name=platforms::$PLATFORMS" - ghcr-latest: + ghcr-base: runs-on: ubuntu-20.04 permissions: packages: write if: success() needs: - pre-setup # transitive, for accessing settings - name: 🐳 ghcr:latest + name: 🐳 ghcr:base steps: - name: Checkout uses: actions/checkout@v3 From 7bb04c020a5b407d0f0385ac36584fcb3fc3d68b Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Thu, 16 May 2024 00:34:26 +0530 Subject: [PATCH 08/18] Include `openssl`, `cryptography` and `paramiko` in default DockerHub image (#1409) * Include `openssl` in docker images to let users try TLS interception using dockerhub images * Include `requirements-tunnel.txt` within docker image to let users try tunneling using docker images * Docker is always using py311+, hardcode for now * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Simply `apk add py-cryptography` * --prefer-binary * Build deps for cryptography * make required by pynacl * Prepare to use base image once it has been published * Prepare image from `ghcr.io/abhinavsingh/proxy.py:base` * --no-cache-dir to avoid pip cache bloating * Optimize base image size * Use find * `-y` * Cut final image `FROM python:3.11-alpine` * Remove global setuptools and local pip too * wheel it too * end and flush * Try `42.0.4` in next base * Full path cleanup * SSL in final copy --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/test-library.yml | 2 -- Dockerfile | 43 ++++++++++++++++++++++-------- DockerfileBase | 2 +- proxy/proxy.py | 39 ++++++++++++++++++--------- requirements-tunnel.txt | 1 + 5 files changed, 60 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index e886eeeed2..c4b9f608ce 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -952,7 +952,6 @@ jobs: with: username: abhinavsingh password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - # TODO: openssl image is not published on DockerHub - name: Push to DockerHub run: >- REGISTRY_URL="abhinavsingh/proxy.py"; @@ -964,7 +963,6 @@ jobs: --platform ${{ needs.pre-setup.outputs.container-platforms }} - --build-arg SKIP_OPENSSL=1 --build-arg PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' diff --git a/Dockerfile b/Dockerfile index 5277069a5a..63306654a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,20 @@ -FROM python:3.11-alpine as base +FROM ghcr.io/abhinavsingh/proxy.py:base as builder LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ - com.abhinavsingh.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ + org.opencontainers.image.title="proxy.py" \ + org.opencontainers.image.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ 😈 TLS interception • 🔒 DNS-over-HTTPS • 🔥 Poor Man's VPN • ⏪ Reverse & ⏩ Forward • \ 👮🏿 \"Proxy Server\" framework • 🌐 \"Web Server\" framework • ➵ ➶ ➷ ➠ \"PubSub\" framework • \ 👷 \"Work\" acceptor & executor framework" \ - com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \ - com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \ + org.opencontainers.url="https://github.com/abhinavsingh/proxy.py" \ + org.opencontainers.image.source="https://github.com/abhinavsingh/proxy.py" \ com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" \ - org.opencontainers.image.source="https://github.com/abhinavsingh/proxy.py" + org.opencontainers.image.licenses="BSD-3-Clause" \ + org.opencontainers.image.authors="Abhinav Singh " \ + org.opencontainers.image.vendor="Abhinav Singh" ENV PYTHONUNBUFFERED 1 +ENV PYTHONDONTWRITEBYTECODE 1 ARG SKIP_OPENSSL ARG PROXYPY_PKG_PATH @@ -18,16 +22,33 @@ ARG PROXYPY_PKG_PATH COPY README.md / COPY $PROXYPY_PKG_PATH / -RUN pip install --upgrade pip && \ - pip install \ +# proxy.py itself needs no external dependencies +# Optionally, include openssl to allow +# users to use TLS interception features using Docker +# Use `--build-arg SKIP_OPENSSL=1` to disable openssl installation +RUN /proxy/venv/bin/pip install --no-compile --no-cache-dir \ + -U pip && \ + /proxy/venv/bin/pip install --no-compile --no-cache-dir \ --no-index \ --find-links file:/// \ proxy.py && \ - rm *.whl - -# Use `--build-arg SKIP_OPENSSL=1` to disable openssl installation -RUN if [[ -z "$SKIP_OPENSSL" ]]; then apk update && apk add openssl; fi + rm *.whl && \ + find . -type d -name '__pycache__' | xargs rm -rf && \ + rm -rf /var/cache/apk/* && \ + rm -rf /root/.cache/ && \ + /proxy/venv/bin/pip uninstall -y wheel setuptools pip && \ + /usr/local/bin/pip uninstall -y wheel setuptools pip +FROM python:3.11-alpine +COPY --from=builder /README.md /README.md +COPY --from=builder /proxy /proxy +RUN if [[ -z "$SKIP_OPENSSL" ]]; then \ + apk update && \ + apk --no-cache add openssl && \ + rm -rf /var/cache/apk/* && \ + rm -rf /root/.cache/; \ + fi +ENV PATH="/proxy/venv/bin:${PATH}" EXPOSE 8899/tcp ENTRYPOINT [ "proxy" ] CMD [ \ diff --git a/DockerfileBase b/DockerfileBase index 99c1ef67ab..6bee9f7d1e 100644 --- a/DockerfileBase +++ b/DockerfileBase @@ -33,7 +33,7 @@ RUN python -m venv /proxy/venv && \ -U pip wheel && \ /proxy/venv/bin/pip install --no-compile --no-cache-dir \ paramiko==3.4.0 \ - cryptography==39.0.1 \ + cryptography==42.0.4 \ --prefer-binary && \ apk del .builddeps && \ find . -type d -name '__pycache__' | xargs rm -rf && \ diff --git a/proxy/proxy.py b/proxy/proxy.py index c607dc29eb..fc102c8ab2 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -402,18 +402,27 @@ def _clear_line() -> None: print('\r' + ' ' * 60, end='', flush=True) def _env(scheme: bytes, host: bytes, port: int) -> Optional[Dict[str, Any]]: - response = client( - scheme=scheme, - host=host, - port=port, - path=b'/env/', - method=httpMethods.BIND, - body='v={0}&u={1}&h={2}'.format( - __version__, - os.environ.get('USER', getpass.getuser()), - socket.gethostname(), - ).encode(), - ) + try: + response = client( + scheme=scheme, + host=host, + port=port, + path=b'/env/', + method=httpMethods.BIND, + body='v={0}&u={1}&h={2}'.format( + __version__, + os.environ.get('USER', getpass.getuser()), + socket.gethostname(), + ).encode(), + ) + except socket.gaierror: + _clear_line() + print( + '\r\033[91mUnable to resolve\033[0m', + end='', + flush=True, + ) + return None if response: if ( response.code is not None @@ -445,7 +454,11 @@ def _env(scheme: bytes, host: bytes, port: int) -> Optional[Dict[str, Any]]: ) else: _clear_line() - print('\r\033[91mUnable to connect\033[0m') + print( + '\r\033[91mUnable to connect\033[0m', + end='', + flush=True, + ) return None def _parse() -> Tuple[str, int]: diff --git a/requirements-tunnel.txt b/requirements-tunnel.txt index a002227042..c80a8cde89 100644 --- a/requirements-tunnel.txt +++ b/requirements-tunnel.txt @@ -3,3 +3,4 @@ paramiko==3.4.0; python_version >= '3.11' types-paramiko==2.11.3; python_version < '3.11' types-paramiko==3.4.0.20240311; python_version >= '3.11' cryptography==36.0.2; python_version <= '3.6' +cryptography==39.0.1; python_version > '3.6' From afa89bc74975cd683f7767a36bbd6864e55db220 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 16 May 2024 01:27:02 +0530 Subject: [PATCH 09/18] Grout (ngrok alternative) using Docker doc --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index b06d7ce81b..2d842c452b 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ - [TLS Interception](#tls-interception) - [TLS Interception With Docker](#tls-interception-with-docker) - [GROUT (NGROK Alternative)](#grout-ngrok-alternative) + - [Grout using Docker](#grout-using-docker) - [How Grout works](#how-grout-works) - [Self-hosted Grout](#self-hosted-grout) - [Proxy Over SSH Tunnel](#proxy-over-ssh-tunnel) @@ -1348,6 +1349,22 @@ SUPPORT: https://jaxl.io ``` +## Grout using Docker + +```console +❯ docker run -it \ + --entrypoint grout \ + --rm -v ~/.proxy:/root/.proxy \ + abhinavsingh/proxy.py:latest \ + http://host.docker.internal:29876 +``` + +Above: + +- We changed `--entrypoint` to `grout` +- We replaced `localhost` with `host.docker.internal`, so that `grout` can route traffic to port `29876` running on the host machine +- *(Optional)* Mount host machine `~/.proxy` folder, so that `grout` credentials can persist across container restarts + ## How Grout works - `grout` infrastructure has 2 components: client and server From e34da54323fe52553e96d7571990337c0cff1f37 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 16 May 2024 08:12:47 +0530 Subject: [PATCH 10/18] Grout (ngrok alternative) using Docker doc From a7077cf8db3bb66a6667a9d968a401e8f805e092 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Sun, 9 Jun 2024 22:52:37 +0530 Subject: [PATCH 11/18] Add `ModifyRequestHeaderPlugin` (#1420) * Add `ModifyRequestHeaderPlugin` * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add to README * Fix lint issues shown by `Python3.11.8` --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- README.md | 26 +++++++++++++++++ proxy/http/server/web.py | 2 +- proxy/plugin/__init__.py | 2 ++ proxy/plugin/modify_request_header.py | 40 +++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 proxy/plugin/modify_request_header.py diff --git a/README.md b/README.md index 2d842c452b..cafa99d656 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ - [Proxy Pool Plugin](#proxypoolplugin) - [Filter By Client IP Plugin](#filterbyclientipplugin) - [Modify Chunk Response Plugin](#modifychunkresponseplugin) + - [Modify Request Header Plugin](#modifyrequestheaderplugin) - [Cloudflare DNS Resolver Plugin](#cloudflarednsresolverplugin) - [Custom DNS Resolver Plugin](#customdnsresolverplugin) - [Custom Network Interface](#customnetworkinterface) @@ -932,6 +933,31 @@ plugin Modify `ModifyChunkResponsePlugin` to your taste. Example, instead of sending hard-coded chunks, parse and modify the original `JSON` chunks received from the upstream server. +### ModifyRequestHeaderPlugin + +This plugin demonstrate how to modify outgoing HTTPS request headers under TLS interception mode. + +Start `proxy.py` as: + +```console +❯ proxy \ + --plugins proxy.plugin.ModifyRequestHeaderPlugin \ + ... [TLS interception flags] ... +``` + +Verify using `curl -x localhost:8899 --cacert ca-cert.pem https://httpbin.org/get`: + +```console +{ + "args": {}, + "headers": { + ... [redacted] ..., + "X-Proxy-Py-Version": "2.4.4rc6.dev15+gf533c711" + }, + ... [redacted] ... +} +``` + ### CloudflareDnsResolverPlugin This plugin uses `Cloudflare` hosted `DNS-over-HTTPS` [API](https://developers.cloudflare.com/1.1.1.1/encrypted-dns/dns-over-https/make-api-requests/dns-json) (json). diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py index f756494380..f3899e890c 100644 --- a/proxy/http/server/web.py +++ b/proxy/http/server/web.py @@ -217,7 +217,7 @@ def on_client_data(self, raw: memoryview) -> None: self.pipeline_request = None def on_response_chunk(self, chunk: List[memoryview]) -> List[memoryview]: - self._response_size += sum([len(c) for c in chunk]) + self._response_size += sum(len(c) for c in chunk) return chunk def on_client_connection_close(self) -> None: diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py index c3ad91945b..74c7e8d4ef 100644 --- a/proxy/plugin/__init__.py +++ b/proxy/plugin/__init__.py @@ -32,6 +32,7 @@ from .filter_by_client_ip import FilterByClientIpPlugin from .filter_by_url_regex import FilterByURLRegexPlugin from .modify_chunk_response import ModifyChunkResponsePlugin +from .modify_request_header import ModifyRequestHeaderPlugin from .redirect_to_custom_server import RedirectToCustomServerPlugin @@ -53,4 +54,5 @@ 'CustomDnsResolverPlugin', 'CloudflareDnsResolverPlugin', 'ProgramNamePlugin', + 'ModifyRequestHeaderPlugin', ] diff --git a/proxy/plugin/modify_request_header.py b/proxy/plugin/modify_request_header.py new file mode 100644 index 0000000000..72735c2781 --- /dev/null +++ b/proxy/plugin/modify_request_header.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +from typing import Optional + +from ..http.proxy import HttpProxyBasePlugin +from ..http.parser import HttpParser +from ..common.utils import bytes_ +from ..common.version import __version__ + + +class ModifyRequestHeaderPlugin(HttpProxyBasePlugin): + """Modify request header before sending to upstream server.""" + + # def before_upstream_connection(self, request: HttpParser) -> Optional[HttpParser]: + # """NOTE: Use this for HTTP only request headers modification.""" + # request.add_header( + # b"x-proxy-py-version", + # bytes_(__version__), + # ) + # return request + + def handle_client_request(self, request: HttpParser) -> Optional[HttpParser]: + """NOTE: This is for HTTPS request headers modification when under TLS interception. + + For HTTPS requests, modification of request under TLS interception WILL NOT WORK + through before_upstream_connection. + """ + request.add_header( + b'x-proxy-py-version', + bytes_(__version__), + ) + return request From 84c36b60c2a0e44c9dd700e27101222d070e44ff Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Sun, 28 Jul 2024 10:43:37 +0530 Subject: [PATCH 12/18] Static route reverse proxy always `needs_upstream` (#1434) * Static route reverse proxy always needs_upstream * reverse hack done in https://github.com/abhinavsingh/proxy.py/pull/1371/files\#diff-8e4998393b40035040fb8494b321b9538b897ed0f25d35aeeb037a99f623abc3 to make tests work * httpbingo is an issue on github workflows --- proxy/http/server/reverse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/proxy/http/server/reverse.py b/proxy/http/server/reverse.py index 4d91bf3a0a..303b627f20 100644 --- a/proxy/http/server/reverse.py +++ b/proxy/http/server/reverse.py @@ -84,6 +84,7 @@ def handle_request(self, request: HttpParser) -> None: self.choice = Url.from_bytes( random.choice(route[1]), ) + needs_upstream = True break # Dynamic routes elif isinstance(route, str): From 2dbc8af0bb96314ae17b208c804ead137ece5c5a Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Mon, 5 Aug 2024 23:01:54 +0530 Subject: [PATCH 13/18] Support Grout Wildcards (#1439) * Support Grout Wildcards * Disable `Upload coverage to Codecov` which is already broken for sometime --- .github/workflows/test-library.yml | 10 +++++----- proxy/proxy.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index c4b9f608ce..69c99a0ce3 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -545,11 +545,11 @@ jobs: --parallel-live --skip-missing-interpreters false --skip-pkg-install - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - flags: pytest, GHA, Python ${{ matrix.python }}, ${{ runner.os }} - verbose: true + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@v3 + # with: + # flags: pytest, GHA, Python ${{ matrix.python }}, ${{ runner.os }} + # verbose: true test-container: runs-on: ubuntu-20.04 diff --git a/proxy/proxy.py b/proxy/proxy.py index fc102c8ab2..faa6cf3412 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -466,10 +466,11 @@ def _parse() -> Tuple[str, int]: parser = argparse.ArgumentParser(add_help=False) parser.add_argument('route', nargs='?', default=None) parser.add_argument('name', nargs='?', default=None) + parser.add_argument('--wildcard', action='store_true', help='Enable wildcard') args, _remaining_args = parser.parse_known_args() grout_tld = default_grout_tld if args.name is not None and '.' in args.name: - grout_tld = args.name.split('.', maxsplit=1)[1] + grout_tld = args.name if args.wildcard else args.name.split('.', maxsplit=1)[1] grout_tld_parts = grout_tld.split(':') tld_host = grout_tld_parts[0] tld_port = 443 From b9fa0d5a4c690693eb57b3a020e098e81fcf518d Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Tue, 6 Aug 2024 10:02:32 +0530 Subject: [PATCH 14/18] Renable Codecov (#1440) * Renable Codecov * Bump codecov-action and provide CODECOV_TOKEN --- .github/workflows/test-library.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 69c99a0ce3..86fe218088 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -545,12 +545,13 @@ jobs: --parallel-live --skip-missing-interpreters false --skip-pkg-install - # - name: Upload coverage to Codecov - # uses: codecov/codecov-action@v3 - # with: - # flags: pytest, GHA, Python ${{ matrix.python }}, ${{ runner.os }} - # verbose: true - + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + flags: pytest, GHA, Python ${{ matrix.python }}, ${{ runner.os }} + verbose: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} test-container: runs-on: ubuntu-20.04 permissions: From 6ac5aedd2631985abc7630bba2b6974e30956707 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Tue, 6 Aug 2024 11:58:58 +0530 Subject: [PATCH 15/18] `Grout Wildcard` documentation (#1441) * Add documentation around `Grout Wildcard` support * Fix spellcheck * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- README.md | 59 ++++++++++++++++++++++++++++++++++++-- docs/spelling_wordlist.txt | 3 ++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cafa99d656..a82ce93fc8 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,10 @@ - [TLS Interception](#tls-interception) - [TLS Interception With Docker](#tls-interception-with-docker) - [GROUT (NGROK Alternative)](#grout-ngrok-alternative) + - [Grout Usage](#grout-usage) + - [Grout Authentication](#grout-authentication) + - [Grout Paths](#grout-paths) + - [Grout Wildcard Domains](#grout-wildcard-domains) - [Grout using Docker](#grout-using-docker) - [How Grout works](#how-grout-works) - [Self-hosted Grout](#self-hosted-grout) @@ -1323,7 +1327,10 @@ with TLS Interception: # GROUT (NGROK Alternative) -`grout` is a drop-in alternative to `ngrok` that comes packaged within `proxy.py` +1. `grout` is a drop-in alternative for `ngrok` and `frp` +2. `grout` comes packaged within `proxy.py` + +## Grout Usage ```console ❯ grout @@ -1375,12 +1382,58 @@ SUPPORT: https://jaxl.io ``` +## Grout Authentication + +Grout supports authentication to protect your files, folders and services from unauthorized +access. Use `--basic-auth` flag to enforce authentication. Example: + +```console +grout /path/to/folder --basic-auth user:pass +grout https://localhost:8080 --basic-auth u:p +``` + +## Grout Paths + +By default, Grout allows access to all paths on the services. Use `--path` flag to restrict +access to only certain paths on your web service. Example: + +```console +grout https://localhost:8080 --path /worker/ +grout https://localhost:8080 --path /webhook/ --path /callback/ +``` + +## Grout Wildcard Domains + +By default, Grout client serves incoming traffic on a dedicated subdomain. +However, some services (e.g. Kubernetes) may want to serve traffic on adhoc subdomains. +Starting a dedicated Grout client for every adhoc subdomain may not be a practical solution. + +For such scenarios, Grout supports wildcard domains. Here is how to configure your own +wildcard domain for use with Grout clients. + +1. Choose a domain e.g. `custom.example.com` +2. Your service wants to serve traffic for `custom.example.com` and `*.custom.example.com` +3. If you plan on using `https://`, you need to setup a load balancer: + - Setup a HTTPS load balancer (LB) + - Configure LB with certificate generated for `custom.example.com` and `*.custom.example.com` + - Point traffic to Grout service public IP addresses +4. Contact Grout team at support@jaxl.com to whitelist `custom.example.com`. Grout team will make + sure you really own the domain and you have configured a valid SSL certificate as described above + +Start Grout with `--wildcard` flag. Example: + +```console +grout https://localhost:8080 custom.example.com --wildcard +2024-08-05 18:24:59,294 - grout - Logged in as someone@gmail.com +2024-08-05 18:25:03,159 - setup - Grouting https://*.custom.domain.com +``` + ## Grout using Docker ```console -❯ docker run -it \ +❯ docker run --rm -it \ --entrypoint grout \ - --rm -v ~/.proxy:/root/.proxy \ + -v ~/.proxy:/root/.proxy \ abhinavsingh/proxy.py:latest \ http://host.docker.internal:29876 ``` diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 89bbd31845..103cac65a4 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -28,3 +28,6 @@ websocket writables www youtube +Kubernetes +adhoc +balancer From 201d02ce72d4a491e10d82e022c1e6bc93c6d68d Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Fri, 9 Aug 2024 23:45:17 +0530 Subject: [PATCH 16/18] Deprecate usage of `ssl.wrap_socket` in favour of `SSLContext.wrap_socket` (#1443) * Remove use of ssl.wrap_socket ssl.wrap_socket() has been deprecated since Python 3.7, and isn't recommended for use, and further, has been removed in Python 3.12. ssl.SSLContext().wrap_socket() is the new path forward, so switch the one callsite and the two test cases to use it instead. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix `SSLContext.wrap_socket` params and reusable `DEFAULT_SSL_CONTEXT_OPTIONS` * Fix test cases * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix e2e tests??? --------- Co-authored-by: Steve Kowalik Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- proxy/common/constants.py | 4 ++++ proxy/common/utils.py | 19 ++++----------- proxy/core/connection/client.py | 23 +++++++++++++------ .../proxy/test_http_proxy_tls_interception.py | 17 +++++++++----- ...ttp_proxy_plugins_with_tls_interception.py | 4 ++-- 5 files changed, 38 insertions(+), 29 deletions(-) diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 673f9a903c..46558227de 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for more details. """ import os +import ssl import sys import time import pathlib @@ -156,6 +157,9 @@ def _env_threadless_compliant() -> bool: DEFAULT_SELECTOR_SELECT_TIMEOUT = 25 / 1000 DEFAULT_WAIT_FOR_TASKS_TIMEOUT = 1 / 1000 DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT = 1 # in seconds +DEFAULT_SSL_CONTEXT_OPTIONS = ( + ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 +) DEFAULT_DEVTOOLS_DOC_URL = 'http://proxy' DEFAULT_DEVTOOLS_FRAME_ID = secrets.token_hex(8) diff --git a/proxy/common/utils.py b/proxy/common/utils.py index 4e2eed8144..9ac8883d07 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -26,7 +26,7 @@ from .types import HostPort from .constants import ( CRLF, COLON, HTTP_1_1, IS_WINDOWS, WHITESPACE, DEFAULT_TIMEOUT, - DEFAULT_THREADLESS, PROXY_AGENT_HEADER_VALUE, + DEFAULT_THREADLESS, PROXY_AGENT_HEADER_VALUE, DEFAULT_SSL_CONTEXT_OPTIONS, ) @@ -219,20 +219,11 @@ def wrap_socket( cafile: Optional[str] = None, ) -> ssl.SSLSocket: """Use this to upgrade server_side socket to TLS.""" - ctx = ssl.create_default_context( - ssl.Purpose.CLIENT_AUTH, - cafile=cafile, - ) - ctx.options |= ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 + ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=cafile) + ctx.options |= DEFAULT_SSL_CONTEXT_OPTIONS ctx.verify_mode = ssl.CERT_NONE - ctx.load_cert_chain( - certfile=certfile, - keyfile=keyfile, - ) - return ctx.wrap_socket( - conn, - server_side=True, - ) + ctx.load_cert_chain(certfile=certfile, keyfile=keyfile) + return ctx.wrap_socket(conn, server_side=True) def new_socket_connection( diff --git a/proxy/core/connection/client.py b/proxy/core/connection/client.py index f241c56a06..4c8ad97cdd 100644 --- a/proxy/core/connection/client.py +++ b/proxy/core/connection/client.py @@ -9,11 +9,12 @@ :license: BSD, see LICENSE for more details. """ import ssl -from typing import Optional +from typing import Any, Dict, Optional from .types import tcpConnectionTypes from .connection import TcpConnection, TcpConnectionUninitializedException from ...common.types import HostPort, TcpOrTlsSocket +from ...common.constants import DEFAULT_SSL_CONTEXT_OPTIONS class TcpClientConnection(TcpConnection): @@ -42,11 +43,19 @@ def connection(self) -> TcpOrTlsSocket: def wrap(self, keyfile: str, certfile: str) -> None: self.connection.setblocking(True) self.flush() - self._conn = ssl.wrap_socket( - self.connection, - server_side=True, - certfile=certfile, - keyfile=keyfile, - ssl_version=ssl.PROTOCOL_TLS, + ctx = ssl.SSLContext( + protocol=( + ssl.PROTOCOL_TLS_CLIENT + if self.tag == 'server' + else ssl.PROTOCOL_TLS_SERVER + ), ) + ctx.options |= DEFAULT_SSL_CONTEXT_OPTIONS + ctx.load_cert_chain(certfile=certfile, keyfile=keyfile) + assert self.addr + kwargs: Dict[str, Any] = {'server_side': True} + if self.tag == 'server': + assert self.addr + kwargs['server_hostname'] = self.addr[0] + self._conn = ctx.wrap_socket(self.connection, **kwargs) self.connection.setblocking(False) diff --git a/tests/http/proxy/test_http_proxy_tls_interception.py b/tests/http/proxy/test_http_proxy_tls_interception.py index 654bbc5fcd..2fbdaef9a0 100644 --- a/tests/http/proxy/test_http_proxy_tls_interception.py +++ b/tests/http/proxy/test_http_proxy_tls_interception.py @@ -62,9 +62,9 @@ async def test_e2e(self, mocker: MockerFixture) -> None: self.mock_ssl_context.return_value.wrap_socket.return_value = upstream_tls_sock # Used for client wrapping - self.mock_ssl_wrap = mocker.patch('ssl.wrap_socket') + self.mock_ssl_wrap = mocker.patch('ssl.SSLContext') client_tls_sock = mock.MagicMock(spec=ssl.SSLSocket) - self.mock_ssl_wrap.return_value = client_tls_sock + self.mock_ssl_wrap.return_value.wrap_socket.return_value = client_tls_sock plain_connection = mock.MagicMock(spec=socket.socket) @@ -251,13 +251,18 @@ async def asyncReturn(val: T) -> T: ) assert self.flags.ca_cert_dir is not None self.mock_ssl_wrap.assert_called_with( - self._conn, - server_side=True, + protocol=ssl.PROTOCOL_TLS_SERVER, + ) + self.mock_ssl_wrap.return_value.load_cert_chain( keyfile=self.flags.ca_signing_key_file, certfile=HttpProxyPlugin.generated_cert_file_path( - self.flags.ca_cert_dir, host, + self.flags.ca_cert_dir, + host, ), - ssl_version=ssl.PROTOCOL_TLS, + ) + self.mock_ssl_wrap.return_value.wrap_socket.assert_called_with( + self._conn, + server_side=True, ) self.assertEqual(self._conn.setblocking.call_count, 2) self.assertEqual( diff --git a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py index 3d8d6a28f4..a0a05b61f8 100644 --- a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py +++ b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py @@ -46,7 +46,7 @@ def _setUp(self, request: Any, mocker: MockerFixture) -> None: 'proxy.http.proxy.server.TcpServerConnection', ) self.mock_ssl_context = mocker.patch('ssl.create_default_context') - self.mock_ssl_wrap = mocker.patch('ssl.wrap_socket') + self.mock_ssl_wrap = mocker.patch('ssl.SSLContext') self.mock_sign_csr.return_value = True self.mock_gen_csr.return_value = True @@ -82,7 +82,7 @@ def _setUp(self, request: Any, mocker: MockerFixture) -> None: self.server_ssl_connection = mocker.MagicMock(spec=ssl.SSLSocket) self.mock_ssl_context.return_value.wrap_socket.return_value = self.server_ssl_connection self.client_ssl_connection = mocker.MagicMock(spec=ssl.SSLSocket) - self.mock_ssl_wrap.return_value = self.client_ssl_connection + self.mock_ssl_wrap.return_value.wrap_socket.return_value = self.client_ssl_connection def has_buffer() -> bool: return cast(bool, self.server.queue.called) From 1e4e87d5154b02d358227ff33c4e2ed5a3791ec1 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Sat, 10 Aug 2024 09:33:49 +0530 Subject: [PATCH 17/18] Support for Python 3.12 (#1444) * Support for Python 3.12 * Use `assert_called_once_with` and not `called_once_with` * lint fix * Fix `test_pac_file_served_from_disk` * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/test-library.yml | 3 ++- proxy/http/handler.py | 4 +-- tests/http/web/test_web_server.py | 41 +++++++++++++++++++----------- tox.ini | 2 +- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 86fe218088..88e9d407d3 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -446,8 +446,9 @@ jobs: # NOTE: The latest and the lowest supported Pythons are prioritized # NOTE: to improve the responsiveness. It's nice to see the most # NOTE: important results first. - - '3.11' + - '3.12' - 3.6 + - '3.11' - '3.10' - 3.9 - 3.8 diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 0bdcfa22c0..5120d5b32c 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -292,9 +292,7 @@ def _parse_first_request(self, data: memoryview) -> bool: return True # Discover which HTTP handler plugin is capable of # handling the current incoming request - klass = self._discover_plugin_klass( - self.request.http_handler_protocol, - ) + klass = self._discover_plugin_klass(self.request.http_handler_protocol) if klass is None: # No matching protocol class found. # Return bad request response and diff --git a/tests/http/web/test_web_server.py b/tests/http/web/test_web_server.py index 71baac4fd6..8100d995bd 100644 --- a/tests/http/web/test_web_server.py +++ b/tests/http/web/test_web_server.py @@ -59,19 +59,6 @@ def test_on_client_connection_called_on_teardown(mocker: MockerFixture) -> None: assert _conn.closed -def mock_selector_for_client_read(self: Any) -> None: - self.mock_selector.return_value.select.return_value = [ - ( - selectors.SelectorKey( - fileobj=self._conn.fileno(), - fd=self._conn.fileno(), - events=selectors.EVENT_READ, - data=None, - ), - selectors.EVENT_READ, - ), - ] - # @mock.patch('socket.fromfd') # def test_on_client_connection_called_on_teardown( # self, mock_fromfd: mock.Mock, @@ -171,16 +158,40 @@ def _setUp(self, request: Any, mocker: MockerFixture) -> None: b'GET / HTTP/1.1', CRLF, ]) - mock_selector_for_client_read(self) + self.mock_selector.return_value.select.side_effect = [ + [ + ( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + ), + ], + [ + ( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_WRITE, + data=None, + ), + selectors.EVENT_WRITE, + ), + ], + ] @pytest.mark.asyncio # type: ignore[misc] async def test_pac_file_served_from_disk(self) -> None: + await self.protocol_handler._run_once() await self.protocol_handler._run_once() self.assertEqual( self.protocol_handler.request.state, httpParserStates.COMPLETE, ) - self._conn.send.called_once_with( + self._conn.send.assert_called_once_with( build_http_response( 200, reason=b'OK', diff --git a/tox.ini b/tox.ini index 5e7d706a4b..37f686ca45 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36,py37,py38,py39,py310,py311 +envlist = py36,py37,py38,py39,py310,py311,py312 isolated_build = true minversion = 3.21.0 From 39854e1d799e198c393d3721e8b75602f72554b0 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Sat, 10 Aug 2024 10:28:53 +0530 Subject: [PATCH 18/18] Use Python 3.12 as default Docker base image (#1445) * Use Python 3.12 as default Docker base image * Add standard OCI labels --- Dockerfile | 14 ++++++++------ DockerfileBase | 16 +++++++++------- README.md | 2 +- setup.cfg | 1 + 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 63306654a7..5be77ec843 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,19 @@ FROM ghcr.io/abhinavsingh/proxy.py:base as builder -LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ - org.opencontainers.image.title="proxy.py" \ - org.opencontainers.image.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ +LABEL org.opencontainers.image.title="proxy.py" \ + org.opencontainers.image.description="💫 Ngrok FRP Alternative • ⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ 😈 TLS interception • 🔒 DNS-over-HTTPS • 🔥 Poor Man's VPN • ⏪ Reverse & ⏩ Forward • \ 👮🏿 \"Proxy Server\" framework • 🌐 \"Web Server\" framework • ➵ ➶ ➷ ➠ \"PubSub\" framework • \ 👷 \"Work\" acceptor & executor framework" \ - org.opencontainers.url="https://github.com/abhinavsingh/proxy.py" \ + org.opencontainers.image.url="https://github.com/abhinavsingh/proxy.py" \ org.opencontainers.image.source="https://github.com/abhinavsingh/proxy.py" \ - com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" \ org.opencontainers.image.licenses="BSD-3-Clause" \ org.opencontainers.image.authors="Abhinav Singh " \ - org.opencontainers.image.vendor="Abhinav Singh" + org.opencontainers.image.vendor="Abhinav Singh" \ + org.opencontainers.image.created="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ + org.opencontainers.image.documentation="https://github.com/abhinavsingh/proxy.py#readme" \ + org.opencontainers.image.ref.name="abhinavsingh/proxy.py" \ + com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" ENV PYTHONUNBUFFERED 1 ENV PYTHONDONTWRITEBYTECODE 1 diff --git a/DockerfileBase b/DockerfileBase index 6bee9f7d1e..1a86018dc3 100644 --- a/DockerfileBase +++ b/DockerfileBase @@ -1,17 +1,19 @@ -FROM python:3.11-alpine +FROM python:3.12-alpine -LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ - org.opencontainers.image.title="proxy.py" \ - org.opencontainers.image.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ +LABEL org.opencontainers.image.title="proxy.py" \ + org.opencontainers.image.description="💫 Ngrok FRP Alternative • ⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ 😈 TLS interception • 🔒 DNS-over-HTTPS • 🔥 Poor Man's VPN • ⏪ Reverse & ⏩ Forward • \ 👮🏿 \"Proxy Server\" framework • 🌐 \"Web Server\" framework • ➵ ➶ ➷ ➠ \"PubSub\" framework • \ 👷 \"Work\" acceptor & executor framework" \ - org.opencontainers.url="https://github.com/abhinavsingh/proxy.py" \ + org.opencontainers.image.url="https://github.com/abhinavsingh/proxy.py" \ org.opencontainers.image.source="https://github.com/abhinavsingh/proxy.py" \ - com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" \ org.opencontainers.image.licenses="BSD-3-Clause" \ org.opencontainers.image.authors="Abhinav Singh " \ - org.opencontainers.image.vendor="Abhinav Singh" + org.opencontainers.image.vendor="Abhinav Singh" \ + org.opencontainers.image.created="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ + org.opencontainers.image.documentation="https://github.com/abhinavsingh/proxy.py#readme" \ + org.opencontainers.image.ref.name="abhinavsingh/proxy.py" \ + com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" ENV PYTHONUNBUFFERED 1 ENV PYTHONDONTWRITEBYTECODE 1 diff --git a/README.md b/README.md index a82ce93fc8..d38bc4ede0 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ [![iOS, iOS Simulator](https://img.shields.io/static/v1?label=tested%20with&message=iOS%20%F0%9F%93%B1%20%7C%20iOS%20Simulator%20%F0%9F%93%B1&color=darkgreen&style=for-the-badge)](https://abhinavsingh.com/proxy-py-a-lightweight-single-file-http-proxy-server-in-python/) [![pypi version](https://img.shields.io/pypi/v/proxy.py?style=flat-square)](https://pypi.org/project/proxy.py/) -[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10%20%7C%203.11&color=blue&style=flat-square)](https://www.python.org/) +[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12&color=blue&style=flat-square)](https://www.python.org/) [![Checked with mypy](https://img.shields.io/static/v1?label=MyPy&message=checked&color=blue&style=flat-square)](http://mypy-lang.org/) [![doc](https://img.shields.io/readthedocs/proxypy/latest?style=flat-square&color=darkgreen)](https://proxypy.readthedocs.io/) diff --git a/setup.cfg b/setup.cfg index 4e555abb3e..5dbde4c3f6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -66,6 +66,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Topic :: Internet Topic :: Internet :: Proxy Servers