diff --git a/.github/workflows/dockerfile-base.yml b/.github/workflows/dockerfile-base.yml new file mode 100644 index 0000000000..f08ddc7e47 --- /dev/null +++ b/.github/workflows/dockerfile-base.yml @@ -0,0 +1,79 @@ +--- +name: base + +on: # yamllint disable-line rule:truthy + workflow_dispatch: + +concurrency: + group: >- + ${{ + github.workflow + }}-${{ + github.event.pull_request.number || github.sha + }} + cancel-in-progress: true + +jobs: + pre-setup: + name: ⚙️ Pre-set global build settings + runs-on: ubuntu-20.04 + defaults: + run: + shell: bash + outputs: + container-platforms: ${{ steps.container.outputs.platforms }} + steps: + - name: Calculate container attributes + id: container + shell: bash + run: >- + PLATFORMS="linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x"; + echo "::set-output name=platforms::$PLATFORMS" + + ghcr-base: + runs-on: ubuntu-20.04 + permissions: + packages: write + if: success() + needs: + - pre-setup # transitive, for accessing settings + name: 🐳 ghcr:base + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.release-commitish }} + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + # See https://github.com/docker/buildx/issues/850#issuecomment-996408167 + with: + version: v0.7.0 + buildkitd-flags: --debug + config: .github/buildkitd.toml + install: true + - name: Enable Multiarch # This slows down arm build by 4-5x + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: Create builder + run: | + docker buildx create --name proxypybuilder + docker buildx use proxypybuilder + docker buildx inspect + docker buildx ls + - name: Push base to GHCR + run: >- + docker buildx build + --push + --platform ${{ + needs.pre-setup.outputs.container-platforms + }} + -t ghcr.io/abhinavsingh/proxy.py:base + -f DockerfileBase . +... diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index e886eeeed2..88e9d407d3 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -446,8 +446,9 @@ jobs: # NOTE: The latest and the lowest supported Pythons are prioritized # NOTE: to improve the responsiveness. It's nice to see the most # NOTE: important results first. - - '3.11' + - '3.12' - 3.6 + - '3.11' - '3.10' - 3.9 - 3.8 @@ -546,11 +547,12 @@ jobs: --skip-missing-interpreters false --skip-pkg-install - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: flags: pytest, GHA, Python ${{ matrix.python }}, ${{ runner.os }} verbose: true - + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} test-container: runs-on: ubuntu-20.04 permissions: @@ -952,7 +954,6 @@ jobs: with: username: abhinavsingh password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - # TODO: openssl image is not published on DockerHub - name: Push to DockerHub run: >- REGISTRY_URL="abhinavsingh/proxy.py"; @@ -964,7 +965,6 @@ jobs: --platform ${{ needs.pre-setup.outputs.container-platforms }} - --build-arg SKIP_OPENSSL=1 --build-arg PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' diff --git a/Dockerfile b/Dockerfile index 5277069a5a..5be77ec843 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,22 @@ -FROM python:3.11-alpine as base +FROM ghcr.io/abhinavsingh/proxy.py:base as builder -LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ - com.abhinavsingh.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ +LABEL org.opencontainers.image.title="proxy.py" \ + org.opencontainers.image.description="💫 Ngrok FRP Alternative • ⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ 😈 TLS interception • 🔒 DNS-over-HTTPS • 🔥 Poor Man's VPN • ⏪ Reverse & ⏩ Forward • \ 👮🏿 \"Proxy Server\" framework • 🌐 \"Web Server\" framework • ➵ ➶ ➷ ➠ \"PubSub\" framework • \ 👷 \"Work\" acceptor & executor framework" \ - com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \ - com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \ - com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" \ - org.opencontainers.image.source="https://github.com/abhinavsingh/proxy.py" + org.opencontainers.image.url="https://github.com/abhinavsingh/proxy.py" \ + org.opencontainers.image.source="https://github.com/abhinavsingh/proxy.py" \ + org.opencontainers.image.licenses="BSD-3-Clause" \ + org.opencontainers.image.authors="Abhinav Singh " \ + org.opencontainers.image.vendor="Abhinav Singh" \ + org.opencontainers.image.created="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ + org.opencontainers.image.documentation="https://github.com/abhinavsingh/proxy.py#readme" \ + org.opencontainers.image.ref.name="abhinavsingh/proxy.py" \ + com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" ENV PYTHONUNBUFFERED 1 +ENV PYTHONDONTWRITEBYTECODE 1 ARG SKIP_OPENSSL ARG PROXYPY_PKG_PATH @@ -18,16 +24,33 @@ ARG PROXYPY_PKG_PATH COPY README.md / COPY $PROXYPY_PKG_PATH / -RUN pip install --upgrade pip && \ - pip install \ +# proxy.py itself needs no external dependencies +# Optionally, include openssl to allow +# users to use TLS interception features using Docker +# Use `--build-arg SKIP_OPENSSL=1` to disable openssl installation +RUN /proxy/venv/bin/pip install --no-compile --no-cache-dir \ + -U pip && \ + /proxy/venv/bin/pip install --no-compile --no-cache-dir \ --no-index \ --find-links file:/// \ proxy.py && \ - rm *.whl - -# Use `--build-arg SKIP_OPENSSL=1` to disable openssl installation -RUN if [[ -z "$SKIP_OPENSSL" ]]; then apk update && apk add openssl; fi + rm *.whl && \ + find . -type d -name '__pycache__' | xargs rm -rf && \ + rm -rf /var/cache/apk/* && \ + rm -rf /root/.cache/ && \ + /proxy/venv/bin/pip uninstall -y wheel setuptools pip && \ + /usr/local/bin/pip uninstall -y wheel setuptools pip +FROM python:3.11-alpine +COPY --from=builder /README.md /README.md +COPY --from=builder /proxy /proxy +RUN if [[ -z "$SKIP_OPENSSL" ]]; then \ + apk update && \ + apk --no-cache add openssl && \ + rm -rf /var/cache/apk/* && \ + rm -rf /root/.cache/; \ + fi +ENV PATH="/proxy/venv/bin:${PATH}" EXPOSE 8899/tcp ENTRYPOINT [ "proxy" ] CMD [ \ diff --git a/DockerfileBase b/DockerfileBase new file mode 100644 index 0000000000..1a86018dc3 --- /dev/null +++ b/DockerfileBase @@ -0,0 +1,43 @@ +FROM python:3.12-alpine + +LABEL org.opencontainers.image.title="proxy.py" \ + org.opencontainers.image.description="💫 Ngrok FRP Alternative • ⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ + 😈 TLS interception • 🔒 DNS-over-HTTPS • 🔥 Poor Man's VPN • ⏪ Reverse & ⏩ Forward • \ + 👮🏿 \"Proxy Server\" framework • 🌐 \"Web Server\" framework • ➵ ➶ ➷ ➠ \"PubSub\" framework • \ + 👷 \"Work\" acceptor & executor framework" \ + org.opencontainers.image.url="https://github.com/abhinavsingh/proxy.py" \ + org.opencontainers.image.source="https://github.com/abhinavsingh/proxy.py" \ + org.opencontainers.image.licenses="BSD-3-Clause" \ + org.opencontainers.image.authors="Abhinav Singh " \ + org.opencontainers.image.vendor="Abhinav Singh" \ + org.opencontainers.image.created="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ + org.opencontainers.image.documentation="https://github.com/abhinavsingh/proxy.py#readme" \ + org.opencontainers.image.ref.name="abhinavsingh/proxy.py" \ + com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" + +ENV PYTHONUNBUFFERED 1 +ENV PYTHONDONTWRITEBYTECODE 1 + +# Install paramiko and cryptography to allow +# users to use tunneling features using Docker +RUN apk update && apk --no-cache add \ + --virtual .builddeps \ + gcc \ + musl-dev \ + libffi-dev \ + openssl-dev \ + python3-dev \ + cargo \ + rust \ + make +RUN python -m venv /proxy/venv && \ + /proxy/venv/bin/pip install --no-compile --no-cache-dir \ + -U pip wheel && \ + /proxy/venv/bin/pip install --no-compile --no-cache-dir \ + paramiko==3.4.0 \ + cryptography==42.0.4 \ + --prefer-binary && \ + apk del .builddeps && \ + find . -type d -name '__pycache__' | xargs rm -rf && \ + rm -rf /var/cache/apk/* && \ + rm -rf /root/.cache/ diff --git a/README.md b/README.md index 29b50a748f..9c259554f7 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ [![iOS, iOS Simulator](https://img.shields.io/static/v1?label=tested%20with&message=iOS%20%F0%9F%93%B1%20%7C%20iOS%20Simulator%20%F0%9F%93%B1&color=darkgreen&style=for-the-badge)](https://abhinavsingh.com/proxy-py-a-lightweight-single-file-http-proxy-server-in-python/) [![pypi version](https://img.shields.io/pypi/v/proxy.py?style=flat-square)](https://pypi.org/project/proxy.py/) -[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10%20%7C%203.11&color=blue&style=flat-square)](https://www.python.org/) +[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12&color=blue&style=flat-square)](https://www.python.org/) [![Checked with mypy](https://img.shields.io/static/v1?label=MyPy&message=checked&color=blue&style=flat-square)](http://mypy-lang.org/) [![doc](https://img.shields.io/readthedocs/proxypy/latest?style=flat-square&color=darkgreen)](https://proxypy.readthedocs.io/) @@ -59,6 +59,7 @@ - [Proxy Pool Plugin](#proxypoolplugin) - [Filter By Client IP Plugin](#filterbyclientipplugin) - [Modify Chunk Response Plugin](#modifychunkresponseplugin) + - [Modify Request Header Plugin](#modifyrequestheaderplugin) - [Cloudflare DNS Resolver Plugin](#cloudflarednsresolverplugin) - [Custom DNS Resolver Plugin](#customdnsresolverplugin) - [Custom Network Interface](#customnetworkinterface) @@ -71,6 +72,14 @@ - [End-to-End Encryption](#end-to-end-encryption) - [TLS Interception](#tls-interception) - [TLS Interception With Docker](#tls-interception-with-docker) +- [GROUT (NGROK Alternative)](#grout-ngrok-alternative) + - [Grout Usage](#grout-usage) + - [Grout Authentication](#grout-authentication) + - [Grout Paths](#grout-paths) + - [Grout Wildcard Domains](#grout-wildcard-domains) + - [Grout using Docker](#grout-using-docker) + - [How Grout works](#how-grout-works) + - [Self-hosted Grout](#self-hosted-grout) - [Proxy Over SSH Tunnel](#proxy-over-ssh-tunnel) - [Proxy Remote Requests Locally](#proxy-remote-requests-locally) - [Proxy Local Requests Remotely](#proxy-local-requests-remotely) @@ -138,6 +147,7 @@ [//]: # (DO-NOT-REMOVE-docs-badges-END) # Features +- [A drop-in alternative to `ngrok`](#grout-ngrok-alternative) - Fast & Scalable - Scale up by using all available cores on the system @@ -927,6 +937,31 @@ plugin Modify `ModifyChunkResponsePlugin` to your taste. Example, instead of sending hard-coded chunks, parse and modify the original `JSON` chunks received from the upstream server. +### ModifyRequestHeaderPlugin + +This plugin demonstrate how to modify outgoing HTTPS request headers under TLS interception mode. + +Start `proxy.py` as: + +```console +❯ proxy \ + --plugins proxy.plugin.ModifyRequestHeaderPlugin \ + ... [TLS interception flags] ... +``` + +Verify using `curl -x localhost:8899 --cacert ca-cert.pem https://httpbin.org/get`: + +```console +{ + "args": {}, + "headers": { + ... [redacted] ..., + "X-Proxy-Py-Version": "2.4.4rc6.dev15+gf533c711" + }, + ... [redacted] ... +} +``` + ### CloudflareDnsResolverPlugin This plugin uses `Cloudflare` hosted `DNS-over-HTTPS` [API](https://developers.cloudflare.com/1.1.1.1/encrypted-dns/dns-over-https/make-api-requests/dns-json) (json). @@ -1292,6 +1327,141 @@ with TLS Interception: } ``` +# GROUT (NGROK Alternative) + +1. `grout` is a drop-in alternative for `ngrok` and `frp` +2. `grout` comes packaged within `proxy.py` + +## Grout Usage + +```console +❯ grout +NAME: + grout - securely tunnel local files, folders and services to public URLs + +USAGE: + grout route [name] + +DESCRIPTION: + grout exposes local networked services behinds NATs and firewalls to the + public internet over a secure tunnel. Share local folders, directories and websites, + build/test webhook consumers and self-host personal services to public URLs. + +EXAMPLES: + Share Files and Folders: + grout C:\path\to\folder # Share a folder on your system + grout /path/to/folder # Share a folder on your system + grout /path/to/folder --basic-auth user:pass # Add authentication for shared folder + grout /path/to/photo.jpg # Share a specific file on your system + + Expose HTTP, HTTPS and Websockets: + grout http://localhost:9090 # Expose HTTP service running on port 9090 + grout https://localhost:8080 # Expose HTTPS service running on port 8080 + grout https://localhost:8080 --path /worker/ # Expose only certain paths of HTTPS service on port 8080 + grout https://localhost:8080 --basic-auth u:p # Add authentication for exposed HTTPS service on port 8080 + + Expose TCP Services: + grout tcp://:6379 # Expose Redis service running locally on port 6379 + grout tcp://:22 # Expose SSH service running locally on port 22 + + Custom URLs: + grout https://localhost:8080 abhinavsingh # Custom URL for HTTPS service running on port 8080 + grout tcp://:22 abhinavsingh # Custom URL for SSH service running locally on port 22 + + Custom Domains: + grout tcp://:5432 abhinavsingh.domain.tld # Custom URL for Postgres service running locally on port 5432 + + Self-hosted solutions: + grout tcp://:5432 abhinavsingh.my.server # Custom URL for Postgres service running locally on port 5432 + +SUPPORT: + Write to us at support@jaxl.com + + Privacy policy and Terms & conditions + https://jaxl.com/privacy/ + + Created by Jaxl™ + https://jaxl.io +``` + +## Grout Authentication + +Grout supports authentication to protect your files, folders and services from unauthorized +access. Use `--basic-auth` flag to enforce authentication. Example: + +```console +grout /path/to/folder --basic-auth user:pass +grout https://localhost:8080 --basic-auth u:p +``` + +## Grout Paths + +By default, Grout allows access to all paths on the services. Use `--path` flag to restrict +access to only certain paths on your web service. Example: + +```console +grout https://localhost:8080 --path /worker/ +grout https://localhost:8080 --path /webhook/ --path /callback/ +``` + +## Grout Wildcard Domains + +By default, Grout client serves incoming traffic on a dedicated subdomain. +However, some services (e.g. Kubernetes) may want to serve traffic on adhoc subdomains. +Starting a dedicated Grout client for every adhoc subdomain may not be a practical solution. + +For such scenarios, Grout supports wildcard domains. Here is how to configure your own +wildcard domain for use with Grout clients. + +1. Choose a domain e.g. `custom.example.com` +2. Your service wants to serve traffic for `custom.example.com` and `*.custom.example.com` +3. If you plan on using `https://`, you need to setup a load balancer: + - Setup a HTTPS load balancer (LB) + - Configure LB with certificate generated for `custom.example.com` and `*.custom.example.com` + - Point traffic to Grout service public IP addresses +4. Contact Grout team at support@jaxl.com to whitelist `custom.example.com`. Grout team will make + sure you really own the domain and you have configured a valid SSL certificate as described above + +Start Grout with `--wildcard` flag. Example: + +```console +grout https://localhost:8080 custom.example.com --wildcard +2024-08-05 18:24:59,294 - grout - Logged in as someone@gmail.com +2024-08-05 18:25:03,159 - setup - Grouting https://*.custom.domain.com +``` + +## Grout using Docker + +```console +❯ docker run --rm -it \ + --entrypoint grout \ + -v ~/.proxy:/root/.proxy \ + abhinavsingh/proxy.py:latest \ + http://host.docker.internal:29876 +``` + +Above: + +- We changed `--entrypoint` to `grout` +- We replaced `localhost` with `host.docker.internal`, so that `grout` can route traffic to port `29876` running on the host machine +- *(Optional)* Mount host machine `~/.proxy` folder, so that `grout` credentials can persist across container restarts + +## How Grout works + +- `grout` infrastructure has 2 components: client and server +- `grout` client has 2 components: a thin and a thick client +- `grout` thin client is part of open source `proxy.py` (BSD 3-Clause License) +- `grout` thick client and servers are hosted at [jaxl.io](https://jaxl.io) + and a copyright of [Jaxl Innovations Private Limited](https://jaxl.com) +- `grout` server has 3 components: a registry server, a reverse proxy server and a tunnel server + +## Self-Hosted `grout` + +- `grout` thick client and servers can also be hosted on your GCP, AWS, Cloud infrastructures +- With a self-hosted version, your traffic flows through the network you control and trust +- `grout` developers at [jaxl.io](https://jaxl.io) provides GCP, AWS, Docker images for self-hosted solutions +- Please drop an email at [support@jaxl.com](mailto:support@jaxl.com) to get started. + # Proxy Over SSH Tunnel **This is a WIP and may not work as documented** @@ -2342,12 +2512,17 @@ To run standalone benchmark for `proxy.py`, use the following command from repo ```console ❯ proxy -h -usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] +usage: -m [-h] [--enable-proxy-protocol] [--threadless] [--threaded] + [--num-workers NUM_WORKERS] [--enable-events] [--enable-conn-pool] + [--key-file KEY_FILE] [--cert-file CERT_FILE] + [--client-recvbuf-size CLIENT_RECVBUF_SIZE] + [--server-recvbuf-size SERVER_RECVBUF_SIZE] + [--max-sendbuf-size MAX_SENDBUF_SIZE] [--timeout TIMEOUT] + [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--tunnel-username TUNNEL_USERNAME] [--tunnel-ssh-key TUNNEL_SSH_KEY] [--tunnel-ssh-key-passphrase TUNNEL_SSH_KEY_PASSPHRASE] - [--tunnel-remote-port TUNNEL_REMOTE_PORT] [--threadless] - [--threaded] [--num-workers NUM_WORKERS] [--enable-events] + [--tunnel-remote-port TUNNEL_REMOTE_PORT] [--local-executor LOCAL_EXECUTOR] [--backlog BACKLOG] [--hostname HOSTNAME] [--hostnames HOSTNAMES [HOSTNAMES ...]] [--port PORT] [--ports PORTS [PORTS ...]] [--port-file PORT_FILE] @@ -2359,10 +2534,6 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--basic-auth BASIC_AUTH] [--enable-ssh-tunnel] [--work-klass WORK_KLASS] [--pid-file PID_FILE] [--openssl OPENSSL] [--data-dir DATA_DIR] [--ssh-listener-klass SSH_LISTENER_KLASS] - [--enable-proxy-protocol] [--enable-conn-pool] [--key-file KEY_FILE] - [--cert-file CERT_FILE] [--client-recvbuf-size CLIENT_RECVBUF_SIZE] - [--server-recvbuf-size SERVER_RECVBUF_SIZE] - [--max-sendbuf-size MAX_SENDBUF_SIZE] [--timeout TIMEOUT] [--disable-http-proxy] [--disable-headers DISABLE_HEADERS] [--ca-key-file CA_KEY_FILE] [--insecure] [--ca-cert-dir CA_CERT_DIR] [--ca-cert-file CA_CERT_FILE] [--ca-file CA_FILE] @@ -2380,10 +2551,45 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--filtered-client-ips FILTERED_CLIENT_IPS] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] -proxy.py v2.4.4rc6.dev172+ge1879403.d20240425 +proxy.py v2.4.4rc6.dev191+gef5a8922 options: -h, --help show this help message and exit + --enable-proxy-protocol + Default: False. If used, will enable proxy protocol. + Only version 1 is currently supported. + --threadless Default: True. Enabled by default on Python 3.8+ (mac, + linux). When disabled a new thread is spawned to + handle each client connection. + --threaded Default: False. Disabled by default on Python < 3.8 + and windows. When enabled a new thread is spawned to + handle each client connection. + --num-workers NUM_WORKERS + Defaults to number of CPU cores. + --enable-events Default: False. Enables core to dispatch lifecycle + events. Plugins can be used to subscribe for core + events. + --enable-conn-pool Default: False. (WIP) Enable upstream connection + pooling. + --key-file KEY_FILE Default: None. Server key file to enable end-to-end + TLS encryption with clients. If used, must also pass + --cert-file. + --cert-file CERT_FILE + Default: None. Server certificate to enable end-to-end + TLS encryption with clients. If used, must also pass + --key-file. + --client-recvbuf-size CLIENT_RECVBUF_SIZE + Default: 128 KB. Maximum amount of data received from + the client in a single recv() operation. + --server-recvbuf-size SERVER_RECVBUF_SIZE + Default: 128 KB. Maximum amount of data received from + the server in a single recv() operation. + --max-sendbuf-size MAX_SENDBUF_SIZE + Default: 64 KB. Maximum amount of data to flush in a + single send() operation. + --timeout TIMEOUT Default: 10.0. Number of seconds after which an + inactive connection must be dropped. Inactivity is + defined by no data sent or received by the client. --tunnel-hostname TUNNEL_HOSTNAME Default: None. Remote hostname or IP address to which SSH tunnel will be established. @@ -2399,17 +2605,6 @@ options: --tunnel-remote-port TUNNEL_REMOTE_PORT Default: 8899. Remote port which will be forwarded locally for proxy. - --threadless Default: True. Enabled by default on Python 3.8+ (mac, - linux). When disabled a new thread is spawned to - handle each client connection. - --threaded Default: False. Disabled by default on Python < 3.8 - and windows. When enabled a new thread is spawned to - handle each client connection. - --num-workers NUM_WORKERS - Defaults to number of CPU cores. - --enable-events Default: False. Enables core to dispatch lifecycle - events. Plugins can be used to subscribe for core - events. --local-executor LOCAL_EXECUTOR Default: 1. Enabled by default. Use 0 to disable. When enabled acceptors will make use of local (same @@ -2465,30 +2660,6 @@ options: --ssh-listener-klass SSH_LISTENER_KLASS Default: proxy.core.ssh.listener.SshTunnelListener. An implementation of BaseSshTunnelListener - --enable-proxy-protocol - Default: False. If used, will enable proxy protocol. - Only version 1 is currently supported. - --enable-conn-pool Default: False. (WIP) Enable upstream connection - pooling. - --key-file KEY_FILE Default: None. Server key file to enable end-to-end - TLS encryption with clients. If used, must also pass - --cert-file. - --cert-file CERT_FILE - Default: None. Server certificate to enable end-to-end - TLS encryption with clients. If used, must also pass - --key-file. - --client-recvbuf-size CLIENT_RECVBUF_SIZE - Default: 128 KB. Maximum amount of data received from - the client in a single recv() operation. - --server-recvbuf-size SERVER_RECVBUF_SIZE - Default: 128 KB. Maximum amount of data received from - the server in a single recv() operation. - --max-sendbuf-size MAX_SENDBUF_SIZE - Default: 64 KB. Maximum amount of data to flush in a - single send() operation. - --timeout TIMEOUT Default: 10.0. Number of seconds after which an - inactive connection must be dropped. Inactivity is - defined by no data sent or received by the client. --disable-http-proxy Default: False. Whether to disable proxy.HttpProxyPlugin. --disable-headers DISABLE_HEADERS diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 89bbd31845..103cac65a4 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -28,3 +28,6 @@ websocket writables www youtube +Kubernetes +adhoc +balancer diff --git a/proxy/__init__.py b/proxy/__init__.py index 08da6e2aa0..421d39e601 100755 --- a/proxy/__init__.py +++ b/proxy/__init__.py @@ -8,11 +8,14 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from .proxy import Proxy, main, sleep_loop, entry_point +from .proxy import Proxy, main, grout, sleep_loop, entry_point from .testing import TestCase __all__ = [ + # Grout entry point. See + # https://jaxl.io/ + 'grout', # PyPi package entry_point. See # https://github.com/abhinavsingh/proxy.py#from-command-line-when-installed-using-pip 'entry_point', diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 9c938c66b9..44a3e15bbe 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for more details. """ import os +import ssl import sys import time import pathlib @@ -157,6 +158,9 @@ def _env_threadless_compliant() -> bool: DEFAULT_SELECTOR_SELECT_TIMEOUT = 25 / 1000 DEFAULT_WAIT_FOR_TASKS_TIMEOUT = 1 / 1000 DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT = 1 # in seconds +DEFAULT_SSL_CONTEXT_OPTIONS = ( + ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 +) DEFAULT_DEVTOOLS_DOC_URL = 'http://proxy' DEFAULT_DEVTOOLS_FRAME_ID = secrets.token_hex(8) diff --git a/proxy/common/utils.py b/proxy/common/utils.py index dc8886a420..bb0e8c6db5 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -29,7 +29,7 @@ from .types import HostPort from .constants import ( CRLF, COLON, HTTP_1_1, IS_WINDOWS, WHITESPACE, DEFAULT_TIMEOUT, - DEFAULT_THREADLESS, PROXY_AGENT_HEADER_VALUE, + DEFAULT_THREADLESS, PROXY_AGENT_HEADER_VALUE, DEFAULT_SSL_CONTEXT_OPTIONS, ) @@ -234,20 +234,11 @@ def wrap_socket( cafile: Optional[str] = None, ) -> ssl.SSLSocket: """Use this to upgrade server_side socket to TLS.""" - ctx = ssl.create_default_context( - ssl.Purpose.CLIENT_AUTH, - cafile=cafile, - ) - ctx.options |= ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 + ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=cafile) + ctx.options |= DEFAULT_SSL_CONTEXT_OPTIONS ctx.verify_mode = ssl.CERT_NONE - ctx.load_cert_chain( - certfile=certfile, - keyfile=keyfile, - ) - return ctx.wrap_socket( - conn, - server_side=True, - ) + ctx.load_cert_chain(certfile=certfile, keyfile=keyfile) + return ctx.wrap_socket(conn, server_side=True) def new_socket_connection( diff --git a/proxy/core/connection/client.py b/proxy/core/connection/client.py index f241c56a06..4c8ad97cdd 100644 --- a/proxy/core/connection/client.py +++ b/proxy/core/connection/client.py @@ -9,11 +9,12 @@ :license: BSD, see LICENSE for more details. """ import ssl -from typing import Optional +from typing import Any, Dict, Optional from .types import tcpConnectionTypes from .connection import TcpConnection, TcpConnectionUninitializedException from ...common.types import HostPort, TcpOrTlsSocket +from ...common.constants import DEFAULT_SSL_CONTEXT_OPTIONS class TcpClientConnection(TcpConnection): @@ -42,11 +43,19 @@ def connection(self) -> TcpOrTlsSocket: def wrap(self, keyfile: str, certfile: str) -> None: self.connection.setblocking(True) self.flush() - self._conn = ssl.wrap_socket( - self.connection, - server_side=True, - certfile=certfile, - keyfile=keyfile, - ssl_version=ssl.PROTOCOL_TLS, + ctx = ssl.SSLContext( + protocol=( + ssl.PROTOCOL_TLS_CLIENT + if self.tag == 'server' + else ssl.PROTOCOL_TLS_SERVER + ), ) + ctx.options |= DEFAULT_SSL_CONTEXT_OPTIONS + ctx.load_cert_chain(certfile=certfile, keyfile=keyfile) + assert self.addr + kwargs: Dict[str, Any] = {'server_side': True} + if self.tag == 'server': + assert self.addr + kwargs['server_hostname'] = self.addr[0] + self._conn = ctx.wrap_socket(self.connection, **kwargs) self.connection.setblocking(False) diff --git a/proxy/core/work/threadless.py b/proxy/core/work/threadless.py index e638940fe4..e8f7339def 100644 --- a/proxy/core/work/threadless.py +++ b/proxy/core/work/threadless.py @@ -370,6 +370,8 @@ async def _run_once(self) -> bool: work_id = task._work_id # type: ignore try: teardown = task.result() + except Exception: + teardown = True finally: if teardown: self._cleanup(work_id) diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 0bdcfa22c0..5120d5b32c 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -292,9 +292,7 @@ def _parse_first_request(self, data: memoryview) -> bool: return True # Discover which HTTP handler plugin is capable of # handling the current incoming request - klass = self._discover_plugin_klass( - self.request.http_handler_protocol, - ) + klass = self._discover_plugin_klass(self.request.http_handler_protocol) if klass is None: # No matching protocol class found. # Return bad request response and diff --git a/proxy/http/server/reverse.py b/proxy/http/server/reverse.py index 4d91bf3a0a..303b627f20 100644 --- a/proxy/http/server/reverse.py +++ b/proxy/http/server/reverse.py @@ -84,6 +84,7 @@ def handle_request(self, request: HttpParser) -> None: self.choice = Url.from_bytes( random.choice(route[1]), ) + needs_upstream = True break # Dynamic routes elif isinstance(route, str): diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py index f756494380..f3899e890c 100644 --- a/proxy/http/server/web.py +++ b/proxy/http/server/web.py @@ -217,7 +217,7 @@ def on_client_data(self, raw: memoryview) -> None: self.pipeline_request = None def on_response_chunk(self, chunk: List[memoryview]) -> List[memoryview]: - self._response_size += sum([len(c) for c in chunk]) + self._response_size += sum(len(c) for c in chunk) return chunk def on_client_connection_close(self) -> None: diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py index c3ad91945b..74c7e8d4ef 100644 --- a/proxy/plugin/__init__.py +++ b/proxy/plugin/__init__.py @@ -32,6 +32,7 @@ from .filter_by_client_ip import FilterByClientIpPlugin from .filter_by_url_regex import FilterByURLRegexPlugin from .modify_chunk_response import ModifyChunkResponsePlugin +from .modify_request_header import ModifyRequestHeaderPlugin from .redirect_to_custom_server import RedirectToCustomServerPlugin @@ -53,4 +54,5 @@ 'CustomDnsResolverPlugin', 'CloudflareDnsResolverPlugin', 'ProgramNamePlugin', + 'ModifyRequestHeaderPlugin', ] diff --git a/proxy/plugin/modify_request_header.py b/proxy/plugin/modify_request_header.py new file mode 100644 index 0000000000..72735c2781 --- /dev/null +++ b/proxy/plugin/modify_request_header.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +from typing import Optional + +from ..http.proxy import HttpProxyBasePlugin +from ..http.parser import HttpParser +from ..common.utils import bytes_ +from ..common.version import __version__ + + +class ModifyRequestHeaderPlugin(HttpProxyBasePlugin): + """Modify request header before sending to upstream server.""" + + # def before_upstream_connection(self, request: HttpParser) -> Optional[HttpParser]: + # """NOTE: Use this for HTTP only request headers modification.""" + # request.add_header( + # b"x-proxy-py-version", + # bytes_(__version__), + # ) + # return request + + def handle_client_request(self, request: HttpParser) -> Optional[HttpParser]: + """NOTE: This is for HTTPS request headers modification when under TLS interception. + + For HTTPS requests, modification of request under TLS interception WILL NOT WORK + through before_upstream_connection. + """ + request.add_header( + b'x-proxy-py-version', + bytes_(__version__), + ) + return request diff --git a/proxy/proxy.py b/proxy/proxy.py index 233893f15f..faa6cf3412 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -10,26 +10,35 @@ """ import os import sys +import gzip +import json import time import pprint import signal +import socket +import getpass import logging import argparse import threading -from typing import TYPE_CHECKING, Any, List, Type, Optional, cast +from typing import TYPE_CHECKING, Any, Dict, List, Type, Tuple, Optional, cast from .core.ssh import SshTunnelListener, SshHttpProtocolHandler from .core.work import ThreadlessPool from .core.event import EventManager +from .http.codes import httpStatusCodes from .common.flag import FlagParser, flags +from .http.client import client from .common.utils import bytes_ from .core.work.fd import RemoteFdExecutor +from .http.methods import httpMethods from .core.acceptor import AcceptorPool from .core.listener import ListenerPool from .core.ssh.base import BaseSshTunnelListener +from .common.plugins import Plugins +from .common.version import __version__ from .common.constants import ( - IS_WINDOWS, DEFAULT_PLUGINS, DEFAULT_VERSION, DEFAULT_LOG_FILE, - DEFAULT_PID_FILE, DEFAULT_LOG_LEVEL, DEFAULT_BASIC_AUTH, + IS_WINDOWS, HTTPS_PROTO, DEFAULT_PLUGINS, DEFAULT_VERSION, + DEFAULT_LOG_FILE, DEFAULT_PID_FILE, DEFAULT_LOG_LEVEL, DEFAULT_BASIC_AUTH, DEFAULT_LOG_FORMAT, DEFAULT_WORK_KLASS, DEFAULT_OPEN_FILE_LIMIT, DEFAULT_ENABLE_DASHBOARD, DEFAULT_ENABLE_SSH_TUNNEL, DEFAULT_SSH_LISTENER_KLASS, @@ -384,3 +393,112 @@ def main(**opts: Any) -> None: def entry_point() -> None: main() + + +def grout() -> None: # noqa: C901 + default_grout_tld = os.environ.get('JAXL_DEFAULT_GROUT_TLD', 'jaxl.io') + + def _clear_line() -> None: + print('\r' + ' ' * 60, end='', flush=True) + + def _env(scheme: bytes, host: bytes, port: int) -> Optional[Dict[str, Any]]: + try: + response = client( + scheme=scheme, + host=host, + port=port, + path=b'/env/', + method=httpMethods.BIND, + body='v={0}&u={1}&h={2}'.format( + __version__, + os.environ.get('USER', getpass.getuser()), + socket.gethostname(), + ).encode(), + ) + except socket.gaierror: + _clear_line() + print( + '\r\033[91mUnable to resolve\033[0m', + end='', + flush=True, + ) + return None + if response: + if ( + response.code is not None + and int(response.code) == httpStatusCodes.OK + and response.body is not None + ): + return cast( + Dict[str, Any], + json.loads( + ( + gzip.decompress(response.body).decode() + if response.has_header(b'content-encoding') + and response.header(b'content-encoding') == b'gzip' + else response.body.decode() + ), + ), + ) + if response.code is None: + _clear_line() + print('\r\033[91mUnable to fetch\033[0m', end='', flush=True) + else: + _clear_line() + print( + '\r\033[91mError code {0}\033[0m'.format( + response.code.decode(), + ), + end='', + flush=True, + ) + else: + _clear_line() + print( + '\r\033[91mUnable to connect\033[0m', + end='', + flush=True, + ) + return None + + def _parse() -> Tuple[str, int]: + """Here we deduce registry host/port based upon input parameters.""" + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('route', nargs='?', default=None) + parser.add_argument('name', nargs='?', default=None) + parser.add_argument('--wildcard', action='store_true', help='Enable wildcard') + args, _remaining_args = parser.parse_known_args() + grout_tld = default_grout_tld + if args.name is not None and '.' in args.name: + grout_tld = args.name if args.wildcard else args.name.split('.', maxsplit=1)[1] + grout_tld_parts = grout_tld.split(':') + tld_host = grout_tld_parts[0] + tld_port = 443 + if len(grout_tld_parts) > 1: + tld_port = int(grout_tld_parts[1]) + return tld_host, tld_port + + tld_host, tld_port = _parse() + env = None + attempts = 0 + try: + while True: + env = _env(scheme=HTTPS_PROTO, host=tld_host.encode(), port=int(tld_port)) + attempts += 1 + if env is not None: + print('\rStarting ...' + ' ' * 30 + '\r', end='', flush=True) + break + time.sleep(1) + _clear_line() + print( + '\rWaiting for connection {0}'.format('.' * (attempts % 4)), + end='', + flush=True, + ) + time.sleep(1) + except KeyboardInterrupt: + sys.exit(1) + + assert env is not None + print('\r' + ' ' * 70 + '\r', end='', flush=True) + Plugins.from_bytes(env['m'].encode(), name='client').grout(env=env['e']) # type: ignore[attr-defined] diff --git a/requirements-tunnel.txt b/requirements-tunnel.txt index a002227042..c80a8cde89 100644 --- a/requirements-tunnel.txt +++ b/requirements-tunnel.txt @@ -3,3 +3,4 @@ paramiko==3.4.0; python_version >= '3.11' types-paramiko==2.11.3; python_version < '3.11' types-paramiko==3.4.0.20240311; python_version >= '3.11' cryptography==36.0.2; python_version <= '3.6' +cryptography==39.0.1; python_version > '3.6' diff --git a/setup.cfg b/setup.cfg index fb14a6515e..5dbde4c3f6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -66,6 +66,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Topic :: Internet Topic :: Internet :: Proxy Servers @@ -111,6 +112,7 @@ install_requires = [options.entry_points] console_scripts = proxy = proxy:entry_point + grout = proxy:grout [options.package_data] proxy = diff --git a/tests/http/proxy/test_http_proxy_tls_interception.py b/tests/http/proxy/test_http_proxy_tls_interception.py index 3b412d190d..96795f4a8d 100644 --- a/tests/http/proxy/test_http_proxy_tls_interception.py +++ b/tests/http/proxy/test_http_proxy_tls_interception.py @@ -67,9 +67,9 @@ def mock_cert(a: Any) -> Any: self.mock_ssl_context.return_value.wrap_socket.return_value = upstream_tls_sock # Used for client wrapping - self.mock_ssl_wrap = mocker.patch('ssl.wrap_socket') + self.mock_ssl_wrap = mocker.patch('ssl.SSLContext') client_tls_sock = mock.MagicMock(spec=ssl.SSLSocket) - self.mock_ssl_wrap.return_value = client_tls_sock + self.mock_ssl_wrap.return_value.wrap_socket.return_value = client_tls_sock plain_connection = mock.MagicMock(spec=socket.socket) @@ -256,13 +256,18 @@ async def asyncReturn(val: T) -> T: ) assert self.flags.ca_cert_dir is not None self.mock_ssl_wrap.assert_called_with( - self._conn, - server_side=True, + protocol=ssl.PROTOCOL_TLS_SERVER, + ) + self.mock_ssl_wrap.return_value.load_cert_chain( keyfile=self.flags.ca_signing_key_file, certfile=HttpProxyPlugin.generated_cert_file_path( - self.flags.ca_cert_dir, host, + self.flags.ca_cert_dir, + host, ), - ssl_version=ssl.PROTOCOL_TLS, + ) + self.mock_ssl_wrap.return_value.wrap_socket.assert_called_with( + self._conn, + server_side=True, ) self.assertEqual(self._conn.setblocking.call_count, 2) self.assertEqual( diff --git a/tests/http/web/test_web_server.py b/tests/http/web/test_web_server.py index 71baac4fd6..8100d995bd 100644 --- a/tests/http/web/test_web_server.py +++ b/tests/http/web/test_web_server.py @@ -59,19 +59,6 @@ def test_on_client_connection_called_on_teardown(mocker: MockerFixture) -> None: assert _conn.closed -def mock_selector_for_client_read(self: Any) -> None: - self.mock_selector.return_value.select.return_value = [ - ( - selectors.SelectorKey( - fileobj=self._conn.fileno(), - fd=self._conn.fileno(), - events=selectors.EVENT_READ, - data=None, - ), - selectors.EVENT_READ, - ), - ] - # @mock.patch('socket.fromfd') # def test_on_client_connection_called_on_teardown( # self, mock_fromfd: mock.Mock, @@ -171,16 +158,40 @@ def _setUp(self, request: Any, mocker: MockerFixture) -> None: b'GET / HTTP/1.1', CRLF, ]) - mock_selector_for_client_read(self) + self.mock_selector.return_value.select.side_effect = [ + [ + ( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_READ, + data=None, + ), + selectors.EVENT_READ, + ), + ], + [ + ( + selectors.SelectorKey( + fileobj=self._conn.fileno(), + fd=self._conn.fileno(), + events=selectors.EVENT_WRITE, + data=None, + ), + selectors.EVENT_WRITE, + ), + ], + ] @pytest.mark.asyncio # type: ignore[misc] async def test_pac_file_served_from_disk(self) -> None: + await self.protocol_handler._run_once() await self.protocol_handler._run_once() self.assertEqual( self.protocol_handler.request.state, httpParserStates.COMPLETE, ) - self._conn.send.called_once_with( + self._conn.send.assert_called_once_with( build_http_response( 200, reason=b'OK', diff --git a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py index 2d6d0d60af..6383817abc 100644 --- a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py +++ b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py @@ -47,7 +47,7 @@ def _setUp(self, request: Any, mocker: MockerFixture) -> None: 'proxy.http.proxy.server.TcpServerConnection', ) self.mock_ssl_context = mocker.patch('ssl.create_default_context') - self.mock_ssl_wrap = mocker.patch('ssl.wrap_socket') + self.mock_ssl_wrap = mocker.patch('ssl.SSLContext') self.mock_sign_csr.return_value = True self.mock_gen_csr.return_value = True @@ -86,7 +86,7 @@ def mock_cert(a: Any) -> Any: self.server_ssl_connection.getpeercert = mock_cert self.mock_ssl_context.return_value.wrap_socket.return_value = self.server_ssl_connection self.client_ssl_connection = mocker.MagicMock(spec=ssl.SSLSocket) - self.mock_ssl_wrap.return_value = self.client_ssl_connection + self.mock_ssl_wrap.return_value.wrap_socket.return_value = self.client_ssl_connection def has_buffer() -> bool: return cast(bool, self.server.queue.called) diff --git a/tox.ini b/tox.ini index 5e7d706a4b..37f686ca45 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36,py37,py38,py39,py310,py311 +envlist = py36,py37,py38,py39,py310,py311,py312 isolated_build = true minversion = 3.21.0