diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 4f49831221..19d214b48f 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -135,7 +135,7 @@ jobs: - name: Set up pip cache if: >- steps.request-check.outputs.release-requested != 'true' - uses: actions/cache@v3.0.4 + uses: actions/cache@v3.0.8 with: path: ${{ steps.pip-cache-dir.outputs.dir }} key: >- @@ -244,7 +244,7 @@ jobs: run: >- echo "::set-output name=dir::$(pip cache dir)" - name: Set up pip cache - uses: actions/cache@v3.0.4 + uses: actions/cache@v3.0.8 with: path: ${{ steps.pip-cache.outputs.dir }} key: >- @@ -369,7 +369,7 @@ jobs: run: >- echo "::set-output name=dir::$(pip cache dir)" - name: Set up pip cache - uses: actions/cache@v3.0.4 + uses: actions/cache@v3.0.8 with: path: ${{ steps.pip-cache.outputs.dir }} key: >- @@ -486,7 +486,7 @@ jobs: run: >- echo "::set-output name=dir::$(pip cache dir)" - name: Set up pip cache - uses: actions/cache@v3.0.4 + uses: actions/cache@v3.0.8 with: path: ${{ steps.pip-cache.outputs.dir }} key: >- @@ -658,27 +658,27 @@ jobs: - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 - brew: - runs-on: ${{ matrix.os }}-latest - name: 🍺 🐍${{ matrix.python }} @ ${{ matrix.os }} - strategy: - matrix: - os: [macOS] - python: ['3.10'] - # max-parallel: 1 - fail-fast: false - steps: - - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - name: Brew - run: | - brew install ./helper/homebrew/develop/proxy.rb - - name: Verify - run: | - proxy -h + # brew: + # runs-on: ${{ matrix.os }}-latest + # name: 🍺 🐍${{ matrix.python }} @ ${{ matrix.os }} + # strategy: + # matrix: + # os: [macOS] + # python: ['3.10'] + # # max-parallel: 1 + # fail-fast: false + # steps: + # - uses: actions/checkout@v3 + # - name: Setup Python + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ matrix.python }} + # - name: Brew + # run: | + # brew install ./helper/homebrew/develop/proxy.rb + # - name: Verify + # run: | + # proxy -h dashboard: runs-on: ${{ matrix.os }}-latest @@ -969,7 +969,7 @@ jobs: - test - lint - dashboard - - brew + # - brew - developer - ghcr-latest - ghcr-openssl diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d62799511..9c5050d667 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -158,6 +158,7 @@ repos: additional_dependencies: - paramiko == 2.11.0 - types-paramiko == 2.7.3 + - types-requests==2.27.30 - cryptography==36.0.2; python_version <= '3.6' - types-setuptools == 57.4.2 args: diff --git a/README.md b/README.md index a0a2f9b49b..9b43bc83eb 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ - [Redirect To Custom Server Plugin](#redirecttocustomserverplugin) - [Filter By Upstream Host Plugin](#filterbyupstreamhostplugin) - [Cache Responses Plugin](#cacheresponsesplugin) + - [Cache By Response Type](#cachebyresponsetype) - [Man-In-The-Middle Plugin](#maninthemiddleplugin) - [Proxy Pool Plugin](#proxypoolplugin) - [Filter By Client IP Plugin](#filterbyclientipplugin) @@ -116,6 +117,8 @@ - [Plugin Developer and Contributor Guide](#plugin-developer-and-contributor-guide) - [High level architecture](#high-level-architecture) - [Everything is a plugin](#everything-is-a-plugin) + - [Managing states for your stateless plugins](#managing-states-for-your-stateless-plugins) + - [Passing processing context between plugins](#passing-processing-context-between-plugins) - [Internal Documentation](#internal-documentation) - [Read The Doc](#read-the-doc) - [pydoc](#pydoc) @@ -778,6 +781,28 @@ Connection: keep-alive } ``` +### CacheByResponseType + +`CacheResponsesPlugin` plugin can also automatically cache responses by `content-type`. +To try this, you must be running under [TLS Interception](#tls-interception) mode +and then pass `--cache-by-content-type` flag. Example: + +```console +❯ proxy \ + --plugins proxy.plugin.CacheResponsesPlugin \ + --cache-by-content-type \ + --ca-key-file ca-key.pem \ + --ca-cert-file ca-cert.pem \ + --ca-signing-key ca-signing-key.pem +``` + +Make a few requests to the proxy server and you shall see data under `~/.proxy/cache` directory. + +You should see 2 folders: + +- `content`: Contains parsed `jpg`, `css`, `js`, `html`, `pdf` etc by content type +- `responses`: Contains raw responses as received _(of-course decrypted because of interception)_ + ### ManInTheMiddlePlugin Modifies upstream server responses. @@ -1694,26 +1719,28 @@ Use `proxy.common.pki` module for: 3. Signing CSR requests using custom CA. ```console -python -m proxy.common.pki -h -usage: pki.py [-h] [--password PASSWORD] [--private-key-path PRIVATE_KEY_PATH] - [--public-key-path PUBLIC_KEY_PATH] [--subject SUBJECT] +❯ python -m proxy.common.pki -h +usage: pki.py [-h] [--password PASSWORD] [--private-key-path PRIVATE_KEY_PATH] [--public-key-path PUBLIC_KEY_PATH] + [--subject SUBJECT] [--csr-path CSR_PATH] [--crt-path CRT_PATH] [--hostname HOSTNAME] [--openssl OPENSSL] action -proxy.py v2.2.0 : PKI Utility +proxy.py v2.4.4rc2.dev12+gdc06ea4 : PKI Utility positional arguments: - action Valid actions: remove_passphrase, gen_private_key, - gen_public_key, gen_csr, sign_csr + action Valid actions: remove_passphrase, gen_private_key, gen_public_key, gen_csr, sign_csr -optional arguments: +options: -h, --help show this help message and exit --password PASSWORD Password to use for encryption. Default: proxy.py --private-key-path PRIVATE_KEY_PATH Private key path --public-key-path PUBLIC_KEY_PATH Public key path - --subject SUBJECT Subject to use for public key generation. Default: - /CN=example.com + --subject SUBJECT Subject to use for public key generation. Default: /CN=localhost + --csr-path CSR_PATH CSR file path. Use with gen_csr and sign_csr action. + --crt-path CRT_PATH Signed certificate path. Use with sign_csr action. + --hostname HOSTNAME Alternative subject names to use during CSR signing. + --openssl OPENSSL Path to openssl binary. By default, we assume openssl is in your PATH ``` ## Internal Documentation @@ -2218,6 +2245,28 @@ Within `proxy.py` everything is a plugin. Use this flag with `--enable-web-server` flag to run `proxy.py` as a programmable http(s) server. +## Managing states for your stateless plugins + +Plugin class instances are created per-request. Most importantly, +plugin instances are created within CPU core context where the request +was received. + +For above reason, global variables in your plugins may work as expected. +Your plugin code by design must be **stateless**. + +To manage global states, you have a couple of options: +1) Make use of Python's [multiprocessing safe data structures](https://python.readthedocs.io/en/latest/library/multiprocessing.html#sharing-state-between-processes) +2) Make use of `proxy.py` in-built [eventing mechanism](https://github.com/abhinavsingh/proxy.py/blob/develop/tutorial/eventing.ipynb) + +## Passing processing context between plugins + +Sometimes, a plugin may need to pass additional context to other plugins after them in the processing chain. Example, this additional +context can also be dumped as part of access logs. + +To pass processing context, make use of plugin's `on_access_log` method. See how [Program Name](https://github.com/abhinavsingh/proxy.py/blob/develop/proxy/plugin/program_name.py) plugin modifies default `client_ip` key in the context and updates it to detected program name. + +As a result, when we enable [Program Name Plugin](#programnameplugin), we see local client program name instead of IP address in the access logs. + ## Development Guide ### Setup Local Environment diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index 06c866ac7f..9881a790d9 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -14,15 +14,15 @@ "@types/js-cookie": "^3.0.2", "@typescript-eslint/eslint-plugin": "^2.34.0", "@typescript-eslint/parser": "^2.34.0", - "chrome-devtools-frontend": "^1.0.980332", + "chrome-devtools-frontend": "^1.0.1029149", "eslint": "^6.8.0", "eslint-config-standard": "^14.1.1", "eslint-plugin-import": "^2.26.0", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^5.0.0", - "http-server": "^14.1.0", - "jasmine": "^4.1.0", + "http-server": "^14.1.1", + "jasmine": "^4.3.0", "jasmine-ts": "^0.4.0", "jquery": "^3.6.0", "js-cookie": "^3.0.1", @@ -32,9 +32,9 @@ "rollup-plugin-copy": "^3.4.0", "rollup-plugin-javascript-obfuscator": "^1.0.4", "rollup-plugin-typescript": "^1.0.1", - "ts-node": "^10.8.0", - "typescript": "^4.5.4", - "ws": "^8.6.0" + "ts-node": "^10.9.1", + "typescript": "^4.7.3", + "ws": "^8.8.0" } }, "node_modules/@babel/code-frame": { @@ -777,9 +777,9 @@ } }, "node_modules/chrome-devtools-frontend": { - "version": "1.0.980332", - "resolved": "https://registry.npmjs.org/chrome-devtools-frontend/-/chrome-devtools-frontend-1.0.980332.tgz", - "integrity": "sha512-+WJU9z+jVvSglXbmKInzxl4SIlWmh7tJm1D66GH04NY6IQP/Ev+6TfLLSrPVXjXwEvKhXOEW50y/mQZt0w3WRQ==", + "version": "1.0.1029149", + "resolved": "https://registry.npmjs.org/chrome-devtools-frontend/-/chrome-devtools-frontend-1.0.1029149.tgz", + "integrity": "sha512-h6VdOoWBZMizUdgnOTw0+RTzQS9KjhZwq3Auq2bM2BM2WdrhUdfuKob6P9UOC9o8xBrs5wO2SEvqevLgqnLttQ==", "dev": true }, "node_modules/cli-cursor": { @@ -2452,9 +2452,9 @@ } }, "node_modules/http-server": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/http-server/-/http-server-14.1.0.tgz", - "integrity": "sha512-5lYsIcZtf6pdR8tCtzAHTWrAveo4liUlJdWc7YafwK/maPgYHs+VNP6KpCClmUnSorJrARVMXqtT055zBv11Yg==", + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/http-server/-/http-server-14.1.1.tgz", + "integrity": "sha512-+cbxadF40UXd9T01zUHgA+rlo2Bg1Srer4+B4NwIHdaGxAGGv59nYRnGGDJ9LBk7alpS0US+J+bLLdQOOkJq4A==", "dev": true, "dependencies": { "basic-auth": "^2.0.1", @@ -2464,7 +2464,7 @@ "html-encoding-sniffer": "^3.0.0", "http-proxy": "^1.18.1", "mime": "^1.6.0", - "minimist": "^1.2.5", + "minimist": "^1.2.6", "opener": "^1.5.1", "portfinder": "^1.0.28", "secure-compare": "3.0.1", @@ -2967,22 +2967,22 @@ "dev": true }, "node_modules/jasmine": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-4.1.0.tgz", - "integrity": "sha512-4VhjbUgwfNS9CBnUMoSWr9tdNgOoOhNIjAD8YRxTn+PmOf4qTSC0Uqhk66dWGnz2vJxtNIU0uBjiwnsp4Ud9VA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-4.3.0.tgz", + "integrity": "sha512-ieBmwkd8L1DXnvSnxx7tecXgA0JDgMXPAwBcqM4lLPedJeI9hTHuWifPynTC+dLe4Y+GkSPSlbqqrmYIgGzYUw==", "dev": true, "dependencies": { "glob": "^7.1.6", - "jasmine-core": "^4.1.0" + "jasmine-core": "^4.3.0" }, "bin": { "jasmine": "bin/jasmine.js" } }, "node_modules/jasmine-core": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-4.1.1.tgz", - "integrity": "sha512-lmUfT5XcK9KKvt3lLYzn93hc4MGzlUBowExFVgzbSW0ZCrdeyS574dfsyfRhxbg81Wj4gk+RxUiTnj7KBfDA1g==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-4.3.0.tgz", + "integrity": "sha512-qybtBUesniQdW6n+QIHMng2vDOHscIC/dEXjW+JzO9+LoAZMb03RCUC5xFOv/btSKPm1xL42fn+RjlU4oB42Lg==", "dev": true }, "node_modules/jasmine-ts": { @@ -4805,9 +4805,9 @@ } }, "node_modules/ts-node": { - "version": "10.8.0", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.8.0.tgz", - "integrity": "sha512-/fNd5Qh+zTt8Vt1KbYZjRHCE9sI5i7nqfD/dzBBRDeVXZXS6kToW6R7tTU6Nd4XavFs0mAVCg29Q//ML7WsZYA==", + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", + "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", "dev": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", @@ -4941,9 +4941,9 @@ } }, "node_modules/typescript": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.4.tgz", - "integrity": "sha512-VgYs2A2QIRuGphtzFV7aQJduJ2gyfTljngLzjpfW9FoYZF6xuw1W0vW9ghCKLfcWrCFxK81CSGRAvS1pn4fIUg==", + "version": "4.7.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.3.tgz", + "integrity": "sha512-WOkT3XYvrpXx4vMMqlD+8R8R37fZkjyLGlxavMc4iB8lrl8L0DeTcHbYgw/v0N/z9wAFsgBhcsF0ruoySS22mA==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -5239,9 +5239,9 @@ } }, "node_modules/ws": { - "version": "8.6.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.6.0.tgz", - "integrity": "sha512-AzmM3aH3gk0aX7/rZLYvjdvZooofDu3fFOzGqcSnQ1tOcTWwhM/o+q++E8mAyVVIyUdajrkzWUGftaVSDLn1bw==", + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.8.0.tgz", + "integrity": "sha512-JDAgSYQ1ksuwqfChJusw1LSJ8BizJ2e/vVu5Lxjq3YvNJNlROv1ui4i+c/kUUrPheBvQl4c5UbERhTwKa6QBJQ==", "dev": true, "engines": { "node": ">=10.0.0" @@ -5954,9 +5954,9 @@ "dev": true }, "chrome-devtools-frontend": { - "version": "1.0.980332", - "resolved": "https://registry.npmjs.org/chrome-devtools-frontend/-/chrome-devtools-frontend-1.0.980332.tgz", - "integrity": "sha512-+WJU9z+jVvSglXbmKInzxl4SIlWmh7tJm1D66GH04NY6IQP/Ev+6TfLLSrPVXjXwEvKhXOEW50y/mQZt0w3WRQ==", + "version": "1.0.1029149", + "resolved": "https://registry.npmjs.org/chrome-devtools-frontend/-/chrome-devtools-frontend-1.0.1029149.tgz", + "integrity": "sha512-h6VdOoWBZMizUdgnOTw0+RTzQS9KjhZwq3Auq2bM2BM2WdrhUdfuKob6P9UOC9o8xBrs5wO2SEvqevLgqnLttQ==", "dev": true }, "cli-cursor": { @@ -7234,9 +7234,9 @@ } }, "http-server": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/http-server/-/http-server-14.1.0.tgz", - "integrity": "sha512-5lYsIcZtf6pdR8tCtzAHTWrAveo4liUlJdWc7YafwK/maPgYHs+VNP6KpCClmUnSorJrARVMXqtT055zBv11Yg==", + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/http-server/-/http-server-14.1.1.tgz", + "integrity": "sha512-+cbxadF40UXd9T01zUHgA+rlo2Bg1Srer4+B4NwIHdaGxAGGv59nYRnGGDJ9LBk7alpS0US+J+bLLdQOOkJq4A==", "dev": true, "requires": { "basic-auth": "^2.0.1", @@ -7246,7 +7246,7 @@ "html-encoding-sniffer": "^3.0.0", "http-proxy": "^1.18.1", "mime": "^1.6.0", - "minimist": "^1.2.5", + "minimist": "^1.2.6", "opener": "^1.5.1", "portfinder": "^1.0.28", "secure-compare": "3.0.1", @@ -7614,19 +7614,19 @@ "dev": true }, "jasmine": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-4.1.0.tgz", - "integrity": "sha512-4VhjbUgwfNS9CBnUMoSWr9tdNgOoOhNIjAD8YRxTn+PmOf4qTSC0Uqhk66dWGnz2vJxtNIU0uBjiwnsp4Ud9VA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-4.3.0.tgz", + "integrity": "sha512-ieBmwkd8L1DXnvSnxx7tecXgA0JDgMXPAwBcqM4lLPedJeI9hTHuWifPynTC+dLe4Y+GkSPSlbqqrmYIgGzYUw==", "dev": true, "requires": { "glob": "^7.1.6", - "jasmine-core": "^4.1.0" + "jasmine-core": "^4.3.0" } }, "jasmine-core": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-4.1.1.tgz", - "integrity": "sha512-lmUfT5XcK9KKvt3lLYzn93hc4MGzlUBowExFVgzbSW0ZCrdeyS574dfsyfRhxbg81Wj4gk+RxUiTnj7KBfDA1g==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-4.3.0.tgz", + "integrity": "sha512-qybtBUesniQdW6n+QIHMng2vDOHscIC/dEXjW+JzO9+LoAZMb03RCUC5xFOv/btSKPm1xL42fn+RjlU4oB42Lg==", "dev": true }, "jasmine-ts": { @@ -9023,9 +9023,9 @@ } }, "ts-node": { - "version": "10.8.0", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.8.0.tgz", - "integrity": "sha512-/fNd5Qh+zTt8Vt1KbYZjRHCE9sI5i7nqfD/dzBBRDeVXZXS6kToW6R7tTU6Nd4XavFs0mAVCg29Q//ML7WsZYA==", + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", + "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", "dev": true, "requires": { "@cspotcode/source-map-support": "^0.8.0", @@ -9115,9 +9115,9 @@ "dev": true }, "typescript": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.4.tgz", - "integrity": "sha512-VgYs2A2QIRuGphtzFV7aQJduJ2gyfTljngLzjpfW9FoYZF6xuw1W0vW9ghCKLfcWrCFxK81CSGRAvS1pn4fIUg==", + "version": "4.7.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.3.tgz", + "integrity": "sha512-WOkT3XYvrpXx4vMMqlD+8R8R37fZkjyLGlxavMc4iB8lrl8L0DeTcHbYgw/v0N/z9wAFsgBhcsF0ruoySS22mA==", "dev": true }, "unbox-primitive": { @@ -9356,9 +9356,9 @@ } }, "ws": { - "version": "8.6.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.6.0.tgz", - "integrity": "sha512-AzmM3aH3gk0aX7/rZLYvjdvZooofDu3fFOzGqcSnQ1tOcTWwhM/o+q++E8mAyVVIyUdajrkzWUGftaVSDLn1bw==", + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.8.0.tgz", + "integrity": "sha512-JDAgSYQ1ksuwqfChJusw1LSJ8BizJ2e/vVu5Lxjq3YvNJNlROv1ui4i+c/kUUrPheBvQl4c5UbERhTwKa6QBJQ==", "dev": true, "requires": {} }, diff --git a/dashboard/package.json b/dashboard/package.json index d3b3c440a3..64d937f00a 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -30,15 +30,15 @@ "@types/js-cookie": "^3.0.2", "@typescript-eslint/eslint-plugin": "^2.34.0", "@typescript-eslint/parser": "^2.34.0", - "chrome-devtools-frontend": "^1.0.980332", + "chrome-devtools-frontend": "^1.0.1029149", "eslint": "^6.8.0", "eslint-config-standard": "^14.1.1", "eslint-plugin-import": "^2.26.0", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^5.0.0", - "http-server": "^14.1.0", - "jasmine": "^4.1.0", + "http-server": "^14.1.1", + "jasmine": "^4.3.0", "jasmine-ts": "^0.4.0", "jquery": "^3.6.0", "js-cookie": "^3.0.1", @@ -48,8 +48,8 @@ "rollup-plugin-copy": "^3.4.0", "rollup-plugin-javascript-obfuscator": "^1.0.4", "rollup-plugin-typescript": "^1.0.1", - "ts-node": "^10.8.0", - "typescript": "^4.5.4", - "ws": "^8.6.0" + "ts-node": "^10.9.1", + "typescript": "^4.7.3", + "ws": "^8.8.0" } } diff --git a/proxy/common/constants.py b/proxy/common/constants.py index a8b154fa53..bd0a40e785 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -22,7 +22,7 @@ SYS_PLATFORM = platform.system() -IS_WINDOWS = SYS_PLATFORM == 'Windows' +IS_WINDOWS = SYS_PLATFORM.lower() in ('windows', 'cygwin') def _env_threadless_compliant() -> bool: diff --git a/proxy/common/flag.py b/proxy/common/flag.py index 01f196568a..f8395a6f62 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -279,6 +279,13 @@ def initialize( args.ca_file, ), ) + args.openssl = cast( + Optional[str], + opts.get( + 'openssl', + args.openssl, + ), + ) args.hostname = cast( IpAddress, diff --git a/proxy/common/pki.py b/proxy/common/pki.py index f8189c0cb9..bdc2c5f3f8 100644 --- a/proxy/common/pki.py +++ b/proxy/common/pki.py @@ -62,10 +62,11 @@ def remove_passphrase( password: str, key_out_path: str, timeout: int = 10, + openssl: str = 'openssl', ) -> bool: """Remove passphrase from a private key.""" command = [ - 'openssl', 'rsa', + openssl, 'rsa', '-passin', 'pass:%s' % password, '-in', key_in_path, '-out', key_out_path, @@ -78,10 +79,11 @@ def gen_private_key( password: str, bits: int = 2048, timeout: int = 10, + openssl: str = 'openssl', ) -> bool: """Generates a private key.""" command = [ - 'openssl', 'genrsa', '-aes256', + openssl, 'genrsa', '-aes256', '-passout', 'pass:%s' % password, '-out', key_path, str(bits), ] @@ -97,11 +99,12 @@ def gen_public_key( extended_key_usage: Optional[str] = None, validity_in_days: int = 365, timeout: int = 10, + openssl: str = 'openssl', ) -> bool: """For a given private key, generates a corresponding public key.""" with ssl_config(alt_subj_names, extended_key_usage) as (config_path, has_extension): command = [ - 'openssl', 'req', '-new', '-x509', '-sha256', + openssl, 'req', '-new', '-x509', '-sha256', '-days', str(validity_in_days), '-subj', subject, '-passin', 'pass:%s' % private_key_password, '-config', config_path, @@ -120,10 +123,11 @@ def gen_csr( password: str, crt_path: str, timeout: int = 10, + openssl: str = 'openssl', ) -> bool: """Generates a CSR based upon existing certificate and key file.""" command = [ - 'openssl', 'x509', '-x509toreq', + openssl, 'x509', '-x509toreq', '-passin', 'pass:%s' % password, '-in', crt_path, '-signkey', key_path, '-out', csr_path, @@ -142,11 +146,12 @@ def sign_csr( extended_key_usage: Optional[str] = None, validity_in_days: int = 365, timeout: int = 10, + openssl: str = 'openssl', ) -> bool: """Sign a CSR using CA key and certificate.""" with ext_file(alt_subj_names, extended_key_usage) as extension_path: command = [ - 'openssl', 'x509', '-req', '-sha256', + openssl, 'x509', '-req', '-sha256', '-CA', ca_crt_path, '-CAkey', ca_key_path, '-passin', 'pass:%s' % ca_key_password, @@ -288,6 +293,12 @@ def run_openssl_command(command: List[str], timeout: int) -> bool: default=None, help='Alternative subject names to use during CSR signing.', ) + parser.add_argument( + '--openssl', + type=str, + default='openssl', + help='Path to openssl binary. By default, we assume openssl is in your PATH', + ) args = parser.parse_args(sys.argv[1:]) # Validation @@ -310,16 +321,19 @@ def run_openssl_command(command: List[str], timeout: int) -> bool: # Execute if args.action == 'gen_private_key': - gen_private_key(args.private_key_path, args.password) + gen_private_key( + args.private_key_path, + args.password, openssl=args.openssl, + ) elif args.action == 'gen_public_key': gen_public_key( args.public_key_path, args.private_key_path, - args.password, args.subject, + args.password, args.subject, openssl=args.openssl, ) elif args.action == 'remove_passphrase': remove_passphrase( args.private_key_path, args.password, - args.private_key_path, + args.private_key_path, openssl=args.openssl, ) elif args.action == 'gen_csr': gen_csr( @@ -327,9 +341,11 @@ def run_openssl_command(command: List[str], timeout: int) -> bool: args.private_key_path, args.password, args.public_key_path, + openssl=args.openssl, ) elif args.action == 'sign_csr': sign_csr( args.csr_path, args.crt_path, args.private_key_path, args.password, args.public_key_path, str(int(time.time())), alt_subj_names=[args.hostname], + openssl=args.openssl, ) diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py index 0dcb017309..91a0cbbc9e 100644 --- a/proxy/http/parser/parser.py +++ b/proxy/http/parser/parser.py @@ -357,7 +357,7 @@ def _process_body(self, raw: memoryview) -> Tuple[bool, memoryview]: # # See TestHttpParser.test_issue_398 scenario self.state = httpParserStates.RCVING_BODY - self.body = raw + self.body = bytes(raw) return False, memoryview(b'') def _process_headers(self, raw: memoryview) -> Tuple[bool, memoryview]: diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 019eb651e7..4838219030 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -673,6 +673,7 @@ def gen_ca_signed_certificate( public_key_path=public_key_path, private_key_path=private_key_path, private_key_password=private_key_password, subject=subject, alt_subj_names=alt_subj_names, validity_in_days=validity_in_days, timeout=timeout, + openssl=self.flags.openssl, ) assert(resp is True) @@ -687,6 +688,7 @@ def gen_ca_signed_certificate( resp = gen_csr( csr_path=csr_path, key_path=private_key_path, password=private_key_password, crt_path=public_key_path, timeout=timeout, + openssl=self.flags.openssl, ) assert(resp is True) @@ -703,6 +705,7 @@ def gen_ca_signed_certificate( ca_key_password=ca_key_password, ca_crt_path=ca_crt_path, serial=str(serial), alt_subj_names=alt_subj_names, validity_in_days=validity_in_days, timeout=timeout, + openssl=self.flags.openssl, ) assert(resp is True) diff --git a/proxy/http/server/plugin.py b/proxy/http/server/plugin.py index 4e290178f4..544d39ab8f 100644 --- a/proxy/http/server/plugin.py +++ b/proxy/http/server/plugin.py @@ -13,8 +13,8 @@ from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union, Optional -from proxy.http.url import Url from ..parser import HttpParser +from ...http.url import Url from ..responses import NOT_FOUND_RESPONSE_PKT, okResponse from ..websocket import WebsocketFrame from ..connection import HttpClientConnection @@ -128,6 +128,20 @@ def on_access_log(self, context: Dict[str, Any]) -> Optional[Dict[str, Any]]: class ReverseProxyBasePlugin(ABC): """ReverseProxy base plugin class.""" + def __init__( + self, + uid: str, + flags: argparse.Namespace, + client: HttpClientConnection, + event_queue: EventQueue, + upstream_conn_pool: Optional['UpstreamConnectionPool'] = None, + ): + self.uid = uid + self.flags = flags + self.client = client + self.event_queue = event_queue + self.upstream_conn_pool = upstream_conn_pool + @abstractmethod def routes(self) -> List[Union[str, Tuple[str, List[bytes]]]]: """List of routes registered by plugin. @@ -147,6 +161,12 @@ def routes(self) -> List[Union[str, Tuple[str, List[bytes]]]]: must return the url to serve.""" raise NotImplementedError() # pragma: no cover + def before_routing(self, request: HttpParser) -> Optional[HttpParser]: + """Plugins can modify request, return response, close connection. + + If None is returned, request will be dropped and closed.""" + return request # pragma: no cover + def handle_route(self, request: HttpParser, pattern: RePattern) -> Url: """Implement this method if you have configured dynamic routes.""" pass diff --git a/proxy/http/server/reverse.py b/proxy/http/server/reverse.py index 6eab123c90..9037370358 100644 --- a/proxy/http/server/reverse.py +++ b/proxy/http/server/reverse.py @@ -40,10 +40,14 @@ def __init__(self, *args: Any, **kwargs: Any): self.choice: Optional[Url] = None self.plugins: List['ReverseProxyBasePlugin'] = [] for klass in self.flags.plugins[b'ReverseProxyBasePlugin']: - plugin: 'ReverseProxyBasePlugin' = klass() + plugin: 'ReverseProxyBasePlugin' = klass( + self.uid, self.flags, self.client, self.event_queue, self.upstream_conn_pool, + ) self.plugins.append(plugin) def handle_upstream_data(self, raw: memoryview) -> None: + # TODO: Parse response and implement plugin hook per parsed response object + # This will give plugins a chance to modify the responses before dispatching to client self.client.queue(raw) def routes(self) -> List[Tuple[int, str]]: @@ -55,6 +59,14 @@ def routes(self) -> List[Tuple[int, str]]: return r def handle_request(self, request: HttpParser) -> None: + # before_routing + for plugin in self.plugins: + r = plugin.before_routing(request) + if r is None: + raise HttpProtocolException('before_routing closed connection') + request = r + + # routes for plugin in self.plugins: for route in plugin.routes(): if isinstance(route, tuple): @@ -71,6 +83,7 @@ def handle_request(self, request: HttpParser) -> None: break else: raise ValueError('Invalid route') + assert self.choice and self.choice.hostname port = self.choice.port or \ DEFAULT_HTTP_PORT \ diff --git a/proxy/proxy.py b/proxy/proxy.py index b9b602d6de..d9d9f89798 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -137,6 +137,14 @@ help='Default: None. Save "parent" process ID to a file.', ) +flags.add_argument( + '--openssl', + type=str, + default='openssl', + help='Default: openssl. Path to openssl binary. ' + + 'By default, assumption is that openssl is in your PATH.', +) + class Proxy: """Proxy is a context manager to control proxy.py library core. diff --git a/pyproject.toml b/pyproject.toml index 7f9f7173cc..4f368f7bda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ requires = [ "setuptools", # Plugins - "setuptools-scm[toml] >= 6", - "setuptools-scm-git-archive >= 1.1", + "setuptools-scm[toml]>=6,!=7.0.0,!=7.0.1,!=7.0.2", + "setuptools-scm-git-archive>=1.1", ] build-backend = "setuptools.build_meta" diff --git a/requirements-release.txt b/requirements-release.txt index 0abc33fb64..6e7b54213a 100644 --- a/requirements-release.txt +++ b/requirements-release.txt @@ -1,2 +1,2 @@ setuptools-scm == 6.3.2 -twine==3.7.1 +twine==3.8.0 diff --git a/requirements-testing.txt b/requirements-testing.txt index 200567b70c..ae1f3813d0 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -8,10 +8,10 @@ pytest-xdist == 2.5.0 pytest-mock==3.6.1 pytest-asyncio==0.16.0 autopep8==1.6.0 -mypy==0.961 +mypy==0.971 py-spy==0.3.12 codecov==2.1.12 -tox==3.25.0 +tox==3.25.1 mccabe==0.6.1 pylint==2.13.7 rope==1.1.1 @@ -21,4 +21,6 @@ h2==4.1.0 hpack==4.0.0 hyperframe==6.0.1 pre-commit==2.16.0 -types-setuptools==57.4.10 +# Types +types-requests==2.28.8 +types-setuptools==64.0.1 diff --git a/requirements-tunnel.txt b/requirements-tunnel.txt index 9682185834..e28eb4ff04 100644 --- a/requirements-tunnel.txt +++ b/requirements-tunnel.txt @@ -1,3 +1,3 @@ paramiko==2.11.0 -types-paramiko==2.8.9 +types-paramiko==2.11.3 cryptography==36.0.2; python_version <= '3.6' diff --git a/tests/test_main.py b/tests/test_main.py index 1d4c11538f..ad35f1d933 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -14,7 +14,9 @@ import unittest from unittest import mock -from proxy.proxy import main, entry_point +import requests + +from proxy.proxy import Proxy, main, entry_point from proxy.common.utils import bytes_ from proxy.core.work.fd import RemoteFdExecutor from proxy.common.constants import ( # noqa: WPS450 @@ -368,3 +370,23 @@ def test_enable_ssh_tunnel( mock_ssh_tunnel_listener.return_value.shutdown.assert_called_once() # shutdown will internally call stop port forward mock_ssh_tunnel_listener.return_value.stop_port_forward.assert_not_called() + + +class TestProxyContextManager(unittest.TestCase): + + def test_proxy_context_manager(self) -> None: + with Proxy(port=8888, num_acceptors=1): + response = requests.get( + 'http://httpbin.org/get', proxies={ + 'http': 'http://127.0.0.1:8888', + 'https': 'http://127.0.0.1:8888', + }, + ) + self.assertEqual(response.status_code, 200) + response = requests.get( + 'https://httpbin.org/get', proxies={ + 'http': 'http://127.0.0.1:8888', + 'https': 'http://127.0.0.1:8888', + }, + ) + self.assertEqual(response.status_code, 200) diff --git a/tox.ini b/tox.ini index 83bc337213..9f59afc2bb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36,py37,py38,py39,py310 +envlist = py36,py37,py38,py39,py310,py311 isolated_build = true minversion = 3.21.0