Add profiling/benchmarks to CI suite (#3997)

Co-authored-by: Adam Sachs <[email protected]>

.github/workflows/backend_checks.yml

@@ -105,6 +105,44 @@ jobs:
       - name: Run Static Check
         run: nox -s ${{ matrix.session_name }}
 
+  ##################
+  ## Performance ##
+  ##################
+  Performance-Checks:
+    needs: Check-Container-Startup
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Download container
+        uses: actions/download-artifact@v3
+        with:
+          name: python-${{ env.DEFAULT_PYTHON_VERSION }}
+          path: /tmp/
+
+      - name: Load image
+        run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set Up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+          cache: "pip"
+
+      - name: Install Rust/Cargo
+        run: curl -y https://sh.rustup.rs -sSf | sh
+
+      - name: Install Drill
+        run: cargo install drill
+
+      - name: Install Nox
+        run: pip install nox>=2022
+
+      - name: Run Performance Tests
+        run: nox -s performance_tests
+
   #################
   ## Misc Checks ##
   #################

CHANGELOG.md

@@ -17,9 +17,14 @@ The types of changes are:
 
 ## [Unreleased](https://github.com/ethyca/fides/compare/2.19.0...main)
 
+### Added
+
+- Added new Performance-related nox commands and included them as part of the CI suite [#3997](https://github.com/ethyca/fides/pull/3997)
+
 ## [2.19.0](https://github.com/ethyca/fides/compare/2.18.0...2.19.0)
 
 ### Added
+
 - Add dictionary suggestions [#3937](https://github.com/ethyca/fides/pull/3937), [#3988](https://github.com/ethyca/fides/pull/3988)
 - Added new endpoints for healthchecks [#3947](https://github.com/ethyca/fides/pull/3947)
 - Added vendor list dropdown [#3857](https://github.com/ethyca/fides/pull/3857)

docs/fides/docs/development/development_tips.md

@@ -136,3 +136,12 @@ nox -s clean
 
 !!! warning
     If you find yourself feeling the need to run this command regularly, open an issue or slack a member of the dev team as it is not expected that this will need to be run regularly.
+
+## Performance and Benchmarking
+
+The following are a few options we have for monitoring and benchmarking application performance:
+
+1. [docker stats](https://docs.docker.com/engine/reference/commandline/stats/) - Running this command will show you the CPU and Memory usage of your containers. This is very handy for quickly checking the memory footprint of an image while running.
+1. [drill](https://github.com/fcsonline/drill) - This is a CLI tool used for load-testing applications. It requires [Rust](https://www.rust-lang.org/tools/install) and OpenSSL. This is used in CI to continually benchmark performance.
+1. `nox -s performance_tests` - This is a convenient combination of the prior two tools that is used in CI but also possible to use locally.
+1. `profile-request` - When in `dev_mode`, Adding `profile-request: true` to any request header will tell the server to profile the request and send back a text response containing the profile data, instead of returning the typical JSON response. This allows arbitrary profiling of any endpoint or feature.

noxfiles/ci_nox.py

@@ -216,6 +216,52 @@ def minimal_config_startup(session: nox.Session) -> None:
     session.run(*start_command, external=True)
 
 
+#################
+## Performance ##
+#################
+@nox.session()
+def performance_tests(session: nox.Session) -> None:
+    """Compose the various performance checks into a single uber-test."""
+    session.notify("teardown")
+    session.run(*START_APP, external=True, silent=True)
+    samples = 2
+    for i in range(samples):
+        session.log(f"Sample {i + 1} of {samples}")
+        load_tests(session)
+        docker_stats(session)
+
+
+@nox.session()
+def docker_stats(session: nox.Session) -> None:
+    """
+    Use the builtin `docker stats` command to show resource usage.
+
+    Run this _last_ to get a better worst-case scenario
+    """
+    session.run(
+        "docker",
+        "stats",
+        "--no-stream",
+        "--format",
+        "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}",
+        external=True,
+    )
+
+
+@nox.session()
+def load_tests(session: nox.Session) -> None:
+    """
+    Load test the application.
+
+    Requires a Rust/Cargo installation and then `cargo install drill`
+
+    https://github.com/fcsonline/drill
+    """
+    session.run(
+        "drill", "-b", "noxfiles/drill.yml", "--quiet", "--stats", external=True
+    )
+
+
 ############
 ## Pytest ##
 ############

noxfiles/drill.yml

@@ -0,0 +1,16 @@
+# This file is used by "drill", a load testing utility
+
+concurrency: 4
+base: 'http://localhost:8080'
+# There is a rate-limiter on the webserver, default is set to 2000 requests/min
+iterations: 400
+rampup: 3
+
+plan:
+  - name: Health Check
+    request:
+      url: /health
+
+  - name: Worker Health Check
+    request:
+      url: /health/workers

requirements.txt

@@ -31,6 +31,7 @@ paramiko==3.1.0
 passlib[bcrypt]==1.7.4
 plotly==5.13.1
 pyarrow==6.0.0
+pyinstrument==4.5.1
 psycopg2-binary==2.9.6
 pydantic==1.10.9
 pydash==6.0.2

src/fides/api/main.py

@@ -10,9 +10,10 @@
 from urllib.parse import unquote
 
 from fastapi import HTTPException, Request, Response, status
-from fastapi.responses import FileResponse
+from fastapi.responses import FileResponse, HTMLResponse
 from fideslog.sdk.python.event import AnalyticsEvent
 from loguru import logger
+from pyinstrument import Profiler
 from starlette.background import BackgroundTask
 from uvicorn import Config, Server
 
@@ -50,6 +51,21 @@
 
 app = create_fides_app()
 
+if CONFIG.dev_mode:
+
+    @app.middleware("http")
+    async def profile_request(request: Request, call_next: Callable) -> Response:
+        profiling = request.headers.get("profile-request", False)
+        if profiling:
+            profiler = Profiler(interval=0.001, async_mode="enabled")
+            profiler.start()
+            await call_next(request)
+            profiler.stop()
+            logger.debug("Request Profiled!")
+            return HTMLResponse(profiler.output_text(timeline=True))
+
+        return await call_next(request)
+
 
 @app.middleware("http")
 async def dispatch_log_request(request: Request, call_next: Callable) -> Response: