tool benchmarking (#190)

Co-authored-by: Bagatur <[email protected]>
langchain-ai · Jul 24, 2024 · 7e16b6d · 7e16b6d
1 parent 22d279a
commit 7e16b6d
Show file tree

Hide file tree

Showing 7 changed files with 2,627 additions and 1 deletion.
diff --git a/.github/workflows/tool_benchmarks.yml b/.github/workflows/tool_benchmarks.yml
@@ -0,0 +1,33 @@
+name: Weekly Tool Benchmarks
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * 0'  # Runs at midnight (00:00) every Sunday (UTC time)
+
+jobs:
+  run_tool_benchmarks:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.12 + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: '3.12'
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Multiverse math benchmark
+        run: python scripts/multiverse_math_benchmark.py
+
+      - name: Query analysis benchmark
+        run: python scripts/query_analysis_benchmark.py
diff --git a/docs/source/notebooks/tool_usage/multiverse_math_benchmark.ipynb b/docs/source/notebooks/tool_usage/multiverse_math_benchmark.ipynb