Python Integration Tests #12
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # This workflow will run all python integrations tests. | |
| # | |
| name: Python Integration Tests | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: ["main"] | |
| merge_group: | |
| branches: ["main"] | |
| schedule: | |
| - cron: "0 0 * * *" # Run at midnight UTC daily | |
| permissions: | |
| contents: read | |
| jobs: | |
| paths-filter: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pythonChanges: ${{ steps.filter.outputs.python}} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| python: | |
| - 'python/**' | |
| # run only if 'python' files were changed | |
| - name: python tests | |
| if: steps.filter.outputs.python == 'true' | |
| run: echo "Python file" | |
| # run only if not 'python' files were changed | |
| - name: not python tests | |
| if: steps.filter.outputs.python != 'true' | |
| run: echo "NOT python file" | |
| python-merge-gate: | |
| name: Python Pre-Merge Integration Tests | |
| needs: paths-filter | |
| if: github.event_name != 'pull_request' && github.event_name != 'schedule' && needs.paths-filter.outputs.pythonChanges == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| max-parallel: 1 | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.11"] | |
| os: [ubuntu-latest] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install poetry | |
| run: pipx install poetry | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: "poetry" | |
| - name: Install dependencies with hnswlib native disabled | |
| if: matrix.os == 'macos-latest' && matrix.python-version == '3.11' | |
| run: | | |
| export HNSWLIB_NO_NATIVE=1 | |
| python -m pip install --upgrade pip setuptools wheel | |
| cd python && poetry install --with tests | |
| - name: Install dependencies with hnswlib native enabled | |
| if: matrix.os != 'macos-latest' || matrix.python-version != '3.11' | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| cd python | |
| poetry install --with tests | |
| - name: Install Ollama | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| curl -fsSL https://ollama.com/install.sh | sh | |
| ollama serve & | |
| sleep 5 | |
| - name: Pull model in Ollama | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| ollama pull ${{ vars.OLLAMA_MODEL }} | |
| ollama list | |
| - name: Run Integration Tests | |
| id: run_tests | |
| shell: bash | |
| env: # Set Azure credentials secret as an input | |
| HNSWLIB_NO_NATIVE: 1 | |
| Python_Integration_Tests: Python_Integration_Tests | |
| AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }} # azure-text-embedding-ada-002 | |
| AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} | |
| AZURE_OPENAI_TEXT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_TEXT_DEPLOYMENT_NAME }} | |
| AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }} | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| BING_API_KEY: ${{ secrets.BING_API_KEY }} | |
| OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_ID }} | |
| OPENAI_TEXT_MODEL_ID: ${{ vars.OPENAI_TEXT_MODEL_ID }} | |
| OPENAI_EMBEDDING_MODEL_ID: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| PINECONE_API_KEY: ${{ secrets.PINECONE__APIKEY }} | |
| POSTGRES_CONNECTION_STRING: ${{secrets.POSTGRES__CONNECTIONSTR}} | |
| AZURE_AI_SEARCH_API_KEY: ${{secrets.AZURE_AI_SEARCH_API_KEY}} | |
| AZURE_AI_SEARCH_ENDPOINT: ${{secrets.AZURE_AI_SEARCH_ENDPOINT}} | |
| MONGODB_ATLAS_CONNECTION_STRING: ${{secrets.MONGODB_ATLAS_CONNECTION_STRING}} | |
| AZURE_KEY_VAULT_ENDPOINT: ${{secrets.AZURE_KEY_VAULT_ENDPOINT}} | |
| AZURE_KEY_VAULT_CLIENT_ID: ${{secrets.AZURE_KEY_VAULT_CLIENT_ID}} | |
| AZURE_KEY_VAULT_CLIENT_SECRET: ${{secrets.AZURE_KEY_VAULT_CLIENT_SECRET}} | |
| ACA_POOL_MANAGEMENT_ENDPOINT: ${{secrets.ACA_POOL_MANAGEMENT_ENDPOINT}} | |
| MISTRALAI_API_KEY: ${{secrets.MISTRALAI_API_KEY}} | |
| MISTRALAI_CHAT_MODEL_ID: ${{ vars.MISTRALAI_CHAT_MODEL_ID }} | |
| OLLAMA_MODEL: "${{ matrix.os == 'ubuntu-latest' && vars.OLLAMA_MODEL || '' }}" # phi3 | |
| run: | | |
| if ${{ matrix.os == 'ubuntu-latest' }}; then | |
| docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest | |
| fi | |
| cd python | |
| poetry run pytest ./tests/integration -v | |
| poetry run pytest ./tests/samples -v | |
| python-integration-tests: | |
| needs: paths-filter | |
| if: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && needs.paths-filter.outputs.pythonChanges == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| max-parallel: 1 | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12"] | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install poetry | |
| run: pipx install poetry | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: "poetry" | |
| - name: Install dependencies with hnswlib native disabled | |
| if: matrix.os == 'macos-latest' && matrix.python-version == '3.11' | |
| run: | | |
| export HNSWLIB_NO_NATIVE=1 | |
| python -m pip install --upgrade pip setuptools wheel | |
| cd python && poetry install --with tests | |
| - name: Install dependencies with hnswlib native enabled | |
| if: matrix.os != 'macos-latest' || matrix.python-version != '3.11' | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| cd python && poetry install --with tests | |
| - name: Install Ollama | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| curl -fsSL https://ollama.com/install.sh | sh | |
| ollama serve & | |
| sleep 5 | |
| - name: Pull model in Ollama | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| ollama pull ${{ vars.OLLAMA_MODEL }} | |
| ollama list | |
| - name: Run Integration Tests | |
| id: run_tests | |
| shell: bash | |
| env: # Set Azure credentials secret as an input | |
| HNSWLIB_NO_NATIVE: 1 | |
| Python_Integration_Tests: Python_Integration_Tests | |
| AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }} # azure-text-embedding-ada-002 | |
| AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} | |
| AZURE_OPENAI_TEXT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_TEXT_DEPLOYMENT_NAME }} | |
| AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }} | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| BING_API_KEY: ${{ secrets.BING_API_KEY }} | |
| OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_ID }} | |
| OPENAI_TEXT_MODEL_ID: ${{ vars.OPENAI_TEXT_MODEL_ID }} | |
| OPENAI_EMBEDDING_MODEL_ID: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| PINECONE_API_KEY: ${{ secrets.PINECONE__APIKEY }} | |
| POSTGRES_CONNECTION_STRING: ${{secrets.POSTGRES__CONNECTIONSTR}} | |
| AZURE_AI_SEARCH_API_KEY: ${{secrets.AZURE_AI_SEARCH_API_KEY}} | |
| AZURE_AI_SEARCH_ENDPOINT: ${{secrets.AZURE_AI_SEARCH_ENDPOINT}} | |
| MONGODB_ATLAS_CONNECTION_STRING: ${{secrets.MONGODB_ATLAS_CONNECTION_STRING}} | |
| AZURE_KEY_VAULT_ENDPOINT: ${{secrets.AZURE_KEY_VAULT_ENDPOINT}} | |
| AZURE_KEY_VAULT_CLIENT_ID: ${{secrets.AZURE_KEY_VAULT_CLIENT_ID}} | |
| AZURE_KEY_VAULT_CLIENT_SECRET: ${{secrets.AZURE_KEY_VAULT_CLIENT_SECRET}} | |
| ACA_POOL_MANAGEMENT_ENDPOINT: ${{secrets.ACA_POOL_MANAGEMENT_ENDPOINT}} | |
| MISTRALAI_API_KEY: ${{secrets.MISTRALAI_API_KEY}} | |
| MISTRALAI_CHAT_MODEL_ID: ${{ vars.MISTRALAI_CHAT_MODEL_ID }} | |
| OLLAMA_MODEL: "${{ matrix.os == 'ubuntu-latest' && vars.OLLAMA_MODEL || '' }}" # phi3 | |
| run: | | |
| if ${{ matrix.os == 'ubuntu-latest' }}; then | |
| docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest | |
| fi | |
| cd python | |
| poetry run pytest ./tests/integration -v | |
| poetry run pytest ./tests/samples -v | |
| # This final job is required to satisfy the merge queue. It must only run (or succeed) if no tests failed | |
| python-integration-tests-check: | |
| if: always() | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 1 | |
| fail-fast: false | |
| needs: [python-merge-gate, python-integration-tests] | |
| steps: | |
| - name: Get Date | |
| shell: bash | |
| run: | | |
| echo "date=$(date +'%m/%d/%Y %H:%M:%S')" >> "$GITHUB_ENV" | |
| - name: Run Type is Daily | |
| if: ${{ github.event_name == 'schedule' }} | |
| shell: bash | |
| run: | | |
| echo "run_type=Daily" >> "$GITHUB_ENV" | |
| - name: Run Type is Manual | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| shell: bash | |
| run: | | |
| echo "run_type=Manual" >> "$GITHUB_ENV" | |
| - name: Run Type is ${{ github.event_name }} | |
| if: ${{ github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'}} | |
| shell: bash | |
| run: | | |
| echo "run_type=${{ github.event_name }}" >> "$GITHUB_ENV" | |
| - name: Fail workflow if tests failed | |
| id: check_tests_failed | |
| if: contains(join(needs.*.result, ','), 'failure') | |
| uses: actions/github-script@v6 | |
| with: | |
| script: core.setFailed('Integration Tests Failed!') | |
| - name: Fail workflow if tests cancelled | |
| id: check_tests_cancelled | |
| if: contains(join(needs.*.result, ','), 'cancelled') | |
| uses: actions/github-script@v6 | |
| with: | |
| script: core.setFailed('Integration Tests Cancelled!') | |
| - name: Microsoft Teams Notification | |
| uses: skitionek/notify-microsoft-teams@master | |
| if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' | |
| with: | |
| webhook_url: ${{ secrets.MSTEAMS_WEBHOOK }} | |
| dry_run: ${{ env.run_type != 'Daily' && env.run_type != 'Manual'}} | |
| job: ${{ toJson(job) }} | |
| steps: ${{ toJson(steps) }} | |
| overwrite: "{title: ` ${{ env.run_type }}: ${{ env.date }} `, text: ` ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`}" | |
| - name: Microsoft Teams Notification (Dry Run) | |
| uses: skitionek/notify-microsoft-teams@master | |
| if: github.ref != 'refs/heads/main' | |
| with: | |
| webhook_url: NONE | |
| dry_run: ${{ env.run_type != 'Daily' && env.run_type != 'Manual'}} | |
| job: ${{ toJson(job) }} | |
| steps: ${{ toJson(steps) }} | |
| overwrite: "{title: ` ${{ env.run_type }}: ${{ env.date }} `, text: ` ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`}" |