fix(dataloader): add support in dataloader for csv and parquet #1924
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| jobs: | |
| build: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macOS-latest] | |
| python-version: ["3.10", "3.11"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Poetry (Unix) | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| curl -sSL https://install.python-poetry.org | python3 - | |
| echo 'export PATH="$HOME/.local/bin:$PATH"' >> $GITHUB_ENV | |
| - name: Install Poetry (Windows) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| (Invoke-WebRequest -Uri https://install.python-poetry.org -UseBasicParsing).Content | python - | |
| echo "C:\\Users\\runneradmin\\AppData\\Roaming\\Python\\Scripts" >> $env:GITHUB_PATH | |
| - name: Verify Poetry Installation | |
| run: poetry --version | |
| - name: Clear Poetry Cache | |
| run: poetry cache clear pypi --all | |
| - name: Install future | |
| run: pip wheel --use-pep517 "future==0.18.3" | |
| - name: Install dependencies | |
| run: poetry install --all-extras --with dev --verbose | |
| - name: Install extension dependencies | |
| run: | | |
| # Install LLM extension dependencies | |
| for dir in extensions/llms/*/; do | |
| if [ -f "$dir/pyproject.toml" ]; then | |
| echo "Installing dependencies for $dir" | |
| cd "$dir" | |
| poetry install --all-extras --with dev | |
| cd - | |
| fi | |
| done | |
| # Install connector extension dependencies | |
| for dir in extensions/connectors/*/; do | |
| if [ -f "$dir/pyproject.toml" ]; then | |
| echo "Installing dependencies for $dir" | |
| cd "$dir" | |
| poetry install --all-extras --with dev | |
| cd - | |
| fi | |
| done | |
| # Install enterprise extension dependencies | |
| for dir in extensions/ee/*/*/; do | |
| if [ -f "$dir/pyproject.toml" ]; then | |
| echo "Installing dependencies for $dir" | |
| cd "$dir" | |
| poetry install --all-extras --with dev | |
| cd - | |
| fi | |
| done | |
| - name: Install extension dependencies (Windows) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| # Install LLM extension dependencies | |
| Get-ChildItem -Path extensions/llms -Directory | ForEach-Object { | |
| $projFile = Join-Path $_.FullName "pyproject.toml" | |
| if (Test-Path $projFile) { | |
| Write-Host "Installing dependencies for $_" | |
| Push-Location $_.FullName | |
| poetry install --all-extras --with dev | |
| Pop-Location | |
| } | |
| } | |
| # Install connector extension dependencies | |
| Get-ChildItem -Path extensions/connectors -Directory | ForEach-Object { | |
| $projFile = Join-Path $_.FullName "pyproject.toml" | |
| if (Test-Path $projFile) { | |
| Write-Host "Installing dependencies for $_" | |
| Push-Location $_.FullName | |
| poetry install --all-extras --with dev | |
| Pop-Location | |
| } | |
| } | |
| # Install enterprise extension dependencies | |
| Get-ChildItem -Path extensions/ee -Recurse -Directory -Depth 2 | ForEach-Object { | |
| $projFile = Join-Path $_.FullName "pyproject.toml" | |
| if (Test-Path $projFile) { | |
| Write-Host "Installing dependencies for $_" | |
| Push-Location $_.FullName | |
| poetry install --all-extras --with dev | |
| Pop-Location | |
| } | |
| } | |
| - name: Lint with ruff | |
| run: make format_diff | |
| - name: Spellcheck | |
| run: make spell_check | |
| - name: Run core tests | |
| run: make test_core | |
| - name: Run extension tests | |
| if: matrix.os != 'windows-latest' | |
| run: make test_extensions | |
| - name: Run extension tests (Windows) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| # Run LLM extension tests | |
| Get-ChildItem -Path extensions/llms -Directory | ForEach-Object { | |
| $testDir = Join-Path $_.FullName "tests" | |
| if (Test-Path $testDir) { | |
| Write-Host "Running tests for $_" | |
| Push-Location $_.FullName | |
| poetry run pytest tests/ | |
| Pop-Location | |
| } | |
| } | |
| # Run connector extension tests | |
| Get-ChildItem -Path extensions/connectors -Directory | ForEach-Object { | |
| $testDir = Join-Path $_.FullName "tests" | |
| if (Test-Path $testDir) { | |
| Write-Host "Running tests for $_" | |
| Push-Location $_.FullName | |
| poetry run pytest tests/ | |
| Pop-Location | |
| } | |
| } | |
| # Run enterprise extension tests | |
| Get-ChildItem -Path extensions/ee -Recurse -Directory -Depth 2 | ForEach-Object { | |
| $testDir = Join-Path $_.FullName "tests" | |
| if (Test-Path $testDir) { | |
| Write-Host "Running tests for $_" | |
| Push-Location $_.FullName | |
| poetry run pytest tests/ | |
| Pop-Location | |
| } | |
| } | |
| - name: Run code coverage | |
| continue-on-error: true | |
| run: | | |
| poetry run coverage run --source=pandasai,extensions -m pytest tests extensions/*/tests extensions/ee/*/tests --ignore=tests/integration_tests | |
| poetry run coverage xml | |
| - name: Report coverage | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: ./coverage.xml | |
| flags: unittests | |
| name: codecov-umbrella | |
| fail_ci_if_error: false |