Merge branch 'master' into docs/config_providers_update

.github/workflows/lint.yml

@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   get_docs_changes:
     uses: ./.github/workflows/get_docs_changes.yml
@@ -17,9 +21,10 @@ jobs:
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        os:
+          - ubuntu-latest
         python-version: ["3.8.x", "3.9.x", "3.10.x", "3.11.x"]
 
     defaults:
@@ -75,4 +80,4 @@ jobs:
       - name: Check matrix job results
         if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
         run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1      
+          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1

.github/workflows/test_airflow.yml

@@ -7,6 +7,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   get_docs_changes:
     uses: ./.github/workflows/get_docs_changes.yml

.github/workflows/test_build_images.yml

@@ -7,6 +7,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   get_docs_changes:
     uses: ./.github/workflows/get_docs_changes.yml

.github/workflows/test_common.yml

@@ -1,4 +1,3 @@
-
 name: test common
 
 on:
@@ -8,6 +7,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   RUNTIME__LOG_LEVEL: ERROR
 
@@ -92,6 +95,19 @@ jobs:
         name: Run smoke tests with minimum deps Windows
         shell: cmd
 
+      - name: Install pyarrow
+        run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk
+
+      - run: |
+          poetry run pytest tests/pipeline/test_pipeline_extra.py -k arrow
+        if: runner.os != 'Windows'
+        name: Run pipeline tests with pyarrow but no pandas installed
+      - run: |
+          poetry run pytest tests/pipeline/test_pipeline_extra.py -k arrow
+        if: runner.os == 'Windows'
+        name: Run pipeline tests with pyarrow but no pandas installed Windows
+        shell: cmd
+
       - name: Install pipeline dependencies
         run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk --with pipeline
 

.github/workflows/test_dbt_cloud.yml

@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   # all credentials must be present to be passed to dbt cloud
   DBT_CLOUD__ACCOUNT_ID: ${{ secrets.DBT_CLOUD__ACCOUNT_ID }}

.github/workflows/test_dbt_runner.yml

@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
 
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}

.github/workflows/test_destination_athena.yml

@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 

.github/workflows/test_destination_athena_iceberg.yml

@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 

.github/workflows/test_destination_bigquery.yml

@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 

.github/workflows/test_destination_databricks.yml

@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 

.github/workflows/test_destination_mssql.yml

@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
 
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}

.github/workflows/test_destination_qdrant.yml

@@ -7,6 +7,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
@@ -28,7 +32,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        os:
+          - ubuntu-latest
     defaults:
       run:
         shell: bash
@@ -64,13 +69,7 @@ jobs:
         run: poetry install --no-interaction -E qdrant -E parquet --with sentry-sdk --with pipeline
       - run: |
           poetry run pytest tests/load/
-        if: runner.os != 'Windows'
-        name: Run tests Linux/MAC
-      - run: |
-          poetry run pytest tests/load/
-        if: runner.os == 'Windows'
-        name: Run tests Windows
-        shell: cmd
+        name: Run tests Linux
 
   matrix_job_required_check:
     name: Qdrant loader tests

.github/workflows/test_destination_snowflake.yml

@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 

.github/workflows/test_destination_synapse.yml

@@ -7,6 +7,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
@@ -24,7 +28,7 @@ jobs:
   run_loader:
     name: Tests Synapse loader
     needs: get_docs_changes
-    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'    
+    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
       fail-fast: false
       matrix:

.github/workflows/test_destinations.yml

@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
 
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}

.github/workflows/test_doc_snippets.yml

@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
@@ -54,7 +58,7 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E duckdb -E weaviate -E parquet -E qdrant --with docs,sentry-sdk --without airflow
+        run: poetry install --no-interaction -E duckdb -E weaviate -E parquet -E qdrant -E bigquery --with docs,sentry-sdk --without airflow
 
       - name: create secrets.toml
         run: pwd && echo "$DLT_SECRETS_TOML" > docs/website/docs/.dlt/secrets.toml

.github/workflows/test_local_destinations.yml

@@ -10,6 +10,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 

CONTRIBUTING.md

@@ -12,6 +12,15 @@ Thank you for considering contributing to **dlt**! We appreciate your help in ma
 6. [Publishing (Maintainers Only)](#publishing-maintainers-only)
 7. [Resources](#resources)
 
+## Before You Begin
+
+- **Proposing significant changes or enhancements**: If you're thinking about making significant changes, make sure to [submit an issue](https://github.com/dlt-hub/dlt/issues/new/choose) first. This ensures your efforts align with the project's direction and that you don't invest time on a feature that may not be merged.
+
+- **Fixing bugs**:
+  - **Check existing issues**: search [open issues](https://github.com/dlt-hub/dlt/issues) to see if the bug you've found is already reported.
+    - If **not reported**, [create a new issue](https://github.com/dlt-hub/dlt/issues/new/choose). You're more than welcome to fix it and submit a pull request with your solution. Thank you!
+    - If the bug is **already reported**, please leave a comment on that issue stating you're working on fixing it. This helps keep everyone updated and avoids duplicate efforts.
+
 ## Getting Started
 
 To get started, follow these steps:

Makefile

@@ -47,7 +47,8 @@ dev: has-poetry
 	poetry install --all-extras --with airflow --with docs --with providers --with pipeline --with sentry-sdk
 
 lint:
-	./check-package.sh
+	./tools/check-package.sh
+	poetry run python ./tools/check-lockfile.py
 	poetry run mypy --config-file mypy.ini dlt tests
 	poetry run flake8 --max-line-length=200 dlt
 	poetry run flake8 --max-line-length=200 tests --exclude tests/reflection/module_cases
@@ -60,8 +61,9 @@ format:
 	# poetry run isort ./
 
 test-and-lint-snippets:
-	poetry run mypy --config-file mypy.ini docs/website docs/examples
-	poetry run flake8 --max-line-length=200 docs/website docs/examples
+	cd docs/tools && poetry run python check_embedded_snippets.py full
+	poetry run mypy --config-file mypy.ini docs/website docs/examples docs/tools --exclude docs/tools/lint_setup
+	poetry run flake8 --max-line-length=200 docs/website docs/examples docs/tools
 	cd docs/website/docs && poetry run pytest --ignore=node_modules
 
 lint-security:

README.md

@@ -41,19 +41,22 @@ Load chess game data from chess.com API and save it in DuckDB:
 ```python
 import dlt
 from dlt.sources.helpers import requests
+
 # Create a dlt pipeline that will load
 # chess player data to the DuckDB destination
 pipeline = dlt.pipeline(
     pipeline_name='chess_pipeline',
     destination='duckdb',
     dataset_name='player_data'
 )
+
 # Grab some player data from Chess.com API
 data = []
 for player in ['magnuscarlsen', 'rpragchess']:
     response = requests.get(f'https://api.chess.com/pub/player/{player}')
     response.raise_for_status()
     data.append(response.json())
+
 # Extract, normalize, and load the data
 pipeline.run(data, table_name='player')
 ```

dlt/__init__.py

@@ -29,6 +29,8 @@
 
 from dlt import sources
 from dlt.extract.decorators import source, resource, transformer, defer
+from dlt.destinations.decorators import destination
+
 from dlt.pipeline import (
     pipeline as _pipeline,
     run,
@@ -62,6 +64,7 @@
     "resource",
     "transformer",
     "defer",
+    "destination",
     "pipeline",
     "run",
     "attach",

dlt/cli/_dlt.py

@@ -443,6 +443,12 @@ def main() -> int:
     pipe_cmd.add_argument(
         "--list-pipelines", "-l", default=False, action="store_true", help="List local pipelines"
     )
+    pipe_cmd.add_argument(
+        "--hot-reload",
+        default=False,
+        action="store_true",
+        help="Reload streamlit app (for core development)",
+    )
     pipe_cmd.add_argument("pipeline_name", nargs="?", help="Pipeline name")
     pipe_cmd.add_argument("--pipelines-dir", help="Pipelines working directory", default=None)
     pipe_cmd.add_argument(