diff --git a/docs/batch_task.md b/docs/concurrency.md similarity index 55% rename from docs/batch_task.md rename to docs/concurrency.md index 0ced6cc..c3fe8b5 100644 --- a/docs/batch_task.md +++ b/docs/concurrency.md @@ -1,4 +1,8 @@ -# BatchTask +# Concurrency + +Module for advanced concurrency control. + +## BatchTask ::: src.prefecto.concurrency.BatchTask options: diff --git a/docs/index.md b/docs/index.md index b4af537..5006816 100644 --- a/docs/index.md +++ b/docs/index.md @@ -62,3 +62,19 @@ def flow(): df = csv_task(df) return df ``` + +## Extras + +Prefecto includes a number of extras that are not installed by default. + +| Extra | Description | +| --- | --- | +| `pandas` | Adds support for [`pandas.DataFrame`](./serializers/pandas.md) serialization. | +| `polars` | Adds support for [`polars.DataFrame`](./serializers/polars.md) serialization. | +| `moto` | Adds support for mocking AWS's `boto3` with `moto` | + +Extras can be installed with + +```bash +pip install prefecto[extra] +``` diff --git a/docs/serializers/pandas.md b/docs/serializers/pandas.md new file mode 100644 index 0000000..6b9bd11 --- /dev/null +++ b/docs/serializers/pandas.md @@ -0,0 +1,22 @@ +# Pandas + +## PandasSerializer + +Type: `pandas` + +::: src.prefecto.serializers.pandas.PandasSerializer + +## Methods + +`Method` classes for serializing `pandas.DataFrame`. + +| Discriminator | Default Read | Default Write | +| --- | --- | --- | +| `pandas.csv` | `{"index": False}` | `{"index": False}` | +| `pandas.excel` | None | None | +| `pandas.feather` | None | None | +| `pandas.json` | None | None | +| `pandas.jsonl` | None | None | +| `pandas.parquet` | None | None | +| `pandas.pickle` | None | None | +| `pandas.tsv` | `{"sep": "\t", "index": False}` | `{"sep": "\t", "index": False}` | diff --git a/docs/serializers/pandas/methods.md b/docs/serializers/pandas/methods.md deleted file mode 100644 index d3e0868..0000000 --- a/docs/serializers/pandas/methods.md +++ /dev/null @@ -1,67 +0,0 @@ -# Methods - -`Method` classes for serializing `pandas.DataFrame`. - -## CSV - -Discriminator: `pandas.csv` - -::: src.prefecto.serializers.pandas.CSV - options: - show_source: true - -## Excel - -Discriminator: `pandas.excel` - -::: src.prefecto.serializers.pandas.Excel - options: - show_source: true - -## Feather - -Discriminator: `pandas.feather` - -::: src.prefecto.serializers.pandas.Feather - options: - show_source: true - -## JSON - -Discriminator: `pandas.json` - -::: src.prefecto.serializers.pandas.JSON - options: - show_source: true - -## JSONL - -Discriminator: `pandas.jsonl` - -::: src.prefecto.serializers.pandas.JSONL - options: - show_source: true - -## Parquet - -Discriminator: `pandas.parquet` - -::: src.prefecto.serializers.pandas.Parquet - options: - show_source: true - -## Pickle - -Discriminator: `pandas.pickle` - -::: src.prefecto.serializers.pandas.Pickle - options: - show_source: true - -## TSV - -Discriminator: `pandas.tsv` - -::: src.prefecto.serializers.pandas.TSV - options: - show_source: true diff --git a/docs/serializers/pandas/pandas_serializer.md b/docs/serializers/pandas/pandas_serializer.md deleted file mode 100644 index 776e519..0000000 --- a/docs/serializers/pandas/pandas_serializer.md +++ /dev/null @@ -1,7 +0,0 @@ -# PandasSerializer - -Type: `pandas` - -::: src.prefecto.serializers.pandas.PandasSerializer - options: - show_source: true diff --git a/docs/serializers/polars.md b/docs/serializers/polars.md new file mode 100644 index 0000000..5c17b55 --- /dev/null +++ b/docs/serializers/polars.md @@ -0,0 +1,22 @@ +# Polars + +Serialization module for for the `polars.DataFrame` type. + +## PolarsSerializer + +Type: `polars` + +::: src.prefecto.serializers.polars.PolarsSerializer + +## Methods + +`Method` classes for serializing `polars.DataFrame`. + +| Discriminator | Default Read | Default Write | +| --- | --- | --- | +| `polars.csv` | None | None | +| `polars.excel` | None | None | +| `polars.json` | None | None | +| `polars.ndjson` | None | None | +| `polars.parquet` | None | None | +| `polars.tsv` | `{"separator": "\t"}` | `{"separator": "\t"}` | diff --git a/docs/serializers/polars/methods.md b/docs/serializers/polars/methods.md deleted file mode 100644 index fcb4d60..0000000 --- a/docs/serializers/polars/methods.md +++ /dev/null @@ -1,51 +0,0 @@ -# Methods - -`Method` classes for serializing `polars.DataFrame`. - -## CSV - -Discriminator: `polars.csv` - -::: src.prefecto.serializers.polars.CSV - options: - show_source: true - -## Excel - -Discriminator: `polars.excel` - -::: src.prefecto.serializers.polars.Excel - options: - show_source: true - -## JSON - -Discriminator: `polars.json` - -::: src.prefecto.serializers.polars.JSON - options: - show_source: true - -## NDJSON - -Discriminator: `polars.ndjson` - -::: src.prefecto.serializers.polars.NDJSON - options: - show_source: true - -## Parquet - -Discriminator: `polars.parquet` - -::: src.prefecto.serializers.polars.Parquet - options: - show_source: true - -## TSV - -Discriminator: `polars.tsv` - -::: src.prefecto.serializers.polars.TSV - options: - show_source: true diff --git a/docs/serializers/polars/polars_serializer.md b/docs/serializers/polars/polars_serializer.md deleted file mode 100644 index a3a356a..0000000 --- a/docs/serializers/polars/polars_serializer.md +++ /dev/null @@ -1,7 +0,0 @@ -# PolarsSerializer - -Type: `polars` - -::: src.prefecto.serializers.polars.PolarsSerializer - options: - show_source: true diff --git a/mkdocs.yml b/mkdocs.yml index 6b712e5..f35c3f6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,23 +4,20 @@ repo_url: https://github.com/dominictarro/prefecto theme: readthedocs nav: - - Home: index.md - - "Concurrency": - - batch_task.md - - "File Systems": filesystems.md - - "Serialization": - - serializers/extended_serializer.md - - serializers/method.md - - serializers/functions.md - - "Pandas": - - serializers/pandas/pandas_serializer.md - - serializers/pandas/methods.md - - "Polars": - - serializers/polars/polars_serializer.md - - serializers/polars/methods.md - - "Testing": - - testing/s3.md + - "Home": index.md - "License": license.md + - "Reference": + - "Concurrency": concurrency.md + - "File Systems": filesystems.md + - "Serialization": + - serializers/extended_serializer.md + - serializers/method.md + - serializers/functions.md + - "Custom Serializers": + - "Pandas": serializers/pandas.md + - "Polars": serializers/polars.md + - "Testing": + - testing/s3.md plugins: - search: diff --git a/src/prefecto/concurrency.py b/src/prefecto/concurrency.py index 71ca786..4bedfe6 100644 --- a/src/prefecto/concurrency.py +++ b/src/prefecto/concurrency.py @@ -1,5 +1,5 @@ """ -Tools to improve Prefect concurrently. +Tools to improve Prefect concurrency. """ from __future__ import annotations