From 04cd0a165ad485a8fc7649a73344c69a04c279be Mon Sep 17 00:00:00 2001 From: Violetta Mishechkina Date: Tue, 27 Aug 2024 18:42:12 +0200 Subject: [PATCH 01/19] Move sources and destinations to the top level --- .../verified-sources/filesystem.md | 6 +- .../dlt-ecosystem/verified-sources/index.md | 46 ++++-- .../verified-sources/rest_api.md | 2 +- .../verified-sources/sql_database.md | 2 +- docs/website/sidebars.js | 138 ++++++++---------- 5 files changed, 99 insertions(+), 95 deletions(-) diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/filesystem.md b/docs/website/docs/dlt-ecosystem/verified-sources/filesystem.md index 7552a0acb2..e6d19a4214 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/filesystem.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/filesystem.md @@ -1,6 +1,6 @@ --- title: Filesystem -description: dlt verified source for Readers Source and Filesystem +description: AWS S3, Google Cloud Storage, Azure Blob Storage, local files keywords: [readers source and filesystem, filesystem, readers source] --- import Header from './_source-info-header.md'; @@ -123,7 +123,7 @@ For more information, read the 3. You can pass the bucket URL and glob pattern or use `config.toml`. For local filesystems, use `file://` as follows: - + ```toml [sources.filesystem] # use [sources.readers.credentials] for the "readers" source bucket_url='file://Users/admin/Documents/csv_files' @@ -136,7 +136,7 @@ For more information, read the bucket_url='~\Documents\csv_files\' file_glob="*" ``` - + In the example above we use Windows path to current user's Documents folder. Mind that literal toml string (single quotes) was used to conveniently use the backslashes without need to escape. diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/index.md b/docs/website/docs/dlt-ecosystem/verified-sources/index.md index d105dccb9c..4a58c36cc2 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/index.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/index.md @@ -1,31 +1,45 @@ --- -title: Verified sources -description: List of verified sources -keywords: ['verified source'] +title: Sources +description: Available sources +keywords: ['source'] --- -import DocCardList from '@theme/DocCardList'; import Link from '../../_book-onboarding-call.md'; +import DocCardList from '@theme/DocCardList'; +import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; -Choose from our collection of verified sources, developed and maintained by the dlt team and community. Each source is rigorously tested against a real API and provided as Python code for easy customization. - -Planning to use dlt in production and need a source that isn't listed? We're happy to help you build it: . +Planning to use dlt in production and need a source that isn't listed? We're happy to help you build it: . -### Popular sources +### Core sources -- [SQL databases](sql_database). Supports PostgreSQL, MySQL, MS SQL Server, BigQuery, Redshift, and more. -- [REST API generic source](rest_api). Loads data from REST APIs using declarative configuration. -- [OpenAPI source generator](openapi-generator). Generates a source from an OpenAPI 3.x spec using the REST API source. -- [Cloud and local storage](filesystem). Retrieves data from AWS S3, Google Cloud Storage, Azure Blob Storage, local files, and more. + item.label === '30+ SQL Databases' || item.label === 'REST API generic source' || item.label === 'Filesystem' +)} /> -### Full list of verified sources +### Verified sources - +Choose from our collection of verified sources, developed and maintained by the dlt team and community. Each source is rigorously tested against a real API and provided as Python code for easy customization. :::tip -If you're looking for a source that isn't listed and it provides a REST API, be sure to check out our [REST API generic source](rest_api) - source. +If you couldn't find a source implementation, you can easily create your own, check our [tutorial](../../tutorial) to learn how! ::: + item.label !== '30+ SQL Databases' && item.label !== 'REST API generic source'&& item.label !== 'Filesystem' +)} /> + +### What's the difference? + +The main difference between the [core sources](#core-sources) and [verified sources](#verified-sources) lies in their structure. +Core sources are generic collections, meaning they can connect to a variety of systems. For example, [SQL Database source](sql_database) can connect to any +database which supports SQLAlchemy. + +According to our telemetry, core sources are the most widely used among our users! + +It's also important to note that core sources are integrated into the `dlt` core library, +whereas verified sources are maintained in a separate [repository](https://github.com/dlt-hub/verified-sources). +To use a verified source, you need to run the dlt init command, which will download the verified source code to +your working directory. + ### Get help diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/rest_api.md b/docs/website/docs/dlt-ecosystem/verified-sources/rest_api.md index e1cd9ce88e..c15d5cd7e0 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/rest_api.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/rest_api.md @@ -1,6 +1,6 @@ --- title: REST API generic source -description: dlt verified source for REST APIs +description: Loads data from REST APIs using declarative configuration keywords: [rest api, restful api] --- import Header from './_source-info-header.md'; diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/sql_database.md b/docs/website/docs/dlt-ecosystem/verified-sources/sql_database.md index c89a63a524..b1aab8efee 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/sql_database.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/sql_database.md @@ -1,6 +1,6 @@ --- title: 30+ SQL Databases -description: dlt pipeline for SQL Database +description: PostgreSQL, MySQL, MS SQL Server, BigQuery, Redshift, and more keywords: [sql connector, sql database pipeline, sql database] --- import Header from './_source-info-header.md'; diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 921c3c0dc4..1e7f5c0b54 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -44,84 +44,74 @@ const sidebars = { }, { type: 'category', - label: 'Integrations', + label: 'Sources', link: { type: 'doc', - id: 'dlt-ecosystem/index', + id: 'dlt-ecosystem/verified-sources/index', }, items: [ - { - type: 'category', - label: 'Sources', - link: { - type: 'doc', - id: 'dlt-ecosystem/verified-sources/index', - }, - items: [ - 'dlt-ecosystem/verified-sources/airtable', - 'dlt-ecosystem/verified-sources/amazon_kinesis', - 'dlt-ecosystem/verified-sources/arrow-pandas', - 'dlt-ecosystem/verified-sources/asana', - 'dlt-ecosystem/verified-sources/chess', - 'dlt-ecosystem/verified-sources/facebook_ads', - 'dlt-ecosystem/verified-sources/filesystem', - 'dlt-ecosystem/verified-sources/freshdesk', - 'dlt-ecosystem/verified-sources/github', - 'dlt-ecosystem/verified-sources/google_ads', - 'dlt-ecosystem/verified-sources/google_analytics', - 'dlt-ecosystem/verified-sources/google_sheets', - 'dlt-ecosystem/verified-sources/hubspot', - 'dlt-ecosystem/verified-sources/inbox', - 'dlt-ecosystem/verified-sources/jira', - 'dlt-ecosystem/verified-sources/kafka', - 'dlt-ecosystem/verified-sources/matomo', - 'dlt-ecosystem/verified-sources/mongodb', - 'dlt-ecosystem/verified-sources/mux', - 'dlt-ecosystem/verified-sources/notion', - 'dlt-ecosystem/verified-sources/personio', - 'dlt-ecosystem/verified-sources/pg_replication', - 'dlt-ecosystem/verified-sources/pipedrive', - 'dlt-ecosystem/verified-sources/rest_api', - 'dlt-ecosystem/verified-sources/openapi-generator', - 'dlt-ecosystem/verified-sources/salesforce', - 'dlt-ecosystem/verified-sources/scrapy', - 'dlt-ecosystem/verified-sources/shopify', - 'dlt-ecosystem/verified-sources/sql_database', - 'dlt-ecosystem/verified-sources/slack', - 'dlt-ecosystem/verified-sources/strapi', - 'dlt-ecosystem/verified-sources/stripe', - 'dlt-ecosystem/verified-sources/workable', - 'dlt-ecosystem/verified-sources/zendesk' - ] - }, - { - type: 'category', - label: 'Destinations', - link: { - type: 'doc', - id: 'dlt-ecosystem/destinations/index', - }, - items: [ - 'dlt-ecosystem/destinations/bigquery', - 'dlt-ecosystem/destinations/databricks', - 'dlt-ecosystem/destinations/duckdb', - 'dlt-ecosystem/destinations/mssql', - 'dlt-ecosystem/destinations/synapse', - 'dlt-ecosystem/destinations/clickhouse', - 'dlt-ecosystem/destinations/filesystem', - 'dlt-ecosystem/destinations/postgres', - 'dlt-ecosystem/destinations/redshift', - 'dlt-ecosystem/destinations/snowflake', - 'dlt-ecosystem/destinations/athena', - 'dlt-ecosystem/destinations/weaviate', - 'dlt-ecosystem/destinations/lancedb', - 'dlt-ecosystem/destinations/qdrant', - 'dlt-ecosystem/destinations/dremio', - 'dlt-ecosystem/destinations/destination', - 'dlt-ecosystem/destinations/motherduck' - ] - }, - ], + 'dlt-ecosystem/verified-sources/airtable', + 'dlt-ecosystem/verified-sources/amazon_kinesis', + 'dlt-ecosystem/verified-sources/arrow-pandas', + 'dlt-ecosystem/verified-sources/asana', + 'dlt-ecosystem/verified-sources/chess', + 'dlt-ecosystem/verified-sources/facebook_ads', + 'dlt-ecosystem/verified-sources/filesystem', + 'dlt-ecosystem/verified-sources/freshdesk', + 'dlt-ecosystem/verified-sources/github', + 'dlt-ecosystem/verified-sources/google_ads', + 'dlt-ecosystem/verified-sources/google_analytics', + 'dlt-ecosystem/verified-sources/google_sheets', + 'dlt-ecosystem/verified-sources/hubspot', + 'dlt-ecosystem/verified-sources/inbox', + 'dlt-ecosystem/verified-sources/jira', + 'dlt-ecosystem/verified-sources/kafka', + 'dlt-ecosystem/verified-sources/matomo', + 'dlt-ecosystem/verified-sources/mongodb', + 'dlt-ecosystem/verified-sources/mux', + 'dlt-ecosystem/verified-sources/notion', + 'dlt-ecosystem/verified-sources/personio', + 'dlt-ecosystem/verified-sources/pg_replication', + 'dlt-ecosystem/verified-sources/pipedrive', + 'dlt-ecosystem/verified-sources/rest_api', + 'dlt-ecosystem/verified-sources/openapi-generator', + 'dlt-ecosystem/verified-sources/salesforce', + 'dlt-ecosystem/verified-sources/scrapy', + 'dlt-ecosystem/verified-sources/shopify', + 'dlt-ecosystem/verified-sources/sql_database', + 'dlt-ecosystem/verified-sources/slack', + 'dlt-ecosystem/verified-sources/strapi', + 'dlt-ecosystem/verified-sources/stripe', + 'dlt-ecosystem/verified-sources/workable', + 'dlt-ecosystem/verified-sources/zendesk' + ] + }, + { + type: 'category', + label: 'Destinations', + link: { + type: 'doc', + id: 'dlt-ecosystem/destinations/index', + }, + items: [ + 'dlt-ecosystem/destinations/bigquery', + 'dlt-ecosystem/destinations/databricks', + 'dlt-ecosystem/destinations/duckdb', + 'dlt-ecosystem/destinations/mssql', + 'dlt-ecosystem/destinations/synapse', + 'dlt-ecosystem/destinations/clickhouse', + 'dlt-ecosystem/destinations/filesystem', + 'dlt-ecosystem/destinations/postgres', + 'dlt-ecosystem/destinations/redshift', + 'dlt-ecosystem/destinations/snowflake', + 'dlt-ecosystem/destinations/athena', + 'dlt-ecosystem/destinations/weaviate', + 'dlt-ecosystem/destinations/lancedb', + 'dlt-ecosystem/destinations/qdrant', + 'dlt-ecosystem/destinations/dremio', + 'dlt-ecosystem/destinations/destination', + 'dlt-ecosystem/destinations/motherduck' + ] }, { type: 'category', From 5737571b94a366582121b3b2fd2e746384a5c040 Mon Sep 17 00:00:00 2001 From: Violetta Mishechkina Date: Wed, 28 Aug 2024 15:42:51 +0200 Subject: [PATCH 02/19] Update the css --- docs/website/docs/dlt-ecosystem/index.md | 14 +-- .../dlt-ecosystem/verified-sources/index.md | 4 +- docs/website/sidebars.js | 9 ++ docs/website/src/css/custom.css | 86 ++++++++++++++----- docs/website/static/img/Sources-Active-1.svg | 1 + docs/website/static/img/Sources-Active.svg | 1 + .../website/static/img/Sources-Inactive-1.svg | 1 + docs/website/static/img/Sources-Inactive.svg | 1 + 8 files changed, 79 insertions(+), 38 deletions(-) create mode 100644 docs/website/static/img/Sources-Active-1.svg create mode 100644 docs/website/static/img/Sources-Active.svg create mode 100644 docs/website/static/img/Sources-Inactive-1.svg create mode 100644 docs/website/static/img/Sources-Inactive.svg diff --git a/docs/website/docs/dlt-ecosystem/index.md b/docs/website/docs/dlt-ecosystem/index.md index 740a3a3a39..9fc3fb4ed9 100644 --- a/docs/website/docs/dlt-ecosystem/index.md +++ b/docs/website/docs/dlt-ecosystem/index.md @@ -3,16 +3,4 @@ title: Integrations description: List of integrations keywords: ['integrations, sources, destinations'] --- -import DocCardList from '@theme/DocCardList'; -import Link from '../_book-onboarding-call.md'; - -Speed up the process of creating data pipelines by using dlt's multiple pre-built sources and destinations: - -- Each [dlt verified source](verified-sources) allows you to create [pipelines](../general-usage/pipeline) that extract data from a particular source: a database, a cloud service, or an API. -- [Destinations](destinations) are where you want to load your data. dlt supports a variety of destinations, including databases, data warehouses, and data lakes. - - - -:::tip -Most source-destination pairs work seamlessly together. If the merge [write disposition](../general-usage/incremental-loading#choosing-a-write-disposition) is not supported by a destination (for example, [file sytem destination](destinations/filesystem)), dlt will automatically fall back to the [append](../general-usage/incremental-loading#append) write disposition. -::: \ No newline at end of file +Could not remove this page, please ignore it for now \ No newline at end of file diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/index.md b/docs/website/docs/dlt-ecosystem/verified-sources/index.md index 4a58c36cc2..d2bce768b2 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/index.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/index.md @@ -20,7 +20,7 @@ item => item.label === '30+ SQL Databases' || item.label === 'REST API generic s Choose from our collection of verified sources, developed and maintained by the dlt team and community. Each source is rigorously tested against a real API and provided as Python code for easy customization. :::tip -If you couldn't find a source implementation, you can easily create your own, check our [tutorial](../../tutorial) to learn how! +If you couldn't find a source implementation, you can easily create your own, check our [tutorial](../../tutorial/grouping-resources) to learn how! ::: item.label !== '30+ SQL Databases' && item.label !== 'REST API generic s ### What's the difference? The main difference between the [core sources](#core-sources) and [verified sources](#verified-sources) lies in their structure. -Core sources are generic collections, meaning they can connect to a variety of systems. For example, [SQL Database source](sql_database) can connect to any +Core sources are generic collections, meaning they can connect to a variety of systems. For example, the [SQL Database source](sql_database) can connect to any database which supports SQLAlchemy. According to our telemetry, core sources are the most widely used among our users! diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 1e7f5c0b54..ea2ffb7682 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -42,6 +42,15 @@ const sidebars = { 'tutorial/grouping-resources', ] }, + { + type: 'category', + label: 'Integration', + link: { + type: 'doc', + id: 'dlt-ecosystem/index', + }, + items: ['dlt-ecosystem/index'], + }, { type: 'category', label: 'Sources', diff --git a/docs/website/src/css/custom.css b/docs/website/src/css/custom.css index fca5b933ba..e443a6cda2 100644 --- a/docs/website/src/css/custom.css +++ b/docs/website/src/css/custom.css @@ -601,97 +601,137 @@ html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>[ background-image: url(../../static/img/UsingLoadedData-Active-1.svg); } -/* Using dlt */ +/* Sources */ .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>a::before { - background-image: url(../../static/img/GeneralUsage-Inactive.svg); + background-image: url(../../static/img/Sources-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(5)>div>a::before, -.theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/GeneralUsage-Active.svg); +.theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>[aria-expanded="true"]::before { + background-image: url(../../static/img/Sources-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>a::before { - background-image: url(../../static/img/GeneralUsage-Inactive-1.svg); + background-image: url(../../static/img/Sources-Inactive.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(5)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/GeneralUsage-Active-1.svg); + background-image: url(../../static/img/Sources-Active-1.svg); } -/* How-to Guides */ +/* Destinations */ .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>a::before { - background-image: url(../../static/img/Walkthrough-Inactive.svg); + background-image: url(../../static/img/Destinations-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(6)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Walkthrough-Active.svg); + background-image: url(../../static/img/Destinations-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>a::before { - background-image: url(../../static/img/Walkthrough-Inactive-1.svg); + background-image: url(../../static/img/Destinations-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(6)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Walkthrough-Active-1.svg); + background-image: url(../../static/img/Destinations-Active-1.svg); } -/* Code Examples */ +/* Using dlt */ .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>a::before { - background-image: url(../../static/img/Howdltworks-Inactive.svg); + background-image: url(../../static/img/GeneralUsage-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Howdltworks-Active.svg); + background-image: url(../../static/img/GeneralUsage-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>a::before { - background-image: url(../../static/img/Howdltworks-Inactive-1.svg); + background-image: url(../../static/img/GeneralUsage-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Howdltworks-Active-1.svg); + background-image: url(../../static/img/GeneralUsage-Active-1.svg); } -/* Reference */ +/* How-to Guides */ .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>a::before { - background-image: url(../../static/img/Reference-Inactive.svg); + background-image: url(../../static/img/Walkthrough-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(8)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Reference-Active.svg); + background-image: url(../../static/img/Walkthrough-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>a::before { - background-image: url(../../static/img/Reference-Inactive-1.svg); + background-image: url(../../static/img/Walkthrough-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(8)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>[aria-expanded="true"]::before { + background-image: url(../../static/img/Walkthrough-Active-1.svg); +} + +/* Code Examples */ + +.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>a::before { + background-image: url(../../static/img/Howdltworks-Inactive.svg); +} + +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(9)>div>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>[aria-expanded="true"]::before { + background-image: url(../../static/img/Howdltworks-Active.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>a::before { + background-image: url(../../static/img/Howdltworks-Inactive-1.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(9)>div>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>[aria-expanded="true"]::before { + background-image: url(../../static/img/Howdltworks-Active-1.svg); +} + +/* Reference */ + +.theme-doc-sidebar-menu.menu__list>li:nth-child(10)>div>a::before { + background-image: url(../../static/img/Reference-Inactive.svg); +} + +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(10)>div>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(10)>div>[aria-expanded="true"]::before { + background-image: url(../../static/img/Reference-Active.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(10)>div>a::before { + background-image: url(../../static/img/Reference-Inactive-1.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(10)>div>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(10)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Reference-Active-1.svg); } /* Devel / Stable switch */ -.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>a svg { +.theme-doc-sidebar-menu.menu__list>li:nth-child(11)>a svg { display: none; } -.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>a { +.theme-doc-sidebar-menu.menu__list>li:nth-child(11)>a { flex: none; } -.theme-doc-sidebar-menu.menu__list>li:nth-child(9) { +.theme-doc-sidebar-menu.menu__list>li:nth-child(11) { margin-top:30px; padding-top: 10px; justify-content: center; diff --git a/docs/website/static/img/Sources-Active-1.svg b/docs/website/static/img/Sources-Active-1.svg new file mode 100644 index 0000000000..c409ba1c0f --- /dev/null +++ b/docs/website/static/img/Sources-Active-1.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/website/static/img/Sources-Active.svg b/docs/website/static/img/Sources-Active.svg new file mode 100644 index 0000000000..5e13f5d797 --- /dev/null +++ b/docs/website/static/img/Sources-Active.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/website/static/img/Sources-Inactive-1.svg b/docs/website/static/img/Sources-Inactive-1.svg new file mode 100644 index 0000000000..dff6d8f514 --- /dev/null +++ b/docs/website/static/img/Sources-Inactive-1.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/website/static/img/Sources-Inactive.svg b/docs/website/static/img/Sources-Inactive.svg new file mode 100644 index 0000000000..5506061f25 --- /dev/null +++ b/docs/website/static/img/Sources-Inactive.svg @@ -0,0 +1 @@ + \ No newline at end of file From dae7094786aa251115269d9f48fb879f59152223 Mon Sep 17 00:00:00 2001 From: mariarice15 <123215798+mariarice15@users.noreply.github.com> Date: Thu, 5 Sep 2024 12:37:09 +0200 Subject: [PATCH 03/19] Update sidebars.js --- docs/website/sidebars.js | 198 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 188 insertions(+), 10 deletions(-) diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index ea2ffb7682..7d58a765fe 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -29,17 +29,37 @@ function *walkSync(dir) { const sidebars = { tutorialSidebar: [ 'intro', - 'getting-started', { type: 'category', - label: 'Tutorial', - link: { - type: 'doc', - id: 'tutorial/intro', - }, + label: 'Getting Started', items: [ - 'tutorial/load-data-from-an-api', - 'tutorial/grouping-resources', + 'getting-started', + 'reference/explainers/how-dlt-works', + 'reference/installation', + 'reference/command-line-interface', + { + type: 'category', + label: 'Core Sources', + items: [ + //do these exist? + //'walkthroughs/load-data-from-rest-api', + //'walkthroughs/load-data-from-sql-database', + //'walkthroughs/load-data-from-filesystem-cloud', + // Unsure item + 'reference/core-sources', + ] + } + ] + }, + { + type: 'category', + label: 'Core Concepts', + items: [ + 'general-usage/source', + 'general-usage/resource', + 'general-usage/pipeline', + 'general-usage/destination', + 'general-usage/glossary' ] }, { @@ -92,7 +112,17 @@ const sidebars = { 'dlt-ecosystem/verified-sources/strapi', 'dlt-ecosystem/verified-sources/stripe', 'dlt-ecosystem/verified-sources/workable', - 'dlt-ecosystem/verified-sources/zendesk' + 'dlt-ecosystem/verified-sources/zendesk', + //added + 'walkthroughs/add-a-verified-source', + { + type: 'category', + label: 'REST API Helpers', + items: [ + 'general-usage/http/rest-client', + 'general-usage/http/requests', + ] + } ] }, { @@ -119,9 +149,157 @@ const sidebars = { 'dlt-ecosystem/destinations/qdrant', 'dlt-ecosystem/destinations/dremio', 'dlt-ecosystem/destinations/destination', - 'dlt-ecosystem/destinations/motherduck' + 'dlt-ecosystem/destinations/motherduck', + //added + 'walkthroughs/create-new-destination' + ] + }, + { + type: 'category', + label: 'Using dlt', + items: [ + 'walkthroughs/create-a-pipeline', + 'walkthroughs/run-a-pipeline', + 'dlt-ecosystem/visualizations/exploring-the-data', + { + type: 'category', + label: 'Transform the Data', + items: [ + 'general-usage/customising-pipelines/renaming_columns', + 'general-usage/customising-pipelines/pseudonymizing_columns', + 'general-usage/customising-pipelines/removing_columns' + ] + }, + { + type: 'category', + label: 'Configuration & Secrets', + items: [ + 'general-usage/credentials/setup', + 'general-usage/credentials/advanced', + 'general-usage/credentials/complex_types', + // Unsure item + 'walkthroughs/add_credentials', + ] + }, + 'general-usage/schema', + 'general-usage/schema-evolution', + 'walkthroughs/adjust-a-schema', + { + type: 'category', + label: 'Loading Data', + items: [ + 'general-usage/incremental-loading', + 'walkthroughs/add-incremental-configuration-sql', + 'general-usage/full-loading', + ] + } ] }, + { + type: 'category', + label: 'Deploying dlt', + items: [ + 'walkthroughs/share-a-dataset', + { + type: 'category', + label: 'Deploy a pipeline', + link: { + type: 'generated-index', + title: 'Deploy a pipeline', + description: 'Deploy dlt pipelines with different methods.', + slug: 'walkthroughs/deploy-a-pipeline', + }, + items: [ + 'walkthroughs/deploy-a-pipeline/deploy-with-github-actions', + 'walkthroughs/deploy-a-pipeline/deploy-with-airflow-composer', + 'reference/explainers/airflow-gcp-cloud-composer', + 'walkthroughs/deploy-a-pipeline/deploy-with-google-cloud-functions', + 'walkthroughs/deploy-a-pipeline/deploy-gcp-cloud-function-as-webhook', + 'walkthroughs/deploy-a-pipeline/deploy-with-kestra', + 'walkthroughs/deploy-a-pipeline/deploy-with-dagster', + 'walkthroughs/deploy-a-pipeline/deploy-with-prefect', + ] + }, + 'walkthroughs/run-in-production', + ] + }, + { + type: 'category', + label: 'Optimizing dlt', + items: [ + 'reference/performance', + // Unsure item + 'general-usage/state', + ] + }, + { + type: 'category', + label: 'Code examples', + link: { + type: 'generated-index', + title: 'Code examples', + description: 'A list of comprehensive code examples that teach you how to solve real world problem.', + slug: 'examples', + keywords: ['examples'], + }, + items: [ + 'walkthroughs/dispatch-to-multiple-tables', + ], + }, + + { + type: 'category', + label: 'Reference', + link: { + type: 'generated-index', + title: 'Reference', + description: 'The dlthub reference. Learn more about the dlt, CLI, and the telemetry.', + slug: 'reference', + keywords: ['reference'], + }, + items: [ + 'reference/telemetry', + 'reference/frequently-asked-questions', + // Unsure item + 'reference/destination-tables', + 'general-usage/naming-convention', + 'general-usage/staging', + { + type: 'category', + label: 'File Formats', + items: [ + 'dlt-ecosystem/file-formats/jsonl', + 'dlt-ecosystem/file-formats/parquet', + 'dlt-ecosystem/file-formats/csv', + 'dlt-ecosystem/file-formats/insert-format', + ] + } + ], + }, + ] +}; + + + + + + + + + { + type: 'category', + label: 'Tutorial', + link: { + type: 'doc', + id: 'tutorial/intro', + }, + items: [ + 'tutorial/load-data-from-an-api', + 'tutorial/grouping-resources', + ] + }, + + { type: 'category', label: 'Using dlt', From 7a07996e879796dabf6b433ba88e2a860b641dcb Mon Sep 17 00:00:00 2001 From: mariarice15 <123215798+mariarice15@users.noreply.github.com> Date: Thu, 5 Sep 2024 13:27:09 +0200 Subject: [PATCH 04/19] Update sidebars.js --- docs/website/sidebars.js | 268 +++++---------------------------------- 1 file changed, 35 insertions(+), 233 deletions(-) diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 7d58a765fe..c2c9fe2c27 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -163,8 +163,25 @@ const sidebars = { 'dlt-ecosystem/visualizations/exploring-the-data', { type: 'category', - label: 'Transform the Data', + label: 'Transform the data', + link: { + type: 'generated-index', + title: 'Transform the data', + description: 'If you want to transform the data after loading, you can use one of the following methods: dbt, SQL, Pandas.', + slug: 'dlt-ecosystem/transformations', + keywords: ['transformations'], + }, items: [ + { + type: 'category', + label: 'Transforming data with dbt', + items: [ + 'dlt-ecosystem/transformations/dbt/dbt', + 'dlt-ecosystem/transformations/dbt/dbt_cloud', + ] + }, + 'dlt-ecosystem/transformations/sql', + 'dlt-ecosystem/transformations/pandas', 'general-usage/customising-pipelines/renaming_columns', 'general-usage/customising-pipelines/pseudonymizing_columns', 'general-usage/customising-pipelines/removing_columns' @@ -220,7 +237,23 @@ const sidebars = { 'walkthroughs/deploy-a-pipeline/deploy-with-prefect', ] }, - 'walkthroughs/run-in-production', + { + type: 'category', + label: 'Run in production', + link: { + type: 'generated-index', + title: 'Run in production', + description: 'How to run dlt in production.', + slug: 'running-in-production', + keywords: ['production'], + }, + items: [ + 'running-in-production/running', + 'running-in-production/monitoring', + 'running-in-production/alerting', + 'running-in-production/tracing', + ], + }, ] }, { @@ -281,237 +314,6 @@ const sidebars = { - - - - - - { - type: 'category', - label: 'Tutorial', - link: { - type: 'doc', - id: 'tutorial/intro', - }, - items: [ - 'tutorial/load-data-from-an-api', - 'tutorial/grouping-resources', - ] - }, - - - { - type: 'category', - label: 'Using dlt', - link: { - type: 'generated-index', - title: 'Using dlt', - slug: 'general-usage', - keywords: ['concepts', 'usage'], - }, - items: [ - 'reference/explainers/how-dlt-works', - 'general-usage/resource', - 'general-usage/source', - { - type: 'category', - label: 'Configuration and secrets', - link: { - type: 'doc', - id: 'general-usage/credentials/index', - }, - items: [ - 'general-usage/credentials/setup', - 'general-usage/credentials/advanced', - 'general-usage/credentials/complex_types', - ] - }, - 'general-usage/pipeline', - 'general-usage/destination', - 'general-usage/destination-tables', - { - type: 'category', - label: 'REST API helpers', - link: { - type: 'doc', - id: 'general-usage/http/overview', - }, - items: [ - 'general-usage/http/rest-client', - 'general-usage/http/requests', - ] - }, - 'dlt-ecosystem/staging', - 'general-usage/state', - 'general-usage/incremental-loading', - 'general-usage/full-loading', - 'general-usage/schema', - 'general-usage/naming-convention', - 'general-usage/schema-contracts', - 'general-usage/schema-evolution', - 'build-a-pipeline-tutorial', - 'reference/performance', - { - type: 'category', - label: 'File formats', - link: { - type: 'generated-index', - title: 'File formats', - description: 'Overview of our loader file formats', - slug: 'dlt-ecosystem/file-formats', - keywords: ['destination'], - }, - items: [ - 'dlt-ecosystem/file-formats/jsonl', - 'dlt-ecosystem/file-formats/parquet', - 'dlt-ecosystem/file-formats/csv', - 'dlt-ecosystem/file-formats/insert-format', - ] - }, - ], - }, - { - type: 'category', - label: 'How-to guides', - link: { - type: 'generated-index', - title: 'How-to guides', - description: 'In this section you will find step-by-step instructions for the common tasks.', - slug: 'walkthroughs', - keywords: ['how-to'], - }, - items: [ - 'walkthroughs/create-a-pipeline', - 'walkthroughs/add-a-verified-source', - 'walkthroughs/add-incremental-configuration', - 'walkthroughs/add_credentials', - 'walkthroughs/run-a-pipeline', - 'walkthroughs/adjust-a-schema', - 'walkthroughs/share-a-dataset', - 'dlt-ecosystem/visualizations/exploring-the-data', - { - type: 'category', - label: 'Transform the data', - link: { - type: 'generated-index', - title: 'Transform the data', - description: 'If you want to transform the data after loading, you can use one of the following methods: dbt, SQL, Pandas.', - slug: 'dlt-ecosystem/transformations', - keywords: ['transformations'], - }, - items: [ - { - type: 'category', - label: 'Transforming data with dbt', - items: [ - 'dlt-ecosystem/transformations/dbt/dbt', - 'dlt-ecosystem/transformations/dbt/dbt_cloud', - ] - }, - 'dlt-ecosystem/transformations/sql', - 'dlt-ecosystem/transformations/pandas', - ] - }, - { - type: 'category', - label: 'Deploy a pipeline', - link: { - type: 'generated-index', - title: 'Deploy a pipeline', - description: 'Deploy dlt pipelines with different methods.', - slug: 'walkthroughs/deploy-a-pipeline', - }, - items: [ - 'walkthroughs/deploy-a-pipeline/deploy-with-github-actions', - 'walkthroughs/deploy-a-pipeline/deploy-with-airflow-composer', - 'reference/explainers/airflow-gcp-cloud-composer', - 'walkthroughs/deploy-a-pipeline/deploy-with-google-cloud-functions', - 'walkthroughs/deploy-a-pipeline/deploy-gcp-cloud-function-as-webhook', - 'walkthroughs/deploy-a-pipeline/deploy-with-kestra', - 'walkthroughs/deploy-a-pipeline/deploy-with-dagster', - 'walkthroughs/deploy-a-pipeline/deploy-with-prefect', - ] - }, - { - type: 'category', - label: 'Customise pipelines', - items: [ - 'general-usage/customising-pipelines/renaming_columns', - 'general-usage/customising-pipelines/pseudonymizing_columns', - 'general-usage/customising-pipelines/removing_columns', - ] - }, - { - type: 'category', - label: 'Data enrichments', - items: [ - 'general-usage/data-enrichments/user_agent_device_data_enrichment', - 'general-usage/data-enrichments/currency_conversion_data_enrichment', - 'general-usage/data-enrichments/url-parser-data-enrichment' - ] - }, - { - type: 'category', - label: 'Run in production', - link: { - type: 'generated-index', - title: 'Run in production', - description: 'How to run dlt in production.', - slug: 'running-in-production', - keywords: ['production'], - }, - items: [ - 'running-in-production/running', - 'running-in-production/monitoring', - 'running-in-production/alerting', - 'running-in-production/tracing', - ], - }, - 'walkthroughs/dispatch-to-multiple-tables', - 'walkthroughs/create-new-destination', - 'walkthroughs/zendesk-weaviate', - ], - }, - { - type: 'category', - label: 'Code examples', - link: { - type: 'generated-index', - title: 'Code examples', - description: 'A list of comprehensive code examples that teach you how to solve real world problem.', - slug: 'examples', - keywords: ['examples'], - }, - items: [ - ], - }, - { - type: 'category', - label: 'Reference', - link: { - type: 'generated-index', - title: 'Reference', - description: 'The dlthub reference. Learn more about the dlt, CLI, and the telemetry.', - slug: 'reference', - keywords: ['reference'], - }, - items: [ - 'reference/installation', - 'reference/command-line-interface', - 'reference/telemetry', - 'reference/frequently-asked-questions', - 'general-usage/glossary', - ], - }, - // { - // "API Documentation": [ - // require("./docs/api_reference/sidebar.json") - // ], - // } - ] -}; - - // insert examples for (const item of sidebars.tutorialSidebar) { if (item.label === 'Code examples') { From 5a95b77ef20e0190e794e45bbbc77e2ac5e90023 Mon Sep 17 00:00:00 2001 From: mariarice15 <123215798+mariarice15@users.noreply.github.com> Date: Thu, 5 Sep 2024 15:50:45 +0200 Subject: [PATCH 05/19] Update sidebars.js --- docs/website/sidebars.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index c2c9fe2c27..b0a281da87 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -8,7 +8,7 @@ Create as many sidebars as you want. */ - + // @ts-check const fs = require('fs'); const path = require('path'); From 75e16568d4fa4012f491777f05233981b9ed6e63 Mon Sep 17 00:00:00 2001 From: mariarice15 <123215798+mariarice15@users.noreply.github.com> Date: Thu, 5 Sep 2024 15:56:28 +0200 Subject: [PATCH 06/19] Update sidebars.js --- docs/website/sidebars.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index b0a281da87..9de1590f3f 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -46,7 +46,7 @@ const sidebars = { //'walkthroughs/load-data-from-sql-database', //'walkthroughs/load-data-from-filesystem-cloud', // Unsure item - 'reference/core-sources', + //'reference/core-sources', ] } ] @@ -206,7 +206,7 @@ const sidebars = { label: 'Loading Data', items: [ 'general-usage/incremental-loading', - 'walkthroughs/add-incremental-configuration-sql', + 'walkthroughs/add-incremental-configuration', 'general-usage/full-loading', ] } @@ -294,9 +294,9 @@ const sidebars = { 'reference/telemetry', 'reference/frequently-asked-questions', // Unsure item - 'reference/destination-tables', + 'general-usage/destination-tables', 'general-usage/naming-convention', - 'general-usage/staging', + 'dlt-ecosystem/staging', { type: 'category', label: 'File Formats', From c3a298306515a4ccefccc594715452a14cac13a4 Mon Sep 17 00:00:00 2001 From: mariarice15 <123215798+mariarice15@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:00:24 +0200 Subject: [PATCH 07/19] Update sidebars.js --- docs/website/sidebars.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 9de1590f3f..462739ab89 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -37,6 +37,8 @@ const sidebars = { 'reference/explainers/how-dlt-works', 'reference/installation', 'reference/command-line-interface', + /** + to be added: { type: 'category', label: 'Core Sources', @@ -49,6 +51,7 @@ const sidebars = { //'reference/core-sources', ] } + **/ ] }, { From 75daafa5afbb11f36378660f2446f3a265d08e22 Mon Sep 17 00:00:00 2001 From: mariarice15 <123215798+mariarice15@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:08:57 +0200 Subject: [PATCH 08/19] Update sidebars.js --- docs/website/sidebars.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 462739ab89..eff702d30a 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -190,9 +190,13 @@ const sidebars = { 'general-usage/customising-pipelines/removing_columns' ] }, - { + { type: 'category', - label: 'Configuration & Secrets', + label: 'Configuration and secrets', + link: { + type: 'doc', + id: 'general-usage/credentials/index', + }, items: [ 'general-usage/credentials/setup', 'general-usage/credentials/advanced', From 2d961ab67d69cb1c57837ed5b173c46cbbf16ef6 Mon Sep 17 00:00:00 2001 From: Maria Rice Date: Fri, 6 Sep 2024 14:55:12 +0200 Subject: [PATCH 09/19] reorder --- docs/website/sidebars.js | 82 ++++++++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 25 deletions(-) diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index eff702d30a..3fcf064c7e 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -8,7 +8,7 @@ Create as many sidebars as you want. */ - + // @ts-check const fs = require('fs'); const path = require('path'); @@ -37,12 +37,11 @@ const sidebars = { 'reference/explainers/how-dlt-works', 'reference/installation', 'reference/command-line-interface', - /** - to be added: { type: 'category', label: 'Core Sources', items: [ + 'tutorial/load-data-from-an-api', //do these exist? //'walkthroughs/load-data-from-rest-api', //'walkthroughs/load-data-from-sql-database', @@ -51,7 +50,6 @@ const sidebars = { //'reference/core-sources', ] } - **/ ] }, { @@ -65,15 +63,6 @@ const sidebars = { 'general-usage/glossary' ] }, - { - type: 'category', - label: 'Integration', - link: { - type: 'doc', - id: 'dlt-ecosystem/index', - }, - items: ['dlt-ecosystem/index'], - }, { type: 'category', label: 'Sources', @@ -120,12 +109,16 @@ const sidebars = { 'walkthroughs/add-a-verified-source', { type: 'category', - label: 'REST API Helpers', + label: 'REST API helpers', + link: { + type: 'doc', + id: 'general-usage/http/overview', + }, items: [ 'general-usage/http/rest-client', 'general-usage/http/requests', ] - } + }, ] }, { @@ -157,9 +150,15 @@ const sidebars = { 'walkthroughs/create-new-destination' ] }, - { + { type: 'category', label: 'Using dlt', + link: { + type: 'generated-index', + title: 'Using dlt', + slug: 'general-usage', + keywords: ['concepts', 'usage'], + }, items: [ 'walkthroughs/create-a-pipeline', 'walkthroughs/run-a-pipeline', @@ -202,12 +201,22 @@ const sidebars = { 'general-usage/credentials/advanced', 'general-usage/credentials/complex_types', // Unsure item - 'walkthroughs/add_credentials', + 'walkthroughs/add_credentials' + ] + }, + { + type: 'category', + label: 'Schema', + link: { + type: 'doc', + id: 'general-usage/schema', + }, + items: [ + 'general-usage/schema-contracts', + 'general-usage/schema-evolution', + 'walkthroughs/adjust-a-schema', ] }, - 'general-usage/schema', - 'general-usage/schema-evolution', - 'walkthroughs/adjust-a-schema', { type: 'category', label: 'Loading Data', @@ -286,7 +295,6 @@ const sidebars = { 'walkthroughs/dispatch-to-multiple-tables', ], }, - { type: 'category', label: 'Reference', @@ -306,21 +314,31 @@ const sidebars = { 'dlt-ecosystem/staging', { type: 'category', - label: 'File Formats', + label: 'File formats', + link: { + type: 'generated-index', + title: 'File formats', + description: 'Overview of our loader file formats', + slug: 'dlt-ecosystem/file-formats', + keywords: ['destination'], + }, items: [ 'dlt-ecosystem/file-formats/jsonl', 'dlt-ecosystem/file-formats/parquet', 'dlt-ecosystem/file-formats/csv', 'dlt-ecosystem/file-formats/insert-format', ] - } + }, ], }, + // { + // "API Documentation": [ + // require("./docs/api_reference/sidebar.json") + // ], + // } ] }; - - // insert examples for (const item of sidebars.tutorialSidebar) { if (item.label === 'Code examples') { @@ -360,3 +378,17 @@ if (process.env.IS_MASTER_BRANCH) { } module.exports = sidebars; + /* + blog: + 'build-a-pipeline-tutorial', + 'walkthroughs/zendesk-weaviate', + { + type: 'category', + label: 'Data enrichments', + items: [ + 'general-usage/data-enrichments/user_agent_device_data_enrichment', + 'general-usage/data-enrichments/currency_conversion_data_enrichment', + 'general-usage/data-enrichments/url-parser-data-enrichment' + ] + }, + */ \ No newline at end of file From 85ab078e5ba4751148255ff4fcb572d4919bf8ae Mon Sep 17 00:00:00 2001 From: Maria Rice Date: Fri, 6 Sep 2024 15:06:26 +0200 Subject: [PATCH 10/19] error fix --- docs/website/sidebars.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 3fcf064c7e..370ed0deae 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -149,6 +149,20 @@ const sidebars = { //added 'walkthroughs/create-new-destination' ] + }, + { + //ignore: could not remove + type: 'category', + label: 'How-to guides', + link: { + type: 'generated-index', + title: 'How-to guides', + description: 'In this section you will find step-by-step instructions for the common tasks.', + slug: 'walkthroughs', + keywords: ['how-to'], + }, + items: [ + ] }, { type: 'category', From 0f3bc4510137e49e373b82e0c05960aac7b78495 Mon Sep 17 00:00:00 2001 From: Maria Rice Date: Fri, 6 Sep 2024 15:17:50 +0200 Subject: [PATCH 11/19] error fix --- docs/website/sidebars.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 370ed0deae..7c25e3cd7f 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -160,7 +160,7 @@ const sidebars = { description: 'In this section you will find step-by-step instructions for the common tasks.', slug: 'walkthroughs', keywords: ['how-to'], - }, + }, items: [ ] }, From 138c491eaed0cdcfa5277daaf3a3e9c877d132c2 Mon Sep 17 00:00:00 2001 From: Maria Rice Date: Fri, 6 Sep 2024 15:25:07 +0200 Subject: [PATCH 12/19] re-add walkthroughs --- docs/website/sidebars.js | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 7c25e3cd7f..88139c4534 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -162,6 +162,7 @@ const sidebars = { keywords: ['how-to'], }, items: [ + 'docs/walkthroughs' ] }, { From c1440f6e243d1df5168d139064ccb322165eb205 Mon Sep 17 00:00:00 2001 From: Maria Rice Date: Fri, 6 Sep 2024 15:41:29 +0200 Subject: [PATCH 13/19] re-add walkthroughs (in all) --- docs/website/sidebars.js | 150 +++++++++++++++++++++++++++++++-------- 1 file changed, 121 insertions(+), 29 deletions(-) diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 88139c4534..f625876a5c 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -149,21 +149,6 @@ const sidebars = { //added 'walkthroughs/create-new-destination' ] - }, - { - //ignore: could not remove - type: 'category', - label: 'How-to guides', - link: { - type: 'generated-index', - title: 'How-to guides', - description: 'In this section you will find step-by-step instructions for the common tasks.', - slug: 'walkthroughs', - keywords: ['how-to'], - }, - items: [ - 'docs/walkthroughs' - ] }, { type: 'category', @@ -351,6 +336,111 @@ const sidebars = { // require("./docs/api_reference/sidebar.json") // ], // } + + { + type: 'category', + label: 'How-to guides', + link: { + type: 'generated-index', + title: 'How-to guides', + description: 'In this section you will find step-by-step instructions for the common tasks.', + slug: 'walkthroughs', + keywords: ['how-to'], + }, + items: [ + 'walkthroughs/create-a-pipeline', + 'walkthroughs/add-a-verified-source', + 'walkthroughs/add-incremental-configuration', + 'walkthroughs/add_credentials', + 'walkthroughs/run-a-pipeline', + 'walkthroughs/adjust-a-schema', + 'walkthroughs/share-a-dataset', + 'dlt-ecosystem/visualizations/exploring-the-data', + { + type: 'category', + label: 'Transform the data', + link: { + type: 'generated-index', + title: 'Transform the data', + description: 'If you want to transform the data after loading, you can use one of the following methods: dbt, SQL, Pandas.', + slug: 'dlt-ecosystem/transformations', + keywords: ['transformations'], + }, + items: [ + { + type: 'category', + label: 'Transforming data with dbt', + items: [ + 'dlt-ecosystem/transformations/dbt/dbt', + 'dlt-ecosystem/transformations/dbt/dbt_cloud', + ] + }, + 'dlt-ecosystem/transformations/sql', + 'dlt-ecosystem/transformations/pandas', + ] + }, + { + type: 'category', + label: 'Deploy a pipeline', + link: { + type: 'generated-index', + title: 'Deploy a pipeline', + description: 'Deploy dlt pipelines with different methods.', + slug: 'walkthroughs/deploy-a-pipeline', + }, + items: [ + 'walkthroughs/deploy-a-pipeline/deploy-with-github-actions', + 'walkthroughs/deploy-a-pipeline/deploy-with-airflow-composer', + 'reference/explainers/airflow-gcp-cloud-composer', + 'walkthroughs/deploy-a-pipeline/deploy-with-google-cloud-functions', + 'walkthroughs/deploy-a-pipeline/deploy-gcp-cloud-function-as-webhook', + 'walkthroughs/deploy-a-pipeline/deploy-with-kestra', + 'walkthroughs/deploy-a-pipeline/deploy-with-dagster', + 'walkthroughs/deploy-a-pipeline/deploy-with-prefect', + ] + }, + { + type: 'category', + label: 'Customise pipelines', + items: [ + 'general-usage/customising-pipelines/renaming_columns', + 'general-usage/customising-pipelines/pseudonymizing_columns', + 'general-usage/customising-pipelines/removing_columns', + ] + }, + { + type: 'category', + label: 'Data enrichments', + items: [ + 'general-usage/data-enrichments/user_agent_device_data_enrichment', + 'general-usage/data-enrichments/currency_conversion_data_enrichment', + 'general-usage/data-enrichments/url-parser-data-enrichment' + ] + }, + { + type: 'category', + label: 'Run in production', + link: { + type: 'generated-index', + title: 'Run in production', + description: 'How to run dlt in production.', + slug: 'running-in-production', + keywords: ['production'], + }, + items: [ + 'running-in-production/running', + 'running-in-production/monitoring', + 'running-in-production/alerting', + 'running-in-production/tracing', + ], + }, + 'walkthroughs/dispatch-to-multiple-tables', + 'walkthroughs/create-new-destination', + 'walkthroughs/zendesk-weaviate', + ], + }, + + ] }; @@ -393,17 +483,19 @@ if (process.env.IS_MASTER_BRANCH) { } module.exports = sidebars; - /* - blog: - 'build-a-pipeline-tutorial', - 'walkthroughs/zendesk-weaviate', - { - type: 'category', - label: 'Data enrichments', - items: [ - 'general-usage/data-enrichments/user_agent_device_data_enrichment', - 'general-usage/data-enrichments/currency_conversion_data_enrichment', - 'general-usage/data-enrichments/url-parser-data-enrichment' - ] - }, - */ \ No newline at end of file + + +/* +blog: +'build-a-pipeline-tutorial', +'walkthroughs/zendesk-weaviate', +{ + type: 'category', + label: 'Data enrichments', + items: [ + 'general-usage/data-enrichments/user_agent_device_data_enrichment', + 'general-usage/data-enrichments/currency_conversion_data_enrichment', + 'general-usage/data-enrichments/url-parser-data-enrichment' + ] +}, +*/ \ No newline at end of file From 898a5963a337a3aad1c77001adaaa0e175dbc853 Mon Sep 17 00:00:00 2001 From: Maria Rice Date: Tue, 10 Sep 2024 16:11:02 +0200 Subject: [PATCH 14/19] updates to sidebar structure re: notion comments --- docs/website/docs/reference/performance.md | 2 +- .../docs/walkthroughs/share-a-dataset.md | 2 +- docs/website/sidebars.js | 26 ++++--------------- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/docs/website/docs/reference/performance.md b/docs/website/docs/reference/performance.md index 0ee62acec7..184017988d 100644 --- a/docs/website/docs/reference/performance.md +++ b/docs/website/docs/reference/performance.md @@ -1,5 +1,5 @@ --- -title: Performance +title: Optimizing dlt description: Scale-up, parallelize and finetune dlt pipelines keywords: [scaling, parallelism, finetuning] --- diff --git a/docs/website/docs/walkthroughs/share-a-dataset.md b/docs/website/docs/walkthroughs/share-a-dataset.md index 75ba856468..fe7c44f594 100644 --- a/docs/website/docs/walkthroughs/share-a-dataset.md +++ b/docs/website/docs/walkthroughs/share-a-dataset.md @@ -1,5 +1,5 @@ --- -title: 'Share a dataset: DuckDB -> BigQuery' +title: 'Moving from local to production' description: Share a local dataset by moving it to BigQuery keywords: [how to, share a dataset] --- diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index f625876a5c..bb720221f5 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -28,15 +28,12 @@ function *walkSync(dir) { /** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ const sidebars = { tutorialSidebar: [ - 'intro', { type: 'category', label: 'Getting Started', items: [ 'getting-started', - 'reference/explainers/how-dlt-works', 'reference/installation', - 'reference/command-line-interface', { type: 'category', label: 'Core Sources', @@ -46,8 +43,6 @@ const sidebars = { //'walkthroughs/load-data-from-rest-api', //'walkthroughs/load-data-from-sql-database', //'walkthroughs/load-data-from-filesystem-cloud', - // Unsure item - //'reference/core-sources', ] } ] @@ -56,10 +51,12 @@ const sidebars = { type: 'category', label: 'Core Concepts', items: [ + 'reference/explainers/how-dlt-works', 'general-usage/source', 'general-usage/resource', 'general-usage/pipeline', 'general-usage/destination', + 'general-usage/state', 'general-usage/glossary' ] }, @@ -219,7 +216,7 @@ const sidebars = { }, { type: 'category', - label: 'Loading Data', + label: 'Loading Behavior', items: [ 'general-usage/incremental-loading', 'walkthroughs/add-incremental-configuration', @@ -272,15 +269,7 @@ const sidebars = { }, ] }, - { - type: 'category', - label: 'Optimizing dlt', - items: [ - 'reference/performance', - // Unsure item - 'general-usage/state', - ] - }, + 'reference/performance', { type: 'category', label: 'Code examples', @@ -306,6 +295,7 @@ const sidebars = { keywords: ['reference'], }, items: [ + 'reference/command-line-interface', 'reference/telemetry', 'reference/frequently-asked-questions', // Unsure item @@ -331,12 +321,6 @@ const sidebars = { }, ], }, - // { - // "API Documentation": [ - // require("./docs/api_reference/sidebar.json") - // ], - // } - { type: 'category', label: 'How-to guides', From 7410d67de49fe9fe07c0351713d709971f9987f7 Mon Sep 17 00:00:00 2001 From: Maria Rice Date: Tue, 10 Sep 2024 16:31:22 +0200 Subject: [PATCH 15/19] added back intro --- docs/website/sidebars.js | 72 +++------------------------------------- 1 file changed, 4 insertions(+), 68 deletions(-) diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index bb720221f5..39f7f01f0b 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -28,6 +28,7 @@ function *walkSync(dir) { /** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ const sidebars = { tutorialSidebar: [ + 'intro', { type: 'category', label: 'Getting Started', @@ -340,49 +341,6 @@ const sidebars = { 'walkthroughs/adjust-a-schema', 'walkthroughs/share-a-dataset', 'dlt-ecosystem/visualizations/exploring-the-data', - { - type: 'category', - label: 'Transform the data', - link: { - type: 'generated-index', - title: 'Transform the data', - description: 'If you want to transform the data after loading, you can use one of the following methods: dbt, SQL, Pandas.', - slug: 'dlt-ecosystem/transformations', - keywords: ['transformations'], - }, - items: [ - { - type: 'category', - label: 'Transforming data with dbt', - items: [ - 'dlt-ecosystem/transformations/dbt/dbt', - 'dlt-ecosystem/transformations/dbt/dbt_cloud', - ] - }, - 'dlt-ecosystem/transformations/sql', - 'dlt-ecosystem/transformations/pandas', - ] - }, - { - type: 'category', - label: 'Deploy a pipeline', - link: { - type: 'generated-index', - title: 'Deploy a pipeline', - description: 'Deploy dlt pipelines with different methods.', - slug: 'walkthroughs/deploy-a-pipeline', - }, - items: [ - 'walkthroughs/deploy-a-pipeline/deploy-with-github-actions', - 'walkthroughs/deploy-a-pipeline/deploy-with-airflow-composer', - 'reference/explainers/airflow-gcp-cloud-composer', - 'walkthroughs/deploy-a-pipeline/deploy-with-google-cloud-functions', - 'walkthroughs/deploy-a-pipeline/deploy-gcp-cloud-function-as-webhook', - 'walkthroughs/deploy-a-pipeline/deploy-with-kestra', - 'walkthroughs/deploy-a-pipeline/deploy-with-dagster', - 'walkthroughs/deploy-a-pipeline/deploy-with-prefect', - ] - }, { type: 'category', label: 'Customise pipelines', @@ -400,31 +358,9 @@ const sidebars = { 'general-usage/data-enrichments/currency_conversion_data_enrichment', 'general-usage/data-enrichments/url-parser-data-enrichment' ] - }, - { - type: 'category', - label: 'Run in production', - link: { - type: 'generated-index', - title: 'Run in production', - description: 'How to run dlt in production.', - slug: 'running-in-production', - keywords: ['production'], - }, - items: [ - 'running-in-production/running', - 'running-in-production/monitoring', - 'running-in-production/alerting', - 'running-in-production/tracing', - ], - }, - 'walkthroughs/dispatch-to-multiple-tables', - 'walkthroughs/create-new-destination', - 'walkthroughs/zendesk-weaviate', - ], - }, - - + } + ] + } ] }; From 298689c14b500bf6e77d77379fa5471294d37228 Mon Sep 17 00:00:00 2001 From: Anton Burnashev Date: Sat, 14 Sep 2024 16:07:05 +0200 Subject: [PATCH 16/19] Sync titles --- docs/website/docs/reference/performance.md | 2 +- docs/website/docs/walkthroughs/share-a-dataset.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/website/docs/reference/performance.md b/docs/website/docs/reference/performance.md index b34923291b..58951702a5 100644 --- a/docs/website/docs/reference/performance.md +++ b/docs/website/docs/reference/performance.md @@ -4,7 +4,7 @@ description: Scale-up, parallelize and finetune dlt pipelines keywords: [scaling, parallelism, finetuning] --- -# Performance +# Optimizing dlt ## Yield pages instead of rows diff --git a/docs/website/docs/walkthroughs/share-a-dataset.md b/docs/website/docs/walkthroughs/share-a-dataset.md index fe7c44f594..3e231b8274 100644 --- a/docs/website/docs/walkthroughs/share-a-dataset.md +++ b/docs/website/docs/walkthroughs/share-a-dataset.md @@ -4,7 +4,7 @@ description: Share a local dataset by moving it to BigQuery keywords: [how to, share a dataset] --- -# Share a dataset: DuckDB -> BigQuery +# Moving from local to production In previous how-to guides you used the local stack to create and run your pipeline. This saved you the headache of setting up cloud account, credentials and often also money. Our choice for local From fee8c503f9a79aacef7beffe487858e7b2d27e8a Mon Sep 17 00:00:00 2001 From: Anton Burnashev Date: Sat, 14 Sep 2024 17:15:29 +0200 Subject: [PATCH 17/19] Fix new sidebar --- docs/website/sidebars.js | 77 +++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 4f8d9998da..61e75f2ce5 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -45,7 +45,7 @@ const sidebars = { }, { type: 'category', - label: 'Core Concepts', + label: 'Core concepts', items: [ 'reference/explainers/how-dlt-works', 'general-usage/source', @@ -71,18 +71,18 @@ const sidebars = { 'dlt-ecosystem/verified-sources/chess', 'dlt-ecosystem/verified-sources/facebook_ads', { - type: 'category', - label: 'Filesystem & buckets', - description: 'AWS S3, GCP, Azure, local files', - link: { - type: 'doc', - id: 'dlt-ecosystem/verified-sources/filesystem/index', - }, - items: [ - 'dlt-ecosystem/verified-sources/filesystem/basic', - 'dlt-ecosystem/verified-sources/filesystem/advanced', - ] - }, + type: 'category', + label: 'Filesystem & cloud storage', + description: 'AWS S3, Google Cloud Storage, Azure Blob Storage, local file system', + link: { + type: 'doc', + id: 'dlt-ecosystem/verified-sources/filesystem/index', + }, + items: [ + 'dlt-ecosystem/verified-sources/filesystem/basic', + 'dlt-ecosystem/verified-sources/filesystem/advanced', + ] + }, 'dlt-ecosystem/verified-sources/freshdesk', 'dlt-ecosystem/verified-sources/github', 'dlt-ecosystem/verified-sources/google_ads', @@ -136,7 +136,20 @@ const sidebars = { 'dlt-ecosystem/verified-sources/strapi', 'dlt-ecosystem/verified-sources/stripe', 'dlt-ecosystem/verified-sources/workable', - 'dlt-ecosystem/verified-sources/zendesk' + 'dlt-ecosystem/verified-sources/zendesk', + { + type: 'category', + label: 'REST API helpers', + link: { + type: 'doc', + id: 'general-usage/http/overview', + }, + items: [ + 'general-usage/http/rest-client', + 'general-usage/http/requests', + ] + }, + 'walkthroughs/add-a-verified-source', ] }, { @@ -164,16 +177,15 @@ const sidebars = { 'dlt-ecosystem/destinations/dremio', 'dlt-ecosystem/destinations/destination', 'dlt-ecosystem/destinations/motherduck', - //added 'walkthroughs/create-new-destination' ] }, - { + { type: 'category', - label: 'Core concepts', + label: 'Using dlt', link: { type: 'generated-index', - title: 'Core concepts', + title: 'Using dlt', slug: 'general-usage', keywords: ['concepts', 'usage'], }, @@ -207,7 +219,7 @@ const sidebars = { 'general-usage/customising-pipelines/removing_columns' ] }, - { + { type: 'category', label: 'Configuration and secrets', link: { @@ -311,14 +323,13 @@ const sidebars = { link: { type: 'generated-index', title: 'Reference', - description: 'The dlthub reference. Learn more about the dlt, CLI, and the telemetry.', + description: 'Learn more about the dlt, CLI, and the telemetry.', slug: 'reference', keywords: ['reference'], }, items: [ 'reference/command-line-interface', 'reference/telemetry', - 'reference/frequently-asked-questions', // Unsure item 'general-usage/destination-tables', 'general-usage/naming-convention', @@ -340,8 +351,10 @@ const sidebars = { 'dlt-ecosystem/file-formats/insert-format', ] }, + 'reference/frequently-asked-questions', ], }, + /* { type: 'category', label: 'How-to guides', @@ -353,23 +366,6 @@ const sidebars = { keywords: ['how-to'], }, items: [ - 'walkthroughs/create-a-pipeline', - 'walkthroughs/add-a-verified-source', - 'walkthroughs/add-incremental-configuration', - 'walkthroughs/add_credentials', - 'walkthroughs/run-a-pipeline', - 'walkthroughs/adjust-a-schema', - 'walkthroughs/share-a-dataset', - 'dlt-ecosystem/visualizations/exploring-the-data', - { - type: 'category', - label: 'Customise pipelines', - items: [ - 'general-usage/customising-pipelines/renaming_columns', - 'general-usage/customising-pipelines/pseudonymizing_columns', - 'general-usage/customising-pipelines/removing_columns', - ] - }, { type: 'category', label: 'Data enrichments', @@ -378,9 +374,10 @@ const sidebars = { 'general-usage/data-enrichments/currency_conversion_data_enrichment', 'general-usage/data-enrichments/url-parser-data-enrichment' ] - } + } ] - } + } + */ ] }; From 8b09b98d0f00cb2142e24283216ad8b227e1b514 Mon Sep 17 00:00:00 2001 From: Anton Burnashev Date: Sat, 14 Sep 2024 17:36:24 +0200 Subject: [PATCH 18/19] Adjust iconts --- docs/website/src/css/custom.css | 100 ++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 30 deletions(-) diff --git a/docs/website/src/css/custom.css b/docs/website/src/css/custom.css index 4d016a9a7f..baff5ba024 100644 --- a/docs/website/src/css/custom.css +++ b/docs/website/src/css/custom.css @@ -707,123 +707,163 @@ html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(1)>div>[ background-image: url(../../static/img/GettingStarted-Active-1.svg); } -/* Sources */ +/* Core concepts */ .theme-doc-sidebar-menu.menu__list>li:nth-child(2)>div>a::before { - background-image: url(../../static/img/Sources-Inactive.svg); + background-image: url(../../static/img/UserGuide-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(2)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(2)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Sources-Active.svg); + background-image: url(../../static/img/UserGuide-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(2)>div>a::before { - background-image: url(../../static/img/Sources-Inactive-1.svg); + background-image: url(../../static/img/UserGuide-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(2)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(2)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Sources-Active-1.svg); + background-image: url(../../static/img/UserGuide-Active-1.svg); } -/* Destinations */ +/* Sources */ .theme-doc-sidebar-menu.menu__list>li:nth-child(3)>div>a::before { - background-image: url(../../static/img/Destinations-Inactive.svg); + background-image: url(../../static/img/Sources-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(3)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(3)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Destinations-Active.svg); + background-image: url(../../static/img/Sources-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(3)>div>a::before { - background-image: url(../../static/img/Destinations-Inactive-1.svg); + background-image: url(../../static/img/Sources-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(3)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(3)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Destinations-Active-1.svg); + background-image: url(../../static/img/Sources-Active-1.svg); } -/* Using dlt */ +/* Destinations */ .theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>a::before { - background-image: url(../../static/img/GeneralUsage-Inactive.svg); + background-image: url(../../static/img/Destinations-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(4)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/GeneralUsage-Active.svg); + background-image: url(../../static/img/Destinations-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>a::before { - background-image: url(../../static/img/GeneralUsage-Inactive-1.svg); + background-image: url(../../static/img/Destinations-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(4)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/GeneralUsage-Active-1.svg); + background-image: url(../../static/img/Destinations-Active-1.svg); } -/* How-to Guides */ +/* Using dlt */ .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>a::before { - background-image: url(../../static/img/Walkthrough-Inactive.svg); + background-image: url(../../static/img/GeneralUsage-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(5)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Walkthrough-Active.svg); + background-image: url(../../static/img/GeneralUsage-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>a::before { - background-image: url(../../static/img/Walkthrough-Inactive-1.svg); + background-image: url(../../static/img/GeneralUsage-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(5)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Walkthrough-Active-1.svg); + background-image: url(../../static/img/GeneralUsage-Active-1.svg); } -/* Code Examples */ +/* Deploying dlt */ .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>a::before { - background-image: url(../../static/img/Howdltworks-Inactive.svg); + background-image: url(../../static/img/UsingLoadedData-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(6)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Howdltworks-Active.svg); + background-image: url(../../static/img/UsingLoadedData-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>a::before { - background-image: url(../../static/img/Howdltworks-Inactive-1.svg); + background-image: url(../../static/img/UsingLoadedData-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(6)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>[aria-expanded="true"]::before { + background-image: url(../../static/img/UsingLoadedData-Active-1.svg); +} + +/* Optimizing dlt */ + +.theme-doc-sidebar-menu.menu__list>li:nth-child(7)>a::before { + background-image: url(../../static/img/Installation-Inactive.svg); +} + +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(7)>a.menu__link--active::before { + background-image: url(../../static/img/Installation-Active.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>a::before { + background-image: url(../../static/img/Installation-Inactive-1.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>a.menu__link--active::before { + background-image: url(../../static/img/Installation-Active-1.svg); +} + +/* Code Examples */ + +.theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>a::before { + background-image: url(../../static/img/Howdltworks-Inactive.svg); +} + +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(8)>div>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>[aria-expanded="true"]::before { + background-image: url(../../static/img/Howdltworks-Active.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>a::before { + background-image: url(../../static/img/Howdltworks-Inactive-1.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(8)>div>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Howdltworks-Active-1.svg); } /* Reference */ -.theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>a::before { +.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>a::before { background-image: url(../../static/img/Reference-Inactive.svg); } -.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>div>a::before, -.theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>[aria-expanded="true"]::before { +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(9)>div>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Reference-Active.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>a::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>a::before { background-image: url(../../static/img/Reference-Inactive-1.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>div>a::before, -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>[aria-expanded="true"]::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(9)>div>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Reference-Active-1.svg); } From d389010271a9fd6c126bdd2b86d833c73596dd5e Mon Sep 17 00:00:00 2001 From: Anton Burnashev Date: Sat, 14 Sep 2024 17:45:50 +0200 Subject: [PATCH 19/19] Remove the category link --- docs/website/docs/tutorial/load-data-from-an-api.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/website/docs/tutorial/load-data-from-an-api.md b/docs/website/docs/tutorial/load-data-from-an-api.md index 1718ef642b..1e40531691 100644 --- a/docs/website/docs/tutorial/load-data-from-an-api.md +++ b/docs/website/docs/tutorial/load-data-from-an-api.md @@ -574,9 +574,8 @@ Interested in learning more? Here are some suggestions: - [Run in production: inspecting, tracing, retry policies and cleaning up](../running-in-production/running). - [Run resources in parallel, optimize buffers and local storage](../reference/performance.md) - [Use REST API client helpers](../general-usage/http/rest-client.md) to simplify working with REST APIs. -3. Check out our [how-to guides](../walkthroughs) to get answers to some common questions. -4. Explore [destinations](../dlt-ecosystem/destinations/) and [sources](../dlt-ecosystem/verified-sources/) provided by us and community. -5. Explore the [Examples](../examples) section to see how dlt can be used in real-world scenarios +3. Explore [destinations](../dlt-ecosystem/destinations/) and [sources](../dlt-ecosystem/verified-sources/) provided by us and community. +4. Explore the [Examples](../examples) section to see how dlt can be used in real-world scenarios