From 91656c560107ede4011b53b74cb0d4b4c369bd33 Mon Sep 17 00:00:00 2001 From: Violetta Mishechkina Date: Tue, 17 Sep 2024 11:05:54 +0200 Subject: [PATCH 1/2] Bring back links, redirects, reorder --- .../website/docs/build-a-pipeline-tutorial.md | 8 +-- .../dlt-ecosystem/destinations/bigquery.md | 2 + .../dlt-ecosystem/verified-sources/index.md | 2 +- .../verified-sources/rest_api/basic.md | 2 +- docs/website/netlify.toml | 16 +++-- docs/website/sidebars.js | 58 +++++++++---------- 6 files changed, 47 insertions(+), 41 deletions(-) diff --git a/docs/website/docs/build-a-pipeline-tutorial.md b/docs/website/docs/build-a-pipeline-tutorial.md index 128f4ccc88..de1a4d647f 100644 --- a/docs/website/docs/build-a-pipeline-tutorial.md +++ b/docs/website/docs/build-a-pipeline-tutorial.md @@ -48,8 +48,8 @@ normalize, and evolve your data schemas, enabling seamless data integration and For example, let's consider a scenario where you want to load a list of objects into a DuckDB table named "three". With `dlt`, you can create a pipeline and run it with just a few lines of code: -1. [Create a pipeline](walkthroughs/create-a-pipeline.md) to the [destination](dlt-ecosystem/destinations). -1. Give this pipeline data and [run it](walkthroughs/run-a-pipeline.md). +1. [Create a pipeline](./walkthroughs/create-a-pipeline.md) to the [destination](dlt-ecosystem/destinations). +1. Give this pipeline data and [run it](./walkthroughs/run-a-pipeline.md). ```py import dlt @@ -377,7 +377,7 @@ processing and loading the data. Exporting schema files enables you to modify th adjustments to the schema as needed. You can then import the modified schema files back into `dlt` to use them in your pipeline. -Read more: [Adjust a schema docs.](walkthroughs/adjust-a-schema.md) +Read more: [Adjust a schema docs.](./walkthroughs/adjust-a-schema.md) ## Governance Support in `dlt` Pipelines @@ -399,7 +399,7 @@ define the structure of normalized data and guide the processing and loading of predefined schemas, pipelines maintain data integrity and facilitate standardized data handling practices. -Read more: [Adjust a schema docs.](walkthroughs/adjust-a-schema.md) +Read more: [Adjust a schema docs.](./walkthroughs/adjust-a-schema.md) ### Schema evolution diff --git a/docs/website/docs/dlt-ecosystem/destinations/bigquery.md b/docs/website/docs/dlt-ecosystem/destinations/bigquery.md index 3dd625212f..324c712dfc 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/bigquery.md +++ b/docs/website/docs/dlt-ecosystem/destinations/bigquery.md @@ -362,5 +362,7 @@ bigquery_adapter(my_resource, partition="partition_column_name") my_resource = bigquery_adapter(my_resource, partition="partition_column_name") ``` +Refer to the [full API specification](../../api_reference/destinations/impl/bigquery/bigquery_adapter) for more details. + diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/index.md b/docs/website/docs/dlt-ecosystem/verified-sources/index.md index 505488b42f..a3d2ba00a7 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/index.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/index.md @@ -24,7 +24,7 @@ If you couldn't find a source implementation, you can easily create your own, ch ::: item.label !== '30+ SQL Databases' && item.label !== 'REST API generic source'&& item.label !== 'Filesystem & buckets' +item => item.label !== '30+ SQL Databases' && item.label !== 'REST APIs' && item.label !== 'Filesystem & cloud storage' )} /> ### What's the difference between core and verified sources? diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/rest_api/basic.md b/docs/website/docs/dlt-ecosystem/verified-sources/rest_api/basic.md index 1a31eb864b..e301128dc1 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/rest_api/basic.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/rest_api/basic.md @@ -309,7 +309,7 @@ A resource configuration is used to define a [dlt resource](../../../general-usa - `include_from_parent`: A list of fields from the parent resource to be included in the resource output. See the [resource relationships](#include-fields-from-the-parent-resource) section for more details. - `selected`: A flag to indicate if the resource is selected for loading. This could be useful when you want to load data only from child resources and not from the parent resource. -You can also pass additional resource parameters that will be used to configure the dlt resource. +You can also pass additional resource parameters that will be used to configure the dlt resource. See [dlt resource API reference](../../../api_reference/extract/decorators#resource) for more details. ### Endpoint configuration diff --git a/docs/website/netlify.toml b/docs/website/netlify.toml index 919ad09aea..76c0a15c03 100644 --- a/docs/website/netlify.toml +++ b/docs/website/netlify.toml @@ -6,18 +6,27 @@ to = "/docs/intro" from = "/docs" to = "/docs/intro" +[[redirects]] +from = "/docs/getting-started" +to = "/docs/intro" + [[redirects]] from = "/docs/dlt-ecosystem" to = "/docs/dlt-ecosystem/verified-sources" [[redirects]] from = "/docs/general-usage/credentials/config_providers" -to = "/docs/general-usage/credentials" +to = "/docs/general-usage/credentials/setup" [[redirects]] from = "/docs/general-usage/credentials/configuration" to = "/docs/general-usage/credentials/setup" +[[redirects]] +from = "/docs/general-usage/credentials/config_specs" +to = "/docs/general-usage/credentials/complex_types" +status = 301 + [[redirects]] from = "/docs/tutorial/intro" to = "docs/tutorial/load-data-from-an-api" @@ -25,8 +34,3 @@ to = "docs/tutorial/load-data-from-an-api" [[redirects]] from = "/docs/tutorial/grouping-resources" to = "docs/tutorial/load-data-from-an-api" - -[[redirects]] -from = "/docs/general-usage/credentials/config_specs" -to = "/docs/general-usage/credentials" -status = 301 diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 61e75f2ce5..710c3ac57c 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -64,12 +64,6 @@ const sidebars = { id: 'dlt-ecosystem/verified-sources/index', }, items: [ - 'dlt-ecosystem/verified-sources/airtable', - 'dlt-ecosystem/verified-sources/amazon_kinesis', - 'dlt-ecosystem/verified-sources/arrow-pandas', - 'dlt-ecosystem/verified-sources/asana', - 'dlt-ecosystem/verified-sources/chess', - 'dlt-ecosystem/verified-sources/facebook_ads', { type: 'category', label: 'Filesystem & cloud storage', @@ -83,22 +77,6 @@ const sidebars = { 'dlt-ecosystem/verified-sources/filesystem/advanced', ] }, - 'dlt-ecosystem/verified-sources/freshdesk', - 'dlt-ecosystem/verified-sources/github', - 'dlt-ecosystem/verified-sources/google_ads', - 'dlt-ecosystem/verified-sources/google_analytics', - 'dlt-ecosystem/verified-sources/google_sheets', - 'dlt-ecosystem/verified-sources/hubspot', - 'dlt-ecosystem/verified-sources/inbox', - 'dlt-ecosystem/verified-sources/jira', - 'dlt-ecosystem/verified-sources/kafka', - 'dlt-ecosystem/verified-sources/matomo', - 'dlt-ecosystem/verified-sources/mongodb', - 'dlt-ecosystem/verified-sources/mux', - 'dlt-ecosystem/verified-sources/notion', - 'dlt-ecosystem/verified-sources/personio', - 'dlt-ecosystem/verified-sources/pg_replication', - 'dlt-ecosystem/verified-sources/pipedrive', { type: 'category', label: 'REST APIs', @@ -112,26 +90,48 @@ const sidebars = { 'dlt-ecosystem/verified-sources/rest_api/advanced', ] }, - 'dlt-ecosystem/verified-sources/openapi-generator', - 'dlt-ecosystem/verified-sources/salesforce', - 'dlt-ecosystem/verified-sources/scrapy', - 'dlt-ecosystem/verified-sources/shopify', { type: 'category', label: '30+ SQL Databases', description: 'PostgreSQL, MySQL, MS SQL, BigQuery, Redshift, and more', - link: { + link: { type: 'doc', id: 'dlt-ecosystem/verified-sources/sql_database/index', - }, + }, items: [ 'dlt-ecosystem/verified-sources/sql_database/setup', 'dlt-ecosystem/verified-sources/sql_database/configuration', 'dlt-ecosystem/verified-sources/sql_database/usage', 'dlt-ecosystem/verified-sources/sql_database/troubleshooting', - 'dlt-ecosystem/verified-sources/sql_database/advanced' + 'dlt-ecosystem/verified-sources/sql_database/advanced', ] }, + 'dlt-ecosystem/verified-sources/airtable', + 'dlt-ecosystem/verified-sources/amazon_kinesis', + 'dlt-ecosystem/verified-sources/arrow-pandas', + 'dlt-ecosystem/verified-sources/asana', + 'dlt-ecosystem/verified-sources/chess', + 'dlt-ecosystem/verified-sources/facebook_ads', + 'dlt-ecosystem/verified-sources/freshdesk', + 'dlt-ecosystem/verified-sources/github', + 'dlt-ecosystem/verified-sources/google_ads', + 'dlt-ecosystem/verified-sources/google_analytics', + 'dlt-ecosystem/verified-sources/google_sheets', + 'dlt-ecosystem/verified-sources/hubspot', + 'dlt-ecosystem/verified-sources/inbox', + 'dlt-ecosystem/verified-sources/jira', + 'dlt-ecosystem/verified-sources/kafka', + 'dlt-ecosystem/verified-sources/matomo', + 'dlt-ecosystem/verified-sources/mongodb', + 'dlt-ecosystem/verified-sources/mux', + 'dlt-ecosystem/verified-sources/notion', + 'dlt-ecosystem/verified-sources/personio', + 'dlt-ecosystem/verified-sources/pg_replication', + 'dlt-ecosystem/verified-sources/pipedrive', + 'dlt-ecosystem/verified-sources/openapi-generator', + 'dlt-ecosystem/verified-sources/salesforce', + 'dlt-ecosystem/verified-sources/scrapy', + 'dlt-ecosystem/verified-sources/shopify', 'dlt-ecosystem/verified-sources/slack', 'dlt-ecosystem/verified-sources/strapi', 'dlt-ecosystem/verified-sources/stripe', From f483ee68c8891256a1cb26ea35d9e78c19983f32 Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 17 Sep 2024 11:39:51 +0200 Subject: [PATCH 2/2] re-enable throw on broken links clear cache in the right places in preprocess docs --- docs/website/docusaurus.config.js | 2 +- docs/website/tools/preprocess_docs.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/website/docusaurus.config.js b/docs/website/docusaurus.config.js index 88ec2eb017..662ed0d5d0 100644 --- a/docs/website/docusaurus.config.js +++ b/docs/website/docusaurus.config.js @@ -50,7 +50,7 @@ const config = { tagline: 'data load tool', url: 'https://dlthub.com', baseUrl: process.env.DOCUSAURUS_BASE_URL || '/docs', - onBrokenLinks: 'warn', + onBrokenLinks: 'throw', onBrokenMarkdownLinks: 'throw', favicon: 'img/favicon.ico', staticDirectories: ['public', 'static'], diff --git a/docs/website/tools/preprocess_docs.js b/docs/website/tools/preprocess_docs.js index 8347aff4cb..c5c0c33246 100644 --- a/docs/website/tools/preprocess_docs.js +++ b/docs/website/tools/preprocess_docs.js @@ -358,6 +358,7 @@ function syncExamples() { console.log(`Synced ${count} examples`) } +fs.rmSync(MD_TARGET_DIR, {force: true, recursive: true}) syncExamples(); preprocess_docs();