Skip to content

Commit

Permalink
Updated
Browse files Browse the repository at this point in the history
  • Loading branch information
dat-a-man committed Feb 5, 2024
1 parent 8196f2f commit a310e95
Showing 1 changed file with 6 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@ Let's create a sample pipeline demonstrating the process of removing a column.
```python
import dlt

# This function creates a dummy data source.
@dlt.source
def dummy_source():
# This function creates a dummy data source.
@dlt.resource(write_disposition='replace')
@dlt.resource(write_disposition="replace")
def dummy_data():
for i in range(3):
yield {"id": i, "name": f"Jane Washington {i}", "country_code": 40 + i}

for i in range(3):
yield {'id': i, 'name': f'Jane Washington {i}', 'country_code': 40 + i}
return dummy_data()
```
This function creates three columns `id`, `name` and `country_code`.
Expand All @@ -35,7 +35,6 @@ Let's create a sample pipeline demonstrating the process of removing a column.
from typing import Dict, List, Optional

def remove_columns(doc: Dict, remove_columns: Optional[List[str]] = None) -> Dict:

if remove_columns is None:
remove_columns = []

Expand All @@ -62,10 +61,8 @@ Let's create a sample pipeline demonstrating the process of removing a column.
data_source = dummy_source()

# Modify this source instance's resource
data_source = (
data_source.dummy_data.add_map(
lambda doc: remove_columns(doc, remove_columns_list)
)
data_source = data_source.dummy_data.add_map(
lambda doc: remove_columns(doc, remove_columns_list)
)
```
1. You can optionally inspect the result:
Expand All @@ -87,7 +84,6 @@ Let's create a sample pipeline demonstrating the process of removing a column.
destination='bigquery',
dataset_name='filtered_data'
)

# Run the pipeline with the transformed source
load_info = pipeline.run(data_source)
print(load_info)
Expand Down

0 comments on commit a310e95

Please sign in to comment.