Merge pull request #8 from ekoepplin/addtoactions

add to actions

.github/workflows/run_newsapi_articles_workflow.yml

@@ -5,6 +5,7 @@ name: Run newsapi_articles pipeline from newsapi_pipeline.py
   workflow_dispatch: null
 env:
   DESTINATION__FILESYSTEM__DATASET_NAME: newsapi
+  NEWSAPI_PIPELINE__DESTINATION__SCHEMA_NAME: "ingest_newsapi_v1"
   DESTINATION__FILESYSTEM__BUCKET_URL: ${{ secrets.BUCKET_URL }}
   NEWSAPI__DESTINATION__ATHENA__QUERY_RESULT_BUCKET: ${{ secrets.ATHENA__QUERY_RESULT_BUCKET }}
   NEWSAPI__DESTINATION__BUCKET_URL: ${{ secrets.NEWSAPI__DESTINATION__BUCKET_URL_PROD }}

newsapi_pipeline.py

@@ -1,5 +1,6 @@
 import argparse
 from datetime import datetime, timedelta
+from pathlib import Path
 
 import dlt
 from loguru import logger  # Import Loguru
@@ -9,6 +10,8 @@
 today = datetime.utcnow().date()
 before_yesterday = today - timedelta(days=2)
 
+target_schema_name: str = dlt.config[f"{Path(__file__).stem}.destination.schema_name"]
+
 
 # Define a resource for fetching articles from the US
 @dlt.resource(table_name="articles_us_en", write_disposition="append")
@@ -139,7 +142,7 @@ def run_pipeline(destination="filesystem", full_refresh=False):
     pipeline = dlt.pipeline(
         pipeline_name="newsapi_articles",
         destination=destination,
-        dataset_name="newsapi_data",
+        dataset_name=target_schema_name,
     )
 
     load_info = pipeline.run(