feat(satellite): add Sentinel-5P L2 products integration

BREAKING CHANGE: add new data fetcher for Sentinel-5P products

feat:
- Add all level 2 products support (O3, CH4, CO, NO2, HCHO, SO2, Cloud, Aerosol)
- Add processing mode options (NRTI, OFFL, RPRO)
- Implement ESA data hub API connection

improvements:
- Add robust error handling
- Add logging system
- Add progress tracking for downloads

app_run.py

@@ -1,17 +1,12 @@
 import logging
-import sys
-import os
-from datetime import datetime
 import tkinter as tk
 from tkinter import messagebox, ttk
 from tkcalendar import DateEntry
 import threading
 
 # 導入你的處理模組
 from src.api.sentinel_api import S5PFetcher
-from src.processing.data_processor import S5Processor
-from src.utils.logger import setup_logging
-from src.config.settings import setup_directory_structure
+from src.config import setup_directory_structure
 
 
 class SatelliteApp:

main.py

@@ -1,27 +1,29 @@
 """主程式"""
 import logging
+from datetime import datetime
 
 from src.api.sentinel_api import S5PFetcher
 from src.processing.data_processor import S5Processor
-from src.utils.logger import setup_logging
-from src.utils.richer import rich_print
-from src.utils.catalog import ProductClassInput, ProductTypeInput, ProductType
-from src.config.settings import setup_directory_structure, FILTER_BOUNDARY
-from src.visualization.gif_nc import animate_data
+
+from src.config.richer import rich_print
+from src.config.catalog import ClassInput, TypeInput, PRODUCT_CONFIGS
+from src.config import setup_directory_structure, FILTER_BOUNDARY
 
 
 logger = logging.getLogger(__name__)
 
 
-def fetch_data(file_class: ProductClassInput,
-               file_type: ProductTypeInput,
-               start_date: str,
-               end_date: str):
+def fetch_data(file_class: ClassInput,
+               file_type: TypeInput,
+               start_date: str | datetime,
+               end_date: str | datetime):
     """下載數據的工作流程"""
     try:
+        rich_print(
+            f"正在獲取 sentinel-5p 衛星數據 ({PRODUCT_CONFIGS[file_type].display_name}) from {start_date} to {end_date} ...")
+
         fetcher = S5PFetcher(max_workers=3)
 
-        rich_print(f"正在獲取 sentinel-5p 衛星數據 ({ProductType[file_type].display_name}) from {start_date} to {end_date} ...")
         products = fetcher.fetch_data(
             file_class=file_class,
             file_type=file_type,
@@ -33,7 +35,8 @@ def fetch_data(file_class: ProductClassInput,
 
         if products:
             if rich_print("是否要下載數據？", confirm=True):
-                rich_print(f"開始下載 sentinel-5p 衛星數據 ({ProductType[file_type].display_name}) from {start_date} to {end_date} ...")
+                rich_print(
+                    f"開始下載 sentinel-5p 衛星數據 ({PRODUCT_CONFIGS[file_type].display_name}) from {start_date} to {end_date} ...")
                 fetcher.parallel_download(products)
                 rich_print("數據下載完成！")
             else:
@@ -47,29 +50,29 @@ def fetch_data(file_class: ProductClassInput,
         logger.error(error_message)
 
 
-def process_data(file_class: ProductClassInput,
-                 file_type: ProductTypeInput,
-                 start_date: str,
-                 end_date: str):
+def process_data(file_class: ClassInput,
+                 file_type: TypeInput,
+                 start_date: str | datetime,
+                 end_date: str | datetime):
     """處理數據的工作流程"""
     try:
         if rich_print("是否要處理數據？", confirm=True):
-            rich_print(f"建立 sentinel-5p 衛星數據 ({ProductType[file_type].display_name}) 處理器 ...")
+            rich_print(
+                f"正在處理 sentinel-5p 衛星數據 ({PRODUCT_CONFIGS[file_type].display_name}) from {start_date} to {end_date} ...")
 
             processor = S5Processor(
                 interpolation_method='kdtree',
                 resolution=0.02,
                 mask_qc_value=0.5
             )
 
-            rich_print(f"正在處理 sentinel-5p 衛星數據 ({ProductType[file_type].display_name}) from {start_date} to {end_date} ...")
-
             processor.process_each_data(
                 file_class=file_class,
                 file_type=file_type,
                 start_date=start_date,
                 end_date=end_date,
-                use_taiwan_mask=True)
+            )
+
             rich_print("數據完成處理")
         else:
             rich_print("已取消處理操作")
@@ -82,20 +85,21 @@ def process_data(file_class: ProductClassInput,
 
 def main():
     # 設定參數
-    start, end = '2024-01-01', '2024-11-30'
+    start, end = '2024-03-01', '2024-03-02'
+    file_class: ClassInput = 'OFFL'
+    file_type: TypeInput = 'NO2___'
 
     # 設定輸入輸出配置
-    setup_logging()
-    setup_directory_structure(file_type='NO2___', start_date=start, end_date=end)
+    setup_directory_structure(file_type=file_type, start_date=start, end_date=end)
 
     # 下載數據
-    # fetch_data(file_class='OFFL', file_type='NO2___', start_date=start, end_date=end)
+    # fetch_data(file_class=file_class, file_type=file_type, start_date=start, end_date=end)
 
-    # 處理數據
-    process_data(file_class='OFFL', file_type='NO2___', start_date=start, end_date=end)
+    # 處理與繪製數據
+    process_data(file_class=file_class, file_type=file_type, start_date=start, end_date=end)
 
     # 動畫
-    # animate_data(file_type='NO2___', start_date=start, end_date=end)
+    # animate_data(file_type=file_type, start_date=start, end_date=end)
 
 
 if __name__ == "__main__":

src/api/sentinel_api.py

@@ -25,8 +25,8 @@
     RAW_DATA_DIR,
     DEFAULT_TIMEOUT
 )
-from src.utils.richer import console, rich_print, DisplayManager
-from src.utils.catalog import ProductClassInput, ProductTypeInput, ProductType
+from src.config.richer import console, rich_print, DisplayManager
+from src.config.catalog import ClassInput, TypeInput, PRODUCT_CONFIGS
 
 
 logger = logging.getLogger(__name__)
@@ -64,8 +64,8 @@ def __init__(self, max_workers: int = 5):
         }
 
     def fetch_data(self,
-                   file_class: ProductClassInput,
-                   file_type: ProductTypeInput,
+                   file_class: ClassInput,
+                   file_type: TypeInput,
                    start_date: str,
                    end_date: str,
                    boundary: tuple[float, float, float, float] | None = None,
@@ -98,7 +98,7 @@ def fetch_data(self,
 
             file_class = '' if file_class == '*' else file_class
             file_type = '' if file_type == '*' else file_type
-            self.file_type = ProductType[file_type]
+            self.file_type = file_type
 
             # 構建基本篩選條件
             base_filter = (

src/config/__init__.py

@@ -0,0 +1,100 @@
+import logging
+from pathlib import Path
+from datetime import datetime
+
+from src.config.settings import RAW_DATA_DIR, PROCESSED_DATA_DIR, FIGURE_DIR, LOGS_DIR, FILTER_BOUNDARY
+from src.config.catalog import TypeInput
+
+
+__all__ = ['setup_directory_structure', 'FILTER_BOUNDARY']
+
+
+def setup_logging():
+    """設置日誌配置"""
+    # 確保日誌目錄存在
+    log_dir = Path(LOGS_DIR)
+    log_dir.mkdir(parents=True, exist_ok=True)
+
+    # 創建日誌檔案路徑
+    log_file = log_dir / f"Satellite_S5P_{datetime.now().strftime('%Y%m')}.log"
+
+    # 配置基本設定
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(name).10s - %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S',
+        handlers=[
+            logging.FileHandler(log_file, encoding='utf-8'),
+            logging.StreamHandler()  # 同時輸出到控制台
+        ]
+    )
+
+
+def setup_directory_structure(file_type: TypeInput,
+                              start_date: str | datetime,
+                              end_date: str | datetime):
+    """確保所有必要的目錄存在"""
+    directories = [RAW_DATA_DIR, PROCESSED_DATA_DIR, FIGURE_DIR, LOGS_DIR]
+    for directory in directories:
+        try:
+            directory.mkdir(parents=True, exist_ok=True)
+        except Exception as e:
+            raise
+
+    setup_logging()
+
+    """依照開始和結束時間設定資料夾結構"""
+    start = datetime.strptime(start_date, '%Y-%m-%d')
+    end = datetime.strptime(end_date, '%Y-%m-%d')
+
+    # 遍歷範圍內的所有月份
+    current_date = start
+    while current_date <= end:
+        product_type = file_type
+        year = current_date.strftime('%Y')
+        month = current_date.strftime('%m')
+
+        # 構建每個月份的 figure、processed 和 raw 路徑
+        for base_dir in [FIGURE_DIR, PROCESSED_DATA_DIR, RAW_DATA_DIR]:
+            month_dir = base_dir / product_type / year / month
+            month_dir.mkdir(parents=True, exist_ok=True)
+
+        # 移動到下個月
+        if month == "12":
+            current_date = current_date.replace(year=current_date.year + 1, month=1, day=1)
+        else:
+            current_date = current_date.replace(month=current_date.month + 1, day=1)
+
+
+""" I/O structure
+Main Folder (Sentinel_data)
+├── figure
+│   ├── NO2
+│   │   ├── 2023
+│   │   │   ├── 01
+│   │   │   └── ...
+│   │   └── 2024
+│   │       ├── 01
+│   │       └── ...
+│   └── ...
+├── logs
+│   └── Satellite_S5P_202411.log
+├── processed
+│   ├── NO2
+│   │   ├── 2023
+│   │   │   ├── 01
+│   │   │   └── ...
+│   │   └── 2024
+│   │       ├── 01
+│   │       └── ...
+│   └── ...
+└── raw
+    ├── NO2
+    │   ├── 2023
+    │   │   ├── 01
+    │   │   └── ...
+    │   └── 2024
+    │       ├── 01
+    │       └── ...
+    └── ...
+"""