refactor(sentinel): restructure program and add Sentinel-5P API integ…

…ration

BREAKING CHANGE: complete program architecture rewrite for better maintainability and extensibility

feat:
- Implement Sentinel-5P API data retrieval
- Add token authentication with auto-refresh mechanism
- Implement data download with retry logic
- Add HDF5 data format support

refactor:
- Modularize program structure, separate data fetching and processing
- Enhance error handling and logging system
- Optimize file reading and data processing workflow
- Add code documentation and comments

technical:
- Use requests Session for connection efficiency
- Implement streaming data download
- Add progress bar visualization
- Improve data quality filtering mechanism

improvements:
- Add auto-installation of dependencies
- Enhance NO2 data visualization
- Add statistical analysis functionality
- Optimize memory usage

.gitattributes

@@ -0,0 +1,2 @@
+*.txt text
+*.png binary

.github/workflows/cleanup.yml

@@ -0,0 +1,103 @@
+# .gitHub/workflows/cleanup.yml
+name: Repository Cleanup
+
+on:
+  workflow_dispatch:
+    inputs:
+      action_type:
+        description: '選擇要執行的操作'
+        required: true
+        type: choice
+        options:
+          - 'Cleanup Workflow'
+          - 'Cleanup Deployments'
+      workflow_status:
+        description: '要清理的工作流程狀態 (僅在選擇 Cleanup Workflow 時需要)'
+        required: false
+        type: choice
+        options:
+          - 'disabled'  # 已停用的工作流程
+          - 'active'    # 活躍的工作流程
+          - 'all'       # 所有工作流程
+      environment:
+        description: '要清理的部署環境 (僅在選擇 Cleanup Deployments 時需要)'
+        required: false
+        type: choice
+        options:
+          - 'all'
+          - 'github-pages'
+          - 'pypi'
+
+jobs:
+  cleanup-workflows:
+    if: ${{ github.event.inputs.action_type == 'Cleanup Workflow' }}
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+    steps:
+      - name: Cleanup workflows
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const status = '${{ github.event.inputs.workflow_status }}';
+            console.log(`Cleaning up workflows with status: ${status}`);
+            
+            // 獲取所有工作流程
+            const workflows = await github.rest.actions.listRepoWorkflows({
+              owner: context.repo.owner,
+              repo: context.repo.repo
+            });
+            
+            for (const workflow of workflows.data.workflows) {
+              // 根據選擇的狀態過濾工作流程
+              if (status === 'all' || 
+                 (status === 'disabled' && !workflow.state === 'active') ||
+                 (status === 'active' && workflow.state === 'active')) {
+            
+                console.log(`Processing workflow: ${workflow.name} (${workflow.state})`);
+            
+                // 獲取此工作流程的所有運行
+                const runs = await github.rest.actions.listWorkflowRuns({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  workflow_id: workflow.id,
+                });
+            
+                // 刪除運行
+                console.log(`Found ${runs.data.total_count} runs to delete`);
+                for (const run of runs.data.workflow_runs) {
+                  console.log(`Deleting run #${run.run_number} of ${workflow.name}`);
+                  await github.rest.actions.deleteWorkflowRun({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    run_id: run.id
+                  });
+                }
+              }
+            }
+            console.log('Cleanup completed');
+
+  cleanup-deployments:
+    if: ${{ github.event.inputs.action_type == 'Cleanup Deployments' }}
+    runs-on: ubuntu-latest
+    permissions:
+      deployments: write
+      actions: write
+      contents: write
+    steps:
+      - name: Delete github-pages deployments
+        if: ${{ github.event.inputs.environment == 'github-pages' || github.event.inputs.environment == 'all' }}
+        uses: strumwolf/delete-deployment-environment@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          environment: github-pages
+          onlyRemoveDeployments: true
+
+      - name: Delete pypi deployments
+        if: ${{ github.event.inputs.environment == 'pypi' || github.event.inputs.environment == 'all' }}
+        uses: strumwolf/delete-deployment-environment@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          environment: pypi
+          onlyRemoveDeployments: true

.github/workflows/pytest.yml

@@ -0,0 +1,49 @@
+name: Python Tests
+
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python-version: [ "3.11", "3.12" ]
+        os: [ ubuntu-latest ]
+
+      fail-fast: false
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.XX
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'  # 啟用 pip 緩存加速安裝
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .
+          pip install -e ".[test]"
+
+      - name: Run tests with coverage
+        run: |
+          pytest tests/ -m "not requires_data" \
+          --cov=AeroViz \
+                --cov-report=term-missing \
+                --cov-report=xml \
+                -v
+
+      - name: Upload coverage reports
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report-${{ matrix.python-version }}-${{ github.sha }}
+          path: coverage.xml
+          if-no-files-found: error

.gitignore

@@ -0,0 +1,165 @@
+
+# MacOX product
+.DS_store
+temp/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Chih-Yu Chan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,31 @@
+# Satellite_S5P
+### NO2 Data Processing Script
+
+This Python script is designed to process and analyze nitrogen dioxide (NO2) data from satellite observations. Here's a breakdown of its main components:
+
+### Data Extraction:
+The script uses the extract_data function to filter and extract relevant NO2 data from a given dataset based on predefined geographical conditions (latitude and longitude ranges) and a quality threshold (qa_value).
+
+### Data Interpolation:
+The interp_data function utilizes a KD-tree algorithm to find the nearest neighbors in a 2D array of latitude and longitude. It then performs linear interpolation using the map_coordinates function to estimate NO2 values at specified target coordinates.
+
+### Main Loop:
+
+The script iterates through folders corresponding to different months of a specified year, each containing NetCDF files of satellite observations.
+For each NetCDF file, it extracts and interpolates NO2 data and stores the results.
+
+### Data Aggregation:
+The script aggregates the processed NO2 data from multiple files within a month, calculating the average.
+
+### Output:
+The processed NO2 data is stored in NetCDF format, with each file representing a month of observations.
+
+### Folder Skipping:
+The script checks if the output NetCDF file already exists. If it does, it skips processing that particular folder to avoid unnecessary computations.
+
+### Usage:
+The script is designed to be run for a specific year, processing monthly satellite observations of NO2 data and generating NetCDF files for further analysis or visualization.
+
+### Dependencies:
+The script relies on various Python libraries, including NumPy, xarray, scipy, and netCDF4, for efficient data manipulation and processing.
+This script is particularly useful for researchers or analysts working with satellite-derived NO2 data, allowing them to extract, interpolate, and aggregate information for further study.

data/shapefiles/stations/空氣品質監測站位置圖_121_10704.prj

@@ -0,0 +1 @@
+PROJCS["TWD_1997_TM_Taiwan",GEOGCS["GCS_TWD_1997",DATUM["D_TWD_1997",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",250000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",121.0],PARAMETER["Scale_Factor",0.9999],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]]

data/shapefiles/stations/空氣品質監測站位置圖_121_10704.shp.xml

@@ -0,0 +1 @@
+<metadata xml:lang="zh"><Esri><CreaDate>20180425</CreaDate><CreaTime>13123900</CreaTime><ArcGISFormat>1.0</ArcGISFormat><SyncOnce>FALSE</SyncOnce><DataProperties><itemProps><itemName Sync="TRUE">空氣品質監測站位置圖_121_10704</itemName><imsContentType Sync="TRUE">002</imsContentType><itemLocation><linkage Sync="TRUE">file://\\TCGIS\D$\ExportSDE\1070425\shp\空氣品質監測站位置圖_121_10704.shp</linkage><protocol Sync="TRUE">Local Area Network</protocol></itemLocation><itemSize Sync="TRUE">0.000</itemSize></itemProps><coordRef><type Sync="TRUE">Projected</type><geogcsn Sync="TRUE">GCS_TWD_1997</geogcsn><csUnits Sync="TRUE">Linear Unit: Meter (1.000000)</csUnits><projcsn Sync="TRUE">TWD_1997_TM_Taiwan</projcsn><peXml Sync="TRUE">&lt;ProjectedCoordinateSystem xsi:type='typens:ProjectedCoordinateSystem' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns:xs='http://www.w3.org/2001/XMLSchema' xmlns:typens='http://www.esri.com/schemas/ArcGIS/10.0'&gt;&lt;WKT&gt;PROJCS[&amp;quot;TWD_1997_TM_Taiwan&amp;quot;,GEOGCS[&amp;quot;GCS_TWD_1997&amp;quot;,DATUM[&amp;quot;D_TWD_1997&amp;quot;,SPHEROID[&amp;quot;GRS_1980&amp;quot;,6378137.0,298.257222101]],PRIMEM[&amp;quot;Greenwich&amp;quot;,0.0],UNIT[&amp;quot;Degree&amp;quot;,0.0174532925199433]],PROJECTION[&amp;quot;Transverse_Mercator&amp;quot;],PARAMETER[&amp;quot;False_Easting&amp;quot;,250000.0],PARAMETER[&amp;quot;False_Northing&amp;quot;,0.0],PARAMETER[&amp;quot;Central_Meridian&amp;quot;,121.0],PARAMETER[&amp;quot;Scale_Factor&amp;quot;,0.9999],PARAMETER[&amp;quot;Latitude_Of_Origin&amp;quot;,0.0],UNIT[&amp;quot;Meter&amp;quot;,1.0],AUTHORITY[&amp;quot;ESRI&amp;quot;,102443]]&lt;/WKT&gt;&lt;XOrigin&gt;-5372600&lt;/XOrigin&gt;&lt;YOrigin&gt;-10001100&lt;/YOrigin&gt;&lt;XYScale&gt;450310428.58990461&lt;/XYScale&gt;&lt;ZOrigin&gt;-100000&lt;/ZOrigin&gt;&lt;ZScale&gt;10000&lt;/ZScale&gt;&lt;MOrigin&gt;-100000&lt;/MOrigin&gt;&lt;MScale&gt;10000&lt;/MScale&gt;&lt;XYTolerance&gt;0.001&lt;/XYTolerance&gt;&lt;ZTolerance&gt;0.001&lt;/ZTolerance&gt;&lt;MTolerance&gt;0.001&lt;/MTolerance&gt;&lt;HighPrecision&gt;true&lt;/HighPrecision&gt;&lt;WKID&gt;102443&lt;/WKID&gt;&lt;/ProjectedCoordinateSystem&gt;</peXml></coordRef><lineage><Process ToolSource="C:\Program Files (x86)\ArcGIS\Desktop10.0\ArcToolbox\Toolboxes\Data Management Tools.tbx\CopyFeatures" Date="20180426" Time="151053">CopyFeatures "Database Connections\sde@dc@[email protected]\epavec10_101.SDE.AIR_121\epavec10_101.SDE.空氣品質監測站位置圖_121_10704" D:\ExportSDE\1070425\shp\空氣品質監測站位置圖_121_10704.shp # 0 0 0</Process></lineage></DataProperties><SyncDate>20180426</SyncDate><SyncTime>15105300</SyncTime><ModDate>20180426</ModDate><ModTime>15105300</ModTime></Esri><dataIdInfo><envirDesc Sync="TRUE">Microsoft Windows Server 2008 R2 Version 6.1 (Build 7601) Service Pack 1; ESRI ArcGIS 10.0.0.2414</envirDesc><dataLang><languageCode value="zho" Sync="TRUE" country="TWN"/><countryCode value="TWN" Sync="TRUE"/></dataLang><idCitation><resTitle Sync="TRUE">空氣品質監測站位置圖_121_10704</resTitle><presForm><PresFormCd value="005" Sync="TRUE"/></presForm></idCitation><spatRpType><SpatRepTypCd value="001" Sync="TRUE"/></spatRpType><descKeys><thesaName uuidref="723f6998-058e-11dc-8314-0800200c9a66"></thesaName><keyword Sync="TRUE">002</keyword></descKeys></dataIdInfo><mdLang><languageCode value="zho" Sync="TRUE"/><countryCode value="TWN" Sync="TRUE"/></mdLang><distInfo><distFormat><formatName Sync="TRUE">Shapefile</formatName></distFormat><distTranOps><transSize Sync="TRUE">0.000</transSize></distTranOps></distInfo><mdHrLv><ScopeCd value="005" Sync="TRUE"/></mdHrLv><mdHrLvName Sync="TRUE">dataset</mdHrLvName><refSysInfo><RefSystem><refSysID><identCode code="102443" Sync="TRUE"/><idCodeSpace Sync="TRUE">ESRI</idCodeSpace><idVersion Sync="TRUE">10.0.0</idVersion></refSysID></RefSystem></refSysInfo><spatRepInfo><VectSpatRep><geometObjs Name="epavec10_101.SDE.空氣品質監測站位置圖_121_10704"><geoObjTyp><GeoObjTypCd value="004" Sync="TRUE"/><geoObjCnt Sync="TRUE">0</geoObjCnt></geoObjTyp><geoObjCnt Sync="TRUE">0</geoObjCnt></geometObjs><topLvl><TopoLevCd value="001" Sync="TRUE"/></topLvl></VectSpatRep></spatRepInfo><spdoinfo><ptvctinf><esriterm Name="空氣品質監測站位置圖_121_10704"><efeatyp Sync="TRUE">Simple</efeatyp><efeageom Sync="TRUE">1</efeageom><esritopo Sync="TRUE">FALSE</esritopo><efeacnt Sync="TRUE">0</efeacnt><spindex Sync="TRUE">FALSE</spindex><linrefer Sync="TRUE">FALSE</linrefer></esriterm></ptvctinf></spdoinfo><eainfo><detailed Name="空氣品質監測站位置圖_121_10704"><enttyp><enttypl Sync="TRUE">空氣品質監測站位置圖_121_10704</enttypl><enttypt Sync="TRUE">Feature Class</enttypt><enttypc Sync="TRUE">0</enttypc></enttyp><attr><attrlabl Sync="TRUE">FID</attrlabl><attalias Sync="TRUE">FID</attalias><attrtype Sync="TRUE">OID</attrtype><attwidth Sync="TRUE">4</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale><attrdef Sync="TRUE">Internal feature number.</attrdef><attrdefs Sync="TRUE">ESRI</attrdefs><attrdomv><udom Sync="TRUE">Sequential unique whole numbers that are automatically generated.</udom></attrdomv></attr><attr><attrlabl Sync="TRUE">SiteName</attrlabl><attalias Sync="TRUE">SiteName</attalias><attrtype Sync="TRUE">String</attrtype><attwidth Sync="TRUE">254</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale></attr><attr><attrlabl Sync="TRUE">SiteEngNam</attrlabl><attalias Sync="TRUE">SiteEngNam</attalias><attrtype Sync="TRUE">String</attrtype><attwidth Sync="TRUE">254</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale></attr><attr><attrlabl Sync="TRUE">AreaName</attrlabl><attalias Sync="TRUE">AreaName</attalias><attrtype Sync="TRUE">String</attrtype><attwidth Sync="TRUE">254</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale></attr><attr><attrlabl Sync="TRUE">County</attrlabl><attalias Sync="TRUE">County</attalias><attrtype Sync="TRUE">String</attrtype><attwidth Sync="TRUE">254</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale></attr><attr><attrlabl Sync="TRUE">Township</attrlabl><attalias Sync="TRUE">Township</attalias><attrtype Sync="TRUE">String</attrtype><attwidth Sync="TRUE">254</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale></attr><attr><attrlabl Sync="TRUE">SiteAddres</attrlabl><attalias Sync="TRUE">SiteAddres</attalias><attrtype Sync="TRUE">String</attrtype><attwidth Sync="TRUE">254</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale></attr><attr><attrlabl Sync="TRUE">TWD97Lon</attrlabl><attalias Sync="TRUE">TWD97Lon</attalias><attrtype Sync="TRUE">Double</attrtype><attwidth Sync="TRUE">19</attwidth><atprecis Sync="TRUE">18</atprecis><attscale Sync="TRUE">11</attscale></attr><attr><attrlabl Sync="TRUE">TWD97Lat</attrlabl><attalias Sync="TRUE">TWD97Lat</attalias><attrtype Sync="TRUE">Double</attrtype><attwidth Sync="TRUE">19</attwidth><atprecis Sync="TRUE">18</atprecis><attscale Sync="TRUE">11</attscale></attr><attr><attrlabl Sync="TRUE">SiteType</attrlabl><attalias Sync="TRUE">SiteType</attalias><attrtype Sync="TRUE">String</attrtype><attwidth Sync="TRUE">254</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale></attr><attr><attrlabl Sync="TRUE">Shape</attrlabl><attalias Sync="TRUE">Shape</attalias><attrtype Sync="TRUE">Geometry</attrtype><attwidth Sync="TRUE">0</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale><attrdef Sync="TRUE">Feature geometry.</attrdef><attrdefs Sync="TRUE">Esri</attrdefs><attrdomv><udom Sync="TRUE">Coordinates defining the features.</udom></attrdomv></attr></detailed></eainfo><mdDateSt Sync="TRUE">20180426</mdDateSt></metadata>

data/shapefiles/taiwan/COUNTY_MOI_1090820.CPG

@@ -0,0 +1 @@
+UTF-8

data/shapefiles/taiwan/COUNTY_MOI_1090820.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_TWD_1997",DATUM["D_TWD_1997",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]]