Merge pull request #268 from hotosm/feature/metrics

Feature : Download Metrics for rawdataapi

API/download_metrics.py

@@ -0,0 +1,62 @@
+# Standard library imports
+from datetime import datetime
+
+# Third party imports
+from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi_versioning import version
+
+# Reader imports
+from src.app import DownloadMetrics
+
+from .auth import admin_required
+
+router = APIRouter(prefix="/metrics", tags=["Metrics"])
+
+
+@router.get("/summary")
+@version(1)
+def get_stats(
+    start_date: str = Query(
+        ...,
+        description="Start date (YYYY-MM-DD)",
+        regex=r"^\d{4}-\d{2}-\d{2}$",
+        example="2023-04-01",
+    ),
+    end_date: str = Query(
+        ...,
+        description="End date (YYYY-MM-DD)",
+        regex=r"^\d{4}-\d{2}-\d{2}$",
+        example="2023-04-30",
+    ),
+    group_by: str = Query(
+        "day",
+        description="Group by: day, month, or quarter",
+        regex=r"^(day|month|quarter|year)$",
+    ),
+    _: bool = Depends(admin_required),
+):
+    """
+    Retrieve download metrics summary statistics.
+
+    - **start_date**: The start date for the metrics, in the format "YYYY-MM-DD".
+    - **end_date**: The end date for the metrics, in the format "YYYY-MM-DD".
+    - **group_by**: The time period to group the metrics by. Can be "day", "month", "quarter", or "year".
+
+    The API requires admin authentication to access.
+    """
+    if group_by not in ["day", "month", "quarter", "year"]:
+        raise HTTPException(
+            status_code=400, detail={"error": "Invalid group_by parameter"}
+        )
+
+    try:
+        start_date_obj = datetime.strptime(start_date, "%Y-%m-%d")
+        end_date_obj = datetime.strptime(end_date, "%Y-%m-%d")
+    except ValueError:
+        raise HTTPException(
+            status_code=400,
+            detail={"error": "Invalid date format, expected YYYY-MM-DD"},
+        )
+
+    metrics = DownloadMetrics()
+    return metrics.get_summary_stats(start_date, end_date, group_by)

API/main.py

@@ -32,6 +32,7 @@
 from src.config import (
     ENABLE_CUSTOM_EXPORTS,
     ENABLE_HDX_EXPORTS,
+    ENABLE_METRICS_APIS,
     ENABLE_POLYGON_STATISTICS_ENDPOINTS,
     EXPORT_PATH,
     LIMITER,
@@ -56,6 +57,9 @@
 if ENABLE_POLYGON_STATISTICS_ENDPOINTS:
     from .stats import router as stats_router
 
+if ENABLE_METRICS_APIS:
+    from .download_metrics import router as metrics_router
+
 if ENABLE_HDX_EXPORTS:
     from .hdx import router as hdx_router
 
@@ -89,6 +93,8 @@
     app.include_router(custom_exports_router)
 if ENABLE_POLYGON_STATISTICS_ENDPOINTS:
     app.include_router(stats_router)
+if ENABLE_METRICS_APIS:
+    app.include_router(metrics_router)
 if ENABLE_HDX_EXPORTS:
     app.include_router(hdx_router)
 

docs/src/installation/configurations.md

@@ -81,6 +81,7 @@ The following are the different configuration options that are accepted.
 | `SENTRY_DSN` | `SENTRY_DSN` | `[SENTRY]` | _none_ | Sentry Data Source Name | OPTIONAL |
 | `SENTRY_RATE` | `SENTRY_RATE` | `[SENTRY]` | `1.0` | Sample rate percentage for shipping errors to sentry; Allowed values between 0 (0%) to 1 (100%)| OPTIONAL |
 | `ENABLE_HDX_EXPORTS` | `ENABLE_HDX_EXPORTS` | `[HDX]` | False | Enables hdx related endpoints and imports | OPTIONAL |
+| `ENABLE_METRICS_APIS` | `ENABLE_METRICS_APIS` | `[API_CONFIG]` | False | Enables download metrics related endpoints , Require different setup of metrics populator | OPTIONAL |
 | `HDX_SITE` | `HDX_SITE` | `[HDX]` | 'demo' | HDX site to point , By default demo site , use prod for production | CONDITIONAL |
 | `HDX_API_KEY` | `HDX_API_KEY` | `[HDX]` | None | Your API Secret key for hdx upload , should have write access and it is compulsory if ENABLE_HDX_EXPORTS is True | CONDITIONAL |
 | `HDX_OWNER_ORG` | `HDX_OWNER_ORG` | `[HDX]` | None | Your HDX organization ID| CONDITIONAL |
@@ -138,6 +139,7 @@ API Tokens have expiry date, It is `important to update API Tokens manually each
 | `DUCK_DB_MEMORY_LIMIT` | `[API_CONFIG]` | Yes | Yes |
 | `DUCK_DB_THREAD_LIMIT` | `[API_CONFIG]` | Yes | Yes |
 | `ENABLE_CUSTOM_EXPORTS` | `[API_CONFIG]` | Yes | Yes |
+| `ENABLE_METRICS_APIS` | `[API_CONFIG]` | Yes | No |
 | `CELERY_BROKER_URL` | `[CELERY]` | Yes | Yes |
 | `CELERY_RESULT_BACKEND` | `[CELERY]` | Yes | Yes |
 | `WORKER_PREFETCH_MULTIPLIER` | `[CELERY]` | Yes | Yes |

src/app.py

@@ -2201,3 +2201,43 @@ def delete_hdx(self, hdx_id: int):
         if result:
             return dict(result[0])
         raise HTTPException(status_code=404, detail="HDX item not found")
+
+
+class DownloadMetrics:
+    def __init__(self) -> None:
+        """
+        Initializes an instance of the DownloadMetrics class, connecting to the database.
+        """
+        dbdict = get_db_connection_params()
+        self.d_b = Database(dbdict)
+        self.con, self.cur = self.d_b.connect()
+
+    def get_summary_stats(self, start_date, end_date, group_by):
+        """
+        Get summary metrics for raw-data-api downlaods
+        """
+
+        select_query = f"""
+            SELECT
+                date_trunc('{group_by}', date) as kwdate,
+                SUM((summary->>'downloads_count')::numeric) as total_downloads_count,
+                SUM((summary->>'uploads_count')::numeric) as total_uploads_count,
+                SUM((summary->>'unique_users')::numeric) as total_unique_users,
+                SUM((summary->>'unique_downloads')::numeric) as total_unique_downloads,
+                SUM((summary->>'interactions_count')::numeric) as total_interactions_count,
+                SUM((summary->>'upload_size')::numeric) as total_upload_size,
+                SUM((summary->>'download_size')::numeric) as total_download_size
+            FROM
+                metrics
+            WHERE
+                date BETWEEN '{start_date}' AND '{end_date}'
+            GROUP BY
+                kwdate
+            ORDER BY
+                kwdate
+        """
+
+        self.cur.execute(select_query)
+        result = self.cur.fetchall()
+        self.d_b.close_conn()
+        return [dict(item) for item in result]

src/config.py

@@ -281,6 +281,12 @@ def not_raises(func, *args, **kwargs):
 )
 
 
+ENABLE_METRICS_APIS = get_bool_env_var(
+    "ENABLE_METRICS_APIS",
+    config.getboolean("API_CONFIG", "ENABLE_METRICS_APIS", fallback=False),
+)
+
+
 if ENABLE_HDX_EXPORTS:
     try:
         hdx_credentials = os.environ["REMOTE_HDX"]