Merge pull request #77 from skalenetwork/fix-metrics-collector

Update metrics collector timeframe

.github/workflows/test_metrics.yml

@@ -13,6 +13,11 @@ jobs:
     env:
       ETH_ENDPOINT: ${{ secrets.ETH_ENDPOINT }}
       PYTHONPATH: ${{ github.workspace }}/metrics
+      MYSQL_USER: test
+      MYSQL_PASSWORD: test
+      MYSQL_ROOT_PASSWORD: test
+      MYSQL_HOST: 127.0.0.1
+      MYSQL_DATABASE: metrics
     steps:
     - uses: actions/checkout@v4
       with:
@@ -26,5 +31,9 @@ jobs:
       run: pip install -r requirements.txt && pip install -r requirements-dev.txt
     - name: Lint with ruff
       run: ruff check src/
+    - name: Run MySQL container
+      run: docker compose up -d db
+    - name: Prepare database
+      run: python tests/prepare_db.py
     - name: Run metrics tests
       run: pytest tests/

metrics/requirements-dev.txt

@@ -3,4 +3,5 @@ pytest==8.3.3
 Faker==28.4.1
 pytest-aiohttp==1.0.5
 eth-typing==4.0.0
-eth-utils==4.0.0
+eth-utils==4.0.0
+types-peewee==3.17.7.20241017

metrics/src/collector.py

@@ -21,7 +21,7 @@
 import logging
 import asyncio
 import aiohttp
-from datetime import datetime, timedelta
+from datetime import datetime, date, timedelta
 from typing import Tuple, Optional, Dict, List
 from aiohttp import ClientError, ClientSession
 
@@ -62,7 +62,7 @@ def get_empty_address_counter() -> AddressCounter:
         'token_transfers_count': '0',
         'transactions_count': '0',
         'validations_count': '0',
-        'transactions_last_day': 0,
+        'transactions_today': 0,
         'transactions_last_7_days': 0,
         'transactions_last_30_days': 0,
     }
@@ -81,37 +81,42 @@ async def fetch_address_data(
         response.raise_for_status()
 
         current_data: Dict = await response.json()
+        logger.debug(f'Explorer response for {address}: {json.dumps(current_data, indent=2)}')
 
         await update_transaction_counts(chain_name, app_name, address, current_data)
-
-        today = datetime.now().date()
-        yesterday = today - timedelta(days=1)
-        week_ago = today - timedelta(days=7)
-        month_ago = today - timedelta(days=30)
-
-        transactions_last_day = await get_address_transaction_counts(
-            chain_name, app_name, address, yesterday, yesterday
-        )
-        transactions_last_7_days = await get_address_transaction_counts(
-            chain_name, app_name, address, week_ago, yesterday
-        )
-        transactions_last_30_days = await get_address_transaction_counts(
-            chain_name, app_name, address, month_ago, yesterday
-        )
-
-        result: AddressCounter = {
-            'gas_usage_count': str(current_data.get('gas_usage_count', '0')),
-            'token_transfers_count': str(current_data.get('token_transfers_count', '0')),
-            'transactions_count': str(current_data.get('transactions_count', '0')),
-            'validations_count': str(current_data.get('validations_count', '0')),
-            'transactions_last_day': transactions_last_day,
-            'transactions_last_7_days': transactions_last_7_days,
-            'transactions_last_30_days': transactions_last_30_days,
-        }
+        result = await get_db_counts(current_data, chain_name, app_name, address)
         logger.info(f'Fetched data for {address} at {url}: {result}')
         return result
 
 
+async def get_db_counts(
+    current_data: Dict, chain_name: str, app_name: str, address: str
+) -> AddressCounter:
+    today = date.today()
+    tomorrow = today + timedelta(days=1)
+    week_ago = today - timedelta(days=7)
+    month_ago = today - timedelta(days=30)
+
+    transactions_today = await get_address_transaction_counts(
+        chain_name, app_name, address, today, tomorrow
+    )
+    transactions_last_7_days = await get_address_transaction_counts(
+        chain_name, app_name, address, week_ago, today
+    )
+    transactions_last_30_days = await get_address_transaction_counts(
+        chain_name, app_name, address, month_ago, today
+    )
+    return {
+        'gas_usage_count': str(current_data.get('gas_usage_count', '0')),
+        'token_transfers_count': str(current_data.get('token_transfers_count', '0')),
+        'transactions_count': str(current_data.get('transactions_count', '0')),
+        'validations_count': str(current_data.get('validations_count', '0')),
+        'transactions_today': transactions_today,
+        'transactions_last_7_days': transactions_last_7_days,
+        'transactions_last_30_days': transactions_last_30_days,
+    }
+
+
 async def get_address_counters(
     session: ClientSession, network: str, chain_name: str, app_name: str, address: str
 ) -> AddressCounter:

metrics/src/metrics_types.py

@@ -27,7 +27,7 @@ class AddressCounter(TypedDict):
     token_transfers_count: str
     transactions_count: str
     validations_count: str
-    transactions_last_day: int
+    transactions_today: int
     transactions_last_7_days: int
     transactions_last_30_days: int
 

metrics/tests/conftest.py

@@ -1,9 +1,11 @@
 import os
 import json
 import pytest
+from datetime import date
 from faker import Faker
 from aiohttp import web
-
+from unittest.mock import patch
+from tests.prepare_db import load_test_data
 
 fake = Faker()
 
@@ -41,9 +43,11 @@
 }
 
 TEST_NETWORK = 'testnet'
-TEST_CHAIN = 'chain2'
-TEST_ADDRESS = '0x1234'
+TEST_CHAIN = 'test-chain'
+TEST_ADDRESS = '0x1234567890123456789012345678901234567890'
 TEST_APP = 'test-app'
+MOCK_DATE = date(2024, 5, 4)
+TEST_DATA = load_test_data()
 
 
 @pytest.fixture
@@ -61,6 +65,11 @@ def sample_metadata():
     return SAMPLE_METADATA
 
 
+@pytest.fixture
+def sample_counters():
+    return TEST_DATA
+
+
 def load_counters():
     current_dir = os.path.dirname(os.path.abspath(__file__))
     json_file_path = os.path.join(current_dir, 'counters.json')
@@ -86,14 +95,14 @@ def latest_day_counters():
 @pytest.fixture
 def mock_db_data():
     return {
-        'transactions_last_day': 50,
+        'transactions_today': 50,
         'transactions_last_7_days': 300,
         'transactions_last_30_days': 1000,
     }
 
 
 @pytest.fixture
-def mock_address_data():
+def address_data():
     return {
         'gas_usage_count': '16935',
         'token_transfers_count': '174',
@@ -102,6 +111,13 @@ def mock_address_data():
     }
 
 
+@pytest.fixture
+def mock_today():
+    with patch('src.collector.date') as mock_date:
+        mock_date.today.return_value = MOCK_DATE
+        yield mock_date
+
+
 @pytest.fixture
 def mock_chain_stats_data():
     return CHAIN_STATS