feat(stats): Chart migration 7

stats/config/charts.json

@@ -69,6 +69,10 @@
             "title": "Number of deployed contracts today",
             "description": "Number of deployed contracts today"
         },
+        "new_contracts_24h": {
+            "title": "Contracts indexed within last 24h",
+            "description": "(24h)"
+        },
         "total_contracts": {
             "title": "Total contracts",
             "description": "Number of contracts"
@@ -77,6 +81,10 @@
             "title": "Number of verified contracts today",
             "description": "Number of contracts verified today"
         },
+        "new_verified_contracts_24h": {
+            "title": "Number of verified contracts within last 24h",
+            "description": "(24h)"
+        },
         "total_verified_contracts": {
             "title": "Total verified contracts",
             "description": "Number of verified contracts"

stats/config/layout.json

@@ -4,6 +4,8 @@
         "completed_txns",
         "last_new_contracts",
         "last_new_verified_contracts",
+        "new_contracts_24h",
+        "new_verified_contracts_24h",
         "total_accounts",
         "total_addresses",
         "total_blocks",

stats/config/update_groups.json

@@ -33,12 +33,13 @@
         "new_blocks_group": "0 0 8 * * * *",
         "txns_fee_group": "0 0 7 * * * *",
         "txns_success_rate_group": "0 0 19 * * * *",
-        "new_accounts_group": "0 0 5 * * * *",
+        "new_accounts_group": "0 0 4 * * * *",
         "new_contracts_group": "0 20 */3 * * * *",
         "new_txns_group": "0 10 */3 * * * *",
         "new_verified_contracts_group": "0 30 */3 * * * *",
         "native_coin_holders_growth_group": "0 0 7,17,22 * * * *",
         "new_native_coin_transfers_group": "0 0 3,13 * * * *",
-        "txns_stats_24h_group": "0 30 * * * * *"
+        "txns_stats_24h_group": "0 30 * * * * *",
+        "verified_contracts_page_group": "0 15,45 * * * * *"
     }
 }

stats/config/utils/durations/.gitignore

@@ -0,0 +1 @@
+*.csv

stats/config/utils/durations/README.md

@@ -0,0 +1,10 @@
+# Durations config
+
+This is a script folder to ensure an accurate visualization for `find_free_timeslot` script.
+
+## Usage
+
+0. Install dependencies from `../requirements.txt`: `pip install -r ../requirements.txt`
+1. Get data fetch time statistics (e.g. from grafana) (example: `data.csv.example`)
+2. Run the script (preferably from this folder, to correctly use default parameters) (see `--help` for details)
+3. Enjoy newly generated `durations.json` in `find_free_timeslot` script.

stats/config/utils/durations/data.csv.example

@@ -0,0 +1,2 @@
+"Time","accountsGrowth_DAY","accountsGrowth_MONTH","accountsGrowth_WEEK","accountsGrowth_YEAR","activeAccounts_DAY","averageBlockRewards_DAY","averageBlockRewards_MONTH","averageBlockRewards_WEEK","averageBlockRewards_YEAR","averageBlockSize_DAY","averageBlockSize_MONTH","averageBlockSize_WEEK","averageBlockSize_YEAR","averageBlockTime_DAY","averageGasLimit_DAY","averageGasLimit_MONTH","averageGasLimit_WEEK","averageGasLimit_YEAR","averageGasPrice_DAY","averageGasPrice_MONTH","averageGasPrice_WEEK","averageGasPrice_YEAR","averageTxnFee_DAY","averageTxnFee_MONTH","averageTxnFee_WEEK","averageTxnFee_YEAR","completedTxns_DAY","gasUsedGrowth_DAY","gasUsedGrowth_MONTH","gasUsedGrowth_WEEK","gasUsedGrowth_YEAR","lastNewVerifiedContracts_DAY","newAccounts_DAY","newAccounts_MONTH","newAccounts_WEEK","newAccounts_YEAR","newBlockRewards_DAY","newBlocks_DAY","newBlocks_MONTH","newBlocks_WEEK","newBlocks_YEAR","newNativeCoinTransfers_DAY","newNativeCoinTransfers_MONTH","newNativeCoinTransfers_WEEK","newNativeCoinTransfers_YEAR","newTxns_DAY","newTxns_MONTH","newTxns_WEEK","newTxns_YEAR","newVerifiedContracts_DAY","newVerifiedContracts_MONTH","newVerifiedContracts_WEEK","newVerifiedContracts_YEAR","totalAccounts_DAY","totalAddresses_DAY","totalBlocks_DAY","totalNativeCoinTransfers_DAY","totalTokens_DAY","totalTxns_DAY","totalVerifiedContracts_DAY","txnsFee_DAY","txnsFee_MONTH","txnsFee_WEEK","txnsFee_YEAR","txnsGrowth_DAY","txnsGrowth_MONTH","txnsGrowth_WEEK","txnsGrowth_YEAR","txnsSuccessRate_DAY","txnsSuccessRate_MONTH","txnsSuccessRate_WEEK","txnsSuccessRate_YEAR","verifiedContractsGrowth_DAY","verifiedContractsGrowth_MONTH","verifiedContractsGrowth_WEEK","verifiedContractsGrowth_YEAR"
+1735022010000,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34,12.34

stats/config/utils/durations/durations.json

@@ -0,0 +1,42 @@
+{
+    "active_accounts_group": 1,
+    "average_block_time_group": 1,
+    "completed_txns_group": 19,
+    "pending_txns_group": 1,
+    "total_addresses_group": 3,
+    "total_blocks_group": 1,
+    "total_txns_group": 1,
+    "total_operational_txns_group": 1,
+    "total_tokens_group": 1,
+    "yesterday_txns_group": 1,
+    "active_recurring_accounts_daily_recurrence60_days_group": 1,
+    "active_recurring_accounts_monthly_recurrence60_days_group": 1,
+    "active_recurring_accounts_weekly_recurrence60_days_group": 1,
+    "active_recurring_accounts_yearly_recurrence60_days_group": 1,
+    "active_recurring_accounts_daily_recurrence90_days_group": 1,
+    "active_recurring_accounts_monthly_recurrence90_days_group": 1,
+    "active_recurring_accounts_weekly_recurrence90_days_group": 1,
+    "active_recurring_accounts_yearly_recurrence90_days_group": 1,
+    "active_recurring_accounts_daily_recurrence120_days_group": 1,
+    "active_recurring_accounts_monthly_recurrence120_days_group": 1,
+    "active_recurring_accounts_weekly_recurrence120_days_group": 1,
+    "active_recurring_accounts_yearly_recurrence120_days_group": 1,
+    "new_txns_window_group": 1,
+    "average_block_rewards_group": 1,
+    "average_block_size_group": 1,
+    "average_gas_limit_group": 1,
+    "average_gas_price_group": 1,
+    "average_txn_fee_group": 1,
+    "gas_used_growth_group": 1,
+    "native_coin_supply_group": 1,
+    "new_blocks_group": 1,
+    "txns_fee_group": 1,
+    "txns_success_rate_group": 2,
+    "txns_stats24h_group": 1,
+    "new_accounts_group": 102,
+    "new_contracts_group": 1,
+    "new_txns_group": 1,
+    "new_verified_contracts_group": 1,
+    "native_coin_holders_growth_group": 1,
+    "new_native_coin_transfers_group": 1
+}

stats/config/utils/durations/durations_producer.py

@@ -0,0 +1,203 @@
+import pandas as pd
+import re
+import json
+from typing import Dict, List, Set, Optional
+import typer
+from pathlib import Path
+
+
+def convert_camel_to_snake(name: str) -> str:
+    """Convert camelCase to snake_case."""
+    name = re.sub("([a-z0-9])([A-Z])", r"\1_\2", name)
+    return name.lower()
+
+
+def normalize_period(period: str) -> str:
+    """Normalize period names (WEEK -> Weekly, etc)."""
+    period_map = {"WEEK": "Weekly", "MONTH": "Monthly", "YEAR": "Yearly", "DAY": ""}
+    return period_map.get(period, period)
+
+
+def parse_rust_groups(rust_file_path: str) -> Dict[str, List[str]]:
+    """Parse Rust file to extract update groups and their charts."""
+    with open(rust_file_path, "r") as f:
+        content = f.read()
+
+    # Extract singleton groups
+    singleton_pattern = r"singleton_groups!\(([\s\S]*?)\);"
+    singleton_match = re.search(singleton_pattern, content)
+
+    groups = {}
+    if singleton_match:
+        # Extract individual chart names, skipping comments
+        charts = re.findall(
+            r"^\s*([A-Za-z0-9]+),?\s*(?://.*)?$", singleton_match.group(1), re.MULTILINE
+        )
+
+        # Create group names and entries for singleton groups
+        for chart in charts:
+            group_name = f"{chart}Group"
+            groups[group_name] = [chart]
+
+    # Extract complex groups
+    group_pattern = (
+        r"construct_update_group!\((\w+)\s*\{[\s\S]*?charts:\s*\[([\s\S]*?)\]"
+    )
+    complex_groups = re.finditer(group_pattern, content)
+
+    for match in complex_groups:
+        group_name = match.group(1)
+        # Extract chart names, handling possible comments
+        charts = re.findall(r"([A-Za-z0-9]+),", match.group(2))
+        if charts:
+            groups[group_name] = charts
+
+    return groups
+
+
+def process_durations(
+    csv_path: Path, rust_path: Path, output_path: Path, verbose: bool = False
+) -> Dict[str, int]:
+    """Process duration data and create config file."""
+    if verbose:
+        print(f"Reading duration data from {csv_path}")
+
+    # Read first row of CSV
+    df = pd.read_csv(csv_path, nrows=1)
+
+    # Get duration columns (skip 'Time' column)
+    duration_cols = [col for col in df.columns if col != "Time"]
+
+    if verbose:
+        print(f"Found {len(duration_cols)} duration columns")
+
+    # Create mapping of chart names to durations
+    chart_durations = {}
+    for col in duration_cols:
+        # Split column name into chart and period
+        parts = col.split("_")
+        if len(parts) == 2:
+            chart_name, period = parts
+
+            # Convert to camelCase and normalize period if present
+            camel_chart = "".join(
+                word.capitalize()
+                for word in convert_camel_to_snake(chart_name).split("_")
+            )
+            if period in ["WEEK", "MONTH", "YEAR", "DAY"]:
+                camel_chart += normalize_period(period)
+
+            # Store duration (convert to milliseconds and round to nearest minute)
+            duration_mins = round(
+                float(df[col].iloc[0]) / 60
+            )  # assuming duration is in seconds
+            chart_durations[camel_chart] = duration_mins
+
+            if verbose:
+                print(f"Processed chart {camel_chart}: {duration_mins} minutes")
+
+    if verbose:
+        print(f"\nParsing group definitions from {rust_path}")
+
+    # Parse group definitions
+    groups = parse_rust_groups(rust_path)
+
+    if verbose:
+        print(f"Found {len(groups)} update groups")
+
+    # Calculate group durations
+    group_durations = {}
+    for group_name, charts in groups.items():
+        total_duration = 0
+        missing_charts = []
+        matched_charts = []
+
+        for chart in charts:
+            if chart in chart_durations:
+                total_duration += chart_durations[chart]
+                matched_charts.append(chart)
+            else:
+                missing_charts.append(chart)
+
+        # Convert group name to snake_case for consistency with visualizer
+        snake_group = convert_camel_to_snake(group_name)
+        group_durations[snake_group] = max(
+            1, total_duration
+        )  # ensure at least 1 minute
+
+        if verbose:
+            print(f"\nGroup: {snake_group}")
+            print(f"Total duration: {group_durations[snake_group]} minutes")
+            if missing_charts:
+                print(
+                    f"Warning: Missing duration data for charts: {', '.join(missing_charts)}"
+                )
+                if matched_charts:
+                    print(
+                        f"Duration data found for charts: {', '.join(matched_charts)}"
+                    )
+                else:
+                    print(f"No charts in the group had duration data")
+
+    # Save to JSON file
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(output_path, "w") as f:
+        json.dump(group_durations, f, indent=2)
+
+    if verbose:
+        print(f"\nSaved durations configuration to {output_path}")
+
+    return group_durations
+
+
+def main(
+    csv_path: Path = typer.Argument(
+        ...,
+        help="Path to CSV file with duration data",
+        exists=True,
+        dir_okay=False,
+        readable=True,
+    ),
+    rust_path: Path = typer.Option(
+        Path("../../../stats/src/update_groups.rs"),
+        help="Path to Rust file with group definitions",
+        exists=True,
+        dir_okay=False,
+        readable=True,
+    ),
+    output_path: Path = typer.Option(
+        Path("durations.json"),
+        "--output",
+        "-o",
+        help="Path for output JSON file",
+        writable=True,
+    ),
+    verbose: bool = typer.Option(
+        False, "--verbose", "-v", help="Enable verbose output"
+    ),
+    print_durations: bool = typer.Option(
+        False, "--print", "-p", help="Print calculated durations"
+    ),
+):
+    """
+    Process update durations from CSV data and group definitions from Rust code.
+
+    This tool reads duration data from a CSV file and group definitions from a Rust source file,
+    calculates total durations for each update group, and saves the results to a JSON file
+    that can be used by the visualization tool.
+    """
+    try:
+        durations = process_durations(csv_path, rust_path, output_path, verbose)
+
+        if print_durations:
+            print("\nCalculated durations:")
+            for group, duration in sorted(durations.items()):
+                print(f"{group}: {duration} minutes")
+
+    except Exception as e:
+        typer.echo(f"Error: {str(e)}", err=True)
+        raise typer.Exit(code=1)
+
+
+if __name__ == "__main__":
+    typer.run(main)