Merge branch 'main' into complete-call-request

crawlers/deploy/deploy-monitoring.bash

@@ -39,7 +39,7 @@ chown ubuntu:ubuntu "/home/ubuntu/monitoring"
 echo
 echo
 echo -e "${PREFIX_INFO} Retrieving monitoring deployment parameters"
-AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+crawlers:true,monitoring:true > "${PARAMETERS_ENV_MONITORING_PATH}"
+AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+service:true,monitoring:true > "${PARAMETERS_ENV_MONITORING_PATH}"
 chmod 0640 "${PARAMETERS_ENV_MONITORING_PATH}"
 
 echo
@@ -49,7 +49,7 @@ echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_MONITORIN
 
 echo
 echo
-echo -e "${PREFIX_INFO} Add AWS default region to monitring parameters"
+echo -e "${PREFIX_INFO} Add AWS default region to monitoring parameters"
 echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> "${PARAMETERS_ENV_MONITORING_PATH}"
 
 echo

crawlers/deploy/deploy.bash

@@ -47,17 +47,13 @@ ETHEREUM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="ethereum-historical-crawl-events.ti
 POLYGON_SYNCHRONIZE_SERVICE="polygon-synchronize.service"
 POLYGON_MISSING_SERVICE_FILE="polygon-missing.service"
 POLYGON_MISSING_TIMER_FILE="polygon-missing.timer"
-POLYGON_STATISTICS_SERVICE_FILE="polygon-statistics.service"
-POLYGON_STATISTICS_TIMER_FILE="polygon-statistics.timer"
 POLYGON_MOONWORM_CRAWLER_SERVICE_FILE="polygon-moonworm-crawler.service"
 POLYGON_STATE_SERVICE_FILE="polygon-state.service"
 POLYGON_STATE_TIMER_FILE="polygon-state.timer"
 POLYGON_STATE_CLEAN_SERVICE_FILE="polygon-state-clean.service"
 POLYGON_STATE_CLEAN_TIMER_FILE="polygon-state-clean.timer"
 POLYGON_METADATA_SERVICE_FILE="polygon-metadata.service"
 POLYGON_METADATA_TIMER_FILE="polygon-metadata.timer"
-POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE="polygon-cu-reports-tokenonomics.service"
-POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE="polygon-cu-reports-tokenonomics.timer"
 POLYGON_CU_NFT_DASHBOARD_SERVICE_FILE="polygon-cu-nft-dashboard.service"
 POLYGON_CU_NFT_DASHBOARD_TIMER_FILE="polygon-cu-nft-dashboard.timer"
 POLYGON_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="polygon-historical-crawl-transactions.service"
@@ -86,8 +82,6 @@ MUMBAI_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="mumbai-historical-crawl-events.timer"
 XDAI_SYNCHRONIZE_SERVICE="xdai-synchronize.service"
 XDAI_MISSING_SERVICE_FILE="xdai-missing.service"
 XDAI_MISSING_TIMER_FILE="xdai-missing.timer"
-XDAI_STATISTICS_SERVICE_FILE="xdai-statistics.service"
-XDAI_STATISTICS_TIMER_FILE="xdai-statistics.timer"
 XDAI_MOONWORM_CRAWLER_SERVICE_FILE="xdai-moonworm-crawler.service"
 XDai_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="xdai-historical-crawl-transactions.service"
 XDai_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="xdai-historical-crawl-transactions.timer"
@@ -98,8 +92,6 @@ XDai_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="xdai-historical-crawl-events.timer"
 WYRM_SYNCHRONIZE_SERVICE="wyrm-synchronize.service"
 WYRM_MISSING_SERVICE_FILE="wyrm-missing.service"
 WYRM_MISSING_TIMER_FILE="wyrm-missing.timer"
-WYRM_STATISTICS_SERVICE_FILE="wyrm-statistics.service"
-WYRM_STATISTICS_TIMER_FILE="wyrm-statistics.timer"
 WYRM_MOONWORM_CRAWLER_SERVICE_FILE="wyrm-moonworm-crawler.service"
 WYRM_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="wyrm-historical-crawl-transactions.service"
 WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="wyrm-historical-crawl-transactions.timer"
@@ -115,6 +107,10 @@ ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="zksync-era-historical-cra
 ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="zksync-era-historical-crawl-transactions.timer"
 ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="zksync-era-historical-crawl-events.service"
 ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-historical-crawl-events.timer"
+ZKSYNC_ERA_STATE_SERVICE_FILE="zksync-era-state.service"
+ZKSYNC_ERA_STATE_TIMER_FILE="zksync-era-state.timer"
+ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE="zksync-era-state-clean.service"
+ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE="zksync-era-state-clean.timer"
 
 # ZkSync Era testnet
 ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE="zksync-era-testnet-synchronize.service"
@@ -252,14 +248,6 @@ cp "${SCRIPT_DIR}/${POLYGON_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/u
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_MISSING_TIMER_FILE}"
 
-echo
-echo
-echo -e "${PREFIX_INFO} Replacing existing Polygon statistics dashbord service and timer with: ${POLYGON_STATISTICS_SERVICE_FILE}, ${POLYGON_STATISTICS_TIMER_FILE}"
-chmod 644 "${SCRIPT_DIR}/${POLYGON_STATISTICS_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYGON_STATISTICS_TIMER_FILE}"
-cp "${SCRIPT_DIR}/${POLYGON_STATISTICS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_STATISTICS_SERVICE_FILE}"
-cp "${SCRIPT_DIR}/${POLYGON_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_STATISTICS_TIMER_FILE}"
-XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
-XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_STATISTICS_TIMER_FILE}"
 
 echo
 echo
@@ -296,14 +284,6 @@ cp "${SCRIPT_DIR}/${POLYGON_METADATA_TIMER_FILE}" "/home/ubuntu/.config/systemd/
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_METADATA_TIMER_FILE}"
 
-echo
-echo
-echo -e "${PREFIX_INFO} Replacing existing Polygon CU reports tokenonomics service and timer with: ${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}, ${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}"
-chmod 644 "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}"
-cp "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}"
-cp "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}"
-XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
-XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}"
 
 echo
 echo
@@ -419,14 +399,6 @@ cp "${SCRIPT_DIR}/${XDAI_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XDAI_MISSING_TIMER_FILE}"
 
-echo
-echo
-echo -e "${PREFIX_INFO} Replacing existing XDai statistics dashbord service and timer with: ${XDAI_STATISTICS_SERVICE_FILE}, ${XDAI_STATISTICS_TIMER_FILE}"
-chmod 644 "${SCRIPT_DIR}/${XDAI_STATISTICS_SERVICE_FILE}" "${SCRIPT_DIR}/${XDAI_STATISTICS_TIMER_FILE}"
-cp "${SCRIPT_DIR}/${XDAI_STATISTICS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XDAI_STATISTICS_SERVICE_FILE}"
-cp "${SCRIPT_DIR}/${XDAI_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XDAI_STATISTICS_TIMER_FILE}"
-XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
-XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XDAI_STATISTICS_TIMER_FILE}"
 
 echo
 echo
@@ -471,15 +443,6 @@ cp "${SCRIPT_DIR}/${WYRM_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_MISSING_TIMER_FILE}"
 
-echo
-echo
-echo -e "${PREFIX_INFO} Replacing existing Wyrm statistics dashbord service and timer with: ${WYRM_STATISTICS_SERVICE_FILE}, ${WYRM_STATISTICS_TIMER_FILE}"
-chmod 644 "${SCRIPT_DIR}/${WYRM_STATISTICS_SERVICE_FILE}" "${SCRIPT_DIR}/${WYRM_STATISTICS_TIMER_FILE}"
-cp "${SCRIPT_DIR}/${WYRM_STATISTICS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_STATISTICS_SERVICE_FILE}"
-cp "${SCRIPT_DIR}/${WYRM_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_STATISTICS_TIMER_FILE}"
-XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
-XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_STATISTICS_TIMER_FILE}"
-
 echo
 echo
 echo -e "${PREFIX_INFO} Replacing existing Wyrm moonworm crawler service definition with ${WYRM_MOONWORM_CRAWLER_SERVICE_FILE}"
@@ -550,6 +513,23 @@ cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubunt
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
 
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing ZkSync Era state service and timer with: ${ZKSYNC_ERA_STATE_SERVICE_FILE}, ${ZKSYNC_ERA_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000"  systemctl --user restart --no-block "${ZKSYNC_ERA_STATE_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing ZkSync Era state clean service and timer with: ${ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE}, ${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000"  systemctl --user restart --no-block "${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}"
 
 # ZkSync Era testnet
 echo

crawlers/deploy/wyrm-statistics.service → crawlers/deploy/zksync-era-state-clean.service

@@ -1,11 +1,11 @@
 [Unit]
-Description=Update Wyrm statistics dashboards
+Description=Execute state clean labels crawler
 After=network.target
 
 [Service]
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.stats_worker.dashboard --access-id "${NB_CONTROLLER_ACCESS_ID}" generate --blockchain wyrm
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain zksync_era -N 10000
 CPUWeight=60
-SyslogIdentifier=wyrm-statistics
+SyslogIdentifier=zksync-era-state-clean

crawlers/deploy/zksync-era-state-clean.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Zksync Era state clean labels crawler each 25m
+
+[Timer]
+OnBootSec=50s
+OnUnitActiveSec=25m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/zksync-era-state.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Execute state crawler
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync_era --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json
+CPUWeight=60
+SyslogIdentifier=zksync-era-state

crawlers/deploy/zksync-era-state.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Zksync Era state crawler each 10m
+
+[Timer]
+OnBootSec=15s
+OnUnitActiveSec=10m
+
+[Install]
+WantedBy=timers.target

crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py

@@ -7,12 +7,14 @@
 import requests  # type: ignore
 from bugout.data import BugoutSearchResult
 
-from .utils import get_results_for_moonstream_query
+from .utils import get_results_for_moonstream_query, leaderboard_push_batch
 from ..settings import (
     MOONSTREAM_ADMIN_ACCESS_TOKEN,
     MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID,
     MOONSTREAM_API_URL,
     MOONSTREAM_ENGINE_URL,
+    MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE,
+    MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS,
 )
 
 from ..settings import bugout_client as bc
@@ -35,10 +37,15 @@ def handle_leaderboards(args: argparse.Namespace) -> None:
 
     ### get leaderboard journal
 
+    leaderboard_push_batch_size = args.leaderboard_push_batch_size
+
+    leaderboard_push_timeout_seconds = args.leaderboard_push_timeout_seconds
+
     query = "#leaderboard #status:active"
 
-    if args.leaderboard_id:  # way to run only one leaderboard
-        query += f" #leaderboard_id:{args.leaderboard_id}"
+    if args.leaderboard_id:  # way to run only one leaderboard without status:active
+        query = f"#leaderboard #leaderboard_id:{args.leaderboard_id}"
+
     try:
         leaderboards = bc.search(
             token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
@@ -116,26 +123,33 @@ def handle_leaderboards(args: argparse.Namespace) -> None:
             "Content-Type": "application/json",
         }
 
-        try:
-            leaderboard_api_response = requests.put(
-                leaderboard_push_api_url,
-                json=query_results["data"],
-                headers=leaderboard_api_headers,
-                timeout=10,
+        if len(query_results["data"]) > leaderboard_push_batch_size:
+            logger.info(
+                f"Pushing {len(query_results['data'])} scores to leaderboard {leaderboard_id} in batches of {leaderboard_push_batch_size}"
             )
-        except Exception as e:
-            logger.error(
-                f"Could not push results to leaderboard API: {e} for leaderboard {leaderboard_id}"
+            leaderboard_push_batch(
+                leaderboard_id,
+                leaderboard_data,
+                query_results["data"],
+                leaderboard_api_headers,
+                leaderboard_push_batch_size,
+                timeout=leaderboard_push_timeout_seconds,
             )
-            continue
 
-        try:
-            leaderboard_api_response.raise_for_status()
-        except requests.exceptions.HTTPError as http_error:
-            logger.error(
-                f"Could not push results to leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}"
-            )
-            continue
+        else:
+            try:
+                leaderboard_api_response = requests.put(
+                    leaderboard_push_api_url,
+                    json=query_results["data"],
+                    headers=leaderboard_api_headers,
+                    timeout=leaderboard_push_timeout_seconds,
+                )
+                leaderboard_api_response.raise_for_status()
+            except requests.exceptions.HTTPError as http_error:
+                logger.error(
+                    f"Could not push results to leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}"
+                )
+                continue
 
         ### get leaderboard from leaderboard API
 
@@ -213,6 +227,18 @@ def main():
         required=True,
         help="Moonstream Access Token to use for Moonstream Query API requests",
     )
+    leaderboard_generator_parser.add_argument(
+        "--leaderboard-push-batch-size",
+        type=int,
+        default=MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE,
+        help="Number of scores to push to leaderboard API at once",
+    )
+    leaderboard_generator_parser.add_argument(
+        "--leaderboard-push-timeout-seconds",
+        type=int,
+        default=MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS,
+        help="Timeout for leaderboard API requests",
+    )
 
     leaderboard_generator_parser.set_defaults(func=handle_leaderboards)