Merge branch 'flare-rewards' into 'main'

Flare rewards

See merge request flarenetwork/ftso-scaling!135

libs/ftso-core/src/configs/networks.ts

@@ -640,6 +640,28 @@ export const FINALIZATION_VOTER_SELECTION_THRESHOLD_WEIGHT_BIPS = () => {
  */
 export const CAPPED_STAKING_FEE_BIPS = 2000;
 
-export const CALCULATIONS_FOLDER = () => "calculations";
+export const CALCULATIONS_FOLDER = () => {
+  const network = process.env.NETWORK as networks;
+  switch (network) {
+    case "from-env":
+      return "calculations/from-env";
+    case "coston":
+      return "calculations/coston";
+    case "coston2":
+      return "calculations/coston2";
+    case "songbird":
+      return "calculations/songbird";
+    case "flare":
+      return "calculations/flare";
+    case "local-test":
+      return "calculations/local-test";
+    default:
+      // Ensure exhaustive checking
+      // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars
+      ((_: never): void => {})(network);
+  }
+};
+
+export const FEEDS_RENAMING_FILE = () => "libs/ftso-core/src/reward-calculation/feeds-renaming.json";
 
 export const FUTURE_VOTING_ROUNDS = () => 30;

libs/ftso-core/src/reward-calculation/feeds-renaming.json

@@ -0,0 +1,8 @@
+[
+    {
+        "oldName": "MATIC/USD",
+        "newName": "POL/USD",
+        "oldFeedId": "0x014d415449432f5553440000000000000000000000",
+        "newFeedId": "0x01504f4c2f55534400000000000000000000000000"
+    }
+]

libs/ftso-core/src/reward-calculation/reward-calculation.ts

@@ -5,6 +5,7 @@ import {
   FINALIZATION_VOTER_SELECTION_THRESHOLD_WEIGHT_BIPS,
   FTSO2_PROTOCOL_ID,
   PENALTY_FACTOR,
+  FEEDS_RENAMING_FILE,
 } from "../configs/networks";
 import { calculateMedianResults } from "../ftso-calculation/ftso-median";
 import { ClaimType, IMergeableRewardClaim, IPartialRewardClaim, IRewardClaim, RewardClaim } from "../utils/RewardClaim";
@@ -49,6 +50,7 @@ import { calculateFastUpdatesClaims } from "./reward-fast-updates";
 import { calculatePenalties } from "./reward-penalties";
 import { calculateSigningRewards } from "./reward-signing";
 import { ILogger } from "../utils/ILogger";
+import { existsSync, readFileSync } from "fs";
 
 /**
  * Initializes reward epoch storage for the given reward epoch.
@@ -259,8 +261,19 @@ export async function partialRewardClaimsForVotingRound(
       } as FUFeedValue);
     }
     const signingPolicyAddressesSubmitted = data.fastUpdatesData.signingPolicyAddressesSubmitted;
+    // Renaming on Fast Updates happens after renaming for Ftso Scaling
+    const feedRenamingMap: Map<string, string> = new Map<string, string>();
+    if (existsSync(FEEDS_RENAMING_FILE())) {
+      const feedRenamingData = JSON.parse(readFileSync(FEEDS_RENAMING_FILE(), "utf8"));
+      for (const feed of feedRenamingData) {
+        feedRenamingMap.set(feed.oldFeedId, feed.newFeedId);
+      }
+    }
     for (const [feedId, offers] of fuFeedOffers.entries()) {
-      const medianResult = medianCalculationMap.get(feedId);
+      const medianResult =
+        medianCalculationMap.get(feedId) == undefined
+          ? medianCalculationMap.get(feedRenamingMap.get(feedId))
+          : medianCalculationMap.get(feedId);
       if (medianResult === undefined) {
         // This should never happen
         throw new Error("Critical error: Median result is undefined");

libs/ftso-core/src/utils/stat-info/granulated-partial-offers-map.ts

@@ -73,7 +73,7 @@ export function createRewardCalculationFolders(
   calculationFolder = CALCULATIONS_FOLDER()
 ): void {
   if (!existsSync(calculationFolder)) {
-    mkdirSync(calculationFolder);
+    mkdirSync(calculationFolder, { recursive: true });
   }
   const rewardEpochFolder = path.join(
     calculationFolder,

libs/ftso-core/src/utils/stat-info/reward-distribution-data.ts

@@ -11,6 +11,7 @@ import Web3 from "web3";
 
 export interface IRewardDistributionData {
   rewardEpochId: number;
+  network: string;
   rewardClaims: IRewardClaimWithProof[];
   noOfWeightBasedClaims: number;
   merkleRoot: string;
@@ -41,6 +42,7 @@ export function serializeRewardDistributionData(
   ).length;
   const result: IRewardDistributionData = {
     rewardEpochId,
+    network: process.env.NETWORK!,
     rewardClaims: rewardClaimsWithProof,
     noOfWeightBasedClaims,
     merkleRoot,

libs/ftso-core/src/utils/stat-info/reward-epoch-info.ts

@@ -70,7 +70,7 @@ export function serializeRewardEpochInfo(
   calculationFolder = CALCULATIONS_FOLDER()
 ): void {
   if (!existsSync(calculationFolder)) {
-    mkdirSync(calculationFolder);
+    mkdirSync(calculationFolder, { recursive: true });
   }
   const rewardEpochFolder = path.join(
     calculationFolder,

scripts/analytics/run/extract-reward-data.ts

@@ -49,7 +49,7 @@ async function main() {
     REWARD_DISTRIBUTION_DATA_FILE, 
     REWARD_EPOCH_INFO_FILE
   ];
-
+  process.env.NETWORK = network;
   const sourceFolder = path.join(CALCULATIONS_FOLDER(), `${rewardEpochId}`);
 
   for(const fileName of files) {

scripts/rewards/README.md

@@ -171,12 +171,14 @@ When finalizing the following [reward detail tags](../../libs/ftso-core/src/rewa
 **Detailed reward calculation data** can be obtained using reward calculation scripts.
 - [`coston-db.sh`](./coston-db.sh)
 - [`songbird-db.sh`](./songbird-db.sh)
+- [`flare-db.sh`](./flare-db.sh)
 The data is calculated into the folder `calculations`. For each reward epoch `rewardEpochId` the data are stored into the folder
 `calculations/<rewardEpochId>` and within that folder the specific data for each voting round (`votingRoundId`) are stored in the folders 
 `calculations/<rewardEpochId>/<votingRoundId>`.
 
 ### Public reward data
 
+Data calculated by Flare Newtworks are published on [https://github.com/flare-foundation/fsp-rewards](https://github.com/flare-foundation/fsp-rewards).
 Public reward data includes the following files:
 - `reward-epoch-info.json` - extracted data about reward epoch (offers, signing policy, feeds, boundaries, etc.).
 - `reward-distribution-data.json` - all aggregated reward claims in the order as they are put into the Merkle tree, together with Merkle proofs and all pieces of data that are necessary to reconstruct the Merkle tree (one can use [MerkleTree.ts](../../libs/ftso-core/src/utils/MerkleTree.ts) lib).
@@ -206,9 +208,15 @@ In folders `calculations/<rewardEpochId>/<votingRoundId>`:
 Claiming is done using the function [`claim(...)`](https://gitlab.com/flarenetwork/flare-smart-contracts-v2/-/blob/main/contracts/userInterfaces/IRewardManager.sol?ref_type=heads#L85) on `RewardManager` smart contract.
 Delegators and stake delegators can use the claim function without any proofs, once somebody (usually this is done by a dedicated bot, but it may be done essentially by anyone) initializes all the reward claims of claim type `2-WNAT` and `3-MIRROR` (the latter is relevant on Flare and Coston2 only). Initialization is done using (multiple) calls of function [`initialiseWeightBasedClaims(...)`](https://gitlab.com/flarenetwork/flare-smart-contracts-v2/-/blob/main/contracts/userInterfaces/IRewardManager.sol?ref_type=heads#L115). Once all the proofs of those types of claims for some reward epoch id are initialized, claiming without proofs by delegators and stake delegators are enabled for that reward epoch id. Note that such claims actually claim all the rewards for the reward owner address, also unclaimed rewards from previous reward epochs, up to the claimed one.
 
-On the other hand, data providers wanting to get their fees (claim type `1-FEE` and direct claims `0-DIRECT`) need to provide an array of claims with Merkle proofs as the last parameter of `claim(...)` function. If this is done using one of the explorers, the claims with Merkle proofs structs should be encoded as array tuples. Such encoding that can be immediately used in explorer interface is provided in files `reward-distribution-data-tuples.json`. For example:
+On the other hand, data providers wanting to get their fees (claim type `1-FEE` and direct claims `0-DIRECT`) need to provide an array of claims with Merkle proofs as the last parameter of `claim(...)` function. 
 
-Consider the following file [`rewards-data/songbird/196/reward-distribution-data-tuples.json`](../../rewards-data/songbird/196/reward-distribution-data-tuples.json). The first tuple-encoded claim in the array under the key `rewardClaims` looks like this:
+On the explorers, function `4. claim` can be used:
+- [Songbird explorer](https://songbird-explorer.flare.network/address/0x8A80583BD5A5Cd8f68De585163259D61Ea8dc904/write-contract#address-tabs),
+- [Flare explorer](https://flare-explorer.flare.network/address/0xC8f55c5aA2C752eE285Bd872855C749f4ee6239B/write-contract#address-tabs).
+
+In case a new version of the explorer is used, data from `reward-distribution-data.json` can be entered into interface that allows input per struct fields. In the older version of the explorer the claims with Merkle proofs structs should be encoded as array tuples. Such encoding that can be immediately used in explorer interface is provided in files `reward-distribution-data-tuples.json`. For example:
+
+Consider the following file [`songbird/196/reward-distribution-data-tuples.json`](https://raw.githubusercontent.com/flare-foundation/fsp-rewards/refs/heads/main/songbird/196/reward-distribution-data-tuples.json) in the [reward publication repository](https://github.com/flare-foundation/fsp-rewards). The first tuple-encoded claim in the array under the key `rewardClaims` looks like this:
 
 ```
 [
@@ -230,8 +238,6 @@ Consider the following file [`rewards-data/songbird/196/reward-distribution-data
 ]
 ```
 
-[Songbird explorer](https://songbird-explorer.flare.network/address/0x8A80583BD5A5Cd8f68De585163259D61Ea8dc904/write-contract#address-tabs) can be used for claiming (function `4. claim`).
-
 The following parameters should be provided:
 - `_rewardOwner` - `beneficiary` in the list of `_proofs`.
 - `_recipient` - any address.

scripts/rewards/flare-db.sh

@@ -0,0 +1,42 @@
+# Reward calculation for Songbird network
+# Setup the correct DB connection and run the script, e.g.
+# ./scripts/rewards/flare-db.sh
+
+export NETWORK=flare
+
+export DB_REQUIRED_INDEXER_HISTORY_TIME_SEC=86400
+export VOTING_ROUND_HISTORY_SIZE=10000
+export INDEXER_TOP_TIMEOUT=1000
+export DB_HOST=127.0.0.1
+export DB_PORT=3336
+export DB_USERNAME=root
+export DB_PASSWORD=root
+export DB_NAME=flare_ftso_indexer
+
+export REMOVE_ANNOYING_MESSAGES=true
+
+
+# check here: https://songbird-explorer.flare.network/address/0x421c69E22f48e14Fc2d2Ee3812c59bfb81c38516/read-contract#address-tabs
+# 9.  getCurrentRewardEpochId, and use one epoch less for test (required indexer history is 4 epochs/ 14 days)
+# export REWARD_EPOCH_ID=2349
+
+# COMPILATION
+yarn nest build ftso-reward-calculation-process
+
+# ---------------------------------------------------------------------------------------------------------------------------
+# Calculating all reward data from the starting reward epoch id. The calculation of claims is parallelized. 
+# In the current (ongoing) reward epoch the calculation is switched to incremental, as data becomes available. 
+# If the data for a specific reward epoch id is already available, the calculation is skipped.
+export FROM_REWARD_EPOCH_ID=228
+node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -g -o -c -a -y -b 100 -w 10 -d $FROM_REWARD_EPOCH_ID -m 10000
+
+# Incremental calculation
+# node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -l -b 80 -w 5 -m 10000
+
+# ---------------------------------------------------------------------------------------------------------------------------
+# Recoverable sequential calculation
+# node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -i -a -c -o -d $FROM_REWARD_EPOCH_ID -m 10000
+# ---------------------------------------------------------------------------------------------------------------------------
+# Calculating for specific reward epoch id
+# export SPECIFIC_REWARD_EPOCH_ID=2380
+# node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -i -a -c -b 10 -w 24 -r $SPECIFIC_REWARD_EPOCH_ID -m 10000