From e5d65ecb12f59176ec0f4da7a82b4175a1ba262e Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Mon, 23 Sep 2024 16:09:43 +0200 Subject: [PATCH 01/28] fdc rewards wip --- abi/FdcHub.json | 674 ++++++++++++++++++ .../ftso-reward-calculator-process.command.ts | 8 + .../src/interfaces/OptionalCommandOptions.ts | 1 + .../src/libs/calculator-utils.ts | 2 + .../src/libs/claim-utils.ts | 5 +- .../src/libs/offer-utils.ts | 12 +- .../src/libs/reward-claims-calculation.ts | 4 +- .../src/libs/reward-data-calculation.ts | 20 +- .../src/services/calculator.service.ts | 6 +- libs/ftso-core/src/DataManager.ts | 4 + libs/ftso-core/src/DataManagerForRewarding.ts | 92 ++- .../src/IndexerClientForRewarding.ts | 51 ++ libs/ftso-core/src/configs/contracts.ts | 12 +- libs/ftso-core/src/configs/networks.ts | 70 +- .../src/data-calculation-interfaces.ts | 33 + .../src/events/AttestationRequest.ts | 28 + .../src/events/FDCInflationRewardsOffered.ts | 53 ++ .../src/events/FUInflationRewardsOffered.ts | 2 +- .../reward-calculation/reward-calculation.ts | 11 +- .../src/reward-calculation/reward-offers.ts | 36 + libs/ftso-core/src/utils/ABICache.ts | 4 + .../ftso-core/src/utils/PartialRewardOffer.ts | 7 + .../src/utils/stat-info/constants.ts | 1 + .../granulated-partial-offers-map.ts | 43 +- .../stat-info/reward-calculation-data.ts | 23 +- .../src/utils/stat-info/reward-epoch-info.ts | 6 +- test/libs/unit/generator-rewards.test.ts | 10 +- 27 files changed, 1185 insertions(+), 33 deletions(-) create mode 100644 abi/FdcHub.json create mode 100644 libs/ftso-core/src/events/AttestationRequest.ts create mode 100644 libs/ftso-core/src/events/FDCInflationRewardsOffered.ts diff --git a/abi/FdcHub.json b/abi/FdcHub.json new file mode 100644 index 00000000..0d807538 --- /dev/null +++ b/abi/FdcHub.json @@ -0,0 +1,674 @@ +{ + "_format": "hh-sol-artifact-1", + "contractName": "FdcHub", + "sourceName": "contracts/fdc/implementation/FdcHub.sol", + "abi": [ + { + "inputs": [ + { + "internalType": "contract IGovernanceSettings", + "name": "_governanceSettings", + "type": "address" + }, + { + "internalType": "address", + "name": "_initialGovernance", + "type": "address" + }, + { + "internalType": "address", + "name": "_addressUpdater", + "type": "address" + }, + { + "internalType": "uint8", + "name": "_requestsOffsetSeconds", + "type": "uint8" + } + ], + "stateMutability": "nonpayable", + "type": "constructor" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "bytes", + "name": "data", + "type": "bytes" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "fee", + "type": "uint256" + } + ], + "name": "AttestationRequest", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "uint256", + "name": "authorizedAmountWei", + "type": "uint256" + } + ], + "name": "DailyAuthorizedInflationSet", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "bytes4", + "name": "selector", + "type": "bytes4" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "allowedAfterTimestamp", + "type": "uint256" + }, + { + "indexed": false, + "internalType": "bytes", + "name": "encodedCall", + "type": "bytes" + } + ], + "name": "GovernanceCallTimelocked", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "address", + "name": "initialGovernance", + "type": "address" + } + ], + "name": "GovernanceInitialised", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "address", + "name": "governanceSettings", + "type": "address" + } + ], + "name": "GovernedProductionModeEntered", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "uint256", + "name": "amountReceivedWei", + "type": "uint256" + } + ], + "name": "InflationReceived", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "uint24", + "name": "rewardEpochId", + "type": "uint24" + }, + { + "components": [ + { + "internalType": "bytes32", + "name": "attestationType", + "type": "bytes32" + }, + { + "internalType": "bytes32", + "name": "source", + "type": "bytes32" + }, + { + "internalType": "uint24", + "name": "inflationShare", + "type": "uint24" + }, + { + "internalType": "uint8", + "name": "minRequestsThreshold", + "type": "uint8" + }, + { + "internalType": "uint224", + "name": "mode", + "type": "uint224" + } + ], + "indexed": false, + "internalType": "struct IFdcInflationConfigurations.FdcConfiguration[]", + "name": "fdcConfigurations", + "type": "tuple[]" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "amount", + "type": "uint256" + } + ], + "name": "InflationRewardsOffered", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "uint8", + "name": "requestsOffsetSeconds", + "type": "uint8" + } + ], + "name": "RequestsOffsetSet", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "bytes4", + "name": "selector", + "type": "bytes4" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "timestamp", + "type": "uint256" + } + ], + "name": "TimelockedGovernanceCallCanceled", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "bytes4", + "name": "selector", + "type": "bytes4" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "timestamp", + "type": "uint256" + } + ], + "name": "TimelockedGovernanceCallExecuted", + "type": "event" + }, + { + "inputs": [ + { + "internalType": "bytes4", + "name": "_selector", + "type": "bytes4" + } + ], + "name": "cancelGovernanceCall", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "dailyAuthorizedInflation", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "bytes4", + "name": "_selector", + "type": "bytes4" + } + ], + "name": "executeGovernanceCall", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "fdcInflationConfigurations", + "outputs": [ + { + "internalType": "contract IFdcInflationConfigurations", + "name": "", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "fdcRequestFeeConfigurations", + "outputs": [ + { + "internalType": "contract IFdcRequestFeeConfigurations", + "name": "", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "flareSystemsManager", + "outputs": [ + { + "internalType": "contract IIFlareSystemsManager", + "name": "", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getAddressUpdater", + "outputs": [ + { + "internalType": "address", + "name": "_addressUpdater", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getContractName", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "stateMutability": "pure", + "type": "function" + }, + { + "inputs": [], + "name": "getExpectedBalance", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getInflationAddress", + "outputs": [ + { + "internalType": "address", + "name": "", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getTokenPoolSupplyData", + "outputs": [ + { + "internalType": "uint256", + "name": "_lockedFundsWei", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "_totalInflationAuthorizedWei", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "_totalClaimedWei", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "governance", + "outputs": [ + { + "internalType": "address", + "name": "", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "governanceSettings", + "outputs": [ + { + "internalType": "contract IGovernanceSettings", + "name": "", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "contract IGovernanceSettings", + "name": "_governanceSettings", + "type": "address" + }, + { + "internalType": "address", + "name": "_initialGovernance", + "type": "address" + } + ], + "name": "initialise", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "_address", + "type": "address" + } + ], + "name": "isExecutor", + "outputs": [ + { + "internalType": "bool", + "name": "", + "type": "bool" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "lastInflationAuthorizationReceivedTs", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "lastInflationReceivedTs", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "productionMode", + "outputs": [ + { + "internalType": "bool", + "name": "", + "type": "bool" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "receiveInflation", + "outputs": [], + "stateMutability": "payable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "bytes", + "name": "_data", + "type": "bytes" + } + ], + "name": "requestAttestation", + "outputs": [], + "stateMutability": "payable", + "type": "function" + }, + { + "inputs": [], + "name": "requestsOffsetSeconds", + "outputs": [ + { + "internalType": "uint8", + "name": "", + "type": "uint8" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "rewardManager", + "outputs": [ + { + "internalType": "contract IIRewardManager", + "name": "", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_toAuthorizeWei", + "type": "uint256" + } + ], + "name": "setDailyAuthorizedInflation", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint8", + "name": "_requestsOffsetSeconds", + "type": "uint8" + } + ], + "name": "setRequestsOffset", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "switchToProductionMode", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "bytes4", + "name": "selector", + "type": "bytes4" + } + ], + "name": "timelockedCalls", + "outputs": [ + { + "internalType": "uint256", + "name": "allowedAfterTimestamp", + "type": "uint256" + }, + { + "internalType": "bytes", + "name": "encodedCall", + "type": "bytes" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "totalInflationAuthorizedWei", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "totalInflationReceivedWei", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "totalInflationRewardsOfferedWei", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint24", + "name": "_currentRewardEpochId", + "type": "uint24" + }, + { + "internalType": "uint64", + "name": "_currentRewardEpochExpectedEndTs", + "type": "uint64" + }, + { + "internalType": "uint64", + "name": "_rewardEpochDurationSeconds", + "type": "uint64" + } + ], + "name": "triggerRewardEpochSwitchover", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "bytes32[]", + "name": "_contractNameHashes", + "type": "bytes32[]" + }, + { + "internalType": "address[]", + "name": "_contractAddresses", + "type": "address[]" + } + ], + "name": "updateContractAddresses", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + } + ], + "bytecode": "0x60806040523480156200001157600080fd5b5060405162002a3b38038062002a3b83398101604081905262000034916200026d565b83838380808484620000478282620000db565b50620000739050817f714f205b2abd25bef1d06a1af944e38c113fe6160375c4e1d6d5cf28848e771955565b5050600d805460ff60a01b1916600160a01b60ff8716908102919091179091556040519081527f5d5d031078427b5a6de8e1f2973a0edacfca00cc17bb10fc99b4c726df1f1f4c93506020019150620000c99050565b60405180910390a150505050620002da565b600054600160a01b900460ff16156200013b5760405162461bcd60e51b815260206004820152601460248201527f696e697469616c6973656420213d2066616c736500000000000000000000000060448201526064015b60405180910390fd5b6001600160a01b038216620001935760405162461bcd60e51b815260206004820152601860248201527f676f7665726e616e63652073657474696e6773207a65726f0000000000000000604482015260640162000132565b6001600160a01b038116620001de5760405162461bcd60e51b815260206004820152601060248201526f5f676f7665726e616e6365207a65726f60801b604482015260640162000132565b600080546001600160a01b038481166001600160a81b031990921691909117600160a01b17909155600180549183166001600160a01b0319909216821790556040519081527f9789733827840833afc031fb2ef9ab6894271f77bad2085687cf4ae5c7bee4db9060200160405180910390a15050565b6001600160a01b03811681146200026a57600080fd5b50565b600080600080608085870312156200028457600080fd5b8451620002918162000254565b6020860151909450620002a48162000254565b6040860151909350620002b78162000254565b606086015190925060ff81168114620002cf57600080fd5b939692955090935050565b61275180620002ea6000396000f3fe6080604052600436106101d85760003560e01c806391f2567911610102578063debfda3011610095578063ef88bf1311610064578063ef88bf131461054d578063f5a983831461056d578063f5f5ba7214610582578063faae7fc9146105b757600080fd5b8063debfda30146104be578063e17f212e146104ee578063e27395631461050f578063ed39d3f81461052f57600080fd5b8063b00c0b76116100d1578063b00c0b7614610452578063bd76b69c14610472578063bfda608614610488578063d0c1c393146104a857600080fd5b806391f25679146103d457806394d019f1146103f4578063a5555aea14610427578063af04cd3b1461043d57600080fd5b80635267a15d1161017a5780636238f354116101495780636238f3541461035d57806367fc402914610370578063708e34ce1461039057806374e6310e146103a657600080fd5b80635267a15d146102d45780635aa6e675146103085780635ff270791461031d57806362354e031461033d57600080fd5b806312afcf0b116101b657806312afcf0b146102445780632dafdbbf14610268578063473252c41461029e5780634c5a1d28146102b457600080fd5b806306201f1d146101dd5780630f4ef8a6146101e7578063116ea70214610224575b600080fd5b6101e56105d7565b005b3480156101f357600080fd5b50600d54610207906001600160a01b031681565b6040516001600160a01b0390911681526020015b60405180910390f35b34801561023057600080fd5b50600c54610207906001600160a01b031681565b34801561025057600080fd5b5061025a60065481565b60405190815260200161021b565b34801561027457600080fd5b50610283600354600a54600092565b6040805193845260208401929092529082015260600161021b565b3480156102aa57600080fd5b5061025a60055481565b3480156102c057600080fd5b50600b54610207906001600160a01b031681565b3480156102e057600080fd5b507f714f205b2abd25bef1d06a1af944e38c113fe6160375c4e1d6d5cf28848e771954610207565b34801561031457600080fd5b50610207610639565b34801561032957600080fd5b506101e5610338366004611d8f565b6106d5565b34801561034957600080fd5b50600054610207906001600160a01b031681565b6101e561036b366004611db9565b610960565b34801561037c57600080fd5b506101e561038b366004611d8f565b610d3c565b34801561039c57600080fd5b5061025a60075481565b3480156103b257600080fd5b506103c66103c1366004611d8f565b610e1d565b60405161021b929190611e7a565b3480156103e057600080fd5b506101e56103ef366004611ec1565b610ec2565b34801561040057600080fd5b50600d5461041590600160a01b900460ff1681565b60405160ff909116815260200161021b565b34801561043357600080fd5b5061025a60045481565b34801561044957600080fd5b5061025a610f27565b34801561045e57600080fd5b506101e561046d366004612026565b610f31565b34801561047e57600080fd5b5061025a600a5481565b34801561049457600080fd5b506101e56104a33660046120ed565b61100c565b3480156104b457600080fd5b5061025a60035481565b3480156104ca57600080fd5b506104de6104d936600461210a565b61115f565b604051901515815260200161021b565b3480156104fa57600080fd5b506000546104de90600160a81b900460ff1681565b34801561051b57600080fd5b506101e561052a366004612127565b6111e8565b34801561053b57600080fd5b506008546001600160a01b0316610207565b34801561055957600080fd5b506101e5610568366004612140565b611240565b34801561057957600080fd5b506101e56113a6565b34801561058e57600080fd5b506040805180820182526006815265233231a43ab160d11b6020820152905161021b9190612179565b3480156105c357600080fd5b50600954610207906001600160a01b031681565b6105df61146c565b6105e76114d4565b346004546105f591906121a2565b600455426006556040513481527f95c4e29cc99bc027cfc3cd719d6fd973d5f0317061885fbb322b9b17d8d35d379060200160405180910390a161063761151f565b565b60008054600160a81b900460ff1661065b57506001546001600160a01b031690565b60008054906101000a90046001600160a01b03166001600160a01b031663732524946040518163ffffffff1660e01b8152600401602060405180830381865afa1580156106ac573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906106d091906121b5565b905090565b6106de3361115f565b61071f5760405162461bcd60e51b815260206004820152600d60248201526c37b7363c9032bc32b1baba37b960991b60448201526064015b60405180910390fd5b6001600160e01b03198116600090815260026020526040812080549091036107895760405162461bcd60e51b815260206004820152601a60248201527f74696d656c6f636b3a20696e76616c69642073656c6563746f720000000000006044820152606401610716565b80544210156107da5760405162461bcd60e51b815260206004820152601960248201527f74696d656c6f636b3a206e6f7420616c6c6f77656420796574000000000000006044820152606401610716565b60008160010180546107eb906121d2565b80601f0160208091040260200160405190810160405280929190818152602001828054610817906121d2565b80156108645780601f1061083957610100808354040283529160200191610864565b820191906000526020600020905b81548152906001019060200180831161084757829003601f168201915b505050506001600160e01b03198516600090815260026020526040812081815592935090506108966001830182611d41565b50506000805460ff60b01b1916600160b01b17815560405130906108bb90849061220c565b6000604051808303816000865af19150503d80600081146108f8576040519150601f19603f3d011682016040523d82523d6000602084013e6108fd565b606091505b50506000805460ff60b01b19169055604080516001600160e01b0319871681524260208201529192507fa7326b57fc9cfe267aaea5e7f0b01757154d265620a0585819416ee9ddd2c438910160405180910390a161095a81611566565b50505050565b61096861146c565b600c54604051630507923b60e11b81526000916001600160a01b031690630a0f24769061099b9086908690600401612251565b602060405180830381865afa1580156109b8573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906109dc9190612265565b905080341015610a545760405162461bcd60e51b815260206004820152603d60248201527f66656520746f206c6f772c2063616c6c2067657452657175657374466565207460448201527f6f20676574207468652072657175697265642066656520616d6f756e740000006064820152608401610716565b60095460408051637056269760e01b815290516000926001600160a01b03169163705626979160048083019260209291908290030181865afa158015610a9e573d6000803e3d6000fd5b505050506040513d601f19601f82011682018060405250810190610ac2919061227e565b90506000600960009054906101000a90046001600160a01b03166001600160a01b031663ed54fd636040518163ffffffff1660e01b8152600401602060405180830381865afa158015610b19573d6000803e3d6000fd5b505050506040513d601f19601f82011682018060405250810190610b3d919061229b565b600d54909150610b5790600160a01b900460ff16826122b8565b6001600160401b03164210610c87576009546001600160a01b03166375d2187a610b828460016122df565b6040516001600160e01b031960e084901b16815262ffffff9091166004820152602401602060405180830381865afa925050508015610bde575060408051601f3d908101601f19168201909252610bdb918101906122fb565b60015b15610c8757600960009054906101000a90046001600160a01b03166001600160a01b0316634134520b6040518163ffffffff1660e01b8152600401602060405180830381865afa158015610c36573d6000803e3d6000fd5b505050506040513d601f19601f82011682018060405250810190610c5a91906122fb565b610c65906001612321565b63ffffffff168163ffffffff1611610c8557610c826001846122df565b92505b505b600d5460405163a02e86e560e01b815262ffffff84166004820152600060248201526001600160a01b039091169063a02e86e59034906044016000604051808303818588803b158015610cd957600080fd5b505af1158015610ced573d6000803e3d6000fd5b50505050507f251377668af6553101c9bb094ba89c0c536783e005e203625e6cd57345918cc9858534604051610d259392919061233e565b60405180910390a1505050610d3861151f565b5050565b610d44611583565b6001600160e01b031981166000908152600260205260408120549003610dac5760405162461bcd60e51b815260206004820152601a60248201527f74696d656c6f636b3a20696e76616c69642073656c6563746f720000000000006044820152606401610716565b604080516001600160e01b0319831681524260208201527f7735b2391c38a81419c513e30ca578db7158eadd7101511b23e221c654d19cf8910160405180910390a16001600160e01b03198116600090815260026020526040812081815590610e186001830182611d41565b505050565b60026020526000908152604090208054600182018054919291610e3f906121d2565b80601f0160208091040260200160405190810160405280929190818152602001828054610e6b906121d2565b8015610eb85780601f10610e8d57610100808354040283529160200191610eb8565b820191906000526020600020905b815481529060010190602001808311610e9b57829003601f168201915b5050505050905082565b6009546001600160a01b03163314610f1c5760405162461bcd60e51b815260206004820152601960248201527f6f6e6c7920666c6172652073797374656d206d616e61676572000000000000006044820152606401610716565b610e188383836115dd565b60006106d06117b4565b7f714f205b2abd25bef1d06a1af944e38c113fe6160375c4e1d6d5cf28848e7719546001600160a01b0316336001600160a01b031614610faa5760405162461bcd60e51b815260206004820152601460248201527337b7363c9030b2323932b9b9903ab83230ba32b960611b6044820152606401610716565b611002610fde83836040518060400160405280600e81526020016d20b2323932b9b9aab83230ba32b960911b8152506117c6565b7f714f205b2abd25bef1d06a1af944e38c113fe6160375c4e1d6d5cf28848e771955565b610d3882826118a3565b600054600160b01b900460ff168061102e5750600054600160a81b900460ff16155b156111515761103b6119cd565b600960009054906101000a90046001600160a01b03166001600160a01b0316635a8320886040518163ffffffff1660e01b8152600401602060405180830381865afa15801561108e573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906110b2919061229b565b6001600160401b03168160ff16106110fd5760405162461bcd60e51b815260206004820152600e60248201526d1a5b9d985b1a59081bd9999cd95d60921b6044820152606401610716565b600d805460ff60a01b1916600160a01b60ff8416908102919091179091556040519081527f5d5d031078427b5a6de8e1f2973a0edacfca00cc17bb10fc99b4c726df1f1f4c9060200160405180910390a150565b61115c600036611a05565b50565b60008054600160a01b900460ff1680156111e25750600054604051630debfda360e41b81526001600160a01b0384811660048301529091169063debfda3090602401602060405180830381865afa1580156111be573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906111e29190612362565b92915050565b6111f06114d4565b60078190556003546112039082906121a2565b600355426005556040518181527f187f32a0f765499f15b3bb52ed0aebf6015059f230f2ace7e701e60a476695959060200160405180910390a150565b600054600160a01b900460ff16156112915760405162461bcd60e51b8152602060048201526014602482015273696e697469616c6973656420213d2066616c736560601b6044820152606401610716565b6001600160a01b0382166112e75760405162461bcd60e51b815260206004820152601860248201527f676f7665726e616e63652073657474696e6773207a65726f00000000000000006044820152606401610716565b6001600160a01b0381166113305760405162461bcd60e51b815260206004820152601060248201526f5f676f7665726e616e6365207a65726f60801b6044820152606401610716565b600080546001600160a01b038481166001600160a81b031990921691909117600160a01b17909155600180549183166001600160a01b0319909216821790556040519081527f9789733827840833afc031fb2ef9ab6894271f77bad2085687cf4ae5c7bee4db9060200160405180910390a15050565b6113ae611583565b600054600160a81b900460ff16156114085760405162461bcd60e51b815260206004820152601a60248201527f616c726561647920696e2070726f64756374696f6e206d6f64650000000000006044820152606401610716565b600180546001600160a01b031916905560008054600160a81b60ff60a81b198216179091556040516001600160a01b0390911681527f83af113638b5422f9e977cebc0aaf0eaf2188eb9a8baae7f9d46c42b33a1560c9060200160405180910390a1565b6000346114776117b4565b61148191906121a2565b905047818111156114c45761dead6108fc61149c8484612384565b6040518115909202916000818181858888f19350505050158015610e18573d6000803e3d6000fd5b81811015610d3857610d38612397565b6008546001600160a01b031633146106375760405162461bcd60e51b815260206004820152600e60248201526d696e666c6174696f6e206f6e6c7960901b6044820152606401610716565b6115276117b4565b47146106375760405162461bcd60e51b815260206004820152600e60248201526d6f7574206f662062616c616e636560901b6044820152606401610716565b3d604051818101604052816000823e821561157f578181f35b8181fd5b61158b610639565b6001600160a01b0316336001600160a01b0316146106375760405162461bcd60e51b815260206004820152600f60248201526e6f6e6c7920676f7665726e616e636560881b6044820152606401610716565b60006115ea8260026123ad565b6115f490846122b8565b6001600160401b03169050600061162d6201518060065461161591906121a2565b61161f85876122b8565b6001600160401b0316611b51565b9050600061165f6001600160401b0385166116488585612384565b600a546004546116589190612384565b9190611b67565b9050600061166e8760016122df565b90508062ffffff167fedcf03eed469135e307ec8dc425dc2c49560d3014b724a532f6f468fcc975df8600b60009054906101000a90046001600160a01b03166001600160a01b03166345bd2e196040518163ffffffff1660e01b8152600401600060405180830381865afa1580156116ea573d6000803e3d6000fd5b505050506040513d6000823e601f3d908101601f1916820160405261171291908101906123d8565b846040516117219291906124d1565b60405180910390a281600a600082825461173b91906121a2565b9091555050600d5460405163a02e86e560e01b815262ffffff83166004820152600160248201526001600160a01b039091169063a02e86e59084906044016000604051808303818588803b15801561179257600080fd5b505af11580156117a6573d6000803e3d6000fd5b505050505050505050505050565b6000600a546004546106d09190612384565b600080826040516020016117da9190612179565b6040516020818303038152906040528051906020012090506000805b86518110156118525786818151811061181157611811612557565b602002602001015183036118405785818151811061183157611831612557565b60200260200101519150611852565b8061184a8161256d565b9150506117f6565b506001600160a01b0381166118985760405162461bcd60e51b815260206004820152600c60248201526b61646472657373207a65726f60a01b6044820152606401610716565b9150505b9392505050565b6118ad8282611c8d565b6118ed82826040518060400160405280601a81526020017f466463496e666c6174696f6e436f6e66696775726174696f6e730000000000008152506117c6565b600b60006101000a8154816001600160a01b0302191690836001600160a01b0316021790555061195382826040518060400160405280601b81526020017f46646352657175657374466565436f6e66696775726174696f6e7300000000008152506117c6565b600c60006101000a8154816001600160a01b0302191690836001600160a01b031602179055506119a982826040518060400160405280600d81526020016c2932bbb0b93226b0b730b3b2b960991b8152506117c6565b600d80546001600160a01b0319166001600160a01b03929092169190911790555050565b600054600160b01b900460ff16156119fd573330146119ee576119ee612397565b6000805460ff60b01b19169055565b610637611583565b611a0d611583565b6000805460408051636221a54b60e01b81529051853593926001600160a01b031691636221a54b9160048083019260209291908290030181865afa158015611a59573d6000803e3d6000fd5b505050506040513d601f19601f82011682018060405250810190611a7d9190612265565b90506000611a8b82426121a2565b9050604051806040016040528082815260200186868080601f01602080910402602001604051908101604052809392919081815260200183838082843760009201829052509390945250506001600160e01b03198616815260026020908152604090912083518155908301519091506001820190611b0990826125d4565b509050507fed948300a3694aa01d4a6b258bfd664350193d770c0b51f8387277f6d83ea3b683828787604051611b429493929190612693565b60405180910390a15050505050565b6000818311611b60578161189c565b5090919050565b6000808211611bab5760405162461bcd60e51b815260206004820152601060248201526f4469766973696f6e206279207a65726f60801b6044820152606401610716565b83600003611bbb5750600061189c565b83830283858281611bce57611bce6126c6565b0403611bec57828181611be357611be36126c6565b0491505061189c565b506000611bf983866126dc565b90506000611c0784876126f0565b90506000611c1585876126dc565b90506000611c2386886126f0565b905085611c308285612704565b611c3a91906126dc565b611c448385612704565b611c4e8387612704565b88611c598689612704565b611c639190612704565b611c6d91906121a2565b611c7791906121a2565b611c8191906121a2565b98975050505050505050565b611c978282611cf1565b611ccd828260405180604001604052806013815260200172233630b932a9bcb9ba32b6b9a6b0b730b3b2b960691b8152506117c6565b600980546001600160a01b0319166001600160a01b03929092169190911790555050565b611d1d82826040518060400160405280600981526020016824b7333630ba34b7b760b91b8152506117c6565b600880546001600160a01b0319166001600160a01b03929092169190911790555050565b508054611d4d906121d2565b6000825580601f10611d5d575050565b601f01602090049060005260206000209081019061115c91905b80821115611d8b5760008155600101611d77565b5090565b600060208284031215611da157600080fd5b81356001600160e01b03198116811461189c57600080fd5b60008060208385031215611dcc57600080fd5b82356001600160401b0380821115611de357600080fd5b818501915085601f830112611df757600080fd5b813581811115611e0657600080fd5b866020828501011115611e1857600080fd5b60209290920196919550909350505050565b60005b83811015611e45578181015183820152602001611e2d565b50506000910152565b60008151808452611e66816020860160208601611e2a565b601f01601f19169290920160200192915050565b828152604060208201526000611e936040830184611e4e565b949350505050565b62ffffff8116811461115c57600080fd5b6001600160401b038116811461115c57600080fd5b600080600060608486031215611ed657600080fd5b8335611ee181611e9b565b92506020840135611ef181611eac565b91506040840135611f0181611eac565b809150509250925092565b634e487b7160e01b600052604160045260246000fd5b60405160a081016001600160401b0381118282101715611f4457611f44611f0c565b60405290565b604051601f8201601f191681016001600160401b0381118282101715611f7257611f72611f0c565b604052919050565b60006001600160401b03821115611f9357611f93611f0c565b5060051b60200190565b6001600160a01b038116811461115c57600080fd5b600082601f830112611fc357600080fd5b81356020611fd8611fd383611f7a565b611f4a565b82815260059290921b84018101918181019086841115611ff757600080fd5b8286015b8481101561201b57803561200e81611f9d565b8352918301918301611ffb565b509695505050505050565b6000806040838503121561203957600080fd5b82356001600160401b038082111561205057600080fd5b818501915085601f83011261206457600080fd5b81356020612074611fd383611f7a565b82815260059290921b8401810191818101908984111561209357600080fd5b948201945b838610156120b157853582529482019490820190612098565b965050860135925050808211156120c757600080fd5b506120d485828601611fb2565b9150509250929050565b60ff8116811461115c57600080fd5b6000602082840312156120ff57600080fd5b813561189c816120de565b60006020828403121561211c57600080fd5b813561189c81611f9d565b60006020828403121561213957600080fd5b5035919050565b6000806040838503121561215357600080fd5b823561215e81611f9d565b9150602083013561216e81611f9d565b809150509250929050565b60208152600061189c6020830184611e4e565b634e487b7160e01b600052601160045260246000fd5b808201808211156111e2576111e261218c565b6000602082840312156121c757600080fd5b815161189c81611f9d565b600181811c908216806121e657607f821691505b60208210810361220657634e487b7160e01b600052602260045260246000fd5b50919050565b6000825161221e818460208701611e2a565b9190910192915050565b81835281816020850137506000828201602090810191909152601f909101601f19169091010190565b602081526000611e93602083018486612228565b60006020828403121561227757600080fd5b5051919050565b60006020828403121561229057600080fd5b815161189c81611e9b565b6000602082840312156122ad57600080fd5b815161189c81611eac565b6001600160401b038281168282160390808211156122d8576122d861218c565b5092915050565b62ffffff8181168382160190808211156122d8576122d861218c565b60006020828403121561230d57600080fd5b815163ffffffff8116811461189c57600080fd5b63ffffffff8181168382160190808211156122d8576122d861218c565b604081526000612352604083018587612228565b9050826020830152949350505050565b60006020828403121561237457600080fd5b8151801515811461189c57600080fd5b818103818111156111e2576111e261218c565b634e487b7160e01b600052600160045260246000fd5b6001600160401b038181168382160280821691908281146123d0576123d061218c565b505092915050565b600060208083850312156123eb57600080fd5b82516001600160401b0381111561240157600080fd5b8301601f8101851361241257600080fd5b8051612420611fd382611f7a565b81815260a0918202830184019184820191908884111561243f57600080fd5b938501935b838510156124c55780858a03121561245c5760008081fd5b612464611f22565b85518152868601518782015260408087015161247f81611e9b565b90820152606086810151612492816120de565b908201526080868101516001600160e01b03811681146124b25760008081fd5b9082015283529384019391850191612444565b50979650505050505050565b6040808252835182820181905260009190606090818501906020808901865b838110156125435781518051865283810151848701528781015162ffffff16888701528681015160ff16878701526080908101516001600160e01b03169086015260a090940193908201906001016124f0565b505095909501959095525092949350505050565b634e487b7160e01b600052603260045260246000fd5b60006001820161257f5761257f61218c565b5060010190565b601f821115610e1857600081815260208120601f850160051c810160208610156125ad5750805b601f850160051c820191505b818110156125cc578281556001016125b9565b505050505050565b81516001600160401b038111156125ed576125ed611f0c565b612601816125fb84546121d2565b84612586565b602080601f831160018114612636576000841561261e5750858301515b600019600386901b1c1916600185901b1785556125cc565b600085815260208120601f198616915b8281101561266557888601518255948401946001909101908401612646565b50858210156126835787850151600019600388901b60f8161c191681555b5050505050600190811b01905550565b63ffffffff60e01b851681528360208201526060604082015260006126bc606083018486612228565b9695505050505050565b634e487b7160e01b600052601260045260246000fd5b6000826126eb576126eb6126c6565b500490565b6000826126ff576126ff6126c6565b500690565b80820281158282048414176111e2576111e261218c56fea2646970667358221220a46f0657f639e56c214d7ad7de679384f5f991aa0f939781ee8c2f428f5bf30b64736f6c63430008140033", + "deployedBytecode": "0x6080604052600436106101d85760003560e01c806391f2567911610102578063debfda3011610095578063ef88bf1311610064578063ef88bf131461054d578063f5a983831461056d578063f5f5ba7214610582578063faae7fc9146105b757600080fd5b8063debfda30146104be578063e17f212e146104ee578063e27395631461050f578063ed39d3f81461052f57600080fd5b8063b00c0b76116100d1578063b00c0b7614610452578063bd76b69c14610472578063bfda608614610488578063d0c1c393146104a857600080fd5b806391f25679146103d457806394d019f1146103f4578063a5555aea14610427578063af04cd3b1461043d57600080fd5b80635267a15d1161017a5780636238f354116101495780636238f3541461035d57806367fc402914610370578063708e34ce1461039057806374e6310e146103a657600080fd5b80635267a15d146102d45780635aa6e675146103085780635ff270791461031d57806362354e031461033d57600080fd5b806312afcf0b116101b657806312afcf0b146102445780632dafdbbf14610268578063473252c41461029e5780634c5a1d28146102b457600080fd5b806306201f1d146101dd5780630f4ef8a6146101e7578063116ea70214610224575b600080fd5b6101e56105d7565b005b3480156101f357600080fd5b50600d54610207906001600160a01b031681565b6040516001600160a01b0390911681526020015b60405180910390f35b34801561023057600080fd5b50600c54610207906001600160a01b031681565b34801561025057600080fd5b5061025a60065481565b60405190815260200161021b565b34801561027457600080fd5b50610283600354600a54600092565b6040805193845260208401929092529082015260600161021b565b3480156102aa57600080fd5b5061025a60055481565b3480156102c057600080fd5b50600b54610207906001600160a01b031681565b3480156102e057600080fd5b507f714f205b2abd25bef1d06a1af944e38c113fe6160375c4e1d6d5cf28848e771954610207565b34801561031457600080fd5b50610207610639565b34801561032957600080fd5b506101e5610338366004611d8f565b6106d5565b34801561034957600080fd5b50600054610207906001600160a01b031681565b6101e561036b366004611db9565b610960565b34801561037c57600080fd5b506101e561038b366004611d8f565b610d3c565b34801561039c57600080fd5b5061025a60075481565b3480156103b257600080fd5b506103c66103c1366004611d8f565b610e1d565b60405161021b929190611e7a565b3480156103e057600080fd5b506101e56103ef366004611ec1565b610ec2565b34801561040057600080fd5b50600d5461041590600160a01b900460ff1681565b60405160ff909116815260200161021b565b34801561043357600080fd5b5061025a60045481565b34801561044957600080fd5b5061025a610f27565b34801561045e57600080fd5b506101e561046d366004612026565b610f31565b34801561047e57600080fd5b5061025a600a5481565b34801561049457600080fd5b506101e56104a33660046120ed565b61100c565b3480156104b457600080fd5b5061025a60035481565b3480156104ca57600080fd5b506104de6104d936600461210a565b61115f565b604051901515815260200161021b565b3480156104fa57600080fd5b506000546104de90600160a81b900460ff1681565b34801561051b57600080fd5b506101e561052a366004612127565b6111e8565b34801561053b57600080fd5b506008546001600160a01b0316610207565b34801561055957600080fd5b506101e5610568366004612140565b611240565b34801561057957600080fd5b506101e56113a6565b34801561058e57600080fd5b506040805180820182526006815265233231a43ab160d11b6020820152905161021b9190612179565b3480156105c357600080fd5b50600954610207906001600160a01b031681565b6105df61146c565b6105e76114d4565b346004546105f591906121a2565b600455426006556040513481527f95c4e29cc99bc027cfc3cd719d6fd973d5f0317061885fbb322b9b17d8d35d379060200160405180910390a161063761151f565b565b60008054600160a81b900460ff1661065b57506001546001600160a01b031690565b60008054906101000a90046001600160a01b03166001600160a01b031663732524946040518163ffffffff1660e01b8152600401602060405180830381865afa1580156106ac573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906106d091906121b5565b905090565b6106de3361115f565b61071f5760405162461bcd60e51b815260206004820152600d60248201526c37b7363c9032bc32b1baba37b960991b60448201526064015b60405180910390fd5b6001600160e01b03198116600090815260026020526040812080549091036107895760405162461bcd60e51b815260206004820152601a60248201527f74696d656c6f636b3a20696e76616c69642073656c6563746f720000000000006044820152606401610716565b80544210156107da5760405162461bcd60e51b815260206004820152601960248201527f74696d656c6f636b3a206e6f7420616c6c6f77656420796574000000000000006044820152606401610716565b60008160010180546107eb906121d2565b80601f0160208091040260200160405190810160405280929190818152602001828054610817906121d2565b80156108645780601f1061083957610100808354040283529160200191610864565b820191906000526020600020905b81548152906001019060200180831161084757829003601f168201915b505050506001600160e01b03198516600090815260026020526040812081815592935090506108966001830182611d41565b50506000805460ff60b01b1916600160b01b17815560405130906108bb90849061220c565b6000604051808303816000865af19150503d80600081146108f8576040519150601f19603f3d011682016040523d82523d6000602084013e6108fd565b606091505b50506000805460ff60b01b19169055604080516001600160e01b0319871681524260208201529192507fa7326b57fc9cfe267aaea5e7f0b01757154d265620a0585819416ee9ddd2c438910160405180910390a161095a81611566565b50505050565b61096861146c565b600c54604051630507923b60e11b81526000916001600160a01b031690630a0f24769061099b9086908690600401612251565b602060405180830381865afa1580156109b8573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906109dc9190612265565b905080341015610a545760405162461bcd60e51b815260206004820152603d60248201527f66656520746f206c6f772c2063616c6c2067657452657175657374466565207460448201527f6f20676574207468652072657175697265642066656520616d6f756e740000006064820152608401610716565b60095460408051637056269760e01b815290516000926001600160a01b03169163705626979160048083019260209291908290030181865afa158015610a9e573d6000803e3d6000fd5b505050506040513d601f19601f82011682018060405250810190610ac2919061227e565b90506000600960009054906101000a90046001600160a01b03166001600160a01b031663ed54fd636040518163ffffffff1660e01b8152600401602060405180830381865afa158015610b19573d6000803e3d6000fd5b505050506040513d601f19601f82011682018060405250810190610b3d919061229b565b600d54909150610b5790600160a01b900460ff16826122b8565b6001600160401b03164210610c87576009546001600160a01b03166375d2187a610b828460016122df565b6040516001600160e01b031960e084901b16815262ffffff9091166004820152602401602060405180830381865afa925050508015610bde575060408051601f3d908101601f19168201909252610bdb918101906122fb565b60015b15610c8757600960009054906101000a90046001600160a01b03166001600160a01b0316634134520b6040518163ffffffff1660e01b8152600401602060405180830381865afa158015610c36573d6000803e3d6000fd5b505050506040513d601f19601f82011682018060405250810190610c5a91906122fb565b610c65906001612321565b63ffffffff168163ffffffff1611610c8557610c826001846122df565b92505b505b600d5460405163a02e86e560e01b815262ffffff84166004820152600060248201526001600160a01b039091169063a02e86e59034906044016000604051808303818588803b158015610cd957600080fd5b505af1158015610ced573d6000803e3d6000fd5b50505050507f251377668af6553101c9bb094ba89c0c536783e005e203625e6cd57345918cc9858534604051610d259392919061233e565b60405180910390a1505050610d3861151f565b5050565b610d44611583565b6001600160e01b031981166000908152600260205260408120549003610dac5760405162461bcd60e51b815260206004820152601a60248201527f74696d656c6f636b3a20696e76616c69642073656c6563746f720000000000006044820152606401610716565b604080516001600160e01b0319831681524260208201527f7735b2391c38a81419c513e30ca578db7158eadd7101511b23e221c654d19cf8910160405180910390a16001600160e01b03198116600090815260026020526040812081815590610e186001830182611d41565b505050565b60026020526000908152604090208054600182018054919291610e3f906121d2565b80601f0160208091040260200160405190810160405280929190818152602001828054610e6b906121d2565b8015610eb85780601f10610e8d57610100808354040283529160200191610eb8565b820191906000526020600020905b815481529060010190602001808311610e9b57829003601f168201915b5050505050905082565b6009546001600160a01b03163314610f1c5760405162461bcd60e51b815260206004820152601960248201527f6f6e6c7920666c6172652073797374656d206d616e61676572000000000000006044820152606401610716565b610e188383836115dd565b60006106d06117b4565b7f714f205b2abd25bef1d06a1af944e38c113fe6160375c4e1d6d5cf28848e7719546001600160a01b0316336001600160a01b031614610faa5760405162461bcd60e51b815260206004820152601460248201527337b7363c9030b2323932b9b9903ab83230ba32b960611b6044820152606401610716565b611002610fde83836040518060400160405280600e81526020016d20b2323932b9b9aab83230ba32b960911b8152506117c6565b7f714f205b2abd25bef1d06a1af944e38c113fe6160375c4e1d6d5cf28848e771955565b610d3882826118a3565b600054600160b01b900460ff168061102e5750600054600160a81b900460ff16155b156111515761103b6119cd565b600960009054906101000a90046001600160a01b03166001600160a01b0316635a8320886040518163ffffffff1660e01b8152600401602060405180830381865afa15801561108e573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906110b2919061229b565b6001600160401b03168160ff16106110fd5760405162461bcd60e51b815260206004820152600e60248201526d1a5b9d985b1a59081bd9999cd95d60921b6044820152606401610716565b600d805460ff60a01b1916600160a01b60ff8416908102919091179091556040519081527f5d5d031078427b5a6de8e1f2973a0edacfca00cc17bb10fc99b4c726df1f1f4c9060200160405180910390a150565b61115c600036611a05565b50565b60008054600160a01b900460ff1680156111e25750600054604051630debfda360e41b81526001600160a01b0384811660048301529091169063debfda3090602401602060405180830381865afa1580156111be573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906111e29190612362565b92915050565b6111f06114d4565b60078190556003546112039082906121a2565b600355426005556040518181527f187f32a0f765499f15b3bb52ed0aebf6015059f230f2ace7e701e60a476695959060200160405180910390a150565b600054600160a01b900460ff16156112915760405162461bcd60e51b8152602060048201526014602482015273696e697469616c6973656420213d2066616c736560601b6044820152606401610716565b6001600160a01b0382166112e75760405162461bcd60e51b815260206004820152601860248201527f676f7665726e616e63652073657474696e6773207a65726f00000000000000006044820152606401610716565b6001600160a01b0381166113305760405162461bcd60e51b815260206004820152601060248201526f5f676f7665726e616e6365207a65726f60801b6044820152606401610716565b600080546001600160a01b038481166001600160a81b031990921691909117600160a01b17909155600180549183166001600160a01b0319909216821790556040519081527f9789733827840833afc031fb2ef9ab6894271f77bad2085687cf4ae5c7bee4db9060200160405180910390a15050565b6113ae611583565b600054600160a81b900460ff16156114085760405162461bcd60e51b815260206004820152601a60248201527f616c726561647920696e2070726f64756374696f6e206d6f64650000000000006044820152606401610716565b600180546001600160a01b031916905560008054600160a81b60ff60a81b198216179091556040516001600160a01b0390911681527f83af113638b5422f9e977cebc0aaf0eaf2188eb9a8baae7f9d46c42b33a1560c9060200160405180910390a1565b6000346114776117b4565b61148191906121a2565b905047818111156114c45761dead6108fc61149c8484612384565b6040518115909202916000818181858888f19350505050158015610e18573d6000803e3d6000fd5b81811015610d3857610d38612397565b6008546001600160a01b031633146106375760405162461bcd60e51b815260206004820152600e60248201526d696e666c6174696f6e206f6e6c7960901b6044820152606401610716565b6115276117b4565b47146106375760405162461bcd60e51b815260206004820152600e60248201526d6f7574206f662062616c616e636560901b6044820152606401610716565b3d604051818101604052816000823e821561157f578181f35b8181fd5b61158b610639565b6001600160a01b0316336001600160a01b0316146106375760405162461bcd60e51b815260206004820152600f60248201526e6f6e6c7920676f7665726e616e636560881b6044820152606401610716565b60006115ea8260026123ad565b6115f490846122b8565b6001600160401b03169050600061162d6201518060065461161591906121a2565b61161f85876122b8565b6001600160401b0316611b51565b9050600061165f6001600160401b0385166116488585612384565b600a546004546116589190612384565b9190611b67565b9050600061166e8760016122df565b90508062ffffff167fedcf03eed469135e307ec8dc425dc2c49560d3014b724a532f6f468fcc975df8600b60009054906101000a90046001600160a01b03166001600160a01b03166345bd2e196040518163ffffffff1660e01b8152600401600060405180830381865afa1580156116ea573d6000803e3d6000fd5b505050506040513d6000823e601f3d908101601f1916820160405261171291908101906123d8565b846040516117219291906124d1565b60405180910390a281600a600082825461173b91906121a2565b9091555050600d5460405163a02e86e560e01b815262ffffff83166004820152600160248201526001600160a01b039091169063a02e86e59084906044016000604051808303818588803b15801561179257600080fd5b505af11580156117a6573d6000803e3d6000fd5b505050505050505050505050565b6000600a546004546106d09190612384565b600080826040516020016117da9190612179565b6040516020818303038152906040528051906020012090506000805b86518110156118525786818151811061181157611811612557565b602002602001015183036118405785818151811061183157611831612557565b60200260200101519150611852565b8061184a8161256d565b9150506117f6565b506001600160a01b0381166118985760405162461bcd60e51b815260206004820152600c60248201526b61646472657373207a65726f60a01b6044820152606401610716565b9150505b9392505050565b6118ad8282611c8d565b6118ed82826040518060400160405280601a81526020017f466463496e666c6174696f6e436f6e66696775726174696f6e730000000000008152506117c6565b600b60006101000a8154816001600160a01b0302191690836001600160a01b0316021790555061195382826040518060400160405280601b81526020017f46646352657175657374466565436f6e66696775726174696f6e7300000000008152506117c6565b600c60006101000a8154816001600160a01b0302191690836001600160a01b031602179055506119a982826040518060400160405280600d81526020016c2932bbb0b93226b0b730b3b2b960991b8152506117c6565b600d80546001600160a01b0319166001600160a01b03929092169190911790555050565b600054600160b01b900460ff16156119fd573330146119ee576119ee612397565b6000805460ff60b01b19169055565b610637611583565b611a0d611583565b6000805460408051636221a54b60e01b81529051853593926001600160a01b031691636221a54b9160048083019260209291908290030181865afa158015611a59573d6000803e3d6000fd5b505050506040513d601f19601f82011682018060405250810190611a7d9190612265565b90506000611a8b82426121a2565b9050604051806040016040528082815260200186868080601f01602080910402602001604051908101604052809392919081815260200183838082843760009201829052509390945250506001600160e01b03198616815260026020908152604090912083518155908301519091506001820190611b0990826125d4565b509050507fed948300a3694aa01d4a6b258bfd664350193d770c0b51f8387277f6d83ea3b683828787604051611b429493929190612693565b60405180910390a15050505050565b6000818311611b60578161189c565b5090919050565b6000808211611bab5760405162461bcd60e51b815260206004820152601060248201526f4469766973696f6e206279207a65726f60801b6044820152606401610716565b83600003611bbb5750600061189c565b83830283858281611bce57611bce6126c6565b0403611bec57828181611be357611be36126c6565b0491505061189c565b506000611bf983866126dc565b90506000611c0784876126f0565b90506000611c1585876126dc565b90506000611c2386886126f0565b905085611c308285612704565b611c3a91906126dc565b611c448385612704565b611c4e8387612704565b88611c598689612704565b611c639190612704565b611c6d91906121a2565b611c7791906121a2565b611c8191906121a2565b98975050505050505050565b611c978282611cf1565b611ccd828260405180604001604052806013815260200172233630b932a9bcb9ba32b6b9a6b0b730b3b2b960691b8152506117c6565b600980546001600160a01b0319166001600160a01b03929092169190911790555050565b611d1d82826040518060400160405280600981526020016824b7333630ba34b7b760b91b8152506117c6565b600880546001600160a01b0319166001600160a01b03929092169190911790555050565b508054611d4d906121d2565b6000825580601f10611d5d575050565b601f01602090049060005260206000209081019061115c91905b80821115611d8b5760008155600101611d77565b5090565b600060208284031215611da157600080fd5b81356001600160e01b03198116811461189c57600080fd5b60008060208385031215611dcc57600080fd5b82356001600160401b0380821115611de357600080fd5b818501915085601f830112611df757600080fd5b813581811115611e0657600080fd5b866020828501011115611e1857600080fd5b60209290920196919550909350505050565b60005b83811015611e45578181015183820152602001611e2d565b50506000910152565b60008151808452611e66816020860160208601611e2a565b601f01601f19169290920160200192915050565b828152604060208201526000611e936040830184611e4e565b949350505050565b62ffffff8116811461115c57600080fd5b6001600160401b038116811461115c57600080fd5b600080600060608486031215611ed657600080fd5b8335611ee181611e9b565b92506020840135611ef181611eac565b91506040840135611f0181611eac565b809150509250925092565b634e487b7160e01b600052604160045260246000fd5b60405160a081016001600160401b0381118282101715611f4457611f44611f0c565b60405290565b604051601f8201601f191681016001600160401b0381118282101715611f7257611f72611f0c565b604052919050565b60006001600160401b03821115611f9357611f93611f0c565b5060051b60200190565b6001600160a01b038116811461115c57600080fd5b600082601f830112611fc357600080fd5b81356020611fd8611fd383611f7a565b611f4a565b82815260059290921b84018101918181019086841115611ff757600080fd5b8286015b8481101561201b57803561200e81611f9d565b8352918301918301611ffb565b509695505050505050565b6000806040838503121561203957600080fd5b82356001600160401b038082111561205057600080fd5b818501915085601f83011261206457600080fd5b81356020612074611fd383611f7a565b82815260059290921b8401810191818101908984111561209357600080fd5b948201945b838610156120b157853582529482019490820190612098565b965050860135925050808211156120c757600080fd5b506120d485828601611fb2565b9150509250929050565b60ff8116811461115c57600080fd5b6000602082840312156120ff57600080fd5b813561189c816120de565b60006020828403121561211c57600080fd5b813561189c81611f9d565b60006020828403121561213957600080fd5b5035919050565b6000806040838503121561215357600080fd5b823561215e81611f9d565b9150602083013561216e81611f9d565b809150509250929050565b60208152600061189c6020830184611e4e565b634e487b7160e01b600052601160045260246000fd5b808201808211156111e2576111e261218c565b6000602082840312156121c757600080fd5b815161189c81611f9d565b600181811c908216806121e657607f821691505b60208210810361220657634e487b7160e01b600052602260045260246000fd5b50919050565b6000825161221e818460208701611e2a565b9190910192915050565b81835281816020850137506000828201602090810191909152601f909101601f19169091010190565b602081526000611e93602083018486612228565b60006020828403121561227757600080fd5b5051919050565b60006020828403121561229057600080fd5b815161189c81611e9b565b6000602082840312156122ad57600080fd5b815161189c81611eac565b6001600160401b038281168282160390808211156122d8576122d861218c565b5092915050565b62ffffff8181168382160190808211156122d8576122d861218c565b60006020828403121561230d57600080fd5b815163ffffffff8116811461189c57600080fd5b63ffffffff8181168382160190808211156122d8576122d861218c565b604081526000612352604083018587612228565b9050826020830152949350505050565b60006020828403121561237457600080fd5b8151801515811461189c57600080fd5b818103818111156111e2576111e261218c565b634e487b7160e01b600052600160045260246000fd5b6001600160401b038181168382160280821691908281146123d0576123d061218c565b505092915050565b600060208083850312156123eb57600080fd5b82516001600160401b0381111561240157600080fd5b8301601f8101851361241257600080fd5b8051612420611fd382611f7a565b81815260a0918202830184019184820191908884111561243f57600080fd5b938501935b838510156124c55780858a03121561245c5760008081fd5b612464611f22565b85518152868601518782015260408087015161247f81611e9b565b90820152606086810151612492816120de565b908201526080868101516001600160e01b03811681146124b25760008081fd5b9082015283529384019391850191612444565b50979650505050505050565b6040808252835182820181905260009190606090818501906020808901865b838110156125435781518051865283810151848701528781015162ffffff16888701528681015160ff16878701526080908101516001600160e01b03169086015260a090940193908201906001016124f0565b505095909501959095525092949350505050565b634e487b7160e01b600052603260045260246000fd5b60006001820161257f5761257f61218c565b5060010190565b601f821115610e1857600081815260208120601f850160051c810160208610156125ad5750805b601f850160051c820191505b818110156125cc578281556001016125b9565b505050505050565b81516001600160401b038111156125ed576125ed611f0c565b612601816125fb84546121d2565b84612586565b602080601f831160018114612636576000841561261e5750858301515b600019600386901b1c1916600185901b1785556125cc565b600085815260208120601f198616915b8281101561266557888601518255948401946001909101908401612646565b50858210156126835787850151600019600388901b60f8161c191681555b5050505050600190811b01905550565b63ffffffff60e01b851681528360208201526060604082015260006126bc606083018486612228565b9695505050505050565b634e487b7160e01b600052601260045260246000fd5b6000826126eb576126eb6126c6565b500490565b6000826126ff576126ff6126c6565b500690565b80820281158282048414176111e2576111e261218c56fea2646970667358221220a46f0657f639e56c214d7ad7de679384f5f991aa0f939781ee8c2f428f5bf30b64736f6c63430008140033", + "linkReferences": {}, + "deployedLinkReferences": {} +} diff --git a/apps/ftso-reward-calculation-process/src/commands/ftso-reward-calculator-process.command.ts b/apps/ftso-reward-calculation-process/src/commands/ftso-reward-calculator-process.command.ts index 41a3e4e1..c3ef8be9 100644 --- a/apps/ftso-reward-calculation-process/src/commands/ftso-reward-calculator-process.command.ts +++ b/apps/ftso-reward-calculation-process/src/commands/ftso-reward-calculator-process.command.ts @@ -158,6 +158,14 @@ export class FtsoRewardCalculationProcessCommand extends CommandRunner { return JSON.parse(val); } + @Option({ + flags: "-z, --useFDCData [boolean]", + description: "Extracts data for FDC rewarding", + }) + parseUseFDCDataMode(val: string): boolean { + return JSON.parse(val); + } + @Option({ flags: "-l, --incrementalCalculation [boolean]", description: "Start incremental calculation for current reward epoch", diff --git a/apps/ftso-reward-calculation-process/src/interfaces/OptionalCommandOptions.ts b/apps/ftso-reward-calculation-process/src/interfaces/OptionalCommandOptions.ts index 7525c612..afa5de67 100644 --- a/apps/ftso-reward-calculation-process/src/interfaces/OptionalCommandOptions.ts +++ b/apps/ftso-reward-calculation-process/src/interfaces/OptionalCommandOptions.ts @@ -20,6 +20,7 @@ export interface OptionalCommandOptions { isWorker?: boolean; recoveryMode?: boolean; useFastUpdatesData?: boolean; + useFDCData?: boolean; tempRewardEpochFolder?: boolean; incrementalCalculation?: boolean; } diff --git a/apps/ftso-reward-calculation-process/src/libs/calculator-utils.ts b/apps/ftso-reward-calculation-process/src/libs/calculator-utils.ts index e5639d62..dd427b9f 100644 --- a/apps/ftso-reward-calculation-process/src/libs/calculator-utils.ts +++ b/apps/ftso-reward-calculation-process/src/libs/calculator-utils.ts @@ -114,6 +114,7 @@ export async function calculationOfRewardCalculationDataForRange( retryDelayMs: number, logger: Logger, useFastUpdatesData: boolean, + useFDCData: boolean, tempRewardEpochFolder = false, calculationFolder = CALCULATIONS_FOLDER() ) { @@ -131,6 +132,7 @@ export async function calculationOfRewardCalculationDataForRange( RANDOM_GENERATION_BENCHING_WINDOW(), dataManager, useFastUpdatesData, + useFDCData, tempRewardEpochFolder, calculationFolder ); diff --git a/apps/ftso-reward-calculation-process/src/libs/claim-utils.ts b/apps/ftso-reward-calculation-process/src/libs/claim-utils.ts index 62ae34e1..1e91ebb1 100644 --- a/apps/ftso-reward-calculation-process/src/libs/claim-utils.ts +++ b/apps/ftso-reward-calculation-process/src/libs/claim-utils.ts @@ -45,6 +45,7 @@ export async function calculateClaimsAndAggregate( retryDelayMs: number, logger: Logger, useFastUpdatesData: boolean, + useFDCData: boolean, keepRetrying = false ) { let done = false; @@ -62,6 +63,7 @@ export async function calculateClaimsAndAggregate( false, // don't merge true, //serializeResults useFastUpdatesData, + useFDCData, logger ); if (aggregateClaims) { @@ -139,7 +141,8 @@ export async function calculateAndAggregateRemainingClaims( true, // aggregate options.retryDelayMs, logger, - false //options.useFastUpdatesData + false, //options.useFastUpdatesData + false, //options.useFDCData ); logger.log(`Claims calculated for voting round ${tmpVotingRoundId}.`); diff --git a/apps/ftso-reward-calculation-process/src/libs/offer-utils.ts b/apps/ftso-reward-calculation-process/src/libs/offer-utils.ts index b4a18a0b..79f57d44 100644 --- a/apps/ftso-reward-calculation-process/src/libs/offer-utils.ts +++ b/apps/ftso-reward-calculation-process/src/libs/offer-utils.ts @@ -1,14 +1,16 @@ import { + granulatedPartialOfferMapForFDC, granulatedPartialOfferMapForFastUpdates, granulatedPartialOfferMapForRandomFeedSelection, } from "../../../../libs/ftso-core/src/reward-calculation/reward-offers"; import { + IFDCPartialRewardOfferForRound, IFUPartialRewardOfferForRound, IPartialRewardOfferForRound, } from "../../../../libs/ftso-core/src/utils/PartialRewardOffer"; import { RewardEpochDuration } from "../../../../libs/ftso-core/src/utils/RewardEpochDuration"; -import { FU_OFFERS_FILE, OFFERS_FILE } from "../../../../libs/ftso-core/src/utils/stat-info/constants"; -import { serializeGranulatedPartialOfferMap } from "../../../../libs/ftso-core/src/utils/stat-info/granulated-partial-offers-map"; +import { FDC_OFFERS_FILE, FU_OFFERS_FILE, OFFERS_FILE } from "../../../../libs/ftso-core/src/utils/stat-info/constants"; +import { serializeGranulatedPartialOfferMap, serializeGranulatedPartialOfferMapForFDC } from "../../../../libs/ftso-core/src/utils/stat-info/granulated-partial-offers-map"; import { RewardEpochInfo, deserializeRewardEpochInfo, @@ -56,6 +58,12 @@ export async function fullRoundOfferCalculation(options: OptionalCommandOptions) > = granulatedPartialOfferMapForFastUpdates(rewardEpochInfo, randomNumbers); serializeGranulatedPartialOfferMap(rewardEpochDuration, fuRewardOfferMap, false, FU_OFFERS_FILE); } + + if (options.useFDCData) { + const fdcRewardOfferMap: Map + = granulatedPartialOfferMapForFDC(rewardEpochInfo); + serializeGranulatedPartialOfferMapForFDC(rewardEpochDuration, fdcRewardOfferMap, false, FDC_OFFERS_FILE); + } } export function initializeTemplateOffers(rewardEpochInfo: RewardEpochInfo, endVotingRoundId: number) { diff --git a/apps/ftso-reward-calculation-process/src/libs/reward-claims-calculation.ts b/apps/ftso-reward-calculation-process/src/libs/reward-claims-calculation.ts index 0ae51d58..5b26e3fc 100644 --- a/apps/ftso-reward-calculation-process/src/libs/reward-claims-calculation.ts +++ b/apps/ftso-reward-calculation-process/src/libs/reward-claims-calculation.ts @@ -54,6 +54,7 @@ export async function runCalculateRewardClaimsTopJob(options: OptionalCommandOpt endVotingRoundId: endBatch, isWorker: true, useFastUpdatesData: options.useFastUpdatesData, + useFDCData: options.useFDCData, }; // logger.log(batchOptions); @@ -90,7 +91,8 @@ export async function runCalculateRewardClaimWorker( false, // don't aggregate options.retryDelayMs, logger, - options.useFastUpdatesData + options.useFastUpdatesData, + options.useFDCData ); recordProgress(rewardEpochId); } diff --git a/apps/ftso-reward-calculation-process/src/libs/reward-data-calculation.ts b/apps/ftso-reward-calculation-process/src/libs/reward-data-calculation.ts index d7290fab..33f24618 100644 --- a/apps/ftso-reward-calculation-process/src/libs/reward-data-calculation.ts +++ b/apps/ftso-reward-calculation-process/src/libs/reward-data-calculation.ts @@ -20,6 +20,7 @@ import { destroyStorage } from "../../../../libs/ftso-core/src/utils/stat-info/s import { OptionalCommandOptions } from "../interfaces/OptionalCommandOptions"; import { calculationOfRewardCalculationDataForRange } from "./calculator-utils"; import { RewardEpochManager } from "../../../../libs/ftso-core/src/RewardEpochManager"; +import { FDCInflationRewardsOffered } from "../../../../libs/ftso-core/src/events/FDCInflationRewardsOffered"; export async function runCalculateRewardCalculationTopJob( indexerClient: IndexerClientForRewarding, @@ -42,6 +43,7 @@ export async function runCalculateRewardCalculationTopJob( let fuInflationRewardsOffered: FUInflationRewardsOffered | undefined; let fuIncentivesOfferedData: IncentiveOffered[] | undefined; + let fdcInflationRewardsOffered: FDCInflationRewardsOffered | undefined; if (options.useFastUpdatesData) { const fuInflationRewardsOfferedResponse = await indexerClient.getFUInflationRewardsOfferedEvents( @@ -64,11 +66,25 @@ export async function runCalculateRewardCalculationTopJob( } fuIncentivesOfferedData = fuIncentivesOfferedResponse.data; } + if(options.useFDCData) { + const fdcInflationRewardsOfferedResponse = await indexerClient.getFDCInflationRewardsOfferedEvents( + rewardEpoch.previousRewardEpochStartedEvent.startVotingRoundId, + rewardEpoch.signingPolicy.startVotingRoundId - 1 + ); + if (fdcInflationRewardsOfferedResponse.status !== BlockAssuranceResult.OK) { + throw new Error(`Error while fetching FDCInflationRewardsOffered events for reward epoch ${rewardEpochId}`); + } + fdcInflationRewardsOffered = fdcInflationRewardsOfferedResponse.data.find(x => x.rewardEpochId === rewardEpochId); + if (fdcInflationRewardsOffered === undefined) { + throw new Error(`No FDCInflationRewardsOffered event found for reward epoch ${rewardEpochId}`); + } + } const rewardEpochInfo = getRewardEpochInfo( rewardEpoch, rewardEpochDuration.endVotingRoundId, fuInflationRewardsOffered, - fuIncentivesOfferedData + fuIncentivesOfferedData, + fdcInflationRewardsOffered ); serializeRewardEpochInfo(rewardEpochId, rewardEpochInfo, options.tempRewardEpochFolder); setRewardCalculationStatus( @@ -121,6 +137,7 @@ export async function runCalculateRewardCalculationTopJob( endVotingRoundId: endBatch, isWorker: true, useFastUpdatesData: options.useFastUpdatesData, + useFDCData: options.useFDCData, tempRewardEpochFolder: options.tempRewardEpochFolder, }; // logger.log(batchOptions); @@ -148,6 +165,7 @@ export async function runCalculateRewardCalculationDataWorker( options.retryDelayMs, logger, options.useFastUpdatesData, + options.useFDCData, options.tempRewardEpochFolder ); logger.log( diff --git a/apps/ftso-reward-calculation-process/src/services/calculator.service.ts b/apps/ftso-reward-calculation-process/src/services/calculator.service.ts index 54ba96d8..2fa66d65 100644 --- a/apps/ftso-reward-calculation-process/src/services/calculator.service.ts +++ b/apps/ftso-reward-calculation-process/src/services/calculator.service.ts @@ -162,7 +162,8 @@ export class CalculatorService { state.votingRoundId, options.retryDelayMs, logger, - options.useFastUpdatesData + options.useFastUpdatesData, + options.useFDCData ); // The call above may create additional folder state.maxVotingRoundIdFolder = Math.max(state.maxVotingRoundIdFolder, state.votingRoundId); @@ -205,6 +206,7 @@ export class CalculatorService { batchSize: options.batchSize, numberOfWorkers: options.numberOfWorkers, useFastUpdatesData: options.useFastUpdatesData, + useFDCData: options.useFDCData, } as OptionalCommandOptions; const rewardEpochDuration = await runCalculateRewardCalculationTopJob( this.indexerClient, @@ -217,6 +219,7 @@ export class CalculatorService { newOptions.tempRewardEpochFolder = true; newOptions.useExpectedEndIfNoSigningPolicyAfter = true; newOptions.useFastUpdatesData = false; + newOptions.useFDCData = false; const rewardEpochDuration2 = await runCalculateRewardCalculationTopJob( this.indexerClient, this.rewardEpochManager, @@ -234,6 +237,7 @@ export class CalculatorService { batchSize: options.batchSize, numberOfWorkers: options.numberOfWorkers, useFastUpdatesData: options.useFastUpdatesData, + useFDCData: options.useFDCData, } as OptionalCommandOptions; await runCalculateRewardClaimsTopJob(adaptedOptions); } diff --git a/libs/ftso-core/src/DataManager.ts b/libs/ftso-core/src/DataManager.ts index 2c474dce..a8c45c87 100644 --- a/libs/ftso-core/src/DataManager.ts +++ b/libs/ftso-core/src/DataManager.ts @@ -67,6 +67,8 @@ export interface CommitsAndReveals { export interface CommitAndRevealSubmissionsMappingsForRange { votingRoundIdToCommits: Map; votingRoundIdToReveals: Map; + submit1?: SubmissionData[]; + submit2?: SubmissionData[]; } export interface SignAndFinalizeSubmissionData { @@ -279,6 +281,8 @@ export class DataManager { data: { votingRoundIdToCommits, votingRoundIdToReveals, + submit1: commitSubmissionResponse.data, + submit2: revealSubmissionResponse.data, }, }; } diff --git a/libs/ftso-core/src/DataManagerForRewarding.ts b/libs/ftso-core/src/DataManagerForRewarding.ts index 3cebef63..8d46859e 100644 --- a/libs/ftso-core/src/DataManagerForRewarding.ts +++ b/libs/ftso-core/src/DataManagerForRewarding.ts @@ -1,16 +1,19 @@ import { DataAvailabilityStatus, DataManager, DataMangerResponse, SignAndFinalizeSubmissionData } from "./DataManager"; -import { BlockAssuranceResult } from "./IndexerClient"; +import { BlockAssuranceResult, SubmissionData } from "./IndexerClient"; import { IndexerClientForRewarding } from "./IndexerClientForRewarding"; import { RewardEpoch } from "./RewardEpoch"; import { RewardEpochManager } from "./RewardEpochManager"; import { ContractMethodNames } from "./configs/contracts"; -import { ADDITIONAL_REWARDED_FINALIZATION_WINDOWS, EPOCH_SETTINGS, FTSO2_PROTOCOL_ID } from "./configs/networks"; +import { ADDITIONAL_REWARDED_FINALIZATION_WINDOWS, EPOCH_SETTINGS, FDC_PROTOCOL_ID, FTSO2_PROTOCOL_ID } from "./configs/networks"; import { DataForCalculations, DataForRewardCalculation, + FDCDataForVotingRound, FastUpdatesDataForVotingRound, } from "./data-calculation-interfaces"; import { ILogger } from "./utils/ILogger"; +import { errorString } from "./utils/error"; +import { Address } from "./voting-types"; /** * Helps in extracting data in a consistent way for FTSO scaling feed median calculations, random number calculation and rewarding. @@ -85,6 +88,7 @@ export class DataManagerForRewarding extends DataManager { } for (let votingRoundId = firstVotingRoundId; votingRoundId <= lastVotingRoundId; votingRoundId++) { + //////// FTSO Scaling //////// const commits = mappingsResponse.data.votingRoundIdToCommits.get(votingRoundId) || []; const reveals = mappingsResponse.data.votingRoundIdToReveals.get(votingRoundId) || []; // this.logger.debug(`Commits for voting round ${votingRoundId}: ${JSON.stringify(commits)}`); @@ -110,11 +114,14 @@ export class DataManagerForRewarding extends DataManager { if (!process.env.REMOVE_ANNOYING_MESSAGES) { this.logger.debug(`Valid reveals from: ${JSON.stringify(Array.from(partialData.validEligibleReveals.keys()))}`); } + //////// FDC //////// + const validEligibleBitVotes: Map = this.extractValidEligibleBitVotes(mappingsResponse.data.submit2, rewardEpoch); const dataForRound = { ...partialData, randomGenerationBenchingWindow, benchingWindowRevealOffenders, rewardEpoch, + validEligibleBitVotes } as DataForCalculations; result.push(dataForRound); } @@ -142,7 +149,8 @@ export class DataManagerForRewarding extends DataManager { firstVotingRoundId: number, lastVotingRoundId: number, randomGenerationBenchingWindow: number, - useFastUpdatesData: boolean + useFastUpdatesData: boolean, + useFDCData: boolean ): Promise> { const dataForCalculationsResponse = await this.getDataForCalculationsForVotingRoundRange( firstVotingRoundId, @@ -164,6 +172,7 @@ export class DataManagerForRewarding extends DataManager { }; } let fastUpdatesData: FastUpdatesDataForVotingRound[] = []; + let fdcData: FDCDataForVotingRound[] = []; if (useFastUpdatesData) { const fastUpdatesDataResponse = await this.getFastUpdatesDataForVotingRoundRange( firstVotingRoundId, @@ -176,6 +185,20 @@ export class DataManagerForRewarding extends DataManager { } fastUpdatesData = fastUpdatesDataResponse.data; } + if (useFDCData) { + const fdcDataResponse = await this.getFDCDataForVotingRoundRange( + firstVotingRoundId, + lastVotingRoundId + ); + if (fdcDataResponse.status !== DataAvailabilityStatus.OK) { + return { + status: fdcDataResponse.status, + }; + } + fdcData = fdcDataResponse.data; + } + + /// const result: DataForRewardCalculation[] = []; let startIndexSignatures = 0; let endIndexSignatures = 0; @@ -236,6 +259,7 @@ export class DataManagerForRewarding extends DataManager { finalizations, firstSuccessfulFinalization, fastUpdatesData: fastUpdatesData[votingRoundId - firstVotingRoundId], + fdcData: fdcData[votingRoundId - firstVotingRoundId], }; result.push(dataForRound); } @@ -331,4 +355,66 @@ export class DataManagerForRewarding extends DataManager { data: result, }; } + + public async getFDCDataForVotingRoundRange( + firstVotingRoundId: number, + lastVotingRoundId: number + ): Promise> { + + const attestationRequestsResponse = await this.indexerClient.getAttestationRequestEvents(firstVotingRoundId, lastVotingRoundId); + if (attestationRequestsResponse.status !== BlockAssuranceResult.OK) { + return { + status: DataAvailabilityStatus.NOT_OK, + }; + } + // const feedUpdates = await this.indexerClient.getFastUpdateFeedsSubmittedEvents( + // firstVotingRoundId, + // lastVotingRoundId + // ); + // if (feedUpdates.status !== BlockAssuranceResult.OK) { + // return { + // status: DataAvailabilityStatus.NOT_OK, + // }; + // } + const result: FDCDataForVotingRound[] = []; + for (let votingRoundId = firstVotingRoundId; votingRoundId <= lastVotingRoundId; votingRoundId++) { + // const fastUpdateFeeds = attestationRequestsResponse.data[votingRoundId - firstVotingRoundId]; + // const fastUpdateSubmissions = feedUpdates.data[votingRoundId - firstVotingRoundId]; + const value: any = {//FDCDataForVotingRound = { + votingRoundId, + // attestationRequests: attestationRequestsResponse.data[votingRoundId - firstVotingRoundId], + }; + result.push(value); + } + return { + status: DataAvailabilityStatus.OK, + data: result, + }; + } + + public extractValidEligibleBitVotes(submissionDataArray: SubmissionData[], rewardEpoch: RewardEpoch): Map { + const voterToLastBitVote = new Map(); + for (const submission of submissionDataArray) { + for (const message of submission.messages) { + if ( + message.protocolId === FDC_PROTOCOL_ID && + message.votingRoundId + 1 === submission.votingEpochIdFromTimestamp + ) { + try { + const submitAddress = submission.submitAddress.toLowerCase(); + if (rewardEpoch.isEligibleSubmitAddress(submitAddress)) { + voterToLastBitVote.set(submitAddress, message.payload); + } else { + if (!process.env.REMOVE_ANNOYING_MESSAGES) { + this.logger.warn(`Non-eligible commit found for address ${submitAddress}`); + } + } + } catch (e) { + this.logger.warn(`Unparsable reveal message: ${message.payload}, error: ${errorString(e)}`); + } + } + } + } + return voterToLastBitVote; + } } diff --git a/libs/ftso-core/src/IndexerClientForRewarding.ts b/libs/ftso-core/src/IndexerClientForRewarding.ts index bedaf466..e7f4801d 100644 --- a/libs/ftso-core/src/IndexerClientForRewarding.ts +++ b/libs/ftso-core/src/IndexerClientForRewarding.ts @@ -6,6 +6,8 @@ import { FastUpdateFeeds } from "./events/FastUpdateFeeds"; import { FastUpdateFeedsSubmitted } from "./events/FastUpdateFeedsSubmitted"; import { IncentiveOffered } from "./events/IncentiveOffered"; import { FUInflationRewardsOffered } from "./events/FUInflationRewardsOffered"; +import { FDCInflationRewardsOffered } from "./events/FDCInflationRewardsOffered"; +import { AttestationRequest } from "./events/AttestationRequest"; export class IndexerClientForRewarding extends IndexerClient { constructor( @@ -148,4 +150,53 @@ export class IndexerClientForRewarding extends IndexerClient { data, }; } + + /** + * Extract AttestationRequest events from the indexer that match the range of voting rounds. + */ + public async getAttestationRequestEvents( + startVotingRoundId: number, + endVotingRoundId: number + ): Promise> { + const startTime = EPOCH_SETTINGS().votingEpochStartSec(startVotingRoundId); + // strictly containing in the range + const endTime = EPOCH_SETTINGS().votingEpochStartSec(endVotingRoundId + 1) - 1; + const eventName = AttestationRequest.eventName; + const status = await this.ensureEventRange(startTime, endTime); + const result = await this.queryEvents(CONTRACTS.FdcHub, eventName, startTime, endTime); + if (status !== BlockAssuranceResult.OK) { + return { status }; + } + + const data = result.map(event => AttestationRequest.fromRawEvent(event)); + return { + status, + data, + }; + } + + + /** + * Extract FDCInflationRewardsOffered events from the indexer that match the range of voting rounds. + */ + public async getFDCInflationRewardsOfferedEvents( + startVotingRoundId: number, + endVotingRoundId: number + ): Promise> { + const startTime = EPOCH_SETTINGS().votingEpochStartSec(startVotingRoundId); + // strictly containing in the range + const endTime = EPOCH_SETTINGS().votingEpochStartSec(endVotingRoundId + 1) - 1; + const eventName = FDCInflationRewardsOffered.eventName; + const status = await this.ensureEventRange(startTime, endTime); + const result = await this.queryEvents(CONTRACTS.FdcHub, eventName, startTime, endTime); + if (status !== BlockAssuranceResult.OK) { + return { status }; + } + const data = result.map(event => FDCInflationRewardsOffered.fromRawEvent(event)); + return { + status, + data, + }; + } + } diff --git a/libs/ftso-core/src/configs/contracts.ts b/libs/ftso-core/src/configs/contracts.ts index 08d9f2a4..8483db8f 100644 --- a/libs/ftso-core/src/configs/contracts.ts +++ b/libs/ftso-core/src/configs/contracts.ts @@ -45,6 +45,11 @@ interface FastUpdateIncentiveManagerDefinition { address: ContractAddress; } +interface FdcHubDefinition { + name: "FdcHub"; + address: ContractAddress; +} + export type ContractDefinitions = | FlareSystemsManagerDefinition | FtsoRewardOffersManagerDefinition @@ -56,7 +61,8 @@ export type ContractDefinitions = | ProtocolMerkleStructsDefinition | FtsoMerkleStructsDefinition | FastUpdaterDefinition - | FastUpdateIncentiveManagerDefinition; + | FastUpdateIncentiveManagerDefinition + | FdcHubDefinition; export type ContractDefinitionsNames = | FlareSystemsManagerDefinition["name"] @@ -69,7 +75,8 @@ export type ContractDefinitionsNames = | ProtocolMerkleStructsDefinition["name"] | FtsoMerkleStructsDefinition["name"] | FastUpdaterDefinition["name"] - | FastUpdateIncentiveManagerDefinition["name"]; + | FastUpdateIncentiveManagerDefinition["name"] + | FdcHubDefinition["name"]; export enum ContractMethodNames { submit1 = "submit1", @@ -103,4 +110,5 @@ export interface NetworkContractAddresses { ProtocolMerkleStructs: ProtocolMerkleStructsDefinition; FastUpdater: FastUpdaterDefinition; FastUpdateIncentiveManager: FastUpdateIncentiveManagerDefinition; + FdcHub: FdcHubDefinition; } diff --git a/libs/ftso-core/src/configs/networks.ts b/libs/ftso-core/src/configs/networks.ts index e4a606c5..31900052 100644 --- a/libs/ftso-core/src/configs/networks.ts +++ b/libs/ftso-core/src/configs/networks.ts @@ -15,6 +15,7 @@ const TEST_CONFIG: NetworkContractAddresses = { ProtocolMerkleStructs: { name: "ProtocolMerkleStructs", address: "" }, FastUpdater: { name: "FastUpdater", address: "" }, FastUpdateIncentiveManager: { name: "FastUpdateIncentiveManager", address: "" }, + FdcHub: { name: "FdcHub", address: "" }, }; const COSTON_CONFIG: NetworkContractAddresses = { @@ -32,6 +33,7 @@ const COSTON_CONFIG: NetworkContractAddresses = { name: "FastUpdateIncentiveManager", address: "0x8c45666369B174806E1AB78D989ddd79a3267F3b", }, + FdcHub: { name: "FdcHub", address: "0x1c78A073E3BD2aCa4cc327d55FB0cD4f0549B55b" }, }; const COSTON2_CONFIG: NetworkContractAddresses = { @@ -49,6 +51,7 @@ const COSTON2_CONFIG: NetworkContractAddresses = { name: "FastUpdateIncentiveManager", address: "0xC71C1C6E6FB31eF6D948B2C074fA0d38a07D4f68", }, + FdcHub: { name: "FdcHub", address: "" }, }; const SONGBIRD_CONFIG: NetworkContractAddresses = { @@ -66,6 +69,7 @@ const SONGBIRD_CONFIG: NetworkContractAddresses = { name: "FastUpdateIncentiveManager", address: "0x596C70Ad6fFFdb9b6158F1Dfd0bc32cc72B82006", }, + FdcHub: { name: "FdcHub", address: "" }, }; const FLARE_CONFIG: NetworkContractAddresses = { @@ -83,6 +87,7 @@ const FLARE_CONFIG: NetworkContractAddresses = { name: "FastUpdateIncentiveManager", address: "0xd648e8ACA486Ce876D641A0F53ED1F2E9eF4885D", }, + FdcHub: { name: "FdcHub", address: "" }, }; export type networks = "local-test" | "from-env" | "coston2" | "coston" | "songbird" | "flare"; @@ -132,6 +137,22 @@ const contracts = () => { !isValidContractAddress(process.env.FTSO_CA_VOTER_REGISTRY_ADDRESS) ) throw new Error("FTSO_CA_VOTER_REGISTRY_ADDRESS value is not valid contract address"); + if ( + !process.env.FTSO_CA_FAST_UPDATER_ADDRESS || + !isValidContractAddress(process.env.FTSO_CA_FAST_UPDATER_ADDRESS) + ) + throw new Error("FTSO_CA_FAST_UPDATER_ADDRESS value is not valid contract address"); + if ( + !process.env.FTSO_CA_FAST_UPDATE_INCENTIVE_MANAGER_ADDRESS || + !isValidContractAddress(process.env.FTSO_CA_FAST_UPDATE_INCENTIVE_MANAGER_ADDRESS) + ) + throw new Error("FTSO_CA_FAST_UPDATE_INCENTIVE_MANAGER_ADDRESS value is not valid contract address"); + if ( + !process.env.FTSO_CA_FDC_HUB_ADDRESS || + !isValidContractAddress(process.env.FTSO_CA_FDC_HUB_ADDRESS) + ) + throw new Error("FTSO_CA_FDC_HUB_ADDRESS value is not valid contract address"); + const CONTRACT_CONFIG: NetworkContractAddresses = { FlareSystemsManager: { name: "FlareSystemsManager", address: process.env.FTSO_CA_FTSO_SYSTEMS_MANAGER_ADDRESS }, FtsoRewardOffersManager: { @@ -153,13 +174,14 @@ const contracts = () => { name: "FastUpdateIncentiveManager", address: process.env.FTSO_CA_FAST_UPDATE_INCENTIVE_MANAGER_ADDRESS, }, + FdcHub: { name: "FdcHub", address: process.env.FTSO_CA_FDC_HUB_ADDRESS }, }; return CONTRACT_CONFIG; } default: // Ensure exhaustive checking // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars - ((_: never): void => {})(network); + ((_: never): void => { })(network); } }; @@ -185,7 +207,7 @@ const ftso2ProtocolId = () => { default: // Ensure exhaustive checking // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars - ((_: never): void => {})(network); + ((_: never): void => { })(network); } }; @@ -205,13 +227,35 @@ const ftso2FastUpdatesProtocolId = () => { default: // Ensure exhaustive checking // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars - ((_: never): void => {})(network); + ((_: never): void => { })(network); } }; // Protocol id for FTSO2 fast updates export const FTSO2_FAST_UPDATES_PROTOCOL_ID = ftso2FastUpdatesProtocolId(); + +const FDCProtocolId = () => { + const network = process.env.NETWORK as networks; + switch (network) { + case "coston": + case "from-env": + case "local-test": + case "coston2": + case "songbird": + case "flare": + return 200; + default: + // Ensure exhaustive checking + // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars + ((_: never): void => { })(network); + } +}; + +// Protocol id for FDC +export const FDC_PROTOCOL_ID = FDCProtocolId(); + + const epochSettings = () => { const network = process.env.NETWORK as networks; switch (network) { @@ -266,7 +310,7 @@ const epochSettings = () => { default: // Ensure exhaustive checking // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars - ((_: never): void => {})(network); + ((_: never): void => { })(network); } }; @@ -310,7 +354,7 @@ const randomGenerationBenchingWindow = () => { default: // Ensure exhaustive checking // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars - ((_: never): void => {})(network); + ((_: never): void => { })(network); } }; @@ -345,7 +389,7 @@ const initialRewardEpochId = () => { default: // Ensure exhaustive checking // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars - ((_: never): void => {})(network); + ((_: never): void => { })(network); } }; @@ -364,7 +408,7 @@ const burnAddress = () => { default: // Ensure exhaustive checking // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars - ((_: never): void => {})(network); + ((_: never): void => { })(network); } }; @@ -388,7 +432,7 @@ const additionalRewardFinalizationWindows = () => { default: // Ensure exhaustive checking // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars - ((_: never): void => {})(network); + ((_: never): void => { })(network); } }; @@ -457,7 +501,7 @@ const penaltyFactor = () => { default: // Ensure exhaustive checking // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars - ((_: never): void => {})(network); + ((_: never): void => { })(network); } }; @@ -493,7 +537,7 @@ const gracePeriodForSignaturesDurationSec = () => { default: // Ensure exhaustive checking // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars - ((_: never): void => {})(network); + ((_: never): void => { })(network); } }; @@ -531,7 +575,7 @@ const gracePeriodForFinalizationDurationSec = () => { default: // Ensure exhaustive checking // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars - ((_: never): void => {})(network); + ((_: never): void => { })(network); } }; @@ -584,7 +628,7 @@ const minimalRewardedNonConsensusDepositedSignaturesPerHashBips = () => { default: // Ensure exhaustive checking // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars - ((_: never): void => {})(network); + ((_: never): void => { })(network); } }; @@ -619,7 +663,7 @@ const finalizationVoterSelectionThresholdWeightBips = () => { default: // Ensure exhaustive checking // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars - ((_: never): void => {})(network); + ((_: never): void => { })(network); } }; diff --git a/libs/ftso-core/src/data-calculation-interfaces.ts b/libs/ftso-core/src/data-calculation-interfaces.ts index 6c35651a..8a636f09 100644 --- a/libs/ftso-core/src/data-calculation-interfaces.ts +++ b/libs/ftso-core/src/data-calculation-interfaces.ts @@ -1,6 +1,7 @@ import { ISignaturePayload } from "../../fsp-utils/src/SignaturePayload"; import { GenericSubmissionData, ParsedFinalizationData } from "./IndexerClient"; import { RewardEpoch } from "./RewardEpoch"; +import { AttestationRequest } from "./events/AttestationRequest"; import { IRevealData } from "./utils/RevealData"; import { Address, Feed, MessageHash } from "./voting-types"; @@ -26,15 +27,21 @@ export interface DataForCalculations extends DataForCalculationsPartial { benchingWindowRevealOffenders: Set
; // Reward epoch rewardEpoch: RewardEpoch; + // valid eligible bit-votes + validEligibleBitVotes?: Map; } export interface DataForRewardCalculation { + // FTSO Scaling dataForCalculations: DataForCalculations; signatures: Map[]>; finalizations: ParsedFinalizationData[]; // might be undefined, if such finalization does not exist in an observed range firstSuccessfulFinalization?: ParsedFinalizationData; + // FAST UPDATES fastUpdatesData?: FastUpdatesDataForVotingRound; + // FDC + fdcData?: FDCDataForVotingRound; } export interface FastUpdatesDataForVotingRound { @@ -44,6 +51,32 @@ export interface FastUpdatesDataForVotingRound { signingPolicyAddressesSubmitted: string[]; } +export interface FDCDataForVotingRound { + votingRoundId: number; + // List of attestation requests to be processed + attestationRequests: AttestationRequest[]; + // List of bitvotes for the consensus bitvote + // Only last bitvote for each data provider is included + // submit address is used to assign to data provider + bitVotes: GenericSubmissionData[]; + // signature data, include the unsigned message, which should be consensus bitvote + // Might be multiple signatures by the same data provider + // All signatures for FDC protocol in the observed range are included and sorted in the order of arrival + signatures: GenericSubmissionData[]; + // First successful finalization + // might be undefined, if such finalization does not exist in an observed range + // If defined then we can check if signatures are correct + firstSuccessfulFinalization?: ParsedFinalizationData; + // All finalizations in the observed range + finalizations: ParsedFinalizationData[]; + // Filtered signatures that match the first finalized protocol Merkle root message + // One per eligible data provider + rewardedSignatures?: GenericSubmissionData[]; + // Majority bitvote attached to the finalized signatures + // All signers that have unmatching majority bitvote are considered as offenders + majorityBitVote?: string; +} + export interface FUFeedValue { feedId: string; value: bigint; diff --git a/libs/ftso-core/src/events/AttestationRequest.ts b/libs/ftso-core/src/events/AttestationRequest.ts new file mode 100644 index 00000000..baa5fdbf --- /dev/null +++ b/libs/ftso-core/src/events/AttestationRequest.ts @@ -0,0 +1,28 @@ +import { CONTRACTS } from "../configs/networks"; +import { decodeEvent } from "../utils/EncodingUtils"; +import { RawEventConstructible } from "./RawEventConstructible"; + + +export class AttestationRequest extends RawEventConstructible { + static eventName = "AttestationRequest"; + constructor(data: any) { + super(); + this.data = data.data; + this.fee = BigInt(data.fee); + } + + static fromRawEvent(event: any): AttestationRequest { + return decodeEvent( + CONTRACTS.FdcHub.name, + AttestationRequest.eventName, + event, + (data: any) => new AttestationRequest(data) + ); + } + + // Feed values in the order of feedIds + data: string; + + // feed decimals + fee: bigint; +} diff --git a/libs/ftso-core/src/events/FDCInflationRewardsOffered.ts b/libs/ftso-core/src/events/FDCInflationRewardsOffered.ts new file mode 100644 index 00000000..1897d564 --- /dev/null +++ b/libs/ftso-core/src/events/FDCInflationRewardsOffered.ts @@ -0,0 +1,53 @@ +import { CONTRACTS } from "../configs/networks"; +import { decodeEvent } from "../utils/EncodingUtils"; +import { RawEventConstructible } from "./RawEventConstructible"; + +export interface FdcConfiguration { + // attestation type + attestationType: string; + // source + source: string; + // inflation share for this configuration + inflationShare: number; + // minimal reward eligibility threshold in number of request + minRequestsThreshold: number; + // mode (additional settings interpreted on the client side off-chain) + mode: bigint; +} + +export class FDCInflationRewardsOffered extends RawEventConstructible { + static eventName = "InflationRewardsOffered"; + constructor(data: any) { + super(); + this.rewardEpochId = Number(data.rewardEpochId); + this.fdcConfigurations = data.fdcConfigurations.map((v: any) => { + const config: FdcConfiguration = { + attestationType: v.attestationType, + source: v.source, + inflationShare: Number(v.inflationShare), + minRequestsThreshold: Number(v.minRequestsThreshold), + mode: BigInt(v.mode), + }; + return config; + }); + this.amount = BigInt(data.amount); + } + + static fromRawEvent(event: any): FDCInflationRewardsOffered { + return decodeEvent( + CONTRACTS.FdcHub.name, + FDCInflationRewardsOffered.eventName, + event, + (data: any) => new FDCInflationRewardsOffered(data) + ); + } + + // reward epoch id + rewardEpochId: number; + + // fdc configurations + fdcConfigurations: FdcConfiguration[]; + + // amount (in wei) of reward in native coin + amount: bigint; +} diff --git a/libs/ftso-core/src/events/FUInflationRewardsOffered.ts b/libs/ftso-core/src/events/FUInflationRewardsOffered.ts index 47809cbc..d715651f 100644 --- a/libs/ftso-core/src/events/FUInflationRewardsOffered.ts +++ b/libs/ftso-core/src/events/FUInflationRewardsOffered.ts @@ -42,6 +42,6 @@ export class FUInflationRewardsOffered extends RawEventConstructible { // Feed values in the order of feedIds feedConfigurations: FastUpdateFeedConfiguration[]; - // feed decimals + // amount amount: bigint; } diff --git a/libs/ftso-core/src/reward-calculation/reward-calculation.ts b/libs/ftso-core/src/reward-calculation/reward-calculation.ts index 6cb5193d..cebfafbb 100644 --- a/libs/ftso-core/src/reward-calculation/reward-calculation.ts +++ b/libs/ftso-core/src/reward-calculation/reward-calculation.ts @@ -89,6 +89,7 @@ export async function partialRewardClaimsForVotingRound( merge = true, serializeResults = false, useFastUpdatesData = false, + useFDCData = false, logger: ILogger = console, calculationFolder = CALCULATIONS_FOLDER() ): Promise { @@ -294,6 +295,12 @@ export async function partialRewardClaimsForVotingRound( } } } + + if(useFDCData) { + // TODO + // ddd + } + if (serializeResults) { serializePartialClaimsForVotingRoundId(rewardEpochId, votingRoundId, allRewardClaims, calculationFolder); } @@ -363,6 +370,7 @@ export async function prepareDataForRewardCalculationsForRange( randomGenerationBenchingWindow: number, dataManager: DataManagerForRewarding, useFastUpdatesData: boolean, + useFDCData: boolean, tempRewardEpochFolder = false, calculationFolder = CALCULATIONS_FOLDER() ) { @@ -370,7 +378,8 @@ export async function prepareDataForRewardCalculationsForRange( firstVotingRoundId, lastVotingRoundId, randomGenerationBenchingWindow, - useFastUpdatesData + useFastUpdatesData, + useFDCData ); if (rewardDataForCalculationResponse.status !== DataAvailabilityStatus.OK) { throw new Error(`Data availability status is not OK: ${rewardDataForCalculationResponse.status}`); diff --git a/libs/ftso-core/src/reward-calculation/reward-offers.ts b/libs/ftso-core/src/reward-calculation/reward-offers.ts index 0a601c5c..1ded6d24 100644 --- a/libs/ftso-core/src/reward-calculation/reward-offers.ts +++ b/libs/ftso-core/src/reward-calculation/reward-offers.ts @@ -1,6 +1,7 @@ import { BURN_ADDRESS, FINALIZATION_BIPS, SIGNING_BIPS, TOTAL_BIPS } from "../configs/networks"; import { InflationRewardsOffered } from "../events"; import { + IFDCPartialRewardOfferForRound, IFUPartialRewardOfferForRound, IPartialRewardOfferForEpoch, IPartialRewardOfferForRound, @@ -292,3 +293,38 @@ export function granulatedPartialOfferMapForFastUpdates( } return rewardOfferMap; } + +export function granulatedPartialOfferMapForFDC( + rewardEpochInfo: RewardEpochInfo, +): Map { + const startVotingRoundId = rewardEpochInfo.signingPolicy.startVotingRoundId; + const endVotingRoundId = rewardEpochInfo.endVotingRoundId; + if (startVotingRoundId === undefined || endVotingRoundId === undefined) { + throw new Error("Start or end voting round id is undefined"); + } + + // Calculate total amount of rewards for the reward epoch + let totalAmount = rewardEpochInfo.fdcInflationRewardsOffered.amount; + + if (process.env.TEST_FDC_INFLATION_REWARD_AMOUNT) { + totalAmount = BigInt(process.env.TEST_FDC_INFLATION_REWARD_AMOUNT); + } + // Create a map of votingRoundId -> rewardOffer + const rewardOfferMap = new Map(); + const numberOfVotingRounds = endVotingRoundId - startVotingRoundId + 1; + const sharePerOne: bigint = totalAmount / BigInt(numberOfVotingRounds); + const remainder: number = Number(totalAmount % BigInt(numberOfVotingRounds)); + + for (let votingRoundId = startVotingRoundId; votingRoundId <= endVotingRoundId; votingRoundId++) { + let amount = sharePerOne + (votingRoundId - startVotingRoundId < remainder ? 1n : 0n); + + // Create adapted offer with selected feed + const feedOfferForVoting: IFDCPartialRewardOfferForRound = { + votingRoundId, + amount, + }; + + rewardOfferMap.set(votingRoundId, feedOfferForVoting); + } + return rewardOfferMap; +} diff --git a/libs/ftso-core/src/utils/ABICache.ts b/libs/ftso-core/src/utils/ABICache.ts index bb4006a1..571c1ef8 100644 --- a/libs/ftso-core/src/utils/ABICache.ts +++ b/libs/ftso-core/src/utils/ABICache.ts @@ -17,6 +17,8 @@ import { FastUpdateFeedsSubmitted } from "../events/FastUpdateFeedsSubmitted"; import { FastUpdateFeeds } from "../events/FastUpdateFeeds"; import { FUInflationRewardsOffered } from "../events/FUInflationRewardsOffered"; import { IncentiveOffered } from "../events/IncentiveOffered"; +import { AttestationRequest } from "../events/AttestationRequest"; +import { FDCInflationRewardsOffered } from "../events/FDCInflationRewardsOffered"; type AbiItem = AbiFunctionFragment | AbiEventFragment; @@ -62,6 +64,8 @@ export class ABICache { ["FastUpdater", undefined, FastUpdateFeeds.eventName], ["FastUpdateIncentiveManager", undefined, FUInflationRewardsOffered.eventName], ["FastUpdateIncentiveManager", undefined, IncentiveOffered.eventName], + ["FdcHub", undefined, AttestationRequest.eventName], + ["FdcHub", undefined, FDCInflationRewardsOffered.eventName], ]; for (const [contractName, functionName, eventName] of cachedABIs) { diff --git a/libs/ftso-core/src/utils/PartialRewardOffer.ts b/libs/ftso-core/src/utils/PartialRewardOffer.ts index 2ec7228c..ef91ff6b 100644 --- a/libs/ftso-core/src/utils/PartialRewardOffer.ts +++ b/libs/ftso-core/src/utils/PartialRewardOffer.ts @@ -39,6 +39,13 @@ export interface IFUPartialRewardOfferForRound { shouldBeBurned?: boolean; } +export interface IFDCPartialRewardOfferForRound { + votingRoundId: number; + amount: bigint; + shouldBeBurned?: boolean; +} + + export namespace PartialRewardOffer { export function fromRewardOffered(rewardOffer: RewardsOffered): IPartialRewardOfferForEpoch { return { diff --git a/libs/ftso-core/src/utils/stat-info/constants.ts b/libs/ftso-core/src/utils/stat-info/constants.ts index 9060bc92..36921af2 100644 --- a/libs/ftso-core/src/utils/stat-info/constants.ts +++ b/libs/ftso-core/src/utils/stat-info/constants.ts @@ -1,5 +1,6 @@ export const OFFERS_FILE = "offers.json"; export const FU_OFFERS_FILE = "fast-updates-offers.json"; +export const FDC_OFFERS_FILE = "fdc-offers.json"; export const CLAIMS_FILE = "claims.json"; export const AGGREGATED_CLAIMS_FILE = "aggregated-claims.json"; export const FEED_VALUES_FILE = "feed-values.json"; diff --git a/libs/ftso-core/src/utils/stat-info/granulated-partial-offers-map.ts b/libs/ftso-core/src/utils/stat-info/granulated-partial-offers-map.ts index e2a4fc64..3235f590 100644 --- a/libs/ftso-core/src/utils/stat-info/granulated-partial-offers-map.ts +++ b/libs/ftso-core/src/utils/stat-info/granulated-partial-offers-map.ts @@ -1,10 +1,10 @@ import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs"; import path from "path/posix"; import { CALCULATIONS_FOLDER } from "../../configs/networks"; -import { IFUPartialRewardOfferForRound, IPartialRewardOfferForRound } from "../PartialRewardOffer"; +import { IFDCPartialRewardOfferForRound, IFUPartialRewardOfferForRound, IPartialRewardOfferForRound } from "../PartialRewardOffer"; import { RewardEpochDuration } from "../RewardEpochDuration"; import { bigIntReplacer, bigIntReviver } from "../big-number-serialization"; -import { FU_OFFERS_FILE, OFFERS_FILE, TEMP_REWARD_EPOCH_FOLDER_PREFIX } from "./constants"; +import { FDC_OFFERS_FILE, FU_OFFERS_FILE, OFFERS_FILE, TEMP_REWARD_EPOCH_FOLDER_PREFIX } from "./constants"; export interface FeedOffers { readonly feedId: string; @@ -63,6 +63,45 @@ export function serializeGranulatedPartialOfferMap( } } +/** + * Serializes granulated partial offer map to disk. + * It creates necessary folders and structure of form + * `///OFFERS_FILE` + * The `OFFERS_FILE` files contain relevant granulated offers for all feeds. + */ +export function serializeGranulatedPartialOfferMapForFDC( + rewardEpochDuration: RewardEpochDuration, + rewardOfferMap: Map, + regenerate = true, + file = FDC_OFFERS_FILE, + calculationFolder = CALCULATIONS_FOLDER() +): void { + if (!existsSync(calculationFolder)) { + mkdirSync(calculationFolder); + } + const rewardEpochFolder = path.join(calculationFolder, `${rewardEpochDuration.rewardEpochId}`); + if (regenerate && existsSync(rewardEpochFolder)) { + rmSync(rewardEpochFolder, { recursive: true }); + } + if (!existsSync(rewardEpochFolder)) { + mkdirSync(rewardEpochFolder); + } + for (let i = rewardEpochDuration.startVotingRoundId; i <= rewardEpochDuration.endVotingRoundId; i++) { + const votingRoundFolder = path.join(rewardEpochFolder, `${i}`); + if (!existsSync(votingRoundFolder)) { + mkdirSync(votingRoundFolder); + } + const votingRoundOffer = rewardOfferMap.get(i); + if (!votingRoundOffer) { + throw new Error(`Critical error: No offer for voting round ${i}`); + } + const offersPath = path.join(votingRoundFolder, file); + writeFileSync(offersPath, JSON.stringify(votingRoundOffer, bigIntReplacer)); + } +} + + + /** * Creates necessary folders for reward epoch calculations. These include * the `//` folders. diff --git a/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts b/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts index 6e0a36f3..de5a8e38 100644 --- a/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts +++ b/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts @@ -5,7 +5,7 @@ import { ISignaturePayload } from "../../../../fsp-utils/src/SignaturePayload"; import { GenericSubmissionData, ParsedFinalizationData } from "../../IndexerClient"; import { VoterWeights } from "../../RewardEpoch"; import { CALCULATIONS_FOLDER } from "../../configs/networks"; -import { DataForRewardCalculation, FastUpdatesDataForVotingRound } from "../../data-calculation-interfaces"; +import { DataForRewardCalculation, FDCDataForVotingRound, FastUpdatesDataForVotingRound } from "../../data-calculation-interfaces"; import { Address, Feed, MedianCalculationResult, MessageHash, RandomCalculationResult } from "../../voting-types"; import { IRevealData } from "../RevealData"; import { bigIntReplacer, bigIntReviver } from "../big-number-serialization"; @@ -17,6 +17,10 @@ export interface RevealRecords { data: IRevealData; } +export interface BitVoteRecords { + submitAddress: string; + data: string; +} export interface VoterWeightData { submitAddress: string; weight: bigint; @@ -32,6 +36,7 @@ export interface SDataForCalculation { randomGenerationBenchingWindow: number; benchingWindowRevealOffenders: string[]; feedOrder: Feed[]; + validEligibleBitVotes: BitVoteRecords[]; // Not serialized, reconstructed on augmentation validEligibleRevealsMap?: Map; revealOffendersSet?: Set; @@ -41,13 +46,19 @@ export interface SDataForCalculation { signingAddressToSubmitAddress?: Map; votersWeightsMap?: Map; signerToSigningWeight?: Map; + validEligibleBitVotesMap?: Map; } export function prepareDataForCalculations(rewardEpochId: number, data: DataForRewardCalculation): SDataForCalculation { const validEligibleReveals: RevealRecords[] = []; + const validEligibleBitVotes: BitVoteRecords[] = []; for (const [submitAddress, revealData] of data.dataForCalculations.validEligibleReveals.entries()) { validEligibleReveals.push({ submitAddress, data: revealData }); } + for (const [submitAddress, bitVote] of data.dataForCalculations.validEligibleBitVotes.entries()) { + validEligibleBitVotes.push({ submitAddress, data: bitVote }); + } + const voterMedianVotingWeights: VoterWeightData[] = []; for (const [submitAddress, weight] of data.dataForCalculations.voterMedianVotingWeights.entries()) { voterMedianVotingWeights.push({ submitAddress, weight }); @@ -62,6 +73,7 @@ export function prepareDataForCalculations(rewardEpochId: number, data: DataForR randomGenerationBenchingWindow: data.dataForCalculations.randomGenerationBenchingWindow, benchingWindowRevealOffenders: [...data.dataForCalculations.benchingWindowRevealOffenders], feedOrder: data.dataForCalculations.feedOrder, + validEligibleBitVotes, }; return result; } @@ -87,6 +99,7 @@ export interface SDataForRewardCalculation { medianCalculationResults: MedianCalculationResult[]; randomResult: SimplifiedRandomCalculationResult; fastUpdatesData?: FastUpdatesDataForVotingRound; + fdcData?: FDCDataForVotingRound; // usually added after results of the next voting round are known nextVotingRoundRandomResult?: string; eligibleFinalizers: string[]; @@ -148,6 +161,7 @@ export function serializeDataForRewardCalculation( randomResult: simplifyRandomCalculationResult(randomResult), eligibleFinalizers: eligibleFinalizationRewardVotersInGracePeriod, fastUpdatesData: rewardCalculationData.fastUpdatesData, + fdcData: rewardCalculationData.fdcData, }; writeFileSync(rewardCalculationsDataPath, JSON.stringify(data, bigIntReplacer)); } @@ -182,6 +196,12 @@ function augmentDataForCalculation(data: SDataForCalculation, rewardEpochInfo: R for (const reveal of data.validEligibleReveals) { validEligibleRevealsMap.set(reveal.submitAddress.toLowerCase(), reveal.data); } + + const validEligibleBitVoteMap = new Map(); + for (const bitVote of data.validEligibleBitVotes) { + validEligibleBitVoteMap.set(bitVote.submitAddress.toLowerCase(), bitVote.data); + } + const revealOffendersSet = new Set(data.revealOffenders); const voterMedianVotingWeightsSet = new Map(); for (const voter of data.voterMedianVotingWeights) { @@ -191,6 +211,7 @@ function augmentDataForCalculation(data: SDataForCalculation, rewardEpochInfo: R data.benchingWindowRevealOffenders.map(address => address.toLowerCase()) ); data.validEligibleRevealsMap = validEligibleRevealsMap; + data.validEligibleBitVotesMap = validEligibleBitVoteMap; data.revealOffendersSet = revealOffendersSet; data.voterMedianVotingWeightsSet = voterMedianVotingWeightsSet; data.benchingWindowRevealOffendersSet = benchingWindowRevealOffendersSet; diff --git a/libs/ftso-core/src/utils/stat-info/reward-epoch-info.ts b/libs/ftso-core/src/utils/stat-info/reward-epoch-info.ts index c8560e29..b998c30a 100644 --- a/libs/ftso-core/src/utils/stat-info/reward-epoch-info.ts +++ b/libs/ftso-core/src/utils/stat-info/reward-epoch-info.ts @@ -9,6 +9,7 @@ import { bigIntReplacer, bigIntReviver } from "../big-number-serialization"; import { REWARD_EPOCH_INFO_FILE, TEMP_REWARD_EPOCH_FOLDER_PREFIX } from "./constants"; import { FUInflationRewardsOffered } from "../../events/FUInflationRewardsOffered"; import { IncentiveOffered } from "../../events/IncentiveOffered"; +import { FDCInflationRewardsOffered } from "../../events/FDCInflationRewardsOffered"; export interface RewardEpochInfo { rewardEpochId: number; @@ -23,13 +24,15 @@ export interface RewardEpochInfo { endVotingRoundId?: number; fuInflationRewardsOffered?: FUInflationRewardsOffered; fuIncentivesOffered?: IncentiveOffered[]; + fdcInflationRewardsOffered?: FDCInflationRewardsOffered; } export function getRewardEpochInfo( rewardEpoch: RewardEpoch, endVotingRoundId?: number, fuInflationRewardsOffered?: FUInflationRewardsOffered, - fuIncentivesOffered?: IncentiveOffered[] + fuIncentivesOffered?: IncentiveOffered[], + fdcInflationRewardsOffered?: FDCInflationRewardsOffered ): RewardEpochInfo { const voterRegistrationInfo: FullVoterRegistrationInfo[] = []; for (const signingAddress of rewardEpoch.signingPolicy.voters) { @@ -54,6 +57,7 @@ export function getRewardEpochInfo( endVotingRoundId, fuInflationRewardsOffered, fuIncentivesOffered, + fdcInflationRewardsOffered, }; return result; } diff --git a/test/libs/unit/generator-rewards.test.ts b/test/libs/unit/generator-rewards.test.ts index 091767ef..d8ea02b0 100644 --- a/test/libs/unit/generator-rewards.test.ts +++ b/test/libs/unit/generator-rewards.test.ts @@ -605,7 +605,8 @@ describe(`generator-rewards, ${getTestFile(__filename)}`, () => { true, // prepare data for reward calculations merge, serializeResults, - false // do not use fast updates data + false, // do not use fast updates data + false // do not use FDC updates data ); claims.push(...rewardClaims); } @@ -663,7 +664,8 @@ describe(`generator-rewards, ${getTestFile(__filename)}`, () => { true, // prepare data for reward calculations merge, serializeResults, - false // do not use fast updates data + false, // do not use fast updates data + false // do not use FDC updates data ); logStatus(ProgressType.CLAIM_CALCULATION); } @@ -771,7 +773,9 @@ export async function rewardClaimsForRewardEpoch( rewardOfferMap.get(votingRoundId), true, // prepareData merge, - serialize + serialize, + false, // do not use fast updates data + false // do not use FDC updates data ); allRewardClaims.push(...rewardClaims); if (merge) { From 3c528fbbdb0fe869ae1fb9c77ae37442ad3d30bb Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Fri, 27 Sep 2024 08:43:17 +0200 Subject: [PATCH 02/28] fdc rewarding wip --- .../ftso-core/src/IndexerClientForRewarding.ts | 18 ++++++++++++++---- .../ftso-core/src/events/AttestationRequest.ts | 4 ++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/libs/ftso-core/src/IndexerClientForRewarding.ts b/libs/ftso-core/src/IndexerClientForRewarding.ts index e7f4801d..9b7c742a 100644 --- a/libs/ftso-core/src/IndexerClientForRewarding.ts +++ b/libs/ftso-core/src/IndexerClientForRewarding.ts @@ -46,8 +46,7 @@ export class IndexerClientForRewarding extends IndexerClient { processed = event.votingRoundId; } else { throw new Error( - `FastUpdateFeeds events are not continuous from ${startVotingRoundId} to ${endVotingRoundId}: expected ${ - processed + 1 + `FastUpdateFeeds events are not continuous from ${startVotingRoundId} to ${endVotingRoundId}: expected ${processed + 1 }, got ${event.votingRoundId}` ); } @@ -157,7 +156,7 @@ export class IndexerClientForRewarding extends IndexerClient { public async getAttestationRequestEvents( startVotingRoundId: number, endVotingRoundId: number - ): Promise> { + ): Promise> { const startTime = EPOCH_SETTINGS().votingEpochStartSec(startVotingRoundId); // strictly containing in the range const endTime = EPOCH_SETTINGS().votingEpochStartSec(endVotingRoundId + 1) - 1; @@ -168,7 +167,18 @@ export class IndexerClientForRewarding extends IndexerClient { return { status }; } - const data = result.map(event => AttestationRequest.fromRawEvent(event)); + const allAttestationRequests = result.map(event => AttestationRequest.fromRawEvent(event)); + const data: AttestationRequest[][] = []; + let i = 0; + for (let votingRoundId = startVotingRoundId; votingRoundId <= endVotingRoundId; votingRoundId++) { + const attestationRequestsInVotingRound: AttestationRequest[] = []; + const votingEpochEndTime = EPOCH_SETTINGS().votingEpochStartSec(votingRoundId + 1) - 1; + while (i < allAttestationRequests.length && allAttestationRequests[i].timestamp <= votingEpochEndTime) { + attestationRequestsInVotingRound.push(allAttestationRequests[i]); + i++; + } + data.push(attestationRequestsInVotingRound); + } return { status, data, diff --git a/libs/ftso-core/src/events/AttestationRequest.ts b/libs/ftso-core/src/events/AttestationRequest.ts index baa5fdbf..21419a1e 100644 --- a/libs/ftso-core/src/events/AttestationRequest.ts +++ b/libs/ftso-core/src/events/AttestationRequest.ts @@ -9,6 +9,7 @@ export class AttestationRequest extends RawEventConstructible { super(); this.data = data.data; this.fee = BigInt(data.fee); + this.timestamp = data.timestamp; } static fromRawEvent(event: any): AttestationRequest { @@ -25,4 +26,7 @@ export class AttestationRequest extends RawEventConstructible { // feed decimals fee: bigint; + + // timestamp + timestamp: number; } From ac96a67293fe6bef5cce63f0df77e502c6905fcf Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Fri, 27 Sep 2024 08:44:17 +0200 Subject: [PATCH 03/28] wip --- libs/ftso-core/src/DataManagerForRewarding.ts | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/libs/ftso-core/src/DataManagerForRewarding.ts b/libs/ftso-core/src/DataManagerForRewarding.ts index 8d46859e..71b92e40 100644 --- a/libs/ftso-core/src/DataManagerForRewarding.ts +++ b/libs/ftso-core/src/DataManagerForRewarding.ts @@ -367,22 +367,13 @@ export class DataManagerForRewarding extends DataManager { status: DataAvailabilityStatus.NOT_OK, }; } - // const feedUpdates = await this.indexerClient.getFastUpdateFeedsSubmittedEvents( - // firstVotingRoundId, - // lastVotingRoundId - // ); - // if (feedUpdates.status !== BlockAssuranceResult.OK) { - // return { - // status: DataAvailabilityStatus.NOT_OK, - // }; - // } const result: FDCDataForVotingRound[] = []; for (let votingRoundId = firstVotingRoundId; votingRoundId <= lastVotingRoundId; votingRoundId++) { // const fastUpdateFeeds = attestationRequestsResponse.data[votingRoundId - firstVotingRoundId]; // const fastUpdateSubmissions = feedUpdates.data[votingRoundId - firstVotingRoundId]; - const value: any = {//FDCDataForVotingRound = { + const value: FDCDataForVotingRound = { votingRoundId, - // attestationRequests: attestationRequestsResponse.data[votingRoundId - firstVotingRoundId], + attestationRequests: attestationRequestsResponse.data[votingRoundId - firstVotingRoundId], }; result.push(value); } From 230a90146f00a49270880db87ef6a04e550fcd03 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Thu, 17 Oct 2024 23:04:06 +0200 Subject: [PATCH 04/28] wip --- libs/ftso-core/src/DataManager.ts | 6 ++- libs/ftso-core/src/DataManagerForRewarding.ts | 49 +++++++++++++++---- .../src/IndexerClientForRewarding.ts | 2 +- libs/ftso-core/src/configs/networks.ts | 2 + .../src/data-calculation-interfaces.ts | 4 +- 5 files changed, 51 insertions(+), 12 deletions(-) diff --git a/libs/ftso-core/src/DataManager.ts b/libs/ftso-core/src/DataManager.ts index a8c45c87..eb10a39e 100644 --- a/libs/ftso-core/src/DataManager.ts +++ b/libs/ftso-core/src/DataManager.ts @@ -355,6 +355,7 @@ export class DataManager { rewardEpoch: RewardEpoch, submissions: SubmissionData[], protocolId = FTSO2_PROTOCOL_ID, + providedMessageHash: MessageHash | undefined = undefined, logger: ILogger ): Map[]> { const signatureMap = new Map>>(); @@ -366,7 +367,10 @@ export class DataManager { signaturePayload.message.votingRoundId === votingRoundId && signaturePayload.message.protocolId === protocolId ) { - const messageHash = ProtocolMessageMerkleRoot.hash(signaturePayload.message); + // - Override the messageHash if provided + // - Require + xxx + let messageHash = providedMessageHash ?? ProtocolMessageMerkleRoot.hash(signaturePayload.message); signaturePayload.messageHash = messageHash; const signer = ECDSASignature.recoverSigner(messageHash, signaturePayload.signature).toLowerCase(); if (!rewardEpoch.isEligibleSignerAddress(signer)) { diff --git a/libs/ftso-core/src/DataManagerForRewarding.ts b/libs/ftso-core/src/DataManagerForRewarding.ts index cdbbb342..5c6ba8c4 100644 --- a/libs/ftso-core/src/DataManagerForRewarding.ts +++ b/libs/ftso-core/src/DataManagerForRewarding.ts @@ -1,3 +1,4 @@ +import { first } from "rxjs"; import { DataAvailabilityStatus, DataManager, DataMangerResponse, SignAndFinalizeSubmissionData } from "./DataManager"; import { BlockAssuranceResult, SubmissionData } from "./IndexerClient"; import { IndexerClientForRewarding } from "./IndexerClientForRewarding"; @@ -10,6 +11,7 @@ import { DataForRewardCalculation, FDCDataForVotingRound, FastUpdatesDataForVotingRound, + PartialFDCDataForVotingRound, } from "./data-calculation-interfaces"; import { ILogger } from "./utils/ILogger"; import { errorString } from "./utils/error"; @@ -172,7 +174,8 @@ export class DataManagerForRewarding extends DataManager { }; } let fastUpdatesData: FastUpdatesDataForVotingRound[] = []; - let fdcData: FDCDataForVotingRound[] = []; + let partialFdcData: PartialFDCDataForVotingRound[] = []; + if (useFastUpdatesData) { const fastUpdatesDataResponse = await this.getFastUpdatesDataForVotingRoundRange( firstVotingRoundId, @@ -186,16 +189,16 @@ export class DataManagerForRewarding extends DataManager { fastUpdatesData = fastUpdatesDataResponse.data; } if (useFDCData) { - const fdcDataResponse = await this.getFDCDataForVotingRoundRange( + const partialFdcDataResponse = await this.getFDCDataForVotingRoundRange( firstVotingRoundId, lastVotingRoundId ); - if (fdcDataResponse.status !== DataAvailabilityStatus.OK) { + if (partialFdcDataResponse.status !== DataAvailabilityStatus.OK) { return { - status: fdcDataResponse.status, + status: partialFdcDataResponse.status, }; } - fdcData = fdcDataResponse.data; + partialFdcData = partialFdcDataResponse.data; } /// @@ -253,13 +256,41 @@ export class DataManagerForRewarding extends DataManager { FTSO2_PROTOCOL_ID ); const firstSuccessfulFinalization = finalizations.find(finalization => finalization.successfulOnChain); + + const fdcSignatures = DataManager.extractSignatures( + votingRoundId, + rewardEpoch, + votingRoundSignatures, + FDC_PROTOCOL_ID, + this.logger + ); + const fdcFinalizations = this.extractFinalizations( + votingRoundId, + rewardEpoch, + votingRoundFinalizations, + FTSO2_PROTOCOL_ID + ); + const fdcFirstSuccessfulFinalization = fdcFinalizations.find(finalization => finalization.successfulOnChain); + + const partialData = partialFdcData[votingRoundId - firstVotingRoundId]; + if(partialData.votingRoundId !== votingRoundId) { + throw new Error(`Voting round id mismatch: ${partialData.votingRoundId} !== ${votingRoundId}`); + } + const fdcData: FDCDataForVotingRound = { + ...partialData, + bitVotes: dataForCalculations.validEligibleBitVotes, + signatures: [], // TODO + finalizations: fdcFinalizations, + firstSuccessfulFinalization: fdcFirstSuccessfulFinalization + } + const dataForRound: DataForRewardCalculation = { dataForCalculations, signatures, finalizations, firstSuccessfulFinalization, fastUpdatesData: fastUpdatesData[votingRoundId - firstVotingRoundId], - fdcData: fdcData[votingRoundId - firstVotingRoundId], + fdcData }; result.push(dataForRound); } @@ -369,7 +400,7 @@ export class DataManagerForRewarding extends DataManager { public async getFDCDataForVotingRoundRange( firstVotingRoundId: number, lastVotingRoundId: number - ): Promise> { + ): Promise> { const attestationRequestsResponse = await this.indexerClient.getAttestationRequestEvents(firstVotingRoundId, lastVotingRoundId); if (attestationRequestsResponse.status !== BlockAssuranceResult.OK) { @@ -377,11 +408,11 @@ export class DataManagerForRewarding extends DataManager { status: DataAvailabilityStatus.NOT_OK, }; } - const result: FDCDataForVotingRound[] = []; + const result: PartialFDCDataForVotingRound[] = []; for (let votingRoundId = firstVotingRoundId; votingRoundId <= lastVotingRoundId; votingRoundId++) { // const fastUpdateFeeds = attestationRequestsResponse.data[votingRoundId - firstVotingRoundId]; // const fastUpdateSubmissions = feedUpdates.data[votingRoundId - firstVotingRoundId]; - const value: FDCDataForVotingRound = { + const value: PartialFDCDataForVotingRound = { votingRoundId, attestationRequests: attestationRequestsResponse.data[votingRoundId - firstVotingRoundId], }; diff --git a/libs/ftso-core/src/IndexerClientForRewarding.ts b/libs/ftso-core/src/IndexerClientForRewarding.ts index 934cf81f..c9d2c21d 100644 --- a/libs/ftso-core/src/IndexerClientForRewarding.ts +++ b/libs/ftso-core/src/IndexerClientForRewarding.ts @@ -111,7 +111,7 @@ export class IndexerClientForRewarding extends IndexerClient { endVotingRoundId: number ): Promise> { const startTime = EPOCH_SETTINGS().votingEpochStartSec(startVotingRoundId); - const endTime = EPOCH_SETTINGS().votingEpochStartSec(endVotingRoundId + 1); + const endTime = EPOCH_SETTINGS().votingEpochStartSec(endVotingRoundId + 1) - 1; const eventName = FastUpdateFeedsSubmitted.eventName; const status = await this.ensureEventRange(startTime, endTime); if (status !== BlockAssuranceResult.OK) { diff --git a/libs/ftso-core/src/configs/networks.ts b/libs/ftso-core/src/configs/networks.ts index f804e854..a92f5473 100644 --- a/libs/ftso-core/src/configs/networks.ts +++ b/libs/ftso-core/src/configs/networks.ts @@ -712,3 +712,5 @@ export const FUTURE_VOTING_ROUNDS = () => 30; export const COSTON_FAST_UPDATER_SWITCH_VOTING_ROUND_ID = 779191; export const SONGBIRD_FAST_UPDATER_SWITCH_VOTING_ROUND_ID = 99999999999999; // temporary set to very high to not have effect + +export const WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH = "WRONG_SIGNATURE"; diff --git a/libs/ftso-core/src/data-calculation-interfaces.ts b/libs/ftso-core/src/data-calculation-interfaces.ts index 8a636f09..349bfa8e 100644 --- a/libs/ftso-core/src/data-calculation-interfaces.ts +++ b/libs/ftso-core/src/data-calculation-interfaces.ts @@ -51,10 +51,12 @@ export interface FastUpdatesDataForVotingRound { signingPolicyAddressesSubmitted: string[]; } -export interface FDCDataForVotingRound { +export interface PartialFDCDataForVotingRound { votingRoundId: number; // List of attestation requests to be processed attestationRequests: AttestationRequest[]; +} +export interface FDCDataForVotingRound extends PartialFDCDataForVotingRound{ // List of bitvotes for the consensus bitvote // Only last bitvote for each data provider is included // submit address is used to assign to data provider From 7af01578005485b3697e2354205f39e0e928db2d Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Wed, 23 Oct 2024 14:51:11 +0200 Subject: [PATCH 05/28] consolidation of data collection --- .../src/libs/calculator-utils.ts | 2 +- libs/ftso-core/src/DataManager.ts | 41 +++++++-- libs/ftso-core/src/DataManagerForRewarding.ts | 88 +++++++++++-------- libs/ftso-core/src/RewardEpoch.ts | 11 +++ .../src/data-calculation-interfaces.ts | 22 ++++- .../src/events/AttestationRequest.ts | 9 +- .../stat-info/reward-calculation-data.ts | 64 +++++++++----- 7 files changed, 166 insertions(+), 71 deletions(-) diff --git a/apps/ftso-reward-calculation-process/src/libs/calculator-utils.ts b/apps/ftso-reward-calculation-process/src/libs/calculator-utils.ts index dd427b9f..830e3c6d 100644 --- a/apps/ftso-reward-calculation-process/src/libs/calculator-utils.ts +++ b/apps/ftso-reward-calculation-process/src/libs/calculator-utils.ts @@ -138,7 +138,7 @@ export async function calculationOfRewardCalculationDataForRange( ); done = true; } catch (e) { - // console.log(e); + console.log(e); logger.error( `Error while calculating reward calculation data for voting rounds ${firstVotingRoundId}-${lastVotingRoundId} in reward epoch ${rewardEpochId}: ${e}` ); diff --git a/libs/ftso-core/src/DataManager.ts b/libs/ftso-core/src/DataManager.ts index eb10a39e..58e8edc5 100644 --- a/libs/ftso-core/src/DataManager.ts +++ b/libs/ftso-core/src/DataManager.ts @@ -19,6 +19,7 @@ import { EPOCH_SETTINGS, FTSO2_PROTOCOL_ID, GENESIS_REWARD_EPOCH_START_EVENT, + WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH, } from "./configs/networks"; import { DataForCalculations, @@ -28,8 +29,8 @@ import { import { CommitData, ICommitData } from "./utils/CommitData"; import { ILogger } from "./utils/ILogger"; import { IRevealData, RevealData } from "./utils/RevealData"; -import { Address, Feed, MessageHash } from "./voting-types"; import { errorString } from "./utils/error"; +import { Address, Feed, MessageHash } from "./voting-types"; /** * Data availability status for data manager responses. @@ -200,6 +201,7 @@ export class DataManager { dataForCalculationsResponse.data.rewardEpoch, signaturesResponse.data.signatures, FTSO2_PROTOCOL_ID, + undefined, this.logger ); const finalizations = this.extractFinalizations( @@ -369,16 +371,41 @@ export class DataManager { ) { // - Override the messageHash if provided // - Require - xxx + let messageHash = providedMessageHash ?? ProtocolMessageMerkleRoot.hash(signaturePayload.message); - signaturePayload.messageHash = messageHash; + const signer = ECDSASignature.recoverSigner(messageHash, signaturePayload.signature).toLowerCase(); - if (!rewardEpoch.isEligibleSignerAddress(signer)) { + // submit signature address should match the signingPolicyAddress + const expectedSigner = rewardEpoch.getSigningAddressFromSubmitSignatureAddress(submission.submitAddress.toLowerCase()); + // if the expected signer is not found, the signature is not valid for rewarding + if (!expectedSigner) { continue; } - signaturePayload.signer = signer; - signaturePayload.weight = rewardEpoch.signerToSigningWeight(signer); - signaturePayload.index = rewardEpoch.signerToVotingPolicyIndex(signer); + // In case of FTSO Scaling, we have message hash as a part of payload + // the signer is incorrect + if (!providedMessageHash) { + if (!rewardEpoch.isEligibleSignerAddress(signer)) { + continue; + } + signaturePayload.messageHash = messageHash; + signaturePayload.signer = signer; + signaturePayload.weight = rewardEpoch.signerToSigningWeight(signer); + signaturePayload.index = rewardEpoch.signerToVotingPolicyIndex(signer); + } else { + if (signer !== expectedSigner) { + if (!rewardEpoch.isEligibleSignerAddress(expectedSigner)) { + continue; + } + // wrong signature by eligible signer + signaturePayload.messageHash = WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH; + } else { + signaturePayload.messageHash = providedMessageHash; + } + signaturePayload.signer = expectedSigner; + signaturePayload.weight = rewardEpoch.signerToSigningWeight(expectedSigner); + signaturePayload.index = rewardEpoch.signerToVotingPolicyIndex(expectedSigner); + } + if ( signaturePayload.weight === undefined || signaturePayload.signer === undefined || diff --git a/libs/ftso-core/src/DataManagerForRewarding.ts b/libs/ftso-core/src/DataManagerForRewarding.ts index 5c6ba8c4..2545168e 100644 --- a/libs/ftso-core/src/DataManagerForRewarding.ts +++ b/libs/ftso-core/src/DataManagerForRewarding.ts @@ -1,6 +1,7 @@ -import { first } from "rxjs"; +import { RelayMessage } from "../../fsp-utils/src/RelayMessage"; +import { ISignaturePayload } from "../../fsp-utils/src/SignaturePayload"; import { DataAvailabilityStatus, DataManager, DataMangerResponse, SignAndFinalizeSubmissionData } from "./DataManager"; -import { BlockAssuranceResult, SubmissionData } from "./IndexerClient"; +import { BlockAssuranceResult, GenericSubmissionData, SubmissionData } from "./IndexerClient"; import { IndexerClientForRewarding } from "./IndexerClientForRewarding"; import { RewardEpoch } from "./RewardEpoch"; import { RewardEpochManager } from "./RewardEpochManager"; @@ -15,7 +16,7 @@ import { } from "./data-calculation-interfaces"; import { ILogger } from "./utils/ILogger"; import { errorString } from "./utils/error"; -import { Address } from "./voting-types"; +import { Address, MessageHash } from "./voting-types"; /** * Helps in extracting data in a consistent way for FTSO scaling feed median calculations, random number calculation and rewarding. @@ -117,13 +118,13 @@ export class DataManagerForRewarding extends DataManager { this.logger.debug(`Valid reveals from: ${JSON.stringify(Array.from(partialData.validEligibleReveals.keys()))}`); } //////// FDC //////// - const validEligibleBitVotes: Map = this.extractValidEligibleBitVotes(mappingsResponse.data.submit2, rewardEpoch); + const validEligibleBitVotes: SubmissionData[] = this.extractSubmissionsWithValidEligibleBitVotes(mappingsResponse.data.submit2, rewardEpoch); const dataForRound = { ...partialData, randomGenerationBenchingWindow, benchingWindowRevealOffenders, rewardEpoch, - validEligibleBitVotes + validEligibleBitVoteSubmissions: validEligibleBitVotes } as DataForCalculations; result.push(dataForRound); } @@ -247,6 +248,7 @@ export class DataManagerForRewarding extends DataManager { rewardEpoch, votingRoundSignatures, FTSO2_PROTOCOL_ID, + undefined, this.logger ); const finalizations = this.extractFinalizations( @@ -257,31 +259,43 @@ export class DataManagerForRewarding extends DataManager { ); const firstSuccessfulFinalization = finalizations.find(finalization => finalization.successfulOnChain); - const fdcSignatures = DataManager.extractSignatures( - votingRoundId, - rewardEpoch, - votingRoundSignatures, - FDC_PROTOCOL_ID, - this.logger - ); - const fdcFinalizations = this.extractFinalizations( - votingRoundId, - rewardEpoch, - votingRoundFinalizations, - FTSO2_PROTOCOL_ID - ); - const fdcFirstSuccessfulFinalization = fdcFinalizations.find(finalization => finalization.successfulOnChain); + let fdcData: FDCDataForVotingRound | undefined; - const partialData = partialFdcData[votingRoundId - firstVotingRoundId]; - if(partialData.votingRoundId !== votingRoundId) { - throw new Error(`Voting round id mismatch: ${partialData.votingRoundId} !== ${votingRoundId}`); - } - const fdcData: FDCDataForVotingRound = { - ...partialData, - bitVotes: dataForCalculations.validEligibleBitVotes, - signatures: [], // TODO - finalizations: fdcFinalizations, - firstSuccessfulFinalization: fdcFirstSuccessfulFinalization + if (useFDCData) { + const fdcFinalizations = this.extractFinalizations( + votingRoundId, + rewardEpoch, + votingRoundFinalizations, + FDC_PROTOCOL_ID + ); + const fdcFirstSuccessfulFinalization = fdcFinalizations.find(finalization => finalization.successfulOnChain); + let fdcSignatures = new Map[]>; + if (fdcFirstSuccessfulFinalization) { + if (!fdcFirstSuccessfulFinalization.messages.protocolMessageMerkleRoot) { + throw new Error(`Protocol message merkle root is missing for FDC finalization ${fdcFirstSuccessfulFinalization.messages.protocolMessageHash}`); + } + RelayMessage.augment(fdcFirstSuccessfulFinalization.messages); + fdcSignatures = DataManager.extractSignatures( + votingRoundId, + rewardEpoch, + votingRoundSignatures, + FDC_PROTOCOL_ID, + fdcFirstSuccessfulFinalization.messages.protocolMessageHash, + this.logger + ); + } + + const partialData = partialFdcData[votingRoundId - firstVotingRoundId]; + if (partialData.votingRoundId !== votingRoundId) { + throw new Error(`Voting round id mismatch: ${partialData.votingRoundId} !== ${votingRoundId}`); + } + fdcData = { + ...partialData, + bitVotes: dataForCalculations.validEligibleBitVoteSubmissions, + signaturesMap: fdcSignatures, + finalizations: fdcFinalizations, + firstSuccessfulFinalization: fdcFirstSuccessfulFinalization, + } } const dataForRound: DataForRewardCalculation = { @@ -374,7 +388,7 @@ export class DataManagerForRewarding extends DataManager { const fastUpdateFeeds = feedValuesResponse.data[votingRoundId - firstVotingRoundId]; const fastUpdateSubmissions = feedUpdates.data[votingRoundId - firstVotingRoundId]; // Handles the 'undefined' value in fastUpdateFeeds - this can happen on FastUpdater contract change - if(!fastUpdateFeeds) { + if (!fastUpdateFeeds) { throw new Error(`FastUpdateFeeds is undefined for voting round ${votingRoundId}`); } @@ -424,8 +438,12 @@ export class DataManagerForRewarding extends DataManager { }; } - public extractValidEligibleBitVotes(submissionDataArray: SubmissionData[], rewardEpoch: RewardEpoch): Map { - const voterToLastBitVote = new Map(); + /** + * Extracts all submissions that have in payload a valid eligible bit vote for the given reward epoch. + * Note that payloads may contain messages from other protocols! + */ + public extractSubmissionsWithValidEligibleBitVotes(submissionDataArray: SubmissionData[], rewardEpoch: RewardEpoch): SubmissionData[] { + const voterToLastBitVote = new Map(); for (const submission of submissionDataArray) { for (const message of submission.messages) { if ( @@ -435,10 +453,10 @@ export class DataManagerForRewarding extends DataManager { try { const submitAddress = submission.submitAddress.toLowerCase(); if (rewardEpoch.isEligibleSubmitAddress(submitAddress)) { - voterToLastBitVote.set(submitAddress, message.payload); + voterToLastBitVote.set(submitAddress, submission); } else { if (!process.env.REMOVE_ANNOYING_MESSAGES) { - this.logger.warn(`Non-eligible commit found for address ${submitAddress}`); + this.logger.warn(`Non-eligible submit1 found for address ${submitAddress}`); } } } catch (e) { @@ -447,6 +465,6 @@ export class DataManagerForRewarding extends DataManager { } } } - return voterToLastBitVote; + return [...voterToLastBitVote.values()]; } } diff --git a/libs/ftso-core/src/RewardEpoch.ts b/libs/ftso-core/src/RewardEpoch.ts index cad5520d..61a19344 100644 --- a/libs/ftso-core/src/RewardEpoch.ts +++ b/libs/ftso-core/src/RewardEpoch.ts @@ -38,6 +38,8 @@ export class RewardEpoch { readonly submitAddressToVoter = new Map(); // delegateAddress => identityAddress readonly delegationAddressToVoter = new Map(); + // submitSignaturesAddress => signingAddress + readonly submitSignatureAddressToSigningAddress = new Map(); readonly submitAddressToCappedWeight = new Map(); readonly submitAddressToVoterRegistrationInfo = new Map(); @@ -136,6 +138,11 @@ export class RewardEpoch { fullVoterRegistrationInfo.voterRegistrationInfo.wNatCappedWeight ); + this.submitSignatureAddressToSigningAddress.set( + fullVoterRegistrationInfo.voterRegistered.submitSignaturesAddress.toLowerCase(), + fullVoterRegistrationInfo.voterRegistered.signingPolicyAddress.toLowerCase() + ) + this.submitAddressToVoterRegistrationInfo.set( fullVoterRegistrationInfo.voterRegistered.submitAddress.toLowerCase(), fullVoterRegistrationInfo @@ -185,6 +192,10 @@ export class RewardEpoch { return !!this.signingAddressToVoter.get(signerAddress.toLowerCase()); } + public getSigningAddressFromSubmitSignatureAddress(submitSignatureAddress: Address): Address | undefined { + return this.submitSignatureAddressToSigningAddress.get(submitSignatureAddress.toLowerCase()); + } + /** * Returns weight for participation in median voting. * @param submissionAddress diff --git a/libs/ftso-core/src/data-calculation-interfaces.ts b/libs/ftso-core/src/data-calculation-interfaces.ts index 349bfa8e..87a614c3 100644 --- a/libs/ftso-core/src/data-calculation-interfaces.ts +++ b/libs/ftso-core/src/data-calculation-interfaces.ts @@ -1,8 +1,9 @@ import { ISignaturePayload } from "../../fsp-utils/src/SignaturePayload"; -import { GenericSubmissionData, ParsedFinalizationData } from "./IndexerClient"; +import { GenericSubmissionData, ParsedFinalizationData, SubmissionData } from "./IndexerClient"; import { RewardEpoch } from "./RewardEpoch"; import { AttestationRequest } from "./events/AttestationRequest"; import { IRevealData } from "./utils/RevealData"; +import { HashSignatures } from "./utils/stat-info/reward-calculation-data"; import { Address, Feed, MessageHash } from "./voting-types"; export interface DataForCalculationsPartial { @@ -28,7 +29,7 @@ export interface DataForCalculations extends DataForCalculationsPartial { // Reward epoch rewardEpoch: RewardEpoch; // valid eligible bit-votes - validEligibleBitVotes?: Map; + validEligibleBitVoteSubmissions?: SubmissionData[]; } export interface DataForRewardCalculation { @@ -60,17 +61,18 @@ export interface FDCDataForVotingRound extends PartialFDCDataForVotingRound{ // List of bitvotes for the consensus bitvote // Only last bitvote for each data provider is included // submit address is used to assign to data provider - bitVotes: GenericSubmissionData[]; + bitVotes: SubmissionData[]; // signature data, include the unsigned message, which should be consensus bitvote // Might be multiple signatures by the same data provider // All signatures for FDC protocol in the observed range are included and sorted in the order of arrival - signatures: GenericSubmissionData[]; + signaturesMap: Map[]> // First successful finalization // might be undefined, if such finalization does not exist in an observed range // If defined then we can check if signatures are correct firstSuccessfulFinalization?: ParsedFinalizationData; // All finalizations in the observed range finalizations: ParsedFinalizationData[]; + // ----- These data is added after the reward calculation for log ------ // Filtered signatures that match the first finalized protocol Merkle root message // One per eligible data provider rewardedSignatures?: GenericSubmissionData[]; @@ -79,6 +81,18 @@ export interface FDCDataForVotingRound extends PartialFDCDataForVotingRound{ majorityBitVote?: string; } +export interface SFDCDataForVotingRound { + votingRoundId: number; + attestationRequests: AttestationRequest[]; + bitVotes: SubmissionData[]; + signatures: HashSignatures[]; + firstSuccessfulFinalization?: ParsedFinalizationData; + finalizations: ParsedFinalizationData[]; + signaturesMap?: Map[]>; + rewardedSignatures?: GenericSubmissionData[]; + majorityBitVote?: string; +} + export interface FUFeedValue { feedId: string; value: bigint; diff --git a/libs/ftso-core/src/events/AttestationRequest.ts b/libs/ftso-core/src/events/AttestationRequest.ts index 21419a1e..71424c01 100644 --- a/libs/ftso-core/src/events/AttestationRequest.ts +++ b/libs/ftso-core/src/events/AttestationRequest.ts @@ -5,11 +5,14 @@ import { RawEventConstructible } from "./RawEventConstructible"; export class AttestationRequest extends RawEventConstructible { static eventName = "AttestationRequest"; - constructor(data: any) { + constructor(data: any, timestamp: number) { super(); + if(timestamp === undefined) { + throw new Error("Timestamp is required"); + } this.data = data.data; this.fee = BigInt(data.fee); - this.timestamp = data.timestamp; + this.timestamp = timestamp; } static fromRawEvent(event: any): AttestationRequest { @@ -17,7 +20,7 @@ export class AttestationRequest extends RawEventConstructible { CONTRACTS.FdcHub.name, AttestationRequest.eventName, event, - (data: any) => new AttestationRequest(data) + (data: any, entity: any) => new AttestationRequest(data, entity.timestamp) ); } diff --git a/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts b/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts index de5a8e38..039ebccc 100644 --- a/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts +++ b/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts @@ -5,7 +5,7 @@ import { ISignaturePayload } from "../../../../fsp-utils/src/SignaturePayload"; import { GenericSubmissionData, ParsedFinalizationData } from "../../IndexerClient"; import { VoterWeights } from "../../RewardEpoch"; import { CALCULATIONS_FOLDER } from "../../configs/networks"; -import { DataForRewardCalculation, FDCDataForVotingRound, FastUpdatesDataForVotingRound } from "../../data-calculation-interfaces"; +import { DataForRewardCalculation, FDCDataForVotingRound, FastUpdatesDataForVotingRound, SFDCDataForVotingRound } from "../../data-calculation-interfaces"; import { Address, Feed, MedianCalculationResult, MessageHash, RandomCalculationResult } from "../../voting-types"; import { IRevealData } from "../RevealData"; import { bigIntReplacer, bigIntReviver } from "../big-number-serialization"; @@ -17,10 +17,6 @@ export interface RevealRecords { data: IRevealData; } -export interface BitVoteRecords { - submitAddress: string; - data: string; -} export interface VoterWeightData { submitAddress: string; weight: bigint; @@ -36,7 +32,6 @@ export interface SDataForCalculation { randomGenerationBenchingWindow: number; benchingWindowRevealOffenders: string[]; feedOrder: Feed[]; - validEligibleBitVotes: BitVoteRecords[]; // Not serialized, reconstructed on augmentation validEligibleRevealsMap?: Map; revealOffendersSet?: Set; @@ -46,18 +41,13 @@ export interface SDataForCalculation { signingAddressToSubmitAddress?: Map; votersWeightsMap?: Map; signerToSigningWeight?: Map; - validEligibleBitVotesMap?: Map; } export function prepareDataForCalculations(rewardEpochId: number, data: DataForRewardCalculation): SDataForCalculation { const validEligibleReveals: RevealRecords[] = []; - const validEligibleBitVotes: BitVoteRecords[] = []; for (const [submitAddress, revealData] of data.dataForCalculations.validEligibleReveals.entries()) { validEligibleReveals.push({ submitAddress, data: revealData }); } - for (const [submitAddress, bitVote] of data.dataForCalculations.validEligibleBitVotes.entries()) { - validEligibleBitVotes.push({ submitAddress, data: bitVote }); - } const voterMedianVotingWeights: VoterWeightData[] = []; for (const [submitAddress, weight] of data.dataForCalculations.voterMedianVotingWeights.entries()) { @@ -72,8 +62,7 @@ export function prepareDataForCalculations(rewardEpochId: number, data: DataForR voterMedianVotingWeights, randomGenerationBenchingWindow: data.dataForCalculations.randomGenerationBenchingWindow, benchingWindowRevealOffenders: [...data.dataForCalculations.benchingWindowRevealOffenders], - feedOrder: data.dataForCalculations.feedOrder, - validEligibleBitVotes, + feedOrder: data.dataForCalculations.feedOrder }; return result; } @@ -99,7 +88,7 @@ export interface SDataForRewardCalculation { medianCalculationResults: MedianCalculationResult[]; randomResult: SimplifiedRandomCalculationResult; fastUpdatesData?: FastUpdatesDataForVotingRound; - fdcData?: FDCDataForVotingRound; + fdcData?: SFDCDataForVotingRound; // usually added after results of the next voting round are known nextVotingRoundRandomResult?: string; eligibleFinalizers: string[]; @@ -146,12 +135,36 @@ export function serializeDataForRewardCalculation( }; hashSignatures.push(hashRecord); } + + + let fdcData: SFDCDataForVotingRound | undefined; + + if (rewardCalculationData.fdcData) { + const fdcHashSignatures: HashSignatures[] = []; + for (const [hash, signatures] of rewardCalculationData.fdcData.signaturesMap.entries()) { + const hashRecord: HashSignatures = { + hash, + signatures, + }; + fdcHashSignatures.push(hashRecord); + } + fdcData = { + votingRoundId: rewardCalculationData.fdcData.votingRoundId, + attestationRequests: rewardCalculationData.fdcData.attestationRequests, + bitVotes: rewardCalculationData.fdcData.bitVotes, + signatures: fdcHashSignatures, + firstSuccessfulFinalization: rewardCalculationData.fdcData.firstSuccessfulFinalization, + finalizations: rewardCalculationData.fdcData.finalizations, + } + } + for (const finalization of rewardCalculationData.finalizations) { RelayMessage.augment(finalization.messages); } if (rewardCalculationData.firstSuccessfulFinalization?.messages) { RelayMessage.augment(rewardCalculationData.firstSuccessfulFinalization?.messages); } + const data: SDataForRewardCalculation = { dataForCalculations: prepareDataForCalculations(rewardEpochId, rewardCalculationData), signatures: hashSignatures, @@ -161,7 +174,7 @@ export function serializeDataForRewardCalculation( randomResult: simplifyRandomCalculationResult(randomResult), eligibleFinalizers: eligibleFinalizationRewardVotersInGracePeriod, fastUpdatesData: rewardCalculationData.fastUpdatesData, - fdcData: rewardCalculationData.fdcData, + fdcData }; writeFileSync(rewardCalculationsDataPath, JSON.stringify(data, bigIntReplacer)); } @@ -197,11 +210,6 @@ function augmentDataForCalculation(data: SDataForCalculation, rewardEpochInfo: R validEligibleRevealsMap.set(reveal.submitAddress.toLowerCase(), reveal.data); } - const validEligibleBitVoteMap = new Map(); - for (const bitVote of data.validEligibleBitVotes) { - validEligibleBitVoteMap.set(bitVote.submitAddress.toLowerCase(), bitVote.data); - } - const revealOffendersSet = new Set(data.revealOffenders); const voterMedianVotingWeightsSet = new Map(); for (const voter of data.voterMedianVotingWeights) { @@ -211,7 +219,6 @@ function augmentDataForCalculation(data: SDataForCalculation, rewardEpochInfo: R data.benchingWindowRevealOffenders.map(address => address.toLowerCase()) ); data.validEligibleRevealsMap = validEligibleRevealsMap; - data.validEligibleBitVotesMap = validEligibleBitVoteMap; data.revealOffendersSet = revealOffendersSet; data.voterMedianVotingWeightsSet = voterMedianVotingWeightsSet; data.benchingWindowRevealOffendersSet = benchingWindowRevealOffendersSet; @@ -269,8 +276,23 @@ export function augmentDataForRewardCalculation( signaturesMap.set(hashSignature.hash, hashSignature.signatures); } data.signaturesMap = signaturesMap; + augmentFdcDataForRewardCalculation(data.fdcData); +} + +/** + * After deserialization, the data is augmented with additional maps and sets for easier access. + */ +export function augmentFdcDataForRewardCalculation( + data: SFDCDataForVotingRound, +): void { + const signaturesMap = new Map[]>(); + for (const hashSignature of data.signatures) { + signaturesMap.set(hashSignature.hash, hashSignature.signatures); + } + data.signaturesMap = signaturesMap; } + export function deserializeDataForRewardCalculation( rewardEpochId: number, votingRoundId: number, From 122a02ba946bdcc7361bbf7b48d01c43f10de0df Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Thu, 24 Oct 2024 22:58:39 +0200 Subject: [PATCH 06/28] reward data calculations --- libs/ftso-core/src/DataManagerForRewarding.ts | 162 +++++++++++++++++- libs/ftso-core/src/RewardEpoch.ts | 19 +- .../src/data-calculation-interfaces.ts | 53 ++++-- .../src/events/AttestationRequest.ts | 3 + .../stat-info/reward-calculation-data.ts | 3 + 5 files changed, 224 insertions(+), 16 deletions(-) diff --git a/libs/ftso-core/src/DataManagerForRewarding.ts b/libs/ftso-core/src/DataManagerForRewarding.ts index 2545168e..5bf80dd1 100644 --- a/libs/ftso-core/src/DataManagerForRewarding.ts +++ b/libs/ftso-core/src/DataManagerForRewarding.ts @@ -6,11 +6,15 @@ import { IndexerClientForRewarding } from "./IndexerClientForRewarding"; import { RewardEpoch } from "./RewardEpoch"; import { RewardEpochManager } from "./RewardEpochManager"; import { ContractMethodNames } from "./configs/contracts"; -import { ADDITIONAL_REWARDED_FINALIZATION_WINDOWS, EPOCH_SETTINGS, FDC_PROTOCOL_ID, FTSO2_PROTOCOL_ID } from "./configs/networks"; +import { ADDITIONAL_REWARDED_FINALIZATION_WINDOWS, EPOCH_SETTINGS, FDC_PROTOCOL_ID, FTSO2_PROTOCOL_ID, WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH } from "./configs/networks"; import { DataForCalculations, DataForRewardCalculation, FDCDataForVotingRound, + FDCEligibleSigner, + FDCOffender, + FDCOffense, + FDCRewardData, FastUpdatesDataForVotingRound, PartialFDCDataForVotingRound, } from "./data-calculation-interfaces"; @@ -260,6 +264,7 @@ export class DataManagerForRewarding extends DataManager { const firstSuccessfulFinalization = finalizations.find(finalization => finalization.successfulOnChain); let fdcData: FDCDataForVotingRound | undefined; + let fdcRewardData: FDCRewardData | undefined; if (useFDCData) { const fdcFinalizations = this.extractFinalizations( @@ -275,26 +280,34 @@ export class DataManagerForRewarding extends DataManager { throw new Error(`Protocol message merkle root is missing for FDC finalization ${fdcFirstSuccessfulFinalization.messages.protocolMessageHash}`); } RelayMessage.augment(fdcFirstSuccessfulFinalization.messages); + const consensusMessageHash = fdcFirstSuccessfulFinalization.messages.protocolMessageHash; fdcSignatures = DataManager.extractSignatures( votingRoundId, rewardEpoch, votingRoundSignatures, FDC_PROTOCOL_ID, - fdcFirstSuccessfulFinalization.messages.protocolMessageHash, + consensusMessageHash, this.logger ); + fdcRewardData = DataManagerForRewarding.extractFDCRewardData( + consensusMessageHash, + dataForCalculations.validEligibleBitVoteSubmissions, + fdcSignatures, + rewardEpoch + ) } - + const partialData = partialFdcData[votingRoundId - firstVotingRoundId]; if (partialData.votingRoundId !== votingRoundId) { throw new Error(`Voting round id mismatch: ${partialData.votingRoundId} !== ${votingRoundId}`); - } + } fdcData = { ...partialData, bitVotes: dataForCalculations.validEligibleBitVoteSubmissions, signaturesMap: fdcSignatures, finalizations: fdcFinalizations, firstSuccessfulFinalization: fdcFirstSuccessfulFinalization, + ...fdcRewardData } } @@ -441,6 +454,7 @@ export class DataManagerForRewarding extends DataManager { /** * Extracts all submissions that have in payload a valid eligible bit vote for the given reward epoch. * Note that payloads may contain messages from other protocols! + * If data provider submits multiple bitvotes, the last submission is considered */ public extractSubmissionsWithValidEligibleBitVotes(submissionDataArray: SubmissionData[], rewardEpoch: RewardEpoch): SubmissionData[] { const voterToLastBitVote = new Map(); @@ -467,4 +481,144 @@ export class DataManagerForRewarding extends DataManager { } return [...voterToLastBitVote.values()]; } + + /** + * Given finalized messageHash it calculates consensus bitvote, filters out eligible signers and determines + * offenders. + */ + public static extractFDCRewardData( + messageHash: string, + bitVoteSubmissions: SubmissionData[], + fdcSignatures: Map[]>, + rewardEpoch: RewardEpoch, + ): FDCRewardData | undefined { + const voteCounter = new Map(); + // + const eligibleSigners: FDCEligibleSigner[] = []; + const offenseMap = new Map(); + if (!messageHash) { + throw new Error("Consensus message hash is required"); + } + const signatures = fdcSignatures.get(messageHash); + if (!signatures) { + // TODO: log warning + return undefined; + } + for (const signature of signatures) { + const consensusBitVoteCandidate = signature.messages.unsignedMessage?.toLowerCase(); + if (!consensusBitVoteCandidate) { + continue; + } + voteCounter.set(consensusBitVoteCandidate, (voteCounter.get(consensusBitVoteCandidate) || 0) + signature.messages.weight) + } + const maxCount = Math.max(...voteCounter.values()); + const maxHashes = [...voteCounter.entries()].filter(([_, count]) => count === maxCount).map(([hash, _]) => hash); + maxHashes.sort(); + // if it happens there are multiple maxHashes we take the first in lexicographical order + const consensusBitVote = maxHashes[0]; + + // TODO: + // should we require 50%+ weight on maxHash? + // const consensusBitVoteWeight = voteCounter.get(consensusBitVote); + // if(consensusBitVoteWeight < rewardEpoch.signingPolicy.threshold) { + // return undefined; + // } + + const submitSignatureAddressToBitVote = new Map(); + for (const submission of bitVoteSubmissions) { + const submitSignatureAddress = rewardEpoch.getSubmitSignatureAddressFromSubmitAddress(submission.submitAddress).toLowerCase(); + const message = submission.messages.find(m => m.protocolId === FDC_PROTOCOL_ID); + if (message && message.payload) { + submitSignatureAddressToBitVote.set(submitSignatureAddress, message.payload.toLowerCase()); + } + } + + const submitSignatureSenders = new Set
(); + + for (const signature of signatures) { + // too late + if (signature.relativeTimestamp >= 90) { + continue; + } + const submitSignatureAddress = signature.submitAddress.toLowerCase() + submitSignatureSenders.add(submitSignatureAddress); + const bitVote = submitSignatureAddressToBitVote.get(submitSignatureAddress); + const eligibleSigner: FDCEligibleSigner = { + submitSignatureAddress: signature.submitAddress.toLowerCase(), + relativeTimestamp: signature.relativeTimestamp, + bitVote, + dominatesConsensusBitVote: DataManagerForRewarding.isConsensusVoteDominated(consensusBitVote, bitVote), + weight: signature.messages.weight, + } + eligibleSigners.push(eligibleSigner); + } + + for (const submission of bitVoteSubmissions) { + const submitSignatureAddress = rewardEpoch.getSubmitSignatureAddressFromSubmitAddress(submission.submitAddress).toLowerCase(); + if (!submitSignatureSenders.has(submitSignatureAddress)) { + const offender: FDCOffender = { + submitSignatureAddress, + offenses: [FDCOffense.NO_REVEAL_ON_BITVOTE] + } + offenseMap.set(submitSignatureAddress, offender); + } + } + + const wrongSignatures = fdcSignatures.get(WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH); + if (wrongSignatures) { + for (const signature of wrongSignatures) { + const submitSignatureAddress = signature.submitAddress.toLowerCase(); + if(!rewardEpoch.isEligibleSubmitSignatureAddress(submitSignatureAddress)) { + continue; + } + const offender = offenseMap.get(submitSignatureAddress) || { + submitSignatureAddress, + offenses: [] + } + offender.offenses.push(FDCOffense.WRONG_SIGNATURE); + offenseMap.set(submitSignatureAddress, offender); + } + } + + for(const signature of signatures) { + const submitSignatureAddress = signature.submitAddress.toLowerCase(); + const consensusBitVoteCandidate = signature.messages.unsignedMessage?.toLowerCase(); + if(consensusBitVoteCandidate !== consensusBitVote) { + const offender = offenseMap.get(submitSignatureAddress) || { + submitSignatureAddress, + offenses: [] + } + offender.offenses.push(FDCOffense.BAD_CONSENSUS_BITVOTE_CANDIDATE); + offenseMap.set(submitSignatureAddress, offender); + } + } + + const fdcOffenders = [...offenseMap.values()]; + fdcOffenders.sort((a, b) => a.submitSignatureAddress.localeCompare(b.submitSignatureAddress)); + const result: FDCRewardData = { + eligibleSigners, + consensusBitVote, + fdcOffenders + }; + + return result; + } + + public static isConsensusVoteDominated(consensusBitVote: string, bitVote?: string): boolean { + if (!bitVote) { + return false; + } + let h1 = consensusBitVote.startsWith("0x") ? consensusBitVote.slice(2) : consensusBitVote; + let h2 = bitVote.startsWith("0x") ? bitVote.slice(2) : bitVote; + if (h1.length !== h2.length) { + const mLen = Math.max(h1.length, h2.length); + h1 = h1.padEnd(mLen, "0"); + h2 = h2.padEnd(mLen, "0"); + } + const buf1 = Buffer.from(h1, "hex"); + const buf2 = Buffer.from(h2, "hex"); + // AND operation + const bufResult = buf1.map((b, i) => b & buf2[i]); + return buf1.equals(bufResult); + } } diff --git a/libs/ftso-core/src/RewardEpoch.ts b/libs/ftso-core/src/RewardEpoch.ts index 61a19344..23033bd1 100644 --- a/libs/ftso-core/src/RewardEpoch.ts +++ b/libs/ftso-core/src/RewardEpoch.ts @@ -40,6 +40,8 @@ export class RewardEpoch { readonly delegationAddressToVoter = new Map(); // submitSignaturesAddress => signingAddress readonly submitSignatureAddressToSigningAddress = new Map(); + // submitSignaturesAddress => identityAddress + readonly submitSignatureAddressToVoter = new Map(); readonly submitAddressToCappedWeight = new Map(); readonly submitAddressToVoterRegistrationInfo = new Map(); @@ -139,10 +141,15 @@ export class RewardEpoch { ); this.submitSignatureAddressToSigningAddress.set( - fullVoterRegistrationInfo.voterRegistered.submitSignaturesAddress.toLowerCase(), + fullVoterRegistrationInfo.voterRegistered.submitSignaturesAddress.toLowerCase(), fullVoterRegistrationInfo.voterRegistered.signingPolicyAddress.toLowerCase() ) + this.submitSignatureAddressToVoter.set( + fullVoterRegistrationInfo.voterRegistered.submitSignaturesAddress.toLowerCase(), + voter + ); + this.submitAddressToVoterRegistrationInfo.set( fullVoterRegistrationInfo.voterRegistered.submitAddress.toLowerCase(), fullVoterRegistrationInfo @@ -192,10 +199,18 @@ export class RewardEpoch { return !!this.signingAddressToVoter.get(signerAddress.toLowerCase()); } + public isEligibleSubmitSignatureAddress(submitSignatureAddress: Address): boolean { + return !!this.submitSignatureAddressToVoter.get(submitSignatureAddress.toLowerCase()); + } + public getSigningAddressFromSubmitSignatureAddress(submitSignatureAddress: Address): Address | undefined { return this.submitSignatureAddressToSigningAddress.get(submitSignatureAddress.toLowerCase()); } - + + public getSubmitSignatureAddressFromSubmitAddress(submitAddress: Address): Address | undefined { + return this.submitAddressToVoterRegistrationInfo.get(submitAddress.toLowerCase())?.voterRegistered.submitSignaturesAddress; + } + /** * Returns weight for participation in median voting. * @param submissionAddress diff --git a/libs/ftso-core/src/data-calculation-interfaces.ts b/libs/ftso-core/src/data-calculation-interfaces.ts index 87a614c3..92a75b7f 100644 --- a/libs/ftso-core/src/data-calculation-interfaces.ts +++ b/libs/ftso-core/src/data-calculation-interfaces.ts @@ -57,7 +57,46 @@ export interface PartialFDCDataForVotingRound { // List of attestation requests to be processed attestationRequests: AttestationRequest[]; } -export interface FDCDataForVotingRound extends PartialFDCDataForVotingRound{ + +/** + * Represents a signer, that: + * - has submitted a signature for voting round N, before the end of the voting round N + 1 + * - signature signed the consensus bitvote + * - might or might not have submitted a bitvote + */ +export interface FDCEligibleSigner { + submitSignatureAddress: string; + // Relative timestamp in voting epoch N + 1 + relativeTimestamp: number; + bitVote?: string; + dominatesConsensusBitVote: boolean; + weight: number; +} + + +export enum FDCOffense { + NO_REVEAL_ON_BITVOTE = "NO_REVEAL_ON_BITVOTE", + WRONG_SIGNATURE = "WRONG_SIGNATURE", + BAD_CONSENSUS_BITVOTE_CANDIDATE = "BAD_CONSENSUS_BITVOTE_CANDIDATE", +} +export interface FDCOffender { + submitSignatureAddress: string; + offenses: FDCOffense[]; +} + +export interface FDCRewardData { + // ----- These data is added after the reward calculation for log ------ + // Filtered signatures that match the first finalized protocol Merkle root message + // One per eligible data provider + eligibleSigners?: FDCEligibleSigner[]; + // Majority bitvote attached to the finalized signatures + // All signers that have unmatching majority bitvote are considered as offenders + consensusBitVote?: string; + // FDC offenders + fdcOffenders?: FDCOffender[]; +} + +export interface FDCDataForVotingRound extends PartialFDCDataForVotingRound, FDCRewardData { // List of bitvotes for the consensus bitvote // Only last bitvote for each data provider is included // submit address is used to assign to data provider @@ -72,13 +111,6 @@ export interface FDCDataForVotingRound extends PartialFDCDataForVotingRound{ firstSuccessfulFinalization?: ParsedFinalizationData; // All finalizations in the observed range finalizations: ParsedFinalizationData[]; - // ----- These data is added after the reward calculation for log ------ - // Filtered signatures that match the first finalized protocol Merkle root message - // One per eligible data provider - rewardedSignatures?: GenericSubmissionData[]; - // Majority bitvote attached to the finalized signatures - // All signers that have unmatching majority bitvote are considered as offenders - majorityBitVote?: string; } export interface SFDCDataForVotingRound { @@ -88,9 +120,10 @@ export interface SFDCDataForVotingRound { signatures: HashSignatures[]; firstSuccessfulFinalization?: ParsedFinalizationData; finalizations: ParsedFinalizationData[]; + eligibleSigners: FDCEligibleSigner[]; + consensusBitVote: string; + fdcOffenders: FDCOffender[]; signaturesMap?: Map[]>; - rewardedSignatures?: GenericSubmissionData[]; - majorityBitVote?: string; } export interface FUFeedValue { diff --git a/libs/ftso-core/src/events/AttestationRequest.ts b/libs/ftso-core/src/events/AttestationRequest.ts index 71424c01..85bdf1e0 100644 --- a/libs/ftso-core/src/events/AttestationRequest.ts +++ b/libs/ftso-core/src/events/AttestationRequest.ts @@ -32,4 +32,7 @@ export class AttestationRequest extends RawEventConstructible { // timestamp timestamp: number; + + // confirmed + confirmed: boolean = false; } diff --git a/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts b/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts index 039ebccc..c4186f99 100644 --- a/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts +++ b/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts @@ -155,6 +155,9 @@ export function serializeDataForRewardCalculation( signatures: fdcHashSignatures, firstSuccessfulFinalization: rewardCalculationData.fdcData.firstSuccessfulFinalization, finalizations: rewardCalculationData.fdcData.finalizations, + eligibleSigners: rewardCalculationData.fdcData.eligibleSigners, + consensusBitVote: rewardCalculationData.fdcData.consensusBitVote, + fdcOffenders: rewardCalculationData.fdcData.fdcOffenders, } } From a6e8d4fb09ad279c02e0646342df0c3f844c9b43 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Fri, 25 Oct 2024 20:41:43 +0200 Subject: [PATCH 07/28] fdc reward calculation data --- libs/ftso-core/src/DataManager.ts | 4 - libs/ftso-core/src/DataManagerForRewarding.ts | 126 +++++++++++++----- .../src/data-calculation-interfaces.ts | 9 +- .../stat-info/reward-calculation-data.ts | 1 + 4 files changed, 103 insertions(+), 37 deletions(-) diff --git a/libs/ftso-core/src/DataManager.ts b/libs/ftso-core/src/DataManager.ts index 58e8edc5..9b4f3882 100644 --- a/libs/ftso-core/src/DataManager.ts +++ b/libs/ftso-core/src/DataManager.ts @@ -68,8 +68,6 @@ export interface CommitsAndReveals { export interface CommitAndRevealSubmissionsMappingsForRange { votingRoundIdToCommits: Map; votingRoundIdToReveals: Map; - submit1?: SubmissionData[]; - submit2?: SubmissionData[]; } export interface SignAndFinalizeSubmissionData { @@ -283,8 +281,6 @@ export class DataManager { data: { votingRoundIdToCommits, votingRoundIdToReveals, - submit1: commitSubmissionResponse.data, - submit2: revealSubmissionResponse.data, }, }; } diff --git a/libs/ftso-core/src/DataManagerForRewarding.ts b/libs/ftso-core/src/DataManagerForRewarding.ts index 5bf80dd1..66822d49 100644 --- a/libs/ftso-core/src/DataManagerForRewarding.ts +++ b/libs/ftso-core/src/DataManagerForRewarding.ts @@ -18,6 +18,7 @@ import { FastUpdatesDataForVotingRound, PartialFDCDataForVotingRound, } from "./data-calculation-interfaces"; +import { AttestationRequest } from "./events/AttestationRequest"; import { ILogger } from "./utils/ILogger"; import { errorString } from "./utils/error"; import { Address, MessageHash } from "./voting-types"; @@ -122,7 +123,7 @@ export class DataManagerForRewarding extends DataManager { this.logger.debug(`Valid reveals from: ${JSON.stringify(Array.from(partialData.validEligibleReveals.keys()))}`); } //////// FDC //////// - const validEligibleBitVotes: SubmissionData[] = this.extractSubmissionsWithValidEligibleBitVotes(mappingsResponse.data.submit2, rewardEpoch); + const validEligibleBitVotes: SubmissionData[] = this.extractSubmissionsWithValidEligibleBitVotes(reveals, rewardEpoch); const dataForRound = { ...partialData, randomGenerationBenchingWindow, @@ -265,6 +266,7 @@ export class DataManagerForRewarding extends DataManager { let fdcData: FDCDataForVotingRound | undefined; let fdcRewardData: FDCRewardData | undefined; + let consensusBitVoteIndices: number[] = []; if (useFDCData) { const fdcFinalizations = this.extractFinalizations( @@ -301,13 +303,17 @@ export class DataManagerForRewarding extends DataManager { if (partialData.votingRoundId !== votingRoundId) { throw new Error(`Voting round id mismatch: ${partialData.votingRoundId} !== ${votingRoundId}`); } + if (partialData && partialData.nonDuplicationIndices && fdcRewardData && fdcRewardData.consensusBitVote !== undefined) { + consensusBitVoteIndices = DataManagerForRewarding.bitVoteIndicesNum(fdcRewardData.consensusBitVote, partialData.nonDuplicationIndices); + } fdcData = { ...partialData, bitVotes: dataForCalculations.validEligibleBitVoteSubmissions, signaturesMap: fdcSignatures, finalizations: fdcFinalizations, firstSuccessfulFinalization: fdcFirstSuccessfulFinalization, - ...fdcRewardData + ...fdcRewardData, + consensusBitVoteIndices, } } @@ -437,11 +443,11 @@ export class DataManagerForRewarding extends DataManager { } const result: PartialFDCDataForVotingRound[] = []; for (let votingRoundId = firstVotingRoundId; votingRoundId <= lastVotingRoundId; votingRoundId++) { - // const fastUpdateFeeds = attestationRequestsResponse.data[votingRoundId - firstVotingRoundId]; - // const fastUpdateSubmissions = feedUpdates.data[votingRoundId - firstVotingRoundId]; + const attestationRequests = attestationRequestsResponse.data[votingRoundId - firstVotingRoundId], const value: PartialFDCDataForVotingRound = { votingRoundId, - attestationRequests: attestationRequestsResponse.data[votingRoundId - firstVotingRoundId], + attestationRequests, + nonDuplicationIndices: DataManagerForRewarding.uniqueRequestsIndices(attestationRequests), }; result.push(value); } @@ -492,7 +498,7 @@ export class DataManagerForRewarding extends DataManager { fdcSignatures: Map[]>, rewardEpoch: RewardEpoch, ): FDCRewardData | undefined { - const voteCounter = new Map(); + const voteCounter = new Map(); // const eligibleSigners: FDCEligibleSigner[] = []; const offenseMap = new Map(); @@ -506,23 +512,28 @@ export class DataManagerForRewarding extends DataManager { } for (const signature of signatures) { const consensusBitVoteCandidate = signature.messages.unsignedMessage?.toLowerCase(); - if (!consensusBitVoteCandidate) { + if (!consensusBitVoteCandidate || consensusBitVoteCandidate.length < 6) { continue; } - voteCounter.set(consensusBitVoteCandidate, (voteCounter.get(consensusBitVoteCandidate) || 0) + signature.messages.weight) + const bitVoteNum = BigInt("0x" + consensusBitVoteCandidate.slice(6)); + // Note that 0n is also a legit consensus bitvote meaning no confirmations (but might not be rewarded) + voteCounter.set(bitVoteNum, (voteCounter.get(bitVoteNum) || 0) + signature.messages.weight) } - const maxCount = Math.max(...voteCounter.values()); - const maxHashes = [...voteCounter.entries()].filter(([_, count]) => count === maxCount).map(([hash, _]) => hash); - maxHashes.sort(); - // if it happens there are multiple maxHashes we take the first in lexicographical order - const consensusBitVote = maxHashes[0]; + let consensusBitVote: bigint | undefined; + if (voteCounter.size > 0) { + const maxCount = Math.max(...voteCounter.values()); + const maxBitVotes = [...voteCounter.entries()].filter(([_, count]) => count === maxCount).map(([bitVote, _]) => bitVote); + maxBitVotes.sort(); + // if it happens there are multiple maxHashes we take the first in lexicographical order + consensusBitVote = maxBitVotes[0]; - // TODO: - // should we require 50%+ weight on maxHash? - // const consensusBitVoteWeight = voteCounter.get(consensusBitVote); - // if(consensusBitVoteWeight < rewardEpoch.signingPolicy.threshold) { - // return undefined; - // } + // TODO: + // should we require 50%+ weight on maxHash? + // const consensusBitVoteWeight = voteCounter.get(consensusBitVote); + // if(consensusBitVoteWeight < rewardEpoch.signingPolicy.threshold) { + // return undefined; + // } + } const submitSignatureAddressToBitVote = new Map(); for (const submission of bitVoteSubmissions) { @@ -547,7 +558,7 @@ export class DataManagerForRewarding extends DataManager { submitSignatureAddress: signature.submitAddress.toLowerCase(), relativeTimestamp: signature.relativeTimestamp, bitVote, - dominatesConsensusBitVote: DataManagerForRewarding.isConsensusVoteDominated(consensusBitVote, bitVote), + dominatesConsensusBitVote: consensusBitVote === undefined ? undefined : DataManagerForRewarding.isConsensusVoteDominated(consensusBitVote, bitVote), weight: signature.messages.weight, } eligibleSigners.push(eligibleSigner); @@ -568,7 +579,7 @@ export class DataManagerForRewarding extends DataManager { if (wrongSignatures) { for (const signature of wrongSignatures) { const submitSignatureAddress = signature.submitAddress.toLowerCase(); - if(!rewardEpoch.isEligibleSubmitSignatureAddress(submitSignatureAddress)) { + if (!rewardEpoch.isEligibleSubmitSignatureAddress(submitSignatureAddress)) { continue; } const offender = offenseMap.get(submitSignatureAddress) || { @@ -579,15 +590,21 @@ export class DataManagerForRewarding extends DataManager { offenseMap.set(submitSignatureAddress, offender); } } - - for(const signature of signatures) { + for (const signature of signatures) { const submitSignatureAddress = signature.submitAddress.toLowerCase(); const consensusBitVoteCandidate = signature.messages.unsignedMessage?.toLowerCase(); - if(consensusBitVoteCandidate !== consensusBitVote) { + if (!consensusBitVoteCandidate) { + continue; + } + let isOffense = consensusBitVoteCandidate.length < 6; + if (!isOffense) { + isOffense = BigInt("0x" + consensusBitVoteCandidate.slice(6)) !== consensusBitVote; + } + if (isOffense) { const offender = offenseMap.get(submitSignatureAddress) || { submitSignatureAddress, offenses: [] - } + } offender.offenses.push(FDCOffense.BAD_CONSENSUS_BITVOTE_CANDIDATE); offenseMap.set(submitSignatureAddress, offender); } @@ -598,22 +615,69 @@ export class DataManagerForRewarding extends DataManager { const result: FDCRewardData = { eligibleSigners, consensusBitVote, - fdcOffenders + fdcOffenders }; return result; } - public static isConsensusVoteDominated(consensusBitVote: string, bitVote?: string): boolean { + public static uniqueRequestsIndices(attestationRequests: AttestationRequest[]): number[] { + const encountered = new Set(); + const result: number[] = []; + for (let i = 0; i < attestationRequests.length; i++) { + const request = attestationRequests[i]; + if (!encountered.has(request.data)) { + result.push(i); + encountered.add(request.data); + } + } + return result; + } + + public static bitVoteIndices(bitVote: string, indices: number[]): number[] | undefined { + if (!bitVote || bitVote.length < 4) { + return undefined + } + const length = parseInt(bitVote.slice(2, 4), 16); + if (length !== indices.length) { + throw new Error(`Bitvote length mismatch: ${length} !== ${indices.length}`); + } + + const result: number[] = []; + let bitVoteNum = BigInt("0x" + bitVote.slice(4)); + return DataManagerForRewarding.bitVoteIndicesNum(bitVoteNum, indices); + } + + public static bitVoteIndicesNum(bitVoteNum: bigint, indices: number[]): number[] { + const result: number[] = []; + for (let i = 0; i < indices.length; i++) { + if (bitVoteNum % 2n === 1n) { + result.push(indices[i]); + } + bitVoteNum /= 2n; + } + if (bitVoteNum !== 0n) { + throw new Error(`bitVoteNum not fully consumed: ${bitVoteNum}`); + } + return result; + } + + public static isConsensusVoteDominated(consensusBitVote: bigint, bitVote?: string): boolean { if (!bitVote) { return false; } - let h1 = consensusBitVote.startsWith("0x") ? consensusBitVote.slice(2) : consensusBitVote; - let h2 = bitVote.startsWith("0x") ? bitVote.slice(2) : bitVote; + // Remove 0x prefix and first 2 bytes, used for the length + let h1 = consensusBitVote.toString(16); + // Ensure even length + if (h1.length % 2 !== 0) { + h1 = "0" + h1; + } + // This one is always even length + let h2 = bitVote.startsWith("0x") ? bitVote.slice(6) : bitVote.slice(4); if (h1.length !== h2.length) { const mLen = Math.max(h1.length, h2.length); - h1 = h1.padEnd(mLen, "0"); - h2 = h2.padEnd(mLen, "0"); + h1 = h1.padStart(mLen, "0"); + h2 = h2.padStart(mLen, "0"); } const buf1 = Buffer.from(h1, "hex"); const buf2 = Buffer.from(h2, "hex"); diff --git a/libs/ftso-core/src/data-calculation-interfaces.ts b/libs/ftso-core/src/data-calculation-interfaces.ts index 92a75b7f..93b8ed17 100644 --- a/libs/ftso-core/src/data-calculation-interfaces.ts +++ b/libs/ftso-core/src/data-calculation-interfaces.ts @@ -56,6 +56,8 @@ export interface PartialFDCDataForVotingRound { votingRoundId: number; // List of attestation requests to be processed attestationRequests: AttestationRequest[]; + // list of non-duplication indices + nonDuplicationIndices: number[]; } /** @@ -91,9 +93,11 @@ export interface FDCRewardData { eligibleSigners?: FDCEligibleSigner[]; // Majority bitvote attached to the finalized signatures // All signers that have unmatching majority bitvote are considered as offenders - consensusBitVote?: string; + consensusBitVote?: bigint; // FDC offenders fdcOffenders?: FDCOffender[]; + // Consensus bitVote indices + consensusBitVoteIndices?: number[]; } export interface FDCDataForVotingRound extends PartialFDCDataForVotingRound, FDCRewardData { @@ -121,8 +125,9 @@ export interface SFDCDataForVotingRound { firstSuccessfulFinalization?: ParsedFinalizationData; finalizations: ParsedFinalizationData[]; eligibleSigners: FDCEligibleSigner[]; - consensusBitVote: string; + consensusBitVote: bigint; fdcOffenders: FDCOffender[]; + consensusBitVoteIndices: number[]; signaturesMap?: Map[]>; } diff --git a/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts b/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts index c4186f99..f6a2124e 100644 --- a/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts +++ b/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts @@ -157,6 +157,7 @@ export function serializeDataForRewardCalculation( finalizations: rewardCalculationData.fdcData.finalizations, eligibleSigners: rewardCalculationData.fdcData.eligibleSigners, consensusBitVote: rewardCalculationData.fdcData.consensusBitVote, + consensusBitVoteIndices: rewardCalculationData.fdcData.consensusBitVoteIndices, fdcOffenders: rewardCalculationData.fdcData.fdcOffenders, } } From d047dbbf6b48d30dbc9ad35671c7c1f82f71d060 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Sat, 26 Oct 2024 00:47:39 +0200 Subject: [PATCH 08/28] bitvote mappings --- libs/ftso-core/src/DataManagerForRewarding.ts | 38 +++++++++++-------- .../src/data-calculation-interfaces.ts | 2 +- .../src/events/AttestationRequest.ts | 3 ++ 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/libs/ftso-core/src/DataManagerForRewarding.ts b/libs/ftso-core/src/DataManagerForRewarding.ts index 66822d49..7b431f14 100644 --- a/libs/ftso-core/src/DataManagerForRewarding.ts +++ b/libs/ftso-core/src/DataManagerForRewarding.ts @@ -304,7 +304,13 @@ export class DataManagerForRewarding extends DataManager { throw new Error(`Voting round id mismatch: ${partialData.votingRoundId} !== ${votingRoundId}`); } if (partialData && partialData.nonDuplicationIndices && fdcRewardData && fdcRewardData.consensusBitVote !== undefined) { - consensusBitVoteIndices = DataManagerForRewarding.bitVoteIndicesNum(fdcRewardData.consensusBitVote, partialData.nonDuplicationIndices); + consensusBitVoteIndices = DataManagerForRewarding.bitVoteIndicesNum(fdcRewardData.consensusBitVote, partialData.nonDuplicationIndices.length); + for (const bitVoteIndex of consensusBitVoteIndices) { + for (const [i, originalIndex] of partialData.nonDuplicationIndices[bitVoteIndex].entries()) { + partialData.attestationRequests[originalIndex].confirmed = true; + partialData.attestationRequests[originalIndex].duplicate = i > 0; + } + } } fdcData = { ...partialData, @@ -443,7 +449,7 @@ export class DataManagerForRewarding extends DataManager { } const result: PartialFDCDataForVotingRound[] = []; for (let votingRoundId = firstVotingRoundId; votingRoundId <= lastVotingRoundId; votingRoundId++) { - const attestationRequests = attestationRequestsResponse.data[votingRoundId - firstVotingRoundId], + const attestationRequests = attestationRequestsResponse.data[votingRoundId - firstVotingRoundId]; const value: PartialFDCDataForVotingRound = { votingRoundId, attestationRequests, @@ -621,38 +627,40 @@ export class DataManagerForRewarding extends DataManager { return result; } - public static uniqueRequestsIndices(attestationRequests: AttestationRequest[]): number[] { - const encountered = new Set(); - const result: number[] = []; + public static uniqueRequestsIndices(attestationRequests: AttestationRequest[]): number[][] { + const encountered = new Map(); + const result: number[][] = []; for (let i = 0; i < attestationRequests.length; i++) { const request = attestationRequests[i]; - if (!encountered.has(request.data)) { - result.push(i); - encountered.add(request.data); + if (!encountered.get(request.data)) { + encountered.set(request.data, i); + result.push([i]); + } else { + result[encountered.get(request.data)].push(i); } } return result; } - public static bitVoteIndices(bitVote: string, indices: number[]): number[] | undefined { + public static bitVoteIndices(bitVote: string, len: number): number[] | undefined { if (!bitVote || bitVote.length < 4) { return undefined } const length = parseInt(bitVote.slice(2, 4), 16); - if (length !== indices.length) { - throw new Error(`Bitvote length mismatch: ${length} !== ${indices.length}`); + if (length !== len) { + throw new Error(`Bitvote length mismatch: ${length} !== ${len}`); } const result: number[] = []; let bitVoteNum = BigInt("0x" + bitVote.slice(4)); - return DataManagerForRewarding.bitVoteIndicesNum(bitVoteNum, indices); + return DataManagerForRewarding.bitVoteIndicesNum(bitVoteNum, len); } - public static bitVoteIndicesNum(bitVoteNum: bigint, indices: number[]): number[] { + public static bitVoteIndicesNum(bitVoteNum: bigint, len: number): number[] { const result: number[] = []; - for (let i = 0; i < indices.length; i++) { + for (let i = 0; i < len; i++) { if (bitVoteNum % 2n === 1n) { - result.push(indices[i]); + result.push(i); } bitVoteNum /= 2n; } diff --git a/libs/ftso-core/src/data-calculation-interfaces.ts b/libs/ftso-core/src/data-calculation-interfaces.ts index 93b8ed17..ca75b56b 100644 --- a/libs/ftso-core/src/data-calculation-interfaces.ts +++ b/libs/ftso-core/src/data-calculation-interfaces.ts @@ -57,7 +57,7 @@ export interface PartialFDCDataForVotingRound { // List of attestation requests to be processed attestationRequests: AttestationRequest[]; // list of non-duplication indices - nonDuplicationIndices: number[]; + nonDuplicationIndices: number[][]; } /** diff --git a/libs/ftso-core/src/events/AttestationRequest.ts b/libs/ftso-core/src/events/AttestationRequest.ts index 85bdf1e0..da5db39b 100644 --- a/libs/ftso-core/src/events/AttestationRequest.ts +++ b/libs/ftso-core/src/events/AttestationRequest.ts @@ -35,4 +35,7 @@ export class AttestationRequest extends RawEventConstructible { // confirmed confirmed: boolean = false; + + // duplicate + duplicate: boolean = false; } From 14e81a52bea7bd39f6317a3eaf666c6c16dae5c1 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Mon, 28 Oct 2024 09:51:21 +0100 Subject: [PATCH 09/28] extracting number of attestation req per type in round --- .../src/libs/fdc-utils.ts | 94 +++++++++++++++++++ .../src/services/calculator.service.ts | 6 +- .../src/events/AttestationRequest.ts | 12 ++- .../src/utils/stat-info/constants.ts | 1 + .../stat-info/reward-calculation-data.ts | 6 +- 5 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 apps/ftso-reward-calculation-process/src/libs/fdc-utils.ts diff --git a/apps/ftso-reward-calculation-process/src/libs/fdc-utils.ts b/apps/ftso-reward-calculation-process/src/libs/fdc-utils.ts new file mode 100644 index 00000000..5c34be12 --- /dev/null +++ b/apps/ftso-reward-calculation-process/src/libs/fdc-utils.ts @@ -0,0 +1,94 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; +import path from "path/posix"; +import { CALCULATIONS_FOLDER } from "../../../../libs/ftso-core/src/configs/networks"; +import { FDC_ATTESTATION_APPEARANCES_FILE } from "../../../../libs/ftso-core/src/utils/stat-info/constants"; +import { deserializeDataForRewardCalculation } from "../../../../libs/ftso-core/src/utils/stat-info/reward-calculation-data"; +import { deserializeRewardEpochInfo } from "../../../../libs/ftso-core/src/utils/stat-info/reward-epoch-info"; +import { AttestationRequest } from "../../../../libs/ftso-core/src/events/AttestationRequest"; + +export interface FDCAttestationRequestAppearances { + attestationRequestId: string; + attestationType: string; + source: string; + count: number; +} + +export function calculateAttestationTypeAppearances(rewardEpochId: number): void { + const rewardEpochInfo = deserializeRewardEpochInfo(rewardEpochId); + + const attestationTypeCount = new Map(); + for ( + let votingRoundId = rewardEpochInfo.signingPolicy.startVotingRoundId; + votingRoundId < rewardEpochInfo.endVotingRoundId; + votingRoundId++ + ) { + const currentCalculationData = deserializeDataForRewardCalculation( + rewardEpochId, + votingRoundId + ); + if (!currentCalculationData) { + throw new Error(`Missing reward calculation data for voting round ${votingRoundId}`); + } + const attestationRequests = currentCalculationData?.fdcData?.attestationRequests; + if (!attestationRequests) { + continue; + } + for (const attestationRequest of attestationRequests) { + if (attestationRequest.confirmed && !attestationRequest.duplicate) { + const id = AttestationRequest.getId(attestationRequest); + if (id) { + attestationTypeCount.set(id, (attestationTypeCount.get(id) || 0) + 1); + } + } + } + } + const appearances: FDCAttestationRequestAppearances[] = []; + for (const [attestationRequestId, appearancesCount] of attestationTypeCount.entries()) { + const appearance: FDCAttestationRequestAppearances = { + attestationRequestId, + count: appearancesCount, + attestationType: Buffer.from(attestationRequestId.slice(2, 66), "hex").toString("utf8").replaceAll("\0", ""), + source: Buffer.from(attestationRequestId.slice(66, 130), "hex").toString("utf8").replaceAll("\0", "") + } + appearances.push(appearance); + } + serializeAttestationRequestAppearances(appearances, rewardEpochId); +} + + +/** + * Writes the data regarding attestation request appearances. + * The data is stored in + * `//FDC_ATTESTATION_APPEARANCES_FILE`. + */ +export function serializeAttestationRequestAppearances( + appearances: FDCAttestationRequestAppearances[], + rewardEpochId: number, + calculationFolder = CALCULATIONS_FOLDER() +): void { + const rewardEpochFolder = path.join( + calculationFolder, + `${rewardEpochId}` + ); + if (!existsSync(rewardEpochFolder)) { + mkdirSync(rewardEpochFolder); + } + const appearancesPath = path.join(rewardEpochFolder, FDC_ATTESTATION_APPEARANCES_FILE); + writeFileSync(appearancesPath, JSON.stringify(appearances)); +} + + +export function deserializeAttestationRequestAppearances( + rewardEpochId: number, + calculationFolder = CALCULATIONS_FOLDER() +): FDCAttestationRequestAppearances[] { + const rewardEpochFolder = path.join( + calculationFolder, + `${rewardEpochId}` + ); + const appearancesPath = path.join(rewardEpochFolder, FDC_ATTESTATION_APPEARANCES_FILE); + const data = JSON.parse( + readFileSync(appearancesPath, "utf-8") + ) as FDCAttestationRequestAppearances[]; + return data; +} \ No newline at end of file diff --git a/apps/ftso-reward-calculation-process/src/services/calculator.service.ts b/apps/ftso-reward-calculation-process/src/services/calculator.service.ts index 2fa66d65..86293a36 100644 --- a/apps/ftso-reward-calculation-process/src/services/calculator.service.ts +++ b/apps/ftso-reward-calculation-process/src/services/calculator.service.ts @@ -45,6 +45,7 @@ import { runRandomNumberFixing } from "../libs/random-number-fixing-utils"; import { runCalculateRewardClaimsTopJob } from "../libs/reward-claims-calculation"; import { runCalculateRewardCalculationTopJob } from "../libs/reward-data-calculation"; import { getIncrementalCalculationsFeedSelections, serializeIncrementalCalculationsFeedSelections } from "../../../../libs/ftso-core/src/utils/stat-info/incremental-calculation-temp-selected-feeds"; +import { calculateAttestationTypeAppearances } from "../libs/fdc-utils"; if (process.env.FORCE_NOW) { const newNow = parseInt(process.env.FORCE_NOW) * 1000; @@ -173,12 +174,12 @@ export class CalculatorService { const feedSelections = getIncrementalCalculationsFeedSelections(rewardEpochDuration.rewardEpochId, state.nextVotingRoundIdWithNoSecureRandom - 1); serializeIncrementalCalculationsFeedSelections(feedSelections); - + await calculateAndAggregateRemainingClaims(this.dataManager, state, options, logger); const tempClaimData = getIncrementalCalculationsTempRewards(rewardEpochDuration.rewardEpochId, state.nextVotingRoundForClaimCalculation - 1); serializeIncrementalCalculationsTempRewards(tempClaimData); - + recordProgress(rewardEpochId); state.votingRoundId++; @@ -228,6 +229,7 @@ export class CalculatorService { logger.log(rewardEpochDuration2); await runRandomNumberFixing(options.rewardEpochId, FUTURE_VOTING_ROUNDS()); destroyStorage(options.rewardEpochId + 1, true); + calculateAttestationTypeAppearances(options.rewardEpochId); } async fullRoundClaimCalculation(options: OptionalCommandOptions): Promise { diff --git a/libs/ftso-core/src/events/AttestationRequest.ts b/libs/ftso-core/src/events/AttestationRequest.ts index da5db39b..4e3b5340 100644 --- a/libs/ftso-core/src/events/AttestationRequest.ts +++ b/libs/ftso-core/src/events/AttestationRequest.ts @@ -7,7 +7,7 @@ export class AttestationRequest extends RawEventConstructible { static eventName = "AttestationRequest"; constructor(data: any, timestamp: number) { super(); - if(timestamp === undefined) { + if (timestamp === undefined) { throw new Error("Timestamp is required"); } this.data = data.data; @@ -24,6 +24,14 @@ export class AttestationRequest extends RawEventConstructible { ); } + static getId(attestationRequest: AttestationRequest): string { + // 0x + 64 bytes in hex + if (attestationRequest.data.length < 130) { + return undefined; + } + return attestationRequest.data.substring(0, 130); + } + // Feed values in the order of feedIds data: string; @@ -31,7 +39,7 @@ export class AttestationRequest extends RawEventConstructible { fee: bigint; // timestamp - timestamp: number; + timestamp: number; // confirmed confirmed: boolean = false; diff --git a/libs/ftso-core/src/utils/stat-info/constants.ts b/libs/ftso-core/src/utils/stat-info/constants.ts index 36921af2..41ad165a 100644 --- a/libs/ftso-core/src/utils/stat-info/constants.ts +++ b/libs/ftso-core/src/utils/stat-info/constants.ts @@ -1,6 +1,7 @@ export const OFFERS_FILE = "offers.json"; export const FU_OFFERS_FILE = "fast-updates-offers.json"; export const FDC_OFFERS_FILE = "fdc-offers.json"; +export const FDC_ATTESTATION_APPEARANCES_FILE = "fdc-attestation-appearances.json"; export const CLAIMS_FILE = "claims.json"; export const AGGREGATED_CLAIMS_FILE = "aggregated-claims.json"; export const FEED_VALUES_FILE = "feed-values.json"; diff --git a/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts b/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts index f6a2124e..5b7868e6 100644 --- a/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts +++ b/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts @@ -5,7 +5,7 @@ import { ISignaturePayload } from "../../../../fsp-utils/src/SignaturePayload"; import { GenericSubmissionData, ParsedFinalizationData } from "../../IndexerClient"; import { VoterWeights } from "../../RewardEpoch"; import { CALCULATIONS_FOLDER } from "../../configs/networks"; -import { DataForRewardCalculation, FDCDataForVotingRound, FastUpdatesDataForVotingRound, SFDCDataForVotingRound } from "../../data-calculation-interfaces"; +import { DataForRewardCalculation, FastUpdatesDataForVotingRound, SFDCDataForVotingRound } from "../../data-calculation-interfaces"; import { Address, Feed, MedianCalculationResult, MessageHash, RandomCalculationResult } from "../../voting-types"; import { IRevealData } from "../RevealData"; import { bigIntReplacer, bigIntReviver } from "../big-number-serialization"; @@ -136,7 +136,7 @@ export function serializeDataForRewardCalculation( hashSignatures.push(hashRecord); } - + let fdcData: SFDCDataForVotingRound | undefined; if (rewardCalculationData.fdcData) { @@ -158,7 +158,7 @@ export function serializeDataForRewardCalculation( eligibleSigners: rewardCalculationData.fdcData.eligibleSigners, consensusBitVote: rewardCalculationData.fdcData.consensusBitVote, consensusBitVoteIndices: rewardCalculationData.fdcData.consensusBitVoteIndices, - fdcOffenders: rewardCalculationData.fdcData.fdcOffenders, + fdcOffenders: rewardCalculationData.fdcData.fdcOffenders, } } From 72739e703e3fe44a3e9a558b9b23fae63d2b0db6 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:09:09 +0100 Subject: [PATCH 10/28] fdc rewarding implemented --- .../src/libs/offer-utils.ts | 7 +- libs/ftso-core/src/DataManager.ts | 2 + libs/ftso-core/src/DataManagerForRewarding.ts | 12 ++ libs/ftso-core/src/RewardEpoch.ts | 22 +++ .../src/data-calculation-interfaces.ts | 6 + .../reward-calculation/RewardTypePrefix.ts | 1 + .../fdc/reward-fdc-penalties.ts | 40 +++++ .../fdc/reward-fdc-signing.ts | 146 ++++++++++++++++++ .../reward-calculation/reward-calculation.ts | 66 +++++++- .../reward-calculation/reward-finalization.ts | 23 +-- .../src/reward-calculation/reward-offers.ts | 78 ++++++++-- .../reward-calculation/reward-penalties.ts | 3 +- .../reward-signing-split.ts | 13 +- .../src/reward-calculation/reward-signing.ts | 5 +- .../src/reward-calculation/reward-utils.ts | 5 +- .../ftso-core/src/utils/PartialRewardOffer.ts | 43 +++++- .../granulated-partial-offers-map.ts | 23 ++- .../stat-info/reward-calculation-data.ts | 2 + scripts/analytics/run/offer-check.ts | 80 ++++++++++ scripts/rewards/coston-db.sh | 4 +- test/libs/unit/reward-offers.test.ts | 4 +- 21 files changed, 528 insertions(+), 57 deletions(-) create mode 100644 libs/ftso-core/src/reward-calculation/fdc/reward-fdc-penalties.ts create mode 100644 libs/ftso-core/src/reward-calculation/fdc/reward-fdc-signing.ts create mode 100644 scripts/analytics/run/offer-check.ts diff --git a/apps/ftso-reward-calculation-process/src/libs/offer-utils.ts b/apps/ftso-reward-calculation-process/src/libs/offer-utils.ts index 79f57d44..1f624828 100644 --- a/apps/ftso-reward-calculation-process/src/libs/offer-utils.ts +++ b/apps/ftso-reward-calculation-process/src/libs/offer-utils.ts @@ -4,9 +4,8 @@ import { granulatedPartialOfferMapForRandomFeedSelection, } from "../../../../libs/ftso-core/src/reward-calculation/reward-offers"; import { - IFDCPartialRewardOfferForRound, IFUPartialRewardOfferForRound, - IPartialRewardOfferForRound, + IPartialRewardOfferForRound } from "../../../../libs/ftso-core/src/utils/PartialRewardOffer"; import { RewardEpochDuration } from "../../../../libs/ftso-core/src/utils/RewardEpochDuration"; import { FDC_OFFERS_FILE, FU_OFFERS_FILE, OFFERS_FILE } from "../../../../libs/ftso-core/src/utils/stat-info/constants"; @@ -60,9 +59,9 @@ export async function fullRoundOfferCalculation(options: OptionalCommandOptions) } if (options.useFDCData) { - const fdcRewardOfferMap: Map + const fdcRewardOfferMap: Map = granulatedPartialOfferMapForFDC(rewardEpochInfo); - serializeGranulatedPartialOfferMapForFDC(rewardEpochDuration, fdcRewardOfferMap, false, FDC_OFFERS_FILE); + serializeGranulatedPartialOfferMapForFDC(rewardEpochDuration, fdcRewardOfferMap, false, FDC_OFFERS_FILE); } } diff --git a/libs/ftso-core/src/DataManager.ts b/libs/ftso-core/src/DataManager.ts index 9b4f3882..ae571ca9 100644 --- a/libs/ftso-core/src/DataManager.ts +++ b/libs/ftso-core/src/DataManager.ts @@ -556,6 +556,7 @@ export class DataManager { const revealOffenders = this.getRevealOffenders(commitsAndReveals.votingRoundId, eligibleCommits, eligibleReveals); const voterMedianVotingWeights = new Map(); const orderedVotersSubmissionAddresses = rewardEpoch.orderedVotersSubmitAddresses; + const orderedVotersSubmitSignatureAddresses = rewardEpoch.orderedVotersSubmitSignatureAddresses; for (const submitAddress of orderedVotersSubmissionAddresses) { voterMedianVotingWeights.set(submitAddress, rewardEpoch.ftsoMedianVotingWeight(submitAddress)); } @@ -563,6 +564,7 @@ export class DataManager { const result: DataForCalculationsPartial = { votingRoundId: commitsAndReveals.votingRoundId, orderedVotersSubmitAddresses: orderedVotersSubmissionAddresses, + orderedVotersSubmitSignatureAddresses, validEligibleReveals, revealOffenders, voterMedianVotingWeights, diff --git a/libs/ftso-core/src/DataManagerForRewarding.ts b/libs/ftso-core/src/DataManagerForRewarding.ts index 7b431f14..aef5137f 100644 --- a/libs/ftso-core/src/DataManagerForRewarding.ts +++ b/libs/ftso-core/src/DataManagerForRewarding.ts @@ -562,6 +562,8 @@ export class DataManagerForRewarding extends DataManager { const bitVote = submitSignatureAddressToBitVote.get(submitSignatureAddress); const eligibleSigner: FDCEligibleSigner = { submitSignatureAddress: signature.submitAddress.toLowerCase(), + timestamp: signature.timestamp, + votingEpochIdFromTimestamp: signature.votingEpochIdFromTimestamp, relativeTimestamp: signature.relativeTimestamp, bitVote, dominatesConsensusBitVote: consensusBitVote === undefined ? undefined : DataManagerForRewarding.isConsensusVoteDominated(consensusBitVote, bitVote), @@ -572,9 +574,12 @@ export class DataManagerForRewarding extends DataManager { for (const submission of bitVoteSubmissions) { const submitSignatureAddress = rewardEpoch.getSubmitSignatureAddressFromSubmitAddress(submission.submitAddress).toLowerCase(); + const submissionAddress = rewardEpoch.getSubmitAddressFromSubmitSignatureAddress(submitSignatureAddress).toLowerCase(); if (!submitSignatureSenders.has(submitSignatureAddress)) { const offender: FDCOffender = { submitSignatureAddress, + submissionAddress, + weight: rewardEpoch.getSigningWeightForSubmitSignatureAddress(submitSignatureAddress), offenses: [FDCOffense.NO_REVEAL_ON_BITVOTE] } offenseMap.set(submitSignatureAddress, offender); @@ -585,11 +590,14 @@ export class DataManagerForRewarding extends DataManager { if (wrongSignatures) { for (const signature of wrongSignatures) { const submitSignatureAddress = signature.submitAddress.toLowerCase(); + const submissionAddress = rewardEpoch.getSubmitAddressFromSubmitSignatureAddress(submitSignatureAddress).toLowerCase(); if (!rewardEpoch.isEligibleSubmitSignatureAddress(submitSignatureAddress)) { continue; } const offender = offenseMap.get(submitSignatureAddress) || { submitSignatureAddress, + submissionAddress, + weight: rewardEpoch.getSigningWeightForSubmitSignatureAddress(submitSignatureAddress), offenses: [] } offender.offenses.push(FDCOffense.WRONG_SIGNATURE); @@ -598,10 +606,12 @@ export class DataManagerForRewarding extends DataManager { } for (const signature of signatures) { const submitSignatureAddress = signature.submitAddress.toLowerCase(); + const submissionAddress = rewardEpoch.getSubmitAddressFromSubmitSignatureAddress(submitSignatureAddress).toLowerCase(); const consensusBitVoteCandidate = signature.messages.unsignedMessage?.toLowerCase(); if (!consensusBitVoteCandidate) { continue; } + // 0x + 2 bytes length let isOffense = consensusBitVoteCandidate.length < 6; if (!isOffense) { isOffense = BigInt("0x" + consensusBitVoteCandidate.slice(6)) !== consensusBitVote; @@ -609,6 +619,8 @@ export class DataManagerForRewarding extends DataManager { if (isOffense) { const offender = offenseMap.get(submitSignatureAddress) || { submitSignatureAddress, + submissionAddress, + weight: rewardEpoch.getSigningWeightForSubmitSignatureAddress(submitSignatureAddress), offenses: [] } offender.offenses.push(FDCOffense.BAD_CONSENSUS_BITVOTE_CANDIDATE); diff --git a/libs/ftso-core/src/RewardEpoch.ts b/libs/ftso-core/src/RewardEpoch.ts index 23033bd1..88653331 100644 --- a/libs/ftso-core/src/RewardEpoch.ts +++ b/libs/ftso-core/src/RewardEpoch.ts @@ -24,6 +24,7 @@ export interface VoterWeights { export class RewardEpoch { readonly orderedVotersSubmitAddresses: Address[] = []; + readonly orderedVotersSubmitSignatureAddresses: Address[] = []; public readonly rewardOffers: RewardOffers; public readonly signingPolicy: SigningPolicyInitialized; @@ -155,6 +156,7 @@ export class RewardEpoch { fullVoterRegistrationInfo ); this.orderedVotersSubmitAddresses.push(fullVoterRegistrationInfo.voterRegistered.submitAddress); + this.orderedVotersSubmitSignatureAddresses.push(fullVoterRegistrationInfo.voterRegistered.submitSignaturesAddress); this.signingAddressToDelegationAddress.set( voterSigningAddress, @@ -211,6 +213,26 @@ export class RewardEpoch { return this.submitAddressToVoterRegistrationInfo.get(submitAddress.toLowerCase())?.voterRegistered.submitSignaturesAddress; } + public getSubmitAddressFromSubmitSignatureAddress(submitSignatureAddress: Address): Address | undefined { + const voterAddress = this.submitSignatureAddressToVoter.get(submitSignatureAddress.toLowerCase()); + if (!voterAddress) { + return undefined; + } + return this.voterToRegistrationInfo.get(voterAddress.toLowerCase())?.voterRegistered.submitAddress; + } + + public getSigningWeightForSubmitSignatureAddress(submitSignatureAddress: Address): number | undefined { + const voterAddress = this.submitSignatureAddressToVoter.get(submitSignatureAddress.toLowerCase()); + if (!voterAddress) { + return undefined; + } + const signingAddress = this.voterToRegistrationInfo.get(voterAddress.toLowerCase())?.voterRegistered.signingPolicyAddress; + if (!signingAddress) { + return undefined; + } + return this.signingAddressToSigningWeight.get(signingAddress); + } + /** * Returns weight for participation in median voting. * @param submissionAddress diff --git a/libs/ftso-core/src/data-calculation-interfaces.ts b/libs/ftso-core/src/data-calculation-interfaces.ts index ca75b56b..dd0982ea 100644 --- a/libs/ftso-core/src/data-calculation-interfaces.ts +++ b/libs/ftso-core/src/data-calculation-interfaces.ts @@ -11,6 +11,8 @@ export interface DataForCalculationsPartial { votingRoundId: number; // Ordered list of submitAddresses matching the order in the signing policy orderedVotersSubmitAddresses: Address[]; + // Ordered list of submitSignatureAddresses matching the order in the signing policy + orderedVotersSubmitSignatureAddresses: Address[]; // Reveals from eligible submitAddresses that match to existing commits validEligibleReveals: Map; // submitAddresses of eligible voters that committed but withheld or provided wrong reveals in the voting round @@ -68,6 +70,8 @@ export interface PartialFDCDataForVotingRound { */ export interface FDCEligibleSigner { submitSignatureAddress: string; + votingEpochIdFromTimestamp: number; + timestamp: number; // Relative timestamp in voting epoch N + 1 relativeTimestamp: number; bitVote?: string; @@ -82,8 +86,10 @@ export enum FDCOffense { BAD_CONSENSUS_BITVOTE_CANDIDATE = "BAD_CONSENSUS_BITVOTE_CANDIDATE", } export interface FDCOffender { + submissionAddress: string; submitSignatureAddress: string; offenses: FDCOffense[]; + weight: number; } export interface FDCRewardData { diff --git a/libs/ftso-core/src/reward-calculation/RewardTypePrefix.ts b/libs/ftso-core/src/reward-calculation/RewardTypePrefix.ts index cc33e8e5..2abf2479 100644 --- a/libs/ftso-core/src/reward-calculation/RewardTypePrefix.ts +++ b/libs/ftso-core/src/reward-calculation/RewardTypePrefix.ts @@ -9,4 +9,5 @@ export enum RewardTypePrefix { REVEAL_OFFENDERS = "Reveal offenders", FAST_UPDATES_ACCURACY = "Fast updates accuracy", FULL_OFFER_CLAIM_BACK = "Full offer claim back", + PARTIAL_FDC_OFFER_CLAIM_BACK = "Partial FDC offer claim back", } diff --git a/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-penalties.ts b/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-penalties.ts new file mode 100644 index 00000000..a9aa1a1c --- /dev/null +++ b/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-penalties.ts @@ -0,0 +1,40 @@ +import { VoterWeights } from "../../RewardEpoch"; +import { FDC_PROTOCOL_ID } from "../../configs/networks"; +import { IPartialRewardOfferForRound } from "../../utils/PartialRewardOffer"; +import { IPartialRewardClaim } from "../../utils/RewardClaim"; +import { SDataForRewardCalculation } from "../../utils/stat-info/reward-calculation-data"; +import { RewardEpochInfo } from "../../utils/stat-info/reward-epoch-info"; +import { Address } from "../../voting-types"; +import { RewardTypePrefix } from "../RewardTypePrefix"; +import { generateSigningWeightBasedClaimsForVoter } from "../reward-signing-split"; + +/** + * * Given a @param offer, @param penaltyFactor and @param votersWeights penalty claims for offenders. + * The penalty amount is proportional to the weight of the offender. + */ +export function calculateFdcPenalties( + offer: IPartialRewardOfferForRound, + rewardEpochInfo: RewardEpochInfo, + data: SDataForRewardCalculation, + penaltyFactor: bigint, + votersWeights: Map, + penaltyType: RewardTypePrefix +): IPartialRewardClaim[] { + const penaltyClaims: IPartialRewardClaim[] = []; + if(!data.fdcData.fdcOffenders) { + return penaltyClaims; + } + const totalWeight = BigInt(rewardEpochInfo.signingPolicy.weights.reduce((acc, weight) => acc + weight, 0)); + + for (const offender of data.fdcData.fdcOffenders) { + const voterWeights = votersWeights.get(offender.submissionAddress)!; + let penalty = 0n; + if (offender.weight > 0) { + penalty = (-BigInt(offender.weight) * offer.amount * penaltyFactor) / totalWeight; + } + if (penalty > 0n) { + penaltyClaims.push(...generateSigningWeightBasedClaimsForVoter(penalty, offer, voterWeights, penaltyType, FDC_PROTOCOL_ID)); + } + } + return penaltyClaims; +} diff --git a/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-signing.ts b/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-signing.ts new file mode 100644 index 00000000..df77f157 --- /dev/null +++ b/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-signing.ts @@ -0,0 +1,146 @@ +import { EPOCH_SETTINGS, FDC_PROTOCOL_ID, FINALIZATION_BIPS, TOTAL_BIPS } from "../../configs/networks"; +import { FDCEligibleSigner } from "../../data-calculation-interfaces"; +import { IPartialRewardOfferForRound } from "../../utils/PartialRewardOffer"; +import { ClaimType, IPartialRewardClaim } from "../../utils/RewardClaim"; +import { SDataForRewardCalculation } from "../../utils/stat-info/reward-calculation-data"; +import { RewardEpochInfo } from "../../utils/stat-info/reward-epoch-info"; +import { Address } from "../../voting-types"; +import { RewardTypePrefix } from "../RewardTypePrefix"; +import { SigningRewardClaimType } from "../reward-signing"; +import { generateSigningWeightBasedClaimsForVoter } from "../reward-signing-split"; +import { isSignatureBeforeTimestamp, isSignatureInGracePeriod } from "../reward-utils"; + +/** + * A split of partial reward offer into three parts: + */ +export interface SplitFDCRewardOffer { + readonly signingRewardOffer: IPartialRewardOfferForRound; + readonly finalizationRewardOffer: IPartialRewardOfferForRound; +} + + +/** + * Splits a FDC partial reward offer into two parts: signing and finalization. + * These split offers are used as inputs into reward calculation for specific types + * of rewards. + */ +export function splitFDCRewardOfferByTypes(offer: IPartialRewardOfferForRound): SplitFDCRewardOffer { + const forFinalization = (offer.amount * FINALIZATION_BIPS()) / TOTAL_BIPS; + const forSigning = offer.amount - forFinalization; + + const result: SplitFDCRewardOffer = { + signingRewardOffer: { + ...offer, + amount: forSigning, + }, + finalizationRewardOffer: { + ...offer, + amount: forFinalization, + }, + }; + return result; +} + + +/** + * Given an offer and data for reward calculation it calculates signing rewards for the offer. + * The reward is distributed to signers that deposited signatures in the grace period or before the timestamp of the first successful finalization. + * If a successful finalization for the votingRoundId does not happen before the end of the voting epoch + * votingRoundId + 1 + ADDITIONAL_REWARDED_FINALIZATION_WINDOWS, then the data about the finalization does not enter this function. + * In this case rewards can be still paid out if there is (are) a signed hash which has more than certain percentage of + * the total weight of the voting weight deposits. + * TODO: think through whether to reward only in grace period or up to the end of the voting epoch id of votingRoundId + 1. + */ +export function calculateSigningRewardsForFDC( + offer: IPartialRewardOfferForRound, + data: SDataForRewardCalculation, + rewardEpochInfo: RewardEpochInfo +): IPartialRewardClaim[] { + const votingRoundId = data.dataForCalculations.votingRoundId; + if (!data.fdcData?.firstSuccessfulFinalization) { + // burn all + const backClaim: IPartialRewardClaim = { + votingRoundId, + beneficiary: offer.claimBackAddress.toLowerCase(), + amount: offer.amount, + claimType: ClaimType.DIRECT, + offerIndex: offer.offerIndex, + feedId: offer.feedId, + protocolTag: "" + FDC_PROTOCOL_ID, + rewardTypeTag: RewardTypePrefix.SIGNING, + rewardDetailTag: SigningRewardClaimType.NO_TIMELY_FINALIZATION, + }; + return [backClaim]; + } + + const orderedSubmitSignatureAddresses = data.dataForCalculations.orderedVotersSubmitSignatureAddresses; + const totalWeight = rewardEpochInfo.signingPolicy.weights.reduce((acc, weight) => acc + weight, 0); + const signingAddressToVoter = new Map(); + + const deadlineTimestamp = Math.min( + data.fdcData.firstSuccessfulFinalization.timestamp, + EPOCH_SETTINGS().votingEpochEndSec(votingRoundId + 1) + ); + + // signingAddressToVoter will map only onto signers that get the reward + for (const voter of data.fdcData.eligibleSigners) { + if (isSignatureInGracePeriod(votingRoundId, voter) || + isSignatureBeforeTimestamp(votingRoundId, voter, deadlineTimestamp)) { + signingAddressToVoter.set(voter.submitSignatureAddress.toLowerCase(), voter); + } + } + + const allClaims: IPartialRewardClaim[] = []; + let undistributedWeight = BigInt(totalWeight); + let undistributedAmount = offer.amount; + for (let i = 0; i < orderedSubmitSignatureAddresses.length; i++) { + const submitSignatureAddress = orderedSubmitSignatureAddresses[i]; + const submitAddress = data.dataForCalculations.orderedVotersSubmitAddresses[i]; + const voterData = signingAddressToVoter.get(submitSignatureAddress) + if (voterData) { + let voterAmount = BigInt(voterData.weight) * undistributedAmount / undistributedWeight; + undistributedAmount -= voterAmount; + undistributedWeight -= BigInt(voterData.weight); + const voterWeights = data.dataForCalculations.votersWeightsMap!.get(submitAddress); + if(!voterData.dominatesConsensusBitVote) { + // burn 20% + const burnAmount = 200000n * voterAmount / 1000000n; + voterAmount -= burnAmount; + if(burnAmount > 0n) { + const burnClaim: IPartialRewardClaim = { + votingRoundId, + beneficiary: submitAddress.toLowerCase(), + amount: burnAmount, + claimType: ClaimType.DIRECT, + offerIndex: offer.offerIndex, + feedId: offer.feedId, + protocolTag: "" + FDC_PROTOCOL_ID, + rewardTypeTag: RewardTypePrefix.SIGNING, + rewardDetailTag: SigningRewardClaimType.NON_DOMINATING_BITVOTE, + }; + allClaims.push(burnClaim); + } + } + allClaims.push( + ...generateSigningWeightBasedClaimsForVoter(voterAmount, offer, voterWeights, RewardTypePrefix.SIGNING, FDC_PROTOCOL_ID) + ); + } + } + + // claim back + if (undistributedAmount > 0n) { + const backClaim: IPartialRewardClaim = { + votingRoundId, + beneficiary: offer.claimBackAddress.toLowerCase(), + amount: undistributedAmount, + claimType: ClaimType.DIRECT, + offerIndex: offer.offerIndex, + feedId: offer.feedId, + protocolTag: "" + FDC_PROTOCOL_ID, + rewardTypeTag: RewardTypePrefix.SIGNING, + rewardDetailTag: SigningRewardClaimType.CLAIM_BACK_OF_NON_SIGNERS_SHARE, + }; + allClaims.push(backClaim); + } + return allClaims; +} diff --git a/libs/ftso-core/src/reward-calculation/reward-calculation.ts b/libs/ftso-core/src/reward-calculation/reward-calculation.ts index ede88d2e..255fbd30 100644 --- a/libs/ftso-core/src/reward-calculation/reward-calculation.ts +++ b/libs/ftso-core/src/reward-calculation/reward-calculation.ts @@ -3,6 +3,7 @@ import { RewardEpochManager } from "../RewardEpochManager"; import { BURN_ADDRESS, CALCULATIONS_FOLDER, + FDC_PROTOCOL_ID, FEEDS_RENAMING_FILE, FINALIZATION_VOTER_SELECTION_THRESHOLD_WEIGHT_BIPS, FTSO2_FAST_UPDATES_PROTOCOL_ID, @@ -39,6 +40,7 @@ import { createRewardCalculationFolders, deserializeGranulatedPartialOfferMap, deserializeGranulatedPartialOfferMapForFastUpdates, + deserializeOffersForFDC, } from "../utils/stat-info/granulated-partial-offers-map"; import { deserializePartialClaimsForVotingRoundId, @@ -53,6 +55,8 @@ import { deserializeRewardEpochInfo } from "../utils/stat-info/reward-epoch-info import { FastUpdatesRewardClaimType, calculateFastUpdatesClaims } from "./reward-fast-updates"; import { calculatePenalties } from "./reward-penalties"; import { calculateSigningRewards } from "./reward-signing"; +import { calculateSigningRewardsForFDC, splitFDCRewardOfferByTypes } from "./fdc/reward-fdc-signing"; +import { calculateFdcPenalties } from "./fdc/reward-fdc-penalties"; /** * Initializes reward epoch storage for the given reward epoch. @@ -188,6 +192,9 @@ export async function partialRewardClaimsForVotingRound( const finalizationRewardClaims = calculateFinalizationRewardClaims( splitOffers.finalizationRewardOffer, + FTSO2_PROTOCOL_ID, + data.firstSuccessfulFinalization, + data.finalizations, data, new Set(data.eligibleFinalizers), medianEligibleVoters @@ -253,7 +260,7 @@ export async function partialRewardClaimsForVotingRound( // feedId: offer.feedId, // should be undefined protocolTag: "" + FTSO2_FAST_UPDATES_PROTOCOL_ID, rewardTypeTag: RewardTypePrefix.FULL_OFFER_CLAIM_BACK, - rewardDetailTag: FastUpdatesRewardClaimType.CONTRACT_CHANGE, + rewardDetailTag: FastUpdatesRewardClaimType.CONTRACT_CHANGE, }); } } @@ -339,9 +346,60 @@ export async function partialRewardClaimsForVotingRound( } } - if(useFDCData) { - // TODO - // ddd + if (useFDCData) { + // read offers + const offers = deserializeOffersForFDC(rewardEpochId, votingRoundId, calculationFolder); + for (const offer of offers) { + // We set the claim back address to burn address by default + offer.claimBackAddress = BURN_ADDRESS.toLowerCase(); + if (offer.shouldBeBurned) { + const fullOfferBackClaim: IPartialRewardClaim = { + votingRoundId, + beneficiary: offer.claimBackAddress, + amount: offer.amount, + claimType: ClaimType.DIRECT, + // offerIndex: offer.offerIndex, + protocolTag: "" + FDC_PROTOCOL_ID, + rewardTypeTag: RewardTypePrefix.PARTIAL_FDC_OFFER_CLAIM_BACK, + rewardDetailTag: "", // no additional tag + }; + allRewardClaims.push(fullOfferBackClaim); + continue; + } + const splitOffers = splitFDCRewardOfferByTypes(offer); + + const fdCFinalizationRewardClaims = calculateFinalizationRewardClaims( + splitOffers.finalizationRewardOffer, + FDC_PROTOCOL_ID, + data.fdcData.firstSuccessfulFinalization, + data.fdcData.finalizations, + data, + new Set(data.eligibleFinalizers), + new Set(data.eligibleFinalizers) + ); + + const fdcSigningRewardClaims = calculateSigningRewardsForFDC( + splitOffers.signingRewardOffer, + data, + rewardEpochInfo + ); + + const fdcPenalties = calculateFdcPenalties( + offer, + rewardEpochInfo, + data, + PENALTY_FACTOR(), + data.dataForCalculations.votersWeightsMap!, + RewardTypePrefix.REVEAL_OFFENDERS + ) + + allRewardClaims.push(...fdCFinalizationRewardClaims); + allRewardClaims.push(...fdcSigningRewardClaims); + allRewardClaims.push(...fdcPenalties); + if (merge) { + allRewardClaims = RewardClaim.merge(allRewardClaims); + } + } } if (serializeResults) { diff --git a/libs/ftso-core/src/reward-calculation/reward-finalization.ts b/libs/ftso-core/src/reward-calculation/reward-finalization.ts index 8d5c1e69..e43eec28 100644 --- a/libs/ftso-core/src/reward-calculation/reward-finalization.ts +++ b/libs/ftso-core/src/reward-calculation/reward-finalization.ts @@ -1,4 +1,4 @@ -import { FTSO2_PROTOCOL_ID } from "../configs/networks"; +import { ParsedFinalizationData } from "../IndexerClient"; import { IPartialRewardOfferForRound } from "../utils/PartialRewardOffer"; import { ClaimType, IPartialRewardClaim } from "../utils/RewardClaim"; import { SDataForRewardCalculation } from "../utils/stat-info/reward-calculation-data"; @@ -28,11 +28,14 @@ const BURN_NON_ELIGIBLE_REWARDS = true; */ export function calculateFinalizationRewardClaims( offer: IPartialRewardOfferForRound, + protocolId: number, + firstSuccessfulFinalization: ParsedFinalizationData | undefined, + finalizations: ParsedFinalizationData[], data: SDataForRewardCalculation, eligibleFinalizationRewardVotersInGracePeriod: Set
, // signing addresses of the voters that are eligible for finalization reward eligibleVoters: Set
// signing addresses of the voters that are eligible for finalization reward ): IPartialRewardClaim[] { - if (!data.firstSuccessfulFinalization) { + if (!firstSuccessfulFinalization) { const backClaim: IPartialRewardClaim = { votingRoundId: offer.votingRoundId, beneficiary: offer.claimBackAddress.toLowerCase(), @@ -40,7 +43,7 @@ export function calculateFinalizationRewardClaims( claimType: ClaimType.DIRECT, offerIndex: offer.offerIndex, feedId: offer.feedId, - protocolTag: "" + FTSO2_PROTOCOL_ID, + protocolTag: "" + protocolId, rewardTypeTag: RewardTypePrefix.FINALIZATION, rewardDetailTag: FinalizationRewardClaimType.NO_FINALIZATION, }; @@ -48,21 +51,21 @@ export function calculateFinalizationRewardClaims( } const votingRoundId = data.dataForCalculations.votingRoundId; // No voter provided finalization in grace period. Whoever finalizes gets the full reward. - if (isFinalizationOutsideOfGracePeriod(votingRoundId, data.firstSuccessfulFinalization!)) { + if (isFinalizationOutsideOfGracePeriod(votingRoundId, firstSuccessfulFinalization!)) { const otherFinalizerClaim: IPartialRewardClaim = { votingRoundId: offer.votingRoundId, - beneficiary: data.firstSuccessfulFinalization!.submitAddress.toLowerCase(), + beneficiary: firstSuccessfulFinalization!.submitAddress.toLowerCase(), amount: offer.amount, claimType: ClaimType.DIRECT, offerIndex: offer.offerIndex, feedId: offer.feedId, - protocolTag: "" + FTSO2_PROTOCOL_ID, + protocolTag: "" + protocolId, rewardTypeTag: RewardTypePrefix.FINALIZATION, rewardDetailTag: FinalizationRewardClaimType.OUTSIDE_OF_GRACE_PERIOD, }; return [otherFinalizerClaim]; } - const gracePeriodFinalizations = data.finalizations.filter(finalization => + const gracePeriodFinalizations = finalizations.filter(finalization => isFinalizationInGracePeriodAndEligible(votingRoundId, eligibleFinalizationRewardVotersInGracePeriod, finalization) ); // Rewarding of first successful finalizations outside of the grace period are already handled above @@ -76,7 +79,7 @@ export function calculateFinalizationRewardClaims( claimType: ClaimType.DIRECT, offerIndex: offer.offerIndex, feedId: offer.feedId, - protocolTag: "" + FTSO2_PROTOCOL_ID, + protocolTag: "" + protocolId, rewardTypeTag: RewardTypePrefix.FINALIZATION, rewardDetailTag: FinalizationRewardClaimType.FINALIZED_BUT_NO_ELIGIBLE_VOTERS, }; @@ -124,7 +127,7 @@ export function calculateFinalizationRewardClaims( undistributedAmount -= amount; undistributedSigningRewardWeight -= 1n; resultClaims.push( - ...generateSigningWeightBasedClaimsForVoter(amount, offer, voterWeight, RewardTypePrefix.FINALIZATION) + ...generateSigningWeightBasedClaimsForVoter(amount, offer, voterWeight, RewardTypePrefix.FINALIZATION, protocolId) ); } @@ -144,7 +147,7 @@ export function calculateFinalizationRewardClaims( claimType: ClaimType.DIRECT, offerIndex: offer.offerIndex, feedId: offer.feedId, - protocolTag: "" + FTSO2_PROTOCOL_ID, + protocolTag: "" + protocolId, rewardTypeTag: RewardTypePrefix.FINALIZATION, rewardDetailTag: FinalizationRewardClaimType.CLAIM_BACK_FOR_UNDISTRIBUTED_REWARDS, }); diff --git a/libs/ftso-core/src/reward-calculation/reward-offers.ts b/libs/ftso-core/src/reward-calculation/reward-offers.ts index 1ded6d24..32231f21 100644 --- a/libs/ftso-core/src/reward-calculation/reward-offers.ts +++ b/libs/ftso-core/src/reward-calculation/reward-offers.ts @@ -1,11 +1,11 @@ +import { deserializeAttestationRequestAppearances } from "../../../../apps/ftso-reward-calculation-process/src/libs/fdc-utils"; import { BURN_ADDRESS, FINALIZATION_BIPS, SIGNING_BIPS, TOTAL_BIPS } from "../configs/networks"; import { InflationRewardsOffered } from "../events"; import { - IFDCPartialRewardOfferForRound, IFUPartialRewardOfferForRound, IPartialRewardOfferForEpoch, IPartialRewardOfferForRound, - PartialRewardOffer, + PartialRewardOffer } from "../utils/PartialRewardOffer"; import { RewardEpochDuration } from "../utils/RewardEpochDuration"; import { OFFERS_FILE } from "../utils/stat-info/constants"; @@ -13,15 +13,16 @@ import { deserializeGranulatedPartialOfferMap, serializeGranulatedPartialOfferMap, } from "../utils/stat-info/granulated-partial-offers-map"; +import { deserializeDataForRewardCalculation } from "../utils/stat-info/reward-calculation-data"; import { RewardEpochInfo } from "../utils/stat-info/reward-epoch-info"; /** * A split of partial reward offer into three parts: */ -export interface SplitRewardOffer { - readonly medianRewardOffer: T; - readonly signingRewardOffer: T; - readonly finalizationRewardOffer: T; +export interface SplitRewardOffer { + readonly medianRewardOffer: IPartialRewardOfferForRound; + readonly signingRewardOffer: IPartialRewardOfferForRound; + readonly finalizationRewardOffer: IPartialRewardOfferForRound; } /** @@ -209,11 +210,11 @@ export function fixOffersForRandomFeedSelection( * These split offers are used as inputs into reward calculation for specific types * of rewards. */ -export function splitRewardOfferByTypes(offer: T): SplitRewardOffer { +export function splitRewardOfferByTypes(offer: IPartialRewardOfferForRound): SplitRewardOffer { const forSigning = (offer.amount * SIGNING_BIPS()) / TOTAL_BIPS; const forFinalization = (offer.amount * FINALIZATION_BIPS()) / TOTAL_BIPS; const forMedian = offer.amount - forSigning - forFinalization; - const result: SplitRewardOffer = { + const result: SplitRewardOffer = { medianRewardOffer: { ...offer, amount: forMedian, @@ -296,35 +297,80 @@ export function granulatedPartialOfferMapForFastUpdates( export function granulatedPartialOfferMapForFDC( rewardEpochInfo: RewardEpochInfo, -): Map { +): Map { const startVotingRoundId = rewardEpochInfo.signingPolicy.startVotingRoundId; const endVotingRoundId = rewardEpochInfo.endVotingRoundId; if (startVotingRoundId === undefined || endVotingRoundId === undefined) { throw new Error("Start or end voting round id is undefined"); } + const attestationCount = deserializeAttestationRequestAppearances(rewardEpochInfo.rewardEpochId); + const attestationCountMap = new Map(); + for (const attestation of attestationCount) { + attestationCountMap.set(attestation.attestationRequestId, attestation.count); + } + + let totalWeight = 0; + let burnWeight = 0; + for(const config of rewardEpochInfo.fdcInflationRewardsOffered.fdcConfigurations) { + const id = (config.attestationType + config.source.slice(2)).toLowerCase(); + const appearances = attestationCountMap.get(id) || 0; + totalWeight += config.inflationShare; + if(appearances < config.minRequestsThreshold) { + burnWeight += config.inflationShare; + } + } // Calculate total amount of rewards for the reward epoch - let totalAmount = rewardEpochInfo.fdcInflationRewardsOffered.amount; + const totalBurnAmount = (BigInt(burnWeight)*rewardEpochInfo.fdcInflationRewardsOffered.amount)/BigInt(totalWeight); + // const totalBurnAmount = 0n; + let totalAmount = rewardEpochInfo.fdcInflationRewardsOffered.amount - totalBurnAmount; if (process.env.TEST_FDC_INFLATION_REWARD_AMOUNT) { totalAmount = BigInt(process.env.TEST_FDC_INFLATION_REWARD_AMOUNT); } // Create a map of votingRoundId -> rewardOffer - const rewardOfferMap = new Map(); + const rewardOfferMap = new Map(); const numberOfVotingRounds = endVotingRoundId - startVotingRoundId + 1; const sharePerOne: bigint = totalAmount / BigInt(numberOfVotingRounds); const remainder: number = Number(totalAmount % BigInt(numberOfVotingRounds)); + const burnSharePerOne: bigint = totalBurnAmount / BigInt(numberOfVotingRounds); + const burnShareRemainder: number = Number(totalBurnAmount % BigInt(numberOfVotingRounds)); for (let votingRoundId = startVotingRoundId; votingRoundId <= endVotingRoundId; votingRoundId++) { - let amount = sharePerOne + (votingRoundId - startVotingRoundId < remainder ? 1n : 0n); + const amount = sharePerOne + (votingRoundId - startVotingRoundId < remainder ? 1n : 0n); + const burnAmount = burnSharePerOne + (votingRoundId - startVotingRoundId < burnShareRemainder ? 1n : 0n); - // Create adapted offer with selected feed - const feedOfferForVoting: IFDCPartialRewardOfferForRound = { + const roundDataForCalculation = deserializeDataForRewardCalculation(rewardEpochInfo.rewardEpochId, votingRoundId); + const attestationRequests = roundDataForCalculation?.fdcData?.attestationRequests; + if(!attestationRequests) { + throw new Error(`Missing attestation requests for voting round ${votingRoundId}`); + } + let feeAmount = 0n; + let feeBurnAmount = 0n; + for(const attestationRequest of attestationRequests) { + if(attestationRequest.confirmed) { + feeAmount += attestationRequest.fee; + } else { + feeBurnAmount += attestationRequest.fee; + } + } + const offerForVotingRound: IPartialRewardOfferForRound = { votingRoundId, - amount, + amount: amount + feeAmount, + feeAmount, + feeBurnAmount, }; - rewardOfferMap.set(votingRoundId, feedOfferForVoting); + const burnOfferForVotingRound: IPartialRewardOfferForRound = { + votingRoundId, + amount: burnAmount + feeBurnAmount, + shouldBeBurned: true, + } + + rewardOfferMap.set(votingRoundId, [ + offerForVotingRound, + burnOfferForVotingRound + ]); } return rewardOfferMap; } diff --git a/libs/ftso-core/src/reward-calculation/reward-penalties.ts b/libs/ftso-core/src/reward-calculation/reward-penalties.ts index eaeaa1d4..f4c10aa0 100644 --- a/libs/ftso-core/src/reward-calculation/reward-penalties.ts +++ b/libs/ftso-core/src/reward-calculation/reward-penalties.ts @@ -1,4 +1,5 @@ import { VoterWeights } from "../RewardEpoch"; +import { FTSO2_PROTOCOL_ID } from "../configs/networks"; import { IPartialRewardOfferForRound } from "../utils/PartialRewardOffer"; import { IPartialRewardClaim } from "../utils/RewardClaim"; import { Address } from "../voting-types"; @@ -42,7 +43,7 @@ export function calculatePenalties( } penalty = (-voterWeight * offer.amount * penaltyFactor) / totalWeight; } - penaltyClaims.push(...generateSigningWeightBasedClaimsForVoter(penalty, offer, voterWeights, penaltyType)); + penaltyClaims.push(...generateSigningWeightBasedClaimsForVoter(penalty, offer, voterWeights, penaltyType, FTSO2_PROTOCOL_ID)); } return penaltyClaims; } diff --git a/libs/ftso-core/src/reward-calculation/reward-signing-split.ts b/libs/ftso-core/src/reward-calculation/reward-signing-split.ts index 5fbffca5..80148efd 100644 --- a/libs/ftso-core/src/reward-calculation/reward-signing-split.ts +++ b/libs/ftso-core/src/reward-calculation/reward-signing-split.ts @@ -1,5 +1,5 @@ import { VoterWeights } from "../RewardEpoch"; -import { CAPPED_STAKING_FEE_BIPS, FTSO2_PROTOCOL_ID, TOTAL_BIPS } from "../configs/networks"; +import { CAPPED_STAKING_FEE_BIPS, TOTAL_BIPS } from "../configs/networks"; import { IPartialRewardOfferForRound } from "../utils/PartialRewardOffer"; import { ClaimType, IPartialRewardClaim } from "../utils/RewardClaim"; import { RewardTypePrefix } from "./RewardTypePrefix"; @@ -25,7 +25,8 @@ export function generateSigningWeightBasedClaimsForVoter( amount: bigint, offer: IPartialRewardOfferForRound, voterWeights: VoterWeights, - rewardType: RewardTypePrefix + rewardType: RewardTypePrefix, + protocolId: number ): IPartialRewardClaim[] { const rewardClaims: IPartialRewardClaim[] = []; let stakedWeight = 0n; @@ -43,7 +44,7 @@ export function generateSigningWeightBasedClaimsForVoter( claimType: ClaimType.DIRECT, offerIndex: offer.offerIndex, feedId: offer.feedId, - protocolTag: "" + FTSO2_PROTOCOL_ID, + protocolTag: "" + protocolId, rewardTypeTag: rewardType, rewardDetailTag: SigningWeightRewardClaimType.NO_VOTER_WEIGHT, } as IPartialRewardClaim, @@ -66,7 +67,7 @@ export function generateSigningWeightBasedClaimsForVoter( claimType: ClaimType.FEE, offerIndex: offer.offerIndex, feedId: offer.feedId, - protocolTag: "" + FTSO2_PROTOCOL_ID, + protocolTag: "" + protocolId, rewardTypeTag: rewardType, rewardDetailTag: SigningWeightRewardClaimType.FEE_FOR_DELEGATION_AND_STAKING, } as IPartialRewardClaim); @@ -80,7 +81,7 @@ export function generateSigningWeightBasedClaimsForVoter( claimType: ClaimType.WNAT, offerIndex: offer.offerIndex, feedId: offer.feedId, - protocolTag: "" + FTSO2_PROTOCOL_ID, + protocolTag: "" + protocolId, rewardTypeTag: rewardType, rewardDetailTag: SigningWeightRewardClaimType.DELEGATION_COMMUNITY_REWARD, } as IPartialRewardClaim); @@ -111,7 +112,7 @@ export function generateSigningWeightBasedClaimsForVoter( claimType: ClaimType.MIRROR, offerIndex: offer.offerIndex, feedId: offer.feedId, - protocolTag: "" + FTSO2_PROTOCOL_ID, + protocolTag: "" + protocolId, rewardTypeTag: rewardType, rewardDetailTag: SigningWeightRewardClaimType.NODE_COMMUNITY_REWARD, } as IPartialRewardClaim); diff --git a/libs/ftso-core/src/reward-calculation/reward-signing.ts b/libs/ftso-core/src/reward-calculation/reward-signing.ts index 29981668..6465bd45 100644 --- a/libs/ftso-core/src/reward-calculation/reward-signing.ts +++ b/libs/ftso-core/src/reward-calculation/reward-signing.ts @@ -24,6 +24,9 @@ export enum SigningRewardClaimType { NO_WEIGHT_OF_ELIGIBLE_SIGNERS = "NO_WEIGHT_OF_ELIGIBLE_SIGNERS", CLAIM_BACK_DUE_TO_NON_ELIGIBLE_SIGNER = "CLAIM_BACK_DUE_TO_NON_ELIGIBLE_SIGNER", CLAIM_BACK_NO_CLAIMS = "CLAIM_BACK_NO_CLAIMS", + NO_TIMELY_FINALIZATION = "NO_TIMELY_FINALIZATION", + CLAIM_BACK_OF_NON_SIGNERS_SHARE = "CLAIM_BACK_OF_NON_SIGNERS_SHARE", + NON_DOMINATING_BITVOTE = "NON_DOMINATING_BITVOTE", } /** * Given an offer and data for reward calculation it calculates signing rewards for the offer. @@ -169,7 +172,7 @@ export function calculateSigningRewards( resultClaims.push(backClaim); } else { resultClaims.push( - ...generateSigningWeightBasedClaimsForVoter(amount, offer, voterWeights, RewardTypePrefix.SIGNING) + ...generateSigningWeightBasedClaimsForVoter(amount, offer, voterWeights, RewardTypePrefix.SIGNING, FTSO2_PROTOCOL_ID) ); } } diff --git a/libs/ftso-core/src/reward-calculation/reward-utils.ts b/libs/ftso-core/src/reward-calculation/reward-utils.ts index 8a87b980..d143dd71 100644 --- a/libs/ftso-core/src/reward-calculation/reward-utils.ts +++ b/libs/ftso-core/src/reward-calculation/reward-utils.ts @@ -6,6 +6,7 @@ import { GRACE_PERIOD_FOR_FINALIZATION_DURATION_SEC, GRACE_PERIOD_FOR_SIGNATURES_DURATION_SEC, } from "../configs/networks"; +import { FDCEligibleSigner } from "../data-calculation-interfaces"; import { Address } from "../voting-types"; /** @@ -20,7 +21,7 @@ export function medianRewardDistributionWeight(voterWeights: VoterWeights): bigi */ export function isSignatureInGracePeriod( votingRoundId: number, - signatureSubmission: GenericSubmissionData + signatureSubmission: GenericSubmissionData | FDCEligibleSigner ) { return ( signatureSubmission.votingEpochIdFromTimestamp == votingRoundId + 1 && @@ -35,7 +36,7 @@ export function isSignatureInGracePeriod( */ export function isSignatureBeforeTimestamp( votingRoundId: number, - signatureSubmission: GenericSubmissionData, + signatureSubmission: GenericSubmissionData | FDCEligibleSigner, timestamp: number ) { return ( diff --git a/libs/ftso-core/src/utils/PartialRewardOffer.ts b/libs/ftso-core/src/utils/PartialRewardOffer.ts index ef91ff6b..d7d8f392 100644 --- a/libs/ftso-core/src/utils/PartialRewardOffer.ts +++ b/libs/ftso-core/src/utils/PartialRewardOffer.ts @@ -25,10 +25,28 @@ export interface IPartialRewardOfferForEpoch { offerIndex: number; } -export interface IPartialRewardOfferForRound extends IPartialRewardOfferForEpoch { +export interface IPartialRewardOfferForRound { // voting round id votingRoundId: number; + // hex encoded feed id + feedId?: string; + // amount (in wei) of reward in native coin + amount: bigint; + // minimal reward eligibility turnout threshold in BIPS (basis points) + minRewardedTurnoutBIPS?: number; + // primary band reward share in PPM (parts per million) + primaryBandRewardSharePPM?: number; + // secondary band width in PPM (parts per million) in relation to the median + secondaryBandWidthPPM?: number; + // address that can claim undistributed part of the reward (or burn address) + claimBackAddress?: Address; + // indicates if the reward is from inflation + isInflation?: boolean; + // Reward offer index - link to the initial reward offer + offerIndex?: number; shouldBeBurned?: boolean; + feeAmount?: bigint; + feeBurnAmount?: bigint; } export interface IFUPartialRewardOfferForRound { @@ -39,12 +57,6 @@ export interface IFUPartialRewardOfferForRound { shouldBeBurned?: boolean; } -export interface IFDCPartialRewardOfferForRound { - votingRoundId: number; - amount: bigint; - shouldBeBurned?: boolean; -} - export namespace PartialRewardOffer { export function fromRewardOffered(rewardOffer: RewardsOffered): IPartialRewardOfferForEpoch { @@ -89,4 +101,21 @@ export namespace PartialRewardOffer { } return rewardOffers; } + + export function remapToPartialOfferForRound( + partialOffer: IPartialRewardOfferForEpoch, + votingRoundId: number + ): IPartialRewardOfferForRound { + return { + votingRoundId, + feedId: partialOffer.feedId, + amount: partialOffer.amount, + minRewardedTurnoutBIPS: partialOffer.minRewardedTurnoutBIPS, + primaryBandRewardSharePPM: partialOffer.primaryBandRewardSharePPM, + secondaryBandWidthPPM: partialOffer.secondaryBandWidthPPM, + claimBackAddress: partialOffer.claimBackAddress, + isInflation: partialOffer.isInflation, + offerIndex: partialOffer.offerIndex, + }; + } } diff --git a/libs/ftso-core/src/utils/stat-info/granulated-partial-offers-map.ts b/libs/ftso-core/src/utils/stat-info/granulated-partial-offers-map.ts index 126bf8cf..1ba09227 100644 --- a/libs/ftso-core/src/utils/stat-info/granulated-partial-offers-map.ts +++ b/libs/ftso-core/src/utils/stat-info/granulated-partial-offers-map.ts @@ -1,7 +1,7 @@ import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs"; import path from "path/posix"; import { CALCULATIONS_FOLDER } from "../../configs/networks"; -import { IFDCPartialRewardOfferForRound, IFUPartialRewardOfferForRound, IPartialRewardOfferForRound } from "../PartialRewardOffer"; +import { IFUPartialRewardOfferForRound, IPartialRewardOfferForRound } from "../PartialRewardOffer"; import { RewardEpochDuration } from "../RewardEpochDuration"; import { bigIntReplacer, bigIntReviver } from "../big-number-serialization"; import { FDC_OFFERS_FILE, FU_OFFERS_FILE, OFFERS_FILE, TEMP_REWARD_EPOCH_FOLDER_PREFIX } from "./constants"; @@ -71,7 +71,7 @@ export function serializeGranulatedPartialOfferMap( */ export function serializeGranulatedPartialOfferMapForFDC( rewardEpochDuration: RewardEpochDuration, - rewardOfferMap: Map, + rewardOfferMap: Map, regenerate = true, file = FDC_OFFERS_FILE, calculationFolder = CALCULATIONS_FOLDER() @@ -176,3 +176,22 @@ export function deserializeGranulatedPartialOfferMapForFastUpdates( } return feedOffers; } + +export function deserializeOffersForFDC( + rewardEpochId: number, + votingRoundId: number, + calculationFolder = CALCULATIONS_FOLDER() +): IPartialRewardOfferForRound[] { + const rewardEpochFolder = path.join(calculationFolder, `${rewardEpochId}`); + const votingRoundFolder = path.join(rewardEpochFolder, `${votingRoundId}`); + const offersPath = path.join(votingRoundFolder, FDC_OFFERS_FILE); + if (!existsSync(offersPath)) { + throw new Error(`Critical error: No FDC offers for voting round ${votingRoundId}`); + } + const offersPerVotingRound: IPartialRewardOfferForRound[] = JSON.parse( + readFileSync(offersPath, "utf8"), + bigIntReviver + ); + return offersPerVotingRound; +} + diff --git a/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts b/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts index 5b7868e6..48ef322e 100644 --- a/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts +++ b/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts @@ -26,6 +26,7 @@ export interface SDataForCalculation { rewardEpochId: number; votingRoundId: number; orderedVotersSubmitAddresses: string[]; + orderedVotersSubmitSignatureAddresses: string[]; validEligibleReveals: RevealRecords[]; revealOffenders: string[]; voterMedianVotingWeights: VoterWeightData[]; @@ -57,6 +58,7 @@ export function prepareDataForCalculations(rewardEpochId: number, data: DataForR rewardEpochId, votingRoundId: data.dataForCalculations.votingRoundId, orderedVotersSubmitAddresses: data.dataForCalculations.orderedVotersSubmitAddresses, + orderedVotersSubmitSignatureAddresses: data.dataForCalculations.orderedVotersSubmitSignatureAddresses, validEligibleReveals, revealOffenders: [...data.dataForCalculations.revealOffenders], voterMedianVotingWeights, diff --git a/scripts/analytics/run/offer-check.ts b/scripts/analytics/run/offer-check.ts new file mode 100644 index 00000000..b31e16ff --- /dev/null +++ b/scripts/analytics/run/offer-check.ts @@ -0,0 +1,80 @@ +import path from "path/posix"; +import { deserializeGranulatedPartialOfferMap, deserializeGranulatedPartialOfferMapForFastUpdates, deserializeOffersForFDC } from "../../../libs/ftso-core/src/utils/stat-info/granulated-partial-offers-map"; +import { deserializeRewardEpochInfo } from "../../../libs/ftso-core/src/utils/stat-info/reward-epoch-info"; +import { deserializeDataForRewardCalculation } from "../../../libs/ftso-core/src/utils/stat-info/reward-calculation-data"; + +async function main() { + if (!process.argv[2]) { + throw new Error("no rewardEpochId"); + } + if (!process.argv[3]) { + throw new Error("no network"); + } + + const rewardEpochId = parseInt(process.argv[2]); + const network = process.argv[3]; + + const calculationFolder = path.join("calculations", network); + const rewardEpochInfo = deserializeRewardEpochInfo(rewardEpochId, false, calculationFolder); + let ftsoScalingFunds = 0n; + for (let offer of rewardEpochInfo.rewardOffers.rewardOffers) { + ftsoScalingFunds += offer.amount; + } + for (let offer of rewardEpochInfo.rewardOffers.inflationOffers) { + ftsoScalingFunds += offer.amount; + } + + let fastUpdatesFunds = rewardEpochInfo.fuInflationRewardsOffered.amount; + for (let incentive of rewardEpochInfo.fuIncentivesOffered) { + fastUpdatesFunds += incentive.offerAmount; + } + + let fdcFunds = rewardEpochInfo.fdcInflationRewardsOffered.amount + + let ftsoScalingOfferAmount = 0n; + let fastUpdatesOfferAmount = 0n; + let fdcOfferAmount = 0n; + for (let votingRoundId = rewardEpochInfo.signingPolicy.startVotingRoundId; votingRoundId <= rewardEpochInfo.endVotingRoundId; votingRoundId++) { + const ftsoOfferClaims = deserializeGranulatedPartialOfferMap(rewardEpochId, votingRoundId, calculationFolder); + for (let [_, offers] of ftsoOfferClaims.entries()) { + for (let offer of offers) { + ftsoScalingOfferAmount += offer.amount; + } + } + const fuFeedOffers = deserializeGranulatedPartialOfferMapForFastUpdates(rewardEpochId, votingRoundId, calculationFolder); + for (let [_, offers] of fuFeedOffers.entries()) { + for (let offer of offers) { + fastUpdatesOfferAmount += offer.amount; + } + } + + const offers = deserializeOffersForFDC(rewardEpochId, votingRoundId, calculationFolder); + for (let offer of offers) { + fdcOfferAmount += offer.amount; + } + const data = deserializeDataForRewardCalculation( + rewardEpochId, + votingRoundId, + false, + calculationFolder + ); + + for (let attestationRequest of data.fdcData.attestationRequests) { + fdcFunds += attestationRequest.fee; + } + } + + console.log(`FTSO Scaling Funds: ${ftsoScalingFunds - ftsoScalingOfferAmount}`); + console.log(`Fast Updates Funds: ${fastUpdatesFunds - fastUpdatesOfferAmount}`); + console.log(`FDC Funds: ${fdcFunds - fdcOfferAmount}`); +} + +main() + .then(() => { + console.dir("Done"); + process.exit(0); + }) + .catch(e => { + console.error(e); + process.exit(1); + }); diff --git a/scripts/rewards/coston-db.sh b/scripts/rewards/coston-db.sh index a1d6d105..4c1faad2 100755 --- a/scripts/rewards/coston-db.sh +++ b/scripts/rewards/coston-db.sh @@ -29,8 +29,8 @@ yarn nest build ftso-reward-calculation-process # Calculating all reward data from the starting reward epoch id. The calculation of claims is parallelized. # In the current (ongoing) reward epoch the calculation is switched to incremental, as data becomes available. # If the data for a specific reward epoch id is already available, the calculation is skipped. -export FROM_REWARD_EPOCH_ID=2773 -node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -g -o -c -a -y -b 40 -w 10 -d $FROM_REWARD_EPOCH_ID -m 10000 +export FROM_REWARD_EPOCH_ID=3250 +node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -g -o -c -a -y -z -b 40 -w 10 -d $FROM_REWARD_EPOCH_ID -m 10000 # --------------------------------------------------------------------------------------------------------------------------- # single reward epoch calculation diff --git a/test/libs/unit/reward-offers.test.ts b/test/libs/unit/reward-offers.test.ts index 1974424a..84919af4 100644 --- a/test/libs/unit/reward-offers.test.ts +++ b/test/libs/unit/reward-offers.test.ts @@ -72,7 +72,7 @@ describe(`Reward offers, ${getTestFile(__filename)}`, function () { const partialRewardOffer = PartialRewardOffer.fromRewardOffered(rewardOffered); - const splitRewardOffers = splitRewardOfferByTypes(partialRewardOffer); + const splitRewardOffers = splitRewardOfferByTypes(PartialRewardOffer.remapToPartialOfferForRound(partialRewardOffer, 1)); const finalization = splitRewardOffers.finalizationRewardOffer; const signing = splitRewardOffers.signingRewardOffer; @@ -87,7 +87,7 @@ describe(`Reward offers, ${getTestFile(__filename)}`, function () { const partialRewardOffers = distributeInflationRewardOfferToFeeds(rewardOffered); - const splitRewardOffers = splitRewardOfferByTypes(partialRewardOffers[0]); + const splitRewardOffers = splitRewardOfferByTypes(PartialRewardOffer.remapToPartialOfferForRound(partialRewardOffers[0], 1)); const finalization = splitRewardOffers.finalizationRewardOffer; const signing = splitRewardOffers.signingRewardOffer; From 9bd03a36c2a9dbf75bec0393f6db46e7a86f5aec Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:58:34 +0100 Subject: [PATCH 11/28] fix --- libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts b/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts index 48ef322e..7b902970 100644 --- a/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts +++ b/libs/ftso-core/src/utils/stat-info/reward-calculation-data.ts @@ -282,7 +282,9 @@ export function augmentDataForRewardCalculation( signaturesMap.set(hashSignature.hash, hashSignature.signatures); } data.signaturesMap = signaturesMap; - augmentFdcDataForRewardCalculation(data.fdcData); + if (data.fdcData) { + augmentFdcDataForRewardCalculation(data.fdcData); + } } /** From f5c0f1ad48ae16bc9dc0cd11860bf4c8ec454381 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:23:38 +0100 Subject: [PATCH 12/28] parsing signature messages of type 1 --- libs/fsp-utils/src/SignaturePayload.ts | 30 +++++++++++++++++++++----- libs/ftso-core/src/DataManager.ts | 12 +++++++++-- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/libs/fsp-utils/src/SignaturePayload.ts b/libs/fsp-utils/src/SignaturePayload.ts index e9664699..cadc00d3 100644 --- a/libs/fsp-utils/src/SignaturePayload.ts +++ b/libs/fsp-utils/src/SignaturePayload.ts @@ -54,18 +54,38 @@ export namespace SignaturePayload { if (!/^[0-9a-f]*$/.test(encodedSignaturePayloadInternal)) { throw Error(`Invalid format - not hex string: ${encodedSignaturePayload}`); } - if (encodedSignaturePayloadInternal.length < 2 + 38 * 2 + 65 * 2) { + // if (encodedSignaturePayloadInternal.length < 2 + 38 * 2 + 65 * 2) { + if (encodedSignaturePayloadInternal.length < 2 + 65 * 2) { throw Error(`Invalid format - too short: ${encodedSignaturePayload}`); } const type = "0x" + encodedSignaturePayloadInternal.slice(0, 2); - const message = "0x" + encodedSignaturePayloadInternal.slice(2, 2 + 38 * 2); - const signature = "0x" + encodedSignaturePayloadInternal.slice(2 + 38 * 2, 2 + 38 * 2 + 65 * 2); - const unsignedMessage = encodedSignaturePayloadInternal.slice(2 + 38 * 2 + 65 * 2); + if (type === "0x00" && encodedSignaturePayloadInternal.length < 2 + 38 * 2 + 65 * 2) { + throw Error(`Invalid format - too short type 0 signature: ${encodedSignaturePayload}`); + } + if (type === "0x01" && encodedSignaturePayloadInternal.length < 2 + 65 * 2) { + throw Error(`Invalid format - too short type 1 signature: ${encodedSignaturePayload}`); + } + if (type !== "0x00" && type !== "0x01") { + throw Error(`Invalid format - unknown type: ${type}`); + } + let message: string | undefined; + let signature: string; + let unsignedMessage: string; + if (type === "0x00") { + message = "0x" + encodedSignaturePayloadInternal.slice(2, 2 + 38 * 2); + signature = "0x" + encodedSignaturePayloadInternal.slice(2 + 38 * 2, 2 + 38 * 2 + 65 * 2); + unsignedMessage = encodedSignaturePayloadInternal.slice(2 + 38 * 2 + 65 * 2); + } + if (type === "0x01") { + signature = "0x" + encodedSignaturePayloadInternal.slice(2, 2 + 65 * 2); + unsignedMessage = encodedSignaturePayloadInternal.slice(2 + 65 * 2); + } const result: ISignaturePayload = { type, - message: ProtocolMessageMerkleRoot.decode(message), + message: message ? ProtocolMessageMerkleRoot.decode(message) : undefined, signature: ECDSASignature.decode(signature), }; + if (unsignedMessage.length > 0) { result.unsignedMessage = "0x" + unsignedMessage; } diff --git a/libs/ftso-core/src/DataManager.ts b/libs/ftso-core/src/DataManager.ts index ae571ca9..e0b97c0c 100644 --- a/libs/ftso-core/src/DataManager.ts +++ b/libs/ftso-core/src/DataManager.ts @@ -1,3 +1,4 @@ +import { sign } from "crypto"; import { ECDSASignature } from "../../fsp-utils/src/ECDSASignature"; import { ProtocolMessageMerkleRoot } from "../../fsp-utils/src/ProtocolMessageMerkleRoot"; import { RelayMessage } from "../../fsp-utils/src/RelayMessage"; @@ -361,9 +362,15 @@ export class DataManager { for (const message of submission.messages) { try { const signaturePayload = SignaturePayload.decode(message.payload); + if(signaturePayload.message && signaturePayload.message.protocolId !== message.protocolId) { + throw new Error(`Protocol id mismatch in signed message. Expected ${message.protocolId}, got ${signaturePayload.message.protocolId}`); + } + if(signaturePayload.message && signaturePayload.message.votingRoundId !== message.votingRoundId) { + throw new Error(`Voting round id mismatch in signed message. Expected ${message.votingRoundId}, got ${signaturePayload.message.votingRoundId}`); + } if ( - signaturePayload.message.votingRoundId === votingRoundId && - signaturePayload.message.protocolId === protocolId + message.votingRoundId === votingRoundId && + message.protocolId === protocolId ) { // - Override the messageHash if provided // - Require @@ -422,6 +429,7 @@ export class DataManager { signatures.set(signer, submissionData); } } catch (e) { + console.log(e) logger.warn(`Issues with parsing submission message: ${e.message}`); } } From 9457ccfd0d75765af2b9f89cab20a88eeef3635e Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:29:12 +0100 Subject: [PATCH 13/28] test fix --- test/utils/mini-finalizer/MiniFinalizer.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/utils/mini-finalizer/MiniFinalizer.ts b/test/utils/mini-finalizer/MiniFinalizer.ts index 0a831a6c..cf88723f 100644 --- a/test/utils/mini-finalizer/MiniFinalizer.ts +++ b/test/utils/mini-finalizer/MiniFinalizer.ts @@ -170,6 +170,7 @@ export class MiniFinalizer { rewardEpoch, signatures, protocolId, + undefined, this.logger ); if (!finalizationMap) { From 2a651054bc816a2a83d0fa7fbe01c6c09bb7ad7b Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Tue, 5 Nov 2024 16:23:06 +0100 Subject: [PATCH 14/28] factorization, tag fixes --- .../src/libs/calculator-utils.ts | 2 +- .../src/libs/fdc-utils.ts | 2 +- libs/fsp-utils/src/SignaturePayload.ts | 1 - libs/ftso-core/src/DataManager.ts | 185 +------- libs/ftso-core/src/DataManagerForRewarding.ts | 408 ++++++++---------- .../src/events/AttestationRequest.ts | 15 +- .../src/events/FDCInflationRewardsOffered.ts | 3 + .../src/events/FUInflationRewardsOffered.ts | 3 + .../reward-calculation/RewardTypePrefix.ts | 2 + .../src/reward-calculation/fdc/fdc-utils.ts | 232 ++++++++++ .../fdc/reward-fdc-signing.ts | 8 +- .../reward-calculation/reward-calculation.ts | 10 +- .../reward-calculation/reward-finalization.ts | 13 +- scripts/analytics/run/offer-check.ts | 5 + test/libs/unit/generator-rewards.test.ts | 47 +- test/utils/mini-finalizer/MiniFinalizer.ts | 3 +- 16 files changed, 489 insertions(+), 450 deletions(-) create mode 100644 libs/ftso-core/src/reward-calculation/fdc/fdc-utils.ts diff --git a/apps/ftso-reward-calculation-process/src/libs/calculator-utils.ts b/apps/ftso-reward-calculation-process/src/libs/calculator-utils.ts index 830e3c6d..dd427b9f 100644 --- a/apps/ftso-reward-calculation-process/src/libs/calculator-utils.ts +++ b/apps/ftso-reward-calculation-process/src/libs/calculator-utils.ts @@ -138,7 +138,7 @@ export async function calculationOfRewardCalculationDataForRange( ); done = true; } catch (e) { - console.log(e); + // console.log(e); logger.error( `Error while calculating reward calculation data for voting rounds ${firstVotingRoundId}-${lastVotingRoundId} in reward epoch ${rewardEpochId}: ${e}` ); diff --git a/apps/ftso-reward-calculation-process/src/libs/fdc-utils.ts b/apps/ftso-reward-calculation-process/src/libs/fdc-utils.ts index 5c34be12..4a5efb56 100644 --- a/apps/ftso-reward-calculation-process/src/libs/fdc-utils.ts +++ b/apps/ftso-reward-calculation-process/src/libs/fdc-utils.ts @@ -35,7 +35,7 @@ export function calculateAttestationTypeAppearances(rewardEpochId: number): void } for (const attestationRequest of attestationRequests) { if (attestationRequest.confirmed && !attestationRequest.duplicate) { - const id = AttestationRequest.getId(attestationRequest); + const id = AttestationRequest.getPrefix(attestationRequest); if (id) { attestationTypeCount.set(id, (attestationTypeCount.get(id) || 0) + 1); } diff --git a/libs/fsp-utils/src/SignaturePayload.ts b/libs/fsp-utils/src/SignaturePayload.ts index cadc00d3..4d788b54 100644 --- a/libs/fsp-utils/src/SignaturePayload.ts +++ b/libs/fsp-utils/src/SignaturePayload.ts @@ -54,7 +54,6 @@ export namespace SignaturePayload { if (!/^[0-9a-f]*$/.test(encodedSignaturePayloadInternal)) { throw Error(`Invalid format - not hex string: ${encodedSignaturePayload}`); } - // if (encodedSignaturePayloadInternal.length < 2 + 38 * 2 + 65 * 2) { if (encodedSignaturePayloadInternal.length < 2 + 65 * 2) { throw Error(`Invalid format - too short: ${encodedSignaturePayload}`); } diff --git a/libs/ftso-core/src/DataManager.ts b/libs/ftso-core/src/DataManager.ts index e0b97c0c..594a43de 100644 --- a/libs/ftso-core/src/DataManager.ts +++ b/libs/ftso-core/src/DataManager.ts @@ -1,8 +1,4 @@ -import { sign } from "crypto"; -import { ECDSASignature } from "../../fsp-utils/src/ECDSASignature"; -import { ProtocolMessageMerkleRoot } from "../../fsp-utils/src/ProtocolMessageMerkleRoot"; import { RelayMessage } from "../../fsp-utils/src/RelayMessage"; -import { ISignaturePayload, SignaturePayload } from "../../fsp-utils/src/SignaturePayload"; import { SigningPolicy } from "../../fsp-utils/src/SigningPolicy"; import { BlockAssuranceResult, @@ -19,8 +15,7 @@ import { ADDITIONAL_REWARDED_FINALIZATION_WINDOWS, EPOCH_SETTINGS, FTSO2_PROTOCOL_ID, - GENESIS_REWARD_EPOCH_START_EVENT, - WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH, + GENESIS_REWARD_EPOCH_START_EVENT } from "./configs/networks"; import { DataForCalculations, @@ -31,7 +26,7 @@ import { CommitData, ICommitData } from "./utils/CommitData"; import { ILogger } from "./utils/ILogger"; import { IRevealData, RevealData } from "./utils/RevealData"; import { errorString } from "./utils/error"; -import { Address, Feed, MessageHash } from "./voting-types"; +import { Address, Feed } from "./voting-types"; /** * Data availability status for data manager responses. @@ -162,65 +157,6 @@ export class DataManager { }; } - /** - * Provides the data for reward calculation given the voting round id and the random generation benching window. - * Since calculation of rewards takes place when all the data is surely on the blockchain, no timeout queries are relevant here. - * The data for reward calculation is composed of: - * - data for median calculation - * - signatures for the given voting round id in given rewarding window - * - finalizations for the given voting round id in given rewarding window - * Data for median calculation is used to calculate the median feed value for each feed in the rewarding boundaries. - * The data also contains the RewardEpoch objects, which contains all reward offers. - * Signatures and finalizations are used to calculate the rewards for signature deposition and finalizations. - * Each finalization is checked if it is valid and finalizable. Note that only one such finalization is fully executed on chain, while - * others are reverted. Nevertheless, all finalizations in rewarded window are considered for the reward calculation, since a certain - * subset is eligible for a reward if submitted in due time. - */ - public async getDataForRewardCalculation( - votingRoundId: number, - randomGenerationBenchingWindow: number - ): Promise> { - const dataForCalculationsResponse = await this.getDataForCalculations( - votingRoundId, - randomGenerationBenchingWindow - ); - if (dataForCalculationsResponse.status !== DataAvailabilityStatus.OK) { - return { - status: dataForCalculationsResponse.status, - }; - } - const signaturesResponse = await this.getSignAndFinalizeSubmissionDataForVotingRound(votingRoundId); - if (signaturesResponse.status !== DataAvailabilityStatus.OK) { - return { - status: signaturesResponse.status, - }; - } - const signatures = DataManager.extractSignatures( - votingRoundId, - dataForCalculationsResponse.data.rewardEpoch, - signaturesResponse.data.signatures, - FTSO2_PROTOCOL_ID, - undefined, - this.logger - ); - const finalizations = this.extractFinalizations( - votingRoundId, - dataForCalculationsResponse.data.rewardEpoch, - signaturesResponse.data.finalizations, - FTSO2_PROTOCOL_ID - ); - const firstSuccessfulFinalization = finalizations.find(finalization => finalization.successfulOnChain); - return { - status: DataAvailabilityStatus.OK, - data: { - dataForCalculations: dataForCalculationsResponse.data, - signatures, - finalizations, - firstSuccessfulFinalization, - }, - }; - } - /** * Creates a pair of mappings * 1. votingRoundId -> commit submissions, chronologically ordered @@ -335,123 +271,6 @@ export class DataManager { }; } - /** - * Extract signature payloads for the given voting round id from the given submissions. - * Each signature is filtered out for the correct voting round id, protocol id and eligible signer. - * Signatures are returned in the form of a map - * from message hash to a list of signatures to submission data containing parsed signature payload. - * The last signed message for a specific message hash is considered. - * ASSUMPTION: all signature submissions for voting round id, hence contained , - * between reveal deadline for votingRoundId (hence in voting epoch votingRoundId + 1) and - * the end of the voting epoch votingRoundId + 1 + ADDITIONAL_REWARDED_FINALIZATION_WINDOWS - * @param votingRoundId - * @param rewardEpoch - * @param submissions - * @returns - */ - public static extractSignatures( - votingRoundId: number, - rewardEpoch: RewardEpoch, - submissions: SubmissionData[], - protocolId = FTSO2_PROTOCOL_ID, - providedMessageHash: MessageHash | undefined = undefined, - logger: ILogger - ): Map[]> { - const signatureMap = new Map>>(); - for (const submission of submissions) { - for (const message of submission.messages) { - try { - const signaturePayload = SignaturePayload.decode(message.payload); - if(signaturePayload.message && signaturePayload.message.protocolId !== message.protocolId) { - throw new Error(`Protocol id mismatch in signed message. Expected ${message.protocolId}, got ${signaturePayload.message.protocolId}`); - } - if(signaturePayload.message && signaturePayload.message.votingRoundId !== message.votingRoundId) { - throw new Error(`Voting round id mismatch in signed message. Expected ${message.votingRoundId}, got ${signaturePayload.message.votingRoundId}`); - } - if ( - message.votingRoundId === votingRoundId && - message.protocolId === protocolId - ) { - // - Override the messageHash if provided - // - Require - - let messageHash = providedMessageHash ?? ProtocolMessageMerkleRoot.hash(signaturePayload.message); - - const signer = ECDSASignature.recoverSigner(messageHash, signaturePayload.signature).toLowerCase(); - // submit signature address should match the signingPolicyAddress - const expectedSigner = rewardEpoch.getSigningAddressFromSubmitSignatureAddress(submission.submitAddress.toLowerCase()); - // if the expected signer is not found, the signature is not valid for rewarding - if (!expectedSigner) { - continue; - } - // In case of FTSO Scaling, we have message hash as a part of payload - // the signer is incorrect - if (!providedMessageHash) { - if (!rewardEpoch.isEligibleSignerAddress(signer)) { - continue; - } - signaturePayload.messageHash = messageHash; - signaturePayload.signer = signer; - signaturePayload.weight = rewardEpoch.signerToSigningWeight(signer); - signaturePayload.index = rewardEpoch.signerToVotingPolicyIndex(signer); - } else { - if (signer !== expectedSigner) { - if (!rewardEpoch.isEligibleSignerAddress(expectedSigner)) { - continue; - } - // wrong signature by eligible signer - signaturePayload.messageHash = WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH; - } else { - signaturePayload.messageHash = providedMessageHash; - } - signaturePayload.signer = expectedSigner; - signaturePayload.weight = rewardEpoch.signerToSigningWeight(expectedSigner); - signaturePayload.index = rewardEpoch.signerToVotingPolicyIndex(expectedSigner); - } - - if ( - signaturePayload.weight === undefined || - signaturePayload.signer === undefined || - signaturePayload.index === undefined - ) { - // assert: this should never happen - throw new Error( - `Critical error: signerToSigningWeight or signerToDelegationAddress is not defined for signer ${signer}` - ); - } - const signatures = - signatureMap.get(messageHash) || new Map>(); - const submissionData: GenericSubmissionData = { - ...submission, - messages: signaturePayload, - }; - signatureMap.set(messageHash, signatures); - signatures.set(signer, submissionData); - } - } catch (e) { - console.log(e) - logger.warn(`Issues with parsing submission message: ${e.message}`); - } - } - } - const result = new Map[]>(); - for (const [hash, sigMap] of signatureMap.entries()) { - const values = [...sigMap.values()]; - DataManager.sortSubmissionDataArray(values); - // consider only the first sent signature of a sender as rewardable - const existingSenderAddresses = new Set
(); - const filteredSignatureSubmissions: GenericSubmissionData[] = []; - for (const submission of values) { - if (!existingSenderAddresses.has(submission.submitAddress.toLowerCase())) { - filteredSignatureSubmissions.push(submission); - existingSenderAddresses.add(submission.submitAddress.toLowerCase()); - } - } - result.set(hash, filteredSignatureSubmissions); - } - return result; - } - /** * Given submissions of finalizations eligible for voting round @param votingRoundId and matching reward epoch @param rewardEpoch to the * voting round id, extract finalizations which match voting round id, given protocol id and are parsable and finalizeable (would cause finalisation) diff --git a/libs/ftso-core/src/DataManagerForRewarding.ts b/libs/ftso-core/src/DataManagerForRewarding.ts index aef5137f..2fb8f786 100644 --- a/libs/ftso-core/src/DataManagerForRewarding.ts +++ b/libs/ftso-core/src/DataManagerForRewarding.ts @@ -1,5 +1,7 @@ +import { ECDSASignature } from "../../fsp-utils/src/ECDSASignature"; +import { ProtocolMessageMerkleRoot } from "../../fsp-utils/src/ProtocolMessageMerkleRoot"; import { RelayMessage } from "../../fsp-utils/src/RelayMessage"; -import { ISignaturePayload } from "../../fsp-utils/src/SignaturePayload"; +import { ISignaturePayload, SignaturePayload } from "../../fsp-utils/src/SignaturePayload"; import { DataAvailabilityStatus, DataManager, DataMangerResponse, SignAndFinalizeSubmissionData } from "./DataManager"; import { BlockAssuranceResult, GenericSubmissionData, SubmissionData } from "./IndexerClient"; import { IndexerClientForRewarding } from "./IndexerClientForRewarding"; @@ -11,14 +13,11 @@ import { DataForCalculations, DataForRewardCalculation, FDCDataForVotingRound, - FDCEligibleSigner, - FDCOffender, - FDCOffense, FDCRewardData, FastUpdatesDataForVotingRound, - PartialFDCDataForVotingRound, + PartialFDCDataForVotingRound } from "./data-calculation-interfaces"; -import { AttestationRequest } from "./events/AttestationRequest"; +import { bitVoteIndicesNum, extractFDCRewardData, uniqueRequestsIndices } from "./reward-calculation/fdc/fdc-utils"; import { ILogger } from "./utils/ILogger"; import { errorString } from "./utils/error"; import { Address, MessageHash } from "./voting-types"; @@ -38,6 +37,66 @@ export class DataManagerForRewarding extends DataManager { super(indexerClient, rewardEpochManager, logger); } + /** + * Provides the data for reward calculation given the voting round id and the random generation benching window. + * Since calculation of rewards takes place when all the data is surely on the blockchain, no timeout queries are relevant here. + * The data for reward calculation is composed of: + * - data for median calculation + * - signatures for the given voting round id in given rewarding window + * - finalizations for the given voting round id in given rewarding window + * Data for median calculation is used to calculate the median feed value for each feed in the rewarding boundaries. + * The data also contains the RewardEpoch objects, which contains all reward offers. + * Signatures and finalizations are used to calculate the rewards for signature deposition and finalizations. + * Each finalization is checked if it is valid and finalizable. Note that only one such finalization is fully executed on chain, while + * others are reverted. Nevertheless, all finalizations in rewarded window are considered for the reward calculation, since a certain + * subset is eligible for a reward if submitted in due time. + */ + public async getDataForRewardCalculation( + votingRoundId: number, + randomGenerationBenchingWindow: number + ): Promise> { + const dataForCalculationsResponse = await this.getDataForCalculations( + votingRoundId, + randomGenerationBenchingWindow + ); + if (dataForCalculationsResponse.status !== DataAvailabilityStatus.OK) { + return { + status: dataForCalculationsResponse.status, + }; + } + const signaturesResponse = await this.getSignAndFinalizeSubmissionDataForVotingRound(votingRoundId); + if (signaturesResponse.status !== DataAvailabilityStatus.OK) { + return { + status: signaturesResponse.status, + }; + } + const signatures = DataManagerForRewarding.extractSignatures( + votingRoundId, + dataForCalculationsResponse.data.rewardEpoch, + signaturesResponse.data.signatures, + FTSO2_PROTOCOL_ID, + undefined, + this.logger + ); + const finalizations = this.extractFinalizations( + votingRoundId, + dataForCalculationsResponse.data.rewardEpoch, + signaturesResponse.data.finalizations, + FTSO2_PROTOCOL_ID + ); + const firstSuccessfulFinalization = finalizations.find(finalization => finalization.successfulOnChain); + return { + status: DataAvailabilityStatus.OK, + data: { + dataForCalculations: dataForCalculationsResponse.data, + signatures, + finalizations, + firstSuccessfulFinalization, + }, + }; + } + + /** * Prepare data for median calculation and rewarding given the voting round id and the random generation benching window. * - queries relevant commits and reveals from chain indexer database @@ -138,6 +197,122 @@ export class DataManagerForRewarding extends DataManager { data: result as DataForCalculations[], }; } + /** + * Extract signature payloads for the given voting round id from the given submissions. + * Each signature is filtered out for the correct voting round id, protocol id and eligible signer. + * Signatures are returned in the form of a map + * from message hash to a list of signatures to submission data containing parsed signature payload. + * The last signed message for a specific message hash is considered. + * ASSUMPTION: all signature submissions for voting round id, hence contained , + * between reveal deadline for votingRoundId (hence in voting epoch votingRoundId + 1) and + * the end of the voting epoch votingRoundId + 1 + ADDITIONAL_REWARDED_FINALIZATION_WINDOWS + * @param votingRoundId + * @param rewardEpoch + * @param submissions + * @returns + */ + public static extractSignatures( + votingRoundId: number, + rewardEpoch: RewardEpoch, + submissions: SubmissionData[], + protocolId = FTSO2_PROTOCOL_ID, + providedMessageHash: MessageHash | undefined = undefined, + logger: ILogger + ): Map[]> { + const signatureMap = new Map>>(); + for (const submission of submissions) { + for (const message of submission.messages) { + try { + const signaturePayload = SignaturePayload.decode(message.payload); + if (signaturePayload.message && signaturePayload.message.protocolId !== message.protocolId) { + throw new Error(`Protocol id mismatch in signed message. Expected ${message.protocolId}, got ${signaturePayload.message.protocolId}`); + } + if (signaturePayload.message && signaturePayload.message.votingRoundId !== message.votingRoundId) { + throw new Error(`Voting round id mismatch in signed message. Expected ${message.votingRoundId}, got ${signaturePayload.message.votingRoundId}`); + } + if ( + message.votingRoundId === votingRoundId && + message.protocolId === protocolId + ) { + // - Override the messageHash if provided + // - Require + + let messageHash = providedMessageHash ?? ProtocolMessageMerkleRoot.hash(signaturePayload.message); + + const signer = ECDSASignature.recoverSigner(messageHash, signaturePayload.signature).toLowerCase(); + // submit signature address should match the signingPolicyAddress + const expectedSigner = rewardEpoch.getSigningAddressFromSubmitSignatureAddress(submission.submitAddress.toLowerCase()); + // if the expected signer is not found, the signature is not valid for rewarding + if (!expectedSigner) { + continue; + } + // In case of FTSO Scaling, we have message hash as a part of payload + // the signer is incorrect + if (!providedMessageHash) { + if (!rewardEpoch.isEligibleSignerAddress(signer)) { + continue; + } + signaturePayload.messageHash = messageHash; + signaturePayload.signer = signer; + signaturePayload.weight = rewardEpoch.signerToSigningWeight(signer); + signaturePayload.index = rewardEpoch.signerToVotingPolicyIndex(signer); + } else { + if (signer !== expectedSigner) { + if (!rewardEpoch.isEligibleSignerAddress(expectedSigner)) { + continue; + } + // wrong signature by eligible signer + signaturePayload.messageHash = WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH; + } else { + signaturePayload.messageHash = providedMessageHash; + } + signaturePayload.signer = expectedSigner; + signaturePayload.weight = rewardEpoch.signerToSigningWeight(expectedSigner); + signaturePayload.index = rewardEpoch.signerToVotingPolicyIndex(expectedSigner); + } + + if ( + signaturePayload.weight === undefined || + signaturePayload.signer === undefined || + signaturePayload.index === undefined + ) { + // assert: this should never happen + throw new Error( + `Critical error: signerToSigningWeight or signerToDelegationAddress is not defined for signer ${signer}` + ); + } + const signatures = + signatureMap.get(messageHash) || new Map>(); + const submissionData: GenericSubmissionData = { + ...submission, + messages: signaturePayload, + }; + signatureMap.set(messageHash, signatures); + signatures.set(signer, submissionData); + } + } catch (e) { + console.log(e) + logger.warn(`Issues with parsing submission message: ${e.message}`); + } + } + } + const result = new Map[]>(); + for (const [hash, sigMap] of signatureMap.entries()) { + const values = [...sigMap.values()]; + DataManager.sortSubmissionDataArray(values); + // consider only the first sent signature of a sender as rewardable + const existingSenderAddresses = new Set
(); + const filteredSignatureSubmissions: GenericSubmissionData[] = []; + for (const submission of values) { + if (!existingSenderAddresses.has(submission.submitAddress.toLowerCase())) { + filteredSignatureSubmissions.push(submission); + existingSenderAddresses.add(submission.submitAddress.toLowerCase()); + } + } + result.set(hash, filteredSignatureSubmissions); + } + return result; + } /** * Provides the data for reward calculation given the voting round id and the random generation benching window. @@ -248,7 +423,7 @@ export class DataManagerForRewarding extends DataManager { startIndexFinalizations, endIndexFinalizations ); - const signatures = DataManager.extractSignatures( + const signatures = DataManagerForRewarding.extractSignatures( votingRoundId, rewardEpoch, votingRoundSignatures, @@ -283,7 +458,7 @@ export class DataManagerForRewarding extends DataManager { } RelayMessage.augment(fdcFirstSuccessfulFinalization.messages); const consensusMessageHash = fdcFirstSuccessfulFinalization.messages.protocolMessageHash; - fdcSignatures = DataManager.extractSignatures( + fdcSignatures = DataManagerForRewarding.extractSignatures( votingRoundId, rewardEpoch, votingRoundSignatures, @@ -291,7 +466,7 @@ export class DataManagerForRewarding extends DataManager { consensusMessageHash, this.logger ); - fdcRewardData = DataManagerForRewarding.extractFDCRewardData( + fdcRewardData = extractFDCRewardData( consensusMessageHash, dataForCalculations.validEligibleBitVoteSubmissions, fdcSignatures, @@ -304,7 +479,7 @@ export class DataManagerForRewarding extends DataManager { throw new Error(`Voting round id mismatch: ${partialData.votingRoundId} !== ${votingRoundId}`); } if (partialData && partialData.nonDuplicationIndices && fdcRewardData && fdcRewardData.consensusBitVote !== undefined) { - consensusBitVoteIndices = DataManagerForRewarding.bitVoteIndicesNum(fdcRewardData.consensusBitVote, partialData.nonDuplicationIndices.length); + consensusBitVoteIndices = bitVoteIndicesNum(fdcRewardData.consensusBitVote, partialData.nonDuplicationIndices.length); for (const bitVoteIndex of consensusBitVoteIndices) { for (const [i, originalIndex] of partialData.nonDuplicationIndices[bitVoteIndex].entries()) { partialData.attestationRequests[originalIndex].confirmed = true; @@ -453,7 +628,7 @@ export class DataManagerForRewarding extends DataManager { const value: PartialFDCDataForVotingRound = { votingRoundId, attestationRequests, - nonDuplicationIndices: DataManagerForRewarding.uniqueRequestsIndices(attestationRequests), + nonDuplicationIndices: uniqueRequestsIndices(attestationRequests), }; result.push(value); } @@ -494,215 +669,4 @@ export class DataManagerForRewarding extends DataManager { return [...voterToLastBitVote.values()]; } - /** - * Given finalized messageHash it calculates consensus bitvote, filters out eligible signers and determines - * offenders. - */ - public static extractFDCRewardData( - messageHash: string, - bitVoteSubmissions: SubmissionData[], - fdcSignatures: Map[]>, - rewardEpoch: RewardEpoch, - ): FDCRewardData | undefined { - const voteCounter = new Map(); - // - const eligibleSigners: FDCEligibleSigner[] = []; - const offenseMap = new Map(); - if (!messageHash) { - throw new Error("Consensus message hash is required"); - } - const signatures = fdcSignatures.get(messageHash); - if (!signatures) { - // TODO: log warning - return undefined; - } - for (const signature of signatures) { - const consensusBitVoteCandidate = signature.messages.unsignedMessage?.toLowerCase(); - if (!consensusBitVoteCandidate || consensusBitVoteCandidate.length < 6) { - continue; - } - const bitVoteNum = BigInt("0x" + consensusBitVoteCandidate.slice(6)); - // Note that 0n is also a legit consensus bitvote meaning no confirmations (but might not be rewarded) - voteCounter.set(bitVoteNum, (voteCounter.get(bitVoteNum) || 0) + signature.messages.weight) - } - let consensusBitVote: bigint | undefined; - if (voteCounter.size > 0) { - const maxCount = Math.max(...voteCounter.values()); - const maxBitVotes = [...voteCounter.entries()].filter(([_, count]) => count === maxCount).map(([bitVote, _]) => bitVote); - maxBitVotes.sort(); - // if it happens there are multiple maxHashes we take the first in lexicographical order - consensusBitVote = maxBitVotes[0]; - - // TODO: - // should we require 50%+ weight on maxHash? - // const consensusBitVoteWeight = voteCounter.get(consensusBitVote); - // if(consensusBitVoteWeight < rewardEpoch.signingPolicy.threshold) { - // return undefined; - // } - } - - const submitSignatureAddressToBitVote = new Map(); - for (const submission of bitVoteSubmissions) { - const submitSignatureAddress = rewardEpoch.getSubmitSignatureAddressFromSubmitAddress(submission.submitAddress).toLowerCase(); - const message = submission.messages.find(m => m.protocolId === FDC_PROTOCOL_ID); - if (message && message.payload) { - submitSignatureAddressToBitVote.set(submitSignatureAddress, message.payload.toLowerCase()); - } - } - - const submitSignatureSenders = new Set
(); - - for (const signature of signatures) { - // too late - if (signature.relativeTimestamp >= 90) { - continue; - } - const submitSignatureAddress = signature.submitAddress.toLowerCase() - submitSignatureSenders.add(submitSignatureAddress); - const bitVote = submitSignatureAddressToBitVote.get(submitSignatureAddress); - const eligibleSigner: FDCEligibleSigner = { - submitSignatureAddress: signature.submitAddress.toLowerCase(), - timestamp: signature.timestamp, - votingEpochIdFromTimestamp: signature.votingEpochIdFromTimestamp, - relativeTimestamp: signature.relativeTimestamp, - bitVote, - dominatesConsensusBitVote: consensusBitVote === undefined ? undefined : DataManagerForRewarding.isConsensusVoteDominated(consensusBitVote, bitVote), - weight: signature.messages.weight, - } - eligibleSigners.push(eligibleSigner); - } - - for (const submission of bitVoteSubmissions) { - const submitSignatureAddress = rewardEpoch.getSubmitSignatureAddressFromSubmitAddress(submission.submitAddress).toLowerCase(); - const submissionAddress = rewardEpoch.getSubmitAddressFromSubmitSignatureAddress(submitSignatureAddress).toLowerCase(); - if (!submitSignatureSenders.has(submitSignatureAddress)) { - const offender: FDCOffender = { - submitSignatureAddress, - submissionAddress, - weight: rewardEpoch.getSigningWeightForSubmitSignatureAddress(submitSignatureAddress), - offenses: [FDCOffense.NO_REVEAL_ON_BITVOTE] - } - offenseMap.set(submitSignatureAddress, offender); - } - } - - const wrongSignatures = fdcSignatures.get(WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH); - if (wrongSignatures) { - for (const signature of wrongSignatures) { - const submitSignatureAddress = signature.submitAddress.toLowerCase(); - const submissionAddress = rewardEpoch.getSubmitAddressFromSubmitSignatureAddress(submitSignatureAddress).toLowerCase(); - if (!rewardEpoch.isEligibleSubmitSignatureAddress(submitSignatureAddress)) { - continue; - } - const offender = offenseMap.get(submitSignatureAddress) || { - submitSignatureAddress, - submissionAddress, - weight: rewardEpoch.getSigningWeightForSubmitSignatureAddress(submitSignatureAddress), - offenses: [] - } - offender.offenses.push(FDCOffense.WRONG_SIGNATURE); - offenseMap.set(submitSignatureAddress, offender); - } - } - for (const signature of signatures) { - const submitSignatureAddress = signature.submitAddress.toLowerCase(); - const submissionAddress = rewardEpoch.getSubmitAddressFromSubmitSignatureAddress(submitSignatureAddress).toLowerCase(); - const consensusBitVoteCandidate = signature.messages.unsignedMessage?.toLowerCase(); - if (!consensusBitVoteCandidate) { - continue; - } - // 0x + 2 bytes length - let isOffense = consensusBitVoteCandidate.length < 6; - if (!isOffense) { - isOffense = BigInt("0x" + consensusBitVoteCandidate.slice(6)) !== consensusBitVote; - } - if (isOffense) { - const offender = offenseMap.get(submitSignatureAddress) || { - submitSignatureAddress, - submissionAddress, - weight: rewardEpoch.getSigningWeightForSubmitSignatureAddress(submitSignatureAddress), - offenses: [] - } - offender.offenses.push(FDCOffense.BAD_CONSENSUS_BITVOTE_CANDIDATE); - offenseMap.set(submitSignatureAddress, offender); - } - } - - const fdcOffenders = [...offenseMap.values()]; - fdcOffenders.sort((a, b) => a.submitSignatureAddress.localeCompare(b.submitSignatureAddress)); - const result: FDCRewardData = { - eligibleSigners, - consensusBitVote, - fdcOffenders - }; - - return result; - } - - public static uniqueRequestsIndices(attestationRequests: AttestationRequest[]): number[][] { - const encountered = new Map(); - const result: number[][] = []; - for (let i = 0; i < attestationRequests.length; i++) { - const request = attestationRequests[i]; - if (!encountered.get(request.data)) { - encountered.set(request.data, i); - result.push([i]); - } else { - result[encountered.get(request.data)].push(i); - } - } - return result; - } - - public static bitVoteIndices(bitVote: string, len: number): number[] | undefined { - if (!bitVote || bitVote.length < 4) { - return undefined - } - const length = parseInt(bitVote.slice(2, 4), 16); - if (length !== len) { - throw new Error(`Bitvote length mismatch: ${length} !== ${len}`); - } - - const result: number[] = []; - let bitVoteNum = BigInt("0x" + bitVote.slice(4)); - return DataManagerForRewarding.bitVoteIndicesNum(bitVoteNum, len); - } - - public static bitVoteIndicesNum(bitVoteNum: bigint, len: number): number[] { - const result: number[] = []; - for (let i = 0; i < len; i++) { - if (bitVoteNum % 2n === 1n) { - result.push(i); - } - bitVoteNum /= 2n; - } - if (bitVoteNum !== 0n) { - throw new Error(`bitVoteNum not fully consumed: ${bitVoteNum}`); - } - return result; - } - - public static isConsensusVoteDominated(consensusBitVote: bigint, bitVote?: string): boolean { - if (!bitVote) { - return false; - } - // Remove 0x prefix and first 2 bytes, used for the length - let h1 = consensusBitVote.toString(16); - // Ensure even length - if (h1.length % 2 !== 0) { - h1 = "0" + h1; - } - // This one is always even length - let h2 = bitVote.startsWith("0x") ? bitVote.slice(6) : bitVote.slice(4); - if (h1.length !== h2.length) { - const mLen = Math.max(h1.length, h2.length); - h1 = h1.padStart(mLen, "0"); - h2 = h2.padStart(mLen, "0"); - } - const buf1 = Buffer.from(h1, "hex"); - const buf2 = Buffer.from(h2, "hex"); - // AND operation - const bufResult = buf1.map((b, i) => b & buf2[i]); - return buf1.equals(bufResult); - } } diff --git a/libs/ftso-core/src/events/AttestationRequest.ts b/libs/ftso-core/src/events/AttestationRequest.ts index 4e3b5340..7b8abeba 100644 --- a/libs/ftso-core/src/events/AttestationRequest.ts +++ b/libs/ftso-core/src/events/AttestationRequest.ts @@ -2,7 +2,10 @@ import { CONTRACTS } from "../configs/networks"; import { decodeEvent } from "../utils/EncodingUtils"; import { RawEventConstructible } from "./RawEventConstructible"; - +/** + * Represents and event emitted on submission of an attestation request on + * FdcHub smart contract. + */ export class AttestationRequest extends RawEventConstructible { static eventName = "AttestationRequest"; constructor(data: any, timestamp: number) { @@ -24,7 +27,11 @@ export class AttestationRequest extends RawEventConstructible { ); } - static getId(attestationRequest: AttestationRequest): string { + /** + * Gets the attestation request prefix. + * The prefix consists of the first 64 bytes of the data. + */ + static getPrefix(attestationRequest: AttestationRequest): string { // 0x + 64 bytes in hex if (attestationRequest.data.length < 130) { return undefined; @@ -41,9 +48,9 @@ export class AttestationRequest extends RawEventConstructible { // timestamp timestamp: number; - // confirmed + // whether the attestation got confirmed in the voting round confirmed: boolean = false; - // duplicate + // whether the attestations is a duplicate duplicate: boolean = false; } diff --git a/libs/ftso-core/src/events/FDCInflationRewardsOffered.ts b/libs/ftso-core/src/events/FDCInflationRewardsOffered.ts index 1897d564..c0d98624 100644 --- a/libs/ftso-core/src/events/FDCInflationRewardsOffered.ts +++ b/libs/ftso-core/src/events/FDCInflationRewardsOffered.ts @@ -15,6 +15,9 @@ export interface FdcConfiguration { mode: bigint; } +/** + * Represents an event emitted on offering inflation rewards on FdcHub smart contract. + */ export class FDCInflationRewardsOffered extends RawEventConstructible { static eventName = "InflationRewardsOffered"; constructor(data: any) { diff --git a/libs/ftso-core/src/events/FUInflationRewardsOffered.ts b/libs/ftso-core/src/events/FUInflationRewardsOffered.ts index d715651f..b7ad7179 100644 --- a/libs/ftso-core/src/events/FUInflationRewardsOffered.ts +++ b/libs/ftso-core/src/events/FUInflationRewardsOffered.ts @@ -11,6 +11,9 @@ export interface FastUpdateFeedConfiguration { inflationShare: number; } +/** + * Represents an event emitted on offering inflation rewards on FastUpdateIncentiveManager smart contract. + */ export class FUInflationRewardsOffered extends RawEventConstructible { static eventName = "InflationRewardsOffered"; constructor(data: any) { diff --git a/libs/ftso-core/src/reward-calculation/RewardTypePrefix.ts b/libs/ftso-core/src/reward-calculation/RewardTypePrefix.ts index 2abf2479..5c45d789 100644 --- a/libs/ftso-core/src/reward-calculation/RewardTypePrefix.ts +++ b/libs/ftso-core/src/reward-calculation/RewardTypePrefix.ts @@ -10,4 +10,6 @@ export enum RewardTypePrefix { FAST_UPDATES_ACCURACY = "Fast updates accuracy", FULL_OFFER_CLAIM_BACK = "Full offer claim back", PARTIAL_FDC_OFFER_CLAIM_BACK = "Partial FDC offer claim back", + FDC_SIGNING = "FDC signing", + FDC_FINALIZATION = "FDC finalization", } diff --git a/libs/ftso-core/src/reward-calculation/fdc/fdc-utils.ts b/libs/ftso-core/src/reward-calculation/fdc/fdc-utils.ts new file mode 100644 index 00000000..5c252172 --- /dev/null +++ b/libs/ftso-core/src/reward-calculation/fdc/fdc-utils.ts @@ -0,0 +1,232 @@ +import { ISignaturePayload } from "../../../../fsp-utils/src/SignaturePayload"; +import { GenericSubmissionData, SubmissionData } from "../../IndexerClient"; +import { RewardEpoch } from "../../RewardEpoch"; +import { FDC_PROTOCOL_ID, WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH } from "../../configs/networks"; +import { FDCEligibleSigner, FDCOffender, FDCOffense, FDCRewardData } from "../../data-calculation-interfaces"; +import { AttestationRequest } from "../../events/AttestationRequest"; +import { Address, MessageHash } from "../../voting-types"; + +/** + * Given a list of attestation request events it calculates the list of indices of same requests. + * The first index in such sub-list is the representative of the same requests. + */ +export function uniqueRequestsIndices(attestationRequests: AttestationRequest[]): number[][] { + const encountered = new Map(); + const result: number[][] = []; + for (let i = 0; i < attestationRequests.length; i++) { + const request = attestationRequests[i]; + if (!encountered.get(request.data)) { + encountered.set(request.data, i); + result.push([i]); + } else { + result[encountered.get(request.data)].push(i); + } + } + return result; +} + +/** + * Given a bitvote string (bytes in hex) it returns the indices of accepted attestation requests. + */ +export function bitVoteIndices(bitVote: string, len: number): number[] | undefined { + if (!bitVote || bitVote.length < 4) { + return undefined + } + const length = parseInt(bitVote.slice(2, 4), 16); + if (length !== len) { + throw new Error(`Bitvote length mismatch: ${length} !== ${len}`); + } + + const result: number[] = []; + let bitVoteNum = BigInt("0x" + bitVote.slice(4)); + return bitVoteIndicesNum(bitVoteNum, len); +} + +/** + * Given a number representing a bitvote it returns the indices of accepted attestation requests. + */ +export function bitVoteIndicesNum(bitVoteNum: bigint, len: number): number[] { + const result: number[] = []; + for (let i = 0; i < len; i++) { + if (bitVoteNum % 2n === 1n) { + result.push(i); + } + bitVoteNum /= 2n; + } + if (bitVoteNum !== 0n) { + throw new Error(`bitVoteNum not fully consumed: ${bitVoteNum}`); + } + return result; +} + +/** + * Returns true if the string encoded bitvote dominates the number encoded consensus bitvote. + */ +function isConsensusVoteDominated(consensusBitVote: bigint, bitVote?: string): boolean { + if (!bitVote) { + return false; + } + // Remove 0x prefix and first 2 bytes, used for the length + let h1 = consensusBitVote.toString(16); + // Ensure even length + if (h1.length % 2 !== 0) { + h1 = "0" + h1; + } + // This one is always even length + let h2 = bitVote.startsWith("0x") ? bitVote.slice(6) : bitVote.slice(4); + if (h1.length !== h2.length) { + const mLen = Math.max(h1.length, h2.length); + h1 = h1.padStart(mLen, "0"); + h2 = h2.padStart(mLen, "0"); + } + const buf1 = Buffer.from(h1, "hex"); + const buf2 = Buffer.from(h2, "hex"); + // AND operation + const bufResult = buf1.map((b, i) => b & buf2[i]); + return buf1.equals(bufResult); +} + +/** + * Given finalized messageHash it calculates consensus bitvote, filters out eligible signers and determines + * offenders. + */ +export function extractFDCRewardData( + messageHash: string, + bitVoteSubmissions: SubmissionData[], + fdcSignatures: Map[]>, + rewardEpoch: RewardEpoch, +): FDCRewardData | undefined { + const voteCounter = new Map(); + // + const eligibleSigners: FDCEligibleSigner[] = []; + const offenseMap = new Map(); + if (!messageHash) { + throw new Error("Consensus message hash is required"); + } + const signatures = fdcSignatures.get(messageHash); + if (!signatures) { + // TODO: log warning + return undefined; + } + for (const signature of signatures) { + const consensusBitVoteCandidate = signature.messages.unsignedMessage?.toLowerCase(); + if (!consensusBitVoteCandidate || consensusBitVoteCandidate.length < 6) { + continue; + } + const bitVoteNum = BigInt("0x" + consensusBitVoteCandidate.slice(6)); + // Note that 0n is also a legit consensus bitvote meaning no confirmations (but might not be rewarded) + voteCounter.set(bitVoteNum, (voteCounter.get(bitVoteNum) || 0) + signature.messages.weight) + } + let consensusBitVote: bigint | undefined; + if (voteCounter.size > 0) { + const maxCount = Math.max(...voteCounter.values()); + const maxBitVotes = [...voteCounter.entries()].filter(([_, count]) => count === maxCount).map(([bitVote, _]) => bitVote); + maxBitVotes.sort(); + // if it happens there are multiple maxHashes we take the first in lexicographical order + consensusBitVote = maxBitVotes[0]; + + // TODO: + // should we require 50%+ weight on maxHash? + // const consensusBitVoteWeight = voteCounter.get(consensusBitVote); + // if(consensusBitVoteWeight < rewardEpoch.signingPolicy.threshold) { + // return undefined; + // } + } + + const submitSignatureAddressToBitVote = new Map(); + for (const submission of bitVoteSubmissions) { + const submitSignatureAddress = rewardEpoch.getSubmitSignatureAddressFromSubmitAddress(submission.submitAddress).toLowerCase(); + const message = submission.messages.find(m => m.protocolId === FDC_PROTOCOL_ID); + if (message && message.payload) { + submitSignatureAddressToBitVote.set(submitSignatureAddress, message.payload.toLowerCase()); + } + } + + const submitSignatureSenders = new Set
(); + + for (const signature of signatures) { + // too late + if (signature.relativeTimestamp >= 90) { + continue; + } + const submitSignatureAddress = signature.submitAddress.toLowerCase() + submitSignatureSenders.add(submitSignatureAddress); + const bitVote = submitSignatureAddressToBitVote.get(submitSignatureAddress); + const eligibleSigner: FDCEligibleSigner = { + submitSignatureAddress: signature.submitAddress.toLowerCase(), + timestamp: signature.timestamp, + votingEpochIdFromTimestamp: signature.votingEpochIdFromTimestamp, + relativeTimestamp: signature.relativeTimestamp, + bitVote, + dominatesConsensusBitVote: consensusBitVote === undefined ? undefined : isConsensusVoteDominated(consensusBitVote, bitVote), + weight: signature.messages.weight, + } + eligibleSigners.push(eligibleSigner); + } + + for (const submission of bitVoteSubmissions) { + const submitSignatureAddress = rewardEpoch.getSubmitSignatureAddressFromSubmitAddress(submission.submitAddress).toLowerCase(); + const submissionAddress = rewardEpoch.getSubmitAddressFromSubmitSignatureAddress(submitSignatureAddress).toLowerCase(); + if (!submitSignatureSenders.has(submitSignatureAddress)) { + const offender: FDCOffender = { + submitSignatureAddress, + submissionAddress, + weight: rewardEpoch.getSigningWeightForSubmitSignatureAddress(submitSignatureAddress), + offenses: [FDCOffense.NO_REVEAL_ON_BITVOTE] + } + offenseMap.set(submitSignatureAddress, offender); + } + } + + const wrongSignatures = fdcSignatures.get(WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH); + if (wrongSignatures) { + for (const signature of wrongSignatures) { + const submitSignatureAddress = signature.submitAddress.toLowerCase(); + const submissionAddress = rewardEpoch.getSubmitAddressFromSubmitSignatureAddress(submitSignatureAddress).toLowerCase(); + if (!rewardEpoch.isEligibleSubmitSignatureAddress(submitSignatureAddress)) { + continue; + } + const offender = offenseMap.get(submitSignatureAddress) || { + submitSignatureAddress, + submissionAddress, + weight: rewardEpoch.getSigningWeightForSubmitSignatureAddress(submitSignatureAddress), + offenses: [] + } + offender.offenses.push(FDCOffense.WRONG_SIGNATURE); + offenseMap.set(submitSignatureAddress, offender); + } + } + for (const signature of signatures) { + const submitSignatureAddress = signature.submitAddress.toLowerCase(); + const submissionAddress = rewardEpoch.getSubmitAddressFromSubmitSignatureAddress(submitSignatureAddress).toLowerCase(); + const consensusBitVoteCandidate = signature.messages.unsignedMessage?.toLowerCase(); + if (!consensusBitVoteCandidate) { + continue; + } + // 0x + 2 bytes length + let isOffense = consensusBitVoteCandidate.length < 6; + if (!isOffense) { + isOffense = BigInt("0x" + consensusBitVoteCandidate.slice(6)) !== consensusBitVote; + } + if (isOffense) { + const offender = offenseMap.get(submitSignatureAddress) || { + submitSignatureAddress, + submissionAddress, + weight: rewardEpoch.getSigningWeightForSubmitSignatureAddress(submitSignatureAddress), + offenses: [] + } + offender.offenses.push(FDCOffense.BAD_CONSENSUS_BITVOTE_CANDIDATE); + offenseMap.set(submitSignatureAddress, offender); + } + } + + const fdcOffenders = [...offenseMap.values()]; + fdcOffenders.sort((a, b) => a.submitSignatureAddress.localeCompare(b.submitSignatureAddress)); + const result: FDCRewardData = { + eligibleSigners, + consensusBitVote, + fdcOffenders + }; + + return result; +} diff --git a/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-signing.ts b/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-signing.ts index df77f157..4268d3ee 100644 --- a/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-signing.ts +++ b/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-signing.ts @@ -67,7 +67,7 @@ export function calculateSigningRewardsForFDC( offerIndex: offer.offerIndex, feedId: offer.feedId, protocolTag: "" + FDC_PROTOCOL_ID, - rewardTypeTag: RewardTypePrefix.SIGNING, + rewardTypeTag: RewardTypePrefix.FDC_SIGNING, rewardDetailTag: SigningRewardClaimType.NO_TIMELY_FINALIZATION, }; return [backClaim]; @@ -115,14 +115,14 @@ export function calculateSigningRewardsForFDC( offerIndex: offer.offerIndex, feedId: offer.feedId, protocolTag: "" + FDC_PROTOCOL_ID, - rewardTypeTag: RewardTypePrefix.SIGNING, + rewardTypeTag: RewardTypePrefix.FDC_SIGNING, rewardDetailTag: SigningRewardClaimType.NON_DOMINATING_BITVOTE, }; allClaims.push(burnClaim); } } allClaims.push( - ...generateSigningWeightBasedClaimsForVoter(voterAmount, offer, voterWeights, RewardTypePrefix.SIGNING, FDC_PROTOCOL_ID) + ...generateSigningWeightBasedClaimsForVoter(voterAmount, offer, voterWeights, RewardTypePrefix.FDC_SIGNING, FDC_PROTOCOL_ID) ); } } @@ -137,7 +137,7 @@ export function calculateSigningRewardsForFDC( offerIndex: offer.offerIndex, feedId: offer.feedId, protocolTag: "" + FDC_PROTOCOL_ID, - rewardTypeTag: RewardTypePrefix.SIGNING, + rewardTypeTag: RewardTypePrefix.FDC_SIGNING, rewardDetailTag: SigningRewardClaimType.CLAIM_BACK_OF_NON_SIGNERS_SHARE, }; allClaims.push(backClaim); diff --git a/libs/ftso-core/src/reward-calculation/reward-calculation.ts b/libs/ftso-core/src/reward-calculation/reward-calculation.ts index 255fbd30..cf4e27d5 100644 --- a/libs/ftso-core/src/reward-calculation/reward-calculation.ts +++ b/libs/ftso-core/src/reward-calculation/reward-calculation.ts @@ -91,7 +91,7 @@ export async function partialRewardClaimsForVotingRound( rewardEpochId: number, votingRoundId: number, randomGenerationBenchingWindow: number, - dataManager: DataManager, + dataManager: DataManagerForRewarding, feedOffersParam: Map | undefined, prepareData = true, merge = true, @@ -197,7 +197,8 @@ export async function partialRewardClaimsForVotingRound( data.finalizations, data, new Set(data.eligibleFinalizers), - medianEligibleVoters + medianEligibleVoters, + RewardTypePrefix.FINALIZATION ); // Calculate penalties for reveal withdrawal offenders @@ -375,7 +376,8 @@ export async function partialRewardClaimsForVotingRound( data.fdcData.finalizations, data, new Set(data.eligibleFinalizers), - new Set(data.eligibleFinalizers) + new Set(data.eligibleFinalizers), + RewardTypePrefix.FDC_FINALIZATION ); const fdcSigningRewardClaims = calculateSigningRewardsForFDC( @@ -412,7 +414,7 @@ export async function prepareDataForRewardCalculations( rewardEpochId: number, votingRoundId: number, randomGenerationBenchingWindow: number, - dataManager: DataManager, + dataManager: DataManagerForRewarding, calculationFolder = CALCULATIONS_FOLDER() ) { const rewardDataForCalculationResponse = await dataManager.getDataForRewardCalculation( diff --git a/libs/ftso-core/src/reward-calculation/reward-finalization.ts b/libs/ftso-core/src/reward-calculation/reward-finalization.ts index e43eec28..dbec97be 100644 --- a/libs/ftso-core/src/reward-calculation/reward-finalization.ts +++ b/libs/ftso-core/src/reward-calculation/reward-finalization.ts @@ -33,7 +33,8 @@ export function calculateFinalizationRewardClaims( finalizations: ParsedFinalizationData[], data: SDataForRewardCalculation, eligibleFinalizationRewardVotersInGracePeriod: Set
, // signing addresses of the voters that are eligible for finalization reward - eligibleVoters: Set
// signing addresses of the voters that are eligible for finalization reward + eligibleVoters: Set
, // signing addresses of the voters that are eligible for finalization reward + rewardType: RewardTypePrefix ): IPartialRewardClaim[] { if (!firstSuccessfulFinalization) { const backClaim: IPartialRewardClaim = { @@ -44,7 +45,7 @@ export function calculateFinalizationRewardClaims( offerIndex: offer.offerIndex, feedId: offer.feedId, protocolTag: "" + protocolId, - rewardTypeTag: RewardTypePrefix.FINALIZATION, + rewardTypeTag: rewardType, rewardDetailTag: FinalizationRewardClaimType.NO_FINALIZATION, }; return [backClaim]; @@ -60,7 +61,7 @@ export function calculateFinalizationRewardClaims( offerIndex: offer.offerIndex, feedId: offer.feedId, protocolTag: "" + protocolId, - rewardTypeTag: RewardTypePrefix.FINALIZATION, + rewardTypeTag: rewardType, rewardDetailTag: FinalizationRewardClaimType.OUTSIDE_OF_GRACE_PERIOD, }; return [otherFinalizerClaim]; @@ -80,7 +81,7 @@ export function calculateFinalizationRewardClaims( offerIndex: offer.offerIndex, feedId: offer.feedId, protocolTag: "" + protocolId, - rewardTypeTag: RewardTypePrefix.FINALIZATION, + rewardTypeTag: rewardType, rewardDetailTag: FinalizationRewardClaimType.FINALIZED_BUT_NO_ELIGIBLE_VOTERS, }; return [burnClaim]; @@ -127,7 +128,7 @@ export function calculateFinalizationRewardClaims( undistributedAmount -= amount; undistributedSigningRewardWeight -= 1n; resultClaims.push( - ...generateSigningWeightBasedClaimsForVoter(amount, offer, voterWeight, RewardTypePrefix.FINALIZATION, protocolId) + ...generateSigningWeightBasedClaimsForVoter(amount, offer, voterWeight, rewardType, protocolId) ); } @@ -148,7 +149,7 @@ export function calculateFinalizationRewardClaims( offerIndex: offer.offerIndex, feedId: offer.feedId, protocolTag: "" + protocolId, - rewardTypeTag: RewardTypePrefix.FINALIZATION, + rewardTypeTag: rewardType, rewardDetailTag: FinalizationRewardClaimType.CLAIM_BACK_FOR_UNDISTRIBUTED_REWARDS, }); } diff --git a/scripts/analytics/run/offer-check.ts b/scripts/analytics/run/offer-check.ts index b31e16ff..229d6999 100644 --- a/scripts/analytics/run/offer-check.ts +++ b/scripts/analytics/run/offer-check.ts @@ -34,6 +34,7 @@ async function main() { let ftsoScalingOfferAmount = 0n; let fastUpdatesOfferAmount = 0n; let fdcOfferAmount = 0n; + let fdcOfferBurn = 0n; for (let votingRoundId = rewardEpochInfo.signingPolicy.startVotingRoundId; votingRoundId <= rewardEpochInfo.endVotingRoundId; votingRoundId++) { const ftsoOfferClaims = deserializeGranulatedPartialOfferMap(rewardEpochId, votingRoundId, calculationFolder); for (let [_, offers] of ftsoOfferClaims.entries()) { @@ -51,6 +52,9 @@ async function main() { const offers = deserializeOffersForFDC(rewardEpochId, votingRoundId, calculationFolder); for (let offer of offers) { fdcOfferAmount += offer.amount; + if(offer.shouldBeBurned) { + fdcOfferBurn += offer.amount; + } } const data = deserializeDataForRewardCalculation( rewardEpochId, @@ -67,6 +71,7 @@ async function main() { console.log(`FTSO Scaling Funds: ${ftsoScalingFunds - ftsoScalingOfferAmount}`); console.log(`Fast Updates Funds: ${fastUpdatesFunds - fastUpdatesOfferAmount}`); console.log(`FDC Funds: ${fdcFunds - fdcOfferAmount}`); + console.log(`FDC Offer Burn: ${fdcOfferBurn}`); } main() diff --git a/test/libs/unit/generator-rewards.test.ts b/test/libs/unit/generator-rewards.test.ts index d8ea02b0..c21f487d 100644 --- a/test/libs/unit/generator-rewards.test.ts +++ b/test/libs/unit/generator-rewards.test.ts @@ -1,7 +1,6 @@ import FakeTimers from "@sinonjs/fake-timers"; import { expect } from "chai"; import { DataSource, EntityManager } from "typeorm"; -import { DataManager } from "../../../libs/ftso-core/src/DataManager"; import { IndexerClient } from "../../../libs/ftso-core/src/IndexerClient"; import { RewardEpochManager } from "../../../libs/ftso-core/src/RewardEpochManager"; import { @@ -49,7 +48,18 @@ import { setupEpochSettings, } from "../../utils/test-epoch-settings"; +import { DataManagerForRewarding } from "../../../libs/ftso-core/src/DataManagerForRewarding"; +import { IndexerClientForRewarding } from "../../../libs/ftso-core/src/IndexerClientForRewarding"; +import { RewardOffers } from "../../../libs/ftso-core/src/events"; +import { + IPartialRewardOfferForEpoch, + IPartialRewardOfferForRound, + PartialRewardOffer, +} from "../../../libs/ftso-core/src/utils/PartialRewardOffer"; +import { RewardEpochDuration } from "../../../libs/ftso-core/src/utils/RewardEpochDuration"; import { deserializeAggregatedClaimsForVotingRoundId } from "../../../libs/ftso-core/src/utils/stat-info/aggregated-claims"; +import { OFFERS_FILE } from "../../../libs/ftso-core/src/utils/stat-info/constants"; +import { serializeGranulatedPartialOfferMap } from "../../../libs/ftso-core/src/utils/stat-info/granulated-partial-offers-map"; import { ProgressType, printProgress, @@ -65,15 +75,6 @@ import { } from "../../../libs/ftso-core/src/utils/stat-info/reward-epoch-info"; import { destroyStorage } from "../../../libs/ftso-core/src/utils/stat-info/storage"; import { toFeedId } from "../../utils/generators"; -import { - IPartialRewardOfferForEpoch, - IPartialRewardOfferForRound, - PartialRewardOffer, -} from "../../../libs/ftso-core/src/utils/PartialRewardOffer"; -import { RewardEpochDuration } from "../../../libs/ftso-core/src/utils/RewardEpochDuration"; -import { OFFERS_FILE } from "../../../libs/ftso-core/src/utils/stat-info/constants"; -import { serializeGranulatedPartialOfferMap } from "../../../libs/ftso-core/src/utils/stat-info/granulated-partial-offers-map"; -import { RewardOffers } from "../../../libs/ftso-core/src/events"; // Ensure that the networks are not loaded @@ -241,9 +242,9 @@ describe(`generator-rewards, ${getTestFile(__filename)}`, () => { 2 * EPOCH_SETTINGS().rewardEpochDurationInVotingEpochs * EPOCH_SETTINGS().votingEpochDurationSeconds; const earliestTimestamp = Math.floor(clock.Date.now() / 1000) - requiredHistoryTimeSec; logger.log("Earliest timestamp", earliestTimestamp); - const indexerClient = new IndexerClient(entityManager, requiredHistoryTimeSec, console); + const indexerClient = new IndexerClientForRewarding(entityManager, requiredHistoryTimeSec, console); const rewardEpochManger = new RewardEpochManager(indexerClient); - const dataManager = new DataManager(indexerClient, rewardEpochManger, console); + const dataManager = new DataManagerForRewarding(indexerClient, rewardEpochManger, console); const votingRoundId = EPOCH_SETTINGS().expectedFirstVotingRoundForRewardEpoch(rewardEpochId); const benchingWindowRevealOffenders = 1; @@ -302,9 +303,9 @@ describe(`generator-rewards, ${getTestFile(__filename)}`, () => { 2 * EPOCH_SETTINGS().rewardEpochDurationInVotingEpochs * EPOCH_SETTINGS().votingEpochDurationSeconds; const earliestTimestamp = Math.floor(clock.Date.now() / 1000) - requiredHistoryTimeSec; logger.log("Earliest timestamp", earliestTimestamp); - const indexerClient = new IndexerClient(entityManager, requiredHistoryTimeSec, console); + const indexerClient = new IndexerClientForRewarding(entityManager, requiredHistoryTimeSec, console); const rewardEpochManger = new RewardEpochManager(indexerClient); - const dataManager = new DataManager(indexerClient, rewardEpochManger, console); + const dataManager = new DataManagerForRewarding(indexerClient, rewardEpochManger, console); const votingRoundId = EPOCH_SETTINGS().expectedFirstVotingRoundForRewardEpoch(rewardEpochId); const benchingWindowRevealOffenders = 1; @@ -391,9 +392,9 @@ describe(`generator-rewards, ${getTestFile(__filename)}`, () => { 2 * EPOCH_SETTINGS().rewardEpochDurationInVotingEpochs * EPOCH_SETTINGS().votingEpochDurationSeconds; const earliestTimestamp = Math.floor(clock.Date.now() / 1000) - requiredHistoryTimeSec; logger.log("Earliest timestamp", earliestTimestamp); - const indexerClient = new IndexerClient(entityManager, requiredHistoryTimeSec, console); + const indexerClient = new IndexerClientForRewarding(entityManager, requiredHistoryTimeSec, console); const rewardEpochManger = new RewardEpochManager(indexerClient); - const dataManager = new DataManager(indexerClient, rewardEpochManger, console); + const dataManager = new DataManagerForRewarding(indexerClient, rewardEpochManger, console); const votingRoundId = EPOCH_SETTINGS().expectedFirstVotingRoundForRewardEpoch(rewardEpochId); const benchingWindowRevealOffenders = 1; @@ -452,9 +453,9 @@ describe(`generator-rewards, ${getTestFile(__filename)}`, () => { 2 * EPOCH_SETTINGS().rewardEpochDurationInVotingEpochs * EPOCH_SETTINGS().votingEpochDurationSeconds; const earliestTimestamp = Math.floor(clock.Date.now() / 1000) - requiredHistoryTimeSec; logger.log("Earliest timestamp", earliestTimestamp); - const indexerClient = new IndexerClient(entityManager, requiredHistoryTimeSec, console); + const indexerClient = new IndexerClientForRewarding(entityManager, requiredHistoryTimeSec, console); const rewardEpochManger = new RewardEpochManager(indexerClient); - const dataManager = new DataManager(indexerClient, rewardEpochManger, console); + const dataManager = new DataManagerForRewarding(indexerClient, rewardEpochManger, console); const votingRoundId = EPOCH_SETTINGS().expectedFirstVotingRoundForRewardEpoch(rewardEpochId); const benchingWindowRevealOffenders = 1; @@ -523,9 +524,9 @@ describe(`generator-rewards, ${getTestFile(__filename)}`, () => { 2 * EPOCH_SETTINGS().rewardEpochDurationInVotingEpochs * EPOCH_SETTINGS().votingEpochDurationSeconds; const earliestTimestamp = Math.floor(clock.Date.now() / 1000) - requiredHistoryTimeSec; logger.log("Earliest timestamp", earliestTimestamp); - const indexerClient = new IndexerClient(entityManager, requiredHistoryTimeSec, console); + const indexerClient = new IndexerClientForRewarding(entityManager, requiredHistoryTimeSec, console); const rewardEpochManger = new RewardEpochManager(indexerClient); - const dataManager = new DataManager(indexerClient, rewardEpochManger, console); + const dataManager = new DataManagerForRewarding(indexerClient, rewardEpochManger, console); const votingRoundId = EPOCH_SETTINGS().expectedFirstVotingRoundForRewardEpoch(rewardEpochId); const benchingWindowRevealOffenders = 1; @@ -568,9 +569,9 @@ describe(`generator-rewards, ${getTestFile(__filename)}`, () => { 2 * EPOCH_SETTINGS().rewardEpochDurationInVotingEpochs * EPOCH_SETTINGS().votingEpochDurationSeconds; const earliestTimestamp = Math.floor(clock.Date.now() / 1000) - requiredHistoryTimeSec; logger.log("Earliest timestamp", earliestTimestamp); - const indexerClient = new IndexerClient(entityManager, requiredHistoryTimeSec, console); + const indexerClient = new IndexerClientForRewarding(entityManager, requiredHistoryTimeSec, console); const rewardEpochManger = new RewardEpochManager(indexerClient); - const dataManager = new DataManager(indexerClient, rewardEpochManger, console); + const dataManager = new DataManagerForRewarding(indexerClient, rewardEpochManger, console); const votingRoundId = EPOCH_SETTINGS().expectedFirstVotingRoundForRewardEpoch(rewardEpochId); const rewardEpoch = await rewardEpochManger.getRewardEpochForVotingEpochId(votingRoundId); @@ -739,7 +740,7 @@ export async function initializeRewardEpochStorageOld( export async function rewardClaimsForRewardEpoch( rewardEpochId: number, randomGenerationBenchingWindow: number, - dataManager: DataManager, + dataManager: DataManagerForRewarding, rewardEpochManager: RewardEpochManager, merge = true, serialize = false, diff --git a/test/utils/mini-finalizer/MiniFinalizer.ts b/test/utils/mini-finalizer/MiniFinalizer.ts index cf88723f..2010e877 100644 --- a/test/utils/mini-finalizer/MiniFinalizer.ts +++ b/test/utils/mini-finalizer/MiniFinalizer.ts @@ -9,6 +9,7 @@ import { RewardEpoch } from "../../../libs/ftso-core/src/RewardEpoch"; import { RewardEpochManager } from "../../../libs/ftso-core/src/RewardEpochManager"; import { CONTRACTS, EPOCH_SETTINGS, FTSO2_PROTOCOL_ID } from "../../../libs/ftso-core/src/configs/networks"; +import { DataManagerForRewarding } from "../../../libs/ftso-core/src/DataManagerForRewarding"; import { ContractMethodNames } from "../../../libs/ftso-core/src/configs/contracts"; import { TLPTransaction } from "../../../libs/ftso-core/src/orm/entities"; import { RandomVoterSelector } from "../../../libs/ftso-core/src/reward-calculation/RandomVoterSelector"; @@ -165,7 +166,7 @@ export class MiniFinalizer { } const signatures = submitSignaturesSubmissionResponse.data; DataManager.sortSubmissionDataArray(signatures); - const finalizationMap = DataManager.extractSignatures( + const finalizationMap = DataManagerForRewarding.extractSignatures( votingRoundId, rewardEpoch, signatures, From 787d791ab07cb709702d4abc6b4a4068ec891b3c Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Wed, 6 Nov 2024 08:27:52 +0100 Subject: [PATCH 15/28] fix --- .../libs/{fdc-utils.ts => attestation-type-appearances.ts} | 7 ++++++- .../src/services/calculator.service.ts | 2 +- libs/ftso-core/src/reward-calculation/reward-offers.ts | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) rename apps/ftso-reward-calculation-process/src/libs/{fdc-utils.ts => attestation-type-appearances.ts} (95%) diff --git a/apps/ftso-reward-calculation-process/src/libs/fdc-utils.ts b/apps/ftso-reward-calculation-process/src/libs/attestation-type-appearances.ts similarity index 95% rename from apps/ftso-reward-calculation-process/src/libs/fdc-utils.ts rename to apps/ftso-reward-calculation-process/src/libs/attestation-type-appearances.ts index 4a5efb56..95ab90ff 100644 --- a/apps/ftso-reward-calculation-process/src/libs/fdc-utils.ts +++ b/apps/ftso-reward-calculation-process/src/libs/attestation-type-appearances.ts @@ -13,6 +13,9 @@ export interface FDCAttestationRequestAppearances { count: number; } +/** + * Calculates the number of appearances of each attestation type in the given reward epoch. + */ export function calculateAttestationTypeAppearances(rewardEpochId: number): void { const rewardEpochInfo = deserializeRewardEpochInfo(rewardEpochId); @@ -77,7 +80,9 @@ export function serializeAttestationRequestAppearances( writeFileSync(appearancesPath, JSON.stringify(appearances)); } - +/** + * Deserializes attestation request appearances data. + */ export function deserializeAttestationRequestAppearances( rewardEpochId: number, calculationFolder = CALCULATIONS_FOLDER() diff --git a/apps/ftso-reward-calculation-process/src/services/calculator.service.ts b/apps/ftso-reward-calculation-process/src/services/calculator.service.ts index 86293a36..cb2a44f6 100644 --- a/apps/ftso-reward-calculation-process/src/services/calculator.service.ts +++ b/apps/ftso-reward-calculation-process/src/services/calculator.service.ts @@ -45,7 +45,7 @@ import { runRandomNumberFixing } from "../libs/random-number-fixing-utils"; import { runCalculateRewardClaimsTopJob } from "../libs/reward-claims-calculation"; import { runCalculateRewardCalculationTopJob } from "../libs/reward-data-calculation"; import { getIncrementalCalculationsFeedSelections, serializeIncrementalCalculationsFeedSelections } from "../../../../libs/ftso-core/src/utils/stat-info/incremental-calculation-temp-selected-feeds"; -import { calculateAttestationTypeAppearances } from "../libs/fdc-utils"; +import { calculateAttestationTypeAppearances } from "../libs/attestation-type-appearances"; if (process.env.FORCE_NOW) { const newNow = parseInt(process.env.FORCE_NOW) * 1000; diff --git a/libs/ftso-core/src/reward-calculation/reward-offers.ts b/libs/ftso-core/src/reward-calculation/reward-offers.ts index 32231f21..351514e4 100644 --- a/libs/ftso-core/src/reward-calculation/reward-offers.ts +++ b/libs/ftso-core/src/reward-calculation/reward-offers.ts @@ -1,4 +1,4 @@ -import { deserializeAttestationRequestAppearances } from "../../../../apps/ftso-reward-calculation-process/src/libs/fdc-utils"; +import { deserializeAttestationRequestAppearances } from "../../../../apps/ftso-reward-calculation-process/src/libs/attestation-type-appearances"; import { BURN_ADDRESS, FINALIZATION_BIPS, SIGNING_BIPS, TOTAL_BIPS } from "../configs/networks"; import { InflationRewardsOffered } from "../events"; import { From 66a27ad93e1678b96e613432d1de61efba087c19 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:36:38 +0100 Subject: [PATCH 16/28] comments --- .../src/reward-calculation/reward-offers.ts | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/libs/ftso-core/src/reward-calculation/reward-offers.ts b/libs/ftso-core/src/reward-calculation/reward-offers.ts index 351514e4..550177e1 100644 --- a/libs/ftso-core/src/reward-calculation/reward-offers.ts +++ b/libs/ftso-core/src/reward-calculation/reward-offers.ts @@ -38,6 +38,11 @@ export function distributeInflationRewardOfferToFeeds( throw new Error(`Mode ${inflationRewardOffer.mode} is not supported`); } +/** + * Adapts community reward offer to default data. + * Community reward offers can have specific data about primary and seconary bands, which are ignored and + * overridden by default values. + */ export function adaptCommunityRewardOffer(rewardOffer: IPartialRewardOfferForEpoch): void { rewardOffer.minRewardedTurnoutBIPS = 0; rewardOffer.primaryBandRewardSharePPM = 1000000; @@ -158,6 +163,13 @@ export function granulatedPartialOfferMapForRandomFeedSelection( return rewardOfferMap; } +/** + * Prepares FTSO Scaling offers according to random number choice. + * Among all the offers for all available feeds one feed is selected using the + * provided random number. + * This is done for the range of voting rounds. + * If the random number is undefined, the offer is marked for burning. + */ export function fixOffersForRandomFeedSelection( rewardEpochId: number, startVotingRoundId: number, @@ -231,6 +243,9 @@ export function splitRewardOfferByTypes(offer: IPartialRewardOfferForRound): Spl return result; } +/** + * Creates a map of partial reward offers for each voting round in the reward epoch for fast updates. + */ export function granulatedPartialOfferMapForFastUpdates( rewardEpochInfo: RewardEpochInfo, randomNumbers: (bigint | undefined)[], @@ -295,6 +310,9 @@ export function granulatedPartialOfferMapForFastUpdates( return rewardOfferMap; } +/** + * Creates a map of partial reward offers for each voting round in the reward epoch for FDC. + */ export function granulatedPartialOfferMapForFDC( rewardEpochInfo: RewardEpochInfo, ): Map { @@ -311,18 +329,17 @@ export function granulatedPartialOfferMapForFDC( let totalWeight = 0; let burnWeight = 0; - for(const config of rewardEpochInfo.fdcInflationRewardsOffered.fdcConfigurations) { + for (const config of rewardEpochInfo.fdcInflationRewardsOffered.fdcConfigurations) { const id = (config.attestationType + config.source.slice(2)).toLowerCase(); const appearances = attestationCountMap.get(id) || 0; totalWeight += config.inflationShare; - if(appearances < config.minRequestsThreshold) { + if (appearances < config.minRequestsThreshold) { burnWeight += config.inflationShare; } } // Calculate total amount of rewards for the reward epoch - const totalBurnAmount = (BigInt(burnWeight)*rewardEpochInfo.fdcInflationRewardsOffered.amount)/BigInt(totalWeight); - // const totalBurnAmount = 0n; + const totalBurnAmount = (BigInt(burnWeight) * rewardEpochInfo.fdcInflationRewardsOffered.amount) / BigInt(totalWeight); let totalAmount = rewardEpochInfo.fdcInflationRewardsOffered.amount - totalBurnAmount; if (process.env.TEST_FDC_INFLATION_REWARD_AMOUNT) { @@ -342,15 +359,16 @@ export function granulatedPartialOfferMapForFDC( const roundDataForCalculation = deserializeDataForRewardCalculation(rewardEpochInfo.rewardEpochId, votingRoundId); const attestationRequests = roundDataForCalculation?.fdcData?.attestationRequests; - if(!attestationRequests) { + if (!attestationRequests) { throw new Error(`Missing attestation requests for voting round ${votingRoundId}`); } let feeAmount = 0n; let feeBurnAmount = 0n; - for(const attestationRequest of attestationRequests) { - if(attestationRequest.confirmed) { + for (const attestationRequest of attestationRequests) { + if (attestationRequest.confirmed) { feeAmount += attestationRequest.fee; } else { + // fees for unconfirmed requests are burned feeBurnAmount += attestationRequest.fee; } } @@ -368,7 +386,7 @@ export function granulatedPartialOfferMapForFDC( } rewardOfferMap.set(votingRoundId, [ - offerForVotingRound, + offerForVotingRound, burnOfferForVotingRound ]); } From 02c47d983c3206332628532a8e38f435ae77ac94 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:54:51 +0100 Subject: [PATCH 17/28] penalty fix --- libs/ftso-core/src/RewardEpoch.ts | 6 ++-- .../reward-calculation/RewardTypePrefix.ts | 1 + .../src/reward-calculation/fdc/fdc-utils.ts | 34 ++++++++++++++----- .../fdc/reward-fdc-penalties.ts | 2 +- .../reward-calculation/reward-calculation.ts | 2 +- 5 files changed, 32 insertions(+), 13 deletions(-) diff --git a/libs/ftso-core/src/RewardEpoch.ts b/libs/ftso-core/src/RewardEpoch.ts index 88653331..80ca892c 100644 --- a/libs/ftso-core/src/RewardEpoch.ts +++ b/libs/ftso-core/src/RewardEpoch.ts @@ -210,7 +210,7 @@ export class RewardEpoch { } public getSubmitSignatureAddressFromSubmitAddress(submitAddress: Address): Address | undefined { - return this.submitAddressToVoterRegistrationInfo.get(submitAddress.toLowerCase())?.voterRegistered.submitSignaturesAddress; + return this.submitAddressToVoterRegistrationInfo.get(submitAddress.toLowerCase())?.voterRegistered.submitSignaturesAddress.toLowerCase(); } public getSubmitAddressFromSubmitSignatureAddress(submitSignatureAddress: Address): Address | undefined { @@ -218,7 +218,7 @@ export class RewardEpoch { if (!voterAddress) { return undefined; } - return this.voterToRegistrationInfo.get(voterAddress.toLowerCase())?.voterRegistered.submitAddress; + return this.voterToRegistrationInfo.get(voterAddress.toLowerCase())?.voterRegistered.submitAddress.toLowerCase(); } public getSigningWeightForSubmitSignatureAddress(submitSignatureAddress: Address): number | undefined { @@ -226,7 +226,7 @@ export class RewardEpoch { if (!voterAddress) { return undefined; } - const signingAddress = this.voterToRegistrationInfo.get(voterAddress.toLowerCase())?.voterRegistered.signingPolicyAddress; + const signingAddress = this.voterToRegistrationInfo.get(voterAddress.toLowerCase())?.voterRegistered.signingPolicyAddress.toLowerCase(); if (!signingAddress) { return undefined; } diff --git a/libs/ftso-core/src/reward-calculation/RewardTypePrefix.ts b/libs/ftso-core/src/reward-calculation/RewardTypePrefix.ts index 5c45d789..9683f3d7 100644 --- a/libs/ftso-core/src/reward-calculation/RewardTypePrefix.ts +++ b/libs/ftso-core/src/reward-calculation/RewardTypePrefix.ts @@ -12,4 +12,5 @@ export enum RewardTypePrefix { PARTIAL_FDC_OFFER_CLAIM_BACK = "Partial FDC offer claim back", FDC_SIGNING = "FDC signing", FDC_FINALIZATION = "FDC finalization", + FDC_OFFENDERS = "FDC offenders", } diff --git a/libs/ftso-core/src/reward-calculation/fdc/fdc-utils.ts b/libs/ftso-core/src/reward-calculation/fdc/fdc-utils.ts index 5c252172..8bd4eb9f 100644 --- a/libs/ftso-core/src/reward-calculation/fdc/fdc-utils.ts +++ b/libs/ftso-core/src/reward-calculation/fdc/fdc-utils.ts @@ -73,6 +73,7 @@ function isConsensusVoteDominated(consensusBitVote: bigint, bitVote?: string): b h1 = "0" + h1; } // This one is always even length + // first 2-bytes are skipped (length) let h2 = bitVote.startsWith("0x") ? bitVote.slice(6) : bitVote.slice(4); if (h1.length !== h2.length) { const mLen = Math.max(h1.length, h2.length); @@ -81,35 +82,47 @@ function isConsensusVoteDominated(consensusBitVote: bigint, bitVote?: string): b } const buf1 = Buffer.from(h1, "hex"); const buf2 = Buffer.from(h2, "hex"); - // AND operation + // AND operation should not decrease the number of 1s const bufResult = buf1.map((b, i) => b & buf2[i]); return buf1.equals(bufResult); } /** - * Given finalized messageHash it calculates consensus bitvote, filters out eligible signers and determines + * Given finalized messageHash it calculates consensus bit-vote, filters out eligible signers and determines * offenders. + * Message hash of the finalized (consensus) message is required or exception is thrown. + * The @param fdcSignatures is expected to be a map containing at most 2 keys: + * - messageHash + * - WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH + * If there are no signatures for the messageHash, the function returns undefined. */ export function extractFDCRewardData( messageHash: string, bitVoteSubmissions: SubmissionData[], fdcSignatures: Map[]>, rewardEpoch: RewardEpoch, -): FDCRewardData | undefined { +): FDCRewardData { + // consensus bitvote -> weight const voteCounter = new Map(); - // + // List of records about signers which are eligible for the reward. + // Note that a subset of those is later rewarded const eligibleSigners: FDCEligibleSigner[] = []; + // submitSignatureAddress -> FDCOffender const offenseMap = new Map(); if (!messageHash) { throw new Error("Consensus message hash is required"); } const signatures = fdcSignatures.get(messageHash); if (!signatures) { - // TODO: log warning - return undefined; + return { + eligibleSigners: [], + consensusBitVote: undefined, + fdcOffenders: [] + }; } for (const signature of signatures) { const consensusBitVoteCandidate = signature.messages.unsignedMessage?.toLowerCase(); + // first 2 bytes are length if (!consensusBitVoteCandidate || consensusBitVoteCandidate.length < 6) { continue; } @@ -122,7 +135,7 @@ export function extractFDCRewardData( const maxCount = Math.max(...voteCounter.values()); const maxBitVotes = [...voteCounter.entries()].filter(([_, count]) => count === maxCount).map(([bitVote, _]) => bitVote); maxBitVotes.sort(); - // if it happens there are multiple maxHashes we take the first in lexicographical order + // if it happens there are multiple maxHashes we take the first in order consensusBitVote = maxBitVotes[0]; // TODO: @@ -135,7 +148,7 @@ export function extractFDCRewardData( const submitSignatureAddressToBitVote = new Map(); for (const submission of bitVoteSubmissions) { - const submitSignatureAddress = rewardEpoch.getSubmitSignatureAddressFromSubmitAddress(submission.submitAddress).toLowerCase(); + const submitSignatureAddress = rewardEpoch.getSubmitSignatureAddressFromSubmitAddress(submission.submitAddress.toLowerCase()).toLowerCase(); const message = submission.messages.find(m => m.protocolId === FDC_PROTOCOL_ID); if (message && message.payload) { submitSignatureAddressToBitVote.set(submitSignatureAddress, message.payload.toLowerCase()); @@ -203,6 +216,7 @@ export function extractFDCRewardData( if (!consensusBitVoteCandidate) { continue; } + // Offense is either wrong bitvote message or not matching the consensus bitvote // 0x + 2 bytes length let isOffense = consensusBitVoteCandidate.length < 6; if (!isOffense) { @@ -221,7 +235,11 @@ export function extractFDCRewardData( } const fdcOffenders = [...offenseMap.values()]; + // Fix the orders for determinism fdcOffenders.sort((a, b) => a.submitSignatureAddress.localeCompare(b.submitSignatureAddress)); + for(const offender of fdcOffenders) { + offender.offenses.sort(); + } const result: FDCRewardData = { eligibleSigners, consensusBitVote, diff --git a/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-penalties.ts b/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-penalties.ts index a9aa1a1c..aa499c33 100644 --- a/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-penalties.ts +++ b/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-penalties.ts @@ -32,7 +32,7 @@ export function calculateFdcPenalties( if (offender.weight > 0) { penalty = (-BigInt(offender.weight) * offer.amount * penaltyFactor) / totalWeight; } - if (penalty > 0n) { + if (penalty < 0n) { penaltyClaims.push(...generateSigningWeightBasedClaimsForVoter(penalty, offer, voterWeights, penaltyType, FDC_PROTOCOL_ID)); } } diff --git a/libs/ftso-core/src/reward-calculation/reward-calculation.ts b/libs/ftso-core/src/reward-calculation/reward-calculation.ts index cf4e27d5..8097ba88 100644 --- a/libs/ftso-core/src/reward-calculation/reward-calculation.ts +++ b/libs/ftso-core/src/reward-calculation/reward-calculation.ts @@ -392,7 +392,7 @@ export async function partialRewardClaimsForVotingRound( data, PENALTY_FACTOR(), data.dataForCalculations.votersWeightsMap!, - RewardTypePrefix.REVEAL_OFFENDERS + RewardTypePrefix.FDC_OFFENDERS ) allRewardClaims.push(...fdCFinalizationRewardClaims); From 09f5040a37b4c1d0fe0edb6a191bb1d043e5d58a Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Wed, 6 Nov 2024 16:01:53 +0100 Subject: [PATCH 18/28] burn on empty consensus bitvote --- .../fdc/reward-fdc-signing.ts | 25 ++++++++++++++++--- .../src/reward-calculation/reward-signing.ts | 1 + 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-signing.ts b/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-signing.ts index 4268d3ee..099ea1f5 100644 --- a/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-signing.ts +++ b/libs/ftso-core/src/reward-calculation/fdc/reward-fdc-signing.ts @@ -57,6 +57,7 @@ export function calculateSigningRewardsForFDC( rewardEpochInfo: RewardEpochInfo ): IPartialRewardClaim[] { const votingRoundId = data.dataForCalculations.votingRoundId; + // if no successful finalization, nothing to decide - burn all if (!data.fdcData?.firstSuccessfulFinalization) { // burn all const backClaim: IPartialRewardClaim = { @@ -73,6 +74,22 @@ export function calculateSigningRewardsForFDC( return [backClaim]; } + if (data.fdcData.consensusBitVote === undefined || data.fdcData.consensusBitVote === 0n) { + // burn all + const backClaim: IPartialRewardClaim = { + votingRoundId, + beneficiary: offer.claimBackAddress.toLowerCase(), + amount: offer.amount, + claimType: ClaimType.DIRECT, + offerIndex: offer.offerIndex, + feedId: offer.feedId, + protocolTag: "" + FDC_PROTOCOL_ID, + rewardTypeTag: RewardTypePrefix.FDC_SIGNING, + rewardDetailTag: SigningRewardClaimType.EMPTY_BITVOTE, + }; + return [backClaim]; + } + const orderedSubmitSignatureAddresses = data.dataForCalculations.orderedVotersSubmitSignatureAddresses; const totalWeight = rewardEpochInfo.signingPolicy.weights.reduce((acc, weight) => acc + weight, 0); const signingAddressToVoter = new Map(); @@ -102,14 +119,14 @@ export function calculateSigningRewardsForFDC( undistributedAmount -= voterAmount; undistributedWeight -= BigInt(voterData.weight); const voterWeights = data.dataForCalculations.votersWeightsMap!.get(submitAddress); - if(!voterData.dominatesConsensusBitVote) { + if (!voterData.dominatesConsensusBitVote) { // burn 20% const burnAmount = 200000n * voterAmount / 1000000n; voterAmount -= burnAmount; - if(burnAmount > 0n) { + if (burnAmount > 0n) { const burnClaim: IPartialRewardClaim = { votingRoundId, - beneficiary: submitAddress.toLowerCase(), + beneficiary: offer.claimBackAddress.toLowerCase(), amount: burnAmount, claimType: ClaimType.DIRECT, offerIndex: offer.offerIndex, @@ -123,7 +140,7 @@ export function calculateSigningRewardsForFDC( } allClaims.push( ...generateSigningWeightBasedClaimsForVoter(voterAmount, offer, voterWeights, RewardTypePrefix.FDC_SIGNING, FDC_PROTOCOL_ID) - ); + ); } } diff --git a/libs/ftso-core/src/reward-calculation/reward-signing.ts b/libs/ftso-core/src/reward-calculation/reward-signing.ts index 6465bd45..142340d4 100644 --- a/libs/ftso-core/src/reward-calculation/reward-signing.ts +++ b/libs/ftso-core/src/reward-calculation/reward-signing.ts @@ -27,6 +27,7 @@ export enum SigningRewardClaimType { NO_TIMELY_FINALIZATION = "NO_TIMELY_FINALIZATION", CLAIM_BACK_OF_NON_SIGNERS_SHARE = "CLAIM_BACK_OF_NON_SIGNERS_SHARE", NON_DOMINATING_BITVOTE = "NON_DOMINATING_BITVOTE", + EMPTY_BITVOTE = "EMPTY_BITVOTE", } /** * Given an offer and data for reward calculation it calculates signing rewards for the offer. From c7ad231d309da825cc0c5a89827dbece535812b5 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Mon, 25 Nov 2024 08:26:41 +0100 Subject: [PATCH 19/28] some fixes, interface for staking rewarding --- libs/ftso-core/src/DataManagerForRewarding.ts | 1 - .../src/IndexerClientForRewarding.ts | 8 +-- .../src/utils/interfacing/input-interfaces.ts | 61 +++++++++++++++++++ scripts/analytics/run/offer-check.ts | 17 +++--- 4 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 libs/ftso-core/src/utils/interfacing/input-interfaces.ts diff --git a/libs/ftso-core/src/DataManagerForRewarding.ts b/libs/ftso-core/src/DataManagerForRewarding.ts index 2fb8f786..6230d7cf 100644 --- a/libs/ftso-core/src/DataManagerForRewarding.ts +++ b/libs/ftso-core/src/DataManagerForRewarding.ts @@ -382,7 +382,6 @@ export class DataManagerForRewarding extends DataManager { partialFdcData = partialFdcDataResponse.data; } - /// const result: DataForRewardCalculation[] = []; let startIndexSignatures = 0; let endIndexSignatures = 0; diff --git a/libs/ftso-core/src/IndexerClientForRewarding.ts b/libs/ftso-core/src/IndexerClientForRewarding.ts index 1ae1226c..1b093a92 100644 --- a/libs/ftso-core/src/IndexerClientForRewarding.ts +++ b/libs/ftso-core/src/IndexerClientForRewarding.ts @@ -180,11 +180,11 @@ export class IndexerClientForRewarding extends IndexerClient { const endTime = EPOCH_SETTINGS().votingEpochStartSec(endVotingRoundId + 1) - 1; const eventName = IncentiveOffered.eventName; const status = await this.ensureEventRange(startTime, endTime); - const result = await this.queryEvents(CONTRACTS.FastUpdateIncentiveManager, eventName, startTime, endTime); if (status !== BlockAssuranceResult.OK) { return { status }; } + const result = await this.queryEvents(CONTRACTS.FastUpdateIncentiveManager, eventName, startTime, endTime); const data = result.map(event => IncentiveOffered.fromRawEvent(event)); return { status, @@ -227,10 +227,10 @@ export class IndexerClientForRewarding extends IndexerClient { const endTime = EPOCH_SETTINGS().votingEpochStartSec(endVotingRoundId + 1) - 1; const eventName = AttestationRequest.eventName; const status = await this.ensureEventRange(startTime, endTime); - const result = await this.queryEvents(CONTRACTS.FdcHub, eventName, startTime, endTime); if (status !== BlockAssuranceResult.OK) { return { status }; } + const result = await this.queryEvents(CONTRACTS.FdcHub, eventName, startTime, endTime); const allAttestationRequests = result.map(event => AttestationRequest.fromRawEvent(event)); const data: AttestationRequest[][] = []; @@ -262,11 +262,11 @@ export class IndexerClientForRewarding extends IndexerClient { // strictly containing in the range const endTime = EPOCH_SETTINGS().votingEpochStartSec(endVotingRoundId + 1) - 1; const eventName = FDCInflationRewardsOffered.eventName; - const status = await this.ensureEventRange(startTime, endTime); - const result = await this.queryEvents(CONTRACTS.FdcHub, eventName, startTime, endTime); + const status = await this.ensureEventRange(startTime, endTime); if (status !== BlockAssuranceResult.OK) { return { status }; } + const result = await this.queryEvents(CONTRACTS.FdcHub, eventName, startTime, endTime); const data = result.map(event => FDCInflationRewardsOffered.fromRawEvent(event)); return { status, diff --git a/libs/ftso-core/src/utils/interfacing/input-interfaces.ts b/libs/ftso-core/src/utils/interfacing/input-interfaces.ts new file mode 100644 index 00000000..83c4c0b0 --- /dev/null +++ b/libs/ftso-core/src/utils/interfacing/input-interfaces.ts @@ -0,0 +1,61 @@ +export interface Delegator { + // pChain address, like flare123l344hlugpg0r2ntdl6fn45qyp0f5m2xakc0r + pAddress: string; + // cChain address, like 0x4485B10aD3ff29066938922059c5CB1e5e8Ee8b6 + cAddress: string; + // as string in GWei + amount: string; + // as string in GWei + delegatorRewardAmount: string; +} + +export interface ValidatorInfo { + // node id in form: NodeID-2a7BPY7UeJv2njMuyUHfBSTeQCYZj6bwV + nodeId: string; + // bonding address in form: flare1uz66xddzplexwfdsxrzxsnlwlucyfsuax00crd + bondingAddress: string; + // self bond in GWei + selfBond: string; + // ftso address in form "0xfe532cB6Fb3C47940aeA7BeAd4d61C5e041D950e", + ftsoAddress: string; + // end of stake in unix time + stakeEnd: number; + // string of p-chain addresses (in form flare1uz66xddzplexwfdsxrzxsnlwlucyfsuax00crd) + pChainAddress: string[]; + // fee in GWei + fee: number; + // group number + group: number; + // is the validator eligible for staking rewards + eligible: boolean; + // data provider name + ftsoName: string; + // Boosting eligibility bond in GWei + BEB: string; + // Boost delegations in GWei + boostDelegations: string; + // boost in GWei + boost: string; + // self delegations in GWei + selfDelegations: string; + // other delegations in GWei + normalDelegations: string; + // total self bond in GWei + totalSelfBond: string; + // list of delegators + delegators: Delegator[]; + // total stake amount in GWei“ + totalStakeAmount: string; + // C-chain address in form of 0xaDEDCd23941E479b4736B38e271Eb926596BBe3d + cChainAddress: string; + // overboost in GWei + overboost: string; + // reward weight in GWei + rewardingWeight: string; + // capped weight in GWei + cappedWeight: string; + // node reward amount in wei + nodeRewardAmount: string; + // validator reward amount in wei + validatorRewardAmount: string; +} \ No newline at end of file diff --git a/scripts/analytics/run/offer-check.ts b/scripts/analytics/run/offer-check.ts index 229d6999..5d500f11 100644 --- a/scripts/analytics/run/offer-check.ts +++ b/scripts/analytics/run/offer-check.ts @@ -29,7 +29,8 @@ async function main() { fastUpdatesFunds += incentive.offerAmount; } - let fdcFunds = rewardEpochInfo.fdcInflationRewardsOffered.amount + let fdcFunds = rewardEpochInfo.fdcInflationRewardsOffered?.amount || 0n; + const noFDC = rewardEpochInfo.fdcInflationRewardsOffered === undefined; let ftsoScalingOfferAmount = 0n; let fastUpdatesOfferAmount = 0n; @@ -49,10 +50,10 @@ async function main() { } } - const offers = deserializeOffersForFDC(rewardEpochId, votingRoundId, calculationFolder); + const offers = noFDC ? [] : deserializeOffersForFDC(rewardEpochId, votingRoundId, calculationFolder); for (let offer of offers) { fdcOfferAmount += offer.amount; - if(offer.shouldBeBurned) { + if (offer.shouldBeBurned) { fdcOfferBurn += offer.amount; } } @@ -63,13 +64,15 @@ async function main() { calculationFolder ); - for (let attestationRequest of data.fdcData.attestationRequests) { - fdcFunds += attestationRequest.fee; + if (!noFDC) { + for (let attestationRequest of data.fdcData.attestationRequests) { + fdcFunds += attestationRequest.fee; + } } } - console.log(`FTSO Scaling Funds: ${ftsoScalingFunds - ftsoScalingOfferAmount}`); - console.log(`Fast Updates Funds: ${fastUpdatesFunds - fastUpdatesOfferAmount}`); + console.log(`FTSO Scaling Funds: ${ftsoScalingOfferAmount}$ {ftsoScalingFunds - ftsoScalingOfferAmount}`); + console.log(`Fast Updates Funds: ${fastUpdatesOfferAmount} ${fastUpdatesFunds - fastUpdatesOfferAmount}`); console.log(`FDC Funds: ${fdcFunds - fdcOfferAmount}`); console.log(`FDC Offer Burn: ${fdcOfferBurn}`); } From e8b0eeb17b46763dbb29089f17b2396a18561bd0 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Mon, 25 Nov 2024 08:45:31 +0100 Subject: [PATCH 20/28] getting staking files --- .gitignore | 3 +- .../src/utils/interfacing/input-interfaces.ts | 7 ++++ scripts/get-staking-data.sh | 4 ++ scripts/rewards/songbird-db-local.sh | 42 +++++++++++++++++++ scripts/rewards/songbird-db.sh | 2 +- 5 files changed, 56 insertions(+), 2 deletions(-) create mode 100755 scripts/get-staking-data.sh create mode 100755 scripts/rewards/songbird-db-local.sh diff --git a/.gitignore b/.gitignore index 3d477baa..657425af 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ coverage .DS_Store calculations test-db -exports-csv \ No newline at end of file +exports-csv +staking-data \ No newline at end of file diff --git a/libs/ftso-core/src/utils/interfacing/input-interfaces.ts b/libs/ftso-core/src/utils/interfacing/input-interfaces.ts index 83c4c0b0..57cf34dc 100644 --- a/libs/ftso-core/src/utils/interfacing/input-interfaces.ts +++ b/libs/ftso-core/src/utils/interfacing/input-interfaces.ts @@ -1,3 +1,5 @@ +import { readFileSync } from "fs"; + export interface Delegator { // pChain address, like flare123l344hlugpg0r2ntdl6fn45qyp0f5m2xakc0r pAddress: string; @@ -58,4 +60,9 @@ export interface ValidatorInfo { nodeRewardAmount: string; // validator reward amount in wei validatorRewardAmount: string; +} + +export function readStakingInfo(fname: string): ValidatorInfo[] { + const data = readFileSync(fname, 'utf8'); + return JSON.parse(data); } \ No newline at end of file diff --git a/scripts/get-staking-data.sh b/scripts/get-staking-data.sh new file mode 100755 index 00000000..9fe26a10 --- /dev/null +++ b/scripts/get-staking-data.sh @@ -0,0 +1,4 @@ +wget "https://raw.githubusercontent.com/flare-foundation/reward-scripts/refs/heads/main/generated-files/reward-epoch-$1/nodes-data.json" +mkdir -p staking-data +rm -f staking-data/$1-nodes-data.json +mv nodes-data.json staking-data/$1-nodes-data.json \ No newline at end of file diff --git a/scripts/rewards/songbird-db-local.sh b/scripts/rewards/songbird-db-local.sh new file mode 100755 index 00000000..7d684e63 --- /dev/null +++ b/scripts/rewards/songbird-db-local.sh @@ -0,0 +1,42 @@ +# Reward calculation for Songbird network +# Setup the correct DB connection and run the script, e.g. +# ./scripts/rewards/songbird-db.sh + +export NETWORK=songbird + +export DB_REQUIRED_INDEXER_HISTORY_TIME_SEC=86400 +export VOTING_ROUND_HISTORY_SIZE=10000 +export INDEXER_TOP_TIMEOUT=1000 +export DB_HOST=127.0.0.1 +export DB_PORT=3306 +export DB_USERNAME=ftso-indxr-sgb-rdr +export DB_PASSWORD=$(gcloud secrets versions access latest --project flare-network-production --secret="ftso_v2_c_chain_indexer_sgb_db_reader_password") +export DB_NAME=flare_ftso_indexer_songbird + +export REMOVE_ANNOYING_MESSAGES=true + + +# check here: https://songbird-explorer.flare.network/address/0x421c69E22f48e14Fc2d2Ee3812c59bfb81c38516/read-contract#address-tabs +# 9. getCurrentRewardEpochId, and use one epoch less for test (required indexer history is 4 epochs/ 14 days) +# export REWARD_EPOCH_ID=2349 + +# COMPILATION +yarn nest build ftso-reward-calculation-process + +# --------------------------------------------------------------------------------------------------------------------------- +# Calculating all reward data from the starting reward epoch id. The calculation of claims is parallelized. +# In the current (ongoing) reward epoch the calculation is switched to incremental, as data becomes available. +# If the data for a specific reward epoch id is already available, the calculation is skipped. +export FROM_REWARD_EPOCH_ID=196 +node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -g -o -c -a -y -b 100 -w 10 -d $FROM_REWARD_EPOCH_ID -m 10000 + +# Incremental calculation +# node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -l -b 80 -w 5 -m 10000 + +# --------------------------------------------------------------------------------------------------------------------------- +# Recoverable sequential calculation +# node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -i -a -c -o -d $FROM_REWARD_EPOCH_ID -m 10000 +# --------------------------------------------------------------------------------------------------------------------------- +# Calculating for specific reward epoch id +# export SPECIFIC_REWARD_EPOCH_ID=2380 +# node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -i -a -c -b 10 -w 24 -r $SPECIFIC_REWARD_EPOCH_ID -m 10000 \ No newline at end of file diff --git a/scripts/rewards/songbird-db.sh b/scripts/rewards/songbird-db.sh index dd1ec724..3d8e894f 100755 --- a/scripts/rewards/songbird-db.sh +++ b/scripts/rewards/songbird-db.sh @@ -8,7 +8,7 @@ export DB_REQUIRED_INDEXER_HISTORY_TIME_SEC=86400 export VOTING_ROUND_HISTORY_SIZE=10000 export INDEXER_TOP_TIMEOUT=1000 export DB_HOST=127.0.0.1 -export DB_PORT=3336 +export DB_PORT=3337 export DB_USERNAME=root export DB_PASSWORD=root export DB_NAME=flare_ftso_indexer From f8d77921ce5608346574db1368c401e8810ae38f Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Fri, 29 Nov 2024 08:30:27 +0100 Subject: [PATCH 21/28] aux functions --- libs/ftso-core/src/configs/networks.ts | 50 ++++++++++- .../src/utils/interfacing/input-interfaces.ts | 90 +++++++++++++++++-- package.json | 1 + scripts/get-staking-data.sh | 5 +- yarn.lock | 36 ++------ 5 files changed, 146 insertions(+), 36 deletions(-) diff --git a/libs/ftso-core/src/configs/networks.ts b/libs/ftso-core/src/configs/networks.ts index d25b0823..32907d3e 100644 --- a/libs/ftso-core/src/configs/networks.ts +++ b/libs/ftso-core/src/configs/networks.ts @@ -716,6 +716,54 @@ export const COSTON_FAST_UPDATER_SWITCH_VOTING_ROUND_ID = 779191; // set to start voting round id of epoch 234 // on Songbird there was no missing event for the voting round // Only used to filter out the old events -export const SONGBIRD_FAST_UPDATER_SWITCH_VOTING_ROUND_ID = 786240; +export const SONGBIRD_FAST_UPDATER_SWITCH_VOTING_ROUND_ID = 786240; export const WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH = "WRONG_SIGNATURE"; + +export const STAKING_DATA_BASE_FOLDER = "staking-data"; + +export const STAKING_DATA_FOLDER = () => { + const network = process.env.NETWORK as networks; + const STAKING_DATA_BASE_FOLDER = "staking-data"; + switch (network) { + case "from-env": + return `${STAKING_DATA_BASE_FOLDER}/from-env`; + case "coston": + return `${STAKING_DATA_BASE_FOLDER}/coston`; + case "coston2": + return `${STAKING_DATA_BASE_FOLDER}/coston2`; + case "songbird": + return `${STAKING_DATA_BASE_FOLDER}/songbird`; + case "flare": + return `${STAKING_DATA_BASE_FOLDER}/flare`; + case "local-test": + return `${STAKING_DATA_BASE_FOLDER}/local-test`; + default: + // Ensure exhaustive checking + // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars + ((_: never): void => { })(network); + } +}; + +export const PASSES_DATA_FOLDER = () => { + const network = process.env.NETWORK as networks; + const PASSES_DATA_BASE_FOLDER = "passes-data"; + switch (network) { + case "from-env": + return `${PASSES_DATA_BASE_FOLDER}/from-env`; + case "coston": + return `${PASSES_DATA_BASE_FOLDER}/coston`; + case "coston2": + return `${PASSES_DATA_BASE_FOLDER}/coston2`; + case "songbird": + return `${PASSES_DATA_BASE_FOLDER}/songbird`; + case "flare": + return `${PASSES_DATA_BASE_FOLDER}/flare`; + case "local-test": + return `${PASSES_DATA_BASE_FOLDER}/local-test`; + default: + // Ensure exhaustive checking + // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars + ((_: never): void => { })(network); + } +}; \ No newline at end of file diff --git a/libs/ftso-core/src/utils/interfacing/input-interfaces.ts b/libs/ftso-core/src/utils/interfacing/input-interfaces.ts index 57cf34dc..bd92783e 100644 --- a/libs/ftso-core/src/utils/interfacing/input-interfaces.ts +++ b/libs/ftso-core/src/utils/interfacing/input-interfaces.ts @@ -1,4 +1,7 @@ -import { readFileSync } from "fs"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; +import { base58 } from '@scure/base'; +import path from "path/posix"; +import { PASSES_DATA_FOLDER, STAKING_DATA_FOLDER } from "../../configs/networks"; export interface Delegator { // pChain address, like flare123l344hlugpg0r2ntdl6fn45qyp0f5m2xakc0r @@ -19,7 +22,7 @@ export interface ValidatorInfo { // self bond in GWei selfBond: string; // ftso address in form "0xfe532cB6Fb3C47940aeA7BeAd4d61C5e041D950e", - ftsoAddress: string; + ftsoAddress: string; // end of stake in unix time stakeEnd: number; // string of p-chain addresses (in form flare1uz66xddzplexwfdsxrzxsnlwlucyfsuax00crd) @@ -49,7 +52,7 @@ export interface ValidatorInfo { // total stake amount in GWei“ totalStakeAmount: string; // C-chain address in form of 0xaDEDCd23941E479b4736B38e271Eb926596BBe3d - cChainAddress: string; + cChainAddress: string; // overboost in GWei overboost: string; // reward weight in GWei @@ -60,9 +63,86 @@ export interface ValidatorInfo { nodeRewardAmount: string; // validator reward amount in wei validatorRewardAmount: string; + // Node id as 20-byte hex string + nodeId20Byte?: string; } -export function readStakingInfo(fname: string): ValidatorInfo[] { +export enum MinimalConditionFailureType { + // Providers must submit a value estimate that lies within a 0.5% band around the consensus median value + // in 80% of voting rounds within a reward epoch. + FTSO_SCALING_FAILURE = "FTSO_SCALING_FAILURE", + // Providers must submit at least 80% of their expected number of updates within a reward epoch, + // unless they have very low weight, defined as < 0.2% of the total active weight. + FAST_UPDATES_FAILURE = "FAST_UPDATES_FAILURE", + // Providers must meet 80% total uptime in the reward epoch with at least 1M FLR in active self-bond. + // However, in order to earn passes, the provider must have at least 3M FLR in active self-bond and 15M + // in active stake. Providers with 80% total uptime and at least 1M FLR in active self-bond but + // not meeting both the 3M FLR active self-bond and 15M active stake requirements neither earn + // nor lose passes, and still receive eligible rewards. + STAKING_FAILURE = "STAKING_AVAILABILITY", +} + +export interface MinimalConditionFailure { + // protocol id + protocolId: number; + // failure id + failureId: MinimalConditionFailureType; +} + +export interface DataProviderPasses { + // epoch id in string + rewardEpochId: string; + // voter identity address in lowercase + voterAddress: string; + // number of passes. A number between 0 and 3 + passes: number; + // failures + failures?: MinimalConditionFailure[]; +} + +/** + * Reads the staking info for a given reward epoch id. + * The data is stored in the staking data folder. + */ +export function readStakingInfo( + rewardEpochId: number, + stakingDataFolder = STAKING_DATA_FOLDER() +): ValidatorInfo[] { + const fname = path.join(stakingDataFolder, `${rewardEpochId}-nodes-data.json`); + const data = readFileSync(fname, 'utf8'); + const result: ValidatorInfo[] = JSON.parse(data); + for(let validatorInfo of result) { + // "NodeID-2a7BPY7UeJv2njMuyUHfBSTeQCYZj6bwV" + // Checksum is not validated + validatorInfo.nodeId20Byte = Buffer.from(base58.decode(validatorInfo.nodeId.slice(7)).subarray(0, -4)).toString("hex"); + } + return result; +} + +/** + * Reads the passes info for a given reward epoch id. + * The data is stored in the passes data folder. + */ +export function readPassesInfo( + rewardEpochId: number, + passesDataFolder = PASSES_DATA_FOLDER() +): DataProviderPasses[] { + const fname = path.join(passesDataFolder, `${rewardEpochId}-passes-data.json`); const data = readFileSync(fname, 'utf8'); return JSON.parse(data); -} \ No newline at end of file +} + +/** + * Writes the staking info for a given reward epoch id. + */ +export function writePassesInfo( + rewardEpochId: number, + data: DataProviderPasses, + passesDataFolder = PASSES_DATA_FOLDER() +): void { + if (!existsSync(passesDataFolder)) { + mkdirSync(passesDataFolder, { recursive: true }); + } + const fname = path.join(passesDataFolder, `${rewardEpochId}-passes-data.json`); + writeFileSync(fname, JSON.stringify(data)); +} diff --git a/package.json b/package.json index 01777201..bb279c18 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "@nestjs/platform-express": "^10.0.0", "@nestjs/swagger": "^7.1.17", "@nestjs/typeorm": "^10.0.1", + "@scure/base": "^1.2.1", "axios": "^1.6.5", "class-transformer": "^0.5.1", "ethers": "^6.6.0", diff --git a/scripts/get-staking-data.sh b/scripts/get-staking-data.sh index 9fe26a10..51859fe6 100755 --- a/scripts/get-staking-data.sh +++ b/scripts/get-staking-data.sh @@ -1,4 +1,5 @@ wget "https://raw.githubusercontent.com/flare-foundation/reward-scripts/refs/heads/main/generated-files/reward-epoch-$1/nodes-data.json" mkdir -p staking-data -rm -f staking-data/$1-nodes-data.json -mv nodes-data.json staking-data/$1-nodes-data.json \ No newline at end of file +mkdir -p staking-data/flare +rm -f staking-data/flare/$1-nodes-data.json +mv nodes-data.json staking-data/flare/$1-nodes-data.json \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 6244d00f..fc59e55c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -632,6 +632,11 @@ resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== +"@scure/base@^1.2.1": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@scure/base/-/base-1.2.1.tgz#dd0b2a533063ca612c17aa9ad26424a2ff5aa865" + integrity sha512-DGmGtC8Tt63J5GfHgfl5CuAXh96VF/LD8K9Hr/Gv0J2lAoRGlPOMpqMpMbCTOoOJMZCk2Xt+DskdDyn6dEFdzQ== + "@scure/base@~1.1.4": version "1.1.5" resolved "https://registry.yarnpkg.com/@scure/base/-/base-1.1.5.tgz#1d85d17269fe97694b9c592552dd9e5e33552157" @@ -5563,16 +5568,7 @@ streamsearch@^1.1.0: resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764" integrity sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg== -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -string-width@4.2.3, "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3, string-width@^5.0.1, string-width@^5.1.2: +"string-width-cjs@npm:string-width@^4.2.0", string-width@4.2.3, "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3, string-width@^5.0.1, string-width@^5.1.2: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -5595,14 +5591,7 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -6486,7 +6475,7 @@ workerpool@^9.1.0: resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-9.1.0.tgz#cb1856aaa66ee97a3dfec97e4e362365c253512b" integrity sha512-+wRWfm9yyJghvXLSHMQj3WXDxHbibHAQmRrWbqKBfy0RjftZNeQaW+Std5bSYc83ydkrxoPTPOWVlXUR9RWJdQ== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -6504,15 +6493,6 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From 837ef6c8ced4b992af593eeac2c1a174007aa557 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Mon, 2 Dec 2024 08:47:52 +0100 Subject: [PATCH 22/28] minimal conditions wip --- .../minimal-conditions/minimal-conditions.ts | 238 ++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts diff --git a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts new file mode 100644 index 00000000..cf254c76 --- /dev/null +++ b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts @@ -0,0 +1,238 @@ +import { readStakingInfo } from "../../utils/interfacing/input-interfaces"; +import { deserializeDataForRewardCalculation } from "../../utils/stat-info/reward-calculation-data"; +import { deserializeRewardEpochInfo } from "../../utils/stat-info/reward-epoch-info"; + +const TOTAL_PPM = 1000000n; +const FTSO_SCALING_AVAILABILITY_THRESHOLD_PPM = 800000n; // 80% +const FTSO_SCALING_CLOSENESS_THRESHOLD_PPM = 5000n; // 0.5% +const FU_THRESHOLD_PPM = 800000n; // 60% +const FU_CONSIDERATION_THRESHOLD_PPM = 2000n; // 0.2% of the weight +const STAKING_UPTIME_THRESHOLD_PPM = 800000n; // 80% + +export interface ConditionSummary { + // whether a minimal condition is met + conditionMet: boolean; + // allows for a pass to be earned, if condition is met + // if any criteria has this flag on true, the pass cannot be earned + obstructsPass?: boolean; +} + +export interface FeedHits { + // feed id + feedId: string; + // out of 3360 + hits: number; +} + +export interface FtsoScalingConditionSummary extends ConditionSummary { + totalHits: number; + feedHits: FeedHits[]; +} + +export interface FUConditionSummary extends ConditionSummary { + // total updates by all providers in the reward epoch + totalUpdatesByAll: number; + // updates by the provider in the reward epoch + updates: number; + //exempt due to low weight + tooLowWeight: boolean; + // expected PPM share based on the share of the weight + expectedUpdatesPPM: bigint; +} + +export interface NodeStakingConditions { + // node id as 20 byte hex string + nodeId: string; + // uptime sufficient + uptimeOk: boolean; + // self bond in GWei + selfBond: bigint; + // mirrored stake in GWei + totalMirroredStake: bigint; + // total stake amount in GWei + totalStakeAmount: bigint; +} + +export interface StakingConditionSummary extends ConditionSummary { + // total self bond in Gwei + totalSelfBond: bigint; + // total stake amount in Gwei + stake: bigint; + // stake with uptime + stakeWithUptime: bigint; + // node conditions + nodeConditions: NodeStakingConditions[]; +} + +export interface DataProviderConditions { + // voter identity address + voterAddress: string; + // voter index + voterIndex: number; + // passes held + passesHeld: number; + // strikes + strikes: number; + // ftso scaling conditions + ftsoScaling: FtsoScalingConditionSummary; + // fast update conditions + fastUpdates: FUConditionSummary; + // staking conditions + staking: StakingConditionSummary; +} + +export function calculateMinimalConditions( + rewardEpochId: number, +): DataProviderConditions[] { + const rewardEpochInfo = deserializeRewardEpochInfo(rewardEpochId); + const submitAddressToVoter = new Map(); + const signingPolicyAddressToVoter = new Map(); + const voterToVoterIndex = new Map(); + const voterToFtsoScalingConditionSummary = new Map(); + const voterToFUConditionSummary = new Map(); + const voterToStakingConditionSummary = new Map(); + const nodeIdToVoter = new Map(); + const nodeIdToNodeStakingCondition = new Map(); + const submitAddressToFeedToHits = new Map>(); + + const totalSigningWeight = rewardEpochInfo.signingPolicy.weights.reduce((acc, weight) => acc + weight, 0); + + for (let i = 0; i < rewardEpochInfo.voterRegistrationInfo.length; i++) { + const voter = rewardEpochInfo.voterRegistrationInfo[i].voterRegistered.voter.toLowerCase(); + const signingWeight = rewardEpochInfo.signingPolicy.weights[i]; + const submissionAddress = rewardEpochInfo.voterRegistrationInfo[i].voterRegistered.submitAddress.toLowerCase(); + const signingPolicyAddress = rewardEpochInfo.voterRegistrationInfo[i].voterRegistered.signingPolicyAddress.toLowerCase(); + submitAddressToVoter.set(submissionAddress, voter); + signingPolicyAddressToVoter.set(signingPolicyAddress, voter); + voterToVoterIndex.set(voter, i); + const ftsoScalingConditionSummary: FtsoScalingConditionSummary = { + conditionMet: false, + totalHits: 0, + feedHits: [], + } + voterToFtsoScalingConditionSummary.set(voter, ftsoScalingConditionSummary); + const expectedUpdatesPPM = (BigInt(signingWeight) * TOTAL_PPM) / BigInt(totalSigningWeight), + const fuConditionSummary: FUConditionSummary = { + conditionMet: false, + totalUpdatesByAll: 0, + updates: 0, + expectedUpdatesPPM, + tooLowWeight: expectedUpdatesPPM < FU_CONSIDERATION_THRESHOLD_PPM, + } + voterToFUConditionSummary.set(voter, fuConditionSummary); + const stakingConditionSummary: StakingConditionSummary = { + conditionMet: false, + totalSelfBond: 0n, + stake: 0n, + stakeWithUptime: 0n, + nodeConditions: [], + } + + voterToStakingConditionSummary.set(voter, stakingConditionSummary); + for (let j = 0; j < rewardEpochInfo.voterRegistrationInfo[i].voterRegistrationInfo.nodeIds.length; j++) { + const nodeId = rewardEpochInfo.voterRegistrationInfo[i].voterRegistrationInfo.nodeIds[j]; + const stake = rewardEpochInfo.voterRegistrationInfo[i].voterRegistrationInfo.nodeWeights[j]; + // will be updated later from staking file + const nodeCondition: NodeStakingConditions = { + nodeId: nodeId, + uptimeOk: false, + selfBond: 0n, + totalMirroredStake: stake, + totalStakeAmount: 0n, + } + stakingConditionSummary.nodeConditions.push(nodeCondition); + nodeIdToVoter.set(nodeId, voter); + nodeIdToNodeStakingCondition.set(nodeId, nodeCondition); + } + const voterFeedHits = new Map(); + for (let feedInfo of rewardEpochInfo.canonicalFeedOrder) { + const feedHits: FeedHits = { + feedId: feedInfo.id, + hits: 0, + } + voterFeedHits.set(feedInfo.id, feedHits); + } + submitAddressToFeedToHits.set(submissionAddress, voterFeedHits); + } + + // Reading staking info data for reward epoch and updating node conditions + const validatorInfoList = readStakingInfo(rewardEpochId); + for (const validatorInfo of validatorInfoList) { + const nodeId = validatorInfo.nodeId20Byte; + const condition = nodeIdToNodeStakingCondition.get(nodeId); + if (condition === undefined) { + // TODO: log properly + console.log(`Node ${nodeId} not found in the voter registration info`); + continue; + } + condition.selfBond = BigInt(validatorInfo.selfBond); + condition.totalStakeAmount = BigInt(validatorInfo.totalStakeAmount); + // TODO: check if this will stay so + condition.uptimeOk = validatorInfo.eligible; + } + + // Checking staking conditions + for (const [voter, stakingConditionSummary] of voterToStakingConditionSummary.entries()) { + for (const nodeCondition of stakingConditionSummary.nodeConditions) { + if (nodeCondition.uptimeOk) { + stakingConditionSummary.stakeWithUptime += nodeCondition.totalStakeAmount; + } + stakingConditionSummary.totalSelfBond += nodeCondition.selfBond; + stakingConditionSummary.stake += nodeCondition.totalStakeAmount; + } + // STAKING_UPTIME_THRESHOLD_PPM (80%) of total weight must have sufficient uptime + stakingConditionSummary.conditionMet = + TOTAL_PPM * stakingConditionSummary.stakeWithUptime >= STAKING_UPTIME_THRESHOLD_PPM * stakingConditionSummary.stake; + } + + // Processing by voting rounds + let totalFUUpdates = 0; + for (let votingRoundId = rewardEpochInfo.signingPolicy.startVotingRoundId; votingRoundId <= rewardEpochInfo.endVotingRoundId; votingRoundId++) { + const rewardCalculationData = deserializeDataForRewardCalculation(rewardEpochId, votingRoundId); + + // Fast updates checks + if (!rewardCalculationData?.fastUpdatesData?.signingPolicyAddressesSubmitted) { + continue; + } + for (const signingPolicyAddress of rewardCalculationData.fastUpdatesData.signingPolicyAddressesSubmitted) { + totalFUUpdates++; + const voter = signingPolicyAddressToVoter.get(signingPolicyAddress); + if (!voter) { + // sanity check + throw new Error(`Voter not found for signing policy address ${signingPolicyAddress}`); + } + const fuConditionSummary = voterToFUConditionSummary.get(voter); + fuConditionSummary.updates++; + } + // FTSO Scaling checks + for (let feedRecord of rewardCalculationData.medianCalculationResults) { + const feedId = feedRecord.feed.id; + if (feedRecord.data.finalMedian.isEmpty) { + continue; + } + const median = feedRecord.data.finalMedian.value; + const delta = BigInt(median) * FTSO_SCALING_CLOSENESS_THRESHOLD_PPM / TOTAL_PPM; + const low = median - Number(delta); + const high = median + Number(delta); + + if (feedRecord.feedValues.length !== feedRecord.votersSubmitAddresses.length) { + // sanity check + throw new Error(`Feed values and voters submit addresses length mismatch for feed ${feedId}`); + } + for (let i = 0; i < feedRecord.feedValues.length; i++) { + // TODO: write logic + } + } + } + + // TODO: go over all data providers and check minimal conditions for fast updates + for (const [voter, fuConditionSummary] of voterToFUConditionSummary.entries()) { + fuConditionSummary.totalUpdatesByAll = totalFUUpdates; + fuConditionSummary.conditionMet = TOTAL_PPM * BigInt(fuConditionSummary.updates) >= FU_THRESHOLD_PPM * BigInt(totalFUUpdates); + } + + // TODO: go over all data providers and check minimal conditions for ftso scaling + + // TODO: assemble all DataProviderConditions for each data provider +} + From 74c33c1de5ed7ea0ebf5f1c81ba7b8ef0a947ef5 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Tue, 3 Dec 2024 22:44:15 +0100 Subject: [PATCH 23/28] minimal conditions script --- .../ftso-reward-calculator-process.command.ts | 9 + .../src/interfaces/OptionalCommandOptions.ts | 1 + .../src/services/calculator.service.ts | 14 ++ .../minimal-conditions}/input-interfaces.ts | 26 +- .../minimal-conditions-constants.ts | 21 ++ .../minimal-conditions-interfaces.ts | 84 +++++++ .../minimal-conditions/minimal-conditions.ts | 238 +++++++++++------- scripts/rewards/flare-min-conditions.sh | 38 +++ 8 files changed, 338 insertions(+), 93 deletions(-) rename libs/ftso-core/src/{utils/interfacing => reward-calculation/minimal-conditions}/input-interfaces.ts (83%) create mode 100644 libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-constants.ts create mode 100644 libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts create mode 100755 scripts/rewards/flare-min-conditions.sh diff --git a/apps/ftso-reward-calculation-process/src/commands/ftso-reward-calculator-process.command.ts b/apps/ftso-reward-calculation-process/src/commands/ftso-reward-calculator-process.command.ts index c3ef8be9..dbe21916 100644 --- a/apps/ftso-reward-calculation-process/src/commands/ftso-reward-calculator-process.command.ts +++ b/apps/ftso-reward-calculation-process/src/commands/ftso-reward-calculator-process.command.ts @@ -173,4 +173,13 @@ export class FtsoRewardCalculationProcessCommand extends CommandRunner { parseIncrementalCalculation(val: string): boolean { return JSON.parse(val); } + + @Option({ + flags: "-f, --minimalConditions [number]", + description: "process minimal conditions", + }) + parseMinimalConditions(val: string): boolean { + return JSON.parse(val); + } + } diff --git a/apps/ftso-reward-calculation-process/src/interfaces/OptionalCommandOptions.ts b/apps/ftso-reward-calculation-process/src/interfaces/OptionalCommandOptions.ts index afa5de67..fa36fb2d 100644 --- a/apps/ftso-reward-calculation-process/src/interfaces/OptionalCommandOptions.ts +++ b/apps/ftso-reward-calculation-process/src/interfaces/OptionalCommandOptions.ts @@ -23,4 +23,5 @@ export interface OptionalCommandOptions { useFDCData?: boolean; tempRewardEpochFolder?: boolean; incrementalCalculation?: boolean; + minimalConditions?: boolean; } diff --git a/apps/ftso-reward-calculation-process/src/services/calculator.service.ts b/apps/ftso-reward-calculation-process/src/services/calculator.service.ts index cb2a44f6..b45d238f 100644 --- a/apps/ftso-reward-calculation-process/src/services/calculator.service.ts +++ b/apps/ftso-reward-calculation-process/src/services/calculator.service.ts @@ -46,6 +46,8 @@ import { runCalculateRewardClaimsTopJob } from "../libs/reward-claims-calculatio import { runCalculateRewardCalculationTopJob } from "../libs/reward-data-calculation"; import { getIncrementalCalculationsFeedSelections, serializeIncrementalCalculationsFeedSelections } from "../../../../libs/ftso-core/src/utils/stat-info/incremental-calculation-temp-selected-feeds"; import { calculateAttestationTypeAppearances } from "../libs/attestation-type-appearances"; +import { calculateMinimalConditions } from "../../../../libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions"; +import { writeDataProviderConditions } from "../../../../libs/ftso-core/src/reward-calculation/minimal-conditions/input-interfaces"; if (process.env.FORCE_NOW) { const newNow = parseInt(process.env.FORCE_NOW) * 1000; @@ -293,6 +295,14 @@ export class CalculatorService { await this.fullRoundAggregateClaims(options); } } + + processMinimalConditions(options: OptionalCommandOptions): void { + if (options.rewardEpochId === undefined) { + throw new Error("Reward epoch id is required for minimal conditions calculation"); + } + const result = calculateMinimalConditions(options.rewardEpochId, false); + writeDataProviderConditions(options.rewardEpochId, result); + } /** * Returns a list of all (merged) reward claims for the given reward epoch. * Calculation can be quite intensive. @@ -300,6 +310,10 @@ export class CalculatorService { async run(options: OptionalCommandOptions): Promise { const logger = new Logger(); logger.log(options); + if (options.minimalConditions) { + this.processMinimalConditions(options); + return; + } if (options.rewardEpochId !== undefined) { await this.processOneRewardEpoch(options); return; diff --git a/libs/ftso-core/src/utils/interfacing/input-interfaces.ts b/libs/ftso-core/src/reward-calculation/minimal-conditions/input-interfaces.ts similarity index 83% rename from libs/ftso-core/src/utils/interfacing/input-interfaces.ts rename to libs/ftso-core/src/reward-calculation/minimal-conditions/input-interfaces.ts index bd92783e..7854178f 100644 --- a/libs/ftso-core/src/utils/interfacing/input-interfaces.ts +++ b/libs/ftso-core/src/reward-calculation/minimal-conditions/input-interfaces.ts @@ -1,7 +1,9 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; import { base58 } from '@scure/base'; import path from "path/posix"; -import { PASSES_DATA_FOLDER, STAKING_DATA_FOLDER } from "../../configs/networks"; +import { CALCULATIONS_FOLDER, PASSES_DATA_FOLDER, STAKING_DATA_FOLDER } from "../../configs/networks"; +import { DataProviderConditions } from "./minimal-conditions-interfaces"; +import { bigIntReplacer } from "../../utils/big-number-serialization"; export interface Delegator { // pChain address, like flare123l344hlugpg0r2ntdl6fn45qyp0f5m2xakc0r @@ -114,7 +116,7 @@ export function readStakingInfo( for(let validatorInfo of result) { // "NodeID-2a7BPY7UeJv2njMuyUHfBSTeQCYZj6bwV" // Checksum is not validated - validatorInfo.nodeId20Byte = Buffer.from(base58.decode(validatorInfo.nodeId.slice(7)).subarray(0, -4)).toString("hex"); + validatorInfo.nodeId20Byte = "0x" + Buffer.from(base58.decode(validatorInfo.nodeId.slice(7)).subarray(0, -4)).toString("hex"); } return result; } @@ -126,8 +128,11 @@ export function readStakingInfo( export function readPassesInfo( rewardEpochId: number, passesDataFolder = PASSES_DATA_FOLDER() -): DataProviderPasses[] { +): DataProviderPasses[] | undefined { const fname = path.join(passesDataFolder, `${rewardEpochId}-passes-data.json`); + if(!existsSync(fname)) { + return undefined; + } const data = readFileSync(fname, 'utf8'); return JSON.parse(data); } @@ -146,3 +151,18 @@ export function writePassesInfo( const fname = path.join(passesDataFolder, `${rewardEpochId}-passes-data.json`); writeFileSync(fname, JSON.stringify(data)); } + +/** + * Writes the staking info for a given reward epoch id. + */ +export function writeDataProviderConditions( + rewardEpochId: number, + data: DataProviderConditions[], + calculationFolder = CALCULATIONS_FOLDER() +): void { + if (!existsSync(calculationFolder)) { + mkdirSync(calculationFolder, { recursive: true }); + } + const fname = path.join(calculationFolder, `${rewardEpochId}`, `minimal-conditions.json`); + writeFileSync(fname, JSON.stringify(data, bigIntReplacer)); +} diff --git a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-constants.ts b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-constants.ts new file mode 100644 index 00000000..905ffca6 --- /dev/null +++ b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-constants.ts @@ -0,0 +1,21 @@ +// FTSO anchor feeds: Providers must submit a value estimate that lies within a 0.5% band around the consensus median value in 80% +// of voting rounds within a reward epoch. +// +// FTSO block-latency feeds: Providers must submit at least 80% of their expected number of updates within a reward epoch, +// unless they have very low weight, defined as < 0.2% of the total active weight. +// +// Staking: Providers must meet 80% total uptime in the reward epoch with at least 1M FLR in active self-bond. +// However, in order to earn passes, the provider must have at least 3M FLR in active self-bond and 15M in active stake. +// Providers with 80% total uptime and at least 1M FLR in active self-bond but not meeting both the 3M FLR active self-bond +// and 15M active stake requirements neither earn nor lose passes, and still receive eligible rewards. + +export const TOTAL_PPM = 1000000n; +export const FTSO_SCALING_AVAILABILITY_THRESHOLD_PPM = 800000n; // 80% +export const FTSO_SCALING_CLOSENESS_THRESHOLD_PPM = 5000n; // 0.5% +export const FU_THRESHOLD_PPM = 800000n; // 60% +export const FU_CONSIDERATION_THRESHOLD_PPM = 2000n; // 0.2% of the weight +export const STAKING_UPTIME_THRESHOLD_PPM = 800000n; // 80% +export const STAKING_MIN_SELF_BOND_GWEI = 1000000000000000n; // 1M FLR +export const STAKING_MIN_DESIRED_SELF_BOND_GWEI = 3000000000000000n; // 3M FLR +export const STAKING_MIN_DESIRED_STAKE_GWEI = 15000000000000000n; // 15M FLR +export const MAX_NUMBER_OF_PASSES = 3; // 3 passes diff --git a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts new file mode 100644 index 00000000..47406a90 --- /dev/null +++ b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts @@ -0,0 +1,84 @@ + +export interface ConditionSummary { + // whether a minimal condition is met + conditionMet: boolean; + // allows for a pass to be earned, if condition is met + // if any criteria has this flag on true, the pass cannot be earned + obstructsPass?: boolean; +} + +export interface FeedHits { + // feed id + feedId: string; + // hits out of totalHits + feedHits: number; + // all feed hits + totalHits: number; +} + +export interface FtsoScalingConditionSummary extends ConditionSummary { + // total number of feed values, equals number of voting rounds in the reward epoch times number of feeds + allPossibleHits: number; + // stat telling how many feed values were not empty + totalHits: number; + // feed hit stats for each feed in the canonical feed order + feedHits: FeedHits[]; +} + +export interface FUConditionSummary extends ConditionSummary { + // total updates by all providers in the reward epoch + totalUpdatesByAll: number; + // updates by the provider in the reward epoch + updates: number; + //exempt due to low weight + tooLowWeight: boolean; + // expected PPM share based on the share of the weight + expectedUpdatesPPM: bigint; +} + +export interface NodeStakingConditions { + // node id as 20 byte hex string + nodeId: string; + // uptime sufficient + uptimeOk: boolean; + // self bond in GWei + selfBond: bigint; + // mirrored stake in GWei + totalMirroredStake: bigint; + // total stake amount in GWei + totalStakeAmount: bigint; +} + +export interface StakingConditionSummary extends ConditionSummary { + // total self bond in Gwei + totalSelfBond: bigint; + // total stake amount in Gwei + stake: bigint; + // stake with uptime + stakeWithUptime: bigint; + // node conditions + nodeConditions: NodeStakingConditions[]; +} + +export interface DataProviderConditions { + // voter identity address + voterAddress: string; + // voter index + voterIndex: number; + // passes held + passesHeld: number; + // strikes + strikes: number; + // pass earned + passEarned: boolean; + // eligible for reward + eligibleForReward: boolean; + // new number of passes after the reward epoch calculation + newNumberOfPasses: number; + // ftso scaling conditions + ftsoScaling: FtsoScalingConditionSummary; + // fast update conditions + fastUpdates: FUConditionSummary; + // staking conditions + staking: StakingConditionSummary; +} diff --git a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts index cf254c76..07792757 100644 --- a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts +++ b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts @@ -1,91 +1,41 @@ -import { readStakingInfo } from "../../utils/interfacing/input-interfaces"; +import { DataProviderPasses, readPassesInfo, readStakingInfo } from "./input-interfaces"; import { deserializeDataForRewardCalculation } from "../../utils/stat-info/reward-calculation-data"; import { deserializeRewardEpochInfo } from "../../utils/stat-info/reward-epoch-info"; - -const TOTAL_PPM = 1000000n; -const FTSO_SCALING_AVAILABILITY_THRESHOLD_PPM = 800000n; // 80% -const FTSO_SCALING_CLOSENESS_THRESHOLD_PPM = 5000n; // 0.5% -const FU_THRESHOLD_PPM = 800000n; // 60% -const FU_CONSIDERATION_THRESHOLD_PPM = 2000n; // 0.2% of the weight -const STAKING_UPTIME_THRESHOLD_PPM = 800000n; // 80% - -export interface ConditionSummary { - // whether a minimal condition is met - conditionMet: boolean; - // allows for a pass to be earned, if condition is met - // if any criteria has this flag on true, the pass cannot be earned - obstructsPass?: boolean; -} - -export interface FeedHits { - // feed id - feedId: string; - // out of 3360 - hits: number; -} - -export interface FtsoScalingConditionSummary extends ConditionSummary { - totalHits: number; - feedHits: FeedHits[]; -} - -export interface FUConditionSummary extends ConditionSummary { - // total updates by all providers in the reward epoch - totalUpdatesByAll: number; - // updates by the provider in the reward epoch - updates: number; - //exempt due to low weight - tooLowWeight: boolean; - // expected PPM share based on the share of the weight - expectedUpdatesPPM: bigint; -} - -export interface NodeStakingConditions { - // node id as 20 byte hex string - nodeId: string; - // uptime sufficient - uptimeOk: boolean; - // self bond in GWei - selfBond: bigint; - // mirrored stake in GWei - totalMirroredStake: bigint; - // total stake amount in GWei - totalStakeAmount: bigint; -} - -export interface StakingConditionSummary extends ConditionSummary { - // total self bond in Gwei - totalSelfBond: bigint; - // total stake amount in Gwei - stake: bigint; - // stake with uptime - stakeWithUptime: bigint; - // node conditions - nodeConditions: NodeStakingConditions[]; -} - -export interface DataProviderConditions { - // voter identity address - voterAddress: string; - // voter index - voterIndex: number; - // passes held - passesHeld: number; - // strikes - strikes: number; - // ftso scaling conditions - ftsoScaling: FtsoScalingConditionSummary; - // fast update conditions - fastUpdates: FUConditionSummary; - // staking conditions - staking: StakingConditionSummary; -} +import { + FTSO_SCALING_AVAILABILITY_THRESHOLD_PPM, + FTSO_SCALING_CLOSENESS_THRESHOLD_PPM, + FU_CONSIDERATION_THRESHOLD_PPM, FU_THRESHOLD_PPM, + MAX_NUMBER_OF_PASSES, + STAKING_MIN_DESIRED_SELF_BOND_GWEI, + STAKING_MIN_DESIRED_STAKE_GWEI, + STAKING_MIN_SELF_BOND_GWEI, + STAKING_UPTIME_THRESHOLD_PPM, + TOTAL_PPM +} from "./minimal-conditions-constants"; +import { + DataProviderConditions, + FUConditionSummary, + FeedHits, + FtsoScalingConditionSummary, + NodeStakingConditions, + StakingConditionSummary +} from "./minimal-conditions-interfaces"; export function calculateMinimalConditions( rewardEpochId: number, + requirePassesFile: boolean ): DataProviderConditions[] { const rewardEpochInfo = deserializeRewardEpochInfo(rewardEpochId); + // returns undefined if the file is not found + let passesInputData = readPassesInfo(rewardEpochId - 1); + if (requirePassesFile && passesInputData === undefined) { + throw new Error(`Passes file not found for reward epoch ${rewardEpochId - 1}`); + } + if (!passesInputData) { + passesInputData = []; + } const submitAddressToVoter = new Map(); + const voterToSubmitAddress = new Map(); const signingPolicyAddressToVoter = new Map(); const voterToVoterIndex = new Map(); const voterToFtsoScalingConditionSummary = new Map(); @@ -94,30 +44,41 @@ export function calculateMinimalConditions( const nodeIdToVoter = new Map(); const nodeIdToNodeStakingCondition = new Map(); const submitAddressToFeedToHits = new Map>(); + const voterToPassesInputData = new Map(); const totalSigningWeight = rewardEpochInfo.signingPolicy.weights.reduce((acc, weight) => acc + weight, 0); + const numberOfVotingRounds = (rewardEpochInfo.endVotingRoundId - rewardEpochInfo.signingPolicy.startVotingRoundId + 1); + + for (let dataProviderPasses of passesInputData) { + let voter = dataProviderPasses.voterAddress.toLowerCase(); + voterToPassesInputData.set(voter, dataProviderPasses); + } + for (let i = 0; i < rewardEpochInfo.voterRegistrationInfo.length; i++) { const voter = rewardEpochInfo.voterRegistrationInfo[i].voterRegistered.voter.toLowerCase(); const signingWeight = rewardEpochInfo.signingPolicy.weights[i]; const submissionAddress = rewardEpochInfo.voterRegistrationInfo[i].voterRegistered.submitAddress.toLowerCase(); const signingPolicyAddress = rewardEpochInfo.voterRegistrationInfo[i].voterRegistered.signingPolicyAddress.toLowerCase(); submitAddressToVoter.set(submissionAddress, voter); + voterToSubmitAddress.set(voter, submissionAddress); signingPolicyAddressToVoter.set(signingPolicyAddress, voter); voterToVoterIndex.set(voter, i); const ftsoScalingConditionSummary: FtsoScalingConditionSummary = { + allPossibleHits: numberOfVotingRounds * rewardEpochInfo.canonicalFeedOrder.length, conditionMet: false, totalHits: 0, feedHits: [], } voterToFtsoScalingConditionSummary.set(voter, ftsoScalingConditionSummary); - const expectedUpdatesPPM = (BigInt(signingWeight) * TOTAL_PPM) / BigInt(totalSigningWeight), + const proportionUpdatesPPM = (BigInt(signingWeight) * TOTAL_PPM) / BigInt(totalSigningWeight); + const expectedUpdatesPPM = (BigInt(signingWeight) * FU_THRESHOLD_PPM) / BigInt(totalSigningWeight); const fuConditionSummary: FUConditionSummary = { conditionMet: false, totalUpdatesByAll: 0, updates: 0, expectedUpdatesPPM, - tooLowWeight: expectedUpdatesPPM < FU_CONSIDERATION_THRESHOLD_PPM, + tooLowWeight: proportionUpdatesPPM < FU_CONSIDERATION_THRESHOLD_PPM, } voterToFUConditionSummary.set(voter, fuConditionSummary); const stakingConditionSummary: StakingConditionSummary = { @@ -148,7 +109,8 @@ export function calculateMinimalConditions( for (let feedInfo of rewardEpochInfo.canonicalFeedOrder) { const feedHits: FeedHits = { feedId: feedInfo.id, - hits: 0, + feedHits: 0, + totalHits: numberOfVotingRounds } voterFeedHits.set(feedInfo.id, feedHits); } @@ -172,7 +134,7 @@ export function calculateMinimalConditions( } // Checking staking conditions - for (const [voter, stakingConditionSummary] of voterToStakingConditionSummary.entries()) { + for (const [_, stakingConditionSummary] of voterToStakingConditionSummary.entries()) { for (const nodeCondition of stakingConditionSummary.nodeConditions) { if (nodeCondition.uptimeOk) { stakingConditionSummary.stakeWithUptime += nodeCondition.totalStakeAmount; @@ -181,13 +143,19 @@ export function calculateMinimalConditions( stakingConditionSummary.stake += nodeCondition.totalStakeAmount; } // STAKING_UPTIME_THRESHOLD_PPM (80%) of total weight must have sufficient uptime - stakingConditionSummary.conditionMet = + const uptimeOk = TOTAL_PPM * stakingConditionSummary.stakeWithUptime >= STAKING_UPTIME_THRESHOLD_PPM * stakingConditionSummary.stake; + + stakingConditionSummary.conditionMet = uptimeOk && stakingConditionSummary.totalSelfBond >= STAKING_MIN_SELF_BOND_GWEI; + stakingConditionSummary.obstructsPass = + stakingConditionSummary.totalSelfBond < STAKING_MIN_DESIRED_SELF_BOND_GWEI || stakingConditionSummary.stake < STAKING_MIN_DESIRED_STAKE_GWEI; } // Processing by voting rounds let totalFUUpdates = 0; + let nonEmptyFeedValues = 0; for (let votingRoundId = rewardEpochInfo.signingPolicy.startVotingRoundId; votingRoundId <= rewardEpochInfo.endVotingRoundId; votingRoundId++) { + if (votingRoundId % 100 === 0) console.log(votingRoundId); const rewardCalculationData = deserializeDataForRewardCalculation(rewardEpochId, votingRoundId); // Fast updates checks @@ -204,6 +172,7 @@ export function calculateMinimalConditions( const fuConditionSummary = voterToFUConditionSummary.get(voter); fuConditionSummary.updates++; } + // FTSO Scaling checks for (let feedRecord of rewardCalculationData.medianCalculationResults) { const feedId = feedRecord.feed.id; @@ -211,6 +180,7 @@ export function calculateMinimalConditions( continue; } const median = feedRecord.data.finalMedian.value; + nonEmptyFeedValues++; const delta = BigInt(median) * FTSO_SCALING_CLOSENESS_THRESHOLD_PPM / TOTAL_PPM; const low = median - Number(delta); const high = median + Number(delta); @@ -220,19 +190,107 @@ export function calculateMinimalConditions( throw new Error(`Feed values and voters submit addresses length mismatch for feed ${feedId}`); } for (let i = 0; i < feedRecord.feedValues.length; i++) { - // TODO: write logic + const submitAddress = feedRecord.votersSubmitAddresses[i]; + const feedValue = feedRecord.feedValues[i]; + const feedHits = submitAddressToFeedToHits.get(submitAddress)?.get(feedId); + if (!feedHits) { + // sanity check + throw new Error(`Feed hits not found for submit address ${submitAddress} and feed ${feedId}`); + } + if (feedRecord.data.finalMedian.decimals !== feedValue.decimals) { + // sanity check + throw new Error(`Decimals mismatch for feed ${feedId}`); + } + // boundaries included + if (feedValue.value >= low && feedValue.value <= high) { + feedHits.feedHits++; + } } } } - // TODO: go over all data providers and check minimal conditions for fast updates + // go over all data providers and check minimal conditions for fast updates for (const [voter, fuConditionSummary] of voterToFUConditionSummary.entries()) { fuConditionSummary.totalUpdatesByAll = totalFUUpdates; - fuConditionSummary.conditionMet = TOTAL_PPM * BigInt(fuConditionSummary.updates) >= FU_THRESHOLD_PPM * BigInt(totalFUUpdates); + fuConditionSummary.conditionMet = fuConditionSummary.tooLowWeight || + TOTAL_PPM * BigInt(fuConditionSummary.updates) >= fuConditionSummary.expectedUpdatesPPM * BigInt(totalFUUpdates) + } + + for (const [voter, ftsoScalingConditionSummary] of voterToFtsoScalingConditionSummary.entries()) { + for (let feed of rewardEpochInfo.canonicalFeedOrder) { + const feedId = feed.id; + const feedHits = submitAddressToFeedToHits.get(voterToSubmitAddress.get(voter))?.get(feedId); + if (!feedHits) { + // sanity check + throw new Error(`Feed hits not found for voter ${voter} and feed ${feedId}`); + } + ftsoScalingConditionSummary.feedHits.push(feedHits); + ftsoScalingConditionSummary.totalHits += feedHits.feedHits; + } + ftsoScalingConditionSummary.conditionMet = + TOTAL_PPM * BigInt(ftsoScalingConditionSummary.totalHits) >= FTSO_SCALING_AVAILABILITY_THRESHOLD_PPM * BigInt(ftsoScalingConditionSummary.allPossibleHits); } - // TODO: go over all data providers and check minimal conditions for ftso scaling + const dataProviderConditions: DataProviderConditions[] = []; + for (const regInfo of rewardEpochInfo.voterRegistrationInfo) { + const voter = regInfo.voterRegistered.voter.toLowerCase(); + const ftsoScaling = voterToFtsoScalingConditionSummary.get(voter); + const fastUpdates = voterToFUConditionSummary.get(voter); + const staking = voterToStakingConditionSummary.get(voter); + const voterIndex = voterToVoterIndex.get(voter); + + if (ftsoScaling === undefined) { + throw new Error(`FTSO scaling condition summary not found for voter ${voter}`); + } + if (fastUpdates === undefined) { + throw new Error(`Fast updates condition summary not found for voter ${voter}`); + } + if (staking === undefined) { + throw new Error(`Staking condition summary not found for voter ${voter}`); + } + if (voterIndex === undefined) { + throw new Error(`Voter index not found for voter ${voter}`); + } + const passes = voterToPassesInputData.get(voter); + const passesHeld = passes?.passes ?? 0; + // assemble all DataProviderConditions for each data provider + let passEarned = false; + if (ftsoScaling.conditionMet && fastUpdates.conditionMet && staking.conditionMet && !staking.obstructsPass) { + passEarned = true; + } + let strikes = 0; + if (!ftsoScaling.conditionMet) { + strikes++; + } + if (!fastUpdates.conditionMet) { + strikes++; + } + if (!staking.conditionMet) { + strikes++; + } + + const eligibleForReward = passesHeld - strikes >= 0; + // Cannot go below zero + let newNumberOfPasses = Math.max(passesHeld - strikes, 0); + if (passEarned) { + newNumberOfPasses++; + } + newNumberOfPasses = Math.min(newNumberOfPasses, MAX_NUMBER_OF_PASSES); + const dataProviderCondition: DataProviderConditions = { + voterAddress: voter, + voterIndex: voterToVoterIndex.get(voter), + passesHeld, + passEarned, + strikes, + eligibleForReward, + newNumberOfPasses, + ftsoScaling, + fastUpdates, + staking + } + dataProviderConditions.push(dataProviderCondition); + } - // TODO: assemble all DataProviderConditions for each data provider + return dataProviderConditions; } diff --git a/scripts/rewards/flare-min-conditions.sh b/scripts/rewards/flare-min-conditions.sh new file mode 100755 index 00000000..f5329b0a --- /dev/null +++ b/scripts/rewards/flare-min-conditions.sh @@ -0,0 +1,38 @@ +# Reward calculation for Coston network +# Setup the correct DB connection and run the script, e.g. +# ./scripts/rewards/coston-db.sh + +export NETWORK=flare +export DB_REQUIRED_INDEXER_HISTORY_TIME_SEC=86400 +export VOTING_ROUND_HISTORY_SIZE=10000 +export INDEXER_TOP_TIMEOUT=1000 +export DB_HOST=127.0.0.1 +export DB_PORT=3336 +export DB_USERNAME=root +export DB_PASSWORD=root +export DB_NAME=flare_ftso_indexer + +export REMOVE_ANNOYING_MESSAGES=true +export ALLOW_IDENTITY_ADDRESS_SIGNING=true +# Start reward epoch id on Coston +export START_REWARD_EPOCH_ID=2344 + +# COMPILATION +yarn nest build ftso-reward-calculation-process + +# --------------------------------------------------------------------------------------------------------------------------- +# Calculating all reward data from the starting reward epoch id. The calculation of claims is parallelized. +# In the current (ongoing) reward epoch the calculation is switched to incremental, as data becomes available. +# If the data for a specific reward epoch id is already available, the calculation is skipped. +export FROM_REWARD_EPOCH_ID=246 +node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -f -r $FROM_REWARD_EPOCH_ID + +# --------------------------------------------------------------------------------------------------------------------------- +# single reward epoch calculation +# export REWARD_EPOCH_ID=2773 +# node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -g -o -c -a -y -b 40 -w 10 -r $REWARD_EPOCH_ID -m 10000 + +# --------------------------------------------------------------------------------------------------------------------------- +# Incremental calculation for the current reward epoch +# node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -l -b 40 -w 10 -m 10000 + From bbd7f38598b86f9f0857164207c332cb9a0ac321 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:26:05 +0100 Subject: [PATCH 24/28] refactoring min conditions --- .../src/services/calculator.service.ts | 2 +- .../minimal-conditions/input-interfaces.ts | 168 ------------------ .../minimal-conditions-data.ts | 71 ++++++++ .../minimal-conditions-interfaces.ts | 110 ++++++++++++ .../minimal-conditions/minimal-conditions.ts | 3 +- 5 files changed, 184 insertions(+), 170 deletions(-) delete mode 100644 libs/ftso-core/src/reward-calculation/minimal-conditions/input-interfaces.ts create mode 100644 libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-data.ts diff --git a/apps/ftso-reward-calculation-process/src/services/calculator.service.ts b/apps/ftso-reward-calculation-process/src/services/calculator.service.ts index b45d238f..21bfb272 100644 --- a/apps/ftso-reward-calculation-process/src/services/calculator.service.ts +++ b/apps/ftso-reward-calculation-process/src/services/calculator.service.ts @@ -47,7 +47,7 @@ import { runCalculateRewardCalculationTopJob } from "../libs/reward-data-calcula import { getIncrementalCalculationsFeedSelections, serializeIncrementalCalculationsFeedSelections } from "../../../../libs/ftso-core/src/utils/stat-info/incremental-calculation-temp-selected-feeds"; import { calculateAttestationTypeAppearances } from "../libs/attestation-type-appearances"; import { calculateMinimalConditions } from "../../../../libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions"; -import { writeDataProviderConditions } from "../../../../libs/ftso-core/src/reward-calculation/minimal-conditions/input-interfaces"; +import { writeDataProviderConditions } from "../../../../libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-data"; if (process.env.FORCE_NOW) { const newNow = parseInt(process.env.FORCE_NOW) * 1000; diff --git a/libs/ftso-core/src/reward-calculation/minimal-conditions/input-interfaces.ts b/libs/ftso-core/src/reward-calculation/minimal-conditions/input-interfaces.ts deleted file mode 100644 index 7854178f..00000000 --- a/libs/ftso-core/src/reward-calculation/minimal-conditions/input-interfaces.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; -import { base58 } from '@scure/base'; -import path from "path/posix"; -import { CALCULATIONS_FOLDER, PASSES_DATA_FOLDER, STAKING_DATA_FOLDER } from "../../configs/networks"; -import { DataProviderConditions } from "./minimal-conditions-interfaces"; -import { bigIntReplacer } from "../../utils/big-number-serialization"; - -export interface Delegator { - // pChain address, like flare123l344hlugpg0r2ntdl6fn45qyp0f5m2xakc0r - pAddress: string; - // cChain address, like 0x4485B10aD3ff29066938922059c5CB1e5e8Ee8b6 - cAddress: string; - // as string in GWei - amount: string; - // as string in GWei - delegatorRewardAmount: string; -} - -export interface ValidatorInfo { - // node id in form: NodeID-2a7BPY7UeJv2njMuyUHfBSTeQCYZj6bwV - nodeId: string; - // bonding address in form: flare1uz66xddzplexwfdsxrzxsnlwlucyfsuax00crd - bondingAddress: string; - // self bond in GWei - selfBond: string; - // ftso address in form "0xfe532cB6Fb3C47940aeA7BeAd4d61C5e041D950e", - ftsoAddress: string; - // end of stake in unix time - stakeEnd: number; - // string of p-chain addresses (in form flare1uz66xddzplexwfdsxrzxsnlwlucyfsuax00crd) - pChainAddress: string[]; - // fee in GWei - fee: number; - // group number - group: number; - // is the validator eligible for staking rewards - eligible: boolean; - // data provider name - ftsoName: string; - // Boosting eligibility bond in GWei - BEB: string; - // Boost delegations in GWei - boostDelegations: string; - // boost in GWei - boost: string; - // self delegations in GWei - selfDelegations: string; - // other delegations in GWei - normalDelegations: string; - // total self bond in GWei - totalSelfBond: string; - // list of delegators - delegators: Delegator[]; - // total stake amount in GWei“ - totalStakeAmount: string; - // C-chain address in form of 0xaDEDCd23941E479b4736B38e271Eb926596BBe3d - cChainAddress: string; - // overboost in GWei - overboost: string; - // reward weight in GWei - rewardingWeight: string; - // capped weight in GWei - cappedWeight: string; - // node reward amount in wei - nodeRewardAmount: string; - // validator reward amount in wei - validatorRewardAmount: string; - // Node id as 20-byte hex string - nodeId20Byte?: string; -} - -export enum MinimalConditionFailureType { - // Providers must submit a value estimate that lies within a 0.5% band around the consensus median value - // in 80% of voting rounds within a reward epoch. - FTSO_SCALING_FAILURE = "FTSO_SCALING_FAILURE", - // Providers must submit at least 80% of their expected number of updates within a reward epoch, - // unless they have very low weight, defined as < 0.2% of the total active weight. - FAST_UPDATES_FAILURE = "FAST_UPDATES_FAILURE", - // Providers must meet 80% total uptime in the reward epoch with at least 1M FLR in active self-bond. - // However, in order to earn passes, the provider must have at least 3M FLR in active self-bond and 15M - // in active stake. Providers with 80% total uptime and at least 1M FLR in active self-bond but - // not meeting both the 3M FLR active self-bond and 15M active stake requirements neither earn - // nor lose passes, and still receive eligible rewards. - STAKING_FAILURE = "STAKING_AVAILABILITY", -} - -export interface MinimalConditionFailure { - // protocol id - protocolId: number; - // failure id - failureId: MinimalConditionFailureType; -} - -export interface DataProviderPasses { - // epoch id in string - rewardEpochId: string; - // voter identity address in lowercase - voterAddress: string; - // number of passes. A number between 0 and 3 - passes: number; - // failures - failures?: MinimalConditionFailure[]; -} - -/** - * Reads the staking info for a given reward epoch id. - * The data is stored in the staking data folder. - */ -export function readStakingInfo( - rewardEpochId: number, - stakingDataFolder = STAKING_DATA_FOLDER() -): ValidatorInfo[] { - const fname = path.join(stakingDataFolder, `${rewardEpochId}-nodes-data.json`); - const data = readFileSync(fname, 'utf8'); - const result: ValidatorInfo[] = JSON.parse(data); - for(let validatorInfo of result) { - // "NodeID-2a7BPY7UeJv2njMuyUHfBSTeQCYZj6bwV" - // Checksum is not validated - validatorInfo.nodeId20Byte = "0x" + Buffer.from(base58.decode(validatorInfo.nodeId.slice(7)).subarray(0, -4)).toString("hex"); - } - return result; -} - -/** - * Reads the passes info for a given reward epoch id. - * The data is stored in the passes data folder. - */ -export function readPassesInfo( - rewardEpochId: number, - passesDataFolder = PASSES_DATA_FOLDER() -): DataProviderPasses[] | undefined { - const fname = path.join(passesDataFolder, `${rewardEpochId}-passes-data.json`); - if(!existsSync(fname)) { - return undefined; - } - const data = readFileSync(fname, 'utf8'); - return JSON.parse(data); -} - -/** - * Writes the staking info for a given reward epoch id. - */ -export function writePassesInfo( - rewardEpochId: number, - data: DataProviderPasses, - passesDataFolder = PASSES_DATA_FOLDER() -): void { - if (!existsSync(passesDataFolder)) { - mkdirSync(passesDataFolder, { recursive: true }); - } - const fname = path.join(passesDataFolder, `${rewardEpochId}-passes-data.json`); - writeFileSync(fname, JSON.stringify(data)); -} - -/** - * Writes the staking info for a given reward epoch id. - */ -export function writeDataProviderConditions( - rewardEpochId: number, - data: DataProviderConditions[], - calculationFolder = CALCULATIONS_FOLDER() -): void { - if (!existsSync(calculationFolder)) { - mkdirSync(calculationFolder, { recursive: true }); - } - const fname = path.join(calculationFolder, `${rewardEpochId}`, `minimal-conditions.json`); - writeFileSync(fname, JSON.stringify(data, bigIntReplacer)); -} diff --git a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-data.ts b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-data.ts new file mode 100644 index 00000000..61b7b4af --- /dev/null +++ b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-data.ts @@ -0,0 +1,71 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; +import { base58 } from '@scure/base'; +import path from "path/posix"; +import { CALCULATIONS_FOLDER, PASSES_DATA_FOLDER, STAKING_DATA_FOLDER } from "../../configs/networks"; +import { DataProviderConditions, DataProviderPasses, ValidatorInfo } from "./minimal-conditions-interfaces"; +import { bigIntReplacer } from "../../utils/big-number-serialization"; + +/** + * Reads the staking info for a given reward epoch id. + * The data is stored in the staking data folder. + */ +export function readStakingInfo( + rewardEpochId: number, + stakingDataFolder = STAKING_DATA_FOLDER() +): ValidatorInfo[] { + const fname = path.join(stakingDataFolder, `${rewardEpochId}-nodes-data.json`); + const data = readFileSync(fname, 'utf8'); + const result: ValidatorInfo[] = JSON.parse(data); + for(let validatorInfo of result) { + // "NodeID-2a7BPY7UeJv2njMuyUHfBSTeQCYZj6bwV" + // Checksum is not validated + validatorInfo.nodeId20Byte = "0x" + Buffer.from(base58.decode(validatorInfo.nodeId.slice(7)).subarray(0, -4)).toString("hex"); + } + return result; +} + +/** + * Reads the passes info for a given reward epoch id. + * The data is stored in the passes data folder. + */ +export function readPassesInfo( + rewardEpochId: number, + passesDataFolder = PASSES_DATA_FOLDER() +): DataProviderPasses[] | undefined { + const fname = path.join(passesDataFolder, `${rewardEpochId}-passes-data.json`); + if(!existsSync(fname)) { + return undefined; + } + const data = readFileSync(fname, 'utf8'); + return JSON.parse(data); +} + +/** + * Writes the staking info for a given reward epoch id. + */ +export function writePassesInfo( + rewardEpochId: number, + data: DataProviderPasses, + passesDataFolder = PASSES_DATA_FOLDER() +): void { + if (!existsSync(passesDataFolder)) { + mkdirSync(passesDataFolder, { recursive: true }); + } + const fname = path.join(passesDataFolder, `${rewardEpochId}-passes-data.json`); + writeFileSync(fname, JSON.stringify(data)); +} + +/** + * Writes the staking info for a given reward epoch id. + */ +export function writeDataProviderConditions( + rewardEpochId: number, + data: DataProviderConditions[], + calculationFolder = CALCULATIONS_FOLDER() +): void { + if (!existsSync(calculationFolder)) { + mkdirSync(calculationFolder, { recursive: true }); + } + const fname = path.join(calculationFolder, `${rewardEpochId}`, `minimal-conditions.json`); + writeFileSync(fname, JSON.stringify(data, bigIntReplacer)); +} diff --git a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts index 47406a90..abfb76a0 100644 --- a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts +++ b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts @@ -1,3 +1,112 @@ +/////////////////////////////////////////////////////////////////////////////////////////////// +// Structure of input JSON produced by initial script for Staking reward calculation +// The script produces data about uptime and node stakes that is used as input to reward +// calculation to consider minimal conditions. +/////////////////////////////////////////////////////////////////////////////////////////////// +export interface Delegator { + // pChain address, like flare123l344hlugpg0r2ntdl6fn45qyp0f5m2xakc0r + pAddress: string; + // cChain address, like 0x4485B10aD3ff29066938922059c5CB1e5e8Ee8b6 + cAddress: string; + // as string in GWei + amount: string; + // as string in GWei + delegatorRewardAmount: string; +} + +export interface ValidatorInfo { + // node id in form: NodeID-2a7BPY7UeJv2njMuyUHfBSTeQCYZj6bwV + nodeId: string; + // bonding address in form: flare1uz66xddzplexwfdsxrzxsnlwlucyfsuax00crd + bondingAddress: string; + // self bond in GWei + selfBond: string; + // ftso address in form "0xfe532cB6Fb3C47940aeA7BeAd4d61C5e041D950e", + ftsoAddress: string; + // end of stake in unix time + stakeEnd: number; + // string of p-chain addresses (in form flare1uz66xddzplexwfdsxrzxsnlwlucyfsuax00crd) + pChainAddress: string[]; + // fee in GWei + fee: number; + // group number + group: number; + // is the validator eligible for staking rewards + eligible: boolean; + // data provider name + ftsoName: string; + // Boosting eligibility bond in GWei + BEB: string; + // Boost delegations in GWei + boostDelegations: string; + // boost in GWei + boost: string; + // self delegations in GWei + selfDelegations: string; + // other delegations in GWei + normalDelegations: string; + // total self bond in GWei + totalSelfBond: string; + // list of delegators + delegators: Delegator[]; + // total stake amount in GWei“ + totalStakeAmount: string; + // C-chain address in form of 0xaDEDCd23941E479b4736B38e271Eb926596BBe3d + cChainAddress: string; + // overboost in GWei + overboost: string; + // reward weight in GWei + rewardingWeight: string; + // capped weight in GWei + cappedWeight: string; + // node reward amount in wei + nodeRewardAmount: string; + // validator reward amount in wei + validatorRewardAmount: string; + // Node id as 20-byte hex string + nodeId20Byte?: string; +} + +/////////////////////////////////////////////////////////////////////////////////////////////// +// Minimal condition related "passes" JSON types +/////////////////////////////////////////////////////////////////////////////////////////////// + +export enum MinimalConditionFailureType { + // Providers must submit a value estimate that lies within a 0.5% band around the consensus median value + // in 80% of voting rounds within a reward epoch. + FTSO_SCALING_FAILURE = "FTSO_SCALING_FAILURE", + // Providers must submit at least 80% of their expected number of updates within a reward epoch, + // unless they have very low weight, defined as < 0.2% of the total active weight. + FAST_UPDATES_FAILURE = "FAST_UPDATES_FAILURE", + // Providers must meet 80% total uptime in the reward epoch with at least 1M FLR in active self-bond. + // However, in order to earn passes, the provider must have at least 3M FLR in active self-bond and 15M + // in active stake. Providers with 80% total uptime and at least 1M FLR in active self-bond but + // not meeting both the 3M FLR active self-bond and 15M active stake requirements neither earn + // nor lose passes, and still receive eligible rewards. + STAKING_FAILURE = "STAKING_AVAILABILITY", +} + +export interface MinimalConditionFailure { + // protocol id + protocolId: number; + // failure id + failureId: MinimalConditionFailureType; +} + +export interface DataProviderPasses { + // epoch id in string + rewardEpochId: string; + // voter identity address in lowercase + voterAddress: string; + // number of passes. A number between 0 and 3 + passes: number; + // failures + failures?: MinimalConditionFailure[]; +} + +/////////////////////////////////////////////////////////////////////////////////////////////// +// Minimal condition calculation result types +/////////////////////////////////////////////////////////////////////////////////////////////// export interface ConditionSummary { // whether a minimal condition is met @@ -60,6 +169,7 @@ export interface StakingConditionSummary extends ConditionSummary { nodeConditions: NodeStakingConditions[]; } +// Reflects a record about minimal condition calculation for a data provider. export interface DataProviderConditions { // voter identity address voterAddress: string; diff --git a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts index 07792757..d4064ce0 100644 --- a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts +++ b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts @@ -1,4 +1,3 @@ -import { DataProviderPasses, readPassesInfo, readStakingInfo } from "./input-interfaces"; import { deserializeDataForRewardCalculation } from "../../utils/stat-info/reward-calculation-data"; import { deserializeRewardEpochInfo } from "../../utils/stat-info/reward-epoch-info"; import { @@ -12,8 +11,10 @@ import { STAKING_UPTIME_THRESHOLD_PPM, TOTAL_PPM } from "./minimal-conditions-constants"; +import { readPassesInfo, readStakingInfo } from "./minimal-conditions-data"; import { DataProviderConditions, + DataProviderPasses, FUConditionSummary, FeedHits, FtsoScalingConditionSummary, From 3dc9d0bef3d62bc32b3dac18fcc85452f35539bb Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Thu, 5 Dec 2024 22:54:36 +0100 Subject: [PATCH 25/28] readme fix partial --- scripts/rewards/README.md | 75 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/scripts/rewards/README.md b/scripts/rewards/README.md index 4b503ebc..204d7979 100644 --- a/scripts/rewards/README.md +++ b/scripts/rewards/README.md @@ -19,6 +19,11 @@ Reward funds for FTSO fast updates come from: - inflation, - community incentives to increase volatility. +### Flare Data Connector (FDC) protocol +Reward funds for FDC come from: +- inflation, +- fees from attestation requests. + ## Secure reward random number FTSO scaling protocol generates a random number for each voting round. The random number may be secure or not. For the purpose of reward calculation, we assign to each voting round a **secure reward random number** which is defined as the earliest secure random number generated by the protocol in the subsequent voting rounds. For example, consider voting round `N`. Then usually a secure random number is obtained from the voting round `N + 1`, say `r`. Then the secure reward random number for the voting round `N` is `sha256(r, N)`. Note that it may happen that the first secure random number occurs only in the voting round e.g. `N+5`, say `s`. That secure random number is then used for determining the secure reward random numbers of voting rounds `N`, `N + 1`, ..., `N + 4`, as `sha256(s, N)`, `sha256(s, N + 1)`, ..., `sha256(s, N + 4)`, respectively. @@ -73,6 +78,71 @@ Total reward for voting round is distributed according to the accuracy as follow - These rewards are assigned to each provider who has given an update in the 90s period, with each provider receiving a share relative to the amount of updates they have given in the round (or equally - each update gets the same share of rewards). - If in some voting round there are no updates or accuracy is not sufficient, the reward intended for the voting round is burned. +## FDC reward calculation + +FDC reward calculation starts with rewards fund adjustments. Note that reward funds for a reward epoch consist of inflation part and attestation request fees, which belong to specific voting rounds. + +### Reward funds adjustment + +Reward funds adjustment consists of the following steps: +- Counting confirmed attestation requests in the whole reward epoch for each attestation type. +- Burning the share of inflation reward funds for attestation types that did not receive sufficient number of confirmed attestation requests in the reward epoch. +- Burning the fees of not confirmed attestation requests. +- Assigning to each voting round `N` its **voting round reward funds** `R(N)`, consisting of: + - the equal share of remaining inflation funds, + - fees of confirmed attestation requests in the specific round. +- `R(N)` gets further split as follows: + - `Rfdc(N) = 0.9 * R(N)` - 90% is dedicated for rewarding data providers for protocol, except for finalization. + - `Rfin(N) = 0.1 * R(N)` - 10% is dedicated for finalization rewards. + +### FDC protocol participation + +A **correct sign submission** by an eligible data provider meets exactly the following criteria: +- Sent by the `submitSignature` function from the corresponding `submitSignatureAddress`. +- The payload contains a signature of the finalized Merkle root, but without the signed message (to prevent copying), and the **consensus bit-vote candidate**. +- The transaction was sent within the **signing grace period** or before the first finalization of the finalized Merkle root (the timestamp of the finalization included). + +The phases of FDC protocol include: +- **Collect phase** - Attestation requests are collected. Matches commit period of FTSO scaling +- **Choose phase** - Bit-voting takes place. Matches the reveal period of FTSO scaling. +- **Resolve period** - signatures get submitted and when sufficient weight of signatures is deposited on-chain, finalization takes place. Matches the same signing and finalizing periods as with FTSO scaling. + +The grace periods used in the rewarding start at the end of Choose phase and last as follows: +- **signing grace period**: 10s, +- **finalize grace period**: 20s. + +Correct sign submissions for a voting round have attached consensus bit-vote candidates. If there exists a majority (by weight) of consensus bit-vote candidates, it is proclaimed as the **consensus bit-vote**. The majority is calculated based on the total weight of correctly signed submissions. Note that this is not the 50%+ majority of eligible data providers, but just the majority of the correct sign submissions. + +Each data provider sending a correct sign submission is considered to have sent a **correct vote**, if +- the finalized Merkle root for the round exists, +- consensus bit-vote exists. + +### Punishable violations + +**Punishable violations** are defined as follows: +- **No reveal on bit-vote** - bit-vote submitted and dominating the consensus bit-vote, but no reveal. +- **Wrong signature** - reveal not signing the finalized Merkle root. +- **Bad consensus bit-vote candidate** - reveal submitted, signing the finalized Merkle root, but the consensus bit-vote candidate does not match the consensus bit-vote. + +Note that technically an eligible data provider can submit multiple correct sign submissions, some may be correct votes, while some may be punishable violations. + +### Reward distribution for a voting round + +Data providers use signing weight to participate in FDC protocol and rewards are distributed based on their share of signing weight in total signing weight. +The **expected reward amount** for an eligible data provider `i` with the signing weight `w(i)` is calculated as + `Rexp(i, N) = w(i)/W * Rfdc(N)`, where `W` denotes the total signing weight. + +Reward distribution for FDC is carried out as follows: +- If there is no finalized Merkle root for the voting round, `R(N)` gets burned. +- Each eligible data provider that provided a correct vote and committed no punishable violations gets the reward `k * Rexp(i, N)`, where the factor `k` is as follows: + - 1 - bit-vote transaction sent and dominates the consensus bit-vote + - 0.8 - bitvote sent, but it does not dominate the consensus bitvote + - 0.8 - no bit-vote sent +- All the remaining funds from `R(n)` get burned. +- Each eligible data provider that committed punishable violation gets penalized by `30 * Rexp(i, N)`. +- `F(N)` be a set of selected data providers for finalization in the voting round `N` and `WF(N)` their total weight. Each selected data provider `i` who submits the correct finalized Merkle root with signatures for the voting round within the finalization grace period gets `w(i)/WF(N) * Rfin(N)` of the finalization reward. If the finalization is done within the finalization grace period, the rest of `Rfin(N)` gets burned. Otherwise, the first finalizer (might even not be a data provider) gets `Rfin(N)` as a full reward. This may be data provider or not. In case of a data provider, this is a direct claim, not shared by delegators and stakers. + + ## Reward claims Reward claims are records of a part of a reward (or penalization) assignment. There are **aggregated reward claims** and **detailed reward claims**. The reward calculation process creates detailed reward claims for each separate component of rewarding in each voting round. The claims for parts of rewards are positive, while the penalization claims are negative. Note that the sum of all positive (non-penalization) claims matches the total reward fund. Aggregated reward claims are obtained by aggregating the detailed reward claims by beneficiary and claim type (see below). @@ -118,6 +188,11 @@ Possible values of protocolTag are: - `Reveal offenders` - penalization for reveal offenders in FTSO scaling protocol. - `Fast updates accuracy` - accuracy reward for FAST updates protocol. - `Full offer claim back` - full reward offer burning/claim back in FTSO scaling protocol. Happens when a voting round does not receive a secure reward random number and the rewarded feed cannot be chosen. Consequently, this is a reward claim to claim to the burn address. Detail tags are irrelevant in this case and are empty. +- `Partial FDC offer claim back` +- `FDC signing` +- `FDC finalization` +- `FDC offenders` + ### Reward detail tag From 73e8c9b8e599c4304bbcfccd2a0e54114cd694c9 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Fri, 6 Dec 2024 01:56:24 +0100 Subject: [PATCH 26/28] fdc hub contract address for songbird --- libs/ftso-core/src/configs/networks.ts | 2 +- scripts/rewards/songbird-db.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/ftso-core/src/configs/networks.ts b/libs/ftso-core/src/configs/networks.ts index 32907d3e..2b5e1828 100644 --- a/libs/ftso-core/src/configs/networks.ts +++ b/libs/ftso-core/src/configs/networks.ts @@ -69,7 +69,7 @@ const SONGBIRD_CONFIG: NetworkContractAddresses = { name: "FastUpdateIncentiveManager", address: "0x596C70Ad6fFFdb9b6158F1Dfd0bc32cc72B82006", }, - FdcHub: { name: "FdcHub", address: "" }, + FdcHub: { name: "FdcHub", address: "0xCfD4669a505A70c2cE85db8A1c1d14BcDE5a1a06" }, }; const FLARE_CONFIG: NetworkContractAddresses = { diff --git a/scripts/rewards/songbird-db.sh b/scripts/rewards/songbird-db.sh index 3d8e894f..00e990ae 100755 --- a/scripts/rewards/songbird-db.sh +++ b/scripts/rewards/songbird-db.sh @@ -28,7 +28,7 @@ yarn nest build ftso-reward-calculation-process # In the current (ongoing) reward epoch the calculation is switched to incremental, as data becomes available. # If the data for a specific reward epoch id is already available, the calculation is skipped. export FROM_REWARD_EPOCH_ID=196 -node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -g -o -c -a -y -b 100 -w 10 -d $FROM_REWARD_EPOCH_ID -m 10000 +node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -g -o -c -a -y -z -b 100 -w 10 -d $FROM_REWARD_EPOCH_ID -m 10000 # Incremental calculation # node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -l -b 80 -w 5 -m 10000 From e0c47b4a8ddd70fac2659d6a516a42a169af3470 Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Tue, 10 Dec 2024 13:00:22 +0100 Subject: [PATCH 27/28] minimal conditions --- .gitignore | 3 +- .../minimal-conditions-data.ts | 10 ++- .../minimal-conditions-interfaces.ts | 39 ++++++++++- .../minimal-conditions/minimal-conditions.ts | 65 +++++++++++++++++-- scripts/analytics/reward-claims-csv.ts | 3 + scripts/analytics/run/offer-check.ts | 39 +++++++++++ ...king-data.sh => flare-get-staking-data.sh} | 0 scripts/get-listed-data-providers.sh | 4 ++ scripts/rewards/flare-min-conditions.sh | 38 ----------- .../min-conditions/flare-min-conditions.sh | 23 +++++++ .../min-conditions/songbird-min-conditions.sh | 23 +++++++ 11 files changed, 198 insertions(+), 49 deletions(-) rename scripts/{get-staking-data.sh => flare-get-staking-data.sh} (100%) create mode 100755 scripts/get-listed-data-providers.sh delete mode 100755 scripts/rewards/flare-min-conditions.sh create mode 100755 scripts/rewards/min-conditions/flare-min-conditions.sh create mode 100755 scripts/rewards/min-conditions/songbird-min-conditions.sh diff --git a/.gitignore b/.gitignore index 657425af..c113cbb3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ coverage calculations test-db exports-csv -staking-data \ No newline at end of file +staking-data +listed-data-providers \ No newline at end of file diff --git a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-data.ts b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-data.ts index 61b7b4af..3105950f 100644 --- a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-data.ts +++ b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-data.ts @@ -2,7 +2,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; import { base58 } from '@scure/base'; import path from "path/posix"; import { CALCULATIONS_FOLDER, PASSES_DATA_FOLDER, STAKING_DATA_FOLDER } from "../../configs/networks"; -import { DataProviderConditions, DataProviderPasses, ValidatorInfo } from "./minimal-conditions-interfaces"; +import { DataProviderConditions, DataProviderPasses, ListedProviderList, ValidatorInfo } from "./minimal-conditions-interfaces"; import { bigIntReplacer } from "../../utils/big-number-serialization"; /** @@ -67,5 +67,11 @@ export function writeDataProviderConditions( mkdirSync(calculationFolder, { recursive: true }); } const fname = path.join(calculationFolder, `${rewardEpochId}`, `minimal-conditions.json`); - writeFileSync(fname, JSON.stringify(data, bigIntReplacer)); + writeFileSync(fname, JSON.stringify(data, bigIntReplacer, 2)); +} + +export function readListedDataProviders(): ListedProviderList { + const fname = path.join(`listed-data-providers`, `bifrost-wallet.providerlist.json`); + const data = readFileSync(fname, 'utf8'); + return JSON.parse(data); } diff --git a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts index abfb76a0..49bdaad2 100644 --- a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts +++ b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts @@ -117,8 +117,8 @@ export interface ConditionSummary { } export interface FeedHits { - // feed id - feedId: string; + // feed name + feedName: string; // hits out of totalHits feedHits: number; // all feed hits @@ -143,6 +143,8 @@ export interface FUConditionSummary extends ConditionSummary { tooLowWeight: boolean; // expected PPM share based on the share of the weight expectedUpdatesPPM: bigint; + // expected number of updates + expectedUpdates: bigint; } export interface NodeStakingConditions { @@ -171,6 +173,12 @@ export interface StakingConditionSummary extends ConditionSummary { // Reflects a record about minimal condition calculation for a data provider. export interface DataProviderConditions { + // reward epoch id + network: string; + // network name + rewardEpochId: number; + // data provider name + dataProviderName?: string; // voter identity address voterAddress: string; // voter index @@ -192,3 +200,30 @@ export interface DataProviderConditions { // staking conditions staking: StakingConditionSummary; } + +/////////////////////////////////////////////////////////////////////////////////////////////// +// Listed providers for Bifrost wallet types +/////////////////////////////////////////////////////////////////////////////////////////////// + +export interface ListedProviderListVersion { + major: number; + minor: number; + patch: number; +} + +export interface ListedProvider { + chainId: number; + name: string; + description: string; + url: string; + address: string; + logoURI: string; + listed: boolean; +} + +export interface ListedProviderList { + name: string; + timestamp: string; + version: ListedProviderListVersion; + providers: ListedProvider[]; +} \ No newline at end of file diff --git a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts index d4064ce0..7dc7f4d8 100644 --- a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts +++ b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions.ts @@ -11,7 +11,7 @@ import { STAKING_UPTIME_THRESHOLD_PPM, TOTAL_PPM } from "./minimal-conditions-constants"; -import { readPassesInfo, readStakingInfo } from "./minimal-conditions-data"; +import { readListedDataProviders, readPassesInfo, readStakingInfo } from "./minimal-conditions-data"; import { DataProviderConditions, DataProviderPasses, @@ -19,9 +19,32 @@ import { FeedHits, FtsoScalingConditionSummary, NodeStakingConditions, - StakingConditionSummary + StakingConditionSummary, + ValidatorInfo } from "./minimal-conditions-interfaces"; +function networkId() { + if(process.env.NETWORK === "flare") { + return 14; + } + if(process.env.NETWORK === "songbird") { + return 19; + } + throw new Error(`Network ${process.env.NETWORK} not supported`); +} + +function toFeedName(hex: string) { + let result = ""; + for (let i = 4; i < hex.length; i += 2) { + const charHexCode = hex.slice(i, i + 2); + if (charHexCode === "00") { + continue; + } + result += String.fromCharCode(parseInt(charHexCode, 16)); + } + return result; + } + export function calculateMinimalConditions( rewardEpochId: number, requirePassesFile: boolean @@ -43,14 +66,18 @@ export function calculateMinimalConditions( const voterToFUConditionSummary = new Map(); const voterToStakingConditionSummary = new Map(); const nodeIdToVoter = new Map(); + const delegationAddressToVoter = new Map(); const nodeIdToNodeStakingCondition = new Map(); const submitAddressToFeedToHits = new Map>(); const voterToPassesInputData = new Map(); + const voterToName = new Map(); + + const stakingIncluded = process.env.NETWORK === "flare"; const totalSigningWeight = rewardEpochInfo.signingPolicy.weights.reduce((acc, weight) => acc + weight, 0); const numberOfVotingRounds = (rewardEpochInfo.endVotingRoundId - rewardEpochInfo.signingPolicy.startVotingRoundId + 1); - + for (let dataProviderPasses of passesInputData) { let voter = dataProviderPasses.voterAddress.toLowerCase(); voterToPassesInputData.set(voter, dataProviderPasses); @@ -61,9 +88,11 @@ export function calculateMinimalConditions( const signingWeight = rewardEpochInfo.signingPolicy.weights[i]; const submissionAddress = rewardEpochInfo.voterRegistrationInfo[i].voterRegistered.submitAddress.toLowerCase(); const signingPolicyAddress = rewardEpochInfo.voterRegistrationInfo[i].voterRegistered.signingPolicyAddress.toLowerCase(); + const delegationAddress = rewardEpochInfo.voterRegistrationInfo[i].voterRegistrationInfo.delegationAddress.toLowerCase(); submitAddressToVoter.set(submissionAddress, voter); voterToSubmitAddress.set(voter, submissionAddress); signingPolicyAddressToVoter.set(signingPolicyAddress, voter); + delegationAddressToVoter.set(delegationAddress, voter); voterToVoterIndex.set(voter, i); const ftsoScalingConditionSummary: FtsoScalingConditionSummary = { allPossibleHits: numberOfVotingRounds * rewardEpochInfo.canonicalFeedOrder.length, @@ -79,6 +108,7 @@ export function calculateMinimalConditions( totalUpdatesByAll: 0, updates: 0, expectedUpdatesPPM, + expectedUpdates: 0n, tooLowWeight: proportionUpdatesPPM < FU_CONSIDERATION_THRESHOLD_PPM, } voterToFUConditionSummary.set(voter, fuConditionSummary); @@ -109,7 +139,7 @@ export function calculateMinimalConditions( const voterFeedHits = new Map(); for (let feedInfo of rewardEpochInfo.canonicalFeedOrder) { const feedHits: FeedHits = { - feedId: feedInfo.id, + feedName: toFeedName(feedInfo.id), feedHits: 0, totalHits: numberOfVotingRounds } @@ -118,14 +148,29 @@ export function calculateMinimalConditions( submitAddressToFeedToHits.set(submissionAddress, voterFeedHits); } + const dataProviderData = readListedDataProviders(); + for(let provider of dataProviderData.providers) { + if(provider.chainId !== networkId()) { + continue; + } + const delegationAddress = provider.address.toLowerCase(); + const voter = delegationAddressToVoter.get(delegationAddress); + if(voter !== undefined) { + voterToName.set(voter, provider.name); + } + } + // Reading staking info data for reward epoch and updating node conditions - const validatorInfoList = readStakingInfo(rewardEpochId); + let validatorInfoList: ValidatorInfo[] = stakingIncluded ? readStakingInfo(rewardEpochId) : []; + if(!stakingIncluded) { + console.log(`Staking data not relevant for the network ${process.env.NETWORK}`); + } for (const validatorInfo of validatorInfoList) { const nodeId = validatorInfo.nodeId20Byte; const condition = nodeIdToNodeStakingCondition.get(nodeId); if (condition === undefined) { // TODO: log properly - console.log(`Node ${nodeId} not found in the voter registration info`); + console.log(`Node ${nodeId} / ${validatorInfo.nodeId} by ${validatorInfo.ftsoName} not found in the voter registration info`); continue; } condition.selfBond = BigInt(validatorInfo.selfBond); @@ -136,6 +181,10 @@ export function calculateMinimalConditions( // Checking staking conditions for (const [_, stakingConditionSummary] of voterToStakingConditionSummary.entries()) { + if(!stakingIncluded) { + stakingConditionSummary.conditionMet = true; + continue; + } for (const nodeCondition of stakingConditionSummary.nodeConditions) { if (nodeCondition.uptimeOk) { stakingConditionSummary.stakeWithUptime += nodeCondition.totalStakeAmount; @@ -213,6 +262,7 @@ export function calculateMinimalConditions( // go over all data providers and check minimal conditions for fast updates for (const [voter, fuConditionSummary] of voterToFUConditionSummary.entries()) { fuConditionSummary.totalUpdatesByAll = totalFUUpdates; + fuConditionSummary.expectedUpdates = fuConditionSummary.expectedUpdatesPPM * BigInt(totalFUUpdates) / TOTAL_PPM; fuConditionSummary.conditionMet = fuConditionSummary.tooLowWeight || TOTAL_PPM * BigInt(fuConditionSummary.updates) >= fuConditionSummary.expectedUpdatesPPM * BigInt(totalFUUpdates) } @@ -278,6 +328,9 @@ export function calculateMinimalConditions( } newNumberOfPasses = Math.min(newNumberOfPasses, MAX_NUMBER_OF_PASSES); const dataProviderCondition: DataProviderConditions = { + rewardEpochId, + network: process.env.NETWORK, + dataProviderName: voterToName.get(voter), voterAddress: voter, voterIndex: voterToVoterIndex.get(voter), passesHeld, diff --git a/scripts/analytics/reward-claims-csv.ts b/scripts/analytics/reward-claims-csv.ts index d447a121..24054bc1 100644 --- a/scripts/analytics/reward-claims-csv.ts +++ b/scripts/analytics/reward-claims-csv.ts @@ -146,6 +146,9 @@ function writeAllClaimsForRewardEpochRange(startRewardEpochId: number, endReward } function decodeFeed(feedIdHex: string): string { + if(!feedIdHex) { + return "------"; + } const name = Buffer.from(feedIdHex.slice(4), "hex").toString("utf8").replaceAll("\0", ""); return name; } diff --git a/scripts/analytics/run/offer-check.ts b/scripts/analytics/run/offer-check.ts index 5d500f11..08117494 100644 --- a/scripts/analytics/run/offer-check.ts +++ b/scripts/analytics/run/offer-check.ts @@ -3,6 +3,21 @@ import { deserializeGranulatedPartialOfferMap, deserializeGranulatedPartialOffer import { deserializeRewardEpochInfo } from "../../../libs/ftso-core/src/utils/stat-info/reward-epoch-info"; import { deserializeDataForRewardCalculation } from "../../../libs/ftso-core/src/utils/stat-info/reward-calculation-data"; + +function extractName(hexInput: string) { + const hex = hexInput.startsWith("0x") ? hexInput.slice(2) : hexInput; + let result = ""; + for (let i = 0; i < hex.length; i += 2) { + const charHexCode = hex.slice(i, i + 2); + if (charHexCode === "00") { + continue; + } + result += String.fromCharCode(parseInt(charHexCode, 16)); + } + return result; +} + + async function main() { if (!process.argv[2]) { throw new Error("no rewardEpochId"); @@ -36,6 +51,8 @@ async function main() { let fastUpdatesOfferAmount = 0n; let fdcOfferAmount = 0n; let fdcOfferBurn = 0n; + let attestationRequestCount = 0; + let acceptedAttestationRequestCount = 0; for (let votingRoundId = rewardEpochInfo.signingPolicy.startVotingRoundId; votingRoundId <= rewardEpochInfo.endVotingRoundId; votingRoundId++) { const ftsoOfferClaims = deserializeGranulatedPartialOfferMap(rewardEpochId, votingRoundId, calculationFolder); for (let [_, offers] of ftsoOfferClaims.entries()) { @@ -65,9 +82,30 @@ async function main() { ); if (!noFDC) { + attestationRequestCount += data.fdcData.attestationRequests.length; + let reqString = ""; + let i = 0; for (let attestationRequest of data.fdcData.attestationRequests) { fdcFunds += attestationRequest.fee; + if (attestationRequest.confirmed) { + acceptedAttestationRequestCount++; + } + const attType = extractName(attestationRequest.data.slice(2, 66)).slice(0, 3); + const attSource = extractName(attestationRequest.data.slice(66, 130)); + const duplicate = attestationRequest.duplicate ? "D" : ""; + const confirmed = attestationRequest.confirmed ? "C" : ""; + reqString += `${i} ${attType}/${attSource} ${confirmed}${duplicate},`; + i++; + } + if(data.fdcData.attestationRequests.length > 0) { + const finalized = data.fdcData.firstSuccessfulFinalization ? "F" : ""; + const consensusBitvote = data.fdcData.consensusBitVoteIndices; + const firstVotingRoundTs = 1658429955; + const time = firstVotingRoundTs + votingRoundId * 90; + const date = `${new Date(time * 1000)}`.replace(" GMT+0100 (Central European Standard Time)", ""); + console.log(`${votingRoundId}: ${data.fdcData.attestationRequests.length} ${finalized} ${consensusBitvote.length}/${data.fdcData.attestationRequests.length} | ${consensusBitvote} | ${date} || ${reqString}`); } + } } @@ -75,6 +113,7 @@ async function main() { console.log(`Fast Updates Funds: ${fastUpdatesOfferAmount} ${fastUpdatesFunds - fastUpdatesOfferAmount}`); console.log(`FDC Funds: ${fdcFunds - fdcOfferAmount}`); console.log(`FDC Offer Burn: ${fdcOfferBurn}`); + console.log(`Total attestation requests: ${attestationRequestCount}, accepted ${acceptedAttestationRequestCount}`); } main() diff --git a/scripts/get-staking-data.sh b/scripts/flare-get-staking-data.sh similarity index 100% rename from scripts/get-staking-data.sh rename to scripts/flare-get-staking-data.sh diff --git a/scripts/get-listed-data-providers.sh b/scripts/get-listed-data-providers.sh new file mode 100755 index 00000000..62087698 --- /dev/null +++ b/scripts/get-listed-data-providers.sh @@ -0,0 +1,4 @@ +wget "https://raw.githubusercontent.com/TowoLabs/ftso-signal-providers/next/bifrost-wallet.providerlist.json" +mkdir -p listed-data-providers +rm -f listed-data-providers/bifrost-wallet.providerlist.json +mv bifrost-wallet.providerlist.json listed-data-providers/bifrost-wallet.providerlist.json \ No newline at end of file diff --git a/scripts/rewards/flare-min-conditions.sh b/scripts/rewards/flare-min-conditions.sh deleted file mode 100755 index f5329b0a..00000000 --- a/scripts/rewards/flare-min-conditions.sh +++ /dev/null @@ -1,38 +0,0 @@ -# Reward calculation for Coston network -# Setup the correct DB connection and run the script, e.g. -# ./scripts/rewards/coston-db.sh - -export NETWORK=flare -export DB_REQUIRED_INDEXER_HISTORY_TIME_SEC=86400 -export VOTING_ROUND_HISTORY_SIZE=10000 -export INDEXER_TOP_TIMEOUT=1000 -export DB_HOST=127.0.0.1 -export DB_PORT=3336 -export DB_USERNAME=root -export DB_PASSWORD=root -export DB_NAME=flare_ftso_indexer - -export REMOVE_ANNOYING_MESSAGES=true -export ALLOW_IDENTITY_ADDRESS_SIGNING=true -# Start reward epoch id on Coston -export START_REWARD_EPOCH_ID=2344 - -# COMPILATION -yarn nest build ftso-reward-calculation-process - -# --------------------------------------------------------------------------------------------------------------------------- -# Calculating all reward data from the starting reward epoch id. The calculation of claims is parallelized. -# In the current (ongoing) reward epoch the calculation is switched to incremental, as data becomes available. -# If the data for a specific reward epoch id is already available, the calculation is skipped. -export FROM_REWARD_EPOCH_ID=246 -node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -f -r $FROM_REWARD_EPOCH_ID - -# --------------------------------------------------------------------------------------------------------------------------- -# single reward epoch calculation -# export REWARD_EPOCH_ID=2773 -# node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -g -o -c -a -y -b 40 -w 10 -r $REWARD_EPOCH_ID -m 10000 - -# --------------------------------------------------------------------------------------------------------------------------- -# Incremental calculation for the current reward epoch -# node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -l -b 40 -w 10 -m 10000 - diff --git a/scripts/rewards/min-conditions/flare-min-conditions.sh b/scripts/rewards/min-conditions/flare-min-conditions.sh new file mode 100755 index 00000000..a102aee2 --- /dev/null +++ b/scripts/rewards/min-conditions/flare-min-conditions.sh @@ -0,0 +1,23 @@ +# Reward calculation for Coston network +# Setup the correct DB connection and run the script, e.g. +# ./scripts/rewards/coston-db.sh + +export NETWORK=flare +export DB_REQUIRED_INDEXER_HISTORY_TIME_SEC=86400 +export VOTING_ROUND_HISTORY_SIZE=10000 +export INDEXER_TOP_TIMEOUT=1000 +export DB_HOST=127.0.0.1 +export DB_PORT=3336 +export DB_USERNAME=root +export DB_PASSWORD=root +export DB_NAME=flare_ftso_indexer + +export REMOVE_ANNOYING_MESSAGES=true +export ALLOW_IDENTITY_ADDRESS_SIGNING=true +# Start reward epoch id on Coston +export START_REWARD_EPOCH_ID=2344 + +# COMPILATION +yarn nest build ftso-reward-calculation-process + +node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -f -r $1 diff --git a/scripts/rewards/min-conditions/songbird-min-conditions.sh b/scripts/rewards/min-conditions/songbird-min-conditions.sh new file mode 100755 index 00000000..713ec6a7 --- /dev/null +++ b/scripts/rewards/min-conditions/songbird-min-conditions.sh @@ -0,0 +1,23 @@ +# Reward calculation for Coston network +# Setup the correct DB connection and run the script, e.g. +# ./scripts/rewards/coston-db.sh + +export NETWORK=songbird +export DB_REQUIRED_INDEXER_HISTORY_TIME_SEC=86400 +export VOTING_ROUND_HISTORY_SIZE=10000 +export INDEXER_TOP_TIMEOUT=1000 +export DB_HOST=127.0.0.1 +export DB_PORT=3336 +export DB_USERNAME=root +export DB_PASSWORD=root +export DB_NAME=flare_ftso_indexer + +export REMOVE_ANNOYING_MESSAGES=true +export ALLOW_IDENTITY_ADDRESS_SIGNING=true +# Start reward epoch id on Coston +export START_REWARD_EPOCH_ID=2344 + +# COMPILATION +yarn nest build ftso-reward-calculation-process + +node dist/apps/ftso-reward-calculation-process/apps/ftso-reward-calculation-process/src/main.js ftso-reward-calculation-process -f -r $1 From ee144e5f6df0065c18b358d3a85e7e72c871b1ab Mon Sep 17 00:00:00 2001 From: alenabelium <33514602+alenabelium@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:13:08 +0100 Subject: [PATCH 28/28] readme fix, some comments --- .../minimal-conditions-interfaces.ts | 71 ++++++++++--------- scripts/rewards/README.md | 53 ++++++++------ 2 files changed, 68 insertions(+), 56 deletions(-) diff --git a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts index 49bdaad2..a038c0f9 100644 --- a/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts +++ b/libs/ftso-core/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts @@ -108,6 +108,36 @@ export interface DataProviderPasses { // Minimal condition calculation result types /////////////////////////////////////////////////////////////////////////////////////////////// +// Reflects a record about minimal condition calculation for a data provider. +export interface DataProviderConditions { + // reward epoch id + network: string; + // network name + rewardEpochId: number; + // data provider name + dataProviderName?: string; + // voter identity address + voterAddress: string; + // voter index + voterIndex: number; + // passes held before the reward epoch calculation + passesHeld: number; + // strikes + strikes: number; + // whether a pass has been earned + passEarned: boolean; + // eligible for reward + eligibleForReward: boolean; + // new number of passes after the reward epoch calculation + newNumberOfPasses: number; + // ftso scaling conditions summary + ftsoScaling: FtsoScalingConditionSummary; + // fast update conditions summary + fastUpdates: FUConditionSummary; + // staking conditions summary + staking: StakingConditionSummary; +} + export interface ConditionSummary { // whether a minimal condition is met conditionMet: boolean; @@ -116,6 +146,7 @@ export interface ConditionSummary { obstructsPass?: boolean; } +// Summary how many times a FTSO scaling feed was in 0.5% band around the consensus median value export interface FeedHits { // feed name feedName: string; @@ -125,28 +156,32 @@ export interface FeedHits { totalHits: number; } +// Summary of FTSO scaling conditions export interface FtsoScalingConditionSummary extends ConditionSummary { // total number of feed values, equals number of voting rounds in the reward epoch times number of feeds allPossibleHits: number; - // stat telling how many feed values were not empty + // stat telling how many feed values were not empty. Percentage of this related to allPossibilities is used to + // calculate meeting the criteria totalHits: number; // feed hit stats for each feed in the canonical feed order feedHits: FeedHits[]; } +// Summary of fast update conditions export interface FUConditionSummary extends ConditionSummary { // total updates by all providers in the reward epoch totalUpdatesByAll: number; // updates by the provider in the reward epoch updates: number; - //exempt due to low weight + // exempt due to low weight, less than 0.2% tooLowWeight: boolean; - // expected PPM share based on the share of the weight + // expected PPM (parts per million) share based on the share of the weight expectedUpdatesPPM: bigint; // expected number of updates expectedUpdates: bigint; } +// Summary of staking conditions export interface NodeStakingConditions { // node id as 20 byte hex string nodeId: string; @@ -171,36 +206,6 @@ export interface StakingConditionSummary extends ConditionSummary { nodeConditions: NodeStakingConditions[]; } -// Reflects a record about minimal condition calculation for a data provider. -export interface DataProviderConditions { - // reward epoch id - network: string; - // network name - rewardEpochId: number; - // data provider name - dataProviderName?: string; - // voter identity address - voterAddress: string; - // voter index - voterIndex: number; - // passes held - passesHeld: number; - // strikes - strikes: number; - // pass earned - passEarned: boolean; - // eligible for reward - eligibleForReward: boolean; - // new number of passes after the reward epoch calculation - newNumberOfPasses: number; - // ftso scaling conditions - ftsoScaling: FtsoScalingConditionSummary; - // fast update conditions - fastUpdates: FUConditionSummary; - // staking conditions - staking: StakingConditionSummary; -} - /////////////////////////////////////////////////////////////////////////////////////////////// // Listed providers for Bifrost wallet types /////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/scripts/rewards/README.md b/scripts/rewards/README.md index 204d7979..c05ce3b9 100644 --- a/scripts/rewards/README.md +++ b/scripts/rewards/README.md @@ -99,32 +99,34 @@ Reward funds adjustment consists of the following steps: A **correct sign submission** by an eligible data provider meets exactly the following criteria: - Sent by the `submitSignature` function from the corresponding `submitSignatureAddress`. -- The payload contains a signature of the finalized Merkle root, but without the signed message (to prevent copying), and the **consensus bit-vote candidate**. +- The payload contains a signature of the finalized Merkle root, but without the signed message (to prevent copying), and the **consensus bit-vote candidate**, as an additional message. - The transaction was sent within the **signing grace period** or before the first finalization of the finalized Merkle root (the timestamp of the finalization included). The phases of FDC protocol include: -- **Collect phase** - Attestation requests are collected. Matches commit period of FTSO scaling +- **Collect phase** - Attestation requests are collected. Matches the commit period of FTSO scaling. - **Choose phase** - Bit-voting takes place. Matches the reveal period of FTSO scaling. -- **Resolve period** - signatures get submitted and when sufficient weight of signatures is deposited on-chain, finalization takes place. Matches the same signing and finalizing periods as with FTSO scaling. +- **Resolve phase** - Signatures get submitted and when sufficient weight of signatures is deposited on-chain, finalization takes place. Matches the same signing and finalizing periods as with FTSO scaling. The grace periods used in the rewarding start at the end of Choose phase and last as follows: - **signing grace period**: 10s, - **finalize grace period**: 20s. -Correct sign submissions for a voting round have attached consensus bit-vote candidates. If there exists a majority (by weight) of consensus bit-vote candidates, it is proclaimed as the **consensus bit-vote**. The majority is calculated based on the total weight of correctly signed submissions. Note that this is not the 50%+ majority of eligible data providers, but just the majority of the correct sign submissions. +Correct sign submissions for a voting round have attached consensus bit-vote candidates. If there exists a majority (by weight) of consensus bit-vote candidates, it is proclaimed as the **consensus bit-vote**. The majority is calculated based on the total weight of correct sign submissions. Note that this is not the 50%+ weight majority of eligible data providers, but just the weight majority of the correct sign submissions. -Each data provider sending a correct sign submission is considered to have sent a **correct vote**, if -- the finalized Merkle root for the round exists, +We say that **finalized Merkle root for a voting round exists** if a successful finalization of the Merkle root for a voting round `i` is done in due time, meaning before the end of the voting epoch `i + 1`. Note that a valid finalization can be done later as well, but for the purpose of rewarding this is not considered as "existence" of the Merkle root. + +Each data provider sending a correct sign submission is considered to have sent a **correct vote** in a voting round, if +- the finalized Merkle root for the voting round exists, - consensus bit-vote exists. ### Punishable violations **Punishable violations** are defined as follows: -- **No reveal on bit-vote** - bit-vote submitted and dominating the consensus bit-vote, but no reveal. +- **No reveal on bit-vote** - bit-vote submitted and dominating the consensus bit-vote, but no reveal. Domination means that a sent bit-vote contains all 1s as the consensus bit-vote. - **Wrong signature** - reveal not signing the finalized Merkle root. - **Bad consensus bit-vote candidate** - reveal submitted, signing the finalized Merkle root, but the consensus bit-vote candidate does not match the consensus bit-vote. -Note that technically an eligible data provider can submit multiple correct sign submissions, some may be correct votes, while some may be punishable violations. +Note that technically an eligible data provider can submit multiple correct sign submissions, some may be correct votes, while some may be punishable violations. Duplication of correct votes is not punishable. ### Reward distribution for a voting round @@ -133,14 +135,14 @@ The **expected reward amount** for an eligible data provider `i` with the signin `Rexp(i, N) = w(i)/W * Rfdc(N)`, where `W` denotes the total signing weight. Reward distribution for FDC is carried out as follows: -- If there is no finalized Merkle root for the voting round, `R(N)` gets burned. +- If finalized Merkle root does not exist for the voting round, `R(N)` gets burned. - Each eligible data provider that provided a correct vote and committed no punishable violations gets the reward `k * Rexp(i, N)`, where the factor `k` is as follows: - - 1 - bit-vote transaction sent and dominates the consensus bit-vote - - 0.8 - bitvote sent, but it does not dominate the consensus bitvote - - 0.8 - no bit-vote sent + - 1 - bit-vote transaction sent and dominates the consensus bit-vote, + - 0.8 - bit-vote sent, but it does not dominate the consensus bit-vote, + - 0.8 - no bit-vote sent. - All the remaining funds from `R(n)` get burned. - Each eligible data provider that committed punishable violation gets penalized by `30 * Rexp(i, N)`. -- `F(N)` be a set of selected data providers for finalization in the voting round `N` and `WF(N)` their total weight. Each selected data provider `i` who submits the correct finalized Merkle root with signatures for the voting round within the finalization grace period gets `w(i)/WF(N) * Rfin(N)` of the finalization reward. If the finalization is done within the finalization grace period, the rest of `Rfin(N)` gets burned. Otherwise, the first finalizer (might even not be a data provider) gets `Rfin(N)` as a full reward. This may be data provider or not. In case of a data provider, this is a direct claim, not shared by delegators and stakers. +- Let `F(N)` be a set of selected data providers for finalization in the voting round `N` and `WF(N)` their total weight. Each selected data provider `i` who submits the correct finalized Merkle root with signatures for the voting round within the finalization grace period gets `w(i)/WF(N) * Rfin(N)` of the finalization reward. If the finalization is done within the finalization grace period, the rest of `Rfin(N)` gets burned. Otherwise, the first finalizer (might even not be a data provider) gets `Rfin(N)` as a full reward as long as the finalization is done before the end of the voting epoch `N + 1`. This may be a data provider or not. In case of a data provider, this is a direct claim, not shared by delegators and stakers. ## Reward claims @@ -188,10 +190,10 @@ Possible values of protocolTag are: - `Reveal offenders` - penalization for reveal offenders in FTSO scaling protocol. - `Fast updates accuracy` - accuracy reward for FAST updates protocol. - `Full offer claim back` - full reward offer burning/claim back in FTSO scaling protocol. Happens when a voting round does not receive a secure reward random number and the rewarded feed cannot be chosen. Consequently, this is a reward claim to claim to the burn address. Detail tags are irrelevant in this case and are empty. -- `Partial FDC offer claim back` -- `FDC signing` -- `FDC finalization` -- `FDC offenders` +- `Partial FDC offer claim back` - Burning the share of inflation reward funds for attestation types in FDC protocol that did not receive sufficient number of confirmed attestation requests in the reward epoch. Also includes the burning of attestation request fees that did not get confirmed. +- `FDC signing` - timely signing reward claim for FDC. +- `FDC finalization` - prompt finalization reward for FDC. +- `FDC offenders` - penalization for FDC offenders. ### Reward detail tag @@ -214,15 +216,19 @@ When rewards are distributed with reward type tags Signing and Finalization to a - `DELEGATION_COMMUNITY_REWARD` - claim for a part of the signing weight of a data provider obtained from delegations. The beneficiary is the delegation address of the data provider. Claim type is `WNAT`. - `NODE_COMMUNITY_REWARD` - claim for a part of the signing weight of a data provider obtained from stakes. The beneficiary is the node id encoded into 20-bytes of a node assigned to a data provider. Claim type is `MIRROR`. -#### For `Signing` +#### For `Signing` and `FDC signing` -In case some parts of funds intended for correct signing and signature deposition get burned, one of the following [detail tags](../../libs/ftso-core/src/reward-calculation/reward-signing.ts) are used. +In case some parts of funds intended for correct signing and signature deposition get burned, one of the following [detail tags](../../libs/ftso-core/src/reward-calculation/reward-signing.ts) are used. - `NO_MOST_FREQUENT_SIGNATURES` - In case that finalization for the voting round N is not done by the end of the voting epoch `N + 1` and there are no signatures the burn claim is generated. - `NO_WEIGHT_OF_ELIGIBLE_SIGNERS` - eligible signers (the ones that earned non-zero reward from accuracy part) have total weight 0. - `CLAIM_BACK_DUE_TO_NON_ELIGIBLE_SIGNER` - reward claim for a non-eligible signer in a voting round is burned and marked with this tag. - `CLAIM_BACK_NO_CLAIMS` - if there are no reward claims by eligible signers, the remaining funds get burned and marked with this tag. +- `NO_TIMELY_FINALIZATION` - No finalization for FDC was done in due time, so no rewards can be calculated, hence all reward funds in a voting round get burned. +- `CLAIM_BACK_OF_NON_SIGNERS_SHARE` - in FDC each eligible signer would get their expected reward corresponding to its share in the total signing weight. This tag indicates the shares of the rewards for data providers which do not meet conditions for getting a reward in a voting round so their shares get burned. +- `NON_DOMINATING_BITVOTE` - if in FDC protocol a data provider submitted a non-dominating bit vote or did not submit a bit-vote a part of its reward (20%) is burned. +- `EMPTY_BITVOTE` - no consensus bit vote in FDC protocol, consequently all rewards were burned. -#### For `Finalization` +#### For `Finalization` and `FDC finalization` When finalizing the following [reward detail tags](../../libs/ftso-core/src/reward-calculation/reward-finalization.ts) are used. - `NO_FINALIZATION` - no finalization was done for the voting round N before the end of the voting epoch `N + 1`. The burn claim is generated. @@ -230,14 +236,15 @@ When finalizing the following [reward detail tags](../../libs/ftso-core/src/rewa - `FINALIZED_BUT_NO_ELIGIBLE_VOTERS` - the finalization was done during the grace period for finalization but by a non-eligible voter, while on the other hand there were no eligible finalizers submitting valid finalization data during the grace period. Hence the burn claim is generated. - `CLAIM_BACK_FOR_UNDISTRIBUTED_REWARDS` - a joint burn claim for remaining funds for rewarding prompt finalizing. E.g. some of data providers selected for finalization did not submit the finalization data or became non-eligible due to 0 accuracy rewards. -#### Fast updates accuracy +#### For `Fast updates accuracy` [Detail tags](../../libs/ftso-core/src/reward-calculation/reward-fast-updates.ts) relevant for FTSOv2 fast updates sub protocol (not FTSO scaling). - `NO_SUBMISSIONS` - no fast updates submissions during the voting epoch so nobody is eligible for rewards. A burn claim is generated. - `NO_MEDIAN_PRICE` - FTSO scaling protocol did not produce a median price so comparison with fast updates price is not possible, hence the burn claim is generated. - `MISSED_BAND` - burn claim for total voting round rewards, when the benchmark fast update price misses the prescribed band around the FTSO scaling price for specific voting round. -- `FEE` - fee part of the reward claim of a data provider -- `PARTICIPATION` - community reward for delegators earned by a data provider +- `FEE` - fee part of the reward claim of a data provider. +- `PARTICIPATION` - community reward for delegators earned by a data provider. +- `CONTRACT_CHANGE` - special case, happening only when changing FastUpdater smart contract if a version of it is not available, so system is temporarily unusable, hence the rewards for specific voting round(s) get burned. ## Calculation data