From 0a013c436fd082c20119a4d5ac16f758f0d6a2be Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 15 Jan 2020 10:45:35 +0800 Subject: [PATCH 01/28] feat: add new columns to daily statistic --- app/models/daily_statistic.rb | 6 ++++++ ...0200115020206_add_new_columns_to_daily_statistic.rb | 10 ++++++++++ db/schema.rb | 8 +++++++- 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20200115020206_add_new_columns_to_daily_statistic.rb diff --git a/app/models/daily_statistic.rb b/app/models/daily_statistic.rb index d097220c3..69c17e69e 100644 --- a/app/models/daily_statistic.rb +++ b/app/models/daily_statistic.rb @@ -22,4 +22,10 @@ class DailyStatistic < ApplicationRecord # mining_reward :string default("0") # deposit_compensation :string default("0") # treasury_amount :string default("0") +# live_cells_count :string default("0") +# dead_cells_count :string default("0") +# avg_hash_rate :string default("0") +# avg_difficulty :string default("0") +# uncle_rate :string default("0") +# total_depositors_count :string default("0") # diff --git a/db/migrate/20200115020206_add_new_columns_to_daily_statistic.rb b/db/migrate/20200115020206_add_new_columns_to_daily_statistic.rb new file mode 100644 index 000000000..2e709f5be --- /dev/null +++ b/db/migrate/20200115020206_add_new_columns_to_daily_statistic.rb @@ -0,0 +1,10 @@ +class AddNewColumnsToDailyStatistic < ActiveRecord::Migration[6.0] + def change + add_column :daily_statistics, :live_cells_count, :string, default: "0" + add_column :daily_statistics, :dead_cells_count, :string, default: "0" + add_column :daily_statistics, :avg_hash_rate, :string, default: "0" + add_column :daily_statistics, :avg_difficulty, :string, default: "0" + add_column :daily_statistics, :uncle_rate, :string, default: "0" + add_column :daily_statistics, :total_depositors_count, :string, default: "0" + end +end diff --git a/db/schema.rb b/db/schema.rb index bcb30a30f..d87511c58 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_01_10_123617) do +ActiveRecord::Schema.define(version: 2020_01_15_020206) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -172,6 +172,12 @@ t.string "mining_reward", default: "0" t.string "deposit_compensation", default: "0" t.string "treasury_amount", default: "0" + t.string "live_cells_count", default: "0" + t.string "dead_cells_count", default: "0" + t.string "avg_hash_rate", default: "0" + t.string "avg_difficulty", default: "0" + t.string "uncle_rate", default: "0" + t.string "total_depositors_count", default: "0" end create_table "dao_contracts", force: :cascade do |t| From 0e239b17a3458310fbb83707155d539dfb2ad02e Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 15 Jan 2020 19:05:11 +0800 Subject: [PATCH 02/28] feat: add more elements to daily chart --- app/models/block.rb | 2 +- app/models/daily_statistic.rb | 2 +- app/serializers/daily_statistic_serializer.rb | 20 +++++++ .../charts/daily_statistic_generator.rb | 53 ++++++++++++++++++- 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/app/models/block.rb b/app/models/block.rb index 1d6367d8a..78bf6b940 100644 --- a/app/models/block.rb +++ b/app/models/block.rb @@ -57,7 +57,7 @@ def difficulty end def block_index_in_epoch - number - start_number + number - start_number end def fraction_epoch diff --git a/app/models/daily_statistic.rb b/app/models/daily_statistic.rb index 69c17e69e..bf1c11724 100644 --- a/app/models/daily_statistic.rb +++ b/app/models/daily_statistic.rb @@ -1,5 +1,5 @@ class DailyStatistic < ApplicationRecord - VALID_INDICATORS = %w(transactions_count addresses_count total_dao_deposit).freeze + VALID_INDICATORS = %w(transactions_count addresses_count total_dao_deposit live_cells_count dead_cells_count avg_hash_rate avg_difficulty uncle_rate).freeze end # == Schema Information diff --git a/app/serializers/daily_statistic_serializer.rb b/app/serializers/daily_statistic_serializer.rb index 9fece7ac4..a68c60386 100644 --- a/app/serializers/daily_statistic_serializer.rb +++ b/app/serializers/daily_statistic_serializer.rb @@ -16,4 +16,24 @@ class DailyStatisticSerializer attribute :total_dao_deposit, if: Proc.new { |_record, params| params && params[:indicator] == "total_dao_deposit" } + + attribute :live_cells_count, if: Proc.new { |_record, params| + params && params[:indicator] == "live_cells_count" + } + + attribute :dead_cells_count, if: Proc.new { |_record, params| + params && params[:indicator] == "dead_cells_count" + } + + attribute :avg_hash_rate, if: Proc.new { |_record, params| + params && params[:indicator] == "avg_hash_rate" + } + + attribute :avg_difficulty, if: Proc.new { |_record, params| + params && params[:indicator] == "avg_difficulty" + } + + attribute :uncle_rate, if: Proc.new { |_record, params| + params && params[:indicator] == "uncle_rate" + } end diff --git a/app/services/charts/daily_statistic_generator.rb b/app/services/charts/daily_statistic_generator.rb index 1400d49ba..4f506f339 100644 --- a/app/services/charts/daily_statistic_generator.rb +++ b/app/services/charts/daily_statistic_generator.rb @@ -11,6 +11,8 @@ def initialize(datetime = nil, from_scratch = false) def call daily_ckb_transactions_count = CkbTransaction.created_after(started_at).created_before(ended_at).count + return if daily_ckb_transactions_count.zero? + cell_outputs = CellOutput.where.not(cell_type: "normal") mining_reward = Block.where("timestamp <= ?", ended_at).sum(:secondary_reward) deposit_compensation = unclaimed_compensation(cell_outputs, current_tip_block) + claimed_compensation(cell_outputs) @@ -23,13 +25,62 @@ def call dao_depositors_count: dao_depositors_count, unclaimed_compensation: unclaimed_compensation(cell_outputs, current_tip_block), claimed_compensation: claimed_compensation(cell_outputs), average_deposit_time: average_deposit_time(cell_outputs), mining_reward: mining_reward, deposit_compensation: deposit_compensation, treasury_amount: treasury_amount(cell_outputs, current_tip_block), - estimated_apc: estimated_apc) + estimated_apc: estimated_apc, live_cells_count: live_cells_count, dead_cells_count: dead_cells_count, avg_hash_rate: avg_hash_rate, + avg_difficulty: avg_difficulty, uncle_rate: uncle_rate) end private attr_reader :datetime, :from_scratch + def live_cells_count + CellOutput.unconsumed_at(ended_at).count + end + + def dead_cells_count + CellOutput.consumed_before(ended_at).count + end + + def total_blocks_count + @total_blocks_count ||= Block.created_after(started_at).created_before(ended_at).count + end + + def first_blocks_of_each_epoch + Block.created_after(started_at).created_before(ended_at).select("distinct on (epoch) * ").order(:epoch, :number).to_a + end + + def avg_hash_rate + first_block_for_the_day = Block.created_after(started_at).created_before(ended_at).recent.last + last_block_for_the_day = Block.created_after(started_at).created_before(ended_at).recent.first + total_block_time = last_block_for_the_day.timestamp - first_block_for_the_day.timestamp + + BigDecimal(total_difficulties_for_the_day) / total_block_time + end + + def avg_difficulty + BigDecimal(total_difficulties_for_the_day) / total_blocks_count + end + + def total_difficulties_for_the_day + @total_difficulties ||= + first_blocks_of_each_epoch.each_with_index.reduce(0) do |memo, (block, index)| + case index + when 0 + last_block_number_in_epoch = block.number + block.length - 1 + block.difficulty * (last_block_number_in_epoch - block.number + 1) + when first_blocks_of_each_epoch.size - 1 + block.difficulty * (block.block_index_in_epoch + 1) + else + block.difficulty * block.length + end + end + end + + def uncle_rate + uncles_count = Block.created_after(started_at).created_before(ended_at).sum(:uncles_count) + BigDecimal(uncles_count) / total_blocks_count + end + def processed_addresses_count if from_scratch Address.created_before(ended_at).count From 7abdb16c056595748f0f4d2738300511c538131b Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 15 Jan 2020 19:11:30 +0800 Subject: [PATCH 03/28] chore: just return 90 records for epoch statistic --- app/controllers/api/v1/epoch_statistics_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/v1/epoch_statistics_controller.rb b/app/controllers/api/v1/epoch_statistics_controller.rb index 4073c5251..720353bc2 100644 --- a/app/controllers/api/v1/epoch_statistics_controller.rb +++ b/app/controllers/api/v1/epoch_statistics_controller.rb @@ -4,7 +4,7 @@ class EpochStatisticsController < ApplicationController before_action :validate_query_params def show - epoch_statistics = EpochStatistic.order(id: :desc).reverse + epoch_statistics = EpochStatistic.order(epoch_number: :desc).limit(90).reverse render json: EpochStatisticSerializer.new(epoch_statistics, { params: { indicator: params[:id] } }) end From 70f3efd1d77b1e49c5e2ff46a9262a89b92dbe40 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 15 Jan 2020 19:28:00 +0800 Subject: [PATCH 04/28] chore: update daily statistic validations --- app/controllers/validations/daily_statistic.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/validations/daily_statistic.rb b/app/controllers/validations/daily_statistic.rb index dc98c6d19..69a5bf726 100644 --- a/app/controllers/validations/daily_statistic.rb +++ b/app/controllers/validations/daily_statistic.rb @@ -25,7 +25,7 @@ def error_object attr_accessor :query_key def query_key_format_must_be_correct - if query_key.blank? || !query_key.in?(::DailyStatistic::VALID_INDICATORS) + if query_key.blank? || !query_key.split("-").map { |item| item.in?(::DailyStatistic::VALID_INDICATORS) }.all?(true) errors.add(:query_key, "indicator name is invalid") end end From 1090c7ff3faca9ba17e967f7d3ec30c0797d8613 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 15 Jan 2020 19:31:44 +0800 Subject: [PATCH 05/28] chore: adjust test --- test/services/charts/daily_statistic_generator_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/services/charts/daily_statistic_generator_test.rb b/test/services/charts/daily_statistic_generator_test.rb index 768e6c310..83d844443 100644 --- a/test/services/charts/daily_statistic_generator_test.rb +++ b/test/services/charts/daily_statistic_generator_test.rb @@ -4,7 +4,7 @@ module Charts class DailyStatisticGeneratorTest < ActiveSupport::TestCase test "should create daily statistic record" do block = create(:block, dao: "0xaff1568bbe49672f8a02516252ab2300df8c9e15dad428000035a1d671700007", timestamp: (Time.current - 1.day).end_of_day.to_i * 1000) - tx = create(:ckb_transaction, block: block) + tx = create(:ckb_transaction, block: block, block_timestamp: block.timestamp) create(:cell_output, cell_type: "nervos_dao_deposit", generated_by: tx, ckb_transaction: tx, block: block, capacity: 10**8 * 1000, block_timestamp: (Time.current - 1.day).end_of_day.to_i * 1000) assert_difference -> { ::DailyStatistic.count }, 1 do Charts::DailyStatisticGenerator.new.call From 8803845fa7f08dcce4fc0b946cfdc35f1cadae11 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 15 Jan 2020 19:38:00 +0800 Subject: [PATCH 06/28] chore: update daily statistic serializer --- app/models/daily_statistic.rb | 2 +- app/serializers/daily_statistic_serializer.rb | 14 +++++++++----- app/services/charts/daily_statistic_generator.rb | 8 ++++++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/app/models/daily_statistic.rb b/app/models/daily_statistic.rb index bf1c11724..b189e9177 100644 --- a/app/models/daily_statistic.rb +++ b/app/models/daily_statistic.rb @@ -1,5 +1,5 @@ class DailyStatistic < ApplicationRecord - VALID_INDICATORS = %w(transactions_count addresses_count total_dao_deposit live_cells_count dead_cells_count avg_hash_rate avg_difficulty uncle_rate).freeze + VALID_INDICATORS = %w(transactions_count addresses_count total_dao_deposit live_cells_count dead_cells_count avg_hash_rate avg_difficulty uncle_rate total_depositors_count).freeze end # == Schema Information diff --git a/app/serializers/daily_statistic_serializer.rb b/app/serializers/daily_statistic_serializer.rb index a68c60386..88ddaec5b 100644 --- a/app/serializers/daily_statistic_serializer.rb +++ b/app/serializers/daily_statistic_serializer.rb @@ -18,22 +18,26 @@ class DailyStatisticSerializer } attribute :live_cells_count, if: Proc.new { |_record, params| - params && params[:indicator] == "live_cells_count" + params && params[:indicator].include?("live_cells_count") } attribute :dead_cells_count, if: Proc.new { |_record, params| - params && params[:indicator] == "dead_cells_count" + params && params[:indicator].include?("dead_cells_count") } attribute :avg_hash_rate, if: Proc.new { |_record, params| - params && params[:indicator] == "avg_hash_rate" + params && params[:indicator].include?("avg_hash_rate") } attribute :avg_difficulty, if: Proc.new { |_record, params| - params && params[:indicator] == "avg_difficulty" + params && params[:indicator].include?("avg_difficulty") } attribute :uncle_rate, if: Proc.new { |_record, params| - params && params[:indicator] == "uncle_rate" + params && params[:indicator].include?("uncle_rate") + } + + attribute :total_depositors_count, if: Proc.new { |_record, params| + params && params[:indicator].include?("total_depositors_count") } end diff --git a/app/services/charts/daily_statistic_generator.rb b/app/services/charts/daily_statistic_generator.rb index 4f506f339..9c1e72f86 100644 --- a/app/services/charts/daily_statistic_generator.rb +++ b/app/services/charts/daily_statistic_generator.rb @@ -26,7 +26,7 @@ def call claimed_compensation: claimed_compensation(cell_outputs), average_deposit_time: average_deposit_time(cell_outputs), mining_reward: mining_reward, deposit_compensation: deposit_compensation, treasury_amount: treasury_amount(cell_outputs, current_tip_block), estimated_apc: estimated_apc, live_cells_count: live_cells_count, dead_cells_count: dead_cells_count, avg_hash_rate: avg_hash_rate, - avg_difficulty: avg_difficulty, uncle_rate: uncle_rate) + avg_difficulty: avg_difficulty, uncle_rate: uncle_rate, total_depositors_count: total_depositors_count) end private @@ -107,7 +107,11 @@ def total_dao_deposit end def dao_depositors_count - DaoEvent.processed.new_dao_depositor.created_before(ended_at).count - DaoEvent.processed.take_away_all_deposit.created_before(ended_at).count + total_depositors_count - DaoEvent.processed.take_away_all_deposit.created_before(ended_at).count + end + + def total_depositors_count + @total_depositors_count ||= DaoEvent.processed.new_dao_depositor.created_before(ended_at).count end def to_be_counted_date From 36250cc1bc9a6ceecd933b42d7962ba062aa2713 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 16 Jan 2020 12:54:50 +0800 Subject: [PATCH 07/28] chore: update live, dead cells count calculation --- app/services/charts/daily_statistic_generator.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/services/charts/daily_statistic_generator.rb b/app/services/charts/daily_statistic_generator.rb index 9c1e72f86..2cb690f0b 100644 --- a/app/services/charts/daily_statistic_generator.rb +++ b/app/services/charts/daily_statistic_generator.rb @@ -34,11 +34,11 @@ def call attr_reader :datetime, :from_scratch def live_cells_count - CellOutput.unconsumed_at(ended_at).count + CellOutput.generated_before(ended_at).unconsumed_at(ended_at).count end def dead_cells_count - CellOutput.consumed_before(ended_at).count + CellOutput.generated_before(ended_at).consumed_before(ended_at).count end def total_blocks_count @@ -143,7 +143,7 @@ def claimed_compensation(cell_outputs) end def phase1_dao_interests(cell_outputs) - cell_outputs.nervos_dao_withdrawing.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| + cell_outputs.nervos_dao_withdrawing.generated_before(ended_at).unconsumed_at(ended_at).or().reduce(0) do |memo, nervos_dao_withdrawing_cell| memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell) end end From 6c4b86db42702fc4951af3b7ce8bb40ede551dd5 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 16 Jan 2020 12:57:36 +0800 Subject: [PATCH 08/28] chore: update daily statistic serializer --- app/serializers/daily_statistic_serializer.rb | 6 +++--- app/services/charts/daily_statistic_generator.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/serializers/daily_statistic_serializer.rb b/app/serializers/daily_statistic_serializer.rb index 88ddaec5b..d3aa9b91f 100644 --- a/app/serializers/daily_statistic_serializer.rb +++ b/app/serializers/daily_statistic_serializer.rb @@ -6,15 +6,15 @@ class DailyStatisticSerializer end attribute :transactions_count, if: Proc.new { |_record, params| - params && params[:indicator] == "transactions_count" + params && params[:indicator].include?("transactions_count") } attribute :addresses_count, if: Proc.new { |_record, params| - params && params[:indicator] == "addresses_count" + params && params[:indicator].include?("addresses_count") } attribute :total_dao_deposit, if: Proc.new { |_record, params| - params && params[:indicator] == "total_dao_deposit" + params && params[:indicator].include?("total_dao_deposit") } attribute :live_cells_count, if: Proc.new { |_record, params| diff --git a/app/services/charts/daily_statistic_generator.rb b/app/services/charts/daily_statistic_generator.rb index 2cb690f0b..f81bc96df 100644 --- a/app/services/charts/daily_statistic_generator.rb +++ b/app/services/charts/daily_statistic_generator.rb @@ -143,7 +143,7 @@ def claimed_compensation(cell_outputs) end def phase1_dao_interests(cell_outputs) - cell_outputs.nervos_dao_withdrawing.generated_before(ended_at).unconsumed_at(ended_at).or().reduce(0) do |memo, nervos_dao_withdrawing_cell| + cell_outputs.nervos_dao_withdrawing.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell) end end From 482e074d45cc50fdb27e7bb54dbc613232d8afa2 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 16 Jan 2020 18:32:29 +0800 Subject: [PATCH 09/28] chore: change total difficulties calculation --- .../charts/daily_statistic_generator.rb | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/app/services/charts/daily_statistic_generator.rb b/app/services/charts/daily_statistic_generator.rb index f81bc96df..fd4471438 100644 --- a/app/services/charts/daily_statistic_generator.rb +++ b/app/services/charts/daily_statistic_generator.rb @@ -45,8 +45,8 @@ def total_blocks_count @total_blocks_count ||= Block.created_after(started_at).created_before(ended_at).count end - def first_blocks_of_each_epoch - Block.created_after(started_at).created_before(ended_at).select("distinct on (epoch) * ").order(:epoch, :number).to_a + def epoch_numbers_for_the_day + Block.created_after(started_at).created_before(ended_at).distinct(:epoch).pluck(:epoch) end def avg_hash_rate @@ -63,16 +63,10 @@ def avg_difficulty def total_difficulties_for_the_day @total_difficulties ||= - first_blocks_of_each_epoch.each_with_index.reduce(0) do |memo, (block, index)| - case index - when 0 - last_block_number_in_epoch = block.number + block.length - 1 - block.difficulty * (last_block_number_in_epoch - block.number + 1) - when first_blocks_of_each_epoch.size - 1 - block.difficulty * (block.block_index_in_epoch + 1) - else - block.difficulty * block.length - end + epoch_numbers_for_the_day.reduce(0) do |memo, epoch_number| + first_block_of_the_epoch = Block.created_after(started_at).created_before(ended_at).where(epoch: epoch_number).recent.last + last_block_of_the_epoch = Block.created_after(started_at).created_before(ended_at).where(epoch: epoch_number).recent.first + memo + first_block_of_the_epoch.difficulty * (last_block_of_the_epoch.number - first_block_of_the_epoch.number + 1) end end @@ -123,7 +117,7 @@ def started_at end def ended_at - @ended_at ||= time_in_milliseconds(to_be_counted_date.end_of_day) + @ended_at ||= time_in_milliseconds(to_be_counted_date.end_of_day) - 1 end def unclaimed_compensation(cell_outputs, current_tip_block) From 288e165a76321c66632ccfd10300cbb35bd1f15b Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 17 Jan 2020 16:26:22 +0800 Subject: [PATCH 10/28] chore: support incremental update --- app/models/cell_output.rb | 1 + app/models/dao_event.rb | 1 + .../charts/daily_statistic_generator.rb | 65 +++++++++++++++---- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/app/models/cell_output.rb b/app/models/cell_output.rb index 4245cda15..18907ba8c 100644 --- a/app/models/cell_output.rb +++ b/app/models/cell_output.rb @@ -16,6 +16,7 @@ class CellOutput < ApplicationRecord attribute :tx_hash, :ckb_hash + scope :consumed_after, -> (block_timestamp) { where("consumed_block_timestamp >= ?", block_timestamp) } scope :consumed_before, -> (block_timestamp) { where("consumed_block_timestamp <= ?", block_timestamp) } scope :unconsumed_at, -> (block_timestamp) { where("consumed_block_timestamp > ? or consumed_block_timestamp is null", block_timestamp) } scope :generated_after, -> (block_timestamp) { where("block_timestamp >= ?", block_timestamp) } diff --git a/app/models/dao_event.rb b/app/models/dao_event.rb index e26949a39..ed28753af 100644 --- a/app/models/dao_event.rb +++ b/app/models/dao_event.rb @@ -7,6 +7,7 @@ class DaoEvent < ApplicationRecord belongs_to :ckb_transaction belongs_to :address + scope :created_after, ->(block_timestamp) { where("block_timestamp >= ?", block_timestamp) } scope :created_before, ->(block_timestamp) { where("block_timestamp <= ?", block_timestamp) } end diff --git a/app/services/charts/daily_statistic_generator.rb b/app/services/charts/daily_statistic_generator.rb index fd4471438..132fd477d 100644 --- a/app/services/charts/daily_statistic_generator.rb +++ b/app/services/charts/daily_statistic_generator.rb @@ -34,11 +34,23 @@ def call attr_reader :datetime, :from_scratch def live_cells_count - CellOutput.generated_before(ended_at).unconsumed_at(ended_at).count + if from_scratch + CellOutput.generated_before(ended_at).unconsumed_at(ended_at).count + else + CellOutput.generated_after(started_at).generated_before(ended_at).count + yesterday_daily_statistic.live_cells_count.to_i - dead_cells_count_today + end + end + + def dead_cells_count_today + @dead_cells_count_today ||= CellOutput.consumed_after(started_at).consumed_before(ended_at).count end def dead_cells_count - CellOutput.generated_before(ended_at).consumed_before(ended_at).count + if from_scratch + CellOutput.generated_before(ended_at).consumed_before(ended_at).count + else + dead_cells_count_today + yesterday_daily_statistic.dead_cells_count.to_i + end end def total_blocks_count @@ -79,7 +91,7 @@ def processed_addresses_count if from_scratch Address.created_before(ended_at).count else - Address.created_after(started_at).created_before(ended_at).count + latest_daily_statistic.addresses_count.to_i + Address.created_after(started_at).created_before(ended_at).count + yesterday_daily_statistic.addresses_count.to_i end end @@ -95,17 +107,37 @@ def current_tip_block end def total_dao_deposit - deposit_amount = DaoEvent.processed.deposit_to_dao.created_before(ended_at).sum(:value) - withdraw_amount = DaoEvent.processed.withdraw_from_dao.created_before(ended_at).sum(:value) - deposit_amount - withdraw_amount + if from_scratch + deposit_amount = DaoEvent.processed.deposit_to_dao.created_before(ended_at).sum(:value) + withdraw_amount = DaoEvent.processed.withdraw_from_dao.created_before(ended_at).sum(:value) + deposit_amount - withdraw_amount + else + deposit_amount_today = DaoEvent.processed.deposit_to_dao.created_after(started_at).created_before(ended_at).sum(:value) + withdraw_amount_today = DaoEvent.processed.withdraw_from_dao.created_after(started_at).created_before(ended_at).sum(:value) + deposit_amount_today - withdraw_amount_today + yesterday_daily_statistic.total_dao_deposit.to_i + end end def dao_depositors_count - total_depositors_count - DaoEvent.processed.take_away_all_deposit.created_before(ended_at).count + if from_scratch + total_depositors_count - DaoEvent.processed.take_away_all_deposit.created_before(ended_at).count + else + withdrawals_today = DaoEvent.processed.take_away_all_deposit.created_after(started_at).created_before(ended_at).count + new_depositors_today = DaoEvent.processed.new_dao_depositor.created_after(started_at).created_before(ended_at).count + new_depositors_today - withdrawals_today + yesterday_daily_statistic.dao_depositors_count.to_i + end end def total_depositors_count - @total_depositors_count ||= DaoEvent.processed.new_dao_depositor.created_before(ended_at).count + @total_depositors_count ||= + begin + if from_scratch + DaoEvent.processed.new_dao_depositor.created_before(ended_at).count + else + new_depositors_count_today = DaoEvent.processed.new_dao_depositor.created_after(started_at).created_before(ended_at).count + new_depositors_count_today + yesterday_daily_statistic.total_depositors_count.to_i + end + end end def to_be_counted_date @@ -130,8 +162,17 @@ def unclaimed_compensation(cell_outputs, current_tip_block) def claimed_compensation(cell_outputs) @claimed_compensation ||= begin - cell_outputs.nervos_dao_withdrawing.consumed_before(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| - memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell) + if from_scratch + cell_outputs.nervos_dao_withdrawing.consumed_before(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| + memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell) + end + else + claimed_compensation_today = + cell_outputs.nervos_dao_withdrawing.consumed_after(started_at).consumed_before(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| + memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell) + end + + claimed_compensation_today + yesterday_daily_statistic.claimed_compensation.to_i end end end @@ -182,8 +223,8 @@ def time_in_milliseconds(time) (time.to_f * 1000).floor end - def latest_daily_statistic - ::DailyStatistic.order(created_at_unixtimestamp: :desc).first || OpenStruct.new(addresses_count: 0, total_dao_deposit: 0, dao_depositors_count: 0, unclaimed_compensation: 0, claimed_compensation: 0, average_deposit_time: 0, mining_reward: 0, deposit_compensation: 0, treasury_amount: 0) + def yesterday_daily_statistic + @yesterday_daily_statistic ||= ::DailyStatistic.find_by(created_at_unixtimestamp: to_be_counted_date.yesterday.beginning_of_day.to_i) || OpenStruct.new(addresses_count: 0, total_dao_deposit: 0, dao_depositors_count: 0, unclaimed_compensation: 0, claimed_compensation: 0, average_deposit_time: 0, mining_reward: 0, deposit_compensation: 0, treasury_amount: 0, total_depositors_count: 0, live_cells_count: 0, dead_cells_count: 0) end end end From 0326be760e1ad36b8041591f7c2be429071fdc3c Mon Sep 17 00:00:00 2001 From: shaojunda Date: Mon, 20 Jan 2020 14:38:20 +0800 Subject: [PATCH 11/28] Update app/controllers/validations/daily_statistic.rb Co-Authored-By: Quake Wang --- app/controllers/validations/daily_statistic.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/validations/daily_statistic.rb b/app/controllers/validations/daily_statistic.rb index 69a5bf726..c0e7f1c7c 100644 --- a/app/controllers/validations/daily_statistic.rb +++ b/app/controllers/validations/daily_statistic.rb @@ -25,7 +25,7 @@ def error_object attr_accessor :query_key def query_key_format_must_be_correct - if query_key.blank? || !query_key.split("-").map { |item| item.in?(::DailyStatistic::VALID_INDICATORS) }.all?(true) + if query_key.blank? || !(query_key.split("-") - ::DailyStatistic::VALID_INDICATORS).empty? errors.add(:query_key, "indicator name is invalid") end end From 268858db07f0ed07618e3778bb073f77d2ea3d67 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Tue, 21 Jan 2020 16:38:05 +0800 Subject: [PATCH 12/28] feat: add claimed and unclaimed compensation --- app/models/ckb_sync/node_data_processor.rb | 4 ++-- app/models/dao_contract.rb | 9 +++------ app/serializers/dao_contract_serializer.rb | 3 +++ ...claimed_and_unclaimed_compensation_to_dao_contract.rb | 6 ++++++ db/schema.rb | 5 +++-- 5 files changed, 17 insertions(+), 10 deletions(-) create mode 100644 db/migrate/20200121083529_add_claimed_and_unclaimed_compensation_to_dao_contract.rb diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index a8711144b..b94a13918 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -78,7 +78,7 @@ def process_take_away_all_deposit(dao_contract, dao_events) def process_issue_interest(dao_contract, dao_events) issue_interest_dao_events = dao_events.where(event_type: "issue_interest") issue_interest_dao_events.each do |event| - dao_contract.increment!(:interest_granted, event.value) + dao_contract.increment!(:claimed_compensation, event.value) address = event.address address.increment!(:interest, event.value) event.processed! @@ -190,7 +190,7 @@ def revert_take_away_all_deposit(dao_contract, dao_events) def revert_issue_interest(dao_contract, dao_events) issue_interest_dao_events = dao_events.where(event_type: "issue_interest") issue_interest_dao_events.each do |event| - dao_contract.decrement!(:interest_granted, event.value) + dao_contract.decrement!(:claimed_compensation, event.value) address = event.address address.decrement!(:interest, event.value) event.reverted! diff --git a/app/models/dao_contract.rb b/app/models/dao_contract.rb index a23aa2bdc..0bde7a612 100644 --- a/app/models/dao_contract.rb +++ b/app/models/dao_contract.rb @@ -1,5 +1,5 @@ class DaoContract < ApplicationRecord - validates :total_deposit, :interest_granted, :deposit_transactions_count, :withdraw_transactions_count, :depositors_count, :total_depositors_count, presence: true, numericality: { greater_than_or_equal_to: 0 } + validates :total_deposit, :claimed_compensation, :deposit_transactions_count, :withdraw_transactions_count, :depositors_count, :total_depositors_count, presence: true, numericality: { greater_than_or_equal_to: 0 } CONTRACT_NAME = "nervos_dao".freeze GENESIS_ISSUANCE = 336 * 10**8 ANNUAL_PRIMARY_ISSUANCE_BASE = GENESIS_ISSUANCE / 8 @@ -65,10 +65,6 @@ def unclaimed_compensation latest_daily_statistic.unclaimed_compensation end - def claimed_compensation - latest_daily_statistic.claimed_compensation - end - def average_deposit_time latest_daily_statistic.average_deposit_time end @@ -134,11 +130,12 @@ def secondary_issuance(start_epoch) # # id :bigint not null, primary key # total_deposit :decimal(30, ) default(0) -# interest_granted :decimal(30, ) default(0) +# claimed_compensation :decimal(30, ) default(0) # deposit_transactions_count :bigint default(0) # withdraw_transactions_count :bigint default(0) # depositors_count :integer default(0) # total_depositors_count :bigint default(0) # created_at :datetime not null # updated_at :datetime not null +# unclaimed_compensation :decimal(30, ) # diff --git a/app/serializers/dao_contract_serializer.rb b/app/serializers/dao_contract_serializer.rb index ef7080e21..bd449c532 100644 --- a/app/serializers/dao_contract_serializer.rb +++ b/app/serializers/dao_contract_serializer.rb @@ -40,4 +40,7 @@ class DaoContractSerializer attribute :estimated_apc do |object| object.estimated_apc.to_s end + attribute :claimed_compensation do |object| + object.claimed_compensation.to_s + end end diff --git a/db/migrate/20200121083529_add_claimed_and_unclaimed_compensation_to_dao_contract.rb b/db/migrate/20200121083529_add_claimed_and_unclaimed_compensation_to_dao_contract.rb new file mode 100644 index 000000000..d7eef62ce --- /dev/null +++ b/db/migrate/20200121083529_add_claimed_and_unclaimed_compensation_to_dao_contract.rb @@ -0,0 +1,6 @@ +class AddClaimedAndUnclaimedCompensationToDaoContract < ActiveRecord::Migration[6.0] + def change + rename_column :dao_contracts, :interest_granted, :claimed_compensation + add_column :dao_contracts, :unclaimed_compensation, :decimal, precision: 30, scale: 0 + end +end diff --git a/db/schema.rb b/db/schema.rb index d87511c58..688824b83 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_01_15_020206) do +ActiveRecord::Schema.define(version: 2020_01_21_083529) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -182,13 +182,14 @@ create_table "dao_contracts", force: :cascade do |t| t.decimal "total_deposit", precision: 30, default: "0" - t.decimal "interest_granted", precision: 30, default: "0" + t.decimal "claimed_compensation", precision: 30, default: "0" t.bigint "deposit_transactions_count", default: 0 t.bigint "withdraw_transactions_count", default: 0 t.integer "depositors_count", default: 0 t.bigint "total_depositors_count", default: 0 t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false + t.decimal "unclaimed_compensation", precision: 30 end create_table "dao_events", force: :cascade do |t| From e00ca492a6f4afb1cd6d71368a984b6938952d0c Mon Sep 17 00:00:00 2001 From: shaojunda Date: Tue, 21 Jan 2020 17:30:06 +0800 Subject: [PATCH 13/28] chore: adjust tests --- test/models/ckb_sync/node_data_processor_test.rb | 8 ++++---- test/models/dao_contract_test.rb | 16 ++++------------ 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index 90e0768e9..91e851c0f 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -761,7 +761,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase tx = fake_dao_withdraw_transaction(node_block) withdraw_amount = tx.cell_outputs.nervos_dao_withdrawing.first.capacity - assert_difference -> { DaoContract.default_contract.reload.interest_granted }, "0x174876ebe8".hex - withdraw_amount do + assert_difference -> { DaoContract.default_contract.reload.claimed_compensation }, "0x174876ebe8".hex - withdraw_amount do node_data_processor.process_block(node_block) end @@ -835,7 +835,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase init_deposit_transactions_count = 2 init_withdraw_transactions_count = 1 init_total_depositors_count = 2 - dao_contract.update(total_deposit: init_total_deposit, depositors_count: init_depositors_count, interest_granted: init_interest_granted, deposit_transactions_count: init_deposit_transactions_count, withdraw_transactions_count: init_withdraw_transactions_count, total_depositors_count: init_total_depositors_count) + dao_contract.update(total_deposit: init_total_deposit, depositors_count: init_depositors_count, claimed_compensation: init_interest_granted, deposit_transactions_count: init_deposit_transactions_count, withdraw_transactions_count: init_withdraw_transactions_count, total_depositors_count: init_total_depositors_count) prepare_node_data(HAS_UNCLES_BLOCK_NUMBER) local_block = Block.find_by(number: HAS_UNCLES_BLOCK_NUMBER) local_block.update(block_hash: "0x419c632366c8eb9635acbb39ea085f7552ae62e1fdd480893375334a0f37d1bx") @@ -845,7 +845,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase dao_contract.reload assert_equal init_total_deposit, dao_contract.total_deposit assert_equal init_depositors_count, dao_contract.depositors_count - assert_equal init_interest_granted, dao_contract.interest_granted + assert_equal init_interest_granted, dao_contract.claimed_compensation assert_equal init_deposit_transactions_count, dao_contract.deposit_transactions_count assert_equal init_withdraw_transactions_count, dao_contract.withdraw_transactions_count assert_equal init_total_depositors_count, dao_contract.total_depositors_count @@ -1086,7 +1086,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase local_block.update(block_hash: "0x419c632366c8eb9635acbb39ea085f7552ae62e1fdd480893375334a0f37d1bx") VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}", record: :new_episodes) do - assert_difference -> { DaoContract.default_contract.reload.interest_granted }, -1000 do + assert_difference -> { DaoContract.default_contract.reload.claimed_compensation }, -1000 do node_data_processor.call end diff --git a/test/models/dao_contract_test.rb b/test/models/dao_contract_test.rb index 23aa0cc49..6a624a843 100644 --- a/test/models/dao_contract_test.rb +++ b/test/models/dao_contract_test.rb @@ -5,8 +5,8 @@ class DaoContractTest < ActiveSupport::TestCase should validate_presence_of(:total_deposit) should validate_numericality_of(:total_deposit). is_greater_than_or_equal_to(0) - should validate_presence_of(:interest_granted) - should validate_numericality_of(:interest_granted). + should validate_presence_of(:claimed_compensation) + should validate_numericality_of(:claimed_compensation). is_greater_than_or_equal_to(0) should validate_presence_of(:deposit_transactions_count) should validate_numericality_of(:deposit_transactions_count). @@ -24,8 +24,8 @@ class DaoContractTest < ActiveSupport::TestCase test "should have correct columns" do dao_contract = create(:dao_contract) - expected_attributes = %w(created_at deposit_transactions_count depositors_count id interest_granted total_deposit total_depositors_count updated_at withdraw_transactions_count) - assert_equal expected_attributes, dao_contract.attributes.keys.sort + expected_attributes = %w(created_at deposit_transactions_count depositors_count id claimed_compensation total_deposit total_depositors_count updated_at withdraw_transactions_count unclaimed_compensation) + assert_equal expected_attributes.sort, dao_contract.attributes.keys.sort end test "estimated apc when deposit period is less than one year" do @@ -99,14 +99,6 @@ class DaoContractTest < ActiveSupport::TestCase assert_equal latest_daily_statistic.unclaimed_compensation, dao_contract.unclaimed_compensation end - test "claimed_compensation should return beginning of today value" do - dao_contract = create(:dao_contract, total_deposit: 10**21 * 100) - create(:daily_statistic) - latest_daily_statistic = DailyStatistic.order(id: :desc).first - - assert_equal latest_daily_statistic.claimed_compensation, dao_contract.claimed_compensation - end - test "average_deposit_time should return beginning of today value" do dao_contract = create(:dao_contract, total_deposit: 10**21 * 100) create(:daily_statistic) From ba0e1057059a461325951e25bd7ba82e2241731c Mon Sep 17 00:00:00 2001 From: shaojunda Date: Tue, 21 Jan 2020 17:49:33 +0800 Subject: [PATCH 14/28] feat: add unlaimed compenstaion generator worker --- ...ntract_unclaimed_compensation_generator.rb | 29 +++++++++++++++++++ config/schedule.yml | 6 +++- 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 app/workers/dao_contract_unclaimed_compensation_generator.rb diff --git a/app/workers/dao_contract_unclaimed_compensation_generator.rb b/app/workers/dao_contract_unclaimed_compensation_generator.rb new file mode 100644 index 000000000..791137d4b --- /dev/null +++ b/app/workers/dao_contract_unclaimed_compensation_generator.rb @@ -0,0 +1,29 @@ +class DaoContractUnclaimedCompensationGenerator + include Sidekiq::Worker + + def perform + DaoContract.default_contract.update(unclaimed_compensation: cal_unclaimed_compensation) + end + + private + + def cal_unclaimed_compensation + phase1_dao_interests + unmade_dao_interests + end + + def phase1_dao_interests + CellOutput.nervos_dao_withdrawing.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| + memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell) + end + end + + def unmade_dao_interests + CellOutput.nervos_dao_deposit.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, cell_output| + dao = cell_output.block.dao + tip_dao = current_tip_block.dao + parse_dao = CkbUtils.parse_dao(dao) + tip_parse_dao = CkbUtils.parse_dao(tip_dao) + memo + (cell_output.capacity * tip_parse_dao.ar_i / parse_dao.ar_i) - cell_output.capacity + end + end +end diff --git a/config/schedule.yml b/config/schedule.yml index 074dee849..a08a6cc9a 100644 --- a/config/schedule.yml +++ b/config/schedule.yml @@ -16,4 +16,8 @@ epoch_statistic: chart_forked_event_processor: cron: "30 0 * * *" class: "Charts::ForkedEventProcessor" - queue: critical \ No newline at end of file + queue: critical + +cal_unclaimed_compensation: + cron: "0 */1 * * *" + class: "DaoContractUnclaimedCompensationGenerator" From 62e534a96f28c0c3fef70da45633985de36cc05f Mon Sep 17 00:00:00 2001 From: shaojunda Date: Tue, 21 Jan 2020 17:51:54 +0800 Subject: [PATCH 15/28] refactor: remove unnecessary params --- .../charts/daily_statistic_generator.rb | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/app/services/charts/daily_statistic_generator.rb b/app/services/charts/daily_statistic_generator.rb index 132fd477d..d612d321c 100644 --- a/app/services/charts/daily_statistic_generator.rb +++ b/app/services/charts/daily_statistic_generator.rb @@ -13,18 +13,17 @@ def call daily_ckb_transactions_count = CkbTransaction.created_after(started_at).created_before(ended_at).count return if daily_ckb_transactions_count.zero? - cell_outputs = CellOutput.where.not(cell_type: "normal") mining_reward = Block.where("timestamp <= ?", ended_at).sum(:secondary_reward) - deposit_compensation = unclaimed_compensation(cell_outputs, current_tip_block) + claimed_compensation(cell_outputs) + deposit_compensation = unclaimed_compensation + claimed_compensation estimated_apc = DaoContract.default_contract.estimated_apc(current_tip_block.fraction_epoch) block_timestamp = Block.created_after(started_at).created_before(ended_at).recent.pick(:timestamp) addresses_count = processed_addresses_count daily_statistic = ::DailyStatistic.find_or_create_by!(created_at_unixtimestamp: to_be_counted_date.to_i) daily_statistic.update(block_timestamp: block_timestamp, transactions_count: daily_ckb_transactions_count, addresses_count: addresses_count, total_dao_deposit: total_dao_deposit, - dao_depositors_count: dao_depositors_count, unclaimed_compensation: unclaimed_compensation(cell_outputs, current_tip_block), - claimed_compensation: claimed_compensation(cell_outputs), average_deposit_time: average_deposit_time(cell_outputs), - mining_reward: mining_reward, deposit_compensation: deposit_compensation, treasury_amount: treasury_amount(cell_outputs, current_tip_block), + dao_depositors_count: dao_depositors_count, unclaimed_compensation: unclaimed_compensation, + claimed_compensation: claimed_compensation, average_deposit_time: average_deposit_time, + mining_reward: mining_reward, deposit_compensation: deposit_compensation, treasury_amount: treasury_amount, estimated_apc: estimated_apc, live_cells_count: live_cells_count, dead_cells_count: dead_cells_count, avg_hash_rate: avg_hash_rate, avg_difficulty: avg_difficulty, uncle_rate: uncle_rate, total_depositors_count: total_depositors_count) end @@ -152,23 +151,23 @@ def ended_at @ended_at ||= time_in_milliseconds(to_be_counted_date.end_of_day) - 1 end - def unclaimed_compensation(cell_outputs, current_tip_block) + def unclaimed_compensation @unclaimed_compensation ||= begin - phase1_dao_interests(cell_outputs) + unmade_dao_interests(cell_outputs, current_tip_block) + phase1_dao_interests + unmade_dao_interests end end - def claimed_compensation(cell_outputs) + def claimed_compensation @claimed_compensation ||= begin if from_scratch - cell_outputs.nervos_dao_withdrawing.consumed_before(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| + CellOutput.nervos_dao_withdrawing.consumed_before(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell) end else claimed_compensation_today = - cell_outputs.nervos_dao_withdrawing.consumed_after(started_at).consumed_before(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| + CellOutput.nervos_dao_withdrawing.consumed_after(started_at).consumed_before(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell) end @@ -177,16 +176,16 @@ def claimed_compensation(cell_outputs) end end - def phase1_dao_interests(cell_outputs) - cell_outputs.nervos_dao_withdrawing.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| + def phase1_dao_interests + CellOutput.nervos_dao_withdrawing.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell) end end - def unmade_dao_interests(cell_outputs, current_tip_block) + def unmade_dao_interests @unmade_dao_interests ||= begin - cell_outputs.nervos_dao_deposit.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, cell_output| + CellOutput.nervos_dao_deposit.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, cell_output| dao = cell_output.block.dao tip_dao = current_tip_block.dao parse_dao = CkbUtils.parse_dao(dao) @@ -196,16 +195,16 @@ def unmade_dao_interests(cell_outputs, current_tip_block) end end - def average_deposit_time(cell_outputs) + def average_deposit_time interest_bearing_deposits = 0 uninterest_bearing_deposits = 0 - sum_interest_bearing = cell_outputs.nervos_dao_withdrawing.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| + sum_interest_bearing = CellOutput.nervos_dao_withdrawing.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| nervos_dao_withdrawing_cell_generated_tx = nervos_dao_withdrawing_cell.generated_by nervos_dao_deposit_cell = nervos_dao_withdrawing_cell_generated_tx.cell_inputs.order(:id)[nervos_dao_withdrawing_cell.cell_index].previous_cell_output interest_bearing_deposits += nervos_dao_deposit_cell.capacity memo + nervos_dao_deposit_cell.capacity * (nervos_dao_withdrawing_cell.block_timestamp - nervos_dao_deposit_cell.block_timestamp) / MILLISECONDS_IN_DAY end - sum_uninterest_bearing = cell_outputs.nervos_dao_deposit.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_deposit_cell| + sum_uninterest_bearing = CellOutput.nervos_dao_deposit.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_deposit_cell| uninterest_bearing_deposits += nervos_dao_deposit_cell.capacity memo + nervos_dao_deposit_cell.capacity * (ended_at - nervos_dao_deposit_cell.block_timestamp) / MILLISECONDS_IN_DAY @@ -214,9 +213,9 @@ def average_deposit_time(cell_outputs) (sum_interest_bearing + sum_uninterest_bearing) / (interest_bearing_deposits + uninterest_bearing_deposits) end - def treasury_amount(cell_outputs, current_tip_block) + def treasury_amount parse_dao = CkbUtils.parse_dao(current_tip_block.dao) - parse_dao.s_i - unmade_dao_interests(cell_outputs, current_tip_block) + parse_dao.s_i - unmade_dao_interests end def time_in_milliseconds(time) From e42b0722b002855052174a5423a17e681345b98c Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 22 Jan 2020 14:07:17 +0800 Subject: [PATCH 16/28] feat: add compensation and lock period --- app/models/ckb_transaction.rb | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/app/models/ckb_transaction.rb b/app/models/ckb_transaction.rb index bdbdf1834..4e2ed320b 100644 --- a/app/models/ckb_transaction.rb +++ b/app/models/ckb_transaction.rb @@ -89,21 +89,26 @@ def normal_tx_display_inputs(previews) previous_cell_output = cell_input.previous_cell_output display_input = { id: previous_cell_output.id, from_cellbase: false, capacity: previous_cell_output.capacity, address_hash: previous_cell_output.address_hash, generated_tx_hash: previous_cell_output.generated_by.tx_hash, cell_type: previous_cell_output.cell_type } display_input.merge!(attributes_for_dao_input(previous_cell_output)) if previous_cell_output.nervos_dao_withdrawing? - display_input.merge!(attributes_for_dao_input(cell_outputs[index])) if previous_cell_output.nervos_dao_deposit? + display_input.merge!(attributes_for_dao_input(cell_outputs[index]), false) if previous_cell_output.nervos_dao_deposit? CkbUtils.hash_value_to_s(display_input) end end end - def attributes_for_dao_input(nervos_dao_withdrawing_cell) + def attributes_for_dao_input(nervos_dao_withdrawing_cell, is_phase2 = true) nervos_dao_withdrawing_cell_generated_tx = nervos_dao_withdrawing_cell.generated_by nervos_dao_deposit_cell = nervos_dao_withdrawing_cell_generated_tx.cell_inputs.order(:id)[nervos_dao_withdrawing_cell.cell_index].previous_cell_output - started_block_number = Block.find(nervos_dao_deposit_cell.block.id).number - ended_block_number = Block.find(block_id).number + compensation_started_block = Block.select(:number, :timestamp).find(nervos_dao_deposit_cell.block.id) + compensation_ended_block = Block.select(:number, :timestamp).find(nervos_dao_withdrawing_cell_generated_tx.block_id) interest = CkbUtils.dao_interest(nervos_dao_withdrawing_cell) + attributes = { compensation_started_block_number: compensation_started_block.number, compensation_ended_block_number: compensation_ended_block.number, compensation_started_timestamp: compensation_ended_block.timestamp, compensation_ended_timestamp: compensation_started_timestamp.timestamp, interest: interest } + if is_phase2 + locked_until_block = Block.select(:number, :timestamp).find(block_id) + attributes.merge({ locked_until_block_number: locked_until_block.number, locked_until_block_timestamp: locked_until_block.timestamp }) + end - CkbUtils.hash_value_to_s({ started_block_number: started_block_number, ended_block_number: ended_block_number, interest: interest, dao_type_hash: ENV["DAO_TYPE_HASH"] }) + CkbUtils.hash_value_to_s(attributes) end def cellbase_display_inputs From 81469df70fbd5d43a2b6f40aa5f49f3c78148876 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 22 Jan 2020 14:27:59 +0800 Subject: [PATCH 17/28] feat: add average deposit time to address --- app/models/address.rb | 1 + .../20200122060907_add_average_deposit_time_to_address.rb | 5 +++++ db/schema.rb | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20200122060907_add_average_deposit_time_to_address.rb diff --git a/app/models/address.rb b/app/models/address.rb index 7d2baa0b4..ac8d023fa 100644 --- a/app/models/address.rb +++ b/app/models/address.rb @@ -80,6 +80,7 @@ def special? # visible :boolean default(TRUE) # live_cells_count :decimal(30, ) default(0) # mined_blocks_count :integer default(0) +# average_deposit_time :decimal(, ) # # Indexes # diff --git a/db/migrate/20200122060907_add_average_deposit_time_to_address.rb b/db/migrate/20200122060907_add_average_deposit_time_to_address.rb new file mode 100644 index 000000000..ab12e192f --- /dev/null +++ b/db/migrate/20200122060907_add_average_deposit_time_to_address.rb @@ -0,0 +1,5 @@ +class AddAverageDepositTimeToAddress < ActiveRecord::Migration[6.0] + def change + add_column :addresses, :average_deposit_time, :decimal + end +end diff --git a/db/schema.rb b/db/schema.rb index 688824b83..c4b54f25a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_01_21_083529) do +ActiveRecord::Schema.define(version: 2020_01_22_060907) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -38,6 +38,7 @@ t.boolean "visible", default: true t.decimal "live_cells_count", precision: 30, default: "0" t.integer "mined_blocks_count", default: 0 + t.decimal "average_deposit_time" t.index ["address_hash"], name: "index_addresses_on_address_hash" t.index ["lock_hash"], name: "index_addresses_on_lock_hash", unique: true end From 06181385c0da6118e851109ce09d31de367e6315 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 22 Jan 2020 14:28:13 +0800 Subject: [PATCH 18/28] feat: add address average deposit time generator --- app/models/ckb_transaction.rb | 6 ++-- .../address_average_deposit_time_generator.rb | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 app/workers/address_average_deposit_time_generator.rb diff --git a/app/models/ckb_transaction.rb b/app/models/ckb_transaction.rb index 4e2ed320b..bbfefc5f8 100644 --- a/app/models/ckb_transaction.rb +++ b/app/models/ckb_transaction.rb @@ -89,7 +89,7 @@ def normal_tx_display_inputs(previews) previous_cell_output = cell_input.previous_cell_output display_input = { id: previous_cell_output.id, from_cellbase: false, capacity: previous_cell_output.capacity, address_hash: previous_cell_output.address_hash, generated_tx_hash: previous_cell_output.generated_by.tx_hash, cell_type: previous_cell_output.cell_type } display_input.merge!(attributes_for_dao_input(previous_cell_output)) if previous_cell_output.nervos_dao_withdrawing? - display_input.merge!(attributes_for_dao_input(cell_outputs[index]), false) if previous_cell_output.nervos_dao_deposit? + display_input.merge!(attributes_for_dao_input(cell_outputs[index], false)) if previous_cell_output.nervos_dao_deposit? CkbUtils.hash_value_to_s(display_input) end @@ -102,10 +102,10 @@ def attributes_for_dao_input(nervos_dao_withdrawing_cell, is_phase2 = true) compensation_started_block = Block.select(:number, :timestamp).find(nervos_dao_deposit_cell.block.id) compensation_ended_block = Block.select(:number, :timestamp).find(nervos_dao_withdrawing_cell_generated_tx.block_id) interest = CkbUtils.dao_interest(nervos_dao_withdrawing_cell) - attributes = { compensation_started_block_number: compensation_started_block.number, compensation_ended_block_number: compensation_ended_block.number, compensation_started_timestamp: compensation_ended_block.timestamp, compensation_ended_timestamp: compensation_started_timestamp.timestamp, interest: interest } + attributes = { compensation_started_block_number: compensation_started_block.number, compensation_ended_block_number: compensation_ended_block.number, compensation_started_timestamp: compensation_ended_block.timestamp, compensation_ended_timestamp: compensation_ended_block.timestamp, interest: interest } if is_phase2 locked_until_block = Block.select(:number, :timestamp).find(block_id) - attributes.merge({ locked_until_block_number: locked_until_block.number, locked_until_block_timestamp: locked_until_block.timestamp }) + attributes.merge!({ locked_until_block_number: locked_until_block.number, locked_until_block_timestamp: locked_until_block.timestamp }) end CkbUtils.hash_value_to_s(attributes) diff --git a/app/workers/address_average_deposit_time_generator.rb b/app/workers/address_average_deposit_time_generator.rb new file mode 100644 index 000000000..5671d250b --- /dev/null +++ b/app/workers/address_average_deposit_time_generator.rb @@ -0,0 +1,34 @@ +class AddressAverageDepositTimeGenerator + include Sidekiq::Worker + + def perform(address_id) + address = Address.find(address_id) + address.update(average_deposit_time: average_deposit_time) + end + + private + + def average_deposit_time + interest_bearing_deposits = 0 + uninterest_bearing_deposits = 0 + milliseconds_in_day = BigDecimal(24 * 60 * 60 * 1000) + ended_at = time_in_milliseconds(Time.current) + sum_interest_bearing = address.cell_outputs.nervos_dao_withdrawing.unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| + nervos_dao_withdrawing_cell_generated_tx = nervos_dao_withdrawing_cell.generated_by + nervos_dao_deposit_cell = nervos_dao_withdrawing_cell_generated_tx.cell_inputs.order(:id)[nervos_dao_withdrawing_cell.cell_index].previous_cell_output + interest_bearing_deposits += nervos_dao_deposit_cell.capacity + memo + nervos_dao_deposit_cell.capacity * (nervos_dao_withdrawing_cell.block_timestamp - nervos_dao_deposit_cell.block_timestamp) / milliseconds_in_day + end + sum_uninterest_bearing = address.cell_outputs.nervos_dao_deposit.unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_deposit_cell| + uninterest_bearing_deposits += nervos_dao_deposit_cell.capacity + + memo + nervos_dao_deposit_cell.capacity * (ended_at - nervos_dao_deposit_cell.block_timestamp) / milliseconds_in_day + end + + (sum_interest_bearing + sum_uninterest_bearing) / (interest_bearing_deposits + uninterest_bearing_deposits) + end + + def time_in_milliseconds(time) + (time.to_f * 1000).floor + end +end From dae8f24ff78231e4985e8e31f62a58d9bfb492db Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 22 Jan 2020 15:27:40 +0800 Subject: [PATCH 19/28] chore: update api doc --- doc/api.raml | 45 +++++++++++++++++++++++++++++++++++---------- public/api_doc.html | 28 +++++++++++++++++----------- 2 files changed, 52 insertions(+), 21 deletions(-) diff --git a/doc/api.raml b/doc/api.raml index 56882ecb1..668d9d967 100644 --- a/doc/api.raml +++ b/doc/api.raml @@ -127,9 +127,12 @@ types: }, "pending_reward_blocks_count": { "type": integer - } + }, "lock_info": { "type": lock_info_attributes + }, + "average_deposit_time": { + "type": string } } } @@ -222,12 +225,28 @@ types: "cell_type": { type: "string" }, - "started_block_number": { - type: integer, + "compensation_started_block_number": { + type: string, required: false }, - "ended_block_number": { - type: integer, + "compensation_ended_block_number": { + type: string, + required: false + }, + "compensation_started_timestamp": { + type: string, + required: false + }, + "compensation_ended_timestamp": { + type: string, + required: false + }, + "locked_until_block_number": { + type: string, + required: false + }, + "locked_until_block_timestamp": { + type: string, required: false }, "interest": { @@ -1259,8 +1278,12 @@ types: "address_hash": "0xddbd21e6e51674bb961bb4c5f3cee9faa5da30e64be10628dc1cef292cbae323", "capacity": 100, "generated_tx_hash": "0x3abd21e6e51674bb961bb4c5f3cee9faa5da30e64be10628dc1cef292cbae321", - "started_block_number": 10, - "ended_block_number": 15, + "compensation_started_block_number": "10", + "compensation_ended_block_number": "15", + "compensation_started_timestamp": "1579592269056" + "compensation_ended_timestamp": "1579592269056", + "locked_until_block_number": "608395", + "locked_until_block_timestamp": "1579665264926", "interest": 100000, "cell_type": "dao", "dao_type_hash": "0xa20df8e80518e9b2eabc1a0efb0ebe1de83f8df9c867edf99d0c5895654fcde1", @@ -2022,7 +2045,8 @@ types: "epoch_number": "1000", "epoch_index": "789", "estimated_unlock_time": "1588259390000" - } + }, + "average_deposit_time": "35.780193591686053852850446361429702611876162589037114569656468227898621305950821369" } } } @@ -2502,7 +2526,8 @@ types: "code_hash": "0x0000000000000000000000000000000000000000000000000000000000000001" }, "dao_deposit": "10000", - "interest": "100" + "interest": "100", + "average_deposit_time": "35.780193591686053852850446361429702611876162589037114569656468227898621305950821369" } } } @@ -3320,4 +3345,4 @@ types: example: | { "25374730228.31050194" - } \ No newline at end of file + } diff --git a/public/api_doc.html b/public/api_doc.html index 56fb953c2..3e41a5f5a 100644 --- a/public/api_doc.html +++ b/public/api_doc.html @@ -371,7 +371,7 @@ "last": "https://explorer.nervos.org/api/v1/transactions?page=28909&page_size=15" } } -
get

Returns a transaction by transaction hash

get

Returns a transaction by transaction hash

/address_transactions

get

Returns transactions related to an address

/address_transactions

get

Returns transactions related to an address

/block_transactions

get

Returns transactions in the block

/block_transactions

get

Returns transactions in the block

/addresses

get

Returns address info

/addresses

get

Returns address info

/contract_transactions

get

Returns transactions under the contract

/contract_transactions

get

Returns transactions under the contract

/dao_contract_transactions

get

Returns the transaction related to dao contract by given transaction hash

/dao_contract_transactions

get

Returns the transaction related to dao contract by given transaction hash

/address_dao_transactions

get

Returns transactions under the address by given address hash

/address_dao_transactions

get

Returns transactions under the address by given address hash