From 3823620464c8c45e2104aa9a20883fc355dbe28b Mon Sep 17 00:00:00 2001 From: shaojunda Date: Mon, 3 Aug 2020 21:45:07 +0800 Subject: [PATCH 01/76] refactor: move route not found to errors controller --- app/controllers/application_controller.rb | 7 ------- app/controllers/errors_controller.rb | 5 +++++ config/routes.rb | 3 +-- test/controllers/errors_controller_test.rb | 7 +++++++ 4 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 app/controllers/errors_controller.rb create mode 100644 test/controllers/errors_controller_test.rb diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 02448e75e..9b03f780e 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -2,18 +2,11 @@ class ApplicationController < ActionController::API before_action :check_header_info, :set_raven_context rescue_from Api::V1::Exceptions::Error, with: :api_error - rescue_from ActionController::RoutingError do |exception| - render json: { message: exception.message }, status: :not_found - end def homepage render json: { message: "Please read more API info at https://github.com/nervosnetwork/ckb-explorer/" } end - def catch_404 - raise ActionController::RoutingError.new(params[:path]) - end - private def set_raven_context diff --git a/app/controllers/errors_controller.rb b/app/controllers/errors_controller.rb new file mode 100644 index 000000000..3717f8cab --- /dev/null +++ b/app/controllers/errors_controller.rb @@ -0,0 +1,5 @@ +class ErrorsController < ActionController::API + def routing_error + render json: { message: "Not Found" }, status: :not_found + end +end diff --git a/config/routes.rb b/config/routes.rb index b046b4a1b..42fea2d10 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -7,7 +7,6 @@ Sidekiq::Web.set :session_secret, Rails.application.credentials[:secret_key_base] root "application#homepage" - namespace :api do namespace :v1 do namespace :external do @@ -46,5 +45,5 @@ end end - match "*path", to: "application#catch_404", via: :all + match "/:anything" => "errors#routing_error", via: :all, constraints: { anything: /.*/ } end diff --git a/test/controllers/errors_controller_test.rb b/test/controllers/errors_controller_test.rb new file mode 100644 index 000000000..63c6dd51d --- /dev/null +++ b/test/controllers/errors_controller_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class ErrorsControllerTest < ActionDispatch::IntegrationTest + # test "the truth" do + # assert true + # end +end From ea1798af3b05efa039a39a4212ffcbeae8f0476c Mon Sep 17 00:00:00 2001 From: shaojunda Date: Mon, 3 Aug 2020 22:40:19 +0800 Subject: [PATCH 02/76] test: routing error --- .../api/v1/application_controller_test.rb | 2 +- test/controllers/errors_controller_test.rb | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/test/controllers/api/v1/application_controller_test.rb b/test/controllers/api/v1/application_controller_test.rb index c02f0091c..e206efbbe 100644 --- a/test/controllers/api/v1/application_controller_test.rb +++ b/test/controllers/api/v1/application_controller_test.rb @@ -14,7 +14,7 @@ class ApplicationControllerTest < ActionDispatch::IntegrationTest valid_get "/app" assert_response :not_found - assert_equal "app", json["message"] + assert_equal "Not Found", json["message"] end end end diff --git a/test/controllers/errors_controller_test.rb b/test/controllers/errors_controller_test.rb index 63c6dd51d..1857d2c6b 100644 --- a/test/controllers/errors_controller_test.rb +++ b/test/controllers/errors_controller_test.rb @@ -1,7 +1,16 @@ -require 'test_helper' +require "test_helper" class ErrorsControllerTest < ActionDispatch::IntegrationTest - # test "the truth" do - # assert true - # end + test "should return 404 status code when path is not exist" do + post "/" + + assert_response :not_found + end + + test "should return Not Found message when path is not exist" do + post "/oop" + + assert_response :not_found + assert_equal "Not Found", json["message"] + end end From 0bf5f76d68e687bf475854a21899a70c946e4d52 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Tue, 4 Aug 2020 19:44:48 +0800 Subject: [PATCH 03/76] feat: add table record count --- .../fast_jsonapi/pagination_meta_generator.rb | 14 ++++++++++---- app/models/table_record_count.rb | 17 +++++++++++++++++ ...20200803152507_create_table_record_counts.rb | 12 ++++++++++++ db/schema.rb | 10 +++++++++- test/models/table_record_count_test.rb | 7 +++++++ 5 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 app/models/table_record_count.rb create mode 100644 db/migrate/20200803152507_create_table_record_counts.rb create mode 100644 test/models/table_record_count_test.rb diff --git a/app/lib/fast_jsonapi/pagination_meta_generator.rb b/app/lib/fast_jsonapi/pagination_meta_generator.rb index 037c938c6..4a1c3a17b 100644 --- a/app/lib/fast_jsonapi/pagination_meta_generator.rb +++ b/app/lib/fast_jsonapi/pagination_meta_generator.rb @@ -3,13 +3,19 @@ class PaginationMetaGenerator DEFAULT_PAGE = 1 DEFAULT_PER_PAGE = 20 - def initialize(request:, records:, page:, page_size:) + def initialize(request:, records:, page:, page_size:, records_counter: nil) @url = request.base_url + request.path + query_string(request.query_parameters) @page = page.to_i @page_size = limit_page_size(records, page_size.to_i) - @total_pages = records.total_pages @records = records - @hash = { links: {}, meta: { total: records.total_count, page_size: @page_size } } + @records_counter = records_counter.presence || records + @total_count = @records_counter.total_count + @total_pages = total_pages + @hash = { links: {}, meta: { total: @total_count, page_size: @page_size } } + end + + def total_pages + (total_count / @page_size).ceil end def call @@ -30,7 +36,7 @@ def call private - attr_reader :page, :page_size, :records, :total_pages + attr_reader :page, :page_size, :records, :records_counter, :total_count attr_accessor :url, :hash def generate_url(page) diff --git a/app/models/table_record_count.rb b/app/models/table_record_count.rb new file mode 100644 index 000000000..7d2be7989 --- /dev/null +++ b/app/models/table_record_count.rb @@ -0,0 +1,17 @@ +class TableRecordCount < ApplicationRecord +end + +# == Schema Information +# +# Table name: table_record_counts +# +# id :bigint not null, primary key +# table_name :string +# count :bigint +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_table_record_counts_on_table_name_and_count (table_name,count) +# diff --git a/db/migrate/20200803152507_create_table_record_counts.rb b/db/migrate/20200803152507_create_table_record_counts.rb new file mode 100644 index 000000000..f344d9a0d --- /dev/null +++ b/db/migrate/20200803152507_create_table_record_counts.rb @@ -0,0 +1,12 @@ +class CreateTableRecordCounts < ActiveRecord::Migration[6.0] + def change + create_table :table_record_counts do |t| + t.string :table_name + t.bigint :count + + t.timestamps + end + + add_index :table_record_counts, [:table_name, :count] + end +end diff --git a/db/schema.rb b/db/schema.rb index 43c1abdc3..35ef6564c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_07_29_174146) do +ActiveRecord::Schema.define(version: 2020_08_03_152507) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -349,6 +349,14 @@ t.index ["block_number"], name: "index_mining_infos_on_block_number" end + create_table "table_record_counts", force: :cascade do |t| + t.string "table_name" + t.bigint "count" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["table_name", "count"], name: "index_table_record_counts_on_table_name_and_count" + end + create_table "transaction_propagation_delays", force: :cascade do |t| t.string "tx_hash" t.integer "created_at_unixtimestamp" diff --git a/test/models/table_record_count_test.rb b/test/models/table_record_count_test.rb new file mode 100644 index 000000000..f7515659b --- /dev/null +++ b/test/models/table_record_count_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class TableRecordCountTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end From dbf7fa565fa83e856e6728bc6ae414482392a1ea Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 5 Aug 2020 17:39:43 +0800 Subject: [PATCH 04/76] feat: add ckb_transactions_count to udts --- app/models/udt.rb | 39 ++++++++++--------- ...3044_add_ckb_transactions_count_to_udts.rb | 5 +++ db/schema.rb | 3 +- 3 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 db/migrate/20200805093044_add_ckb_transactions_count_to_udts.rb diff --git a/app/models/udt.rb b/app/models/udt.rb index 65c055b58..f9584418b 100644 --- a/app/models/udt.rb +++ b/app/models/udt.rb @@ -34,25 +34,26 @@ def type_script # # Table name: udts # -# id :bigint not null, primary key -# code_hash :binary -# hash_type :string -# args :string -# type_hash :string -# full_name :string -# symbol :string -# decimal :integer -# description :string -# icon_file :string -# operator_website :string -# addresses_count :decimal(30, ) default(0) -# total_amount :decimal(40, ) default(0) -# udt_type :integer -# published :boolean default(FALSE) -# created_at :datetime not null -# updated_at :datetime not null -# block_timestamp :decimal(30, ) -# issuer_address :binary +# id :bigint not null, primary key +# code_hash :binary +# hash_type :string +# args :string +# type_hash :string +# full_name :string +# symbol :string +# decimal :integer +# description :string +# icon_file :string +# operator_website :string +# addresses_count :decimal(30, ) default(0) +# total_amount :decimal(40, ) default(0) +# udt_type :integer +# published :boolean default(FALSE) +# created_at :datetime not null +# updated_at :datetime not null +# block_timestamp :decimal(30, ) +# issuer_address :binary +# ckb_transactions_count :decimal(30, ) default(0) # # Indexes # diff --git a/db/migrate/20200805093044_add_ckb_transactions_count_to_udts.rb b/db/migrate/20200805093044_add_ckb_transactions_count_to_udts.rb new file mode 100644 index 000000000..896ed8d72 --- /dev/null +++ b/db/migrate/20200805093044_add_ckb_transactions_count_to_udts.rb @@ -0,0 +1,5 @@ +class AddCkbTransactionsCountToUdts < ActiveRecord::Migration[6.0] + def change + add_column :udts, :ckb_transactions_count, :decimal, precision: 30, default: 0 + end +end diff --git a/db/schema.rb b/db/schema.rb index 35ef6564c..ee750c3f0 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_08_03_152507) do +ActiveRecord::Schema.define(version: 2020_08_05_093044) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -413,6 +413,7 @@ t.datetime "updated_at", precision: 6, null: false t.decimal "block_timestamp", precision: 30 t.binary "issuer_address" + t.decimal "ckb_transactions_count", precision: 30, default: "0" t.index ["type_hash"], name: "index_udts_on_type_hash", unique: true end From b2ff6ae7171594d7c092f3032948e7e8c6693e80 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 5 Aug 2020 17:40:44 +0800 Subject: [PATCH 05/76] test: should update udt's ckb_transactions_count after process block data --- test/models/ckb_sync/node_data_processor_test.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index 13967e5a3..cbd6fba99 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -2722,6 +2722,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase outputs1 = [ CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock1, type: udt_script1), CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock2, type: udt_script2), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock2, type: udt_script2), CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock3) ] miner_lock = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") @@ -2734,7 +2735,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase transactions = [ CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: cellbase_inputs, outputs: cellbase_outputs, outputs_data: %w[0x], witnesses: ["0x590000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd800000000"]), CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs, outputs: outputs, outputs_data: %W[#{CKB::Utils.generate_sudt_amount(1000)} #{CKB::Utils.generate_sudt_amount(1000)} 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]), - CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs1, outputs: outputs1, outputs_data: %W[#{CKB::Utils.generate_sudt_amount(1000)} #{CKB::Utils.generate_sudt_amount(1000)} 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]), + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs1, outputs: outputs1, outputs_data: %W[#{CKB::Utils.generate_sudt_amount(1000)} #{CKB::Utils.generate_sudt_amount(1000)} #{CKB::Utils.generate_sudt_amount(1000)} 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]), ] node_block = CKB::Types::Block.new(uncles: [], proposals: [], transactions: transactions, header: header) block = node_data_processor.process_block(node_block) @@ -2745,6 +2746,8 @@ class NodeDataProcessorTest < ActiveSupport::TestCase assert_equal [udt1.id, udt2.id], tx.contained_udt_ids assert_equal [udt1.id, udt2.id], tx1.contained_udt_ids + assert_equal 2, udt1.ckb_transactions_count + assert_equal 2, udt2.ckb_transactions_count end test "#process_block should update tx's contained_udt_ids when there are udt cells in inputs" do @@ -2821,6 +2824,8 @@ class NodeDataProcessorTest < ActiveSupport::TestCase assert_equal [udt1.id, udt2.id], tx.contained_udt_ids assert_equal [udt1.id, udt2.id], tx1.contained_udt_ids + assert_equal 3, udt1.ckb_transactions_count + assert_equal 2, udt2.ckb_transactions_count end private From e9076c3279bfa2bab0c6bd8604a44d9439622602 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 5 Aug 2020 17:43:26 +0800 Subject: [PATCH 06/76] feat: update udt's ckb_transactions_count on build cell_output --- app/models/ckb_sync/node_data_processor.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index a5c04793b..2334cfe10 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -402,6 +402,7 @@ def build_ckb_transactions(local_block, transactions, outputs, new_dao_depositor ckb_transaction.contained_address_ids += address_ids.to_a ckb_transaction.tags += tags.to_a ckb_transaction.contained_udt_ids += udt_ids.to_a + Udt.where(id: udt_ids.to_a).map { |udt| udt.increment!(:ckb_transactions_count) } ckb_transaction end @@ -455,7 +456,8 @@ def build_cell_outputs(node_outputs, ckb_transaction, addresses, outputs_data, o if cell_output.udt? udt_infos << { type_script: output.type, address: address } tags << "udt" - udt_ids << Udt.find_or_create_by!(type_hash: output.type.compute_hash, code_hash: ENV["SUDT_CELL_TYPE_HASH"], udt_type: "sudt").id + udt = Udt.find_or_create_by!(type_hash: output.type.compute_hash, code_hash: ENV["SUDT_CELL_TYPE_HASH"], udt_type: "sudt") + udt_ids << udt.id end if cell_output.nervos_dao_deposit? || cell_output.nervos_dao_withdrawing? tags << "dao" From 4baef105af6e2123c7a941bf6b887635b5549593 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 5 Aug 2020 23:54:22 +0800 Subject: [PATCH 07/76] test: should update udt's ckb_transactions_count when inputs has udt cells --- test/models/ckb_sync/node_data_processor_test.rb | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index cbd6fba99..c61679f44 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -2705,7 +2705,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase ] inputs1 = [ CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx4.tx_hash, index: 0)), - CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx5.tx_hash, index: 0)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx5.tx_hash, index: 0)) ] lock1 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") lock2 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") @@ -2735,7 +2735,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase transactions = [ CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: cellbase_inputs, outputs: cellbase_outputs, outputs_data: %w[0x], witnesses: ["0x590000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd800000000"]), CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs, outputs: outputs, outputs_data: %W[#{CKB::Utils.generate_sudt_amount(1000)} #{CKB::Utils.generate_sudt_amount(1000)} 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]), - CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs1, outputs: outputs1, outputs_data: %W[#{CKB::Utils.generate_sudt_amount(1000)} #{CKB::Utils.generate_sudt_amount(1000)} #{CKB::Utils.generate_sudt_amount(1000)} 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]), + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs1, outputs: outputs1, outputs_data: %W[#{CKB::Utils.generate_sudt_amount(1000)} #{CKB::Utils.generate_sudt_amount(1000)} #{CKB::Utils.generate_sudt_amount(1000)} 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]) ] node_block = CKB::Types::Block.new(uncles: [], proposals: [], transactions: transactions, header: header) block = node_data_processor.process_block(node_block) @@ -2780,6 +2780,11 @@ class NodeDataProcessorTest < ActiveSupport::TestCase create(:type_script, args: udt_script1.args, code_hash: ENV["SUDT_CELL_TYPE_HASH"], hash_type: "data", cell_output: output3) create(:type_script, args: udt_script1.args, code_hash: ENV["SUDT_CELL_TYPE_HASH"], hash_type: "data", cell_output: output4) create(:type_script, args: udt_script2.args, code_hash: ENV["SUDT_CELL_TYPE_HASH"], hash_type: "data", cell_output: output5) + output1.update(type_hash: CKB::Types::Script.new(output1.type_script.to_node_type).compute_hash) + output2.update(type_hash: CKB::Types::Script.new(output2.type_script.to_node_type).compute_hash) + output3.update(type_hash: CKB::Types::Script.new(output3.type_script.to_node_type).compute_hash) + output4.update(type_hash: CKB::Types::Script.new(output4.type_script.to_node_type).compute_hash) + output5.update(type_hash: CKB::Types::Script.new(output5.type_script.to_node_type).compute_hash) Address.create(lock_hash: udt_script1.args, address_hash: "0x#{SecureRandom.hex(32)}") Address.create(lock_hash: udt_script2.args, address_hash: "0x#{SecureRandom.hex(32)}") @@ -2812,7 +2817,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase transactions = [ CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: cellbase_inputs, outputs: cellbase_outputs, outputs_data: %w[0x], witnesses: ["0x590000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd800000000"]), CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs, outputs: outputs, outputs_data: %w[0x 0x 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]), - CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs1, outputs: outputs, outputs_data: %w[0x 0x 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]), + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs1, outputs: outputs, outputs_data: %w[0x 0x 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]) ] node_block = CKB::Types::Block.new(uncles: [], proposals: [], transactions: transactions, header: header) block = node_data_processor.process_block(node_block) @@ -2824,7 +2829,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase assert_equal [udt1.id, udt2.id], tx.contained_udt_ids assert_equal [udt1.id, udt2.id], tx1.contained_udt_ids - assert_equal 3, udt1.ckb_transactions_count + assert_equal 2, udt1.ckb_transactions_count assert_equal 2, udt2.ckb_transactions_count end From a4b16b6baa23d39162137b87e08aa91b7184f700 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 00:01:16 +0800 Subject: [PATCH 08/76] feat: update udt's ckb_transactions_count when inputs has udt cells --- app/models/ckb_sync/node_data_processor.rb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index 2334cfe10..7f0407ff9 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -586,10 +586,19 @@ def update_tx_fee_related_data(local_block, input_capacities, udt_infos) account_books << account_book end + udt_ids = updated_ckb_transactions.pluck(:contained_udt_ids).flatten + udt_counts = udt_ids.each_with_object(Hash.new(0)) { |udt_id, counts| counts[udt_id] += 1 } + udt_counts_value = + udt_counts.map do |udt_id, count| + udt = Udt.find(udt_id) + { id: udt_id, ckb_transactions_count: count, created_at: udt.created_at, updated_at: Time.current } + end + CellInput.import!(updated_inputs, validate: false, on_duplicate_key_update: [:previous_cell_output_id]) CellOutput.import!(updated_outputs, validate: false, on_duplicate_key_update: [:consumed_by_id, :status, :consumed_block_timestamp]) AccountBook.import!(account_books, validate: false) CkbTransaction.upsert_all(updated_ckb_transactions.uniq { |tx| tx[:id] }) + Udt.upsert_all(udt_counts_value) if udt_counts_value.present? end input_cache_keys = updated_inputs.map(&:cache_keys) output_cache_keys = updated_outputs.map(&:cache_keys) @@ -618,7 +627,7 @@ def update_ckb_transaction(consumed_tx, address_id, previous_cell_output, update consumed_tx.contained_address_ids << address_id if previous_cell_output.udt? consumed_tx.tags << "udt" - consumed_tx.contained_udt_ids << Udt.find_or_create_by!(type_hash: previous_cell_output.node_output.type.compute_hash, code_hash: ENV["SUDT_CELL_TYPE_HASH"], udt_type: "sudt").id + consumed_tx.contained_udt_ids << Udt.find_or_create_by!(type_hash: previous_cell_output.type_hash, code_hash: ENV["SUDT_CELL_TYPE_HASH"], udt_type: "sudt").id end if previous_cell_output.nervos_dao_withdrawing? consumed_tx.tags << "dao" From 14e4c27b33f01e01a512bdeec96c459b35847881 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 01:19:58 +0800 Subject: [PATCH 09/76] feat: recalculate udts ckb_transactions_count when block forked --- app/models/ckb_sync/node_data_processor.rb | 16 +- .../ckb_sync/node_data_processor_test.rb | 154 ++++++++++++++++++ 2 files changed, 168 insertions(+), 2 deletions(-) diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index 7f0407ff9..745d31228 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -199,6 +199,7 @@ def invalid_block(local_tip_block) revert_dao_contract_related_operations(local_tip_block) revert_mining_info(local_tip_block) udt_type_hashes = local_tip_block.cell_outputs.udt.pluck(:type_hash).uniq + recalculate_udt_transactions_count(local_tip_block) local_tip_block.invalid! recalculate_udt_accounts(udt_type_hashes, local_tip_block) local_tip_block.contained_addresses.each(&method(:update_address_balance_and_ckb_transactions_count)) @@ -210,6 +211,18 @@ def invalid_block(local_tip_block) end end + def recalculate_udt_transactions_count(local_tip_block) + udt_ids = local_tip_block.ckb_transactions.where("tags @> array[?]::varchar[]", ["udt"]).pluck(:contained_udt_ids).flatten + udt_counts = udt_ids.each_with_object(Hash.new(0)) { |udt_id, counts| counts[udt_id] += 1 } + udt_counts_value = + udt_counts.map do |udt_id, count| + udt = Udt.find(udt_id) + { id: udt_id, ckb_transactions_count: udt.ckb_transactions_count - count, created_at: udt.created_at, updated_at: Time.current } + end + + Udt.upsert_all(udt_counts_value) if udt_counts_value.present? + end + def recalculate_udt_accounts(udt_type_hashes, local_tip_block) return if udt_type_hashes.blank? @@ -402,7 +415,6 @@ def build_ckb_transactions(local_block, transactions, outputs, new_dao_depositor ckb_transaction.contained_address_ids += address_ids.to_a ckb_transaction.tags += tags.to_a ckb_transaction.contained_udt_ids += udt_ids.to_a - Udt.where(id: udt_ids.to_a).map { |udt| udt.increment!(:ckb_transactions_count) } ckb_transaction end @@ -591,7 +603,7 @@ def update_tx_fee_related_data(local_block, input_capacities, udt_infos) udt_counts_value = udt_counts.map do |udt_id, count| udt = Udt.find(udt_id) - { id: udt_id, ckb_transactions_count: count, created_at: udt.created_at, updated_at: Time.current } + { id: udt_id, ckb_transactions_count: udt.ckb_transactions_count + count, created_at: udt.created_at, updated_at: Time.current } end CellInput.import!(updated_inputs, validate: false, on_duplicate_key_update: [:previous_cell_output_id]) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index c61679f44..6d1cc6f72 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -2833,6 +2833,160 @@ class NodeDataProcessorTest < ActiveSupport::TestCase assert_equal 2, udt2.ckb_transactions_count end + test "should recalculate udts ckb transactions count when block is invalid and outputs has udt cell" do + block1 = create(:block, :with_block_hash, number: DEFAULT_NODE_BLOCK_NUMBER - 2) + tx1 = create(:ckb_transaction, block: block1) + block2 = create(:block, :with_block_hash, number: DEFAULT_NODE_BLOCK_NUMBER - 1) + tx2 = create(:ckb_transaction, block: block2) + tx3 = create(:ckb_transaction, block: block2) + tx4 = create(:ckb_transaction, block: block2) + tx5 = create(:ckb_transaction, block: block2) + input_address1 = create(:address) + input_address2 = create(:address) + input_address3 = create(:address) + input_address4 = create(:address) + input_address5 = create(:address) + create(:cell_output, ckb_transaction: tx1, generated_by: tx1, block: block1, capacity: 50000 * 10**8, tx_hash: tx1.tx_hash, cell_index: 0, address: input_address1) + create(:cell_output, ckb_transaction: tx2, generated_by: tx2, block: block2, capacity: 60000 * 10**8, tx_hash: tx2.tx_hash, cell_index: 1, address: input_address2) + create(:cell_output, ckb_transaction: tx3, generated_by: tx3, block: block2, capacity: 70000 * 10**8, tx_hash: tx3.tx_hash, cell_index: 2, address: input_address3) + create(:cell_output, ckb_transaction: tx4, generated_by: tx4, block: block1, capacity: 50000 * 10**8, tx_hash: tx4.tx_hash, cell_index: 0, address: input_address4) + create(:cell_output, ckb_transaction: tx5, generated_by: tx5, block: block2, capacity: 60000 * 10**8, tx_hash: tx5.tx_hash, cell_index: 0, address: input_address5) + header = CKB::Types::BlockHeader.new(compact_target: "0x1000", hash: "0x#{SecureRandom.hex(32)}", number: DEFAULT_NODE_BLOCK_NUMBER, parent_hash: "0x#{SecureRandom.hex(32)}", nonce: 1757392074788233522, timestamp: CkbUtils.time_in_milliseconds(Time.current), transactions_root: "0x#{SecureRandom.hex(32)}", proposals_hash: "0x#{SecureRandom.hex(32)}", uncles_hash: "0x#{SecureRandom.hex(32)}", version: 0, epoch: 1, dao: "0x01000000000000000000c16ff286230000a3a65e97fd03000057c138586f0000") + inputs = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx1.tx_hash, index: 0)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx2.tx_hash, index: 1)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx3.tx_hash, index: 2)) + ] + inputs1 = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx4.tx_hash, index: 0)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx5.tx_hash, index: 0)) + ] + lock1 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + lock2 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + lock3 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + udt_script1 = CKB::Types::Script.new(code_hash: ENV["SUDT_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(32)}") + udt_script2 = CKB::Types::Script.new(code_hash: ENV["SUDT_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(32)}") + Address.create(lock_hash: udt_script1.args, address_hash: "0x#{SecureRandom.hex(32)}") + Address.create(lock_hash: udt_script2.args, address_hash: "0x#{SecureRandom.hex(32)}") + outputs = [ + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock1, type: udt_script1), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock2, type: udt_script2), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock3) + ] + outputs1 = [ + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock1, type: udt_script1), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock2, type: udt_script2), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock2, type: udt_script2), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock3) + ] + miner_lock = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + cellbase_inputs = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: "0x0000000000000000000000000000000000000000000000000000000000000000", index: 4294967295), since: 3000) + ] + cellbase_outputs = [ + CKB::Types::Output.new(capacity: 200986682127, lock: miner_lock) + ] + transactions = [ + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: cellbase_inputs, outputs: cellbase_outputs, outputs_data: %w[0x], witnesses: ["0x590000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd800000000"]), + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs, outputs: outputs, outputs_data: %W[#{CKB::Utils.generate_sudt_amount(1000)} #{CKB::Utils.generate_sudt_amount(1000)} 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]), + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs1, outputs: outputs1, outputs_data: %W[#{CKB::Utils.generate_sudt_amount(1000)} #{CKB::Utils.generate_sudt_amount(1000)} #{CKB::Utils.generate_sudt_amount(1000)} 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]) + ] + node_block = CKB::Types::Block.new(uncles: [], proposals: [], transactions: transactions, header: header) + block = node_data_processor.process_block(node_block) + CkbSync::Api.any_instance.stubs(:get_tip_block_number).returns(block.number + 1) + VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}", record: :new_episodes) do + node_data_processor.call + end + + udt1 = Udt.find_by(args: udt_script1.args) + udt2 = Udt.find_by(args: udt_script2.args) + + assert_equal 0, udt1.reload.ckb_transactions_count + assert_equal 0, udt2.reload.ckb_transactions_count + end + + test "should recalculate udts ckb transactions count when block is invalid and inputs has udt cell" do + block1 = create(:block, :with_block_hash, number: DEFAULT_NODE_BLOCK_NUMBER - 2) + tx1 = create(:ckb_transaction, block: block1) + block2 = create(:block, :with_block_hash, number: DEFAULT_NODE_BLOCK_NUMBER - 1) + tx2 = create(:ckb_transaction, block: block2) + tx3 = create(:ckb_transaction, block: block2) + tx4 = create(:ckb_transaction, block: block2) + tx5 = create(:ckb_transaction, block: block2) + input_address1 = create(:address) + input_address2 = create(:address) + input_address3 = create(:address) + input_address4 = create(:address) + input_address5 = create(:address) + address1_lock = create(:lock_script, address: input_address1, args: "0x#{SecureRandom.hex(20)}", code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type") + address2_lock = create(:lock_script, address: input_address2, args: "0x#{SecureRandom.hex(20)}", code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type") + address3_lock = create(:lock_script, address: input_address3, args: "0x#{SecureRandom.hex(20)}", code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type") + address4_lock = create(:lock_script, address: input_address3, args: "0x#{SecureRandom.hex(20)}", code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type") + address5_lock = create(:lock_script, address: input_address3, args: "0x#{SecureRandom.hex(20)}", code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type") + output1 = create(:cell_output, ckb_transaction: tx1, generated_by: tx1, block: block1, capacity: 50000 * 10**8, tx_hash: tx1.tx_hash, cell_index: 0, address: input_address1, cell_type: "udt", lock_script: address1_lock) + output2 = create(:cell_output, ckb_transaction: tx2, generated_by: tx2, block: block2, capacity: 60000 * 10**8, tx_hash: tx2.tx_hash, cell_index: 1, address: input_address2, cell_type: "udt", lock_script: address2_lock) + output3 = create(:cell_output, ckb_transaction: tx3, generated_by: tx3, block: block2, capacity: 70000 * 10**8, tx_hash: tx3.tx_hash, cell_index: 2, address: input_address3, cell_type: "udt", lock_script: address3_lock) + output4 = create(:cell_output, ckb_transaction: tx4, generated_by: tx4, block: block2, capacity: 70000 * 10**8, tx_hash: tx4.tx_hash, cell_index: 0, address: input_address4, cell_type: "udt", lock_script: address4_lock) + output5 = create(:cell_output, ckb_transaction: tx5, generated_by: tx5, block: block2, capacity: 70000 * 10**8, tx_hash: tx5.tx_hash, cell_index: 0, address: input_address5, cell_type: "udt", lock_script: address5_lock) + udt_script1 = CKB::Types::Script.new(code_hash: ENV["SUDT_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(32)}") + udt_script2 = CKB::Types::Script.new(code_hash: ENV["SUDT_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(32)}") + create(:type_script, args: udt_script1.args, code_hash: ENV["SUDT_CELL_TYPE_HASH"], hash_type: "data", cell_output: output1) + create(:type_script, args: udt_script2.args, code_hash: ENV["SUDT_CELL_TYPE_HASH"], hash_type: "data", cell_output: output2) + create(:type_script, args: udt_script1.args, code_hash: ENV["SUDT_CELL_TYPE_HASH"], hash_type: "data", cell_output: output3) + create(:type_script, args: udt_script1.args, code_hash: ENV["SUDT_CELL_TYPE_HASH"], hash_type: "data", cell_output: output4) + create(:type_script, args: udt_script2.args, code_hash: ENV["SUDT_CELL_TYPE_HASH"], hash_type: "data", cell_output: output5) + output1.update(type_hash: CKB::Types::Script.new(output1.type_script.to_node_type).compute_hash) + output2.update(type_hash: CKB::Types::Script.new(output2.type_script.to_node_type).compute_hash) + output3.update(type_hash: CKB::Types::Script.new(output3.type_script.to_node_type).compute_hash) + output4.update(type_hash: CKB::Types::Script.new(output4.type_script.to_node_type).compute_hash) + output5.update(type_hash: CKB::Types::Script.new(output5.type_script.to_node_type).compute_hash) + Address.create(lock_hash: udt_script1.args, address_hash: "0x#{SecureRandom.hex(32)}") + Address.create(lock_hash: udt_script2.args, address_hash: "0x#{SecureRandom.hex(32)}") + + header = CKB::Types::BlockHeader.new(compact_target: "0x1000", hash: "0x#{SecureRandom.hex(32)}", number: DEFAULT_NODE_BLOCK_NUMBER, parent_hash: "0x#{SecureRandom.hex(32)}", nonce: 1757392074788233522, timestamp: CkbUtils.time_in_milliseconds(Time.current), transactions_root: "0x#{SecureRandom.hex(32)}", proposals_hash: "0x#{SecureRandom.hex(32)}", uncles_hash: "0x#{SecureRandom.hex(32)}", version: 0, epoch: 1, dao: "0x01000000000000000000c16ff286230000a3a65e97fd03000057c138586f0000") + inputs = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx1.tx_hash, index: 0)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx2.tx_hash, index: 1)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx3.tx_hash, index: 2)) + ] + inputs1 = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx4.tx_hash, index: 0)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx5.tx_hash, index: 0)) + ] + lock1 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + lock2 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + lock3 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + + outputs = [ + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock1), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock2), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock3) + ] + miner_lock = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + cellbase_inputs = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: "0x0000000000000000000000000000000000000000000000000000000000000000", index: 4294967295), since: 3000) + ] + cellbase_outputs = [ + CKB::Types::Output.new(capacity: 200986682127, lock: miner_lock) + ] + transactions = [ + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: cellbase_inputs, outputs: cellbase_outputs, outputs_data: %w[0x], witnesses: ["0x590000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd800000000"]), + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs, outputs: outputs, outputs_data: %w[0x 0x 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]), + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs1, outputs: outputs, outputs_data: %w[0x 0x 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]) + ] + node_block = CKB::Types::Block.new(uncles: [], proposals: [], transactions: transactions, header: header) + block = node_data_processor.process_block(node_block) + udt1 = Udt.find_by(args: udt_script1.args) + udt2 = Udt.find_by(args: udt_script2.args) + + CkbSync::Api.any_instance.stubs(:get_tip_block_number).returns(block.number + 1) + VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}", record: :new_episodes) do + node_data_processor.call + end + assert_equal 0, udt1.reload.ckb_transactions_count + assert_equal 0, udt2.reload.ckb_transactions_count + end + private def node_data_processor From 961974969e8b20bd5a26814d115b417837a43c17 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 01:40:23 +0800 Subject: [PATCH 10/76] test: record counters base instance should respond to total_count --- test/models/record_counters/base_test.rb | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 test/models/record_counters/base_test.rb diff --git a/test/models/record_counters/base_test.rb b/test/models/record_counters/base_test.rb new file mode 100644 index 000000000..e745a6994 --- /dev/null +++ b/test/models/record_counters/base_test.rb @@ -0,0 +1,8 @@ +require "test_helper" + +class BaseTest < ActiveSupport::TestCase + test "should respond to total_count" do + base = RecordCounters::Base.new + assert_respond_to base, :total_count + end +end \ No newline at end of file From 8c38d366540b3330fe079b994e22deb2ebf4c36e Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 01:40:39 +0800 Subject: [PATCH 11/76] feat: add base record counter --- app/models/record_counters/base.rb | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 app/models/record_counters/base.rb diff --git a/app/models/record_counters/base.rb b/app/models/record_counters/base.rb new file mode 100644 index 000000000..6169cb55b --- /dev/null +++ b/app/models/record_counters/base.rb @@ -0,0 +1,7 @@ +module RecordCounters + class Base + def total_count + raise "Not implemented" + end + end +end From 1a2041314854976dbe5aaacb57d7969e9d510272 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 02:02:36 +0800 Subject: [PATCH 12/76] test: udt transactions counter --- app/models/record_counters/base.rb | 2 +- .../record_counters/udt_transactions_test.rb | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 test/models/record_counters/udt_transactions_test.rb diff --git a/app/models/record_counters/base.rb b/app/models/record_counters/base.rb index 6169cb55b..28ff2d6fd 100644 --- a/app/models/record_counters/base.rb +++ b/app/models/record_counters/base.rb @@ -1,7 +1,7 @@ module RecordCounters class Base def total_count - raise "Not implemented" + raise "Not Implemented" end end end diff --git a/test/models/record_counters/udt_transactions_test.rb b/test/models/record_counters/udt_transactions_test.rb new file mode 100644 index 000000000..78cf188d6 --- /dev/null +++ b/test/models/record_counters/udt_transactions_test.rb @@ -0,0 +1,15 @@ +require "test_helper" + +class UdtTransactionsTest < ActiveSupport::TestCase + test "should respond to total_count" do + udt = create(:udt, published: true) + udt_transactions_counter = RecordCounters::UdtTransactions.new(udt) + assert_respond_to udt_transactions_counter, :total_count + end + + test "total_count should return udt's ckb transactions count" do + udt = create(:udt, published: true) + udt_transactions_counter = RecordCounters::UdtTransactions.new(udt) + assert_equal udt.ckb_transactions_count, udt_transactions_counter.total_count + end +end \ No newline at end of file From ac455ead2e5f13c82413bf0c26e38ad80d5e8a4f Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 02:02:50 +0800 Subject: [PATCH 13/76] feat: add udt transactions counter --- app/models/record_counters/udt_transactions.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 app/models/record_counters/udt_transactions.rb diff --git a/app/models/record_counters/udt_transactions.rb b/app/models/record_counters/udt_transactions.rb new file mode 100644 index 000000000..e7568839e --- /dev/null +++ b/app/models/record_counters/udt_transactions.rb @@ -0,0 +1,15 @@ +module RecordCounters + class UdtTransactions < Base + def initialize(udt) + @udt = udt + end + + def total_count + udt.ckb_transactions_count + end + + private + + attr_reader :udt + end +end From dcb9d6ba5f0cc40bebbe8a4b7a81ed823f9a2df4 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 02:08:49 +0800 Subject: [PATCH 14/76] chore: use udt_transactions_counter on pagination --- app/controllers/api/v1/udt_transactions_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/v1/udt_transactions_controller.rb b/app/controllers/api/v1/udt_transactions_controller.rb index ee6fb91db..1d4f99538 100644 --- a/app/controllers/api/v1/udt_transactions_controller.rb +++ b/app/controllers/api/v1/udt_transactions_controller.rb @@ -9,7 +9,8 @@ def show ckb_transactions = udt.ckb_transactions.select(:id, :tx_hash, :block_id, :block_number, :block_timestamp, :is_cellbase).recent.page(@page).per(@page_size) json = Rails.cache.realize(ckb_transactions.cache_key, version: ckb_transactions.cache_version) do - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: @page, page_size: @page_size).call + records_counter = RecordCounters::UdtTransactions.new(udt) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: @page, page_size: @page_size, records_counter: records_counter).call CkbTransactionsSerializer.new(ckb_transactions, options.merge(params: { previews: true })).serialized_json end From 24d31b251eb2bc9f7980b5d813b15be43aa1293a Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 02:09:13 +0800 Subject: [PATCH 15/76] chore: adjust udt factory --- test/factories/udt.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/factories/udt.rb b/test/factories/udt.rb index 5cdb0ca47..48c82f3a5 100644 --- a/test/factories/udt.rb +++ b/test/factories/udt.rb @@ -17,6 +17,7 @@ transaction1 = create(:ckb_transaction, block: block, contained_udt_ids: [udt.id], tags: ["udt"]) create(:cell_output, block: block, ckb_transaction: transaction, generated_by: transaction, consumed_by: transaction1, type_hash: udt.type_hash, cell_type: "udt", data: "0x000050ad321ea12e0000000000000000") end + udt.update(ckb_transactions_count: 40) end end end From 0cab96fd92ef4a6d64a546dcddf5db6bd6af9464 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 02:18:22 +0800 Subject: [PATCH 16/76] feat: add task to fill ckb_transactions_count to udt --- .../fill_ckb_transactions_count_to_udt.rake | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 lib/tasks/migration/fill_ckb_transactions_count_to_udt.rake diff --git a/lib/tasks/migration/fill_ckb_transactions_count_to_udt.rake b/lib/tasks/migration/fill_ckb_transactions_count_to_udt.rake new file mode 100644 index 000000000..79e5fa582 --- /dev/null +++ b/lib/tasks/migration/fill_ckb_transactions_count_to_udt.rake @@ -0,0 +1,21 @@ +namespace :migration do + desc "Usage: RAILS_ENV=production bundle exec rake migration:fill_ckb_transactions_count_to_udt" + task fill_ckb_transactions_count_to_udt: :environment do + progress_bar = ProgressBar.create({ + total: Udt.count, + format: "%e %B %p%% %c/%C" + }) + + values = + Udt.all.map do |udt| + progress_bar.increment + ckb_transactions_count = udt.ckb_transactions.count + + { id: udt.id, ckb_transactions_count: ckb_transactions_count, created_at: udt.created_at, updated_at: Time.current } + end + + Udt.upsert_all(values) if values.present? + + puts "done" + end +end From 4231860fee2be275ba8752a12db4e11a4f42baac Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 14:01:48 +0800 Subject: [PATCH 17/76] feat: add ckb_transactions_count to dao_contracts --- app/models/dao_contract.rb | 1 + ...0806060029_add_ckb_transactions_count_to_dao_contracts.rb | 5 +++++ db/schema.rb | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20200806060029_add_ckb_transactions_count_to_dao_contracts.rb diff --git a/app/models/dao_contract.rb b/app/models/dao_contract.rb index 014416dce..bbd6eebb8 100644 --- a/app/models/dao_contract.rb +++ b/app/models/dao_contract.rb @@ -130,4 +130,5 @@ def secondary_issuance(start_epoch) # created_at :datetime not null # updated_at :datetime not null # unclaimed_compensation :decimal(30, ) +# ckb_transactions_count :decimal(30, ) default(0) # diff --git a/db/migrate/20200806060029_add_ckb_transactions_count_to_dao_contracts.rb b/db/migrate/20200806060029_add_ckb_transactions_count_to_dao_contracts.rb new file mode 100644 index 000000000..2938948f6 --- /dev/null +++ b/db/migrate/20200806060029_add_ckb_transactions_count_to_dao_contracts.rb @@ -0,0 +1,5 @@ +class AddCkbTransactionsCountToDaoContracts < ActiveRecord::Migration[6.0] + def change + add_column :dao_contracts, :ckb_transactions_count, :decimal, precision: 30, default: 0 + end +end diff --git a/db/schema.rb b/db/schema.rb index ee750c3f0..e57b3090d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_08_05_093044) do +ActiveRecord::Schema.define(version: 2020_08_06_060029) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -244,6 +244,7 @@ t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false t.decimal "unclaimed_compensation", precision: 30 + t.decimal "ckb_transactions_count", precision: 30, default: "0" end create_table "dao_events", force: :cascade do |t| From 3d60d6b21c3b7c723ddd53ded7291aacbec444ed Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 14:05:35 +0800 Subject: [PATCH 18/76] test: should update dao contract ckb_transactions_count on process_block --- test/models/ckb_sync/node_data_processor_test.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index 6d1cc6f72..c09d3bb82 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -2196,6 +2196,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase assert_equal ["dao"], tx.tags assert_equal ["dao"], tx1.tags + assert_equal 2, DaoContract.default_contract.ckb_transactions_count end test "should update tx's tags when output have nervos_dao_withdrawing cells" do @@ -2260,6 +2261,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase tx1 = block.ckb_transactions.where(is_cellbase: false).second assert_equal ["dao"], tx.tags assert_equal ["dao"], tx1.tags + assert_equal 2, DaoContract.default_contract.ckb_transactions_count end test "should update tx's tags when input have nervos_dao_withdrawing cells" do @@ -2337,6 +2339,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase assert_equal ["dao"], tx.tags assert_equal ["dao"], tx1.tags + assert_equal 2, DaoContract.default_contract.ckb_transactions_count end test "should update tx's tags when output have udt cells" do @@ -2432,6 +2435,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase tx = block.ckb_transactions.where(is_cellbase: false).first assert_equal %w[dao udt], tx.tags + assert_equal 1, DaoContract.default_contract.ckb_transactions_count end test "should update tx's tags when output have udt cells and nervos_dao_withdrawing cell" do @@ -2480,6 +2484,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase tx = block.ckb_transactions.where(is_cellbase: false).first assert_equal %w[dao udt], tx.tags + assert_equal 1, DaoContract.default_contract.ckb_transactions_count end test "should update tx's tags when input have udt cells" do @@ -2655,6 +2660,8 @@ class NodeDataProcessorTest < ActiveSupport::TestCase assert_equal %w[dao udt], tx.tags assert_equal %w[dao udt], tx1.tags + + assert_equal 2, DaoContract.default_contract.ckb_transactions_count end test "#process_block should not update tx's tags when there aren't dao cells and udt cells" do From 050a3d538915ced83f26b912e24faa399d498db2 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 14:17:02 +0800 Subject: [PATCH 19/76] feat: update dao contract ckb_transactions_count on process_block --- app/models/ckb_sync/node_data_processor.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index 745d31228..b0aa9588b 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -606,6 +606,9 @@ def update_tx_fee_related_data(local_block, input_capacities, udt_infos) { id: udt_id, ckb_transactions_count: udt.ckb_transactions_count + count, created_at: udt.created_at, updated_at: Time.current } end + dao_tx_count = updated_ckb_transactions.select { |tx| tx[:tags].include?("dao") }.count + DaoContract.default_contract.increment!(:ckb_transactions_count, dao_tx_count) + CellInput.import!(updated_inputs, validate: false, on_duplicate_key_update: [:previous_cell_output_id]) CellOutput.import!(updated_outputs, validate: false, on_duplicate_key_update: [:consumed_by_id, :status, :consumed_block_timestamp]) AccountBook.import!(account_books, validate: false) From 5c9495b7446debf25ed3883a4fa7365feb92697f Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 14:20:29 +0800 Subject: [PATCH 20/76] refactor: extract udt_counts_value method --- app/models/ckb_sync/node_data_processor.rb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index b0aa9588b..161f65ed4 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -598,13 +598,7 @@ def update_tx_fee_related_data(local_block, input_capacities, udt_infos) account_books << account_book end - udt_ids = updated_ckb_transactions.pluck(:contained_udt_ids).flatten - udt_counts = udt_ids.each_with_object(Hash.new(0)) { |udt_id, counts| counts[udt_id] += 1 } - udt_counts_value = - udt_counts.map do |udt_id, count| - udt = Udt.find(udt_id) - { id: udt_id, ckb_transactions_count: udt.ckb_transactions_count + count, created_at: udt.created_at, updated_at: Time.current } - end + udt_counts_value = udt_counts_value(updated_ckb_transactions) dao_tx_count = updated_ckb_transactions.select { |tx| tx[:tags].include?("dao") }.count DaoContract.default_contract.increment!(:ckb_transactions_count, dao_tx_count) @@ -621,6 +615,15 @@ def update_tx_fee_related_data(local_block, input_capacities, udt_infos) end end + def udt_counts_value(updated_ckb_transactions) + udt_ids = updated_ckb_transactions.pluck(:contained_udt_ids).flatten + udt_counts = udt_ids.each_with_object(Hash.new(0)) { |udt_id, counts| counts[udt_id] += 1 } + udt_counts.map do |udt_id, count| + udt = Udt.find(udt_id) + {id: udt_id, ckb_transactions_count: udt.ckb_transactions_count + count, created_at: udt.created_at, updated_at: Time.current} + end + end + def flush_caches(cache_keys) cache_keys.each_slice(400) do |keys| $redis.pipelined do From 5ce56619defb351bc196134b5966553c773b2c98 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 14:27:41 +0800 Subject: [PATCH 21/76] test: should recalculate dao contract ckb_transactions_count when block is invalid and has dao txs --- .../ckb_sync/node_data_processor_test.rb | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index c09d3bb82..f6736b036 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -2199,6 +2199,71 @@ class NodeDataProcessorTest < ActiveSupport::TestCase assert_equal 2, DaoContract.default_contract.ckb_transactions_count end + test "should recalculate dao contract ckb_transactions_count when block is invalid and has dao txs" do + block1 = create(:block, :with_block_hash, number: DEFAULT_NODE_BLOCK_NUMBER - 2) + tx1 = create(:ckb_transaction, block: block1) + block2 = create(:block, :with_block_hash, number: DEFAULT_NODE_BLOCK_NUMBER - 1) + tx2 = create(:ckb_transaction, block: block2) + tx3 = create(:ckb_transaction, block: block2) + tx4 = create(:ckb_transaction, block: block2) + tx5 = create(:ckb_transaction, block: block2) + input_address1 = create(:address) + input_address2 = create(:address) + input_address3 = create(:address) + input_address4 = create(:address) + input_address5 = create(:address) + create(:cell_output, ckb_transaction: tx1, generated_by: tx1, block: block1, capacity: 50000 * 10**8, tx_hash: tx1.tx_hash, cell_index: 0, address: input_address1) + create(:cell_output, ckb_transaction: tx2, generated_by: tx2, block: block2, capacity: 60000 * 10**8, tx_hash: tx2.tx_hash, cell_index: 1, address: input_address2) + create(:cell_output, ckb_transaction: tx3, generated_by: tx3, block: block2, capacity: 70000 * 10**8, tx_hash: tx3.tx_hash, cell_index: 2, address: input_address3) + create(:cell_output, ckb_transaction: tx4, generated_by: tx4, block: block2, capacity: 70000 * 10**8, tx_hash: tx4.tx_hash, cell_index: 0, address: input_address4) + create(:cell_output, ckb_transaction: tx5, generated_by: tx5, block: block2, capacity: 70000 * 10**8, tx_hash: tx5.tx_hash, cell_index: 0, address: input_address5) + header = CKB::Types::BlockHeader.new(compact_target: "0x1000", hash: "0x#{SecureRandom.hex(32)}", number: DEFAULT_NODE_BLOCK_NUMBER, parent_hash: "0x#{SecureRandom.hex(32)}", nonce: 1757392074788233522, timestamp: CkbUtils.time_in_milliseconds(Time.current), transactions_root: "0x#{SecureRandom.hex(32)}", proposals_hash: "0x#{SecureRandom.hex(32)}", uncles_hash: "0x#{SecureRandom.hex(32)}", version: 0, epoch: 1, dao: "0x01000000000000000000c16ff286230000a3a65e97fd03000057c138586f0000") + inputs = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx1.tx_hash, index: 0)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx2.tx_hash, index: 1)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx3.tx_hash, index: 2)) + ] + inputs1 = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx4.tx_hash, index: 0)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx5.tx_hash, index: 0)) + ] + lock1 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + lock2 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + lock3 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + dao_type = CKB::Types::Script.new(code_hash: ENV["DAO_TYPE_HASH"], hash_type: "type", args: "0x") + outputs = [ + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock1, type: dao_type), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock2, type: dao_type), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock3) + ] + outputs1 = [ + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock1, type: dao_type), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock2, type: dao_type), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock3) + ] + miner_lock = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + cellbase_inputs = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: "0x0000000000000000000000000000000000000000000000000000000000000000", index: 4294967295), since: 3000) + ] + cellbase_outputs = [ + CKB::Types::Output.new(capacity: 200986682127, lock: miner_lock) + ] + transactions = [ + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: cellbase_inputs, outputs: cellbase_outputs, outputs_data: %w[0x], witnesses: ["0x590000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd800000000"]), + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs, outputs: outputs, outputs_data: %w[0x0000000000000000 0x0000000000000000 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]), + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs1, outputs: outputs1, outputs_data: %w[0x0000000000000000 0x0000000000000000 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]) + ] + node_block = CKB::Types::Block.new(uncles: [], proposals: [], transactions: transactions, header: header) + block = node_data_processor.process_block(node_block) + CkbSync::Api.any_instance.stubs(:get_tip_block_number).returns(block.number + 1) + + VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}", record: :new_episodes) do + assert_changes -> { DaoContract.default_contract.ckb_transactions_count }, from: 2, to: 0 do + node_data_processor.call + end + end + end + test "should update tx's tags when output have nervos_dao_withdrawing cells" do block1 = create(:block, :with_block_hash, number: DEFAULT_NODE_BLOCK_NUMBER - 2) tx1 = create(:ckb_transaction, block: block1) From e02f7e8519b6929de4fce3bca21d5ccead92cc82 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 14:28:48 +0800 Subject: [PATCH 22/76] feat: recalculate dao contract ckb_transactions_count when block is invalid and has dao txs --- app/models/ckb_sync/node_data_processor.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index 161f65ed4..6cdd0db37 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -200,6 +200,7 @@ def invalid_block(local_tip_block) revert_mining_info(local_tip_block) udt_type_hashes = local_tip_block.cell_outputs.udt.pluck(:type_hash).uniq recalculate_udt_transactions_count(local_tip_block) + recalculate_dao_contract_transactions_count(local_tip_block) local_tip_block.invalid! recalculate_udt_accounts(udt_type_hashes, local_tip_block) local_tip_block.contained_addresses.each(&method(:update_address_balance_and_ckb_transactions_count)) @@ -211,6 +212,11 @@ def invalid_block(local_tip_block) end end + def recalculate_dao_contract_transactions_count(local_tip_block) + dao_transactions_count = local_tip_block.ckb_transactions.where("tags @> array[?]::varchar[]", ["dao"]).count + DaoContract.default_contract.decrement!(:ckb_transactions_count, dao_transactions_count) if dao_transactions_count > 0 + end + def recalculate_udt_transactions_count(local_tip_block) udt_ids = local_tip_block.ckb_transactions.where("tags @> array[?]::varchar[]", ["udt"]).pluck(:contained_udt_ids).flatten udt_counts = udt_ids.each_with_object(Hash.new(0)) { |udt_id, counts| counts[udt_id] += 1 } From 64278a8e086971729a40c482d66b4379e7f3341c Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 14:31:04 +0800 Subject: [PATCH 23/76] feat: add task to fill ckb_transactions_count to dao_contract --- .../fill_ckb_transactions_count_to_dao_contract.rake | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 lib/tasks/migration/fill_ckb_transactions_count_to_dao_contract.rake diff --git a/lib/tasks/migration/fill_ckb_transactions_count_to_dao_contract.rake b/lib/tasks/migration/fill_ckb_transactions_count_to_dao_contract.rake new file mode 100644 index 000000000..9e9514546 --- /dev/null +++ b/lib/tasks/migration/fill_ckb_transactions_count_to_dao_contract.rake @@ -0,0 +1,9 @@ +namespace :migration do + desc "Usage: RAILS_ENV=production bundle exec rake migration:fill_ckb_transactions_count_to_dao_contract" + task fill_ckb_transactions_count_to_dao_contract: :environment do + dao_transactions_count = CkbTransaction.where("tags @> array[?]::varchar[]", ["dao"]).count + DaoContract.default_contract.update(ckb_transactions_count: dao_transactions_count) + + puts "done" + end +end From 750df27b4e7347b46f2c9467813dd28acf159d5e Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 14:45:26 +0800 Subject: [PATCH 24/76] test: dao_transactions counter --- test/models/record_counters/dao_transactions.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 test/models/record_counters/dao_transactions.rb diff --git a/test/models/record_counters/dao_transactions.rb b/test/models/record_counters/dao_transactions.rb new file mode 100644 index 000000000..1cac2e529 --- /dev/null +++ b/test/models/record_counters/dao_transactions.rb @@ -0,0 +1,15 @@ +require "test_helper" + +class DaoTransactionsTest < ActiveSupport::TestCase + test "should respond to total_count" do + dao_contract = DaoContract.default_contract + dao_transactions_counter = RecordCounters::DaoTransactions.new(dao_contract) + assert_respond_to dao_transactions_counter, :total_count + end + + test "total_count should return dao_contract ckb transactions count" do + dao_contract = DaoContract.default_contract + dao_transactions_counter = RecordCounters::DaoTransactions.new(dao_contract) + assert_equal dao_contract.ckb_transactions_count, dao_transactions_counter.total_count + end +end \ No newline at end of file From 2c6a6af260823ce98b8c093b4045a245b25c6f9a Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 14:45:40 +0800 Subject: [PATCH 25/76] feat: add dao transactions counter --- app/models/record_counters/dao_transactions.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 app/models/record_counters/dao_transactions.rb diff --git a/app/models/record_counters/dao_transactions.rb b/app/models/record_counters/dao_transactions.rb new file mode 100644 index 000000000..ed9c55ec9 --- /dev/null +++ b/app/models/record_counters/dao_transactions.rb @@ -0,0 +1,15 @@ +module RecordCounters + class DaoTransactions + def initialize(dao_contract) + @dao_contract = dao_contract + end + + def total_count + dao_contract.ckb_transactions_count + end + + private + + attr_reader :dao_contract + end +end From 440e486ae0280b7833c645ee7ff9da97fbc48069 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 15:04:20 +0800 Subject: [PATCH 26/76] feat: use dao transactions counter --- app/controllers/api/v1/contract_transactions_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/v1/contract_transactions_controller.rb b/app/controllers/api/v1/contract_transactions_controller.rb index 318e7e029..bc99141b3 100644 --- a/app/controllers/api/v1/contract_transactions_controller.rb +++ b/app/controllers/api/v1/contract_transactions_controller.rb @@ -10,7 +10,8 @@ def show ckb_transactions = dao_contract.ckb_transactions.select(:id, :tx_hash, :block_id, :block_number, :block_timestamp, :is_cellbase).recent.page(@page).per(@page_size) json = Rails.cache.realize(ckb_transactions.cache_key, version: ckb_transactions.cache_version) do - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: @page, page_size: @page_size).call + records_counter = RecordCounters::DaoTransactions.new(dao_contract) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: @page, page_size: @page_size, records_counter: records_counter).call CkbTransactionsSerializer.new(ckb_transactions, options.merge(params: { previews: true })).serialized_json end From 8b7622357a6c9213bb336ebda97447d3977d127e Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 15:04:40 +0800 Subject: [PATCH 27/76] chore: adjust fake dao deposit transaction --- test/test_helper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_helper.rb b/test/test_helper.rb index 6f58752a6..eb1d9740e 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -244,6 +244,7 @@ def expected_ranking(address1, address2, address3) def fake_dao_deposit_transaction(dao_cell_count, address) block = create(:block, :with_block_hash) + DaoContract.default_contract.update(ckb_transactions_count: dao_cell_count) dao_cell_count.times do |number| if number % 2 == 0 ckb_transaction1 = create(:ckb_transaction, tx_hash: "0x#{SecureRandom.hex(32)}", block: block, address: address, contained_address_ids: [address.id], tags: ["dao"]) From b32050cce91c60d84b369777ef3e167e985310af Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 15:15:52 +0800 Subject: [PATCH 28/76] feat: add dao_transactions_count to addresses --- app/models/address.rb | 1 + ...0806071500_add_dao_ckb_transactions_count_to_addresses.rb | 5 +++++ db/schema.rb | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20200806071500_add_dao_ckb_transactions_count_to_addresses.rb diff --git a/app/models/address.rb b/app/models/address.rb index 309f65ce1..afe6a54c3 100644 --- a/app/models/address.rb +++ b/app/models/address.rb @@ -133,6 +133,7 @@ def unmade_dao_interests # average_deposit_time :decimal(, ) # unclaimed_compensation :decimal(30, ) # is_depositor :boolean default(FALSE) +# dao_transactions_count :decimal(30, ) default(0) # # Indexes # diff --git a/db/migrate/20200806071500_add_dao_ckb_transactions_count_to_addresses.rb b/db/migrate/20200806071500_add_dao_ckb_transactions_count_to_addresses.rb new file mode 100644 index 000000000..22ce6d923 --- /dev/null +++ b/db/migrate/20200806071500_add_dao_ckb_transactions_count_to_addresses.rb @@ -0,0 +1,5 @@ +class AddDaoCkbTransactionsCountToAddresses < ActiveRecord::Migration[6.0] + def change + add_column :addresses, :dao_transactions_count, :decimal, precision: 30, default: 0 + end +end diff --git a/db/schema.rb b/db/schema.rb index e57b3090d..971c855b3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_08_06_060029) do +ActiveRecord::Schema.define(version: 2020_08_06_071500) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -41,6 +41,7 @@ t.decimal "average_deposit_time" t.decimal "unclaimed_compensation", precision: 30 t.boolean "is_depositor", default: false + t.decimal "dao_transactions_count", precision: 30, default: "0" t.index ["address_hash"], name: "index_addresses_on_address_hash" t.index ["is_depositor"], name: "index_addresses_on_is_depositor", where: "(is_depositor = true)" t.index ["lock_hash"], name: "index_addresses_on_lock_hash", unique: true From 1aad7704dd9ddb940e884c59d1da336b4cc86f59 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 16:34:43 +0800 Subject: [PATCH 29/76] fix: wrong ckb_dao_transactions and ckb_udt_transactions --- app/models/address.rb | 4 ++-- app/models/ckb_sync/node_data_processor.rb | 16 ++++++++++++---- app/models/ckb_transaction.rb | 4 ++++ ...43_add_dao_address_ids_to_ckb_transactions.rb | 11 +++++++++++ db/schema.rb | 6 +++++- 5 files changed, 34 insertions(+), 7 deletions(-) create mode 100644 db/migrate/20200806081043_add_dao_address_ids_to_ckb_transactions.rb diff --git a/app/models/address.rb b/app/models/address.rb index afe6a54c3..e96d09c46 100644 --- a/app/models/address.rb +++ b/app/models/address.rb @@ -22,14 +22,14 @@ def custom_ckb_transactions end def ckb_dao_transactions - CkbTransaction.where("contained_address_ids @> array[?]::bigint[]", [id]).where("tags @> array[?]::varchar[]", ["dao"]) + CkbTransaction.where("dao_address_ids @> array[?]::bigint[]", [id]) end def ckb_udt_transactions(type_hash) udt = Udt.where(type_hash: type_hash).select(:id).first return [] if udt.blank? - CkbTransaction.where("contained_address_ids @> array[?]::bigint[]", [id]).where("contained_udt_ids @> array[?]::bigint[]", [udt.id]) + CkbTransaction.where("udt_address_ids @> array[?]::bigint[]", [id]).where("contained_udt_ids @> array[?]::bigint[]", [udt.id]) end def lock_info diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index 6cdd0db37..4840dcbe4 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -414,13 +414,17 @@ def build_ckb_transactions(local_block, transactions, outputs, new_dao_depositor address_ids = Set.new tags = Set.new udt_ids = Set.new + dao_address_ids = Set.new + udt_address_ids = Set.new ckb_transaction = build_ckb_transaction(local_block, transaction, transaction_index) build_cell_inputs(transaction.inputs, ckb_transaction) - build_cell_outputs(transaction.outputs, ckb_transaction, addresses, transaction.outputs_data, outputs, new_dao_depositor_events, udt_infos, address_ids, tags, udt_ids) + build_cell_outputs(transaction.outputs, ckb_transaction, addresses, transaction.outputs_data, outputs, new_dao_depositor_events, udt_infos, address_ids, tags, udt_ids, dao_address_ids, udt_address_ids) ckb_transaction.addresses << addresses.to_a ckb_transaction.contained_address_ids += address_ids.to_a ckb_transaction.tags += tags.to_a ckb_transaction.contained_udt_ids += udt_ids.to_a + ckb_transaction.dao_address_ids += dao_address_ids.to_a + ckb_transaction.udt_address_ids += udt_address_ids.to_a ckb_transaction end @@ -464,7 +468,7 @@ def from_cell_base?(node_input) node_input.previous_output.tx_hash == CellOutput::SYSTEM_TX_HASH end - def build_cell_outputs(node_outputs, ckb_transaction, addresses, outputs_data, outputs, new_dao_depositor_events, udt_infos, address_ids, tags, udt_ids) + def build_cell_outputs(node_outputs, ckb_transaction, addresses, outputs_data, outputs, new_dao_depositor_events, udt_infos, address_ids, tags, udt_ids, dao_address_ids, udt_address_ids) node_outputs.each_with_index.map do |output, cell_index| address = Address.find_or_create_address(output.lock, ckb_transaction.block_timestamp) addresses << address @@ -476,9 +480,11 @@ def build_cell_outputs(node_outputs, ckb_transaction, addresses, outputs_data, o tags << "udt" udt = Udt.find_or_create_by!(type_hash: output.type.compute_hash, code_hash: ENV["SUDT_CELL_TYPE_HASH"], udt_type: "sudt") udt_ids << udt.id + udt_address_ids << address.id end if cell_output.nervos_dao_deposit? || cell_output.nervos_dao_withdrawing? tags << "dao" + dao_address_ids << address.id end build_deposit_dao_events(address, cell_output, ckb_transaction, new_dao_depositor_events) @@ -652,16 +658,18 @@ def update_ckb_transaction(consumed_tx, address_id, previous_cell_output, update if previous_cell_output.udt? consumed_tx.tags << "udt" consumed_tx.contained_udt_ids << Udt.find_or_create_by!(type_hash: previous_cell_output.type_hash, code_hash: ENV["SUDT_CELL_TYPE_HASH"], udt_type: "sudt").id + consumed_tx.udt_address_ids << previous_cell_output.address_id end if previous_cell_output.nervos_dao_withdrawing? consumed_tx.tags << "dao" + consumed_tx.dao_address_ids << previous_cell_output.address_id end if tx.present? tx[:contained_address_ids] = (tx[:contained_address_ids] << consumed_tx.contained_address_ids).flatten.uniq tx[:tags] = (tx[:tags] << consumed_tx.tags).flatten.uniq tx[:contained_udt_ids] = (tx[:contained_udt_ids] << consumed_tx.contained_udt_ids).flatten.uniq else - updated_ckb_transactions << { id: consumed_tx.id, contained_udt_ids: consumed_tx.contained_udt_ids.uniq, contained_address_ids: consumed_tx.contained_address_ids.uniq, tags: consumed_tx.tags.uniq, created_at: consumed_tx.created_at, updated_at: Time.current } + updated_ckb_transactions << { id: consumed_tx.id, dao_address_ids: consumed_tx.dao_address_ids.uniq, udt_address_ids: consumed_tx.udt_address_ids.uniq, contained_udt_ids: consumed_tx.contained_udt_ids.uniq, contained_address_ids: consumed_tx.contained_address_ids.uniq, tags: consumed_tx.tags.uniq, created_at: consumed_tx.created_at, updated_at: Time.current } end end @@ -673,7 +681,7 @@ def update_previous_cell_output_status(ckb_transaction_id, previous_cell_output, def update_address_balance_and_ckb_transactions_count(address) address.balance = address.cell_outputs.live.sum(:capacity) - address.ckb_transactions_count = AccountBook.where(address: address).select(:ckb_transaction_id).distinct.count + address.ckb_transactions_count = address.custom_ckb_transactions.count address.live_cells_count = address.cell_outputs.live.count address.save! end diff --git a/app/models/ckb_transaction.rb b/app/models/ckb_transaction.rb index 5da2fa368..448c05508 100644 --- a/app/models/ckb_transaction.rb +++ b/app/models/ckb_transaction.rb @@ -154,6 +154,8 @@ def recover_dead_cell # contained_address_ids :bigint default([]), is an Array # tags :string default([]), is an Array # contained_udt_ids :bigint default([]), is an Array +# dao_address_ids :bigint default([]), is an Array +# udt_address_ids :bigint default([]), is an Array # # Indexes # @@ -161,7 +163,9 @@ def recover_dead_cell # index_ckb_transactions_on_block_timestamp_and_id (block_timestamp DESC NULLS LAST,id DESC) # index_ckb_transactions_on_contained_address_ids (contained_address_ids) USING gin # index_ckb_transactions_on_contained_udt_ids (contained_udt_ids) USING gin +# index_ckb_transactions_on_dao_address_ids (dao_address_ids) USING gin # index_ckb_transactions_on_is_cellbase (is_cellbase) # index_ckb_transactions_on_tags (tags) USING gin # index_ckb_transactions_on_tx_hash_and_block_id (tx_hash,block_id) UNIQUE +# index_ckb_transactions_on_udt_address_ids (udt_address_ids) USING gin # diff --git a/db/migrate/20200806081043_add_dao_address_ids_to_ckb_transactions.rb b/db/migrate/20200806081043_add_dao_address_ids_to_ckb_transactions.rb new file mode 100644 index 000000000..cbb71a56f --- /dev/null +++ b/db/migrate/20200806081043_add_dao_address_ids_to_ckb_transactions.rb @@ -0,0 +1,11 @@ +class AddDaoAddressIdsToCkbTransactions < ActiveRecord::Migration[6.0] + disable_ddl_transaction! + + def change + add_column :ckb_transactions, :dao_address_ids, :bigint, array: true, default: [] + add_column :ckb_transactions, :udt_address_ids, :bigint, array: true, default: [] + + add_index :ckb_transactions, :dao_address_ids, using: :gin, algorithm: :concurrently + add_index :ckb_transactions, :udt_address_ids, using: :gin, algorithm: :concurrently + end +end diff --git a/db/schema.rb b/db/schema.rb index 971c855b3..ea5f37c66 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_08_06_071500) do +ActiveRecord::Schema.define(version: 2020_08_06_081043) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -186,13 +186,17 @@ t.bigint "contained_address_ids", default: [], array: true t.string "tags", default: [], array: true t.bigint "contained_udt_ids", default: [], array: true + t.bigint "dao_address_ids", default: [], array: true + t.bigint "udt_address_ids", default: [], array: true t.index ["block_id", "block_timestamp"], name: "index_ckb_transactions_on_block_id_and_block_timestamp" t.index ["block_timestamp", "id"], name: "index_ckb_transactions_on_block_timestamp_and_id", order: { block_timestamp: "DESC NULLS LAST", id: :desc } t.index ["contained_address_ids"], name: "index_ckb_transactions_on_contained_address_ids", using: :gin t.index ["contained_udt_ids"], name: "index_ckb_transactions_on_contained_udt_ids", using: :gin + t.index ["dao_address_ids"], name: "index_ckb_transactions_on_dao_address_ids", using: :gin t.index ["is_cellbase"], name: "index_ckb_transactions_on_is_cellbase" t.index ["tags"], name: "index_ckb_transactions_on_tags", using: :gin t.index ["tx_hash", "block_id"], name: "index_ckb_transactions_on_tx_hash_and_block_id", unique: true + t.index ["udt_address_ids"], name: "index_ckb_transactions_on_udt_address_ids", using: :gin end create_table "daily_statistics", force: :cascade do |t| From 50f55cecb8cae3e15ec4e1a5a4315bd4467fe02e Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 16:35:24 +0800 Subject: [PATCH 30/76] test: dao_transactions count --- .../ckb_sync/node_data_processor_test.rb | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index f6736b036..7cce6db63 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -1348,6 +1348,81 @@ class NodeDataProcessorTest < ActiveSupport::TestCase end end + test "#process_block should update block's contained address's balance" do + prepare_node_data(12) + local_block = Block.find_by(number: 12) + origin_balance = local_block.contained_addresses.sum(:balance) + VCR.use_cassette("blocks/13") do + new_local_block = node_data_processor.call + assert_equal origin_balance + new_local_block.cell_outputs.sum(:capacity), new_local_block.contained_addresses.sum(:balance) + end + end + + test "#process_block should update block's contained address's dao_ckb_transactions_count" do + block1 = create(:block, :with_block_hash, number: DEFAULT_NODE_BLOCK_NUMBER - 2) + tx1 = create(:ckb_transaction, block: block1) + block2 = create(:block, :with_block_hash, number: DEFAULT_NODE_BLOCK_NUMBER - 1) + tx2 = create(:ckb_transaction, block: block2) + tx3 = create(:ckb_transaction, block: block2) + tx4 = create(:ckb_transaction, block: block2) + tx5 = create(:ckb_transaction, block: block2) + input_address1 = create(:address) + input_address2 = create(:address) + input_address3 = create(:address) + input_address4 = create(:address) + input_address5 = create(:address) + create(:cell_output, ckb_transaction: tx1, generated_by: tx1, block: block1, capacity: 50000 * 10**8, tx_hash: tx1.tx_hash, cell_index: 0, address: input_address1) + create(:cell_output, ckb_transaction: tx2, generated_by: tx2, block: block2, capacity: 60000 * 10**8, tx_hash: tx2.tx_hash, cell_index: 1, address: input_address2) + create(:cell_output, ckb_transaction: tx3, generated_by: tx3, block: block2, capacity: 70000 * 10**8, tx_hash: tx3.tx_hash, cell_index: 2, address: input_address3) + create(:cell_output, ckb_transaction: tx4, generated_by: tx4, block: block2, capacity: 70000 * 10**8, tx_hash: tx4.tx_hash, cell_index: 0, address: input_address4) + create(:cell_output, ckb_transaction: tx5, generated_by: tx5, block: block2, capacity: 70000 * 10**8, tx_hash: tx5.tx_hash, cell_index: 0, address: input_address5) + header = CKB::Types::BlockHeader.new(compact_target: "0x1000", hash: "0x#{SecureRandom.hex(32)}", number: DEFAULT_NODE_BLOCK_NUMBER, parent_hash: "0x#{SecureRandom.hex(32)}", nonce: 1757392074788233522, timestamp: CkbUtils.time_in_milliseconds(Time.current), transactions_root: "0x#{SecureRandom.hex(32)}", proposals_hash: "0x#{SecureRandom.hex(32)}", uncles_hash: "0x#{SecureRandom.hex(32)}", version: 0, epoch: 1, dao: "0x01000000000000000000c16ff286230000a3a65e97fd03000057c138586f0000") + inputs = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx1.tx_hash, index: 0)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx2.tx_hash, index: 1)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx3.tx_hash, index: 2)) + ] + inputs1 = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx4.tx_hash, index: 0)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx5.tx_hash, index: 0)) + ] + lock1 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + lock2 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + lock3 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + dao_type = CKB::Types::Script.new(code_hash: ENV["DAO_TYPE_HASH"], hash_type: "type", args: "0x") + outputs = [ + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock1, type: dao_type), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock2, type: dao_type), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock3) + ] + outputs1 = [ + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock1, type: dao_type), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock2, type: dao_type), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock3) + ] + miner_lock = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + cellbase_inputs = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: "0x0000000000000000000000000000000000000000000000000000000000000000", index: 4294967295), since: 3000) + ] + cellbase_outputs = [ + CKB::Types::Output.new(capacity: 200986682127, lock: miner_lock) + ] + transactions = [ + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: cellbase_inputs, outputs: cellbase_outputs, outputs_data: %w[0x], witnesses: ["0x590000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd800000000"]), + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs, outputs: outputs, outputs_data: %w[0x0000000000000000 0x0000000000000000 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]), + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs1, outputs: outputs1, outputs_data: %w[0x0000000000000000 0x0000000000000000 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]) + ] + node_block = CKB::Types::Block.new(uncles: [], proposals: [], transactions: transactions, header: header) + node_data_processor.process_block(node_block) + address1 = Address.find_by(lock_hash: lock1.compute_hash) + address2 = Address.find_by(lock_hash: lock2.compute_hash) + address3 = Address.find_by(lock_hash: lock3.compute_hash) + + assert_equal 2, address1.dao_transactions_count + assert_equal 2, address2.dao_transactions_count + assert_equal 0, address3.dao_transactions_count + end + test "#process_block should update abandoned block's contained address's live cells count" do prepare_node_data(12) local_block = Block.find_by(number: 12) From d51d4294b6e204af8161fc895e6e47cbbe59aa6c Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 16:38:09 +0800 Subject: [PATCH 31/76] feat: update address dao transactions count --- app/models/ckb_sync/node_data_processor.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index 4840dcbe4..53386b75e 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -683,6 +683,7 @@ def update_address_balance_and_ckb_transactions_count(address) address.balance = address.cell_outputs.live.sum(:capacity) address.ckb_transactions_count = address.custom_ckb_transactions.count address.live_cells_count = address.cell_outputs.live.count + address.dao_transactions_count = address.ckb_dao_transactions.count address.save! end From b124d9b2d38769ce5a61ea56d184b979cc5f413d Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 16:42:19 +0800 Subject: [PATCH 32/76] test: should recalculate block's contained address's dao_ckb_transactions_count when block is invalid --- .../ckb_sync/node_data_processor_test.rb | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index 7cce6db63..92ff8809d 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -1423,6 +1423,73 @@ class NodeDataProcessorTest < ActiveSupport::TestCase assert_equal 0, address3.dao_transactions_count end + test "should recalculate block's contained address's dao_ckb_transactions_count when block is invalid" do + block1 = create(:block, :with_block_hash, number: DEFAULT_NODE_BLOCK_NUMBER - 2) + tx1 = create(:ckb_transaction, block: block1) + block2 = create(:block, :with_block_hash, number: DEFAULT_NODE_BLOCK_NUMBER - 1) + tx2 = create(:ckb_transaction, block: block2) + tx3 = create(:ckb_transaction, block: block2) + tx4 = create(:ckb_transaction, block: block2) + tx5 = create(:ckb_transaction, block: block2) + input_address1 = create(:address) + input_address2 = create(:address) + input_address3 = create(:address) + input_address4 = create(:address) + input_address5 = create(:address) + create(:cell_output, ckb_transaction: tx1, generated_by: tx1, block: block1, capacity: 50000 * 10**8, tx_hash: tx1.tx_hash, cell_index: 0, address: input_address1) + create(:cell_output, ckb_transaction: tx2, generated_by: tx2, block: block2, capacity: 60000 * 10**8, tx_hash: tx2.tx_hash, cell_index: 1, address: input_address2) + create(:cell_output, ckb_transaction: tx3, generated_by: tx3, block: block2, capacity: 70000 * 10**8, tx_hash: tx3.tx_hash, cell_index: 2, address: input_address3) + create(:cell_output, ckb_transaction: tx4, generated_by: tx4, block: block2, capacity: 70000 * 10**8, tx_hash: tx4.tx_hash, cell_index: 0, address: input_address4) + create(:cell_output, ckb_transaction: tx5, generated_by: tx5, block: block2, capacity: 70000 * 10**8, tx_hash: tx5.tx_hash, cell_index: 0, address: input_address5) + header = CKB::Types::BlockHeader.new(compact_target: "0x1000", hash: "0x#{SecureRandom.hex(32)}", number: DEFAULT_NODE_BLOCK_NUMBER, parent_hash: "0x#{SecureRandom.hex(32)}", nonce: 1757392074788233522, timestamp: CkbUtils.time_in_milliseconds(Time.current), transactions_root: "0x#{SecureRandom.hex(32)}", proposals_hash: "0x#{SecureRandom.hex(32)}", uncles_hash: "0x#{SecureRandom.hex(32)}", version: 0, epoch: 1, dao: "0x01000000000000000000c16ff286230000a3a65e97fd03000057c138586f0000") + inputs = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx1.tx_hash, index: 0)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx2.tx_hash, index: 1)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx3.tx_hash, index: 2)) + ] + inputs1 = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx4.tx_hash, index: 0)), + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: tx5.tx_hash, index: 0)) + ] + lock1 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + lock2 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + lock3 = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + dao_type = CKB::Types::Script.new(code_hash: ENV["DAO_TYPE_HASH"], hash_type: "type", args: "0x") + outputs = [ + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock1, type: dao_type), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock2, type: dao_type), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock3) + ] + outputs1 = [ + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock1, type: dao_type), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock2, type: dao_type), + CKB::Types::Output.new(capacity: 40000 * 10**8, lock: lock3) + ] + miner_lock = CKB::Types::Script.new(code_hash: ENV["SECP_CELL_TYPE_HASH"], hash_type: "type", args: "0x#{SecureRandom.hex(20)}") + cellbase_inputs = [ + CKB::Types::Input.new(previous_output: CKB::Types::OutPoint.new(tx_hash: "0x0000000000000000000000000000000000000000000000000000000000000000", index: 4294967295), since: 3000) + ] + cellbase_outputs = [ + CKB::Types::Output.new(capacity: 200986682127, lock: miner_lock) + ] + transactions = [ + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: cellbase_inputs, outputs: cellbase_outputs, outputs_data: %w[0x], witnesses: ["0x590000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd800000000"]), + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs, outputs: outputs, outputs_data: %w[0x0000000000000000 0x0000000000000000 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]), + CKB::Types::Transaction.new(hash: "0x#{SecureRandom.hex(32)}", cell_deps: [], header_deps: [], inputs: inputs1, outputs: outputs1, outputs_data: %w[0x0000000000000000 0x0000000000000000 0x], witnesses: ["0x5d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce801140000003954acece65096bfa81258983ddb83915fc56bd804000000123456780000000000000000"]) + ] + node_block = CKB::Types::Block.new(uncles: [], proposals: [], transactions: transactions, header: header) + block = node_data_processor.process_block(node_block) + address1 = Address.find_by(lock_hash: lock1.compute_hash) + address2 = Address.find_by(lock_hash: lock2.compute_hash) + address3 = Address.find_by(lock_hash: lock3.compute_hash) + CkbSync::Api.any_instance.stubs(:get_tip_block_number).returns(block.number + 1) + node_data_processor.call + + assert_equal 0, address1.reload.dao_transactions_count + assert_equal 0, address2.reload.dao_transactions_count + assert_equal 0, address3.reload.dao_transactions_count + end + test "#process_block should update abandoned block's contained address's live cells count" do prepare_node_data(12) local_block = Block.find_by(number: 12) From 98640258fc58b08d1405ecc8a14ff7e5a61a0984 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 16:47:44 +0800 Subject: [PATCH 33/76] test: address dao transactions counter --- .../record_counters/address_dao_transactions.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 test/models/record_counters/address_dao_transactions.rb diff --git a/test/models/record_counters/address_dao_transactions.rb b/test/models/record_counters/address_dao_transactions.rb new file mode 100644 index 000000000..6deb098bb --- /dev/null +++ b/test/models/record_counters/address_dao_transactions.rb @@ -0,0 +1,15 @@ +require "test_helper" + +class AddressDaoTransactionsTest < ActiveSupport::TestCase + test "should respond to total_count" do + address = create(:address) + address_dao_transactions_counter = RecordCounters::AddressDaoTransactions.new(address) + assert_respond_to address_dao_transactions_counter, :total_count + end + + test "total_count should return dao_contract ckb transactions count" do + address = create(:address) + address_dao_transactions_counter = RecordCounters::AddressDaoTransactions.new(address) + assert_equal address.dao_transactions_count, address_dao_transactions_counter.total_count + end +end From 9b8f19aabc8de297461342fa19282e5f8f61df61 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 16:47:56 +0800 Subject: [PATCH 34/76] feat: add address dao transactions counter --- .../record_counters/address_dao_transactions.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 app/models/record_counters/address_dao_transactions.rb diff --git a/app/models/record_counters/address_dao_transactions.rb b/app/models/record_counters/address_dao_transactions.rb new file mode 100644 index 000000000..7cdb1be89 --- /dev/null +++ b/app/models/record_counters/address_dao_transactions.rb @@ -0,0 +1,15 @@ +module RecordCounters + class AddressDaoTransactions + def initialize(address) + @address = address + end + + def total_count + address.dao_transactions_count + end + + private + + attr_reader :address + end +end From 840351f74062bd45c94c67b0dc6d194974d69c45 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 16:50:10 +0800 Subject: [PATCH 35/76] chore: format code --- app/models/ckb_sync/node_data_processor.rb | 4 ++-- test/controllers/api/v1/blocks_controller_test.rb | 1 - test/models/record_counters/base_test.rb | 2 +- test/models/record_counters/dao_transactions.rb | 2 +- test/models/record_counters/udt_transactions_test.rb | 2 +- test/models/table_record_count_test.rb | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index 53386b75e..e95acfe05 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -632,7 +632,7 @@ def udt_counts_value(updated_ckb_transactions) udt_counts = udt_ids.each_with_object(Hash.new(0)) { |udt_id, counts| counts[udt_id] += 1 } udt_counts.map do |udt_id, count| udt = Udt.find(udt_id) - {id: udt_id, ckb_transactions_count: udt.ckb_transactions_count + count, created_at: udt.created_at, updated_at: Time.current} + { id: udt_id, ckb_transactions_count: udt.ckb_transactions_count + count, created_at: udt.created_at, updated_at: Time.current } end end @@ -669,7 +669,7 @@ def update_ckb_transaction(consumed_tx, address_id, previous_cell_output, update tx[:tags] = (tx[:tags] << consumed_tx.tags).flatten.uniq tx[:contained_udt_ids] = (tx[:contained_udt_ids] << consumed_tx.contained_udt_ids).flatten.uniq else - updated_ckb_transactions << { id: consumed_tx.id, dao_address_ids: consumed_tx.dao_address_ids.uniq, udt_address_ids: consumed_tx.udt_address_ids.uniq, contained_udt_ids: consumed_tx.contained_udt_ids.uniq, contained_address_ids: consumed_tx.contained_address_ids.uniq, tags: consumed_tx.tags.uniq, created_at: consumed_tx.created_at, updated_at: Time.current } + updated_ckb_transactions << { id: consumed_tx.id, dao_address_ids: consumed_tx.dao_address_ids.uniq, udt_address_ids: consumed_tx.udt_address_ids.uniq, contained_udt_ids: consumed_tx.contained_udt_ids.uniq, contained_address_ids: consumed_tx.contained_address_ids.uniq, tags: consumed_tx.tags.uniq, created_at: consumed_tx.created_at, updated_at: Time.current } end end diff --git a/test/controllers/api/v1/blocks_controller_test.rb b/test/controllers/api/v1/blocks_controller_test.rb index cf9047a31..aa74070b1 100644 --- a/test/controllers/api/v1/blocks_controller_test.rb +++ b/test/controllers/api/v1/blocks_controller_test.rb @@ -48,7 +48,6 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest test "should get serialized objects" do 15.times do |number| create(:block, :with_block_hash, timestamp: 2.days.ago.to_i + number) - end block_timestamps = Block.recent.limit(ENV["HOMEPAGE_BLOCK_RECORDS_COUNT"].to_i).pluck(:timestamp) blocks = Block.where(timestamp: block_timestamps).select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count, :live_cell_changes).recent diff --git a/test/models/record_counters/base_test.rb b/test/models/record_counters/base_test.rb index e745a6994..c4684b1e5 100644 --- a/test/models/record_counters/base_test.rb +++ b/test/models/record_counters/base_test.rb @@ -5,4 +5,4 @@ class BaseTest < ActiveSupport::TestCase base = RecordCounters::Base.new assert_respond_to base, :total_count end -end \ No newline at end of file +end diff --git a/test/models/record_counters/dao_transactions.rb b/test/models/record_counters/dao_transactions.rb index 1cac2e529..8c6866290 100644 --- a/test/models/record_counters/dao_transactions.rb +++ b/test/models/record_counters/dao_transactions.rb @@ -12,4 +12,4 @@ class DaoTransactionsTest < ActiveSupport::TestCase dao_transactions_counter = RecordCounters::DaoTransactions.new(dao_contract) assert_equal dao_contract.ckb_transactions_count, dao_transactions_counter.total_count end -end \ No newline at end of file +end diff --git a/test/models/record_counters/udt_transactions_test.rb b/test/models/record_counters/udt_transactions_test.rb index 78cf188d6..7d1662595 100644 --- a/test/models/record_counters/udt_transactions_test.rb +++ b/test/models/record_counters/udt_transactions_test.rb @@ -12,4 +12,4 @@ class UdtTransactionsTest < ActiveSupport::TestCase udt_transactions_counter = RecordCounters::UdtTransactions.new(udt) assert_equal udt.ckb_transactions_count, udt_transactions_counter.total_count end -end \ No newline at end of file +end diff --git a/test/models/table_record_count_test.rb b/test/models/table_record_count_test.rb index f7515659b..730e65519 100644 --- a/test/models/table_record_count_test.rb +++ b/test/models/table_record_count_test.rb @@ -1,4 +1,4 @@ -require 'test_helper' +require "test_helper" class TableRecordCountTest < ActiveSupport::TestCase # test "the truth" do From 6497f41c527d29fbffd23f643e2e0c5d5d9b1d77 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 16:56:31 +0800 Subject: [PATCH 36/76] feat: use address dao transaction counter --- app/controllers/api/v1/address_dao_transactions_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/v1/address_dao_transactions_controller.rb b/app/controllers/api/v1/address_dao_transactions_controller.rb index 339b2a204..6e4b0466e 100644 --- a/app/controllers/api/v1/address_dao_transactions_controller.rb +++ b/app/controllers/api/v1/address_dao_transactions_controller.rb @@ -11,7 +11,8 @@ def show ckb_dao_transactions = address.ckb_dao_transactions.select(:id, :tx_hash, :block_id, :block_number, :block_timestamp, :is_cellbase).recent.page(@page).per(@page_size) json = Rails.cache.realize(ckb_dao_transactions.cache_key, version: ckb_dao_transactions.cache_version) do - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_dao_transactions, page: @page, page_size: @page_size).call + records_counter = RecordCounters::AddressDaoTransactions.new(address) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_dao_transactions, page: @page, page_size: @page_size, records_counter: records_counter).call CkbTransactionsSerializer.new(ckb_dao_transactions, options.merge(params: { previews: true })).serialized_json end From a29f4896c465b1ccc1a32548dea990eb3a813de0 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 16:56:38 +0800 Subject: [PATCH 37/76] chore: adjust tests --- .../v1/address_dao_transactions_controller_test.rb | 14 +++++++++----- test/test_helper.rb | 5 +++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/test/controllers/api/v1/address_dao_transactions_controller_test.rb b/test/controllers/api/v1/address_dao_transactions_controller_test.rb index ab2afc4e6..004805c0d 100644 --- a/test/controllers/api/v1/address_dao_transactions_controller_test.rb +++ b/test/controllers/api/v1/address_dao_transactions_controller_test.rb @@ -150,8 +150,8 @@ class AddressDaoTransactionsControllerTest < ActionDispatch::IntegrationTest address_dao_transactions = address.ckb_dao_transactions.order(block_timestamp: :desc).page(page).per(page_size) valid_get api_v1_address_dao_transaction_url(address.address_hash), params: { page: page } - - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_dao_transactions, page: page, page_size: page_size).call + records_counter = RecordCounters::AddressDaoTransactions.new(address) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_dao_transactions, page: page, page_size: page_size, records_counter: records_counter).call response_transaction = CkbTransactionsSerializer.new(address_dao_transactions, options.merge(params: { previews: true })).serialized_json assert_equal response_transaction, response.body @@ -167,7 +167,8 @@ class AddressDaoTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_address_dao_transaction_url(address.address_hash), params: { page_size: page_size } - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_dao_transactions, page: page, page_size: page_size).call + records_counter = RecordCounters::AddressDaoTransactions.new(address) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_dao_transactions, page: page, page_size: page_size, records_counter: records_counter).call response_transaction = CkbTransactionsSerializer.new(address_dao_transactions, options.merge(params: { previews: true })).serialized_json assert_equal response_transaction, response.body @@ -182,7 +183,9 @@ class AddressDaoTransactionsControllerTest < ActionDispatch::IntegrationTest address_dao_transactions = address.ckb_dao_transactions.order(block_timestamp: :desc).page(page).per(page_size) valid_get api_v1_address_dao_transaction_url(address.address_hash), params: { page: page, page_size: page_size } - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_dao_transactions, page: page, page_size: page_size).call + + records_counter = RecordCounters::AddressDaoTransactions.new(address) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_dao_transactions, page: page, page_size: page_size, records_counter: records_counter).call response_transaction = CkbTransactionsSerializer.new(address_dao_transactions, options.merge(params: { previews: true })).serialized_json assert_equal response_transaction, response.body @@ -197,7 +200,8 @@ class AddressDaoTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_address_dao_transaction_url(address.address_hash), params: { page: page, page_size: page_size } - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_dao_transactions, page: page, page_size: page_size).call + records_counter = RecordCounters::AddressDaoTransactions.new(address) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_dao_transactions, page: page, page_size: page_size, records_counter: records_counter).call response_transaction = CkbTransactionsSerializer.new(address_dao_transactions, options.merge(params: { previews: true })).serialized_json assert_equal [], json["data"] diff --git a/test/test_helper.rb b/test/test_helper.rb index eb1d9740e..74d3472bd 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -245,12 +245,13 @@ def expected_ranking(address1, address2, address3) def fake_dao_deposit_transaction(dao_cell_count, address) block = create(:block, :with_block_hash) DaoContract.default_contract.update(ckb_transactions_count: dao_cell_count) + address.update(dao_transactions_count: dao_cell_count) dao_cell_count.times do |number| if number % 2 == 0 - ckb_transaction1 = create(:ckb_transaction, tx_hash: "0x#{SecureRandom.hex(32)}", block: block, address: address, contained_address_ids: [address.id], tags: ["dao"]) + ckb_transaction1 = create(:ckb_transaction, tx_hash: "0x#{SecureRandom.hex(32)}", block: block, address: address, dao_address_ids: [address.id], contained_address_ids: [address.id], tags: ["dao"]) create(:cell_output, ckb_transaction: ckb_transaction1, cell_index: number, tx_hash: "0x498315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", generated_by: ckb_transaction1, block: block, capacity: 10**8 * 1000, cell_type: "nervos_dao_deposit", address: address) else - ckb_transaction2 = create(:ckb_transaction, tx_hash: "0x#{SecureRandom.hex(32)}", block: block, address: address, contained_address_ids: [address.id], tags: ["dao"]) + ckb_transaction2 = create(:ckb_transaction, tx_hash: "0x#{SecureRandom.hex(32)}", block: block, address: address, dao_address_ids: [address.id], contained_address_ids: [address.id], tags: ["dao"]) create(:cell_output, ckb_transaction: ckb_transaction2, cell_index: number, tx_hash: "0x498315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", generated_by: ckb_transaction2, block: block, capacity: 10**8 * 1000, cell_type: "nervos_dao_deposit", address: address) end end From a964e1d357cdc38c3a58dabd8ad71155d19e3449 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 6 Aug 2020 22:45:43 +0800 Subject: [PATCH 38/76] feat: add two tasks 1. fill dao_address_ids and udt_address_ids to ckb_transactions 2. fill dao transactions count to address --- ...s_tags_and_udt_ids_to_ckb_transaction.rake | 2 +- ...nd_udt_address_ids_to_ckb_transaction.rake | 44 +++++++++++++++++++ ...ill_dao_transactions_count_to_address.rake | 21 +++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 lib/tasks/migration/fill_dao_address_ids_and_udt_address_ids_to_ckb_transaction.rake create mode 100644 lib/tasks/migration/fill_dao_transactions_count_to_address.rake diff --git a/lib/tasks/migration/fill_address_ids_tags_and_udt_ids_to_ckb_transaction.rake b/lib/tasks/migration/fill_address_ids_tags_and_udt_ids_to_ckb_transaction.rake index 24c78d2e4..247497ac7 100644 --- a/lib/tasks/migration/fill_address_ids_tags_and_udt_ids_to_ckb_transaction.rake +++ b/lib/tasks/migration/fill_address_ids_tags_and_udt_ids_to_ckb_transaction.rake @@ -16,7 +16,7 @@ namespace :migration do if tx.inputs.udt.present? tx.tags << "udt" - type_hashes = tx.outputs.udt.pluck(:type_hash).uniq + type_hashes = tx.inputs.udt.pluck(:type_hash).uniq tx.contained_udt_ids += Udt.where(type_hash: type_hashes).pluck(:id) end diff --git a/lib/tasks/migration/fill_dao_address_ids_and_udt_address_ids_to_ckb_transaction.rake b/lib/tasks/migration/fill_dao_address_ids_and_udt_address_ids_to_ckb_transaction.rake new file mode 100644 index 000000000..ef660f332 --- /dev/null +++ b/lib/tasks/migration/fill_dao_address_ids_and_udt_address_ids_to_ckb_transaction.rake @@ -0,0 +1,44 @@ +namespace :migration do + desc "Usage: RAILS_ENV=production bundle exec rake migration:fill_dao_address_ids_and_udt_address_ids_to_ckb_transaction" + task fill_dao_address_ids_and_udt_address_ids_to_ckb_transaction: :environment do + progress_bar = ProgressBar.create(total: CkbTransaction.where("tags @> array[?]::varchar[]", ["dao"]).count, format: "%e %B %p%% %c/%C") + CkbTransaction.order(:id).where("tags @> array[?]::varchar[]", ["dao"]).find_in_batches do |txs| + values = txs.map do |tx| + if tx.outputs.where(cell_type: %w(nervos_dao_deposit nervos_dao_withdrawing)).present? + tx.dao_address_ids += tx.outputs.where(cell_type: %w(nervos_dao_deposit nervos_dao_withdrawing)).pluck(:address_id).uniq + end + + if tx.inputs.nervos_dao_withdrawing.present? + tx.dao_address_ids += tx.inputs.nervos_dao_withdrawing.pluck(:address_id).uniq + end + progress_bar.increment + + { id: tx.id, dao_address_ids: tx.dao_address_ids.uniq, created_at: tx.created_at, updated_at: Time.current } + end + + CkbTransaction.upsert_all(values) if values.present? + end + + puts "dao_address_ids done" + + progress_bar = ProgressBar.create(total: CkbTransaction.where("tags @> array[?]::varchar[]", ["udt"]).count, format: "%e %B %p%% %c/%C") + CkbTransaction.order(:id).where("tags @> array[?]::varchar[]", ["udt"]).find_in_batches do |txs| + values = txs.map do |tx| + if tx.outputs.udt.present? + tx.udt_address_ids += tx.outputs.udt.pluck(:address_id).uniq + end + + if tx.inputs.udt.present? + tx.udt_address_ids += tx.inputs.udt.pluck(:address_id).uniq + end + + progress_bar.increment + + { id: tx.id, udt_address_ids: tx.udt_address_ids.uniq, created_at: tx.created_at, updated_at: Time.current } + end + + CkbTransaction.upsert_all(values) if values.present? + end + puts "udt_address_ids done" + end +end diff --git a/lib/tasks/migration/fill_dao_transactions_count_to_address.rake b/lib/tasks/migration/fill_dao_transactions_count_to_address.rake new file mode 100644 index 000000000..ba66aeef8 --- /dev/null +++ b/lib/tasks/migration/fill_dao_transactions_count_to_address.rake @@ -0,0 +1,21 @@ +namespace :migration do + desc "Usage: RAILS_ENV=production bundle exec rake migration:fill_dao_transactions_count_to_address" + task fill_dao_transactions_count_to_address: :environment do + progress_bar = ProgressBar.create({ + total: Address.count, + format: "%e %B %p%% %c/%C" + }) + + values = + Address.all.map do |address| + progress_bar.increment + dao_transactions_count = address.ckb_dao_transactions.count + + { id: udt.id, dao_transactions_count: dao_transactions_count, created_at: udt.created_at, updated_at: Time.current } + end + + Address.upsert_all(values) if values.present? + + puts "done" + end +end From e5a66797d08a2a41cb4a7f6905dfc006d66b618a Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 00:12:03 +0800 Subject: [PATCH 39/76] refactor: use udt_id directly --- .../api/v1/address_udt_transactions_controller.rb | 2 +- app/models/address.rb | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/v1/address_udt_transactions_controller.rb b/app/controllers/api/v1/address_udt_transactions_controller.rb index a3e1c1d1e..9bfbf20c0 100644 --- a/app/controllers/api/v1/address_udt_transactions_controller.rb +++ b/app/controllers/api/v1/address_udt_transactions_controller.rb @@ -12,7 +12,7 @@ def show udt = Udt.find_by(type_hash: params[:type_hash], published: true) raise Api::V1::Exceptions::UdtNotFoundError if udt.blank? - ckb_dao_transactions = address.ckb_udt_transactions(params[:type_hash]).select(:id, :tx_hash, :block_id, :block_number, :block_timestamp, :is_cellbase).recent.page(@page).per(@page_size) + ckb_dao_transactions = address.ckb_udt_transactions(udt.id).select(:id, :tx_hash, :block_id, :block_number, :block_timestamp, :is_cellbase).recent.page(@page).per(@page_size) json = Rails.cache.realize(ckb_dao_transactions.cache_key, version: ckb_dao_transactions.cache_version) do options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_dao_transactions, page: @page, page_size: @page_size).call diff --git a/app/models/address.rb b/app/models/address.rb index e96d09c46..2d9cac5ed 100644 --- a/app/models/address.rb +++ b/app/models/address.rb @@ -25,11 +25,8 @@ def ckb_dao_transactions CkbTransaction.where("dao_address_ids @> array[?]::bigint[]", [id]) end - def ckb_udt_transactions(type_hash) - udt = Udt.where(type_hash: type_hash).select(:id).first - return [] if udt.blank? - - CkbTransaction.where("udt_address_ids @> array[?]::bigint[]", [id]).where("contained_udt_ids @> array[?]::bigint[]", [udt.id]) + def ckb_udt_transactions(udt_id) + CkbTransaction.where("udt_address_ids @> array[?]::bigint[]", [id]).where("contained_udt_ids @> array[?]::bigint[]", [udt_id]) end def lock_info From 9a3527b3f8f5531c6fa14d0d1bba0de4ae6dd903 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 00:13:48 +0800 Subject: [PATCH 40/76] chore: adjust tests --- .../api/v1/address_udt_transactions_controller_test.rb | 10 +++++----- test/factories/address.rb | 4 ++-- test/models/address_test.rb | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/controllers/api/v1/address_udt_transactions_controller_test.rb b/test/controllers/api/v1/address_udt_transactions_controller_test.rb index 8d74005ee..49504ee48 100644 --- a/test/controllers/api/v1/address_udt_transactions_controller_test.rb +++ b/test/controllers/api/v1/address_udt_transactions_controller_test.rb @@ -77,7 +77,7 @@ class AddressUdtTransactionsControllerTest < ActionDispatch::IntegrationTest page_size = 10 udt = create(:udt, published: true) address = create(:address, :with_udt_transactions, udt: udt) - ckb_udt_transactions = address.ckb_udt_transactions(udt.type_hash).order(block_timestamp: :desc).page(page).per(page_size) + ckb_udt_transactions = address.ckb_udt_transactions(udt.id).order(block_timestamp: :desc).page(page).per(page_size) valid_get api_v1_address_udt_transaction_url(address.address_hash, type_hash: udt.type_hash) @@ -168,7 +168,7 @@ class AddressUdtTransactionsControllerTest < ActionDispatch::IntegrationTest udt = create(:udt, published: true) address = create(:address, :with_udt_transactions, transactions_count: 30, udt: udt) - address_udt_transactions = address.ckb_udt_transactions(udt.type_hash).order(block_timestamp: :desc).page(page).per(page_size) + address_udt_transactions = address.ckb_udt_transactions(udt.id).order(block_timestamp: :desc).page(page).per(page_size) valid_get api_v1_address_udt_transaction_url(address.address_hash, type_hash: udt.type_hash), params: { page: page } @@ -184,7 +184,7 @@ class AddressUdtTransactionsControllerTest < ActionDispatch::IntegrationTest page_size = 12 udt = create(:udt, published: true) address = create(:address, :with_udt_transactions, transactions_count: 15, udt: udt) - address_udt_transactions = address.ckb_udt_transactions(udt.type_hash).order(block_timestamp: :desc).page(page).per(page_size) + address_udt_transactions = address.ckb_udt_transactions(udt.id).order(block_timestamp: :desc).page(page).per(page_size) valid_get api_v1_address_udt_transaction_url(address.address_hash, type_hash: udt.type_hash), params: { page_size: page_size } @@ -200,7 +200,7 @@ class AddressUdtTransactionsControllerTest < ActionDispatch::IntegrationTest page_size = 5 udt = create(:udt, published: true) address = create(:address, :with_udt_transactions, transactions_count: 30, udt: udt) - address_udt_transactions = address.ckb_udt_transactions(udt.type_hash).order(block_timestamp: :desc).page(page).per(page_size) + address_udt_transactions = address.ckb_udt_transactions(udt.id).order(block_timestamp: :desc).page(page).per(page_size) valid_get api_v1_address_udt_transaction_url(address.address_hash, type_hash: udt.type_hash), params: { page: page, page_size: page_size } options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_udt_transactions, page: page, page_size: page_size).call @@ -215,7 +215,7 @@ class AddressUdtTransactionsControllerTest < ActionDispatch::IntegrationTest udt = create(:udt, published: true) address = create(:address, :with_udt_transactions, udt: udt) - address_udt_transactions = address.ckb_udt_transactions(udt.type_hash).order(block_timestamp: :desc).page(page).per(page_size) + address_udt_transactions = address.ckb_udt_transactions(udt.id).order(block_timestamp: :desc).page(page).per(page_size) valid_get api_v1_address_udt_transaction_url(address.address_hash, type_hash: udt.type_hash), params: { page: page, page_size: page_size } diff --git a/test/factories/address.rb b/test/factories/address.rb index d7bfaa684..b07abf75a 100644 --- a/test/factories/address.rb +++ b/test/factories/address.rb @@ -48,8 +48,8 @@ after(:create) do |address, evaluator| evaluator.transactions_count.times do block = create(:block, :with_block_hash) - transaction = create(:ckb_transaction, block: block, contained_address_ids: [address.id], tags: ["udt"], contained_udt_ids: [evaluator.udt.id]) - transaction1 = create(:ckb_transaction, block: block, contained_address_ids: [address.id], tags: ["udt"], contained_udt_ids: [evaluator.udt.id]) + transaction = create(:ckb_transaction, block: block, udt_address_ids: [address.id], contained_address_ids: [address.id], tags: ["udt"], contained_udt_ids: [evaluator.udt.id]) + transaction1 = create(:ckb_transaction, block: block, udt_address_ids: [address.id], contained_address_ids: [address.id], tags: ["udt"], contained_udt_ids: [evaluator.udt.id]) create(:cell_output, address: address, block: block, ckb_transaction: transaction, generated_by: transaction, consumed_by: transaction1, type_hash: evaluator.udt.type_hash, cell_type: "udt", data: "0x000050ad321ea12e0000000000000000") address.ckb_transactions << transaction address.ckb_transactions << transaction1 diff --git a/test/models/address_test.rb b/test/models/address_test.rb index 491d448a7..9854c269f 100644 --- a/test/models/address_test.rb +++ b/test/models/address_test.rb @@ -196,14 +196,14 @@ class AddressTest < ActiveSupport::TestCase ckb_transaction_ids = CellOutput.select("ckb_transaction_id").from("(#{sql}) as cell_outputs") expected_ckb_transactions = CkbTransaction.where(id: ckb_transaction_ids.distinct).recent - assert_equal expected_ckb_transactions.pluck(:id), address.ckb_udt_transactions(udt.type_hash).recent.pluck(:id) + assert_equal expected_ckb_transactions.pluck(:id), address.ckb_udt_transactions(udt.id).recent.pluck(:id) end test "#ckb_udt_transactions should return an empty array when there aren't udt cells" do udt = create(:udt) address = create(:address) - assert_equal [], address.ckb_udt_transactions(udt.type_hash) + assert_equal [], address.ckb_udt_transactions(udt.id) end test "#ckb_udt_transactions should return an empty array when udt not exist" do From 6d1ada3f08e22e47460aa8e435c2e27c0c736bea Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 00:14:08 +0800 Subject: [PATCH 41/76] test: address udt transactions counter --- .../record_counters/address_udt_transactions.rb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 test/models/record_counters/address_udt_transactions.rb diff --git a/test/models/record_counters/address_udt_transactions.rb b/test/models/record_counters/address_udt_transactions.rb new file mode 100644 index 000000000..3341efd11 --- /dev/null +++ b/test/models/record_counters/address_udt_transactions.rb @@ -0,0 +1,17 @@ +require "test_helper" + +class AddressUdtTransactionsTest < ActiveSupport::TestCase + test "should respond to total_count" do + address = create(:address) + udt = create(:udt, published: true) + address_udt_transactions_counter = RecordCounters::AddressUdtTransactions.new(address, udt.id) + assert_respond_to address_udt_transactions_counter, :total_count + end + + test "total_count should return dao_contract ckb transactions count" do + address = create(:address) + udt = create(:udt, published: true) + address_udt_transactions_counter = RecordCounters::AddressUdtTransactions.new(address, udt.id) + assert_equal address.ckb_udt_transactions(udt.id).count, address_udt_transactions_counter.total_count + end +end From 66b792522fdf9a50b63b66e8e524666ee456ab64 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 00:14:18 +0800 Subject: [PATCH 42/76] feat: add address udt transactions counter --- .../record_counters/address_udt_transactions.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 app/models/record_counters/address_udt_transactions.rb diff --git a/app/models/record_counters/address_udt_transactions.rb b/app/models/record_counters/address_udt_transactions.rb new file mode 100644 index 000000000..b7eea8ba8 --- /dev/null +++ b/app/models/record_counters/address_udt_transactions.rb @@ -0,0 +1,16 @@ +module RecordCounters + class AddressUdtTransactions + def initialize(address, udt_id) + @address = address + @udt_id = udt_id + end + + def total_count + address.ckb_udt_transactions(udt_id).count + end + + private + + attr_reader :address, :udt_id + end +end From c0fbedb47a6520c3a3e7824cb2bb1bf473e81361 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 00:15:17 +0800 Subject: [PATCH 43/76] feat: use address udt transactions counter --- app/controllers/api/v1/address_udt_transactions_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/v1/address_udt_transactions_controller.rb b/app/controllers/api/v1/address_udt_transactions_controller.rb index 9bfbf20c0..f2ec5b158 100644 --- a/app/controllers/api/v1/address_udt_transactions_controller.rb +++ b/app/controllers/api/v1/address_udt_transactions_controller.rb @@ -15,7 +15,8 @@ def show ckb_dao_transactions = address.ckb_udt_transactions(udt.id).select(:id, :tx_hash, :block_id, :block_number, :block_timestamp, :is_cellbase).recent.page(@page).per(@page_size) json = Rails.cache.realize(ckb_dao_transactions.cache_key, version: ckb_dao_transactions.cache_version) do - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_dao_transactions, page: @page, page_size: @page_size).call + records_counter = RecordCounters::AddressUdtTransactions.new(address, udt.id) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_dao_transactions, page: @page, page_size: @page_size, records_counter: records_counter).call CkbTransactionsSerializer.new(ckb_dao_transactions, options.merge(params: { previews: true })).serialized_json end From 70bdbc0f1cc2eade9fffa2398b30b62e8ffeded5 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 11:26:58 +0800 Subject: [PATCH 44/76] chore: adjust tests --- test/models/address_test.rb | 10 +++++----- test/models/ckb_sync/node_data_processor_test.rb | 4 +++- test/models/dao_contract_test.rb | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/test/models/address_test.rb b/test/models/address_test.rb index 9854c269f..e39980905 100644 --- a/test/models/address_test.rb +++ b/test/models/address_test.rb @@ -130,7 +130,7 @@ class AddressTest < ActiveSupport::TestCase 30.times do |number| block = create(:block, :with_block_hash) contained_address_ids = number % 2 == 0 ? [address.id] : [address1.id] - tx = create(:ckb_transaction, block: block, tags: ["dao"], contained_address_ids: contained_address_ids) + tx = create(:ckb_transaction, block: block, tags: ["dao"], dao_address_ids: [contained_address_ids], contained_address_ids: contained_address_ids) AccountBook.create(address: address, ckb_transaction: tx) cell_type = number % 2 == 0 ? "nervos_dao_deposit" : "nervos_dao_withdrawing" cell_output_address = number % 2 == 0 ? address : address1 @@ -155,11 +155,11 @@ class AddressTest < ActiveSupport::TestCase 30.times do |number| block = create(:block, :with_block_hash) if number % 2 == 0 - tx = create(:ckb_transaction, block: block, tags: ["udt"], contained_udt_ids: [udt.id], contained_address_ids: [address.id]) + tx = create(:ckb_transaction, block: block, tags: ["udt"], contained_udt_ids: [udt.id], udt_address_ids: [address.id], contained_address_ids: [address.id]) create(:cell_output, block: block, ckb_transaction: tx, cell_type: "udt", type_hash: udt.type_hash, generated_by: tx, address: address) else - tx = create(:ckb_transaction, block: block, tags: ["udt"], contained_udt_ids: [udt.id], contained_address_ids: [address.id]) - tx1 = create(:ckb_transaction, block: block, tags: ["udt"], contained_udt_ids: [udt.id], contained_address_ids: [address.id]) + tx = create(:ckb_transaction, block: block, tags: ["udt"], contained_udt_ids: [udt.id], udt_address_ids: [address.id], contained_address_ids: [address.id]) + tx1 = create(:ckb_transaction, block: block, tags: ["udt"], contained_udt_ids: [udt.id], udt_address_ids: [address.id], contained_address_ids: [address.id]) create(:cell_output, block: block, ckb_transaction: tx1, cell_type: "udt", type_hash: udt.type_hash, generated_by: tx1, address: address) create(:cell_output, block: block, ckb_transaction: tx, cell_type: "udt", type_hash: udt.type_hash, generated_by: tx, consumed_by_id: tx1, address: address) end @@ -209,6 +209,6 @@ class AddressTest < ActiveSupport::TestCase test "#ckb_udt_transactions should return an empty array when udt not exist" do address = create(:address) - assert_equal [], address.ckb_udt_transactions("0x11") + assert_equal [], address.ckb_udt_transactions(123) end end diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index 92ff8809d..a88cf6bec 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -1483,7 +1483,9 @@ class NodeDataProcessorTest < ActiveSupport::TestCase address2 = Address.find_by(lock_hash: lock2.compute_hash) address3 = Address.find_by(lock_hash: lock3.compute_hash) CkbSync::Api.any_instance.stubs(:get_tip_block_number).returns(block.number + 1) - node_data_processor.call + VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}", record: :new_episodes) do + node_data_processor.call + end assert_equal 0, address1.reload.dao_transactions_count assert_equal 0, address2.reload.dao_transactions_count diff --git a/test/models/dao_contract_test.rb b/test/models/dao_contract_test.rb index 2336038e9..143c78cc7 100644 --- a/test/models/dao_contract_test.rb +++ b/test/models/dao_contract_test.rb @@ -24,7 +24,7 @@ class DaoContractTest < ActiveSupport::TestCase test "should have correct columns" do dao_contract = create(:dao_contract) - expected_attributes = %w(created_at deposit_transactions_count depositors_count id claimed_compensation total_deposit total_depositors_count updated_at withdraw_transactions_count unclaimed_compensation) + expected_attributes = %w(created_at deposit_transactions_count depositors_count id claimed_compensation total_deposit total_depositors_count updated_at withdraw_transactions_count unclaimed_compensation ckb_transactions_count) assert_equal expected_attributes.sort, dao_contract.attributes.keys.sort end From d3c61c71b6b293433cefa36b54f7c2b5ce2d50a8 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 11:57:27 +0800 Subject: [PATCH 45/76] test: should update table_record_counts block count after block has been processed --- test/factories/table_record_count.rb | 14 ++++++++++++++ test/models/ckb_sync/node_data_processor_test.rb | 11 +++++++++++ 2 files changed, 25 insertions(+) create mode 100644 test/factories/table_record_count.rb diff --git a/test/factories/table_record_count.rb b/test/factories/table_record_count.rb new file mode 100644 index 000000000..03fc8b7b5 --- /dev/null +++ b/test/factories/table_record_count.rb @@ -0,0 +1,14 @@ +FactoryBot.define do + factory :table_record_count do + table_name { "" } + count { 0 } + + trait :block_counter do + table_name { "blocks" } + end + + trait :ckb_transactions_counter do + table_name { "ckb_transactions" } + end + end +end diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index a88cf6bec..f0163cee3 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -23,6 +23,17 @@ class NodeDataProcessorTest < ActiveSupport::TestCase end end + test "should update table_record_counts block count after block has been processed" do + block_counter = create(:table_record_count, :block_counter) + VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}") do + node_block = CkbSync::Api.instance.get_block_by_number(DEFAULT_NODE_BLOCK_NUMBER) + create(:block, :with_block_hash, number: node_block.header.number - 1) + assert_difference -> { block_counter.reload.count }, 1 do + node_data_processor.process_block(node_block) + end + end + end + test "#process_block created block's attribute value should equal with the node block's attribute value" do CkbSync::Api.any_instance.stubs(:get_epoch_by_number).returns( CKB::Types::Epoch.new( From 06c6ab4ea2fdd391df2e665e36301cb7a5bc35d6 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 11:58:01 +0800 Subject: [PATCH 46/76] feat: update table_record_counts block count after block has been processed --- app/models/ckb_sync/node_data_processor.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index e95acfe05..95e5946b8 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -47,6 +47,8 @@ def process_block(node_block) update_dao_contract_related_info(local_block) end + block_counter = TableRecordCount.find_by(table_name: "blocks") + block_counter.increment!(:count) local_block end From 2056986c2a25d3537961a54ef1ba4e78835547f1 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 12:03:04 +0800 Subject: [PATCH 47/76] test: should recalculate table_record_counts block count when block is invalid --- test/models/ckb_sync/node_data_processor_test.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index f0163cee3..bd78fc400 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -1334,6 +1334,19 @@ class NodeDataProcessorTest < ActiveSupport::TestCase end end + test "should recalculate table_record_counts block count when block is invalid" do + block_counter = create(:table_record_count, :block_counter) + prepare_node_data(12) + local_block = Block.find_by(number: 12) + local_block.update(block_hash: "0x419c632366c8eb9635acbb39ea085f7552ae62e1fdd480893375334a0f37d1bx") + + VCR.use_cassette("blocks/13") do + assert_difference -> { block_counter.reload.count }, -1 do + node_data_processor.call + end + end + end + test "#process_block should update abandoned block's contained address's balance" do prepare_node_data(12) local_block = Block.find_by(number: 12) From 15f9cd4375659ef57082b03a575122e210b8bd16 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 12:03:15 +0800 Subject: [PATCH 48/76] feat: recalculate table_record_counts block count when block is invalid --- app/models/ckb_sync/node_data_processor.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index 95e5946b8..82bf2e34b 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -209,6 +209,8 @@ def invalid_block(local_tip_block) revert_block_rewards(local_tip_block) ForkedEvent.create!(block_number: local_tip_block.number, epoch_number: local_tip_block.epoch, block_timestamp: local_tip_block.timestamp) Charts::BlockStatisticGenerator.new(local_tip_block.number).call + block_counter = TableRecordCount.find_by(table_name: "blocks") + block_counter.decrement!(:count) local_tip_block end From d3b2f52d4bab2de311939b1ca8e0e953d774259d Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 12:16:20 +0800 Subject: [PATCH 49/76] test: should update ckb transactions count after block has been processed --- test/models/ckb_sync/node_data_processor_test.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index bd78fc400..2a4918796 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -34,6 +34,18 @@ class NodeDataProcessorTest < ActiveSupport::TestCase end end + test "should update table_record_counts ckb transactions count after block has been processed" do + create(:table_record_count, :block_counter) + ckb_transaction_counter = create(:table_record_count, :ckb_transactions_counter) + VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}") do + node_block = CkbSync::Api.instance.get_block_by_number(DEFAULT_NODE_BLOCK_NUMBER) + create(:block, :with_block_hash, number: node_block.header.number - 1) + assert_difference -> { ckb_transaction_counter.reload.count }, node_block.transactions.count do + node_data_processor.process_block(node_block) + end + end + end + test "#process_block created block's attribute value should equal with the node block's attribute value" do CkbSync::Api.any_instance.stubs(:get_epoch_by_number).returns( CKB::Types::Epoch.new( From f17d75919e931256b873670c8c725d7d76d77dc8 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 12:19:11 +0800 Subject: [PATCH 50/76] feat: update ckb transactions count after block has been processed --- app/models/ckb_sync/node_data_processor.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index 82bf2e34b..f24abdcc1 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -45,10 +45,13 @@ def process_block(node_block) DaoEvent.import!(dao_events, validate: false) update_dao_contract_related_info(local_block) + block_counter = TableRecordCount.find_by(table_name: "blocks") + block_counter.increment!(:count) + ckb_transaction_counter = TableRecordCount.find_by(table_name: "ckb_transactions") + ckb_transaction_counter.increment!(:count, ckb_transactions.count) end - block_counter = TableRecordCount.find_by(table_name: "blocks") - block_counter.increment!(:count) + local_block end From 5835e4bca71fe935e41d6a6a994ae0f5c0f86eaf Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 12:25:32 +0800 Subject: [PATCH 51/76] test: should recalculate table_record_counts ckb_transactions count when block is invalid --- test/models/ckb_sync/node_data_processor_test.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index 2a4918796..54422c405 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -1359,6 +1359,21 @@ class NodeDataProcessorTest < ActiveSupport::TestCase end end + test "should recalculate table_record_counts ckb_transactions count when block is invalid" do + create(:table_record_count, :block_counter) + ckb_transactions_counter = create(:table_record_count, :ckb_transactions_counter) + + prepare_node_data(12) + local_block = Block.find_by(number: 12) + local_block.update(block_hash: "0x419c632366c8eb9635acbb39ea085f7552ae62e1fdd480893375334a0f37d1bx") + + VCR.use_cassette("blocks/13") do + assert_difference -> { ckb_transactions_counter.reload.count }, -local_block.ckb_transactions.count do + node_data_processor.call + end + end + end + test "#process_block should update abandoned block's contained address's balance" do prepare_node_data(12) local_block = Block.find_by(number: 12) From 8b3d4f1dfc751ed97f2c14b4b17f63dc305f4217 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 12:27:46 +0800 Subject: [PATCH 52/76] feat: recalculate table_record_counts ckb_transactions count when block is invalid --- app/models/ckb_sync/node_data_processor.rb | 23 +++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index f24abdcc1..91b3bb3fd 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -45,18 +45,21 @@ def process_block(node_block) DaoEvent.import!(dao_events, validate: false) update_dao_contract_related_info(local_block) - block_counter = TableRecordCount.find_by(table_name: "blocks") - block_counter.increment!(:count) - ckb_transaction_counter = TableRecordCount.find_by(table_name: "ckb_transactions") - ckb_transaction_counter.increment!(:count, ckb_transactions.count) + increase_records_count(ckb_transactions) end - local_block end private + def increase_records_count(ckb_transactions) + block_counter = TableRecordCount.find_by(table_name: "blocks") + block_counter.increment!(:count) + ckb_transaction_counter = TableRecordCount.find_by(table_name: "ckb_transactions") + ckb_transaction_counter.increment!(:count, ckb_transactions.count) + end + def update_udt_info(udt_infos) return if udt_infos.blank? @@ -206,19 +209,25 @@ def invalid_block(local_tip_block) udt_type_hashes = local_tip_block.cell_outputs.udt.pluck(:type_hash).uniq recalculate_udt_transactions_count(local_tip_block) recalculate_dao_contract_transactions_count(local_tip_block) + decrease_records_count(local_tip_block) local_tip_block.invalid! recalculate_udt_accounts(udt_type_hashes, local_tip_block) local_tip_block.contained_addresses.each(&method(:update_address_balance_and_ckb_transactions_count)) revert_block_rewards(local_tip_block) ForkedEvent.create!(block_number: local_tip_block.number, epoch_number: local_tip_block.epoch, block_timestamp: local_tip_block.timestamp) Charts::BlockStatisticGenerator.new(local_tip_block.number).call - block_counter = TableRecordCount.find_by(table_name: "blocks") - block_counter.decrement!(:count) local_tip_block end end + def decrease_records_count(local_tip_block) + block_counter = TableRecordCount.find_by(table_name: "blocks") + block_counter.decrement!(:count) + ckb_transaction_counter = TableRecordCount.find_by(table_name: "ckb_transactions") + ckb_transaction_counter.decrement!(:count, local_tip_block.ckb_transactions.count) + end + def recalculate_dao_contract_transactions_count(local_tip_block) dao_transactions_count = local_tip_block.ckb_transactions.where("tags @> array[?]::varchar[]", ["dao"]).count DaoContract.default_contract.decrement!(:ckb_transactions_count, dao_transactions_count) if dao_transactions_count > 0 From 02a2f298e63feb9b656d5dbf09497402860e158b Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 12:45:04 +0800 Subject: [PATCH 53/76] test: block transactions counter --- test/models/record_counters/block_transactions.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 test/models/record_counters/block_transactions.rb diff --git a/test/models/record_counters/block_transactions.rb b/test/models/record_counters/block_transactions.rb new file mode 100644 index 000000000..fc687e135 --- /dev/null +++ b/test/models/record_counters/block_transactions.rb @@ -0,0 +1,15 @@ +require "test_helper" + +class BlockTransactionsTest < ActiveSupport::TestCase + test "should respond to total_count" do + block = create(:block, :with_block_hash) + block_transactions_counter = RecordCounters::BlockTransactions.new(block) + assert_respond_to block_transactions_counter, :total_count + end + + test "total_count should return dao_contract ckb transactions count" do + block = create(:block, :with_block_hash) + block_transactions_counter = RecordCounters::BlockTransactions.new(block) + assert_equal block.ckb_transactions_count, block_transactions_counter.total_count + end +end From 5ea2e44272ad81c78514d42097daf444d44d21fb Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 12:45:18 +0800 Subject: [PATCH 54/76] feat: add block transactions counter --- app/models/record_counters/block_transactions.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 app/models/record_counters/block_transactions.rb diff --git a/app/models/record_counters/block_transactions.rb b/app/models/record_counters/block_transactions.rb new file mode 100644 index 000000000..d4938c6eb --- /dev/null +++ b/app/models/record_counters/block_transactions.rb @@ -0,0 +1,15 @@ +module RecordCounters + class BlockTransactions + def initialize(block) + @block = block + end + + def total_count + block.ckb_transactions_count + end + + private + + attr_reader :block + end +end From e705e9fe59692713ee7cb25e798d9cefe41b100d Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 12:49:49 +0800 Subject: [PATCH 55/76] feat: use block transactions counter --- app/controllers/api/v1/block_transactions_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/v1/block_transactions_controller.rb b/app/controllers/api/v1/block_transactions_controller.rb index 45db75803..5302bb349 100644 --- a/app/controllers/api/v1/block_transactions_controller.rb +++ b/app/controllers/api/v1/block_transactions_controller.rb @@ -9,7 +9,8 @@ def show ckb_transactions = block.ckb_transactions.select(:id, :tx_hash, :block_id, :block_number, :block_timestamp, :is_cellbase).order(:id).page(@page).per(@page_size) json = Rails.cache.realize(ckb_transactions.cache_key, version: ckb_transactions.cache_version) do - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: @page, page_size: @page_size).call + records_counter = RecordCounters::BlockTransactions.new(block) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: @page, page_size: @page_size, records_counter: records_counter).call CkbTransactionsSerializer.new(ckb_transactions, options.merge(params: { previews: true })).serialized_json end From 46f2e9d16622d074073a7ff5e5dd7c417fecd571 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 12:50:08 +0800 Subject: [PATCH 56/76] chore: adjust block transactions controller test --- .../api/v1/block_transactions_controller_test.rb | 16 +++++++++++----- test/factories/block.rb | 1 + 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/test/controllers/api/v1/block_transactions_controller_test.rb b/test/controllers/api/v1/block_transactions_controller_test.rb index 18530e66d..17b533951 100644 --- a/test/controllers/api/v1/block_transactions_controller_test.rb +++ b/test/controllers/api/v1/block_transactions_controller_test.rb @@ -79,8 +79,10 @@ class BlockTransactionsControllerTest < ActionDispatch::IntegrationTest block = create(:block, :with_ckb_transactions) valid_get api_v1_block_transaction_url(block.block_hash) + ckb_transactions = block.ckb_transactions.order(:id).page(page).per(page_size) - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: page, page_size: page_size).call + records_counter = RecordCounters::BlockTransactions.new(block) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: page, page_size: page_size, records_counter: records_counter).call assert_equal CkbTransactionsSerializer.new(ckb_transactions, options).serialized_json, response.body end @@ -160,7 +162,8 @@ class BlockTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_block_transaction_url(block.block_hash), params: { page: page } - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: block_ckb_transactions, page: page, page_size: page_size).call + records_counter = RecordCounters::BlockTransactions.new(block) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: block_ckb_transactions, page: page, page_size: page_size, records_counter: records_counter).call response_transaction = CkbTransactionsSerializer.new(block_ckb_transactions, options).serialized_json assert_equal response_transaction, response.body @@ -175,7 +178,8 @@ class BlockTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_block_transaction_url(block.block_hash), params: { page_size: page_size } - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: block_ckb_transactions, page: page, page_size: page_size).call + records_counter = RecordCounters::BlockTransactions.new(block) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: block_ckb_transactions, page: page, page_size: page_size, records_counter: records_counter).call response_transaction = CkbTransactionsSerializer.new(block_ckb_transactions, options).serialized_json assert_equal response_transaction, response.body @@ -190,7 +194,8 @@ class BlockTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_block_transaction_url(block.block_hash), params: { page: page, page_size: page_size } - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: block_ckb_transactions, page: page, page_size: page_size).call + records_counter = RecordCounters::BlockTransactions.new(block) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: block_ckb_transactions, page: page, page_size: page_size, records_counter: records_counter).call response_transaction = CkbTransactionsSerializer.new(block_ckb_transactions, options).serialized_json assert_equal response_transaction, response.body @@ -204,7 +209,8 @@ class BlockTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_block_transaction_url(block.block_hash), params: { page: page, page_size: page_size } - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: block_ckb_transactions, page: page, page_size: page_size).call + records_counter = RecordCounters::BlockTransactions.new(block) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: block_ckb_transactions, page: page, page_size: page_size, records_counter: records_counter).call response_transaction = CkbTransactionsSerializer.new(block_ckb_transactions, options).serialized_json assert_equal [], json["data"] diff --git a/test/factories/block.rb b/test/factories/block.rb index fa90a9a94..306fbecbe 100644 --- a/test/factories/block.rb +++ b/test/factories/block.rb @@ -47,6 +47,7 @@ trait :with_ckb_transactions do after(:create) do |block, evaluator| create_list(:ckb_transaction, evaluator.transactions_count, block: block) + block.update(ckb_transactions_count: evaluator.transactions_count) end end end From 195ee485e3d875f98d5ceca101d71c0f15b8bef9 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 13:12:05 +0800 Subject: [PATCH 57/76] feat: add task to init table record count --- lib/tasks/migration/init_table_record_count.rake | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 lib/tasks/migration/init_table_record_count.rake diff --git a/lib/tasks/migration/init_table_record_count.rake b/lib/tasks/migration/init_table_record_count.rake new file mode 100644 index 000000000..5d39c73f6 --- /dev/null +++ b/lib/tasks/migration/init_table_record_count.rake @@ -0,0 +1,12 @@ +namespace :migration do + desc "Usage: RAILS_ENV=production bundle exec rake migration:init_table_record_count" + task init_table_record_count: :environment do + ApplicationRecord.transaction do + ActiveRecord::Base.connection.execute('LOCK blocks IN ACCESS EXCLUSIVE MODE') + TableRecordCount.create(table_name: "blocks", count: Block.count) + TableRecordCount.create(table_name: "ckb_transactions", count: CkbTransaction.count) + end + + puts "done" + end +end From 6d1fa494bfbebe9dd1d02697f37e77e8ffbe6a65 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 13:16:23 +0800 Subject: [PATCH 58/76] test: address transactions counter --- .../record_counters/address_dao_transactions.rb | 2 +- .../record_counters/address_transactions.rb | 15 +++++++++++++++ .../record_counters/address_udt_transactions.rb | 2 +- test/models/record_counters/block_transactions.rb | 2 +- .../record_counters/udt_transactions_test.rb | 2 +- 5 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 test/models/record_counters/address_transactions.rb diff --git a/test/models/record_counters/address_dao_transactions.rb b/test/models/record_counters/address_dao_transactions.rb index 6deb098bb..211686e6a 100644 --- a/test/models/record_counters/address_dao_transactions.rb +++ b/test/models/record_counters/address_dao_transactions.rb @@ -7,7 +7,7 @@ class AddressDaoTransactionsTest < ActiveSupport::TestCase assert_respond_to address_dao_transactions_counter, :total_count end - test "total_count should return dao_contract ckb transactions count" do + test "total_count should return address dao transactions count" do address = create(:address) address_dao_transactions_counter = RecordCounters::AddressDaoTransactions.new(address) assert_equal address.dao_transactions_count, address_dao_transactions_counter.total_count diff --git a/test/models/record_counters/address_transactions.rb b/test/models/record_counters/address_transactions.rb new file mode 100644 index 000000000..412a58c92 --- /dev/null +++ b/test/models/record_counters/address_transactions.rb @@ -0,0 +1,15 @@ +require "test_helper" + +class AddressTransactionsTest < ActiveSupport::TestCase + test "should respond to total_count" do + address = create(:address) + address_transactions_counter = RecordCounters::AddressTransactions.new(address) + assert_respond_to address_transactions_counter, :total_count + end + + test "total_count should return address ckb transactions count" do + address = create(:address) + address_transactions_counter = RecordCounters::AddressTransactions.new(address) + assert_equal address.ckb_transactions_count, address_transactions_counter.total_count + end +end diff --git a/test/models/record_counters/address_udt_transactions.rb b/test/models/record_counters/address_udt_transactions.rb index 3341efd11..27a803baa 100644 --- a/test/models/record_counters/address_udt_transactions.rb +++ b/test/models/record_counters/address_udt_transactions.rb @@ -8,7 +8,7 @@ class AddressUdtTransactionsTest < ActiveSupport::TestCase assert_respond_to address_udt_transactions_counter, :total_count end - test "total_count should return dao_contract ckb transactions count" do + test "total_count should return address udt transactions count" do address = create(:address) udt = create(:udt, published: true) address_udt_transactions_counter = RecordCounters::AddressUdtTransactions.new(address, udt.id) diff --git a/test/models/record_counters/block_transactions.rb b/test/models/record_counters/block_transactions.rb index fc687e135..2e13e07b7 100644 --- a/test/models/record_counters/block_transactions.rb +++ b/test/models/record_counters/block_transactions.rb @@ -7,7 +7,7 @@ class BlockTransactionsTest < ActiveSupport::TestCase assert_respond_to block_transactions_counter, :total_count end - test "total_count should return dao_contract ckb transactions count" do + test "total_count should return block ckb transactions count" do block = create(:block, :with_block_hash) block_transactions_counter = RecordCounters::BlockTransactions.new(block) assert_equal block.ckb_transactions_count, block_transactions_counter.total_count diff --git a/test/models/record_counters/udt_transactions_test.rb b/test/models/record_counters/udt_transactions_test.rb index 7d1662595..b31a5d2ff 100644 --- a/test/models/record_counters/udt_transactions_test.rb +++ b/test/models/record_counters/udt_transactions_test.rb @@ -7,7 +7,7 @@ class UdtTransactionsTest < ActiveSupport::TestCase assert_respond_to udt_transactions_counter, :total_count end - test "total_count should return udt's ckb transactions count" do + test "total_count should return udt ckb transactions count" do udt = create(:udt, published: true) udt_transactions_counter = RecordCounters::UdtTransactions.new(udt) assert_equal udt.ckb_transactions_count, udt_transactions_counter.total_count From 604bf23188b258cff9308f91cb775001fd70f3a1 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 13:16:34 +0800 Subject: [PATCH 59/76] feat: add address transactions counter --- .../record_counters/address_transactions.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 app/models/record_counters/address_transactions.rb diff --git a/app/models/record_counters/address_transactions.rb b/app/models/record_counters/address_transactions.rb new file mode 100644 index 000000000..3811be91d --- /dev/null +++ b/app/models/record_counters/address_transactions.rb @@ -0,0 +1,15 @@ +module RecordCounters + class AddressTransactions + def initialize(address) + @address = address + end + + def total_count + address.ckb_transactions_count + end + + private + + attr_reader :address + end +end From e7b2533a0fb862bd2fb9a9a35c9d310f895e1745 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 13:20:50 +0800 Subject: [PATCH 60/76] feat: use address transactions counter --- app/controllers/api/v1/address_transactions_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/v1/address_transactions_controller.rb b/app/controllers/api/v1/address_transactions_controller.rb index caef4b99f..aa8a0cdf0 100644 --- a/app/controllers/api/v1/address_transactions_controller.rb +++ b/app/controllers/api/v1/address_transactions_controller.rb @@ -12,7 +12,8 @@ def show json = Rails.cache.realize(@ckb_transactions.cache_key, version: @ckb_transactions.cache_version) do - @options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: @ckb_transactions, page: @page, page_size: @page_size).call + records_counter = RecordCounters::AddressTransactions.new(@address) + @options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: @ckb_transactions, page: @page, page_size: @page_size, records_counter: records_counter).call json_result end From ed290dd1f94425c96bc3ca088c7c8f46569c4139 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 13:21:05 +0800 Subject: [PATCH 61/76] chore: adjust address transactions controller tests --- .../address_transactions_controller_test.rb | 28 +++++++++++-------- test/factories/address.rb | 1 + 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/test/controllers/api/v1/address_transactions_controller_test.rb b/test/controllers/api/v1/address_transactions_controller_test.rb index 476331d20..784bafade 100644 --- a/test/controllers/api/v1/address_transactions_controller_test.rb +++ b/test/controllers/api/v1/address_transactions_controller_test.rb @@ -72,9 +72,10 @@ class AddressTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_address_transaction_url(address.address_hash) - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: page, page_size: page_size).call + records_counter = RecordCounters::AddressTransactions.new(address) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: page, page_size: page_size, records_counter: records_counter).call - assert_equal CkbTransactionsSerializer.new(ckb_transactions, options.merge({ params: { previews: true, address: address } })).serialized_json, response.body + assert_equal CkbTransactionsSerializer.new(ckb_transactions, options.merge(params: { previews: true, address: address })).serialized_json, response.body end test "should return corresponding ckb transactions with given lock hash" do @@ -85,9 +86,10 @@ class AddressTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_address_transaction_url(address.lock_hash) - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: page, page_size: page_size).call + records_counter = RecordCounters::AddressTransactions.new(address) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: page, page_size: page_size, records_counter: records_counter).call - assert_equal CkbTransactionsSerializer.new(ckb_transactions, options.merge({ params: { previews: true, address: address } })).serialized_json, response.body + assert_equal CkbTransactionsSerializer.new(ckb_transactions, options.merge(params: { previews: true, address: address })).serialized_json, response.body end test "should contain right keys in the serialized object when call show" do @@ -179,7 +181,7 @@ class AddressTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_address_transaction_url(address.address_hash), params: { page: page } options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_ckb_transactions, page: page, page_size: page_size).call - response_transaction = CkbTransactionsSerializer.new(address_ckb_transactions, options.merge({ params: { previews: true, address: address } })).serialized_json + response_transaction = CkbTransactionsSerializer.new(address_ckb_transactions, options.merge(params: { previews: true, address: address })).serialized_json assert_equal response_transaction, response.body assert_equal page_size, json["data"].size @@ -193,8 +195,9 @@ class AddressTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_address_transaction_url(address.address_hash), params: { page_size: page_size } - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_ckb_transactions, page: page, page_size: page_size).call - response_transaction = CkbTransactionsSerializer.new(address_ckb_transactions, options.merge({ params: { previews: true, address: address } })).serialized_json + records_counter = RecordCounters::AddressTransactions.new(address) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_ckb_transactions, page: page, page_size: page_size, records_counter: records_counter).call + response_transaction = CkbTransactionsSerializer.new(address_ckb_transactions, options.merge(params: { previews: true, address: address })).serialized_json assert_equal response_transaction, response.body assert_equal page_size, json["data"].size @@ -207,8 +210,10 @@ class AddressTransactionsControllerTest < ActionDispatch::IntegrationTest address_ckb_transactions = address.ckb_transactions.order(block_timestamp: :desc).page(page).per(page_size) valid_get api_v1_address_transaction_url(address.address_hash), params: { page: page, page_size: page_size } - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_ckb_transactions, page: page, page_size: page_size).call - response_transaction = CkbTransactionsSerializer.new(address_ckb_transactions, options.merge({ params: { previews: true, address: address } })).serialized_json + + records_counter = RecordCounters::AddressTransactions.new(address) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_ckb_transactions, page: page, page_size: page_size, records_counter: records_counter).call + response_transaction = CkbTransactionsSerializer.new(address_ckb_transactions, options.merge(params: { previews: true, address: address })).serialized_json assert_equal response_transaction, response.body end @@ -221,8 +226,9 @@ class AddressTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_address_transaction_url(address.address_hash), params: { page: page, page_size: page_size } - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_ckb_transactions, page: page, page_size: page_size).call - response_transaction = CkbTransactionsSerializer.new(address_ckb_transactions, options.merge({ params: { previews: true, address: address } })).serialized_json + records_counter = RecordCounters::AddressTransactions.new(address) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: address_ckb_transactions, page: page, page_size: page_size, records_counter: records_counter).call + response_transaction = CkbTransactionsSerializer.new(address_ckb_transactions, options.merge(params: { previews: true, address: address })).serialized_json assert_equal [], json["data"] assert_equal response_transaction, response.body diff --git a/test/factories/address.rb b/test/factories/address.rb index b07abf75a..7cf329800 100644 --- a/test/factories/address.rb +++ b/test/factories/address.rb @@ -40,6 +40,7 @@ tx.contained_address_ids << address.id end address.ckb_transactions << ckb_transactions + address.update(ckb_transactions_count: address.ckb_transactions.count) end end From 025b9d73b78367c06fe79825901f61986f3c6614 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 13:33:47 +0800 Subject: [PATCH 62/76] test: blocks counter --- test/models/record_counters/blocks.rb | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 test/models/record_counters/blocks.rb diff --git a/test/models/record_counters/blocks.rb b/test/models/record_counters/blocks.rb new file mode 100644 index 000000000..dea207b88 --- /dev/null +++ b/test/models/record_counters/blocks.rb @@ -0,0 +1,31 @@ +require "test_helper" + +class BlockTransactionsTest < ActiveSupport::TestCase + test "should respond to total_count" do + create(:block, :with_block_hash) + blocks_counter = RecordCounters::Blocks.new + assert_respond_to blocks_counter, :total_count + end + + test "total_count should return blocks count" do + create(:table_record_count, :block_counter) + create(:table_record_count, :ckb_transactions_counter) + blocks_counter = RecordCounters::Blocks.new + + VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}") do + CkbSync::Api.any_instance.stubs(:get_epoch_by_number).returns( + CKB::Types::Epoch.new( + compact_target: "0x1000", + length: "0x07d0", + number: "0x0", + start_number: "0x0" + ) + ) + node_block = CkbSync::Api.instance.get_block_by_number(DEFAULT_NODE_BLOCK_NUMBER) + create(:block, :with_block_hash, number: node_block.header.number - 1) + TableRecordCount.find_by(table_name: "blocks").increment!(:count) + CkbSync::NodeDataProcessor.new.process_block(node_block) + assert_equal Block.count, blocks_counter.total_count + end + end +end From 36f9ab4c39efe5d66b2d831c4001a158afe30fba Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 13:33:58 +0800 Subject: [PATCH 63/76] feat: add blocks counter --- app/models/record_counters/blocks.rb | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 app/models/record_counters/blocks.rb diff --git a/app/models/record_counters/blocks.rb b/app/models/record_counters/blocks.rb new file mode 100644 index 000000000..bd5f30d4e --- /dev/null +++ b/app/models/record_counters/blocks.rb @@ -0,0 +1,7 @@ +module RecordCounters + class Blocks + def total_count + TableRecordCount.find_by(table_name: "blocks").count + end + end +end From 44693de4c0bbd43b570c0918c824a073a4d4d225 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 13:39:23 +0800 Subject: [PATCH 64/76] test: blocks counter --- test/controllers/api/v1/blocks_controller_test.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/controllers/api/v1/blocks_controller_test.rb b/test/controllers/api/v1/blocks_controller_test.rb index aa74070b1..5b870aaa0 100644 --- a/test/controllers/api/v1/blocks_controller_test.rb +++ b/test/controllers/api/v1/blocks_controller_test.rb @@ -123,6 +123,7 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest page_size = 30 create_list(:block, 15, :with_block_hash) create(:block) + create(:table_record_count, :block_counter, count: 16) valid_get api_v1_blocks_url, params: { page: page, page_size: page_size } @@ -160,6 +161,7 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest test "should return the corresponding blocks when page and page_size are set" do create_list(:block, 15, :with_block_hash) + create(:table_record_count, :block_counter, count: 15) page = 2 page_size = 5 block_timestamps = Block.recent.select(:timestamp).page(page).per(page_size) @@ -167,19 +169,22 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_blocks_url, params: { page: page, page_size: page_size } - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: block_timestamps, page: page, page_size: page_size).call + records_counter = RecordCounters::Blocks.new + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: block_timestamps, page: page, page_size: page_size, records_counter: records_counter).call response_blocks = BlockListSerializer.new(blocks, options).serialized_json assert_equal response_blocks, response.body end test "should return empty array when there is no blocks" do + create(:table_record_count, :block_counter) page = 2 page_size = 5 blocks = Block.order(timestamp: :desc).page(page).per(page_size) valid_get api_v1_blocks_url, params: { page: page, page_size: page_size } - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: blocks, page: page, page_size: page_size).call + records_counter = RecordCounters::Blocks.new + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: blocks, page: page, page_size: page_size, records_counter: records_counter).call response_blocks = BlockSerializer.new(blocks, options).serialized_json assert_equal [], json["data"] @@ -190,6 +195,7 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest page = 2 page_size = 3 create_list(:block, 30, :with_block_hash) + create(:table_record_count, :block_counter, count: 30) links = { self: "#{api_v1_blocks_url}?page=2&page_size=3", first: "#{api_v1_blocks_url}?page_size=3", @@ -207,12 +213,14 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest page = 1 page_size = 30 create_list(:block, 30, :with_block_hash) + create(:table_record_count, :block_counter, count: 30) valid_get api_v1_blocks_url, params: { page: page, page_size: page_size } assert_equal page_size, json.dig("meta", "total") end test "should return pagination links that only contain self in response body when there is no blocks" do + create(:table_record_count, :block_counter) page = 1 page_size = 10 links = { From 79dc5a89f75ba936104238fbe94461ee2fc8d275 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 13:39:36 +0800 Subject: [PATCH 65/76] feat: use blocks counter --- app/controllers/api/v1/blocks_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/v1/blocks_controller.rb b/app/controllers/api/v1/blocks_controller.rb index 64e0097fb..20a4ca1ae 100644 --- a/app/controllers/api/v1/blocks_controller.rb +++ b/app/controllers/api/v1/blocks_controller.rb @@ -15,7 +15,8 @@ def index blocks = Block.recent.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count, :live_cell_changes).page(@page).per(@page_size) json = Rails.cache.realize(blocks.cache_key, version: blocks.cache_version, race_condition_ttl: 3.seconds) do - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: blocks, page: @page, page_size: @page_size).call + records_counter = RecordCounters::Blocks.new + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: blocks, page: @page, page_size: @page_size, records_counter: records_counter).call BlockListSerializer.new(blocks, options).serialized_json end end From 52df93415b8db1b308e0ec0079e15b2686779c9a Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 13:45:39 +0800 Subject: [PATCH 66/76] test: transactions counter --- test/models/record_counters/transactions.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 test/models/record_counters/transactions.rb diff --git a/test/models/record_counters/transactions.rb b/test/models/record_counters/transactions.rb new file mode 100644 index 000000000..4baf88d40 --- /dev/null +++ b/test/models/record_counters/transactions.rb @@ -0,0 +1,15 @@ +require "test_helper" + +class TransactionsTest < ActiveSupport::TestCase + test "should respond to total_count" do + transactions_counter = RecordCounters::Transactions.new + assert_respond_to transactions_counter, :total_count + end + + test "total_count should return block ckb transactions count" do + create(:ckb_transaction) + create(:table_record_count, :ckb_transactions_counter, count: CkbTransaction.count) + transactions_counter = RecordCounters::Transactions.new + assert_equal CkbTransaction.count, transactions_counter.total_count + end +end From 91b422be4386ea129614b037e4bb9c222c2cdec2 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 13:45:45 +0800 Subject: [PATCH 67/76] feat: transactions counter --- app/models/record_counters/transactions.rb | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 app/models/record_counters/transactions.rb diff --git a/app/models/record_counters/transactions.rb b/app/models/record_counters/transactions.rb new file mode 100644 index 000000000..6231815e8 --- /dev/null +++ b/app/models/record_counters/transactions.rb @@ -0,0 +1,7 @@ +module RecordCounters + class Transactions + def total_count + TableRecordCount.find_by(table_name: "ckb_transactions").count + end + end +end From e963be858e28510b7ad265e47a84ed9e12d22e2a Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 13:49:31 +0800 Subject: [PATCH 68/76] test: transactions counter --- .../api/v1/ckb_transactions_controller_test.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/controllers/api/v1/ckb_transactions_controller_test.rb b/test/controllers/api/v1/ckb_transactions_controller_test.rb index 4b351a7ae..4ec4f97de 100644 --- a/test/controllers/api/v1/ckb_transactions_controller_test.rb +++ b/test/controllers/api/v1/ckb_transactions_controller_test.rb @@ -91,6 +91,8 @@ class CkbTransactionsControllerTest < ActionDispatch::IntegrationTest end test "should contain right keys in the serialized object when call show" do + create(:table_record_count, :block_counter) + create(:table_record_count, :ckb_transactions_counter) prepare_node_data(8) ckb_transaction = CkbTransaction.last @@ -101,6 +103,8 @@ class CkbTransactionsControllerTest < ActionDispatch::IntegrationTest end test "returned income should be null" do + create(:table_record_count, :block_counter) + create(:table_record_count, :ckb_transactions_counter) prepare_node_data(8) ckb_transaction = CkbTransaction.last @@ -302,13 +306,16 @@ class CkbTransactionsControllerTest < ActionDispatch::IntegrationTest test "should return the corresponding blocks when page and page_size are set" do block = create(:block, :with_block_hash) create_list(:ckb_transaction, 15, block: block) + create(:table_record_count, :block_counter, count: Block.count) + create(:table_record_count, :ckb_transactions_counter, count: CkbTransaction.count) page = 2 page_size = 5 ckb_transactions = CkbTransaction.order(block_timestamp: :desc).page(page).per(page_size) valid_get api_v1_ckb_transactions_url, params: { page: page, page_size: page_size } - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: page, page_size: page_size).call + records_counter = RecordCounters::Transactions.new + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: page, page_size: page_size, records_counter: records_counter).call response_ckb_transactions = CkbTransactionListSerializer.new(ckb_transactions, options).serialized_json assert_equal response_ckb_transactions, response.body end From def5e457cb691a07204484251e977fc352311526 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 13:49:38 +0800 Subject: [PATCH 69/76] feat: use transactions counter --- app/controllers/api/v1/ckb_transactions_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/v1/ckb_transactions_controller.rb b/app/controllers/api/v1/ckb_transactions_controller.rb index 629045b6b..62642b99b 100644 --- a/app/controllers/api/v1/ckb_transactions_controller.rb +++ b/app/controllers/api/v1/ckb_transactions_controller.rb @@ -16,7 +16,8 @@ def index ckb_transactions = CkbTransaction.recent.normal.page(@page).per(@page_size).select(:id, :tx_hash, :block_number, :block_timestamp, :live_cell_changes, :capacity_involved) json = Rails.cache.realize(ckb_transactions.cache_key, version: ckb_transactions.cache_version, race_condition_ttl: 3.seconds) do - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: @page, page_size: @page_size).call + records_counter = RecordCounters::Transactions.new + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: @page, page_size: @page_size, records_counter: records_counter).call CkbTransactionListSerializer.new(ckb_transactions, options).serialized_json end render json: json From dd53fef84c2c644aaafcc3eb923dbb7046edaf7a Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 14:04:48 +0800 Subject: [PATCH 70/76] chore: update fill dao transactions count to address task --- lib/tasks/migration/fill_dao_transactions_count_to_address.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/migration/fill_dao_transactions_count_to_address.rake b/lib/tasks/migration/fill_dao_transactions_count_to_address.rake index ba66aeef8..5d0480a2d 100644 --- a/lib/tasks/migration/fill_dao_transactions_count_to_address.rake +++ b/lib/tasks/migration/fill_dao_transactions_count_to_address.rake @@ -11,7 +11,7 @@ namespace :migration do progress_bar.increment dao_transactions_count = address.ckb_dao_transactions.count - { id: udt.id, dao_transactions_count: dao_transactions_count, created_at: udt.created_at, updated_at: Time.current } + { id: address.id, dao_transactions_count: dao_transactions_count, created_at: address.created_at, updated_at: Time.current } end Address.upsert_all(values) if values.present? From ad328de4665b3cff575144d1a8ee2914bad189a7 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 15:44:01 +0800 Subject: [PATCH 71/76] chore: adjust tests --- .../v1/dao_contract_transactions_controller_test.rb | 5 +++++ .../api/v1/external/stats_controller_test.rb | 2 ++ test/models/address_test.rb | 5 +++++ test/models/block_test.rb | 5 +++++ test/models/ckb_sync/node_data_processor_test.rb | 12 ++++++------ test/models/ckb_transaction_test.rb | 5 +++++ test/models/lock_script_test.rb | 5 +++++ test/models/type_script_test.rb | 5 +++++ test/models/uncle_block_test.rb | 5 +++++ test/utils/ckb_utils_test.rb | 2 ++ 10 files changed, 45 insertions(+), 6 deletions(-) diff --git a/test/controllers/api/v1/dao_contract_transactions_controller_test.rb b/test/controllers/api/v1/dao_contract_transactions_controller_test.rb index ee37353c7..a3751a5da 100644 --- a/test/controllers/api/v1/dao_contract_transactions_controller_test.rb +++ b/test/controllers/api/v1/dao_contract_transactions_controller_test.rb @@ -2,6 +2,11 @@ module Api module V1 class DaoContractTransactionsControllerTest < ActionDispatch::IntegrationTest + setup do + create(:table_record_count, :block_counter) + create(:table_record_count, :ckb_transactions_counter) + end + test "should set right content type when call index" do ckb_transaction = create(:ckb_transaction) diff --git a/test/controllers/api/v1/external/stats_controller_test.rb b/test/controllers/api/v1/external/stats_controller_test.rb index b07ae6db3..3ec53faea 100644 --- a/test/controllers/api/v1/external/stats_controller_test.rb +++ b/test/controllers/api/v1/external/stats_controller_test.rb @@ -5,6 +5,8 @@ module V1 module External class StatsControllerTest < ActionDispatch::IntegrationTest test "should return tip block number when call show action and id is equal to tip_block_number" do + create(:table_record_count, :block_counter) + create(:table_record_count, :ckb_transactions_counter) prepare_node_data get api_v1_external_stat_url("tip_block_number") diff --git a/test/models/address_test.rb b/test/models/address_test.rb index e39980905..0833edf16 100644 --- a/test/models/address_test.rb +++ b/test/models/address_test.rb @@ -1,6 +1,11 @@ require "test_helper" class AddressTest < ActiveSupport::TestCase + setup do + create(:table_record_count, :block_counter) + create(:table_record_count, :ckb_transactions_counter) + end + context "associations" do should have_many(:account_books) should have_many(:ckb_transactions). diff --git a/test/models/block_test.rb b/test/models/block_test.rb index 1daf6c78a..54b6915c3 100644 --- a/test/models/block_test.rb +++ b/test/models/block_test.rb @@ -1,6 +1,11 @@ require "test_helper" class BlockTest < ActiveSupport::TestCase + setup do + create(:table_record_count, :block_counter) + create(:table_record_count, :ckb_transactions_counter) + end + context "associations" do should have_many(:ckb_transactions) should have_many(:uncle_blocks) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index 54422c405..b05e35f4d 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -11,6 +11,8 @@ class NodeDataProcessorTest < ActiveSupport::TestCase start_number: "0x0" ) ) + create(:table_record_count, :block_counter) + create(:table_record_count, :ckb_transactions_counter) end test "#process_block should create one block" do @@ -24,7 +26,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase end test "should update table_record_counts block count after block has been processed" do - block_counter = create(:table_record_count, :block_counter) + block_counter = TableRecordCount.find_by(table_name: "blocks") VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}") do node_block = CkbSync::Api.instance.get_block_by_number(DEFAULT_NODE_BLOCK_NUMBER) create(:block, :with_block_hash, number: node_block.header.number - 1) @@ -35,8 +37,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase end test "should update table_record_counts ckb transactions count after block has been processed" do - create(:table_record_count, :block_counter) - ckb_transaction_counter = create(:table_record_count, :ckb_transactions_counter) + ckb_transaction_counter = TableRecordCount.find_by(table_name: "ckb_transactions") VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}") do node_block = CkbSync::Api.instance.get_block_by_number(DEFAULT_NODE_BLOCK_NUMBER) create(:block, :with_block_hash, number: node_block.header.number - 1) @@ -1347,7 +1348,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase end test "should recalculate table_record_counts block count when block is invalid" do - block_counter = create(:table_record_count, :block_counter) + block_counter = TableRecordCount.find_by(table_name: "blocks") prepare_node_data(12) local_block = Block.find_by(number: 12) local_block.update(block_hash: "0x419c632366c8eb9635acbb39ea085f7552ae62e1fdd480893375334a0f37d1bx") @@ -1360,8 +1361,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase end test "should recalculate table_record_counts ckb_transactions count when block is invalid" do - create(:table_record_count, :block_counter) - ckb_transactions_counter = create(:table_record_count, :ckb_transactions_counter) + ckb_transactions_counter = TableRecordCount.find_by(table_name: "ckb_transactions") prepare_node_data(12) local_block = Block.find_by(number: 12) diff --git a/test/models/ckb_transaction_test.rb b/test/models/ckb_transaction_test.rb index 97d93b41b..34ed59199 100644 --- a/test/models/ckb_transaction_test.rb +++ b/test/models/ckb_transaction_test.rb @@ -1,6 +1,11 @@ require "test_helper" class CkbTransactionTest < ActiveSupport::TestCase + setup do + create(:table_record_count, :block_counter) + create(:table_record_count, :ckb_transactions_counter) + end + context "associations" do should belong_to(:block) should have_many(:account_books) diff --git a/test/models/lock_script_test.rb b/test/models/lock_script_test.rb index 8bc64bd0d..393bf4fd1 100644 --- a/test/models/lock_script_test.rb +++ b/test/models/lock_script_test.rb @@ -1,6 +1,11 @@ require "test_helper" class LockScriptTest < ActiveSupport::TestCase + setup do + create(:table_record_count, :block_counter) + create(:table_record_count, :ckb_transactions_counter) + end + context "associations" do should belong_to(:address) end diff --git a/test/models/type_script_test.rb b/test/models/type_script_test.rb index 7fea6c8dd..f294ece13 100644 --- a/test/models/type_script_test.rb +++ b/test/models/type_script_test.rb @@ -1,6 +1,11 @@ require "test_helper" class TypeScriptTest < ActiveSupport::TestCase + setup do + create(:table_record_count, :block_counter) + create(:table_record_count, :ckb_transactions_counter) + end + context "associations" do should belong_to(:cell_output) end diff --git a/test/models/uncle_block_test.rb b/test/models/uncle_block_test.rb index 1720a8701..f1237bbd9 100644 --- a/test/models/uncle_block_test.rb +++ b/test/models/uncle_block_test.rb @@ -1,6 +1,11 @@ require "test_helper" class UncleBlockTest < ActiveSupport::TestCase + setup do + create(:table_record_count, :block_counter) + create(:table_record_count, :ckb_transactions_counter) + end + context "associations" do should belong_to(:block) end diff --git a/test/utils/ckb_utils_test.rb b/test/utils/ckb_utils_test.rb index 865fbcaba..e7d2d8a76 100644 --- a/test/utils/ckb_utils_test.rb +++ b/test/utils/ckb_utils_test.rb @@ -10,6 +10,8 @@ class CkbUtilsTest < ActiveSupport::TestCase start_number: "0x0" ) ) + create(:table_record_count, :block_counter) + create(:table_record_count, :ckb_transactions_counter) end test ".generate_address should return mainnet address when mode is mainnet" do From 6ac80b489e6d2df498d1d427523d3e0002f2e697 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 7 Aug 2020 18:01:50 +0800 Subject: [PATCH 72/76] chore: update init table record count task --- lib/tasks/migration/init_table_record_count.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/migration/init_table_record_count.rake b/lib/tasks/migration/init_table_record_count.rake index 5d39c73f6..89b1d24a6 100644 --- a/lib/tasks/migration/init_table_record_count.rake +++ b/lib/tasks/migration/init_table_record_count.rake @@ -4,7 +4,7 @@ namespace :migration do ApplicationRecord.transaction do ActiveRecord::Base.connection.execute('LOCK blocks IN ACCESS EXCLUSIVE MODE') TableRecordCount.create(table_name: "blocks", count: Block.count) - TableRecordCount.create(table_name: "ckb_transactions", count: CkbTransaction.count) + TableRecordCount.create(table_name: "ckb_transactions", count: CkbTransaction.normal.count) end puts "done" From ac045cafff6b6ba555e728cccb9829a1110102c4 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Sun, 9 Aug 2020 08:56:17 +0800 Subject: [PATCH 73/76] chore: add .gitattributes file --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..b1f122afc --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +public/* linguist-documentation \ No newline at end of file From a6b95e2fab798d3606984c8cb4b34ff7bbfe9f8e Mon Sep 17 00:00:00 2001 From: shaojunda Date: Sun, 9 Aug 2020 08:56:17 +0800 Subject: [PATCH 74/76] chore: add .gitattributes file --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index b1f122afc..763342cd1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -public/* linguist-documentation \ No newline at end of file +>>>>>>> chore: add .gitattributes file From 9f0b4591e50d753311f88b8076c44fac112daee5 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Mon, 10 Aug 2020 11:03:37 +0800 Subject: [PATCH 75/76] chore(dep): upgrade concurrent-ruby and spring --- .gitattributes | 2 +- Gemfile.lock | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.gitattributes b/.gitattributes index 763342cd1..461ad91e2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ ->>>>>>> chore: add .gitattributes file +public/* linguist-documentation diff --git a/Gemfile.lock b/Gemfile.lock index e31a78eb9..32f21a3db 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -96,7 +96,7 @@ GEM simplecov url coderay (1.1.2) - concurrent-ruby (1.1.6) + concurrent-ruby (1.1.7) config (1.7.2) activesupport (>= 3.0) deep_merge (~> 1.2, >= 1.2.1) @@ -160,7 +160,7 @@ GEM activesupport (>= 4.2.0) hashdiff (1.0.0) hiredis (0.6.3) - i18n (1.8.3) + i18n (1.8.5) concurrent-ruby (~> 1.0) jaro_winkler (1.5.2) json (2.3.1) @@ -316,8 +316,7 @@ GEM json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.2) - spring (2.0.2) - activesupport (>= 4.2) + spring (2.1.0) spring-watcher-listen (2.0.1) listen (>= 2.7, < 4.0) spring (>= 1.2, < 3.0) @@ -343,7 +342,7 @@ GEM websocket-driver (0.7.2) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) - zeitwerk (2.3.0) + zeitwerk (2.4.0) PLATFORMS ruby From e8c2bc21ba417f8b4c6958118dd8c76b896d209f Mon Sep 17 00:00:00 2001 From: shaojunda Date: Mon, 10 Aug 2020 14:03:37 +0800 Subject: [PATCH 76/76] chore: Bump version to v0.11.1 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fbec1c83..73787a170 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# [v0.11.1](https://github.com/shaojunda/ckb-explorer/compare/v0.11.0...v0.11.1) (2020-08-12) + +### Performance Improvements + +* [#716](https://github.com/nervosnetwork/ckb-explorer/pull/716): use record counter to replace count(*) + # [v0.11.0](https://github.com/nervosnetwork/ckb-explorer/compare/v0.10.1...v0.11.0) (2020-07-30)