Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

synchronous by chunk instead of header #10

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

synchronous by chunk instead of header #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b205727
complete bcd chunk sync
Apr 20, 2018
686b051
fix checkpoints error
Apr 20, 2018
13c227c
fix backward sync
Apr 20, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 130 additions & 31 deletions lib/blockchain.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,8 @@ def verify_header(self, header, prev_hash, target):
raise BaseException("prev hash mismatch: %s vs %s" % (prev_hash, header.get('prev_block_hash')))
if bitcoin.NetworkConstants.TESTNET:
return
if header.get('block_height') == NetworkConstants.BITCOIN_DIAMOND_FORK_BLOCK_HEIGHT and hash_header(header) != NetworkConstants.BITCOIN_DIAMOND_FORK_BLOCK_HASH:
if (header.get('block_height') == NetworkConstants.BITCOIN_DIAMOND_FORK_BLOCK_HEIGHT and
hash_header(header) != NetworkConstants.BITCOIN_DIAMOND_FORK_BLOCK_HASH):
raise BaseException("block at height %i is not bitcoin diamond chain fork block. hash %s" % (header.get('block_height'), hash_header(header)))
bits = self.target_to_bits(target)
if bits != header.get('bits'):
Expand All @@ -170,11 +171,31 @@ def verify_header(self, header, prev_hash, target):
def verify_chunk(self, index, data):
num = len(data) // 80
prev_hash = self.get_hash(index * 2016 - 1)
# target = self.get_target(index-1)
target = self.get_target(index * 2016)
for i in range(num):
raw_header = data[i*80:(i+1) * 80]
header = deserialize_header(raw_header, index*2016 + i)
height = index * 2016 + i
if height >= NetworkConstants.BITCOIN_DIAMOND_FORK_BLOCK_HEIGHT:
if height == NetworkConstants.BITCOIN_DIAMOND_FORK_BLOCK_HEIGHT:
target = MAX_TARGET
elif height < NetworkConstants.BITCOIN_DIAMOND_FORK_BLOCK_HEIGHT + 72:
target = MAX_TARGET // 256
else:
pass

raw_header = data[i * 80:(i+1) * 80]
header = deserialize_header(raw_header, height)
self.verify_header(header, prev_hash, target)
prev_hash = hash_header(header)

def verify_chip(self, chip_index, data):
num = len(data) // 80
prev_header = self.read_header(chip_index * 72 - 1 + NetworkConstants.BITCOIN_DIAMOND_FORK_BLOCK_HEIGHT)
prev_hash = hash_header(prev_header)
target = self.get_target(chip_index * 72 + NetworkConstants.BITCOIN_DIAMOND_FORK_BLOCK_HEIGHT)
for i in range(num):
height = chip_index * 72 + i + NetworkConstants.BITCOIN_DIAMOND_FORK_BLOCK_HEIGHT
raw_header = data[i * 80:(i+1) * 80]
header = deserialize_header(raw_header, height)
self.verify_header(header, prev_hash, target)
prev_hash = hash_header(header)

Expand All @@ -184,6 +205,7 @@ def path(self):
return os.path.join(d, filename)

def save_chunk(self, index, chunk):
self.print_error("save_chunk at:", index)
filename = self.path()
d = (index * 2016 - self.checkpoint) * 80
if d < 0:
Expand All @@ -192,6 +214,16 @@ def save_chunk(self, index, chunk):
self.write(chunk, d)
self.swap_with_parent()

def save_chip(self, chip_index, chip):
self.print_error("save_chip at:", chip_index)
file_name = self.path()
d = (chip_index * 72 - self.checkpoint + NetworkConstants.BITCOIN_DIAMOND_FORK_BLOCK_HEIGHT) * 80
if d < 0:
chip = chip[-d:]
d = 0
self.write(chip, d)
self.swap_with_parent()

def swap_with_parent(self):
if self.parent_id is None:
return
Expand Down Expand Up @@ -230,7 +262,7 @@ def write(self, data, offset):
filename = self.path()
with self.lock:
with open(filename, 'rb+') as f:
if offset != self._size*80:
if offset != self._size * 80:
f.seek(offset)
f.truncate()
f.seek(offset)
Expand All @@ -241,11 +273,20 @@ def write(self, data, offset):

def save_header(self, header):
delta = header.get('block_height') - self.checkpoint
self.print_error("save_header at:", delta)
data = bfh(serialize_header(header))
assert delta == self.size()
assert len(data) == 80
self.write(data, delta*80)
self.swap_with_parent()

def save_chip_header(self, header):
delta = header.get('block_height') - self.checkpoint
self.print_error("save_chip_header at:", delta)
data = bfh(serialize_header(header))
assert len(data) == 80
self.write(data, delta * 80)
self.swap_with_parent()

def read_header(self, height):
assert self.parent_id != self.checkpoint
Expand All @@ -265,32 +306,46 @@ def read_header(self, height):
return None
return deserialize_header(h, height)

def get_hash(self, height):
def get_hash(self, height, ignore_checkpoints = False):
if height == -1:
return '0000000000000000000000000000000000000000000000000000000000000000'
elif height == 0:
return bitcoin.NetworkConstants.GENESIS
elif height < len(self.checkpoints) * 2016:
assert (height+1) % 2016 == 0
index = height // 2016
h, t = self.checkpoints[index]
return h
elif (height < len(self.checkpoints) * 2016) and not ignore_checkpoints:
if (height+1) % 2016 == 0:
index = height // 2016
h, t, _ = self.checkpoints[index]
return h
else:
return hash_header(self.read_header(height))
pass

return hash_header(self.read_header(height))

def get_target(self, height):
def get_target(self, height, ignore_checkpoints = False):
# compute target from chunk x, used in chunk x+1
if bitcoin.NetworkConstants.TESTNET:
return 0, 0

index = height // 2016 - 1
if height < NetworkConstants.BITCOIN_DIAMOND_FORK_BLOCK_HEIGHT:
index = height // 2016 - 1
if index == -1:
return MAX_TARGET
if index < len(self.checkpoints) and not ignore_checkpoints:
h, t, _ = self.checkpoints[index]
return t

else:
return self.get_bcd_target(height)
if index == -1:
return MAX_TARGET
if index < len(self.checkpoints):
h, t = self.checkpoints[index]
return t
if index < len(self.checkpoints) and not ignore_checkpoints:
h, t, d_t = self.checkpoints[index]
if height < (index + 1) * 2016 + 3:
return t
elif height < (index + 1) * 2016 + 75:
return d_t
else:
pass

return self.get_bcd_target(height)

# new target
first = self.read_header(index * 2016)
last = self.read_header(index * 2016 + 2015)
Expand Down Expand Up @@ -324,7 +379,6 @@ def get_bcd_target(self, height):
new_target = min(MAX_TARGET, (target * nActualTimespan) // nTargetTimespan)
return new_target


def bits_to_target(self, bits):
bitsN = (bits >> 24) & 0xff
if not (bitsN >= 0x03 and bitsN <= 0x1d):
Expand All @@ -347,40 +401,85 @@ def target_to_bits(self, target):
def can_connect(self, header, check_height=True):
height = header['block_height']
if check_height and self.height() != height - 1:
#self.print_error("cannot connect at height", height)
self.print_error("cannot connect at height:", height)
return False
if height == 0:
return hash_header(header) == bitcoin.NetworkConstants.GENESIS
try:
prev_hash = self.get_hash(height - 1)
except:
except BaseException as e:
self.print_error("get prev hash of header failed:", str(e))
return False
if prev_hash != header.get('prev_block_hash'):
prev_block_hash = header.get('prev_block_hash')
if prev_hash != prev_block_hash:
self.print_error("local prev hash (%s) mismatch remote prev hash (%s)" % (prev_hash, prev_block_hash))
return False
target = self.get_target(height)
try:
self.verify_header(header, prev_hash, target)
except BaseException as e:
self.print_error("verify_header failed:", str(e))
return False
return True

def connect_chunk(self, idx, hexdata):
try:
data = bfh(hexdata)
self.verify_chunk(idx, data)
#self.print_error("validated chunk %d" % idx)
self.save_chunk(idx, data)
if idx <= NetworkConstants.BITCOIN_DIAMOND_FORK_BLOCK_HEIGHT // 2016:
self.verify_chunk(idx, data)
self.print_error("validated chunk %d" % idx)
self.save_chunk(idx, data)
else:
lead_headers = [data[i:i+80] for i in range(0, len(data[:3*80]), 80)]
tail_headers = [data[-69*80:][i:i+80] for i in range(0, len(data[-69*80:]), 80)]
left_data = data[3*80:-69*80]

for i, lead_header in enumerate(lead_headers):
height = idx * 2016 + i
lead_header = deserialize_header(lead_header, height)
if self.can_connect(lead_header, False):
self.print_error("validated header %d in chip" % height)
self.save_chip_header(lead_header)
else:
self.print_error("invalidate header %d in chip" % height)

chips = [left_data[i:i+72*80] for i in range(0, len(left_data), 72*80)]
for i, chip in enumerate(chips):
cix = (idx - 246) * 2016 // 72 + i + 1
self.verify_chip(cix, chip)
self.print_error("validated chip %d" % cix)
self.save_chip(cix, chip)

for i, tail_header in enumerate(tail_headers):
height = idx * 2016 + len(lead_headers) + len(left_data) // 80 + i
tail_header = deserialize_header(tail_header, height)
if self.can_connect(tail_header, False):
self.print_error("validated header %d in chip" % height)
self.save_chip_header(tail_header)
else:
self.print_error("invalidate header %d in chip" % height)

return True
except BaseException as e:
self.print_error('verify_chunk failed', str(e))
return False

def get_checkpoints(self):
# for each chunk, store the hash of the last block and the target after the chunk
file_name = self.path()
if os.path.exists(file_name):
with open(file_name, 'rb') as f:
if f.read(80) == b'\x00' * 80:
raise BaseException('header file is fake, but it must be real!')
else:
raise BaseException('block header file: %s is not exists' % file_name)

cp = []
n = self.height() // 2016
n = self.height() // 2016 - 1
for index in range(n):
h = self.get_hash((index+1) * 2016 -1)
target = self.get_target(index) # todo
cp.append((h, target))
height = (index + 1) * 2016 - 1
h = self.get_hash(height, True)
target = self.get_target(height + 1, True)
delta_target = self.get_target(height + 4, True)
cp.append((h, target, delta_target))
return cp
Loading