diff --git a/stream/build.gradle.kts b/stream/build.gradle.kts index 64f5f971c..20a1bdfab 100644 --- a/stream/build.gradle.kts +++ b/stream/build.gradle.kts @@ -33,7 +33,7 @@ tasks.withType().configureEach { tasks.cloneHederaProtobufs { // uncomment below to use a specific tag // tag = "v0.53.0" or a specific commit like "0047255" - tag = "1033f10" + tag = "eab8b58e30336512bcf387c803e6fc86b6ebe010" // uncomment below to use a specific branch // branch = "main" diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/Record2BlockCommand.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/Record2BlockCommand.java index b6a794669..988d75b01 100644 --- a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/Record2BlockCommand.java +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/Record2BlockCommand.java @@ -16,6 +16,7 @@ package com.hedera.block.tools.commands.record2blocks; +import static com.hedera.block.tools.commands.record2blocks.mirrornode.FetchBlockQuery.getPreviousHashForBlock; import static com.hedera.block.tools.commands.record2blocks.util.BlockWriter.writeBlock; import static com.hedera.block.tools.commands.record2blocks.util.RecordFileDates.blockTimeLongToInstant; @@ -23,15 +24,21 @@ import com.hedera.block.tools.commands.record2blocks.model.BlockInfo; import com.hedera.block.tools.commands.record2blocks.model.BlockTimes; import com.hedera.block.tools.commands.record2blocks.model.ChainFile; -import com.hedera.block.tools.commands.record2blocks.model.SignatureFile; +import com.hedera.block.tools.commands.record2blocks.model.ParsedSignatureFile; +import com.hedera.block.tools.commands.record2blocks.model.RecordFileVersionInfo; import com.hedera.block.tools.commands.record2blocks.util.BlockWriter.BlockPath; import com.hedera.hapi.block.stream.Block; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.hapi.block.stream.BlockItem.ItemOneOfType; import com.hedera.hapi.block.stream.RecordFileItem; +import com.hedera.hapi.block.stream.RecordFileSignature; +import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.hapi.node.base.BlockHashAlgorithm; +import com.hedera.hapi.node.base.SemanticVersion; import com.hedera.hapi.node.base.Timestamp; +import com.hedera.hapi.streams.SidecarFile; import com.hedera.pbj.runtime.OneOf; +import com.hedera.pbj.runtime.ParseException; import com.hedera.pbj.runtime.io.buffer.Bytes; import com.hedera.pbj.runtime.io.stream.WritableStreamingData; import java.io.IOException; @@ -42,7 +49,7 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; +import java.util.HexFormat; import java.util.List; import picocli.CommandLine.Command; import picocli.CommandLine.Help.Ansi; @@ -50,6 +57,15 @@ /** * Command line command that converts a record stream to blocks + *

+ * Example block ranges for testing: + *

+ * Record files start at V2 at block 0 then change to V5 at block 12370838 and V6 at block 38210031 + *

*/ @SuppressWarnings("FieldCanBeLocal") @Command(name = "record2block", description = "Converts a record stream files into blocks") @@ -137,6 +153,14 @@ public void run() { } // map the block_times.bin file final BlockTimes blockTimes = new BlockTimes(blockTimesFile); + // get previous block hash + Bytes previousBlockHash; + if (startBlock == 0) { + previousBlockHash = Bytes.wrap(new byte[48]); // empty hash for first block + } else { + // get previous block hash from mirror node + previousBlockHash = getPreviousHashForBlock(startBlock); + } // iterate over the blocks Instant currentHour = null; List currentHoursFiles = null; @@ -166,35 +190,48 @@ public void run() { // print block info System.out.println(" " + blockInfo); // now we need to download the most common record file - // we will use the GCP bucket to download the file byte[] recordFileBytes = blockInfo.mostCommonRecordFile().chainFile().download(mainNetBucket); + // parse version information out of record file + final RecordFileVersionInfo recordFileVersionInfo = RecordFileVersionInfo.parse(recordFileBytes); + // download and parse all signature files - SignatureFile[] signatureFileBytes = blockInfo.signatureFiles().stream() + ParsedSignatureFile[] signatureFileBytes = blockInfo.signatureFiles().stream() .parallel() - .map(chainFile -> chainFile.download(mainNetBucket)) - .map(SignatureFile::parse) - .toArray(SignatureFile[]::new); - for (SignatureFile signatureFile : signatureFileBytes) { - // System.out.println(" signatureFile = " + signatureFile); - } + .map(cf -> ParsedSignatureFile.downloadAndParse(cf, mainNetBucket)) + .toArray(ParsedSignatureFile[]::new); + // convert signature files to list of RecordFileSignatures + final List recordFileSignatures = Arrays.stream(signatureFileBytes) + .map(sigFile -> new RecordFileSignature(Bytes.wrap(sigFile.signature()), sigFile.nodeId())) + .toList(); // download most common sidecar file - List sideCars = new ArrayList<>(); - // byte[] sidecarFileBytes = blockInfo.getMostCommonSidecarFileBytes(mainNetBucket); + List sideCars = blockInfo.sidecarFiles().values().stream() + .map(sidecarFile -> { + byte[] sidecarFileBytes = sidecarFile.mostCommonSidecarFile().chainFile().download(mainNetBucket); + try { + return SidecarFile.PROTOBUF.parse(Bytes.wrap(sidecarFileBytes)); + } catch (ParseException e) { + throw new RuntimeException(e); + } + }).toList(); // build new Block File + final BlockHeader blockHeader = new BlockHeader( + recordFileVersionInfo.hapiProtoVersion(), + recordFileVersionInfo.hapiProtoVersion(), + blockNumber, + previousBlockHash, + new Timestamp(blockTimeInstant.getEpochSecond(), blockTimeInstant.getNano()), + BlockHashAlgorithm.SHA2_384); final RecordFileItem recordFileItem = new RecordFileItem( - blockInfo.blockNum(), new Timestamp(blockTimeInstant.getEpochSecond(), blockTimeInstant.getNano()), Bytes.wrap(recordFileBytes), - sideCars, - BlockHashAlgorithm.SHA2_384, - Arrays.stream(signatureFileBytes) - .map(sigFile -> Bytes.wrap(sigFile.signature())) - .toList()); - final Block block = new Block(Collections.singletonList( + sideCars,recordFileSignatures + ); + final Block block = new Block(List.of( + new BlockItem(new OneOf<>(ItemOneOfType.BLOCK_HEADER, blockHeader)), new BlockItem(new OneOf<>(ItemOneOfType.RECORD_FILE, recordFileItem)))); // write block to disk final BlockPath blockPath = writeBlock(blocksDir, block); @@ -211,6 +248,8 @@ public void run() { } System.out.println(Ansi.AUTO.string("@|bold,yellow Wrote block json to|@ " + blockJsonPath)); } + // update previous block hash + previousBlockHash = recordFileVersionInfo.blockHash(); } } catch (IOException e) { diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/gcp/MainNetBucket.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/gcp/MainNetBucket.java index 79e3b1ae3..0c995bf35 100644 --- a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/gcp/MainNetBucket.java +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/gcp/MainNetBucket.java @@ -24,10 +24,12 @@ import com.google.cloud.storage.StorageOptions; import com.hedera.block.tools.commands.record2blocks.model.ChainFile; import com.hedera.block.tools.commands.record2blocks.util.RecordFileDates; +import java.io.ByteArrayInputStream; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -95,19 +97,53 @@ public MainNetBucket(boolean cacheEnabled, Path cacheDir, int minNodeAccountId, * @return the bytes of the file */ public byte[] download(String path) { + try { + final Path cachedFilePath = cacheDir.resolve(path); + byte[] rawBytes; + if (cacheEnabled && Files.exists(cachedFilePath)) { + rawBytes = Files.readAllBytes(cachedFilePath); + } else { + rawBytes = STREAMS_BUCKET.get(path).getContent(); + if (cacheEnabled) { + Files.createDirectories(cachedFilePath.getParent()); + Path tempCachedFilePath = Files.createTempFile(cacheDir, null, ".tmp"); + Files.write(tempCachedFilePath, rawBytes); + Files.move(tempCachedFilePath, cachedFilePath); + } + } + // if file is gzipped, unzip it + if (path.endsWith(".gz")) { + try (GZIPInputStream gzipInputStream = new GZIPInputStream(new ByteArrayInputStream(rawBytes))) { + return gzipInputStream.readAllBytes(); + } + } else { + return rawBytes; + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Download a file from GCP as a stream, caching if CACHE_ENABLED is true. This is designed to be thread safe. + * + * @param path the path to the file in the bucket + * @return the stream of the file + */ + public java.io.InputStream downloadStreaming(String path) { try { Path cachedFilePath = cacheDir.resolve(path); if (cacheEnabled && Files.exists(cachedFilePath)) { - return Files.readAllBytes(cachedFilePath); + return Files.newInputStream(cachedFilePath, StandardOpenOption.READ); } else { - byte[] bytes = STREAMS_BUCKET.get(path).getContent(); + final byte[] bytes = STREAMS_BUCKET.get(path).getContent(); if (cacheEnabled) { Files.createDirectories(cachedFilePath.getParent()); Path tempCachedFilePath = Files.createTempFile(cacheDir, null, ".tmp"); Files.write(tempCachedFilePath, bytes); Files.move(tempCachedFilePath, cachedFilePath); } - return bytes; + return new ByteArrayInputStream(bytes); } } catch (Exception e) { throw new RuntimeException(e); diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/mirrornode/FetchBlockQuery.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/mirrornode/FetchBlockQuery.java index b3b192f14..4b20453a3 100644 --- a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/mirrornode/FetchBlockQuery.java +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/mirrornode/FetchBlockQuery.java @@ -18,10 +18,12 @@ import com.google.gson.Gson; import com.google.gson.JsonObject; +import com.hedera.pbj.runtime.io.buffer.Bytes; import java.io.InputStreamReader; import java.io.Reader; import java.net.URI; import java.net.URL; +import java.util.HexFormat; /** * Query Mirror Node and fetch block information @@ -40,6 +42,19 @@ public static String getRecordFileNameForBlock(long blockNumber) { return json.get("name").getAsString(); } + /** + * Get the previous hash for a block number from the mirror node. + * + * @param blockNumber the block number + * @return the record file name + */ + public static Bytes getPreviousHashForBlock(long blockNumber) { + final String url = "https://mainnet-public.mirrornode.hedera.com/api/v1/blocks/" + blockNumber; + final JsonObject json = readUrl(url); + final String hashStr = json.get("previous_hash").getAsString(); + return Bytes.wrap(HexFormat.of().parseHex(hashStr.substring(2))); // remove 0x prefix and parse + } + /** * Read a URL and return the JSON object. * diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/ChainFile.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/ChainFile.java index b664962c1..3849a04c3 100644 --- a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/ChainFile.java +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/ChainFile.java @@ -20,6 +20,7 @@ import static com.hedera.block.tools.commands.record2blocks.util.RecordFileDates.extractRecordFileTime; import com.hedera.block.tools.commands.record2blocks.gcp.MainNetBucket; +import java.io.InputStream; import java.io.Serializable; import java.util.regex.Pattern; @@ -87,6 +88,16 @@ public byte[] download(MainNetBucket mainNetBucket) { return mainNetBucket.download(path); } + /** + * Downloads the file from the bucket as a stream. + * + * @param mainNetBucket the main net bucket that contains the file + * @return the file as a stream + */ + public InputStream downloadStreaming(MainNetBucket mainNetBucket) { + return mainNetBucket.downloadStreaming(path); + } + /** * Enum for the kind of file. */ diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/ParsedSignatureFile.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/ParsedSignatureFile.java new file mode 100644 index 000000000..f289a71fc --- /dev/null +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/ParsedSignatureFile.java @@ -0,0 +1,317 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.tools.commands.record2blocks.model; + +import com.hedera.block.tools.commands.record2blocks.gcp.MainNetBucket; +import com.hedera.hapi.streams.SignatureFile; +import com.hedera.pbj.runtime.ParseException; +import com.hedera.pbj.runtime.io.stream.ReadableStreamingData; +import java.io.DataInputStream; +import java.io.IOException; +import java.util.HexFormat; + +/** + * SignatureFile represents a Hedera record file signature file. There have been 3 versions of the signature files used + * since OA which are V3, V5 and V6. The below tables describe the content that can be parsed from a record signature + * file for each version. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Signature File Format V3
NameType (Bytes)Description
File Hash MarkerbyteValue: 4
File Hashbyte[48]SHA384 hash of corresponding *.rcd file
Signature MarkerbyteValue: 3
Length of Signatureint (4)Byte size of the following signature bytes
Signaturebyte[]Signature bytes
+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Signature File Format V5
NameType (Bytes)Description
Signature File Format VersionbyteValue: 5
Object Stream Signature Versionint (4) Value: 1
This defines the format of the remainder of the signature file. This version number is used when parsing a + * signature file with methods defined in swirlds-common package
Entire Hash of the corresponding stream filebyte[48]SHA384 Hash of the entire corresponding stream file
Signature on hash bytes of Entire Hashbyte[]A signature object generated by signing the hash bytes of Entire Hash. See ` Signature ` table below for + * details
Metadata Hash of the corresponding stream filebyte[48]Metadata Hash of the corresponding stream file
Signature on hash bytes of Metadata Hashbyte[]A signature object generated by signing the hash bytes of Metadata Hash
+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Signature File Format V5 - Signature Object
NameType (Bytes)Description
Class IDlong (8)Value: 0x13dc4b399b245c69
Class Versionint (4)Value: 1
SignatureTypeint (4)Value: 1 - Denotes SHA384withRSA
Length of Signatureint (4)Size of the signature in bytes
CheckSumint (4)101 - length of signature bytes
Signature bytesbyte[]Serialized Signature bytes
+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Signature File Format V6
NameType (Bytes)Description
Signature File Format VersionbyteValue: 6
Protobuf Encodedbyte[]Rest of signature file is a protobuf serialized message of type com.hedera.hapi.streams.SignatureFile
+ * + * @param nodeId Node ID of the node that signed the file + * @param fileHash SHA384 hash of corresponding *.rcd file + * @param signature Signature bytes or RSA signature of the file hash, signed by the node's private key + */ +public record ParsedSignatureFile(int nodeId, byte[] fileHash, byte[] signature) { + /** + * The marker for the file hash in a V3 signature file. This is the first byte so also acts like a version number. + */ + public static final byte V2_FILE_HASH_MARKER = 4; + public static final byte FILE_VERSION_5 = 5; + public static final byte FILE_VERSION_6 = 6; + public static final byte V3_SIGNATURE_MARKER = 3; + + /** + * toString for debugging, prints the file hash and signature in hex format. + * + * @return the string representation of the SignatureFile + */ + @Override + public String toString() { + final HexFormat hexFormat = HexFormat.of(); + return "SignatureFile[" + + "nodeId=" + nodeId + ", " + + "fileHash=" + + hexFormat.formatHex(fileHash) + ", signature=" + + hexFormat.formatHex(signature) + ']'; + } + + /** + * Download and parse a SignatureFile from a ChainFile. + * + * @param signatureChainFile the chain file for the signature file + * @param mainNetBucket the bucket to download from + * @return the parsed SignatureFile + */ + public static ParsedSignatureFile downloadAndParse(ChainFile signatureChainFile, MainNetBucket mainNetBucket) { + // first download + try(DataInputStream in = new DataInputStream(signatureChainFile.downloadStreaming(mainNetBucket))) { + // extract node ID from file path. This depends on the fixed relationship between node account ids and node ids. + final int nodeId = signatureChainFile.nodeAccountId() - 3; + // now parse + final int firstByte = in.read(); + // the first byte is either the file hash marker or a version number in V6 record stream + switch(firstByte) { + case V2_FILE_HASH_MARKER: + final byte[] fileHash = new byte[48]; + in.readFully(fileHash); + if (in.read() != V3_SIGNATURE_MARKER) { + throw new IllegalArgumentException("Invalid signature marker"); + } + final int signatureLength = in.readInt(); + final byte[] signature = new byte[signatureLength]; + in.readFully(signature); + return new ParsedSignatureFile(nodeId, fileHash, signature); + case FILE_VERSION_5: + // check the object stream signature version should be 1 + if (in.readInt() != 1) { + throw new IllegalArgumentException("Invalid object stream signature version"); + } + // read hash object - hash bytes + final byte[] entireFileHash = readHashObject(in); + // read signature object - class id + if (in.readLong() != 0x13dc4b399b245c69L) { + throw new IllegalArgumentException("Invalid signature object class ID"); + } + // read signature object - class version + if (in.readInt() != 1) { + throw new IllegalArgumentException("Invalid signature object class version"); + } + // read signature object - signature type - An RSA signature as specified by the FIPS 186-4 + if (in.readInt() != 1) { + throw new IllegalArgumentException("Invalid signature type"); + } + // read signature object - length of signature + final int signatureLengthV5 = in.readInt(); + // read and check signature object - checksum + if (in.readInt() != 101 - signatureLengthV5) { + throw new IllegalArgumentException("Invalid checksum"); + } + // read signature object - signature bytes + final byte[] signatureV5 = new byte[signatureLengthV5]; + in.readFully(signatureV5); + // we only care about the file metadata hash and the signature so can stop parsing here + return new ParsedSignatureFile(nodeId, entireFileHash, signatureV5); + case FILE_VERSION_6: + // everything from here on is protobuf encoded + try { + SignatureFile signatureFile = SignatureFile.PROTOBUF.parse(new ReadableStreamingData(in)); + return new ParsedSignatureFile( + nodeId, + signatureFile.fileSignature().hashObject().hash().toByteArray(), + signatureFile.fileSignature().signature().toByteArray()); + } catch (ParseException e) { + throw new RuntimeException("Error protobuf parsing V6 signature file", e); + } + default: + throw new IllegalArgumentException("Invalid first byte [" + firstByte + "] expected " + + V2_FILE_HASH_MARKER + " or " + FILE_VERSION_6); + } + } catch (IOException e) { + throw new RuntimeException("Error downloading or parsing signature file", e); + } + } + + /** The size of a hash object in bytes */ + public static final int HASH_OBJECT_SIZE_BYTES = Long.BYTES + Integer.BYTES + Integer.BYTES + Integer.BYTES + 48; + + /** + * Read a hash object from a data input stream in SelfSerializable SHA384 format. + * + * @param in the data input stream + * @return the hash bytes + * @throws IOException if an error occurs reading the hash object + */ + public static byte[] readHashObject(DataInputStream in) throws IOException { + // read hash class id + if (in.readLong() != 0xf422da83a251741eL) { + throw new IllegalArgumentException("Invalid hash class ID"); + } + // read hash class version + if(in.readInt() != 1) { + throw new IllegalArgumentException("Invalid hash class version"); + } + // read hash object, starting with digest type SHA384 + if (in.readInt() != 0x58ff811b) { + throw new IllegalArgumentException("Invalid digest type not SHA384"); + } + // read hash object - length of hash + if (in.readInt() != 48) { + throw new IllegalArgumentException("Invalid hash length"); + } + // read hash object - hash bytes + final byte[] entireFileHash = new byte[48]; + in.readFully(entireFileHash); + return entireFileHash; + } +} diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/RecordFileVersionInfo.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/RecordFileVersionInfo.java new file mode 100644 index 000000000..2e332935a --- /dev/null +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/RecordFileVersionInfo.java @@ -0,0 +1,81 @@ +package com.hedera.block.tools.commands.record2blocks.model; + +import static com.hedera.block.tools.commands.record2blocks.model.ParsedSignatureFile.HASH_OBJECT_SIZE_BYTES; +import static com.hedera.block.tools.commands.record2blocks.model.ParsedSignatureFile.readHashObject; + +import com.hedera.hapi.node.base.SemanticVersion; +import com.hedera.hapi.streams.RecordStreamFile; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import com.hedera.pbj.runtime.io.stream.ReadableStreamingData; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.security.MessageDigest; + +/** + * Represents the version & block hash information of a record file. + * + * @param hapiProtoVersion the HAPI protocol version + * @param blockHash the block hash + */ +public record RecordFileVersionInfo ( + SemanticVersion hapiProtoVersion, + Bytes blockHash +) { + /* The length of the header in a v2 record file */ + private static final int V2_HEADER_LENGTH = Integer.BYTES + Integer.BYTES + 1 + 48; + + /** + * Parses the record file to extract the HAPI protocol version and the block hash. + * + * @param recordFile the record file bytes to parse + * @return the record file version info + */ + public static RecordFileVersionInfo parse(byte[] recordFile) { + try(DataInputStream in = new DataInputStream(new ByteArrayInputStream(recordFile))) { + final int recordFormatVersion = in.readInt(); + return switch (recordFormatVersion) { + case 2 -> { + final int hapiMajorVersion = in.readInt(); + final SemanticVersion hapiProtoVersion = new SemanticVersion( + hapiMajorVersion, 0, 0, null, null); + // The hash for v2 files is the hash(header, hash(content)) this is different to other versions + MessageDigest digest = MessageDigest.getInstance("SHA-384"); + digest.update(recordFile, V2_HEADER_LENGTH, recordFile.length - V2_HEADER_LENGTH); + final byte[] contentHash = digest.digest(); + digest.update(recordFile, 0, V2_HEADER_LENGTH); + digest.update(contentHash); + yield new RecordFileVersionInfo( + hapiProtoVersion, + Bytes.wrap(digest.digest()) + ); + } + case 5 -> { + final int hapiMajorVersion = in.readInt(); + final int hapiMinorVersion = in.readInt(); + final int hapiPatchVersion = in.readInt(); + final SemanticVersion hapiProtoVersion = new SemanticVersion( + hapiMajorVersion, hapiMinorVersion, hapiPatchVersion, null, null); + // skip to last hash object + in.skipBytes(in.available() - HASH_OBJECT_SIZE_BYTES); + final byte[] endHashObject = readHashObject(in); + yield new RecordFileVersionInfo( + hapiProtoVersion, + Bytes.wrap(endHashObject) + ); + } + case 6 -> { + final RecordStreamFile recordStreamFile = RecordStreamFile.PROTOBUF.parse(new ReadableStreamingData( + in)); + yield new RecordFileVersionInfo( + recordStreamFile.hapiProtoVersion(), + recordStreamFile.endObjectRunningHash().hash() + ); + } + default -> + throw new UnsupportedOperationException("Unsupported record format version: " + recordFormatVersion); + }; + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/SignatureFile.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/SignatureFile.java deleted file mode 100644 index fe142c4ac..000000000 --- a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/SignatureFile.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (C) 2024 Hedera Hashgraph, LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.hedera.block.tools.commands.record2blocks.model; - -import java.util.HexFormat; - -/** - * SignatureFile represents a Hedera record file signature file. - * The below table describes the content that can be parsed from a record signature file. - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
Signature File Format
NameType (Bytes)Description
File Hash MarkerbyteValue: 4
File Hashbyte[48]SHA384 hash of corresponding *.rcd file
Signature MarkerbyteValue: 3
Length of Signatureint (4)Byte size of the following signature bytes
Signaturebyte[]Signature bytes
- * - * @param fileHash SHA384 hash of corresponding *.rcd file - * @param signature Signature bytes or RSA signature of the file hash, signed by the node's private key - */ -public record SignatureFile(byte[] fileHash, byte[] signature) { - public static final byte FILE_HASH_MARKER = 4; - public static final byte SIGNATURE_MARKER = 3; - - /** - * toString for debugging, prints the file hash and signature in hex format. - * - * @return the string representation of the SignatureFile - */ - @Override - public String toString() { - final HexFormat hexFormat = HexFormat.of(); - return "SignatureFile[" + "fileHash=" - + hexFormat.formatHex(fileHash) + ", signature=" - + hexFormat.formatHex(signature) + ']'; - } - - /** - * Parse a SignatureFile from a byte array. - * - * @param bytes the byte array to parse - * @return the parsed SignatureFile - */ - public static SignatureFile parse(byte[] bytes) { - int index = 0; - if (bytes[index++] != FILE_HASH_MARKER) { - throw new IllegalArgumentException("Invalid file hash marker"); - } - final byte[] fileHash = new byte[48]; - System.arraycopy(bytes, index, fileHash, 0, fileHash.length); - index += fileHash.length; - if (bytes[index++] != SIGNATURE_MARKER) { - throw new IllegalArgumentException("Invalid signature marker"); - } - final int signatureLength = (bytes[index++] & 0xFF) << 24 - | (bytes[index++] & 0xFF) << 16 - | (bytes[index++] & 0xFF) << 8 - | (bytes[index++] & 0xFF); - final byte[] signature = new byte[signatureLength]; - System.arraycopy(bytes, index, signature, 0, signature.length); - return new SignatureFile(fileHash, signature); - } -} diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/util/BlockWriter.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/util/BlockWriter.java index 993d081f8..ee271c64e 100644 --- a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/util/BlockWriter.java +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/util/BlockWriter.java @@ -67,13 +67,7 @@ public record BlockPath(Path dirPath, String zipFileName, String blockNumStr, St public static BlockPath writeBlock(final Path baseDirectory, final Block block) throws IOException { // get block number from block header final var firstBlockItem = block.items().getFirst(); - final long blockNumber = - switch (firstBlockItem.item().kind()) { - case BLOCK_HEADER -> firstBlockItem.blockHeader().number(); - case RECORD_FILE -> firstBlockItem.recordFile().number(); - default -> throw new IllegalArgumentException( - "Block first item is not a block header or record file"); - }; + final long blockNumber = firstBlockItem.blockHeader().number(); // // convert block number to string // final String blockNumberStr = BLOCK_NUMBER_FORMAT.format(blockNumber); // // split string into digits for zip and for directories