From d39bd03acdedb359e37ef4a4b9df6b27aca59fd5 Mon Sep 17 00:00:00 2001 From: jasperpotts Date: Fri, 20 Dec 2024 09:26:42 -0800 Subject: [PATCH] Build fix(fingers crossed) and review changes Signed-off-by: jasperpotts --- settings.gradle.kts | 5 +--- tool.sh | 2 +- tools/build.gradle.kts | 27 +++++++++++++------ .../record2blocks/Record2BlockCommand.java | 15 +++++------ .../record2blocks/gcp/AddNewerBlockTimes.java | 1 - .../mirrornode/FetchBlockQuery.java | 6 ++++- .../mirrornode/FetchMirrorNodeRecordsCsv.java | 24 +++++++++++++++++ .../record2blocks/model/BlockInfo.java | 7 +++++ .../model/ParsedSignatureFile.java | 2 ++ .../record2blocks/model/RecordFileInfo.java | 3 ++- .../record2blocks/util/BlockWriter.java | 3 +++ 11 files changed, 71 insertions(+), 24 deletions(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index d9b73fcbb..b83e4f619 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -49,7 +49,7 @@ dependencyResolutionManagement { val protobufVersion = "4.28.2" val helidonVersion = "4.1.1" val grpcIoVersion = "1.65.1" - var pbjVersion = "0.9.11" + val pbjVersion = "0.9.11" // Compile time dependencies version("io.helidon.webserver.http2", helidonVersion) @@ -68,9 +68,6 @@ dependencyResolutionManagement { version("com.google.auto.service", "1.1.1") version("org.hyperledger.besu.nativelib.secp256k1", "0.8.2") version("info.picocli", "4.7.6") - version("com.github.luben.zstd_jni", "1.5.6-8") - version("com.google.cloud.core", "2.48.0") - version("com.google.cloud.storage", "2.45.0") // gRPC dependencies for the stream subproject version("io.grpc", grpcIoVersion) diff --git a/tool.sh b/tool.sh index 0c0b87284..251ec9dc7 100755 --- a/tool.sh +++ b/tool.sh @@ -13,4 +13,4 @@ JAR=$(find build/libs -name 'tools-*-all.jar') # run the command line tool built jar file forwarding all arguments java -jar $JAR "$@" # change back to the original directory -popd > /dev/null \ No newline at end of file +popd > /dev/null diff --git a/tools/build.gradle.kts b/tools/build.gradle.kts index 1a6b4015a..f3a093f58 100644 --- a/tools/build.gradle.kts +++ b/tools/build.gradle.kts @@ -1,3 +1,5 @@ +import com.github.jengelman.gradle.plugins.shadow.internal.DefaultDependencyFilter +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar import org.gradlex.javamodule.dependencies.tasks.ModuleDirectivesScopeCheck /* @@ -29,7 +31,7 @@ application { } // Generate Manifest with Main-Class and Implementation-Title -tasks.withType { +tasks.withType().configureEach { manifest { attributes( "Main-Class" to application.mainClass.get(), @@ -39,13 +41,6 @@ tasks.withType { } } -// Switch compilation from modules back to classpath because 3rd party libraries are not modularized -tasks.compileJava { - // Do not use '--module-path' despite `module-info.java` - modularity.inferModulePath = false - // Do not compile module-info, we use it only to extract dependencies for now - exclude("module-info.java") -} // Allow non-module Jar extraJavaModuleInfo { @@ -73,4 +68,20 @@ dependencies { implementation("info.picocli:picocli:4.7.6") // depend on peer streams gradle module to get access to protobuf generated classes implementation(project(":stream")) +} + +tasks.withType().configureEach { + group = "shadow" + + // There is an issue in the shadow plugin that it automatically accesses the + // files in 'runtimeClasspath' while Gradle is building the task graph. + // See: https://github.com/GradleUp/shadow/issues/882 + dependencyFilter = NoResolveDependencyFilter() +} + +// Disable dependency resolution as it conflicts with shadow plugin +class NoResolveDependencyFilter : DefaultDependencyFilter(project) { + override fun resolve(configuration: FileCollection): FileCollection { + return configuration + } } \ No newline at end of file diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/Record2BlockCommand.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/Record2BlockCommand.java index e88bcd256..58767a836 100644 --- a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/Record2BlockCommand.java +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/Record2BlockCommand.java @@ -59,14 +59,13 @@ /** * Command line command that converts a record stream to blocks *

- * Example block ranges for testing: - *

    - *
  • -s 0 -e 10 - Record File v2
    • - *
  • -s 12877843 -e 12877853 - Record File v5
    • - *
  • -s 72756872 -e 72756882 - Record File v6 with sidecars
    • - *
- * Record files start at V2 at block 0 then change to V5 at block 12370838 and V6 at block 38210031 - *

+ * Example block ranges for testing: + *
    + *
  • -s 0 -e 10 - Record File v2
    • + *
  • -s 12877843 -e 12877853 - Record File v5
    • + *
  • -s 72756872 -e 72756882 - Record File v6 with sidecars
    • + *
+ * Record files start at V2 at block 0 then change to V5 at block 12370838 and V6 at block 38210031 */ @SuppressWarnings({"FieldCanBeLocal", "CallToPrintStackTrace"}) @Command(name = "record2block", description = "Converts a record stream files into blocks") diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/gcp/AddNewerBlockTimes.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/gcp/AddNewerBlockTimes.java index bb275e18d..a9523821e 100644 --- a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/gcp/AddNewerBlockTimes.java +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/gcp/AddNewerBlockTimes.java @@ -137,7 +137,6 @@ public void run() { if (!blockFileNameFromMirrorNode.equals(lastRecordFileName)) { throw new RuntimeException("Last block of day number mismatch"); } - // System.exit(0); // next day day = day.plus(1, ChronoUnit.DAYS); } diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/mirrornode/FetchBlockQuery.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/mirrornode/FetchBlockQuery.java index 5111a13b2..23ac92f9e 100644 --- a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/mirrornode/FetchBlockQuery.java +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/mirrornode/FetchBlockQuery.java @@ -72,7 +72,11 @@ private static JsonObject readUrl(String url) { } } - /** Test main method */ + /** + * Test main method + * + * @param args the command line arguments + */ public static void main(String[] args) { System.out.println("Fetching block query..."); int blockNumber = 69333000; diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/mirrornode/FetchMirrorNodeRecordsCsv.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/mirrornode/FetchMirrorNodeRecordsCsv.java index 33eaedc97..9a64b556b 100644 --- a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/mirrornode/FetchMirrorNodeRecordsCsv.java +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/mirrornode/FetchMirrorNodeRecordsCsv.java @@ -107,12 +107,25 @@ public static class ProgressOutputStream extends FilterOutputStream { private final String name; private long bytesWritten = 0; + /** + * Create new progress output stream. + * + * @param out the output stream to wrap + * @param size the size of the output stream + * @param name the name of the output stream + */ public ProgressOutputStream(OutputStream out, long size, String name) { super(out); this.size = size; this.name = name; } + /** + * Write a byte to the output stream. + * + * @param b the byte to write + * @throws IOException if an error occurs writing the byte + */ @Override public void write(int b) throws IOException { super.write(b); @@ -120,6 +133,14 @@ public void write(int b) throws IOException { printProgress(); } + /** + * Write a byte array to the output stream. + * + * @param b the byte array to write + * @param off the offset in the byte array to start writing + * @param len the number of bytes to write + * @throws IOException if an error occurs writing the byte array + */ @Override public void write(byte[] b, int off, int len) throws IOException { super.write(b, off, len); @@ -127,6 +148,9 @@ public void write(byte[] b, int off, int len) throws IOException { printProgress(); } + /** + * Print the progress of the output stream to the console. + */ private void printProgress() { if (bytesWritten % MB == 0) { System.out.printf( diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/BlockInfo.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/BlockInfo.java index f4a0332d6..b0a28a653 100644 --- a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/BlockInfo.java +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/BlockInfo.java @@ -26,6 +26,13 @@ /** * BlockInfo represents a Hedera block with its associated record files, sidecar files and signature files. + * + * @param blockNum the block number + * @param blockTime the block time + * @param recordFiles the record files associated with the block + * @param mostCommonRecordFile the record file with the most occurrences + * @param sidecarFiles the sidecar files associated with the block + * @param signatureFiles the signature files associated with the block */ @SuppressWarnings("unused") public record BlockInfo( diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/ParsedSignatureFile.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/ParsedSignatureFile.java index b2df6bb57..1d43b2171 100644 --- a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/ParsedSignatureFile.java +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/ParsedSignatureFile.java @@ -105,6 +105,7 @@ * byte[48] * Metadata Hash of the corresponding stream file * + * * Signature on hash bytes of Metadata Hash * byte[] * A signature object generated by signing the hash bytes of Metadata Hash @@ -148,6 +149,7 @@ * int (4) * 101 - length of signature bytes * + * * Signature bytes * byte[] * Serialized Signature bytes diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/RecordFileInfo.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/RecordFileInfo.java index c0d447fc5..a5b7c3d76 100644 --- a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/RecordFileInfo.java +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/model/RecordFileInfo.java @@ -28,7 +28,7 @@ import java.security.MessageDigest; /** - * Represents the version & block hash information of a record file. + * Represents the version and block hash information of a record file. *

* The old record file formats are documented in the * @@ -39,6 +39,7 @@ * * @param hapiProtoVersion the HAPI protocol version * @param blockHash the block hash + * @param recordFileContents the record file contents */ public record RecordFileInfo(SemanticVersion hapiProtoVersion, Bytes blockHash, byte[] recordFileContents) { /* The length of the header in a v2 record file */ diff --git a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/util/BlockWriter.java b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/util/BlockWriter.java index 1faabbd97..ba3814a8e 100644 --- a/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/util/BlockWriter.java +++ b/tools/src/main/java/com/hedera/block/tools/commands/record2blocks/util/BlockWriter.java @@ -61,6 +61,7 @@ public record BlockPath(Path dirPath, String zipFileName, String blockNumStr, St * @param baseDirectory The base directory for the block files * @param block The block to write * @throws IOException If an error occurs writing the block + * @return The path to the block file */ public static BlockPath writeBlock(final Path baseDirectory, final Block block) throws IOException { // get block number from block header @@ -115,6 +116,8 @@ private static BlockPath computeBlockPath(final Path baseDirectory, long blockNu /** * Simple main method to test the block path computation + * + * @param args The command line arguments */ public static void main(String[] args) { for (long blockNumber = 0; blockNumber < 3002; blockNumber++) {