From 7f48aec28a9be5c44033ad5ef85daf70bcdf2ca0 Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Wed, 14 Aug 2024 15:50:03 -0600 Subject: [PATCH] refactor: configuration to use Platform SDK across all project (#101) Signed-off-by: Alfredo Gutierrez --- .gitignore | 1 + README.md | 30 ++- .../block/server/BlockStreamService.java | 7 +- .../com/hedera/block/server/Constants.java | 9 - .../java/com/hedera/block/server/Server.java | 31 +-- .../config/BlockNodeConfigExtension.java | 14 +- .../block/server/consumer/ConsumerConfig.java | 29 +++ .../ConsumerStreamResponseObserver.java | 11 +- .../mediator/LiveStreamMediatorImpl.java | 1 + .../storage/{Util.java => FileUtils.java} | 34 ++- .../storage/PersistenceStorageConfig.java | 61 +++++ .../storage/read/BlockAsDirReader.java | 8 +- .../storage/read/BlockAsDirReaderBuilder.java | 24 +- .../storage/write/BlockAsDirWriter.java | 45 +--- .../write/BlockAsDirWriterBuilder.java | 33 +-- server/src/main/java/module-info.java | 1 - server/src/main/resources/application.yaml | 0 .../block/server/BlockStreamServiceIT.java | 52 ++-- .../block/server/BlockStreamServiceTest.java | 43 +--- .../server/config/TestConfigBuilder.java | 241 ++++++++++++++++++ .../ConsumerStreamResponseObserverTest.java | 45 ++-- .../mediator/LiveStreamMediatorImplTest.java | 49 ++-- .../storage/PersistenceStorageConfigTest.java | 90 +++++++ .../storage/read/BlockAsDirReaderTest.java | 76 +++--- .../storage/remove/BlockAsDirRemoverTest.java | 33 ++- .../storage/write/BlockAsDirWriterTest.java | 86 ++----- .../ProducerBlockItemObserverTest.java | 20 +- .../block/server/util/TestConfigUtil.java | 71 ++++++ server/src/test/resources/app.properties | 2 + 29 files changed, 787 insertions(+), 360 deletions(-) create mode 100644 server/src/main/java/com/hedera/block/server/consumer/ConsumerConfig.java rename server/src/main/java/com/hedera/block/server/persistence/storage/{Util.java => FileUtils.java} (56%) create mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java delete mode 100644 server/src/main/resources/application.yaml create mode 100644 server/src/test/java/com/hedera/block/server/config/TestConfigBuilder.java create mode 100644 server/src/test/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfigTest.java create mode 100644 server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java create mode 100644 server/src/test/resources/app.properties diff --git a/.gitignore b/.gitignore index 594c0f46a..54f5cbbf4 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,4 @@ gradle-app.setting .DS_Store # .env files server/docker/.env +/server/data/ diff --git a/README.md b/README.md index e118ae70c..9828504f5 100644 --- a/README.md +++ b/README.md @@ -30,19 +30,31 @@ Please do not file a public ticket mentioning the vulnerability. Refer to the se --- -# Running Locally +# Usage -1) Create a local temp directory. For example, use `mktemp -d -t block-stream-temp-dir` to create a directory -2) Configuration variables -``` -export BLOCKNODE_STORAGE_ROOT_PATH= # You can add this to your .zshrc, etc -``` -3) Optional Configuration variables +## Configuration + +| Environment Variable | Description | Default Value | +|---------------------------------|---------------------------------------------------------------------------------------------------------------|---------------| +| persistence.storage.rootPath | The root path for the storage, if not provided will attempt to create a `data` on the working dir of the app. | ./data | +| consumer.timeoutThresholdMillis | Time to wait for subscribers before disconnecting in milliseconds | 1500 | + + + +# Starting locally: +```bash +./gradlew run ``` -export BLOCKNODE_SERVER_CONSUMER_TIMEOUT_THRESHOLD="" #Default is 1500 + +In debug mode, you can attach a debugger to the port 5005. +```bash +./gradlew run --debug-jvm ``` -3) ./gradlew run # ./gradlew run --debug-jvm to run in debug mode +Also you can run on docker locally: +```bash +./gradlew startDockerContainer +``` # Running Tests 1) ./gradlew build diff --git a/server/src/main/java/com/hedera/block/server/BlockStreamService.java b/server/src/main/java/com/hedera/block/server/BlockStreamService.java index 5b0f1d80d..8b2a0a43b 100644 --- a/server/src/main/java/com/hedera/block/server/BlockStreamService.java +++ b/server/src/main/java/com/hedera/block/server/BlockStreamService.java @@ -44,7 +44,6 @@ public class BlockStreamService implements GrpcService { private final System.Logger LOGGER = System.getLogger(getClass().getName()); - private final long timeoutThresholdMillis; private final ItemAckBuilder itemAckBuilder; private final StreamMediator> streamMediator; private final ServiceStatus serviceStatus; @@ -55,8 +54,6 @@ public class BlockStreamService implements GrpcService { * Constructor for the BlockStreamService class. It initializes the BlockStreamService with the * given parameters. * - * @param timeoutThresholdMillis the timeout threshold in milliseconds for the producer to - * publish block items * @param itemAckBuilder the item acknowledgement builder to send responses back to the producer * @param streamMediator the stream mediator to proxy block items from the producer to the * subscribers and manage the subscription lifecycle for subscribers @@ -66,7 +63,6 @@ public class BlockStreamService implements GrpcService { * stop the service and web server in the event of an unrecoverable exception */ BlockStreamService( - final long timeoutThresholdMillis, @NonNull final ItemAckBuilder itemAckBuilder, @NonNull final StreamMediator> @@ -74,7 +70,6 @@ public class BlockStreamService implements GrpcService { @NonNull final BlockReader blockReader, @NonNull final ServiceStatus serviceStatus, @NonNull final BlockNodeContext blockNodeContext) { - this.timeoutThresholdMillis = timeoutThresholdMillis; this.itemAckBuilder = itemAckBuilder; this.streamMediator = streamMediator; this.blockReader = blockReader; @@ -143,7 +138,7 @@ void subscribeBlockStream( @NonNull final var streamObserver = new ConsumerStreamResponseObserver( - timeoutThresholdMillis, + blockNodeContext, Clock.systemDefaultZone(), streamMediator, subscribeStreamResponseObserver); diff --git a/server/src/main/java/com/hedera/block/server/Constants.java b/server/src/main/java/com/hedera/block/server/Constants.java index 65eaf30ce..d21f62852 100644 --- a/server/src/main/java/com/hedera/block/server/Constants.java +++ b/server/src/main/java/com/hedera/block/server/Constants.java @@ -22,15 +22,6 @@ public final class Constants { private Constants() {} - /** Constant mapped to the root path config key where the block files are stored */ - @NonNull - public static final String BLOCKNODE_STORAGE_ROOT_PATH_KEY = "blocknode.storage.root.path"; - - /** Constant mapped to the timeout for stream consumers in milliseconds */ - @NonNull - public static final String BLOCKNODE_SERVER_CONSUMER_TIMEOUT_THRESHOLD_KEY = - "blocknode.server.consumer.timeout.threshold"; - /** Constant mapped to the name of the service in the .proto file */ @NonNull public static final String SERVICE_NAME = "BlockStreamGrpcService"; diff --git a/server/src/main/java/com/hedera/block/server/Server.java b/server/src/main/java/com/hedera/block/server/Server.java index 980c8efd3..53dd1e0fa 100644 --- a/server/src/main/java/com/hedera/block/server/Server.java +++ b/server/src/main/java/com/hedera/block/server/Server.java @@ -17,21 +17,19 @@ package com.hedera.block.server; import static com.hedera.block.protos.BlockStreamService.*; -import static com.hedera.block.server.Constants.BLOCKNODE_SERVER_CONSUMER_TIMEOUT_THRESHOLD_KEY; -import static com.hedera.block.server.Constants.BLOCKNODE_STORAGE_ROOT_PATH_KEY; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.config.BlockNodeContextFactory; import com.hedera.block.server.data.ObjectEvent; import com.hedera.block.server.mediator.LiveStreamMediatorBuilder; import com.hedera.block.server.mediator.StreamMediator; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.producer.ItemAckBuilder; import edu.umd.cs.findbugs.annotations.NonNull; -import io.helidon.config.Config; import io.helidon.webserver.WebServer; import io.helidon.webserver.grpc.GrpcRouting; import java.io.IOException; @@ -56,17 +54,11 @@ public static void main(final String[] args) { // init context, metrics, and configuration. @NonNull final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); - // Set the global configuration - @NonNull final Config config = Config.create(); - Config.global(config); - @NonNull final ServiceStatus serviceStatus = new ServiceStatusImpl(); @NonNull final BlockWriter blockWriter = - BlockAsDirWriterBuilder.newBuilder( - BLOCKNODE_STORAGE_ROOT_PATH_KEY, config, blockNodeContext) - .build(); + BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); @NonNull final StreamMediator> streamMediator = LiveStreamMediatorBuilder.newBuilder( @@ -75,13 +67,16 @@ public static void main(final String[] args) { @NonNull final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(BLOCKNODE_STORAGE_ROOT_PATH_KEY, config) + BlockAsDirReaderBuilder.newBuilder( + blockNodeContext + .configuration() + .getConfigData(PersistenceStorageConfig.class)) .build(); @NonNull final BlockStreamService blockStreamService = buildBlockStreamService( - config, streamMediator, blockReader, serviceStatus, blockNodeContext); + streamMediator, blockReader, serviceStatus, blockNodeContext); @NonNull final GrpcRouting.Builder grpcRouting = @@ -104,7 +99,6 @@ public static void main(final String[] args) { @NonNull private static BlockStreamService buildBlockStreamService( - @NonNull final Config config, @NonNull final StreamMediator> streamMediator, @@ -112,16 +106,7 @@ private static BlockStreamService buildBlockStreamService( @NonNull final ServiceStatus serviceStatus, @NonNull final BlockNodeContext blockNodeContext) { - // Get Timeout threshold from configuration - final long consumerTimeoutThreshold = - config.get(BLOCKNODE_SERVER_CONSUMER_TIMEOUT_THRESHOLD_KEY).asLong().orElse(1500L); - return new BlockStreamService( - consumerTimeoutThreshold, - new ItemAckBuilder(), - streamMediator, - blockReader, - serviceStatus, - blockNodeContext); + new ItemAckBuilder(), streamMediator, blockReader, serviceStatus, blockNodeContext); } } diff --git a/server/src/main/java/com/hedera/block/server/config/BlockNodeConfigExtension.java b/server/src/main/java/com/hedera/block/server/config/BlockNodeConfigExtension.java index 731030d4b..d99c08b9f 100644 --- a/server/src/main/java/com/hedera/block/server/config/BlockNodeConfigExtension.java +++ b/server/src/main/java/com/hedera/block/server/config/BlockNodeConfigExtension.java @@ -17,6 +17,8 @@ package com.hedera.block.server.config; import com.google.auto.service.AutoService; +import com.hedera.block.server.consumer.ConsumerConfig; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.swirlds.common.config.BasicCommonConfig; import com.swirlds.common.metrics.config.MetricsConfig; import com.swirlds.common.metrics.platform.prometheus.PrometheusConfig; @@ -28,6 +30,11 @@ @AutoService(ConfigurationExtension.class) public class BlockNodeConfigExtension implements ConfigurationExtension { + /** Explicitly defined constructor. */ + public BlockNodeConfigExtension() { + super(); + } + /** * {@inheritDoc} * @@ -36,6 +43,11 @@ public class BlockNodeConfigExtension implements ConfigurationExtension { @NonNull @Override public Set> getConfigDataTypes() { - return Set.of(BasicCommonConfig.class, MetricsConfig.class, PrometheusConfig.class); + return Set.of( + BasicCommonConfig.class, + MetricsConfig.class, + PrometheusConfig.class, + ConsumerConfig.class, + PersistenceStorageConfig.class); } } diff --git a/server/src/main/java/com/hedera/block/server/consumer/ConsumerConfig.java b/server/src/main/java/com/hedera/block/server/consumer/ConsumerConfig.java new file mode 100644 index 000000000..fc4d0124f --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/consumer/ConsumerConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.consumer; + +import com.swirlds.config.api.ConfigData; +import com.swirlds.config.api.ConfigProperty; + +/** + * Use this configuration across the consumer package. + * + * @param timeoutThresholdMillis after this time of inactivity, the consumer will be considered + * timed out and will be disconnected + */ +@ConfigData("consumer") +public record ConsumerConfig(@ConfigProperty(defaultValue = "1500") long timeoutThresholdMillis) {} diff --git a/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java b/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java index 6beec8878..91a9f6cc5 100644 --- a/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java +++ b/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java @@ -19,6 +19,7 @@ import static com.hedera.block.protos.BlockStreamService.BlockItem; import static com.hedera.block.protos.BlockStreamService.SubscribeStreamResponse; +import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.data.ObjectEvent; import com.hedera.block.server.mediator.SubscriptionHandler; import com.lmax.disruptor.EventHandler; @@ -66,15 +67,14 @@ public class ConsumerStreamResponseObserver * SubscribeStreamResponse events from the Disruptor and passing them to the downstream consumer * via the subscribeStreamResponseObserver. * - * @param timeoutThresholdMillis the timeout threshold in milliseconds for the producer to - * publish block items + * @param context contains the context with metrics and configuration for the application * @param producerLivenessClock the clock to use to determine the producer liveness * @param subscriptionHandler the subscription handler to use to manage the subscription * lifecycle * @param subscribeStreamResponseObserver the observer to use to send responses to the consumer */ public ConsumerStreamResponseObserver( - final long timeoutThresholdMillis, + @NonNull final BlockNodeContext context, @NonNull final InstantSource producerLivenessClock, @NonNull final SubscriptionHandler> @@ -82,7 +82,10 @@ public ConsumerStreamResponseObserver( @NonNull final StreamObserver subscribeStreamResponseObserver) { - this.timeoutThresholdMillis = timeoutThresholdMillis; + this.timeoutThresholdMillis = + context.configuration() + .getConfigData(ConsumerConfig.class) + .timeoutThresholdMillis(); this.subscriptionHandler = subscriptionHandler; // The ServerCallStreamObserver can be configured with Runnable handlers to diff --git a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java index 09645e440..015eb09e7 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java +++ b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java @@ -88,6 +88,7 @@ class LiveStreamMediatorImpl // Initialize and start the disruptor @NonNull final Disruptor> disruptor = + // TODO: replace ring buffer size with a configurable value, create a MediatorConfig new Disruptor<>(ObjectEvent::new, 1024, DaemonThreadFactory.INSTANCE); this.ringBuffer = disruptor.start(); this.executor = Executors.newCachedThreadPool(DaemonThreadFactory.INSTANCE); diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/Util.java b/server/src/main/java/com/hedera/block/server/persistence/storage/FileUtils.java similarity index 56% rename from server/src/main/java/com/hedera/block/server/persistence/storage/Util.java rename to server/src/main/java/com/hedera/block/server/persistence/storage/FileUtils.java index 5dec622c0..680d15fb4 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/Util.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/FileUtils.java @@ -17,14 +17,20 @@ package com.hedera.block.server.persistence.storage; import edu.umd.cs.findbugs.annotations.NonNull; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; import java.util.Set; -/** Util methods provide common functionality for the storage package. */ -public final class Util { - private Util() {} +/** FileUtils methods provide common functionality for the storage package. */ +public final class FileUtils { + + private static final System.Logger LOGGER = System.getLogger(FileUtils.class.getName()); + + private FileUtils() {} /** * Default file permissions defines the file and directory for the storage package. @@ -42,4 +48,26 @@ private Util() {} PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_EXECUTE)); + + /** + * Use this to create a Dir if it does not exist with the given permissions and log the result. + * + * @param blockNodePath the path to create + * @param logLevel the log level to use + * @param perms the permissions to use when creating the directory + * @throws IOException if the directory cannot be created + */ + public static void createPathIfNotExists( + @NonNull final Path blockNodePath, + @NonNull final System.Logger.Level logLevel, + @NonNull FileAttribute> perms) + throws IOException { + // Initialize the Block directory if it does not exist + if (Files.notExists(blockNodePath)) { + Files.createDirectory(blockNodePath, perms); + LOGGER.log(logLevel, "Created block node root directory: " + blockNodePath); + } else { + LOGGER.log(logLevel, "Using existing block node root directory: " + blockNodePath); + } + } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java b/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java new file mode 100644 index 000000000..eb91065f2 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage; + +import com.swirlds.config.api.ConfigData; +import com.swirlds.config.api.ConfigProperty; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Use this configuration across the persistent storage package + * + * @param rootPath provides the root path for saving block data, if you want to override it need to + * set it as persistence.storage.rootPath + */ +@ConfigData("persistence.storage") +public record PersistenceStorageConfig(@ConfigProperty(defaultValue = "") String rootPath) { + + /** + * Constructor to set the default root path if not provided, it will be set to the data + * directory in the current working directory + */ + public PersistenceStorageConfig { + // verify rootPath prop + Path path = Path.of(rootPath); + if (rootPath.isEmpty()) { + path = Paths.get(rootPath).toAbsolutePath().resolve("data"); + } + // Check if absolute + if (!path.isAbsolute()) { + throw new IllegalArgumentException(rootPath + " Root path must be absolute"); + } + // Create Directory if it does not exist + if (Files.notExists(path)) { + try { + FileUtils.createPathIfNotExists( + path, System.Logger.Level.ERROR, FileUtils.defaultPerms); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + rootPath = path.toString(); + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReader.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReader.java index 7c5c8f156..de1fc4b0c 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReader.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReader.java @@ -21,8 +21,8 @@ import static com.hedera.block.protos.BlockStreamService.BlockItem; import static com.hedera.block.server.Constants.BLOCK_FILE_EXTENSION; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import edu.umd.cs.findbugs.annotations.NonNull; -import io.helidon.config.Config; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -49,18 +49,16 @@ class BlockAsDirReader implements BlockReader { * Constructor for the BlockAsDirReader class. It initializes the BlockAsDirReader with the * given parameters. * - * @param key the key to retrieve the block node root path from the configuration * @param config the configuration to retrieve the block node root path * @param filePerms the file permissions to set on the block node root path */ BlockAsDirReader( - @NonNull final String key, - @NonNull final Config config, + @NonNull final PersistenceStorageConfig config, @NonNull final FileAttribute> filePerms) { LOGGER.log(System.Logger.Level.INFO, "Initializing FileSystemBlockReader"); - @NonNull final Path blockNodeRootPath = Path.of(config.get(key).asString().get()); + @NonNull final Path blockNodeRootPath = Path.of(config.rootPath()); LOGGER.log(System.Logger.Level.INFO, config.toString()); LOGGER.log(System.Logger.Level.INFO, "Block Node Root Path: " + blockNodeRootPath); diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java index c0a3f9d54..b61d9fe93 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java @@ -18,9 +18,9 @@ import static com.hedera.block.protos.BlockStreamService.Block; -import com.hedera.block.server.persistence.storage.Util; +import com.hedera.block.server.persistence.storage.FileUtils; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import edu.umd.cs.findbugs.annotations.NonNull; -import io.helidon.config.Config; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; import java.util.Set; @@ -32,27 +32,23 @@ */ public class BlockAsDirReaderBuilder { - private final String key; - private final Config config; - private FileAttribute> filePerms = Util.defaultPerms; + private FileAttribute> filePerms = FileUtils.defaultPerms; + private final PersistenceStorageConfig config; - private BlockAsDirReaderBuilder(@NonNull final String key, @NonNull final Config config) { - this.key = key; + private BlockAsDirReaderBuilder(@NonNull PersistenceStorageConfig config) { this.config = config; } /** * Creates a new block reader builder using the minimum required parameters. * - * @param key is required to read pertinent configuration info. * @param config is required to supply pertinent configuration info for the block reader to * access storage. * @return a block reader builder configured with required parameters. */ @NonNull - public static BlockAsDirReaderBuilder newBuilder( - @NonNull final String key, @NonNull final Config config) { - return new BlockAsDirReaderBuilder(key, config); + public static BlockAsDirReaderBuilder newBuilder(@NonNull PersistenceStorageConfig config) { + return new BlockAsDirReaderBuilder(config); } /** @@ -60,8 +56,8 @@ public static BlockAsDirReaderBuilder newBuilder( * and directories. * *

By default, the block reader will use the permissions defined in {@link - * Util#defaultPerms}. This method is primarily used for testing purposes. Default values should - * be sufficient for production use. + * FileUtils#defaultPerms}. This method is primarily used for testing purposes. Default values + * should be sufficient for production use. * * @param filePerms the file permissions to use when managing block files and directories. * @return a block reader builder configured with required parameters. @@ -80,6 +76,6 @@ public BlockAsDirReaderBuilder filePerms( */ @NonNull public BlockReader build() { - return new BlockAsDirReader(key, config, filePerms); + return new BlockAsDirReader(config, filePerms); } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java index c04965454..df59417ed 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java @@ -17,14 +17,14 @@ package com.hedera.block.server.persistence.storage.write; import static com.hedera.block.protos.BlockStreamService.BlockItem; -import static com.hedera.block.server.Constants.BLOCKNODE_STORAGE_ROOT_PATH_KEY; import static com.hedera.block.server.Constants.BLOCK_FILE_EXTENSION; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.metrics.MetricsService; +import com.hedera.block.server.persistence.storage.FileUtils; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.remove.BlockRemover; import edu.umd.cs.findbugs.annotations.NonNull; -import io.helidon.config.Config; import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; @@ -54,19 +54,14 @@ class BlockAsDirWriter implements BlockWriter { private final BlockNodeContext blockNodeContext; /** - * Constructor for the BlockAsDirWriter class. It initializes the BlockAsDirWriter with the - * given key, config, block remover, and file permissions. + * Use the corresponding builder to construct a new BlockAsDirWriter with the given parameters. * - * @param key the key to use to retrieve the block node root path from the config - * @param config the config to use to retrieve the block node root path - * @param blockRemover the block remover to use to remove blocks if there is an exception while - * writing a partial block - * @param filePerms the file permissions to set on the block node root path + * @param blockRemover the block remover to use for removing blocks + * @param filePerms the file permissions to use for writing blocks + * @param blockNodeContext the block node context to use for writing blocks * @throws IOException if an error occurs while initializing the BlockAsDirWriter */ BlockAsDirWriter( - @NonNull final String key, - @NonNull final Config config, @NonNull final BlockRemover blockRemover, @NonNull final FileAttribute> filePerms, @NonNull final BlockNodeContext blockNodeContext) @@ -74,22 +69,18 @@ class BlockAsDirWriter implements BlockWriter { LOGGER.log(System.Logger.Level.INFO, "Initializing FileSystemBlockStorage"); - final Path blockNodeRootPath = Path.of(config.get(key).asString().get()); + PersistenceStorageConfig config = + blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); + final Path blockNodeRootPath = Path.of(config.rootPath()); - LOGGER.log(System.Logger.Level.INFO, config.toString()); LOGGER.log(System.Logger.Level.INFO, "Block Node Root Path: " + blockNodeRootPath); this.blockNodeRootPath = blockNodeRootPath; this.blockRemover = blockRemover; this.filePerms = filePerms; - if (!blockNodeRootPath.isAbsolute()) { - throw new IllegalArgumentException( - BLOCKNODE_STORAGE_ROOT_PATH_KEY + " must be an absolute path"); - } - // Initialize the block node root directory if it does not exist - createPath(blockNodeRootPath, System.Logger.Level.INFO); + FileUtils.createPathIfNotExists(blockNodeRootPath, System.Logger.Level.INFO, filePerms); this.blockNodeContext = blockNodeContext; } @@ -150,7 +141,7 @@ protected void write(@NonNull final Path blockItemFilePath, @NonNull final Block final FileOutputStream fos = new FileOutputStream(blockItemFilePath.toString())) { blockItem.writeTo(fos); LOGGER.log( - System.Logger.Level.INFO, + System.Logger.Level.DEBUG, "Successfully wrote the block item file: {0}", blockItemFilePath); } catch (IOException e) { @@ -172,7 +163,7 @@ private void resetState(@NonNull final BlockItem blockItem) throws IOException { repairPermissions(blockNodeRootPath); // Construct the path to the block directory - createPath(calculateBlockPath(), System.Logger.Level.DEBUG); + FileUtils.createPathIfNotExists(calculateBlockPath(), System.Logger.Level.DEBUG, filePerms); // Reset blockNodeFileNameIndex = 0; @@ -215,16 +206,4 @@ private Path calculateBlockItemPath() { private Path calculateBlockPath() { return blockNodeRootPath.resolve(currentBlockDir); } - - private void createPath( - @NonNull final Path blockNodePath, @NonNull final System.Logger.Level logLevel) - throws IOException { - // Initialize the Block directory if it does not exist - if (Files.notExists(blockNodePath)) { - Files.createDirectory(blockNodePath, filePerms); - LOGGER.log(logLevel, "Created block node root directory: " + blockNodePath); - } else { - LOGGER.log(logLevel, "Using existing block node root directory: " + blockNodePath); - } - } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java index 579e8d5e5..f0f1e8fe1 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java @@ -19,11 +19,11 @@ import static com.hedera.block.protos.BlockStreamService.BlockItem; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.persistence.storage.Util; +import com.hedera.block.server.persistence.storage.FileUtils; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.remove.BlockAsDirRemover; import com.hedera.block.server.persistence.storage.remove.BlockRemover; import edu.umd.cs.findbugs.annotations.NonNull; -import io.helidon.config.Config; import java.io.IOException; import java.nio.file.Path; import java.nio.file.attribute.FileAttribute; @@ -37,39 +37,30 @@ */ public class BlockAsDirWriterBuilder { - private final String key; - private final Config config; private final BlockNodeContext blockNodeContext; - private FileAttribute> filePerms = Util.defaultPerms; + private FileAttribute> filePerms = FileUtils.defaultPerms; private BlockRemover blockRemover; - private BlockAsDirWriterBuilder( - @NonNull final String key, - @NonNull final Config config, - @NonNull final BlockNodeContext blockNodeContext) { - this.key = key; - this.config = config; + private BlockAsDirWriterBuilder(@NonNull final BlockNodeContext blockNodeContext) { this.blockNodeContext = blockNodeContext; + PersistenceStorageConfig config = + blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); + this.blockRemover = - new BlockAsDirRemover(Path.of(config.get(key).asString().get()), Util.defaultPerms); + new BlockAsDirRemover(Path.of(config.rootPath()), FileUtils.defaultPerms); } /** * Creates a new block writer builder using the minimum required parameters. * - * @param key is required to read pertinent configuration info. - * @param config is required to supply pertinent configuration info for the block writer to - * access storage. * @param blockNodeContext is required to provide metrics reporting mechanisms . * @return a block writer builder configured with required parameters. */ @NonNull public static BlockAsDirWriterBuilder newBuilder( - @NonNull final String key, - @NonNull final Config config, @NonNull final BlockNodeContext blockNodeContext) { - return new BlockAsDirWriterBuilder(key, config, blockNodeContext); + return new BlockAsDirWriterBuilder(blockNodeContext); } /** @@ -77,8 +68,8 @@ public static BlockAsDirWriterBuilder newBuilder( * and directories. * *

By default, the block writer will use the permissions defined in {@link - * Util#defaultPerms}. This method is primarily used for testing purposes. Default values should - * be sufficient for production use. + * FileUtils#defaultPerms}. This method is primarily used for testing purposes. Default values + * should be sufficient for production use. * * @param filePerms the file permissions to use when managing block files and directories. * @return a block writer builder configured with required parameters. @@ -114,6 +105,6 @@ public BlockAsDirWriterBuilder blockRemover(@NonNull BlockRemover blockRemover) */ @NonNull public BlockWriter build() throws IOException { - return new BlockAsDirWriter(key, config, blockRemover, filePerms, blockNodeContext); + return new BlockAsDirWriter(blockRemover, filePerms, blockNodeContext); } } diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index c78b82458..895ff1c28 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -11,7 +11,6 @@ requires com.swirlds.metrics.api; requires io.grpc.stub; requires io.helidon.common; - requires io.helidon.config; requires io.helidon.webserver.grpc; requires io.helidon.webserver; requires static com.github.spotbugs.annotations; diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/server/src/test/java/com/hedera/block/server/BlockStreamServiceIT.java b/server/src/test/java/com/hedera/block/server/BlockStreamServiceIT.java index e867c5f92..0fddbe65b 100644 --- a/server/src/test/java/com/hedera/block/server/BlockStreamServiceIT.java +++ b/server/src/test/java/com/hedera/block/server/BlockStreamServiceIT.java @@ -23,11 +23,11 @@ import static org.mockito.Mockito.*; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.config.BlockNodeContextFactory; import com.hedera.block.server.data.ObjectEvent; import com.hedera.block.server.mediator.LiveStreamMediatorBuilder; import com.hedera.block.server.mediator.StreamMediator; -import com.hedera.block.server.persistence.storage.Util; +import com.hedera.block.server.persistence.storage.FileUtils; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.persistence.storage.remove.BlockAsDirRemover; @@ -35,13 +35,11 @@ import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.producer.ItemAckBuilder; +import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; import com.lmax.disruptor.BatchEventProcessor; import com.lmax.disruptor.EventHandler; import io.grpc.stub.StreamObserver; -import io.helidon.config.Config; -import io.helidon.config.MapConfigSource; -import io.helidon.config.spi.ConfigSource; import io.helidon.webserver.WebServer; import java.io.IOException; import java.nio.file.Files; @@ -85,10 +83,10 @@ public class BlockStreamServiceIT { @Mock private BlockWriter blockWriter; private static final String TEMP_DIR = "block-node-unit-test-dir"; - private static final String JUNIT = "my-junit-test"; private Path testPath; - private Config testConfig; + private BlockNodeContext blockNodeContext; + private PersistenceStorageConfig testConfig; private static final int testTimeout = 200; @@ -97,9 +95,10 @@ public void setUp() throws IOException { testPath = Files.createTempDirectory(TEMP_DIR); LOGGER.log(System.Logger.Level.INFO, "Created temp directory: " + testPath.toString()); - Map testProperties = Map.of(JUNIT, testPath.toString()); - ConfigSource testConfigSource = MapConfigSource.builder().map(testProperties).build(); - testConfig = Config.builder(testConfigSource).build(); + blockNodeContext = + TestConfigUtil.getTestBlockNodeContext( + Map.of("persistence.storage.rootPath", testPath.toString())); + testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); } @AfterEach @@ -111,10 +110,8 @@ public void tearDown() { public void testPublishBlockStreamRegistrationAndExecution() throws IOException, NoSuchAlgorithmException { - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockStreamService blockStreamService = new BlockStreamService( - 1500L, new ItemAckBuilder(), streamMediator, blockReader, @@ -159,7 +156,8 @@ public void testSubscribeBlockStream() throws IOException { final ServiceStatus serviceStatus = new ServiceStatusImpl(); serviceStatus.setWebServer(webServer); - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + final var streamMediator = LiveStreamMediatorBuilder.newBuilder(blockWriter, blockNodeContext, serviceStatus) .build(); @@ -167,7 +165,6 @@ public void testSubscribeBlockStream() throws IOException { // Build the BlockStreamService final BlockStreamService blockStreamService = new BlockStreamService( - 2000L, new ItemAckBuilder(), streamMediator, blockReader, @@ -321,7 +318,7 @@ public void testSubAndUnsubWhileStreaming() throws IOException { EventHandler>, BatchEventProcessor>> subscribers = new LinkedHashMap<>(); - final var streamMediator = buildStreamMediator(subscribers, Util.defaultPerms); + final var streamMediator = buildStreamMediator(subscribers, FileUtils.defaultPerms); final var blockStreamService = buildBlockStreamService(streamMediator, blockReader, serviceStatus); @@ -498,7 +495,7 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep // Now verify the block was removed from the file system. final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(JUNIT, testConfig).build(); + BlockAsDirReaderBuilder.newBuilder(testConfig).build(); final Optional blockOpt = blockReader.read(1); assertTrue(blockOpt.isEmpty()); @@ -524,8 +521,9 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep .onNext(expectedSubscriberStreamNotAvailable); } - private void removeRootPathWritePerms(final Config config) throws IOException { - final Path blockNodeRootPath = Path.of(config.get(JUNIT).asString().get()); + private void removeRootPathWritePerms(final PersistenceStorageConfig config) + throws IOException { + final Path blockNodeRootPath = Path.of(config.rootPath()); Files.setPosixFilePermissions(blockNodeRootPath, TestUtils.getNoWrite().value()); } @@ -568,7 +566,7 @@ private static SubscribeStreamResponse buildSubscribeStreamResponse(BlockItem bl private BlockStreamService buildBlockStreamService() throws IOException { final var streamMediator = - buildStreamMediator(new ConcurrentHashMap<>(32), Util.defaultPerms); + buildStreamMediator(new ConcurrentHashMap<>(32), FileUtils.defaultPerms); return buildBlockStreamService(streamMediator, blockReader, serviceStatus); } @@ -583,12 +581,10 @@ private StreamMediator> buildStr // Initialize with concrete a concrete BlockReader, BlockWriter and Mediator final BlockRemover blockRemover = - new BlockAsDirRemover( - Path.of(testConfig.get(JUNIT).asString().get()), Util.defaultPerms); + new BlockAsDirRemover(Path.of(testConfig.rootPath()), FileUtils.defaultPerms); - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockWriter blockWriter = - BlockAsDirWriterBuilder.newBuilder(JUNIT, testConfig, blockNodeContext) + BlockAsDirWriterBuilder.newBuilder(blockNodeContext) .blockRemover(blockRemover) .filePerms(filePerms) .build(); @@ -607,13 +603,9 @@ private BlockStreamService buildBlockStreamService( final ServiceStatus serviceStatus) throws IOException { - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + return new BlockStreamService( - 2000, - new ItemAckBuilder(), - streamMediator, - blockReader, - serviceStatus, - blockNodeContext); + new ItemAckBuilder(), streamMediator, blockReader, serviceStatus, blockNodeContext); } } diff --git a/server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java b/server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java index 277185764..ad4bf1ac1 100644 --- a/server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java @@ -28,20 +28,18 @@ import com.google.protobuf.Descriptors; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.config.BlockNodeContextFactory; import com.hedera.block.server.data.ObjectEvent; import com.hedera.block.server.mediator.StreamMediator; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.producer.ItemAckBuilder; +import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; import io.grpc.stub.ServerCalls; import io.grpc.stub.StreamObserver; -import io.helidon.config.Config; -import io.helidon.config.MapConfigSource; -import io.helidon.config.spi.ConfigSource; import io.helidon.webserver.grpc.GrpcService; import java.io.IOException; import java.nio.file.Files; @@ -60,8 +58,6 @@ @ExtendWith(MockitoExtension.class) public class BlockStreamServiceTest { - private final long TIMEOUT_THRESHOLD_MILLIS = 50L; - @Mock private StreamObserver responseObserver; @Mock private ItemAckBuilder itemAckBuilder; @@ -75,19 +71,20 @@ public class BlockStreamServiceTest { private final System.Logger LOGGER = System.getLogger(getClass().getName()); private static final String TEMP_DIR = "block-node-unit-test-dir"; - private static final String JUNIT = "my-junit-test"; private Path testPath; - private Config testConfig; + private BlockNodeContext blockNodeContext; + private PersistenceStorageConfig config; @BeforeEach public void setUp() throws IOException { testPath = Files.createTempDirectory(TEMP_DIR); LOGGER.log(System.Logger.Level.INFO, "Created temp directory: " + testPath.toString()); - Map testProperties = Map.of(JUNIT, testPath.toString()); - ConfigSource testConfigSource = MapConfigSource.builder().map(testProperties).build(); - testConfig = Config.builder(testConfigSource).build(); + blockNodeContext = + TestConfigUtil.getTestBlockNodeContext( + Map.of("persistence.storage.rootPath", testPath.toString())); + config = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); } @AfterEach @@ -98,10 +95,8 @@ public void tearDown() { @Test public void testServiceName() throws IOException, NoSuchAlgorithmException { - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockStreamService blockStreamService = new BlockStreamService( - TIMEOUT_THRESHOLD_MILLIS, itemAckBuilder, streamMediator, blockReader, @@ -118,10 +113,8 @@ public void testServiceName() throws IOException, NoSuchAlgorithmException { @Test public void testProto() throws IOException, NoSuchAlgorithmException { - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockStreamService blockStreamService = new BlockStreamService( - TIMEOUT_THRESHOLD_MILLIS, itemAckBuilder, streamMediator, blockReader, @@ -140,12 +133,9 @@ public void testProto() throws IOException, NoSuchAlgorithmException { @Test void testSingleBlockHappyPath() throws IOException { - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(JUNIT, testConfig).build(); - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); + final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); final BlockStreamService blockStreamService = new BlockStreamService( - TIMEOUT_THRESHOLD_MILLIS, itemAckBuilder, streamMediator, blockReader, @@ -157,7 +147,7 @@ void testSingleBlockHappyPath() throws IOException { // Generate and persist a block final BlockWriter blockWriter = - BlockAsDirWriterBuilder.newBuilder(JUNIT, testConfig, blockNodeContext).build(); + BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); final List blockItems = generateBlockItems(1); for (BlockItem blockItem : blockItems) { blockWriter.write(blockItem); @@ -186,8 +176,6 @@ void testSingleBlockHappyPath() throws IOException { @Test void testSingleBlockNotFoundPath() throws IOException { - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); - // Get the block so we can verify the response payload when(blockReader.read(1)).thenReturn(Optional.empty()); @@ -201,7 +189,6 @@ void testSingleBlockNotFoundPath() throws IOException { // Call the service final BlockStreamService blockStreamService = new BlockStreamService( - TIMEOUT_THRESHOLD_MILLIS, itemAckBuilder, streamMediator, blockReader, @@ -216,12 +203,10 @@ void testSingleBlockNotFoundPath() throws IOException { } @Test - void testSingleBlockServiceNotAvailable() throws IOException { + void testSingleBlockServiceNotAvailable() { - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockStreamService blockStreamService = new BlockStreamService( - TIMEOUT_THRESHOLD_MILLIS, itemAckBuilder, streamMediator, blockReader, @@ -242,10 +227,8 @@ void testSingleBlockServiceNotAvailable() throws IOException { @Test public void testSingleBlockIOExceptionPath() throws IOException { - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockStreamService blockStreamService = new BlockStreamService( - TIMEOUT_THRESHOLD_MILLIS, itemAckBuilder, streamMediator, blockReader, @@ -266,12 +249,10 @@ public void testSingleBlockIOExceptionPath() throws IOException { } @Test - public void testUpdateInvokesRoutingWithLambdas() throws IOException { + public void testUpdateInvokesRoutingWithLambdas() { - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockStreamService blockStreamService = new BlockStreamService( - TIMEOUT_THRESHOLD_MILLIS, itemAckBuilder, streamMediator, blockReader, diff --git a/server/src/test/java/com/hedera/block/server/config/TestConfigBuilder.java b/server/src/test/java/com/hedera/block/server/config/TestConfigBuilder.java new file mode 100644 index 000000000..fc627f50e --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/config/TestConfigBuilder.java @@ -0,0 +1,241 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.config; + +import com.swirlds.common.config.singleton.ConfigurationHolder; +import com.swirlds.common.threading.locks.AutoClosableLock; +import com.swirlds.common.threading.locks.Locks; +import com.swirlds.common.threading.locks.locked.Locked; +import com.swirlds.config.api.ConfigData; +import com.swirlds.config.api.Configuration; +import com.swirlds.config.api.ConfigurationBuilder; +import com.swirlds.config.api.converter.ConfigConverter; +import com.swirlds.config.api.source.ConfigSource; +import com.swirlds.config.api.validation.ConfigValidator; +import com.swirlds.config.extensions.sources.SimpleConfigSource; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.util.Objects; + +/** + * Helper for use the config in test and change the config for specific tests. Instance can be used + * per class or per test. + */ +public class TestConfigBuilder { + + private final AutoClosableLock configLock = Locks.createAutoLock(); + + private Configuration configuration = null; + + private final ConfigurationBuilder builder; + + /** + * Creates a new instance and automatically registers all config data records (see {@link + * ConfigData}) on classpath / modulepath that are part of the packages {@code com.swirlds} and + * {@code com.hedera}. + */ + public TestConfigBuilder() { + this(true); + } + + /** + * Creates a new instance and automatically registers all given config data records. This call + * will not do a class scan for config data records on classpath / modulepath like some of the + * other constructors do. + * + * @param dataTypes + */ + public TestConfigBuilder(@Nullable final Class dataTypes) { + this(false); + if (dataTypes != null) { + this.builder.withConfigDataTypes(dataTypes); + } + } + + /** + * Creates a new instance and automatically registers all config data records (see {@link + * ConfigData}) on classpath / modulepath that are part of the packages {@code com.swirlds} and + * {@code com.hedera}. if the {@code registerAllTypes} param is true. + * + * @param registerAllTypes if true all config data records on classpath will automatically be + * registered + */ + public TestConfigBuilder(final boolean registerAllTypes) { + if (registerAllTypes) { + this.builder = ConfigurationBuilder.create().autoDiscoverExtensions(); + } else { + this.builder = ConfigurationBuilder.create(); + } + } + + /** + * Sets the value for the config. + * + * @param propertyName name of the property + * @param value the value + * @return the {@link TestConfigBuilder} instance (for fluent API) + */ + @NonNull + public TestConfigBuilder withValue( + @NonNull final String propertyName, @Nullable final String value) { + return withSource(new SimpleConfigSource(propertyName, value)); + } + + /** + * Sets the value for the config. + * + * @param propertyName name of the property + * @param value the value + * @return the {@link TestConfigBuilder} instance (for fluent API) + */ + @NonNull + public TestConfigBuilder withValue(@NonNull final String propertyName, final int value) { + return withSource(new SimpleConfigSource(propertyName, value)); + } + + /** + * Sets the value for the config. + * + * @param propertyName name of the property + * @param value the value + * @return the {@link TestConfigBuilder} instance (for fluent API) + */ + @NonNull + public TestConfigBuilder withValue(@NonNull final String propertyName, final double value) { + return withSource(new SimpleConfigSource(propertyName, value)); + } + + /** + * Sets the value for the config. + * + * @param propertyName name of the property + * @param value the value + * @return the {@link TestConfigBuilder} instance (for fluent API) + */ + @NonNull + public TestConfigBuilder withValue(@NonNull final String propertyName, final long value) { + return withSource(new SimpleConfigSource(propertyName, value)); + } + + /** + * Sets the value for the config. + * + * @param propertyName name of the property + * @param value the value + * @return the {@link TestConfigBuilder} instance (for fluent API) + */ + @NonNull + public TestConfigBuilder withValue(@NonNull final String propertyName, final boolean value) { + return withSource(new SimpleConfigSource(propertyName, value)); + } + + /** + * Sets the value for the config. + * + * @param propertyName name of the property + * @param value the value + * @return the {@link TestConfigBuilder} instance (for fluent API) + */ + @NonNull + public TestConfigBuilder withValue( + @NonNull final String propertyName, @NonNull final Object value) { + Objects.requireNonNull(value, "value must not be null"); + return withSource(new SimpleConfigSource(propertyName, value.toString())); + } + + /** + * This method returns the {@link Configuration} instance. If the method is called for the first + * time the {@link Configuration} instance will be created. All values that have been set (see + * {@link #withValue(String, int)}) methods will be part of the config. Next to this the config + * will support all config data record types (see {@link ConfigData}) that are on the classpath. + * + * @return the created configuration + */ + @NonNull + public Configuration getOrCreateConfig() { + try (final Locked ignore = configLock.lock()) { + if (configuration == null) { + configuration = builder.build(); + ConfigurationHolder.getInstance().setConfiguration(configuration); + } + return configuration; + } + } + + private void checkConfigState() { + try (final Locked ignore = configLock.lock()) { + if (configuration != null) { + throw new IllegalStateException("Configuration already created!"); + } + } + } + + /** + * Adds the given config source to the builder + * + * @param configSource the config source that will be added + * @return the {@link TestConfigBuilder} instance (for fluent API) + */ + @NonNull + public TestConfigBuilder withSource(@NonNull final ConfigSource configSource) { + checkConfigState(); + builder.withSource(configSource); + return this; + } + + /** + * Adds the given config converter to the builder + * + * @param converterType the type of the config converter + * @param converter the config converter that will be added + * @return the {@link TestConfigBuilder} instance (for fluent API) + */ + @NonNull + public TestConfigBuilder withConverter( + @NonNull final Class converterType, @NonNull final ConfigConverter converter) { + checkConfigState(); + builder.withConverter(converterType, converter); + return this; + } + + /** + * Adds the given config validator to the builder + * + * @param validator the config validator that will be added + * @return the {@link TestConfigBuilder} instance (for fluent API) + */ + @NonNull + public TestConfigBuilder withValidator(@NonNull final ConfigValidator validator) { + checkConfigState(); + builder.withValidator(validator); + return this; + } + + /** + * Adds the given config data type to the builder + * + * @param type the config data type that will be added + * @param the type of the config data type + * @return the {@link TestConfigBuilder} instance (for fluent API) + */ + @NonNull + public TestConfigBuilder withConfigDataType(@NonNull final Class type) { + checkConfigState(); + builder.withConfigDataType(type); + return this; + } +} diff --git a/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java b/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java index bce43f3ea..5fd2c44bc 100644 --- a/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java +++ b/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java @@ -20,12 +20,16 @@ import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; import static org.mockito.Mockito.*; +import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.data.ObjectEvent; import com.hedera.block.server.mediator.StreamMediator; +import com.hedera.block.server.util.TestConfigUtil; import io.grpc.stub.ServerCallStreamObserver; import io.grpc.stub.StreamObserver; +import java.io.IOException; import java.time.InstantSource; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -44,6 +48,16 @@ public class ConsumerStreamResponseObserverTest { @Mock private ServerCallStreamObserver serverCallStreamObserver; @Mock private InstantSource testClock; + final BlockNodeContext testContext; + + public ConsumerStreamResponseObserverTest() throws IOException { + this.testContext = + TestConfigUtil.getTestBlockNodeContext( + Map.of( + TestConfigUtil.CONSUMER_TIMEOUT_THRESHOLD_KEY, + String.valueOf(TIMEOUT_THRESHOLD_MILLIS))); + } + @Test public void testProducerTimeoutWithinWindow() { @@ -51,10 +65,7 @@ public void testProducerTimeoutWithinWindow() { final var consumerBlockItemObserver = new ConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, - testClock, - streamMediator, - responseStreamObserver); + testContext, testClock, streamMediator, responseStreamObserver); final BlockHeader blockHeader = BlockHeader.newBuilder().setBlockNumber(1).build(); final BlockItem blockItem = BlockItem.newBuilder().setHeader(blockHeader).build(); @@ -81,10 +92,7 @@ public void testProducerTimeoutOutsideWindow() throws InterruptedException { final var consumerBlockItemObserver = new ConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, - testClock, - streamMediator, - responseStreamObserver); + testContext, testClock, streamMediator, responseStreamObserver); consumerBlockItemObserver.onEvent(objectEvent, 0, true); verify(streamMediator).unsubscribe(consumerBlockItemObserver); @@ -98,7 +106,7 @@ public void testHandlersSetOnObserver() throws InterruptedException { when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS); new ConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, testClock, streamMediator, serverCallStreamObserver); + testContext, testClock, streamMediator, serverCallStreamObserver); verify(serverCallStreamObserver, timeout(50).times(1)).setOnCloseHandler(any()); verify(serverCallStreamObserver, timeout(50).times(1)).setOnCancelHandler(any()); @@ -109,10 +117,7 @@ public void testResponseNotPermittedAfterCancel() { final TestConsumerStreamResponseObserver consumerStreamResponseObserver = new TestConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, - testClock, - streamMediator, - serverCallStreamObserver); + testContext, testClock, streamMediator, serverCallStreamObserver); final List blockItems = generateBlockItems(1); final SubscribeStreamResponse subscribeStreamResponse = @@ -137,10 +142,7 @@ public void testResponseNotPermittedAfterClose() { final TestConsumerStreamResponseObserver consumerStreamResponseObserver = new TestConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, - testClock, - streamMediator, - serverCallStreamObserver); + testContext, testClock, streamMediator, serverCallStreamObserver); final List blockItems = generateBlockItems(1); final SubscribeStreamResponse subscribeStreamResponse = @@ -169,10 +171,7 @@ public void testConsumerNotToSendBeforeBlockHeader() { final var consumerBlockItemObserver = new ConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, - testClock, - streamMediator, - responseStreamObserver); + testContext, testClock, streamMediator, responseStreamObserver); // Send non-header BlockItems to validate that the observer does not send them for (int i = 1; i <= 10; i++) { @@ -209,12 +208,12 @@ public void testConsumerNotToSendBeforeBlockHeader() { private static class TestConsumerStreamResponseObserver extends ConsumerStreamResponseObserver { public TestConsumerStreamResponseObserver( - long timeoutThresholdMillis, + BlockNodeContext context, InstantSource producerLivenessClock, StreamMediator> subscriptionHandler, StreamObserver subscribeStreamResponseObserver) { super( - timeoutThresholdMillis, + context, producerLivenessClock, subscriptionHandler, subscribeStreamResponseObserver); diff --git a/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java b/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java index 0227d6b4e..1d8eb1774 100644 --- a/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java +++ b/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java @@ -24,9 +24,11 @@ import com.hedera.block.server.ServiceStatusImpl; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.config.BlockNodeContextFactory; +import com.hedera.block.server.consumer.ConsumerConfig; import com.hedera.block.server.consumer.ConsumerStreamResponseObserver; import com.hedera.block.server.data.ObjectEvent; import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.util.TestConfigUtil; import com.lmax.disruptor.EventHandler; import edu.umd.cs.findbugs.annotations.NonNull; import io.grpc.stub.ServerCallStreamObserver; @@ -34,6 +36,7 @@ import java.io.IOException; import java.time.InstantSource; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -60,6 +63,17 @@ public class LiveStreamMediatorImplTest { private static final int testTimeout = 200; + private final ConsumerConfig consumerConfig = new ConsumerConfig(TIMEOUT_THRESHOLD_MILLIS); + private final BlockNodeContext testContext; + + public LiveStreamMediatorImplTest() throws IOException { + this.testContext = + TestConfigUtil.getTestBlockNodeContext( + Map.of( + TestConfigUtil.CONSUMER_TIMEOUT_THRESHOLD_KEY, + String.valueOf(TIMEOUT_THRESHOLD_MILLIS))); + } + @Test public void testUnsubscribeEach() throws InterruptedException, IOException { @@ -139,15 +153,15 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) final var concreteObserver1 = new ConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, testClock, streamMediator, streamObserver1); + testContext, testClock, streamMediator, streamObserver1); final var concreteObserver2 = new ConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, testClock, streamMediator, streamObserver2); + testContext, testClock, streamMediator, streamObserver2); final var concreteObserver3 = new ConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, testClock, streamMediator, streamObserver3); + testContext, testClock, streamMediator, streamObserver3); // Set up the subscribers streamMediator.subscribe(concreteObserver1); @@ -196,15 +210,15 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) final var concreteObserver1 = new ConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, testClock, streamMediator, streamObserver1); + testContext, testClock, streamMediator, streamObserver1); final var concreteObserver2 = new ConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, testClock, streamMediator, streamObserver2); + testContext, testClock, streamMediator, streamObserver2); final var concreteObserver3 = new ConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, testClock, streamMediator, streamObserver3); + testContext, testClock, streamMediator, streamObserver3); // Set up the subscribers streamMediator.subscribe(concreteObserver1); @@ -232,10 +246,7 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) final var testConsumerBlockItemObserver = new TestConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, - testClock, - streamMediator, - serverCallStreamObserver); + testContext, testClock, streamMediator, serverCallStreamObserver); streamMediator.subscribe(testConsumerBlockItemObserver); assertTrue(streamMediator.isSubscribed(testConsumerBlockItemObserver)); @@ -271,10 +282,7 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) final var testConsumerBlockItemObserver = new TestConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, - testClock, - streamMediator, - serverCallStreamObserver); + testContext, testClock, streamMediator, serverCallStreamObserver); streamMediator.subscribe(testConsumerBlockItemObserver); assertTrue(streamMediator.isSubscribed(testConsumerBlockItemObserver)); @@ -340,10 +348,7 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) .build(); final var testConsumerBlockItemObserver = new TestConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, - testClock, - streamMediator, - serverCallStreamObserver); + testContext, testClock, streamMediator, serverCallStreamObserver); // Confirm the observer is not subscribed assertFalse(streamMediator.isSubscribed(testConsumerBlockItemObserver)); @@ -357,16 +362,12 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) private static class TestConsumerStreamResponseObserver extends ConsumerStreamResponseObserver { public TestConsumerStreamResponseObserver( - long timeoutThresholdMillis, + BlockNodeContext context, final InstantSource producerLivenessClock, final StreamMediator> streamMediator, final StreamObserver responseStreamObserver) { - super( - timeoutThresholdMillis, - producerLivenessClock, - streamMediator, - responseStreamObserver); + super(context, producerLivenessClock, streamMediator, responseStreamObserver); } @NonNull diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfigTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfigTest.java new file mode 100644 index 000000000..bcde8e3ea --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfigTest.java @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; + +class PersistenceStorageConfigTest { + + final String TEMP_DIR = "block-node-unit-test-dir"; + + @Test + void testPersistenceStorageConfig_happyPath() throws IOException { + + Path testPath = Files.createTempDirectory(TEMP_DIR); + + PersistenceStorageConfig persistenceStorageConfig = + new PersistenceStorageConfig(testPath.toString()); + assertEquals(testPath.toString(), persistenceStorageConfig.rootPath()); + } + + @Test + void testPersistenceStorageConfig_emptyRootPath() throws IOException { + final String expectedDefaultRootPath = + Paths.get("").toAbsolutePath().resolve("data").toString(); + // delete if exists + deleteDirectory(Paths.get(expectedDefaultRootPath)); + + PersistenceStorageConfig persistenceStorageConfig = new PersistenceStorageConfig(""); + assertEquals(expectedDefaultRootPath, persistenceStorageConfig.rootPath()); + } + + @Test + void persistenceStorageConfig_throwsExceptionForRelativePath() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> new PersistenceStorageConfig("relative/path")); + assertEquals("relative/path Root path must be absolute", exception.getMessage()); + } + + @Test + void persistenceStorageConfig_throwsRuntimeExceptionOnIOException() { + Path invalidPath = Paths.get("/invalid/path"); + + RuntimeException exception = + assertThrows( + RuntimeException.class, + () -> new PersistenceStorageConfig(invalidPath.toString())); + assertInstanceOf(IOException.class, exception.getCause()); + } + + public static void deleteDirectory(Path path) throws IOException { + if (!Files.exists(path)) { + return; + } + try (Stream walk = Files.walk(path)) { + walk.sorted(Comparator.reverseOrder()) + .forEach( + p -> { + try { + Files.delete(p); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java index aec8c6ba0..a7d8deaa1 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java @@ -25,16 +25,14 @@ import static org.mockito.Mockito.spy; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.config.BlockNodeContextFactory; -import com.hedera.block.server.persistence.storage.Util; +import com.hedera.block.server.persistence.storage.FileUtils; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.util.PersistTestUtils; +import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; import edu.umd.cs.findbugs.annotations.NonNull; -import io.helidon.config.Config; -import io.helidon.config.MapConfigSource; -import io.helidon.config.spi.ConfigSource; import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; @@ -53,19 +51,21 @@ public class BlockAsDirReaderTest { private final System.Logger LOGGER = System.getLogger(getClass().getName()); private static final String TEMP_DIR = "block-node-unit-test-dir"; - private static final String JUNIT = "my-junit-test"; private Path testPath; - private Config testConfig; + + private BlockNodeContext blockNodeContext; + private PersistenceStorageConfig config; @BeforeEach public void setUp() throws IOException { testPath = Files.createTempDirectory(TEMP_DIR); LOGGER.log(System.Logger.Level.INFO, "Created temp directory: " + testPath.toString()); - final Map testProperties = Map.of(JUNIT, testPath.toString()); - final ConfigSource testConfigSource = MapConfigSource.builder().map(testProperties).build(); - testConfig = Config.builder(testConfigSource).build(); + blockNodeContext = + TestConfigUtil.getTestBlockNodeContext( + Map.of("persistence.storage.rootPath", testPath.toString())); + config = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); } @AfterEach @@ -79,8 +79,7 @@ public void tearDown() { @Test public void testReadBlockDoesNotExist() throws IOException { - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(JUNIT, testConfig).build(); + final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); final Optional blockOpt = blockReader.read(10000); assertTrue(blockOpt.isEmpty()); } @@ -89,19 +88,17 @@ public void testReadBlockDoesNotExist() throws IOException { public void testReadPermsRepairSucceeded() throws IOException { final List blockItems = PersistTestUtils.generateBlockItems(1); - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockWriter blockWriter = - BlockAsDirWriterBuilder.newBuilder(JUNIT, testConfig, blockNodeContext).build(); + BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); for (BlockItem blockItem : blockItems) { blockWriter.write(blockItem); } // Make the block unreadable - removeBlockReadPerms(1, testConfig); + removeBlockReadPerms(1, config); // The default BlockReader will attempt to repair the permissions and should succeed - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(JUNIT, testConfig).build(); + final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); final Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); assertEquals(10, blockOpt.get().getBlockItemsList().size()); @@ -111,20 +108,19 @@ public void testReadPermsRepairSucceeded() throws IOException { public void testRemoveBlockReadPermsRepairFailed() throws IOException { final List blockItems = PersistTestUtils.generateBlockItems(1); - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockWriter blockWriter = - BlockAsDirWriterBuilder.newBuilder(JUNIT, testConfig, blockNodeContext).build(); + BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); for (BlockItem blockItem : blockItems) { blockWriter.write(blockItem); } // Make the block unreadable - removeBlockReadPerms(1, testConfig); + removeBlockReadPerms(1, config); // For this test, build the Reader with ineffective repair permissions to // simulate a failed repair (root changed the perms, etc.) final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(JUNIT, testConfig) + BlockAsDirReaderBuilder.newBuilder(config) .filePerms(TestUtils.getNoPerms()) .build(); final Optional blockOpt = blockReader.read(1); @@ -135,31 +131,28 @@ public void testRemoveBlockReadPermsRepairFailed() throws IOException { public void testRemoveBlockItemReadPerms() throws IOException { final List blockItems = PersistTestUtils.generateBlockItems(1); - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockWriter blockWriter = - BlockAsDirWriterBuilder.newBuilder(JUNIT, testConfig, blockNodeContext).build(); + BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); for (BlockItem blockItem : blockItems) { blockWriter.write(blockItem); } - removeBlockItemReadPerms(1, 1, testConfig); + removeBlockItemReadPerms(1, 1, config); - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(JUNIT, testConfig).build(); + final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); assertThrows(IOException.class, () -> blockReader.read(1)); } @Test public void testPathIsNotDirectory() throws IOException { final List blockItems = PersistTestUtils.generateBlockItems(1); - final Path blockNodeRootPath = Path.of(testConfig.get(JUNIT).asString().get()); + final Path blockNodeRootPath = Path.of(config.rootPath()); // Write a file named "1" where a directory should be writeFileToPath(blockNodeRootPath.resolve(Path.of("1")), blockItems.getFirst()); // Should return empty because the path is not a directory - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(JUNIT, testConfig).build(); + final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); final Optional blockOpt = blockReader.read(1); assertTrue(blockOpt.isEmpty()); } @@ -169,19 +162,18 @@ public void testRepairReadPermsFails() throws IOException { final List blockItems = PersistTestUtils.generateBlockItems(1); - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockWriter blockWriter = - BlockAsDirWriterBuilder.newBuilder(JUNIT, testConfig, blockNodeContext).build(); + BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); for (final BlockItem blockItem : blockItems) { blockWriter.write(blockItem); } - removeBlockReadPerms(1, testConfig); + removeBlockReadPerms(1, config); // Use a spy on a subclass of the BlockAsDirReader to proxy calls // to the actual methods but to also throw an IOException when // the setPerm method is called. - final TestBlockAsDirReader blockReader = spy(new TestBlockAsDirReader(JUNIT, testConfig)); + final TestBlockAsDirReader blockReader = spy(new TestBlockAsDirReader(config)); doThrow(IOException.class).when(blockReader).setPerm(any(), any()); final Optional blockOpt = blockReader.read(1); @@ -192,12 +184,12 @@ public void testRepairReadPermsFails() throws IOException { public void testBlockNodePathReadFails() throws IOException { // Remove read perm on the root path - removePathReadPerms(Path.of(testConfig.get(JUNIT).asString().get())); + removePathReadPerms(Path.of(config.rootPath())); // Use a spy on a subclass of the BlockAsDirReader to proxy calls // to the actual methods but to also throw an IOException when // the setPerm method is called. - final TestBlockAsDirReader blockReader = spy(new TestBlockAsDirReader(JUNIT, testConfig)); + final TestBlockAsDirReader blockReader = spy(new TestBlockAsDirReader(config)); doThrow(IOException.class).when(blockReader).setPerm(any(), any()); final Optional blockOpt = blockReader.read(1); @@ -212,9 +204,9 @@ private void writeFileToPath(final Path path, final BlockItem blockItem) throws } } - public static void removeBlockReadPerms(int blockNumber, final Config config) + public static void removeBlockReadPerms(int blockNumber, final PersistenceStorageConfig config) throws IOException { - final Path blockNodeRootPath = Path.of(config.get(JUNIT).asString().get()); + final Path blockNodeRootPath = Path.of(config.rootPath()); final Path blockPath = blockNodeRootPath.resolve(String.valueOf(blockNumber)); removePathReadPerms(blockPath); } @@ -223,9 +215,9 @@ static void removePathReadPerms(final Path path) throws IOException { Files.setPosixFilePermissions(path, TestUtils.getNoRead().value()); } - private void removeBlockItemReadPerms(int blockNumber, int blockItem, Config config) - throws IOException { - final Path blockNodeRootPath = Path.of(config.get(JUNIT).asString().get()); + private void removeBlockItemReadPerms( + int blockNumber, int blockItem, PersistenceStorageConfig config) throws IOException { + final Path blockNodeRootPath = Path.of(config.rootPath()); final Path blockPath = blockNodeRootPath.resolve(String.valueOf(blockNumber)); final Path blockItemPath = blockPath.resolve(blockItem + BLOCK_FILE_EXTENSION); Files.setPosixFilePermissions(blockItemPath, TestUtils.getNoRead().value()); @@ -234,8 +226,8 @@ private void removeBlockItemReadPerms(int blockNumber, int blockItem, Config con // TestBlockAsDirReader overrides the setPerm() method to allow a test spy to simulate an // IOException while allowing the real setPerm() method to remain protected. private static final class TestBlockAsDirReader extends BlockAsDirReader { - public TestBlockAsDirReader(String key, Config config) { - super(key, config, Util.defaultPerms); + public TestBlockAsDirReader(PersistenceStorageConfig config) { + super(config, FileUtils.defaultPerms); } @Override diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java index 32767b63a..ec9ab0b8e 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java @@ -21,17 +21,15 @@ import com.hedera.block.protos.BlockStreamService.Block; import com.hedera.block.protos.BlockStreamService.BlockItem; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.config.BlockNodeContextFactory; -import com.hedera.block.server.persistence.storage.Util; +import com.hedera.block.server.persistence.storage.FileUtils; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.util.PersistTestUtils; +import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; -import io.helidon.config.Config; -import io.helidon.config.MapConfigSource; -import io.helidon.config.spi.ConfigSource; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -46,19 +44,20 @@ public class BlockAsDirRemoverTest { private final System.Logger LOGGER = System.getLogger(getClass().getName()); private static final String TEMP_DIR = "block-node-unit-test-dir"; - private static final String JUNIT = "my-junit-test"; private Path testPath; - private Config testConfig; + private BlockNodeContext blockNodeContext; + private PersistenceStorageConfig testConfig; @BeforeEach public void setUp() throws IOException { testPath = Files.createTempDirectory(TEMP_DIR); LOGGER.log(System.Logger.Level.INFO, "Created temp directory: " + testPath.toString()); - Map testProperties = Map.of(JUNIT, testPath.toString()); - ConfigSource testConfigSource = MapConfigSource.builder().map(testProperties).build(); - testConfig = Config.builder(testConfigSource).build(); + blockNodeContext = + TestConfigUtil.getTestBlockNodeContext( + Map.of("persistence.storage.rootPath", testPath.toString())); + testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); } @Test @@ -67,20 +66,19 @@ public void testRemoveNonExistentBlock() throws IOException { // Write a block final List blockItems = PersistTestUtils.generateBlockItems(1); - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockWriter blockWriter = - BlockAsDirWriterBuilder.newBuilder(JUNIT, testConfig, blockNodeContext).build(); + BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); for (final BlockItem blockItem : blockItems) { blockWriter.write(blockItem); } // Remove a block that does not exist - final BlockRemover blockRemover = new BlockAsDirRemover(testPath, Util.defaultPerms); + final BlockRemover blockRemover = new BlockAsDirRemover(testPath, FileUtils.defaultPerms); blockRemover.remove(2); // Verify the block was not removed final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(JUNIT, testConfig).build(); + BlockAsDirReaderBuilder.newBuilder(testConfig).build(); Optional blockOpt = blockReader.read(1); assert (blockOpt.isPresent()); assertEquals( @@ -100,9 +98,8 @@ public void testRemoveBlockWithPermException() throws IOException { // Write a block final List blockItems = PersistTestUtils.generateBlockItems(1); - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockWriter blockWriter = - BlockAsDirWriterBuilder.newBuilder(JUNIT, testConfig, blockNodeContext).build(); + BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); for (final BlockItem blockItem : blockItems) { blockWriter.write(blockItem); } @@ -113,14 +110,14 @@ public void testRemoveBlockWithPermException() throws IOException { // Verify the block was not removed final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(JUNIT, testConfig).build(); + BlockAsDirReaderBuilder.newBuilder(testConfig).build(); Optional blockOpt = blockReader.read(1); assert (blockOpt.isPresent()); assertEquals( blockItems.getFirst().getHeader(), blockOpt.get().getBlockItems(0).getHeader()); // Now remove the block - blockRemover = new BlockAsDirRemover(testPath, Util.defaultPerms); + blockRemover = new BlockAsDirRemover(testPath, FileUtils.defaultPerms); blockRemover.remove(1); // Verify the block is removed diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java index 31c4aea4d..4d01af248 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java @@ -23,18 +23,16 @@ import static org.mockito.Mockito.*; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.config.BlockNodeContextFactory; -import com.hedera.block.server.persistence.storage.Util; +import com.hedera.block.server.persistence.storage.FileUtils; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.persistence.storage.remove.BlockAsDirRemover; import com.hedera.block.server.persistence.storage.remove.BlockRemover; import com.hedera.block.server.util.PersistTestUtils; +import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; import edu.umd.cs.findbugs.annotations.NonNull; -import io.helidon.config.Config; -import io.helidon.config.MapConfigSource; -import io.helidon.config.spi.ConfigSource; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -53,19 +51,21 @@ public class BlockAsDirWriterTest { private final System.Logger LOGGER = System.getLogger(getClass().getName()); private static final String TEMP_DIR = "block-node-unit-test-dir"; - private static final String JUNIT = "my-junit-test"; + private static final String PERSISTENCE_STORAGE_ROOT_PATH_KEY = "persistence.storage.rootPath"; private Path testPath; - private Config testConfig; + private BlockNodeContext blockNodeContext; + private PersistenceStorageConfig testConfig; @BeforeEach public void setUp() throws IOException { testPath = Files.createTempDirectory(TEMP_DIR); LOGGER.log(System.Logger.Level.INFO, "Created temp directory: " + testPath.toString()); - Map testProperties = Map.of(JUNIT, testPath.toString()); - ConfigSource testConfigSource = MapConfigSource.builder().map(testProperties).build(); - testConfig = Config.builder(testConfigSource).build(); + blockNodeContext = + TestConfigUtil.getTestBlockNodeContext( + Map.of(PERSISTENCE_STORAGE_ROOT_PATH_KEY, testPath.toString())); + testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); } @AfterEach @@ -77,36 +77,20 @@ public void tearDown() { } } - @Test - public void testConstructorWithInvalidPath() throws IOException { - final Map testProperties = Map.of(JUNIT, "invalid-path"); - final ConfigSource testConfigSource = MapConfigSource.builder().map(testProperties).build(); - final Config testConfig = Config.builder(testConfigSource).build(); - - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); - assertThrows( - IllegalArgumentException.class, - () -> - BlockAsDirWriterBuilder.newBuilder(JUNIT, testConfig, blockNodeContext) - .build()); - } - @Test public void testWriterAndReaderHappyPath() throws IOException { // Write a block final List blockItems = PersistTestUtils.generateBlockItems(1); - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockWriter blockWriter = - BlockAsDirWriterBuilder.newBuilder(JUNIT, testConfig, blockNodeContext).build(); + BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); for (BlockItem blockItem : blockItems) { blockWriter.write(blockItem); } // Confirm the block - BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(JUNIT, testConfig).build(); + BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(testConfig).build(); Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); @@ -135,9 +119,8 @@ public void testRemoveBlockWritePerms() throws IOException { final List blockItems = PersistTestUtils.generateBlockItems(1); - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockWriter blockWriter = - BlockAsDirWriterBuilder.newBuilder(JUNIT, testConfig, blockNodeContext).build(); + BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); // Change the permissions on the block node root directory removeRootWritePerms(testConfig); @@ -147,8 +130,7 @@ public void testRemoveBlockWritePerms() throws IOException { blockWriter.write(blockItems.getFirst()); // Confirm we're able to read 1 block item - BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(JUNIT, testConfig).build(); + BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(testConfig).build(); Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); assertEquals(1, blockOpt.get().getBlockItemsList().size()); @@ -181,14 +163,12 @@ public void testUnrecoverableIOExceptionOnWrite() throws IOException { final List blockItems = PersistTestUtils.generateBlockItems(1); final BlockRemover blockRemover = - new BlockAsDirRemover( - Path.of(testConfig.get(JUNIT).asString().get()), Util.defaultPerms); + new BlockAsDirRemover(Path.of(testConfig.rootPath()), FileUtils.defaultPerms); // Use a spy to simulate an IOException when the first block item is written - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockWriter blockWriter = spy( - BlockAsDirWriterBuilder.newBuilder(JUNIT, testConfig, blockNodeContext) + BlockAsDirWriterBuilder.newBuilder(blockNodeContext) .blockRemover(blockRemover) .build()); doThrow(IOException.class).when(blockWriter).write(blockItems.getFirst()); @@ -199,9 +179,8 @@ public void testUnrecoverableIOExceptionOnWrite() throws IOException { public void testRemoveRootDirReadPerm() throws IOException { final List blockItems = PersistTestUtils.generateBlockItems(1); - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final BlockWriter blockWriter = - BlockAsDirWriterBuilder.newBuilder(JUNIT, testConfig, blockNodeContext).build(); + BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); // Write the first block item to create the block // directory @@ -218,8 +197,7 @@ public void testRemoveRootDirReadPerm() throws IOException { blockWriter.write(blockItems.get(i)); } - BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(JUNIT, testConfig).build(); + BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(testConfig).build(); Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); assertEquals(10, blockOpt.get().getBlockItemsList().size()); @@ -229,22 +207,16 @@ public void testRemoveRootDirReadPerm() throws IOException { public void testPartialBlockRemoval() throws IOException { final List blockItems = PersistTestUtils.generateBlockItems(3); final BlockRemover blockRemover = - new BlockAsDirRemover( - Path.of(testConfig.get(JUNIT).asString().get()), Util.defaultPerms); + new BlockAsDirRemover(Path.of(testConfig.rootPath()), FileUtils.defaultPerms); // Use a spy of TestBlockAsDirWriter to proxy block items to the real writer // for the first 22 block items. Then simulate an IOException on the 23rd block item // thrown from a protected write method in the real class. This should trigger the // blockRemover instance to remove the partially written block. - final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); final TestBlockAsDirWriter blockWriter = spy( new TestBlockAsDirWriter( - JUNIT, - testConfig, - blockRemover, - Util.defaultPerms, - blockNodeContext)); + blockRemover, FileUtils.defaultPerms, blockNodeContext)); for (int i = 0; i < 23; i++) { // Prepare the block writer to call the actual write method @@ -266,7 +238,7 @@ public void testPartialBlockRemoval() throws IOException { // Verify the partially written block was removed final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(JUNIT, testConfig).build(); + BlockAsDirReaderBuilder.newBuilder(testConfig).build(); Optional blockOpt = blockReader.read(3); assertTrue(blockOpt.isEmpty()); @@ -282,19 +254,19 @@ public void testPartialBlockRemoval() throws IOException { assertEquals(2, blockOpt.get().getBlockItems(0).getHeader().getBlockNumber()); } - private void removeRootWritePerms(final Config config) throws IOException { - final Path blockNodeRootPath = Path.of(config.get(JUNIT).asString().get()); + private void removeRootWritePerms(final PersistenceStorageConfig config) throws IOException { + final Path blockNodeRootPath = Path.of(config.rootPath()); Files.setPosixFilePermissions(blockNodeRootPath, TestUtils.getNoWrite().value()); } - private void removeRootReadPerms(final Config config) throws IOException { - final Path blockNodeRootPath = Path.of(config.get(JUNIT).asString().get()); + private void removeRootReadPerms(final PersistenceStorageConfig config) throws IOException { + final Path blockNodeRootPath = Path.of(config.rootPath()); Files.setPosixFilePermissions(blockNodeRootPath, TestUtils.getNoRead().value()); } - private void removeBlockAllPerms(final int blockNumber, final Config config) + private void removeBlockAllPerms(final int blockNumber, final PersistenceStorageConfig config) throws IOException { - final Path blockNodeRootPath = Path.of(config.get(JUNIT).asString().get()); + final Path blockNodeRootPath = Path.of(config.rootPath()); final Path blockPath = blockNodeRootPath.resolve(String.valueOf(blockNumber)); Files.setPosixFilePermissions(blockPath, TestUtils.getNoPerms().value()); } @@ -303,13 +275,11 @@ private void removeBlockAllPerms(final int blockNumber, final Config config) // IOException while allowing the real write() method to remain protected. private static final class TestBlockAsDirWriter extends BlockAsDirWriter { public TestBlockAsDirWriter( - final String key, - final Config config, final BlockRemover blockRemover, final FileAttribute> filePerms, final BlockNodeContext blockNodeContext) throws IOException { - super(key, config, blockRemover, filePerms, blockNodeContext); + super(blockRemover, filePerms, blockNodeContext); } @Override diff --git a/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java b/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java index d21f9393c..b8fb861f7 100644 --- a/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java +++ b/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java @@ -20,6 +20,7 @@ import static com.hedera.block.protos.BlockStreamService.PublishStreamResponse.ItemAcknowledgement; import static com.hedera.block.server.producer.Util.getFakeHash; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; +import static com.hedera.block.server.util.TestConfigUtil.CONSUMER_TIMEOUT_THRESHOLD_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.*; @@ -30,16 +31,19 @@ import com.hedera.block.server.ServiceStatusImpl; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.config.BlockNodeContextFactory; +import com.hedera.block.server.consumer.ConsumerConfig; import com.hedera.block.server.consumer.ConsumerStreamResponseObserver; import com.hedera.block.server.data.ObjectEvent; import com.hedera.block.server.mediator.LiveStreamMediatorBuilder; import com.hedera.block.server.mediator.StreamMediator; import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.util.TestConfigUtil; import io.grpc.stub.StreamObserver; import java.io.IOException; import java.security.NoSuchAlgorithmException; import java.time.InstantSource; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -108,21 +112,27 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) // Mock a clock with 2 different return values in response to anticipated // millis() calls. Here the second call will always be inside the timeout window. - long TIMEOUT_THRESHOLD_MILLIS = 100L; + final BlockNodeContext testContext = + TestConfigUtil.getTestBlockNodeContext( + Map.of(CONSUMER_TIMEOUT_THRESHOLD_KEY, "100")); + final ConsumerConfig consumerConfig = + testContext.configuration().getConfigData(ConsumerConfig.class); + long TEST_TIME = 1_719_427_664_950L; - when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS); + when(testClock.millis()) + .thenReturn(TEST_TIME, TEST_TIME + consumerConfig.timeoutThresholdMillis()); final var concreteObserver1 = new ConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, testClock, streamMediator, streamObserver1); + testContext, testClock, streamMediator, streamObserver1); final var concreteObserver2 = new ConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, testClock, streamMediator, streamObserver2); + testContext, testClock, streamMediator, streamObserver2); final var concreteObserver3 = new ConsumerStreamResponseObserver( - TIMEOUT_THRESHOLD_MILLIS, testClock, streamMediator, streamObserver3); + testContext, testClock, streamMediator, streamObserver3); // Set up the subscribers streamMediator.subscribe(concreteObserver1); diff --git a/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java b/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java new file mode 100644 index 000000000..30f3e1219 --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.util; + +import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.config.BlockNodeContextFactory; +import com.hedera.block.server.config.TestConfigBuilder; +import com.hedera.block.server.consumer.ConsumerConfig; +import com.swirlds.config.api.Configuration; +import com.swirlds.config.extensions.sources.ClasspathFileConfigSource; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Map; + +public class TestConfigUtil { + + public static final String CONSUMER_TIMEOUT_THRESHOLD_KEY = "consumer.timeoutThresholdMillis"; + + private static final String TEST_APP_PROPERTIES_FILE = "app.properties"; + + private TestConfigUtil() {} + + @NonNull + public static BlockNodeContext getTestBlockNodeContext( + @NonNull Map customProperties) throws IOException { + + // we still use the BlockNodeContextFactory to create the BlockNodeContext temporally, + // but we will replace the configuration with a test configuration + // sooner we will need to create a metrics mock, and never use the BlockNodeContextFactory. + BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); + + // create test configuration + TestConfigBuilder testConfigBuilder = + new TestConfigBuilder(true) + .withSource( + new ClasspathFileConfigSource(Path.of(TEST_APP_PROPERTIES_FILE))); + + for (Map.Entry entry : customProperties.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + testConfigBuilder = testConfigBuilder.withValue(key, value); + } + + testConfigBuilder = testConfigBuilder.withConfigDataType(ConsumerConfig.class); + + Configuration testConfiguration = testConfigBuilder.getOrCreateConfig(); + + return new BlockNodeContext( + blockNodeContext.metrics(), blockNodeContext.metricsService(), testConfiguration); + } + + public static BlockNodeContext getTestBlockNodeContext() throws IOException { + return getTestBlockNodeContext(Collections.emptyMap()); + } +} diff --git a/server/src/test/resources/app.properties b/server/src/test/resources/app.properties new file mode 100644 index 000000000..ff6694edb --- /dev/null +++ b/server/src/test/resources/app.properties @@ -0,0 +1,2 @@ +# Test Properties File +prometheus.endpointEnabled=false