Merge pull request kmgowda#144 from kmgowda/kmg-redpanda-1

Redpanda Performance Benchmarking.

Signed-off-by: Keshava Munegowda <[email protected]>

build.gradle

@@ -253,6 +253,7 @@ dependencies {
 	compile project(':driver-postgresql')
 	compile project(':driver-derby')
 	compile project(':driver-mssql')
+	compile project(':driver-redpanda')
 }
 
 //create a single Jar with all dependencies

dockers/sbk

@@ -37,6 +37,7 @@ COPY --chown=root:root driver-bookkeeper ${APP_HOME}/driver-bookkeeper
 COPY --chown=root:root driver-rabbitmq ${APP_HOME}/driver-rabbitmq
 COPY --chown=root:root driver-rocketmq ${APP_HOME}/driver-rocketmq
 COPY --chown=root:root driver-pulsar ${APP_HOME}/driver-pulsar
+COPY --chown=root:root driver-redpanda ${APP_HOME}/driver-redpanda
 COPY --chown=root:root driver-kafka ${APP_HOME}/driver-kafka
 COPY --chown=root:root driver-pravega ${APP_HOME}/driver-pravega
 COPY --chown=root:root driver-nats ${APP_HOME}/driver-nats

dockers/sbk-redpanda

@@ -0,0 +1,54 @@
+##
+# Copyright (c) KMG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+##
+
+# Building Container
+FROM gradle:6.0.1-jdk11 as GradleBuilder
+USER root
+
+COPY ca-certificates/* /usr/local/share/ca-certificates/
+RUN update-ca-certificates
+
+ENV APP_NAME=sbk
+ENV SBK_PROJECT=driver-redpanda
+ENV APP_HOME=/opt/${APP_NAME}
+
+WORKDIR /opt/sbk
+
+COPY --chown=root:root gradle ${APP_HOME}/gradle
+COPY --chown=root:root build.gradle ${APP_HOME}/build.gradle
+COPY --chown=root:root gradle.properties ${APP_HOME}/gradle.properties
+COPY --chown=root:root settings.gradle ${APP_HOME}/settings.gradle
+COPY --chown=root:root gradlew ${APP_HOME}/gradlew
+COPY --chown=root:root checkstyle ${APP_HOME}/checkstyle
+COPY --chown=root:root sbk-api ${APP_HOME}/sbk-api
+
+# Copy the SBK storage drivers
+COPY --chown=root:root driver-kafka ${APP_HOME}/driver-kafka
+COPY --chown=root:root driver-redpanda ${APP_HOME}/driver-redpanda
+
+
+
+ENV GRADLE_USER_HOME=/opt/SBK
+RUN gradle  :${SBK_PROJECT}:build --no-daemon --info --stacktrace
+
+# Runtime Container
+FROM openjdk:11-jre
+ENV APP_NAME=sbk
+ENV SBK_PROJECT=driver-redpanda
+ENV APP_HOME=/opt/${APP_NAME}
+
+COPY --from=GradleBuilder ${APP_HOME}/${SBK_PROJECT}/build/distributions/${APP_NAME}-*.tar /opt/${APP_NAME}.tar
+
+RUN tar -xvf /opt/${APP_NAME}.tar -C /opt/.
+RUN mv /opt/${APP_NAME}-* /opt/${APP_NAME}
+
+EXPOSE 9718
+
+ENTRYPOINT ["/opt/sbk/bin/sbk-redpanda"]

driver-kafka/src/main/java/io/sbk/Kafka/Kafka.java

@@ -39,12 +39,12 @@ public class Kafka implements Storage<byte[]> {
     private Properties consumerConfig;
     private KafkaTopicHandler topicHandler;
 
-    @Override
-    public void addArgs(final Parameters params) throws IllegalArgumentException {
+
+    public void addArgs(final Parameters params, String configFile) throws IllegalArgumentException {
         final ObjectMapper mapper = new ObjectMapper(new JavaPropsFactory())
                 .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
         try {
-            config = mapper.readValue(Objects.requireNonNull(Kafka.class.getClassLoader().getResourceAsStream(CONFIGFILE)),
+            config = mapper.readValue(Objects.requireNonNull(Kafka.class.getClassLoader().getResourceAsStream(configFile)),
                     KafkaConfig.class);
         } catch (Exception ex) {
             ex.printStackTrace();
@@ -59,6 +59,11 @@ public void addArgs(final Parameters params) throws IllegalArgumentException {
                 "Create (recreate) the topic, valid only for writers, default: " + config.create);
     }
 
+    @Override
+    public void addArgs(final Parameters params) throws IllegalArgumentException {
+        addArgs(params, CONFIGFILE);
+    }
+
     @Override
     public void parseArgs(final Parameters params) throws IllegalArgumentException {
         config.topicName =  params.getOptionValue("topic", config.topicName);
@@ -79,7 +84,7 @@ private Properties createProducerConfig(Parameters params) {
         props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
         props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
         // Enabling the producer IDEMPOTENCE is must, to compare between Kafka and Pravega.
-        props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true);
+        props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, config.idempotence);
         return props;
     }
 

driver-kafka/src/main/java/io/sbk/Kafka/KafkaConfig.java

@@ -16,4 +16,5 @@ public class KafkaConfig {
     public short replica;
     public short sync;
     public boolean create;
+    public boolean idempotence;
 }

driver-kafka/src/main/resources/kafka.properties

@@ -24,3 +24,6 @@ sync=1
 
 #Create the topic
 create=false
+
+#Idempotence
+idempotence=true