Update to use the Binary Distribution for Kafka 0.8.2.1

README.md

@@ -1,8 +1,81 @@
 kafka-deb-packaging
 ===================
 
-Simple debian packaging for Apache Kafka
+Simple debian packaging for Apache Kafka and Zookeeper, great for Ubuntu 14.04LTS
+
+### Changelog
+* 2015-Mar-18 : Updated for latest Kafka 0.8.2.1 and use sbt in system path.
+* 2015-Mar-30 : Final Test, ready for git pull, packaged Zookeeper init.d!
+
+# Usage
+
+```sh
+$ ./dist_kakfa.sh
+```
+
+# Sample Output
+
+```sh
+~/git/kafka-deb-packaging$ ./dist_kafka.sh
+
+--2015-03-30 17:39:01--  http://mirror.sdunix.com/apache/kafka/0.8.2.1/kafka_2.10-0.8.2.1.tgz
+Resolving mirror.sdunix.com (mirror.sdunix.com)... 74.206.97.82
+Connecting to mirror.sdunix.com (mirror.sdunix.com)|74.206.97.82|:80... connected.
+HTTP request sent, awaiting response... 200 OK
+Length: 16162559 (15M) [application/x-gzip]
+Saving to: ‘kafka_2.10-0.8.2.1.tgz’
+
+100%[========================>] 16,162,559   340KB/s   in 44s
+
+2015-03-30 17:39:45 (361 KB/s) - ‘kafka_2.10-0.8.2.1.tgz’ saved [16162559/16162559]
+
+~/git/kafka-deb-packaging/tmp ~/git/kafka-deb-packaging
+Created package {:path=>"kafka_0.8.2.1-5_all.deb"}
+~/git/kafka-deb-packaging
+```
+
+# Installation
+
+```sh
+$ dpkg -i kafka_0.8.2.1-10_all.deb
+```
+
+or if you have your own repo:
+
+```sh
+$ ~/gpg-agent-headless.sh
+$ reprepro -b /var/repositories/ includedeb trusty $@
+$ apt-get install kafka
+```
+Note: Installs and runs as user 'app'. Easy to change for your needs.
+
+## Post install
+
+```sh
+$ sudo update-rc.d zookeeper defaults 20
+
+$ sudo update-rc.d kafka defaults 25
+
+ Adding system startup for /etc/init.d/kafka ...
+   /etc/rc0.d/K25kafka -> ../init.d/kafka
+   /etc/rc1.d/K25kafka -> ../init.d/kafka
+   /etc/rc6.d/K25kafka -> ../init.d/kafka
+   /etc/rc2.d/S25kafka -> ../init.d/kafka
+   /etc/rc3.d/S25kafka -> ../init.d/kafka
+   /etc/rc4.d/S25kafka -> ../init.d/kafka
+   /etc/rc5.d/S25kafka -> ../init.d/kafka
+```
+
+## System Check
+
+```sh
+$ /usr/lib/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic installtest --partitions 1 --replication-factor 1
+
+Created topic "installtest".
+```
+
+## Tested Platforms
+
+* Ubuntu 14.04LTS, Amazon EC2
 
-## Changelog
-2015-Mar-18 : Updated for latest Kafka 0.8.2.1 and use sbt in system path.
 

build_kafka.sh

@@ -1,45 +1,67 @@
 #!/bin/bash
 # 2015-Mar-18 Updated to latest Kafka stable: 0.8.2.1
+# 2015-Mar-20 Added the init.d script and changed to use binary download of scala 2.10, Kafka 0.8.2.1
+# 2015-Dec-07 Upgraded to Kafka 0.8.2.2 distro
+
 set -e
 set -u
 name=kafka
-version=0.8.2.1
+version=0.8.2.2
+scala_version=2.10
 description="Apache Kafka is a distributed publish-subscribe messaging system."
 url="https://kafka.apache.org/"
 arch="all"
 section="misc"
 license="Apache Software License 2.0"
-package_version="-1"
+package_version="-10"
+bin_package="kafka_${scala_version}-${version}.tgz"
 src_package="kafka-${version}-src.tgz"
-download_url="http://mirror.sdunix.com/apache/kafka/${version}/${src_package}"
+src_download_url="http://mirror.sdunix.com/apache/kafka/${version}/${src_package}"
 origdir="$(pwd)"
 
 #_ MAIN _#
 rm -rf ${name}*.deb
 if [[ ! -f "${src_package}" ]]; then
-  wget ${download_url}
+  wget -c ${src_download_url}
 fi
 mkdir -p tmp && pushd tmp
 rm -rf kafka
 mkdir -p kafka
 cd kafka
 mkdir -p build/usr/lib/kafka
+mkdir -p build/usr/lib/kafka/logs
 mkdir -p build/etc/default
 mkdir -p build/etc/init
+mkdir -p build/etc/init.d
 mkdir -p build/etc/kafka
 mkdir -p build/var/log/kafka
+#mkdir -p build/var/run/kafka
 
 cp ${origdir}/kafka-broker.default build/etc/default/kafka-broker
 cp ${origdir}/kafka-broker.upstart.conf build/etc/init/kafka-broker.conf
+cp ${origdir}/kafka-broker.init.d build/etc/init.d/kafka
+cp ${origdir}/zookeeper.init.d build/etc/init.d/zookeeper
 
+# This code line uses the src packages
 tar zxf ${origdir}/${src_package}
 cd kafka-${version}-src
-sbt update
-sbt package
-mv config/log4j.properties config/server.properties ../build/etc/kafka
+# updated to gradle build as per Kafka README
+gradle
+./gradlew releaseTarGz
+cp ./core/build/distributions/${bin_package} ${origdir}/${bin_package}
+cd ..
+
+# Now resume regular binary exploding
+# Updated to use the Binary package
+rm -rf kafka_${scala_version}-${version}
+tar zxf ${origdir}/${bin_package}
+cd kafka_${scala_version}-${version}
+cp config/server.properties ../build/etc/kafka
+cp config/zookeeper.properties ../build/etc/kakfa/zookeeper.properties
 mv * ../build/usr/lib/kafka
 cd ../build
 
+
 fpm -t deb \
     -n ${name} \
     -v ${version}${package_version} \

dist_kafka.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+# 2015-Mar-18 Updated to latest Kafka stable: 0.8.2.1
+# 2015-Mar-20 Added the init.d script and changed to use binary download of scala 2.10, Kafka 0.8.2.1
+# 2015-Dec-07 Upgraded to Kafka 0.8.2.2 distro
+
+set -e
+set -u
+app_user=app
+name=kafka
+version=0.8.2.2
+scala_version=2.10
+package_version="-10"
+description="Apache Kafka is a distributed publish-subscribe messaging system."
+url="https://kafka.apache.org/"
+arch="all"
+section="misc"
+license="Apache Software License 2.0"
+bin_package="kafka_${scala_version}-${version}.tgz"
+bin_download_url="http://mirror.sdunix.com/apache/kafka/${version}/${bin_package}"
+origdir="$(pwd)"
+
+#_ MAIN _#
+rm -rf ${name}*.deb
+if [[ ! -f "${bin_package}" ]]; then
+  wget -c ${bin_download_url}
+fi
+mkdir -p tmp && pushd tmp
+rm -rf kafka
+mkdir -p kafka
+cd kafka
+mkdir -p build/usr/lib/kafka
+mkdir -p build/usr/lib/kafka/logs
+mkdir -p build/etc/default
+mkdir -p build/etc/init
+mkdir -p build/etc/init.d
+mkdir -p build/etc/kafka
+mkdir -p build/var/log/kafka
+#mkdir -p build/var/run/kafka
+
+cp ${origdir}/kafka-broker.default build/etc/default/kafka-broker
+cp ${origdir}/kafka-broker.upstart.conf build/etc/init/kafka-broker.conf
+cp ${origdir}/kafka-broker.init.d build/etc/init.d/kafka
+cp ${origdir}/zookeeper.init.d build/etc/init.d/zookeeper
+
+# Updated to use the Binary package
+rm -rf kafka_${scala_version}-${version}
+tar zxf ${origdir}/${bin_package}
+cd kafka_${scala_version}-${version}
+# mv config/log4j.properties config/server.properties ../build/etc/kafka
+cp config/server.properties ../build/etc/kafka
+cp config/zookeeper.properties ../build/etc/kafka/zookeeper.properties
+mv * ../build/usr/lib/kafka
+cd ../build
+
+fpm -t deb \
+    --deb-user ${app_user} \
+    -n ${name} \
+    -v ${version}${package_version} \
+    --description "${description}" \
+    --url="{$url}" \
+    -a ${arch} \
+    --category ${section} \
+    --vendor "" \
+    --license "${license}" \
+    -m "${USER}@localhost" \
+    --prefix=/ \
+    -s dir \
+    -- .
+mv kafka*.deb ${origdir}
+popd

kafka-broker.init.d

@@ -0,0 +1,137 @@
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides:          kafka
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Starts Kafka server
+# Description:       Starts Kafka server
+### END INIT INFO
+
+# Authors:            Josh Skidmore <[email protected]>
+#                     Brandon Brown <[email protected]>
+#                     Brandon Froehlich <[email protected]>
+#                     Taylor Brockman <[email protected]>
+
+# Revision History:   2015-02-12 - Initial commit
+# 	         2015-03-06 - Porting to nop-bigress
+#                2015-03-20 - Contribution to tomdz/kafka-deb-packaging
+#                2015-03-30 - Changed PID directory after running in AWS prod
+
+# Variables
+PATH=$PATH
+DESC="Apache Kafka Broker"
+NAME=kafka
+DAEMON=/usr/lib/kafka/bin/kafka-server-start.sh
+PIDFILE=/var/run/kafka-server.pid
+USER=app
+CONFIG=/etc/kafka/server.properties
+LOG=/var/log/kafka/kafka-server.log
+
+
+# Load the VERBOSE setting and other rcS variables
+. /lib/init/vars.sh
+
+# Define LSB log_* functions.
+# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
+# and status_of_proc is working.
+. /lib/lsb/init-functions
+
+
+#
+# Function that starts the daemon/service
+#
+do_start()
+{
+  # Return
+  #   0 if daemon has been started
+  #   1 if daemon was already running
+  #   2 if daemon could not be started
+
+  start-stop-daemon --chuid $USER --background --start --quiet --make-pidfile --pidfile $PIDFILE --no-close --startas $DAEMON -- $CONFIG >> $LOG 2>&1 || return 2
+}
+
+
+#
+# Function that stops the daemon/service
+#
+do_stop()
+{
+  #   0 if daemon has been stopped
+  #   1 if daemon was already stopped
+  #   2 if daemon could not be stopped
+  #   other if a failure occurred
+
+  start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE
+  RETVAL="$?"
+
+  # Many daemons don't delete their pidfiles when they exit.
+  rm -f $PIDFILE
+  return "$RETVAL"
+}
+
+
+case "$1" in
+  start)
+  log_daemon_msg "Starting $DESC" "$NAME"
+  do_start
+  case "$?" in
+    0|1) log_end_msg 0 ;;
+    2) log_end_msg 1 ;;
+  esac
+  ;;
+
+  stop)
+  log_daemon_msg "Stopping $DESC" "$NAME"
+  do_stop
+  case "$?" in
+    0) log_end_msg 0 ;;
+    1|2) log_end_msg 1 ;;
+  esac
+  ;;
+
+  # status)
+  # status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
+  # ;;
+
+  #reload|force-reload)
+  #
+  # If do_reload() is not implemented then leave this commented out
+  # and leave 'force-reload' as an alias for 'restart'.
+  #
+  #log_daemon_msg "Reloading $DESC" "$NAME"
+  #do_reload
+  #log_end_msg $?
+  #;;
+
+  restart|force-reload)
+  #
+  # If the "reload" option is implemented then remove the
+  # 'force-reload' alias
+  #
+  log_daemon_msg "Restarting $DESC" "$NAME"
+  do_stop
+  case "$?" in
+    0|1)
+    do_start
+    case "$?" in
+      0) log_end_msg 0 ;;
+      1) log_end_msg 1 ;; # Old process is still running
+      *) log_end_msg 1 ;; # Failed to start
+    esac
+    ;;
+    *)
+    # Failed to stop
+    log_end_msg 1
+    ;;
+  esac
+  ;;
+  *)
+  #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2
+  echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2
+  exit 3
+  ;;
+esac
+
+: