diff --git a/README.md b/README.md index 9180c10..5ff0415 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,81 @@ kafka-deb-packaging =================== -Simple debian packaging for Apache Kafka +Simple debian packaging for Apache Kafka and Zookeeper, great for Ubuntu 14.04LTS + +### Changelog +* 2015-Mar-18 : Updated for latest Kafka 0.8.2.1 and use sbt in system path. +* 2015-Mar-30 : Final Test, ready for git pull, packaged Zookeeper init.d! + +# Usage + +```sh +$ ./dist_kakfa.sh +``` + +# Sample Output + +```sh +~/git/kafka-deb-packaging$ ./dist_kafka.sh + +--2015-03-30 17:39:01-- http://mirror.sdunix.com/apache/kafka/0.8.2.1/kafka_2.10-0.8.2.1.tgz +Resolving mirror.sdunix.com (mirror.sdunix.com)... 74.206.97.82 +Connecting to mirror.sdunix.com (mirror.sdunix.com)|74.206.97.82|:80... connected. +HTTP request sent, awaiting response... 200 OK +Length: 16162559 (15M) [application/x-gzip] +Saving to: ‘kafka_2.10-0.8.2.1.tgz’ + +100%[========================>] 16,162,559 340KB/s in 44s + +2015-03-30 17:39:45 (361 KB/s) - ‘kafka_2.10-0.8.2.1.tgz’ saved [16162559/16162559] + +~/git/kafka-deb-packaging/tmp ~/git/kafka-deb-packaging +Created package {:path=>"kafka_0.8.2.1-5_all.deb"} +~/git/kafka-deb-packaging +``` + +# Installation + +```sh +$ dpkg -i kafka_0.8.2.1-10_all.deb +``` + +or if you have your own repo: + +```sh +$ ~/gpg-agent-headless.sh +$ reprepro -b /var/repositories/ includedeb trusty $@ +$ apt-get install kafka +``` +Note: Installs and runs as user 'app'. Easy to change for your needs. + +## Post install + +```sh +$ sudo update-rc.d zookeeper defaults 20 + +$ sudo update-rc.d kafka defaults 25 + + Adding system startup for /etc/init.d/kafka ... + /etc/rc0.d/K25kafka -> ../init.d/kafka + /etc/rc1.d/K25kafka -> ../init.d/kafka + /etc/rc6.d/K25kafka -> ../init.d/kafka + /etc/rc2.d/S25kafka -> ../init.d/kafka + /etc/rc3.d/S25kafka -> ../init.d/kafka + /etc/rc4.d/S25kafka -> ../init.d/kafka + /etc/rc5.d/S25kafka -> ../init.d/kafka +``` + +## System Check + +```sh +$ /usr/lib/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic installtest --partitions 1 --replication-factor 1 + +Created topic "installtest". +``` + +## Tested Platforms + +* Ubuntu 14.04LTS, Amazon EC2 -## Changelog -2015-Mar-18 : Updated for latest Kafka 0.8.2.1 and use sbt in system path. diff --git a/build_kafka.sh b/build_kafka.sh index 480f4b4..3cda9d9 100755 --- a/build_kafka.sh +++ b/build_kafka.sh @@ -1,45 +1,67 @@ #!/bin/bash # 2015-Mar-18 Updated to latest Kafka stable: 0.8.2.1 +# 2015-Mar-20 Added the init.d script and changed to use binary download of scala 2.10, Kafka 0.8.2.1 +# 2015-Dec-07 Upgraded to Kafka 0.8.2.2 distro + set -e set -u name=kafka -version=0.8.2.1 +version=0.8.2.2 +scala_version=2.10 description="Apache Kafka is a distributed publish-subscribe messaging system." url="https://kafka.apache.org/" arch="all" section="misc" license="Apache Software License 2.0" -package_version="-1" +package_version="-10" +bin_package="kafka_${scala_version}-${version}.tgz" src_package="kafka-${version}-src.tgz" -download_url="http://mirror.sdunix.com/apache/kafka/${version}/${src_package}" +src_download_url="http://mirror.sdunix.com/apache/kafka/${version}/${src_package}" origdir="$(pwd)" #_ MAIN _# rm -rf ${name}*.deb if [[ ! -f "${src_package}" ]]; then - wget ${download_url} + wget -c ${src_download_url} fi mkdir -p tmp && pushd tmp rm -rf kafka mkdir -p kafka cd kafka mkdir -p build/usr/lib/kafka +mkdir -p build/usr/lib/kafka/logs mkdir -p build/etc/default mkdir -p build/etc/init +mkdir -p build/etc/init.d mkdir -p build/etc/kafka mkdir -p build/var/log/kafka +#mkdir -p build/var/run/kafka cp ${origdir}/kafka-broker.default build/etc/default/kafka-broker cp ${origdir}/kafka-broker.upstart.conf build/etc/init/kafka-broker.conf +cp ${origdir}/kafka-broker.init.d build/etc/init.d/kafka +cp ${origdir}/zookeeper.init.d build/etc/init.d/zookeeper +# This code line uses the src packages tar zxf ${origdir}/${src_package} cd kafka-${version}-src -sbt update -sbt package -mv config/log4j.properties config/server.properties ../build/etc/kafka +# updated to gradle build as per Kafka README +gradle +./gradlew releaseTarGz +cp ./core/build/distributions/${bin_package} ${origdir}/${bin_package} +cd .. + +# Now resume regular binary exploding +# Updated to use the Binary package +rm -rf kafka_${scala_version}-${version} +tar zxf ${origdir}/${bin_package} +cd kafka_${scala_version}-${version} +cp config/server.properties ../build/etc/kafka +cp config/zookeeper.properties ../build/etc/kakfa/zookeeper.properties mv * ../build/usr/lib/kafka cd ../build + fpm -t deb \ -n ${name} \ -v ${version}${package_version} \ diff --git a/dist_kafka.sh b/dist_kafka.sh new file mode 100755 index 0000000..9389116 --- /dev/null +++ b/dist_kafka.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# 2015-Mar-18 Updated to latest Kafka stable: 0.8.2.1 +# 2015-Mar-20 Added the init.d script and changed to use binary download of scala 2.10, Kafka 0.8.2.1 +# 2015-Dec-07 Upgraded to Kafka 0.8.2.2 distro + +set -e +set -u +app_user=app +name=kafka +version=0.8.2.2 +scala_version=2.10 +package_version="-10" +description="Apache Kafka is a distributed publish-subscribe messaging system." +url="https://kafka.apache.org/" +arch="all" +section="misc" +license="Apache Software License 2.0" +bin_package="kafka_${scala_version}-${version}.tgz" +bin_download_url="http://mirror.sdunix.com/apache/kafka/${version}/${bin_package}" +origdir="$(pwd)" + +#_ MAIN _# +rm -rf ${name}*.deb +if [[ ! -f "${bin_package}" ]]; then + wget -c ${bin_download_url} +fi +mkdir -p tmp && pushd tmp +rm -rf kafka +mkdir -p kafka +cd kafka +mkdir -p build/usr/lib/kafka +mkdir -p build/usr/lib/kafka/logs +mkdir -p build/etc/default +mkdir -p build/etc/init +mkdir -p build/etc/init.d +mkdir -p build/etc/kafka +mkdir -p build/var/log/kafka +#mkdir -p build/var/run/kafka + +cp ${origdir}/kafka-broker.default build/etc/default/kafka-broker +cp ${origdir}/kafka-broker.upstart.conf build/etc/init/kafka-broker.conf +cp ${origdir}/kafka-broker.init.d build/etc/init.d/kafka +cp ${origdir}/zookeeper.init.d build/etc/init.d/zookeeper + +# Updated to use the Binary package +rm -rf kafka_${scala_version}-${version} +tar zxf ${origdir}/${bin_package} +cd kafka_${scala_version}-${version} +# mv config/log4j.properties config/server.properties ../build/etc/kafka +cp config/server.properties ../build/etc/kafka +cp config/zookeeper.properties ../build/etc/kafka/zookeeper.properties +mv * ../build/usr/lib/kafka +cd ../build + +fpm -t deb \ + --deb-user ${app_user} \ + -n ${name} \ + -v ${version}${package_version} \ + --description "${description}" \ + --url="{$url}" \ + -a ${arch} \ + --category ${section} \ + --vendor "" \ + --license "${license}" \ + -m "${USER}@localhost" \ + --prefix=/ \ + -s dir \ + -- . +mv kafka*.deb ${origdir} +popd diff --git a/kafka-broker.init.d b/kafka-broker.init.d new file mode 100755 index 0000000..79dcb95 --- /dev/null +++ b/kafka-broker.init.d @@ -0,0 +1,137 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: kafka +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts Kafka server +# Description: Starts Kafka server +### END INIT INFO + +# Authors: Josh Skidmore +# Brandon Brown +# Brandon Froehlich +# Taylor Brockman + +# Revision History: 2015-02-12 - Initial commit +# 2015-03-06 - Porting to nop-bigress +# 2015-03-20 - Contribution to tomdz/kafka-deb-packaging +# 2015-03-30 - Changed PID directory after running in AWS prod + +# Variables +PATH=$PATH +DESC="Apache Kafka Broker" +NAME=kafka +DAEMON=/usr/lib/kafka/bin/kafka-server-start.sh +PIDFILE=/var/run/kafka-server.pid +USER=app +CONFIG=/etc/kafka/server.properties +LOG=/var/log/kafka/kafka-server.log + + +# Load the VERBOSE setting and other rcS variables +. /lib/init/vars.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + + +# +# Function that starts the daemon/service +# +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + + start-stop-daemon --chuid $USER --background --start --quiet --make-pidfile --pidfile $PIDFILE --no-close --startas $DAEMON -- $CONFIG >> $LOG 2>&1 || return 2 +} + + +# +# Function that stops the daemon/service +# +do_stop() +{ + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE + RETVAL="$?" + + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 ;; + 2) log_end_msg 1 ;; + esac + ;; + + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0) log_end_msg 0 ;; + 1|2) log_end_msg 1 ;; + esac + ;; + + # status) + # status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? + # ;; + + #reload|force-reload) + # + # If do_reload() is not implemented then leave this commented out + # and leave 'force-reload' as an alias for 'restart'. + # + #log_daemon_msg "Reloading $DESC" "$NAME" + #do_reload + #log_end_msg $? + #;; + + restart|force-reload) + # + # If the "reload" option is implemented then remove the + # 'force-reload' alias + # + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2 + exit 3 + ;; +esac + +: diff --git a/zookeeper.init.d b/zookeeper.init.d new file mode 100755 index 0000000..6e2645c --- /dev/null +++ b/zookeeper.init.d @@ -0,0 +1,139 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: zookeeper +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: centralized coordination service +# Description: ZooKeeper is a centralized service for maintaining +# configuration information, naming, providing distributed +# synchronization, and providing group services. +### END INIT INFO + +# Authors: Josh Skidmore +# Brandon Brown +# Brandon Froehlich +# Taylor Brockman + +# Revision History: 2015-02-12 - Initial commit +# 2015-03-06 - Porting to nop-bigress +# 2015-03-20 - Contribution to tomdz/kafka-deb-packaging +# 2015-03-30 - Changed PID directory after running in AWS prod + +# Variables +PATH=$PATH +DESC="Apache Zookeeper Server" +NAME=zookeeper +DAEMON=/usr/lib/kafka/bin/zookeeper-server-start.sh +PIDFILE=/var/run/zookeeper-server.pid +USER=app +CONFIG=/etc/kafka/zookeeper.properties +LOG=/var/log/zookeeper-server.log + + +# Load the VERBOSE setting and other rcS variables +. /lib/init/vars.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + + +# +# Function that starts the daemon/service +# +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + + start-stop-daemon --chuid $USER --background --start --quiet --make-pidfile --pidfile $PIDFILE --no-close --startas $DAEMON -- $CONFIG >> $LOG 2>&1 || return 2 +} + + +# +# Function that stops the daemon/service +# +do_stop() +{ + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE + RETVAL="$?" + + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 ;; + 2) log_end_msg 1 ;; + esac + ;; + + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0) log_end_msg 0 ;; + 1|2) log_end_msg 1 ;; + esac + ;; + + # status) + # status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? + # ;; + + #reload|force-reload) + # + # If do_reload() is not implemented then leave this commented out + # and leave 'force-reload' as an alias for 'restart'. + # + #log_daemon_msg "Reloading $DESC" "$NAME" + #do_reload + #log_end_msg $? + #;; + + restart|force-reload) + # + # If the "reload" option is implemented then remove the + # 'force-reload' alias + # + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2 + exit 3 + ;; +esac + +: