From 97c409a9b80e01c4cd300bb465be7b813d7abc9f Mon Sep 17 00:00:00 2001 From: Taylor Brockman Date: Fri, 20 Mar 2015 09:59:41 -0400 Subject: [PATCH 1/9] Included a Ubuntu 14.04.* LTS compatible init.d script. Added a binary distribution script and updated the souce to use the gradle build. --- build_kafka.sh | 30 ++++++++-- dist_kafka.sh | 64 +++++++++++++++++++++ kafka-broker.init.d | 135 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 223 insertions(+), 6 deletions(-) create mode 100755 dist_kafka.sh create mode 100755 kafka-broker.init.d diff --git a/build_kafka.sh b/build_kafka.sh index 480f4b4..2d35c20 100755 --- a/build_kafka.sh +++ b/build_kafka.sh @@ -1,23 +1,27 @@ #!/bin/bash # 2015-Mar-18 Updated to latest Kafka stable: 0.8.2.1 +# 2015-Mar-20 Added the init.d script and changed to use binary download of scala 2.10, Kafka 0.8.2.1 + set -e set -u name=kafka version=0.8.2.1 +scala_version=2.10 description="Apache Kafka is a distributed publish-subscribe messaging system." url="https://kafka.apache.org/" arch="all" section="misc" license="Apache Software License 2.0" -package_version="-1" +package_version="-2" +bin_package="kafka_${scala_version}-${version}.tgz" src_package="kafka-${version}-src.tgz" -download_url="http://mirror.sdunix.com/apache/kafka/${version}/${src_package}" +src_download_url="http://mirror.sdunix.com/apache/kafka/${version}/${src_package}" origdir="$(pwd)" #_ MAIN _# rm -rf ${name}*.deb if [[ ! -f "${src_package}" ]]; then - wget ${download_url} + wget -c ${src_download_url} fi mkdir -p tmp && pushd tmp rm -rf kafka @@ -26,20 +30,34 @@ cd kafka mkdir -p build/usr/lib/kafka mkdir -p build/etc/default mkdir -p build/etc/init +mkdir -p build/etc/init.d mkdir -p build/etc/kafka mkdir -p build/var/log/kafka +mkdir -p build/var/run/kafka cp ${origdir}/kafka-broker.default build/etc/default/kafka-broker cp ${origdir}/kafka-broker.upstart.conf build/etc/init/kafka-broker.conf +cp ${origdir}/kafka-broker.init.d build/etc/init.d/kafka +# This code line uses the src packages tar zxf ${origdir}/${src_package} cd kafka-${version}-src -sbt update -sbt package -mv config/log4j.properties config/server.properties ../build/etc/kafka +# updated to gradle build as per Kafka README +gradle +./gradlew releaseTarGz +cp ./core/build/distributions/${bin_package} ${origdir}/${bin_package} +cd .. + +# Now resume regular binary exploding +# Updated to use the Binary package +rm -rf kafka_${scala_version}-${version} +tar zxf ${origdir}/${bin_package} +cd kafka_${scala_version}-${version} +cp config/server.properties ../build/etc/kafka mv * ../build/usr/lib/kafka cd ../build + fpm -t deb \ -n ${name} \ -v ${version}${package_version} \ diff --git a/dist_kafka.sh b/dist_kafka.sh new file mode 100755 index 0000000..2cb3e2e --- /dev/null +++ b/dist_kafka.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# 2015-Mar-18 Updated to latest Kafka stable: 0.8.2.1 +# 2015-Mar-20 Added the init.d script and changed to use binary download of scala 2.10, Kafka 0.8.2.1 + +set -e +set -u +name=kafka +version=0.8.2.1 +scala_version=2.10 +package_version="-3" +description="Apache Kafka is a distributed publish-subscribe messaging system." +url="https://kafka.apache.org/" +arch="all" +section="misc" +license="Apache Software License 2.0" +bin_package="kafka_${scala_version}-${version}.tgz" +bin_download_url="http://mirror.sdunix.com/apache/kafka/${version}/${bin_package}" +origdir="$(pwd)" + +#_ MAIN _# +rm -rf ${name}*.deb +if [[ ! -f "${bin_package}" ]]; then + wget -c ${bin_download_url} +fi +mkdir -p tmp && pushd tmp +rm -rf kafka +mkdir -p kafka +cd kafka +mkdir -p build/usr/lib/kafka +mkdir -p build/etc/default +mkdir -p build/etc/init +mkdir -p build/etc/init.d +mkdir -p build/etc/kafka +mkdir -p build/var/log/kafka +mkdir -p build/var/run/kafka + +cp ${origdir}/kafka-broker.default build/etc/default/kafka-broker +cp ${origdir}/kafka-broker.upstart.conf build/etc/init/kafka-broker.conf +cp ${origdir}/kafka-broker.init.d build/etc/init.d/kafka + +# Updated to use the Binary package +rm -rf kafka_${scala_version}-${version} +tar zxf ${origdir}/${bin_package} +cd kafka_${scala_version}-${version} +# mv config/log4j.properties config/server.properties ../build/etc/kafka +cp config/server.properties ../build/etc/kafka +mv * ../build/usr/lib/kafka +cd ../build + +fpm -t deb \ + -n ${name} \ + -v ${version}${package_version} \ + --description "${description}" \ + --url="{$url}" \ + -a ${arch} \ + --category ${section} \ + --vendor "" \ + --license "${license}" \ + -m "${USER}@localhost" \ + --prefix=/ \ + -s dir \ + -- . +mv kafka*.deb ${origdir} +popd diff --git a/kafka-broker.init.d b/kafka-broker.init.d new file mode 100755 index 0000000..cda18ea --- /dev/null +++ b/kafka-broker.init.d @@ -0,0 +1,135 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: kafka +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts Kafka server +# Description: Starts Kafka server +### END INIT INFO + +# Authors: Josh Skidmore +# Brandon Brown +# Taylor Brockman + +# Revision History: 2015-02-12 - Initial commit +# 2015-03-06 - Porting to nop-bigress +# 2015-03-20 - Contribution to tomdz/kafka-deb-packaging + +# Variables +PATH=$PATH +#DESC="Apache Kafka queue" +NAME=kafka +DAEMON=/usr/lib/kafka/bin/kafka-server-start.sh +PIDFILE=/var/run/kafka-server.pid +USER=app +CONFIG=/etc/kafka/server.properties +LOG=/var/log/kafka-server.log + + +# Load the VERBOSE setting and other rcS variables +. /lib/init/vars.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + + +# +# Function that starts the daemon/service +# +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + + start-stop-daemon --chuid $USER --background --start --quiet --make-pidfile --pidfile $PIDFILE --no-close --startas $DAEMON -- $CONFIG >> $LOG 2>&1 || return 2 +} + + +# +# Function that stops the daemon/service +# +do_stop() +{ + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE + RETVAL="$?" + + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 ;; + 2) log_end_msg 1 ;; + esac + ;; + + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0) log_end_msg 0 ;; + 1|2) log_end_msg 1 ;; + esac + ;; + + # status) + # status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? + # ;; + + #reload|force-reload) + # + # If do_reload() is not implemented then leave this commented out + # and leave 'force-reload' as an alias for 'restart'. + # + #log_daemon_msg "Reloading $DESC" "$NAME" + #do_reload + #log_end_msg $? + #;; + + restart|force-reload) + # + # If the "reload" option is implemented then remove the + # 'force-reload' alias + # + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2 + exit 3 + ;; +esac + +: From 33d04f5724c7457b1195529b078085e25b2d4bdf Mon Sep 17 00:00:00 2001 From: Taylor Brockman Date: Fri, 20 Mar 2015 10:13:39 -0400 Subject: [PATCH 2/9] Package Revision 5: Correction of system pid and log locations to match init.d script --- build_kafka.sh | 3 ++- dist_kafka.sh | 3 ++- kafka-broker.init.d | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/build_kafka.sh b/build_kafka.sh index 2d35c20..f9c2f7d 100755 --- a/build_kafka.sh +++ b/build_kafka.sh @@ -12,7 +12,7 @@ url="https://kafka.apache.org/" arch="all" section="misc" license="Apache Software License 2.0" -package_version="-2" +package_version="-5" bin_package="kafka_${scala_version}-${version}.tgz" src_package="kafka-${version}-src.tgz" src_download_url="http://mirror.sdunix.com/apache/kafka/${version}/${src_package}" @@ -28,6 +28,7 @@ rm -rf kafka mkdir -p kafka cd kafka mkdir -p build/usr/lib/kafka +mkdir -p build/usr/lib/kafka/logs mkdir -p build/etc/default mkdir -p build/etc/init mkdir -p build/etc/init.d diff --git a/dist_kafka.sh b/dist_kafka.sh index 2cb3e2e..c90095b 100755 --- a/dist_kafka.sh +++ b/dist_kafka.sh @@ -7,7 +7,7 @@ set -u name=kafka version=0.8.2.1 scala_version=2.10 -package_version="-3" +package_version="-5" description="Apache Kafka is a distributed publish-subscribe messaging system." url="https://kafka.apache.org/" arch="all" @@ -27,6 +27,7 @@ rm -rf kafka mkdir -p kafka cd kafka mkdir -p build/usr/lib/kafka +mkdir -p build/usr/lib/kafka/logs mkdir -p build/etc/default mkdir -p build/etc/init mkdir -p build/etc/init.d diff --git a/kafka-broker.init.d b/kafka-broker.init.d index cda18ea..a605007 100755 --- a/kafka-broker.init.d +++ b/kafka-broker.init.d @@ -22,10 +22,10 @@ PATH=$PATH #DESC="Apache Kafka queue" NAME=kafka DAEMON=/usr/lib/kafka/bin/kafka-server-start.sh -PIDFILE=/var/run/kafka-server.pid +PIDFILE=/var/run/kafka/kafka-server.pid USER=app CONFIG=/etc/kafka/server.properties -LOG=/var/log/kafka-server.log +LOG=/var/log/kafka/kafka-server.log # Load the VERBOSE setting and other rcS variables From 3289f029cf8e425da1924dc744075919895c7b14 Mon Sep 17 00:00:00 2001 From: Taylor Brockman Date: Mon, 30 Mar 2015 17:36:35 -0400 Subject: [PATCH 3/9] Added pidfile location fix for ubuntu 14.04LTS --- kafka-broker.init.d | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kafka-broker.init.d b/kafka-broker.init.d index a605007..79dcb95 100755 --- a/kafka-broker.init.d +++ b/kafka-broker.init.d @@ -11,18 +11,20 @@ # Authors: Josh Skidmore # Brandon Brown +# Brandon Froehlich # Taylor Brockman # Revision History: 2015-02-12 - Initial commit # 2015-03-06 - Porting to nop-bigress # 2015-03-20 - Contribution to tomdz/kafka-deb-packaging +# 2015-03-30 - Changed PID directory after running in AWS prod # Variables PATH=$PATH -#DESC="Apache Kafka queue" +DESC="Apache Kafka Broker" NAME=kafka DAEMON=/usr/lib/kafka/bin/kafka-server-start.sh -PIDFILE=/var/run/kafka/kafka-server.pid +PIDFILE=/var/run/kafka-server.pid USER=app CONFIG=/etc/kafka/server.properties LOG=/var/log/kafka/kafka-server.log From 3f6f7e61a7a07cc70d4b6632e8cacd42068927d7 Mon Sep 17 00:00:00 2001 From: Taylor Brockman Date: Mon, 30 Mar 2015 17:41:07 -0400 Subject: [PATCH 4/9] More readme, sample output --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index 9180c10..cf7e67d 100644 --- a/README.md +++ b/README.md @@ -5,4 +5,28 @@ Simple debian packaging for Apache Kafka ## Changelog 2015-Mar-18 : Updated for latest Kafka 0.8.2.1 and use sbt in system path. +2015-Mar-30 : Final Test, ready for git pull +# Tested Systems + +Ubuntu 14.04LTS, Amazon EC2 + + +# Sample Output + +~/git/kafka-deb-packaging$ ./dist_kafka.sh + +--2015-03-30 17:39:01-- http://mirror.sdunix.com/apache/kafka/0.8.2.1/kafka_2.10-0.8.2.1.tgz +Resolving mirror.sdunix.com (mirror.sdunix.com)... 74.206.97.82 +Connecting to mirror.sdunix.com (mirror.sdunix.com)|74.206.97.82|:80... connected. +HTTP request sent, awaiting response... 200 OK +Length: 16162559 (15M) [application/x-gzip] +Saving to: ‘kafka_2.10-0.8.2.1.tgz’ + +100%[========================>] 16,162,559 340KB/s in 44s + +2015-03-30 17:39:45 (361 KB/s) - ‘kafka_2.10-0.8.2.1.tgz’ saved [16162559/16162559] + +~/git/kafka-deb-packaging/tmp ~/git/kafka-deb-packaging +Created package {:path=>"kafka_0.8.2.1-5_all.deb"} +~/git/kafka-deb-packaging From 475329d4e17bd17ce6fcc44e25b9c9585ab17a12 Mon Sep 17 00:00:00 2001 From: Taylor Brockman Date: Mon, 30 Mar 2015 17:57:35 -0400 Subject: [PATCH 5/9] Remvoed erroneous creation of /var/run --- build_kafka.sh | 4 ++-- dist_kafka.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build_kafka.sh b/build_kafka.sh index f9c2f7d..64540dd 100755 --- a/build_kafka.sh +++ b/build_kafka.sh @@ -12,7 +12,7 @@ url="https://kafka.apache.org/" arch="all" section="misc" license="Apache Software License 2.0" -package_version="-5" +package_version="-7" bin_package="kafka_${scala_version}-${version}.tgz" src_package="kafka-${version}-src.tgz" src_download_url="http://mirror.sdunix.com/apache/kafka/${version}/${src_package}" @@ -34,7 +34,7 @@ mkdir -p build/etc/init mkdir -p build/etc/init.d mkdir -p build/etc/kafka mkdir -p build/var/log/kafka -mkdir -p build/var/run/kafka +#mkdir -p build/var/run/kafka cp ${origdir}/kafka-broker.default build/etc/default/kafka-broker cp ${origdir}/kafka-broker.upstart.conf build/etc/init/kafka-broker.conf diff --git a/dist_kafka.sh b/dist_kafka.sh index c90095b..1240dd2 100755 --- a/dist_kafka.sh +++ b/dist_kafka.sh @@ -7,7 +7,7 @@ set -u name=kafka version=0.8.2.1 scala_version=2.10 -package_version="-5" +package_version="-7" description="Apache Kafka is a distributed publish-subscribe messaging system." url="https://kafka.apache.org/" arch="all" @@ -33,7 +33,7 @@ mkdir -p build/etc/init mkdir -p build/etc/init.d mkdir -p build/etc/kafka mkdir -p build/var/log/kafka -mkdir -p build/var/run/kafka +#mkdir -p build/var/run/kafka cp ${origdir}/kafka-broker.default build/etc/default/kafka-broker cp ${origdir}/kafka-broker.upstart.conf build/etc/init/kafka-broker.conf From ae5e7ade9169ba5017f0e007ba3cdf05da936501 Mon Sep 17 00:00:00 2001 From: Taylor Brockman Date: Mon, 30 Mar 2015 18:06:11 -0400 Subject: [PATCH 6/9] Packgage install as the 'app' user --- dist_kafka.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dist_kafka.sh b/dist_kafka.sh index 1240dd2..d0f0749 100755 --- a/dist_kafka.sh +++ b/dist_kafka.sh @@ -4,6 +4,7 @@ set -e set -u +app_user=app name=kafka version=0.8.2.1 scala_version=2.10 @@ -49,6 +50,7 @@ mv * ../build/usr/lib/kafka cd ../build fpm -t deb \ + --deb-user ${app_user} \ -n ${name} \ -v ${version}${package_version} \ --description "${description}" \ From 3b23fa6e6c7272fcdd51498411c6febd4788c541 Mon Sep 17 00:00:00 2001 From: Taylor Brockman Date: Mon, 30 Mar 2015 18:26:28 -0400 Subject: [PATCH 7/9] Updates to include kafka as well --- README.md | 35 ++++++++++++++++++++++++++++++++--- build_kafka.sh | 4 +++- dist_kafka.sh | 4 +++- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cf7e67d..b303dc5 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,40 @@ kafka-deb-packaging =================== -Simple debian packaging for Apache Kafka +Simple debian packaging for Apache Kafka and Zookeeper. -## Changelog +### Changelog 2015-Mar-18 : Updated for latest Kafka 0.8.2.1 and use sbt in system path. -2015-Mar-30 : Final Test, ready for git pull +2015-Mar-30 : Final Test, ready for git pull, packaged Zookeeper init.d! + +# Usage + +$ ./dist_kakfa.sh + +# Installation + +$ dpkg -i kafka_0.8.2.1-10_all.deb + +or + +$ apt-get install kafka + +Note: Installs and runs as user 'app'. Easy to change for your needs. + +## Post install + +$ sudo update-rc.d zookeeper defaults 20 + +$ sudo update-rc.d kafka defaults 25 + + Adding system startup for /etc/init.d/kafka ... + /etc/rc0.d/K25kafka -> ../init.d/kafka + /etc/rc1.d/K25kafka -> ../init.d/kafka + /etc/rc6.d/K25kafka -> ../init.d/kafka + /etc/rc2.d/S25kafka -> ../init.d/kafka + /etc/rc3.d/S25kafka -> ../init.d/kafka + /etc/rc4.d/S25kafka -> ../init.d/kafka + /etc/rc5.d/S25kafka -> ../init.d/kafka # Tested Systems diff --git a/build_kafka.sh b/build_kafka.sh index 64540dd..e8f7d99 100755 --- a/build_kafka.sh +++ b/build_kafka.sh @@ -12,7 +12,7 @@ url="https://kafka.apache.org/" arch="all" section="misc" license="Apache Software License 2.0" -package_version="-7" +package_version="-10" bin_package="kafka_${scala_version}-${version}.tgz" src_package="kafka-${version}-src.tgz" src_download_url="http://mirror.sdunix.com/apache/kafka/${version}/${src_package}" @@ -39,6 +39,7 @@ mkdir -p build/var/log/kafka cp ${origdir}/kafka-broker.default build/etc/default/kafka-broker cp ${origdir}/kafka-broker.upstart.conf build/etc/init/kafka-broker.conf cp ${origdir}/kafka-broker.init.d build/etc/init.d/kafka +cp ${origdir}/zookeeper.init.d build/etc/init.d/zookeeper # This code line uses the src packages tar zxf ${origdir}/${src_package} @@ -55,6 +56,7 @@ rm -rf kafka_${scala_version}-${version} tar zxf ${origdir}/${bin_package} cd kafka_${scala_version}-${version} cp config/server.properties ../build/etc/kafka +cp config/zookeeper.properties ../build/etc/kakfa/zookeeper.properties mv * ../build/usr/lib/kafka cd ../build diff --git a/dist_kafka.sh b/dist_kafka.sh index d0f0749..170a292 100755 --- a/dist_kafka.sh +++ b/dist_kafka.sh @@ -8,7 +8,7 @@ app_user=app name=kafka version=0.8.2.1 scala_version=2.10 -package_version="-7" +package_version="-10" description="Apache Kafka is a distributed publish-subscribe messaging system." url="https://kafka.apache.org/" arch="all" @@ -39,6 +39,7 @@ mkdir -p build/var/log/kafka cp ${origdir}/kafka-broker.default build/etc/default/kafka-broker cp ${origdir}/kafka-broker.upstart.conf build/etc/init/kafka-broker.conf cp ${origdir}/kafka-broker.init.d build/etc/init.d/kafka +cp ${origdir}/zookeeper.init.d build/etc/init.d/zookeeper # Updated to use the Binary package rm -rf kafka_${scala_version}-${version} @@ -46,6 +47,7 @@ tar zxf ${origdir}/${bin_package} cd kafka_${scala_version}-${version} # mv config/log4j.properties config/server.properties ../build/etc/kafka cp config/server.properties ../build/etc/kafka +cp config/zookeeper.properties ../build/etc/kafka/zookeeper.properties mv * ../build/usr/lib/kafka cd ../build From a11aad1b9ca4f1ab910604955a044fe8604bedb6 Mon Sep 17 00:00:00 2001 From: Taylor Brockman Date: Mon, 30 Mar 2015 18:31:43 -0400 Subject: [PATCH 8/9] Updated documentation --- README.md | 64 ++++++++++++++-------- zookeeper.init.d | 139 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 22 deletions(-) create mode 100755 zookeeper.init.d diff --git a/README.md b/README.md index b303dc5..5ff0415 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,57 @@ kafka-deb-packaging =================== -Simple debian packaging for Apache Kafka and Zookeeper. +Simple debian packaging for Apache Kafka and Zookeeper, great for Ubuntu 14.04LTS ### Changelog -2015-Mar-18 : Updated for latest Kafka 0.8.2.1 and use sbt in system path. -2015-Mar-30 : Final Test, ready for git pull, packaged Zookeeper init.d! +* 2015-Mar-18 : Updated for latest Kafka 0.8.2.1 and use sbt in system path. +* 2015-Mar-30 : Final Test, ready for git pull, packaged Zookeeper init.d! # Usage +```sh $ ./dist_kakfa.sh +``` + +# Sample Output + +```sh +~/git/kafka-deb-packaging$ ./dist_kafka.sh + +--2015-03-30 17:39:01-- http://mirror.sdunix.com/apache/kafka/0.8.2.1/kafka_2.10-0.8.2.1.tgz +Resolving mirror.sdunix.com (mirror.sdunix.com)... 74.206.97.82 +Connecting to mirror.sdunix.com (mirror.sdunix.com)|74.206.97.82|:80... connected. +HTTP request sent, awaiting response... 200 OK +Length: 16162559 (15M) [application/x-gzip] +Saving to: ‘kafka_2.10-0.8.2.1.tgz’ + +100%[========================>] 16,162,559 340KB/s in 44s + +2015-03-30 17:39:45 (361 KB/s) - ‘kafka_2.10-0.8.2.1.tgz’ saved [16162559/16162559] + +~/git/kafka-deb-packaging/tmp ~/git/kafka-deb-packaging +Created package {:path=>"kafka_0.8.2.1-5_all.deb"} +~/git/kafka-deb-packaging +``` # Installation +```sh $ dpkg -i kafka_0.8.2.1-10_all.deb +``` -or +or if you have your own repo: +```sh +$ ~/gpg-agent-headless.sh +$ reprepro -b /var/repositories/ includedeb trusty $@ $ apt-get install kafka - +``` Note: Installs and runs as user 'app'. Easy to change for your needs. ## Post install +```sh $ sudo update-rc.d zookeeper defaults 20 $ sudo update-rc.d kafka defaults 25 @@ -35,27 +64,18 @@ $ sudo update-rc.d kafka defaults 25 /etc/rc3.d/S25kafka -> ../init.d/kafka /etc/rc4.d/S25kafka -> ../init.d/kafka /etc/rc5.d/S25kafka -> ../init.d/kafka +``` -# Tested Systems +## System Check -Ubuntu 14.04LTS, Amazon EC2 +```sh +$ /usr/lib/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic installtest --partitions 1 --replication-factor 1 +Created topic "installtest". +``` -# Sample Output +## Tested Platforms -~/git/kafka-deb-packaging$ ./dist_kafka.sh +* Ubuntu 14.04LTS, Amazon EC2 ---2015-03-30 17:39:01-- http://mirror.sdunix.com/apache/kafka/0.8.2.1/kafka_2.10-0.8.2.1.tgz -Resolving mirror.sdunix.com (mirror.sdunix.com)... 74.206.97.82 -Connecting to mirror.sdunix.com (mirror.sdunix.com)|74.206.97.82|:80... connected. -HTTP request sent, awaiting response... 200 OK -Length: 16162559 (15M) [application/x-gzip] -Saving to: ‘kafka_2.10-0.8.2.1.tgz’ - -100%[========================>] 16,162,559 340KB/s in 44s - -2015-03-30 17:39:45 (361 KB/s) - ‘kafka_2.10-0.8.2.1.tgz’ saved [16162559/16162559] -~/git/kafka-deb-packaging/tmp ~/git/kafka-deb-packaging -Created package {:path=>"kafka_0.8.2.1-5_all.deb"} -~/git/kafka-deb-packaging diff --git a/zookeeper.init.d b/zookeeper.init.d new file mode 100755 index 0000000..6e2645c --- /dev/null +++ b/zookeeper.init.d @@ -0,0 +1,139 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: zookeeper +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: centralized coordination service +# Description: ZooKeeper is a centralized service for maintaining +# configuration information, naming, providing distributed +# synchronization, and providing group services. +### END INIT INFO + +# Authors: Josh Skidmore +# Brandon Brown +# Brandon Froehlich +# Taylor Brockman + +# Revision History: 2015-02-12 - Initial commit +# 2015-03-06 - Porting to nop-bigress +# 2015-03-20 - Contribution to tomdz/kafka-deb-packaging +# 2015-03-30 - Changed PID directory after running in AWS prod + +# Variables +PATH=$PATH +DESC="Apache Zookeeper Server" +NAME=zookeeper +DAEMON=/usr/lib/kafka/bin/zookeeper-server-start.sh +PIDFILE=/var/run/zookeeper-server.pid +USER=app +CONFIG=/etc/kafka/zookeeper.properties +LOG=/var/log/zookeeper-server.log + + +# Load the VERBOSE setting and other rcS variables +. /lib/init/vars.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + + +# +# Function that starts the daemon/service +# +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + + start-stop-daemon --chuid $USER --background --start --quiet --make-pidfile --pidfile $PIDFILE --no-close --startas $DAEMON -- $CONFIG >> $LOG 2>&1 || return 2 +} + + +# +# Function that stops the daemon/service +# +do_stop() +{ + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE + RETVAL="$?" + + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 ;; + 2) log_end_msg 1 ;; + esac + ;; + + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0) log_end_msg 0 ;; + 1|2) log_end_msg 1 ;; + esac + ;; + + # status) + # status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? + # ;; + + #reload|force-reload) + # + # If do_reload() is not implemented then leave this commented out + # and leave 'force-reload' as an alias for 'restart'. + # + #log_daemon_msg "Reloading $DESC" "$NAME" + #do_reload + #log_end_msg $? + #;; + + restart|force-reload) + # + # If the "reload" option is implemented then remove the + # 'force-reload' alias + # + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2 + exit 3 + ;; +esac + +: From 67bb78c79c118515eb5473bd8cccd2862aaeabd7 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 7 Dec 2015 16:09:52 -0500 Subject: [PATCH 9/9] REvision to latest Kafka 0.8.2.2 --- build_kafka.sh | 3 ++- dist_kafka.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/build_kafka.sh b/build_kafka.sh index e8f7d99..3cda9d9 100755 --- a/build_kafka.sh +++ b/build_kafka.sh @@ -1,11 +1,12 @@ #!/bin/bash # 2015-Mar-18 Updated to latest Kafka stable: 0.8.2.1 # 2015-Mar-20 Added the init.d script and changed to use binary download of scala 2.10, Kafka 0.8.2.1 +# 2015-Dec-07 Upgraded to Kafka 0.8.2.2 distro set -e set -u name=kafka -version=0.8.2.1 +version=0.8.2.2 scala_version=2.10 description="Apache Kafka is a distributed publish-subscribe messaging system." url="https://kafka.apache.org/" diff --git a/dist_kafka.sh b/dist_kafka.sh index 170a292..9389116 100755 --- a/dist_kafka.sh +++ b/dist_kafka.sh @@ -1,12 +1,13 @@ #!/bin/bash # 2015-Mar-18 Updated to latest Kafka stable: 0.8.2.1 # 2015-Mar-20 Added the init.d script and changed to use binary download of scala 2.10, Kafka 0.8.2.1 +# 2015-Dec-07 Upgraded to Kafka 0.8.2.2 distro set -e set -u app_user=app name=kafka -version=0.8.2.1 +version=0.8.2.2 scala_version=2.10 package_version="-10" description="Apache Kafka is a distributed publish-subscribe messaging system."