From 5c2bb4bbf7c99f38d59dceb58ca797eaaa60f9d1 Mon Sep 17 00:00:00 2001 From: Ravi Peri Date: Thu, 13 Apr 2017 23:54:06 -0700 Subject: [PATCH 01/10] Initial version of custom action script to install Mono on HDInsight clusters --- install-mono/README.md | 39 ++++++++++++ install-mono/install-mono.bash | 111 +++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 install-mono/README.md create mode 100644 install-mono/install-mono.bash diff --git a/install-mono/README.md b/install-mono/README.md new file mode 100644 index 0000000..0e70236 --- /dev/null +++ b/install-mono/README.md @@ -0,0 +1,39 @@ +# hdinsight Install-Mono script-action +Script action to install Mono (https://www.mono-project.com) on a HDInsight cluster node. + +This script can be used to install the default latest stable Mono version (4.8.1) or a choice of Mono version +passed in as an argument to the script. + +The script will uninstall any existing versions of Mono, if different from the one specified. + +## Installed Packages +The following packages are installed: + mono-complete + ca-certificates-mono + +## Installation instructions + +1. Create HDInsight cluster (>3.4 version) + +2. If running a Storm cluster, Stop SCP.Net services from Ambari on all nodes. + +3. Run script action: `install-mono.sh` on this cluster. + + Go to Azure portal > open cluster blade > open Script Actions tile > click Submit new and follow instructions. The script action is provided in this repository. + + When you are on the "Submit script action" blade, you will see "Bash script URI" field. You need to make sure that the `install-mono.sh` is stored in an Azure Storage Blob, and make the link public. + + OR + + You can just add https://raw.githubusercontent.com/hdinsight/script-actions/master/install-mono/install-mono.sh to "Bash script URI". + + The script installs version 4.8.1 by default. + If a custom Mono version is required, users can pass in the version number as the first argument to the script. + Example: + install-mono.sh 4.8.0 + install-mono.sh 5.0.1 + + The version passed in must be present on the following page, as an available version. Else the script will fail. + https://download.mono-project.com/repo/debian/dists/wheezy/snapshots/ + +4. Mono is now installed and ready for use. diff --git a/install-mono/install-mono.bash b/install-mono/install-mono.bash new file mode 100644 index 0000000..32fec69 --- /dev/null +++ b/install-mono/install-mono.bash @@ -0,0 +1,111 @@ +#!/bin/bash + +################################################################## +# Custom action script to install/upgrade Mono ( +# a. Remove any existing version of Mono installed on a node +# b. Install specified versioni of Mono (http://www.mono-project.com) on a HDInsight cluster's Node. +################################################################## + +set -e + +log(){ + local message="${1}" + local current_time=$(date "+%Y.%m.%d-%H.%M.%S") + echo "Install-Mono: ${current_time}: ${message}" +} + +validate_installation(){ + log "Validating Mono installation for target version $1" + local MATCH_STR="Mono JIT compiler version $1" + log "Validating Mono Version $TARGET_VERSION is installed properly" + local INSTALLED_VERSION_STR=$(mono --version 2>&1 | grep -w "Mono JIT compiler version") + if [[ $INSTALLED_VERSION_STR != $MATCH_STR* ]] ; then + log "Failed to detect mono version $1 on machine." + exit 1 + fi + + if [ -f /tmp/tlstest.cs ]; then + rm -f /tmp/tlstest.cs + fi + wget https://raw.github.com/mono/mono/master/mcs/class/Mono.Security/Test/tools/tlstest/tlstest.cs -O /tmp/tlstest.cs + + local CMD="mono /tmp/tlstest.exe https://www.nuget.org" + mcs /tmp/tlstest.cs /r:System.dll /r:Mono.Security.dll + local RET_VAL=$($CMD 2>&1 | sed '/^\s*$/d') + if [[ "$RET_VAL" != "https://www.nuget.org" ]]; then + log "Mono validation failed. Result from $CMD: $RET_VAL" + exit 1 + fi + log "Successfully validated that Mono is installed correctly" +} + +if [ "$(id -u)" != "0" ]; then + log "This script is supposed to be invoked as root." + exit 1 +fi + +# Note: Mono version will default to 4.8 +declare MONO_VERSION="4.8.0" + +# Note: It is expected that user's supplying Mono versions validated the version +# exists under http://download.mono-project.com/repo/debian/wheezy/snapshots folder. +if [ "$1" != "" ]; then + MONO_VERSION="$1" +fi + +log "Target Mono version to install: $MONO_VERSION" + +REPO_CHECK_URL="https://download.mono-project.com/repo/debian/dists/wheezy/snapshots/$MONO_VERSION/" +if ! curl -s --head "$REPO_CHECK_URL" | head -n 1 | grep 200 >/dev/null; then + log "Mono Version $MONO_VERSION not found at $REPO_CHECK_URL" + exit 1 +fi +log "Checking for existing Mono version" +if type "mono" > /dev/null; then + INSTALLED_VERSION_STR=$(mono --version 2>&1 | grep -w "Mono JIT compiler version") + log "Installed Version of Mono: ${INSTALLED_VERSION_STR}" + + MATCH_STR="Mono JIT compiler version $MONO_VERSION " + if [[ $INSTALLED_VERSION_STR != $MATCH_STR* ]] ; then + log "Uninstalling current version, before installing target version" + apt-get -y remove --purge --auto-remove mono-runtime + apt-get -y remove --purge --auto-remove mono-complete + apt-get -y remove --purge --auto-remove ca-certificates-mono + else + log "Installed version of mono $INSTALLED_VERSION_STR matches target version $MONO_VERSION." + validate_installation $MONO_VERSION + + exit 0 + fi +else + log "Mono not found on the system" +fi + +log "Looking for existing xamarian repo sources" +if [ -f /etc/apt/sources.list.d/mono-xamarin.list ]; then + log "Removing xamarian repo sources" + rm /etc/apt/sources.list.d/mono-xamarin.list +fi +log "Removed existing xamarian repo sources" + +log "Adding Mono GPG keys" +apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF + +MONO_RESOURCE_LIST_URL="http://download.mono-project.com/repo/debian wheezy/snapshots/$MONO_VERSION main" +log "Adding Mono package resource list $MONO_RESOURCE_LIST_URL" +echo "deb $MONO_RESOURCE_LIST_URL" | tee /etc/apt/sources.list.d/mono-xamarin.list + +log "Updating package list" +apt-get update + +log "Installing mono-complete package" +apt-get install -y mono-complete +log "Installed mono-complete package" + +log "Installing ca-certificates-mono package" +apt-get install -y ca-certificates-mono +log "Installed ca-certificates-mono package" + +validate_installation $MONO_VERSION + +log "Finished installing Mono version $MONO_VERSION" \ No newline at end of file From 2673aa1fd1384bd0f97d450b2fc06f8061412feb Mon Sep 17 00:00:00 2001 From: Ravi Peri Date: Thu, 13 Apr 2017 23:55:44 -0700 Subject: [PATCH 02/10] Fix comments --- install-mono/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install-mono/README.md b/install-mono/README.md index 0e70236..ddff5d9 100644 --- a/install-mono/README.md +++ b/install-mono/README.md @@ -8,8 +8,8 @@ The script will uninstall any existing versions of Mono, if different from the o ## Installed Packages The following packages are installed: - mono-complete - ca-certificates-mono +1. mono-complete +2. ca-certificates-mono ## Installation instructions From 5e2e549c893b9f1ba49ebd09ef3aa1f44e710d5d Mon Sep 17 00:00:00 2001 From: Ravi Peri Date: Thu, 13 Apr 2017 23:56:36 -0700 Subject: [PATCH 03/10] Fix comments --- install-mono/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install-mono/README.md b/install-mono/README.md index ddff5d9..469700f 100644 --- a/install-mono/README.md +++ b/install-mono/README.md @@ -30,8 +30,8 @@ The following packages are installed: The script installs version 4.8.1 by default. If a custom Mono version is required, users can pass in the version number as the first argument to the script. Example: - install-mono.sh 4.8.0 - install-mono.sh 5.0.1 + a. install-mono.sh 4.8.0 + b. install-mono.sh 5.0.1 The version passed in must be present on the following page, as an available version. Else the script will fail. https://download.mono-project.com/repo/debian/dists/wheezy/snapshots/ From 4b897f6c0b9fbd9208582babe739040e37c1f2ad Mon Sep 17 00:00:00 2001 From: Ravi Peri Date: Fri, 14 Apr 2017 09:36:19 -0700 Subject: [PATCH 04/10] Update default version to 4.8.1 --- install-mono/README.md | 8 ++++---- install-mono/install-mono.bash | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/install-mono/README.md b/install-mono/README.md index 469700f..eb6333a 100644 --- a/install-mono/README.md +++ b/install-mono/README.md @@ -8,8 +8,8 @@ The script will uninstall any existing versions of Mono, if different from the o ## Installed Packages The following packages are installed: -1. mono-complete -2. ca-certificates-mono +* mono-complete +* ca-certificates-mono ## Installation instructions @@ -30,8 +30,8 @@ The following packages are installed: The script installs version 4.8.1 by default. If a custom Mono version is required, users can pass in the version number as the first argument to the script. Example: - a. install-mono.sh 4.8.0 - b. install-mono.sh 5.0.1 + * install-mono.sh 4.8.0 + * install-mono.sh 5.0.1 The version passed in must be present on the following page, as an available version. Else the script will fail. https://download.mono-project.com/repo/debian/dists/wheezy/snapshots/ diff --git a/install-mono/install-mono.bash b/install-mono/install-mono.bash index 32fec69..51ed1d9 100644 --- a/install-mono/install-mono.bash +++ b/install-mono/install-mono.bash @@ -1,13 +1,16 @@ #!/bin/bash ################################################################## -# Custom action script to install/upgrade Mono ( +# Custom action script to install/upgrade Mono # a. Remove any existing version of Mono installed on a node # b. Install specified versioni of Mono (http://www.mono-project.com) on a HDInsight cluster's Node. ################################################################## set -e +# Note: Mono version will default to 4.8 +declare MONO_VERSION="4.8.1" + log(){ local message="${1}" local current_time=$(date "+%Y.%m.%d-%H.%M.%S") @@ -44,9 +47,6 @@ if [ "$(id -u)" != "0" ]; then exit 1 fi -# Note: Mono version will default to 4.8 -declare MONO_VERSION="4.8.0" - # Note: It is expected that user's supplying Mono versions validated the version # exists under http://download.mono-project.com/repo/debian/wheezy/snapshots folder. if [ "$1" != "" ]; then From cb9354f9944fd2f230073de5f76d9ed86a473251 Mon Sep 17 00:00:00 2001 From: Ravi Peri Date: Tue, 18 Apr 2017 00:32:25 -0700 Subject: [PATCH 05/10] Storm scripts to add hadoop hdfs jars to extlib --- install-storm-hdfs-extlib/instal-storm-hdfs-extlib.bash | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 install-storm-hdfs-extlib/instal-storm-hdfs-extlib.bash diff --git a/install-storm-hdfs-extlib/instal-storm-hdfs-extlib.bash b/install-storm-hdfs-extlib/instal-storm-hdfs-extlib.bash new file mode 100644 index 0000000..32aa4f3 --- /dev/null +++ b/install-storm-hdfs-extlib/instal-storm-hdfs-extlib.bash @@ -0,0 +1,6 @@ +#!/bin/bash + +sudo ln -s /usr/hdp/current/hadoop-client/*.jar /usr/hdp/current/storm-client/extlib/ +sudo ln -s /usr/hdp/current/hadoop-client/lib/*.jar /usr/hdp/current/storm-client/extlib/ +sudo ln -s /usr/hdp/current/hadoop-hdfs-client/*.jar /usr/hdp/current/storm-client/extlib/ +sudo ln -s /usr/hdp/current/hadoop-hdfs-client/lib/*.jar /usr/hdp/current/storm-client/extlib/ \ No newline at end of file From 0e2849c3ec70742881187e012d02cf13c29858db Mon Sep 17 00:00:00 2001 From: Ravi Peri Date: Tue, 18 Apr 2017 00:35:48 -0700 Subject: [PATCH 06/10] overwrite existing symlinks --- install-storm-hdfs-extlib/instal-storm-hdfs-extlib.bash | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install-storm-hdfs-extlib/instal-storm-hdfs-extlib.bash b/install-storm-hdfs-extlib/instal-storm-hdfs-extlib.bash index 32aa4f3..097c3cf 100644 --- a/install-storm-hdfs-extlib/instal-storm-hdfs-extlib.bash +++ b/install-storm-hdfs-extlib/instal-storm-hdfs-extlib.bash @@ -1,6 +1,6 @@ #!/bin/bash -sudo ln -s /usr/hdp/current/hadoop-client/*.jar /usr/hdp/current/storm-client/extlib/ -sudo ln -s /usr/hdp/current/hadoop-client/lib/*.jar /usr/hdp/current/storm-client/extlib/ -sudo ln -s /usr/hdp/current/hadoop-hdfs-client/*.jar /usr/hdp/current/storm-client/extlib/ -sudo ln -s /usr/hdp/current/hadoop-hdfs-client/lib/*.jar /usr/hdp/current/storm-client/extlib/ \ No newline at end of file +sudo ln -sf /usr/hdp/current/hadoop-client/*.jar /usr/hdp/current/storm-client/extlib/ +sudo ln -sf /usr/hdp/current/hadoop-client/lib/*.jar /usr/hdp/current/storm-client/extlib/ +sudo ln -sf /usr/hdp/current/hadoop-hdfs-client/*.jar /usr/hdp/current/storm-client/extlib/ +sudo ln -sf /usr/hdp/current/hadoop-hdfs-client/lib/*.jar /usr/hdp/current/storm-client/extlib/ \ No newline at end of file From 06f24d7224e0109c83b546f3e4c44199cd7e14aa Mon Sep 17 00:00:00 2001 From: Ravi Peri Date: Tue, 18 Apr 2017 00:42:30 -0700 Subject: [PATCH 07/10] Fix typo in filename --- ...stal-storm-hdfs-extlib.bash => install-storm-hdfs-extlib.bash} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename install-storm-hdfs-extlib/{instal-storm-hdfs-extlib.bash => install-storm-hdfs-extlib.bash} (100%) diff --git a/install-storm-hdfs-extlib/instal-storm-hdfs-extlib.bash b/install-storm-hdfs-extlib/install-storm-hdfs-extlib.bash similarity index 100% rename from install-storm-hdfs-extlib/instal-storm-hdfs-extlib.bash rename to install-storm-hdfs-extlib/install-storm-hdfs-extlib.bash From 01b10e645f95b9bb94ede03e1629ffb113a55a72 Mon Sep 17 00:00:00 2001 From: Ravi Peri Date: Wed, 19 Apr 2017 11:28:39 -0700 Subject: [PATCH 08/10] Script to fix scpwebapi port --- storm-scp/fix-scpwebapi-port.bash | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 storm-scp/fix-scpwebapi-port.bash diff --git a/storm-scp/fix-scpwebapi-port.bash b/storm-scp/fix-scpwebapi-port.bash new file mode 100644 index 0000000..1debcba --- /dev/null +++ b/storm-scp/fix-scpwebapi-port.bash @@ -0,0 +1,3 @@ +#!/bin/bash +cp /usr/hdp/current/storm-client/scpwebapi/resources/scpwebapi-humboldt.yaml /usr/hdp/current/storm-client/resources/scpwebapi-humboldt.yaml.bak +sed -e "s/scp.webapi.ui.port: 80/scp.webapi.ui.port: 8888/g" /usr/hdp/current/storm-client/resources/scpwebapi-humboldt.yaml >/usr/hdp/current/storm-client/resources/scpwebapi-humboldt.yaml \ No newline at end of file From f63f4d28a8e289849f58634eeec3528dd64687c8 Mon Sep 17 00:00:00 2001 From: Ravi Peri Date: Wed, 19 Apr 2017 11:30:43 -0700 Subject: [PATCH 09/10] Fix config file path --- storm-scp/fix-scpwebapi-port.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storm-scp/fix-scpwebapi-port.bash b/storm-scp/fix-scpwebapi-port.bash index 1debcba..2f117cf 100644 --- a/storm-scp/fix-scpwebapi-port.bash +++ b/storm-scp/fix-scpwebapi-port.bash @@ -1,3 +1,3 @@ #!/bin/bash -cp /usr/hdp/current/storm-client/scpwebapi/resources/scpwebapi-humboldt.yaml /usr/hdp/current/storm-client/resources/scpwebapi-humboldt.yaml.bak -sed -e "s/scp.webapi.ui.port: 80/scp.webapi.ui.port: 8888/g" /usr/hdp/current/storm-client/resources/scpwebapi-humboldt.yaml >/usr/hdp/current/storm-client/resources/scpwebapi-humboldt.yaml \ No newline at end of file +cp /usr/hdp/current/storm-client/scpwebapi/resources/scpwebapi-humboldt.yaml /usr/hdp/current/storm-client/scpwebapi/resources/scpwebapi-humboldt.yaml.bak +sed -e "s/scp.webapi.ui.port: 80/scp.webapi.ui.port: 8888/g" /usr/hdp/current/storm-client/scpwebapi/resources/scpwebapi-humboldt.yaml >/usr/hdp/current/storm-client/scpwebapi/resources/scpwebapi-humboldt.yaml \ No newline at end of file From 14867602af87ef62ed7130ac161556d303b7ba3c Mon Sep 17 00:00:00 2001 From: Ravi Peri Date: Wed, 19 Apr 2017 11:58:42 -0700 Subject: [PATCH 10/10] fix backup logic to write to timestamped file. Fix sed logic --- storm-scp/fix-scpwebapi-port.bash | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/storm-scp/fix-scpwebapi-port.bash b/storm-scp/fix-scpwebapi-port.bash index 2f117cf..e8063cb 100644 --- a/storm-scp/fix-scpwebapi-port.bash +++ b/storm-scp/fix-scpwebapi-port.bash @@ -1,3 +1,6 @@ #!/bin/bash -cp /usr/hdp/current/storm-client/scpwebapi/resources/scpwebapi-humboldt.yaml /usr/hdp/current/storm-client/scpwebapi/resources/scpwebapi-humboldt.yaml.bak -sed -e "s/scp.webapi.ui.port: 80/scp.webapi.ui.port: 8888/g" /usr/hdp/current/storm-client/scpwebapi/resources/scpwebapi-humboldt.yaml >/usr/hdp/current/storm-client/scpwebapi/resources/scpwebapi-humboldt.yaml \ No newline at end of file +cp /usr/hdp/current/storm-client/scpwebapi/resources/scpwebapi-humboldt.yaml /usr/hdp/current/storm-client/scpwebapi/resources/scpwebapi-humboldt.yaml.$(date +"%s") +sed -e "s/scp.webapi.ui.port: 80/scp.webapi.ui.port: 8888/g" /usr/hdp/current/storm-client/scpwebapi/resources/scpwebapi-humboldt.yaml > /tmp/scpwebapi-humboldt.yaml.new +if [ $? == 0 ]; then + cp -f /tmp/scpwebapi-humboldt.yaml.new /usr/hdp/current/storm-client/scpwebapi/resources/scpwebapi-humboldt.yaml +fi \ No newline at end of file