Fix Packer and upgrade Py3 as dependency

CONTRIBUTING.adoc

@@ -22,6 +22,10 @@ to ensure the following steps have been taken:
 
 - [ ] If you changed the workflows used in the workshop, have you implemented the corresponding flow automation and tests to validate it?
 
+### For Deployment related changes:
+
+- [ ] If you changed the Deployment Automation (Terraform, Setup scripts, etc.) used in the workshop, have you tested both Packer AMI and Base AMI deployment?
+
 ## Testing
 
 In order to ensure that changes to this repository don't inadvertently affect the flow used in the workshops we provide automated tests of the workflow. *Please ensure that all tests run successfully before submitting PRs.*
@@ -44,21 +48,31 @@ ENABLE_KERBEROS=no
 cd /tmp/resources
 ----
 
-. Retrieve the schema file used in the workshop and save it locally
+. If you have opted not to deploy the CDSW model set SKIP_CDSW to true
 +
 [source,shell]
 ----
-wget https://raw.githubusercontent.com/asdaraujo/edge2ai-workshop/master/sensor.avsc
+export SKIP_CDSW=true
 ----
 
 . Run the tests
 +
 [source,shell]
 ----
-export SCHEMA_FILE=$PWD/sensor.avsc
+source /opt/rh/rh-python36/enable
 pytest tests
 ----
 
+### Supplying your own schema
+
+If you want to supply your own schema for testing, save it locally and export the path as SCHEMA_FILE
+
+[source,shell]
+----
+wget https://raw.githubusercontent.com/asdaraujo/edge2ai-workshop/master/sensor.avsc
+export SCHEMA_FILE=$PWD/sensor.avsc
+----
+
 ### Keeping a flow after the test run
 
 The test scripts will automate the full workshop flow setup, run the tests and then tear everything down. If you want to keep the flow after the tests, set the following environment variable before running the tests:

setup/terraform/.env.template

@@ -1,6 +1,9 @@
 # Set the number of clusters you want to deploy
 export TF_VAR_cluster_count=1
 
+# Set limit on how many clusteres should be deployed in parallel
+export TF_VAR_parallelism=25
+
 # Set your admin parameters
 export TF_VAR_owner=<CHANGE_ME>
 export TF_VAR_web_server_admin_email=<CHANGE_ME>

setup/terraform/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.6-alpine
+FROM golang:alpine
 
 LABEL maintainer="Dan Chaffelson <[email protected]>"
 LABEL site="https://github.com/asdaraujo/edge2ai-workshop/tree/master/setup"
@@ -7,21 +7,25 @@ ENV PYTHONUNBUFFERED=0
 ENV TZ=${TZ:-"Europe/London"}
 ENV ENV=${ENV:-'/setup/terraform/.env'}
 ENV STACK=${STACK:-'/setup/terraform/resources/stack.sh'}
-ENV TERRAFORMVERSION=${TERRAFORMVERSION:-0.12.19}
+ENV TERRAFORMVERSION=${TERRAFORMVERSION:-0.12.20}
+ENV PACKERVERSION=${PACKERVERSION:-1.5.1}
 
 RUN apk update && apk upgrade \
-    && apk add --no-cache git python3-dev linux-headers wget unzip bash openssh jq curl tzdata
+    && apk add --no-cache git python3-dev py-pip git linux-headers wget unzip bash openssh jq curl tzdata
+
+RUN cp /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 RUN pip install --upgrade pip \
-    && cp /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+    && pip install --no-cache jinja2 pyyaml
 
 RUN wget https://releases.hashicorp.com/terraform/${TERRAFORMVERSION}/terraform_${TERRAFORMVERSION}_linux_amd64.zip \
     && unzip terraform_${TERRAFORMVERSION}_linux_amd64.zip \
     && mv terraform /usr/local/bin/terraform \
     && rm terraform_${TERRAFORMVERSION}_linux_amd64.zip
 
-RUN pip install --no-cache jinja2 pyyaml
-
+RUN curl -sSLo /tmp/packer.zip https://releases.hashicorp.com/packer/${PACKERVERSION}/packer_${PACKERVERSION}_linux_amd64.zip \
+    && unzip /tmp/packer.zip -d /usr/local/bin \
+    && rm /tmp/packer.zip
 
 WORKDIR /edge2ai-workshop/setup/terraform
 

setup/terraform/launch.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 BASE_DIR=$(cd $(dirname $0); pwd -L)
-source $BASE_DIR/common.sh
+source ${BASE_DIR}/common.sh
 
-mkdir -p $BASE_DIR/logs
+mkdir -p "${BASE_DIR}"/logs
 (
 set -o errexit
 set -o nounset
@@ -20,22 +20,28 @@ check_python_modules
 
 # Check if enddate is close
 WARNING_THRESHOLD_DAYS=2
-DATE_CHECK=$(remaining_days "$TF_VAR_enddate")
-if [ "$DATE_CHECK" -le "0" ]; then
+DATE_CHECK=$(python -c "
+from datetime import datetime, timedelta
+dt = datetime.now()
+dt = dt.replace(hour=0, minute=0, second=0, microsecond=0)
+print((datetime.strptime('$TF_VAR_enddate', '%m%d%Y') - dt).days)
+")
+
+if [ "${DATE_CHECK}" -le "0" ]; then
   echo 'ERROR: The expiration date for your environment is either set for today or already in the past.'
   echo '       Please update "TF_VAR_endddate" in .env.'"$NAMESPACE"' and try again.'
   exit 1
 elif [ "$DATE_CHECK" -le "$WARNING_THRESHOLD_DAYS" ]; then
   echo -n "WARNING: Your environment will expire in less than $WARNING_THRESHOLD_DAYS days. Do you really want to continue? "
   read CONFIRM
-  CONFIRM=$(echo $CONFIRM | tr a-z A-Z)
+  CONFIRM=$(echo "${CONFIRM}" | tr a-z A-Z)
   if [ "$CONFIRM" != "Y" -a "$CONFIRM" != "YES" ]; then
     echo 'Please update "TF_VAR_endddate" in .env.'"$NAMESPACE"' and try again.'
     exit 1
   fi
 fi
 
-mkdir -p $NAMESPACE_DIR
+mkdir -p "${NAMESPACE_DIR}"
 
 # Perform a quick configuration sanity check before calling Terraform
 source $BASE_DIR/resources/common.sh
@@ -59,17 +65,17 @@ if [ -s $NAMESPACE_DIR/terraform.state ]; then
     jq -r '.values[]?.resources[]? | select(.type == "aws_security_group").values.id | "\"\(.)\""' | \
     tr "\n" "," | sed 's/,$//')]"
 fi
-terraform apply -auto-approve -parallelism=20 -refresh=true -state=$NAMESPACE_DIR/terraform.state
+terraform apply -auto-approve -parallelism="${TF_VAR_parallelism}" -refresh=true -state=$NAMESPACE_DIR/terraform.state
 
 log "Deployment completed successfully"
 
 echo ""
 echo "Instances:"
-$BASE_DIR/list-details.sh $NAMESPACE
+"${BASE_DIR}/list-details.sh" "${NAMESPACE}"
 
 echo ""
 echo "Health checks:"
-$BASE_DIR/check-services.sh $NAMESPACE
+"${BASE_DIR}/check-services.sh" "${NAMESPACE}"
 
 if [ "$TF_VAR_deploy_cdsw_model" == "true" ]; then
   echo "${C_YELLOW}    NOTE: CDSW model is being deployed in the background."
@@ -81,6 +87,6 @@ fi
 
 echo ""
 echo "Uploading instance details to Web Server:"
-$BASE_DIR/upload-instance-details.sh $NAMESPACE
+"${BASE_DIR}/upload-instance-details.sh" "${NAMESPACE}"
 
 ) 2>&1 | tee $BASE_DIR/logs/setup.log.${1:-unknown}.$(date +%Y%m%d%H%M%S)

setup/terraform/packer.json

@@ -9,7 +9,8 @@
     "source_ami": "{{user `base_ami`}}",
     "instance_type": "{{user `instance_type`}}",
     "ssh_username": "{{user `ssh_username`}}",
-    "ami_name": "{{user `owner`}}-{{user `namespace`}}-packer-{{timestamp}}",
+    "ami_name": "{{user `owner`}}-{{user `namespace`}}-cdfworkshop-{{timestamp}}",
+    "ena_support": true,
     "launch_block_device_mappings": [
         {
           "device_name": "/dev/sda1",
@@ -18,12 +19,6 @@
           "delete_on_termination": true
         }
       ],
-    "ami_block_device_mappings": [ {
-         "device_name": "/dev/sdf",
-         "volume_size": 200,
-         "volume_type": "gp2",
-         "delete_on_termination": true
-     }],
     "tags": {
       "Name": "{{user `owner`}}-{{user `namespace`}}-packer-{{timestamp}}",
       "owner": "{{user `owner`}}",

setup/terraform/provider.tf

@@ -7,4 +7,4 @@ provider "aws" {
 
 provider "null" {
   version = "~> 2.1"
-}
+}

setup/terraform/resources/cdsw_setup.py

@@ -1,4 +1,6 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
 
 import requests
 import sys

setup/terraform/resources/create_cluster.py

@@ -1,4 +1,8 @@
-from __future__ import print_function
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""Cluster creation controls for Cloudera Manager submission"""
+
 from cm_client.rest import ApiException
 from collections import namedtuple
 from datetime import datetime
@@ -140,7 +144,7 @@ def wait(self, cmd, timeout=None):
         try:
             cmd_api_instance = cm_client.CommandsResourceApi(self.api_client)
             while True:
-                cmd = cmd_api_instance.read_command(long(cmd.id))
+                cmd = cmd_api_instance.read_command(int(cmd.id))
                 print(datetime.strftime(datetime.now(), '%c'))
                 print_cmd(cmd)
                 if not cmd.active: