run-cluster

#!/bin/sh
#Runs your script on the cluster using spark2 and Python 3. 
#This requires to export the 2 environment variables below, and to add --conf spark.yarn.appMasterEnv.SPARK_HOME=/usr/hdp/current/spark2-client to your spark-submit
#For Python 3 we need to set HDP_VERSION, and PYSPARK_PYTHON
zip -r histogram.zip histogram/
spark-submit --master yarn --deploy-mode cluster \
--py-files histogram.zip spark.py