From e43d79b11c91f2ad2ddd7ae0bdc4339217ba7034 Mon Sep 17 00:00:00 2001
From: Hiroshi Tanaka <hirtanak@gmail.com>
Date: Sat, 31 Aug 2019 13:37:52 +0900
Subject: [PATCH] initial commit

---
 LICENSE                                       |   21 +
 README.md                                     |   94 ++
 project.ini                                   |   31 +
 .../default/chef/roles/pbspro_execute_role.rb |    9 +
 specs/default/chef/roles/pbspro_login_role.rb |    9 +
 .../default/chef/roles/pbspro_master_role.rb  |   14 +
 .../chef/site-cookbooks/pbspro/README.md      |   29 +
 .../pbspro/attributes/default.rb              |   35 +
 .../pbspro/files/default/autostart.py         |  606 +++++++++
 .../pbspro/files/default/autostart_hook.py    |   61 +
 .../pbspro/files/default/autostart_test.py    | 1210 +++++++++++++++++
 .../pbspro/files/default/autostop.rb          |   74 +
 .../pbspro/files/default/config               |    1 +
 .../pbspro/files/default/doqmgr.sh            |   40 +
 .../pbspro/files/default/logging_init.py      |   40 +
 .../pbspro/files/default/mockpbs.py           |  131 ++
 .../pbspro/files/default/pbs_driver.py        |  301 ++++
 .../pbspro/files/default/pbscc.py             |  239 ++++
 .../site-cookbooks/pbspro/files/default/pi.py |   76 ++
 .../site-cookbooks/pbspro/files/default/pi.sh |    8 +
 .../pbspro/files/default/runpi.sh             |    9 +
 .../pbspro/files/default/sched.config         |   28 +
 .../pbspro/files/default/submit_hook.py       |  259 ++++
 .../pbspro/files/default/submit_hook_test.py  |  269 ++++
 .../chef/site-cookbooks/pbspro/metadata.rb    |   12 +
 .../pbspro/recipes/autostart.rb               |   60 +
 .../site-cookbooks/pbspro/recipes/autostop.rb |   19 +
 .../site-cookbooks/pbspro/recipes/default.rb  |    3 +
 .../site-cookbooks/pbspro/recipes/execute.rb  |  130 ++
 .../site-cookbooks/pbspro/recipes/login.rb    |   32 +
 .../pbspro/recipes/scheduler.rb               |   51 +
 .../site-cookbooks/pbspro/recipes/skel.rb     |   22 +
 .../pbspro/recipes/submit_hook.rb             |   51 +
 .../pbspro/spec/default_spec.rb               |  110 ++
 .../site-cookbooks/pbspro/spec/spec_helper.rb |   20 +
 .../pbspro/templates/default/mom_config.erb   |    1 +
 .../pbspro/templates/default/pbs.conf.erb     |    9 +
 .../templates/default/pbsmastercron.rb.erb    |   16 +
 .../pbspro/templates/default/server_name.erb  |    1 +
 .../templates/default/server_name_exec.erb    |    1 +
 specs/default/chefstar.tar.gz                 |  Bin 0 -> 35656 bytes
 specs/default/cluster-init/files/README.txt   |    5 +
 specs/default/cluster-init/files/runpbs.sh    |   14 +
 specs/default/cluster-init/files/script.sh    |   19 +
 .../default/cluster-init/files/starccmrun.sh  |   15 +
 .../cluster-init/files/starccmsetup.sh        |    8 +
 specs/default/cluster-init/scripts/README.txt |   10 +
 specs/default/cluster-init/tests/README.txt   |    5 +
 .../execute/chef/roles/pbspro_execute_role.rb |    9 +
 specs/execute/chef/roles/pbspro_login_role.rb |    9 +
 .../execute/chef/roles/pbspro_master_role.rb  |   14 +
 .../chef/site-cookbooks/pbspro/README.md      |   29 +
 .../pbspro/attributes/default.rb              |   35 +
 .../pbspro/files/default/autostart.py         |  606 +++++++++
 .../pbspro/files/default/autostart_hook.py    |   61 +
 .../pbspro/files/default/autostart_test.py    | 1210 +++++++++++++++++
 .../pbspro/files/default/autostop.rb          |   74 +
 .../pbspro/files/default/config               |    1 +
 .../pbspro/files/default/doqmgr.sh            |   40 +
 .../pbspro/files/default/logging_init.py      |   40 +
 .../pbspro/files/default/mockpbs.py           |  131 ++
 .../pbspro/files/default/pbs_driver.py        |  301 ++++
 .../pbspro/files/default/pbscc.py             |  239 ++++
 .../site-cookbooks/pbspro/files/default/pi.py |   76 ++
 .../site-cookbooks/pbspro/files/default/pi.sh |    8 +
 .../pbspro/files/default/runpi.sh             |    9 +
 .../pbspro/files/default/sched.config         |   28 +
 .../pbspro/files/default/submit_hook.py       |  259 ++++
 .../pbspro/files/default/submit_hook_test.py  |  269 ++++
 .../chef/site-cookbooks/pbspro/metadata.rb    |   12 +
 .../pbspro/recipes/autostart.rb               |   60 +
 .../site-cookbooks/pbspro/recipes/autostop.rb |   19 +
 .../site-cookbooks/pbspro/recipes/default.rb  |    3 +
 .../site-cookbooks/pbspro/recipes/execute.rb  |  130 ++
 .../site-cookbooks/pbspro/recipes/login.rb    |   32 +
 .../pbspro/recipes/scheduler.rb               |   51 +
 .../site-cookbooks/pbspro/recipes/skel.rb     |   22 +
 .../pbspro/recipes/submit_hook.rb             |   51 +
 .../pbspro/spec/default_spec.rb               |  110 ++
 .../site-cookbooks/pbspro/spec/spec_helper.rb |   20 +
 .../pbspro/templates/default/mom_config.erb   |    1 +
 .../pbspro/templates/default/pbs.conf.erb     |    9 +
 .../templates/default/pbsmastercron.rb.erb    |   16 +
 .../pbspro/templates/default/server_name.erb  |    1 +
 .../templates/default/server_name_exec.erb    |    1 +
 specs/execute/cluster-init/files/README.txt   |    5 +
 specs/execute/cluster-init/files/runpbs.sh    |   14 +
 specs/execute/cluster-init/files/script.sh    |   19 +
 .../execute/cluster-init/files/starccmrun.sh  |   15 +
 .../cluster-init/files/starccmsetup.sh        |    8 +
 .../cluster-init/scripts/10.execute.sh        |   70 +
 specs/execute/cluster-init/tests/README.txt   |    5 +
 specs/master/cluster-init/files/starccmrun.sh |   22 +
 .../scripts/10.install_starccm.sh             |  110 ++
 specs/master/cluster-init/tests/README.txt    |    5 +
 templates/pbs_extended_nfs_starccm.txt        |  281 ++++
 96 files changed, 8748 insertions(+)
 create mode 100644 LICENSE
 create mode 100644 README.md
 create mode 100644 project.ini
 create mode 100644 specs/default/chef/roles/pbspro_execute_role.rb
 create mode 100644 specs/default/chef/roles/pbspro_login_role.rb
 create mode 100644 specs/default/chef/roles/pbspro_master_role.rb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/README.md
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/attributes/default.rb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/autostart.py
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/autostart_hook.py
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/autostart_test.py
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/autostop.rb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/config
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/doqmgr.sh
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/logging_init.py
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/mockpbs.py
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/pbs_driver.py
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/pbscc.py
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/pi.py
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/pi.sh
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/runpi.sh
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/sched.config
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/submit_hook.py
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/files/default/submit_hook_test.py
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/metadata.rb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/recipes/autostart.rb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/recipes/autostop.rb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/recipes/default.rb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/recipes/execute.rb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/recipes/login.rb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/recipes/scheduler.rb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/recipes/skel.rb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/recipes/submit_hook.rb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/spec/default_spec.rb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/spec/spec_helper.rb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/templates/default/mom_config.erb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/templates/default/pbs.conf.erb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/templates/default/pbsmastercron.rb.erb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/templates/default/server_name.erb
 create mode 100644 specs/default/chef/site-cookbooks/pbspro/templates/default/server_name_exec.erb
 create mode 100644 specs/default/chefstar.tar.gz
 create mode 100644 specs/default/cluster-init/files/README.txt
 create mode 100644 specs/default/cluster-init/files/runpbs.sh
 create mode 100644 specs/default/cluster-init/files/script.sh
 create mode 100644 specs/default/cluster-init/files/starccmrun.sh
 create mode 100644 specs/default/cluster-init/files/starccmsetup.sh
 create mode 100644 specs/default/cluster-init/scripts/README.txt
 create mode 100644 specs/default/cluster-init/tests/README.txt
 create mode 100644 specs/execute/chef/roles/pbspro_execute_role.rb
 create mode 100644 specs/execute/chef/roles/pbspro_login_role.rb
 create mode 100644 specs/execute/chef/roles/pbspro_master_role.rb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/README.md
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/attributes/default.rb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/autostart.py
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/autostart_hook.py
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/autostart_test.py
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/autostop.rb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/config
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/doqmgr.sh
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/logging_init.py
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/mockpbs.py
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/pbs_driver.py
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/pbscc.py
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/pi.py
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/pi.sh
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/runpi.sh
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/sched.config
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/submit_hook.py
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/files/default/submit_hook_test.py
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/metadata.rb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/recipes/autostart.rb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/recipes/autostop.rb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/recipes/default.rb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/recipes/execute.rb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/recipes/login.rb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/recipes/scheduler.rb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/recipes/skel.rb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/recipes/submit_hook.rb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/spec/default_spec.rb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/spec/spec_helper.rb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/templates/default/mom_config.erb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/templates/default/pbs.conf.erb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/templates/default/pbsmastercron.rb.erb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/templates/default/server_name.erb
 create mode 100644 specs/execute/chef/site-cookbooks/pbspro/templates/default/server_name_exec.erb
 create mode 100644 specs/execute/cluster-init/files/README.txt
 create mode 100644 specs/execute/cluster-init/files/runpbs.sh
 create mode 100644 specs/execute/cluster-init/files/script.sh
 create mode 100644 specs/execute/cluster-init/files/starccmrun.sh
 create mode 100644 specs/execute/cluster-init/files/starccmsetup.sh
 create mode 100644 specs/execute/cluster-init/scripts/10.execute.sh
 create mode 100644 specs/execute/cluster-init/tests/README.txt
 create mode 100644 specs/master/cluster-init/files/starccmrun.sh
 create mode 100644 specs/master/cluster-init/scripts/10.install_starccm.sh
 create mode 100644 specs/master/cluster-init/tests/README.txt
 create mode 100644 templates/pbs_extended_nfs_starccm.txt

diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..2d48557
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright Hiroshi Tanaka. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..f20afb0
--- /dev/null
+++ b/README.md
@@ -0,0 +1,94 @@
+# Azure CycleCloud template for Siemens STAR-CCM+
+
+## Prerequisites
+
+1. Prepaire for your STAR-CCM+ bilnary.
+2. Install CycleCloud CLI
+
+## How to install 
+
+1. tar zxvf cyclecloud-STAR-CCM+<version>.tar.gz
+2. cd cyclecloud-STAR-CCM+<version>
+2. put STAR-CCM+ library/model on <template>/blob directory
+3. Rewrite "Files" attiribute for your binariy in "project.ini" file. 
+4. run "cyclecloud project upload azure-storage" for uploading template to CycleCloud
+5. "cyclecloud import_template -f templates/pbs_extended_nfs_starccm.txt" for register this template to your CycleCloud
+
+## How to run Siemens STAR-CCM+
+
+1. Create Execute Node mannually
+2. Check Node IP Address
+3. Create hosts file for your nodes
+4. qsub ~/starccmrun.sh (sample as below)
+
+<pre><code>
+#!/bin/bash
+#PBS -j oe
+#PBS -l select=2:ncpus=44
+PPN=44
+NP=88
+
+cd ${PBS_O_WORKDIR}
+
+source /etc/profile.d/starccm.sh
+
+INSTALL_DIR="/shared/home/azureuser/apps"
+I_MPI_ROOT=/opt/intel/impi/2018.4.274
+source /opt/intel/impi/2018.4.274/intel64/bin/mpivars.sh
+STARCCMPLUS_VERSION=14.04.011
+PRECISION=-R8
+
+INPUT=/shared/home/azureuser/test1.sim
+export I_MPI_FABRICS=shm:ofa
+export MPI_IB_PKEY=0x800c
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${INSTALL_DIR}/${STARCCMPLUS_VERSION}${PRECISION}/STAR-CCM+${STARCCMPLUS_VERSION}${PRECISION}/star/lib
+
+PODKEY=<removed>
+
+DISPLAY=0:0
+
+CDLMD_LICENSE_FILE=1999@flex.cd-adapco.com
+
+#/opt/intel/impi/2018.4.274/intel64/bin/mpirun -machinefile /shared/home/azureuser/hosts IMB-MPI1 pingpong
+
+${INSTALL_DIR}/${STARCCMPLUS_VERSION}${PRECISION}/STAR-CCM+${STARCCMPLUS_VERSION}${PRECISION}/star/bin/starccm+ -np ${NP} -machinefile /shared/home/azureuser/hosts -licpath ${CDLMD_LICENSE_FILE} -power -podkey ${PODKEY} -mpi intel -mpiflags "-ppn 1 -env I_MPI_DEBUG 5 -env I_MPI_FABRICS shm:ofa" -batch /shared/home/azureuser/solve_SS_b787_images.java -load ${INPUT}
+</pre></code>
+
+
+# Azure CycleCloud用テンプレート:STAR-CCM+(NFS/PBSPro)
+
+[Azure CycleCloud](https://docs.microsoft.com/en-us/azure/cyclecloud/) はMicrosoft Azure上で簡単にCAE/HPC/Deep Learning用のクラスタ環境を構築できるソリューションです。
+
+![Azure CycleCloudの構築・テンプレート構成](https://raw.githubusercontent.com/hirtanak/osspbsdefault/master/AzureCycleCloud-OSSPBSDefault.png "Azure CycleCloudの構築・テンプレート構成")
+
+Azure CyceCloudのインストールに関しては、[こちら](https://docs.microsoft.com/en-us/azure/cyclecloud/quickstart-install-cyclecloud) のドキュメントを参照してください。
+
+STAR-CCM+用のテンプレートになっています。
+以下の構成、特徴を持っています。
+
+1. OSS PBS ProジョブスケジューラをMasterノードにインストール
+2. H16r, H16r_Promo, HC44rs, HB60rsを想定したテンプレート、イメージ
+	 - OpenLogic CentOS 7.6 HPC を利用 
+3. Masterノードに512GB * 2 のNFSストレージサーバを搭載
+	 - Executeノード（計算ノード）からNFSをマウント
+4. MasterノードのIPアドレスを固定設定
+	 - 一旦停止後、再度起動した場合にアクセスする先のIPアドレスが変更されない
+
+![OSS PBS Default テンプレート構成](https://raw.githubusercontent.com/hirtanak/osspbsdefault/master/OSSPBSDefaultDiagram.png "OSS PBS Default テンプレート構成")
+
+OSS PBS Defaultテンプレートインストール方法
+
+前提条件: テンプレートを利用するためには、Azure CycleCloud CLIのインストールと設定が必要です。詳しくは、 [こちら](https://docs.microsoft.com/en-us/azure/cyclecloud/install-cyclecloud-cli) の文書からインストールと展開されたAzure CycleCloudサーバのFQDNの設定が必要です。
+
+1. テンプレート本体をダウンロード
+2. 展開、ディレクトリ移動
+3. cyclecloudコマンドラインからテンプレートインストール 
+   - tar zxvf cyclecloud-STAR-CCM+<version>.tar.gz
+   - cd cyclecloud-STAR-CCM+<version>
+   - cyclecloud project upload azure-storage
+   - cyclecloud import_template -f templates/pbs_extended_nfs_starccm.txt
+4. 削除したい場合、 cyclecloud delete_template STAR-CCM+ コマンドで削除可能
+
+***
+Copyright Hiroshi Tanaka, hirtanak@gmail.com, @hirtanak All rights reserved.
+Use of this source code is governed by MIT license that can be found in the LICENSE file.
diff --git a/project.ini b/project.ini
new file mode 100644
index 0000000..29d1e41
--- /dev/null
+++ b/project.ini
@@ -0,0 +1,31 @@
+[project]
+version = 1.0.1
+type = application
+name = STAR-CCMplus
+
+# You have to put your STAR-CCM+ binary on blobs directory and write this file for uploading azure cyclecloud
+# pbspr files are required for job schedulering.
+[blobs]
+# Files = pbspro-execution-18.1.4-0.x86_64.rpm, pbspro-server-18.1.4-0.x86_64.rpm, pbspro-client-18.1.4-0.x86_64.rpm, <your STAR-CCM+ binary>, <your model>
+Files =  pbspro-execution-18.1.4-0.x86_64.rpm, pbspro-server-18.1.4-0.x86_64.rpm, pbspro-client-18.1.4-0.x86_64.rpm, STAR-CCM+12.04.010_01_linux-x86_64-r8.tar.gz, STAR-CCM+14.04.011_02_linux-x86_64-r8.tar.gz, test1.sim
+
+[spec master]
+run_list = role[pbspro_master_role]
+
+[spec execute]
+run_list = role[pbspro_execute_role]
+
+[spec viz]
+run_list = role[winpost]
+
+[config PBS.template.version]
+Required = True
+Label = PBS cyclecloud template version
+Description = Version of PBS cyclecloud template version to install on the cluster.
+DefaultValue = 1.3.0
+
+[config STAR-CCM+.version]
+Required = True
+Label = STAR-CCM+ version
+Description = Version of STAR-CCM+ to install on the cluster. Package should be named STAR-CCM+<versiion>_<platform><percision>.tar.gz
+DefaultValue = 14.04.011
diff --git a/specs/default/chef/roles/pbspro_execute_role.rb b/specs/default/chef/roles/pbspro_execute_role.rb
new file mode 100644
index 0000000..175600d
--- /dev/null
+++ b/specs/default/chef/roles/pbspro_execute_role.rb
@@ -0,0 +1,9 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+name "pbspro_execute_role"
+description "PBSPro Execute Role"
+run_list("recipe[cshared::client]",
+  "recipe[cuser]",
+  "recipe[pbspro::execute]",
+  "recipe[cganglia::client]")
diff --git a/specs/default/chef/roles/pbspro_login_role.rb b/specs/default/chef/roles/pbspro_login_role.rb
new file mode 100644
index 0000000..1b41949
--- /dev/null
+++ b/specs/default/chef/roles/pbspro_login_role.rb
@@ -0,0 +1,9 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+name "pbspro_login_role"
+description "PBSPro Login Role"
+run_list("recipe[cshared::client]",
+  "recipe[cuser]",
+  "recipe[pbspro::login]",
+  "recipe[cganglia::client]")
diff --git a/specs/default/chef/roles/pbspro_master_role.rb b/specs/default/chef/roles/pbspro_master_role.rb
new file mode 100644
index 0000000..651d353
--- /dev/null
+++ b/specs/default/chef/roles/pbspro_master_role.rb
@@ -0,0 +1,14 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+name "pbspro_master_role"
+description "Open PBSPro Master Role"
+run_list("role[scheduler]",
+  "recipe[cshared::directories]",
+  "recipe[pbspro::skel]",
+  "recipe[cuser]",
+  "recipe[cshared::server]",
+  "recipe[pbspro::scheduler]",
+  "recipe[cganglia::server]")
+
+default_attributes "cyclecloud" => { "discoverable" => true }
diff --git a/specs/default/chef/site-cookbooks/pbspro/README.md b/specs/default/chef/site-cookbooks/pbspro/README.md
new file mode 100644
index 0000000..7d779fe
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/README.md
@@ -0,0 +1,29 @@
+Description
+====
+
+Requirements
+====
+
+Usage
+====
+
+
+Platform
+----
+
+Tested on:
+
+Cookbooks
+----
+
+
+Resources and Providers
+====
+
+TODO
+
+Attributes
+====
+
+TODO
+====
diff --git a/specs/default/chef/site-cookbooks/pbspro/attributes/default.rb b/specs/default/chef/site-cookbooks/pbspro/attributes/default.rb
new file mode 100644
index 0000000..6dfd8a9
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/attributes/default.rb
@@ -0,0 +1,35 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+default[:pbspro][:version] = "18.1.4-0"
+default[:pbspro][:slots] = nil
+
+default[:pbspro][:is_grouped] = true
+
+default[:pbspro][:submit_hook][:__comment__] = "This file was generated by serializing node[:cyclecloud][:pbspro][:submit_hook]."
+default[:pbspro][:submit_hook][:disable_eager_packing] = true
+default[:pbspro][:submit_hook][:enabled] = true
+
+default[:pbspro][:submit_hook][:logging][:level] = "INFO"
+default[:pbspro][:submit_hook][:logging][:filename] = "#{node[:cyclecloud][:bootstrap]}/pbs/submit_hook.log"
+default[:pbspro][:submit_hook][:logging][:filemode] = "a"
+default[:pbspro][:submit_hook][:logging][:format] = "%(asctime)s %(levelname)s %(message)s"
+
+
+default[:pbspro][:autoscale_hook][:__comment__] = "This file was generated by serializing node[:cyclecloud][:pbspro][:autostart_hook]."
+default[:pbspro][:autoscale_hook][:src_dirs] = ["#{node[:cyclecloud][:home]}/system/embedded/lib/python2.7/site-packages",
+                                                "#{node[:cyclecloud][:bootstrap]}/pbs"
+                                            ]
+# pass through to the hook config json file
+default[:pbspro][:autoscale_hook][:cyclecloud_home] = node[:cyclecloud][:home]
+default[:pbspro][:autoscale_hook][:autostart_log_level] = "DEBUG"
+default[:pbspro][:autoscale_hook][:autostart_log_file_level] = "DEBUG"
+
+if node[:cyclecloud][:node][:template] == "master"
+	default[:cyclecloud][:cluster][:autoscale][:idle_time_before_jobs] = 3600
+	default[:cyclecloud][:cluster][:autoscale][:idle_time_after_jobs] = 300
+else
+	# the pbs autoscaler should be removing idle nodes, but if after an hour of idle time assume something is wrong with the autoscaler.
+	default[:cyclecloud][:cluster][:autoscale][:idle_time_before_jobs] = 4 * 3600
+	default[:cyclecloud][:cluster][:autoscale][:idle_time_after_jobs] = 4 * 3600
+end
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/autostart.py b/specs/default/chef/site-cookbooks/pbspro/files/default/autostart.py
new file mode 100644
index 0000000..53dc485
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/autostart.py
@@ -0,0 +1,606 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+import logging_init  # import first to ensure other modules (requests) don't define logging.basicConfig first
+
+import collections
+import json
+import os
+import sys
+import traceback
+
+from cyclecloud import machine, clustersapi, autoscale_util, autoscaler as autoscalerlib,\
+    nodearrays
+from cyclecloud.autoscale_util import Record
+import cyclecloud.config
+from cyclecloud.job import Job, PackingStrategy
+import mockpbs
+import pbs_driver
+from pbscc import InvalidSizeExpressionError
+import pbscc
+from copy import deepcopy
+import numbers
+
+
+class PBSAutostart:
+    '''
+    1) Converts existing pbsnodes into cyclecloud.machine.Machine instances
+    2) Performs a 'true up' of the machine instances reported by the autoscale lib vs the machines discovered in step 1) using InstanceId.
+    3) Converts PBS jobs into cyclecloud.job.Job instances.
+    4) Uses the current state of the cluster (steps 1 and 2) to construct a cyclecloud.autoscaler.Autoscaler instance.
+    5) Feeds in jobs from step 3) into the Autoscaler, which performs a demand calculation of exactly which new machines are needed and which are idle.
+    6) Does a soft shutdown of idle nodes - first it sets them offline - `pbsnodes -o` - then, on subsequent iterations, if no jobs running on the node it
+       performs a shutdown.
+       
+       
+       To disable
+       sudo qmgr -c 'set hook autoscale enabled=false'
+    
+    '''
+    
+    def __init__(self, driver, clusters_api, cc_config, clock=None):
+        self.cc_config = cc_config
+        self.disable_grouping = cc_config.get("cyclecloud.cluster.autoscale.use_node_groups", True) is not True
+        self.driver = driver
+        self.clusters_api = clusters_api
+        self.clock = clock or Clock()
+        self.default_placement_attrs = self.cc_config.get("cyclecloud.placement_group.defaults", {"group_id": "single"})
+        
+    def query_jobs(self):
+        '''
+            Converts PBS jobs into cyclecloud.job.Job instances. It will also compress jobs that have the exact same requirements.
+        '''
+        scheduler_config = self.driver.scheduler_config()
+        scheduler_resources = [] + scheduler_config["resources"]
+        # special case for hostname so we can automatically place jobs onto the appropriate host
+        scheduler_resources.append("hostname")
+        scheduler_resources.append("instance_id")
+        
+        group_jobs = not self.disable_grouping
+        running_autoscale_jobs = []
+        idle_autoscale_jobs = []
+    
+        # get the raw string outputs first, and convert it second. This somewhat limits the
+        # race condition of asking the status of the queue twice.
+        running_raw_jobs_str, running_converter = self.driver.running_jobs()
+        queued_raw_jobs_str, queued_converter = self.driver.queued_jobs()
+        
+        running_raw_jobs = running_converter(running_raw_jobs_str)
+        queued_raw_jobs = queued_converter(queued_raw_jobs_str)
+        
+        raw_jobs = []
+        
+        for raw_job in running_raw_jobs:
+            # it is only running on a single node
+            if '+' not in raw_job["exec_vnode"]:
+                raw_jobs.append(raw_job)
+                continue
+            
+            for vnode in raw_job["exec_vnode"].split("+"):
+                sub_raw_job = deepcopy(raw_job)
+                sub_raw_job["exec_vnode"] = vnode
+                raw_jobs.append(sub_raw_job)
+        
+        for raw_job in queued_raw_jobs:
+            if not raw_job["resource_list"].get("select"):
+                raw_jobs.append(raw_job)
+            else:
+                # pbspro, like many schedulers, allows a varying set requirements for nodes in a single submission.
+                # we will break it apart here as if they had split them individually.
+                
+                place = raw_job["resource_list"].get("place")
+                slot_type = raw_job["resource_list"].get("slot_type")
+
+                chunks = pbscc.parse_select(raw_job)
+                for n, chunk in enumerate(chunks):
+                    # only pay the penalty of copies when we actually have multi-chunk jobs
+                    sub_raw_job = deepcopy(raw_job)
+
+                    if len(chunks) > 1:
+                        sub_raw_job["job_id"] = "%s.%d" % (sub_raw_job["job_id"], n) 
+                    
+                    sub_raw_job["resource_list"] = {}
+                    if place:
+                        sub_raw_job["resource_list"]["place"] = place
+                        
+                    if slot_type:
+                        sub_raw_job["resource_list"]["slot_type"] = slot_type
+                        
+                    sub_raw_job["resource_list"]["select"] = pbscc.format_select(chunk)
+                    chunk["nodect"] = int(chunk["select"])
+                    if "ncpus" not in chunk:
+                        chunk["ncpus"] = "1"
+                    
+                    for key, value in chunk.iteritems():
+                        if key not in ["select", "nodect"]:
+                            try:
+                                value = pbscc.parse_gb_size(key, value) * chunk["nodect"]
+                            except InvalidSizeExpressionError:
+                                pass
+                            
+                            sub_raw_job["resource_list"][key] = value
+                    
+                    sub_raw_job["nodect"] = sub_raw_job["resource_list"]["nodect"] = chunk["nodect"]
+                    raw_jobs.append(sub_raw_job)
+        
+        warnings = set()
+        raw_jobs = [x for x in raw_jobs if x["job_state"].upper() in [pbscc.JOB_STATE_QUEUED,
+                                                                      pbscc.JOB_STATE_RUNNING,
+                                                                      pbscc.JOB_STATE_BATCH]]
+        
+        for raw_job in raw_jobs:
+            pbs_job = mockpbs.mock_job(raw_job)
+            nodect = int(pbs_job.Resource_List["nodect"])
+            
+            if pbs_job["job_state"].upper() == pbscc.JOB_STATE_RUNNING:
+                # update running job
+                live_resources = pbscc.parse_exec_vnode(raw_job["exec_vnode"])
+                for key, value in live_resources.iteritems():
+                    # live resources are calculated on a per node basis, but the Resource_List is based
+                    # on a total basis.
+                    # we will normalize this below
+                    
+                    if isinstance(value, numbers.Number):
+                        pbs_job.Resource_List[key] = value * nodect
+                    else:
+                        pbs_job.Resource_List[key] = value
+                pbs_job["executing_hostname"] = live_resources["hostname"]
+                
+            is_array = bool(pbs_job.get("array", False))
+            
+            slots_per_job = int(pbs_job.Resource_List['ncpus']) / nodect 
+            slot_type = pbs_job.Resource_List["slot_type"]  # can be None, similar to {}.get("key"). It is a pbs class.
+            pbscc.info("found slot_type %s." % slot_type)
+            
+            placement = pbscc.parse_place(pbs_job.Resource_List.get("place"))
+                    
+            # Note: not sure we will ever support anything but group_id for autoscale purposes.
+            # User could pick, say, group=host, which implies an SMP job, not a parallel job.
+            
+            if placement.get("grouping", "group=group_id") != "group=group_id":
+                placement.pop("grouping")
+
+            if placement.get("arrangement", "").lower() in ["scatter", "vscatter"]:
+                pack = "scatter"
+            else:
+                pack = "pack"
+            
+            exclusive = placement.get("sharing", "").lower() in ["excl", "exclhost"]
+            # we may need to support sharing at some point, but it seems that we can ignore it for now.
+            _shared = placement.get("sharing") in ["sharing"]
+            placeby = placement.get("grouping")
+            
+            autoscale_job = Job(name=pbs_job["job_id"],
+                                nodearray=slot_type,
+                                nodes=nodect,
+                                packing_strategy=pack,
+                                exclusive=exclusive,
+                                resources={"ncpus": 0},
+                                executing_hostname=pbs_job.get("executing_hostname"))
+
+            if placeby:
+                autoscale_job.placeby = placeby.split("=", 1)[-1]
+            
+            if is_array:
+                array_count = 0
+                array_tasks = raw_job["array_state_count"]
+
+                # Only grab the first two array task states (queued and running)
+                for ajob in str(array_tasks).split(" ")[:2]:
+                    array_count += int(ajob.split(":")[1])
+
+                # Multiply the number of cpus needed by number of tasks in the array
+                if array_count != 0:
+                    slots_per_job *= array_count
+            else:
+                array_count = 1
+                    
+            # If it's an MPI job and grouping is enabled
+            # we want to use a grouped autoscale_job to get tightly coupled nodes
+
+            if group_jobs and placement.get("grouping"): 
+                autoscale_job['grouped'] = True
+                autoscale_job["nodes"] *= array_count
+                autoscale_job.placeby_value = "single"
+            elif is_array:
+                autoscale_job["nodes"] *= array_count
+
+            autoscale_job.ncpus += slots_per_job
+            
+            for attr, value in pbs_job.Resource_List.iteritems():
+                if attr not in scheduler_resources:
+                    # if it isn't a scheduler level attribute, don't bother 
+                    # considering it for autoscale as the scheduler won't respect it either.
+                    continue
+                try:
+                    value = pbscc.parse_gb_size(attr, value)
+                    value = value / nodect
+                except InvalidSizeExpressionError:
+                    if value.lower() in ["true", "false"]:
+                        value = value.lower() == "true"
+
+                autoscale_job.resources[attr] = value
+                
+            if raw_job["job_state"] == pbscc.JOB_STATE_QUEUED:
+                idle_autoscale_jobs.append(autoscale_job)
+            else:
+                running_autoscale_jobs.append(autoscale_job)
+                
+        for warning in warnings:
+            format_string, values = warning[0], warning[1:]
+            pbscc.error(format_string % values)
+        
+        # leave an option for disabling this in case it causes issues.
+        if self.cc_config.get("pbspro.compress_jobs", False):
+            all_autoscale_jobs = running_autoscale_jobs + compress_queued_jobs(idle_autoscale_jobs)
+        else:
+            all_autoscale_jobs = running_autoscale_jobs + idle_autoscale_jobs
+            
+        return all_autoscale_jobs
+    
+    def fetch_nodearray_definitions(self):
+        '''
+            A wrapper around the autoscale library function to parse Configuration.autoscale.* chef attributes and add
+            the 'ungrouped' attribute to the machine types.
+            
+            See cyclecloud.nodearrays.NodearrayDefinitions for more info.
+        '''
+        nodearray_definitions = machine.fetch_nodearray_definitions(self.clusters_api, self.default_placement_attrs)
+        nodearray_definitions.placement_group_optional = True
+
+        filtered_nodearray_definitions = nodearrays.NodearrayDefinitions()
+        
+        for machinetype in nodearray_definitions:
+            if machinetype.get("disabled", False):
+                continue
+                
+            # ensure that any custom attribute the user specified, like disk = 100G, gets parsed correctly
+            for key, value in machinetype.iteritems():
+                try:
+                    machinetype[key] = pbscc.parse_gb_size(key, value)
+                except InvalidSizeExpressionError:
+                    pass
+                
+            # kludge: there is a strange bug where ungrouped is showing up as a string and not a boolean.
+            if not machinetype.get("group_id"):
+                machinetype["ungrouped"] = "true"
+                filtered_nodearray_definitions.add_machinetype(machinetype)
+            else:
+                machinetype["ungrouped"] = "false"
+                filtered_nodearray_definitions.add_machinetype_with_placement_group(machinetype.get("group_id"), machinetype)
+            
+        return filtered_nodearray_definitions
+                
+    def autoscale(self):
+        '''
+            The main loop described at the top of this class. 
+            Returns machine_requests, idle_machines and total_machines for ease of unit testing.
+        '''
+        pbscc.info("Begin autoscale cycle")
+        
+        nodearray_definitions = self.fetch_nodearray_definitions()
+        
+        pbsnodes_by_hostname, existing_machines, booting_instance_ids, instance_ids_to_shutdown = self.get_existing_machines(nodearray_definitions)
+        
+        start_enabled = "true" == str(self.cc_config.get("cyclecloud.cluster.autoscale.start_enabled", "true")).lower()
+        
+        if not start_enabled:
+            pbscc.warn("cyclecloud.cluster.autoscale.start_enabled is false, new machines will not be allocated.")
+        
+        autoscaler = autoscalerlib.Autoscaler(nodearray_definitions, existing_machines, self.default_placement_attrs, start_enabled)
+        
+        # throttle how many jobs we attempt to match. When pbspro.compress_jobs is true (default) this shouldn't really be an issue
+        # unless the user has over $pbspro.max_unmatched_jobs unique sets of requirements.
+        max_unmatched_jobs = int(self.cc_config.get("pbspro.max_unmatched_jobs", 10000))
+        unmatched_jobs = 0
+        
+        for job in self.query_jobs():
+            if job.executing_hostname:
+                try:
+                    autoscaler.get_machine(hostname=job.executing_hostname).add_job(job, force=True)
+                    continue
+                except RuntimeError as e:
+                    pbscc.error(str(e))
+                    pass
+                    
+            if not autoscaler.add_job(job):
+                unmatched_jobs += 1
+                pbscc.info("Can not match job %s." % job.name)
+                if max_unmatched_jobs > 0 and unmatched_jobs >= max_unmatched_jobs:
+                    pbscc.warn('Maximum number of unmatched jobs reached - %s. To configure this setting, change {"pbspro": "max_unmatched_jobs": N}} in %s' % (unmatched_jobs, pbscc.CONFIG_PATH))
+                    break
+        
+        machine_requests = autoscaler.get_new_machine_requests()
+        idle_machines = autoscaler.get_idle_machines()
+        
+        autoscale_request = autoscale_util.create_autoscale_request(machine_requests)
+        for request_set in autoscale_request["sets"]:
+            configuration = request_set["nodeAttributes"]["Configuration"]
+            
+            if "pbspro" not in configuration:
+                    configuration["pbspro"] = {}
+            
+            configuration["pbspro"]["slot_type"] = request_set["nodearray"]
+            if not request_set.get("placementGroupId"):
+                configuration["pbspro"]["is_grouped"] = False
+            else:
+                configuration["pbspro"]["is_grouped"] = True
+                
+        autoscale_util.scale_up(self.clusters_api, autoscale_request)
+        
+        for r in machine_requests:
+            if r.placeby_value:
+                pbscc.info("Requesting %d %s machines in placement group %s for nodearray %s" % (r.instancecount, r.machinetype, r.placeby_value, r.nodearray))
+            else:
+                pbscc.info("Requesting %d %s machines in nodearray %s" % (r.instancecount, r.machinetype, r.nodearray))
+        
+        if pbscc.is_fine():
+            pbscc.fine("New target state of the cluster, including booting nodes:")
+            
+            for m in autoscaler.machines:
+                pbscc.fine("    %s" % str(m))
+        
+        if instance_ids_to_shutdown:
+            pbscc.info("Shutting down instance ids %s" % instance_ids_to_shutdown.keys())
+            self.clusters_api.shutdown(instance_ids_to_shutdown.keys())
+            
+            for hostname in instance_ids_to_shutdown.itervalues():
+                pbscc.info("Deleting %s" % hostname)
+                self.driver.delete_host(hostname)
+        
+        now = self.clock.time()
+        
+        stop_enabled = "true" == str(self.cc_config.get("cyclecloud.cluster.autoscale.stop_enabled", "true")).lower()
+        
+        if not stop_enabled:
+            pbscc.warn("cyclecloud.cluster.autoscale.stop_enabled is false, idle machines will not be terminated")
+        
+        if stop_enabled:
+            idle_before_threshold = float(self.cc_config.get("cyclecloud.cluster.autoscale.idle_time_before_jobs", 3600))
+            idle_after_threshold = float(self.cc_config.get("cyclecloud.cluster.autoscale.idle_time_after_jobs", 300))
+        
+            for m in idle_machines:
+                if m.get_attr("instance_id", "") not in booting_instance_ids:
+                    pbscc.debug("Could not find instance id in CycleCloud %s" % m.get_attr("instance_id", ""))
+                    continue
+                
+                pbsnode = pbsnodes_by_hostname.get(m.hostname)
+                
+                # the machine may not have converged yet, so
+                if pbsnode:
+                    if "busy" in pbsnode["state"]:
+                        if "down" in pbsnode["state"]:
+                            pbscc.warn("WARNING: %s is down but busy with jobs %s", m.hostname, pbsnode.get("jobs", []))
+                        else:
+                            pbscc.error("WARNING: Falsely determined that %s is idle!" % m.hostname)
+                        continue
+                    
+                    last_state_change_time = pbsnode["last_state_change_time"]
+                    last_used_time = pbsnode.get("last_used_time")
+                    if last_used_time:
+                        # last_used_time can be stale while a job is exiting, e.g. last_state_change_time could be < 5 minutes but
+                        # somehow last_used_time > 5 minutes, causing us to prematurely terminate the node just because a job took a long time
+                        # to exit.
+                        last_used_time = max(last_state_change_time, last_used_time)
+                    else:
+                        last_used_time = self.clock.time()
+
+                    if now - last_used_time > idle_after_threshold:
+                        pbscc.info("Setting %s offline after %s seconds" % (m.hostname, now - last_used_time))
+                        self.driver.set_offline(m.hostname)
+                    elif now - last_state_change_time > idle_before_threshold:
+                        pbscc.info("Setting %s offline after %s seconds" % (m.hostname, now - last_state_change_time))
+                        self.driver.set_offline(m.hostname)
+        
+        pbscc.info("End autoscale cycle")
+        # returned for testing purposes
+        return machine_requests, idle_machines, autoscaler.machines
+    
+    def get_existing_machines(self, nodearray_definitions):
+        '''
+            Queries pbsnodes and CycleCloud to get a sane set of cyclecloud.machine.Machine instances that represent the current state of the cluster.
+        '''
+        pbsnodes = self.driver.pbsnodes().get(None)
+        existing_machines = []
+        
+        booting_instance_ids = autoscale_util.nodes_by_instance_id(self.clusters_api, nodearray_definitions)
+        
+        instance_ids_to_shutdown = Record()
+        
+        nodes_by_instance_id = Record()
+        
+        for pbsnode in pbsnodes.values():
+            inst = self.process_pbsnode(pbsnode, instance_ids_to_shutdown, nodearray_definitions)
+            if not inst:
+                continue
+            existing_machines.append(inst)
+            
+            # we found the pbsnode that matches the cyclecloud node, so let's remove the duplicate 
+            instance_id = inst.get_attr("instance_id", "")
+            if instance_id in booting_instance_ids:
+                booting_instance_ids.pop(instance_id)
+            nodes_by_instance_id[instance_id] = pbsnode
+        
+        for instance_id, node in list(booting_instance_ids.iteritems()):
+            nodearray_name = node["Template"]
+            machinetype_name = node["MachineType"]
+            
+            try:
+                machinetype = nodearray_definitions.get_machinetype(nodearray_name, machinetype_name, node.get("PlacementGroupId"))
+            except KeyError as e:
+                raise ValueError("machine is %s, key is %s, rest is %s" % (nodearray_name, str(e), nodearray_definitions))
+            
+            inst = machine.new_machine_instance(machinetype, hostname=node.get("hostname"), instance_id=node.get("InstanceId"), group_id=node.get("placementGroupId"))
+            existing_machines.append(inst)
+            nodes_by_instance_id[instance_id] = node
+        
+        return pbsnodes, existing_machines, nodes_by_instance_id, instance_ids_to_shutdown
+    
+    def process_pbsnode(self, pbsnode, instance_ids_to_shutdown, nodearray_definitions):
+        '''
+            If the pbsnode is offline, will handle evaluating whether the node can be shutdown. See instance_ids_to_shutdown, which
+            is an OUT parameter here.
+            
+            Otherwise convert the pbsnode into a cyclecloud.machine.Machine instance.
+        '''
+        
+        states = set(pbsnode["state"].split(","))
+        resources = pbsnode["resources_available"]
+        # host has incorrect case
+        hostname = resources["vnode"]
+        
+        instance_id = resources.get("instance_id", autoscale_util.uuid("instanceid"))
+        
+        def try_shutdown_pbsnode():
+            if not instance_id:
+                pbscc.error("instance_id was not defined for host %s, can not shut it down" % hostname)
+            elif "down" in states:
+                # don't immediately remove down nodes, give them time to recover from network failure.
+                remove_down_nodes = float(self.cc_config.get("pbspro.remove_down_nodes", 300))
+                since_down = self.clock.time() - pbsnode["last_state_change_time"]
+                if since_down > remove_down_nodes:
+                    pbscc.error("Removing down node %s after %.0f seconds", hostname, since_down)
+                    instance_ids_to_shutdown[instance_id] = hostname
+                    return True
+                else:
+                    omega = remove_down_nodes - since_down
+                    pbscc.warn("Not removing down node %s for another %.0f seconds", hostname, omega)
+            else:
+                instance_ids_to_shutdown[instance_id] = hostname
+                return True
+            
+            return False
+        
+        if "offline" in states:
+            if not pbsnode.get("jobs", []):
+                pbscc.fine("%s is offline and has no jobs, may be able to shut down" % hostname)
+                if try_shutdown_pbsnode():
+                    return
+            else:
+                pbscc.fine("Host %s is offline but still running jobs" % hostname)
+        
+        # if the node is just in the down state, try to shut it down. 
+        if set(["down"]) == states and try_shutdown_pbsnode():
+            return
+        
+        # just ignore complex down nodes (down,job-busy etc) until PBS decides to change the state.
+        if "down" in states:
+            return
+        
+        # convert relevant resources from bytes to floating point (GB)
+        for key in resources:
+            
+            value = resources[key]
+            if isinstance(value, basestring) and value.lower() in ["true", "false"]:
+                value = value.lower() == "true"
+            elif isinstance(value, list):
+                # TODO will need to support this eventually
+                continue
+            else:
+                try:
+                    resources[key] = pbscc.parse_gb_size(key, resources[key])
+                except InvalidSizeExpressionError:
+                    pass
+        
+        resources["hostname"] = hostname
+        
+        nodearray_name = resources.get("nodearray") or resources.get("slot_type")
+        group_id = resources.get("group_id")
+
+        if resources.get("machinetype") and nodearray_name:
+            machinetype = nodearray_definitions.get_machinetype(nodearray_name, resources.get("machinetype"), group_id)
+        else:
+            # rely solely on resources_available
+            pbscc.debug("machinetype is not defined for host %s, relying only on resources_available" % hostname)
+            machinetype = {"availableCount": 1, "name": "undefined"}
+            
+        inst = machine.new_machine_instance(machinetype, **pbsnode["resources_available"])
+
+        return inst
+    
+
+def compress_queued_jobs(autoscale_jobs):
+    '''
+        assuming nodearray, num nodes, placeby/placeby_value, exclusivity, packing stategy and of course the requested resources.
+        
+        A compressed job will have the name of the first job with ".compressed" appended to it.
+        
+        i.e. the goal is that if someone does
+        `qsub -l mem=1g` 
+        1000 times, we can compress that so that it appears as if they did
+        `qsub -l select=1000:mem=1g`
+        
+        We do not compress SCATTER jobs.
+    '''
+    ret = []
+    compression_buckets = collections.defaultdict(lambda: [])
+    
+    for job in autoscale_jobs:
+        if job.packing_strategy == PackingStrategy.SCATTER:
+            # for now, we will only worry about compressing PACK jobs as an optimization.
+            ret.append(job)
+            continue
+        
+        comp_bucket = (job.nodearray, job.nodes, job.placeby, job.placeby_value, job.exclusive, job.packing_strategy) + tuple(job.resources.items())
+        compression_buckets[comp_bucket].append(job)
+        
+    for comp_bucket, job_list in compression_buckets.iteritems():
+        if len(job_list) == 1:
+            ret.extend(job_list)
+            continue
+        
+        first_job = job_list[0]
+        pseudo_job = Job(first_job.name + ".compressed", first_job.nodes * len(job_list), first_job.nodearray, first_job.exclusive, first_job.packing_strategy,
+                         first_job.resources, first_job.placeby, first_job.placeby_value)
+        ret.append(pseudo_job)
+        
+        pbscc.debug("Compressed jobs matching job id %s from %d jobs down to a single job" % (first_job.name, len(job_list)))
+    
+    return ret
+
+
+class Clock:
+    
+    def time(self):
+        import time
+        return time.time()
+
+
+def _hook():
+    pbscc.set_application_name("cycle_autoscale")
+    # allow local overrides of jetpack.config or allow non-jetpack masters to define the complete set of settings.
+    overrides = {}
+    
+    if os.path.exists(pbscc.CONFIG_PATH):
+        try:
+            pbscc.warn("overrides exist in file %s" % pbscc.CONFIG_PATH)
+            with open(pbscc.CONFIG_PATH) as fr:
+                overrides = json.load(fr)
+        except Exception:
+            pbscc.error(traceback.format_exc())
+            sys.exit(1)
+    else:
+        pbscc.debug("No overrides exist in file %s" % pbscc.CONFIG_PATH)
+    
+    cc_config = cyclecloud.config.new_provider_config(overrides=overrides)
+    
+    if len(sys.argv) < 3:
+        # There are no env variables for this as far as I can tell.
+        bin_dir = "/opt/pbs/bin"
+    else:
+        bin_dir = sys.argv[2]
+        if not os.path.isdir(bin_dir):
+            bin_dir = os.path.dirname(bin_dir)
+    
+    clusters_api = clustersapi.ClustersAPI(cc_config.get("cyclecloud.cluster.name"), cc_config)
+    autostart = PBSAutostart(pbs_driver.PBSDriver(bin_dir), clusters_api, cc_config=cc_config)
+    
+    autostart.autoscale()
+
+
+# Since this is invoked from a hook, __name__ is not "__main__", so we rely on a special env variable. Otherwise unit testing would be impossible.
+if os.getenv("AUTOSTART_HOOK"):
+    try:
+        _hook()
+    except:
+        pbscc.error(traceback.format_exc())
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/autostart_hook.py b/specs/default/chef/site-cookbooks/pbspro/files/default/autostart_hook.py
new file mode 100644
index 0000000..07d528a
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/autostart_hook.py
@@ -0,0 +1,61 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+import json
+import os
+import subprocess
+import traceback
+
+try:
+    import pbs
+except ImportError:
+    import mockpbs as pbs
+
+
+def perform_hook():
+    """
+        See /var/spool/pbs/server_logs/* or /opt/cycle/jetpack/logs/autoscale.log for log messages
+    """
+    try:
+        if not pbs.hook_config_filename:
+            raise RuntimeError("Hook config for this plugin was not defined.")
+
+        with open(pbs.hook_config_filename) as fr:
+            hook_config = json.load(fr)
+            
+        log_dir = os.path.join(hook_config.get("cyclecloud_home"), "logs")
+        
+        if not os.path.exists(log_dir):
+            log_dir = os.getcwd()
+        
+        env_with_src_dirs = {"PYTHONPATH": os.pathsep.join(hook_config.get("src_dirs", [])),
+                             "AUTOSTART_HOOK": "1",
+                             "AUTOSTART_LOG_FILE": os.path.join(log_dir, "autoscale.log"),
+                             "AUTOSTART_LOG_FILE_LEVEL": hook_config.get("autostart_log_file_level") or "DEBUG",
+                             "AUTOSTART_LOG_LEVEL": hook_config.get("autostart_log_level") or "DEBUG"}
+
+        jetpack_config = hook_config.get("jetpack_python")
+        if not jetpack_config:
+            if not hook_config.get("cyclecloud_home"):
+                raise RuntimeError("Please specify either jetpack_python or cyclecloud_home in the hook config.")
+
+            hook_config["jetpack_python"] = os.path.join(hook_config["cyclecloud_home"], "system", "embedded", "bin", "python")
+
+        pbs_bin_dir = os.path.join(pbs.pbs_conf["PBS_EXEC"], "bin")
+
+        cmd = [hook_config["jetpack_python"], "-m", "autostart", pbs.hook_config_filename, pbs_bin_dir]
+        
+        pbs.logmsg(pbs.LOG_DEBUG, "Running %s with environment %s" % (cmd, env_with_src_dirs))
+        
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env_with_src_dirs)
+        stdout, stderr = proc.communicate()
+        pbs.logmsg(pbs.LOG_DEBUG, stderr)
+        if proc.returncode != 0:
+            raise RuntimeError('autostart failed!\n\tstdout="%s"\n\tstderr="%s"' % (stdout, stderr))
+    except:
+        pbs.logmsg(pbs.LOG_ERROR, traceback.format_exc())
+        raise
+
+# hooks must not have a __name__ == "__main__" guard
+perform_hook()
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/autostart_test.py b/specs/default/chef/site-cookbooks/pbspro/files/default/autostart_test.py
new file mode 100644
index 0000000..b3af822
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/autostart_test.py
@@ -0,0 +1,1210 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+import logging_init
+import unittest
+
+from autostart import PBSAutostart
+from cyclecloud import machine, autoscale_util
+from cyclecloud.job import Job
+from cyclecloud.machine import MachineRequest
+from cyclecloud.autoscale_util import Record
+import time
+from itertools import chain
+from cyclecloud.config import InstanceConfig
+import random
+import pbscc
+from pbscc import InvalidSizeExpressionError
+
+
+class MockDriver:
+    def __init__(self, queues=None, jobs=None, hosts=None):
+        self._queues = queues or ["workq"]
+        self._jobs = jobs or {}
+        assert isinstance(self._jobs, dict)
+        self._hosts = hosts or []
+        self._declared_resources = {"resources": ["ncpus", "mem", "arch", "host", "vnode", "aoe", "slot_type", 
+                                                  "group_id", "ungrouped", "instance_id", "ipv4", "disk", "scratch",
+                                                  "swlicense", "graphics", "dyna"]}
+        
+    def queues(self):
+        return self._queues
+    
+    def queued_jobs(self):
+        # TODO multiple queues
+        return [x for x in self._jobs.get("workq", []) if x["job_state"] == "Q"], lambda x: x
+    
+    def running_jobs(self):
+        return [x for x in self._jobs.get("workq", []) if x["job_state"] == "R"], lambda x: x
+    
+    def scheduler_config(self):
+        return self._declared_resources
+    
+    def pbsnodes(self, grouping=None):
+        ret = {None: {}}
+        for host in self._hosts:
+            state = host.get(grouping)
+            if state not in ret:
+                ret[state] = {}
+            ret[state][host["resources_available"]["vnode"]] = host
+        return ret
+    
+    def set_offline(self, hostname):
+        try:
+            host = self.get_host(hostname)
+        except:
+            return
+        if "offline" not in host["state"]:
+            if host["state"] == "free":
+                host["state"] = "offline"
+            else:
+                host["state"] = host["state"] + ",offline"
+                    
+    def delete_host(self, hostname):
+        self._hosts = [x for x in self._hosts if x["resources_available"]["vnode"] != hostname]
+        
+    def get_host(self, hostname):
+        select = [x for x in self._hosts if x["resources_available"]["vnode"].lower() == hostname.lower()]
+        assert len(select) == 1, "%s %s %s" % (select, hostname, [x["hostname"].lower() for x in self._hosts])
+        return select[0]
+        
+    def add_host(self, hostname, state="free", **resources):
+        host = {"host": hostname, "hostname": hostname, "state": state}
+        resources_assigned = resources.pop("resources_assigned", {})
+        if state == "job-busy" and not resources_assigned:
+            raise RuntimeError("define resources_assigned")
+        
+        host["resources_available"] = resources
+        host["resources_available"]["host"] = hostname.upper()
+        host["resources_available"]["vnode"] = host["resources_available"]["host"].lower() 
+        host["resources_assigned"] = resources_assigned
+        host["jobs"] = resources.pop("jobs", [])
+        host["last_state_change_time"] = time.time()
+        self._hosts.append(host)
+        
+
+class MockClustersAPI:
+    
+    def __init__(self, cluster_def, nodes=None):
+        self.cluster_def = cluster_def
+        nodes = nodes or {}
+        self._nodes = nodes
+        if not hasattr(nodes, "keys"):
+            self._nodes = {"execute": []}
+            for node in nodes:
+                # node.get_attr("instance_id", autoscale_util.uuid("instance_id"))] = node
+                self._nodes["execute"].append(node)
+        
+    def status(self, nodes=False):
+        if nodes:
+            self.cluster_def["nodes"] = nodes = []
+            for node_list in self._nodes.itervalues():
+                nodes.extend(node_list)
+        else:
+            self.cluster_def.pop("nodes", [])
+        return self.cluster_def
+    
+    def nodes(self):
+        for key in self._nodes:
+            random.shuffle(self._nodes[key])
+        return self._nodes
+    
+    def add_nodes(self, request):
+        for request_set in request["sets"]:
+            for _ in range(request_set["count"]):
+                select = [x for x in self.cluster_def["nodearrays"] if x["name"] == request_set["nodearray"]]
+                assert select, "%s not in %s" % (request_set["nodearray"], [x["name"] for x in self.cluster_def["nodearrays"]])
+                nodearray = select[0]
+                select = [b for b in nodearray["buckets"] if b["definition"] == request_set["definition"]]
+                assert select, "No matching bucket found for request %s" % request
+                mt = select[0]["virtualMachine"]
+                mt_name = select[0]["definition"]["machineType"]
+                node = Record({"vcpuCount": mt["vcpuCount"],
+                               "memory": mt["Memory"], 
+                               "machineType": mt_name,
+                               "placementGroupId": request_set.get("placementGroupId")})
+                node.update(request_set["nodeAttributes"])
+                node["nodearray"] = request_set["nodearray"]
+                node["Template"] = request_set["nodearray"]
+                node["InstanceId"] = autoscale_util.uuid("instance")
+                
+                if node["nodearray"] not in self._nodes:
+                    self._nodes[node["nodearray"]] = []
+                self._nodes[node["nodearray"]].append(node)
+                
+    def shutdown(self, instance_ids):
+        instance_ids = list(instance_ids)
+        for key in self._nodes:
+            self._nodes[key] = [x for x in self._nodes[key] if x["InstanceId"] not in instance_ids]
+
+
+def _nodearray_definitions(*machinetypes):
+    machinetypes = sorted(machinetypes, key=lambda x: -x.get("priority", 100))
+    cluster_status = {}
+    ret = {}
+    
+    for machinetype in machinetypes:
+        nodearray = machinetype.get("nodearray")
+        if nodearray not in ret:
+            ret[nodearray] = {"nodearray": {},
+                              "name": nodearray,
+                              "buckets": []}
+        
+        name = machinetype.get("machinetype")
+        ret[nodearray]["buckets"].append({"definition": {"machineType": name},
+                                          "virtualMachine": machinetype})
+
+    cluster_status["nodearrays"] = ret.values()
+    return cluster_status
+
+
+class MockClock:
+    def __init__(self, now=0):
+        self.now = now
+        
+    def time(self):
+        return self.now
+    
+    
+class PBSQ:
+    
+    def __init__(self):
+        self.queues = {"workq": []}
+        self.job_id = 1
+        self.hosts = []
+    
+    def qsub(self, job_id=None, job_state="Q", J=None, select_expr=None, **resource_list):
+        assert "select" not in resource_list
+        if not job_id:
+            job_id = str(self.job_id)
+            self.job_id += 1
+            
+        jobdef = {"resource_list": {},
+                  "job_id": job_id,
+                  "job_state": job_state}
+        
+        for key, value in resource_list.iteritems():
+            jobdef["resource_list"][key] = value
+            
+        jobdef["resource_list"]["nodect"] = 1      
+            
+        if "job_state" not in jobdef:
+            jobdef["job_state"] = "Q"
+            
+        if "ncpus" not in jobdef["resource_list"]:
+            jobdef["resource_list"]["ncpus"] = jobdef.get("nodes", 1)
+        
+        if J:
+            jobdef["array"] = True
+            jobdef["array_state_count"] = J
+            
+        if select_expr:
+            jobdef["resource_list"]["select"] = select_expr
+            chunk_totals = {}
+            for chunk in pbscc.parse_select(jobdef):
+                chunk["ncpus"] = chunk.get("ncpus", 1)
+                for key, value in chunk.iteritems():
+                    if key == "select":
+                        continue
+                    try:
+                        if key == "nodect":
+                            value = pbscc.parse_gb_size(key, value)
+                        else:
+                            value = pbscc.parse_gb_size(key, value) * int(chunk["select"])
+                        chunk_totals[key] = chunk_totals.get(key, 0) + value
+                    except InvalidSizeExpressionError:
+                        chunk_totals[key] = value
+                        
+            for key, value in chunk_totals.iteritems():
+                if key != "select":
+                    jobdef["resource_list"][key] = value
+        
+        self.queues["workq"].append(jobdef)
+        
+    def qdel(self, jobid):
+        to_delete = [x for x in self.queues["workq"] if x["job_id"] == str(jobid)]
+        assert len(to_delete) >= 1
+        self.queues["workq"] = [x for x in self.queues["workq"] if x not in to_delete]
+        
+    def query_jobs(self):
+        cluster = MockClustersAPI({})
+        a = PBSAutostart(MockDriver(["workq"], self.queues), cluster, {})
+        return a.query_jobs()
+    
+    def set_running(self, job_id, exec_vnode):
+        '''
+            Testing: change the job state and set exec_vnode expression so that autoscale knows where the jobs are placed
+        '''
+        for job in chain(*self.queues.values()):
+            if str(job["job_id"]) == str(job_id):
+                job["exec_vnode"] = exec_vnode
+                job["job_state"] = "R"
+                return
+        raise AssertionError("Could not find job_id %s" % job_id)
+    
+        
+class Test(unittest.TestCase):
+    def __init__(self, methodName='runTest'):
+        unittest.TestCase.__init__(self, methodName=methodName)
+        self.maxDiff = None
+    
+    def setUp(self):
+        unittest.TestCase.setUp(self)
+        autoscale_util.set_uuid_func(autoscale_util.IncrementingUUID())
+        
+    def tearDown(self):
+        unittest.TestCase.tearDown(self)
+        autoscale_util.set_uuid_func(__import__("uuid").uuid4)
+        
+    def test_qsub(self):
+        q = PBSQ()
+        # q.qsub(nodes=2, ncpus=2)
+        # this ^ is just tranlsated into the following
+        q.qsub(select_expr="2:ncpus=2:mpiprocs=2", place="scatter")
+        q.qsub(select_expr="2:ncpus=2", place="excl")
+        
+        self.assertEquals([Job(name="1", nodes=2, packing_strategy="scatter", exclusive=False, resources={"ncpus": 2}),
+                           Job(name="2", nodes=2, packing_strategy="pack", exclusive=True, resources={"ncpus": 2})],
+                           q.query_jobs())
+        
+        # 4 cpus packed onto one machine and 2 exclusive
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 32, 100, 100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=3)], self._autoscale(q, cluster_def))
+        
+    def test_qsub_scatter_excl(self):
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 32, 100, 100))
+        
+        for strategy in ["scatter", "vscatter"]:
+            for modifier in [":excl", ":exclhost"]:
+                place = strategy + modifier
+                q = PBSQ()
+                q.qsub(select_expr="2:ncpus=2", place=place)
+                self.assertEquals([Job(name="1", nodes=2, packing_strategy="scatter", exclusive=True, resources={"ncpus": 2})],
+                                   q.query_jobs())
+                
+                self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+                
+        for strategy in ["pack", "free"]:
+            for modifier in [":excl", ":exclhost"]:
+                place = strategy + modifier
+                q = PBSQ()
+                q.qsub(select_expr="2:ncpus=2", place=place)
+                pack = "pack" if strategy in ["pack", "free"] else "scatter"
+                self.assertEquals([Job(name="1", nodes=2, packing_strategy=pack, exclusive=True, resources={"ncpus": 2})],
+                                   q.query_jobs())
+                
+                self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+                
+        for scatter in ["scatter", "vscatter"]:
+            for modifier in [":shared", ""]:
+                place = scatter + modifier
+                q = PBSQ()
+                q.qsub(select_expr="2:ncpus=2", place=place)
+                self.assertEquals([Job(name="1", nodes=2, packing_strategy="scatter", exclusive=False, resources={"ncpus": 2})],
+                                   q.query_jobs())
+                
+                self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+                
+        for scatter in ["pack", "free"]:
+            for modifier in [":shared", ""]:
+                place = scatter + modifier
+                q = PBSQ()
+                q.qsub(select_expr="2:ncpus=2", place=place)
+                self.assertEquals([Job(name="1", nodes=2, packing_strategy="pack", exclusive=False, resources={"ncpus": 2})],
+                                   q.query_jobs())
+                
+                self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+            
+    def test_pbsuserguide_too_many_cpus(self):
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a4", 32, 100, 100))
+        
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=33:mem=1G", place="pack", _can_be_added=False)
+        self.assertEquals([], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="33:ncpus=1:mem=1G", place="pack", machinetype="a4")
+        self.assertEquals([self._machine_request(machine_type="a4", count=2)], self._autoscale(q, cluster_def))
+        
+    def _rebalance(self, q, cluster_def=None, hosts=None, cc_config=None):
+        cluster_def = cluster_def or _nodearray_definitions(machine.new_machinetype("execute", "a4", 32, 100, 100))
+        pbs_autostart = PBSAutostart(MockDriver(jobs=q.queues, hosts=hosts), MockClustersAPI(cluster_def), cc_config or {})
+        return pbs_autostart.autoscale()
+    
+    def _autoscale(self, q, cluster_def=None, hosts=None, cc_config=None):
+        return self._rebalance(q, cluster_def, hosts, cc_config)[0]
+    
+    def _idle_nodes(self, q, cluster_def=None, hosts=None, cc_config=None):
+        return [x.hostname for x in self._rebalance(q, cluster_def, hosts, cc_config)[1]]
+    
+    def _machine_request(self, nodearray=None, machine_type="a4", count=1, placeby="", placeby_value=""):
+        nodearray = nodearray or "execute"
+        
+        return MachineRequest(nodearray, machine_type, count, placeby, placeby_value)
+    
+    def _host(self, machinetype, hostname, **resources):
+        machinetype = machinetype or {}
+        host = {"state": resources.pop("state", "idle"),
+                "jobs": resources.pop("jobs", []),
+                "resources_assigned": {},
+                "resources_available": {"vnode": hostname,
+                                        "host": hostname,
+                                        "machinetype": machinetype.get("name", "")}}
+        host["resources_available"].update(machinetype)
+        host["last_state_change_time"] = resources.get("last_state_change_time") or time.time()
+        host["resources_available"].update(resources)
+        return host
+    
+    def test_pbsuserguide_ex1(self):
+        '''
+            1. A job that will fit in a single host but not in any of the vnodes, packed into the fewest vnodes:
+                -l select=1:ncpus=10:mem=20gb
+                -l place=pack
+            In earlier versions, this would have been:
+                -lncpus=10,mem=20gb
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=16:mem=20G", place="pack")
+        self.assertEquals([self._machine_request(count=1)], self._autoscale(q))
+        
+    def test_pbsuserguide_ex2(self):
+        '''
+            2. Request four chunks, each with 1 CPU and 4GB of memory taken from anywhere.
+            -l select=4:ncpus=1:mem=4GB
+            -l place=free
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="4:ncpus=1:mem=4G", place="pack", machinetype="a4")
+        self.assertEquals([self._machine_request(count=1)], self._autoscale(q))
+        
+        q = PBSQ()
+        q.qsub(select_expr="4:ncpus=1:mem=4G", place="pack", machinetype="a4")
+        self.assertEquals([self._machine_request(count=1)], self._autoscale(q))
+        
+    def test_pbsuserguide_ex3(self):
+        '''
+            3. Allocate 4 chunks, each with 1 CPU and 2GB of memory from between one and four vnodes which have an arch of "linux".
+            -l select=4:ncpus=1:mem=2GB:arch=linux -l place=free
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="4:ncpus=1:mem=2G:arch=linux", place="free")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2win", 16, 8, 100, arch="windows"),
+                                             machine.new_machinetype("execute", "a2linux", 16, 8, 100, arch="linux"))
+        self.assertEquals([self._machine_request(machine_type="a2linux", count=1)], self._autoscale(q, cluster_def))
+        
+        # slightly different take, just so that we see the spill over.
+        q.qsub(select_expr="6:ncpus=1:mem=2G:arch=linux", place="free")
+        self.assertEquals([self._machine_request(machine_type="a2linux", count=3)], self._autoscale(q, cluster_def))
+
+    def test_pbsuserguide_ex4(self):
+        '''
+        4. Allocate four chunks on 1 to 4 vnodes where each vnode must have 1 CPU, 3GB of memory and 1 node-locked dyna
+            license available for each chunk.
+            -l select=4:dyna=1:ncpus=1:mem=3GB -l place=free
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="4:dyna=1:ncpus=1:mem=3G", place="free")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2nodyna", 16, 8, 100),
+                                             machine.new_machinetype("execute", "a2dyna", 16, 8, 100, dyna=4))
+        # 3gb with 8gb per machine, so we need 2 machines
+        self.assertEquals([self._machine_request(machine_type="a2dyna", count=2)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="4:dyna=1:ncpus=1:mem=1G", place="free")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2nodyna", 16, 8, 100),
+                                             machine.new_machinetype("execute", "a2dyna", 16, 8, 100, dyna=4))
+        # 1gb with 8gb per machine, so we need 1 machine
+        self.assertEquals([self._machine_request(machine_type="a2dyna", count=1)], self._autoscale(q, cluster_def))
+    
+    @unittest.skip("need shared value to do this correctly")
+    def test_pbsuserguide_ex5(self):
+        '''
+        5. Allocate four chunks on 1 to 4 vnodes, and 4 floating dyna licenses. This assumes "dyna" is specified as a server
+            dynamic resource.
+            -l dyna=4 -l select=4:ncpus=1:mem=3GB -l place=free
+        '''
+        self.fail("we need a shared value to do this correctly")
+        
+        q = PBSQ()
+        q.qsub(select_expr="4:dyna=1, ncpus=2:mem=1G", place="free")
+        dyna_license = machine.NumericValue(2, "dyna")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2nodyna", 8, 8, 100),
+                                             machine.new_machinetype("execute", "a2dyna", 2, 8, 100, dyna=dyna_license))
+        
+        self.assertEquals([self._machine_request(machine_type="a2dyna", count=2)], self._autoscale(q, cluster_def))
+        
+    def test_pbsuserguide_ex6(self):
+        '''
+            6. This selects exactly 4 vnodes where the arch is linux, and each vnode will be on a separate host. Each vnode will
+            have 1 CPU and 2GB of memory allocated to the job.
+            -lselect=4:mem=2GB:ncpus=1:arch=linux -lplace=scatter
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="4:mem=2G:ncpus=1:arch=linux", place="scatter:excl")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 32, 128, 100, arch="linux"))
+        self.assertEquals([self._machine_request(machine_type="a2", count=4)], self._autoscale(q, cluster_def))
+    
+    def test_pbsuserguide_ex6_scatter(self):
+        q = PBSQ()
+        q.qsub(select_expr="4:mem=2G:ncpus=1:arch=linux", place="scatter")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 32, 128, 100, arch="linux"))
+        self.assertEquals([self._machine_request(machine_type="a2", count=4)], self._autoscale(q, cluster_def))
+        
+    def test_pbsuserguide_ex7(self):
+        '''
+            7. This will allocate 3 chunks, each with 1 CPU and 10GB of memory. This will also reserve 100mb of scratch space if
+            scratch is to be accounted . Scratch is assumed to be on a file system common to all hosts. The value of "place"
+            depends on the default which is "place=free".
+            -l scratch=100mb -l select=3:ncpus=1:mem=10GB
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="3:ncpus=1:mem=10G:scratch=100M")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 32, 128, 100, scratch=.333))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 32, 128, 100, scratch=.225))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+
+    def test_pbsuserguide_ex8(self):
+        '''
+            8. This will allocate 2 CPUs and 50GB of memory on a host named zooland. The value of "place" depends on the
+            default which defaults to "place=free":
+            -l select=1:ncpus=2:mem=50gb:host=zooland
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=2:mem=50G:host=zooland")
+        
+        a2_mt = machine.new_machinetype("execute", "a2", 32, 128, 100)
+        cluster_def = _nodearray_definitions(a2_mt)
+        
+        # pbs calls the hostname resource host, so we have to duplicate
+        
+        zooland = self._host(a2_mt, hostname="zooland", host="zooland")
+        notzooland = self._host(a2_mt, hostname="notzooland", host="notzooland")
+        self.assertEquals([], self._autoscale(q, cluster_def))
+        self.assertEquals(["notzooland"], self._idle_nodes(q, cluster_def, hosts=[zooland, notzooland]))
+    
+    def test_pbsuserguide_ex9(self):
+        '''
+           9. This will allocate 1 CPU and 6GB of memory and one host-locked swlicense from each of two hosts:
+            -l select=2:ncpus=1:mem=6gb:swlicense=1
+            -lplace=scatter
+        '''
+        q = PBSQ()
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 32, 100, swlicense=0),
+                                                machine.new_machinetype("execute", "a4", 16, 64, 100, swlicense=1))
+        q.qsub(select_expr="2:ncpus=2:mem=6G:swlicense=1", place="scatter:excl")
+        self.assertEquals([self._machine_request(machine_type="a4", count=2)], self._autoscale(q, cluster_def))
+        
+    def test_pbsuserguide_ex10(self):
+        '''
+           10. Request free placement of 10 CPUs across hosts:
+            -l select=10:ncpus=1
+            -l place=free
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="10:ncpus=1", place="free")
+        
+        a2_mt = machine.new_machinetype("execute", "a2", 32, 128, 100, priority=100)
+        a4_mt = machine.new_machinetype("execute", "a4", 32, 128, 100, priority=50)
+        
+        cluster_def = _nodearray_definitions(a2_mt, a4_mt)
+        
+        # if there aren't any machines, request an a2 as this has the highest priority
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+#         self.assertEquals([MachineRequest("execute", "a2", 1, "")], q.autoscale_requests(cluster_def, existing_machines=[]))
+        
+        existing_a4 = self._host(a4_mt, hostname="host123")
+        
+        # we already have an a4 up, so just use that.
+        self.assertEquals([], self._autoscale(q, cluster_def, hosts=[existing_a4]))
+        
+    def test_pbsuserguide_ex11(self):
+        '''
+           11. Here is an odd-sized job that will fit on a single SGI system, but not on any one node-board. We request an odd
+                number of CPUs that are not shared, so they must be "rounded up":
+                -l select=1:ncpus=3:mem=6gb
+                -l place=pack:excl
+        '''
+        q = PBSQ()
+        # for allocation testing purposes, I'm going to add a free job here too to ensure we don't try to use that one.
+        q.qsub(select_expr="1:ncpus=1:mem=1G", place="free")
+        q.qsub(select_expr="1:ncpus=3:mem=6G", place="pack:excl")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 24, 100, priority=100))
+         
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=1:mem=1G", place="free")
+        q.qsub(select_expr="3:ncpus=3:mem=6G", place="pack:excl")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 24, 100, priority=100))
+        
+        self.assertEquals([self._machine_request(machine_type="a2", count=3)], self._autoscale(q, cluster_def))
+    
+    def test_pbsuserguide_ex12(self):
+        '''
+           12. Here is an odd-sized job that will fit on a single SGI system, but not on any one node-board. We are asking for small
+                number of CPUs but a large amount of memory:
+                -l select=1:ncpus=1:mem=25gb
+                -l place=pack:excl
+        '''
+        #  this isn't really any different than ex11
+        q = PBSQ()
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 50, 100, priority=100))
+        q.qsub(select_expr="1:ncpus=1:mem=25G", place="pack:excl")
+        
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+        q.qsub(select_expr="1:ncpus=1:mem=1G", place="free")
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+        
+    def test_pbsuserguide_ex13(self):
+        '''
+           13. Here is a job that may be run across multiple SGI systems, packed into the fewest vnodes:
+            -l select=2:ncpus=10:mem=12gb
+            -l place=free
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="2:ncpus=10:mem=12G", place="free")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 36, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 20, 36, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+    
+    def test_pbsuserguide_ex14(self):
+        '''
+            14. Submit a job that must be run across multiple SGI systems, packed into the fewest vnodes:
+            -l select=2:ncpus=10:mem=12gb
+            -l place=scatter
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="2:ncpus=10:mem=12G", place="scatter")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 36, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+         
+        # add enough CPUs and it still requests 2 machines
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 24, 36, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="2:ncpus=10:mem=12G", place="scatter")
+        # add another job
+        q.qsub(select_expr="2:ncpus=10:mem=12G", place="scatter")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 24, 36, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+    
+    def test_pbsuserguide_ex15(self):
+        '''
+            15. Request free placement across nodeboards within a single host:
+            -l select=1:ncpus=10:mem=10gb
+            -l place=group=host 
+        '''
+        # group=host is meaningless without existing machines... and group=host is pretty pointless.
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=10:mem=10G", place="group=host")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 36, 100, priority=100, host=123))
+        # we ignore any group that isn't group_id
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+#         self.assertEquals([MachineRequest("execute", "a2", 1, "")], q.autoscale_requests(cluster_def))
+        
+    def test_pbsuserguide_ex16(self):
+        '''
+            16. Request free placement across vnodes on multiple SGI systems:
+            -l select=10:ncpus=1:mem=1gb
+            -l place=free
+        '''
+        # not really that interesting
+        q = PBSQ()
+        q.qsub(select_expr="10:ncpus=1:mem=1G", place="free")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 36, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+    def test_pbsuserguide_ex17(self):
+        '''
+            17. Here is a small job that uses a shared cpuset:
+            -l select=1:ncpus=1:mem=512kb
+            -l place=pack:shared
+        '''
+        # not really that interesting
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=1:mem=1G", place="pack:shared")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 36, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+    def test_pbsuserguide_ex18(self):
+        '''
+            18. Request a special resource available on a limited set of nodeboards, such as a graphics card:
+            -l select= 1:ncpus=2:mem=2gb:graphics=True + 1:ncpus=20:mem=20gb:graphics=False
+            -l place=pack:excl
+        '''
+        q = PBSQ()
+        q.qsub(place="pack:excl",
+                      select_expr="1:ncpus=2:mem=2G:graphics=true+1:ncpus=20:mem=20G:graphics=false")
+
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2g", 16, 10, 100, priority=100, graphics=True),
+                                                machine.new_machinetype("execute", "a2", 32, 36, 100, priority=100, graphics=False))
+        
+        self.assertEquals([self._machine_request(machine_type="a2", count=1),
+                           self._machine_request(machine_type="a2g")], sorted(self._autoscale(q, cluster_def), key=lambda x: x.machinetype))
+        
+    @unittest.skip("same as ex20")
+    def test_pbsuserguide_ex19(self):
+        '''
+            19. Align SMP jobs on c-brick boundaries:
+            -l select=1:ncpus=4:mem=6gb
+            -l place=pack:group=cbrick 
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=4:mem=6G", place="pack:group=cbrick")
+        
+        #  cbrick isn't defined
+        # TODO due to the single placement group hack, cbrick is set to single for this example
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 36, 100, priority=100),
+                                                machine.new_machinetype("execute", "a2c", 16, 36, 100, priority=50, cbrick="abc"))
+        cc_config = {"cyclecloud": {"placement_group": {"defaults": {"cbrick": "ccc"}}}}
+        self.assertEquals([self._machine_request(machine_type="a2c", count=1, placeby="cbrick", placeby_value="ccc")], self._autoscale(q, cluster_def, cc_config=cc_config))
+#         self.assertEquals([MachineRequest("execute", "a2c", 1, "")], q.autoscale_requests(cluster_def))
+    
+    @unittest.skip("This test will only make sense when we are using multiple placement groups")
+    def test_placement_group(self):
+        q = PBSQ()
+        q.qsub(select_expr="4:ncpus=4:mem=6G", place="scatter:excl:group=group_id")
+        
+        # ok, define placement_group _and_ add a placement_group that doesn't have enough machines
+        a2p_def_mt = machine.new_machinetype("execute", "a2p_def", 16, 36, 100, priority=50,
+                                             placeby="group_id", group_id="def")
+        a2p_abc_mt = machine.new_machinetype("execute", "a2p_abc", 16, 36, 100, priority=49,
+                                             placeby="group_id", group_id="abc")
+        cluster_def = _nodearray_definitions(a2p_def_mt, a2p_abc_mt)
+        
+        other_pg = self._host(a2p_abc_mt, hostname="otherpg")
+        
+        self.assertEquals([self._machine_request(nodearray="execute", machine_type="a2p_def", count=3, placeby="group_id", placeby_value="def")], 
+                          self._autoscale(q, cluster_def, [other_pg]))
+        
+        # duplicate test - if you run the same request twice, do you get the same result?
+        self.assertEquals([self._machine_request(nodearray="execute", machine_type="a2p_def", count=3, placeby="group_id", placeby_value="def")], 
+                          self._autoscale(q, cluster_def, [other_pg])) 
+        
+        q.qsub(select_expr="1:ncpus=4:mem=6G", place="scatter:excl:group=group_id")
+        self.assertEquals([self._machine_request(nodearray="execute", machine_type="a2p_def", count=4, placeby="group_id", placeby_value="def")], 
+                          self._autoscale(q, cluster_def, [other_pg]))
+        
+        # bump up the priority of a2p_abc so that one is allocated for the second job
+        cluster_def["execute"]["abc"]["machinetype"]["a2p_abc"]["priority"] = 100
+        self.assertEquals([self._machine_request(nodearray="execute", machine_type="a2p_def", count=3, placeby="group_id", placeby_value="def"),
+                           self._machine_request(nodearray="execute", machine_type="a2p_abc", count=1, placeby="group_id", placeby_value="abc")], 
+                          self._autoscale(q, cluster_def, [other_pg]))
+    
+    @unittest.skip("For now this seems pointless - just use group_id. WIll need to figure out behavior around non-group_id groupings")
+    def test_pbsuserguide_ex20(self):
+        '''
+            20. Align a large job within one router, if it fits within a router:
+            -l select=1:ncpus=100:mem=200gb
+            -l place=pack:group=router 
+        '''
+        self.fail()
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=100:mem=200G", place="pack:group=router")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2r", 100, 200, 100, priority=100,
+                                                                                    placeby="router", router="router-123"),
+                                                machine.new_machinetype("execute", "a2r0", 100, 200, 100, priority=100,
+                                                                                    placeby="router", router="router-123"),
+                                                machine.new_machinetype("execute", "a2", 100, 200, 100, priority=200))
+        
+        self.assertEquals([self._machine_request(machine_type="a2r", count=1, placeby="router", placeby_value="router-123")], self._autoscale(q, cluster_def))
+        self.assertEquals([MachineRequest("execute", "a2r", 1, "router-123")], q.autoscale_requests(cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="5:ncpus=100:mem=200G", place="scatter:excl:group=placement_group")
+        
+        self.assertEquals([MachineRequest("execute", "a2r", 5, "router-123")], q.autoscale_requests(cluster_def))
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2r", 100, 200, 100, priority=100,
+                                                                                     availableCount=4,
+                                                                                    placeby="placement_group", placement_group="router-123"),
+                                                machine.new_machinetype("execute", "a2r0", 100, 200, 100, priority=50,
+                                                                                    placeby="placement_group", placement_group="router-234"),
+                                                machine.new_machinetype("execute", "a2", 100, 200, 100, priority=200))
+        
+        self.assertEquals([MachineRequest("execute", "a2r0", 5, "router-234")], q.autoscale_requests(cluster_def))
+        
+    def test_pbsuserguide_ex22(self):
+        '''
+            22. To submit an MPI job, specify one chunk per MPI task. For a 10-way MPI job with 2gb of memory per MPI task:
+            -l select=10:ncpus=1:mem=2gb
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="10:ncpus=1:mem=2G", place="free")
+
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 32, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="10:ncpus=1:mem=2G", place="free")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 32, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="10:ncpus=1:mem=2G", place="scatter")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 32, 100, priority=100))
+
+        self.assertEquals([self._machine_request(machine_type="a2", count=10)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="10:ncpus=1:mem=2G", place="scatter")
+        q.qsub(select_expr="10:ncpus=1:mem=2G", place="scatter")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 32, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=10)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="10:ncpus=1:mem=2G", place="scatter")
+        q.qsub(select_expr="10:ncpus=1:mem=2G", place="scatter:excl")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 32, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=20)], self._autoscale(q, cluster_def))
+    
+    def test_pbsuserguide_ex23(self):
+        '''
+            23. To submit a non-MPI job (including a 1-CPU job or an OpenMP or shared memory) job, use a single chunk. For a
+            2-CPU job requiring 10gb of memory:
+            -l select=1:ncpus=2:mem=10gb
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=2:mem=10G")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 2, 32, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 1, 32, 100, priority=100))
+        self.assertEquals([], self._autoscale(q, cluster_def))
+        
+    def test_pack_available(self):
+        q = PBSQ()
+        q.qsub(select_expr="2:ncpus=2:mem=2G", place="pack")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 4, 32, 100, priority=100, availableCount=1))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="2:ncpus=2:mem=2G", place="pack")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 2, 32, 100, priority=100, availableCount=2))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+        
+        q.qsub()
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 1, 32, 100, priority=100, availableCount=1))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub()
+        q.qsub()
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 1, 32, 100, priority=100, availableCount=2))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+    
+    def _test_live(self, live_jobstatus, expected):
+        q = PBSQ()
+        q.qsub(select_expr="2:ncpus=2", place="scatter:excl")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 8))
+        
+        mock_driver = MockDriver(["workq"], q.queues)
+        mock_driver.queued_jobs = lambda: (live_jobstatus, lambda x: x)
+        cluster = MockClustersAPI(cluster_def)
+        a = PBSAutostart(mock_driver, cluster, {})
+        
+        autoscale_requests = a.autoscale()[0]
+        
+        if not isinstance(expected, list):
+            expected = [expected]
+            
+        self.assertEquals(expected, autoscale_requests)
+    
+    def test_live(self):
+        queued = [{'job_id': '11.ip-0A030005',
+                      'job_state': 'Q',
+                      'queue': 'workq',
+                      'resource_list': {'ncpus': '2',
+                                        'nodect': '2',
+                                        'place': 'scatter:excl:group=group_id',
+                                        'select': '2:ncpus=1:slot_type=execute'}}]
+        self._test_live(queued, MachineRequest("execute", "a2", 2, "group_id", "single"))
+
+    def test_live2(self):
+        queued = [{'job_id': '27.ip-0A030005',
+                      'job_state': 'Q',
+                      'queue': 'workq',
+                      'resource_list': {'ncpus': '4',
+                                        'nodect': '2',
+                                        'place': 'scatter:group=group_id',
+                                        'select': '2:ncpus=2:slot_type=execute'}},
+                     {'job_id': '28.ip-0A030005',
+                      'job_state': 'Q',
+                      'queue': 'workq',
+                      'resource_list': {'ncpus': '4',
+                                        'nodect': '2',
+                                        'place': 'scatter:group=group_id',
+                                        'select': '2:ncpus=2:slot_type=execute'}}]
+        self._test_live(queued, MachineRequest("execute", "a2", 2, "group_id", "single"))
+        
+    def test_live3(self):
+        queued = [{'job_id': '27.ip-0A030005',
+                      'job_state': 'Q',
+                      'resource_list': {'mem': '2mb',
+                                        'ncpus': '4',
+                                        'nodect': '2',
+                                        'place': 'scatter:group=group_id',
+                                        'select': '2:ncpus=2:slot_type=execute'}}]
+        self._test_live(queued, MachineRequest("execute", "a2", 2, "group_id", "single"))
+        
+    def test_old_style_nodes(self):
+        queued = [{'job_id': '123',
+                   'job_state': 'Q',
+                   'exec_host': 'cazlrss28/0*8',
+                   'exec_vnode': '(cazlrss28:ncpus=8)',
+                   'resource_list': {
+                       'mpiprocs': '8',
+                       'ncpus': '8',
+                       'nodect': '1',
+                       'nodes': '1:ppn8',
+                       'place': 'scatter',
+                       'select': '1:ncpus=8:mpiprocs=8',
+                       'ungrouped': 'true'}}]
+        self._test_live(queued, MachineRequest("execute", "a2", 1, "", ""))
+        
+    def test_forced_assignment(self):
+        queued = [{'job_id': '1',
+                      'job_state': 'Q',
+                      'resource_list': {'mem': '2gb',
+                                        'ncpus': '2',  # too many ncpus
+                                        'nodect': '1',
+                                        'place': 'scatter:group=group_id',
+                                        'select': '1:ncpus=2:slot_type=execute'}}]
+            
+        running = [{'job_id': '2',
+                      'job_state': 'R',
+                      'exec_vnode': '(hostname-0:ncpus=18)',
+                      'resource_list': {'mem': '2gb',
+                                        'ncpus': '18',  # too many ncpus
+                                        'nodect': '1',
+                                        'arbitrary_string': 'abc', # undefined
+                                        'arbitrary_bool': False,  # undefined
+                                        'place': 'scatter:group=group_id',
+                                        'select': '1:ncpus=18:slot_type=execute'}}]
+        
+        hosts = [{"state": "job-busy", "jobs": ['2'], "resources_assigned": {"ncpus": 2}, "resources_available": {"vnode": "hostname-0", "group_id": "single", "ncpus": 16, "mem": 8, "slot_type": "execute", "machinetype": "a2"}},
+                 {"state": "job-idle", "resources_assigned": {}, "resources_available": {"vnode": "hostname-1", "group_id": "single", "ncpus": 16, "mem": 8, "slot_type": "execute", "machinetype": "a2"}}]
+        
+        for host in hosts:
+            host["last_state_change_time"] = time.time()
+        
+        q = PBSQ()
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 8, 100, placeby="group_id", group_id="single"))
+        
+        mock_driver = MockDriver(["workq"], q.queues, hosts=hosts)
+        mock_driver.queued_jobs = lambda: (queued, lambda x: x)
+        mock_driver.running_jobs = lambda: (running, lambda x: x)
+        cluster = MockClustersAPI(cluster_def)
+        a = PBSAutostart(mock_driver, cluster, {})
+        
+        autoscale_requests, idle_machines, all_machines = a.autoscale()
+        
+        self.assertEquals([], autoscale_requests)
+        self.assertEquals([], idle_machines)
+        
+        machine_0 = [x for x in all_machines if x.hostname == "hostname-0"][0]
+        machine_1 = [x for x in all_machines if x.hostname == "hostname-1"][0]
+        self.assertEquals(["1"], machine_1.assigned_job_ids())
+
+        self.assertEquals(["2"], machine_0.assigned_job_ids())
+        
+    def test_live4(self):
+        q = PBSQ()
+        q.qsub(select_expr="2:ncpus=2", place="scatter:group=group_id")
+        q.qsub(select_expr="2:ncpus=2", place="scatter:group=group_id")
+        q.qsub(select_expr="2:ncpus=2", place="scatter:group=group_id")
+        
+        q.qsub(select_expr="2:ncpus=2", place="scatter:excl:group=group_id")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 4, 8))
+        
+        q.qsub(select_expr="2:ncpus=2", place="scatter:group=group_id")
+        pbs_autostart = PBSAutostart(MockDriver(jobs=q.queues), MockClustersAPI(cluster_def), {})
+        self.assertEquals([MachineRequest("execute", "a2", 6, "group_id", "single")], pbs_autostart.autoscale()[0])
+        
+    def test_pbsuserguide_ex8_p2(self):
+        '''
+            8. This will allocate 2 CPUs and 50GB of memory on a host named zooland. The value of "place" depends on the
+            default which defaults to "place=free":
+            -l select=1:ncpus=2:mem=50gb:host=zooland
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=2:mem=2G:host=zooland")
+        
+        a2_mt = machine.new_machinetype("execute", "a2", 32, 128, 100)
+        cluster_def = _nodearray_definitions(a2_mt)
+        
+        zooland = self._host(hostname="zooland", machinetype=a2_mt, ncpus=32, mem=128)
+        notzooland = self._host(hostname="notzooland", machinetype=a2_mt, ncpus=32, mem=128)
+        
+        new, idle, all = self._rebalance(q, cluster_def, [zooland, notzooland])
+        self.assertEquals([], new)
+        self.assertEquals(1, len(idle))
+        self.assertEquals("notzooland", idle[0].get_attr("host"))
+        
+        q.qsub(select_expr="1:ncpus=2:mem=2G:host=notzooland")
+        
+        new, idle, all = self._rebalance(q, cluster_def, [zooland, notzooland])
+        self.assertEquals([], new)
+        self.assertEquals(0, len(idle))
+        
+        q.qdel(1)
+        
+        new, idle, all = self._rebalance(q, cluster_def, [zooland, notzooland])
+        self.assertEquals([], new)
+        self.assertEquals(1, len(idle))
+        self.assertEquals("zooland", idle[0].get_attr("host"))
+        
+    def test_arrays(self):
+        q = PBSQ()
+        q.qsub(J="1:100")
+        # 100 * 1 core / 32 core boxes
+        self.assertEquals([self._machine_request(count=4)], self._autoscale(q))
+        
+        q = PBSQ()
+        q.qsub(J="1:100", ncpus=16)
+        # 100 * 16 cores / 32 core boxes
+        self.assertEquals([self._machine_request(count=50)], self._autoscale(q))
+        
+        q = PBSQ()
+        q.qsub(J="1:25", mem=30)
+        # 25 * 30 GB ram / 100 GB boxes - 3 jobs per box, so 9 boxes are required
+        self.assertEquals([self._machine_request(count=9)], self._autoscale(q))
+        
+    def test_booleans_as_strings(self):
+        ''' caught a bug where booleans show up as strings in the resources. We needed to auto-convert it'''
+        job_status = [{"job_id": "7.ip-0A03000A", 
+                        "job_state": "Q", 
+                        "resource_list": {
+                          "ncpus": "1", 
+                          "nodect": "1", 
+                          "place": "pack", 
+                          "select": "1:ncpus=1:slot_type=execute:ungrouped=True", 
+                          "slot_type": "execute", 
+                          "ungrouped": "True"
+                        }}]
+                          
+        self._test_live(job_status, [self._machine_request(machine_type="a2", count=1)])
+        
+    def test_custom_attribute_on_booting_instance(self):
+        mt = machine.new_machinetype("execute", "a2", 4, 128, 100, availableCount=1000 * 1000 * 100)
+        cluster_def = _nodearray_definitions(mt)
+        q = PBSQ()
+        q.qsub(ungrouped=True)
+        driver = MockDriver(jobs=q.queues)
+        cluster = MockClustersAPI(cluster_def, nodes=[{"MachineType": "a2", "InstanceId": "123", "Template": "execute"}])
+        autoscale_requests, idle_machines, all_machines = PBSAutostart(driver, cluster, {}).autoscale()
+        self.assertEquals([], autoscale_requests)
+        self.assertEquals([], idle_machines)
+        self.assertTrue(list(all_machines)[0].get_attr("ungrouped"))
+        
+    def test_no_nodearray_on_job(self):
+        q = PBSQ()
+        q.qsub()
+        requests = self._autoscale(q)
+        self.assertEquals([self._machine_request(count=1)], requests)
+        
+    def test_host_lifecycle(self):
+        q = PBSQ()
+        cc_config = InstanceConfig({}, {})
+        q.qsub(select_expr="2:ncpus=16", place="scatter:excl:group=group_id")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a4", 32, 100, 100))
+        cluster = MockClustersAPI(cluster_def)
+        driver = MockDriver(jobs=q.queues, hosts=[])
+        
+        def run_test(num_idle, num_request):
+            
+            pbs_autostart = PBSAutostart(driver, cluster, cc_config)
+            
+            autoscale_requests, idle_machines, machines = pbs_autostart.autoscale()
+            expected_requests = [self._machine_request(count=num_request, machine_type="a4", placeby="group_id", placeby_value="single")] if num_request else []
+            self.assertEquals(expected_requests, autoscale_requests)
+            self.assertEquals(num_idle, len(idle_machines))
+            return machines
+        
+        # first time we request 2 machines
+        run_test(num_idle=0, num_request=2)
+        # second time they are already booting
+        machines = run_test(num_idle=0, num_request=0)
+        
+        # create them as hosts, but the jobs still aren't running...
+        for n, m in enumerate(machines):
+            host_attrs = {}
+            for name, attr in m.iterattrs():
+                host_attrs[name] = attr.current_value
+        
+            host_attrs.pop("hostname", "")
+            host_attrs.pop("host", "")
+            host_attrs["instance_id"] = "instance-%s" % n
+        
+            driver.add_host("hostname-%s" % n, state="free", **host_attrs)
+        
+        run_test(num_idle=0, num_request=0)
+        
+        # turns out the job ended up only running on one host... weird! But we need to respect it.
+        host1 = driver.get_host("hostname-0")
+        host1["jobs"] = ["1", "2"]
+        host1["resources_assigned"] = {"ncpus": 16}
+        host1["state"] = "job-busy"
+        host1["last_state_change_time"] = time.time()
+        
+        q.set_running("1", "(hostname-0:ncpus=1)+(hostname-0:ncpus=1)")
+        
+        run_test(num_idle=1, num_request=0)
+        host2 = driver.get_host("hostname-1")
+        host2["last_state_change_time"] = time.time() - 3000
+        # default is 300 seconds for idle machines t hat have run at least one job.
+        host2["last_used_time"] = time.time() - 301
+        
+        self.assertEquals(host2["state"], "free")
+        
+        cc_config.set("cyclecloud.cluster.autoscale.stop_enabled", False)
+        run_test(num_idle=1, num_request=0)
+        self.assertEquals(host2["state"], "free")
+        
+        cc_config.set("cyclecloud.cluster.autoscale.stop_enabled", True)
+        run_test(num_idle=1, num_request=0)
+        self.assertEquals(host2["state"], "offline")
+        # FYI idle nodes are actually terminated, but it still returns this. This isn't part of the API, this is just exposing
+        # the internals of PBS autostart for testing purposes.
+        machines = run_test(num_idle=1, num_request=0)
+        machines = run_test(num_idle=0, num_request=0)
+        # shutdown hostname-2
+        self.assertEquals(1, len(machines))
+        
+    def test_disable_start(self):
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=16:mem=20G", place="pack")
+        cc_config = InstanceConfig({}, {})
+        cc_config.set("cyclecloud.cluster.autoscale.start_enabled", False)
+        self.assertEquals([], self._autoscale(q, cc_config=cc_config))
+    
+    def test_packed_multi_chunk(self):
+        q = PBSQ()
+        cc_config = InstanceConfig({}, {})
+        q.qsub(select_expr="2:mem=15G+2:ncpus=4", place="group=group_id")
+        
+        cluster_def = _nodearray_definitions(
+            machine.new_machinetype("execute", "a2", 2, 16, 100),
+            machine.new_machinetype("execute", "a4", 4, 8, 100))
+        cluster = MockClustersAPI(cluster_def)
+        driver = MockDriver(jobs=q.queues, hosts=[])
+        
+        def run_test(num_idle, num_request):
+            
+            pbs_autostart = PBSAutostart(driver, cluster, cc_config)
+            
+            autoscale_requests, idle_machines, machines = pbs_autostart.autoscale()
+            autoscale_requests = sorted(autoscale_requests, key=lambda x: x.machinetype)
+            expected_requests = [self._machine_request(count=2, machine_type="a2", placeby="group_id", placeby_value="single"),
+                                self._machine_request(count=2, machine_type="a4", placeby="group_id", placeby_value="single")] if num_request else []
+            self.assertEquals(expected_requests, autoscale_requests)
+            self.assertEquals(num_idle, len(idle_machines))
+            return machines
+        
+        # first time we request 2 machines
+        run_test(num_idle=0, num_request=4)
+        # second time they are already booting
+        assert len(run_test(num_idle=0, num_request=0)) == 4
+        
+    def test_chunk_scatter_excl(self):
+        q = PBSQ()
+        cc_config = InstanceConfig({}, {})
+        q.qsub(select_expr="1:ncpus=4+4:ncpus=4", place="scatter:excl:group=group_id")
+        
+        cluster_def = _nodearray_definitions(
+            machine.new_machinetype("execute", "a2", 2, 16, 100),
+            machine.new_machinetype("execute", "a4", 4, 8, 100))
+        cluster = MockClustersAPI(cluster_def)
+        driver = MockDriver(jobs=q.queues, hosts=[])
+        
+        def run_test(num_idle, num_request):
+            
+            pbs_autostart = PBSAutostart(driver, cluster, cc_config)
+            
+            autoscale_requests, idle_machines, machines = pbs_autostart.autoscale()
+            autoscale_requests = sorted(autoscale_requests, key=lambda x: x.machinetype)
+            expected_requests = [self._machine_request(count=5, machine_type="a4", placeby="group_id", placeby_value="single")] if num_request else []
+            self.assertEquals(expected_requests, autoscale_requests)
+            self.assertEquals(num_idle, len(idle_machines))
+            return machines
+        
+        # first time we request 2 machines
+        run_test(num_idle=0, num_request=4)
+        # second time they are already booting
+        assert len(run_test(num_idle=0, num_request=0)) == 5
+        
+    def test_shutdown_down_nodes(self):
+        '''
+        Create three nodes - one that is up and busy, one that is down but presumed busy, and one that is just down.
+        Test that the down node goes away after 100 seconds but that the 'down,job-busy' node does not until it is in the 'down' state.
+        '''
+        mt = machine.new_machinetype("execute", "a2", 4, 128, 100, availableCount=1000 * 1000 * 100)
+        cc_config = InstanceConfig({}, {})
+        cc_config.set("pbspro.remove_down_nodes", "100")
+        clock = MockClock(time.time())
+        
+        q = PBSQ()
+        # create one job for an up and busy node, and one for a down yet busy node 
+        q.qsub(job_id="1")
+        q.qsub(job_id="2")
+        q.set_running("1", "(jobbusy:ncpus=1)")
+        q.set_running("2", "(downjobbusy:ncpus=1)")
+        
+        cluster_def = _nodearray_definitions(mt)
+        busy = self._host(mt, hostname="jobbusy", host="jobbusy", state="job-busy", instance_id="i123", jobs=["1"])
+        down = self._host(mt, hostname="down", host="down", state="down", instance_id="i234", last_state_change_time=clock.time() - 99)
+        # make it so this state is very old - we are asserting that it isn't treated as idle by PBS autostart (though it is returned as isdle by the autoscaler) 
+        downjobbusy = self._host(mt, hostname="downjobbusy", host="downjobbusy", state="down,job-busy", instance_id="i345", jobs=["2"], last_state_change_time=clock.time() - 10000)
+        
+        driver = MockDriver(jobs=q.queues, hosts=[busy, down, downjobbusy])
+        
+        cluster = MockClustersAPI(cluster_def, nodes=[{"MachineType": "a2", "InstanceId": "i123", "Template": "execute", "hostname": "jobbusy"},
+                                                      {"MachineType": "a2", "InstanceId": "i234", "Template": "execute", "hostname": "down"},
+                                                      {"MachineType": "a2", "InstanceId": "i345", "Template": "execute", "hostname": "downjobbusy"}])
+        
+        # we haven't hit the timeout threshold
+        autoscale_requests, idle_machines, all_machines = PBSAutostart(driver, cluster, cc_config, clock=clock).autoscale()
+        self.assertEquals([], autoscale_requests)
+        self.assertEquals(set(["down", "jobbusy", "downjobbusy"]), set([m.hostname for m in all_machines]))
+        self.assertEquals(["down"], [m.hostname for m in idle_machines])
+        
+        # we have hit the timeout threshold, though autoscale will still return the idle_machine here for testing.
+        clock.now += 2
+        autoscale_requests, idle_machines, all_machines = PBSAutostart(driver, cluster, cc_config, clock=clock).autoscale()
+        self.assertEquals([], autoscale_requests)
+        self.assertEquals(set(["down", "jobbusy", "downjobbusy"]), set([m.hostname for m in all_machines]))
+        self.assertEquals(["down"], [m.hostname for m in idle_machines])
+        
+        # rerun autoscale and the down node is gone.
+        autoscale_requests, idle_machines, all_machines = PBSAutostart(driver, cluster, cc_config, clock=clock).autoscale()
+        self.assertEquals([], autoscale_requests)
+        self.assertEquals(set(["jobbusy", "downjobbusy"]), set([m.hostname for m in all_machines]))
+        self.assertEquals([], [m.hostname for m in idle_machines])
+        
+        # the node is now just down, as PBS has rescheduled the job or removed it. Run autoscale twice to check it is removed.
+        downjobbusy["state"] = "down"
+        PBSAutostart(driver, cluster, cc_config, clock=clock).autoscale()
+        autoscale_requests, idle_machines, all_machines = PBSAutostart(driver, cluster, cc_config, clock=clock).autoscale()
+        
+        self.assertEquals([], autoscale_requests)
+        self.assertEquals(set(["jobbusy"]), set([m.hostname for m in all_machines]))
+        self.assertEquals([], [m.hostname for m in idle_machines])
+        
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/autostop.rb b/specs/default/chef/site-cookbooks/pbspro/files/default/autostop.rb
new file mode 100644
index 0000000..7f9a1cf
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/autostop.rb
@@ -0,0 +1,74 @@
+#!/usr/bin/env ruby
+
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+require 'json'
+
+# Arguments
+AUTOSTOP_ENABLED = `jetpack config cyclecloud.cluster.autoscale.stop_enabled`.downcase.strip == "true"
+
+# Short-circuit without error if not enabled
+exit 0 unless AUTOSTOP_ENABLED
+
+IDLE_TIME_AFTER_JOBS = `jetpack config cyclecloud.cluster.autoscale.idle_time_after_jobs`.to_i
+IDLE_TIME_BEFORE_JOBS = `jetpack config cyclecloud.cluster.autoscale.idle_time_before_jobs`.to_i
+
+# Checks to see if we should shutdown
+idle_long_enough = false
+
+# indicates if execute node has ever ran a job
+def been_active?
+  # Shell out to grep with -m 1 for lazy match, as the log files can grow quite large
+  # and we only need to know if one job has ever started.
+  any_job = `egrep -m 1 ';pbs_mom;Job;.+;Started' /var/spool/pbs/mom_logs/*`.strip
+  any_job.length > 0
+end
+
+# indicates if there are currently running jobs
+def active?
+  activejobs = Dir.glob('/var/spool/pbs/mom_priv/jobs/*').count
+  activejobs > 0
+end
+
+# This is our autoscale runtime configuration
+runtime_config = {"idle_start_time" => nil}
+
+
+AUTOSCALE_DATA = '/opt/cycle/jetpack/run/autoscale.json'.freeze
+
+if File.exist?(AUTOSCALE_DATA)
+  file = File.read(AUTOSCALE_DATA)
+  runtime_config.merge!(JSON.parse(file))
+end
+
+if active?
+  runtime_config["idle_start_time"] = nil
+elsif runtime_config["idle_start_time"].nil?
+  runtime_config["idle_start_time"] = Time.now.to_i
+else
+  idle_seconds = Time.now - Time.at(runtime_config["idle_start_time"].to_i)
+
+  # Different timeouts if the node has ever run a job
+  timeout = if been_active?
+              IDLE_TIME_AFTER_JOBS
+            else
+              IDLE_TIME_BEFORE_JOBS
+            end
+
+  idle_long_enough = idle_seconds > timeout
+end
+
+# Write the config information back for next time
+file = File.new(AUTOSCALE_DATA, "w")
+file.puts JSON.pretty_generate(runtime_config)
+file.close
+
+# Do the shutdown
+if idle_long_enough
+  myhost = `hostname`
+  system("bash -lc 'pbsnodes -o #{myhost}'")
+  sleep(5)
+  system("jetpack shutdown --idle")
+end
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/config b/specs/default/chef/site-cookbooks/pbspro/files/default/config
new file mode 100644
index 0000000..6de8e9c
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/config
@@ -0,0 +1 @@
+$usecp *:/shared /shared
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/doqmgr.sh b/specs/default/chef/site-cookbooks/pbspro/files/default/doqmgr.sh
new file mode 100644
index 0000000..a099e7e
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/doqmgr.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+/opt/pbs/bin/qmgr -c 'set server managers = root@*'
+/opt/pbs/bin/qmgr -c 'set server query_other_jobs = true'
+/opt/pbs/bin/qmgr -c 'set server scheduler_iteration = 15'
+
+function create_resource() {
+	/opt/pbs/bin/qmgr -c "list resource $1" >/dev/null  2>/dev/null   || \
+	/opt/pbs/bin/qmgr -c "create resource $1 type=$2, flag=h"
+}
+
+create_resource slot_type string
+create_resource group_id string
+create_resource ungrouped string
+create_resource instance_id string
+create_resource machinetype string
+create_resource nodearray string
+create_resource disk size
+create_resource ngpus size
+
+/opt/pbs/bin/qmgr -c "set queue workq resources_default.slot_type = execute"
+/opt/pbs/bin/qmgr -c "set queue workq resources_default.ungrouped = false"
+/opt/pbs/bin/qmgr -c "set queue workq resources_default.place = scatter"
+/opt/pbs/bin/qmgr -c "set queue workq default_chunk.slot_type = execute"
+/opt/pbs/bin/qmgr -c "set queue workq default_chunk.ungrouped = false"
+/opt/pbs/bin/qmgr -c "set queue workq default_chunk.place = scatter"
+
+/opt/pbs/bin/qmgr -c "create queue htcq resources_default.slot_type = execute"
+/opt/pbs/bin/qmgr -c "set queue htcq queue_type = Execution"
+/opt/pbs/bin/qmgr -c "set queue htcq resources_default.slot_type = execute"
+/opt/pbs/bin/qmgr -c "set queue htcq resources_default.ungrouped = true"
+/opt/pbs/bin/qmgr -c "set queue htcq resources_default.place = pack"
+/opt/pbs/bin/qmgr -c "set queue htcq default_chunk.slot_type = execute"
+/opt/pbs/bin/qmgr -c "set queue htcq default_chunk.ungrouped = true"
+/opt/pbs/bin/qmgr -c "set queue htcq default_chunk.place = pack"
+/opt/pbs/bin/qmgr -c "set queue htcq enabled = true"
+/opt/pbs/bin/qmgr -c "set queue htcq started = true"
\ No newline at end of file
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/logging_init.py b/specs/default/chef/site-cookbooks/pbspro/files/default/logging_init.py
new file mode 100644
index 0000000..a665c80
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/logging_init.py
@@ -0,0 +1,40 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+'''
+The purpose of this class is to call basicConfig before any other module does, specifically requests.
+'''
+import logging
+import logging.handlers
+import os
+import sys
+
+
+log_level_name = os.getenv('AUTOSTART_LOG_LEVEL', "INFO")
+log_file_level_name = os.getenv('AUTOSTART_LOG_FILE_LEVEL', "DEBUG")
+log_file = os.getenv('AUTOSTART_LOG_FILE', "autoscale.log")
+
+if log_level_name.lower() not in ["debug", "info", "warn", "error", "critical"]:
+    log_level = logging.DEBUG
+else:
+    log_level = getattr(logging, log_level_name.upper())
+
+if log_file_level_name.lower() not in ["debug", "info", "warn", "error", "critical"]:
+    log_file_level = logging.DEBUG
+else:
+    log_file_level = getattr(logging, log_level_name.upper())
+
+requests_logger = logging.getLogger("requests.packages.urllib3.connectionpool")
+requests_logger.setLevel(logging.WARN)
+
+stderr = logging.StreamHandler(stream=sys.stderr)
+stderr.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s"))
+stderr.setLevel(log_level)
+
+log_file = logging.handlers.RotatingFileHandler(log_file, maxBytes=1024 * 1024 * 5, backupCount=5)
+log_file.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s"))
+log_file.setLevel(log_file_level)
+
+logging.getLogger().addHandler(stderr)
+logging.getLogger().addHandler(log_file)
+logging.getLogger().setLevel(logging.DEBUG)
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/mockpbs.py b/specs/default/chef/site-cookbooks/pbspro/files/default/mockpbs.py
new file mode 100644
index 0000000..20bd0fb
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/mockpbs.py
@@ -0,0 +1,131 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+import logging
+import os
+import time
+from UserDict import UserDict
+from copy import deepcopy
+
+LOG_DEBUG = logging.DEBUG
+LOG_WARNING = logging.WARN
+LOG_ERROR = logging.ERROR
+
+QUEUEJOB = "QUEUEJOB"
+PERIODIC = "PERIODIC"
+EVENT_ERROR = logging.ERROR
+
+
+def hold_types(h):
+    return h
+
+
+# feel free to change as needed
+hook_config_filename = "hook_config_file.json"
+
+pbs_conf = {"PBS_EXEC": os.getcwd()}
+
+
+def logmsg(level, msg):
+    logging.log(level, msg)
+
+
+class _Repr:
+    def __init__(self, expr):
+        self.expr = expr
+        
+    def __repr__(self):
+        return self.expr
+
+
+def place(expr):
+    return _Repr(expr)
+
+
+def select(expr):
+    return _Repr(expr)
+
+
+_job_queue = []
+
+
+def testing_add_job(job):
+    _job_queue.append(job)
+
+
+class _Event:
+    def __init__(self, job):
+        self.job = job
+    
+    
+def event():
+    return _Event(_job_queue.pop())
+
+
+pbs_str = str
+
+
+class _MockJob:
+    
+    def __init__(self, resource_list=None):
+        # match the behavior of PBS
+        self._data = {}
+        self.Resource_List = self._data["resource_list"] = ResourceList()
+        if resource_list:
+            self.Resource_List.update(resource_list)
+        
+    def get(self, key, default=None):
+        return self._data.get(key, default)
+        
+    def __getitem__(self, attr):
+        return self._data[attr]
+    
+    def __setitem__(self, attr, value):
+        self._data[attr] = value
+        
+    def iteritems(self):
+        return self._data.iteritems()
+    
+    def __contains__(self, key):
+        return key in self._data
+    
+
+class ResourceList(UserDict):
+    
+    def __init__(self, *args, **kwargs):
+        UserDict.__init__(self, *args, **kwargs)
+    
+    def __getitem__(self, key):
+        if key in self:
+            return UserDict.__getitem__(self, key)
+        return None
+    
+    def __setitem__(self, key, item):
+        # same failure occurs with the pbs ResourceList
+        assert item is not None, "attempted to set %s to '%s'" % (key, item)
+        UserDict.__setitem__(self, key, item)
+    
+    def __repr__(self):
+        return UserDict.__repr__(self)
+    
+    def __str__(self):
+        return UserDict.__repr__(self)
+    
+
+def mock_job(raw_job):
+    '''
+    Create a job object that matches the interface (duck typing) of the internal pbs job class, which
+    is only accessible in the submit_hook currently.
+    '''
+    raw_job = deepcopy(raw_job)
+    job = _MockJob(raw_job.pop("resource_list", None))
+    
+    job["job_id"] = raw_job.pop("job_id", str(time.time()))
+    job["job_state"] = raw_job.pop("job_state", "Q")
+    job["array"] = raw_job.pop("array", False)
+   
+    if job["array"]:
+        job["array_state_count"] = raw_job.pop("array_state_count")
+    
+    # remaining things are assumed to be resources
+    return job
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/pbs_driver.py b/specs/default/chef/site-cookbooks/pbspro/files/default/pbs_driver.py
new file mode 100644
index 0000000..7624b7d
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/pbs_driver.py
@@ -0,0 +1,301 @@
+#!/usr/bin/python
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+import cStringIO
+from collections import OrderedDict
+
+import tandem_utils
+import tandem_driver_main
+import json
+from tandem_driver_main import TandemDriver
+import os
+import re
+import pbscc
+
+#     E -     Job is    exiting    after having run.
+#     H -     Job is    held.
+#     Q -     job is    queued,    eligable to run    or routed.
+#     R -     job is    running.
+#     T -     job is    being moved to new location.
+#     W -     job is    waiting    for its    execution time
+#      (-a option) to    be reached.
+#     S -     (Unicos only) job is suspend.
+_PBS_STATES = {"E": "e",
+               "H": "h",
+               "Q": "i",
+               "R": "r",
+               "W": "i",
+               "S": "i"
+               }
+
+_PBS_NOT_FOUND = 153
+
+JOB_STATE_EXITING = "E"
+JOB_STATE_FINISHED = "F"
+JOB_STATE_HELD = "H"
+JOB_STATE_QUEUED = "Q"
+JOB_STATE_RUNNING = "R"
+JOB_STATE_SUSPEND = "S"
+JOB_STATE_TRANSIT = "T"
+JOB_STATE_USER_SUSPEND = "U"
+JOB_STATE_WAITING = "W"
+JOB_STATE_EXPIRED = "X"
+
+
+class PBSDriver(TandemDriver):
+
+    def __init__(self, bin_dir=None, version=None):
+        TandemDriver.__init__(self)
+        self.bin_dir = bin_dir
+        self.version = version if version else self._version()
+        
+    def capabilities(self):
+        return {
+                    "format": ["native"],
+                    "version": self.version,
+                    "schedulerType": "pbs"
+                }
+
+    def _version(self):
+        try:
+            version_line = tandem_utils.check_call(["qmgr", "--version"]).split("\n")[0]
+            return version_line.split("=")[1].strip()
+        except:
+            return "-1"
+
+    def _bin(self, name):
+        return name if self.bin_dir is None else os.path.join(self.bin_dir, name)
+
+    def queues(self):
+        lines = tandem_utils.check_call([self._bin("qstat"), "-Q"]).split("\n")
+        return [x.split()[0] for x in lines[2:] if x.strip()]
+
+    def jobstatus(self, scheduler_name, marker=None, local_job_id_query=None):
+        marker = float(marker) if marker else 0
+        stdout, stderr, code, converter = self._jobstatus()
+        if code == 0:
+            return converter(stdout)
+        elif code == _PBS_NOT_FOUND:
+            return []
+        else:
+            tandem_utils.error_and_exit(stderr)
+            
+    def scheduler_config(self):
+        sched_config = "/var/spool/pbs/sched_priv/sched_config"
+        pattern = re.compile(r"^\s*resources\s*: .+$")
+        try:
+            if os.path.exists(sched_config):
+                for line in open(sched_config):
+                    line = line.strip()
+                    if pattern.match(line):
+                        expr = line.split(":", 1)[1].strip()
+                        expr = expr.replace('\"', "")
+                        toks = expr.split(",")
+                        resources = [x.strip() for x in toks]
+                        return {"resources": resources}
+        except Exception as e: 
+            pbscc.error("Could not parse %s, using default resources. Error was %s" % (sched_config, str(e)))
+        
+        # just return default values
+        return {"resources": ["ncpus", "mem", "arch", "host", "vnode", "aoe", "slot_type", "group_id", "ungrouped", "instance_id", "ipv4", "disk"]}
+            
+    def _jobstatus(self, local_job_id_query=None):
+        stdout, stderr, code = tandem_utils.call(self.qstat_args(local_job_id_query))
+        return stdout, stderr, code, _from_qstat
+
+    def submit(self, scheduler_name, format, submit_data):
+        if format == "tandem":
+            parsed_submit_data = submit_data
+        elif format == "tandem":
+            buf = cStringIO.StringIO()
+            jobs_data = json.loads(submit_data)
+            for job in jobs_data:
+                transform_tandem_job(job, buf)
+            submit_data = buf.getvalue()
+        jobid = tandem_utils.check_call([self._bin("qsub"), "-"], parsed_submit_data)
+        return [job[tandem_utils.LOCAL_JOB_ID_KEY] for job in self.jobstatus(scheduler_name, 0, jobid.strip())]
+    
+    def logs(self, scheduler_name, local_job_id_query):
+        jobs = self.jobstatus(scheduler_name, 0, local_job_id_query)
+        if not jobs:
+            return {}
+        job = jobs[0]
+        
+        stdout_path = job["Output_Path"].replace("\"", "").split(":")[1]
+        stderr_path = job["Error_Path"].replace("\"", "").split(":")[1]
+        return tandem_utils.make_std_logs(stdout_path, stderr_path)
+    
+    def schedstatus(self, scheduler_name):
+        # TODO
+        return [{tandem_utils.SCHED_NAME_KEY: scheduler_name}]
+    
+    def hold(self, scheduler_name, local_job_ids):
+        return self._hold_release_remove(self._bin("qhold"), local_job_ids)
+    
+    def release(self, scheduler_name, local_job_ids):
+        return self._hold_release_remove(self._bin("qrls"), local_job_ids)
+        
+    def remove(self, scheduler_name, local_job_ids):
+        return self._hold_release_remove(self._bin("qdel"), local_job_ids)
+    
+    def _hold_release_remove(self, cmd, local_job_ids):
+        out, err, code = tandem_utils.call([cmd] + local_job_ids)
+        if code == 0:
+            return {"status": "success"}
+        elif code == _PBS_NOT_FOUND:
+            return {"status": "not_found", "details": err}
+        else:
+            return {"status": "error", "details": err}
+        
+    def qstat_args(self, jobid=None):
+        return [self._bin("qstat"), "-f", "-w"] + ([jobid] if jobid else [])
+    
+    def running_jobs(self):
+        return self._get_jobs([self._bin("qstat"), "-f", "-w", "-t", "-r"])
+    
+    def queued_jobs(self):
+        return self._get_jobs([self._bin("qstat"), "-f", "-w", "-i"])
+    
+    def _get_jobs(self, args):
+        stdout, stderr, code = tandem_utils.call(args)
+        if code == 0:
+            return stdout, _from_qstat
+        elif code == _PBS_NOT_FOUND:
+            return "[]", json.loads
+        else:
+            tandem_utils.error_and_exit(stderr)
+    
+    def hosts(self, grouping=None, keyformatter=lambda x: x):
+        return self.pbsnodes(grouping, keyformatter)
+
+    def pbsnodes(self, grouping=None, keyformatter=lambda x: x):
+        stdout, stderr, ret = tandem_utils.call([self._bin("pbsnodes"), "-a", "-F", "json"])
+        
+        if ret == 1 and 'Server has no node list' in stderr:
+            return {None: {}}
+        
+        if ret != 0:
+            raise RuntimeError(stderr)
+        
+        nodes = json.loads(stdout)["nodes"]
+        
+        if not grouping:
+            return {None: nodes}
+        
+        if isinstance(grouping, basestring):
+            grouping = (grouping, )
+        
+        grouped = OrderedDict()
+        for _node_name, node in nodes.iteritems():
+            res_avail = node["resources_available"]
+            
+            def flat_get(node, g):
+                return res_avail.get(g, node.get(g))
+            
+            key = keyformatter(tuple([flat_get(node, g) for g in grouping]))
+            if key not in grouped:
+                grouped[key] = []
+            grouped[key].append(node)
+            
+        return grouped
+    
+    def set_offline(self, hostname):
+        tandem_utils.check_call([self._bin("pbsnodes"), "-o", hostname])
+        
+    def delete_host(self, hostname):
+        tandem_utils.check_call([self._bin("qmgr"), "-c", "delete node %s" % hostname])
+
+    def alter(self, jobs):
+        resources = {}
+        for job in jobs: 
+            resource_arg = ":".join(["%s=%s" % (x, y) for x, y in job.iteritems()])
+            if resource_arg not in resources:
+                resources[resource_arg] = []
+            resources[resource_arg].append(job["job_id"])
+        for resource_arg, job_ids in resources.iteritems():
+            tandem_utils.check_call(". /etc/cluster-setup.sh && qalter %s -l %s" % (",".join([str(j) for j in job_ids]), resource_arg))
+
+    def parse_select(self, job):
+        # Need to detect when slot_type is specified with `-l select=1:slot_type`
+        selectdict = {}
+        select_expression = str(job.Resource_List['select'])
+        
+        for expr in select_expression.split(":"):
+            key_val = expr.split("=", 1)
+            if len(key_val) == 1:
+                # this shouldn't ever happen, but log it in case.
+                if "select" in selectdict:
+                    continue
+                key_val = ("select", key_val[0])
+            selectdict[key_val[0]] = key_val[1]
+            
+        slot_type = selectdict.get('slot_type') or job.Resource_List.get('slot_type')
+        
+        return slot_type, selectdict
+    
+    def parse_place(self, place):
+        '''
+        arrangement is one of free | pack | scatter | vscatter
+        sharing is one of excl | shared | exclhost
+        grouping can have only one instance of group=resource
+        '''
+        placement = {"arrangement": "free"}
+        
+        if not place:
+            return placement
+        
+        toks = place.split(":")
+        
+        for tok in toks:
+            if tok in ["free", "pack", "scatter", "vscatter"]:
+                placement["arrangement"] = tok
+            elif tok in ["excl", "shared", "exclhost"]:
+                placement["sharing"] = tok
+            elif tok.startswith("group="):
+                placement["grouping"] = tok
+        
+        return placement
+
+
+def transform_tandem_job(job, stream):
+    # TODO can do other things like add defaults or translate TandemArguments == Arguments etc.
+    _to_jobad(job, stream)
+    
+
+def _to_jobad(job, stream):
+    for key, value in job.iteritems():
+        stream.write("%s = %s\n" % (key, value))
+    stream.write("queue\n")
+
+
+def _from_qstat(stdout, clz=OrderedDict):
+    ads = [clz()]
+    delim = ":"
+    for line in stdout.strip().split("\n"):
+        line = line.strip()
+        if not line:
+            if ads[-1]:
+                ads.append(clz())
+                delim = ":"
+        else:
+            if delim in line:
+                key, value = [x.strip() for x in line.split(delim, 1)]
+                keys = key.split(".")
+                parent = ads[-1]
+
+                def fmt_key(key):
+                    return key.replace(" ", "_").lower()
+
+                for parent_key in keys[:-1]:
+                    parent_key = fmt_key(parent_key)
+                    if parent_key not in parent:
+                        parent[parent_key] = clz()
+                    parent = parent[parent_key]
+                parent[fmt_key(keys[-1])] = value
+                delim = "="
+    return [ad for ad in ads if ad]
+
+
+if __name__ == "__main__":
+    tandem_driver_main.main(driver=PBSDriver())
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/pbscc.py b/specs/default/chef/site-cookbooks/pbspro/files/default/pbscc.py
new file mode 100644
index 0000000..7f515b7
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/pbscc.py
@@ -0,0 +1,239 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+'''
+Utility class for parsing pbs specific expressions and dealing with logging inside and outside the pbs hook. 
+'''
+import numbers
+import os
+import collections
+import logging
+try:
+    import pbs
+    # there is no info in the actual pbs implementation.
+    pbs_LOG_INFO = pbs.LOG_DEBUG
+except ImportError:
+    import mockpbs as pbs
+    pbs_LOG_INFO = __import__("logging").INFO
+    
+
+_PBS_NOT_FOUND = 153
+    
+JOB_STATE_EXITING = "E"
+JOB_STATE_FINISHED = "F"
+JOB_STATE_HELD = "H"
+JOB_STATE_QUEUED = "Q"
+JOB_STATE_RUNNING = "R"
+JOB_STATE_BATCH = "B"
+JOB_STATE_SUSPEND = "S"
+JOB_STATE_TRANSIT = "T"
+JOB_STATE_USER_SUSPEND = "U"
+JOB_STATE_WAITING = "W"
+JOB_STATE_EXPIRED = "X"
+
+
+CONFIG_PATH = os.path.join(os.getenv("CYCLECLOUD_BOOTSTRAP", "."), "pbs", "config.json")
+    
+
+def parse_select(raw_job):
+    # Need to detect when slot_type is specified with `-l select=1:slot_type`
+    chunks = []
+    if not raw_job["resource_list"]['select']:
+        return {}
+    select_expression = str(raw_job["resource_list"]['select'])
+    
+    for chunk_expr in select_expression.split("+"):
+        chunk = collections.OrderedDict()
+        for expr in chunk_expr.split(":"):
+            key_val = expr.split("=", 1)
+            if len(key_val) == 1:
+                # this shouldn't ever happen, but log it in case.
+                if "select" in chunk:
+                    continue
+                key_val = ("select", key_val[0])
+            chunk[key_val[0]] = key_val[1]
+        chunks.append(chunk)
+    
+    return chunks
+
+
+def format_select(chunk):
+    expr = "%s" % chunk["select"]
+    
+    for key, value in chunk.iteritems():
+        if key == "select":
+            continue
+        expr += ":%s=%s" % (key, value)
+    
+    return expr
+        
+
+def parse_place(place):
+    '''
+    arrangement is one of free | pack | scatter | vscatter
+    sharing is one of excl | shared | exclhost
+    grouping can have only one instance of group=resource
+    '''
+    placement = {"arrangement": "free"}
+    
+    if not place:
+        return placement
+    
+    toks = place.split(":")
+    
+    for tok in toks:
+        if tok in ["free", "pack", "scatter", "vscatter"]:
+            placement["arrangement"] = tok
+        elif tok in ["excl", "shared", "exclhost"]:
+            placement["sharing"] = tok
+        elif tok.startswith("group="):
+            placement["grouping"] = tok
+    
+    return placement
+
+
+def parse_exec_vnode(expr):
+    '''
+    Example:  "(ip-0A030008:ncpus=1)"
+    '''
+    expr = expr[1:-1]
+    host, resource_expr = expr.split(":", 1)
+    resources = {}
+    for res_sub_expr in resource_expr.split(":"):
+        attr, value = res_sub_expr.split("=", 1)
+        try:
+            value = parse_gb_size(attr, value)
+        except InvalidSizeExpressionError:
+            if value.lower() in ["true", "false"]:
+                value = value.lower() == "true"
+            
+        resources[attr] = value
+    resources["hostname"] = host
+    return resources
+
+
+class InvalidSizeExpressionError(RuntimeError):
+    pass
+
+
+def parse_gb_size(attr, value):
+    if isinstance(value, numbers.Number):
+        return value
+    try:
+        
+        value = value.lower()
+        if value.endswith("pb"):
+            value = float(value[:-2]) * 1024
+        elif value.endswith("p"):
+            value = float(value[:-1]) * 1024
+        elif value.endswith("gb"):
+            value = float(value[:-2])
+        elif value.endswith("g"):
+            value = float(value[:-1])
+        elif value.endswith("mb"):
+            value = float(value[:-2]) / 1024
+        elif value.endswith("m"):
+            value = float(value[:-1]) / 1024
+        elif value.endswith("kb"):
+            value = float(value[:-2]) / (1024 * 1024)
+        elif value.endswith("k"):
+            value = float(value[:-1]) / (1024 * 1024)
+        elif value.endswith("b"):
+            value = float(value[:-1]) / (1024 * 1024 * 1024)
+        else:
+            try:
+                value = int(value)
+            except:
+                value = float(value)
+        
+        return value
+    except ValueError:
+        raise InvalidSizeExpressionError("Unsupported size for %s - %s" % (attr, value))
+
+
+__FINE = 0
+__DEBUG = 1
+__INFO = 2
+__WARN = 3
+__ERROR = 4
+
+__log_level = {
+    "fine": __FINE,
+    "debug": __DEBUG,
+    "info": __INFO,
+    "warn": __WARN,
+    "error": __ERROR
+}.get(os.environ.get("AUTOSTART_LOG_LEVEL", "WARN").lower(), __WARN)
+
+
+__log_level_names = {
+    __FINE: "fine",
+    __DEBUG: "debug",
+    __INFO: "info",
+    __WARN: "warn",
+    __ERROR: "error"
+}
+
+
+__application_name = "cycle_plugin"
+
+
+def set_application_name(value):
+    global __application_name
+    __application_name = value
+
+
+def __log(level, msg, interp=None):
+    interp = interp or ()
+    if __log_level <= level:
+        pbs_msg = "%s:%s - %s" % (__application_name, __log_level_names[__log_level], msg)
+        pylog_msg = msg
+        try:
+            pbs_msg = pbs_msg % interp
+            pylog_msg = msg % interp
+        except TypeError as e:
+            pbs.logmsg(__WARN, str(e))
+            
+        if level >= __ERROR:
+            pbs_level = pbs.LOG_ERROR
+            pylog_level = logging.ERROR
+        elif level == __WARN:
+            pbs_level = pbs.LOG_WARNING
+            pylog_level = logging.WARN
+        elif level == __INFO:
+            pbs_level = pbs_LOG_INFO
+            pylog_level = logging.INFO
+        else:
+            pbs_level = pbs.LOG_DEBUG
+            pylog_level = logging.DEBUG
+        pbs.logmsg(pbs_level, pbs_msg)
+        logging.log(pylog_level, pylog_msg)
+
+
+def is_fine():
+    return __log_level == __FINE
+
+
+def fine(msg, *interp):
+    interp = interp or ()
+    __log(__FINE, msg, interp)
+
+
+def debug(msg, *interp):
+    interp = interp or ()
+    __log(__DEBUG, msg, interp)
+
+
+def info(msg, *interp):
+    interp = interp or ()
+    __log(__INFO, msg, interp)
+    
+
+def warn(msg, *interp):
+    interp = interp or ()
+    __log(__WARN, msg, interp)
+
+
+def error(msg, *interp):
+    interp = interp or ()
+    __log(__WARN, msg, interp)
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/pi.py b/specs/default/chef/site-cookbooks/pbspro/files/default/pi.py
new file mode 100644
index 0000000..a8d5e2c
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/pi.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+import getopt, sys, time
+
+
+
+def calcPiNewton(digits):
+# Newtons formula for PI is:
+# PI / 2 = sum_n_from_0_to_inf(n! / (2 * n + 1)!!)
+
+# This can be written as:
+# PI / 2 = 1 + 1/3 * (1 + 2/5 * (1 + 3/7 * (1 + 4/9 * (1 + ... ))))
+
+# This algorithm puts 2 * 1000 on the right side and computes everything from inside out.
+
+    scale = 10000
+    maxarr = int(digits/4) * 14
+    arrinit = 2000
+    carry = 0
+    arr = []
+    output = ''
+
+    # initialize the array
+    arr = [arrinit]*(maxarr+1)
+        
+    for i in range(maxarr, 1, -14):
+        sum = 0
+        for j in range(i, 0, -1):
+            sum = (sum * j) + (scale * arr[j])
+            arr[j] = sum % ( (j*2)-1 )
+            sum = sum / ((j*2)-1)
+        output += "%04d" % (carry + (sum / scale))
+        carry = sum % scale
+    
+    return output
+
+    
+def usage():
+    print 'Calculate pi to x digits.  Usage: python pi.py [digits]'
+
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "h", ["help"])
+    except getopt.GetoptError:
+        # print help information and exit:
+        usage()
+        sys.exit(2)
+
+    for o, a in opts:
+        if o == "-v":
+            verbose = True
+        if o in ("-h", "--help"):
+            usage()
+            sys.exit()
+    
+    if args:
+        digits = int(args[0])
+        if digits < 1:
+            print 'Not a valid number of digits: %s' % digits
+    else:
+        digits = 1000
+        print 'Calculating the default number of pi digits (%s)' % digits
+        
+    # do the calculation
+    start_time = time.time()
+    print calcPiNewton(digits)
+    end_time = time.time()
+    print "TotalTime=%f" %(end_time-start_time)
+        
+    
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/pi.sh b/specs/default/chef/site-cookbooks/pbspro/files/default/pi.sh
new file mode 100644
index 0000000..9147d18
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/pi.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+#PBS -N pitest
+#PBS -j oe
+
+/shared/scratch/pi/pi.py 10000
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/runpi.sh b/specs/default/chef/site-cookbooks/pbspro/files/default/runpi.sh
new file mode 100644
index 0000000..d5cb0d6
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/runpi.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+mkdir -p /shared/scratch/pi
+cp ~/demo/pi.py /shared/scratch/pi
+cp ~/demo/pi.sh /shared/scratch/pi
+cd /shared/scratch/pi
+qsub -J 1-1000 /shared/scratch/pi/pi.sh
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/sched.config b/specs/default/chef/site-cookbooks/pbspro/files/default/sched.config
new file mode 100644
index 0000000..cc8ff05
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/sched.config
@@ -0,0 +1,28 @@
+round_robin: False	all
+by_queue: True		prime
+by_queue: True		non_prime
+strict_ordering: false	ALL
+help_starving_jobs:	true	ALL
+max_starve: 24:00:00
+backfill_prime:	false	ALL
+prime_exempt_anytime_queues:	false
+primetime_prefix: p_
+nonprimetime_prefix: np_
+node_sort_key: "sort_priority HIGH"	ALL
+provision_policy: "aggressive_provision"
+sort_queues:	true	ALL
+resources: "ncpus, mem, arch, host, vnode, aoe, slot_type, group_id, ungrouped, instance_id, machinetype, disk, ngpus, nodearray"
+load_balancing: false	ALL
+smp_cluster_dist: pack
+fair_share: false	ALL
+fairshare_usage_res: cput
+fairshare_entity: euser
+fairshare_decay_time: 24:00:00
+fairshare_decay_factor: 0.5
+preemptive_sched: true	ALL
+preempt_queue_prio:	150
+preempt_prio: "express_queue, normal_jobs"
+preempt_order: "SCR"
+preempt_sort: min_time_since_start
+dedicated_prefix: ded
+log_filter: 3328
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/submit_hook.py b/specs/default/chef/site-cookbooks/pbspro/files/default/submit_hook.py
new file mode 100644
index 0000000..29df8af
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/submit_hook.py
@@ -0,0 +1,259 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+'''
+Note - the pbs module isn't very pythonic, so you'll see things like
+    value = job.Resource_List["attribute"] or 100
+instead of
+    value = job.Resource_List.get("attribute", 100)
+That is because is a metaclass, not a dict.
+
+Also, place and select objects use repr() to convert to a parseable string, but
+so you'll see guards against repr(None) (combined with the above) and 
+
+Quick start:
+    qmgr -c "create hook cycle_sub_hook"
+    qmgr -c "set hook cycle_sub_hook event = queuejob"
+    qmgr -c "create hook cycle_sub_periodic_hook"
+    qmgr -c "set hook cycle_sub_periodic_hook event = periodic"
+
+    # reload source / config
+    qmgr -c "import hook cycle_sub_hook application/x-python default submit_hook.py"
+    qmgr -c "import hook cycle_sub_hook application/x-config default submit_hook.json"
+    qmgr -c "import hook cycle_sub_periodic_hook application/x-python default submit_hook.py"
+    qmgr -c "import hook cycle_sub_periodic_hook application/x-config default submit_hook.json"
+
+Queue setup
+    qmgr -c "set queue <queue_name> resources_default.slot_type = <queue_name>"
+    qmgr -c "set queue <queue_name> resources_default.ungrouped = false"
+    qmgr -c "set queue <queue_name> default_chunk.slot_type = <queue_name>"
+    qmgr -c "set queue <queue_name> default_chunk.ungrouped = false"
+
+See PBS Professional Programmers Guide for detailed information.
+
+See /var/spool/pbs/server_logs/* for log messages
+'''
+
+import json
+import os
+import subprocess
+import traceback
+import sys
+
+
+try:
+    import pbs
+except ImportError:
+    import mockpbs as pbs
+
+
+def debug(msg):
+    pbs.logmsg(pbs.LOG_DEBUG, "cycle_periodic_hook_place - %s" % msg)
+
+
+def error(msg):
+    pbs.logmsg(pbs.EVENT_ERROR, "cycle_periodic_hook_place - %s" % msg)
+
+
+def hold_on_submit(hook_config, job):
+    '''
+    Hold every job so that we can process it properly in the periodic hook.
+    '''
+    pbs.logmsg(pbs.LOG_DEBUG, "cycle_sub_hook - holding job %s with hold_type 'so'" % job.id)
+    job.Hold_Types = pbs.hold_types("so")
+
+
+# Vendored in from pbscc.py
+def parse_place(place):
+    '''
+    arrangement is one of free | pack | scatter | vscatter
+    sharing is one of excl | shared | exclhost
+    grouping can have only one instance of group=resource
+    '''
+    placement = {}
+    
+    if not place:
+        return placement
+    
+    toks = place.split(":")
+    
+    for tok in toks:
+        if tok in ["free", "pack", "scatter", "vscatter"]:
+            placement["arrangement"] = tok
+        elif tok in ["excl", "shared", "exclhost"]:
+            placement["sharing"] = tok
+        elif tok.startswith("group="):
+            placement["grouping"] = tok
+    
+    return placement
+
+
+def format_place(place_dict):
+    parts = []
+    for key in ["arrangement", "sharing", "grouping"]:
+        if place_dict.get(key):
+            parts.append(place_dict[key])
+    return ":".join(parts)        
+
+
+QSELECT_EXE = os.path.join(pbs.pbs_conf['PBS_EXEC'], 'bin', 'qselect')
+QSTAT_EXE = os.path.join(pbs.pbs_conf['PBS_EXEC'], 'bin', 'qstat')
+QALTER_EXE = os.path.join(pbs.pbs_conf['PBS_EXEC'], 'bin', 'qalter')
+QRLS_EXE = os.path.join(pbs.pbs_conf['PBS_EXEC'], 'bin', 'qrls')
+
+
+def periodic_release_hook(hook_config, e):
+    held_jobs_per_iteration = int(hook_config.get("held_jobs_per_iteration", 25))
+
+    # Defined paths to PBS commands
+
+    # Get the jobs in an "so" hold state
+    qselect_cmd = [QSELECT_EXE, "-h", "so"]
+    stdout = run_cmd(qselect_cmd)
+    jobs = stdout.split()
+    debug("Jobs: %s" % jobs)
+
+    # Get the job information
+    if not jobs:
+        debug("No jobs to evaluate")
+        e.accept()
+        return
+
+    # Get Queue defaults information
+    queue_cmd = [QSTAT_EXE, "-Qf", "-F", "json"] 
+    stdout = run_cmd(queue_cmd)
+    qstat_Qf_json = json.loads(stdout)
+   
+    # Get job information
+    job_cmd = [QSTAT_EXE, "-f", "-F", "json"] + jobs[:held_jobs_per_iteration]
+    stdout = run_cmd(job_cmd)
+    qstat_json = json.loads(stdout)
+    jobs = qstat_json["Jobs"]
+    
+    for job_id, job in jobs.iteritems():
+        # Reevaluate each held job
+        debug("Key: %s\nValue: %s" % (job_id, job))
+        if str(job["Resource_List"].get("ungrouped")).lower() == "true":
+            debug("Skipping ungrouped job %s" % job_id)
+            qrls_cmd = [QRLS_EXE, "-h", "so", job_id]
+            run_cmd(qrls_cmd)
+            continue
+        
+        j_queue = job["queue"]
+        j_select = job["Resource_List"].get("select") or "1:ncpus=1"
+        
+        # Check the groupid placement
+        mj_place = job["Resource_List"].get("place")
+            
+        mj_place_dict = parse_place(mj_place)
+        
+        # Assign default placement from queue. If none, assign group=group_id
+        if j_queue in qstat_Qf_json["Queue"]:
+            if "resources_default" in qstat_Qf_json["Queue"][j_queue]:
+                if "place" in qstat_Qf_json["Queue"][j_queue]["resources_default"]:
+                    # only update arrangement (scatter/pack/vscatter/free) and sharing (excl/shared) if _neither_ is specified 
+                    # Do not override grouping if user specified it.
+                    # -l place=scatter is automatically added to -l nodes jobs, so we will remove it here and if it is not 
+                    # overridden by the queue defaults, we will bring it back
+                    if job["Resource_List"].get("nodes"):
+                        mj_place_dict = {}
+                        
+                    default_place = qstat_Qf_json["Queue"][j_queue]["resources_default"]["place"]
+                    default_place_dict = parse_place(default_place)
+                    
+                    # if the user specifies either arrangement or sharing, ignore them as far as defaults are concerned
+                    specified_at_least_one_of_arrangement_or_sharing = len(set(mj_place_dict.keys()) - set(["grouping"])) > 0
+                    if specified_at_least_one_of_arrangement_or_sharing:
+                        default_place_dict.pop("sharing", "")
+                        default_place_dict.pop("arrangement", "")
+                    
+                    # update if not exist from defaults
+                    for k, v in default_place_dict.iteritems():
+                        mj_place_dict[k] = mj_place_dict.get(k, v)
+                        
+                    # the job is a -l nodes job and the queue didn't specify arrangement or sharing
+                    if job["Resource_List"].get("nodes") and mj_place_dict.get("arrangement") is None and mj_place_dict.get("sharing") is None:
+                        mj_place_dict["arrangement"] = "scatter"
+                        
+                    mj_place = format_place(mj_place_dict)
+                    debug("%s" % mj_place)
+                    
+        mj_place = format_place(mj_place_dict)
+        
+        # Double checking group=group_id setting:
+        placement_grouping = None
+        for expr in mj_place.split(":"):
+            placement_grouping = None
+            if "=" in expr:
+                placekey, value = [x.lower().strip() for x in expr.split("=", 2)]
+                if placekey == "group":
+                    placement_grouping = value
+       
+        if placement_grouping is None:
+            debug("The user didn't specify place=group, setting group=group_id")
+            debug("%s - The user didn't specify place=group, setting group=group_id" % mj_place)
+            placement_grouping = "group_id"
+            prefix = ":" if mj_place else ""
+            mj_place = mj_place + prefix + "group=group_id"
+            debug("after update %s" % mj_place)
+
+        if mj_place != job["Resource_List"].get("place"):
+            # Qalter the job
+            qalter_cmd = [QALTER_EXE]
+            debug("New place statement: %s" % mj_place)
+            debug("before qalter %s" % j_select)
+            debug("before qalter %s" % mj_place)
+            qalter_cmd.append("-lselect=%s" % j_select)
+            qalter_cmd.append("-lplace=%s" % mj_place)
+            debug("qalter the job")
+            qalter_cmd.append(job_id)
+            debug("full cmd %s" % qalter_cmd)
+            stdout = run_cmd(qalter_cmd)
+            debug("stdout %s" % stdout)
+            
+        debug("Release the hold on job %s" % job_id)
+        qrls_cmd = [QRLS_EXE, "-h", "so", job_id]
+        run_cmd(qrls_cmd)
+        
+    e.accept()
+
+
+def run_cmd(cmd):
+    debug("Cmd: %s" % cmd)
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = proc.communicate()
+    if proc.returncode != 0:
+        msg = 'cmd failed!\n\tstdout="%s"\n\tstderr="%s"' % (stdout, stderr)
+        error(msg)
+        sys.exit(proc.returncode)
+        
+    return stdout
+
+
+def main():
+    try:
+        hook_config = {}
+        if pbs.hook_config_filename:
+            with open(pbs.hook_config_filename) as fr:
+                hook_config.update(json.load(fr))
+
+        e = pbs.event()
+        if e.type == pbs.QUEUEJOB:
+            j = e.job
+            hold_on_submit(hook_config, j)
+        elif e.type == pbs.PERIODIC:
+            periodic_release_hook(hook_config, e)
+        else:
+            pbs.logmsg(pbs.EVENT_ERROR, "Unknown event type %s" % e.type)
+    except SystemExit:
+        pbs.logmsg(pbs.LOG_DEBUG, "cycle - Exited with SystemExit")
+        raise
+    except:
+        pbs.logmsg(pbs.EVENT_ERROR, "cycle - %s" % traceback.format_exc())
+        raise
+
+
+# another non-pythonic thing - this can't be behind a __name__ == '__main__',
+# as the hook code has to be executable at the load module step.
+if not os.getenv("_UNITTEST_", ""):
+    main()
diff --git a/specs/default/chef/site-cookbooks/pbspro/files/default/submit_hook_test.py b/specs/default/chef/site-cookbooks/pbspro/files/default/submit_hook_test.py
new file mode 100644
index 0000000..99786a7
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/files/default/submit_hook_test.py
@@ -0,0 +1,269 @@
+import unittest
+import os
+import mockpbs
+import sys
+import json
+
+os.environ["_UNITTEST_"] = "true"
+
+from submit_hook import QSELECT_EXE, QSTAT_EXE, QRLS_EXE, QALTER_EXE
+import submit_hook
+
+
+class Job:
+    
+    def __init__(self, job_id=1, Hold_Types=None, queue="workq", **resource_list):
+        self.job_id = str(job_id)
+        self.id = self.job_id
+        self.interactive = False
+        self.Resource_List = mockpbs.ResourceList(resource_list)
+        self.Hold_Types = Hold_Types
+        self.queue = queue
+        
+    def to_dict(self):
+        return {"queue": self.queue, "Resource_List": dict(self.Resource_List)}
+    
+
+def standard_job(**resource_list):
+    resource_list["select"] = resource_list.get("select", "2:ncpus")
+    return Job(Hold_Types="so", **resource_list)
+
+
+class Event:
+    
+    def accept(self):
+        pass
+    
+    
+class MockRunCmd:
+    
+    def __init__(self, responses=None):
+        self.responses = responses or []
+        self.orig_run_cmd = submit_hook.run_cmd
+        
+    def add_call(self, expected, response):
+        if not isinstance(response, basestring):
+            response = json.dumps(response)
+        self.responses.append((expected, response))
+        
+    def __call__(self, cmd):
+        expected_cmd, response = self.responses.pop(0)
+        assert expected_cmd == cmd, "%s != %s" % (expected_cmd, cmd)
+        return response
+    
+    def __enter__(self):
+        submit_hook.run_cmd = self
+        return self
+    
+    def __exit__(self, exception_type, exception, tb):
+        submit_hook.run_cmd = self.orig_run_cmd
+        if exception_type is None:
+            # if an exception is thrown, we expect that some responses aren't returned
+            assert len(self.responses) == 0, "remaining responses %s" % self.responses
+        
+        
+class Queue:
+    
+    def __init__(self, name="workq", **resources_default):
+        self.name = name
+        self.resources_default = resources_default or {}
+        
+    def to_dict(self):
+        return {"resources_default": self.resources_default}
+    
+
+class SubmitHookTest(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def test_run_cmd(self):
+        try:
+            submit_hook.run_cmd(["test"])
+        except SystemExit:
+            pass
+        submit_hook.run_cmd(["test", "fail"])
+        
+    def test_placement_hook(self):
+        select_job = Job(select="2:ncpus=2")
+        plain_job = Job()
+        select_job_i = Job(select="2:ncpus=2")
+        plain_job_i = Job()
+        select_job_i.interactive = True
+        plain_job_i.interactive = True
+        
+        submit_hook.hold_on_submit({}, select_job)
+        submit_hook.hold_on_submit({}, plain_job)
+        submit_hook.hold_on_submit({}, select_job_i)
+        submit_hook.hold_on_submit({}, plain_job_i)
+        
+        self.assertEquals("so", select_job.Hold_Types)
+        self.assertEquals("so", plain_job.Hold_Types)
+        
+        # interactive jobs should be held as well.
+        self.assertEquals("so", select_job_i.Hold_Types)
+        self.assertEquals("so", plain_job_i.Hold_Types)
+        
+    def test_periodic_release_hook_select(self):
+        '''No queue defaults, no user defined group, add group=group_id'''
+        job = Job(Hold_Types="so", select="2:ncpus")
+        q = Queue()
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, "group=group_id")
+            submit_hook.periodic_release_hook({}, Event())
+            
+    def test_periodic_release_hook_select_with_grouping(self):
+        ''' User picked group=host so we should respect that'''
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="scatter:excl:group=host")
+        q = Queue(place="pack")
+        with MockRunCmd() as run_cmd:
+            # no qalter, user defined all 3 place arguments
+            _initialize_run_cmd(run_cmd, q, job, expected_qalter_place=None)
+            submit_hook.periodic_release_hook({}, Event())
+            
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="group=host")
+        q = Queue(place="scatter:excl")
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, expected_qalter_place="scatter:excl:group=host")
+            submit_hook.periodic_release_hook({}, Event())
+            
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="scatter:group=host")
+        q = Queue(place="scatter:excl")
+        with MockRunCmd() as run_cmd:
+            # no qalter, user defined arrangement and grouping arguments
+            _initialize_run_cmd(run_cmd, q, job, expected_qalter_place=None)
+            submit_hook.periodic_release_hook({}, Event())
+            
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="excl:group=host")
+        q = Queue(place="scatter:excl")
+        with MockRunCmd() as run_cmd:
+            # no qalter, user defined sharing and grouping arguments
+            _initialize_run_cmd(run_cmd, q, job, expected_qalter_place=None)
+            submit_hook.periodic_release_hook({}, Event())
+            
+    def test_periodic_release_hook_select_with_place(self):
+        ''' We should respect the user's -l place= when modifying group'''
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="scatter:excl")
+        q = Queue()
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, "scatter:excl:group=group_id")
+            submit_hook.periodic_release_hook({}, Event())
+            
+    def test_periodic_release_hook_nodes(self):
+        ''' We will ignore -lplace=scatter because the user picked -l nodes=, which auto-converts to -l place=scatter'''
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="scatter", nodes="2:ppn=2")
+        q = Queue(place="scatter:excl")
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, "scatter:excl:group=group_id")
+            submit_hook.periodic_release_hook({}, Event())
+            
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="scatter", nodes="2:ppn=2")
+        q = Queue(place="pack:shared")
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, "pack:shared:group=group_id")
+            submit_hook.periodic_release_hook({}, Event())
+            
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="scatter", nodes="2:ppn=2")
+        q = Queue(place="excl")
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, "excl:group=group_id")
+            submit_hook.periodic_release_hook({}, Event())
+            
+    def test_periodic_release_hook_nodes_excl_default(self):
+        ''' We should respect the user's -l place= when modifying group - pack:shared is not overridden by queue default scatter:excl'''
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="pack:shared")
+        q = Queue(place="scatter:excl")
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, "pack:shared:group=group_id")
+            submit_hook.periodic_release_hook({}, Event())
+            
+        # not that arrangement and exclusivity go together here
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="pack")
+        q = Queue(place="scatter:excl")
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, "pack:group=group_id")
+            submit_hook.periodic_release_hook({}, Event())
+            
+    def test_periodic_release_hook_missing_select(self):
+        ''' We convert simple qsubs into select=1:ncpus=1 - note we should set the default to free in this case'''
+        job = Job(Hold_Types="so")
+        q = Queue()
+        q.resources_default["place"] = "scatter:excl"
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, expected_qalter_place="scatter:excl:group=group_id")
+            # nothing to hold
+            submit_hook.periodic_release_hook({}, Event())
+            
+    def test_chunked(self):
+        ''' There shouldn't be anything special about a chunked expression.'''
+        job = Job(Hold_Types="so", select="2:ncpus=2+1:mem=20g")
+        q = Queue()
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, expected_qalter_place="group=group_id")
+            # nothing to hold
+            submit_hook.periodic_release_hook({}, Event())
+            
+    def test_two_queues(self):
+        hpc_job = Job(job_id=1, Hold_Types="so", queue="workq", select="2:ncpus=2+1:mem=20g")
+        hpcq = Queue("workq")
+        # we don't pick up on defaults from the queue, so just set ungrouped on both.
+        htc_job = Job(job_id=2, Hold_Types="so", queue="htcq", ungrouped=True)
+        htcq = Queue("htcq", ungrouped=True, place="pack")
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, [hpcq, htcq], [hpc_job, htc_job],
+                                # we won't bother running qalter because ungrouped==true
+                                expected_qalter_place=["group=group_id", None])
+            # nothing to hold
+            submit_hook.periodic_release_hook({}, Event())
+            
+
+def _initialize_run_cmd(run_cmd, queues, jobs, expected_qalter_place):
+    if not isinstance(queues, list):
+        queues = [queues]
+        
+    if not isinstance(jobs, list):
+        jobs = [jobs]
+        
+    if not isinstance(expected_qalter_place, list):
+        expected_qalter_place = [expected_qalter_place]
+        
+    job_ids = [job.job_id for job in jobs]
+        
+    run_cmd.add_call([QSELECT_EXE, "-h", "so"], " ".join([job.job_id for job in jobs]))
+    
+    queues_response = {"Queue": {}}
+    for q in queues:
+        queues_response["Queue"][q.name] = q.to_dict()
+    
+    run_cmd.add_call([QSTAT_EXE, "-Qf", "-F", "json"], queues_response)
+    
+    jobs_response = {"Jobs": {}}
+    for job in jobs:
+        jobs_response["Jobs"][job.job_id] = job.to_dict()
+        
+    run_cmd.add_call([QSTAT_EXE, "-f", "-F", "json"] + job_ids, jobs_response)
+    
+    if not job_ids:
+        assert not expected_qalter_place
+    
+    for job, eqp in zip(jobs, expected_qalter_place):
+        select = job.Resource_List["select"] or "1:ncpus=1"
+        if eqp:
+            run_cmd.add_call([QALTER_EXE, "-lselect=" + select, "-lplace=" + eqp, job.job_id], "qalter successful")
+        run_cmd.add_call([QRLS_EXE, "-h", "so", job.job_id], "qrls successful")
+        
+
+if __name__ == "__main__":
+    #import sys;sys.argv = ['', 'Test.testName']
+    unittest.main()
\ No newline at end of file
diff --git a/specs/default/chef/site-cookbooks/pbspro/metadata.rb b/specs/default/chef/site-cookbooks/pbspro/metadata.rb
new file mode 100644
index 0000000..02902d2
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/metadata.rb
@@ -0,0 +1,12 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+name             "pbspro"
+maintainer       "Microsoft Corporation"
+license          "MIT"
+description      "Installs/Configures Open PBS Pro"
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
+version          "0.1"
+depends          "tandem"
+%w{ cganglia cshared cuser cyclecloud }.each {|c| depends c}
+
diff --git a/specs/default/chef/site-cookbooks/pbspro/recipes/autostart.rb b/specs/default/chef/site-cookbooks/pbspro/recipes/autostart.rb
new file mode 100644
index 0000000..31f5a16
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/recipes/autostart.rb
@@ -0,0 +1,60 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/pbs/autostart.py" do
+    source "autostart.py"
+    mode "0755"
+    owner "root"
+    group "root"
+end
+
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/pbs/autostart_hook.py" do
+  source "autostart_hook.py"
+  mode "0755"
+  owner "root"
+  group "root"
+end
+
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/pbs/logging_init.py" do
+  source "logging_init.py"
+  mode "0755"
+  owner "root"
+  group "root"
+end
+
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/pbs/mockpbs.py" do
+  source "mockpbs.py"
+  mode "0755"
+  owner "root"
+  group "root"
+end
+
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/pbs/pbscc.py" do
+  source "pbscc.py"
+  mode "0755"
+  owner "root"
+  group "root"
+end
+
+file "#{node[:cyclecloud][:bootstrap]}/pbs/autostart.json" do
+  mode "0644"
+  owner "root"
+  group "root"
+  content Chef::JSONCompat.to_json_pretty(node[:pbspro][:autoscale_hook])
+end
+
+bash "import autoscale hook" do
+  code <<-EOH
+    set -e
+    /opt/pbs/bin/qmgr -c "create hook autoscale" 1>&2 || true
+    /opt/pbs/bin/qmgr -c "import hook autoscale application/x-python default #{node[:cyclecloud][:bootstrap]}/pbs/autostart_hook.py"
+    /opt/pbs/bin/qmgr -c "import hook autoscale application/x-config default #{node[:cyclecloud][:bootstrap]}/pbs/autostart.json"
+    /opt/pbs/bin/qmgr -c "set hook autoscale event = periodic"
+    /opt/pbs/bin/qmgr -c "set hook autoscale freq = 15"
+    touch #{node[:cyclecloud][:bootstrap]}/pbs/autoscalehook.imported
+  EOH
+  creates "#{node[:cyclecloud][:bootstrap]}/pbs/autoscalehook.imported"
+  only_if { node[:cyclecloud][:cluster][:autoscale][:start_enabled] }
+
+  notifies :restart, 'service[pbs]', :delayed
+end
diff --git a/specs/default/chef/site-cookbooks/pbspro/recipes/autostop.rb b/specs/default/chef/site-cookbooks/pbspro/recipes/autostop.rb
new file mode 100644
index 0000000..de4af8b
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/recipes/autostop.rb
@@ -0,0 +1,19 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/autostop.rb" do
+  source "autostop.rb"
+  owner "root"
+  group "root"
+  mode "0700"
+  action :create
+end
+
+# Convert our stop interval into a number of minutes rounded up and run the script on that interval
+interval = (node[:cyclecloud][:cluster][:autoscale][:stop_interval].to_i / 60.0).ceil
+cron "autostop" do
+  minute "*/#{interval}"
+  command "#{node[:cyclecloud][:bootstrap]}/cron_wrapper.sh #{node[:cyclecloud][:bootstrap]}/autostop.rb > /tmp/autostop.out 2>&1"
+  only_if { node[:cyclecloud][:cluster][:autoscale][:stop_enabled] }
+end
diff --git a/specs/default/chef/site-cookbooks/pbspro/recipes/default.rb b/specs/default/chef/site-cookbooks/pbspro/recipes/default.rb
new file mode 100644
index 0000000..053079f
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/recipes/default.rb
@@ -0,0 +1,3 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
diff --git a/specs/default/chef/site-cookbooks/pbspro/recipes/execute.rb b/specs/default/chef/site-cookbooks/pbspro/recipes/execute.rb
new file mode 100644
index 0000000..7f1fe03
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/recipes/execute.rb
@@ -0,0 +1,130 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+pbsprover = node[:pbspro][:version]
+
+package_name = "pbspro-execution-#{pbsprover}.x86_64.rpm"
+
+jetpack_download package_name do
+  project 'STAR-CCMplus'
+end
+
+yum_package package_name do
+  source "#{node['jetpack']['downloads']}/#{package_name}"
+  action :install
+end
+
+slot_type = node[:pbspro][:slot_type] || "execute"
+
+placement_group = node[:cyclecloud][:node][:placement_group] || nil
+is_node_grouped = node[:pbspro][:is_grouped]
+instance_id = node[:cyclecloud][:instance][:id]
+
+custom_resources = Hash.new {}
+if node[:autoscale] then
+    custom_resources = node[:autoscale].to_h
+end
+
+if custom_resources.empty? || custom_resources.nil? then
+	set_custom_resources = "true"
+else
+    custom_resources.delete("disabled")
+	set_custom_resources = custom_resources.map{ |key, value| "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.#{key}=#{value}'"}.join(" && ")
+end
+
+schedint = cluster.scheduler
+slots = node[:pbspro][:slots] || nil
+
+if schedint != nil
+  template "/var/spool/pbs/server_name" do
+    source "server_name_exec.erb"
+    mode "0644"
+    owner "root"
+    group "root"
+    variables(:servername => schedint)
+  end
+
+  template "/var/spool/pbs/mom_priv/config" do
+    source "mom_config.erb"
+    mode "0644"
+    owner "root"
+    group "root"
+    variables(:servername => schedint)
+  end
+
+  template "/etc/pbs.conf" do
+    source "pbs.conf.erb"
+    mode "0644"
+    owner "root"
+    group "root"
+    variables(:servername => schedint)
+  end
+end
+
+node_created_guard = "#{node['cyclecloud']['chefstate']}/pbs.nodecreated"
+
+bash "add-node-to-scheduler" do
+  code lazy {"/opt/pbs/bin/qmgr -c 'c n #{node[:hostname]}'"}
+  only_if do
+    cmd = Mixlib::ShellOut.new('/opt/pbs/bin/pbsnodes -a')
+    list_of_pbs_nodes = cmd.run_command.stdout.strip().split("\n")
+    !list_of_pbs_nodes.include?(node[:hostname])
+  end
+  not_if {::File.exist?(node_created_guard)}
+end
+
+defer_block 'Defer setting core count and slot_type, and start of PBS pbs_mom until end of converge' do
+  execute "set-node-core-count" do
+    command lazy { "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.ncpus=#{slots}'" }
+    only_if { !slots.nil? }
+    not_if {::File.exist?(node_created_guard)}
+  end
+  execute "set-node-free" do
+    command lazy { "/opt/pbs/bin/pbsnodes -r #{node[:hostname]}"}
+    not_if {::File.exist?(node_created_guard)}
+  end
+
+  set_slot_type = "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.slot_type=#{slot_type}'"
+  set_ungrouped = "true"
+
+  if not is_node_grouped then
+  	# user data explicitly says this is an ungrouped node.
+  	set_ungrouped = "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.ungrouped=true'"
+  else
+  	if placement_group then
+  		# user data didn't explicitly say this was an ungrouped node, but we are in a placement group
+  		set_ungrouped = "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.ungrouped=false'"
+  	else
+  		# user data didn't explicitly say this was an ungrouped node, but we aren't in a placement group
+  		set_ungrouped = "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.ungrouped=true'"
+  	end
+  end
+ 
+  set_group_id = "true"
+  if placement_group then
+    set_group_id = "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.group_id=#{placement_group}'"
+  end
+ 
+  set_instance_id = "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.instance_id=#{instance_id}'"
+   
+  execute "set-node-slot_type" do
+    command lazy {<<-EOS
+      #{set_slot_type} && \
+      #{set_group_id} && \
+      #{set_ungrouped} && \
+      #{set_instance_id} && \
+      #{set_custom_resources} && \
+      touch #{node_created_guard}
+      EOS
+    }
+    creates node_created_guard
+    notifies :restart, 'service[pbs]', :immediately
+  end
+end
+
+service "pbs" do
+  action :nothing
+end
+
+include_recipe "pbspro::autostop"
diff --git a/specs/default/chef/site-cookbooks/pbspro/recipes/login.rb b/specs/default/chef/site-cookbooks/pbspro/recipes/login.rb
new file mode 100644
index 0000000..b10a025
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/recipes/login.rb
@@ -0,0 +1,32 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+pbsprover = node[:pbspro][:version]
+
+package_name = "pbspro-client-#{pbsprover}.x86_64.rpm"
+
+jetpack_download package_name do
+  project 'pbspro'
+end
+
+yum_package package_name do
+  source "#{node['jetpack']['downloads']}/#{package_name}"
+  action :install
+end
+
+schedint = cluster.scheduler
+
+if schedint != nil
+  template "/etc/pbs.conf" do
+    source "pbs.conf.erb"
+    mode "0644"
+    owner "root"
+    group "root"
+    variables(:servername => schedint)
+  end
+end
+
+execute 'set_flatuid' do
+  command '/opt/pbs/bin/qmgr -c "set server flatuid=true"'
+end
diff --git a/specs/default/chef/site-cookbooks/pbspro/recipes/scheduler.rb b/specs/default/chef/site-cookbooks/pbspro/recipes/scheduler.rb
new file mode 100644
index 0000000..f3204ab
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/recipes/scheduler.rb
@@ -0,0 +1,51 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+pbsprover = node[:pbspro][:version]
+
+package_name = "pbspro-server-#{pbsprover}.x86_64.rpm"
+
+jetpack_download package_name do
+  project 'STAR-CCMplus'
+end
+
+yum_package package_name do
+  source "#{node['jetpack']['downloads']}/#{package_name}"
+  action :install
+end
+
+directory "#{node[:cyclecloud][:bootstrap]}/pbs" do
+  owner "root"
+  group "root"
+  mode "0755"
+  action :create
+end
+
+cookbook_file "/var/spool/pbs/doqmgr.sh" do
+  source "doqmgr.sh"
+  mode "0755"
+  owner "root"
+  group "root"
+  action :create
+end
+
+cookbook_file "/var/spool/pbs/sched_priv/sched_config" do
+  source "sched.config"
+  owner "root"
+  group "root"
+  mode "0644"
+end
+
+service "pbs" do
+  action [:enable, :start]
+end
+
+execute "serverconfig" do
+  command "/var/spool/pbs/doqmgr.sh && touch /etc/qmgr.config"
+  creates "/etc/qmgr.config"
+  notifies :restart, 'service[pbs]', :delayed
+end
+
+include_recipe "pbspro::autostart"
+include_recipe "pbspro::submit_hook"
diff --git a/specs/default/chef/site-cookbooks/pbspro/recipes/skel.rb b/specs/default/chef/site-cookbooks/pbspro/recipes/skel.rb
new file mode 100644
index 0000000..0bfd96f
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/recipes/skel.rb
@@ -0,0 +1,22 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+directory "/etc/skel/demo" do
+  owner "root"
+  group "root"
+  mode "0755"
+  action :create
+  recursive true
+  not_if "test -d /etc/skel/demo"
+end
+
+%w{pi.py pi.sh runpi.sh}.each do |myfile|
+  cookbook_file "/etc/skel/demo/#{myfile}" do
+    source myfile
+    owner "root"
+    group "root"
+    mode "0755"
+   action :create
+  end
+end
diff --git a/specs/default/chef/site-cookbooks/pbspro/recipes/submit_hook.rb b/specs/default/chef/site-cookbooks/pbspro/recipes/submit_hook.rb
new file mode 100644
index 0000000..4fed74b
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/recipes/submit_hook.rb
@@ -0,0 +1,51 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/pbs/submit_hook.py" do
+    source "submit_hook.py"
+    mode "0755"
+    owner "root"
+    group "root"
+end
+  
+file "#{node[:cyclecloud][:bootstrap]}/pbs/submit_hook.json" do
+    mode "0644"
+    owner "root"
+    group "root"
+    content Chef::JSONCompat.to_json_pretty(node[:pbspro][:submit_hook])
+end
+
+# required by "tandem::install_driver"
+
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/pbs/pbs_driver.py" do
+    source "pbs_driver.py"
+    mode "0755"
+    owner "root"
+    group "root"
+end
+
+
+node.default[:tandem_driver_directory] = "#{node[:cyclecloud][:bootstrap]}/pbs"
+include_recipe "tandem::install_driver"
+
+bash "import submit hook" do
+    code <<-EOH
+    /opt/pbs/bin/qmgr -c "create hook cycle_sub_hook" 2>/dev/null || true
+    /opt/pbs/bin/qmgr -c "set hook cycle_sub_hook event = queuejob" 2>/dev/null || true
+    /opt/pbs/bin/qmgr -c "create hook cycle_sub_periodic_hook" 2>/dev/null || true
+    /opt/pbs/bin/qmgr -c "set hook cycle_sub_periodic_hook event = periodic" 2>/dev/null || true
+    /opt/pbs/bin/qmgr -c "set hook cycle_sub_periodic_hook freq = 15"  2>/dev/null || true
+    
+    set -e
+    /opt/pbs/bin/qmgr -c "import hook cycle_sub_hook application/x-python default #{node[:cyclecloud][:bootstrap]}/pbs/submit_hook.py"
+    /opt/pbs/bin/qmgr -c "import hook cycle_sub_hook application/x-config default #{node[:cyclecloud][:bootstrap]}/pbs/submit_hook.json"
+    /opt/pbs/bin/qmgr -c "import hook cycle_sub_periodic_hook application/x-python default #{node[:cyclecloud][:bootstrap]}/pbs/submit_hook.py"
+    /opt/pbs/bin/qmgr -c "import hook cycle_sub_periodic_hook application/x-config default #{node[:cyclecloud][:bootstrap]}/pbs/submit_hook.json"
+        touch #{node[:cyclecloud][:bootstrap]}/pbs/submithook.imported
+    EOH
+    creates "#{node[:cyclecloud][:bootstrap]}/pbs/submithook.imported"
+    only_if { node[:pbspro][:submit_hook][:enabled] }
+
+    notifies :restart, "service[pbs]", :delayed
+end
diff --git a/specs/default/chef/site-cookbooks/pbspro/spec/default_spec.rb b/specs/default/chef/site-cookbooks/pbspro/spec/default_spec.rb
new file mode 100644
index 0000000..2540752
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/spec/default_spec.rb
@@ -0,0 +1,110 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+require 'spec_helper'
+require_relative "../../cyclecloud/libraries/cycle_cloud_cluster"
+
+describe 'pbspro::execute' do
+  platforms = {
+    'centos' => ['6.8', '7.2.1511']
+  }
+
+  platforms.each do |platform, versions|
+    versions.each do |version|
+      context "On #{platform} #{version}" do
+        cached(:chef_run) do
+          ChefSpec::SoloRunner.new(platform: platform, version: version) do |node|
+            node.default[:cyclecloud][:cluster][:autoscale][:stop_enabled] = true
+            node.override[:pbspro][:version] = "test-version"
+          end.converge(described_recipe)
+        end
+
+        before do
+          pbs_stdout_double = double('pbs_stdout_double')
+          allow(pbs_stdout_double).to receive(:stdout).and_return("")
+          
+          pbsnodes_double = double('pbsnodes_double')
+          allow(pbsnodes_double).to receive(:run_command).and_return(pbs_stdout_double)
+          
+          allow(Mixlib::ShellOut).to receive(:new).with("/opt/pbs/bin/pbsnodes -a").and_return(pbsnodes_double)
+          
+          # here we mock the global `cluster.scheduler` method so that these
+          # tests don't block forever trying to query a nonexistent CycleServer instance
+          allow_any_instance_of(CycleCloudCluster).to receive(:scheduler).and_return('127.0.0.1')
+        end
+
+        it 'downloads pbspro installer' do
+          expect(chef_run).to download_jetpack_download("pbspro-execution-test-version.x86_64.rpm")
+        end
+        
+        it 'adds node to scheduler' do
+          expect(chef_run).to run_bash("add-node-to-scheduler")
+        end
+        
+        it 'creates cron autostop job' do
+          expect(chef_run).to create_cron("autostop")
+        end
+
+      end
+    end
+  end
+end
+
+
+describe 'pbspro::scheduler' do
+  platforms = {
+    'centos' => ['6.8', '7.2.1511']
+  }
+
+  platforms.each do |platform, versions|
+    versions.each do |version|
+      context "On #{platform} #{version}" do
+        cached(:chef_run) do
+          ChefSpec::SoloRunner.new(platform: platform, version: version) do |node|
+            node.default[:cyclecloud][:cluster][:autoscale][:stop_enabled] = true
+            node.override[:pbspro][:version] = "test-version"
+          end.converge(described_recipe)
+        end
+
+        before do
+          pbs_stdout_double = double('pbs_stdout_double')
+          allow(pbs_stdout_double).to receive(:stdout).and_return("")
+          
+          pbsnodes_double = double('pbsnodes_double')
+          allow(pbsnodes_double).to receive(:run_command).and_return(pbs_stdout_double)
+          
+          allow(Mixlib::ShellOut).to receive(:new).with("/opt/pbs/bin/pbsnodes -a").and_return(pbsnodes_double)
+          
+          # here we mock the global `cluster.scheduler` method so that these
+          # tests don't block forever trying to query a nonexistent CycleServer instance
+          allow_any_instance_of(CycleCloudCluster).to receive(:scheduler).and_return('127.0.0.1')
+        end
+
+        it 'downloads pbspro installer' do
+          expect(chef_run).to download_jetpack_download("pbspro-server-test-version.x86_64.rpm")
+        end
+
+        it 'executes serverconfig' do
+          expect(chef_run).to run_execute("serverconfig")
+        end
+
+        it 'downloads pbspro installer' do
+          expect(chef_run).to run_execute("serverconfig")
+        end
+
+        it 'enables and starts the pbs service' do
+          expect(chef_run).to enable_service("pbs")
+          expect(chef_run).to start_service("pbs")
+        end
+
+        ['pbspro::autostart', 'pbspro::submit_hook'].each do |recipe|
+          it "includes #{recipe}" do
+            expect(chef_run).to include_recipe(recipe)
+          end
+        end
+        
+      end
+    end
+  end
+end
+
diff --git a/specs/default/chef/site-cookbooks/pbspro/spec/spec_helper.rb b/specs/default/chef/site-cookbooks/pbspro/spec/spec_helper.rb
new file mode 100644
index 0000000..3c15481
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/spec/spec_helper.rb
@@ -0,0 +1,20 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+require 'chefspec'
+require 'fauxhai'
+
+RSpec.configure do |config|
+  config.log_level = :error
+  config.cookbook_path = ['../../cookbooks', '../../berks-cookbooks']
+end
+
+# /usr/sbin isn't in the PATH when these tests are executed by jenkins
+if !ENV['PATH'].include?('/usr/sbin')
+  ENV['PATH'] = ENV['PATH'] + ':/usr/sbin'
+end
+
+# This makes it possible to test Chef libraries
+Dir['libraries/*.rb'].each { |f| require File.expand_path(f) }
+
+at_exit { ChefSpec::Coverage.report! }
diff --git a/specs/default/chef/site-cookbooks/pbspro/templates/default/mom_config.erb b/specs/default/chef/site-cookbooks/pbspro/templates/default/mom_config.erb
new file mode 100644
index 0000000..6de8e9c
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/templates/default/mom_config.erb
@@ -0,0 +1 @@
+$usecp *:/shared /shared
diff --git a/specs/default/chef/site-cookbooks/pbspro/templates/default/pbs.conf.erb b/specs/default/chef/site-cookbooks/pbspro/templates/default/pbs.conf.erb
new file mode 100644
index 0000000..109941f
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/templates/default/pbs.conf.erb
@@ -0,0 +1,9 @@
+PBS_SERVER=<%= @servername %>
+PBS_START_SERVER=0
+PBS_START_SCHED=0
+PBS_START_COMM=0
+PBS_START_MOM=1
+PBS_EXEC=/opt/pbs
+PBS_HOME=/var/spool/pbs
+PBS_CORE_LIMIT=unlimited
+PBS_SCP=/bin/scp
diff --git a/specs/default/chef/site-cookbooks/pbspro/templates/default/pbsmastercron.rb.erb b/specs/default/chef/site-cookbooks/pbspro/templates/default/pbsmastercron.rb.erb
new file mode 100644
index 0000000..69b72d2
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/templates/default/pbsmastercron.rb.erb
@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+
+if Dir.entries("<%= @torqueroot %>/addpending").size-2 != 0
+  Dir.foreach("<%= @torqueroot %>/addpending") do |fname|
+    next if fname == '.' or fname == '..'
+    f=File.open("<%= @torqueroot %>/addpending/#{fname}")
+    marray=f.readlines
+    slots=marray[0].chop
+    system("/usr/bin/qmgr -c \"create node #{fname} np=#{slots}\"")
+    if $? == 0
+      FileUtils.mv("<%= @torqueroot %>/addpending/#{fname}","<%= @torqueroot %>/addcomplete/#{fname}")
+    end
+  end
+end
diff --git a/specs/default/chef/site-cookbooks/pbspro/templates/default/server_name.erb b/specs/default/chef/site-cookbooks/pbspro/templates/default/server_name.erb
new file mode 100644
index 0000000..f6ab154
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/templates/default/server_name.erb
@@ -0,0 +1 @@
+<%= @node[:ipaddress] %>
diff --git a/specs/default/chef/site-cookbooks/pbspro/templates/default/server_name_exec.erb b/specs/default/chef/site-cookbooks/pbspro/templates/default/server_name_exec.erb
new file mode 100644
index 0000000..a0298d5
--- /dev/null
+++ b/specs/default/chef/site-cookbooks/pbspro/templates/default/server_name_exec.erb
@@ -0,0 +1 @@
+<%= @servername %>
diff --git a/specs/default/chefstar.tar.gz b/specs/default/chefstar.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..b3aa696f57f1a17d068a8567187a00f8ec9e5bfa
GIT binary patch
literal 35656
zcmV(#K;*w4iwFSc?P*;A1MR*2f7>>aIK01Be+5fl&QdDNlKhgcZq>UqaohCTBu$*#
zyN~KGmy&3kjU*~06~|5U{O!-Y;FSdR;@D2QYu;@mlfYmA3<fiU!OZA9I^F%t%|2~-
z9y~a}e>)Ej+UCFV`AcW-V6VM*(A_)e{H5LL9(Fr_@eXbnFM1YP9;Tl6m+<#R8ZELY
ztthwZ`=9g~;rwTD9_@^h<n<8#mt8wKtU3Q(D08@X@Brrj;1FKj&HpVtuKAx2vw50a
z9~rE@{@aJ}e_fdW2knFXyZOJ3XUY7(IC}W_=cCqi{IPA|1RoymSI$2$eS7_P+j~&l
zYhOnPS3UpY^Zz)?Mrl0H<78Ir!&9rhh~6yXG@3@UJQJ^8W#L&Qe%GE)!u&Kzr?nk;
zYPI7i%cHTE%zCxjBS}Ky8^FqvMLLQyFPx3N=V|gb9!IGJd;IM2vs&$8o~Q9}kw?~R
z`uV8`=U?Divi?IgF(0`VY$E^dclTiZce?uz?)d+0JnPN>I64g%lf0D<uhPZ}{tqnw
zu)zQKy712(|G$N2%X^f}FVpz!JooCOhWB$kN|P)(&Eb7IPtp)~hL-nmGV!QT=A{t~
z!rN%vs%?2c#iMAJ0sCLffUkS`dF1{4<XFCj5;Z|NgB}yr$)NWZHUNNm;`Keh^L4A!
z+TUsWj&j)~$ulf9izhY5_c#mA(qu7@##ks%7m=$}wir(1JU9o!hQESfluU7F3j%8Y
z_&m<M(|8hj7h&d|MY9Mx4BZ)CdeF%@oW#G!vomj&jH5wsbUB(tqe-$DpHwu^a`wZ*
zU>s-Ra1sR(?5Jrl4@a+|0o8$3wWAqU-k`&i<m?O^fM3zuXhNNS^7OlBs}a>TaeQXs
zG@{B|?;V2yY$MO|G@PG&K<2S)KxzS;O&gm+3)EP+QB9CZ!kp^etB2VrkEc;1^X}ED
z=hy-IYZ_%J%``H<<{G|mktf+Ggc-UylA$($>nx+15*}GP3dV6tD`McDhVuj_>BDYz
z2@=_KH<}Kk@i-dqPU7M2{4zgJX5H2U=JiN-FbJ97ta%$gYf*r|5!e&pB=a!KfPcV(
zK0EjFgjfmo+#4mc)A-E$ElXxJ?CXq`MlGN*K_*f$T<d_-^99rv=!Nw7=-XG{uQho#
z1nfo`Of5cjVTrHcADGcO2;SHis4@*<ic`P#g=*WXGny>$yNL-h%rP{Jt0EXiuvntt
zw`54<w0GEUUlmk%3STuO02EEKsP@Gc%^-}3r|P9%cAhLIuv8-0M5dq%z&b%l)a|V4
zfvn-dAP`7U7-0Svsh6B`8Enl1B3Zyc8O&UMPSrE-B2D1OMVy~gle)RqH4V?ccmEoP
zNI{Ec<Ih0BSt9=-nO=)AaAW&l7yj=~{=b!H<@^hp{}?)0SO4qo9UkuOAEN$u*zMlU
z|7|>L&%fHjTJy^d+gPsu9X{CW>=)Mmeiz8#PXD`w=ce_)cnUyrPl)M3Jd1PB+w#Qg
zQxIr#6!)Mw46l~}gwzB1e*vWD)zj$BBFgft;f<5ocJ2W!#<NHQYYoFJ9zA0DkRa7$
zbE9Me`!k}R$yX>q%HN543$n8O3DWB*8iHV4tDUCF)YE&s#C95vK<J2?p4joTa2_|c
zz!5C+c+xcGozRqiKn!pGN^&uZJ7JoJ&~~AvmJLic{3049=~#767iZzP1O;RbS@p+c
z*z}&Wn)xzERpsnb0-7eH*Ra;*pYSsnr!nZ~3=sYvjU=2Wv$vqyj9<pTM@R4GKm(}5
zAEjxM8r4T5feMh6tT>LMIsPq+%@&}Sq!}n*qY29V&%b^7P_mXD4bS%WHvQXa01eIn
z9U!{sU7Y18@IfmiEqd_`$j0cin4H$n{4ab5kh4*g5v*>*dmg2zBEpn-+o(Kyi}^N6
zmq@hJVkMw*#Fq6sORV%@9=*4jSuOKGs~;tRDU^?A9?-@dv0r3}@e={_39J}IWzXmq
zblyY3u+$rP+4>P@Q6j{3_Z!};EaJvTi!=qb6k3Bxl^qe3>D3X3%<GWd9XgNDVKG3z
zM>!0Di{jGOLruA|U2gWE;e8iHV?>kDrCFwKLT?CnY+ToGdKc&M=-iv@F&{@$#05T&
z78Bwsh;#HV9OaWsQG6C%=rIj}AIzXt7*njsCGaIm){NI-!+V@yQI9xQcD~5R$;C|R
z3`{9Ig3SQu$)cPFZwf_DCphCf-aiyc?Iizzzv1(&=>dA#VwfQVnqk$5IU#5%q^9vO
zq>I@Mrx*%j{}BolmAbFdd5Kv|ek=ZaoOr@AB;RC<apJw1o~7Q-$lC_QSsm7NC~QIf
z(-0IcUNP44q5lBK^aA3#K~T@4$!XK$C9auRKwNAz3YeMT2jC0+r^zg8^pqZtbz7<s
z=rQWIT}0Zq$TWvzW0Y*2MS0yf*%zarX|M$wY=Fba^)tWe9m7&>pq@8Na{8mt9`y!V
z5uD4$*XSZ5V*FVw2wkQU_-~SWkMKvmQD}=rp#T)|5uJ|(Xd=&`o;^CHe(DO;EeQy_
z_ui*hK|J<*o{tP;68Rr=!)g|R5YtQ2B559tsgV;dpQ~^e?@10SG;H!fOvoyx#NxSj
zejWnxg>NGwa$2!o7EV!#*C&N7CpsC3m$;Xwnr@6)YZX761Fk-$>I|{(4NknTN<as`
zD&n6g09#%*k4AAg0hW_R9;`%IKv|A<K;RbM1@OvnM$2gmD;tOpxDd?~cRG<wEu4ca
zJWpd}vIt-qhAk)zYJJ_8E%{43q>9UVKq@FqJw8k!gjS!Hfpev);({v$T9y!12YPhK
zI#E@?7(<|B8ih2x0Dhkmvs>izMV>KJZ4z%DF~u>5g+InEkIbZKr@27tnZ(nWIj;d5
z<t?l_j^%C;W+<jmgSe|LGP$h*@xz9lU&L}pGW!X*Q3nD*u&Le(oQOA=G$+5)O45T(
zEsP7W#LFsB;&TB~C4Nx5gk0U#iAt;hLJ_69192BF8iGLu>Ibe4Q3Ut+B2qnzzii<=
z$GAjb$rxK8Y;(lEXdkp{z?QlF)i#kXLT9K0KYAC9g11=OKe3xIQI@Mj{M{%Ogs}k8
zaA7xT{ej(k%e!+$^Q~+?iGgW<<u}ThfFK};0yi2V&{iX4l($=f+HXsP<Lavc!7-VI
z*(y*#uQ>EL><S&pe1ZuG)X;C*s(`3LTXJ=Mtw}(OZ@KE0$99A!tp3+1E6*-969hR5
zIBs%*!qM9>y+jg18OPjGsGm~U*VIMlKcs%N9Bl$$r+sXgM&WBD)Nl@pg$HUST90u!
zfS+8#b~g5C%2^tWfeF2h#|zvA%1}!$c%SI67*#6eJIG^_<UxKpUkyQ(1Sn;)KVQsV
z<ANr|s|DQz*@<Venlx!hn_L%1H3D^v4zTJB&Orh^uC&<aAuB^b3gINbMD7eLA%+RM
z0HGPj9%KU=M3xt(Amqe5+!#`k6ZWdr<*WqrIEiKwY401a<3_yDz`%$9;4aI@gjws}
z*!S*vb!U;LH*1vV9-meHC`~7{_WnavM~F_=M85!cAczbbpkI|xsv1`$c2xv4jUp22
zugwry(FC)n6GiKM)GLXdh@edB_|T_;k4!`W08zLu-b%3UsxUVGY&2hFzTCH{TJ<o=
zwx|w!H@J^MoiA}=zK$-NzzrvhNHy0&%@h7N1qq|NX}}fgN<3B1nx5);bzOY&^m2It
zd>ApC-JA`BEdD*JYs?$o{o<%Eg&e&bMe}9mrPYYx(ectkOFu3pJ?Iy0+Ss!zasq?)
zmQ%S=aGe3%Ok}tSQxwx!SIrG+)f5ch5f#2Oge2N9-!V%hI}_|o3urH`H;B@h7Jhv8
zZSeB=;qg)M%d4YTM~^pmDsVqV=r3M9efs3-_csOp+lR-GemGe}I(7<LXg%dEOqf+$
z_%FV4@5Vg2f+LB_E$J^C`~*ay3Baydl<!1k_bm1MRm13#*cS6K>bX*=2Y{tQ6Ht9D
zo!sOHTG*{S1z@*8V)H$_fz=Ffi)wfpCA1MrGb4E>gX{={!R9$UGZ3x@D9Km@j!F%D
zkH(cU5J1Y4Je)9?vZTo6qzpR4ZE6B64VpsG5eU5Na}k)?IFpuvIt@=#TF+Zg>E9(=
zf}5gDeg=%1>B|YjEz_?Ly(}|JDn)u&pe;X8RuZgbPy0ZdP|EDjUI1A@XDI+;m?RUm
zlt_T4FR%}Ohc?p2l4V5BhQXZuG?p&PZQ9_sPa58?T*D<uQo65L)~3h=<brmnVdSB~
zwFyejG@gVh8tmVH;7-7>`VF$s#i%Z!PYoTNw2J%(mSH?QP3rz>vY3qxOhC{@F;~4<
zG7!pCSy<Wh-aRCiw&ND23igAiNgnmsl7**!a=k=o$<2Us;W-hgi|pcqOI6xPiCt(D
z_<WJhlPoHX#1@{Bq@bCguhBdny@pW@fx{4xKGKyuo`|QQ0@CFK@8!?Wk=->3D;VT3
z1vMs`;Jemx?kfk3he2B7Q262om3_D2ec3O)EUz&QaGuN!fCWvc+%yvV?2KMu>p(NW
zZ`4-p!;t0ZGzOpE%HIyT5M2XNc$G!g(DY>${MWA@ED-f3iwt>nzleHv9;RZ@ifCgk
ze2@R)+!go%aoZ^@e>^I}RU_#~fb_zg%#hwZfi*3B%+O2;I|^I2g(WyX19pwaH^S_2
zVNZL2@SufC)a59PKTpc5YIs?$ZOm@vxLIae0P!P;6v!F-*2Yg7>q^N;9X<CIGgz-u
z)@Nq7y0q|VMnfc>Huv$x>S~HQ`s&x}N@~C8zZZ&E&uf2Jr?KK5)VKBx#~#(FSbsx@
z>SWDWt1$V)OKEKE16_9-gPqRFQVt_`I7d79IsisT?7Cg?Jr6Uo6Df&|Uz4QBm4R9v
z!r(JB`JAQUkR&DHs&SDpR1cx@*>mOzBQ%`_2-(ps>4ids2HL0_$Tj4Mcz$Eh>sDy1
zW^cdZEgIp;x;?1dIdRyFw!ELw4mF>!@rp$#vRwhe$RlQ$AN_$xEap>PYPJLv{ZtKn
zfU~#VSlg1t{k~ak4c=-`ac7;~^$CznzD>K@&(ELW4n`BBPGmuf5NF3_PI?i}$RB5s
zLB~CDzoNi_;4>Wyp*s+aDd08-Sj;6#Ys5IaLL@5LU#~0#TdfYZ1v1+xEL!Kl?((EO
zp&@3;ke6!7wHiochdhxj%+&H#xfUMPEWSmY3`yLgo#n?HcoJ<WQEowPsV;>S4M3z7
z$#Gs;rGMdQpm2&VTA{A!k&C;hu8}o`7uS%zyefd9n+jeMB}&Z;0$<3kFS>~ff(RJS
zQ4ojnQH;Rdnwj&U2%k#(*D{-zG2(8aS61eKcdN9mD_X!{&<U_DEumu*N=xJvvdrvf
zA*W#A-2wtp=At%<YDEo)p<3%JvIEP=E^HMp8wDGZ97I|^x0VJl>+L!RRPJ-~HNaUN
zLjo1nHJk!zXzLYMC>gBj1YHmfZpotbpncL*zdOBx>_|fsAtlr;IH1unAcwtg3l<-q
zz`!1m(B*EzlU)Y*h&`qaj(j6v*^I&k`jy35wlLKxJRoqmw(!ATOS*~BxG62$?m1{t
zWj})o4~4Ik<4Is1tvlvi_qaT@o3-cw?q;-UlrPem1A0vrLpPn%C?A~%O3V((WtAKp
zR{HHc^e$2)K45q$Dext~!!%4Uz0<{vJX>)eCXVE>3+zJp>S|{yw8TfJngR0`9FFaM
zA6wa8%;ff}i_2?R(m4{9g=~p3{4$D6CnN3U+IlK~KGvf_1UV&@Yj&Er-G`jRh!$4i
z>$z&MTegqeGAFLa@<xj;Tmi4Da3X~f%@5*9j*g&iw8TxcCFA1qB%z_=K9fBM7OoR|
z>FD36#4A@g4UH>SD?XLGWI<XKF59F-<JtZQhB}#=<O8&W4)g4Ybs9%w;R*mqpuM2e
zZhzlIQ8HsXMn9r7jnE&oB;MF2VzXOIWtz&3-GCs?mMhM+u4~_Ro>mox_Vr{jK8t!d
zg1A1&gES}OjyGJK;bA+xR!bHOWar5R^7X}>oX(}M7Or<TXrXO13JWe1!a1ZwCE8?F
z?t?jUzQTbIJXv%o)>ytGTELG218Cj&z0NMa3?G()J`$k-uNDS{^<Q5yW=*fM3&q`J
zIVNs0!Cgl2_*Hw#G4XFaRY{ULRK;K~L)7T__#=mx<YE^~o3c%cF9^*{b|{b+GMc<^
zYF};Kt?*z(zrs};p>zT87kKLf_8+vUxH-LPz5X^ji)Y5hM|+CPFS3fm5}#kTH#lHP
zC-4B2KWX!6s%r<TTTNU@Ip1VaKASik#$Q361j-d%U;`u)lz`Qp*g25ntHVNRR|^g0
zW;CR%a6T?OSzF*xTOe=^Wq2(kAQ**Jv2^?i6NobGCXFNOqD9)YegDPrS&ru-=qEP9
zlf0IT28`P_efzSFah<_E(9Y>yCL=WM0dR`lc!fRBC+I7D!9H&!XrgOWj(51px&ku%
z=)C3q9i66JvNR40T~VnEOd7mI`TjI(yyRsyBnh9Sb_0jSY!YRe-ha<wOTgPH-oJ^)
zrs2C_F(WL5kc~?emNs9IfEDX@?RL~-JKPc_p<D;8o_O)kI4ddvZI{Ou8HzL?VsnN=
ze4`}zppIHL;%*=y7}sKO?Fs^^D|4~Cfd*|Jc;-)MNr3Vw>eCH|%?uJkws^6a;SF+<
zCU9S`IFU6qL}UoXlTKBx%2CE86c__w^vNL*6z1+LmzC0F?~lTnkUF3i&8Rq@B-0lT
zqRTYpAiZzAHf=n`*L_FfWdlb{Yy0Q$T|8Y(4J)vMATIkvA^ov~_~M0~kx;wD@jfh_
zoV5GVIZ0~o#R|ZC9gaoN^PYbAfW&+++eQ~e`*Ty^_UPHu@1A@gJb!rnL&c<$qicD!
z6nAWkI}%9)!wrh14Bl^V@TJ<;XV;dYm1F_jgh96(TO&X{H??n3vR*{1VKD*mHb8%2
zQ&5n_(f=*aipH;zsU)LCKto|0t&T&7-dL7S?Jqx|krS;P1<kr{%Cm|LBmjq>XmuZS
zt)dfVscY;8apGF$Mp@$=ANl_GD9}D}>Hw8^=$K4|2&tHB$nfjJbDI4f6i3u0e$HK%
zfD+AAdD$D<rlQ8uW|P{-295)AgZ;f=C}h7I!{XOM51Q?j_XTwuKT9WgK6Sg7k-u`<
zk^(0kCNu#BL+5A~F~S!LMq?eVYHf_KfSL;vQ_>V|X8~UJF7EBZ0T{pdPeEh^fea6$
zT$f-`Du<tAaw1`=oe!yc%TC+L)?)D$)d9yYVxoh8^B~|RJgw+mm8Rh8CXfG1C_%`R
zLLut|gU&>ARUlihqf6k;c4tegp(TsgH-b|dG<CZ!o>gOn)*3ciyUaCauOCN~h-r@c
zs3fdXCG$qs7%N32j@8RG48`$6UE0Kpo-LHJT$@-13i5U8T^ibEYL~8eWv$ETh*rAj
zmb6prg6h+FhH95f>%wz^N~B0M5HXZM9g09}nc(Q1PLj~ADlA3a8J7u$4}&NbXp)Wj
z6mIES8`cq=uwAQJH}9~dBj-`;l%yzBEA8u_<N+lG374&5nIt=ohKn<hu*vfcVE|(q
zn;ZjyKEggfqPP~kw3ec>q2l6N(&TsBcUo9z7N)K0#a()_WgSV7<5wbXC-a@iGa&HG
z2=<*UagDpcqVi1DA1<;>UmT%RI5uA#Q!c%J;Ujn3u$mYiwSfQr@CEs0;}#1`l(;>*
zt06`l-;DH-dq{+O%$l-Arf5N=gOiGx!)Li~uSSbCN{ttZbCXMN95Hu5hdZ?XGrmak
zUot&a&!>DU2?!tO%Snh*D*G&vf<;SH)78Ltm#DZtL4_9?s9QD|9!2}B;|?D_cn8fY
zS3Ra%#rA~HFJOy-B+oINWXRSHigm(@Bp}nO>i0-S()*it;K5wb@g7ji5){x460K4N
zRKC#wnsg3K=Li`cFQySFrJ&)W>y+FObf}BpfMWs*bapJz&nZTQH^Gp62z?1U81)Bw
z>UPy|P7!F)b@y;I3x(Yiww&%u&6VYYDl$NMVy8T=uI*^4PrVlT*p`4RO9m?8sEGG6
zal@X4ZIcbSkXXV5tT+t7|3y1XxiJr--QhB6e&dz^Zl&iXY_4epoCg(0v+<H+iY;--
z6S3F2IQ~#R>&2s>6-O*hmp)QcsNtcI`;38uzPL{ss?UvnSwty%#;e#kXlOGe2jK%4
zdRaIlt1<agt`UuicZ@VbV>jx#t0Eay9v3lMc0dOCy51l*P9bbmozl=$#6oZhU1i&q
z5@n2HE=$9@M?6aokU096OFwWSv9tmbqLj$Atdh0dDj7nRBRo%&5gt#7Ds}O1^~u-D
zu2T!ALe$OjH60O3vnc(w5XKS>mAqR|m0OJ*1foVZyo#t=I_ewuFH1boDYr8YS<AcM
zcrl;EcqZkQ#!Jrv{<Tc9E)IlQdu`d@Rf_J77%$5A4vRE1Fn*tCS;$R4M%kt(DG4vJ
z)}6>0{uu@G()2-Q0}-RyKNgXW3xb<@B5#!!EasT3pVb85Jn1o=YWi-1HIIMDKfy-3
zSwf5lh2_ZSC1YyA_{Qfv{~cW}JL5^i7zC{U!8LlsBDsoJhw4C+E`N)kX~g#!g*C2-
zEFTY5t<7ckOqmx*zHS~Ks5^?5oll*V>Cw=4DJ-^Ol+dyAu#9x6Fbb5d)5QT@n?0<v
z5ZtRk?AVffI?s_^(5soHy^$ArEpOph=D5|n{F76Q8OBQHYO*Z|#3(cnMYuhLB)wdm
zN94S%bO5QyNCO{v0ajy3_wH=MqKgX8ULDh&yD6%c6s={6qv10|;R4r$3_NA^0wZ^Y
z>j)!m*}^dX(d8DtF<(^c<*k#ZNr8oH(Of~j48pfzJVAq*6{ZYV1s&1HGvWV6Ub(7-
zva9wz<9e*PP`y-;Fl+Ggu{WWDNV8bPMl7<}T3hM|(kHM|HKmG3%V+R%XBQ1tnR|tm
z!NeOgD`0Jo_c~VQW#Vunaej;#pzAVQUaBm_q^@Y|49}_S>O0>}h^NzNjIrJ@v=wjo
zB!!8EeipL?Fy(OcKxi;u<XS-yF@g78B<XAK6y|wRBw;2H1R2o4NddOnIw?FTN|j1R
zNJX(R4xs6GWU8;ABW-RgK})ZO`c1h9t6dWO7c$tt8cNjWgj&~XpDM*p?;1^`)e7rN
z1&=ARa0$@~L~wHFxni}oAelyIAuX=rZ0;CnFBuoM_dTWKSJ%+crI#6Blc^ja!eX7X
zypJBKs!^&r3fV`#=7E7P_Lj<ZC|E{?Idl2h8=D}N#Q>!IbK)T$J5!*Y7=5LN6I!&y
zI+pVdfzUdv&K`j^Cd5Si2WD#~_Gt43aRMW5>o^=0hhWA&!lBJPxs#J|PF!Im;-VpM
zVvp1i2=3Xk4|ZVO2h1@~8hq9j%DV&W3`b!Y3=A11zGsd8Zj<Pp!Ck#Ni3rfw9kO^w
z`KaNc1Jr~9nU14TjHNJ=DLdH6Fp-gUUu8M3uSQOa5@|F6d1sazn>q3J;bqRP6Tc=)
z4&EsD>fe8ByQh#aKJzMW%~^lQOIUheM_=@kVpxg-f<a2y!@EJ#a+Ql<t1f_<WyOWZ
zA{walIo86KhZ!8jX=344K6y$-Z$W^kFmY?@{LWQW>AGi+<}%NIyX4Kd_?Lx4Gq#C~
z?ONs?OF=G3bK141;gMgI_0<d>C?Tpi8YNiTeZ6L`ra~!0!}A;Bc7chbJs#JgT`xmG
zYlzG4gP|Q3q`54a;6I76GPOW%Rj|%dt!p<+q*82y<DgbHQoY&O1NGilRUXmp{GQjr
zP;N*tJ<rEf3xt4Q;VHJUs(%0ea%IIJ39-u~9M)Me`<46JUg5@`9orutO6DY;a%j#e
zzLAu*g$L7a!IMcwhlq11E@DkEt3SIWilm^F3%nOcfs5Ik4%4M6sl3u2sxGj%r&NE1
zy^ofkuuF3s1TK@GdwxsT^gT8<Fv~NKw)EVzq85QUOTr1cd7}XmcU=(Yk{NE*#?in(
z-eilRw=?mk(X`(=`-g^u4%DQBz(9!rCMsw*fy@#CIwz!n_nF~O$MLw_2uBU<Bj}!J
zu-t*aBZ{;PRC_NUJv=@>dO=c3%bI1N7Hwpg<zd+e!^P-zL{6zjrVi<LKaNLveG*QG
z<FJR4pEiydF5otG^-P{c_xKAzNN{5-WWi_&yo=mIgnwzOutc<^L9YYBDI6vX-E*Y8
z=MNwKH=oOgnY?W~jeqB>m_^Y;8ly@#ic8yGO@Eor#c;uB)yYR%FBkb|#y`bUvHp<j
zh8@;rWEJNiHN3C99D_O3tt;LwX}-0PDoo|TK=j1fPc?%^Ny397AhO?V39S2ipNRe{
z>yp^gDG7Se-CSV7#nv2hTHzwrj;uRqYq4yeMT>D_1mjT^$bB6K(P9cs&-hM)?tL5k
zVgWI?`lT^#`ekwQmIS@gmDHSCkR;WVUIm4#j|K_rL8v?=Ro+M@XU+JuMQAnXl5|f9
z*!RSD(g?_eCjvghTe4-_!`!q7v0<<Yu}Q%{D$5MzbLiP-R<eo|?$60wK&I=00Eh3C
zBLHoCl-*I%CHEQ6kzs`BbO^~9F%@oR+Rt4GWLq2p+r#Tk6S|<2l7<5t&2LeT^DFZE
zpm4`UXUS|weDgR5B&j4KFA|+!kzSRnGg)yfK`>_Bh;lTqi3OEpKvMa6i)`6hy?p&Z
z4^m0;Hmw@E34(;{nzE(xfkYXs1&FrP1PIL0J~G%-hs16636ng5E)$OHr$!WfA;urk
zKP8uZ(^RE$K*<opP_^Vhdzs-N<@FA~w<S)C^q(f)m4<`&OEcL@G366Ub@McNi=k3v
z_6*flU;S!WbBrqn(GSzJw+-)a-k!#L3$Kb|jC9PpgZa4LVqEqZ1wgh(k>-X|5YjQ#
z6OuXdXfn~e+%TSDiduA(-GzOSQUSv|-!V+OsKjT`HL_E%B}kHFoI%mLs8UckH6U3U
z{-RYU%Nrx=P6vZN(2vCLhtHqXSNn*{Lo8L28&N6p0Jxdfs;)C@wc!0@`laxz%*4|#
zfVbPwj;h3uw!D{Uc@%*?@G#+R@*0&CWUV3P4r+P<@uMJ+nv5R=csV!-{HDl>k_sIu
zRM}HZN@y8JFXo+IkdA~m8uCxrz_DzNnF@>{fT;TqUmZVtdHnFjaqz>lXa6m4l~}Aq
z@Y9-Jv&D6_e3gFcTzY@TXY>4zoYs4jB#_JVKeqP|O8H+8;Q#LOKi<Z3)A=7)q+eVN
z#bLK23!~x<I>BH~TIA@BVS=+W8sY|prCCf9dDtvz&G)s16HlynfyM1+^CX#Y0(<OX
zipjdO-TNqh(SAld!LE?McIl(OEdYPgK8gQ|9P^pgl*Q`wSw{=e5`3OAgO}}S?Ehh2
z2F7y(of}<k&-2ORjABq(hN_ln)3YQjXWLFe%P3eWW~n_2AIx)Z@5{d>@vLsZDekV6
zsvUP#A4j(0)`=+=2uW46RFDg-_MxTGh0`GpwkD46k)<OnPlB=k{Qr)Bc=i<KF~299
z%c6Nj7i3l8cE2L}XmKO#VV;Qf>-_ZW`{27LKOJer7%S0V7(I(5{SDBOpo5=|{^#f?
zXrR>7sx<a&e-?@CH*0Xsw#v<X&@?Spq^8djFj-=LnV%;!gSG@o*$|w^lB+4V%5lqH
zxX&jP6@_CWU5a>xo{b=OvjAQm<(=zw-ztWLHc%WCyL_@_B@Rk`IRQEX!90(q6nc3&
zjK*WWvxf3K{G$fa6wrdQFiuS&r@_zI+Q0`1I5_&>qes*Tf-``Org$_lSlSV&wL@qr
z3h|qjjI?Ru<?xoldO%;(?2NEPiX`#^@Gr#S>%EMbCa{V)O=fgES9sR}y3H~Rak<Ul
zF2@-_#D3R$PE42}WcfH*<bCtwljleD9t7U<duXv=ZdJ9#;AC|?|AN|Rk(z?cj$W$<
zBdqQ}*S1y)!C`C22v3>IL2n%dzpb?*bc>0`U;aA#H5Xm>VIaky*i-zIf+5&lXjG~M
zMcj{Gym<Da>8(=Z2~E_QkYqA%is#bm8d<2-Bs>jODw21$2-9)RmL~r6I>!~{e^mOe
zPyX*bXzw3BD9ZnPoz9*7e;dyy$p0?ip#=eHN#tACd_sITO(-i<vfI9lD>9;&J}ZA}
zE+QO7Mc{)gC21!MGF8MAj2sk;OcorS14AhR6~c>3x6TK(9EL;!O~ctZnK}}#&(}=V
zF3sUTgPz1@sGc&17t%!2QEt}ftR!fd7eDa?J9B1-=F?x5q%=b*X5gbk<Qv1Wkc)~i
zPxKEIGlHG~dX!35dz<JQO%Klo8wFT|k#MBoMvJYDVT>_*gO#`Fy^%J!=gGuvu<=h>
z>Iu`)IsQvYCopLSdEDZMg#OX~70u-t#@vtgxM+Gt+(P)Ha8o3nzum`wF={5YJ4*2c
zWtE)mZZx}?@Y>}3B;ou#9#N;qm$T46`Cv;fBJcDfD#fl9XCmgj*UHt#&V1FvKWNH9
z`9*Ty(tS8H8=0m67QLCKNZl$YD5G8b7nBqE4B30v^WNFWSvi2+J^Q%kUo36Dl=gbr
z;Ff9E2Bz-3@v@Z>^@}T%kK*6s+a4@8J<-aGMRk<ckuBHp%{*RWGzPMn(k&(~DKq^>
zj%!`;azLdV(T?=D0X9|PkD|QarTbKI<nZLr9&zJ(Cy$eI#94BzMZ;nQaI6R`Qj|+I
zBBt=H-CxDva0i`H0-(6{m1ycTjUq?7ph7|2j#bj_LRYM7|NWKcH=V%bqlVWEyHYv4
zHnT3TFaE;oY5BCH;vpL-=ydWP>`GKP_uj`~Vfsv}coyQF#!75wbn-^6va>=Hv(I|>
z)IBZ!X1aF|-kE-asxcS3Ck+Sd0C1+$f*KtR=I0olCa>H+lNtJisoom{CsxFJ-V}RA
zZH=q-mT9TSz&{vhs*wfh<ZKq%9*LCrs$2%pdjG-FNCjU4s+VrVzKVHVuD{x<Dh|>e
zO*?e@B^=7^%{t|Isx5a=nZa?r>N>&r8m%Z|s)jAQZS_RGF4QCU^po-oYUlkzg{s>p
z0K)XLaBvRkeu94(Q(}m<#z}6$-=iSlic?qgMitlfoKme(OrII!U#ghA$ix5I(k!EH
zl_=UC&%=yv2n#<hO!t=+jfpmx_aZ+PF08?cy@SedaWXcW^RAP;U*Wu(y!2KWc?dU3
zS0f#G4iu8g9Sq+$${Yy^VPOU;$oj?D^fXkZ&J^cI=i2G3Ng13the!FG85pk!PR3<~
zPnm+T%=5U2AFm2|g=^qs+g1geezllqU}xMoFuGFYbf`9BoHy9br9Q=zWc$5af~O`{
zYws|HO&L7VMyC*;b7@pgEFW-EU0?%sD{`esP3D-2>ixS!4V-BKNP@R?>2+TnB(r=4
zPHf?Fi^h#xh;TCGVFf7O#6^>9;_0}BphGP79UFUjOor6f(2CAD@P$JqkHm1`YbSii
zc$gno56II*JM{@yRgSb^P=gpN@mC4tG&eg3`&pdg4H038aokIs=JHa6QOiVoAYJzz
zfdS;G>R1`?{kNe0OLzMz41xZ8by`I<86@eYsQEMhakAPhV4>KPeOldI=@xu%oj9%^
z+@#gwq|aq4HUr|it6J2)Zs1v6Sx0)<ysi-`;G`U##on|W@w-Yd#13QkNTx!oVyyBp
zIcf>YNqHAqUB|s;TG-RpDMNQMKQ?cAn!BeR$%bNeSVcG5O^VxxQ;jk{Gvl@9ON^_D
z!-@2%bzT#y-#0F-$gY{MkPjqj9*xbLEgTu6ukA29@5I_j<0OsqOO8Y@4u&L4W^qK@
zCWPh0@RoI?<1t&+!)0>rQo<19_Eu5O;W|}D6f}~kQe;FiV4-nm6$6o1iL|r%stR)R
zu;oqN&rrl?D|FGitz}9X2L8s*B#HjL$;hEY1vZJTt&!u^K)<k?^h``Yh~sB_`U}EC
zpIqOiY$(YBWhJ{>&DKPe&$&bWm?anec2U{jz_c?XO}wh^@>VzHHCc)O!&p53_T?|j
zl%Qhs%7Fh~*f{O^rZSvs*fXR2(|cvuIk6kW+eM<wTvZ<_1j6L-3#`K*#cSTpgXrBn
z6>qd9A}Cxq6>^Brb$q>GjN(OYi*c|Ql<Ja->dS8&fs;kiQj&w30gE+7eY`+;R4z=J
zytyaPs4OE*DeAXlA(m5RSx8hT%VM+#1)NVLgD)p{tgWg^3uwePjZUGC^#|HUb^qif
zGFUf~mbD#i{ndc^P{#&Z!gal&5U@`8*b3D-#l|V6iQ8fAM>lXSGnJw$SVinC&3NCa
zvaiO<S+hf8-JB>Pnkrj$N9T*#>wr=_8C#-!=r$LY5W(csEo%l>=ayXw1hA4hqVz%x
zs5pn~l;xf>d2_v`IpD<bpjd;3Lec5!Xp~Ds(^g(OFSH_ve|ePMW!YtyWuBAI>Nvh@
zw&2~TEIwjZBv~wp-fK{WWD({&nn;4*#yf@X)Uecpt6i|K!lAk$d`hJ2ASKy>Rhg1;
zp1)j0O&c*)*G{ZX&y5R}xS-0?!JBb3(Yq{cUB<Ci67czqOHePgV;pD3W3~(6p2#4b
zhZ+ic0~7CbmiA&@1SGpwupP%3M)s$IBqJm;XslokuA7GkVeyWRc8RDfY?>yH`l60~
z3?UQ4S_!Fa83>Ob#hJ4u3gK<`XcUm~epwPfzGc%h1D~Kn3!i@Ac?X)>G$=1pi%8uA
z*KulJgQS>=9FhW3M%TW`VlZ}MVO+c>@0I#}V_=kATn+&yh(}XY!yu`^B2#10u<1C^
zXBzrXckzNi@S=QNSvZg7v!91V#*)>P;^qrkj}2F*M5w8e#cZ6Py7Xph;a`tJWxjTZ
z{?jNwPsUF%IN~;t1y<NLMxq3%Rb5~IEF70i!*`G4Qxrr|iY}4kt9fzRlv_25>19*M
zu>*vSww3^s64whKpUg&_K@Mp5)vG6u>qW;{QfolTdMqs%OVP1Qu7?o}9Qj1S(E0e*
zZ;)wd--$?;2TEBnFnPnA;uqsUYRh}mqN03S(FJV|g_eHTEJ8Oj{+|lPg!x-1879==
z2!#|&BEeg|Fg&`kmPA2mL4LO<p!cTp7=3Wzr!V4T^nK_%3{-09*GpmOig>Hsb*xF`
z!yNwT4QS0-eSnwTDV8T*Rq#}&i&xM!GCE90#Ai81Aea7oA>#GC?uX^q6Lh=X<?Z4d
zt5KAyE8ARd(0yYua%ZTmeGh>V9x$B=PG>UHSt*bscQrD_ok<)eR#>_mVOuCLjyg_5
z4>TN(y)Id=OcPec^sz~<5MW#-SwrZ321#m0f3G1oK7D1q{>C~mb+2TxfC51U=yg4*
z*qv{746J=ig#e+rSeiopxAM2+Olq2pF=KVgm3p*RpbGuqLv%-2H(Xb2Mc@X%QeaAD
z9ZbcbzcrRG{kI|+(lsvLk61@Wf1K;7G+9gfuptF^H=^JYB4w%({Dqxk{)xzi0!e_x
zMEw&2TV=ON-}DrjTqo_}i$6cvb*_QzcyK?4S7qnuE=n=xmr<Q)=%0W`Ef+98TWYxu
zCS_8~hCJ$@noLUa;GdL6Hs>g|I5r<<cxCAfbRK}7Bmrh553u7mCztK7A(uI5#ziX~
zK`VQE9FXpNt-XjD4o2ZD7)C*e_fv)2*fLRgW65>{YynF~w^N0!F=l-nsE?Q0_piuK
zv}q!ZhT%lrvuS#EmC~gjGd@khU9!?s5`7|Xh@$XzZ)MmW!=y(SnBEA3mFy4wx6v%w
z3C$rh`pd4JCBvVEf)$m=9%%jLV+Y!D)6?_!rr^^`u*@Ju(rrxRL@B0BkG_vaYVb~F
z&no(aY=UX$q=;&9A!v6|qAG!2<?`sLfTDfhSHC&H9lS9*VmfW;-xzp6(qx%Kh|b&h
z26Zf38!v&9LZX08k!}oUeP@D|Dqnp&sI2*y?V_xF`sD$0!Hb(;<3+SFhOR4bmA=Sl
zU+MkrsN!%ncOQ^DJ=xJGgssEIWrKCLa0Xetys&;v)1js3aOpw68cOc;!5-fVhkWUJ
zX8_v3RlX1!xuPV+%y4<tuYs2)t0JYVTUSfujp$vcC}k8+HaacuA?J<6GYX2fdK%N&
zkBa_C<cT5i#i0-gFftVHTNx>s?Qq(x8eO2r>DvmAJtNeu+^{*|ZMS_kbd?rjDh4Go
zzdV_Fu)HTIQo|O5r!9aBS$#Pi8buQrM$ycLf^4{1YS<j34dtZHq3B0Z^tghcu}lE1
zH2)Rmzq@MwyDd*JOY|5P&IFiH36SBa8y)Y_^H*e1-T(euoOpJ%_44p_G!xP7W|tS`
zj6r&q_bssuz!b~zNTJ^JF_Tbi<|d@SO(x|}jCvJ4?X9AxJ&IJ2P-?#YrPS25sEIQ?
zMEM1*Rx;&><h>3Q$xMXvR*Co^z=Klw18Vz|c(!=wuSa;$Vh=0!snSi+UH5yV5j~G&
z6fgofU!N8Jf&v0{4x8TBLPDqZeRw}kF0z#YsEW^et;+6|+vP9UaUdewxQZ%n92R90
zjGKXJv*Xj#h!UM5^J?<3pgf<yz=*vfr^%e6j=wdNpSmXg@Cp-uO@p!C5z*4MyuWNM
z?;Gpb*cdP}X$J!y@9TAmZVa@*+4X^9`OrG0wGN7-wa{^0cSx+?LGN6U8ZhvIMWG@<
zf~Vvm(qpIz_BN|r?FdTC6f`T?`Zca}XA8Hn%Aqa>h%gu2%`EIvGfkSI{=Tuew!FPF
zk$&N8_%V-C9S4TP*izb=u6&z)JoyW>u54=g*jeuUE6#FfomuY4x7R$$>+^g1&wnY0
zmF#tlw>%Lywy_HrLyXg&uzn!&JWk}*k18H{u#86@tjQyrEIsSIRoG3*<+7G{%z?eL
z#T57{$ALko6%J4s1NxZQ!sCdj7A;`WX*^PTXo)fMlvWBv>+wdK0*oA2_vMs_YY(iN
z9$IXhtv}_|vuXii@j&hxI`7U?7|=8xQQF>mSBBWR*>(H%tt{JaVbz)g;9`8AjO$jd
ztHV`##GzOgOc)ruKSZi3${i@JkuCs5NkHPYW{4wX(ilb}q<VxYZgaM2w7es`n1z6r
zU>w>?D3(2?84@m>#l@#qoK!vJ%Cl4uCTMZU_i>_$kU`7Mlw8$%1<$h^a0%+(lun?i
zaeuSzp^_4%v(y@1haWrc|2jjya@ZfNVgL^W1EBRQ*PFMu)TGd9o2%3YkR>v1A`a4Y
zNU2#kkQ?l@c-9`DS_sIGF(r7x+@&KL1z7Yg@1_388~0cgA~r+$ML2pdrh}O#oK6Q5
zno$T9%=96)PPAbVk=Wim;|OmXkSIavLpcPfFoHm%zTUGP#t%lW&;8V!f8lhrZP?+i
zz;H^?dPciHySD$Eq<PtPYj1DwT50}@R)<^ZYqxuFt$kf@gI~I?zh1Sjzjm(cF0NtD
zaC=}aVqQ>m1c7HeWAFDQnLsi38Y--t!jdxAkob$%wz-&krNv|)RIn_1&^{aX5UIXE
z=Q_>~B<Sxg$gYJHc7tg}g2~nNv*8{@Z8^<qnC1Z}WRi)n=;F*-DkJnpDvF%Q5!8eQ
zXXbL;x)S}B;r{s4m0H&)t$kmP3cu5TYYb;ez8)xDP(d?ZU7Hh&qq^CS0%dptI8W#M
z15u@Ebn#>z5$CTf*5qHEOHyyEhb03Z8t4%zX~I)MP1%l5dovypUL>9)mZe@*U26|H
zghizw^*e<+SCW*kmkobyc)@5yH&*q$HMPBcVW2tOFSXfeEaxd*i>DlZZ;tCqb>Ne^
zLOD;c+bgKI(>9JUfrWc2?hcY*r!96C462<ir^ZfOlH<*sPipv;mW=CR@c>=KSLs^}
zKiuB{=zcZOgZ5P=9>IhO5$C4D`EfZ7-fTN3X9H<OR^iU*1h9){F|s+Ghhhtg&(86y
z#Pimrz%6e{iv`!D(gb$MZ3UWnV}v&sh-GzaKcKuj=x-}Wraw4w38AW7xL?={_X~R=
z{_1r1T$H$l`*%1=qi}qw&tLbw#hf;GIzwAfuIJFmw|tdNQ*Cqx-lcw3@`+WhE}c%x
z`vEB{<}|_M@eba?A3MBX62trD<@Zm7D%Vs_Fo|%2nMHh%HB7=3B+|b}CV@f&rEOuh
zm=2>9`4TZDz89Qwv}U%bHu*d<y5w^-&?x;BBe`LO|GDFqds#x=lY6PFtahX!roS#X
z<DASwG{8&EGVS1ek<OD0m*$i0sdtv(J5+VYXd@<WjHAn@x+K9*um>&RuM5m#jSD<S
zSCT|g5Dx0{L)W#BWtRFgi+$1dv$$Ei7_iaqzKcH_{^Pf<%#|A6)ZytWv#|Gh&%!lM
z!umHkmov9jeq5dIEigCaRAGh}J^1d+G%VQ6@hPx4gc^F2Fg=61QydemO04A=$mZ39
ztFtrMltoHbWH+#lO(vJrL&F>79L5Z{kXuzH4%*AfaP_k59;{e8pWY7kQM}`mFu}?-
zxXSmfvuCV9z3`^tC2r&CVkVUkm3Kogzs-DT9SSu2&vv?nb#iUGOR1q-afc8J@ZRC_
zmGa3p%j@lGw|yI3-O#O~aycw@AEWiK-|}8AhSNB=mV*#$Kk<6Fh7;FiOX8*v{65E3
zVXBO%07SE7adxf^`#=YvG+-Jtq13wk!>)Nn?SFp5bhBi}Lmz@5M@dbS;b%+2pWECl
z=XOgS!SLxERGJ|Lo5S%UVVcvK7O&W>I(d?FRJ<aiSQ=%7qAQi26U!~nC;>9cGzw?b
ze8vbRi(FZZ)WU4FTC{lWs`E6;^Gk0I`hJe}TbmMfsZ$2w>85U<eTsviPS-p{CJR`6
zHcL?d0mXq3<lT&w3YkdmpVZ*EwuHRWtE+RkYLN)<E>J>UQo(A=J(JVMQt*~YSVYPp
z&+H6(&sMIh+xc(NbNJSTf-5M3l?1cW3#jwJ)TzkgphZalm1$R09PwhH_9}bAL8tq=
zf<#D-W{|f)DNExO$Os{K^djkHrFTi^Yt3Ln_Mi?Y+9g5XC?PA@BnIUq;^3Xhsj)!=
z!i&*4`(4W%o!$t(RFRiwJ8^*8(UbKkl#=(AE}^fw>M|5JV+lcSVyqX0Rx%4!ol-Ax
zM(KX9QOuzmUzIvx6gb5`Sh?h{k6O<JxprJOv!-W_^-Uee%Tv}0!NH!(n=vPMmiKKv
zXlJzPi)0-u7G^Pj$Ges>JjFg1Z(wBT4iuvAcDKFao<(PskBZm%E8eyMDa?EM^K+55
zc;=0EhG{%{?G5paEKK8S%YdZZedqC*z1mm;9Wm&SF%c9RE%JA>9rw|zCy-z{lh%!~
zFNr+9W4?kxHy2r-&|($Ks^=8lO~E3D2Q@@#H0G0uJbD*SLE^e;J6H9-b5kpKRM{#T
z;dxwOX$(iEzqAfJgRwbuqHqd+p+-2M5vF(2r;z>-v)c3g(P-p<`0(Mw$Mb=aGnk5K
zsT5KL+g-i)ngL~W_?M4SEk;++Eu8#=oYL@bKD<h16Z8dnjrVlX)$HOtnvrh;h=+@e
zlvb%S6zD%IaUXm3uI&0>wYp8mDzVB{vT^4+PhffS>h=n_c>$>3fjChwek1vFoJ7PO
z$Tve6-AZ<~usg@$-o&YhDlA(#n=MvFuSVK+`LmB}#~2xG99H9I*y8f7!gl|!*NQE1
zip{vAo`9y&JqIz6<t~_?6}?ww71M2p%FfR|>cme6a54t8$ay3!h*RyRb9@T?ho*FS
z4!%{aX&?u~>E5)Z-HDk^?4bMDWwM~d1=<eTA({1o{30GjO}wdyMb2;w)I~C29sl{y
zJ9u7s^fr@M*5hb5Id1!(3(*Z1)44~?H&P4+)km>JRX_>!pg^;Qx+}<X7H(g#%Z@GM
zyp@eqrR^OMqtS^pDe4uz72FPfsziE}<*D8N+(`He`pTLJF}p8|L)8co*S==ja0)Kp
zQI6va7OJx-nr7Nm4oph9OiEN+-rt{qfJ0ZSK`aLW3|-{V9zTp=SC7&6j=TT?>TE~W
z;_4vxl*OAu5U5oy)LyqOl`HA1gGM%SN`Z6?Em7JP;IA<z=g5sS4F0zI_(GZtDi1IW
zorqz&b+1g_M>oW?O^+QEGH#uW+rQyUY#2nD#FTE^Y1Y_S8CvI2>qK<1DdtbU*daII
zn^MHm_P<`Am-M)M&3=celOw8L7x<ys`H#HOv{WHXg+f?pZ_{HZeAZm&2TRwvLzXG3
z^XpB@%3dE_Pp|*%JV)Ez9|=+JW|w(!Hk(Do`7>a48|%+EHwfLm{m-2HEnjGDd!<o#
zuJKsAyE=$wx8)rtUM8H~X5P=wpP&TQ6fu0*o0{`<ViYC(oQK)#mWPsJ2*a{-5njq#
z9MzzEHZ(3>nzhzE1n+)iMAHJa@*(z3T8F!z;i=@c4qgfJ&D~V}41BAuc?`?0X}3R&
z*GY@*&u1Y2gfnqB5!F|@|ID+|U4J%~@82tf7I*io{hO>Tax<>SGiYy&$4jtL?_f*>
ze2x3|%zHMEW<Nj2KZFw!Z`Tdp{L#dtTs%2)DR208;|eB@NStCz=;T>-PpT>=N#~kE
zn0J*&=8bv&e#RU8HOn$r>gh-LONFD?I*q^;k2Sjm7a-d}c4JhrUolqAqW0MN9O&fp
z9?Pl~RvOFhXCV;sz}=Xe<39;uEqU>Y2JBN&k79ZOalJ|WR;F~qpOhXh5Ay{^348|%
zcpi;Ef(w2s8H++QyC`Iu0Fxh6RxN18NU=5WFOu~28=q_+oM@}zs>R>NG)@44z59Az
zy*PM=+h2*)M9Wo4yHxS8m=esuLrrbd%Ic--mdrN*EHo2*$$VN?wfK}X)qvmyO=7ky
zNd-tBSzbtaFFyk8f&6`9l~_XK8gdy~muRX;)g(xo4Q&4XHYd5??s?mtPAi`8v>&$j
z+U@qic9nUKYEaH0tn^EDIcf#V5RFuqlfKdc4i%JtzfJ1^Yj!JZIp84mkMfPH(Ne=&
zOB@b1fyZl_&``QlHc4`3KVr>ofB0}>PqM*SczT+})MBnZ*F1anS?>2s0TU5u$)f0{
zf$oFPjt2HW1r1#1^mm=pUjfpZjIZw|<8~(FCfA~bTwaqUE5Yro$@27qra%o%hZ{}r
z9Z_5}Q6!Tw%;2S!pL6xL-Q9Edel3F7Mx4pv6AO*P-zRC7b-&(i-~W191-dkeHR`I0
z7|X9474=GFS2bhX(|J5klM&HY1r}6J5#Z87irzXaN)f0|Z$6(bgI6TPB}Fx%NlVw)
zJ=wSZvNjeoqMC^27jHpb3pJ95JbB4i(cftT^D`FdIq}$bO$xk8o^*CKo&?MVJtr{)
z9Ol2~ydGEKQcI|&T<u1<RA)^tg^&6pUCi*ReQ7UQbE;lcmn*D)=^DPHP710xfOiw~
z-1+*GOmvtI<2()1OH4oy<BR~p;RtZq@-TLyeQRn1aKj{-05Fle{%X)}lnOgvuOLJ9
zgF<T(0je~^m>_s(xX3P%4~Z1Q10c$iW>tD%pIU`q-4D&xiGh7hjlQ|}_SD`BZzV)D
zclbbl@bhcjoMo@Dw81+~Pv{=8Qm61<=p-dhuI$xjed*k2UmP=sZsQzyW9EuqyfRC%
zEA{X)YuF8#v7|9^>7|+qc?_RTOkQ?w`AzI!a_e^5z0CT8otFTJ_n)KXtm#pFd3i6a
ziPsX;Uq!Q88KC==s><e7UBg1cHPOO2+(u6IPUo+;u#HbWBgcKeZ!ww^(<Ugvb*>1m
zV}o<_$?qWTsdieDuLQiyXZ01XB7WDb-*(owM#<7yOEK6!-nd<Px94+%^D)-@HK1Pm
z;il7;e_ewyA7*@Lf}EnrI1GM?9ILMm2bxPck<D8+^~!|*9##NqK$X8J+(7Uaz6Fc^
znr@k22lJJVnRm%%mV0@0{~WnyocUZ_IbjUM`XbeUh%^mIafG|CHT<efF%>CdHUq9(
zi7t5Gqdix(lt#mFLZ?%uJ7&&2FDo`yXk}TE4#*_Ytd3|bJ+rlYjKyI=w$7qF00xxS
zeU8euCL?pxnls1F=QUbw=V+OI9Y>RT=W|7W?aAJFrWG%ywOclcR`Q@9ks5#@Sdl8>
zZ|}a>!JM&?w+oDee+`p&S4kMXEJfjFjur}8xxhPz4zv#m+w8g!KX7|HUIFv&fzSMC
zuWh2;Jpiot+8%6pX*dP6u^;dUVeRdBdmK~;ZyLZW%0uy2@ew0zuybN`g&2q2zrU{4
zJw;$>d5OXqD(b?hQ=&H5{k&1QI6Fs|&Eeuqq=c0<z3e=>z<b&u{A36y`r`#`rR-B!
zd%Skb7jF@<ol!CaNeQ&=+}vKJJ#HavC@h|%+HRf)J@l8{F7}m853BvMw5Vm7^Ou<C
zv9L5d_?=}iu^^e()+xyn4iqFewdG}*=M&WGBle!MqEn2nRUq9KlVw#De5^uERF*E9
zY3--dvad-1*!JemU{#bpg54mQ64#A~(9a;5A@x)K8|A@LApOlGvVAK#Y{BQP&D>1=
zv#A}(kk_jDl0EtQv&*Dot0<dUPns0NFYChs70Su6Hm<>NJQ3shgs22B|G~@SXbR+>
zTeSAUn6Ax@DVFx-ve-C{EjMgqF$&`()RD8)O~f0v0H7CCC6k^d22}vr!qlxV66hC(
zcUO6XxaxjG?w92uvpax74lM4|h@yL~fx8(KOcK*0dQnF+vZ*jHjkkTcsa@jw%n>TD
zo4AnK*l>+)uzFQOuOjo4laMZE#9-^Q#WY~C8GngsD>&*|e=Ie4l$M+hY*r0%jb&~u
z@k#@R1yi~QnhuOhsn+OyDdx$(@3x>9ZuKiG3pqk+cwRKgB5z=CebpX~#waXfL2e*=
zI;A3_xr|*mjq*i0E5|D{1lQ9z#pq4(l(L4&-Z*jxl&GoSw&<zfHRWLug@@o$!Zk{7
zB=X#e4Y+7X#`#;>iYqzdmNx>xIkhxJiIf%wCtX5`ltgdNjvx{rinD}POUG76^YskK
z0>#(}T8vVFSXTpEJ{BTQjyS*nV0|M%;S`lrD1hLmtvE+;P`q9MWCI&uJv>#yzgnY3
ziV2jM^vb=qtG4FJysm8P?EA2?=+a^XUkXUL5)do(V&`7wd-uGV8!2&)6OPAh@zEB4
zQ4LfT6x79a-KKZ{zHZeqy;r0OB>jxy{0JI=Y2{3R9PCiK4J0#_aTnw9Uqo>_{?hvv
z>Rm)CY7nRlxN<>n(G){z9bs5Re{pj!^w-qLsq}zPf^jehq$c1AMX`wE9QA~vWa<;p
z55+pH`OqY(bLV22E1zPvGU%EZt~#Ub@P%l6<$CEdLzO4AQ#qjsvb%JUJB2~+uG>>o
z%iBg!uPtR*jx+osm{>+D8qw5@3gbbFn2?leCkIvxZ4x2D__v%0Eiz#Im56l8<5Xt-
z0-Ts_n1{*>?{ZJZxvLxTRa;`)*1Kv8G^BYD%}{N@9gia{Uum*F4Z^n8bItfBr>7H`
za?7;#-Tysddf;>lw6%@$1qZE<(rG*k>DB`7KSo|2W=I&eOdOnW+~GOOSP5n3^*|bI
z%7l6x$qXchBr`Y5Eh<Q1@n?WB)&%j))8ZNh!YgDD)dab8udN=bD{j#s2%InSadI(J
zaW!|ktBtD;amH3H#&L$TAFxDvwPRXkPdSv))~oLiuCJJEM#q4C^~ziS@~Sh&aCy`?
z=~qC3d;_{NeH%&{9o`&#|CLnF_VqAd(?PK%eQa`#lY`zDBV(_v=<Q4An>#I!pPSI~
zocGj3e0^7Y%R=gZoch=;2g=+`X<Y4$xLNaeI_t;ktov8iS-E%An>9AX26{g2SJ|kM
z69>Xl6`-Z+=Gn4}Mf}x%nJwRhGtV85zT?riz@rcDm?S-aS|)i=!6c;~5zv2}cgD;U
z<ww#Wh<Wf>P1_;uTqG8xqig5~8cLeh2U1p0bDLw_dh}D{GFaLyN7@+{*1FX(1;WSg
zsG<=dY}z|ZP)o!hgyAXjosMB!U>l{VvW<0`(txwgwJ`$)-MI()jwh&4#_Ft{Dr~bk
zt>qku+d0g<vVH{oWu7LjG@2%FBV%q6Bio||l&H~b#j@c)b;E+Po*##@6ALp9smkHZ
zqy-vsJ=s)%XYw-2b%~N|7JDqfzFPLhm7luSCx=N53St8;(^;WB9k<pIttE9f+n>#&
zX6mNpv}Y&A5RLjGpQrJ+u<IIEUkn>^-;eoFm^M#x?lv}A9OETz;&;`c_(QZN{<2%b
zIHh=1z-0*R6D?`u+h6}`(g-ENL9<OFLd;`zTywD3UV6!73^hbz9OiFmXko4(Q<mmL
zW@9{xAUo0UvS6dBV?x-obFrA2n~U<9sDR5`Z6T?VW5SsBRRdO>DEpl;e9g)k+uJ|T
zGe++8>y0V0371`dePN2(E@5Z(*(GD$fcrwc#DAfwFENNaP&9N^aSm2K>mkGxy$(XO
zAZ{2Nri*p4QA6aKJ<0%#wsgv~6!TVygv4hzunnKXs?$QnHG7Px9jVEC3{&#Wd6D40
zH;_!r;GNb$oixzW2L-44@(gZ9)8Yl2ljTvOxE;XJ3CJ3@rBhQqFKDkOUZzHYF9euY
zZ#G#Qjd|ouXjrOS_CDXj1Rr1Zz3$ztf09{GBRbYnGm7$%Ez`obbp{er>z{0bZg^ro
zVoGrM<;dgeQ2Iv`#Re2F&O;O>ve9`oUQD8~I`)P}SiT<P@wNBDnpw)*g(uul4bxW&
zfrN^;Tr!7s)=YG!N?qTCeoA7xL8|&>WK@e!y&%BCK|q%<11J{Hf<WAoQYn`3EsVuq
z{#hQN>eJmU&ZC`ClDr<m|FT`yId@OvNtEr1BmZ5N&69a69d6nNO1g)K`}l9?!9m;n
z7oYw1{)4}C_71ua9vmF*9X$9;y9=-O{^GT7qQk{!fh)rE{u2JaNF%)AS5a=&_dn^g
z_2up&OLvFyY&V*{_0q-gvR2#j9wqb3G{(yf^-;t7IUc1+mYnABKAk6Nh$=DW>;m}(
z3lUq5QsRhP-cP_VXBj#Y&BhVuocsC7v3w0BYBh14!`uEXOJ>{H;KTH6fl+3%+QV1J
z&t4usdmbD;efaH9M~{Jb{o}VN$Fo^a7*W=Ce)vZV8Sp5~@O2u`X=&%=o)2xlJWtYm
zXB4NSMVwRK3l!X<G(~q74&x}`)S`D#q76NwBx*$zYqckje>w_|pZt6jJpAtX=tc13
zGuUr7M4XrtijM(V!RUVfKU#Sb#0L0pkG^~M;^-sc4<q1+k=X!FIqq+CNi>Th>_14h
zAiyp_pO?198iAN3v$H^CdhL6ZcOC1;voR&Zq$}-0)!_{p_uUAU+BC!)u9$fcO)bMH
zng!t~hph;6U*dd3m}qxF*`d$UXwJDvcBWp3SokFT9aHe;qjTEQk+3GonMZue8v*RI
z6!v|fENGbW$PrW&%`PH#X_MY+uTek%G^L$OVMyB}Z|vdh5=RI}IU-~TkL~|Ji~ls4
z{>P8W@IP8#{l`nLvF+`?4b$Cho+J}wpu126M(Qlvz5frUJq@xoiDqZev2VO~Et-u>
zgO`)=kM<Mc;b=JH;hfRv>4D>4A{xZwIBlIxl3{(@fzKSa{aq}$dw;vpq5}cDzJZZ&
zz=7#V4QVJv8!POnOm7S31U2#2Oju2n!u}iN(dG5O@n-SlL#@UO{Lw?8=*JI_A42Kv
z-DIBc67B8^g5Cv?yP9-~O|;N8>i3BD>hECATd*jz{6Fh9JlxcgA^<KGLc_bGh>db<
z8UeX{S^x3nv!|_jm}XHOVStc9qY<0KZPy%>dWv!%^t2XDviNj$=@yh-7wj<_bb)kv
z&EmleROEufTCU4@JNz@u>#K1<coZpO3kc}+G(w`2nye(r;#ywFwd^UKw~s{&%i4Tg
zT+%+y;x4)C7_85(gi|EO!RAfY(0n7&lobB=G-j7*$vxv4aG5EQv^PYqN}kHmJ06Of
zNmaAxqCixj5MB5UD%6_es4*>pexF|kXVHw^rR}K@#ercn3w@k$hE~OHPfNXmj!rKr
zMgZ_7Dd7GC-e#A;?x%Hs7-qmSCnIkgR=qgbNxZH1T<gO&8Njni6wT`g4WpjiNt8d@
z4#zFxCGe@ml)LAKp0(wFrr0aBu|ocDxA!^)`G2q7Ik=PmZ{ztl=+bEJ-S6#+95UkH
zyA}Vh@vOc6$H|-NS=!3ZH*aIb`rq4!#a>wd`-gY>|E)Zr|D*1YQp3%w{Uqn3GJ;Ja
zIe9xHj4PRY1SZ9(o#9zTm&Vd0$^YyA_Ubxs7EyW`aAtI&1W}mW)itFhI1Tvv8OnWq
zuXC_ntDP=pBl?>iA_95ksNV42*S>H<_VG%y3hv<jTgUglfz|VNcea25^Sb6A-XDK>
zzgEE!J{Sfhk6g9>t=sfYC*fKD+^>NiRlv+sQ42)8iBbWTxxM1Mx^i7rz&QA>Dq=X#
zR25O~6;%bsaRy>&44Pi4>e+me@hA7}`Z&9EOzu%+MjdM-5PYnqhp8{E6#kWg=^+z_
zkgJ0tcRTb~5V`o)1SG1nkL<^E&IWYlj@e*JovMH$NS*<m=c8+zID$n#Wz{393QK#<
z%GZg=aspzz<P`zQk*13|>jAk2(s97KVvpQ(vq5i!rFOGcg}F~w`qZ28CwSJD|0c=V
z83%=lf!Z!NX=8=_*FD@X>VNx(hn+k5?>3&Bmif@t7Q<JHi<W$kGtRuxB+SUZj3?_D
ztPUj5c*H)j9-mL3x!fbU&YLDEr;(?x%xWHw=sKx9V{TDP@f1+bJwdC5KU?SFYz)#~
zCf_IOS9Y1zYBeYqOrp2ZB;Z)QPyp{!N3*x}Z9ejO_~JPD>Dl+ePe=cA^wTyveLel|
znctwAxLnq({@s(Gjx>zNN8i5s-T<<yCf3oWaj1pa(Bo6jMjt+0L@CItl#OiW4g7I5
zT%4if2HH9C--|FMXB#^4!XKkF&T(A*6F&7;5HZZO7(~LBY|rH~fHOt!5oMc2los<j
zqNLF+jM;VJtii58+$y*}_LAuEfai?dDrF-8<0pDo_Z4O0CI(tf)>@>KNj%(Zjgr|c
zBJ~1KXJGaU@X(j}Cu~=u()#<u7f+$xEQeKQHun-8X{SFhUDq@Es}IE7;*t$f0|9<V
zrfm?y6j1%fzgG{l5gMi&nRl;Fea0>~vL5_7jj{~zZ)6yXNdr_E@)$L0CIJ_hT<b*w
zg9~radM@#m<?%|&yKgV^DC>8C_mlHH{d3SHJN#nK4)zCnja*}oOpv-ynliX!h1qRj
z^bS4lJlHFWNwDsMr8y^hy>U;IwfBFF{|+BE-1_NRvHu?&wA)4d&*7c_$L%~fz5ly+
zWWDq9$y%CBy;lJCaf~5k_>26Tzm5P)`DGkMbNstj!!424Nr}(n7v!2xo|@nA4ShO#
z@#5JF;{*L&tNrro=+)7W&%Q<1J^9<OJwJN!<k{mVkMOPh?bl%YetKL5h>qyd`<GOz
ztbQ)8VF`EN^BNxaokp<l;Gs6V7vfa`^lRXVptW;Yi{hM+*oF`){Nj5$90$!B{Sxps
z9lt;S_GNJNzekV!o{%L*7vp;4gFp<Bon~ju6@mF?XAQL_QYYYF^CQ$^aV>Ze&C{N`
zcNUN`5CkIoEnb{8Zkv%?9=;=ma-Np&tpz~}@I-y1rWhmzKwwQuVclp&lq0P8y(q~y
zc~=&tEC;L&_VETIR~8P}0fuM7kb>|(0YsN?g>wwPPT%zC9KDTZdFAkEIBIm*y@b!u
z#kdd(5dx|gP$%eh0|$&!h0+MkJna2wXUS8I4qo;6F&Xx1g+aHlvO>M~pC+@&Kyr({
zZD?H&qx0}>oTMV2irtZ59Oj`N>5JaKkkEgM>BL$W9u$JiqI-iZjfD+=Q}m)R%i6br
z%k6-mHm0+7vyvlF3akn-2(Kf&NW&XlaSTjiB+PL&)y*0X&_Mu&U{jr{aYo5<8Pow5
zI4Mq5=77@VQ~*U<GN8avIzpl$M6`1wz|z&waxZn!MiDFEGI<!!GKCh5Mi~~oL>XR#
zWH5p{t+|%0zOkHE?}zDGhSAVoU*IpJNeQ5}qDB#XahB|k0O1*(u*ptz(rDNPt2ib|
zTdU@nNDJRs3o#=<<}jXs6yhbL(IU;*I}X<-FaT>nwVOEKFGgtW7M~Les6c|4Je}w4
z4u@8;d!&kQ-^;c!1YKQY;Tq&>oQu6%%Ha)&X1US`<yQnj%#;!rAtO)2i$F7Bc?Vcr
zgbw*8!0<N=Vn>%1iS<R9jwW@AHp%*UF?x;XF|b86WK=2{Y>FlV;Sh~CIsXQ=f{t|P
zfEbQOpnJr_NhEJ5XN%!9#+Yr%YwhmWvQQL=V~{r?`KZ3ocm5{%z5K>e3~eskr0F+f
zA3?!^3@46^!>TOa<4wrax=q}$>*@;+rRw~ti4n(*N_75HuE4L&I8QaP4$WGc2J~Bb
z-xBr23b<AgV+8Or4|?_UEmaECF0QW9<#t#ULnCIpxh3Pq{i)^mkA2qG|4_&fF$OnP
z0hjB4-Teop^Z)(5gFFA9TX_us|M_Kpp3H9C3qUAwqnC90`{bF_?<NzriDgpFdzJ!(
zXiPewQWrx2I}H|jJjn|0m^kU$j{Jg}5rZ$6k*yrl7rtm+R!`GN{sQ_RjZkaD=g8Zk
ze}Ua1Yxgh^OHB9Nanq$37AP&l-|1r35|w@^RXUF*WBK(L@%1<LRVD;P=#%&ikGCk&
z>SBhk@X}hc$Rh>dMG1fm;wuUqmkJCcM3Ha40pwY9;Zc$*oM2Ju@1;r?q3S8c{me!G
z#jz@D^jz7i?}T15$JY&PfSQCc4bkyJw(wH4P=7UxM+xtf4cT6{$WYO0)dJLLUmia^
zK6*(C?h#$6_DgrT{U0b@rsKmebS>NQ;RSw39UuN)RpTXpD82{vn$Udu>^S)D*{i3I
z$(v-aR{QbUw~RaJ<WG*-Bmq4z-hcPx>64c~&}A3@JM+U2M?cZaAIz6znxNOen6F>F
zdMfN4&?n>l%U3U-9}zZyx$*w^#lxpBVR6FyWApv1mq#zmO0Ue1e}AZY@OR(p+4Cna
z7^(mDl@bcWAz~Pq>kLG&Dj6RbQ813vK8vYPHUr`*>SQJ?TjVd?3KY-)De=qxEWo3_
zBz}RYA%9X_Bi3HTtKuC<M1K^{!(lvuB~gyIBg6!E_7tCe%`xe%KMT=Y-am0iB=AM^
zP}Ib_qOfvmlF<o(ag~#id@v?J_O5_^o?hB2k0=*lxK}(eH&}6tF2@FVBv&8!c-oF4
z-_DN2{G`#!<}gTg|JRw{Fq2vu<7C2@#rsgYbHW#BjNy;ojiPz()X4muj&F1h=3P)0
zng2_ZLsqr;)Fu~TJV<d&0tuer<mav55&=pU6CibC9G{~&juUYKw{pCg#t>2e4a%r~
z1IO-{!k`rqAG{M^aX58uQEF|_?VWI9Q892Q8cB4Kqck#NShN!y$7G*|>FY?YKimx_
zA{l5vj&{YR$i>l1Xp)4Of(HL=VB_KojiHVqo#kWLwNRtLGZRcugW-fk)*}MM!=cf%
z3_(~NFY&dVbFaYExfz3CCPrnONjo7K@yzszp;{T{OwvJQ0B-^m0n}NLSze939&5JW
z7D<J^>swTQ;R+Xjrb}FANxiz~bohn*ER9;DWI6{4zMlI3@7L_UQefbB&ue}4Zw5t`
zRyy8{6I4%ktbkUG_8bxV>2@Bn1)#tLHtbd9@eyo<S9@W77myMCYLQA_#~KwdX};Lx
zY;utvW`cGyl8VasPb-bs;=KK<zg^A)U_Lo$O|q6~vbm(VnhP{Lt2L_EFoLVj5hZsO
z9R?q`y!!p2z$3jQ`Uf{Oyt&mYMNb@!n1t*8Bf9H}HX^zK=iSSi-U7q^sj%-x=CUKY
zC3^udIcI@2<7_a98m_QoTO61{^g===du3MSadsaEBEG-KZjMH*baXDG>EOSdE*(E4
z^p8&DRmcUavkNtK(gyJ@p1-9yLg<F-pL}qy<e<n#S93}i=PPXsxFHY&q@5r@qh-Ae
zx>4E<+-x;K-6tSWdIMmCQw8^Dp`<CTQfRid4We9t?@<%xg*386I^)2ok1SpPWpm_J
z(1r^<&QhvkOa3UVV4Muu7V3O0*o($6Ge#$@^0?AS!I<f)WrmWc;VeT812IS9GzjC?
zpV|QJ?!sq0Bp{kLFomk|cx^#oF=ROaqtuiKwJZ*xEzE-a)3Zkpf5O|6!IQ_qe;@td
ziP>#={3L0(KnCq5V=H-6qi`~T7n>_6$U&`^!A|AbTH3k;ib;jYq_Dno?=}vh)B#&&
z%AAEI8Q@08#RmSfMLvgD&*7DSqP774SD#bC=p6;O6B7_DI|~po$W7tMXi(-nVz|8k
z8VGpMHOfu1*@DrWhIEOXXPqq^KYRSFL{IN6BwjuO<urKu@aH2WrCtHrhXQ%v!S@>E
zkyRZ)(ZP8dVagEn{zn5v-CP{_9ao19%*H`fzezOGNmhYH0TWg8h7cJ?lU2h}2?O?u
z>9`sX+EteCR|5cW;(b*$4lAYh_dXLUDy!Lo4oG}MjM-3fH=zNW2u_m)<rwzIQ62)$
z@E*Ej!Y_*V4nWG=umXry0rj@6PW|I?#hG%2=66cE^M#LNS|1RA6LAIt`-cd(T%>k3
z^^sVz^lTUN15u$BjpKhwrrJsUVq<KWyxz9rQY*T;IQzDbeAgDI^uh01)F|PaveQ**
z(I1>(Y)~bUT!T(&@L1BzXlO{7h3OYs`|IfPRC+?1VZ<xuNyZ7NOE5MB!@!|RUbWfc
zqyZXkmSPZ$Y?NsxBx?DNVgzPnSt@Vq<S;@~Wrw1_Y`^6Dei*xAmT(*kJPzF^Nr+H!
za{%mD59Emtm2G@kCZjNh1^R;HJdhAjvI(dk9G=0JS=n$g@VU<pp+$&4a`2a7PB7fa
z$C=9RtEYVkdjns}E9lp<2;gcIcl1c#{V`5&3>g?jPztatwm%No#~~9l)90kvtt_Bu
z+ZbB~ids3~WE5`S)|st8ka|xiz?#u{heKGq>=I)5RCDC(bH-@$?}o#)BOi1Y>08U7
zo-gK;s6Hq)O#^`uN7T@s6d()d6X6Lj#<h&G_%ML7C+x*m^;P<;Aqo|p5ZxEG3KAu7
zW0C7iEPok&#IzZ&q1xE%Iq@|hSCVQFZK6El_{|^HEE%P#$q{el2$$)PEf@wCs`YR}
z#zeW3+X9+4;uso7LA5N{Ta-5jY)WY0>-lU}9{BgNzA&V`YkHT$7zDorAaiM)lnBKF
zD=0?c=c01)X<$}#kWWQH!~0SDj@WD5EKfs+GNFZ5^<uA@O~2*sM)_!0L^j(2id@WF
z*}3=c|L(n^nL;O~or$za`As>xXpQ=f$MiQbpokAfn=}z^m|GpH+<|Z;bvbZuc~3<!
zF^~z-Y3SlSni-d{@$ix_DnTjo)cOaY#sJL}_5aYDBYhjA3v|(wQs$*W-yu4$q8<TR
zSdNc_Z7#a)+7z+#WZh)#--c+YqrD<a!vr7&<7S`isYNQpEEb|XWvb|uDQ`<ejHj@2
z7|lB3eHrPA(*a^Yx0nc+qV|;_Ks!F8@9Uo6n7IwY=&Bgc7R8`aX2<FhUQ@mtv<pj2
zw=&R0PMFuzU!97r$GlXlfzVE}t-ftHJZ2gNH`ZdQ@|;R3PL^pJjo93q$(fB1OwIJW
zA~^{9OU=$AjK|2HYjkgqJk|f8JGt<WxY7;3-in{P*XJS15Jpu@l!SFS#LOS`H}YFs
zZooPi#*gA@le=6p(@_Ty4(D2uYu1T7^@}h^Hvyw3sK-%qO}9Wqc5kff6sQ!`a$RBk
zrxM0Ga<4!rEG_({BNy>)AhZ5*l0`IQi`(;P;g-Ko%Ji;q99Z3^T&l@yt2ik#sy&<y
zYKNwWZr6$VS<#G`=%r0?62}ZVU|r-hEHaq_R7ndZutmDplOjC#D-E`MGfp(i1}~zt
zaUw353dg@m{5tZ&@mQWqWVkX3EXME`eg`U4DKa_z2W(Buu7W(lmcy~tHm^s*(_Mg(
zLn0w>TCyHHc7Icu!g{R>{89%20!Hgz_UlZ$@3Nmbw`><C4W3Sw1x8xgtxFwkH2J-6
z2=Sufkb(>j;A34Fg+Y>vr!35<9>}vmh9#}Y=5e-tK3;VWDApx@D=inmWU$k55GZ_*
z5(=SNK8P!#%=B7R$>UQ|P#mk5TQVlfeUfJ!LJ}z4Hp?e_04C$jM2=9)ae6opQ|4--
z<26UOfJsmD007l(Pj>Us?)lm*?ct&i{D!=MTZWBYVhjS|6OULAdd?Zp=mxA4Nv!nC
zC1<^=o{(huP4$5<e;??I$Tw)_mmo&LC5rVb#(ZE5Dh++3e8oaOinRJnTGG@PjyYG-
zm?D{vGp1m?bF?Dzc-eE2W03QUzIL!`T<NapCjP&p(N%7LuJZrw>^(Rr#sAxDAKv-@
z-o|qq@gH90=pc6~Vm=5?LXHJ^UqXg+@U%LTk=bq>g%d19%9FU7h|*v@=6o9%n-+h|
z8!&i<qUD)!AZCl{&>8Q+@HQ^qP*u(_+V59L4IGx4uF+7aSb2kkQ=1bCP-j9CiSVH%
z9x}`*bOVNp@F-zqOI=&72`7yw)D~F_t;WTrYjoX^coe_|0T&H|x-Yr_Y#cudfy3#M
zN}oQlSL(M9k01SjZ@>NIzN(L&J^k*<_rddr$3JlJ4eju!BXRhT{_oLGM~{Ab_Udu)
z?XzdcF9D_JxNo+|tb}fs7><JDNt12bl=&=sf_)R|dMybQKq=F<93eu9CQH{YwauXH
zx5|oAmHA;cMB~H)7d@x(U1@{($`lT%4sPdq!M3U`%M!M3=}&9epf22`O@kU(ZO&kF
zQt~wZWlXE|yQD%wmtf<!Bz3OA9TCoTKHKLQ0g32|O)Itv#6iKFN{}#<qu?YU*@IRZ
zEJETdl>N2I)zFqoeY3Bw*NVmz(~Kg`5FcSa+0b-yb+d`YbzO6b$x8}m5~qoTW!EL9
z5jE9LlgLM!L)J8fT-6M+h6!W?^T#UFhe?e^lLwO`UcL#)`O%DdAgB1~U5H{y&-46x
zJl|<QZ0~{C{k2YsYLzxTU<Vyk@8~U#f?+H--DE8vTY+ooXhfU|pbMMY0W4@vMr~Yo
zC!c0rzk*?x?<|PAJWglB0E36t&1>`qV_;Ei2JhqXODK1w_9jCzGboe*sF+nmLlj03
zD2HQaI#gqckX=`&r7_DID@>lll<)WJZ#t<YmQXxGpJ|F4tnzM7bx-T&@wJ$bIaINT
z&NY}l$tH~BW%|;TvUuw${abcL&^@vyQT=cZoXIcv1ME^_vCW6Yg(gApRR#KM(A(*r
zG(^NHd-0awtp~2NCb+Zp(OUs%ZM>GlnQky%yK8iCy8d{r1Mc-k=+%utVk`RndJ{Z1
zfV(EXWw;LTC2yf3pEKddGXdQe_*{3UD$uGvYSODzu*?6!7X|jnE-*{ji~nktE#|1U
zVz6l_K>Gt8-paFElc`Xs9@IsT0LwP~lo6jg@LQ<CUHFN?o#5vl{FG6d_OYfJp^)$5
z`KR#=@5V7yzDOrR4dP3JFWyqzL41h~iFXu(5Z_^o;vIVk;!BPI^?{D>Kzjt$F-}3d
z!9RUR6iM7I5RNv!HU(tvw;hBqQ+;6Ed!jE*@s3dM$<D~Ph<i`QP!#X5g`SM3DBcn3
z%H0uzi~|B7`cGoM6f6T<`jo2)B;dtajG?D$(m|wHwr*|<n3f8(lQKlnx}zZ`H*rQ0
z*kP%f9Cl-FI#===UIX|8s7h=Su>c?_;ctD9erTelv|ia6>nA;v7R#75z0$l4j5jAD
zY?cCjiJura{I^<K(KyIo_e4+ZVm7>TA+a{`1}NXtp{qZ|rdlEln4n{MXU8pqkQ$65
z_f20BN(pzl%A}2pmqHo!VrWjg9C1@oq%SFTWm_WH+ZwG=5V)$wXsIk~Oc~$SXvr#8
z=Hjkyuvu_5XuF&-n+j-C5|N%!GrpSvwy7yf9tg%+fPAGGgUsZmeZhjH+K<I)S>9)A
zT)8B9K?@#iu4st_F|n(`WESoMicIioFlh&{LE>iyi+uEIV0rO5Fy(7`*9Eh}{=feD
zPkgn*pTF?=&w~dC)%joc_6z4f-NWww-TBXLJcazPw;KZnWTIq_?%<bMljDX6(jSGB
z(ewCebde{s`Zzv|b9B8#Lpi>oGo<Na5|Y^X{0VR?{0KjGy)IohoCY&?X=tOb1uXgc
z>`RogyC7T7ysx}Y<I67_D$6Ar&xVnQz7ctZ_hc<-9W1}Q2Q};Xv%7mBf9~x*kU#f#
z|0;jBS}m`ES6?OgaB`La=<}&Zc>%j5-?!Ud!pBEE%IZ`LuWc_d5hz-aFUc>(qYyUY
z<O}^N_;{W(a`p)rU$CEs??Q?XjVb^S+wMN8qx;e@iLo+JUY1ox@a~f2YEeL+93*rV
z4*uTW<_0lMX`II()&I_?ZWLF`tSA~dxnE~gzUmZI35pm)#ve4wiBe6k)AV*a`=)Qp
zVycjm^?uX!V!9-_(<x|JT&s@%-1mNK;QbjMr2B~A;J1PaoZm6QprGf~f4kpp>~y?_
z11kO(C`c61CB!J9MfqNPe@qf159(Kp$u3WnAy~;V=Vs|^$<W7OdE9A9EHGlV+>hrl
z%(=Jy2$so$Jd@@zo*%vQc+y)KtPhI!#CrCKd3yu?da^B68Rs(Zm5na2C^Nor<MqWN
zZSg;K3@#2I-VQpwlO_e)@do~RG?_~eU?F>oTHn(@1^sYK;E&a;q?<_pc;iJE6}{GU
z0nO8$n6B_qLBdD?SB^Z8+a~0M3~M`Y3ugkbu!fk6pzj^0#>oIx05Elb=bXcrP`?T^
zR>a7{$}nxCe+LYpuFQik7AJnPj%3i;hztC!c)lbi`)QIxH%X@!#xykTbEO{WD%(Iy
z{1?xIC2p$5J96PW$cQ80Me0(%ZUx9oK&ao#8ikfrm$%TLj`FEWfUuR&_^4d7Xx7$o
z1PxktaeVH_Y`natf1Kpu1asZ?@14T(tIHZYy4eC3TIqDdE8|n|&A0JvBL7___a8p1
z@_%;s4@>f2`!4^>tvt7x|8ML0w=cb&r@&1x#<uwLH!s0#ZhYvrn~hS8@dDJw3WpGG
zKJ_U5^FC{@|8z0CDhpV({`WgY{cmrtdzb&=R-W5j|I^nPTy1CWl~;RhH23~LptWft
z*8Qp?*}1dG*!32Jsdzg-dYv88yWG46`v0lt;!`|puYU?$(YjI=@D=O7b5P3vz2E7y
z@7DiqJSm392+{<YK#${Ye-Tb5wc#Zvndp&<^u-q-YEPrm+gUOT_zT{A80A5Nu6cMV
zos))s@$jdgYN$kE?Ctce3{cVg0(~9m<1~E7-vDNJzt?WV|JN|*6A<5oVfVh!u;>N4
zyG-YK5Y8@9wr7+wQH+bw`*|9j#_vD}4r+jC`TdMOjH4h!|Jm2kCB`4YpHLJHHkaNH
zPrm=b7Y!$GV~S2WPbTpQi-u=sbg20jA&ReljQ~iDR5$fyT#tj8HN9yxMHPE=-W1L?
zZ~1O`nBX-&eWOo?Eo<uhyiGF`Z_~`V+oa^VO-_{C)Tws;8b%ighT#Ni73L?K&I56!
zHW*`i4RTSdorZB55T`T?<6C+ekRFVwd3)XnEfxApG|OS|dLFG7<NY`qg_op`8FO9y
zd>W4OB<*?a)&UH9gcFS;Ox&mEsR0$Ac%*5<dS7%7+Uh;MK?g26c;;d_9_f@r)MUOY
zN;C^)Up{)FU*a_OyeX)JL^4^7gH8HmZ5)m9ERZ5GG9h8M9+BKDM^GTWeEnzHOxEZB
zI#0o-2LUVMzjq&Wi}K(8!QO*A{(l?KZQTEOn&gqUqwZ;lWS((`e$r^<7FNo51dK27
zE|bOf1pV71^HwIAWu&W0`Q|(3GfCs&g3?eVDXNV%WRg)h_L9@(mDx2z188FHM(r4r
zfW(<M1il3Gjz2@-+Ic9Fd(6llNB6mFwTF`|5qGM|ipjpnBA-a+VSP+bQl~KJB1|p*
z4B45EV!LzX74jCZ+2e4w2-7h%e}-9ia)v}+4=`oP6t-@8F@TKn!{jY|K`d&uUl#G`
zHQV3VvQ<pE&3Q-2odbO<@lW3_h{<6br8r3yj3@C{7*u~9_;U~s5*V{J8?uV47Uf%C
z9!sPVZf~B*62I%Qlxjm4M^3Ju815ar?{<XATpB@bzUN;NsEFz70?O&z)&Oe{#&sjT
z4Ejp!fzB|C8b#zNPb#Na`y21oXmtH%MOJmQsDnt>g6GI!zYd`4r0AO?Rh<}JAXIw^
ziyN(A&(q|T9pu6Z{&ALu(`l4u-uDYU^TR_9jyxBQ%>$DbLnwxcrf63bE^_xiU&l_)
zJdyJ>BaUn($S7w+UJPOX$2fc{#YPH@4aIUhVY2J4+-5i03XUG~c*}BU(S9*a<#}cj
zE8oOW{#u3?i}Ijw!!<HEZVH*ADS|d2g6SGL8O=B`(x?6hD9z`&6dNc@IK$LML`TnJ
ztnqdb`~(Chm-1?%#D?!SS}<3Rk>W%<ge@NKVB_~h-Y}J%(%a4w%K1fs2F5B&Qwy;U
z(1$D&{z*nYOHh<3c)*tTKhbQQq(ql|PAxAA|8rlJxw{qf8Q+Q#SI`Wy3&uKx#T=W~
zZVuwQ8TZy47tv{KScVTLB&_++&}Cl8Vn47od4EoqrjTq6vlA*97}aRV`p~6cULO5)
z^ynB9x)!geQbuNx_;s)?v%YShG`(#Q3%B8)H{v#C1Hc?VyfTnHBoGgOIzD=FMHm#K
z3xT}&>E#tcq?2r0vS7ujWl&pWb~Y!L+y?V3k=BpwJ-~Z}>~Fwl@kSL^1uBXI6?6||
zjIb^5adb+8H+Fz>C*np9;`kIK?@Si@KFW!aQ{Zci{4hfslTX~&ql9!UNr>C9qbW)U
z17pI`4}rW9pw@{vOvjiicpqvp!wPkyvSQv8Wm^Pa;VdEag*RINM~Y0xn;M0+bD(R=
z877;v!)^<JPZMqwMj=8y3v>P%_ZM22_Hb_A)@7PEn`H@7#oZ~k&T^F+A3@2KDCM*m
z64a|0Ckdh<#vDWS{&E^%CC?Rvv*szs2E@=cC>Tl3g;W`*uQ&{OuY$NL=2pNrF<y$U
zoO@k#K(k%EU&|5jnr5g%_x;)}??oi1)WgJGw3t-RVuJ?$8x5!!fQYVyNb;&1Ej286
z7ZJj4;9JVLf5K}}XJ}|N%Gcy<7Ap{V`8uA{4zDF`7I*}i2U56^53&<Adx9yM>rDnO
zECfZA<Roeru2mQB8>=6`sZ`U{Yx=_oS^Ar}F)hD4M@3w5rhsvNN8aG{%Z+V$k1%i|
zEoR=i#*P?f)8B-IyR1zrQHV!Wf)I$Xg{HhGzd4hDJOpj$Oe?Hfwjvpl6I^>wkmX=#
z;gCzSq#!NgCNasu1zNV&^}zpy$GLQQ(=Vxves%4EfbY1_iJ*9h)~G&kv|15AX^Z7k
z&Sh=M$#o&#?V{+nD}}vX6bRYuBPHWHN^0V?lTtDTvna+WCV?GOqN4rB2{HTx^p+I0
zP$faf=#2;hxrPwu<$wkZWM{&2+m}*2n$Z?{f;<$XdR}6<HI9A(ML8fcaw9SYf`W7r
zqvesp)uVz$Ay}}Fc*iuCIgP6ZkTD-eGjDiFOW}>R$uw01L)Ij6k91rYG?|0exkN-f
z+v*Ti#bshd_bfin57Lq#SL;>CY#_+gomt3g!L^#L0`ACK71I!Nl4OfyGLI?IB5Yu4
zf^?f6EO*c<vEmeM1*aihVp8G(X6OZ0JBrd-G<LSDh%<nZLIE)ceLV`2)4*slNcfD4
zLtA5<j|}G1nynTF{i-(_phn|2QyYPo-xxm1=7am&h6q$sPkC-Qt+An<EXg)iRlA_-
zR6wYhQ^A&mOh`$>m~+b+a@HeNtLeQ(PU%3aG9s|~Sd%k&jVhr%U!;aXBh@2P(I+MN
zqM_epM#OLo_7PKr7oOkC3JSh56QRb+J+h{&p)wR!6qb~s3a&ezCBuNSD{MBJhGbzW
z<5p|#84}W*Fu9dv2xrq9HfP8okCO$Sy-=8G<N~&cflEHO)}|6v0wX3qj<70W1|pK}
zSB4>44GtfaG5RE5AmEDNsl{NFLTD}rr7XtJ=sH4DO3Q|ftwY%PZYpy2!HSMpGMLz<
zoir-Q;8^Y)g;mEw1%YVFSt?9kfukw90#5!hP%Ej@ISxAQoL!b&2RsTvO35-M0aJPx
z-!5!WxBB^&1o4$<qVH}<^qO;03$9&UTITeq2EJTFC$ML1%ln0+unR_N>ss_lX<N#U
zbmGGHX>=jZQprSuW7;cW?Wi}5PElx3!Qe^p6MEI!72GXo{G~0=-<gOQY|9#Q)?+eT
zqeE}30rgjbb}6|6(&+-kTpWFFRo5-tWGNX<Dhdj)qC5jFCSAp+XeD^T#yaW}8Nrg-
z5-ojm4ZT`BryqtXimiyUJQnSlHnK<4v7|W#22X0}-j+NcYdxoIr<6g9d*3%dK6!p5
zbKMl*%c#s|J{**|p$KMfoi1kNWTc}P&=)qFQ3fJA^9%<A*v6qcrDMV`f1UlB3zTp#
z6@MZo_~$mI?6g`p>~-qcOubS!oC;_ZhS$oWwCa|bVRyAvqWDxCH>vOFS{31;s?Ble
zAd6<8s(gc#2gjCa8hx29)|ExAewsE6t{92qFLFPyV-7_vJ{;x(6ae$bXW!bb{f2?E
zT19@i^Z>CCS;lVV`O%9f&mKQ{R1hTAv4$I99kB?ik49h3UeA(?nQ*`$WO!vTX4b{O
zmzNoE>?7+I->S1p*pab<^kUH<%(T}SRK{qi61wA5SDoOhbL^H769JYPAe<Y)W6f~J
zf#GM#Y)85$u|E=dGV!fpkQ{~){6D-NgU(>yDBG4d%C?&bEfWk6T`q9-Vk)XU!P*$R
zZIP1$P{_u2@?{ZDQZNwnR!yi`Ml90c)zc@($44)Z1J>QlYik?19$v-&FTm?N8#n>3
z@c-LCcu>6mb#T}|xby$JjYqg;z;=_<rL2-Oh;Z029Y0Lxk(v&1G%%@>bVm+pS0lrK
zK|j(L#5(3Oj!*PXujNF;H${R`f$V-vhCO|}!Lfqy8hQ}u%sn6^^gB&Y`!tB8b)?ew
z{fi`h{RWoOeU&JkqS=}^(nZeNfO7bX!wXATp%VO6mo0q7U>xBHU7Dd7x!c<XENL6%
zeI9-F9$(jO6m0;O(}w=TE+h{k=wEvWr#u=7>B|+l9p8(ii=F`sc#y3p^gIQ@?o&fJ
zPH@O53~nE$bRJfB554tHM~n6P_804T1hDDy*@$nQumhdc4FX^Rve>Mms-w3m>wp>=
zAv_iahC(B%a-#arFmo>!GZZ72(k8S%PiC1Q$Rfe1uiRnr8#H+%Qx_FXQjmItFzfX#
z7d`Q|JkQ~H9H@-rO|%QZ=;8GO*^(V+m(nrlxQ?>WlC*)ch&-+wFQ)TMRcusrRcbSJ
z2SSbvaRK!~ji(mm@sS_`644EO3QeL#w5@@|1fqkkSsOQX0)iHVFZ=R%({8|0ICS7m
zYNSX75%L0owiWrJG;;#oB4G8-1p3Y(Ef$pU(hA=VQGYkRe7IJV7Fk(>uLas|!6D0k
zIMZdZSbm;@BuOVPJap^|mnGB4AaG2(v7L*K77YAiZrE6FP9mKTx0`sfhk_7_Y8s-@
zP^}7Od!Sgmbe^(&*`lQ9+BVdIOK~NOh;}O8ok{n0O@p`Tcd+E&e<&0GDpp-S2kKR{
zD)};xz)K>DAAll`VQJT;c+kSX9)-ji6b@OGznVLFqDC_c)5kFHt4d|LB(`G7Qmi}&
z(bfk(f-rLtE>TKU`)fd?h`#Zplx<dFt!;m#cNNz(JLqsWdfbJC;Ye;%{jQ<t04UHK
zblHXjOb~AZNR@VjDo7<(OcVpgvSQ9LEU38eKQxVI8|zk47_42x?Azwen04%h%8cje
z%>u->ItvfFDMRpTtgNOQbJkKnwmfsl$+L$O44~dfZaAPtUPO~gYXd}KBCmvMMZLl@
za9JVDESh%0!KX>-M1GbC!@TgW6SjEF$!43m&76OFv6RHS!U8dt#~Y|ZjNVulWPZ?+
zTb;~Df*07FBhaC+6w4C-f>J>$N3&#^FUFNJE@?Q;H%ORAY;|&2Ro5Ez0K23PBph?#
zy~5$1$D`MBwEO7XE8H&xz2TNA_v)(+Ja8s)KdLW}6np5N($h^k=SEZ^FXyYw(-u+`
z8?l=<Wr5J>?a3=!GK=5lbF2hFVJT;XNtVCX*}nR4uQ$Txq+DsFKVr1jpomX9gc847
z8p<U^Yd9O=*}pnz{HLbi4H(4@j=gjg`m>Gu2K$6g5M0<K{_fZc_y~PF(>{KjxoQe+
zoL;Jv^XuI&Zg~TDFs4tlo0E4?#mtZkqi~p$hs)C4Qr=!x9(ZD>m;OzCP{q&V(K$I~
z?ueVonQ+InKwaBB`c2U^l$p<Gmd>*7dj2Pv@Y@|n6rg(I4%aOVqtV<y(<t5+6@DJ<
z22V#P(mr&}yMnjl88jjtxowYaOE>h)mHF|!$h{suIs3WQb+bidPWIQLT#5Z%Wbs>!
zf^?R^c5oKaaX9+lU&F|Mc0>Lt*V;7BFd1&OnFQCFj2V(qidhds+TkTR*r!z@P;naw
zR@Mvyu4wH&6Sw;0N`gvs!<sIclP5zKt+k?-%U*BFMrC#9!0;sIiM{c8U0>6zm~~nm
z<nRIGg%-n`p$Q5O5tZ!SIPJ%D+ePOT9iqRt2nkOftKkHry(};ejR0dLQEpw8Nq*Jo
zO`~bQ+df+{;kP_>pTlhB7m0}OR-CZ&`N%NZJ3S6~ybB%&>kU6Z*E1mMHVj4%i+!9R
zji7k%!S<?y!DM_fL)~AXn^nIp(8L$z#$8FMF@z!1w&e5N(W7p4kDxquNI~r55kbQQ
zXLP|?xN48MZbBR2a5f2}6aFh8!V^6{X<DFM4~_T*jrfoq#Q-=RISQ9usdwoP_Vd({
zv$y!TX@5{8{wAHbp4`?N-~?Pt*DfuJrntN3?1jz_S@0x~bqgLh{2E<<`PYd}4`n#q
zb^(gG5Iw+uS3+>&T>|MWg>5)L6dL7qBvFzcMdD7jyo+6^BSyKU95yN=1i(L@2qUo!
ze8Y$~!h;(az7g-mg~Fcq{)0~6`iAbea%9WG9I49sD&iYD@Wy@rO-mk98;!bpwDeAY
zv)GgYjBe9g_|uhl8-1huHgOop^G$hU;7&xmIJ*by_M##P08W3?ZrbRK4zqyeG^)bw
zfv?*+$V@{Jy@gfy2GeVJzsK|X>SakR!nlE646vs>uPUydqf^M6dBOjuJU+^y4~Irx
zpzvW}_$R#;n*h87h%{1iI^eV5ui0XRzVxRHQ%7*LS9NK?gqTh;H@ua3BDS<w<bTM_
zD7dn>F<J@wO0-aFcnbBmc}Xiriq9p(#1j8sjCD8;^RSf;Kdy~c@!t<i`5!vn?p^%n
zTX}9e{wKLTnUBxZ<=1cp<nTYJC%!n__G=S{W`KY;d=N>pQ5v&j8NYsl+xTRX?LML#
z4=`Vu_iP@`WQaYCf(&}LS@Fp;P+G%r{kwP)@wQ37#&JsNOEEd&Pe(z}faUz+=;7m^
zk6P35Hte3Di)X4~YP{X*AWrDHm>I9p!!?@vwR;!uz0p}XJDbFytO={vh%EASMPW2a
z7Gv*2i$bit|6}xrCt-~~)NU}C@RIePf`$n~{_aP&u_FH;KDhq(+It84d;9Rd+dk;*
z{lz=DhW@R5{>9h-8uO1QLlm#=svE%y{=c_(a8S(u(YfRQxAWXI|CdxnH}X7x>ph+<
z40@X9oeX;LJ<swqoS%H40H&JY=9j)VPDpYR!F_$}BYm0Tg(|=O;NXD&PA+gy@l$}x
zFZjSgya1kB`#7vrSY1Y~gqRt2*T$`Y+4bNhlKPT%UdF2U^>%QQ*H%jC7$0s26B&)P
zfQNd$ITlwUWYYf_1<~B${{E`wIGPw9$vgrs-Ru4M^4Zfz$#fp(sE{H^OhJ(6mvzR4
z`8MD|?P6L<BE^#iBaS(KRP<+6gf1lt)aYvB-~P69^y~*-b5ZW?ux7cN%ya4!W(#r#
z`Bo5puk+2ncfCLU@RY^NUE2&qt?^kJ*!qeq^&^oe#J|4E+%LgE#XZ-}mm{5TRQ)uK
z(A3%y3^7j@AXKi8HNv2g<FTRp7fcd7-`OVQUjjx{Kbu?zpj*H99AHE;5|fVMH&19Z
z!<(|>6Ym3F8<-_I2b1c-ri9-?OvlX$wx0nYZ~|I*ZyZg+OXvbFfls~JeumH5@?SE)
zvK?Sm{-^FiDgXC_{k#0%xANSy{I~I@ZBUz2z~V0}cTTyfx7+v^UZH?pq{sY=w?phX
zpQ!_U4*~Oai?`u~57t8?0q!)OQBrP7mxcYElbb}k;5hAUP~eQ-m^DPesVRVcuf74N
zOXh*Bd4gMg?CpAo?N+<d8b$G>1~W9%!y&~A!qD^Y?{2-9l|QfyK_Q9P7<z;oT%>sY
zGQ|XYo6ME>jklXm=lT^MOLxEd_s*5sJq&`u?eEz4%{*)Hf0<F^%4XnI_TSEayU70!
z+IRUsZ{@jZ{{N}_clRuR)>!|O<Sd?j%puT9`LEM0uK&)1JNw_QJU1@?F)@Lhg_qxL
zGle)rIe|jq=ryL36qn@18PU#Y5`!MRv-Ms9`Otdz^<i+h-%96GzgGJ#%CTxN24RrG
zCRu<;E`oZPgU#D!7~3rTT`s19DC?*rB~Bp#ZHs2MPX^nvrEDAIA3)5g`@xhnV?K-G
z)+l>rOd7qAc3SjmF^N(-V^#&egh2uc4|MiuI-j6z#ovwc5iUu}y<0Me$`7q59a`p4
zVP9Hh4#gL5!!$-gE35Z7cN9;<HwrC05yy^DXE<OLrVWbn;1qCN#N%ygKVngHyV_F0
z4S1p+`9%5KchcP_d)DCp+3RTXQ9b}G&;QH%f4h5^|MOO!8|VMy80aWZ(o4a~keVoq
zyI^JOsqt)^FR1Yt=VY`<F$J?UHP4b96-|oT;q8pQLX*tF?p?g+B=ej{9+Y_cAr9Zi
ziTB6!60H?~5KFVn7dAZD0J!9bB46g$Yx3X10amG(lJ(xi{I9eA&A2F6VFN4H|9*R~
zsQ>Nlx9{wKxAJ^;_CLyCX{RqO$~>;@rkfuEd)_8Td8M4E5~<C_tt%et85qbDJ<NRj
zW)Y(+(eP6G8ug^?8H`iljH!R+gHr*(H3jz9s~*GG5RaC;E7W8B1>8Vw9CUeT9i07I
zJOlZ9jA`kSbuAeY>nxN<);tI;ABC<y|12a?_PXD|9`SZ}hJm+NpM|cSWqCum>!e}+
zcvM&9X}<0T^!cV&33|N)?5k#lz6u+=4m7TE675bly$K>4WS+j2f!QEe^+$}*hDYZN
zq<C&FPtG?zJ9j`6Y_)Kf;@lqwJ^P^Cc~tJ3N9F$AQTe~*vyS{PO;tDK|JiMKI_&@3
zK5QTEby5C@pLg>AZ9Jc~{g;h}*JS^FdHnFj&Z9>^&nJuQlURWbOUPQ%>_^*2Tuzg_
zZ^Lvqn<vSHwwZB)TWu>lFZf#M_ZvA#UIhhWFQD_pZ^cjE!3zk{r<VA%?y!(j#uj+q
z>WH!gWPz#NDq;VleeuM$a{Ek!7Ea2eqG~X`fB$zOIgq`G-fGm9AmIAAsY_c)<>95$
z>nZ=kvQ{LE7!rejx8eV5K5Oj%Vkz91|Hp%a!$Z0MclO{n`hRrq{6B8x`K<SUrk!2s
zJJgF8Cm}_uYfG{vS&}VV*2-E$wq!Dv><m&8gF!JdmI}9o85tV8kdbA^PE_`|XeMT4
z9mX;YX8B(Cd47Mv@6~yGp7ZLQ&-p9{3R^?6N4&G-7GLg{bl9Qp45!T@%y`dVE*5sw
zQkmj(?=ViPJ9>O5i<~AK&8;zEQ5K;fjV~DYzIt@<3CZ7Dq#b34*G|;Rw&oIM@oHwi
zn2HJ#aq}3CZXU@n`3#e?A4qE;g%i9yKO>NcAM2kTKM30_?*D;#+1^GKt^)lwu?g^>
z8CcWQ_m=WriV5*J?E!SNafYUSDa-s~r9B})7e-}$*beXfe631iCRfKnn}<K_)`!20
z#O`za7nReo3j$F&iE#D=G;z;!256sjj?LC%_KLH4Ed6uy*cxVP$^~bRYe?`}nKFHX
zaBLB2r{AfVP7QC2nw4CL>y3gaivT6fE`m`xDz~2Zk${LxU&oZJbzic29OyCk{7Z#f
zcKAVvvxlpT<+zhPJ<NdELD$zSjeH~jW7Q&SgMOPD+jHZrp6Kz*N4ol5#5WXE`iC}K
ztPChSV5CJ>f4y0TL0L!(roAnHIqK4{g<(Olb?)saWEoyq(YR{9P@)eYg3-<HkMOxh
zaQBmq?tXhwOhqOM@WaL9B|;;fvG9}V+e_I_d59T@;XP@gfh8f4iLf_mVCYQdreIF}
z(`>s$TGFvajP_f3ASIf!xq;mCtoxLuBzD69DJ{R3(DkFhth2UnI5`|?6zl;PSIe+=
zQ6Zge<pKApn`4Q81ts3!H4?g9=sqgyyDe3m`<Nh;jRvou<LLv{+<X7j-U7o^COM*?
zBie;TjN;UUuc{Z)&ha>1{YZ{qZiSe?kORe>YN3mQWKEvFgNuQD&t#Hf!k2yKe0pDV
ztLGeif2nT180%6$K5s#{<(<AkDRcL_2NdN;xbC#CBwF%7=r8ocPhX23r6nT{<I9p1
zxgGc;G`~ldCFN086n$Tz@o_TxP>5k?DyrBakc~feLI_a)>Uh!jp5w|brRtt<UIm%v
z4o99lIr?W$zDcQ6v4q9U@U$9%x9-9ojUlsg%NoEPBTH$F65Wj>6CN5!K?O&4U~CpE
zGR9tWY0p;{#CFZjuaaFjo&Q74$LMkjz`L4X^#H#jUFq$Cz&M;rDkRO13MH4Pzcf7k
zEcATV0}UWQ4RV`eEapWcKW}6DtnH~L1NT~R^w}%yZxDlu1^0Y3?_Qrk0bux#evBOr
zK<~HJ0|e&dv0dZ2LLLK=V+tH9!IC92TPn#?0*Xz!zXR7gfuN*t(Q&Mh<4*<hTTYz7
z3%#zIlnwVTOE6_=r<-`mQN*!Sy<bNul$x>~%4Odk-*|WGe6SQ2-zyy}=t^*2EFAl=
z7c;N?K`AiOW50ZvFUx0_cXgDc>So`%f0JT8ze2x4XZn5VzpW-U+o(`_9$xnBrT8Np
z9C7AmrS|tsNj-l23(unw+PQLv0Lbu;)rXvTD41^y$sCbphB+|1IF_TZCy2c|j~QUa
zN|ozEubFNO;5P&hD&e$h3|gClQYnK@{1M9Rxf56%;jNN_@_dpyU9II<I$oD5*$z`{
zivCTO+F2WIglJAfbaUlIckFY`m2HO!21f#`=z9)mZZF67Chb+c@T+O_q<Y0Kr_|fN
zj2tpuEX5X%oygKy59$7rQk2vk!c|%^(pma#N2y1LhDZmAwd4_tOplcMdzeU*&iQ3|
zi`{93W%K_HY<#>MhfGrs)*@8HNAG)c-V7Wt<p|b>OPkH+-b8{^pMCvLbeIx(kaad-
zc0K_y&VCarXT{f`YjRz;Wt<cg?uxmU8J1$;<#?i4W)qw2(=rx29M}a!V3kXC2MuFC
z&TbZMcF%Esvd(MOdllri$=7Y-d@}Wt?9SGs)F$S4ve3E4(QcWHxo6+&v`&_<P{j9S
znu-KIfjzk%pdpIW4je}nZ602o)Ox?z%;{XU`VZe=|Esm2oh)3=H_)A0ATn=|@bPiu
z@-N7>$h<qa*{()n#zPtT`Tg&jZl5N8`I{1&*YFXC<m(cV+&rHH6Zs2o@&%i^3U5@6
zXmIhFa*IDfT>P>UCfAUvB=VlCVMH-h;}>3n{cz?|w7wTUKqF?US~bn<Z1Bwo?Zl+=
zR}GMW9a&p!5;uI-uJ$b~uz^t@<va!BZ(#M3+8U&U$iP!UK`IIMYv6IvuGH2^4zKBn
z8WScoQnGiZVfsY2T9POwv0I5M$VN~-vNU_;^iIfs>hQYSAbBj>s=Q56lpOUA<-o(^
zY4^r7JqF<}fTl%!EQ@8B5=(x$c`t|XBqodckv2}~)0lIcg87}Vyg6#A<ku*V{U`fd
zjsCmhSbkdG{?^hmz>lpJ4>OBcX{%}<^9|1CeDb3WzlDW1SDLPi4N*SM1$h*<+bmlB
z-J>wc2`{c#S15rFcsoNSFwY^7>n(z}?}abXiBcHYS11~(`x=QJKeRnFh7{g7v+xOZ
zU7sqj37oz)Id^~j)`iwg+Qn?fcsG<DL#uxQ+^#9k9OU@4zi*2f!y^Eg@{sGc>eZc!
z(T;`f--b)X0)Q}3Kf;_rz7A@vRbtyhYaHGI&vCdzrKU3y_WVlv0+B_m{%UoBjhmSP
zw2<i&ZOgFM>*p#aVD#&+0jlvA(BUn0Xmjtgv9;6#NZ^)$HSlkSYS$$uQAg__`f$Cd
zy*#GvZg?=aRJ*e<QjdaR!AvG)F%Hf)`Y){|HsD8zj<Eq_zcjREVT@rh1eV}7Hsmy{
zYkOxy3P?*3IyR49jIHDHL+_u5Hh=UR+c6dxIY`6!7dlk;^E!Wp71S{!zW}p?z@js<
zO0|C?{+rVOWca=={v@l<U&&Tj8=XYSz5wO-fBYRv8QoX5<a#tS<oXlbD9L2?g-V2<
zUmQB8r_Dm&t<_+>I?7Ra$f9#(f1uo(t!F0LZ9ab%>3d@v6XR4ug*0N-1<E!f5e@~Y
z-TuPY9Hqilln|WppO)xgYh-eOIC$4q@%*eD((Eilkj(wu2qy8_))l?&Jt%d>^St#c
zUfghbm{GysWa>9xCmgDMPxNeD4(91_35;Re-KLWE^CeSRVts<PHod2#d_3cf3;wR8
zFf<U<dHCk+b*i>8jaU8(@Dc_I2W{zae>iT4tY>E<?lZ*~!a|AHwj7S2q8b*F^Eh4G
zxf)mpKLU)Gh&@14PM-`77MQ(`1jYEXF#Yq_wwTU_LDMjLGZYRd+;jm7aJ+DWK7$G|
z=w)ioyv?}GiDxd2Q)!hXfKfnVW$qH)*qJj?As2xbs<euJLJPp0c{?=Lt$&6t>TbtJ
ztLQ|~B2cV29@$6HHGR3)?=a;{HC8SRX0Xb`w2JM(#SbTk#erz3dlbu|85Q(^wOR=j
zu_r@*Pr*VLn||)pSbf7Cz&fduxZSdmmK^uZ>`7zLvzjc<N>>2R=wYg^9Rd}NhYduq
z&J{a(a11SseE;jgte=vTFV_a$M6X#?<nz3MMsJ{mPe@376Ue#lBzJ}|`M9NUsRB_}
zSx3QMkWh4T+aMi2Kfb;SQZ)jfjHY^~)t=Uy5>QXM*4hQSqC@euSe1I1bhK-OlKiB-
rfdxA`Q^>WRz-|;Am@QyO(tjC74jmex{r|;j2oN;lT6@4D&BgIA3Y*4Q

literal 0
HcmV?d00001

diff --git a/specs/default/cluster-init/files/README.txt b/specs/default/cluster-init/files/README.txt
new file mode 100644
index 0000000..e110007
--- /dev/null
+++ b/specs/default/cluster-init/files/README.txt
@@ -0,0 +1,5 @@
+
+Files in this directory are automatically synced to any node using this spec. Content here
+can be anything from software packages to config files. Scripts can be used to install
+software packages or move files into the appropriate location on the node.
+            
\ No newline at end of file
diff --git a/specs/default/cluster-init/files/runpbs.sh b/specs/default/cluster-init/files/runpbs.sh
new file mode 100644
index 0000000..b7e85c9
--- /dev/null
+++ b/specs/default/cluster-init/files/runpbs.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+cd ~/
+
+PPN=15
+echo ${1}> NODES
+NODES=`cat ~/NODES`
+NP=`expr ${1} \* ${PPN}`
+echo ${NP} > NP
+
+echo ${NODES} ${NP} ${PPN}
+
+qsub -j oe -l select=${NODES}:ncpus=${PPN} ~/script.sh
+#qsub -j oe ~/script2.sh
diff --git a/specs/default/cluster-init/files/script.sh b/specs/default/cluster-init/files/script.sh
new file mode 100644
index 0000000..6ae7ccd
--- /dev/null
+++ b/specs/default/cluster-init/files/script.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+NODES=`cat ~/NODES`
+NP=`cat ~/NP`
+#PBS -j oe
+#PBS -l select=${NODES}:ncpus=15
+
+LSDYNA_DIR="/shared/home/azureuser/apps"
+#MPI_ROOT="/shared/home/azureuser/apps/platform_mpi/bin"
+MPI_ROOT="/opt/intel/compilers_and_libraries_2016.3.223/linux/mpi/intel64/bin"
+source /opt/intel/compilers_and_libraries/linux/mpi/bin64/mpivars.sh
+source /opt/intel/bin/compilervars.sh intel64
+#export LD_LIBRARY_PATH="/shared/home/azureuser/apps/platform_mpi/lib/linux_amd64"
+#INPUT="/shared/home/azureuser/apps/neon.refined.rev01.k"
+#INPUT="00_Calc.dyn"
+INPUT="01_Model.dyn"
+
+${MPI_ROOT}/mpirun -np ${NP} -ppn 15 -genv I_MPI_DEBUG 5 ${LSDYNA_DIR}/ls-dyna_mpp_s_R9_3_0_x64_redhat54_ifort131_sse2_intelmpi-413 i=${INPUT} > ~/log01.log
+
+
diff --git a/specs/default/cluster-init/files/starccmrun.sh b/specs/default/cluster-init/files/starccmrun.sh
new file mode 100644
index 0000000..1368a79
--- /dev/null
+++ b/specs/default/cluster-init/files/starccmrun.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+#PBS -j oe
+#PBS -l nodes=2:ppn=16
+
+LSDYNA_DIR="/shared/home/azureuser/apps"
+MPI_ROOT="/shared/home/azureuser/apps/platform_mpi/bin"
+INPUT="/shared/home/azureuser/apps/neon.refined.rev01.k"
+
+cd ${PBS_O_WORKDIR}
+NP=$(wc -l ${PBS_NODEFILE} | awk '{print $1}')
+${MPI_ROOT}/mpirun -np ${NP} ${LSDYNA_DIR}/ls-dyna_mpp_s_R9_3_0_x64_redhat54_ifort131_sse2_platformmpi info 
+
+
+#${MPI_ROOT}/mpirun -np ${NP} ${ADVC_BIN} < ${INPUT}
+#${ADVC_DIR}/ADVCSolver ${INPUT} -out-dir ~/ -np ${NP} | tee ADVC-`date +%Y%m%d_%H-%M-%S`.log
diff --git a/specs/default/cluster-init/files/starccmsetup.sh b/specs/default/cluster-init/files/starccmsetup.sh
new file mode 100644
index 0000000..50c43fc
--- /dev/null
+++ b/specs/default/cluster-init/files/starccmsetup.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+LSDYNA_VERSION=$(jetpack config ADVENTURECluster.version)
+exprot ALDE_LICENSE_FILE=$(jetpack config LICENSE)
+export PATH=$PATH:~/apps/
+export MPI_HASIC_UDAPL=ofa-v2-ib0
+
+#qsub -J 1 ~/
diff --git a/specs/default/cluster-init/scripts/README.txt b/specs/default/cluster-init/scripts/README.txt
new file mode 100644
index 0000000..66c952f
--- /dev/null
+++ b/specs/default/cluster-init/scripts/README.txt
@@ -0,0 +1,10 @@
+
+Files in this directory are executed on the host in alphabetical order.
+It is recommended that files are named start with digits to ensure they
+are executed in the correct order, example:
+    - 000_run_me_first.sh
+    - 001_run_me_second.sh
+
+Allowable file extensions on Linux: .sh
+Allowable file extensions on Windows: .bat, .cmd, .exe
+            
\ No newline at end of file
diff --git a/specs/default/cluster-init/tests/README.txt b/specs/default/cluster-init/tests/README.txt
new file mode 100644
index 0000000..a7149f8
--- /dev/null
+++ b/specs/default/cluster-init/tests/README.txt
@@ -0,0 +1,5 @@
+
+Files in this directory contains tests that will be run at cluster start
+when in testing mode. Please see the official documentation for more information
+on cluster testing.
+            
\ No newline at end of file
diff --git a/specs/execute/chef/roles/pbspro_execute_role.rb b/specs/execute/chef/roles/pbspro_execute_role.rb
new file mode 100644
index 0000000..175600d
--- /dev/null
+++ b/specs/execute/chef/roles/pbspro_execute_role.rb
@@ -0,0 +1,9 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+name "pbspro_execute_role"
+description "PBSPro Execute Role"
+run_list("recipe[cshared::client]",
+  "recipe[cuser]",
+  "recipe[pbspro::execute]",
+  "recipe[cganglia::client]")
diff --git a/specs/execute/chef/roles/pbspro_login_role.rb b/specs/execute/chef/roles/pbspro_login_role.rb
new file mode 100644
index 0000000..1b41949
--- /dev/null
+++ b/specs/execute/chef/roles/pbspro_login_role.rb
@@ -0,0 +1,9 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+name "pbspro_login_role"
+description "PBSPro Login Role"
+run_list("recipe[cshared::client]",
+  "recipe[cuser]",
+  "recipe[pbspro::login]",
+  "recipe[cganglia::client]")
diff --git a/specs/execute/chef/roles/pbspro_master_role.rb b/specs/execute/chef/roles/pbspro_master_role.rb
new file mode 100644
index 0000000..651d353
--- /dev/null
+++ b/specs/execute/chef/roles/pbspro_master_role.rb
@@ -0,0 +1,14 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+name "pbspro_master_role"
+description "Open PBSPro Master Role"
+run_list("role[scheduler]",
+  "recipe[cshared::directories]",
+  "recipe[pbspro::skel]",
+  "recipe[cuser]",
+  "recipe[cshared::server]",
+  "recipe[pbspro::scheduler]",
+  "recipe[cganglia::server]")
+
+default_attributes "cyclecloud" => { "discoverable" => true }
diff --git a/specs/execute/chef/site-cookbooks/pbspro/README.md b/specs/execute/chef/site-cookbooks/pbspro/README.md
new file mode 100644
index 0000000..7d779fe
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/README.md
@@ -0,0 +1,29 @@
+Description
+====
+
+Requirements
+====
+
+Usage
+====
+
+
+Platform
+----
+
+Tested on:
+
+Cookbooks
+----
+
+
+Resources and Providers
+====
+
+TODO
+
+Attributes
+====
+
+TODO
+====
diff --git a/specs/execute/chef/site-cookbooks/pbspro/attributes/default.rb b/specs/execute/chef/site-cookbooks/pbspro/attributes/default.rb
new file mode 100644
index 0000000..6dfd8a9
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/attributes/default.rb
@@ -0,0 +1,35 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+default[:pbspro][:version] = "18.1.4-0"
+default[:pbspro][:slots] = nil
+
+default[:pbspro][:is_grouped] = true
+
+default[:pbspro][:submit_hook][:__comment__] = "This file was generated by serializing node[:cyclecloud][:pbspro][:submit_hook]."
+default[:pbspro][:submit_hook][:disable_eager_packing] = true
+default[:pbspro][:submit_hook][:enabled] = true
+
+default[:pbspro][:submit_hook][:logging][:level] = "INFO"
+default[:pbspro][:submit_hook][:logging][:filename] = "#{node[:cyclecloud][:bootstrap]}/pbs/submit_hook.log"
+default[:pbspro][:submit_hook][:logging][:filemode] = "a"
+default[:pbspro][:submit_hook][:logging][:format] = "%(asctime)s %(levelname)s %(message)s"
+
+
+default[:pbspro][:autoscale_hook][:__comment__] = "This file was generated by serializing node[:cyclecloud][:pbspro][:autostart_hook]."
+default[:pbspro][:autoscale_hook][:src_dirs] = ["#{node[:cyclecloud][:home]}/system/embedded/lib/python2.7/site-packages",
+                                                "#{node[:cyclecloud][:bootstrap]}/pbs"
+                                            ]
+# pass through to the hook config json file
+default[:pbspro][:autoscale_hook][:cyclecloud_home] = node[:cyclecloud][:home]
+default[:pbspro][:autoscale_hook][:autostart_log_level] = "DEBUG"
+default[:pbspro][:autoscale_hook][:autostart_log_file_level] = "DEBUG"
+
+if node[:cyclecloud][:node][:template] == "master"
+	default[:cyclecloud][:cluster][:autoscale][:idle_time_before_jobs] = 3600
+	default[:cyclecloud][:cluster][:autoscale][:idle_time_after_jobs] = 300
+else
+	# the pbs autoscaler should be removing idle nodes, but if after an hour of idle time assume something is wrong with the autoscaler.
+	default[:cyclecloud][:cluster][:autoscale][:idle_time_before_jobs] = 4 * 3600
+	default[:cyclecloud][:cluster][:autoscale][:idle_time_after_jobs] = 4 * 3600
+end
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/autostart.py b/specs/execute/chef/site-cookbooks/pbspro/files/default/autostart.py
new file mode 100644
index 0000000..53dc485
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/autostart.py
@@ -0,0 +1,606 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+import logging_init  # import first to ensure other modules (requests) don't define logging.basicConfig first
+
+import collections
+import json
+import os
+import sys
+import traceback
+
+from cyclecloud import machine, clustersapi, autoscale_util, autoscaler as autoscalerlib,\
+    nodearrays
+from cyclecloud.autoscale_util import Record
+import cyclecloud.config
+from cyclecloud.job import Job, PackingStrategy
+import mockpbs
+import pbs_driver
+from pbscc import InvalidSizeExpressionError
+import pbscc
+from copy import deepcopy
+import numbers
+
+
+class PBSAutostart:
+    '''
+    1) Converts existing pbsnodes into cyclecloud.machine.Machine instances
+    2) Performs a 'true up' of the machine instances reported by the autoscale lib vs the machines discovered in step 1) using InstanceId.
+    3) Converts PBS jobs into cyclecloud.job.Job instances.
+    4) Uses the current state of the cluster (steps 1 and 2) to construct a cyclecloud.autoscaler.Autoscaler instance.
+    5) Feeds in jobs from step 3) into the Autoscaler, which performs a demand calculation of exactly which new machines are needed and which are idle.
+    6) Does a soft shutdown of idle nodes - first it sets them offline - `pbsnodes -o` - then, on subsequent iterations, if no jobs running on the node it
+       performs a shutdown.
+       
+       
+       To disable
+       sudo qmgr -c 'set hook autoscale enabled=false'
+    
+    '''
+    
+    def __init__(self, driver, clusters_api, cc_config, clock=None):
+        self.cc_config = cc_config
+        self.disable_grouping = cc_config.get("cyclecloud.cluster.autoscale.use_node_groups", True) is not True
+        self.driver = driver
+        self.clusters_api = clusters_api
+        self.clock = clock or Clock()
+        self.default_placement_attrs = self.cc_config.get("cyclecloud.placement_group.defaults", {"group_id": "single"})
+        
+    def query_jobs(self):
+        '''
+            Converts PBS jobs into cyclecloud.job.Job instances. It will also compress jobs that have the exact same requirements.
+        '''
+        scheduler_config = self.driver.scheduler_config()
+        scheduler_resources = [] + scheduler_config["resources"]
+        # special case for hostname so we can automatically place jobs onto the appropriate host
+        scheduler_resources.append("hostname")
+        scheduler_resources.append("instance_id")
+        
+        group_jobs = not self.disable_grouping
+        running_autoscale_jobs = []
+        idle_autoscale_jobs = []
+    
+        # get the raw string outputs first, and convert it second. This somewhat limits the
+        # race condition of asking the status of the queue twice.
+        running_raw_jobs_str, running_converter = self.driver.running_jobs()
+        queued_raw_jobs_str, queued_converter = self.driver.queued_jobs()
+        
+        running_raw_jobs = running_converter(running_raw_jobs_str)
+        queued_raw_jobs = queued_converter(queued_raw_jobs_str)
+        
+        raw_jobs = []
+        
+        for raw_job in running_raw_jobs:
+            # it is only running on a single node
+            if '+' not in raw_job["exec_vnode"]:
+                raw_jobs.append(raw_job)
+                continue
+            
+            for vnode in raw_job["exec_vnode"].split("+"):
+                sub_raw_job = deepcopy(raw_job)
+                sub_raw_job["exec_vnode"] = vnode
+                raw_jobs.append(sub_raw_job)
+        
+        for raw_job in queued_raw_jobs:
+            if not raw_job["resource_list"].get("select"):
+                raw_jobs.append(raw_job)
+            else:
+                # pbspro, like many schedulers, allows a varying set requirements for nodes in a single submission.
+                # we will break it apart here as if they had split them individually.
+                
+                place = raw_job["resource_list"].get("place")
+                slot_type = raw_job["resource_list"].get("slot_type")
+
+                chunks = pbscc.parse_select(raw_job)
+                for n, chunk in enumerate(chunks):
+                    # only pay the penalty of copies when we actually have multi-chunk jobs
+                    sub_raw_job = deepcopy(raw_job)
+
+                    if len(chunks) > 1:
+                        sub_raw_job["job_id"] = "%s.%d" % (sub_raw_job["job_id"], n) 
+                    
+                    sub_raw_job["resource_list"] = {}
+                    if place:
+                        sub_raw_job["resource_list"]["place"] = place
+                        
+                    if slot_type:
+                        sub_raw_job["resource_list"]["slot_type"] = slot_type
+                        
+                    sub_raw_job["resource_list"]["select"] = pbscc.format_select(chunk)
+                    chunk["nodect"] = int(chunk["select"])
+                    if "ncpus" not in chunk:
+                        chunk["ncpus"] = "1"
+                    
+                    for key, value in chunk.iteritems():
+                        if key not in ["select", "nodect"]:
+                            try:
+                                value = pbscc.parse_gb_size(key, value) * chunk["nodect"]
+                            except InvalidSizeExpressionError:
+                                pass
+                            
+                            sub_raw_job["resource_list"][key] = value
+                    
+                    sub_raw_job["nodect"] = sub_raw_job["resource_list"]["nodect"] = chunk["nodect"]
+                    raw_jobs.append(sub_raw_job)
+        
+        warnings = set()
+        raw_jobs = [x for x in raw_jobs if x["job_state"].upper() in [pbscc.JOB_STATE_QUEUED,
+                                                                      pbscc.JOB_STATE_RUNNING,
+                                                                      pbscc.JOB_STATE_BATCH]]
+        
+        for raw_job in raw_jobs:
+            pbs_job = mockpbs.mock_job(raw_job)
+            nodect = int(pbs_job.Resource_List["nodect"])
+            
+            if pbs_job["job_state"].upper() == pbscc.JOB_STATE_RUNNING:
+                # update running job
+                live_resources = pbscc.parse_exec_vnode(raw_job["exec_vnode"])
+                for key, value in live_resources.iteritems():
+                    # live resources are calculated on a per node basis, but the Resource_List is based
+                    # on a total basis.
+                    # we will normalize this below
+                    
+                    if isinstance(value, numbers.Number):
+                        pbs_job.Resource_List[key] = value * nodect
+                    else:
+                        pbs_job.Resource_List[key] = value
+                pbs_job["executing_hostname"] = live_resources["hostname"]
+                
+            is_array = bool(pbs_job.get("array", False))
+            
+            slots_per_job = int(pbs_job.Resource_List['ncpus']) / nodect 
+            slot_type = pbs_job.Resource_List["slot_type"]  # can be None, similar to {}.get("key"). It is a pbs class.
+            pbscc.info("found slot_type %s." % slot_type)
+            
+            placement = pbscc.parse_place(pbs_job.Resource_List.get("place"))
+                    
+            # Note: not sure we will ever support anything but group_id for autoscale purposes.
+            # User could pick, say, group=host, which implies an SMP job, not a parallel job.
+            
+            if placement.get("grouping", "group=group_id") != "group=group_id":
+                placement.pop("grouping")
+
+            if placement.get("arrangement", "").lower() in ["scatter", "vscatter"]:
+                pack = "scatter"
+            else:
+                pack = "pack"
+            
+            exclusive = placement.get("sharing", "").lower() in ["excl", "exclhost"]
+            # we may need to support sharing at some point, but it seems that we can ignore it for now.
+            _shared = placement.get("sharing") in ["sharing"]
+            placeby = placement.get("grouping")
+            
+            autoscale_job = Job(name=pbs_job["job_id"],
+                                nodearray=slot_type,
+                                nodes=nodect,
+                                packing_strategy=pack,
+                                exclusive=exclusive,
+                                resources={"ncpus": 0},
+                                executing_hostname=pbs_job.get("executing_hostname"))
+
+            if placeby:
+                autoscale_job.placeby = placeby.split("=", 1)[-1]
+            
+            if is_array:
+                array_count = 0
+                array_tasks = raw_job["array_state_count"]
+
+                # Only grab the first two array task states (queued and running)
+                for ajob in str(array_tasks).split(" ")[:2]:
+                    array_count += int(ajob.split(":")[1])
+
+                # Multiply the number of cpus needed by number of tasks in the array
+                if array_count != 0:
+                    slots_per_job *= array_count
+            else:
+                array_count = 1
+                    
+            # If it's an MPI job and grouping is enabled
+            # we want to use a grouped autoscale_job to get tightly coupled nodes
+
+            if group_jobs and placement.get("grouping"): 
+                autoscale_job['grouped'] = True
+                autoscale_job["nodes"] *= array_count
+                autoscale_job.placeby_value = "single"
+            elif is_array:
+                autoscale_job["nodes"] *= array_count
+
+            autoscale_job.ncpus += slots_per_job
+            
+            for attr, value in pbs_job.Resource_List.iteritems():
+                if attr not in scheduler_resources:
+                    # if it isn't a scheduler level attribute, don't bother 
+                    # considering it for autoscale as the scheduler won't respect it either.
+                    continue
+                try:
+                    value = pbscc.parse_gb_size(attr, value)
+                    value = value / nodect
+                except InvalidSizeExpressionError:
+                    if value.lower() in ["true", "false"]:
+                        value = value.lower() == "true"
+
+                autoscale_job.resources[attr] = value
+                
+            if raw_job["job_state"] == pbscc.JOB_STATE_QUEUED:
+                idle_autoscale_jobs.append(autoscale_job)
+            else:
+                running_autoscale_jobs.append(autoscale_job)
+                
+        for warning in warnings:
+            format_string, values = warning[0], warning[1:]
+            pbscc.error(format_string % values)
+        
+        # leave an option for disabling this in case it causes issues.
+        if self.cc_config.get("pbspro.compress_jobs", False):
+            all_autoscale_jobs = running_autoscale_jobs + compress_queued_jobs(idle_autoscale_jobs)
+        else:
+            all_autoscale_jobs = running_autoscale_jobs + idle_autoscale_jobs
+            
+        return all_autoscale_jobs
+    
+    def fetch_nodearray_definitions(self):
+        '''
+            A wrapper around the autoscale library function to parse Configuration.autoscale.* chef attributes and add
+            the 'ungrouped' attribute to the machine types.
+            
+            See cyclecloud.nodearrays.NodearrayDefinitions for more info.
+        '''
+        nodearray_definitions = machine.fetch_nodearray_definitions(self.clusters_api, self.default_placement_attrs)
+        nodearray_definitions.placement_group_optional = True
+
+        filtered_nodearray_definitions = nodearrays.NodearrayDefinitions()
+        
+        for machinetype in nodearray_definitions:
+            if machinetype.get("disabled", False):
+                continue
+                
+            # ensure that any custom attribute the user specified, like disk = 100G, gets parsed correctly
+            for key, value in machinetype.iteritems():
+                try:
+                    machinetype[key] = pbscc.parse_gb_size(key, value)
+                except InvalidSizeExpressionError:
+                    pass
+                
+            # kludge: there is a strange bug where ungrouped is showing up as a string and not a boolean.
+            if not machinetype.get("group_id"):
+                machinetype["ungrouped"] = "true"
+                filtered_nodearray_definitions.add_machinetype(machinetype)
+            else:
+                machinetype["ungrouped"] = "false"
+                filtered_nodearray_definitions.add_machinetype_with_placement_group(machinetype.get("group_id"), machinetype)
+            
+        return filtered_nodearray_definitions
+                
+    def autoscale(self):
+        '''
+            The main loop described at the top of this class. 
+            Returns machine_requests, idle_machines and total_machines for ease of unit testing.
+        '''
+        pbscc.info("Begin autoscale cycle")
+        
+        nodearray_definitions = self.fetch_nodearray_definitions()
+        
+        pbsnodes_by_hostname, existing_machines, booting_instance_ids, instance_ids_to_shutdown = self.get_existing_machines(nodearray_definitions)
+        
+        start_enabled = "true" == str(self.cc_config.get("cyclecloud.cluster.autoscale.start_enabled", "true")).lower()
+        
+        if not start_enabled:
+            pbscc.warn("cyclecloud.cluster.autoscale.start_enabled is false, new machines will not be allocated.")
+        
+        autoscaler = autoscalerlib.Autoscaler(nodearray_definitions, existing_machines, self.default_placement_attrs, start_enabled)
+        
+        # throttle how many jobs we attempt to match. When pbspro.compress_jobs is true (default) this shouldn't really be an issue
+        # unless the user has over $pbspro.max_unmatched_jobs unique sets of requirements.
+        max_unmatched_jobs = int(self.cc_config.get("pbspro.max_unmatched_jobs", 10000))
+        unmatched_jobs = 0
+        
+        for job in self.query_jobs():
+            if job.executing_hostname:
+                try:
+                    autoscaler.get_machine(hostname=job.executing_hostname).add_job(job, force=True)
+                    continue
+                except RuntimeError as e:
+                    pbscc.error(str(e))
+                    pass
+                    
+            if not autoscaler.add_job(job):
+                unmatched_jobs += 1
+                pbscc.info("Can not match job %s." % job.name)
+                if max_unmatched_jobs > 0 and unmatched_jobs >= max_unmatched_jobs:
+                    pbscc.warn('Maximum number of unmatched jobs reached - %s. To configure this setting, change {"pbspro": "max_unmatched_jobs": N}} in %s' % (unmatched_jobs, pbscc.CONFIG_PATH))
+                    break
+        
+        machine_requests = autoscaler.get_new_machine_requests()
+        idle_machines = autoscaler.get_idle_machines()
+        
+        autoscale_request = autoscale_util.create_autoscale_request(machine_requests)
+        for request_set in autoscale_request["sets"]:
+            configuration = request_set["nodeAttributes"]["Configuration"]
+            
+            if "pbspro" not in configuration:
+                    configuration["pbspro"] = {}
+            
+            configuration["pbspro"]["slot_type"] = request_set["nodearray"]
+            if not request_set.get("placementGroupId"):
+                configuration["pbspro"]["is_grouped"] = False
+            else:
+                configuration["pbspro"]["is_grouped"] = True
+                
+        autoscale_util.scale_up(self.clusters_api, autoscale_request)
+        
+        for r in machine_requests:
+            if r.placeby_value:
+                pbscc.info("Requesting %d %s machines in placement group %s for nodearray %s" % (r.instancecount, r.machinetype, r.placeby_value, r.nodearray))
+            else:
+                pbscc.info("Requesting %d %s machines in nodearray %s" % (r.instancecount, r.machinetype, r.nodearray))
+        
+        if pbscc.is_fine():
+            pbscc.fine("New target state of the cluster, including booting nodes:")
+            
+            for m in autoscaler.machines:
+                pbscc.fine("    %s" % str(m))
+        
+        if instance_ids_to_shutdown:
+            pbscc.info("Shutting down instance ids %s" % instance_ids_to_shutdown.keys())
+            self.clusters_api.shutdown(instance_ids_to_shutdown.keys())
+            
+            for hostname in instance_ids_to_shutdown.itervalues():
+                pbscc.info("Deleting %s" % hostname)
+                self.driver.delete_host(hostname)
+        
+        now = self.clock.time()
+        
+        stop_enabled = "true" == str(self.cc_config.get("cyclecloud.cluster.autoscale.stop_enabled", "true")).lower()
+        
+        if not stop_enabled:
+            pbscc.warn("cyclecloud.cluster.autoscale.stop_enabled is false, idle machines will not be terminated")
+        
+        if stop_enabled:
+            idle_before_threshold = float(self.cc_config.get("cyclecloud.cluster.autoscale.idle_time_before_jobs", 3600))
+            idle_after_threshold = float(self.cc_config.get("cyclecloud.cluster.autoscale.idle_time_after_jobs", 300))
+        
+            for m in idle_machines:
+                if m.get_attr("instance_id", "") not in booting_instance_ids:
+                    pbscc.debug("Could not find instance id in CycleCloud %s" % m.get_attr("instance_id", ""))
+                    continue
+                
+                pbsnode = pbsnodes_by_hostname.get(m.hostname)
+                
+                # the machine may not have converged yet, so
+                if pbsnode:
+                    if "busy" in pbsnode["state"]:
+                        if "down" in pbsnode["state"]:
+                            pbscc.warn("WARNING: %s is down but busy with jobs %s", m.hostname, pbsnode.get("jobs", []))
+                        else:
+                            pbscc.error("WARNING: Falsely determined that %s is idle!" % m.hostname)
+                        continue
+                    
+                    last_state_change_time = pbsnode["last_state_change_time"]
+                    last_used_time = pbsnode.get("last_used_time")
+                    if last_used_time:
+                        # last_used_time can be stale while a job is exiting, e.g. last_state_change_time could be < 5 minutes but
+                        # somehow last_used_time > 5 minutes, causing us to prematurely terminate the node just because a job took a long time
+                        # to exit.
+                        last_used_time = max(last_state_change_time, last_used_time)
+                    else:
+                        last_used_time = self.clock.time()
+
+                    if now - last_used_time > idle_after_threshold:
+                        pbscc.info("Setting %s offline after %s seconds" % (m.hostname, now - last_used_time))
+                        self.driver.set_offline(m.hostname)
+                    elif now - last_state_change_time > idle_before_threshold:
+                        pbscc.info("Setting %s offline after %s seconds" % (m.hostname, now - last_state_change_time))
+                        self.driver.set_offline(m.hostname)
+        
+        pbscc.info("End autoscale cycle")
+        # returned for testing purposes
+        return machine_requests, idle_machines, autoscaler.machines
+    
+    def get_existing_machines(self, nodearray_definitions):
+        '''
+            Queries pbsnodes and CycleCloud to get a sane set of cyclecloud.machine.Machine instances that represent the current state of the cluster.
+        '''
+        pbsnodes = self.driver.pbsnodes().get(None)
+        existing_machines = []
+        
+        booting_instance_ids = autoscale_util.nodes_by_instance_id(self.clusters_api, nodearray_definitions)
+        
+        instance_ids_to_shutdown = Record()
+        
+        nodes_by_instance_id = Record()
+        
+        for pbsnode in pbsnodes.values():
+            inst = self.process_pbsnode(pbsnode, instance_ids_to_shutdown, nodearray_definitions)
+            if not inst:
+                continue
+            existing_machines.append(inst)
+            
+            # we found the pbsnode that matches the cyclecloud node, so let's remove the duplicate 
+            instance_id = inst.get_attr("instance_id", "")
+            if instance_id in booting_instance_ids:
+                booting_instance_ids.pop(instance_id)
+            nodes_by_instance_id[instance_id] = pbsnode
+        
+        for instance_id, node in list(booting_instance_ids.iteritems()):
+            nodearray_name = node["Template"]
+            machinetype_name = node["MachineType"]
+            
+            try:
+                machinetype = nodearray_definitions.get_machinetype(nodearray_name, machinetype_name, node.get("PlacementGroupId"))
+            except KeyError as e:
+                raise ValueError("machine is %s, key is %s, rest is %s" % (nodearray_name, str(e), nodearray_definitions))
+            
+            inst = machine.new_machine_instance(machinetype, hostname=node.get("hostname"), instance_id=node.get("InstanceId"), group_id=node.get("placementGroupId"))
+            existing_machines.append(inst)
+            nodes_by_instance_id[instance_id] = node
+        
+        return pbsnodes, existing_machines, nodes_by_instance_id, instance_ids_to_shutdown
+    
+    def process_pbsnode(self, pbsnode, instance_ids_to_shutdown, nodearray_definitions):
+        '''
+            If the pbsnode is offline, will handle evaluating whether the node can be shutdown. See instance_ids_to_shutdown, which
+            is an OUT parameter here.
+            
+            Otherwise convert the pbsnode into a cyclecloud.machine.Machine instance.
+        '''
+        
+        states = set(pbsnode["state"].split(","))
+        resources = pbsnode["resources_available"]
+        # host has incorrect case
+        hostname = resources["vnode"]
+        
+        instance_id = resources.get("instance_id", autoscale_util.uuid("instanceid"))
+        
+        def try_shutdown_pbsnode():
+            if not instance_id:
+                pbscc.error("instance_id was not defined for host %s, can not shut it down" % hostname)
+            elif "down" in states:
+                # don't immediately remove down nodes, give them time to recover from network failure.
+                remove_down_nodes = float(self.cc_config.get("pbspro.remove_down_nodes", 300))
+                since_down = self.clock.time() - pbsnode["last_state_change_time"]
+                if since_down > remove_down_nodes:
+                    pbscc.error("Removing down node %s after %.0f seconds", hostname, since_down)
+                    instance_ids_to_shutdown[instance_id] = hostname
+                    return True
+                else:
+                    omega = remove_down_nodes - since_down
+                    pbscc.warn("Not removing down node %s for another %.0f seconds", hostname, omega)
+            else:
+                instance_ids_to_shutdown[instance_id] = hostname
+                return True
+            
+            return False
+        
+        if "offline" in states:
+            if not pbsnode.get("jobs", []):
+                pbscc.fine("%s is offline and has no jobs, may be able to shut down" % hostname)
+                if try_shutdown_pbsnode():
+                    return
+            else:
+                pbscc.fine("Host %s is offline but still running jobs" % hostname)
+        
+        # if the node is just in the down state, try to shut it down. 
+        if set(["down"]) == states and try_shutdown_pbsnode():
+            return
+        
+        # just ignore complex down nodes (down,job-busy etc) until PBS decides to change the state.
+        if "down" in states:
+            return
+        
+        # convert relevant resources from bytes to floating point (GB)
+        for key in resources:
+            
+            value = resources[key]
+            if isinstance(value, basestring) and value.lower() in ["true", "false"]:
+                value = value.lower() == "true"
+            elif isinstance(value, list):
+                # TODO will need to support this eventually
+                continue
+            else:
+                try:
+                    resources[key] = pbscc.parse_gb_size(key, resources[key])
+                except InvalidSizeExpressionError:
+                    pass
+        
+        resources["hostname"] = hostname
+        
+        nodearray_name = resources.get("nodearray") or resources.get("slot_type")
+        group_id = resources.get("group_id")
+
+        if resources.get("machinetype") and nodearray_name:
+            machinetype = nodearray_definitions.get_machinetype(nodearray_name, resources.get("machinetype"), group_id)
+        else:
+            # rely solely on resources_available
+            pbscc.debug("machinetype is not defined for host %s, relying only on resources_available" % hostname)
+            machinetype = {"availableCount": 1, "name": "undefined"}
+            
+        inst = machine.new_machine_instance(machinetype, **pbsnode["resources_available"])
+
+        return inst
+    
+
+def compress_queued_jobs(autoscale_jobs):
+    '''
+        assuming nodearray, num nodes, placeby/placeby_value, exclusivity, packing stategy and of course the requested resources.
+        
+        A compressed job will have the name of the first job with ".compressed" appended to it.
+        
+        i.e. the goal is that if someone does
+        `qsub -l mem=1g` 
+        1000 times, we can compress that so that it appears as if they did
+        `qsub -l select=1000:mem=1g`
+        
+        We do not compress SCATTER jobs.
+    '''
+    ret = []
+    compression_buckets = collections.defaultdict(lambda: [])
+    
+    for job in autoscale_jobs:
+        if job.packing_strategy == PackingStrategy.SCATTER:
+            # for now, we will only worry about compressing PACK jobs as an optimization.
+            ret.append(job)
+            continue
+        
+        comp_bucket = (job.nodearray, job.nodes, job.placeby, job.placeby_value, job.exclusive, job.packing_strategy) + tuple(job.resources.items())
+        compression_buckets[comp_bucket].append(job)
+        
+    for comp_bucket, job_list in compression_buckets.iteritems():
+        if len(job_list) == 1:
+            ret.extend(job_list)
+            continue
+        
+        first_job = job_list[0]
+        pseudo_job = Job(first_job.name + ".compressed", first_job.nodes * len(job_list), first_job.nodearray, first_job.exclusive, first_job.packing_strategy,
+                         first_job.resources, first_job.placeby, first_job.placeby_value)
+        ret.append(pseudo_job)
+        
+        pbscc.debug("Compressed jobs matching job id %s from %d jobs down to a single job" % (first_job.name, len(job_list)))
+    
+    return ret
+
+
+class Clock:
+    
+    def time(self):
+        import time
+        return time.time()
+
+
+def _hook():
+    pbscc.set_application_name("cycle_autoscale")
+    # allow local overrides of jetpack.config or allow non-jetpack masters to define the complete set of settings.
+    overrides = {}
+    
+    if os.path.exists(pbscc.CONFIG_PATH):
+        try:
+            pbscc.warn("overrides exist in file %s" % pbscc.CONFIG_PATH)
+            with open(pbscc.CONFIG_PATH) as fr:
+                overrides = json.load(fr)
+        except Exception:
+            pbscc.error(traceback.format_exc())
+            sys.exit(1)
+    else:
+        pbscc.debug("No overrides exist in file %s" % pbscc.CONFIG_PATH)
+    
+    cc_config = cyclecloud.config.new_provider_config(overrides=overrides)
+    
+    if len(sys.argv) < 3:
+        # There are no env variables for this as far as I can tell.
+        bin_dir = "/opt/pbs/bin"
+    else:
+        bin_dir = sys.argv[2]
+        if not os.path.isdir(bin_dir):
+            bin_dir = os.path.dirname(bin_dir)
+    
+    clusters_api = clustersapi.ClustersAPI(cc_config.get("cyclecloud.cluster.name"), cc_config)
+    autostart = PBSAutostart(pbs_driver.PBSDriver(bin_dir), clusters_api, cc_config=cc_config)
+    
+    autostart.autoscale()
+
+
+# Since this is invoked from a hook, __name__ is not "__main__", so we rely on a special env variable. Otherwise unit testing would be impossible.
+if os.getenv("AUTOSTART_HOOK"):
+    try:
+        _hook()
+    except:
+        pbscc.error(traceback.format_exc())
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/autostart_hook.py b/specs/execute/chef/site-cookbooks/pbspro/files/default/autostart_hook.py
new file mode 100644
index 0000000..07d528a
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/autostart_hook.py
@@ -0,0 +1,61 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+import json
+import os
+import subprocess
+import traceback
+
+try:
+    import pbs
+except ImportError:
+    import mockpbs as pbs
+
+
+def perform_hook():
+    """
+        See /var/spool/pbs/server_logs/* or /opt/cycle/jetpack/logs/autoscale.log for log messages
+    """
+    try:
+        if not pbs.hook_config_filename:
+            raise RuntimeError("Hook config for this plugin was not defined.")
+
+        with open(pbs.hook_config_filename) as fr:
+            hook_config = json.load(fr)
+            
+        log_dir = os.path.join(hook_config.get("cyclecloud_home"), "logs")
+        
+        if not os.path.exists(log_dir):
+            log_dir = os.getcwd()
+        
+        env_with_src_dirs = {"PYTHONPATH": os.pathsep.join(hook_config.get("src_dirs", [])),
+                             "AUTOSTART_HOOK": "1",
+                             "AUTOSTART_LOG_FILE": os.path.join(log_dir, "autoscale.log"),
+                             "AUTOSTART_LOG_FILE_LEVEL": hook_config.get("autostart_log_file_level") or "DEBUG",
+                             "AUTOSTART_LOG_LEVEL": hook_config.get("autostart_log_level") or "DEBUG"}
+
+        jetpack_config = hook_config.get("jetpack_python")
+        if not jetpack_config:
+            if not hook_config.get("cyclecloud_home"):
+                raise RuntimeError("Please specify either jetpack_python or cyclecloud_home in the hook config.")
+
+            hook_config["jetpack_python"] = os.path.join(hook_config["cyclecloud_home"], "system", "embedded", "bin", "python")
+
+        pbs_bin_dir = os.path.join(pbs.pbs_conf["PBS_EXEC"], "bin")
+
+        cmd = [hook_config["jetpack_python"], "-m", "autostart", pbs.hook_config_filename, pbs_bin_dir]
+        
+        pbs.logmsg(pbs.LOG_DEBUG, "Running %s with environment %s" % (cmd, env_with_src_dirs))
+        
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env_with_src_dirs)
+        stdout, stderr = proc.communicate()
+        pbs.logmsg(pbs.LOG_DEBUG, stderr)
+        if proc.returncode != 0:
+            raise RuntimeError('autostart failed!\n\tstdout="%s"\n\tstderr="%s"' % (stdout, stderr))
+    except:
+        pbs.logmsg(pbs.LOG_ERROR, traceback.format_exc())
+        raise
+
+# hooks must not have a __name__ == "__main__" guard
+perform_hook()
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/autostart_test.py b/specs/execute/chef/site-cookbooks/pbspro/files/default/autostart_test.py
new file mode 100644
index 0000000..b3af822
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/autostart_test.py
@@ -0,0 +1,1210 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+import logging_init
+import unittest
+
+from autostart import PBSAutostart
+from cyclecloud import machine, autoscale_util
+from cyclecloud.job import Job
+from cyclecloud.machine import MachineRequest
+from cyclecloud.autoscale_util import Record
+import time
+from itertools import chain
+from cyclecloud.config import InstanceConfig
+import random
+import pbscc
+from pbscc import InvalidSizeExpressionError
+
+
+class MockDriver:
+    def __init__(self, queues=None, jobs=None, hosts=None):
+        self._queues = queues or ["workq"]
+        self._jobs = jobs or {}
+        assert isinstance(self._jobs, dict)
+        self._hosts = hosts or []
+        self._declared_resources = {"resources": ["ncpus", "mem", "arch", "host", "vnode", "aoe", "slot_type", 
+                                                  "group_id", "ungrouped", "instance_id", "ipv4", "disk", "scratch",
+                                                  "swlicense", "graphics", "dyna"]}
+        
+    def queues(self):
+        return self._queues
+    
+    def queued_jobs(self):
+        # TODO multiple queues
+        return [x for x in self._jobs.get("workq", []) if x["job_state"] == "Q"], lambda x: x
+    
+    def running_jobs(self):
+        return [x for x in self._jobs.get("workq", []) if x["job_state"] == "R"], lambda x: x
+    
+    def scheduler_config(self):
+        return self._declared_resources
+    
+    def pbsnodes(self, grouping=None):
+        ret = {None: {}}
+        for host in self._hosts:
+            state = host.get(grouping)
+            if state not in ret:
+                ret[state] = {}
+            ret[state][host["resources_available"]["vnode"]] = host
+        return ret
+    
+    def set_offline(self, hostname):
+        try:
+            host = self.get_host(hostname)
+        except:
+            return
+        if "offline" not in host["state"]:
+            if host["state"] == "free":
+                host["state"] = "offline"
+            else:
+                host["state"] = host["state"] + ",offline"
+                    
+    def delete_host(self, hostname):
+        self._hosts = [x for x in self._hosts if x["resources_available"]["vnode"] != hostname]
+        
+    def get_host(self, hostname):
+        select = [x for x in self._hosts if x["resources_available"]["vnode"].lower() == hostname.lower()]
+        assert len(select) == 1, "%s %s %s" % (select, hostname, [x["hostname"].lower() for x in self._hosts])
+        return select[0]
+        
+    def add_host(self, hostname, state="free", **resources):
+        host = {"host": hostname, "hostname": hostname, "state": state}
+        resources_assigned = resources.pop("resources_assigned", {})
+        if state == "job-busy" and not resources_assigned:
+            raise RuntimeError("define resources_assigned")
+        
+        host["resources_available"] = resources
+        host["resources_available"]["host"] = hostname.upper()
+        host["resources_available"]["vnode"] = host["resources_available"]["host"].lower() 
+        host["resources_assigned"] = resources_assigned
+        host["jobs"] = resources.pop("jobs", [])
+        host["last_state_change_time"] = time.time()
+        self._hosts.append(host)
+        
+
+class MockClustersAPI:
+    
+    def __init__(self, cluster_def, nodes=None):
+        self.cluster_def = cluster_def
+        nodes = nodes or {}
+        self._nodes = nodes
+        if not hasattr(nodes, "keys"):
+            self._nodes = {"execute": []}
+            for node in nodes:
+                # node.get_attr("instance_id", autoscale_util.uuid("instance_id"))] = node
+                self._nodes["execute"].append(node)
+        
+    def status(self, nodes=False):
+        if nodes:
+            self.cluster_def["nodes"] = nodes = []
+            for node_list in self._nodes.itervalues():
+                nodes.extend(node_list)
+        else:
+            self.cluster_def.pop("nodes", [])
+        return self.cluster_def
+    
+    def nodes(self):
+        for key in self._nodes:
+            random.shuffle(self._nodes[key])
+        return self._nodes
+    
+    def add_nodes(self, request):
+        for request_set in request["sets"]:
+            for _ in range(request_set["count"]):
+                select = [x for x in self.cluster_def["nodearrays"] if x["name"] == request_set["nodearray"]]
+                assert select, "%s not in %s" % (request_set["nodearray"], [x["name"] for x in self.cluster_def["nodearrays"]])
+                nodearray = select[0]
+                select = [b for b in nodearray["buckets"] if b["definition"] == request_set["definition"]]
+                assert select, "No matching bucket found for request %s" % request
+                mt = select[0]["virtualMachine"]
+                mt_name = select[0]["definition"]["machineType"]
+                node = Record({"vcpuCount": mt["vcpuCount"],
+                               "memory": mt["Memory"], 
+                               "machineType": mt_name,
+                               "placementGroupId": request_set.get("placementGroupId")})
+                node.update(request_set["nodeAttributes"])
+                node["nodearray"] = request_set["nodearray"]
+                node["Template"] = request_set["nodearray"]
+                node["InstanceId"] = autoscale_util.uuid("instance")
+                
+                if node["nodearray"] not in self._nodes:
+                    self._nodes[node["nodearray"]] = []
+                self._nodes[node["nodearray"]].append(node)
+                
+    def shutdown(self, instance_ids):
+        instance_ids = list(instance_ids)
+        for key in self._nodes:
+            self._nodes[key] = [x for x in self._nodes[key] if x["InstanceId"] not in instance_ids]
+
+
+def _nodearray_definitions(*machinetypes):
+    machinetypes = sorted(machinetypes, key=lambda x: -x.get("priority", 100))
+    cluster_status = {}
+    ret = {}
+    
+    for machinetype in machinetypes:
+        nodearray = machinetype.get("nodearray")
+        if nodearray not in ret:
+            ret[nodearray] = {"nodearray": {},
+                              "name": nodearray,
+                              "buckets": []}
+        
+        name = machinetype.get("machinetype")
+        ret[nodearray]["buckets"].append({"definition": {"machineType": name},
+                                          "virtualMachine": machinetype})
+
+    cluster_status["nodearrays"] = ret.values()
+    return cluster_status
+
+
+class MockClock:
+    def __init__(self, now=0):
+        self.now = now
+        
+    def time(self):
+        return self.now
+    
+    
+class PBSQ:
+    
+    def __init__(self):
+        self.queues = {"workq": []}
+        self.job_id = 1
+        self.hosts = []
+    
+    def qsub(self, job_id=None, job_state="Q", J=None, select_expr=None, **resource_list):
+        assert "select" not in resource_list
+        if not job_id:
+            job_id = str(self.job_id)
+            self.job_id += 1
+            
+        jobdef = {"resource_list": {},
+                  "job_id": job_id,
+                  "job_state": job_state}
+        
+        for key, value in resource_list.iteritems():
+            jobdef["resource_list"][key] = value
+            
+        jobdef["resource_list"]["nodect"] = 1      
+            
+        if "job_state" not in jobdef:
+            jobdef["job_state"] = "Q"
+            
+        if "ncpus" not in jobdef["resource_list"]:
+            jobdef["resource_list"]["ncpus"] = jobdef.get("nodes", 1)
+        
+        if J:
+            jobdef["array"] = True
+            jobdef["array_state_count"] = J
+            
+        if select_expr:
+            jobdef["resource_list"]["select"] = select_expr
+            chunk_totals = {}
+            for chunk in pbscc.parse_select(jobdef):
+                chunk["ncpus"] = chunk.get("ncpus", 1)
+                for key, value in chunk.iteritems():
+                    if key == "select":
+                        continue
+                    try:
+                        if key == "nodect":
+                            value = pbscc.parse_gb_size(key, value)
+                        else:
+                            value = pbscc.parse_gb_size(key, value) * int(chunk["select"])
+                        chunk_totals[key] = chunk_totals.get(key, 0) + value
+                    except InvalidSizeExpressionError:
+                        chunk_totals[key] = value
+                        
+            for key, value in chunk_totals.iteritems():
+                if key != "select":
+                    jobdef["resource_list"][key] = value
+        
+        self.queues["workq"].append(jobdef)
+        
+    def qdel(self, jobid):
+        to_delete = [x for x in self.queues["workq"] if x["job_id"] == str(jobid)]
+        assert len(to_delete) >= 1
+        self.queues["workq"] = [x for x in self.queues["workq"] if x not in to_delete]
+        
+    def query_jobs(self):
+        cluster = MockClustersAPI({})
+        a = PBSAutostart(MockDriver(["workq"], self.queues), cluster, {})
+        return a.query_jobs()
+    
+    def set_running(self, job_id, exec_vnode):
+        '''
+            Testing: change the job state and set exec_vnode expression so that autoscale knows where the jobs are placed
+        '''
+        for job in chain(*self.queues.values()):
+            if str(job["job_id"]) == str(job_id):
+                job["exec_vnode"] = exec_vnode
+                job["job_state"] = "R"
+                return
+        raise AssertionError("Could not find job_id %s" % job_id)
+    
+        
+class Test(unittest.TestCase):
+    def __init__(self, methodName='runTest'):
+        unittest.TestCase.__init__(self, methodName=methodName)
+        self.maxDiff = None
+    
+    def setUp(self):
+        unittest.TestCase.setUp(self)
+        autoscale_util.set_uuid_func(autoscale_util.IncrementingUUID())
+        
+    def tearDown(self):
+        unittest.TestCase.tearDown(self)
+        autoscale_util.set_uuid_func(__import__("uuid").uuid4)
+        
+    def test_qsub(self):
+        q = PBSQ()
+        # q.qsub(nodes=2, ncpus=2)
+        # this ^ is just tranlsated into the following
+        q.qsub(select_expr="2:ncpus=2:mpiprocs=2", place="scatter")
+        q.qsub(select_expr="2:ncpus=2", place="excl")
+        
+        self.assertEquals([Job(name="1", nodes=2, packing_strategy="scatter", exclusive=False, resources={"ncpus": 2}),
+                           Job(name="2", nodes=2, packing_strategy="pack", exclusive=True, resources={"ncpus": 2})],
+                           q.query_jobs())
+        
+        # 4 cpus packed onto one machine and 2 exclusive
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 32, 100, 100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=3)], self._autoscale(q, cluster_def))
+        
+    def test_qsub_scatter_excl(self):
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 32, 100, 100))
+        
+        for strategy in ["scatter", "vscatter"]:
+            for modifier in [":excl", ":exclhost"]:
+                place = strategy + modifier
+                q = PBSQ()
+                q.qsub(select_expr="2:ncpus=2", place=place)
+                self.assertEquals([Job(name="1", nodes=2, packing_strategy="scatter", exclusive=True, resources={"ncpus": 2})],
+                                   q.query_jobs())
+                
+                self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+                
+        for strategy in ["pack", "free"]:
+            for modifier in [":excl", ":exclhost"]:
+                place = strategy + modifier
+                q = PBSQ()
+                q.qsub(select_expr="2:ncpus=2", place=place)
+                pack = "pack" if strategy in ["pack", "free"] else "scatter"
+                self.assertEquals([Job(name="1", nodes=2, packing_strategy=pack, exclusive=True, resources={"ncpus": 2})],
+                                   q.query_jobs())
+                
+                self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+                
+        for scatter in ["scatter", "vscatter"]:
+            for modifier in [":shared", ""]:
+                place = scatter + modifier
+                q = PBSQ()
+                q.qsub(select_expr="2:ncpus=2", place=place)
+                self.assertEquals([Job(name="1", nodes=2, packing_strategy="scatter", exclusive=False, resources={"ncpus": 2})],
+                                   q.query_jobs())
+                
+                self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+                
+        for scatter in ["pack", "free"]:
+            for modifier in [":shared", ""]:
+                place = scatter + modifier
+                q = PBSQ()
+                q.qsub(select_expr="2:ncpus=2", place=place)
+                self.assertEquals([Job(name="1", nodes=2, packing_strategy="pack", exclusive=False, resources={"ncpus": 2})],
+                                   q.query_jobs())
+                
+                self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+            
+    def test_pbsuserguide_too_many_cpus(self):
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a4", 32, 100, 100))
+        
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=33:mem=1G", place="pack", _can_be_added=False)
+        self.assertEquals([], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="33:ncpus=1:mem=1G", place="pack", machinetype="a4")
+        self.assertEquals([self._machine_request(machine_type="a4", count=2)], self._autoscale(q, cluster_def))
+        
+    def _rebalance(self, q, cluster_def=None, hosts=None, cc_config=None):
+        cluster_def = cluster_def or _nodearray_definitions(machine.new_machinetype("execute", "a4", 32, 100, 100))
+        pbs_autostart = PBSAutostart(MockDriver(jobs=q.queues, hosts=hosts), MockClustersAPI(cluster_def), cc_config or {})
+        return pbs_autostart.autoscale()
+    
+    def _autoscale(self, q, cluster_def=None, hosts=None, cc_config=None):
+        return self._rebalance(q, cluster_def, hosts, cc_config)[0]
+    
+    def _idle_nodes(self, q, cluster_def=None, hosts=None, cc_config=None):
+        return [x.hostname for x in self._rebalance(q, cluster_def, hosts, cc_config)[1]]
+    
+    def _machine_request(self, nodearray=None, machine_type="a4", count=1, placeby="", placeby_value=""):
+        nodearray = nodearray or "execute"
+        
+        return MachineRequest(nodearray, machine_type, count, placeby, placeby_value)
+    
+    def _host(self, machinetype, hostname, **resources):
+        machinetype = machinetype or {}
+        host = {"state": resources.pop("state", "idle"),
+                "jobs": resources.pop("jobs", []),
+                "resources_assigned": {},
+                "resources_available": {"vnode": hostname,
+                                        "host": hostname,
+                                        "machinetype": machinetype.get("name", "")}}
+        host["resources_available"].update(machinetype)
+        host["last_state_change_time"] = resources.get("last_state_change_time") or time.time()
+        host["resources_available"].update(resources)
+        return host
+    
+    def test_pbsuserguide_ex1(self):
+        '''
+            1. A job that will fit in a single host but not in any of the vnodes, packed into the fewest vnodes:
+                -l select=1:ncpus=10:mem=20gb
+                -l place=pack
+            In earlier versions, this would have been:
+                -lncpus=10,mem=20gb
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=16:mem=20G", place="pack")
+        self.assertEquals([self._machine_request(count=1)], self._autoscale(q))
+        
+    def test_pbsuserguide_ex2(self):
+        '''
+            2. Request four chunks, each with 1 CPU and 4GB of memory taken from anywhere.
+            -l select=4:ncpus=1:mem=4GB
+            -l place=free
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="4:ncpus=1:mem=4G", place="pack", machinetype="a4")
+        self.assertEquals([self._machine_request(count=1)], self._autoscale(q))
+        
+        q = PBSQ()
+        q.qsub(select_expr="4:ncpus=1:mem=4G", place="pack", machinetype="a4")
+        self.assertEquals([self._machine_request(count=1)], self._autoscale(q))
+        
+    def test_pbsuserguide_ex3(self):
+        '''
+            3. Allocate 4 chunks, each with 1 CPU and 2GB of memory from between one and four vnodes which have an arch of "linux".
+            -l select=4:ncpus=1:mem=2GB:arch=linux -l place=free
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="4:ncpus=1:mem=2G:arch=linux", place="free")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2win", 16, 8, 100, arch="windows"),
+                                             machine.new_machinetype("execute", "a2linux", 16, 8, 100, arch="linux"))
+        self.assertEquals([self._machine_request(machine_type="a2linux", count=1)], self._autoscale(q, cluster_def))
+        
+        # slightly different take, just so that we see the spill over.
+        q.qsub(select_expr="6:ncpus=1:mem=2G:arch=linux", place="free")
+        self.assertEquals([self._machine_request(machine_type="a2linux", count=3)], self._autoscale(q, cluster_def))
+
+    def test_pbsuserguide_ex4(self):
+        '''
+        4. Allocate four chunks on 1 to 4 vnodes where each vnode must have 1 CPU, 3GB of memory and 1 node-locked dyna
+            license available for each chunk.
+            -l select=4:dyna=1:ncpus=1:mem=3GB -l place=free
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="4:dyna=1:ncpus=1:mem=3G", place="free")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2nodyna", 16, 8, 100),
+                                             machine.new_machinetype("execute", "a2dyna", 16, 8, 100, dyna=4))
+        # 3gb with 8gb per machine, so we need 2 machines
+        self.assertEquals([self._machine_request(machine_type="a2dyna", count=2)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="4:dyna=1:ncpus=1:mem=1G", place="free")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2nodyna", 16, 8, 100),
+                                             machine.new_machinetype("execute", "a2dyna", 16, 8, 100, dyna=4))
+        # 1gb with 8gb per machine, so we need 1 machine
+        self.assertEquals([self._machine_request(machine_type="a2dyna", count=1)], self._autoscale(q, cluster_def))
+    
+    @unittest.skip("need shared value to do this correctly")
+    def test_pbsuserguide_ex5(self):
+        '''
+        5. Allocate four chunks on 1 to 4 vnodes, and 4 floating dyna licenses. This assumes "dyna" is specified as a server
+            dynamic resource.
+            -l dyna=4 -l select=4:ncpus=1:mem=3GB -l place=free
+        '''
+        self.fail("we need a shared value to do this correctly")
+        
+        q = PBSQ()
+        q.qsub(select_expr="4:dyna=1, ncpus=2:mem=1G", place="free")
+        dyna_license = machine.NumericValue(2, "dyna")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2nodyna", 8, 8, 100),
+                                             machine.new_machinetype("execute", "a2dyna", 2, 8, 100, dyna=dyna_license))
+        
+        self.assertEquals([self._machine_request(machine_type="a2dyna", count=2)], self._autoscale(q, cluster_def))
+        
+    def test_pbsuserguide_ex6(self):
+        '''
+            6. This selects exactly 4 vnodes where the arch is linux, and each vnode will be on a separate host. Each vnode will
+            have 1 CPU and 2GB of memory allocated to the job.
+            -lselect=4:mem=2GB:ncpus=1:arch=linux -lplace=scatter
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="4:mem=2G:ncpus=1:arch=linux", place="scatter:excl")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 32, 128, 100, arch="linux"))
+        self.assertEquals([self._machine_request(machine_type="a2", count=4)], self._autoscale(q, cluster_def))
+    
+    def test_pbsuserguide_ex6_scatter(self):
+        q = PBSQ()
+        q.qsub(select_expr="4:mem=2G:ncpus=1:arch=linux", place="scatter")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 32, 128, 100, arch="linux"))
+        self.assertEquals([self._machine_request(machine_type="a2", count=4)], self._autoscale(q, cluster_def))
+        
+    def test_pbsuserguide_ex7(self):
+        '''
+            7. This will allocate 3 chunks, each with 1 CPU and 10GB of memory. This will also reserve 100mb of scratch space if
+            scratch is to be accounted . Scratch is assumed to be on a file system common to all hosts. The value of "place"
+            depends on the default which is "place=free".
+            -l scratch=100mb -l select=3:ncpus=1:mem=10GB
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="3:ncpus=1:mem=10G:scratch=100M")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 32, 128, 100, scratch=.333))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 32, 128, 100, scratch=.225))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+
+    def test_pbsuserguide_ex8(self):
+        '''
+            8. This will allocate 2 CPUs and 50GB of memory on a host named zooland. The value of "place" depends on the
+            default which defaults to "place=free":
+            -l select=1:ncpus=2:mem=50gb:host=zooland
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=2:mem=50G:host=zooland")
+        
+        a2_mt = machine.new_machinetype("execute", "a2", 32, 128, 100)
+        cluster_def = _nodearray_definitions(a2_mt)
+        
+        # pbs calls the hostname resource host, so we have to duplicate
+        
+        zooland = self._host(a2_mt, hostname="zooland", host="zooland")
+        notzooland = self._host(a2_mt, hostname="notzooland", host="notzooland")
+        self.assertEquals([], self._autoscale(q, cluster_def))
+        self.assertEquals(["notzooland"], self._idle_nodes(q, cluster_def, hosts=[zooland, notzooland]))
+    
+    def test_pbsuserguide_ex9(self):
+        '''
+           9. This will allocate 1 CPU and 6GB of memory and one host-locked swlicense from each of two hosts:
+            -l select=2:ncpus=1:mem=6gb:swlicense=1
+            -lplace=scatter
+        '''
+        q = PBSQ()
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 32, 100, swlicense=0),
+                                                machine.new_machinetype("execute", "a4", 16, 64, 100, swlicense=1))
+        q.qsub(select_expr="2:ncpus=2:mem=6G:swlicense=1", place="scatter:excl")
+        self.assertEquals([self._machine_request(machine_type="a4", count=2)], self._autoscale(q, cluster_def))
+        
+    def test_pbsuserguide_ex10(self):
+        '''
+           10. Request free placement of 10 CPUs across hosts:
+            -l select=10:ncpus=1
+            -l place=free
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="10:ncpus=1", place="free")
+        
+        a2_mt = machine.new_machinetype("execute", "a2", 32, 128, 100, priority=100)
+        a4_mt = machine.new_machinetype("execute", "a4", 32, 128, 100, priority=50)
+        
+        cluster_def = _nodearray_definitions(a2_mt, a4_mt)
+        
+        # if there aren't any machines, request an a2 as this has the highest priority
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+#         self.assertEquals([MachineRequest("execute", "a2", 1, "")], q.autoscale_requests(cluster_def, existing_machines=[]))
+        
+        existing_a4 = self._host(a4_mt, hostname="host123")
+        
+        # we already have an a4 up, so just use that.
+        self.assertEquals([], self._autoscale(q, cluster_def, hosts=[existing_a4]))
+        
+    def test_pbsuserguide_ex11(self):
+        '''
+           11. Here is an odd-sized job that will fit on a single SGI system, but not on any one node-board. We request an odd
+                number of CPUs that are not shared, so they must be "rounded up":
+                -l select=1:ncpus=3:mem=6gb
+                -l place=pack:excl
+        '''
+        q = PBSQ()
+        # for allocation testing purposes, I'm going to add a free job here too to ensure we don't try to use that one.
+        q.qsub(select_expr="1:ncpus=1:mem=1G", place="free")
+        q.qsub(select_expr="1:ncpus=3:mem=6G", place="pack:excl")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 24, 100, priority=100))
+         
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=1:mem=1G", place="free")
+        q.qsub(select_expr="3:ncpus=3:mem=6G", place="pack:excl")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 24, 100, priority=100))
+        
+        self.assertEquals([self._machine_request(machine_type="a2", count=3)], self._autoscale(q, cluster_def))
+    
+    def test_pbsuserguide_ex12(self):
+        '''
+           12. Here is an odd-sized job that will fit on a single SGI system, but not on any one node-board. We are asking for small
+                number of CPUs but a large amount of memory:
+                -l select=1:ncpus=1:mem=25gb
+                -l place=pack:excl
+        '''
+        #  this isn't really any different than ex11
+        q = PBSQ()
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 50, 100, priority=100))
+        q.qsub(select_expr="1:ncpus=1:mem=25G", place="pack:excl")
+        
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+        q.qsub(select_expr="1:ncpus=1:mem=1G", place="free")
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+        
+    def test_pbsuserguide_ex13(self):
+        '''
+           13. Here is a job that may be run across multiple SGI systems, packed into the fewest vnodes:
+            -l select=2:ncpus=10:mem=12gb
+            -l place=free
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="2:ncpus=10:mem=12G", place="free")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 36, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 20, 36, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+    
+    def test_pbsuserguide_ex14(self):
+        '''
+            14. Submit a job that must be run across multiple SGI systems, packed into the fewest vnodes:
+            -l select=2:ncpus=10:mem=12gb
+            -l place=scatter
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="2:ncpus=10:mem=12G", place="scatter")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 36, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+         
+        # add enough CPUs and it still requests 2 machines
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 24, 36, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="2:ncpus=10:mem=12G", place="scatter")
+        # add another job
+        q.qsub(select_expr="2:ncpus=10:mem=12G", place="scatter")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 24, 36, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+    
+    def test_pbsuserguide_ex15(self):
+        '''
+            15. Request free placement across nodeboards within a single host:
+            -l select=1:ncpus=10:mem=10gb
+            -l place=group=host 
+        '''
+        # group=host is meaningless without existing machines... and group=host is pretty pointless.
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=10:mem=10G", place="group=host")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 36, 100, priority=100, host=123))
+        # we ignore any group that isn't group_id
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+#         self.assertEquals([MachineRequest("execute", "a2", 1, "")], q.autoscale_requests(cluster_def))
+        
+    def test_pbsuserguide_ex16(self):
+        '''
+            16. Request free placement across vnodes on multiple SGI systems:
+            -l select=10:ncpus=1:mem=1gb
+            -l place=free
+        '''
+        # not really that interesting
+        q = PBSQ()
+        q.qsub(select_expr="10:ncpus=1:mem=1G", place="free")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 36, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+    def test_pbsuserguide_ex17(self):
+        '''
+            17. Here is a small job that uses a shared cpuset:
+            -l select=1:ncpus=1:mem=512kb
+            -l place=pack:shared
+        '''
+        # not really that interesting
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=1:mem=1G", place="pack:shared")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 36, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+    def test_pbsuserguide_ex18(self):
+        '''
+            18. Request a special resource available on a limited set of nodeboards, such as a graphics card:
+            -l select= 1:ncpus=2:mem=2gb:graphics=True + 1:ncpus=20:mem=20gb:graphics=False
+            -l place=pack:excl
+        '''
+        q = PBSQ()
+        q.qsub(place="pack:excl",
+                      select_expr="1:ncpus=2:mem=2G:graphics=true+1:ncpus=20:mem=20G:graphics=false")
+
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2g", 16, 10, 100, priority=100, graphics=True),
+                                                machine.new_machinetype("execute", "a2", 32, 36, 100, priority=100, graphics=False))
+        
+        self.assertEquals([self._machine_request(machine_type="a2", count=1),
+                           self._machine_request(machine_type="a2g")], sorted(self._autoscale(q, cluster_def), key=lambda x: x.machinetype))
+        
+    @unittest.skip("same as ex20")
+    def test_pbsuserguide_ex19(self):
+        '''
+            19. Align SMP jobs on c-brick boundaries:
+            -l select=1:ncpus=4:mem=6gb
+            -l place=pack:group=cbrick 
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=4:mem=6G", place="pack:group=cbrick")
+        
+        #  cbrick isn't defined
+        # TODO due to the single placement group hack, cbrick is set to single for this example
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 36, 100, priority=100),
+                                                machine.new_machinetype("execute", "a2c", 16, 36, 100, priority=50, cbrick="abc"))
+        cc_config = {"cyclecloud": {"placement_group": {"defaults": {"cbrick": "ccc"}}}}
+        self.assertEquals([self._machine_request(machine_type="a2c", count=1, placeby="cbrick", placeby_value="ccc")], self._autoscale(q, cluster_def, cc_config=cc_config))
+#         self.assertEquals([MachineRequest("execute", "a2c", 1, "")], q.autoscale_requests(cluster_def))
+    
+    @unittest.skip("This test will only make sense when we are using multiple placement groups")
+    def test_placement_group(self):
+        q = PBSQ()
+        q.qsub(select_expr="4:ncpus=4:mem=6G", place="scatter:excl:group=group_id")
+        
+        # ok, define placement_group _and_ add a placement_group that doesn't have enough machines
+        a2p_def_mt = machine.new_machinetype("execute", "a2p_def", 16, 36, 100, priority=50,
+                                             placeby="group_id", group_id="def")
+        a2p_abc_mt = machine.new_machinetype("execute", "a2p_abc", 16, 36, 100, priority=49,
+                                             placeby="group_id", group_id="abc")
+        cluster_def = _nodearray_definitions(a2p_def_mt, a2p_abc_mt)
+        
+        other_pg = self._host(a2p_abc_mt, hostname="otherpg")
+        
+        self.assertEquals([self._machine_request(nodearray="execute", machine_type="a2p_def", count=3, placeby="group_id", placeby_value="def")], 
+                          self._autoscale(q, cluster_def, [other_pg]))
+        
+        # duplicate test - if you run the same request twice, do you get the same result?
+        self.assertEquals([self._machine_request(nodearray="execute", machine_type="a2p_def", count=3, placeby="group_id", placeby_value="def")], 
+                          self._autoscale(q, cluster_def, [other_pg])) 
+        
+        q.qsub(select_expr="1:ncpus=4:mem=6G", place="scatter:excl:group=group_id")
+        self.assertEquals([self._machine_request(nodearray="execute", machine_type="a2p_def", count=4, placeby="group_id", placeby_value="def")], 
+                          self._autoscale(q, cluster_def, [other_pg]))
+        
+        # bump up the priority of a2p_abc so that one is allocated for the second job
+        cluster_def["execute"]["abc"]["machinetype"]["a2p_abc"]["priority"] = 100
+        self.assertEquals([self._machine_request(nodearray="execute", machine_type="a2p_def", count=3, placeby="group_id", placeby_value="def"),
+                           self._machine_request(nodearray="execute", machine_type="a2p_abc", count=1, placeby="group_id", placeby_value="abc")], 
+                          self._autoscale(q, cluster_def, [other_pg]))
+    
+    @unittest.skip("For now this seems pointless - just use group_id. WIll need to figure out behavior around non-group_id groupings")
+    def test_pbsuserguide_ex20(self):
+        '''
+            20. Align a large job within one router, if it fits within a router:
+            -l select=1:ncpus=100:mem=200gb
+            -l place=pack:group=router 
+        '''
+        self.fail()
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=100:mem=200G", place="pack:group=router")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2r", 100, 200, 100, priority=100,
+                                                                                    placeby="router", router="router-123"),
+                                                machine.new_machinetype("execute", "a2r0", 100, 200, 100, priority=100,
+                                                                                    placeby="router", router="router-123"),
+                                                machine.new_machinetype("execute", "a2", 100, 200, 100, priority=200))
+        
+        self.assertEquals([self._machine_request(machine_type="a2r", count=1, placeby="router", placeby_value="router-123")], self._autoscale(q, cluster_def))
+        self.assertEquals([MachineRequest("execute", "a2r", 1, "router-123")], q.autoscale_requests(cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="5:ncpus=100:mem=200G", place="scatter:excl:group=placement_group")
+        
+        self.assertEquals([MachineRequest("execute", "a2r", 5, "router-123")], q.autoscale_requests(cluster_def))
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2r", 100, 200, 100, priority=100,
+                                                                                     availableCount=4,
+                                                                                    placeby="placement_group", placement_group="router-123"),
+                                                machine.new_machinetype("execute", "a2r0", 100, 200, 100, priority=50,
+                                                                                    placeby="placement_group", placement_group="router-234"),
+                                                machine.new_machinetype("execute", "a2", 100, 200, 100, priority=200))
+        
+        self.assertEquals([MachineRequest("execute", "a2r0", 5, "router-234")], q.autoscale_requests(cluster_def))
+        
+    def test_pbsuserguide_ex22(self):
+        '''
+            22. To submit an MPI job, specify one chunk per MPI task. For a 10-way MPI job with 2gb of memory per MPI task:
+            -l select=10:ncpus=1:mem=2gb
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="10:ncpus=1:mem=2G", place="free")
+
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 32, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="10:ncpus=1:mem=2G", place="free")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 32, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="10:ncpus=1:mem=2G", place="scatter")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 32, 100, priority=100))
+
+        self.assertEquals([self._machine_request(machine_type="a2", count=10)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="10:ncpus=1:mem=2G", place="scatter")
+        q.qsub(select_expr="10:ncpus=1:mem=2G", place="scatter")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 32, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=10)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="10:ncpus=1:mem=2G", place="scatter")
+        q.qsub(select_expr="10:ncpus=1:mem=2G", place="scatter:excl")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 8, 32, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=20)], self._autoscale(q, cluster_def))
+    
+    def test_pbsuserguide_ex23(self):
+        '''
+            23. To submit a non-MPI job (including a 1-CPU job or an OpenMP or shared memory) job, use a single chunk. For a
+            2-CPU job requiring 10gb of memory:
+            -l select=1:ncpus=2:mem=10gb
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=2:mem=10G")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 2, 32, 100, priority=100))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 1, 32, 100, priority=100))
+        self.assertEquals([], self._autoscale(q, cluster_def))
+        
+    def test_pack_available(self):
+        q = PBSQ()
+        q.qsub(select_expr="2:ncpus=2:mem=2G", place="pack")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 4, 32, 100, priority=100, availableCount=1))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub(select_expr="2:ncpus=2:mem=2G", place="pack")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 2, 32, 100, priority=100, availableCount=2))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+        
+        q.qsub()
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 1, 32, 100, priority=100, availableCount=1))
+        self.assertEquals([self._machine_request(machine_type="a2", count=1)], self._autoscale(q, cluster_def))
+        
+        q = PBSQ()
+        q.qsub()
+        q.qsub()
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 1, 32, 100, priority=100, availableCount=2))
+        self.assertEquals([self._machine_request(machine_type="a2", count=2)], self._autoscale(q, cluster_def))
+    
+    def _test_live(self, live_jobstatus, expected):
+        q = PBSQ()
+        q.qsub(select_expr="2:ncpus=2", place="scatter:excl")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 8))
+        
+        mock_driver = MockDriver(["workq"], q.queues)
+        mock_driver.queued_jobs = lambda: (live_jobstatus, lambda x: x)
+        cluster = MockClustersAPI(cluster_def)
+        a = PBSAutostart(mock_driver, cluster, {})
+        
+        autoscale_requests = a.autoscale()[0]
+        
+        if not isinstance(expected, list):
+            expected = [expected]
+            
+        self.assertEquals(expected, autoscale_requests)
+    
+    def test_live(self):
+        queued = [{'job_id': '11.ip-0A030005',
+                      'job_state': 'Q',
+                      'queue': 'workq',
+                      'resource_list': {'ncpus': '2',
+                                        'nodect': '2',
+                                        'place': 'scatter:excl:group=group_id',
+                                        'select': '2:ncpus=1:slot_type=execute'}}]
+        self._test_live(queued, MachineRequest("execute", "a2", 2, "group_id", "single"))
+
+    def test_live2(self):
+        queued = [{'job_id': '27.ip-0A030005',
+                      'job_state': 'Q',
+                      'queue': 'workq',
+                      'resource_list': {'ncpus': '4',
+                                        'nodect': '2',
+                                        'place': 'scatter:group=group_id',
+                                        'select': '2:ncpus=2:slot_type=execute'}},
+                     {'job_id': '28.ip-0A030005',
+                      'job_state': 'Q',
+                      'queue': 'workq',
+                      'resource_list': {'ncpus': '4',
+                                        'nodect': '2',
+                                        'place': 'scatter:group=group_id',
+                                        'select': '2:ncpus=2:slot_type=execute'}}]
+        self._test_live(queued, MachineRequest("execute", "a2", 2, "group_id", "single"))
+        
+    def test_live3(self):
+        queued = [{'job_id': '27.ip-0A030005',
+                      'job_state': 'Q',
+                      'resource_list': {'mem': '2mb',
+                                        'ncpus': '4',
+                                        'nodect': '2',
+                                        'place': 'scatter:group=group_id',
+                                        'select': '2:ncpus=2:slot_type=execute'}}]
+        self._test_live(queued, MachineRequest("execute", "a2", 2, "group_id", "single"))
+        
+    def test_old_style_nodes(self):
+        queued = [{'job_id': '123',
+                   'job_state': 'Q',
+                   'exec_host': 'cazlrss28/0*8',
+                   'exec_vnode': '(cazlrss28:ncpus=8)',
+                   'resource_list': {
+                       'mpiprocs': '8',
+                       'ncpus': '8',
+                       'nodect': '1',
+                       'nodes': '1:ppn8',
+                       'place': 'scatter',
+                       'select': '1:ncpus=8:mpiprocs=8',
+                       'ungrouped': 'true'}}]
+        self._test_live(queued, MachineRequest("execute", "a2", 1, "", ""))
+        
+    def test_forced_assignment(self):
+        queued = [{'job_id': '1',
+                      'job_state': 'Q',
+                      'resource_list': {'mem': '2gb',
+                                        'ncpus': '2',  # too many ncpus
+                                        'nodect': '1',
+                                        'place': 'scatter:group=group_id',
+                                        'select': '1:ncpus=2:slot_type=execute'}}]
+            
+        running = [{'job_id': '2',
+                      'job_state': 'R',
+                      'exec_vnode': '(hostname-0:ncpus=18)',
+                      'resource_list': {'mem': '2gb',
+                                        'ncpus': '18',  # too many ncpus
+                                        'nodect': '1',
+                                        'arbitrary_string': 'abc', # undefined
+                                        'arbitrary_bool': False,  # undefined
+                                        'place': 'scatter:group=group_id',
+                                        'select': '1:ncpus=18:slot_type=execute'}}]
+        
+        hosts = [{"state": "job-busy", "jobs": ['2'], "resources_assigned": {"ncpus": 2}, "resources_available": {"vnode": "hostname-0", "group_id": "single", "ncpus": 16, "mem": 8, "slot_type": "execute", "machinetype": "a2"}},
+                 {"state": "job-idle", "resources_assigned": {}, "resources_available": {"vnode": "hostname-1", "group_id": "single", "ncpus": 16, "mem": 8, "slot_type": "execute", "machinetype": "a2"}}]
+        
+        for host in hosts:
+            host["last_state_change_time"] = time.time()
+        
+        q = PBSQ()
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 16, 8, 100, placeby="group_id", group_id="single"))
+        
+        mock_driver = MockDriver(["workq"], q.queues, hosts=hosts)
+        mock_driver.queued_jobs = lambda: (queued, lambda x: x)
+        mock_driver.running_jobs = lambda: (running, lambda x: x)
+        cluster = MockClustersAPI(cluster_def)
+        a = PBSAutostart(mock_driver, cluster, {})
+        
+        autoscale_requests, idle_machines, all_machines = a.autoscale()
+        
+        self.assertEquals([], autoscale_requests)
+        self.assertEquals([], idle_machines)
+        
+        machine_0 = [x for x in all_machines if x.hostname == "hostname-0"][0]
+        machine_1 = [x for x in all_machines if x.hostname == "hostname-1"][0]
+        self.assertEquals(["1"], machine_1.assigned_job_ids())
+
+        self.assertEquals(["2"], machine_0.assigned_job_ids())
+        
+    def test_live4(self):
+        q = PBSQ()
+        q.qsub(select_expr="2:ncpus=2", place="scatter:group=group_id")
+        q.qsub(select_expr="2:ncpus=2", place="scatter:group=group_id")
+        q.qsub(select_expr="2:ncpus=2", place="scatter:group=group_id")
+        
+        q.qsub(select_expr="2:ncpus=2", place="scatter:excl:group=group_id")
+        
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a2", 4, 8))
+        
+        q.qsub(select_expr="2:ncpus=2", place="scatter:group=group_id")
+        pbs_autostart = PBSAutostart(MockDriver(jobs=q.queues), MockClustersAPI(cluster_def), {})
+        self.assertEquals([MachineRequest("execute", "a2", 6, "group_id", "single")], pbs_autostart.autoscale()[0])
+        
+    def test_pbsuserguide_ex8_p2(self):
+        '''
+            8. This will allocate 2 CPUs and 50GB of memory on a host named zooland. The value of "place" depends on the
+            default which defaults to "place=free":
+            -l select=1:ncpus=2:mem=50gb:host=zooland
+        '''
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=2:mem=2G:host=zooland")
+        
+        a2_mt = machine.new_machinetype("execute", "a2", 32, 128, 100)
+        cluster_def = _nodearray_definitions(a2_mt)
+        
+        zooland = self._host(hostname="zooland", machinetype=a2_mt, ncpus=32, mem=128)
+        notzooland = self._host(hostname="notzooland", machinetype=a2_mt, ncpus=32, mem=128)
+        
+        new, idle, all = self._rebalance(q, cluster_def, [zooland, notzooland])
+        self.assertEquals([], new)
+        self.assertEquals(1, len(idle))
+        self.assertEquals("notzooland", idle[0].get_attr("host"))
+        
+        q.qsub(select_expr="1:ncpus=2:mem=2G:host=notzooland")
+        
+        new, idle, all = self._rebalance(q, cluster_def, [zooland, notzooland])
+        self.assertEquals([], new)
+        self.assertEquals(0, len(idle))
+        
+        q.qdel(1)
+        
+        new, idle, all = self._rebalance(q, cluster_def, [zooland, notzooland])
+        self.assertEquals([], new)
+        self.assertEquals(1, len(idle))
+        self.assertEquals("zooland", idle[0].get_attr("host"))
+        
+    def test_arrays(self):
+        q = PBSQ()
+        q.qsub(J="1:100")
+        # 100 * 1 core / 32 core boxes
+        self.assertEquals([self._machine_request(count=4)], self._autoscale(q))
+        
+        q = PBSQ()
+        q.qsub(J="1:100", ncpus=16)
+        # 100 * 16 cores / 32 core boxes
+        self.assertEquals([self._machine_request(count=50)], self._autoscale(q))
+        
+        q = PBSQ()
+        q.qsub(J="1:25", mem=30)
+        # 25 * 30 GB ram / 100 GB boxes - 3 jobs per box, so 9 boxes are required
+        self.assertEquals([self._machine_request(count=9)], self._autoscale(q))
+        
+    def test_booleans_as_strings(self):
+        ''' caught a bug where booleans show up as strings in the resources. We needed to auto-convert it'''
+        job_status = [{"job_id": "7.ip-0A03000A", 
+                        "job_state": "Q", 
+                        "resource_list": {
+                          "ncpus": "1", 
+                          "nodect": "1", 
+                          "place": "pack", 
+                          "select": "1:ncpus=1:slot_type=execute:ungrouped=True", 
+                          "slot_type": "execute", 
+                          "ungrouped": "True"
+                        }}]
+                          
+        self._test_live(job_status, [self._machine_request(machine_type="a2", count=1)])
+        
+    def test_custom_attribute_on_booting_instance(self):
+        mt = machine.new_machinetype("execute", "a2", 4, 128, 100, availableCount=1000 * 1000 * 100)
+        cluster_def = _nodearray_definitions(mt)
+        q = PBSQ()
+        q.qsub(ungrouped=True)
+        driver = MockDriver(jobs=q.queues)
+        cluster = MockClustersAPI(cluster_def, nodes=[{"MachineType": "a2", "InstanceId": "123", "Template": "execute"}])
+        autoscale_requests, idle_machines, all_machines = PBSAutostart(driver, cluster, {}).autoscale()
+        self.assertEquals([], autoscale_requests)
+        self.assertEquals([], idle_machines)
+        self.assertTrue(list(all_machines)[0].get_attr("ungrouped"))
+        
+    def test_no_nodearray_on_job(self):
+        q = PBSQ()
+        q.qsub()
+        requests = self._autoscale(q)
+        self.assertEquals([self._machine_request(count=1)], requests)
+        
+    def test_host_lifecycle(self):
+        q = PBSQ()
+        cc_config = InstanceConfig({}, {})
+        q.qsub(select_expr="2:ncpus=16", place="scatter:excl:group=group_id")
+        cluster_def = _nodearray_definitions(machine.new_machinetype("execute", "a4", 32, 100, 100))
+        cluster = MockClustersAPI(cluster_def)
+        driver = MockDriver(jobs=q.queues, hosts=[])
+        
+        def run_test(num_idle, num_request):
+            
+            pbs_autostart = PBSAutostart(driver, cluster, cc_config)
+            
+            autoscale_requests, idle_machines, machines = pbs_autostart.autoscale()
+            expected_requests = [self._machine_request(count=num_request, machine_type="a4", placeby="group_id", placeby_value="single")] if num_request else []
+            self.assertEquals(expected_requests, autoscale_requests)
+            self.assertEquals(num_idle, len(idle_machines))
+            return machines
+        
+        # first time we request 2 machines
+        run_test(num_idle=0, num_request=2)
+        # second time they are already booting
+        machines = run_test(num_idle=0, num_request=0)
+        
+        # create them as hosts, but the jobs still aren't running...
+        for n, m in enumerate(machines):
+            host_attrs = {}
+            for name, attr in m.iterattrs():
+                host_attrs[name] = attr.current_value
+        
+            host_attrs.pop("hostname", "")
+            host_attrs.pop("host", "")
+            host_attrs["instance_id"] = "instance-%s" % n
+        
+            driver.add_host("hostname-%s" % n, state="free", **host_attrs)
+        
+        run_test(num_idle=0, num_request=0)
+        
+        # turns out the job ended up only running on one host... weird! But we need to respect it.
+        host1 = driver.get_host("hostname-0")
+        host1["jobs"] = ["1", "2"]
+        host1["resources_assigned"] = {"ncpus": 16}
+        host1["state"] = "job-busy"
+        host1["last_state_change_time"] = time.time()
+        
+        q.set_running("1", "(hostname-0:ncpus=1)+(hostname-0:ncpus=1)")
+        
+        run_test(num_idle=1, num_request=0)
+        host2 = driver.get_host("hostname-1")
+        host2["last_state_change_time"] = time.time() - 3000
+        # default is 300 seconds for idle machines t hat have run at least one job.
+        host2["last_used_time"] = time.time() - 301
+        
+        self.assertEquals(host2["state"], "free")
+        
+        cc_config.set("cyclecloud.cluster.autoscale.stop_enabled", False)
+        run_test(num_idle=1, num_request=0)
+        self.assertEquals(host2["state"], "free")
+        
+        cc_config.set("cyclecloud.cluster.autoscale.stop_enabled", True)
+        run_test(num_idle=1, num_request=0)
+        self.assertEquals(host2["state"], "offline")
+        # FYI idle nodes are actually terminated, but it still returns this. This isn't part of the API, this is just exposing
+        # the internals of PBS autostart for testing purposes.
+        machines = run_test(num_idle=1, num_request=0)
+        machines = run_test(num_idle=0, num_request=0)
+        # shutdown hostname-2
+        self.assertEquals(1, len(machines))
+        
+    def test_disable_start(self):
+        q = PBSQ()
+        q.qsub(select_expr="1:ncpus=16:mem=20G", place="pack")
+        cc_config = InstanceConfig({}, {})
+        cc_config.set("cyclecloud.cluster.autoscale.start_enabled", False)
+        self.assertEquals([], self._autoscale(q, cc_config=cc_config))
+    
+    def test_packed_multi_chunk(self):
+        q = PBSQ()
+        cc_config = InstanceConfig({}, {})
+        q.qsub(select_expr="2:mem=15G+2:ncpus=4", place="group=group_id")
+        
+        cluster_def = _nodearray_definitions(
+            machine.new_machinetype("execute", "a2", 2, 16, 100),
+            machine.new_machinetype("execute", "a4", 4, 8, 100))
+        cluster = MockClustersAPI(cluster_def)
+        driver = MockDriver(jobs=q.queues, hosts=[])
+        
+        def run_test(num_idle, num_request):
+            
+            pbs_autostart = PBSAutostart(driver, cluster, cc_config)
+            
+            autoscale_requests, idle_machines, machines = pbs_autostart.autoscale()
+            autoscale_requests = sorted(autoscale_requests, key=lambda x: x.machinetype)
+            expected_requests = [self._machine_request(count=2, machine_type="a2", placeby="group_id", placeby_value="single"),
+                                self._machine_request(count=2, machine_type="a4", placeby="group_id", placeby_value="single")] if num_request else []
+            self.assertEquals(expected_requests, autoscale_requests)
+            self.assertEquals(num_idle, len(idle_machines))
+            return machines
+        
+        # first time we request 2 machines
+        run_test(num_idle=0, num_request=4)
+        # second time they are already booting
+        assert len(run_test(num_idle=0, num_request=0)) == 4
+        
+    def test_chunk_scatter_excl(self):
+        q = PBSQ()
+        cc_config = InstanceConfig({}, {})
+        q.qsub(select_expr="1:ncpus=4+4:ncpus=4", place="scatter:excl:group=group_id")
+        
+        cluster_def = _nodearray_definitions(
+            machine.new_machinetype("execute", "a2", 2, 16, 100),
+            machine.new_machinetype("execute", "a4", 4, 8, 100))
+        cluster = MockClustersAPI(cluster_def)
+        driver = MockDriver(jobs=q.queues, hosts=[])
+        
+        def run_test(num_idle, num_request):
+            
+            pbs_autostart = PBSAutostart(driver, cluster, cc_config)
+            
+            autoscale_requests, idle_machines, machines = pbs_autostart.autoscale()
+            autoscale_requests = sorted(autoscale_requests, key=lambda x: x.machinetype)
+            expected_requests = [self._machine_request(count=5, machine_type="a4", placeby="group_id", placeby_value="single")] if num_request else []
+            self.assertEquals(expected_requests, autoscale_requests)
+            self.assertEquals(num_idle, len(idle_machines))
+            return machines
+        
+        # first time we request 2 machines
+        run_test(num_idle=0, num_request=4)
+        # second time they are already booting
+        assert len(run_test(num_idle=0, num_request=0)) == 5
+        
+    def test_shutdown_down_nodes(self):
+        '''
+        Create three nodes - one that is up and busy, one that is down but presumed busy, and one that is just down.
+        Test that the down node goes away after 100 seconds but that the 'down,job-busy' node does not until it is in the 'down' state.
+        '''
+        mt = machine.new_machinetype("execute", "a2", 4, 128, 100, availableCount=1000 * 1000 * 100)
+        cc_config = InstanceConfig({}, {})
+        cc_config.set("pbspro.remove_down_nodes", "100")
+        clock = MockClock(time.time())
+        
+        q = PBSQ()
+        # create one job for an up and busy node, and one for a down yet busy node 
+        q.qsub(job_id="1")
+        q.qsub(job_id="2")
+        q.set_running("1", "(jobbusy:ncpus=1)")
+        q.set_running("2", "(downjobbusy:ncpus=1)")
+        
+        cluster_def = _nodearray_definitions(mt)
+        busy = self._host(mt, hostname="jobbusy", host="jobbusy", state="job-busy", instance_id="i123", jobs=["1"])
+        down = self._host(mt, hostname="down", host="down", state="down", instance_id="i234", last_state_change_time=clock.time() - 99)
+        # make it so this state is very old - we are asserting that it isn't treated as idle by PBS autostart (though it is returned as isdle by the autoscaler) 
+        downjobbusy = self._host(mt, hostname="downjobbusy", host="downjobbusy", state="down,job-busy", instance_id="i345", jobs=["2"], last_state_change_time=clock.time() - 10000)
+        
+        driver = MockDriver(jobs=q.queues, hosts=[busy, down, downjobbusy])
+        
+        cluster = MockClustersAPI(cluster_def, nodes=[{"MachineType": "a2", "InstanceId": "i123", "Template": "execute", "hostname": "jobbusy"},
+                                                      {"MachineType": "a2", "InstanceId": "i234", "Template": "execute", "hostname": "down"},
+                                                      {"MachineType": "a2", "InstanceId": "i345", "Template": "execute", "hostname": "downjobbusy"}])
+        
+        # we haven't hit the timeout threshold
+        autoscale_requests, idle_machines, all_machines = PBSAutostart(driver, cluster, cc_config, clock=clock).autoscale()
+        self.assertEquals([], autoscale_requests)
+        self.assertEquals(set(["down", "jobbusy", "downjobbusy"]), set([m.hostname for m in all_machines]))
+        self.assertEquals(["down"], [m.hostname for m in idle_machines])
+        
+        # we have hit the timeout threshold, though autoscale will still return the idle_machine here for testing.
+        clock.now += 2
+        autoscale_requests, idle_machines, all_machines = PBSAutostart(driver, cluster, cc_config, clock=clock).autoscale()
+        self.assertEquals([], autoscale_requests)
+        self.assertEquals(set(["down", "jobbusy", "downjobbusy"]), set([m.hostname for m in all_machines]))
+        self.assertEquals(["down"], [m.hostname for m in idle_machines])
+        
+        # rerun autoscale and the down node is gone.
+        autoscale_requests, idle_machines, all_machines = PBSAutostart(driver, cluster, cc_config, clock=clock).autoscale()
+        self.assertEquals([], autoscale_requests)
+        self.assertEquals(set(["jobbusy", "downjobbusy"]), set([m.hostname for m in all_machines]))
+        self.assertEquals([], [m.hostname for m in idle_machines])
+        
+        # the node is now just down, as PBS has rescheduled the job or removed it. Run autoscale twice to check it is removed.
+        downjobbusy["state"] = "down"
+        PBSAutostart(driver, cluster, cc_config, clock=clock).autoscale()
+        autoscale_requests, idle_machines, all_machines = PBSAutostart(driver, cluster, cc_config, clock=clock).autoscale()
+        
+        self.assertEquals([], autoscale_requests)
+        self.assertEquals(set(["jobbusy"]), set([m.hostname for m in all_machines]))
+        self.assertEquals([], [m.hostname for m in idle_machines])
+        
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/autostop.rb b/specs/execute/chef/site-cookbooks/pbspro/files/default/autostop.rb
new file mode 100644
index 0000000..7f9a1cf
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/autostop.rb
@@ -0,0 +1,74 @@
+#!/usr/bin/env ruby
+
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+require 'json'
+
+# Arguments
+AUTOSTOP_ENABLED = `jetpack config cyclecloud.cluster.autoscale.stop_enabled`.downcase.strip == "true"
+
+# Short-circuit without error if not enabled
+exit 0 unless AUTOSTOP_ENABLED
+
+IDLE_TIME_AFTER_JOBS = `jetpack config cyclecloud.cluster.autoscale.idle_time_after_jobs`.to_i
+IDLE_TIME_BEFORE_JOBS = `jetpack config cyclecloud.cluster.autoscale.idle_time_before_jobs`.to_i
+
+# Checks to see if we should shutdown
+idle_long_enough = false
+
+# indicates if execute node has ever ran a job
+def been_active?
+  # Shell out to grep with -m 1 for lazy match, as the log files can grow quite large
+  # and we only need to know if one job has ever started.
+  any_job = `egrep -m 1 ';pbs_mom;Job;.+;Started' /var/spool/pbs/mom_logs/*`.strip
+  any_job.length > 0
+end
+
+# indicates if there are currently running jobs
+def active?
+  activejobs = Dir.glob('/var/spool/pbs/mom_priv/jobs/*').count
+  activejobs > 0
+end
+
+# This is our autoscale runtime configuration
+runtime_config = {"idle_start_time" => nil}
+
+
+AUTOSCALE_DATA = '/opt/cycle/jetpack/run/autoscale.json'.freeze
+
+if File.exist?(AUTOSCALE_DATA)
+  file = File.read(AUTOSCALE_DATA)
+  runtime_config.merge!(JSON.parse(file))
+end
+
+if active?
+  runtime_config["idle_start_time"] = nil
+elsif runtime_config["idle_start_time"].nil?
+  runtime_config["idle_start_time"] = Time.now.to_i
+else
+  idle_seconds = Time.now - Time.at(runtime_config["idle_start_time"].to_i)
+
+  # Different timeouts if the node has ever run a job
+  timeout = if been_active?
+              IDLE_TIME_AFTER_JOBS
+            else
+              IDLE_TIME_BEFORE_JOBS
+            end
+
+  idle_long_enough = idle_seconds > timeout
+end
+
+# Write the config information back for next time
+file = File.new(AUTOSCALE_DATA, "w")
+file.puts JSON.pretty_generate(runtime_config)
+file.close
+
+# Do the shutdown
+if idle_long_enough
+  myhost = `hostname`
+  system("bash -lc 'pbsnodes -o #{myhost}'")
+  sleep(5)
+  system("jetpack shutdown --idle")
+end
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/config b/specs/execute/chef/site-cookbooks/pbspro/files/default/config
new file mode 100644
index 0000000..6de8e9c
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/config
@@ -0,0 +1 @@
+$usecp *:/shared /shared
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/doqmgr.sh b/specs/execute/chef/site-cookbooks/pbspro/files/default/doqmgr.sh
new file mode 100644
index 0000000..a099e7e
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/doqmgr.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+/opt/pbs/bin/qmgr -c 'set server managers = root@*'
+/opt/pbs/bin/qmgr -c 'set server query_other_jobs = true'
+/opt/pbs/bin/qmgr -c 'set server scheduler_iteration = 15'
+
+function create_resource() {
+	/opt/pbs/bin/qmgr -c "list resource $1" >/dev/null  2>/dev/null   || \
+	/opt/pbs/bin/qmgr -c "create resource $1 type=$2, flag=h"
+}
+
+create_resource slot_type string
+create_resource group_id string
+create_resource ungrouped string
+create_resource instance_id string
+create_resource machinetype string
+create_resource nodearray string
+create_resource disk size
+create_resource ngpus size
+
+/opt/pbs/bin/qmgr -c "set queue workq resources_default.slot_type = execute"
+/opt/pbs/bin/qmgr -c "set queue workq resources_default.ungrouped = false"
+/opt/pbs/bin/qmgr -c "set queue workq resources_default.place = scatter"
+/opt/pbs/bin/qmgr -c "set queue workq default_chunk.slot_type = execute"
+/opt/pbs/bin/qmgr -c "set queue workq default_chunk.ungrouped = false"
+/opt/pbs/bin/qmgr -c "set queue workq default_chunk.place = scatter"
+
+/opt/pbs/bin/qmgr -c "create queue htcq resources_default.slot_type = execute"
+/opt/pbs/bin/qmgr -c "set queue htcq queue_type = Execution"
+/opt/pbs/bin/qmgr -c "set queue htcq resources_default.slot_type = execute"
+/opt/pbs/bin/qmgr -c "set queue htcq resources_default.ungrouped = true"
+/opt/pbs/bin/qmgr -c "set queue htcq resources_default.place = pack"
+/opt/pbs/bin/qmgr -c "set queue htcq default_chunk.slot_type = execute"
+/opt/pbs/bin/qmgr -c "set queue htcq default_chunk.ungrouped = true"
+/opt/pbs/bin/qmgr -c "set queue htcq default_chunk.place = pack"
+/opt/pbs/bin/qmgr -c "set queue htcq enabled = true"
+/opt/pbs/bin/qmgr -c "set queue htcq started = true"
\ No newline at end of file
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/logging_init.py b/specs/execute/chef/site-cookbooks/pbspro/files/default/logging_init.py
new file mode 100644
index 0000000..a665c80
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/logging_init.py
@@ -0,0 +1,40 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+'''
+The purpose of this class is to call basicConfig before any other module does, specifically requests.
+'''
+import logging
+import logging.handlers
+import os
+import sys
+
+
+log_level_name = os.getenv('AUTOSTART_LOG_LEVEL', "INFO")
+log_file_level_name = os.getenv('AUTOSTART_LOG_FILE_LEVEL', "DEBUG")
+log_file = os.getenv('AUTOSTART_LOG_FILE', "autoscale.log")
+
+if log_level_name.lower() not in ["debug", "info", "warn", "error", "critical"]:
+    log_level = logging.DEBUG
+else:
+    log_level = getattr(logging, log_level_name.upper())
+
+if log_file_level_name.lower() not in ["debug", "info", "warn", "error", "critical"]:
+    log_file_level = logging.DEBUG
+else:
+    log_file_level = getattr(logging, log_level_name.upper())
+
+requests_logger = logging.getLogger("requests.packages.urllib3.connectionpool")
+requests_logger.setLevel(logging.WARN)
+
+stderr = logging.StreamHandler(stream=sys.stderr)
+stderr.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s"))
+stderr.setLevel(log_level)
+
+log_file = logging.handlers.RotatingFileHandler(log_file, maxBytes=1024 * 1024 * 5, backupCount=5)
+log_file.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s"))
+log_file.setLevel(log_file_level)
+
+logging.getLogger().addHandler(stderr)
+logging.getLogger().addHandler(log_file)
+logging.getLogger().setLevel(logging.DEBUG)
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/mockpbs.py b/specs/execute/chef/site-cookbooks/pbspro/files/default/mockpbs.py
new file mode 100644
index 0000000..20bd0fb
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/mockpbs.py
@@ -0,0 +1,131 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+import logging
+import os
+import time
+from UserDict import UserDict
+from copy import deepcopy
+
+LOG_DEBUG = logging.DEBUG
+LOG_WARNING = logging.WARN
+LOG_ERROR = logging.ERROR
+
+QUEUEJOB = "QUEUEJOB"
+PERIODIC = "PERIODIC"
+EVENT_ERROR = logging.ERROR
+
+
+def hold_types(h):
+    return h
+
+
+# feel free to change as needed
+hook_config_filename = "hook_config_file.json"
+
+pbs_conf = {"PBS_EXEC": os.getcwd()}
+
+
+def logmsg(level, msg):
+    logging.log(level, msg)
+
+
+class _Repr:
+    def __init__(self, expr):
+        self.expr = expr
+        
+    def __repr__(self):
+        return self.expr
+
+
+def place(expr):
+    return _Repr(expr)
+
+
+def select(expr):
+    return _Repr(expr)
+
+
+_job_queue = []
+
+
+def testing_add_job(job):
+    _job_queue.append(job)
+
+
+class _Event:
+    def __init__(self, job):
+        self.job = job
+    
+    
+def event():
+    return _Event(_job_queue.pop())
+
+
+pbs_str = str
+
+
+class _MockJob:
+    
+    def __init__(self, resource_list=None):
+        # match the behavior of PBS
+        self._data = {}
+        self.Resource_List = self._data["resource_list"] = ResourceList()
+        if resource_list:
+            self.Resource_List.update(resource_list)
+        
+    def get(self, key, default=None):
+        return self._data.get(key, default)
+        
+    def __getitem__(self, attr):
+        return self._data[attr]
+    
+    def __setitem__(self, attr, value):
+        self._data[attr] = value
+        
+    def iteritems(self):
+        return self._data.iteritems()
+    
+    def __contains__(self, key):
+        return key in self._data
+    
+
+class ResourceList(UserDict):
+    
+    def __init__(self, *args, **kwargs):
+        UserDict.__init__(self, *args, **kwargs)
+    
+    def __getitem__(self, key):
+        if key in self:
+            return UserDict.__getitem__(self, key)
+        return None
+    
+    def __setitem__(self, key, item):
+        # same failure occurs with the pbs ResourceList
+        assert item is not None, "attempted to set %s to '%s'" % (key, item)
+        UserDict.__setitem__(self, key, item)
+    
+    def __repr__(self):
+        return UserDict.__repr__(self)
+    
+    def __str__(self):
+        return UserDict.__repr__(self)
+    
+
+def mock_job(raw_job):
+    '''
+    Create a job object that matches the interface (duck typing) of the internal pbs job class, which
+    is only accessible in the submit_hook currently.
+    '''
+    raw_job = deepcopy(raw_job)
+    job = _MockJob(raw_job.pop("resource_list", None))
+    
+    job["job_id"] = raw_job.pop("job_id", str(time.time()))
+    job["job_state"] = raw_job.pop("job_state", "Q")
+    job["array"] = raw_job.pop("array", False)
+   
+    if job["array"]:
+        job["array_state_count"] = raw_job.pop("array_state_count")
+    
+    # remaining things are assumed to be resources
+    return job
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/pbs_driver.py b/specs/execute/chef/site-cookbooks/pbspro/files/default/pbs_driver.py
new file mode 100644
index 0000000..7624b7d
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/pbs_driver.py
@@ -0,0 +1,301 @@
+#!/usr/bin/python
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+import cStringIO
+from collections import OrderedDict
+
+import tandem_utils
+import tandem_driver_main
+import json
+from tandem_driver_main import TandemDriver
+import os
+import re
+import pbscc
+
+#     E -     Job is    exiting    after having run.
+#     H -     Job is    held.
+#     Q -     job is    queued,    eligable to run    or routed.
+#     R -     job is    running.
+#     T -     job is    being moved to new location.
+#     W -     job is    waiting    for its    execution time
+#      (-a option) to    be reached.
+#     S -     (Unicos only) job is suspend.
+_PBS_STATES = {"E": "e",
+               "H": "h",
+               "Q": "i",
+               "R": "r",
+               "W": "i",
+               "S": "i"
+               }
+
+_PBS_NOT_FOUND = 153
+
+JOB_STATE_EXITING = "E"
+JOB_STATE_FINISHED = "F"
+JOB_STATE_HELD = "H"
+JOB_STATE_QUEUED = "Q"
+JOB_STATE_RUNNING = "R"
+JOB_STATE_SUSPEND = "S"
+JOB_STATE_TRANSIT = "T"
+JOB_STATE_USER_SUSPEND = "U"
+JOB_STATE_WAITING = "W"
+JOB_STATE_EXPIRED = "X"
+
+
+class PBSDriver(TandemDriver):
+
+    def __init__(self, bin_dir=None, version=None):
+        TandemDriver.__init__(self)
+        self.bin_dir = bin_dir
+        self.version = version if version else self._version()
+        
+    def capabilities(self):
+        return {
+                    "format": ["native"],
+                    "version": self.version,
+                    "schedulerType": "pbs"
+                }
+
+    def _version(self):
+        try:
+            version_line = tandem_utils.check_call(["qmgr", "--version"]).split("\n")[0]
+            return version_line.split("=")[1].strip()
+        except:
+            return "-1"
+
+    def _bin(self, name):
+        return name if self.bin_dir is None else os.path.join(self.bin_dir, name)
+
+    def queues(self):
+        lines = tandem_utils.check_call([self._bin("qstat"), "-Q"]).split("\n")
+        return [x.split()[0] for x in lines[2:] if x.strip()]
+
+    def jobstatus(self, scheduler_name, marker=None, local_job_id_query=None):
+        marker = float(marker) if marker else 0
+        stdout, stderr, code, converter = self._jobstatus()
+        if code == 0:
+            return converter(stdout)
+        elif code == _PBS_NOT_FOUND:
+            return []
+        else:
+            tandem_utils.error_and_exit(stderr)
+            
+    def scheduler_config(self):
+        sched_config = "/var/spool/pbs/sched_priv/sched_config"
+        pattern = re.compile(r"^\s*resources\s*: .+$")
+        try:
+            if os.path.exists(sched_config):
+                for line in open(sched_config):
+                    line = line.strip()
+                    if pattern.match(line):
+                        expr = line.split(":", 1)[1].strip()
+                        expr = expr.replace('\"', "")
+                        toks = expr.split(",")
+                        resources = [x.strip() for x in toks]
+                        return {"resources": resources}
+        except Exception as e: 
+            pbscc.error("Could not parse %s, using default resources. Error was %s" % (sched_config, str(e)))
+        
+        # just return default values
+        return {"resources": ["ncpus", "mem", "arch", "host", "vnode", "aoe", "slot_type", "group_id", "ungrouped", "instance_id", "ipv4", "disk"]}
+            
+    def _jobstatus(self, local_job_id_query=None):
+        stdout, stderr, code = tandem_utils.call(self.qstat_args(local_job_id_query))
+        return stdout, stderr, code, _from_qstat
+
+    def submit(self, scheduler_name, format, submit_data):
+        if format == "tandem":
+            parsed_submit_data = submit_data
+        elif format == "tandem":
+            buf = cStringIO.StringIO()
+            jobs_data = json.loads(submit_data)
+            for job in jobs_data:
+                transform_tandem_job(job, buf)
+            submit_data = buf.getvalue()
+        jobid = tandem_utils.check_call([self._bin("qsub"), "-"], parsed_submit_data)
+        return [job[tandem_utils.LOCAL_JOB_ID_KEY] for job in self.jobstatus(scheduler_name, 0, jobid.strip())]
+    
+    def logs(self, scheduler_name, local_job_id_query):
+        jobs = self.jobstatus(scheduler_name, 0, local_job_id_query)
+        if not jobs:
+            return {}
+        job = jobs[0]
+        
+        stdout_path = job["Output_Path"].replace("\"", "").split(":")[1]
+        stderr_path = job["Error_Path"].replace("\"", "").split(":")[1]
+        return tandem_utils.make_std_logs(stdout_path, stderr_path)
+    
+    def schedstatus(self, scheduler_name):
+        # TODO
+        return [{tandem_utils.SCHED_NAME_KEY: scheduler_name}]
+    
+    def hold(self, scheduler_name, local_job_ids):
+        return self._hold_release_remove(self._bin("qhold"), local_job_ids)
+    
+    def release(self, scheduler_name, local_job_ids):
+        return self._hold_release_remove(self._bin("qrls"), local_job_ids)
+        
+    def remove(self, scheduler_name, local_job_ids):
+        return self._hold_release_remove(self._bin("qdel"), local_job_ids)
+    
+    def _hold_release_remove(self, cmd, local_job_ids):
+        out, err, code = tandem_utils.call([cmd] + local_job_ids)
+        if code == 0:
+            return {"status": "success"}
+        elif code == _PBS_NOT_FOUND:
+            return {"status": "not_found", "details": err}
+        else:
+            return {"status": "error", "details": err}
+        
+    def qstat_args(self, jobid=None):
+        return [self._bin("qstat"), "-f", "-w"] + ([jobid] if jobid else [])
+    
+    def running_jobs(self):
+        return self._get_jobs([self._bin("qstat"), "-f", "-w", "-t", "-r"])
+    
+    def queued_jobs(self):
+        return self._get_jobs([self._bin("qstat"), "-f", "-w", "-i"])
+    
+    def _get_jobs(self, args):
+        stdout, stderr, code = tandem_utils.call(args)
+        if code == 0:
+            return stdout, _from_qstat
+        elif code == _PBS_NOT_FOUND:
+            return "[]", json.loads
+        else:
+            tandem_utils.error_and_exit(stderr)
+    
+    def hosts(self, grouping=None, keyformatter=lambda x: x):
+        return self.pbsnodes(grouping, keyformatter)
+
+    def pbsnodes(self, grouping=None, keyformatter=lambda x: x):
+        stdout, stderr, ret = tandem_utils.call([self._bin("pbsnodes"), "-a", "-F", "json"])
+        
+        if ret == 1 and 'Server has no node list' in stderr:
+            return {None: {}}
+        
+        if ret != 0:
+            raise RuntimeError(stderr)
+        
+        nodes = json.loads(stdout)["nodes"]
+        
+        if not grouping:
+            return {None: nodes}
+        
+        if isinstance(grouping, basestring):
+            grouping = (grouping, )
+        
+        grouped = OrderedDict()
+        for _node_name, node in nodes.iteritems():
+            res_avail = node["resources_available"]
+            
+            def flat_get(node, g):
+                return res_avail.get(g, node.get(g))
+            
+            key = keyformatter(tuple([flat_get(node, g) for g in grouping]))
+            if key not in grouped:
+                grouped[key] = []
+            grouped[key].append(node)
+            
+        return grouped
+    
+    def set_offline(self, hostname):
+        tandem_utils.check_call([self._bin("pbsnodes"), "-o", hostname])
+        
+    def delete_host(self, hostname):
+        tandem_utils.check_call([self._bin("qmgr"), "-c", "delete node %s" % hostname])
+
+    def alter(self, jobs):
+        resources = {}
+        for job in jobs: 
+            resource_arg = ":".join(["%s=%s" % (x, y) for x, y in job.iteritems()])
+            if resource_arg not in resources:
+                resources[resource_arg] = []
+            resources[resource_arg].append(job["job_id"])
+        for resource_arg, job_ids in resources.iteritems():
+            tandem_utils.check_call(". /etc/cluster-setup.sh && qalter %s -l %s" % (",".join([str(j) for j in job_ids]), resource_arg))
+
+    def parse_select(self, job):
+        # Need to detect when slot_type is specified with `-l select=1:slot_type`
+        selectdict = {}
+        select_expression = str(job.Resource_List['select'])
+        
+        for expr in select_expression.split(":"):
+            key_val = expr.split("=", 1)
+            if len(key_val) == 1:
+                # this shouldn't ever happen, but log it in case.
+                if "select" in selectdict:
+                    continue
+                key_val = ("select", key_val[0])
+            selectdict[key_val[0]] = key_val[1]
+            
+        slot_type = selectdict.get('slot_type') or job.Resource_List.get('slot_type')
+        
+        return slot_type, selectdict
+    
+    def parse_place(self, place):
+        '''
+        arrangement is one of free | pack | scatter | vscatter
+        sharing is one of excl | shared | exclhost
+        grouping can have only one instance of group=resource
+        '''
+        placement = {"arrangement": "free"}
+        
+        if not place:
+            return placement
+        
+        toks = place.split(":")
+        
+        for tok in toks:
+            if tok in ["free", "pack", "scatter", "vscatter"]:
+                placement["arrangement"] = tok
+            elif tok in ["excl", "shared", "exclhost"]:
+                placement["sharing"] = tok
+            elif tok.startswith("group="):
+                placement["grouping"] = tok
+        
+        return placement
+
+
+def transform_tandem_job(job, stream):
+    # TODO can do other things like add defaults or translate TandemArguments == Arguments etc.
+    _to_jobad(job, stream)
+    
+
+def _to_jobad(job, stream):
+    for key, value in job.iteritems():
+        stream.write("%s = %s\n" % (key, value))
+    stream.write("queue\n")
+
+
+def _from_qstat(stdout, clz=OrderedDict):
+    ads = [clz()]
+    delim = ":"
+    for line in stdout.strip().split("\n"):
+        line = line.strip()
+        if not line:
+            if ads[-1]:
+                ads.append(clz())
+                delim = ":"
+        else:
+            if delim in line:
+                key, value = [x.strip() for x in line.split(delim, 1)]
+                keys = key.split(".")
+                parent = ads[-1]
+
+                def fmt_key(key):
+                    return key.replace(" ", "_").lower()
+
+                for parent_key in keys[:-1]:
+                    parent_key = fmt_key(parent_key)
+                    if parent_key not in parent:
+                        parent[parent_key] = clz()
+                    parent = parent[parent_key]
+                parent[fmt_key(keys[-1])] = value
+                delim = "="
+    return [ad for ad in ads if ad]
+
+
+if __name__ == "__main__":
+    tandem_driver_main.main(driver=PBSDriver())
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/pbscc.py b/specs/execute/chef/site-cookbooks/pbspro/files/default/pbscc.py
new file mode 100644
index 0000000..7f515b7
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/pbscc.py
@@ -0,0 +1,239 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+'''
+Utility class for parsing pbs specific expressions and dealing with logging inside and outside the pbs hook. 
+'''
+import numbers
+import os
+import collections
+import logging
+try:
+    import pbs
+    # there is no info in the actual pbs implementation.
+    pbs_LOG_INFO = pbs.LOG_DEBUG
+except ImportError:
+    import mockpbs as pbs
+    pbs_LOG_INFO = __import__("logging").INFO
+    
+
+_PBS_NOT_FOUND = 153
+    
+JOB_STATE_EXITING = "E"
+JOB_STATE_FINISHED = "F"
+JOB_STATE_HELD = "H"
+JOB_STATE_QUEUED = "Q"
+JOB_STATE_RUNNING = "R"
+JOB_STATE_BATCH = "B"
+JOB_STATE_SUSPEND = "S"
+JOB_STATE_TRANSIT = "T"
+JOB_STATE_USER_SUSPEND = "U"
+JOB_STATE_WAITING = "W"
+JOB_STATE_EXPIRED = "X"
+
+
+CONFIG_PATH = os.path.join(os.getenv("CYCLECLOUD_BOOTSTRAP", "."), "pbs", "config.json")
+    
+
+def parse_select(raw_job):
+    # Need to detect when slot_type is specified with `-l select=1:slot_type`
+    chunks = []
+    if not raw_job["resource_list"]['select']:
+        return {}
+    select_expression = str(raw_job["resource_list"]['select'])
+    
+    for chunk_expr in select_expression.split("+"):
+        chunk = collections.OrderedDict()
+        for expr in chunk_expr.split(":"):
+            key_val = expr.split("=", 1)
+            if len(key_val) == 1:
+                # this shouldn't ever happen, but log it in case.
+                if "select" in chunk:
+                    continue
+                key_val = ("select", key_val[0])
+            chunk[key_val[0]] = key_val[1]
+        chunks.append(chunk)
+    
+    return chunks
+
+
+def format_select(chunk):
+    expr = "%s" % chunk["select"]
+    
+    for key, value in chunk.iteritems():
+        if key == "select":
+            continue
+        expr += ":%s=%s" % (key, value)
+    
+    return expr
+        
+
+def parse_place(place):
+    '''
+    arrangement is one of free | pack | scatter | vscatter
+    sharing is one of excl | shared | exclhost
+    grouping can have only one instance of group=resource
+    '''
+    placement = {"arrangement": "free"}
+    
+    if not place:
+        return placement
+    
+    toks = place.split(":")
+    
+    for tok in toks:
+        if tok in ["free", "pack", "scatter", "vscatter"]:
+            placement["arrangement"] = tok
+        elif tok in ["excl", "shared", "exclhost"]:
+            placement["sharing"] = tok
+        elif tok.startswith("group="):
+            placement["grouping"] = tok
+    
+    return placement
+
+
+def parse_exec_vnode(expr):
+    '''
+    Example:  "(ip-0A030008:ncpus=1)"
+    '''
+    expr = expr[1:-1]
+    host, resource_expr = expr.split(":", 1)
+    resources = {}
+    for res_sub_expr in resource_expr.split(":"):
+        attr, value = res_sub_expr.split("=", 1)
+        try:
+            value = parse_gb_size(attr, value)
+        except InvalidSizeExpressionError:
+            if value.lower() in ["true", "false"]:
+                value = value.lower() == "true"
+            
+        resources[attr] = value
+    resources["hostname"] = host
+    return resources
+
+
+class InvalidSizeExpressionError(RuntimeError):
+    pass
+
+
+def parse_gb_size(attr, value):
+    if isinstance(value, numbers.Number):
+        return value
+    try:
+        
+        value = value.lower()
+        if value.endswith("pb"):
+            value = float(value[:-2]) * 1024
+        elif value.endswith("p"):
+            value = float(value[:-1]) * 1024
+        elif value.endswith("gb"):
+            value = float(value[:-2])
+        elif value.endswith("g"):
+            value = float(value[:-1])
+        elif value.endswith("mb"):
+            value = float(value[:-2]) / 1024
+        elif value.endswith("m"):
+            value = float(value[:-1]) / 1024
+        elif value.endswith("kb"):
+            value = float(value[:-2]) / (1024 * 1024)
+        elif value.endswith("k"):
+            value = float(value[:-1]) / (1024 * 1024)
+        elif value.endswith("b"):
+            value = float(value[:-1]) / (1024 * 1024 * 1024)
+        else:
+            try:
+                value = int(value)
+            except:
+                value = float(value)
+        
+        return value
+    except ValueError:
+        raise InvalidSizeExpressionError("Unsupported size for %s - %s" % (attr, value))
+
+
+__FINE = 0
+__DEBUG = 1
+__INFO = 2
+__WARN = 3
+__ERROR = 4
+
+__log_level = {
+    "fine": __FINE,
+    "debug": __DEBUG,
+    "info": __INFO,
+    "warn": __WARN,
+    "error": __ERROR
+}.get(os.environ.get("AUTOSTART_LOG_LEVEL", "WARN").lower(), __WARN)
+
+
+__log_level_names = {
+    __FINE: "fine",
+    __DEBUG: "debug",
+    __INFO: "info",
+    __WARN: "warn",
+    __ERROR: "error"
+}
+
+
+__application_name = "cycle_plugin"
+
+
+def set_application_name(value):
+    global __application_name
+    __application_name = value
+
+
+def __log(level, msg, interp=None):
+    interp = interp or ()
+    if __log_level <= level:
+        pbs_msg = "%s:%s - %s" % (__application_name, __log_level_names[__log_level], msg)
+        pylog_msg = msg
+        try:
+            pbs_msg = pbs_msg % interp
+            pylog_msg = msg % interp
+        except TypeError as e:
+            pbs.logmsg(__WARN, str(e))
+            
+        if level >= __ERROR:
+            pbs_level = pbs.LOG_ERROR
+            pylog_level = logging.ERROR
+        elif level == __WARN:
+            pbs_level = pbs.LOG_WARNING
+            pylog_level = logging.WARN
+        elif level == __INFO:
+            pbs_level = pbs_LOG_INFO
+            pylog_level = logging.INFO
+        else:
+            pbs_level = pbs.LOG_DEBUG
+            pylog_level = logging.DEBUG
+        pbs.logmsg(pbs_level, pbs_msg)
+        logging.log(pylog_level, pylog_msg)
+
+
+def is_fine():
+    return __log_level == __FINE
+
+
+def fine(msg, *interp):
+    interp = interp or ()
+    __log(__FINE, msg, interp)
+
+
+def debug(msg, *interp):
+    interp = interp or ()
+    __log(__DEBUG, msg, interp)
+
+
+def info(msg, *interp):
+    interp = interp or ()
+    __log(__INFO, msg, interp)
+    
+
+def warn(msg, *interp):
+    interp = interp or ()
+    __log(__WARN, msg, interp)
+
+
+def error(msg, *interp):
+    interp = interp or ()
+    __log(__WARN, msg, interp)
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/pi.py b/specs/execute/chef/site-cookbooks/pbspro/files/default/pi.py
new file mode 100644
index 0000000..a8d5e2c
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/pi.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+import getopt, sys, time
+
+
+
+def calcPiNewton(digits):
+# Newtons formula for PI is:
+# PI / 2 = sum_n_from_0_to_inf(n! / (2 * n + 1)!!)
+
+# This can be written as:
+# PI / 2 = 1 + 1/3 * (1 + 2/5 * (1 + 3/7 * (1 + 4/9 * (1 + ... ))))
+
+# This algorithm puts 2 * 1000 on the right side and computes everything from inside out.
+
+    scale = 10000
+    maxarr = int(digits/4) * 14
+    arrinit = 2000
+    carry = 0
+    arr = []
+    output = ''
+
+    # initialize the array
+    arr = [arrinit]*(maxarr+1)
+        
+    for i in range(maxarr, 1, -14):
+        sum = 0
+        for j in range(i, 0, -1):
+            sum = (sum * j) + (scale * arr[j])
+            arr[j] = sum % ( (j*2)-1 )
+            sum = sum / ((j*2)-1)
+        output += "%04d" % (carry + (sum / scale))
+        carry = sum % scale
+    
+    return output
+
+    
+def usage():
+    print 'Calculate pi to x digits.  Usage: python pi.py [digits]'
+
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "h", ["help"])
+    except getopt.GetoptError:
+        # print help information and exit:
+        usage()
+        sys.exit(2)
+
+    for o, a in opts:
+        if o == "-v":
+            verbose = True
+        if o in ("-h", "--help"):
+            usage()
+            sys.exit()
+    
+    if args:
+        digits = int(args[0])
+        if digits < 1:
+            print 'Not a valid number of digits: %s' % digits
+    else:
+        digits = 1000
+        print 'Calculating the default number of pi digits (%s)' % digits
+        
+    # do the calculation
+    start_time = time.time()
+    print calcPiNewton(digits)
+    end_time = time.time()
+    print "TotalTime=%f" %(end_time-start_time)
+        
+    
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/pi.sh b/specs/execute/chef/site-cookbooks/pbspro/files/default/pi.sh
new file mode 100644
index 0000000..9147d18
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/pi.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+#PBS -N pitest
+#PBS -j oe
+
+/shared/scratch/pi/pi.py 10000
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/runpi.sh b/specs/execute/chef/site-cookbooks/pbspro/files/default/runpi.sh
new file mode 100644
index 0000000..d5cb0d6
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/runpi.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+mkdir -p /shared/scratch/pi
+cp ~/demo/pi.py /shared/scratch/pi
+cp ~/demo/pi.sh /shared/scratch/pi
+cd /shared/scratch/pi
+qsub -J 1-1000 /shared/scratch/pi/pi.sh
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/sched.config b/specs/execute/chef/site-cookbooks/pbspro/files/default/sched.config
new file mode 100644
index 0000000..cc8ff05
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/sched.config
@@ -0,0 +1,28 @@
+round_robin: False	all
+by_queue: True		prime
+by_queue: True		non_prime
+strict_ordering: false	ALL
+help_starving_jobs:	true	ALL
+max_starve: 24:00:00
+backfill_prime:	false	ALL
+prime_exempt_anytime_queues:	false
+primetime_prefix: p_
+nonprimetime_prefix: np_
+node_sort_key: "sort_priority HIGH"	ALL
+provision_policy: "aggressive_provision"
+sort_queues:	true	ALL
+resources: "ncpus, mem, arch, host, vnode, aoe, slot_type, group_id, ungrouped, instance_id, machinetype, disk, ngpus, nodearray"
+load_balancing: false	ALL
+smp_cluster_dist: pack
+fair_share: false	ALL
+fairshare_usage_res: cput
+fairshare_entity: euser
+fairshare_decay_time: 24:00:00
+fairshare_decay_factor: 0.5
+preemptive_sched: true	ALL
+preempt_queue_prio:	150
+preempt_prio: "express_queue, normal_jobs"
+preempt_order: "SCR"
+preempt_sort: min_time_since_start
+dedicated_prefix: ded
+log_filter: 3328
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/submit_hook.py b/specs/execute/chef/site-cookbooks/pbspro/files/default/submit_hook.py
new file mode 100644
index 0000000..29df8af
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/submit_hook.py
@@ -0,0 +1,259 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+'''
+Note - the pbs module isn't very pythonic, so you'll see things like
+    value = job.Resource_List["attribute"] or 100
+instead of
+    value = job.Resource_List.get("attribute", 100)
+That is because is a metaclass, not a dict.
+
+Also, place and select objects use repr() to convert to a parseable string, but
+so you'll see guards against repr(None) (combined with the above) and 
+
+Quick start:
+    qmgr -c "create hook cycle_sub_hook"
+    qmgr -c "set hook cycle_sub_hook event = queuejob"
+    qmgr -c "create hook cycle_sub_periodic_hook"
+    qmgr -c "set hook cycle_sub_periodic_hook event = periodic"
+
+    # reload source / config
+    qmgr -c "import hook cycle_sub_hook application/x-python default submit_hook.py"
+    qmgr -c "import hook cycle_sub_hook application/x-config default submit_hook.json"
+    qmgr -c "import hook cycle_sub_periodic_hook application/x-python default submit_hook.py"
+    qmgr -c "import hook cycle_sub_periodic_hook application/x-config default submit_hook.json"
+
+Queue setup
+    qmgr -c "set queue <queue_name> resources_default.slot_type = <queue_name>"
+    qmgr -c "set queue <queue_name> resources_default.ungrouped = false"
+    qmgr -c "set queue <queue_name> default_chunk.slot_type = <queue_name>"
+    qmgr -c "set queue <queue_name> default_chunk.ungrouped = false"
+
+See PBS Professional Programmers Guide for detailed information.
+
+See /var/spool/pbs/server_logs/* for log messages
+'''
+
+import json
+import os
+import subprocess
+import traceback
+import sys
+
+
+try:
+    import pbs
+except ImportError:
+    import mockpbs as pbs
+
+
+def debug(msg):
+    pbs.logmsg(pbs.LOG_DEBUG, "cycle_periodic_hook_place - %s" % msg)
+
+
+def error(msg):
+    pbs.logmsg(pbs.EVENT_ERROR, "cycle_periodic_hook_place - %s" % msg)
+
+
+def hold_on_submit(hook_config, job):
+    '''
+    Hold every job so that we can process it properly in the periodic hook.
+    '''
+    pbs.logmsg(pbs.LOG_DEBUG, "cycle_sub_hook - holding job %s with hold_type 'so'" % job.id)
+    job.Hold_Types = pbs.hold_types("so")
+
+
+# Vendored in from pbscc.py
+def parse_place(place):
+    '''
+    arrangement is one of free | pack | scatter | vscatter
+    sharing is one of excl | shared | exclhost
+    grouping can have only one instance of group=resource
+    '''
+    placement = {}
+    
+    if not place:
+        return placement
+    
+    toks = place.split(":")
+    
+    for tok in toks:
+        if tok in ["free", "pack", "scatter", "vscatter"]:
+            placement["arrangement"] = tok
+        elif tok in ["excl", "shared", "exclhost"]:
+            placement["sharing"] = tok
+        elif tok.startswith("group="):
+            placement["grouping"] = tok
+    
+    return placement
+
+
+def format_place(place_dict):
+    parts = []
+    for key in ["arrangement", "sharing", "grouping"]:
+        if place_dict.get(key):
+            parts.append(place_dict[key])
+    return ":".join(parts)        
+
+
+QSELECT_EXE = os.path.join(pbs.pbs_conf['PBS_EXEC'], 'bin', 'qselect')
+QSTAT_EXE = os.path.join(pbs.pbs_conf['PBS_EXEC'], 'bin', 'qstat')
+QALTER_EXE = os.path.join(pbs.pbs_conf['PBS_EXEC'], 'bin', 'qalter')
+QRLS_EXE = os.path.join(pbs.pbs_conf['PBS_EXEC'], 'bin', 'qrls')
+
+
+def periodic_release_hook(hook_config, e):
+    held_jobs_per_iteration = int(hook_config.get("held_jobs_per_iteration", 25))
+
+    # Defined paths to PBS commands
+
+    # Get the jobs in an "so" hold state
+    qselect_cmd = [QSELECT_EXE, "-h", "so"]
+    stdout = run_cmd(qselect_cmd)
+    jobs = stdout.split()
+    debug("Jobs: %s" % jobs)
+
+    # Get the job information
+    if not jobs:
+        debug("No jobs to evaluate")
+        e.accept()
+        return
+
+    # Get Queue defaults information
+    queue_cmd = [QSTAT_EXE, "-Qf", "-F", "json"] 
+    stdout = run_cmd(queue_cmd)
+    qstat_Qf_json = json.loads(stdout)
+   
+    # Get job information
+    job_cmd = [QSTAT_EXE, "-f", "-F", "json"] + jobs[:held_jobs_per_iteration]
+    stdout = run_cmd(job_cmd)
+    qstat_json = json.loads(stdout)
+    jobs = qstat_json["Jobs"]
+    
+    for job_id, job in jobs.iteritems():
+        # Reevaluate each held job
+        debug("Key: %s\nValue: %s" % (job_id, job))
+        if str(job["Resource_List"].get("ungrouped")).lower() == "true":
+            debug("Skipping ungrouped job %s" % job_id)
+            qrls_cmd = [QRLS_EXE, "-h", "so", job_id]
+            run_cmd(qrls_cmd)
+            continue
+        
+        j_queue = job["queue"]
+        j_select = job["Resource_List"].get("select") or "1:ncpus=1"
+        
+        # Check the groupid placement
+        mj_place = job["Resource_List"].get("place")
+            
+        mj_place_dict = parse_place(mj_place)
+        
+        # Assign default placement from queue. If none, assign group=group_id
+        if j_queue in qstat_Qf_json["Queue"]:
+            if "resources_default" in qstat_Qf_json["Queue"][j_queue]:
+                if "place" in qstat_Qf_json["Queue"][j_queue]["resources_default"]:
+                    # only update arrangement (scatter/pack/vscatter/free) and sharing (excl/shared) if _neither_ is specified 
+                    # Do not override grouping if user specified it.
+                    # -l place=scatter is automatically added to -l nodes jobs, so we will remove it here and if it is not 
+                    # overridden by the queue defaults, we will bring it back
+                    if job["Resource_List"].get("nodes"):
+                        mj_place_dict = {}
+                        
+                    default_place = qstat_Qf_json["Queue"][j_queue]["resources_default"]["place"]
+                    default_place_dict = parse_place(default_place)
+                    
+                    # if the user specifies either arrangement or sharing, ignore them as far as defaults are concerned
+                    specified_at_least_one_of_arrangement_or_sharing = len(set(mj_place_dict.keys()) - set(["grouping"])) > 0
+                    if specified_at_least_one_of_arrangement_or_sharing:
+                        default_place_dict.pop("sharing", "")
+                        default_place_dict.pop("arrangement", "")
+                    
+                    # update if not exist from defaults
+                    for k, v in default_place_dict.iteritems():
+                        mj_place_dict[k] = mj_place_dict.get(k, v)
+                        
+                    # the job is a -l nodes job and the queue didn't specify arrangement or sharing
+                    if job["Resource_List"].get("nodes") and mj_place_dict.get("arrangement") is None and mj_place_dict.get("sharing") is None:
+                        mj_place_dict["arrangement"] = "scatter"
+                        
+                    mj_place = format_place(mj_place_dict)
+                    debug("%s" % mj_place)
+                    
+        mj_place = format_place(mj_place_dict)
+        
+        # Double checking group=group_id setting:
+        placement_grouping = None
+        for expr in mj_place.split(":"):
+            placement_grouping = None
+            if "=" in expr:
+                placekey, value = [x.lower().strip() for x in expr.split("=", 2)]
+                if placekey == "group":
+                    placement_grouping = value
+       
+        if placement_grouping is None:
+            debug("The user didn't specify place=group, setting group=group_id")
+            debug("%s - The user didn't specify place=group, setting group=group_id" % mj_place)
+            placement_grouping = "group_id"
+            prefix = ":" if mj_place else ""
+            mj_place = mj_place + prefix + "group=group_id"
+            debug("after update %s" % mj_place)
+
+        if mj_place != job["Resource_List"].get("place"):
+            # Qalter the job
+            qalter_cmd = [QALTER_EXE]
+            debug("New place statement: %s" % mj_place)
+            debug("before qalter %s" % j_select)
+            debug("before qalter %s" % mj_place)
+            qalter_cmd.append("-lselect=%s" % j_select)
+            qalter_cmd.append("-lplace=%s" % mj_place)
+            debug("qalter the job")
+            qalter_cmd.append(job_id)
+            debug("full cmd %s" % qalter_cmd)
+            stdout = run_cmd(qalter_cmd)
+            debug("stdout %s" % stdout)
+            
+        debug("Release the hold on job %s" % job_id)
+        qrls_cmd = [QRLS_EXE, "-h", "so", job_id]
+        run_cmd(qrls_cmd)
+        
+    e.accept()
+
+
+def run_cmd(cmd):
+    debug("Cmd: %s" % cmd)
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = proc.communicate()
+    if proc.returncode != 0:
+        msg = 'cmd failed!\n\tstdout="%s"\n\tstderr="%s"' % (stdout, stderr)
+        error(msg)
+        sys.exit(proc.returncode)
+        
+    return stdout
+
+
+def main():
+    try:
+        hook_config = {}
+        if pbs.hook_config_filename:
+            with open(pbs.hook_config_filename) as fr:
+                hook_config.update(json.load(fr))
+
+        e = pbs.event()
+        if e.type == pbs.QUEUEJOB:
+            j = e.job
+            hold_on_submit(hook_config, j)
+        elif e.type == pbs.PERIODIC:
+            periodic_release_hook(hook_config, e)
+        else:
+            pbs.logmsg(pbs.EVENT_ERROR, "Unknown event type %s" % e.type)
+    except SystemExit:
+        pbs.logmsg(pbs.LOG_DEBUG, "cycle - Exited with SystemExit")
+        raise
+    except:
+        pbs.logmsg(pbs.EVENT_ERROR, "cycle - %s" % traceback.format_exc())
+        raise
+
+
+# another non-pythonic thing - this can't be behind a __name__ == '__main__',
+# as the hook code has to be executable at the load module step.
+if not os.getenv("_UNITTEST_", ""):
+    main()
diff --git a/specs/execute/chef/site-cookbooks/pbspro/files/default/submit_hook_test.py b/specs/execute/chef/site-cookbooks/pbspro/files/default/submit_hook_test.py
new file mode 100644
index 0000000..99786a7
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/files/default/submit_hook_test.py
@@ -0,0 +1,269 @@
+import unittest
+import os
+import mockpbs
+import sys
+import json
+
+os.environ["_UNITTEST_"] = "true"
+
+from submit_hook import QSELECT_EXE, QSTAT_EXE, QRLS_EXE, QALTER_EXE
+import submit_hook
+
+
+class Job:
+    
+    def __init__(self, job_id=1, Hold_Types=None, queue="workq", **resource_list):
+        self.job_id = str(job_id)
+        self.id = self.job_id
+        self.interactive = False
+        self.Resource_List = mockpbs.ResourceList(resource_list)
+        self.Hold_Types = Hold_Types
+        self.queue = queue
+        
+    def to_dict(self):
+        return {"queue": self.queue, "Resource_List": dict(self.Resource_List)}
+    
+
+def standard_job(**resource_list):
+    resource_list["select"] = resource_list.get("select", "2:ncpus")
+    return Job(Hold_Types="so", **resource_list)
+
+
+class Event:
+    
+    def accept(self):
+        pass
+    
+    
+class MockRunCmd:
+    
+    def __init__(self, responses=None):
+        self.responses = responses or []
+        self.orig_run_cmd = submit_hook.run_cmd
+        
+    def add_call(self, expected, response):
+        if not isinstance(response, basestring):
+            response = json.dumps(response)
+        self.responses.append((expected, response))
+        
+    def __call__(self, cmd):
+        expected_cmd, response = self.responses.pop(0)
+        assert expected_cmd == cmd, "%s != %s" % (expected_cmd, cmd)
+        return response
+    
+    def __enter__(self):
+        submit_hook.run_cmd = self
+        return self
+    
+    def __exit__(self, exception_type, exception, tb):
+        submit_hook.run_cmd = self.orig_run_cmd
+        if exception_type is None:
+            # if an exception is thrown, we expect that some responses aren't returned
+            assert len(self.responses) == 0, "remaining responses %s" % self.responses
+        
+        
+class Queue:
+    
+    def __init__(self, name="workq", **resources_default):
+        self.name = name
+        self.resources_default = resources_default or {}
+        
+    def to_dict(self):
+        return {"resources_default": self.resources_default}
+    
+
+class SubmitHookTest(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def test_run_cmd(self):
+        try:
+            submit_hook.run_cmd(["test"])
+        except SystemExit:
+            pass
+        submit_hook.run_cmd(["test", "fail"])
+        
+    def test_placement_hook(self):
+        select_job = Job(select="2:ncpus=2")
+        plain_job = Job()
+        select_job_i = Job(select="2:ncpus=2")
+        plain_job_i = Job()
+        select_job_i.interactive = True
+        plain_job_i.interactive = True
+        
+        submit_hook.hold_on_submit({}, select_job)
+        submit_hook.hold_on_submit({}, plain_job)
+        submit_hook.hold_on_submit({}, select_job_i)
+        submit_hook.hold_on_submit({}, plain_job_i)
+        
+        self.assertEquals("so", select_job.Hold_Types)
+        self.assertEquals("so", plain_job.Hold_Types)
+        
+        # interactive jobs should be held as well.
+        self.assertEquals("so", select_job_i.Hold_Types)
+        self.assertEquals("so", plain_job_i.Hold_Types)
+        
+    def test_periodic_release_hook_select(self):
+        '''No queue defaults, no user defined group, add group=group_id'''
+        job = Job(Hold_Types="so", select="2:ncpus")
+        q = Queue()
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, "group=group_id")
+            submit_hook.periodic_release_hook({}, Event())
+            
+    def test_periodic_release_hook_select_with_grouping(self):
+        ''' User picked group=host so we should respect that'''
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="scatter:excl:group=host")
+        q = Queue(place="pack")
+        with MockRunCmd() as run_cmd:
+            # no qalter, user defined all 3 place arguments
+            _initialize_run_cmd(run_cmd, q, job, expected_qalter_place=None)
+            submit_hook.periodic_release_hook({}, Event())
+            
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="group=host")
+        q = Queue(place="scatter:excl")
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, expected_qalter_place="scatter:excl:group=host")
+            submit_hook.periodic_release_hook({}, Event())
+            
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="scatter:group=host")
+        q = Queue(place="scatter:excl")
+        with MockRunCmd() as run_cmd:
+            # no qalter, user defined arrangement and grouping arguments
+            _initialize_run_cmd(run_cmd, q, job, expected_qalter_place=None)
+            submit_hook.periodic_release_hook({}, Event())
+            
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="excl:group=host")
+        q = Queue(place="scatter:excl")
+        with MockRunCmd() as run_cmd:
+            # no qalter, user defined sharing and grouping arguments
+            _initialize_run_cmd(run_cmd, q, job, expected_qalter_place=None)
+            submit_hook.periodic_release_hook({}, Event())
+            
+    def test_periodic_release_hook_select_with_place(self):
+        ''' We should respect the user's -l place= when modifying group'''
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="scatter:excl")
+        q = Queue()
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, "scatter:excl:group=group_id")
+            submit_hook.periodic_release_hook({}, Event())
+            
+    def test_periodic_release_hook_nodes(self):
+        ''' We will ignore -lplace=scatter because the user picked -l nodes=, which auto-converts to -l place=scatter'''
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="scatter", nodes="2:ppn=2")
+        q = Queue(place="scatter:excl")
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, "scatter:excl:group=group_id")
+            submit_hook.periodic_release_hook({}, Event())
+            
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="scatter", nodes="2:ppn=2")
+        q = Queue(place="pack:shared")
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, "pack:shared:group=group_id")
+            submit_hook.periodic_release_hook({}, Event())
+            
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="scatter", nodes="2:ppn=2")
+        q = Queue(place="excl")
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, "excl:group=group_id")
+            submit_hook.periodic_release_hook({}, Event())
+            
+    def test_periodic_release_hook_nodes_excl_default(self):
+        ''' We should respect the user's -l place= when modifying group - pack:shared is not overridden by queue default scatter:excl'''
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="pack:shared")
+        q = Queue(place="scatter:excl")
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, "pack:shared:group=group_id")
+            submit_hook.periodic_release_hook({}, Event())
+            
+        # not that arrangement and exclusivity go together here
+        job = Job(Hold_Types="so", select="2:ncpus=2", place="pack")
+        q = Queue(place="scatter:excl")
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, "pack:group=group_id")
+            submit_hook.periodic_release_hook({}, Event())
+            
+    def test_periodic_release_hook_missing_select(self):
+        ''' We convert simple qsubs into select=1:ncpus=1 - note we should set the default to free in this case'''
+        job = Job(Hold_Types="so")
+        q = Queue()
+        q.resources_default["place"] = "scatter:excl"
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, expected_qalter_place="scatter:excl:group=group_id")
+            # nothing to hold
+            submit_hook.periodic_release_hook({}, Event())
+            
+    def test_chunked(self):
+        ''' There shouldn't be anything special about a chunked expression.'''
+        job = Job(Hold_Types="so", select="2:ncpus=2+1:mem=20g")
+        q = Queue()
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, q, job, expected_qalter_place="group=group_id")
+            # nothing to hold
+            submit_hook.periodic_release_hook({}, Event())
+            
+    def test_two_queues(self):
+        hpc_job = Job(job_id=1, Hold_Types="so", queue="workq", select="2:ncpus=2+1:mem=20g")
+        hpcq = Queue("workq")
+        # we don't pick up on defaults from the queue, so just set ungrouped on both.
+        htc_job = Job(job_id=2, Hold_Types="so", queue="htcq", ungrouped=True)
+        htcq = Queue("htcq", ungrouped=True, place="pack")
+        
+        with MockRunCmd() as run_cmd:
+            _initialize_run_cmd(run_cmd, [hpcq, htcq], [hpc_job, htc_job],
+                                # we won't bother running qalter because ungrouped==true
+                                expected_qalter_place=["group=group_id", None])
+            # nothing to hold
+            submit_hook.periodic_release_hook({}, Event())
+            
+
+def _initialize_run_cmd(run_cmd, queues, jobs, expected_qalter_place):
+    if not isinstance(queues, list):
+        queues = [queues]
+        
+    if not isinstance(jobs, list):
+        jobs = [jobs]
+        
+    if not isinstance(expected_qalter_place, list):
+        expected_qalter_place = [expected_qalter_place]
+        
+    job_ids = [job.job_id for job in jobs]
+        
+    run_cmd.add_call([QSELECT_EXE, "-h", "so"], " ".join([job.job_id for job in jobs]))
+    
+    queues_response = {"Queue": {}}
+    for q in queues:
+        queues_response["Queue"][q.name] = q.to_dict()
+    
+    run_cmd.add_call([QSTAT_EXE, "-Qf", "-F", "json"], queues_response)
+    
+    jobs_response = {"Jobs": {}}
+    for job in jobs:
+        jobs_response["Jobs"][job.job_id] = job.to_dict()
+        
+    run_cmd.add_call([QSTAT_EXE, "-f", "-F", "json"] + job_ids, jobs_response)
+    
+    if not job_ids:
+        assert not expected_qalter_place
+    
+    for job, eqp in zip(jobs, expected_qalter_place):
+        select = job.Resource_List["select"] or "1:ncpus=1"
+        if eqp:
+            run_cmd.add_call([QALTER_EXE, "-lselect=" + select, "-lplace=" + eqp, job.job_id], "qalter successful")
+        run_cmd.add_call([QRLS_EXE, "-h", "so", job.job_id], "qrls successful")
+        
+
+if __name__ == "__main__":
+    #import sys;sys.argv = ['', 'Test.testName']
+    unittest.main()
\ No newline at end of file
diff --git a/specs/execute/chef/site-cookbooks/pbspro/metadata.rb b/specs/execute/chef/site-cookbooks/pbspro/metadata.rb
new file mode 100644
index 0000000..02902d2
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/metadata.rb
@@ -0,0 +1,12 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+name             "pbspro"
+maintainer       "Microsoft Corporation"
+license          "MIT"
+description      "Installs/Configures Open PBS Pro"
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
+version          "0.1"
+depends          "tandem"
+%w{ cganglia cshared cuser cyclecloud }.each {|c| depends c}
+
diff --git a/specs/execute/chef/site-cookbooks/pbspro/recipes/autostart.rb b/specs/execute/chef/site-cookbooks/pbspro/recipes/autostart.rb
new file mode 100644
index 0000000..31f5a16
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/recipes/autostart.rb
@@ -0,0 +1,60 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/pbs/autostart.py" do
+    source "autostart.py"
+    mode "0755"
+    owner "root"
+    group "root"
+end
+
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/pbs/autostart_hook.py" do
+  source "autostart_hook.py"
+  mode "0755"
+  owner "root"
+  group "root"
+end
+
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/pbs/logging_init.py" do
+  source "logging_init.py"
+  mode "0755"
+  owner "root"
+  group "root"
+end
+
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/pbs/mockpbs.py" do
+  source "mockpbs.py"
+  mode "0755"
+  owner "root"
+  group "root"
+end
+
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/pbs/pbscc.py" do
+  source "pbscc.py"
+  mode "0755"
+  owner "root"
+  group "root"
+end
+
+file "#{node[:cyclecloud][:bootstrap]}/pbs/autostart.json" do
+  mode "0644"
+  owner "root"
+  group "root"
+  content Chef::JSONCompat.to_json_pretty(node[:pbspro][:autoscale_hook])
+end
+
+bash "import autoscale hook" do
+  code <<-EOH
+    set -e
+    /opt/pbs/bin/qmgr -c "create hook autoscale" 1>&2 || true
+    /opt/pbs/bin/qmgr -c "import hook autoscale application/x-python default #{node[:cyclecloud][:bootstrap]}/pbs/autostart_hook.py"
+    /opt/pbs/bin/qmgr -c "import hook autoscale application/x-config default #{node[:cyclecloud][:bootstrap]}/pbs/autostart.json"
+    /opt/pbs/bin/qmgr -c "set hook autoscale event = periodic"
+    /opt/pbs/bin/qmgr -c "set hook autoscale freq = 15"
+    touch #{node[:cyclecloud][:bootstrap]}/pbs/autoscalehook.imported
+  EOH
+  creates "#{node[:cyclecloud][:bootstrap]}/pbs/autoscalehook.imported"
+  only_if { node[:cyclecloud][:cluster][:autoscale][:start_enabled] }
+
+  notifies :restart, 'service[pbs]', :delayed
+end
diff --git a/specs/execute/chef/site-cookbooks/pbspro/recipes/autostop.rb b/specs/execute/chef/site-cookbooks/pbspro/recipes/autostop.rb
new file mode 100644
index 0000000..de4af8b
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/recipes/autostop.rb
@@ -0,0 +1,19 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/autostop.rb" do
+  source "autostop.rb"
+  owner "root"
+  group "root"
+  mode "0700"
+  action :create
+end
+
+# Convert our stop interval into a number of minutes rounded up and run the script on that interval
+interval = (node[:cyclecloud][:cluster][:autoscale][:stop_interval].to_i / 60.0).ceil
+cron "autostop" do
+  minute "*/#{interval}"
+  command "#{node[:cyclecloud][:bootstrap]}/cron_wrapper.sh #{node[:cyclecloud][:bootstrap]}/autostop.rb > /tmp/autostop.out 2>&1"
+  only_if { node[:cyclecloud][:cluster][:autoscale][:stop_enabled] }
+end
diff --git a/specs/execute/chef/site-cookbooks/pbspro/recipes/default.rb b/specs/execute/chef/site-cookbooks/pbspro/recipes/default.rb
new file mode 100644
index 0000000..053079f
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/recipes/default.rb
@@ -0,0 +1,3 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
diff --git a/specs/execute/chef/site-cookbooks/pbspro/recipes/execute.rb b/specs/execute/chef/site-cookbooks/pbspro/recipes/execute.rb
new file mode 100644
index 0000000..7f1fe03
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/recipes/execute.rb
@@ -0,0 +1,130 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+pbsprover = node[:pbspro][:version]
+
+package_name = "pbspro-execution-#{pbsprover}.x86_64.rpm"
+
+jetpack_download package_name do
+  project 'STAR-CCMplus'
+end
+
+yum_package package_name do
+  source "#{node['jetpack']['downloads']}/#{package_name}"
+  action :install
+end
+
+slot_type = node[:pbspro][:slot_type] || "execute"
+
+placement_group = node[:cyclecloud][:node][:placement_group] || nil
+is_node_grouped = node[:pbspro][:is_grouped]
+instance_id = node[:cyclecloud][:instance][:id]
+
+custom_resources = Hash.new {}
+if node[:autoscale] then
+    custom_resources = node[:autoscale].to_h
+end
+
+if custom_resources.empty? || custom_resources.nil? then
+	set_custom_resources = "true"
+else
+    custom_resources.delete("disabled")
+	set_custom_resources = custom_resources.map{ |key, value| "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.#{key}=#{value}'"}.join(" && ")
+end
+
+schedint = cluster.scheduler
+slots = node[:pbspro][:slots] || nil
+
+if schedint != nil
+  template "/var/spool/pbs/server_name" do
+    source "server_name_exec.erb"
+    mode "0644"
+    owner "root"
+    group "root"
+    variables(:servername => schedint)
+  end
+
+  template "/var/spool/pbs/mom_priv/config" do
+    source "mom_config.erb"
+    mode "0644"
+    owner "root"
+    group "root"
+    variables(:servername => schedint)
+  end
+
+  template "/etc/pbs.conf" do
+    source "pbs.conf.erb"
+    mode "0644"
+    owner "root"
+    group "root"
+    variables(:servername => schedint)
+  end
+end
+
+node_created_guard = "#{node['cyclecloud']['chefstate']}/pbs.nodecreated"
+
+bash "add-node-to-scheduler" do
+  code lazy {"/opt/pbs/bin/qmgr -c 'c n #{node[:hostname]}'"}
+  only_if do
+    cmd = Mixlib::ShellOut.new('/opt/pbs/bin/pbsnodes -a')
+    list_of_pbs_nodes = cmd.run_command.stdout.strip().split("\n")
+    !list_of_pbs_nodes.include?(node[:hostname])
+  end
+  not_if {::File.exist?(node_created_guard)}
+end
+
+defer_block 'Defer setting core count and slot_type, and start of PBS pbs_mom until end of converge' do
+  execute "set-node-core-count" do
+    command lazy { "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.ncpus=#{slots}'" }
+    only_if { !slots.nil? }
+    not_if {::File.exist?(node_created_guard)}
+  end
+  execute "set-node-free" do
+    command lazy { "/opt/pbs/bin/pbsnodes -r #{node[:hostname]}"}
+    not_if {::File.exist?(node_created_guard)}
+  end
+
+  set_slot_type = "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.slot_type=#{slot_type}'"
+  set_ungrouped = "true"
+
+  if not is_node_grouped then
+  	# user data explicitly says this is an ungrouped node.
+  	set_ungrouped = "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.ungrouped=true'"
+  else
+  	if placement_group then
+  		# user data didn't explicitly say this was an ungrouped node, but we are in a placement group
+  		set_ungrouped = "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.ungrouped=false'"
+  	else
+  		# user data didn't explicitly say this was an ungrouped node, but we aren't in a placement group
+  		set_ungrouped = "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.ungrouped=true'"
+  	end
+  end
+ 
+  set_group_id = "true"
+  if placement_group then
+    set_group_id = "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.group_id=#{placement_group}'"
+  end
+ 
+  set_instance_id = "/opt/pbs/bin/qmgr -c 's n #{node[:hostname]} resources_available.instance_id=#{instance_id}'"
+   
+  execute "set-node-slot_type" do
+    command lazy {<<-EOS
+      #{set_slot_type} && \
+      #{set_group_id} && \
+      #{set_ungrouped} && \
+      #{set_instance_id} && \
+      #{set_custom_resources} && \
+      touch #{node_created_guard}
+      EOS
+    }
+    creates node_created_guard
+    notifies :restart, 'service[pbs]', :immediately
+  end
+end
+
+service "pbs" do
+  action :nothing
+end
+
+include_recipe "pbspro::autostop"
diff --git a/specs/execute/chef/site-cookbooks/pbspro/recipes/login.rb b/specs/execute/chef/site-cookbooks/pbspro/recipes/login.rb
new file mode 100644
index 0000000..b10a025
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/recipes/login.rb
@@ -0,0 +1,32 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+pbsprover = node[:pbspro][:version]
+
+package_name = "pbspro-client-#{pbsprover}.x86_64.rpm"
+
+jetpack_download package_name do
+  project 'pbspro'
+end
+
+yum_package package_name do
+  source "#{node['jetpack']['downloads']}/#{package_name}"
+  action :install
+end
+
+schedint = cluster.scheduler
+
+if schedint != nil
+  template "/etc/pbs.conf" do
+    source "pbs.conf.erb"
+    mode "0644"
+    owner "root"
+    group "root"
+    variables(:servername => schedint)
+  end
+end
+
+execute 'set_flatuid' do
+  command '/opt/pbs/bin/qmgr -c "set server flatuid=true"'
+end
diff --git a/specs/execute/chef/site-cookbooks/pbspro/recipes/scheduler.rb b/specs/execute/chef/site-cookbooks/pbspro/recipes/scheduler.rb
new file mode 100644
index 0000000..f3204ab
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/recipes/scheduler.rb
@@ -0,0 +1,51 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+pbsprover = node[:pbspro][:version]
+
+package_name = "pbspro-server-#{pbsprover}.x86_64.rpm"
+
+jetpack_download package_name do
+  project 'STAR-CCMplus'
+end
+
+yum_package package_name do
+  source "#{node['jetpack']['downloads']}/#{package_name}"
+  action :install
+end
+
+directory "#{node[:cyclecloud][:bootstrap]}/pbs" do
+  owner "root"
+  group "root"
+  mode "0755"
+  action :create
+end
+
+cookbook_file "/var/spool/pbs/doqmgr.sh" do
+  source "doqmgr.sh"
+  mode "0755"
+  owner "root"
+  group "root"
+  action :create
+end
+
+cookbook_file "/var/spool/pbs/sched_priv/sched_config" do
+  source "sched.config"
+  owner "root"
+  group "root"
+  mode "0644"
+end
+
+service "pbs" do
+  action [:enable, :start]
+end
+
+execute "serverconfig" do
+  command "/var/spool/pbs/doqmgr.sh && touch /etc/qmgr.config"
+  creates "/etc/qmgr.config"
+  notifies :restart, 'service[pbs]', :delayed
+end
+
+include_recipe "pbspro::autostart"
+include_recipe "pbspro::submit_hook"
diff --git a/specs/execute/chef/site-cookbooks/pbspro/recipes/skel.rb b/specs/execute/chef/site-cookbooks/pbspro/recipes/skel.rb
new file mode 100644
index 0000000..0bfd96f
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/recipes/skel.rb
@@ -0,0 +1,22 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+directory "/etc/skel/demo" do
+  owner "root"
+  group "root"
+  mode "0755"
+  action :create
+  recursive true
+  not_if "test -d /etc/skel/demo"
+end
+
+%w{pi.py pi.sh runpi.sh}.each do |myfile|
+  cookbook_file "/etc/skel/demo/#{myfile}" do
+    source myfile
+    owner "root"
+    group "root"
+    mode "0755"
+   action :create
+  end
+end
diff --git a/specs/execute/chef/site-cookbooks/pbspro/recipes/submit_hook.rb b/specs/execute/chef/site-cookbooks/pbspro/recipes/submit_hook.rb
new file mode 100644
index 0000000..4fed74b
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/recipes/submit_hook.rb
@@ -0,0 +1,51 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/pbs/submit_hook.py" do
+    source "submit_hook.py"
+    mode "0755"
+    owner "root"
+    group "root"
+end
+  
+file "#{node[:cyclecloud][:bootstrap]}/pbs/submit_hook.json" do
+    mode "0644"
+    owner "root"
+    group "root"
+    content Chef::JSONCompat.to_json_pretty(node[:pbspro][:submit_hook])
+end
+
+# required by "tandem::install_driver"
+
+cookbook_file "#{node[:cyclecloud][:bootstrap]}/pbs/pbs_driver.py" do
+    source "pbs_driver.py"
+    mode "0755"
+    owner "root"
+    group "root"
+end
+
+
+node.default[:tandem_driver_directory] = "#{node[:cyclecloud][:bootstrap]}/pbs"
+include_recipe "tandem::install_driver"
+
+bash "import submit hook" do
+    code <<-EOH
+    /opt/pbs/bin/qmgr -c "create hook cycle_sub_hook" 2>/dev/null || true
+    /opt/pbs/bin/qmgr -c "set hook cycle_sub_hook event = queuejob" 2>/dev/null || true
+    /opt/pbs/bin/qmgr -c "create hook cycle_sub_periodic_hook" 2>/dev/null || true
+    /opt/pbs/bin/qmgr -c "set hook cycle_sub_periodic_hook event = periodic" 2>/dev/null || true
+    /opt/pbs/bin/qmgr -c "set hook cycle_sub_periodic_hook freq = 15"  2>/dev/null || true
+    
+    set -e
+    /opt/pbs/bin/qmgr -c "import hook cycle_sub_hook application/x-python default #{node[:cyclecloud][:bootstrap]}/pbs/submit_hook.py"
+    /opt/pbs/bin/qmgr -c "import hook cycle_sub_hook application/x-config default #{node[:cyclecloud][:bootstrap]}/pbs/submit_hook.json"
+    /opt/pbs/bin/qmgr -c "import hook cycle_sub_periodic_hook application/x-python default #{node[:cyclecloud][:bootstrap]}/pbs/submit_hook.py"
+    /opt/pbs/bin/qmgr -c "import hook cycle_sub_periodic_hook application/x-config default #{node[:cyclecloud][:bootstrap]}/pbs/submit_hook.json"
+        touch #{node[:cyclecloud][:bootstrap]}/pbs/submithook.imported
+    EOH
+    creates "#{node[:cyclecloud][:bootstrap]}/pbs/submithook.imported"
+    only_if { node[:pbspro][:submit_hook][:enabled] }
+
+    notifies :restart, "service[pbs]", :delayed
+end
diff --git a/specs/execute/chef/site-cookbooks/pbspro/spec/default_spec.rb b/specs/execute/chef/site-cookbooks/pbspro/spec/default_spec.rb
new file mode 100644
index 0000000..2540752
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/spec/default_spec.rb
@@ -0,0 +1,110 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+require 'spec_helper'
+require_relative "../../cyclecloud/libraries/cycle_cloud_cluster"
+
+describe 'pbspro::execute' do
+  platforms = {
+    'centos' => ['6.8', '7.2.1511']
+  }
+
+  platforms.each do |platform, versions|
+    versions.each do |version|
+      context "On #{platform} #{version}" do
+        cached(:chef_run) do
+          ChefSpec::SoloRunner.new(platform: platform, version: version) do |node|
+            node.default[:cyclecloud][:cluster][:autoscale][:stop_enabled] = true
+            node.override[:pbspro][:version] = "test-version"
+          end.converge(described_recipe)
+        end
+
+        before do
+          pbs_stdout_double = double('pbs_stdout_double')
+          allow(pbs_stdout_double).to receive(:stdout).and_return("")
+          
+          pbsnodes_double = double('pbsnodes_double')
+          allow(pbsnodes_double).to receive(:run_command).and_return(pbs_stdout_double)
+          
+          allow(Mixlib::ShellOut).to receive(:new).with("/opt/pbs/bin/pbsnodes -a").and_return(pbsnodes_double)
+          
+          # here we mock the global `cluster.scheduler` method so that these
+          # tests don't block forever trying to query a nonexistent CycleServer instance
+          allow_any_instance_of(CycleCloudCluster).to receive(:scheduler).and_return('127.0.0.1')
+        end
+
+        it 'downloads pbspro installer' do
+          expect(chef_run).to download_jetpack_download("pbspro-execution-test-version.x86_64.rpm")
+        end
+        
+        it 'adds node to scheduler' do
+          expect(chef_run).to run_bash("add-node-to-scheduler")
+        end
+        
+        it 'creates cron autostop job' do
+          expect(chef_run).to create_cron("autostop")
+        end
+
+      end
+    end
+  end
+end
+
+
+describe 'pbspro::scheduler' do
+  platforms = {
+    'centos' => ['6.8', '7.2.1511']
+  }
+
+  platforms.each do |platform, versions|
+    versions.each do |version|
+      context "On #{platform} #{version}" do
+        cached(:chef_run) do
+          ChefSpec::SoloRunner.new(platform: platform, version: version) do |node|
+            node.default[:cyclecloud][:cluster][:autoscale][:stop_enabled] = true
+            node.override[:pbspro][:version] = "test-version"
+          end.converge(described_recipe)
+        end
+
+        before do
+          pbs_stdout_double = double('pbs_stdout_double')
+          allow(pbs_stdout_double).to receive(:stdout).and_return("")
+          
+          pbsnodes_double = double('pbsnodes_double')
+          allow(pbsnodes_double).to receive(:run_command).and_return(pbs_stdout_double)
+          
+          allow(Mixlib::ShellOut).to receive(:new).with("/opt/pbs/bin/pbsnodes -a").and_return(pbsnodes_double)
+          
+          # here we mock the global `cluster.scheduler` method so that these
+          # tests don't block forever trying to query a nonexistent CycleServer instance
+          allow_any_instance_of(CycleCloudCluster).to receive(:scheduler).and_return('127.0.0.1')
+        end
+
+        it 'downloads pbspro installer' do
+          expect(chef_run).to download_jetpack_download("pbspro-server-test-version.x86_64.rpm")
+        end
+
+        it 'executes serverconfig' do
+          expect(chef_run).to run_execute("serverconfig")
+        end
+
+        it 'downloads pbspro installer' do
+          expect(chef_run).to run_execute("serverconfig")
+        end
+
+        it 'enables and starts the pbs service' do
+          expect(chef_run).to enable_service("pbs")
+          expect(chef_run).to start_service("pbs")
+        end
+
+        ['pbspro::autostart', 'pbspro::submit_hook'].each do |recipe|
+          it "includes #{recipe}" do
+            expect(chef_run).to include_recipe(recipe)
+          end
+        end
+        
+      end
+    end
+  end
+end
+
diff --git a/specs/execute/chef/site-cookbooks/pbspro/spec/spec_helper.rb b/specs/execute/chef/site-cookbooks/pbspro/spec/spec_helper.rb
new file mode 100644
index 0000000..3c15481
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/spec/spec_helper.rb
@@ -0,0 +1,20 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+require 'chefspec'
+require 'fauxhai'
+
+RSpec.configure do |config|
+  config.log_level = :error
+  config.cookbook_path = ['../../cookbooks', '../../berks-cookbooks']
+end
+
+# /usr/sbin isn't in the PATH when these tests are executed by jenkins
+if !ENV['PATH'].include?('/usr/sbin')
+  ENV['PATH'] = ENV['PATH'] + ':/usr/sbin'
+end
+
+# This makes it possible to test Chef libraries
+Dir['libraries/*.rb'].each { |f| require File.expand_path(f) }
+
+at_exit { ChefSpec::Coverage.report! }
diff --git a/specs/execute/chef/site-cookbooks/pbspro/templates/default/mom_config.erb b/specs/execute/chef/site-cookbooks/pbspro/templates/default/mom_config.erb
new file mode 100644
index 0000000..6de8e9c
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/templates/default/mom_config.erb
@@ -0,0 +1 @@
+$usecp *:/shared /shared
diff --git a/specs/execute/chef/site-cookbooks/pbspro/templates/default/pbs.conf.erb b/specs/execute/chef/site-cookbooks/pbspro/templates/default/pbs.conf.erb
new file mode 100644
index 0000000..109941f
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/templates/default/pbs.conf.erb
@@ -0,0 +1,9 @@
+PBS_SERVER=<%= @servername %>
+PBS_START_SERVER=0
+PBS_START_SCHED=0
+PBS_START_COMM=0
+PBS_START_MOM=1
+PBS_EXEC=/opt/pbs
+PBS_HOME=/var/spool/pbs
+PBS_CORE_LIMIT=unlimited
+PBS_SCP=/bin/scp
diff --git a/specs/execute/chef/site-cookbooks/pbspro/templates/default/pbsmastercron.rb.erb b/specs/execute/chef/site-cookbooks/pbspro/templates/default/pbsmastercron.rb.erb
new file mode 100644
index 0000000..69b72d2
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/templates/default/pbsmastercron.rb.erb
@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+
+if Dir.entries("<%= @torqueroot %>/addpending").size-2 != 0
+  Dir.foreach("<%= @torqueroot %>/addpending") do |fname|
+    next if fname == '.' or fname == '..'
+    f=File.open("<%= @torqueroot %>/addpending/#{fname}")
+    marray=f.readlines
+    slots=marray[0].chop
+    system("/usr/bin/qmgr -c \"create node #{fname} np=#{slots}\"")
+    if $? == 0
+      FileUtils.mv("<%= @torqueroot %>/addpending/#{fname}","<%= @torqueroot %>/addcomplete/#{fname}")
+    end
+  end
+end
diff --git a/specs/execute/chef/site-cookbooks/pbspro/templates/default/server_name.erb b/specs/execute/chef/site-cookbooks/pbspro/templates/default/server_name.erb
new file mode 100644
index 0000000..f6ab154
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/templates/default/server_name.erb
@@ -0,0 +1 @@
+<%= @node[:ipaddress] %>
diff --git a/specs/execute/chef/site-cookbooks/pbspro/templates/default/server_name_exec.erb b/specs/execute/chef/site-cookbooks/pbspro/templates/default/server_name_exec.erb
new file mode 100644
index 0000000..a0298d5
--- /dev/null
+++ b/specs/execute/chef/site-cookbooks/pbspro/templates/default/server_name_exec.erb
@@ -0,0 +1 @@
+<%= @servername %>
diff --git a/specs/execute/cluster-init/files/README.txt b/specs/execute/cluster-init/files/README.txt
new file mode 100644
index 0000000..e110007
--- /dev/null
+++ b/specs/execute/cluster-init/files/README.txt
@@ -0,0 +1,5 @@
+
+Files in this directory are automatically synced to any node using this spec. Content here
+can be anything from software packages to config files. Scripts can be used to install
+software packages or move files into the appropriate location on the node.
+            
\ No newline at end of file
diff --git a/specs/execute/cluster-init/files/runpbs.sh b/specs/execute/cluster-init/files/runpbs.sh
new file mode 100644
index 0000000..b7e85c9
--- /dev/null
+++ b/specs/execute/cluster-init/files/runpbs.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+cd ~/
+
+PPN=15
+echo ${1}> NODES
+NODES=`cat ~/NODES`
+NP=`expr ${1} \* ${PPN}`
+echo ${NP} > NP
+
+echo ${NODES} ${NP} ${PPN}
+
+qsub -j oe -l select=${NODES}:ncpus=${PPN} ~/script.sh
+#qsub -j oe ~/script2.sh
diff --git a/specs/execute/cluster-init/files/script.sh b/specs/execute/cluster-init/files/script.sh
new file mode 100644
index 0000000..6ae7ccd
--- /dev/null
+++ b/specs/execute/cluster-init/files/script.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+NODES=`cat ~/NODES`
+NP=`cat ~/NP`
+#PBS -j oe
+#PBS -l select=${NODES}:ncpus=15
+
+LSDYNA_DIR="/shared/home/azureuser/apps"
+#MPI_ROOT="/shared/home/azureuser/apps/platform_mpi/bin"
+MPI_ROOT="/opt/intel/compilers_and_libraries_2016.3.223/linux/mpi/intel64/bin"
+source /opt/intel/compilers_and_libraries/linux/mpi/bin64/mpivars.sh
+source /opt/intel/bin/compilervars.sh intel64
+#export LD_LIBRARY_PATH="/shared/home/azureuser/apps/platform_mpi/lib/linux_amd64"
+#INPUT="/shared/home/azureuser/apps/neon.refined.rev01.k"
+#INPUT="00_Calc.dyn"
+INPUT="01_Model.dyn"
+
+${MPI_ROOT}/mpirun -np ${NP} -ppn 15 -genv I_MPI_DEBUG 5 ${LSDYNA_DIR}/ls-dyna_mpp_s_R9_3_0_x64_redhat54_ifort131_sse2_intelmpi-413 i=${INPUT} > ~/log01.log
+
+
diff --git a/specs/execute/cluster-init/files/starccmrun.sh b/specs/execute/cluster-init/files/starccmrun.sh
new file mode 100644
index 0000000..1368a79
--- /dev/null
+++ b/specs/execute/cluster-init/files/starccmrun.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+#PBS -j oe
+#PBS -l nodes=2:ppn=16
+
+LSDYNA_DIR="/shared/home/azureuser/apps"
+MPI_ROOT="/shared/home/azureuser/apps/platform_mpi/bin"
+INPUT="/shared/home/azureuser/apps/neon.refined.rev01.k"
+
+cd ${PBS_O_WORKDIR}
+NP=$(wc -l ${PBS_NODEFILE} | awk '{print $1}')
+${MPI_ROOT}/mpirun -np ${NP} ${LSDYNA_DIR}/ls-dyna_mpp_s_R9_3_0_x64_redhat54_ifort131_sse2_platformmpi info 
+
+
+#${MPI_ROOT}/mpirun -np ${NP} ${ADVC_BIN} < ${INPUT}
+#${ADVC_DIR}/ADVCSolver ${INPUT} -out-dir ~/ -np ${NP} | tee ADVC-`date +%Y%m%d_%H-%M-%S`.log
diff --git a/specs/execute/cluster-init/files/starccmsetup.sh b/specs/execute/cluster-init/files/starccmsetup.sh
new file mode 100644
index 0000000..50c43fc
--- /dev/null
+++ b/specs/execute/cluster-init/files/starccmsetup.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+LSDYNA_VERSION=$(jetpack config ADVENTURECluster.version)
+exprot ALDE_LICENSE_FILE=$(jetpack config LICENSE)
+export PATH=$PATH:~/apps/
+export MPI_HASIC_UDAPL=ofa-v2-ib0
+
+#qsub -J 1 ~/
diff --git a/specs/execute/cluster-init/scripts/10.execute.sh b/specs/execute/cluster-init/scripts/10.execute.sh
new file mode 100644
index 0000000..749750e
--- /dev/null
+++ b/specs/execute/cluster-init/scripts/10.execute.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+# Copyright (c) 2019 Hiroshi Tanaka, hirtanak@gmail.com @hirtanak
+set -exuv
+
+echo "starting 10.execute.sh"
+
+# disabling selinux
+echo "disabling selinux"
+setenforce 0
+sed -i -e "s/^SELINUX=enforcing$/SELINUX=disabled/g" /etc/selinux/config
+
+#HOMEDIR=$(jetpack config cuser.home_dir)
+#CUSER=${HOMEDIR##*/}
+CUSER=$(grep "Added user" /opt/cycle/jetpack/logs/jetpackd.log | awk '{print $6}')
+CUSER=${CUSER//\'/}
+CUSER=${CUSER//\`/}
+echo ${CUSER} > /mnt/exports/shared/CUSER
+HOMEDIR=/shared/home/${CUSER}
+
+# default parameters
+STARCCMPLUS_VERSION=14.04.011
+REVISION=02
+STARCCMPLUS_PLATFORM=linux-x86_64
+PRECISION=r8 #r8 is single precison
+# get file name
+STARCCMPLUSFILENAME=$(jetpack config StarccmFileName)
+# set parameters
+STARCCMPLUS_VERSION=${STARCCMPLUSFILENAME:9:9}
+REVISION=${STARCCMPLUSFILENAME:19:2}
+STARCCMPLUS_PLATFORM=${STARCCMPLUSFILENAME:22:12}
+PRECISION=${STARCCMPLUSFILENAME:35:2}
+
+# resource unlimit setting
+CMD1=$(tail -1 /etc/security/limits.conf)
+if [[ $CMD1 != '* soft nofile 65535' ]]; then
+  (echo "* hard memlock unlimited"; echo "* soft memlock unlimited"; echo "* hard nofile 65535"; echo "* soft nofile 65535") >> /etc/security/limits.conf
+fi
+
+# License File Setting
+LICENSE=$(jetpack config LICENSE)
+PODKEY=$(jetpack config PODKEY)
+(echo "export STARCCMPLUS_VERSION=${STARCCMPLUS_VERSION}"; echo "export PRECISION=-${PRECISION^}"; echo "export CDLMD_LICENSE_FILE=${LICENSE}"; echo "export PODKEY=${PODKEY}") > /etc/profile.d/starccm.sh
+chmod a+x /etc/profile.d/starccm.sh
+chown ${CUSER}:${CUSER} /etc/profile.d/starccm.sh
+
+## Checking VM SKU and Cores
+VMSKU=`cat /proc/cpuinfo | grep "model name" | head -1 | awk '{print $7}'`
+CORES=$(grep cpu.cores /proc/cpuinfo | wc -l)
+
+# mandatory packages
+yum install -y perl-Digest-MD5.x86_64 redhat-lsb-core vtk vtk-devel gcc gcc-gcc++
+
+## H16r or H16r_Promo
+if [[ ${CORES} = 16 ]] ; then
+  echo "Proccesing H16r"
+  grep "vm.zone_reclaim_mode = 1" /etc/sysctl.conf || echo "vm.zone_reclaim_mode = 1" >> /etc/sysctl.conf sysctl -p
+fi
+
+## HC/HB set up
+if [[ ${CORES} = 44 ]] ; then
+  echo "Proccesing HC44rs"
+  grep "vm.zone_reclaim_mode = 1" /etc/sysctl.conf || echo "vm.zone_reclaim_mode = 1" >> /etc/sysctl.conf sysctl -p
+fi
+
+if [[ ${CORES} = 60 ]] ; then
+  echo "Proccesing HB60rs"
+  grep "vm.zone_reclaim_mode = 1" /etc/sysctl.conf || echo "vm.zone_reclaim_mode = 1" >> /etc/sysctl.conf sysctl -p
+fi
+
+echo "end of 10.execute.sh"
diff --git a/specs/execute/cluster-init/tests/README.txt b/specs/execute/cluster-init/tests/README.txt
new file mode 100644
index 0000000..a7149f8
--- /dev/null
+++ b/specs/execute/cluster-init/tests/README.txt
@@ -0,0 +1,5 @@
+
+Files in this directory contains tests that will be run at cluster start
+when in testing mode. Please see the official documentation for more information
+on cluster testing.
+            
\ No newline at end of file
diff --git a/specs/master/cluster-init/files/starccmrun.sh b/specs/master/cluster-init/files/starccmrun.sh
new file mode 100644
index 0000000..9a7353b
--- /dev/null
+++ b/specs/master/cluster-init/files/starccmrun.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+#PBS -j oe
+#PBS -l nodes=2:ppn=1
+NP=2
+
+# Sample Command: /opt/CD-adapco/12.02.011/STAR-CCM+12.02.011/star/bin/starccm+ -np 30 -machinefile /home/azureuser/hosts -licpath 1999@flex.cd-adapco.com -power -podkey <removed> -server -rsh ssh ./MODEL1_CCM11-DUCT.sim
+
+source /etc/profile.d/starccm.sh
+
+INSTALL_DIR="/shared/home/azureuser/apps"
+MPI_ROOT="${INSTALL_DIR}/${STARCCMPLUS_VERSION}${PRECISION}/STAR-CCM+${STARCCMPLUS_VERSION}${PRECISION}/mpi/intel/2018.1.163/linux-x86_64/rto/bin64"
+INPUT=/shared/home/azureuser/test1.sim
+export CDLMD_LICENSE_FILE=1999@flex.cd-adapco.com
+
+cd ${PBS_O_WORKDIR}
+
+# you spin up execute node and create hosts file on home directory.
+
+# pingpong
+#/opt/intel/impi/2018.4.274/intel64/bin/mpirun -machinefile /shared/home/azureuser/hosts IMB-MPI1 pingpong
+
+${INSTALL_DIR}/${STARCCMPLUS_VERSION}${PRECISION}/STAR-CCM+${STARCCMPLUS_VERSION}${PRECISION}/star/bin/starccm+ -np ${NP} -machinefile ${INSTALL_DIR}/hosts -licpath ${CDLMD_LICENSE_FILE} -power -podkey ${PODKEY} -server run ${INPUT}
diff --git a/specs/master/cluster-init/scripts/10.install_starccm.sh b/specs/master/cluster-init/scripts/10.install_starccm.sh
new file mode 100644
index 0000000..8293469
--- /dev/null
+++ b/specs/master/cluster-init/scripts/10.install_starccm.sh
@@ -0,0 +1,110 @@
+#!/bin/bash
+# Copyright (c) 2019 Hiroshi Tanaka, hirtanak@gmail.com @hirtanak
+set -exuv
+
+echo "starting 10.master.sh"
+
+export LC_ALL=en_US.UTF-8
+export LANG=en_US.UTF-8
+export LANGUAGE=en_US.UTF-8
+
+# disabling selinux
+echo "disabling selinux"
+setenforce 0
+sed -i -e "s/^SELINUX=enforcing$/SELINUX=disabled/g" /etc/selinux/config
+
+CUSER=$(grep "Added user" /opt/cycle/jetpack/logs/jetpackd.log | awk '{print $6}')
+CUSER=${CUSER//\'/}
+CUSER=${CUSER//\`/}
+echo ${CUSER} > /mnt/exports/shared/CUSER
+HOMEDIR=/shared/home/${CUSER}
+CYCLECLOUD_SPEC_PATH=/mnt/cluster-init/STAR-CCMplus/master
+
+# default parameters
+STARCCMPLUS_VERSION=14.04.011
+REVISION=02
+STARCCMPLUS_PLATFORM=linux-x86_64
+PRECISION=r8 #r8 is single precison
+# get file name
+STARCCMPLUSFILENAME=$(jetpack config StarccmFileName)
+# set parameters
+STARCCMPLUS_VERSION=${STARCCMPLUSFILENAME:9:9}
+REVISION=${STARCCMPLUSFILENAME:19:2}
+STARCCMPLUS_PLATFORM=${STARCCMPLUSFILENAME:22:12}
+PRECISION=${STARCCMPLUSFILENAME:35:2}
+MPI_TYPE=$(jetpack config MPI)
+MODEL=$(jetpack config MODEL)
+
+# Create tempdir
+tmpdir=$(mktemp -d)
+pushd $tmpdir
+
+# Azure VMs that have ephemeral storage mounted at /mnt/exports.
+if [ ! -d ${HOMEDIR}/apps ]; then
+   sudo -u ${CUSER} ln -s /mnt/exports/apps ${HOMEDIR}/apps
+   chown ${CUSER}:${CUSER} /mnt/exports/apps
+fi
+chown ${CUSER}:${CUSER} /mnt/exports/apps | exit 0
+
+# install packages
+yum install -y perl-Digest-MD5.x86_64 redhat-lsb-core vtk vtk-devel gcc gcc-gcc++
+
+# License File Setting
+LICENSE=$(jetpack config LICENSE)
+PODKEY=$(jetpack config PODKEY)
+(echo "export STARCCMPLUS_VERSION=${STARCCMPLUS_VERSION}"; echo "export PRECISION=-${PRECISION^}"; echo "export CDLMD_LICENSE_FILE=${LICENSE}"; echo "export PODKEY=${PODKEY}") > /etc/profile.d/starccm.sh
+chmod a+x /etc/profile.d/starccm.sh
+chown ${CUSER}:${CUSER} /etc/profile.d/starccm.sh
+
+# Don't run if we've already expanded the STAR-CCM+ tarball. Download STAR-CCM+ installer into tempdir and unpack it into the apps directory
+if [[ ! -f ${HOMEDIR}/apps/STAR-CCM+${STARCCMPLUS_VERSION}_${REVISION}_${STARCCMPLUS_PLATFORM}-${PRECISION}.tar.gz ]]; then
+   jetpack download "STAR-CCM+${STARCCMPLUS_VERSION}_${REVISION}_${STARCCMPLUS_PLATFORM}-${PRECISION}.tar.gz" ${HOMEDIR}/apps/STAR-CCM+${STARCCMPLUS_VERSION}_${REVISION}_${STARCCMPLUS_PLATFORM}-${PRECISION}.tar.gz
+   chown ${CUSER}:${CUSER} ${HOMEDIR}/apps/STAR-CCM+${STARCCMPLUS_VERSION}_${REVISION}_${STARCCMPLUS_PLATFORM}-${PRECISION}.tar.gz
+   tar zxfp ${HOMEDIR}/apps/STAR-CCM+${STARCCMPLUS_VERSION}_${REVISION}_${STARCCMPLUS_PLATFORM}-${PRECISION}.tar.gz -C ${HOMEDIR}/apps
+   chown ${CUSER}:${CUSER} ${HOMEDIR}/apps/starccm+_${STARCCMPLUS_VERSION}
+fi
+
+chown -R ${CUSER}:${CUSER} ${HOMEDIR}/apps/starccm+_${STARCCMPLUS_VERSION} | exit 0
+
+source /etc/profile.d/starccm.sh
+if [[ ! -f ${HOMEDIR}/apps/${STARCCMPLUS_VERSION}${PRECISION}/STAR-CCM+${STARCCMPLUS_VERSION}/star/bin/starccm+ ]]; then
+   case ${STARCCMPLUS_VERSION:0:2} in 
+   "12" ) SCRIPT_VERSION="2.5_gnu4.8" ;;
+   "14" ) SCRIPT_VERSION="2.12_gnu7.1" ;;
+   esac
+#   ${HOMEDIR}/apps/starccm+_${STARCCMPLUS_VERSION}/STAR-CCM+14.04.011_02_linux-x86_64-2.12_gnu7.1-r8.sh -i silent -DINSTALLDIR=${HOMEDIR}/apps -DNODOC=true -DINSTALLFLEX=false
+   sudo -u root ${HOMEDIR}/apps/starccm+_${STARCCMPLUS_VERSION}/Disk1/InstData/VM/STARCCMPLUS.bin -i silent -DINSTALLDIR=${HOMEDIR}/apps -DNODOC=true -DINSTALLFLEX=false
+fi
+
+# download standard models
+if [[ ! -f ${HOMEDIR}/${MODEL%%.gz} ]]; then
+   jetpack download ${MODEL} ${HOMEDIR}/ | exit 0
+   gunzip -f -d ${HOMEDIR}/${MODEL} | exit 0
+   chown ${CUSER}:${CUSER} ${HOMEDIR}/${MODEL} | exit 0
+   case ${MODEL} in
+   "TurboCharger-NoRun.sim.gz" ) 
+       gunzip -f -d ${HOMEDIR}/${MODEL} | exit 0
+       chown ${CUSER}:${CUSER} ${HOMEDIR}/${MODEL%%.gz} | exit 0
+       echo "8d4dbbd82a6b468b394ca717c0abc599" > ${HOMEDIR}/modelchecksum
+       chown ${CUSER}:${CUSER} ${HOMEDIR}/modelchecksum | exit 0 ;;
+   esac
+fi
+
+# local file settings
+if [[ ! -f ${HOMEDIR}/starccmrun.sh ]]; then
+   cp ${CYCLECLOUD_SPEC_PATH}/files/starccmrun.sh ${HOMEDIR}/
+   chmod a+rx ${HOMEDIR}/starccmrun.sh
+   chown ${CUSER}:${CUSER} ${HOMEDIR}/starccmrun.sh
+fi
+
+# file settings
+chown -R ${CUSER}:${CUSER} ${HOMEDIR}/apps | exit 0
+chown -R ${CUSER}:${CUSER} ${HOMEDIR}/apps/${STARCCMPLUS_VERSION}${PRECISION} | exit 0
+cp  /opt/cycle/jetpack/logs/cluster-init/STAR-CCMplus/master/scripts/10.install_starccm.sh.out ${HOMEDIR}/ | exit 0
+chown ${CUSER}:${CUSER} ${HOMEDIR}/10.install_starccm.sh.out | exit 0
+
+#clean up
+popd
+rm -rf $tmpdir
+
+echo "end of 10.master.sh"
diff --git a/specs/master/cluster-init/tests/README.txt b/specs/master/cluster-init/tests/README.txt
new file mode 100644
index 0000000..a7149f8
--- /dev/null
+++ b/specs/master/cluster-init/tests/README.txt
@@ -0,0 +1,5 @@
+
+Files in this directory contains tests that will be run at cluster start
+when in testing mode. Please see the official documentation for more information
+on cluster testing.
+            
\ No newline at end of file
diff --git a/templates/pbs_extended_nfs_starccm.txt b/templates/pbs_extended_nfs_starccm.txt
new file mode 100644
index 0000000..47e791e
--- /dev/null
+++ b/templates/pbs_extended_nfs_starccm.txt
@@ -0,0 +1,281 @@
+# Copyright (c) 2019 Hiroshi Tanaka, hirtanak@gmail.com @hirtanak
+# Licensed under the MIT License.
+################################
+## Cluster Configuration File ##
+################################
+
+[cluster STAR-CCM+]
+FormLayout = selectionpanel
+Category = Applications
+IconUrl = https://raw.githubusercontent.com/hirtanak/scripts/master/SiemensStarCCMPluslogo.png
+
+#Autoscale = $Autoscale #Autoscale as Disable
+
+    [[node defaults]]
+    UsePublicNetwork = $UsePublicNetwork
+    Credentials = $Credentials    
+    SubnetId = $SubnetId
+    Region = $Region
+    KeyPairLocation = ~/.ssh/cyclecloud.pem
+#    KeyPairLocation = ~/examples/id_rsaccuser 
+   
+        [[[cluster-init STAR-CCMplus:default:1.0.1]]]
+        Optional = True
+
+        [[[configuration]]]
+        StarccmFileName = $configuration_StarccmFileName
+	LICENSE = $configuration_LICENSE
+        PODKEY = $configuration_PODKEY
+        MPI = $configuration_MPI
+	MODEL = $configuration_Model
+
+    [[node master]]
+    MachineType = $MasterMachineType
+    IsReturnProxy = $ReturnProxy
+    AdditionalClusterInitSpecs = $MasterClusterInitSpecs
+    ImageName = $ImageName
+
+        [[[configuration]]]
+
+        [[[cluster-init STAR-CCMplus:master:1.0.1]]]
+        Optional = True
+
+        [[[network-interface eth0]]]
+        AssociatePublicIpAddress = $UsePublicNetwork
+	StaticPublicIpAddress = true
+
+        [[[input-endpoint ganglia]]]
+        PrivatePort = 8652
+        PublicPort = 8652
+
+        # Add 2 premium disks in a RAID 0 configuration to the NFS export
+        [[[volume nfs-1]]]
+        Size = 512
+        SSD = True
+        Mount = nfs
+        Persistent = true
+
+        [[[volume nfs-2]]]
+        Size = 512
+        SSD = True
+        Mount = nfs
+        Persistent = true
+
+	[[[configuration cyclecloud.mounts.nfs]]]
+	mountpoint = /mnt/exports/apps
+	fs_type = ext4
+	raid_level = 0
+
+        [[[configuration cyclecloud.exports.nfs_data]]]
+        type = nfs
+        export_path = /mnt/exports/apps
+#        raid_level = 0
+
+    [[nodearray execute]]
+    MachineType = $ExecuteMachineType
+    MaxCoreCount = $MaxExecuteCoreCount
+    Interruptible = $UseLowPrio
+    AdditionalClusterInitSpecs = $ExecuteClusterInitSpecs
+    InstallJetpack = true
+    JetpackPlatform = centos-7
+    ImageName = $ImageNameExecute
+    Azure.SingleScaleset = true
+
+        [[[configuration]]]
+
+        [[[cluster-init STAR-CCMplus:execute:1.0.1]]]
+        Optional = True
+
+        [[[network-interface eth0]]]
+        AssociatePublicIpAddress = $ExecuteNodesPublic
+
+        [[[configuration cyclecloud.mounts.nfs_data]]]
+        type = nfs
+        mountpoint = /mnt/exports/apps
+        export_path = /mnt/exports/apps
+
+[parameters About]
+Order = 1
+
+    [[parameters About STAR-CCMplus]]
+
+        [[[parameter STAR-CCMplus]]]
+        HideLabel = true
+        Config.Plugin = pico.widget.HtmlTemplateWidget
+        Config.Template = '''<p>Insert template description here</p>'''
+
+[parameters Required Settings]
+Order = 10
+
+    [[parameters Virtual Machines ]]
+    Description = "The cluster, in this case, has two roles: the scheduler master-node with shared filer and the execute hosts. Configure which VM types to use based on the requirements of your application."
+    Order = 20
+
+        [[[parameter Region]]]
+        Label = Region
+        Description = Deployment Location
+        ParameterType = Cloud.Region
+        DefaultValue = japaneast
+
+        [[[parameter MasterMachineType]]]
+        Label = Master VM Type
+        Description = The VM type for scheduler master and shared filer.
+        ParameterType = Cloud.MachineType
+        DefaultValue = Standard_DS3_v2
+
+        [[[parameter ExecuteMachineType]]]
+        Label = Execute VM Type
+        Description = The VM type for execute nodes
+        ParameterType = Cloud.MachineType
+        DefaultValue = Standard_H16r
+        Config.Multiselect = true
+        Config.Filter := Package in {"Standard_H16r", "Standard_H16r_Promo", "Standard_HB60rs", "StandardHC44rs"}
+        config.plugin = pico.control.AutoCompleteDropdown
+            [[[[list Config.Entries]]]]
+            Name = Standard_H16r
+            Label = Standard_H16r
+            [[[[list Config.Entries]]]]
+            Name = Standard_H16r_Promo
+            Label = Standard_H16r_Promo
+            [[[[list Config.Entries]]]]
+            Name = Standard_HB60rs
+            Label = Standard_HB60rs
+            [[[[list Config.Entries]]]]
+            Name = Standard_HC44rs
+            Label = Standard_HC44rs
+
+    [[parameters Auto-Scaling]]
+    Description = "The cluster can autoscale to the workload, adding execute hosts as jobs are queued. To enable this check the box below and choose the initial and maximum core counts for the cluster"
+    Order = 30
+
+        [[[parameter Autoscale]]]
+        Label = Autoscale
+        DefaultValue = true
+        Widget.Plugin = pico.form.BooleanCheckBox
+        Widget.Label = Start and stop execute instances automatically
+
+        [[[parameter MaxExecuteCoreCount]]]
+        Label = Max Cores
+        Description = The total number of execute cores to start
+        DefaultValue = 128
+        Config.Plugin = pico.form.NumberTextBox
+        Config.MinValue = 0
+        Config.IntegerOnly = true
+
+        [[[parameter UseLowPrio]]]
+        Label = Low Priority
+        DefaultValue = true
+        Widget.Plugin = pico.form.BooleanCheckBox
+        Widget.Label = Use low priority instances for execute hosts
+
+    [[parameters Networking]]
+    Order = 40
+
+        [[[parameter SubnetId]]]
+        Label = Subnet ID
+        Description = Subnet Resource Path (ResourceGroup/VirtualNetwork/Subnet)
+        ParameterType = Azure.Subnet
+        Required = True
+
+
+[parameters Advanced Settings]
+Order = 20
+
+    [[parameters Azure Settings]]
+    Order = 10 
+
+        [[[parameter Credentials]]]
+        Description = The credentials for the cloud provider
+        ParameterType = Cloud.Credentials
+
+    [[parameters STAR-CCMplus Settings ]]
+    Description = "Section for configuring STAR-CCMplus"
+    Order = 5
+
+        [[[parameter configuration_StarccmFileName]]]
+	required = True
+	label = STAR-CCM+ version
+	Description = Version of STAR-CCMplus to install on the cluster. Package should be named STAR-CCM+-<version>_<platform><precision>.tar.gz
+	defaultvalue = STAR-CCM+14.04.011_02_linux-x86_64-r8.tar.gz
+
+        [[[parameter configuration_LICENSE]]]
+    	requied = False
+	label = License Server Address
+	Description = "PoD License Server <Port#>@<IP Address>"
+	defaultvalue = 1999@flex.cd-adapco.com
+
+        [[[parameter configuration_PODKEY]]]
+        requied = False
+	label = PoDKey
+        Description = "PoD License Key"
+        defaultvalue = 22 digits number
+
+        [[[parameter configuration_MPI]]]
+        requied = False
+        label = MPI type
+        Description = MPI setting IMPI, Plarform MPI
+        defaultvalue = impi
+        Config.Filter := Package in {"impi", "platform"}
+        config.plugin = pico.control.AutoCompleteDropdown
+            [[[[list Config.Entries]]]]
+            Name = impi
+            Label = impi
+            [[[[list Config.Entries]]]]
+            Name = platform
+            Label = plarform
+
+        [[[parameter configuration_Model]]]
+        requied = False
+        label = Model Name
+        Description = "Model Name"
+        defaultvalue = test1.sim
+
+    [[parameters Software]]
+    Description = "Specify the scheduling software, and base OS installed on all nodes, and optionally the cluster-init and chef versions from your Locker."
+    Order = 10
+
+        [[[parameter ImageName]]]
+        Label = Master Node OS
+        ParameterType = Cloud.Image
+        Config.OS = linux
+	DefaultValue = OpenLogic:CentOS-HPC:7.6:latest
+
+        [[[parameter MasterClusterInitSpecs]]]
+        Label = Master Cluster-Init
+        DefaultValue = =undefined
+        Description = Cluster init specs to apply to the master node
+        ParameterType = Cloud.ClusterInitSpecs
+
+        [[[parameter ImageNameExecute]]]
+        Label = Compile Node OS
+        ParameterType = Cloud.Image
+        Config.OS = linux
+        DefaultValue = OpenLogic:CentOS-HPC:7.6:latest
+
+        [[[parameter ExecuteClusterInitSpecs]]]
+        Label = Execute Cluster-Init
+        DefaultValue = =undefined
+        Description = Cluster init specs to apply to execute nodes
+        ParameterType = Cloud.ClusterInitSpecs
+	
+    [[parameters Advanced Networking]]
+    Description = Advanced networking settings
+
+        [[[parameter ReturnProxy]]]
+        Label = Return Proxy
+        DefaultValue = true
+        ParameterType = Boolean
+        Config.Label = Use SSH tunnel to connect to CycleCloud (required if direct access is blocked)
+
+        [[[parameter UsePublicNetwork]]]
+        Label = Public Head Node
+        DefaultValue = true
+        ParameterType = Boolean
+        Config.Label = Access master node from the Internet
+
+        [[[parameter ExecuteNodesPublic]]]
+        Label = Public Execute
+        DefaultValue = false
+        ParameterType = Boolean
+        Config.Label = Access execute nodes from the Internet
+        Conditions.Excluded := UsePublicNetwork isnt true