Initialize VolumeCreate._snapshot to None.

starcluster/commands/createvolume.py

@@ -16,6 +16,7 @@
 # along with StarCluster. If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import re
 
 from starcluster import node
 from starcluster import volume
@@ -141,6 +142,9 @@ def execute(self, args):
         kwargs = self.specified_options_dict
         kwargs.update(dict(keypair=keypair, key_location=key_location,
                            host_instance=host_instance))
+        if 'image_id' not in kwargs.keys():
+            zone_short = re.match(".+-.+-[1-9]+", zone).group()
+            kwargs['image_id'] = static.BASE_AMI_64[zone_short]
         vc = volume.VolumeCreator(self.ec2, **kwargs)
         if host_instance:
             vc._validate_host_instance(host_instance, zone)

starcluster/commands/resizevolume.py

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with StarCluster. If not, see <http://www.gnu.org/licenses/>.
 
+import re
+
 from starcluster import node
 from starcluster import volume
 from starcluster import static
@@ -96,6 +98,9 @@ def execute(self, args):
         kwargs = self.specified_options_dict
         kwargs.update(dict(keypair=keypair, key_location=key_location,
                            host_instance=host_instance))
+        if 'image_id' not in kwargs.keys():
+            zone_short = re.match(".+-.+-[1-9]+", zone).group()
+            kwargs['image_id'] = static.BASE_AMI_64[zone_short]
         vc = volume.VolumeCreator(self.ec2, **kwargs)
         if host_instance:
             vc._validate_host_instance(host_instance, zone)

starcluster/plugins/sge.py

@@ -77,7 +77,12 @@ def _create_sge_pe(self, name="orte", nodes=None, queue="all.q"):
             pe_slots = self.slots_per_host * len(nodes)
         if not pe_exists:
             penv = mssh.remote_file("/tmp/pe.txt", "w")
-            penv.write(sge.sge_pe_template % (name, pe_slots))
+            if name == "orte":
+                template = sge.sge_orte_pe_template
+            elif name == "smp":
+                template = sge.sge_smp_pe_template
+            penv.write(template % (name, pe_slots))
+            #penv.write(sge.sge_pe_template % (name, pe_slots))
             penv.close()
             mssh.execute("qconf -Ap %s" % penv.name)
         else:
@@ -155,7 +160,8 @@ def _setup_sge(self):
         for node in self.nodes:
             self.pool.simple_job(self._add_to_sge, (node,), jobid=node.alias)
         self.pool.wait(numtasks=len(self.nodes))
-        self._create_sge_pe()
+        self._create_sge_pe(name="orte")
+        self._create_sge_pe(name="smp")
 
     def _remove_from_sge(self, node):
         master = self._master
@@ -166,7 +172,8 @@ def _remove_from_sge(self, node):
         master.ssh.execute('qconf -de %s' % node.alias)
         node.ssh.execute('pkill -9 sge_execd')
         nodes = filter(lambda n: n.alias != node.alias, self._nodes)
-        self._create_sge_pe(nodes=nodes)
+        self._create_sge_pe(name="orte", nodes=nodes)
+        self._create_sge_pe(name="smp", nodes=nodes)
 
     def run(self, nodes, master, user, user_shell, volumes):
         if not master.ssh.isdir(self.SGE_FRESH):
@@ -192,7 +199,8 @@ def on_add_node(self, node, nodes, master, user, user_shell, volumes):
         self._add_sge_admin_host(node)
         self._add_sge_submit_host(node)
         self._add_to_sge(node)
-        self._create_sge_pe()
+        self._create_sge_pe(name="orte")
+        self._create_sge_pe(name="smp")
 
     def on_remove_node(self, node, nodes, master, user, user_shell, volumes):
         self._nodes = nodes

starcluster/static.py

@@ -81,9 +81,13 @@ def create_sc_config_dirs():
 CRASH_FILE = os.path.join(STARCLUSTER_LOG_DIR, 'crash-report-%d.txt' % PID)
 
 # StarCluster BASE AMIs (us-east-1)
-BASE_AMI_32 = "ami-9bf9c9f2"
-BASE_AMI_64 = "ami-3393a45a"
-BASE_AMI_HVM = "ami-6b211202"
+BASE_AMI_32 = {'us-east-1': "ami-9bf9c9f2", 'us-west-1': "ami-52112317",
+               'us-west-2': "ami-b2badb82"}
+BASE_AMI_64 = {'us-east-1': "ami-3393a45a", 'us-west-1': "ami-56172513",
+               'us-west-2': "ami-04bedf34"}
+BASE_AMI_HVM = {'us-east-1': "ami-6b211202", 'us-west-1': "ami-06172543",
+                'us-west-2': "ami-80bedfb0"}
+
 
 SECURITY_GROUP_PREFIX = "@sc-"
 SECURITY_GROUP_TEMPLATE = SECURITY_GROUP_PREFIX + "%s"
@@ -138,6 +142,12 @@ def create_sc_config_dirs():
     'r3.2xlarge': ['x86_64'],
     'r3.4xlarge': ['x86_64'],
     'r3.8xlarge': ['x86_64'],
+    'r4.large': ['x86_64'],
+    'r4.xlarge': ['x86_64'],
+    'r4.2xlarge': ['x86_64'],
+    'r4.4xlarge': ['x86_64'],
+    'r4.8xlarge': ['x86_64'],
+    'r4.16xlarge': ['x86_64'],
     'cc1.4xlarge': ['x86_64'],
     'cc2.8xlarge': ['x86_64'],
     'cg1.4xlarge': ['x86_64'],
@@ -156,6 +166,12 @@ def create_sc_config_dirs():
     'c4.2xlarge': ['x86_64'],
     'c4.4xlarge': ['x86_64'],
     'c4.8xlarge': ['x86_64'],
+    'c5.large': ['x86_64'],
+    'c5.xlarge': ['x86_64'],
+    'c5.2xlarge': ['x86_64'],
+    'c5.4xlarge': ['x86_64'],
+    'c5.9xlarge': ['x86_64'],
+    'c5.18xlarge': ['x86_64'],
     'i2.xlarge': ['x86_64'],
     'i2.2xlarge': ['x86_64'],
     'i2.4xlarge': ['x86_64'],
@@ -179,12 +195,17 @@ def create_sc_config_dirs():
 CLUSTER_HIMEM_TYPES = ['cr1.8xlarge']
 
 HIMEM_TYPES = ['r3.large', 'r3.xlarge', 'r3.2xlarge', 'r3.4xlarge',
-               'r3.8xlarge']
+               'r3.8xlarge',
+               'r4.large', 'r4.xlarge', 'r4.2xlarge', 'r4.4xlarge',
+               'r4.8xlarge', 'r4.16xlarge']
 
 HI_IO_TYPES = ['hi1.4xlarge']
 
 HI_STORAGE_TYPES = ['hs1.8xlarge']
 
+C5_COMPUTE_TYPES = ['c5.large', 'c5.xlarge', 'c5.2xlarge', 'c5.4xlarge',
+                    'c5.9xlarge', 'c5.18xlarge']
+
 M3_COMPUTE_TYPES = ['c3.large', 'c3.xlarge', 'c3.2xlarge', 'c3.4xlarge',
                     'c3.8xlarge']
 
@@ -197,6 +218,7 @@ def create_sc_config_dirs():
 DENSE_STORAGE_TYPES = ['d2.xlarge', 'd2.2xlarge', 'd2.4xlarge', 'd2.8xlarge']
 
 HVM_ONLY_TYPES = (CLUSTER_COMPUTE_TYPES + CLUSTER_GPU_TYPES +
+                  C5_COMPUTE_TYPES +
                   CLUSTER_HIMEM_TYPES + I2_STORAGE_TYPES + HIMEM_TYPES +
                   T2_INSTANCE_TYPES + DENSE_STORAGE_TYPES)
 

starcluster/templates/config.py

@@ -349,9 +349,9 @@
 # [plugin xvfb]
 # SETUP_CLASS = starcluster.plugins.xvfb.XvfbSetup
 """ % {
-    'x86_ami': static.BASE_AMI_32,
-    'x86_64_ami': static.BASE_AMI_64,
-    'hvm_ami': static.BASE_AMI_HVM,
+    'x86_ami': static.BASE_AMI_32['us-east-1'],
+    'x86_64_ami': static.BASE_AMI_64['us-east-1'],
+    'hvm_ami': static.BASE_AMI_HVM['us-east-1'],
     'instance_types': ', '.join(static.INSTANCE_TYPES.keys()),
     'shells': ', '.join(static.AVAILABLE_SHELLS.keys()),
 }

starcluster/templates/sge.py

@@ -59,7 +59,7 @@
 CSP_MAIL_ADDRESS="[email protected]"
 """
 
-sge_pe_template = """
+sge_orte_pe_template = """
 pe_name           %s
 slots             %s
 user_lists        NONE
@@ -73,6 +73,20 @@
 accounting_summary FALSE
 """
 
+sge_smp_pe_template = """
+pe_name           %s
+slots             %s
+user_lists        NONE
+xuser_lists       NONE
+start_proc_args   /bin/true
+stop_proc_args    /bin/true
+allocation_rule   $pe_slots
+control_slaves    FALSE
+job_is_first_task TRUE
+urgency_slots     min
+accounting_summary FALSE
+"""
+
 sgeprofile_template = """
 export SGE_ROOT="/opt/sge6"
 export SGE_CELL="default"

starcluster/volume.py

@@ -43,21 +43,22 @@ class VolumeCreator(cluster.Cluster):
     """
     def __init__(self, ec2_conn, spot_bid=None, keypair=None,
                  key_location=None, host_instance=None, device='/dev/sdz',
-                 image_id=static.BASE_AMI_32, instance_type="t1.micro",
+                 image_id=static.BASE_AMI_32['us-east-1'], instance_type="t1.micro",
                  shutdown_instance=False, detach_vol=False,
                  mkfs_cmd='mkfs.ext3 -F', resizefs_cmd='resize2fs', **kwargs):
         self._host_instance = host_instance
         self._instance = None
         self._volume = None
         self._aws_block_device = device or '/dev/sdz'
         self._real_device = None
-        self._image_id = image_id or static.BASE_AMI_32
+        self._image_id = image_id or static.BASE_AMI_32['us-east-1']
         self._instance_type = instance_type or 'm1.small'
         self._shutdown = shutdown_instance
         self._detach_vol = detach_vol
         self._mkfs_cmd = mkfs_cmd
         self._resizefs_cmd = resizefs_cmd
         self._alias_tmpl = "volhost-%s"
+        self._snapshot = None
         super(VolumeCreator, self).__init__(
             ec2_conn=ec2_conn, spot_bid=spot_bid, keyname=keypair,
             key_location=key_location, cluster_tag=static.VOLUME_GROUP_NAME,
@@ -206,7 +207,7 @@ def is_valid(self, size, zone, device):
         try:
             self.validate(size, zone, device)
             return True
-        except exception.BaseException, e:
+        except exception.BaseException as e:
             log.error(e.msg)
             return False