From 3b059dd0a5f6d3ccc91837a9ca84e2c488934579 Mon Sep 17 00:00:00 2001 From: Nick Foti Date: Fri, 18 Sep 2015 10:10:06 -0700 Subject: [PATCH 1/9] Initialize VolumeCreate._snapshot to None. --- starcluster/volume.py | 1 + 1 file changed, 1 insertion(+) diff --git a/starcluster/volume.py b/starcluster/volume.py index 1ae2f507e..d0fa67380 100644 --- a/starcluster/volume.py +++ b/starcluster/volume.py @@ -58,6 +58,7 @@ def __init__(self, ec2_conn, spot_bid=None, keypair=None, self._mkfs_cmd = mkfs_cmd self._resizefs_cmd = resizefs_cmd self._alias_tmpl = "volhost-%s" + self._snapshot = None super(VolumeCreator, self).__init__( ec2_conn=ec2_conn, spot_bid=spot_bid, keyname=keypair, key_location=key_location, cluster_tag=static.VOLUME_GROUP_NAME, From ea241a79c7ef07919a276b892080511365ddde7b Mon Sep 17 00:00:00 2001 From: Nick Foti Date: Fri, 20 Nov 2015 13:47:58 -0800 Subject: [PATCH 2/9] Use default AMIs in correct zone when creating and resizing EBS volumes. --- starcluster/commands/createvolume.py | 4 ++++ starcluster/commands/resizevolume.py | 5 +++++ starcluster/static.py | 10 +++++++--- starcluster/templates/config.py | 6 +++--- starcluster/volume.py | 11 ++++++----- 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/starcluster/commands/createvolume.py b/starcluster/commands/createvolume.py index b8d754db2..7a3329f72 100644 --- a/starcluster/commands/createvolume.py +++ b/starcluster/commands/createvolume.py @@ -16,6 +16,7 @@ # along with StarCluster. If not, see . import os +import re from starcluster import node from starcluster import volume @@ -141,6 +142,9 @@ def execute(self, args): kwargs = self.specified_options_dict kwargs.update(dict(keypair=keypair, key_location=key_location, host_instance=host_instance)) + if 'image_id' not in kwargs.keys(): + zone_short = re.match(".+-.+-[1-9]+", zone).group() + kwargs['image_id'] = static.BASE_AMI_64[zone_short] vc = volume.VolumeCreator(self.ec2, **kwargs) if host_instance: vc._validate_host_instance(host_instance, zone) diff --git a/starcluster/commands/resizevolume.py b/starcluster/commands/resizevolume.py index b3aa75a93..f8b41439b 100644 --- a/starcluster/commands/resizevolume.py +++ b/starcluster/commands/resizevolume.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU Lesser General Public License # along with StarCluster. If not, see . +import re + from starcluster import node from starcluster import volume from starcluster import static @@ -96,6 +98,9 @@ def execute(self, args): kwargs = self.specified_options_dict kwargs.update(dict(keypair=keypair, key_location=key_location, host_instance=host_instance)) + if 'image_id' not in kwargs.keys(): + zone_short = re.match(".+-.+-[1-9]+", zone).group() + kwargs['image_id'] = static.BASE_AMI_64[zone_short] vc = volume.VolumeCreator(self.ec2, **kwargs) if host_instance: vc._validate_host_instance(host_instance, zone) diff --git a/starcluster/static.py b/starcluster/static.py index abc40896e..879164bd3 100644 --- a/starcluster/static.py +++ b/starcluster/static.py @@ -81,9 +81,13 @@ def create_sc_config_dirs(): CRASH_FILE = os.path.join(STARCLUSTER_LOG_DIR, 'crash-report-%d.txt' % PID) # StarCluster BASE AMIs (us-east-1) -BASE_AMI_32 = "ami-9bf9c9f2" -BASE_AMI_64 = "ami-3393a45a" -BASE_AMI_HVM = "ami-6b211202" +BASE_AMI_32 = {'us-east-1': "ami-9bf9c9f2", 'us-west-1': "ami-52112317", + 'us-west-2': "ami-b2badb82"} +BASE_AMI_64 = {'us-east-1': "ami-3393a45a", 'us-west-1': "ami-56172513", + 'us-west-2': "ami-04bedf34"} +BASE_AMI_HVM = {'us-east-1': "ami-6b211202", 'us-west-1': "ami-06172543", + 'us-west-2': "ami-80bedfb0"} + SECURITY_GROUP_PREFIX = "@sc-" SECURITY_GROUP_TEMPLATE = SECURITY_GROUP_PREFIX + "%s" diff --git a/starcluster/templates/config.py b/starcluster/templates/config.py index 885d741c3..8490a496d 100644 --- a/starcluster/templates/config.py +++ b/starcluster/templates/config.py @@ -349,9 +349,9 @@ # [plugin xvfb] # SETUP_CLASS = starcluster.plugins.xvfb.XvfbSetup """ % { - 'x86_ami': static.BASE_AMI_32, - 'x86_64_ami': static.BASE_AMI_64, - 'hvm_ami': static.BASE_AMI_HVM, + 'x86_ami': static.BASE_AMI_32['us-east-1'], + 'x86_64_ami': static.BASE_AMI_64['us-east-1'], + 'hvm_ami': static.BASE_AMI_HVM['us-east-1'], 'instance_types': ', '.join(static.INSTANCE_TYPES.keys()), 'shells': ', '.join(static.AVAILABLE_SHELLS.keys()), } diff --git a/starcluster/volume.py b/starcluster/volume.py index d0fa67380..06b2eda5f 100644 --- a/starcluster/volume.py +++ b/starcluster/volume.py @@ -43,15 +43,16 @@ class VolumeCreator(cluster.Cluster): """ def __init__(self, ec2_conn, spot_bid=None, keypair=None, key_location=None, host_instance=None, device='/dev/sdz', - image_id=static.BASE_AMI_32, instance_type="t1.micro", - shutdown_instance=False, detach_vol=False, - mkfs_cmd='mkfs.ext3 -F', resizefs_cmd='resize2fs', **kwargs): + image_id=static.BASE_AMI_32['us-east-1'], + instance_type="t1.micro", shutdown_instance=False, + detach_vol=False, mkfs_cmd='mkfs.ext3 -F', + resizefs_cmd='resize2fs', **kwargs): self._host_instance = host_instance self._instance = None self._volume = None self._aws_block_device = device or '/dev/sdz' self._real_device = None - self._image_id = image_id or static.BASE_AMI_32 + self._image_id = image_id or static.BASE_AMI_32['us-east-1'] self._instance_type = instance_type or 'm1.small' self._shutdown = shutdown_instance self._detach_vol = detach_vol @@ -207,7 +208,7 @@ def is_valid(self, size, zone, device): try: self.validate(size, zone, device) return True - except exception.BaseException, e: + except exception.BaseException as e: log.error(e.msg) return False From d56a255c8302c5dfa9fa0bd3e1209997674608e6 Mon Sep 17 00:00:00 2001 From: Nick Foti Date: Fri, 22 Jan 2016 11:41:29 -0800 Subject: [PATCH 3/9] Default to instances in correct zone when creating and resizing EBS volumens. --- starcluster/volume.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/starcluster/volume.py b/starcluster/volume.py index 06b2eda5f..2e036d87f 100644 --- a/starcluster/volume.py +++ b/starcluster/volume.py @@ -43,10 +43,9 @@ class VolumeCreator(cluster.Cluster): """ def __init__(self, ec2_conn, spot_bid=None, keypair=None, key_location=None, host_instance=None, device='/dev/sdz', - image_id=static.BASE_AMI_32['us-east-1'], - instance_type="t1.micro", shutdown_instance=False, - detach_vol=False, mkfs_cmd='mkfs.ext3 -F', - resizefs_cmd='resize2fs', **kwargs): + image_id=static.BASE_AMI_32['us-east-1'], instance_type="t1.micro", + shutdown_instance=False, detach_vol=False, + mkfs_cmd='mkfs.ext3 -F', resizefs_cmd='resize2fs', **kwargs): self._host_instance = host_instance self._instance = None self._volume = None From b55d2c3d59bd46ef5cd5c54ee76e9ce78ad7e87a Mon Sep 17 00:00:00 2001 From: Nick Foti Date: Mon, 22 Feb 2016 15:58:01 -0800 Subject: [PATCH 4/9] Modified SGE plugin to create an smp parallel environment. --- starcluster/plugins/sge.py | 8 +++++++- starcluster/templates/sge.py | 16 +++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/starcluster/plugins/sge.py b/starcluster/plugins/sge.py index 2a3662708..100d362f8 100644 --- a/starcluster/plugins/sge.py +++ b/starcluster/plugins/sge.py @@ -77,7 +77,12 @@ def _create_sge_pe(self, name="orte", nodes=None, queue="all.q"): pe_slots = self.slots_per_host * len(nodes) if not pe_exists: penv = mssh.remote_file("/tmp/pe.txt", "w") - penv.write(sge.sge_pe_template % (name, pe_slots)) + if name == "orte": + template = sge.sge_orte_pe_template + elif name == "smp": + template = sge.sge_smp_pe_template + penv.write(template % (name, pe_slots)) + #penv.write(sge.sge_pe_template % (name, pe_slots)) penv.close() mssh.execute("qconf -Ap %s" % penv.name) else: @@ -156,6 +161,7 @@ def _setup_sge(self): self.pool.simple_job(self._add_to_sge, (node,), jobid=node.alias) self.pool.wait(numtasks=len(self.nodes)) self._create_sge_pe() + self._create_sge_pe(name="smp") def _remove_from_sge(self, node): master = self._master diff --git a/starcluster/templates/sge.py b/starcluster/templates/sge.py index dcf189941..97ac4bfaa 100644 --- a/starcluster/templates/sge.py +++ b/starcluster/templates/sge.py @@ -59,7 +59,7 @@ CSP_MAIL_ADDRESS="none@none.edu" """ -sge_pe_template = """ +sge_orte_pe_template = """ pe_name %s slots %s user_lists NONE @@ -73,6 +73,20 @@ accounting_summary FALSE """ +sge_smp_pe_template = """ +pe_name %s +slots %s +user_lists NONE +xuser_lists NONE +start_proc_args /bin/true +stop_proc_args /bin/true +allocation_rule $pe_slots +control_slaves FALSE +job_is_first_task TRUE +urgency_slots min +accounting_summary FALSE +""" + sgeprofile_template = """ export SGE_ROOT="/opt/sge6" export SGE_CELL="default" From ca5a80b01240c03c1ebdd6e61ddec2f1888ae188 Mon Sep 17 00:00:00 2001 From: Nick Foti Date: Tue, 30 May 2017 15:23:45 -0700 Subject: [PATCH 5/9] explicitly create orte pe --- starcluster/plugins/sge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/starcluster/plugins/sge.py b/starcluster/plugins/sge.py index 100d362f8..633c976bf 100644 --- a/starcluster/plugins/sge.py +++ b/starcluster/plugins/sge.py @@ -160,7 +160,7 @@ def _setup_sge(self): for node in self.nodes: self.pool.simple_job(self._add_to_sge, (node,), jobid=node.alias) self.pool.wait(numtasks=len(self.nodes)) - self._create_sge_pe() + self._create_sge_pe(name="orte") self._create_sge_pe(name="smp") def _remove_from_sge(self, node): From 848494bbab50c4b223a0d75eb25887c38b3afddc Mon Sep 17 00:00:00 2001 From: Nick Foti Date: Fri, 8 Jun 2018 16:57:58 -0700 Subject: [PATCH 6/9] addnodes and removenodes updates smp grid engine queue too --- starcluster/plugins/sge.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/starcluster/plugins/sge.py b/starcluster/plugins/sge.py index 633c976bf..4055541ac 100644 --- a/starcluster/plugins/sge.py +++ b/starcluster/plugins/sge.py @@ -172,7 +172,8 @@ def _remove_from_sge(self, node): master.ssh.execute('qconf -de %s' % node.alias) node.ssh.execute('pkill -9 sge_execd') nodes = filter(lambda n: n.alias != node.alias, self._nodes) - self._create_sge_pe(nodes=nodes) + self._create_sge_pe(name="orte", nodes=nodes) + self._create_sge_pe(name="smp", nodes=nodes) def run(self, nodes, master, user, user_shell, volumes): if not master.ssh.isdir(self.SGE_FRESH): @@ -198,7 +199,8 @@ def on_add_node(self, node, nodes, master, user, user_shell, volumes): self._add_sge_admin_host(node) self._add_sge_submit_host(node) self._add_to_sge(node) - self._create_sge_pe() + self._create_sge_pe(name="orte") + self._create_sge_pe(name="smp") def on_remove_node(self, node, nodes, master, user, user_shell, volumes): self._nodes = nodes From 4bfb000af6c4a15b713735ecd242b0f5ce673a80 Mon Sep 17 00:00:00 2001 From: Nick Foti Date: Tue, 12 Jun 2018 15:33:27 -0700 Subject: [PATCH 7/9] added c5 instance types to static.py --- starcluster/static.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/starcluster/static.py b/starcluster/static.py index 879164bd3..8c337bbfd 100644 --- a/starcluster/static.py +++ b/starcluster/static.py @@ -160,6 +160,12 @@ def create_sc_config_dirs(): 'c4.2xlarge': ['x86_64'], 'c4.4xlarge': ['x86_64'], 'c4.8xlarge': ['x86_64'], + 'c5.large': ['x86_64'], + 'c5.xlarge': ['x86_64'], + 'c5.2xlarge': ['x86_64'], + 'c5.4xlarge': ['x86_64'], + 'c5.9xlarge': ['x86_64'], + 'c5.18xlarge': ['x86_64'], 'i2.xlarge': ['x86_64'], 'i2.2xlarge': ['x86_64'], 'i2.4xlarge': ['x86_64'], From 1df2c1991a86c6cb6e7edcbd517acb583b096e44 Mon Sep 17 00:00:00 2001 From: Nick Foti Date: Tue, 12 Jun 2018 15:38:26 -0700 Subject: [PATCH 8/9] added c5 instance types to HVM-only types --- starcluster/static.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/starcluster/static.py b/starcluster/static.py index 8c337bbfd..58542e93f 100644 --- a/starcluster/static.py +++ b/starcluster/static.py @@ -195,6 +195,9 @@ def create_sc_config_dirs(): HI_STORAGE_TYPES = ['hs1.8xlarge'] +C5_COMPUTE_TYPES = ['c5.large', 'c5.xlarge', 'c5.2xlarge', 'c5.4xlarge', + 'c5.9xlarge', 'c5.18xlarge'] + M3_COMPUTE_TYPES = ['c3.large', 'c3.xlarge', 'c3.2xlarge', 'c3.4xlarge', 'c3.8xlarge'] @@ -207,6 +210,7 @@ def create_sc_config_dirs(): DENSE_STORAGE_TYPES = ['d2.xlarge', 'd2.2xlarge', 'd2.4xlarge', 'd2.8xlarge'] HVM_ONLY_TYPES = (CLUSTER_COMPUTE_TYPES + CLUSTER_GPU_TYPES + + C5_COMPUTE_TYPES + CLUSTER_HIMEM_TYPES + I2_STORAGE_TYPES + HIMEM_TYPES + T2_INSTANCE_TYPES + DENSE_STORAGE_TYPES) From 685a3b3ecb0c778b4a7637131423205ae02f9875 Mon Sep 17 00:00:00 2001 From: Nick Foti Date: Wed, 13 Jun 2018 11:00:37 -0700 Subject: [PATCH 9/9] added r4 instance types --- starcluster/static.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/starcluster/static.py b/starcluster/static.py index 58542e93f..6f9486186 100644 --- a/starcluster/static.py +++ b/starcluster/static.py @@ -142,6 +142,12 @@ def create_sc_config_dirs(): 'r3.2xlarge': ['x86_64'], 'r3.4xlarge': ['x86_64'], 'r3.8xlarge': ['x86_64'], + 'r4.large': ['x86_64'], + 'r4.xlarge': ['x86_64'], + 'r4.2xlarge': ['x86_64'], + 'r4.4xlarge': ['x86_64'], + 'r4.8xlarge': ['x86_64'], + 'r4.16xlarge': ['x86_64'], 'cc1.4xlarge': ['x86_64'], 'cc2.8xlarge': ['x86_64'], 'cg1.4xlarge': ['x86_64'], @@ -189,7 +195,9 @@ def create_sc_config_dirs(): CLUSTER_HIMEM_TYPES = ['cr1.8xlarge'] HIMEM_TYPES = ['r3.large', 'r3.xlarge', 'r3.2xlarge', 'r3.4xlarge', - 'r3.8xlarge'] + 'r3.8xlarge', + 'r4.large', 'r4.xlarge', 'r4.2xlarge', 'r4.4xlarge', + 'r4.8xlarge', 'r4.16xlarge'] HI_IO_TYPES = ['hi1.4xlarge']