From 65e71f21e451578da1f4217d54326e46acbbfba7 Mon Sep 17 00:00:00 2001 From: Elveskevtar Date: Fri, 17 Aug 2018 14:07:44 -0700 Subject: [PATCH 1/9] Set accounting_summary to TRUE for multi node job journaling Signed-off-by: Elveskevtar --- recipes/_master_sge_config.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/recipes/_master_sge_config.rb b/recipes/_master_sge_config.rb index 0b885a383..7472ccb78 100644 --- a/recipes/_master_sge_config.rb +++ b/recipes/_master_sge_config.rb @@ -55,6 +55,17 @@ ADDHOST end +bash "set_accounting_summary" do + code <<-SETAS + . /opt/sge/default/common/settings.sh + TMPFILE=/tmp/pe.txt + qconf -sp mpi | grep -v 'accounting_summary' > $TMPFILE + echo 'accounting_summary TRUE' >> $TMPFILE + qconf -Mp $TMPFILE + rm $TMPFILE + SETAS +end + template '/opt/cfncluster/scripts/publish_pending' do source 'publish_pending.sge.erb' owner 'root' From b065fe47e3d33fad9773c81e1580d5780044f5a0 Mon Sep 17 00:00:00 2001 From: Sean Smith Date: Fri, 24 Aug 2018 13:50:36 -0700 Subject: [PATCH 2/9] Release 1.5.4 Signed-off-by: Sean Smith --- CHANGELOG.md | 7 +++++++ amis/packer_variables.json | 4 ++-- attributes/default.rb | 2 +- metadata.rb | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f640e7061..3943b8ece 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,13 @@ cfncluster-cookbook CHANGELOG This file is used to list changes made in each version of the cfncluster cookbook. +1.5.4 +----- + +- Set SGE Accounting summary to be true, this reports a single accounting record +for a mpi job + + 1.5.2 ----- diff --git a/amis/packer_variables.json b/amis/packer_variables.json index bc06bb6ed..13bef4c76 100644 --- a/amis/packer_variables.json +++ b/amis/packer_variables.json @@ -1,6 +1,6 @@ { - "cfncluster_version": "1.5.2", - "cfncluster_cookbook_version": "1.5.2", + "cfncluster_version": "1.5.4", + "cfncluster_cookbook_version": "1.5.4", "chef_version": "14.2.0", "ridley_version": "5.1.1", "berkshelf_version": "7.0.4" diff --git a/attributes/default.rb b/attributes/default.rb index 8d5b86a36..75055f206 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -19,7 +19,7 @@ default['cfncluster']['scripts_dir'] = "#{node['cfncluster']['base_dir']}/scripts" default['cfncluster']['license_dir'] = "#{node['cfncluster']['base_dir']}/licenses" # Python packages -default['cfncluster']['cfncluster-node-version'] = '1.5.2' +default['cfncluster']['cfncluster-node-version'] = '1.5.4' default['cfncluster']['cfncluster-supervisor-version'] = '3.3.1' # URLs to software packages used during install receipes # Gridengine software diff --git a/metadata.rb b/metadata.rb index 2f7c626a4..5a34ca38a 100644 --- a/metadata.rb +++ b/metadata.rb @@ -6,7 +6,7 @@ long_description 'Installs/Configures cfncluster' issues_url 'https://github.com/awslabs/cfncluster-cookbook/issues' source_url 'https://github.com/awslabs/cfncluster-cookbook' -version '1.5.2' +version '1.5.4' depends 'build-essential', '~> 8.1.1' depends 'poise-python', '~> 1.7.0' From 2443b03cdff9ba15ff733f56b8ba98b9d98e8609 Mon Sep 17 00:00:00 2001 From: Sean Smith Date: Wed, 15 Aug 2018 13:20:00 -0700 Subject: [PATCH 3/9] Turn off Ganglia Support Ganglia can be turned off by adding to the cluster config: ``` extra_json = { "cfncluster" : { "ganglia_enabled" : "no" } } ``` Signed-off-by: Sean Smith --- attributes/default.rb | 1 + recipes/_compute_base_config.rb | 20 +-- recipes/_ganglia_install.rb | 238 ++++++++++++++++---------------- recipes/_master_base_config.rb | 48 ++++--- 4 files changed, 157 insertions(+), 150 deletions(-) diff --git a/attributes/default.rb b/attributes/default.rb index 75055f206..f33ba5766 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -35,6 +35,7 @@ default['cfncluster']['munge']['munge_version'] = '0.5.12' default['cfncluster']['munge']['munge_url'] = 'https://github.com/dun/munge/archive/munge-0.5.12.tar.gz' # Ganglia +default['cfncluster']['ganglia_enabled'] = 'yes' default['cfncluster']['ganglia']['version'] = '3.7.2' default['cfncluster']['ganglia']['url'] = 'https://github.com/ganglia/monitor-core/archive/3.7.2.tar.gz' default['cfncluster']['ganglia']['web_version'] = '3.7.2' diff --git a/recipes/_compute_base_config.rb b/recipes/_compute_base_config.rb index 7ca0a8799..a361a4292 100644 --- a/recipes/_compute_base_config.rb +++ b/recipes/_compute_base_config.rb @@ -43,16 +43,18 @@ end # Configure Ganglia -template '/etc/ganglia/gmond.conf' do - source 'gmond.conf.erb' - owner 'root' - group 'root' - mode '0644' -end +if node['cfncluster']['ganglia_enabled'] == "yes" + template '/etc/ganglia/gmond.conf' do + source 'gmond.conf.erb' + owner 'root' + group 'root' + mode '0644' + end -service node['cfncluster']['ganglia']['gmond_service'] do - supports restart: true - action %i[enable restart] + service node['cfncluster']['ganglia']['gmond_service'] do + supports restart: true + action %i[enable restart] + end end # Setup cluster user diff --git a/recipes/_ganglia_install.rb b/recipes/_ganglia_install.rb index 63cff8ebd..ccd41db82 100644 --- a/recipes/_ganglia_install.rb +++ b/recipes/_ganglia_install.rb @@ -13,136 +13,138 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. -case node['platform'] -when "redhat", "centos", "amazon", "scientific" # ~FC024 - - package "httpd" - package "apr-devel" - package "libconfuse-devel" - package "expat-devel" - package "rrdtool-devel" - package "pcre-devel" - package "php" - package "php-gd" - - # Get Ganglia tarball - ganglia_tarball = "#{node['cfncluster']['sources_dir']}/monitor-core-#{node['cfncluster']['ganglia']['version']}.tar.gz" - remote_file ganglia_tarball do - source node['cfncluster']['ganglia']['url'] - mode '0644' - # TODO: Add version or checksum checks - not_if { ::File.exist?(ganglia_tarball) } - end +if node['cfncluster']['ganglia_enabled'] == 'yes' + case node['platform'] + when "redhat", "centos", "amazon", "scientific" # ~FC024 + + package "httpd" + package "apr-devel" + package "libconfuse-devel" + package "expat-devel" + package "rrdtool-devel" + package "pcre-devel" + package "php" + package "php-gd" + + # Get Ganglia tarball + ganglia_tarball = "#{node['cfncluster']['sources_dir']}/monitor-core-#{node['cfncluster']['ganglia']['version']}.tar.gz" + remote_file ganglia_tarball do + source node['cfncluster']['ganglia']['url'] + mode '0644' + # TODO: Add version or checksum checks + not_if { ::File.exist?(ganglia_tarball) } + end - ## - # Install Ganglia - bash 'make install' do - user 'root' - group 'root' - cwd Chef::Config[:file_cache_path] - code <<-GANGLIA - tar xf #{ganglia_tarball} - cd monitor-core-#{node['cfncluster']['ganglia']['version']} - ./bootstrap - ./configure --build=x86_64-redhat-linux-gnu --host=x86_64-redhat-linux-gnu --program-prefix= --disable-dependency-tracking \ - --prefix=/usr --exec-prefix=/usr --bindir=/usr/bin --sbindir=/usr/sbin --sysconfdir=/etc --datadir=/usr/share --includedir=/usr/include \ - --libdir=/usr/lib64 --libexecdir=/usr/libexec --localstatedir=/var --sharedstatedir=/var/lib --mandir=/usr/share/man \ - --infodir=/usr/share/info --with-gmetad --enable-status --sysconfdir=/etc/ganglia - CORES=$(grep processor /proc/cpuinfo | wc -l) - make -j $CORES - make install - GANGLIA - # TODO: Fix, so it works for upgrade - creates '/usr/sbin/gmetad' - end + ## + # Install Ganglia + bash 'make install' do + user 'root' + group 'root' + cwd Chef::Config[:file_cache_path] + code <<-GANGLIA + tar xf #{ganglia_tarball} + cd monitor-core-#{node['cfncluster']['ganglia']['version']} + ./bootstrap + ./configure --build=x86_64-redhat-linux-gnu --host=x86_64-redhat-linux-gnu --program-prefix= --disable-dependency-tracking \ + --prefix=/usr --exec-prefix=/usr --bindir=/usr/bin --sbindir=/usr/sbin --sysconfdir=/etc --datadir=/usr/share --includedir=/usr/include \ + --libdir=/usr/lib64 --libexecdir=/usr/libexec --localstatedir=/var --sharedstatedir=/var/lib --mandir=/usr/share/man \ + --infodir=/usr/share/info --with-gmetad --enable-status --sysconfdir=/etc/ganglia + CORES=$(grep processor /proc/cpuinfo | wc -l) + make -j $CORES + make install + GANGLIA + # TODO: Fix, so it works for upgrade + creates '/usr/sbin/gmetad' + end - if node['init_package'] == 'init' - # Setup init.d scripts if not systemd - execute "copy gmetad init script" do - command "cp " \ - "#{Chef::Config[:file_cache_path]}/monitor-core-#{node['cfncluster']['ganglia']['version']}/gmetad/gmetad.init " \ - "/etc/init.d/gmetad" - not_if "test -f /etc/init.d/gmetad" + if node['init_package'] == 'init' + # Setup init.d scripts if not systemd + execute "copy gmetad init script" do + command "cp " \ + "#{Chef::Config[:file_cache_path]}/monitor-core-#{node['cfncluster']['ganglia']['version']}/gmetad/gmetad.init " \ + "/etc/init.d/gmetad" + not_if "test -f /etc/init.d/gmetad" + end + execute "copy gmmond init script" do + command "cp " \ + "#{Chef::Config[:file_cache_path]}/monitor-core-#{node['cfncluster']['ganglia']['version']}/gmond/gmond.init " \ + "/etc/init.d/gmond" + not_if "test -f /etc/init.d/gmond" + end end - execute "copy gmmond init script" do - command "cp " \ - "#{Chef::Config[:file_cache_path]}/monitor-core-#{node['cfncluster']['ganglia']['version']}/gmond/gmond.init " \ - "/etc/init.d/gmond" - not_if "test -f /etc/init.d/gmond" + + # Reload systemd + execute 'systemctl-daemon-reload' do + command '/bin/systemctl --system daemon-reload' + only_if "test -f /bin/systemctl" end - end - # Reload systemd - execute 'systemctl-daemon-reload' do - command '/bin/systemctl --system daemon-reload' - only_if "test -f /bin/systemctl" - end + # Get Ganglia Web tarball + ganglia_web_tarball = "#{node['cfncluster']['sources_dir']}/ganglia-web-#{node['cfncluster']['ganglia']['web_version']}.tar.gz" + remote_file ganglia_web_tarball do + source node['cfncluster']['ganglia']['web_url'] + mode '0644' + # TODO: Add version or checksum checks + not_if { ::File.exist?(ganglia_web_tarball) } + end - # Get Ganglia Web tarball - ganglia_web_tarball = "#{node['cfncluster']['sources_dir']}/ganglia-web-#{node['cfncluster']['ganglia']['web_version']}.tar.gz" - remote_file ganglia_web_tarball do - source node['cfncluster']['ganglia']['web_url'] - mode '0644' - # TODO: Add version or checksum checks - not_if { ::File.exist?(ganglia_web_tarball) } - end + ## + # Install Ganglia Web + bash 'make install' do + user 'root' + group 'root' + cwd Chef::Config[:file_cache_path] + code <<-GANGLIAWEB + tar xf #{ganglia_web_tarball} + cd ganglia-web-#{node['cfncluster']['ganglia']['web_version']} + make install APACHE_USER=#{node['cfncluster']['ganglia']['apache_user']} + GANGLIAWEB + # TODO: Fix, so it works for upgrade + creates '/usr/share/ganglia-webfrontend/index.php' + end - ## - # Install Ganglia Web - bash 'make install' do - user 'root' - group 'root' - cwd Chef::Config[:file_cache_path] - code <<-GANGLIAWEB - tar xf #{ganglia_web_tarball} - cd ganglia-web-#{node['cfncluster']['ganglia']['web_version']} - make install APACHE_USER=#{node['cfncluster']['ganglia']['apache_user']} - GANGLIAWEB - # TODO: Fix, so it works for upgrade - creates '/usr/share/ganglia-webfrontend/index.php' - end + cookbook_file 'ganglia-webfrontend.conf' do + path '/etc/httpd/conf.d/ganglia-webfrontend.conf' + user 'root' + group 'root' + mode '0644' + end - cookbook_file 'ganglia-webfrontend.conf' do - path '/etc/httpd/conf.d/ganglia-webfrontend.conf' - user 'root' - group 'root' - mode '0644' - end + directory '/var/lib/ganglia/rrds' do + owner 'nobody' + group 'nobody' + mode 0755 + recursive true + action :create + end - directory '/var/lib/ganglia/rrds' do - owner 'nobody' - group 'nobody' - mode 0755 - recursive true - action :create - end + # Copy required licensing files + directory "#{node['cfncluster']['license_dir']}/ganglia" + + bash 'copy license stuff' do + user 'root' + group 'root' + cwd Chef::Config[:file_cache_path] + code <<-GANGLIALICENSE + cd monitor-core-#{node['cfncluster']['ganglia']['version']} + cp -v COPYING #{node['cfncluster']['license_dir']}/ganglia/COPYING + GANGLIALICENSE + # TODO: Fix, so it works for upgrade + creates "#{node['cfncluster']['license_dir']}/ganglia/COPYING" + end - # Copy required licensing files - directory "#{node['cfncluster']['license_dir']}/ganglia" - - bash 'copy license stuff' do - user 'root' - group 'root' - cwd Chef::Config[:file_cache_path] - code <<-GANGLIALICENSE - cd monitor-core-#{node['cfncluster']['ganglia']['version']} - cp -v COPYING #{node['cfncluster']['license_dir']}/ganglia/COPYING - GANGLIALICENSE - # TODO: Fix, so it works for upgrade - creates "#{node['cfncluster']['license_dir']}/ganglia/COPYING" - end + when "ubuntu" -when "ubuntu" + package "ganglia-monitor" + package "rrdtool" + package "gmetad" + package "ganglia-webfrontend" - package "ganglia-monitor" - package "rrdtool" - package "gmetad" - package "ganglia-webfrontend" + # Setup ganglia-web.conf apache config + execute "copy ganglia apache conf" do + command "cp /etc/ganglia-webfrontend/apache.conf /etc/apache2/sites-enabled/ganglia.conf" + not_if "test -f /etc/apache2/sites-enabled/ganglia.conf" + end - # Setup ganglia-web.conf apache config - execute "copy ganglia apache conf" do - command "cp /etc/ganglia-webfrontend/apache.conf /etc/apache2/sites-enabled/ganglia.conf" - not_if "test -f /etc/apache2/sites-enabled/ganglia.conf" end - end diff --git a/recipes/_master_base_config.rb b/recipes/_master_base_config.rb index a71339da5..6449fdefb 100644 --- a/recipes/_master_base_config.rb +++ b/recipes/_master_base_config.rb @@ -93,33 +93,35 @@ end # Configure Ganglia on the Master -template '/etc/ganglia/gmetad.conf' do - source 'gmetad.conf.erb' - owner 'root' - group 'root' - mode '0644' -end +if node['cfncluster']['ganglia_enabled'] == 'yes' + template '/etc/ganglia/gmetad.conf' do + source 'gmetad.conf.erb' + owner 'root' + group 'root' + mode '0644' + end -template '/etc/ganglia/gmond.conf' do - source 'gmond.conf.erb' - owner 'root' - group 'root' - mode '0644' -end + template '/etc/ganglia/gmond.conf' do + source 'gmond.conf.erb' + owner 'root' + group 'root' + mode '0644' + end -service "gmetad" do - supports restart: true - action %i[enable restart] -end + service "gmetad" do + supports restart: true + action %i[enable restart] + end -service node['cfncluster']['ganglia']['gmond_service'] do - supports restart: true - action %i[enable restart] -end + service node['cfncluster']['ganglia']['gmond_service'] do + supports restart: true + action %i[enable restart] + end -service node['cfncluster']['ganglia']['httpd_service'] do - supports restart: true - action %i[enable start] + service node['cfncluster']['ganglia']['httpd_service'] do + supports restart: true + action %i[enable start] + end end # Setup cluster user From 6651bdc32546f356062f4afb759b3bcccbc03661 Mon Sep 17 00:00:00 2001 From: Luca Carrogu Date: Mon, 27 Aug 2018 12:01:10 +0200 Subject: [PATCH 4/9] Fix packer build for ubuntu1404 updating awscli version This fix the following errors on ubuntu1404: - awscli 1.15.40 has requirement botocore==1.10.40%!(PACKER_COMMA) but you'll have botocore 1.11.1 which is incompatible. - Cannot uninstall 'urllib3'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall. Signed-off-by: Luca Carrogu --- recipes/base_install.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/recipes/base_install.rb b/recipes/base_install.rb index 13ed344fd..dfaa9bfc0 100644 --- a/recipes/base_install.rb +++ b/recipes/base_install.rb @@ -61,7 +61,10 @@ # Install AWSCLI python_package 'awscli' do action :upgrade - version '1.15.40' + version '1.16.2' + if node['platform'] == 'ubuntu' && node['platform_version'] == "14.04" + install_options '--ignore-installed urllib3' + end end # Install boto3 From 492517ab7f7cf02d469ee9859d4f069c6c75cd3c Mon Sep 17 00:00:00 2001 From: Sean Smith Date: Tue, 28 Aug 2018 10:59:06 -0700 Subject: [PATCH 5/9] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3943b8ece..d5b0847ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ This file is used to list changes made in each version of the cfncluster cookboo - Set SGE Accounting summary to be true, this reports a single accounting record for a mpi job +- Add option to disable ganglia `extra_json = { "cfncluster" : { "ganglia_enabled" : "no" } }` 1.5.2 From 575898f58effcaaaee84a79c488fb09ebecd7a81 Mon Sep 17 00:00:00 2001 From: Sean Smith Date: Tue, 7 Aug 2018 12:09:57 -0700 Subject: [PATCH 6/9] GovCloud Changes Signed-off-by: Sean Smith --- amis/build_ami.sh | 27 +++++++++++++++++++-------- amis/packer_alinux.json | 15 ++++++++++++--- amis/packer_centos6.json | 15 ++++++++++++--- amis/packer_centos7.json | 15 ++++++++++++--- amis/packer_ubuntu1404.json | 15 ++++++++++++--- amis/packer_ubuntu1604.json | 15 ++++++++++++--- 6 files changed, 79 insertions(+), 23 deletions(-) diff --git a/amis/build_ami.sh b/amis/build_ami.sh index 0a1bafddf..5bf7a076d 100755 --- a/amis/build_ami.sh +++ b/amis/build_ami.sh @@ -13,19 +13,20 @@ # # Usage: build_ami.sh [private|public] [] # os: the os to build (supported values: all|centos6|centos7|alinux|ubuntu1404|ubuntu1604) -# region: region to build (supported values: all|us-east-1|...) +# partition: partition to build in (supported values: commercial|govcloud) +# region: region to copy ami too (supported values: all|us-east-1|us-gov-west-1|...) # private|public: specifies AMIs visibility (optional, default is private) # build-date: timestamp to append to the AMIs names (optional) set -e os=$1 -region=$2 -public=$3 -build_date=$4 +partition=$2 +region=$3 +public=$4 +build_date=$5 available_os="centos6 centos7 alinux ubuntu1404 ubuntu1604" -available_regions="$(aws ec2 --region us-east-1 describe-regions --query Regions[].RegionName --output text | tr '\t' ',')" cwd="$(dirname $0)" export VENDOR_PATH="${cwd}/../../vendor/cookbooks" @@ -36,8 +37,8 @@ if [ "x${os}" == "x" ]; then fi if [ "x${region}" == "x" ]; then - echo "Must provide AWS region to build for." - echo "Options: us-east-1 all" + echo "Must provide AWS region to copy ami into" + echo "Options: all us-east-1 us-gov-west-1 ..." exit 1 fi @@ -45,8 +46,18 @@ if [ "${public}" == "public" ]; then export AMI_PERMS="all" fi -# NOTE: the AMI is always built in us-east-1 and then copied to the specified regions +if [ "${partition}" == "commercial" ]; then + export AWS_REGION="us-east-1" +elif [ "${partition}" == "govcloud" ]; then + export AWS_REGION="us-gov-west-1" +else + echo "Must provide AWS partition to build for." + echo "Options: commercial govcloud" + exit 1 +fi + if [ "${region}" == "all" ]; then + available_regions="$(aws ec2 --region ${AWS_REGION} describe-regions --query Regions[].RegionName --output text | tr '\t' ',')" export BUILD_FOR=${available_regions} else export BUILD_FOR=${region} diff --git a/amis/packer_alinux.json b/amis/packer_alinux.json index d44cd629f..9f21fa403 100644 --- a/amis/packer_alinux.json +++ b/amis/packer_alinux.json @@ -13,13 +13,22 @@ "instance_type" : "{{env `AWS_FLAVOR_ID`}}", "subnet_id" : "{{env `AWS_SUBNET_ID`}}", "vpc_id" : "{{env `AWS_VPC_ID`}}", - "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}" + "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}", + "region" : "{{env `AWS_REGION`}}" }, "builders" : [ { "type" : "amazon-ebs", - "region" : "us-east-1", - "source_ami" : "ami-55ef662f", + "region" : "{{user `region`}}", + "source_ami_filter": { + "filters": { + "virtualization-type": "hvm", + "name": "amzn-ami-hvm-*.*.*.*-x86_64-gp2", + "root-device-type": "ebs", + "owner-alias": "amazon" + }, + "most_recent": true + }, "ami_regions" : "{{user `build_for`}}", "ami_groups" : "{{user `ami_perms`}}", "instance_type" : "{{user `instance_type`}}", diff --git a/amis/packer_centos6.json b/amis/packer_centos6.json index 0c8417fa3..765f54bb0 100644 --- a/amis/packer_centos6.json +++ b/amis/packer_centos6.json @@ -13,13 +13,22 @@ "instance_type" : "{{env `AWS_FLAVOR_ID`}}", "subnet_id" : "{{env `AWS_SUBNET_ID`}}", "vpc_id" : "{{env `AWS_VPC_ID`}}", - "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}" + "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}", + "region" : "{{env `AWS_REGION`}}" }, "builders" : [ { "type" : "amazon-ebs", - "region" : "us-east-1", - "source_ami" : "ami-44f1aa3e", + "region" : "{{user `region`}}", + "source_ami_filter": { + "filters": { + "virtualization-type": "hvm", + "name": "CentOS 6.x x86_64 - minimal with cloud-init - *", + "root-device-type": "ebs" + }, + "owners": ["247102896272"], + "most_recent": true + }, "ami_regions" : "{{user `build_for`}}", "ami_groups" : "{{user `ami_perms`}}", "instance_type" : "{{user `instance_type`}}", diff --git a/amis/packer_centos7.json b/amis/packer_centos7.json index 4fd4c4e83..e56e3d1f5 100644 --- a/amis/packer_centos7.json +++ b/amis/packer_centos7.json @@ -13,13 +13,22 @@ "instance_type" : "{{env `AWS_FLAVOR_ID`}}", "subnet_id" : "{{env `AWS_SUBNET_ID`}}", "vpc_id" : "{{env `AWS_VPC_ID`}}", - "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}" + "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}", + "region" : "{{env `AWS_REGION`}}" }, "builders" : [ { "type" : "amazon-ebs", - "region" : "us-east-1", - "source_ami" : "ami-06fea57c", + "region" : "{{user `region`}}", + "source_ami_filter": { + "filters": { + "virtualization-type": "hvm", + "name": "CentOS 7.x x86_64 - minimal with cloud-init - *", + "root-device-type": "ebs" + }, + "owners": ["247102896272"], + "most_recent": true + }, "ami_regions" : "{{user `build_for`}}", "ami_groups" : "{{user `ami_perms`}}", "instance_type" : "{{user `instance_type`}}", diff --git a/amis/packer_ubuntu1404.json b/amis/packer_ubuntu1404.json index b00da023a..b79b90da5 100644 --- a/amis/packer_ubuntu1404.json +++ b/amis/packer_ubuntu1404.json @@ -13,15 +13,24 @@ "instance_type" : "{{env `AWS_FLAVOR_ID`}}", "subnet_id" : "{{env `AWS_SUBNET_ID`}}", "vpc_id" : "{{env `AWS_VPC_ID`}}", - "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}" + "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}", + "region" : "{{env `AWS_REGION`}}" }, "builders" : [ { "type" : "amazon-ebs", - "region" : "us-east-1", + "region" : "{{user `region`}}", + "source_ami_filter": { + "filters": { + "virtualization-type": "hvm", + "name": "ubuntu/images/hvm-ssd/ubuntu-trusty-14.04-amd64-server-*", + "root-device-type": "ebs" + }, + "owners": ["099720109477", "513442679011"], + "most_recent": true + }, "ami_regions" : "{{user `build_for`}}", "ami_groups" : "{{user `ami_perms`}}", - "source_ami" : "ami-c29e1cb8", "instance_type" : "{{user `instance_type`}}", "ssh_username" : "ubuntu", "ssh_pty" : true, diff --git a/amis/packer_ubuntu1604.json b/amis/packer_ubuntu1604.json index 8c4c64787..b0e45d36a 100644 --- a/amis/packer_ubuntu1604.json +++ b/amis/packer_ubuntu1604.json @@ -13,15 +13,24 @@ "instance_type" : "{{env `AWS_FLAVOR_ID`}}", "subnet_id" : "{{env `AWS_SUBNET_ID`}}", "vpc_id" : "{{env `AWS_VPC_ID`}}", - "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}" + "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}", + "region" : "{{env `AWS_REGION`}}" }, "builders" : [ { "type" : "amazon-ebs", - "region" : "us-east-1", + "region" : "{{user `region`}}", + "source_ami_filter": { + "filters": { + "virtualization-type": "hvm", + "name": "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-*", + "root-device-type": "ebs" + }, + "owners": ["099720109477", "513442679011"], + "most_recent": true + }, "ami_regions" : "{{user `build_for`}}", "ami_groups" : "{{user `ami_perms`}}", - "source_ami" : "ami-aa2ea6d0", "instance_type" : "{{user `instance_type`}}", "ssh_username" : "ubuntu", "ssh_pty" : true, From 80369aad1bc89e98c197500447e671c4b23f8d2c Mon Sep 17 00:00:00 2001 From: Sean Smith Date: Tue, 28 Aug 2018 13:57:45 -0700 Subject: [PATCH 7/9] Pin ubuntu1604 version Signed-off-by: Sean Smith --- amis/packer_ubuntu1604.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amis/packer_ubuntu1604.json b/amis/packer_ubuntu1604.json index b0e45d36a..8c4c589f1 100644 --- a/amis/packer_ubuntu1604.json +++ b/amis/packer_ubuntu1604.json @@ -23,7 +23,7 @@ "source_ami_filter": { "filters": { "virtualization-type": "hvm", - "name": "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-*", + "name": "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-20171121.1", "root-device-type": "ebs" }, "owners": ["099720109477", "513442679011"], From 60d01e0f135ab9fd3e95deef7331744edec35e14 Mon Sep 17 00:00:00 2001 From: Luca Carrogu Date: Mon, 28 May 2018 14:23:04 +0200 Subject: [PATCH 8/9] avoid double execution of resources notified by or which subscribes other resources Chef executes resources in the order they appear in a recipe. But also provide a way for: - resource to be notified by other resources - resource to subscribes other resources. The default action for a resource is "run", so it is executed sequentially as it appears in the recipe. If this resource subscribes another resource, it is also executed when the subscribed resource change its state. Same speech for the resource notified by another resource. --- recipes/_master_base_config.rb | 2 ++ recipes/torque_config.rb | 1 + 2 files changed, 3 insertions(+) diff --git a/recipes/_master_base_config.rb b/recipes/_master_base_config.rb index 6449fdefb..5d16c405c 100644 --- a/recipes/_master_base_config.rb +++ b/recipes/_master_base_config.rb @@ -38,6 +38,7 @@ block do wait_for_block_dev(dev_path) end + action :nothing subscribes :run, "execute[attach_volume]", :immediately end @@ -47,6 +48,7 @@ fs_type = setup_disk(dev_path) node.default['cfncluster']['cfn_volume_fs_type'] = fs_type end + action :nothing subscribes :run, "ruby_block[sleeping_for_volume]", :immediately end diff --git a/recipes/torque_config.rb b/recipes/torque_config.rb index e0cbcdc7d..b2a62819c 100644 --- a/recipes/torque_config.rb +++ b/recipes/torque_config.rb @@ -26,6 +26,7 @@ # Run ldconfig execute "run-ldconfig" do command 'ldconfig' + action :nothing end # Set toruqe server_name From 17bba16196dfab059ad8b726db28eed5f0a1fb2f Mon Sep 17 00:00:00 2001 From: Luca Carrogu Date: Wed, 29 Aug 2018 16:50:28 +0200 Subject: [PATCH 9/9] Unpin ubuntu 1604 AMI Signed-off-by: Luca Carrogu --- amis/packer_ubuntu1604.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amis/packer_ubuntu1604.json b/amis/packer_ubuntu1604.json index 8c4c589f1..b0e45d36a 100644 --- a/amis/packer_ubuntu1604.json +++ b/amis/packer_ubuntu1604.json @@ -23,7 +23,7 @@ "source_ami_filter": { "filters": { "virtualization-type": "hvm", - "name": "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-20171121.1", + "name": "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-*", "root-device-type": "ebs" }, "owners": ["099720109477", "513442679011"],