diff --git a/.gitignore b/.gitignore index 4040c6c..65637f2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .bundle Gemfile.lock pkg/* +.*.*~ diff --git a/README.md b/README.md index 4bf59f6..fa1c69a 100644 --- a/README.md +++ b/README.md @@ -37,38 +37,31 @@ roles: - iisserver: - monitoring: - webserver: -data bags: +data_bags: - users: - alice - bob - chuck - data: - - * + - "*" - passwords: - secret secret_key - mysql - rabbitmq nodes: -- serverA: - - role[base] - - -i ~/.ssh/mray.pem -x user --sudo -d ubuntu10.04-gems -- serverB serverC: - - role[base] - - -i ~/.ssh/mray.pem -x user --sudo -d ubuntu10.04-gems -E production -- ec2 3: - - role[webserver] recipe[mysql::client] - - -S mray -i ~/.ssh/mray.pem -x ubuntu -G default -I ami-7000f019 -f m1.small -- rackspace 3: - - recipe[mysql],role[monitoring] - - --image 49 --flavor 2 -- windows_winrm winboxA: - - role[base],role[iisserver] - - -x Administrator -P 'super_secret_password' -- windows_ssh winboxB winboxC: - - role[base],role[iisserver] - - -x Administrator -P 'super_secret_password' + - name: mysql-server + count: 3 + type: ec2 + run_list: + - role[mysql] + - recipe[hello] + options: -S 'key-pair' -i '~/.ssh/key.pem' -x ubuntu -G inside-db -I ami-123fsdf1 -f m1.large + - name: http + run_list: ["role[apache2]"] + options: -i '~/.ssh/key.pem' -x myuser ``` + JSON ---- From the `example.json`: @@ -98,7 +91,7 @@ From the `example.json`: {"monitoring":[]}, {"webserver":[]} ], - "data bags": + "data_bags": [ {"users": [ @@ -225,16 +218,16 @@ knife role from file webserver.rb Data Bags --------- -The `data bags` section of the manifest currently creates the data bags listed with `knife data bag create FOO` where `FOO` is the name of the data bag. Individual items may be added to the data bag as part of a JSON or YAML sequence, the assumption is made that they `.json` files and in the proper `data_bags/FOO` directory. You may also pass a wildcard as an entry to load all matching data bags (ie. `*`). Encrypted data bags are supported by listing `secret filename` as the first item (where `filename` is the secret key to be used). Validation is done to ensure the JSON is properly formatted, the id matches and any secret keys are in the correct locations. Assuming the presence of `dataA.json` and `dataB.json` in the `data_bags/data` directory, the YAML snippet +The `data_bags` section of the manifest currently creates the data bags listed with `knife data bag create FOO` where `FOO` is the name of the data bag. Individual items may be added to the data bag as part of a JSON or YAML sequence, the assumption is made that they `.json` files and in the proper `data_bags/FOO` directory. You may also pass a wildcard as an entry to load all matching data bags (ie. `"*"`). Encrypted data bags are supported by listing `secret filename` as the first item (where `filename` is the secret key to be used). Validation is done to ensure the JSON is properly formatted, the id matches and any secret keys are in the correct locations. Assuming the presence of `dataA.json` and `dataB.json` in the `data_bags/data` directory, the YAML snippet ``` yaml -data bags: +data_bags: - users: - alice - bob - chuck - data: - - * + - "*" - passwords: - secret secret_key - mysql @@ -258,48 +251,64 @@ knife data bag from file passwords rabbitmq.json --secret-file secret_key Nodes ----- -The `nodes` section of the manifest bootstraps a node for each entry where the entry is a hostname or provider and count. A shortcut syntax for bulk-creating nodes with various providers where the line starts with the provider and ends with the number of nodes to be provisioned. Windows nodes need to specify either `windows_winrm` or `windows_ssh` depending on the protocol used, followed by the name of the node(s). Each node requires 2 items after it in a sequence. You may also use the `--parallel` flag from the command line, allowing provider commands to run simultaneously for faster deployment. +The `nodes` section of the manifest bootstraps a node for each entry where the entry is a hostname or provider. + +Use the following list of directives to control the format of the knife command: +* name: + The -N option is passed to every knife command. Currently this isn't configurable so this option is required for the knife command to be generated correctly. + This directive is also used for bootstrap knife commands if the host directive isn't configured. + + Note: The node name is generated with a -## at the end of it. For example, the string "mysql" will actually be turned into mysql-01. This is to allow a number + of servers to be generated using the count directive. It ensures that the node names are different (as required by chef). In the future there will be a + way to disable this behavior. + +* host: + + Hostname or IP of the host to be bootstrapped. This directive is only used for nodes that do not use the "type" directive. It is usefull for boxes that are + not in DNS but need to be bootstrapped. It defaults to the "name" directive if it is not set. + +* count: + Generate of this type of server. This allows you to generate multiple servers of the same type (same run_list and knife options) without having to + configure number of nodes in the config file. + +* type: + Supported types: "bluebox","clodo","cs","ec2","gandi","hp","openstack","rackspace","slicehost","terremark","voxel" + + Type is used to declare which knife plugin to use for server create commands. If type is left blank, bootstrap commands are generated instead of cloud + specific commands. See the example second to see how type affects command generation. + +* run_list: ["role[]", "recipe[]"] + + A list of roles or recipes to add to the run list for this host. This directive has to be a valid yaml to json LIST type. + +* options: + + Additional command line options to be passed to the knife command. -The first item after the node is the run_list and the second are the CLI options used. The run_list may be space or comma-delimited. Validation is performed on the run_list components to ensure that only cookbooks and roles listed in the manifest are used. Validation on the options ensures that any Environments referenced are also listed. You may specify multiple nodes to have the same configuration by listing them separated by a space. The example YAML snippet +Windows support is currently untested (and is probably broken). ``` yaml nodes: -- serverA: - - role[base] - - -i ~/.ssh/mray.pem -x user --sudo -d ubuntu10.04-gems -- serverB serverC: - - role[base] - - -i ~/.ssh/mray.pem -x user --sudo -d ubuntu10.04-gems -E production -- ec2 3: - - role[webserver] recipe[mysql::client] - - -S mray -i ~/.ssh/mray.pem -x ubuntu -G default -I ami-7000f019 -f m1.small -- rackspace 3: - - recipe[mysql],role[monitoring] - - --image 49 --flavor 2 -- windows_winrm winboxA: - - role[base],role[iisserver] - - -x Administrator -P 'super_secret_password' -- windows_ssh winboxB winboxC: - - role[base],role[iisserver] - - -x Administrator -P 'super_secret_password' + - name: mysql-server + count: 3 + type: ec2 + run_list: + - role[mysql] + - recipe[hello] + options: -S 'key-pair' -i '~/.ssh/key.pem' -x ubuntu -G inside-db -I ami-123fsdf1 -f m1.large + - name: http + host: 192.168.1.10 + run_list: ["role[apache2]"] + options: -i '~/.ssh/key.pem' -x myuser ``` produces the knife commands ``` -knife bootstrap serverA -i ~/.ssh/mray.pem -x user --sudo -d ubuntu10.04-gems -r 'role[base]' -knife bootstrap serverB -i ~/.ssh/mray.pem -x user --sudo -d ubuntu10.04-gems -E production -r 'role[base]' -knife bootstrap serverC -i ~/.ssh/mray.pem -x user --sudo -d ubuntu10.04-gems -E production -r 'role[base]' -knife ec2 server create -S mray -i ~/.ssh/mray.pem -x ubuntu -G default -I ami-7000f019 -f m1.small -r 'role[webserver],recipe[mysql::client]' -knife ec2 server create -S mray -i ~/.ssh/mray.pem -x ubuntu -G default -I ami-7000f019 -f m1.small -r 'role[webserver],recipe[mysql::client]' -knife ec2 server create -S mray -i ~/.ssh/mray.pem -x ubuntu -G default -I ami-7000f019 -f m1.small -r 'role[webserver],recipe[mysql::client]' -knife ec2 server create -S mray -i ~/.ssh/mray.pem -x ubuntu -G default -I ami-7000f019 -f m1.small -r 'role[webserver],recipe[mysql::client]' -knife rackspace server create --image 49 --flavor 2 -r 'recipe[mysql],role[monitoring]' -knife rackspace server create --image 49 --flavor 2 -r 'recipe[mysql],role[monitoring]' -knife rackspace server create --image 49 --flavor 2 -r 'recipe[mysql],role[monitoring]' -knife bootstrap windows winrm winboxA -x Administrator -P 'super_secret_password' -r 'role[base],role[iisserver]' -knife bootstrap windows ssh winboxB -x Administrator -P 'super_secret_password' -r 'role[base],role[iisserver]' -knife bootstrap windows ssh winboxC -x Administrator -P 'super_secret_password' -r 'role[base],role[iisserver]' +knife ec2 server create -r role[mysql],recipe[hello] -S 'key-pair' -i '~/.ssh/key.pem' -x ubuntu -G inside-db -I ami-123fsdf1 -f m1.large -N 'mysql-server-01' +knife ec2 server create -r role[mysql],recipe[hello] -S 'key-pair' -i '~/.ssh/key.pem' -x ubuntu -G inside-db -I ami-123fsdf1 -f m1.large -N 'mysql-server-02' +knife ec2 server create -r role[mysql],recipe[hello] -S 'key-pair' -i '~/.ssh/key.pem' -x ubuntu -G inside-db -I ami-123fsdf1 -f m1.large -N 'mysql-server-03' +knife bootstrap '192.168.1.10' -r role[apache2] -i '~/.ssh/key.pem' -x myuser -N 'http-01' ``` Extract @@ -392,7 +401,7 @@ Use the 'install' command with 'knife cookbook site' instead of the default 'dow ------------ Print the version of spiceweasel currently installed. -License and Author +Original License and Author ================== Author: Matt Ray diff --git a/bin/spiceweasel b/bin/spiceweasel index d17ed03..0a164bf 100755 --- a/bin/spiceweasel +++ b/bin/spiceweasel @@ -18,8 +18,10 @@ # limitations under the License. # +require 'rubygems' require 'json' require 'yaml' +require 'pp' require 'spiceweasel' @@ -31,16 +33,17 @@ begin cli = Spiceweasel::CLI.new cli.parse_options DEBUG = cli.config[:debug] - PARALLEL = cli.config[:parallel] + CHEF_PRE_10 = cli.config[:chef_pre_10] SITEINSTALL = cli.config[:siteinstall] - NOVALIDATION = cli.config[:novalidation] + VALIDATION = cli.config[:validation] EXTRACTLOCAL = cli.config[:extractlocal] EXTRACTYAML = cli.config[:extractyaml] EXTRACTJSON = cli.config[:extractjson] + SKIP_NODENAME = cli.config[:skip_nodename] rescue OptionParser::InvalidOption => e STDERR.puts e.message puts cli.opt_parser.to_s - exit(-1) +n exit(-1) end if cli.config[:knifeconfig] @@ -53,7 +56,7 @@ end if EXTRACTLOCAL || EXTRACTJSON || EXTRACTYAML input = Spiceweasel::DirectoryExtractor.parse_objects - STDOUT.puts "DEBUG: extract input: #{input}" if DEBUG + STDOUT.puts "DEBUG: extract input: #{PP.pp(input)}" if DEBUG else begin file = ARGV.last @@ -66,11 +69,16 @@ else STDERR.puts "ERROR: Unknown file type, please use a file ending with either '.json' or '.yml'." exit(-1) end + rescue Psych::SyntaxError => e + STDERR.puts e.message + STDERR.puts "ERROR: Parsing error in #{file}." + exit(-1) rescue JSON::ParserError => e STDERR.puts e.message - STDERR.puts "ERROR: Parsing error in the infrastructure file provided." + STDERR.puts "ERROR: Parsing error in #{file}." exit(-1) - rescue Exception + rescue Exception => e + STDERR.puts e.message STDERR.puts "ERROR: No infrastructure .json or .yml file provided." puts cli.opt_parser.to_s exit(-1) @@ -81,11 +89,15 @@ end create = String.new() delete = String.new() -cookbook_list = Spiceweasel::CookbookList.new(input['cookbooks'], options) -environment_list = Spiceweasel::EnvironmentList.new(input['environments'], cookbook_list, options) -role_list = Spiceweasel::RoleList.new(input['roles'], environment_list, cookbook_list, options) -data_bag_list = Spiceweasel::DataBagList.new(input['data bags'], options) -node_list = Spiceweasel::NodeList.new(input['nodes'], cookbook_list, environment_list, role_list, options) +# This is a total hack to ensure the original input data structure does +# not get modified. (Deep copy) +input_copy = Marshal.load(Marshal.dump(input)) + +cookbook_list = Spiceweasel::CookbookList.new(input_copy['cookbooks'], options) +environment_list = Spiceweasel::EnvironmentList.new(input_copy['environments'], cookbook_list, options) +role_list = Spiceweasel::RoleList.new(input_copy['roles'], environment_list, cookbook_list, options) +data_bag_list = Spiceweasel::DataBagList.new(input_copy['data_bags'], options) +node_list = Spiceweasel::NodeList.new(input_copy['nodes'], cookbook_list, environment_list, role_list, options) create += cookbook_list.create create += environment_list.create diff --git a/example.json b/example.json index a889efd..d6c2041 100644 --- a/example.json +++ b/example.json @@ -22,7 +22,7 @@ {"monitoring":[]}, {"webserver":[]} ], - "data bags": + "data_bags": [ {"users": [ diff --git a/example.yml b/example.yml index 2402a4d..793421d 100644 --- a/example.yml +++ b/example.yml @@ -12,33 +12,25 @@ roles: - iisserver: - monitoring: - webserver: -data bags: +data_bags: - users: - alice - bob - chuck - data: - - * + - "*" - passwords: - secret secret_key - mysql - rabbitmq nodes: -- serverA: - - role[base] - - -i ~/.ssh/mray.pem -x user --sudo -d ubuntu10.04-gems -- serverB serverC: - - role[base] - - -i ~/.ssh/mray.pem -x user --sudo -d ubuntu10.04-gems -E production -- ec2 4: - - role[webserver] recipe[mysql::client] - - -S mray -i ~/.ssh/mray.pem -x ubuntu -G default -I ami-7000f019 -f m1.small -- rackspace 3: - - recipe[mysql],role[monitoring] - - --image 49 --flavor 2 -- windows_winrm winboxA: - - role[base],role[iisserver] - - -x Administrator -P 'super_secret_password' -- windows_ssh winboxB winboxC: - - role[base],role[iisserver] - - -x Administrator -P 'super_secret_password' + - name: mysql-server + count: 3 + type: ec2 + run_list: + - "role[mysql]" + - "recipe[hello]" + options: -S 'key-pair' -i '~/.ssh/key.pem' -x ubuntu -G inside-db -I ami-123fsdf1 -f m1.large + - name: http + run_list: ["role[apache2]"] + options: -i '~/.ssh/key.pem' -x myuser diff --git a/examples/php-quick-start.yml b/examples/php-quick-start.yml index 75e4fa2..15a81a3 100644 --- a/examples/php-quick-start.yml +++ b/examples/php-quick-start.yml @@ -59,7 +59,7 @@ roles: - mediawiki: - mediawiki_load_balancer: -data bags: +data_bags: - apps: - mediawiki diff --git a/lib/spiceweasel/cli.rb b/lib/spiceweasel/cli.rb index c545a4d..aac2b0f 100644 --- a/lib/spiceweasel/cli.rb +++ b/lib/spiceweasel/cli.rb @@ -53,14 +53,14 @@ class Spiceweasel::CLI :long => "--knifeconfig CONFIG", :description => "Specify the knife.rb configuration file" - option :novalidation, - :long => "--novalidation", - :description => "Disable validation", - :boolean => true - - option :parallel, - :long => "--parallel", - :description => "Use the GNU 'parallel' command to parallelize 'knife VENDOR server create' commands that are not order-dependent", + option :validation, + :long => "--validation", + :description => "Enable validation", + :boolean => false + + option :chef_pre_10, + :long => "--chef-pre-10", + :description => "Print knife commands using syntax for pre chef 10 versions of knife", :boolean => true option :rebuild, @@ -82,4 +82,8 @@ class Spiceweasel::CLI :proc => lambda {|v| puts "Spiceweasel: #{Spiceweasel::VERSION}" }, :exit => 0 + option :skip_nodename, + :long => '--skip_nodename', + :boolean => true + end diff --git a/lib/spiceweasel/cookbook_list.rb b/lib/spiceweasel/cookbook_list.rb index 0ba6381..31c31c2 100644 --- a/lib/spiceweasel/cookbook_list.rb +++ b/lib/spiceweasel/cookbook_list.rb @@ -67,13 +67,12 @@ def validateMetadata(cookbook,version) deps.each do |dependency| STDOUT.puts "DEBUG: cookbook #{cookbook} metadata dependency: #{dependency}" if DEBUG line = dependency.split() - cbdep = '' if line[1] =~ /^"/ #ignore variables and versions cbdep = line[1].gsub(/"/,'') cbdep.gsub!(/\,/,'') if cbdep.end_with?(',') + STDOUT.puts "DEBUG: cookbook #{cookbook} metadata depends: #{cbdep}" if DEBUG + @dependencies << cbdep end - STDOUT.puts "DEBUG: cookbook #{cookbook} metadata depends: #{cbdep}" if DEBUG - @dependencies << cbdep end return @cookbook end diff --git a/lib/spiceweasel/directory_extractor.rb b/lib/spiceweasel/directory_extractor.rb index 4614e13..38492e6 100644 --- a/lib/spiceweasel/directory_extractor.rb +++ b/lib/spiceweasel/directory_extractor.rb @@ -20,7 +20,7 @@ class Spiceweasel::DirectoryExtractor def self.parse_objects - objects = {"cookbooks" => nil, "roles" => nil, "environments" => nil, "data bags" => nil, "nodes" => nil} + objects = {"cookbooks" => nil, "roles" => nil, "environments" => nil, "data_bags" => nil, "nodes" => nil} # COOKBOOKS cookbooks = [] Dir.glob("cookbooks/*").each do |cookbook_full_path| @@ -63,7 +63,7 @@ def self.parse_objects end if File.directory?(data_bag_full_path) data_bags << {data_bag => data_bag_items} unless data_bag_items.empty? end - objects["data bags"] = data_bags unless data_bags.empty? + objects["data_bags"] = data_bags unless data_bags.empty? # NODES # TODO: Cant use this yet as node_list.rb doesnt support node from file syntax but expects the node info to be part of the objects passed in # nodes = [] @@ -111,7 +111,7 @@ def self.order_cookbooks_by_dependency(cookbooks) sorted_cookbooks.push(remainders).flatten! else deps = unsorted_cookbooks.collect {|x| x['dependencies'].collect {|x| x['cookbook']} - sorted_cookbooks} - STDERR.puts "ERROR: Dependencies not satisfied or circular dependencies in cookbook(s): #{remainders} depend(s) on #{deps}" + STDERR.puts "ERROR: Dependencies not satisfied or circular dependencies in cookbook(s): #{remainders.join(' ')} depend(s) on #{deps.join(' ')}" exit(-1) end end diff --git a/lib/spiceweasel/node_list.rb b/lib/spiceweasel/node_list.rb index 8b98cdf..4ae1cc4 100644 --- a/lib/spiceweasel/node_list.rb +++ b/lib/spiceweasel/node_list.rb @@ -3,55 +3,74 @@ def initialize(nodes, cookbooks, environments, roles, options = {}) @create = @delete = '' if nodes nodes.each do |node| - nname = node.keys[0] + nname = node["name"] STDOUT.puts "DEBUG: node: '#{nname}'" if DEBUG - #convert spaces to commas, drop multiple commas - run_list = node[nname][0].gsub(/ /,',').gsub(/,+/,',') + + run_list = node["run_list"] STDOUT.puts "DEBUG: node: 'node[nname]' run_list: '#{run_list}'" if DEBUG - validateRunList(nname, run_list, cookbooks, roles) unless NOVALIDATION - noptions = node[nname][1] + validateRunList(nname, run_list, cookbooks, roles) if VALIDATION + + # If were an array, join together to make a string. This allows for yaml + # anchor and alias usage within the config file. + noptions = node["options"].kind_of?(Array) ? node["options"].join(' ') : \ + node["options"] STDOUT.puts "DEBUG: node: 'node[nname]' options: '#{noptions}'" if DEBUG - validateOptions(nname, noptions, environments) unless NOVALIDATION - #provider support - if nname.start_with?("bluebox ","clodo ","cs ","ec2 ","gandi ","hp ","openstack ","rackspace ","slicehost ","terremark ","voxel ") - provider = nname.split() - count = 1 - if (provider.length == 2) - count = provider[1] - end - if PARALLEL - @create += "seq #{count} | parallel -j 0 -v \"" - @create += "knife #{provider[0]}#{options['knife_options']} server create #{noptions}" - if run_list.length > 0 - @create += " -r '#{run_list}'\"\n" - end - else - count.to_i.times do - @create += "knife #{provider[0]}#{options['knife_options']} server create #{noptions}" + validateOptions(nname, noptions, environments) if VALIDATION + + #provider supportp + provider = node["type"] + count = node["count"] || 1 + count.to_i.times do |num| + host = node["host"] + if ["bluebox","clodo","cs","ec2","gandi","hp","openstack","rackspace","slicehost","terremark","voxel"].include?(provider) + if CHEF_PRE_10 + @create << "knife #{provider}#{options['knife_options']} server create \'#{run_list.join("' '")}\' #{noptions}".gsub(/\{\{n\}\}/, "%02d" % (num + 1)) + else + @create << "knife #{provider}#{options['knife_options']} server create -r \'#{run_list.join(',')}\' #{noptions}".gsub(/\{\{n\}\}/, "%02d" % (num + 1)) + end + + # Rotate around the configured AZ's. + if node.key?("availability_zones") + az_num = num % node["availability_zones"].length + @create += "-Z #{node['availability_zones'][az_num]}" + end + + if node.key?("groups") + if CHEF_PRE_10 + @create += " -G #{node['groups'].join(',')}" + else + @create += " -g #{node['groups'].join(',')}" + end + end + + @create << "\n" + + #@delete += "knife #{provider} server delete -y \'#{host}\'\n" + elsif provider == "windows" #windows node bootstrap support + #TODO Fix this section + nodeline = nname.split() + provider = nodeline.shift.split('_') #split on 'windows_ssh' etc + nodeline.each do |server| + @create += "knife bootstrap #{provider[0]} #{provider[1]}#{options['knife_options']} #{server} #{noptions}\n" if run_list.length > 0 @create += " -r '#{run_list}'\n" end + @delete += "knife node#{options['knife_options']} delete #{server} -y\n" end - end - @delete += "knife node#{options['knife_options']} list | xargs knife #{provider[0]} server delete -y\n" - elsif nname.start_with?("windows") #windows node bootstrap support - nodeline = nname.split() - provider = nodeline.shift.split('_') #split on 'windows_ssh' etc - nodeline.each do |server| - @create += "knife bootstrap #{provider[0]} #{provider[1]}#{options['knife_options']} #{server} #{noptions}" - if run_list.length > 0 - @create += " -r '#{run_list}'\n" + #@delete += "knife node#{options['knife_options']} list | xargs knife #{provider[0]} server delete -y\n" + else #node bootstrap support + if CHEF_PRE_10 + @create += "knife bootstrap#{options['knife_options']} \'#{host}\' \'#{run_list.join("' '")}\' #{noptions}".gsub(/\{\{n\}\}/, "%02d" % (num + 1)) + else + @create += "knife bootstrap#{options['knife_options']} \'#{host}\' -r \'#{run_list.join(',')}\' #{noptions}".gsub(/\{\{n\}\}/, "%02d" % (num + 1)) end - @delete += "knife node#{options['knife_options']} delete #{server} -y\n" - end - @delete += "knife node#{options['knife_options']} list | xargs knife #{provider[0]} server delete -y\n" - else #node bootstrap support - nname.split.each do |server| - @create += "knife bootstrap#{options['knife_options']} #{server} #{noptions}" - if run_list.length > 0 - @create += " -r '#{run_list}'\n" + #@delete += "knife node#{options['knife_options']} delete \'#{nodename}\' -y\n" + + if node.key?("groups") + @create += " -g #{node['groups'].join(',')}" end - @delete += "knife node#{options['knife_options']} delete #{server} -y\n" + + @create << "\n" end end end