From 8e4e98347e8613033f4dd509eacc277c581001a9 Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Tue, 16 Jan 2018 21:19:43 -0500 Subject: [PATCH 01/23] python3 --- lang/python/python3.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 lang/python/python3.yml diff --git a/lang/python/python3.yml b/lang/python/python3.yml new file mode 100644 index 00000000..bb4ea991 --- /dev/null +++ b/lang/python/python3.yml @@ -0,0 +1,33 @@ +- hosts: all + become: yes + tasks: + + - name: Install basic utils and build envs. + apt: pkg={{ item }} state=present + with_items: + - git + - unzip + - vim + - build-essential + + - name: Install required system packages. + apt: pkg={{ item }} state=present + with_items: + - python3-dev + - python3-pip + - python3-setuptools + + - name: Set python3 as default python using alternatives + alternatives: + name: python + path: /usr/bin/python3 + + - name: Set pip3 as default pip using alternatives + alternatives: + name: pip + path: /usr/bin/pip3 + + - name: Upgrade pip + pip: + name: pip + extra_args: --upgrade \ No newline at end of file From a8a322f4ed0de63f455a6e9ee6d4f375582a1c89 Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Tue, 16 Jan 2018 22:31:24 -0500 Subject: [PATCH 02/23] Figuring out how alternatives work. --- lang/python/python3.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lang/python/python3.yml b/lang/python/python3.yml index bb4ea991..eae4bcdf 100644 --- a/lang/python/python3.yml +++ b/lang/python/python3.yml @@ -20,11 +20,13 @@ - name: Set python3 as default python using alternatives alternatives: name: python + link: /usr/bin/python path: /usr/bin/python3 - name: Set pip3 as default pip using alternatives alternatives: name: pip + link: /usr/bin/pip path: /usr/bin/pip3 - name: Upgrade pip From d76707bea9f97e4a2a9ace3f17704df9158acc00 Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Tue, 16 Jan 2018 22:51:12 -0500 Subject: [PATCH 03/23] install packages based on right python version. --- tools/jupyter/jupyter.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tools/jupyter/jupyter.yml b/tools/jupyter/jupyter.yml index e0a90c9a..9f120e4d 100644 --- a/tools/jupyter/jupyter.yml +++ b/tools/jupyter/jupyter.yml @@ -6,6 +6,10 @@ # apt: pkg={{ item }} state=present # with_items: + - name: check python version + command: python --version + register: python_version + - name: Install required system packages. apt: pkg={{ item }} state=present with_items: @@ -13,6 +17,21 @@ - python-scipy - python-matplotlib - python-pandas + when: python_version.stdout_lines.startswith('Python 2') + + - name: Install Jupyter + pip: name=jupyter + when: python_version.stdout_lines.startswith('Python 2') + + - name: Install required system packages. + apt: pkg={{ item }} state=present + with_items: + - python3-numpy + - python3-scipy + - python3-matplotlib + - python3-pandas + when: python_version.stdout_lines.startswith('Python 3') - name: Install Jupyter pip: name=jupyter + when: python_version.stdout_lines.startswith('Python 3') From 3f288652bc1c809518855b7225d592bdfc0f421a Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Tue, 16 Jan 2018 22:55:55 -0500 Subject: [PATCH 04/23] use stdout --- tools/jupyter/jupyter.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/jupyter/jupyter.yml b/tools/jupyter/jupyter.yml index 9f120e4d..afd90dcc 100644 --- a/tools/jupyter/jupyter.yml +++ b/tools/jupyter/jupyter.yml @@ -17,11 +17,11 @@ - python-scipy - python-matplotlib - python-pandas - when: python_version.stdout_lines.startswith('Python 2') + when: python_version.stdout.startswith('Python 2') - name: Install Jupyter pip: name=jupyter - when: python_version.stdout_lines.startswith('Python 2') + when: python_version.stdout.startswith('Python 2') - name: Install required system packages. apt: pkg={{ item }} state=present @@ -30,8 +30,8 @@ - python3-scipy - python3-matplotlib - python3-pandas - when: python_version.stdout_lines.startswith('Python 3') + when: python_version.stdout.startswith('Python 3') - name: Install Jupyter pip: name=jupyter - when: python_version.stdout_lines.startswith('Python 3') + when: python_version.stdout.startswith('Python 3') From 9be06e93a3cea000bb306236a782a8b17fee418a Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Tue, 16 Jan 2018 22:59:34 -0500 Subject: [PATCH 05/23] default to python2 if no version given. --- src/lib/bakerlets/lang/python.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/bakerlets/lang/python.js b/src/lib/bakerlets/lang/python.js index b38fb64d..633aabdd 100644 --- a/src/lib/bakerlets/lang/python.js +++ b/src/lib/bakerlets/lang/python.js @@ -10,8 +10,8 @@ class Python extends Bakerlet { super(ansibleSSHConfig); this.name = name; - this.version = version; - + // Default to python2 + this.version = version || 2; } async load(obj, variables) From d78c20c0b68be07a90920faaa3802de55f34c3f7 Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Wed, 17 Jan 2018 08:38:16 -0500 Subject: [PATCH 06/23] Pip install requirements.txt --- src/lib/bakerlets/lang/python.js | 13 +++++++++++++ src/lib/modules/baker.js | 11 +++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/lib/bakerlets/lang/python.js b/src/lib/bakerlets/lang/python.js index 633aabdd..f0f76fda 100644 --- a/src/lib/bakerlets/lang/python.js +++ b/src/lib/bakerlets/lang/python.js @@ -3,6 +3,7 @@ const baker = modules['baker']; const Bakerlet = require('../bakerlet'); const path = require('path'); +const fs = require('fs'); class Python extends Bakerlet { @@ -27,6 +28,18 @@ class Python extends Bakerlet { await baker.runAnsiblePlaybook( {name: this.name}, cmd, this.ansibleSSHConfig, this.verbose, this.variables ); + + // Check for a requirements.txt and then run pip install -r requirements.txt + // TODO: Possible to allow a requirements: parameter in the python object. + // Otherwise, we might just want it in the packages: pip: requirements: path + var localRequirementsPath = path.resolve(this.bakePath, "requirements.txt"); + if( fs.existsSync(localRequirementsPath) ) + { + var vmRequirementsPath = `/${path.basename(this.bakePath)}/requirements.txt`; + await baker.runAnsiblePipInstall( + {name: this.name}, vmRequirementsPath, this.ansibleSSHConfig, this.verbose + ); + } } diff --git a/src/lib/modules/baker.js b/src/lib/modules/baker.js index 84030d1d..2698448a 100644 --- a/src/lib/modules/baker.js +++ b/src/lib/modules/baker.js @@ -373,6 +373,17 @@ module.exports = function(dep) { return ssh.sshExec(`export ANSIBLE_HOST_KEY_CHECKING=false && cd /home/vagrant/baker/${doc.name} && ansible all -m apt -a "pkg=${cmd} update_cache=yes cache_valid_time=86400" -i baker_inventory --private-key id_rsa -u ${vmSSHConfigUser.user} --become`, sshConfig, verbose); } + result.runAnsiblePipInstall = async function(doc, requirements, sshConfig, verbose) { + const { path, vagrant, baker, ssh, boxes } = dep; + + let dir = path.join(boxes, doc.name); + let vm = vagrant.create({ cwd: dir }); + let vmSSHConfigUser = await baker.getSSHConfig(vm); + + return ssh.sshExec(`export ANSIBLE_HOST_KEY_CHECKING=false && cd /home/vagrant/baker/${doc.name} && ansible all -m pip -a "requirements=${requirements}" -i baker_inventory --private-key id_rsa -u ${vmSSHConfigUser.user} --become`, sshConfig, verbose); + } + + result.mkTemplatesDir = async function(doc, sshConfig) { const { path, vagrant, baker, ssh, boxes } = dep; From b3f33b15fdf2f71fd7a51e937eff72426eeede53 Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Wed, 17 Jan 2018 10:47:44 -0500 Subject: [PATCH 07/23] rename bake2 to bake, bake to custom. --- src/lib/commands/bake.js | 14 +++++++++----- src/lib/commands/{bake2.js => custom.js} | 16 ++++++---------- src/test/integration/bake-326-onboard.js | 2 +- src/test/integration/test-python-nb.js | 2 +- src/test/integration/test-r-tidy.js | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) rename src/lib/commands/{bake2.js => custom.js} (79%) diff --git a/src/lib/commands/bake.js b/src/lib/commands/bake.js index 365c62e0..8e7674da 100644 --- a/src/lib/commands/bake.js +++ b/src/lib/commands/bake.js @@ -18,10 +18,16 @@ module.exports = function(dep) { describe: `give a git repository URL which has a baker.yml in it's root directory`, demand: false, type: 'string' + }, + verbose: { + alias: 'v', + describe: `Provide extra output from baking process`, + demand: false, + type: 'boolean' } }; cmd.handler = async function(argv) { - const { local, repo } = argv; + const { local, repo, verbose } = argv; const { path, baker, cloneRepo, validator, print, spinner, spinnerDot } = dep; try{ @@ -39,7 +45,7 @@ module.exports = function(dep) { process.exit(1); } - let validation = await spinner.spinPromise(validator.validateBakerScript(bakePath), 'Validating baker.yml', spinnerDot); + //let validation = await spinner.spinPromise(validator.validateBakerScript(bakePath), 'Validating baker.yml', spinnerDot); try { @@ -53,10 +59,8 @@ module.exports = function(dep) { let sshConfig = await baker.getSSHConfig(ansibleVM); - let baking = baker.bake(sshConfig, ansibleVM, bakePath); - await spinner.spinPromise(baking, 'Baking VM', spinnerDot); + await baker.bake2(sshConfig, ansibleVM, bakePath, verbose); - // print.info('Baking VM finished.'); } catch (err) { print.error(err); } diff --git a/src/lib/commands/bake2.js b/src/lib/commands/custom.js similarity index 79% rename from src/lib/commands/bake2.js rename to src/lib/commands/custom.js index aba3465f..2cb7575c 100644 --- a/src/lib/commands/bake2.js +++ b/src/lib/commands/custom.js @@ -3,7 +3,7 @@ module.exports = function(dep) { let cmd = {}; - cmd.command = 'bake2'; + cmd.command = 'custom'; cmd.desc = 'Bake your VM given local path or repository URL containing the baker.yml'; cmd.builder = { @@ -18,16 +18,10 @@ module.exports = function(dep) { describe: `give a git repository URL which has a baker.yml in it's root directory`, demand: false, type: 'string' - }, - verbose: { - alias: 'v', - describe: `Provide extra output from baking process`, - demand: false, - type: 'boolean' } }; cmd.handler = async function(argv) { - const { local, repo, verbose } = argv; + const { local, repo } = argv; const { path, baker, cloneRepo, validator, print, spinner, spinnerDot } = dep; try{ @@ -45,7 +39,7 @@ module.exports = function(dep) { process.exit(1); } - //let validation = await spinner.spinPromise(validator.validateBakerScript(bakePath), 'Validating baker.yml', spinnerDot); + let validation = await spinner.spinPromise(validator.validateBakerScript(bakePath), 'Validating baker.yml', spinnerDot); try { @@ -59,8 +53,10 @@ module.exports = function(dep) { let sshConfig = await baker.getSSHConfig(ansibleVM); - await baker.bake2(sshConfig, ansibleVM, bakePath, verbose); + let baking = baker.bake(sshConfig, ansibleVM, bakePath); + await spinner.spinPromise(baking, 'Baking VM', spinnerDot); + // print.info('Baking VM finished.'); } catch (err) { print.error(err); } diff --git a/src/test/integration/bake-326-onboard.js b/src/test/integration/bake-326-onboard.js index 2552db88..3d86b0ee 100644 --- a/src/test/integration/bake-326-onboard.js +++ b/src/test/integration/bake-326-onboard.js @@ -15,7 +15,7 @@ describe('baker should create coffeemaker, run it, and destroy it', function() { const tstDir = path.join(os.tmpdir(), 'Onboarding'); fs.remove(tstDir); // echo value for prompt input for password. - var child = child_process.exec('echo 326 | baker bake2 --repo https://github.ncsu.edu/engr-csc326-staff/Onboarding', + var child = child_process.exec('echo 326 | baker bake --repo https://github.ncsu.edu/engr-csc326-staff/Onboarding', {cwd: os.tmpdir() }, function(error, stdout, stderr) { setTimeout( function() { diff --git a/src/test/integration/test-python-nb.js b/src/test/integration/test-python-nb.js index 10b1d269..b67b7d8f 100644 --- a/src/test/integration/test-python-nb.js +++ b/src/test/integration/test-python-nb.js @@ -16,7 +16,7 @@ describe('baker should create python notebook', function() { fs.mkdirpSync(tstDir); fs.copySync('test/resources/baker2/python-notebook.yml', path.join(tstDir,'baker.yml')); // echo value for prompt input for password. - var child = child_process.exec(`baker bake2 --local ${tstDir}`, + var child = child_process.exec(`baker bake --local ${tstDir}`, {cwd: os.tmpdir() }, function(error, stdout, stderr) { console.log(stderr || stdout); diff --git a/src/test/integration/test-r-tidy.js b/src/test/integration/test-r-tidy.js index 64592da2..a43c176a 100644 --- a/src/test/integration/test-r-tidy.js +++ b/src/test/integration/test-r-tidy.js @@ -16,7 +16,7 @@ describe('baker should create r tidy vm', function() { fs.mkdirpSync(tstDir); fs.copySync('test/resources/baker2/r-tidy.yml', path.join(tstDir,'baker.yml')); // echo value for prompt input for password. - var child = child_process.exec(`baker bake2 --local ${tstDir}`, + var child = child_process.exec(`baker bake --local ${tstDir}`, {cwd: os.tmpdir() }, function(error, stdout, stderr) { console.log(stderr || stdout); From f4944eb2abc839a1ce5fd604a8acbca05a590d8a Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Wed, 17 Jan 2018 11:40:08 -0500 Subject: [PATCH 08/23] initial neo4j --- services/neo4j/neo4j.yml | 92 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 services/neo4j/neo4j.yml diff --git a/services/neo4j/neo4j.yml b/services/neo4j/neo4j.yml new file mode 100644 index 00000000..5d66f50e --- /dev/null +++ b/services/neo4j/neo4j.yml @@ -0,0 +1,92 @@ +--- +- hosts: all + + vars_prompt: + + vars: + - neo4j_initial_password: "{{neo4j_password}}" + + handlers: + + - name: Restart Neo4J + become: yes + service: name=neo4j state=restarted enabled=yes + + tasks: + + - name: Check for Neo4J + stat: + path: /usr/bin/neo4j + register: neo4j_check + + - name: Check for NodeJS + stat: + path: /usr/bin/node + register: node_check + + - name: Add Neo4J Key + become: yes + apt_key: + url: http://debian.neo4j.org/neotechnology.gpg.key + state: present + + - name: Add Neo4J Repo + become: yes + apt_repository: + repo: deb http://debian.neo4j.org/repo stable/ + state: present + filename: neo4j + + - name: Download NodeJS Setup Script + become: yes + get_url: + url: https://deb.nodesource.com/setup_9.x + dest: /tmp/nodejs + mode: 0700 + when: not node_check.stat.exists + + - name: Run Setup Script + become: yes + command: /tmp/nodejs + when: not node_check.stat.exists + + - name: Install Packages + become: yes + apt: name="{{ item.key }}={{ item.value }}" state=present update_cache=yes + with_dict: + nodejs: 9.4.0-1nodesource1 + neo4j: 3.3.0 + + - name: Ensure Neo4J Listens on All Devices + become: yes + lineinfile: + path: /etc/neo4j/neo4j.conf + regexp: "{{ item.key }}" + line: "{{ item.key }}={{ item.value }}" + with_dict: + dbms.connector.bolt.listen_address: 0.0.0.0:7687 + dbms.connector.http.listen_address: 0.0.0.0:7474 + dbms.connector.https.listen_address: 0.0.0.0:7473 + notify: + - Restart Neo4J + + - name: Set Initial Neo4J Password + become: yes + command: "neo4j-admin set-initial-password '{{ neo4j_initial_password }}'" + when: not neo4j_check.stat.exists + + - name: Properly Set Neo4J Directory Permissions # Executing sudo neo4j-admin overrides them + become: yes + file: + path: /var/lib/neo4j + owner: neo4j + group: adm + recurse: yes + state: directory + + - name: Start Neo4J + become: yes + service: name=neo4j state=started enabled=yes + + - name: Install Node Modules + npm: path=/vagrant \ No newline at end of file From cc84f11039271fca79d54554426465d9f61cefca Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Wed, 17 Jan 2018 11:40:24 -0500 Subject: [PATCH 09/23] neo4j test --- src/test/resources/baker2/neo4.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/test/resources/baker2/neo4.yml diff --git a/src/test/resources/baker2/neo4.yml b/src/test/resources/baker2/neo4.yml new file mode 100644 index 00000000..0897fa0a --- /dev/null +++ b/src/test/resources/baker2/neo4.yml @@ -0,0 +1,12 @@ +--- +name: neo4j-test +vm: + ports: 7687, 7474, 7473 +vars: + neo4j_password: + prompt: Please type initial neo4j password +lang: + - nodejs9 +services: + - neo4j + From b109a75c64d5b99f254a5f0777558289f96280c6 Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Wed, 17 Jan 2018 12:06:24 -0500 Subject: [PATCH 10/23] trim to just neo4j stuff --- services/neo4j/neo4j.yml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/services/neo4j/neo4j.yml b/services/neo4j/neo4j.yml index 5d66f50e..c0513cf8 100644 --- a/services/neo4j/neo4j.yml +++ b/services/neo4j/neo4j.yml @@ -19,11 +19,6 @@ path: /usr/bin/neo4j register: neo4j_check - - name: Check for NodeJS - stat: - path: /usr/bin/node - register: node_check - - name: Add Neo4J Key become: yes apt_key: @@ -37,24 +32,10 @@ state: present filename: neo4j - - name: Download NodeJS Setup Script - become: yes - get_url: - url: https://deb.nodesource.com/setup_9.x - dest: /tmp/nodejs - mode: 0700 - when: not node_check.stat.exists - - - name: Run Setup Script - become: yes - command: /tmp/nodejs - when: not node_check.stat.exists - - name: Install Packages become: yes apt: name="{{ item.key }}={{ item.value }}" state=present update_cache=yes with_dict: - nodejs: 9.4.0-1nodesource1 neo4j: 3.3.0 - name: Ensure Neo4J Listens on All Devices @@ -87,6 +68,3 @@ - name: Start Neo4J become: yes service: name=neo4j state=started enabled=yes - - - name: Install Node Modules - npm: path=/vagrant \ No newline at end of file From 7c101925a53ad7f851a9f9fb3e647d7984d5481a Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Wed, 17 Jan 2018 12:13:29 -0500 Subject: [PATCH 11/23] Neo4j support. --- src/lib/bakerlets/services/neo4j.js | 36 +++++++++++++++++++ src/test/integration/test-neo4j.js | 29 +++++++++++++++ .../resources/baker2/{neo4.yml => neo4j.yml} | 0 3 files changed, 65 insertions(+) create mode 100644 src/lib/bakerlets/services/neo4j.js create mode 100644 src/test/integration/test-neo4j.js rename src/test/resources/baker2/{neo4.yml => neo4j.yml} (100%) diff --git a/src/lib/bakerlets/services/neo4j.js b/src/lib/bakerlets/services/neo4j.js new file mode 100644 index 00000000..7613bb3d --- /dev/null +++ b/src/lib/bakerlets/services/neo4j.js @@ -0,0 +1,36 @@ +const { commands, modules } = require('../../../baker'); +const baker = modules['baker']; +const ssh = modules['ssh']; + + +const Bakerlet = require('../bakerlet'); +const path = require('path'); + +class Neo4j extends Bakerlet { + + constructor(name,ansibleSSHConfig, version) { + super(ansibleSSHConfig); + + this.name = name; + this.version = version; + + } + + async load(obj, variables) + { + this.variables = variables; + + let playbook = path.resolve(this.remotesPath, `bakerlets-source/services/neo4j/neo4j${this.version}.yml`); + await this.copy(playbook,`/home/vagrant/baker/${this.name}/neo4j${this.version}.yml`); + } + + async install() + { + var cmd = `neo4j${this.version}.yml`; + await baker.runAnsiblePlaybook( + {name: this.name}, cmd, this.ansibleSSHConfig, this.verbose, this.variables + ); + } +} + +module.exports = Neo4j; \ No newline at end of file diff --git a/src/test/integration/test-neo4j.js b/src/test/integration/test-neo4j.js new file mode 100644 index 00000000..66e3e1b3 --- /dev/null +++ b/src/test/integration/test-neo4j.js @@ -0,0 +1,29 @@ +const child_process = require('child_process'); +const request = require('request'); +const chai = require('chai'); +const expect = chai.expect; +const os = require('os'); +const path = require('path'); +const fs = require('fs-extra'); + +describe('baker should create neo4j vm', function() { + this.timeout(2000000); + + // https://github.ncsu.edu/engr-csc326-staff/Onboarding + + it('should setup neo4j', function(done) { + const tstDir = path.join(os.tmpdir(), 'neo4j'); + fs.mkdirpSync(tstDir); + fs.copySync('test/resources/baker2/neo4j.yml', path.join(tstDir,'baker.yml')); + // echo value for prompt input for password. + var child = child_process.exec(`baker bake --local ${tstDir}`, + {cwd: os.tmpdir() }, function(error, stdout, stderr) + { + console.log(stderr || stdout); + done(); + }); + child.stdout.pipe(process.stdout); + }); +}); + + diff --git a/src/test/resources/baker2/neo4.yml b/src/test/resources/baker2/neo4j.yml similarity index 100% rename from src/test/resources/baker2/neo4.yml rename to src/test/resources/baker2/neo4j.yml From 10fd4bedac3dec90f1cd8fea3c84f5f46ed97372 Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Wed, 17 Jan 2018 14:49:17 -0500 Subject: [PATCH 12/23] Delete stray vars_prompt --- services/neo4j/neo4j.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/neo4j/neo4j.yml b/services/neo4j/neo4j.yml index c0513cf8..76c2e213 100644 --- a/services/neo4j/neo4j.yml +++ b/services/neo4j/neo4j.yml @@ -1,8 +1,6 @@ --- - hosts: all - vars_prompt: - vars: - neo4j_initial_password: "{{neo4j_password}}" From 1b2f5ff4dceb624978f631cb9348082c5c8db683 Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Wed, 17 Jan 2018 15:16:51 -0500 Subject: [PATCH 13/23] Confirmed implementation of neo4j. --- src/lib/bakerlets/resolve.js | 21 +++++++++++++++------ src/lib/modules/baker.js | 2 +- src/test/integration/test-neo4j.js | 2 +- src/test/resources/baker2/itrust2.yml | 13 ------------- src/test/resources/baker2/neo4j.yml | 5 +++-- 5 files changed, 20 insertions(+), 23 deletions(-) delete mode 100644 src/test/resources/baker2/itrust2.yml diff --git a/src/lib/bakerlets/resolve.js b/src/lib/bakerlets/resolve.js index 142bf584..67a9c1f1 100644 --- a/src/lib/bakerlets/resolve.js +++ b/src/lib/bakerlets/resolve.js @@ -115,16 +115,25 @@ async function resolve(vmName, bakerScriptPath, remotesPath, dir, bakerlet, extr } else { - let regex = /([a-zA-Z-]+)([0-9]*\.?[0-9]*)/; + // This will correctly match neo4j3.3, java8, python etc. + let regex = /([a-zA-Z-0-9]*)([0-9]+\.?[0-9]*$)|([a-zA-Z-0-9]*)/; mod = dir + "/" + bakerlet; let match = bakerlet.match(regex); - version = null; - if( match.length == 3) + if( match.length == 4) { - mod = dir + "/" + match[1]; - version = match[2]; + if( match[1] === undefined && match[2] === undefined ) + { + // We did not capture anything in the first part of regex. So, we have no version, just the mod. + // This is captured in third group. + mod = dir + "/" + match[3]; + } + else + { + mod = dir + "/" + match[1]; + version = match[2]; + } } - if( verbose ) console.log("Found", mod, version); + if( verbose ) console.log("Found", mod, version, extra_vars); } let classFoo = require(mod) diff --git a/src/lib/modules/baker.js b/src/lib/modules/baker.js index 2698448a..7cf1eb4b 100644 --- a/src/lib/modules/baker.js +++ b/src/lib/modules/baker.js @@ -359,7 +359,7 @@ module.exports = function(dep) { } let extravars = JSON.stringify(flatVars); //let extravars = yaml.dump(variables); - + if( verbose ) console.log( extravars ); return ssh.sshExec(`export ANSIBLE_HOST_KEY_CHECKING=false && cd /home/vagrant/baker/${doc.name} && echo '${extravars}' > playbook.args.json && ansible-playbook -e @playbook.args.json -i baker_inventory ${cmd} --private-key id_rsa -u ${vmSSHConfigUser.user}; rm -f playbook.args.json`, sshConfig, verbose); } diff --git a/src/test/integration/test-neo4j.js b/src/test/integration/test-neo4j.js index 66e3e1b3..9820e400 100644 --- a/src/test/integration/test-neo4j.js +++ b/src/test/integration/test-neo4j.js @@ -16,7 +16,7 @@ describe('baker should create neo4j vm', function() { fs.mkdirpSync(tstDir); fs.copySync('test/resources/baker2/neo4j.yml', path.join(tstDir,'baker.yml')); // echo value for prompt input for password. - var child = child_process.exec(`baker bake --local ${tstDir}`, + var child = child_process.exec(`echo neo | baker bake -v --local ${tstDir}`, {cwd: os.tmpdir() }, function(error, stdout, stderr) { console.log(stderr || stdout); diff --git a/src/test/resources/baker2/itrust2.yml b/src/test/resources/baker2/itrust2.yml deleted file mode 100644 index 87117a52..00000000 --- a/src/test/resources/baker2/itrust2.yml +++ /dev/null @@ -1,13 +0,0 @@ -vars: - - root_db_password: 326 -tools: - - maven -services: - - mysql5.7 -lang: - - java8 -config: - #- copy: mysql to x. - - template: - src: env/templates/hibernate-template.cfg.xml - dest: /Onboarding/CoffeeMaker/src/main/resources/hibernate.cfg.xml diff --git a/src/test/resources/baker2/neo4j.yml b/src/test/resources/baker2/neo4j.yml index 0897fa0a..0fccaa33 100644 --- a/src/test/resources/baker2/neo4j.yml +++ b/src/test/resources/baker2/neo4j.yml @@ -1,10 +1,11 @@ --- name: neo4j-test vm: + ip: 192.168.88.4 ports: 7687, 7474, 7473 vars: - neo4j_password: - prompt: Please type initial neo4j password + - neo4j_password: + prompt: Please type initial neo4j password lang: - nodejs9 services: From 4cb5fe514cf4aeff8e8ded6f1db22666b2a2d42d Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Wed, 17 Jan 2018 16:21:34 -0500 Subject: [PATCH 14/23] Install package.json if present. --- src/lib/bakerlets/lang/nodejs.js | 13 ++++++++++++- src/lib/modules/baker.js | 11 +++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/lib/bakerlets/lang/nodejs.js b/src/lib/bakerlets/lang/nodejs.js index 79a14cb6..18d22524 100644 --- a/src/lib/bakerlets/lang/nodejs.js +++ b/src/lib/bakerlets/lang/nodejs.js @@ -3,6 +3,7 @@ const baker = modules['baker']; const Bakerlet = require('../bakerlet'); const path = require('path'); +const fs = require('fs'); class Nodejs extends Bakerlet { @@ -29,7 +30,17 @@ class Nodejs extends Bakerlet { await baker.runAnsiblePlaybook( {name: this.name}, cmd, this.ansibleSSHConfig, this.verbose, this.variables ); - //console.log(`installed java ${this.version}`); + + var localPackageJsonPath = path.resolve(this.bakePath, "package.json"); + if( fs.existsSync(localPackageJsonPath) ) + { + var vmPackagePath = `/${path.basename(this.bakePath)}`; + if( this.verbose ) console.log(`Attempting to run npm install in vm at ${vmPackagePath}`); + await baker.runAnsibleNpmInstall( + {name: this.name}, vmPackagePath, this.ansibleSSHConfig, this.verbose + ); + } + } diff --git a/src/lib/modules/baker.js b/src/lib/modules/baker.js index 7cf1eb4b..61c07ba9 100644 --- a/src/lib/modules/baker.js +++ b/src/lib/modules/baker.js @@ -383,6 +383,17 @@ module.exports = function(dep) { return ssh.sshExec(`export ANSIBLE_HOST_KEY_CHECKING=false && cd /home/vagrant/baker/${doc.name} && ansible all -m pip -a "requirements=${requirements}" -i baker_inventory --private-key id_rsa -u ${vmSSHConfigUser.user} --become`, sshConfig, verbose); } + result.runAnsibleNpmInstall = async function(doc, packagejson, sshConfig, verbose) { + const { path, vagrant, baker, ssh, boxes } = dep; + + let dir = path.join(boxes, doc.name); + let vm = vagrant.create({ cwd: dir }); + let vmSSHConfigUser = await baker.getSSHConfig(vm); + + return ssh.sshExec(`export ANSIBLE_HOST_KEY_CHECKING=false && cd /home/vagrant/baker/${doc.name} && ansible all -m npm -a "path=${packagejson}" -i baker_inventory --private-key id_rsa -u ${vmSSHConfigUser.user}`, sshConfig, verbose); + } + + result.mkTemplatesDir = async function(doc, sshConfig) { const { path, vagrant, baker, ssh, boxes } = dep; From 1b863291f302a8dc337ac26efeb9a17265104b25 Mon Sep 17 00:00:00 2001 From: Chris Parnin Date: Wed, 17 Jan 2018 16:22:51 -0500 Subject: [PATCH 15/23] Limitation note. --- src/lib/bakerlets/lang/nodejs.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/bakerlets/lang/nodejs.js b/src/lib/bakerlets/lang/nodejs.js index 18d22524..1581e095 100644 --- a/src/lib/bakerlets/lang/nodejs.js +++ b/src/lib/bakerlets/lang/nodejs.js @@ -34,6 +34,9 @@ class Nodejs extends Bakerlet { var localPackageJsonPath = path.resolve(this.bakePath, "package.json"); if( fs.existsSync(localPackageJsonPath) ) { + // There could be some funkyness depending on if there is a package-lock.json, etc: + // and node_modules is manually deleted: + // https://github.com/ansible/ansible/pull/29131 var vmPackagePath = `/${path.basename(this.bakePath)}`; if( this.verbose ) console.log(`Attempting to run npm install in vm at ${vmPackagePath}`); await baker.runAnsibleNpmInstall( From 9ce25edccdd17cfe862b1cacb55e54ad9d61220c Mon Sep 17 00:00:00 2001 From: Samim Mirhosseini Date: Tue, 23 Jan 2018 20:51:50 -0500 Subject: [PATCH 16/23] Baker package support #34: - updated `node-vagrant` version - added baker ssh key - added new commands: - added `package` - added `import` box - added `boxes` - added `bake --box` --- src/baker.js | 2 +- src/config/BaseVM.mustache | 8 +-- src/config/baker_rsa | 51 +++++++++++++++++ src/config/baker_rsa.pub | 1 + src/lib/commands/bake.js | 70 +++++++++++++---------- src/lib/commands/boxes.js | 22 ++++++++ src/lib/commands/import.js | 35 ++++++++++++ src/lib/commands/package.js | 29 ++++++++++ src/lib/commands/setup.js | 22 +++++++- src/lib/modules/baker.js | 105 ++++++++++++++++++++++++++++++++++- src/lib/modules/validator.js | 21 ++++--- src/package.json | 2 +- 12 files changed, 320 insertions(+), 48 deletions(-) create mode 100644 src/config/baker_rsa create mode 100644 src/config/baker_rsa.pub create mode 100644 src/lib/commands/boxes.js create mode 100644 src/lib/commands/import.js create mode 100644 src/lib/commands/package.js diff --git a/src/baker.js b/src/baker.js index 82d95676..899f83c1 100755 --- a/src/baker.js +++ b/src/baker.js @@ -4,7 +4,7 @@ const camelCase = require('camelcase'); const requireDir = require('require-dir'); const fs = Promise.promisifyAll(require('fs-extra')); const mustache = require('mustache'); -const child_process = require('child_process'); +const child_process = Promise.promisifyAll(require('child_process')); const vagrant = Promise.promisifyAll(require('node-vagrant')); const scp2 = require('scp2'); const ssh2 = require('ssh2'); diff --git a/src/config/BaseVM.mustache b/src/config/BaseVM.mustache index f5e454df..5bb501d9 100644 --- a/src/config/BaseVM.mustache +++ b/src/config/BaseVM.mustache @@ -15,6 +15,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # Every Vagrant virtual environment requires a box to build off of. config.vm.box = "{{{vagrant.box}}}" + config.ssh.insert_key = false + config.ssh.private_key_path = ["~/.baker/ansible-srv/keys/baker_rsa", "~/.vagrant.d/insecure_private_key"] + # Disable automatic box update checking. If you disable this, then # boxes will only be checked for updates when the user runs # `vagrant box outdated`. This is not recommended. @@ -88,9 +91,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.provision "shell", inline: "test -e /usr/bin/python || (sudo apt -y update && sudo apt install -y python-minimal)" - # - # View the documentation for the provider you are using for more - # information on available options. - + config.vm.provision "file", source: "~/.baker/ansible-srv/keys/baker_rsa.pub", destination: "~/.ssh/authorized_keys" end diff --git a/src/config/baker_rsa b/src/config/baker_rsa new file mode 100644 index 00000000..7860aa73 --- /dev/null +++ b/src/config/baker_rsa @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAqPmEYSNho56PuTQ/jf6Qb78Owb44ujhLWYFCryX/D/az3FAk +fMOw6Qz06S0jOLKQaiMKBDNQkEZx/Mf1H74a+dbVP++AOCdjIARIwpSPDOWfIqmI +Q1n6EcsKP9rgda6zxyAiDO9ei0RRiA6Kp7fpXh1OMHHx5hzWvPyqgubTcXff1QcX +r3odAiLu/wWt/1FbZ+gk2NrBjKNJiZNCI8yLCuNk4ES+2Tk0H26NU3dw6TgnbdNs +X7ecVwpPSrkcMK43bcmWz/J/MXO/2HEZNKbMBVYjXxGlORG+Nvg069DoQoGnRg/b +n8SW0oeLctlZkcnEAX49NFakDtszzS9ru+hH5DBCP07KVWp7DAoXkbbdwuUCUAr9 +IF57Q0rp84CDr24EoXOuOHiK5lf5IdKC2JFvhYvNM6sf3SHSjHOzqVyv7piW9fGq +NiAbQaE4ja+XJWtV3qw8GVyRKerDyZUo2vcTW9MsNDwH8TIirEf348pmeRO3T2mI +0t2/iyMsY8S/hkq/GUS/t2h+5WLHQGkMHpHrMWjGVEnyg8EVpOVLxPNYKQllpIHE +3KsQhrogQnb4gVPPMl3w7haHYnYGEyd/STPb94Crr5SQLUWgW10iemE42p36M5Zq +WjWCJh1D3qKGNtL5aJZl87NpZWSPoDOvBmaZ5pd3p/NCyWJ4p9upIoFjPVkCAwEA +AQKCAgBRyAn5Fa3BChIXmiEUcVuoqfjTbmR4RJy7YiNLMAGl0Uo13Bf8xp3N/bZf +ULhWTZ41sGW9qLRaT64FoSWTSmg1+XNWsW0GQJHqQgiRHGOr40rE9PZ9WoP8rp90 +TlQKwRZDztqMFiJVFyi6yAb1q75oDZj1O4DPVa/c4hEIr/0wUstjiD4/cMOvcAbq +KO6QvuiVfrauuhmpHrKNwlbliq7VAz+kh8Ey00vV1qTR++ILDmGO9x/hp7UkL1o3 +GSZ6rSconMPAO2ayIYp9kCeZ4wylnI4cCidEWsEMS88ZPw/aeHPkJfKu/e/dTzr/ +yBBgzh1ud5HZzgEzK4aDzWrAFGkOT6Oa3OuqCPmYGZZ/Z8H/lSvRxcnVG9UyjFpe +yjXhnBfsJzhjFLqyIQfoSO31sbNJkmxQ1XmMEOdYwqmK8nhCSJM5WiUDZQ/X6hpc +ophsS4iGExqqB5/LpodZunDdrxJbljD86iJUp34xCs1LPHqcaB3QmUEppIfYrMUk +crUm6OhUy/u5IcKcpeGGATd26XIi9ljs1GwgSjNd1ejO42HLQBXq569bfhhwDfEf +rVoCM9h6jMeiZQXNsOgf4f7d7T8aPHaOSipJVzoEVyM21K/CDIUtdX8SENeDZaEc +Ob7cKlVDuuZkfk1v/KVDhEGVYBpOYPAiHnedSiYoL6J/9RKTvQKCAQEA1YvF0aho +//O55rk5uGOwdcZ9FM+QNoJNCILGDHWlo/AQdNwd1eWLH0Yap4uYn/WJT50VwcL0 +efDyToLDX7GLAZtc/6eUJ7glhQ0u0dUBcIlH6Z8KBZ8BVJIR7i3V2c9FBh0jmLF1 +BvO5ulcGizv3r8+vcVqVU5YF33vaWlcV4iP1GeH1QzAru5wLyKtLr8SrZZ9JL5xR +5xxmNvvxyUmL7tVCoclyMR9X0qluwGriSOziBCL4mUuhvK5UElgmvFFa2kSP/gEx ++D+DyY8FZ3p8NtdZWYAEs+6HXekxTfD6EhbkfB3AWYeu/N3ETRTX64tS+twYSGQT +EPQZNEt34qOKIwKCAQEAypFUD1cWMaHGzRgaXODqmGZj6uwb8kQ2MUDKlJloUt6H +p/9779RYmcR1Yo5W7H0lmcVH12kE0co7Sco7YtSPR1wLgicyjR+Sq+zxyetwqVEu +LDsmraL3BVaTMqRnAaWpHQlbp+NIeeZcnuW3b8EBAEKPy3xujGsHuZ9miumf45qq +4DbsqMB296yhiXQB1yhWDSEHdcnBfHN8aawDEkp6cssdOB85lmBZpwqAuXxH0l/W +Qrhohq7MH1X7CDefwh23Pz4b9V7XxzirNaq7zFs6ceSNuHv11bHV5QRR5YsPh+KG +CdEDzMuao/1/df0TsiztYdC3K2aYaOEejyGv4sH8UwKCAQBJ8jqwHScu6pEHSkCo +jyy9u9v4Zt/DYF+YgOBf1CVlnW21abuTJAeG7tmwBvD1AytnPDgafo314++kLDfH +XU2LYudTSA5Pqr6jUitSUfZLp94VEhOAWs01Ide/qHOTFukJ8vEuoNSrcZ5w3k3P +zRY59SsFj56B8UNbXiIAgoN7aYQoUEyD1ZxvPNv/wwFUfj/z0rKfH/xkkTr780aI +s0UXkRWfvIgkZnwc4LsPOnPdWNnzIMEBJGV/VsaaC5huQaW6S1+pT3SkSCo0k6gF +ay60NuIj0ebO/9w0Mtn16WpO9UptiEfhONDpk0m0f3E9iWNUpv5pou3PQxevOirr +ekINAoIBACfC3v0j2vdjCeK4GHSisWm4r2QtdE7ZlMmWLi187z1U8MvJGkq5I6sL +JP9zcRx6dCb60l81/fwv9fNF/uInVvhq2NdzWjjZObEFkXBRBow1oxqLgcwTcOlb +VQlbu9xW6BsK+zK5KkDDNur5rEgDWm7yoccPZaOqXpnQ8A/US84hTek03r9BCBkV +iZ+xZasV/84T7aLxN0l2YbVcTj4I4IAn3lRlzKf3waFILnw6KN7icOwnxlypcuez +uNKkGHfB3XZMerBvLWutc+3U1YgHYDF661aK/nYzsgiCEJE9+o5xqF3E6ToJvRDz +cVF3m6Ydq3rHvSyHtuLfTWBK/HtGGIECggEBAMXM4KSngNKpTCApjFpvEI3TvJtp +8A2Mvhj/M0pXkBIDxcBFqF/0cYXnGpPc/AgPekh/vEyOOGOSh3dplswXie231Bod +6h8d2I8Pn7dTl/Po1jSMpwXDQu2PlrTr9+zcRZfGgJc4f6nU4PeD5CdUyBCtvJsG +DrnJ2NVUhgRXkCwn8Whe+le9GUKo3athetUZ7lKrbxz0bs2KHniUURpWltfmG+EM +g08w319Gyq9XEdfcdyoxUDCoiyhyV1AYx+tHiMYidkadnKJVXHUC2YapTKtEfK1o +cJmno7sstGUNY0o6+sPve4CIECE4NSMCQdE/K1JPAvF+yZwHlVUZacX+a5M= +-----END RSA PRIVATE KEY----- diff --git a/src/config/baker_rsa.pub b/src/config/baker_rsa.pub new file mode 100644 index 00000000..e0e2ee99 --- /dev/null +++ b/src/config/baker_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCo+YRhI2Gjno+5ND+N/pBvvw7Bvji6OEtZgUKvJf8P9rPcUCR8w7DpDPTpLSM4spBqIwoEM1CQRnH8x/Ufvhr51tU/74A4J2MgBEjClI8M5Z8iqYhDWfoRywo/2uB1rrPHICIM716LRFGIDoqnt+leHU4wcfHmHNa8/KqC5tNxd9/VBxeveh0CIu7/Ba3/UVtn6CTY2sGMo0mJk0IjzIsK42TgRL7ZOTQfbo1Td3DpOCdt02xft5xXCk9KuRwwrjdtyZbP8n8xc7/YcRk0pswFViNfEaU5Eb42+DTr0OhCgadGD9ufxJbSh4ty2VmRycQBfj00VqQO2zPNL2u76EfkMEI/TspVansMCheRtt3C5QJQCv0gXntDSunzgIOvbgShc644eIrmV/kh0oLYkW+Fi80zqx/dIdKMc7OpXK/umJb18ao2IBtBoTiNr5cla1XerDwZXJEp6sPJlSja9xNb0yw0PAfxMiKsR/fjymZ5E7dPaYjS3b+LIyxjxL+GSr8ZRL+3aH7lYsdAaQwekesxaMZUSfKDwRWk5UvE81gpCWWkgcTcqxCGuiBCdviBU88yXfDuFodidgYTJ39JM9v3gKuvlJAtRaBbXSJ6YTjanfozlmpaNYImHUPeooY20vlolmXzs2llZI+gM68GZpnml3en80LJYnin26kigWM9WQ== samim@Samims-MacBook-Pro.local diff --git a/src/lib/commands/bake.js b/src/lib/commands/bake.js index 8e7674da..ea4d723b 100644 --- a/src/lib/commands/bake.js +++ b/src/lib/commands/bake.js @@ -19,6 +19,12 @@ module.exports = function(dep) { demand: false, type: 'string' }, + box: { + alias: 'b', + describe: `give local path to where your baker.yml file is located`, + demand: false, + type: 'string' + }, verbose: { alias: 'v', describe: `Provide extra output from baking process`, @@ -27,43 +33,49 @@ module.exports = function(dep) { } }; cmd.handler = async function(argv) { - const { local, repo, verbose } = argv; + const { local, repo, box, verbose } = argv; const { path, baker, cloneRepo, validator, print, spinner, spinnerDot } = dep; - try{ - let ansibleVM; - let bakePath; + try{ + let ansibleVM; + let bakePath; - if (local) { - bakePath = path.resolve(local); - } else if (repo) { - bakePath = path.resolve(await cloneRepo.cloneRepo(repo)); - } else { - print.error( - `User --local to give local path or --repo to give git repository with baker.yml` - ); - process.exit(1); - } + if( box ){ + bakePath = path.resolve(box); + } + else if (local) { + bakePath = path.resolve(local); + } else if (repo) { + bakePath = path.resolve(await cloneRepo.cloneRepo(repo)); + } else { + print.error( + `User --local to give local path or --repo to give git repository with baker.yml` + ); + process.exit(1); + } - //let validation = await spinner.spinPromise(validator.validateBakerScript(bakePath), 'Validating baker.yml', spinnerDot); + //let validation = await spinner.spinPromise(validator.validateBakerScript(bakePath), 'Validating baker.yml', spinnerDot); - try - { - ansibleVM = await spinner.spinPromise(baker.prepareAnsibleServer(bakePath), 'Preparing Baker control machine', spinnerDot); - } - catch(ex) - { - await spinner.spinPromise(baker.upVM('baker'), `Restarting Baker control machine: `, spinnerDot); - ansibleVM = await spinner.spinPromise(baker.prepareAnsibleServer(bakePath), 'Preparing Baker control machine', spinnerDot); - } + try + { + ansibleVM = await spinner.spinPromise(baker.prepareAnsibleServer(bakePath), 'Preparing Baker control machine', spinnerDot); + } + catch(ex) + { + await spinner.spinPromise(baker.upVM('baker'), `Restarting Baker control machine`, spinnerDot); + ansibleVM = await spinner.spinPromise(baker.prepareAnsibleServer(bakePath), 'Preparing Baker control machine', spinnerDot); + } - let sshConfig = await baker.getSSHConfig(ansibleVM); + let sshConfig = await baker.getSSHConfig(ansibleVM); - await baker.bake2(sshConfig, ansibleVM, bakePath, verbose); + if(box) + await baker.bakeBox(sshConfig, ansibleVM, bakePath, verbose); + else + await baker.bake2(sshConfig, ansibleVM, bakePath, verbose); - } catch (err) { - print.error(err); - } + } catch (err) { + print.error(err); + } }; return cmd; diff --git a/src/lib/commands/boxes.js b/src/lib/commands/boxes.js new file mode 100644 index 00000000..95fd4de6 --- /dev/null +++ b/src/lib/commands/boxes.js @@ -0,0 +1,22 @@ +'use strict'; + +module.exports = function(dep) { + let cmd = {}; + cmd.builder = { }; + cmd.command = 'boxes'; + cmd.desc = `list existing Baker boxes`; + cmd.handler = async function(argv) { + const { baker, print, spinner, spinnerDot } = dep; + const { verbose } = argv; + + try { + await spinner.spinPromise(baker.bakerBoxes(), `Getting list of Baker boxes`, spinnerDot); + } catch (err) { + print.error(err); + } + }; + + return cmd; +}; + + diff --git a/src/lib/commands/import.js b/src/lib/commands/import.js new file mode 100644 index 00000000..f94dcdf5 --- /dev/null +++ b/src/lib/commands/import.js @@ -0,0 +1,35 @@ +'use strict'; + +module.exports = function(dep) { + let cmd = {}; + cmd.builder = { + name: { + alias: 'n', + describe: `Provide name for the imported box`, + demand: false, + type: 'string' + }, + verbose: { + alias: 'v', + describe: `Provide extra output from baking process`, + demand: false, + type: 'boolean' + } + }; + cmd.command = 'import '; + cmd.desc = `import packaged Baker environment`; + cmd.handler = async function(argv) { + const { baker, print, spinner, spinnerDot } = dep; + const { boxPath, name, verbose } = argv; + + try { + await spinner.spinPromise(baker.import(boxPath, verbose), `Importing box: ${boxPath}`, spinnerDot); + } catch (err) { + print.error(err); + } + }; + + return cmd; +}; + + diff --git a/src/lib/commands/package.js b/src/lib/commands/package.js new file mode 100644 index 00000000..c30de403 --- /dev/null +++ b/src/lib/commands/package.js @@ -0,0 +1,29 @@ +'use strict'; + +module.exports = function(dep) { + let cmd = {}; + cmd.builder = { + verbose: { + alias: 'v', + describe: `Provide extra output from baking process`, + demand: false, + type: 'boolean' + } + }; + cmd.command = 'package '; + cmd.desc = `package a Baker environment`; + cmd.handler = async function(argv) { + const { baker, print, spinner, spinnerDot } = dep; + const { VMName, verbose } = argv; + + try { + await spinner.spinPromise(baker.package(VMName, verbose), `Packaging box: ${VMName}`, spinnerDot); + } catch (err) { + print.error(err); + } + }; + + return cmd; +}; + + diff --git a/src/lib/commands/setup.js b/src/lib/commands/setup.js index 98ea6b8b..2f13b854 100644 --- a/src/lib/commands/setup.js +++ b/src/lib/commands/setup.js @@ -15,7 +15,7 @@ module.exports = function (dep) { } cmd.handler = async function (argv) { const { force } = argv - const { baker, print, spinner, spinnerDot, validator } = dep + const { baker, print, spinner, spinnerDot, validator, fs, path, configPath, ansible } = dep try { await spinner.spinPromise(validator.validateDependencies(), 'Checking dependencies', spinnerDot); @@ -29,6 +29,26 @@ module.exports = function (dep) { await spinner.spinPromise(baker.reinstallAnsibleServer(), 'Re-installing Baker control machine', spinnerDot); else await spinner.spinPromise(baker.installAnsibleServer(), 'Installing Baker control machine', spinnerDot); + + await spinner.spinPromise( + fs.copy( + path.resolve(configPath, './baker_rsa.pub'), + path.resolve(ansible, 'keys','baker_rsa.pub') + ), + 'Copying private ssh key', + spinnerDot + ); + + await spinner.spinPromise( + fs.copy( + path.resolve(configPath, './baker_rsa'), + path.resolve(ansible, 'keys','baker_rsa') + ) + , + 'Copying public ssh key', + spinnerDot + ); + } catch (err) { print.error(err); } diff --git a/src/lib/modules/baker.js b/src/lib/modules/baker.js index 7cf1eb4b..b743165b 100644 --- a/src/lib/modules/baker.js +++ b/src/lib/modules/baker.js @@ -463,7 +463,7 @@ module.exports = function(dep) { } result.initVagrantFile = async function(vagrantFilePath, doc, template, scriptPath) { - const { mustache, fs, path, slash } = dep; + const { mustache, fs, path, slash, baker } = dep; if (doc.vm ) { doc.vagrant = doc.vm; @@ -473,7 +473,17 @@ module.exports = function(dep) { await traverse(vagrant); // Defaults - vagrant.box = vagrant.box || "ubuntu/xenial64" + // vagrant.box = vagrant.box || "ubuntu/xenial64" + // TODO: Cleanup this mess + if (vagrant.box && (await baker.boxes()).map(e=>e.name).includes(`${vagrant.box}.baker`)){ + vagrant.box = vagrant.box + '.baker'; + } + else if(vagrant.box && (await baker.boxes()).map(e=>e.name).includes(`${vagrant.box}`)){ + vagrant.box = vagrant.box; + } + else{ + vagrant.box = "ubuntu/xenial64"; + } vagrant.memory = vagrant.memory || "1024" // Adaptor pattern: Support baker2 and baker format @@ -495,7 +505,7 @@ module.exports = function(dep) { } } vagrant.network = network; - + let syncFolders = doc.vagrant.synced_folders || []; @@ -639,5 +649,94 @@ module.exports = function(dep) { } } + result.package = async function(VMName, verbose) { + var { path, boxes, child_process } = dep; + + let dir = path.join(boxes, VMName); + + await child_process.execAsync(`cd ${dir} && vagrant package --output ${path.join(process.cwd(), VMName + '.box')}`, {stdio: ['inherit', 'inherit', 'ignore']}); + } + + result.import = async function(box, name, verbose) { + var { path, boxes, child_process, vagrant } = dep; + + let boxName = name ? name : path.basename(box).split('.')[0]; + + await vagrant.boxAddAsync(path.join(process.cwd(), box), ['--name', boxName + '.baker']) + // await child_process.execAsync(`vagrant box add ${boxName}.baker ${path.join(process.cwd(), box)}`, {stdio: ['inherit', 'inherit', 'ignore']}); + } + + result.boxes = async function() { + const { vagrant, print } = dep; + try { + let boxes = await vagrant.boxListAsync([]); + delete boxes.version; + return boxes; + } catch (err) { + throw err + } + } + + result.bakerBoxes = async function(verbose=true) { + const { vagrant, print, baker } = dep; + + try { + let boxes = await baker.boxes(); + let bakerBoxes = boxes.filter(box => box.name.match(/.baker$/)); + // Hide .baker from the end before printing + bakerBoxes.forEach(box => { + box.name = box.name.split('.')[0]; + }) + + if(verbose){ + if(bakerBoxes == []) + print.info(`\nYou currently don't have any boxes.`) + else + console.table('\nBaker boxes: ', bakerBoxes); + } + return bakerBoxes; + } catch (err) { + throw err + } + } + + result.bakeBox = async function(ansibleSSHConfig, ansibleVM, scriptPath, verbose) { + const { vagrant, boxes, path, fs, spinner, spinnerDot, configPath, baker, yaml} = dep; + + try { + let doc = yaml.safeLoad(await fs.readFile(path.join(scriptPath, 'baker.yml'), 'utf8')); + + let dir = path.join(boxes, doc.name); + try { + await fs.ensureDir(dir); + } catch (err) { + throw `Creating directory failed: ${dir}`; + } + + + let template = await fs.readFile(path.join(configPath, './BaseVM.mustache'), 'utf8'); + + // if box is specified in baker.yml and this box exists, then use it => otherwise bake it + if(doc.vagrant.box && (await baker.bakerBoxes(false)).map(e=>e.name).includes(`${doc.vagrant.box}`)){ + + await baker.initVagrantFile(path.join(dir, 'Vagrantfile'), doc, template, scriptPath); + + let machine = vagrant.create({ cwd: dir }); + machine.on('up-progress', function(data) { + if( verbose ) print.info(data); + }); + await spinner.spinPromise(machine.upAsync(), `Starting VM`, spinnerDot); + } + else { + await baker.bakeBox(sshConfig, ansibleVM, bakePath, verbose); + } + + } catch (err) { + throw err; + } + + return; + } + return result; }; diff --git a/src/lib/modules/validator.js b/src/lib/modules/validator.js index 4c602351..a43a0b59 100644 --- a/src/lib/modules/validator.js +++ b/src/lib/modules/validator.js @@ -12,13 +12,16 @@ module.exports = function(dep) { if(!hasVagrant) reject('Dependencies not found. Make sure you have installed VirtualBox and Vagrant.') - let hasVirtualBox = false; - if(platform === 'darwin' || platform === 'linux'){ - hasbin('virtualbox', (hasVB)=>{ - hasVirtualBox = hasVB; + if(platform == 'darwin' || platform === 'linux'){ + hasbin('virtualbox', (hasVirtualBox)=>{ + if(hasVirtualBox && hasVagrant) + resolve(true); + else + reject('Dependencies not found. Make sure you have installed VirtualBox and Vagrant.') }) } else if(platform === 'win32'){ + let hasVirtualBox = false; try { fs.accessSync(path.resolve(`C:/Program Files/Oracle/VirtualBox`)); hasVirtualBox = true; @@ -30,12 +33,12 @@ module.exports = function(dep) { catch(err){ } } - } - if(hasVirtualBox && hasVagrant) - resolve(true); - else - reject('Dependencies not found. Make sure you have installed VirtualBox and Vagrant.') + if(hasVirtualBox && hasVagrant) + resolve(true); + else + reject('Dependencies not found. Make sure you have installed VirtualBox and Vagrant.'); + } }) }) } diff --git a/src/package.json b/src/package.json index 8170df6e..7178bad3 100644 --- a/src/package.json +++ b/src/package.json @@ -39,7 +39,7 @@ "hasbin": "^1.2.3", "js-yaml": "^3.10.0", "mustache": "^2.3.0", - "node-vagrant": "^1.1.0", + "node-vagrant": "^1.2.2", "ora": "^1.3.0", "prompt": "^1.0.0", "request": "^2.81.0", From 0428a6bc9b98901e32a0074d5619e170247c83fc Mon Sep 17 00:00:00 2001 From: Samim Mirhosseini Date: Wed, 24 Jan 2018 11:39:00 -0500 Subject: [PATCH 17/23] fixed dependency check on Windows --- src/baker.js | 1 + src/lib/modules/validator.js | 64 +++++++++++++++++------------------- src/package.json | 1 + 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/baker.js b/src/baker.js index 899f83c1..5ae75911 100755 --- a/src/baker.js +++ b/src/baker.js @@ -17,6 +17,7 @@ const slash = require('slash'); const git = require('simple-git'); const ora = require('ora'); const hasbin = require('hasbin'); +const drivelist = Promise.promisifyAll(require('drivelist')); require('console.table'); const boxes = path.join(require('os').homedir(), '.baker'); diff --git a/src/lib/modules/validator.js b/src/lib/modules/validator.js index a43a0b59..bc679032 100644 --- a/src/lib/modules/validator.js +++ b/src/lib/modules/validator.js @@ -4,43 +4,41 @@ module.exports = function(dep) { let result = {}; result.validateDependencies = async function(){ - const { hasbin, Promise, fs, path } = dep; + const { hasbin, Promise, fs, path, drivelist } = dep; let platform = process.platform; + let dependencyNotFound = 'Dependencies not found. Make sure you have installed VirtualBox and Vagrant.'; - return new Promise((resolve, reject)=>{ - hasbin('vagrant', (hasVagrant)=>{ - if(!hasVagrant) - reject('Dependencies not found. Make sure you have installed VirtualBox and Vagrant.') - - if(platform == 'darwin' || platform === 'linux'){ - hasbin('virtualbox', (hasVirtualBox)=>{ - if(hasVirtualBox && hasVagrant) - resolve(true); - else - reject('Dependencies not found. Make sure you have installed VirtualBox and Vagrant.') + if (platform == 'darwin' || platform === 'linux') { + hasbin.all( ['vagrant', 'virtualbox'], hasDependencies => { + if(hasDependencies) return true; + else throw dependencyNotFound;} + ); + } + else { + hasbin('vagrant', async function (hasVagrant) { + if(hasVagrant){ + let drives = (await drivelist.listAsync()).map(d => d.mountpoints[0].path); + drives.forEach(drive => { + fs.access(path.resolve(path.join(drive, `/Program Files/Oracle/VirtualBox`)), err => { + if (err){ + fs.access(path.resolve(path.join(drive, `/Program Files (x86)/Oracle/VirtualBox`)), err => { + if(err) { + fs.access(path.resolve(path.join(process.env.PROGRAMFILES, `/Oracle/VirtualBox`)), err => { + if(err){ + throw dependencyNotFound; + } + }); + } + }); + } + return true; + }); }) + } else { + throw dependencyNotFound; } - else if(platform === 'win32'){ - let hasVirtualBox = false; - try { - fs.accessSync(path.resolve(`C:/Program Files/Oracle/VirtualBox`)); - hasVirtualBox = true; - } catch (err) { - try { - fs.accessSync(path.resolve(`C:/Program Files (x86)/Oracle/VirtualBox`)); - hasVirtualBox = true; - } - catch(err){ - } - } - - if(hasVirtualBox && hasVagrant) - resolve(true); - else - reject('Dependencies not found. Make sure you have installed VirtualBox and Vagrant.'); - } - }) - }) + }); + } } result.validateBakerScript = async function(bakerScriptPath) { diff --git a/src/package.json b/src/package.json index 7178bad3..0766d1d4 100644 --- a/src/package.json +++ b/src/package.json @@ -35,6 +35,7 @@ "camelcase": "^4.1.0", "chalk": "^1.1.3", "console.table": "^0.9.1", + "drivelist": "^6.0.4", "fs-extra": "^3.0.1", "hasbin": "^1.2.3", "js-yaml": "^3.10.0", From b18e3e12e13c10e9186d18416eddb0486d2d34bd Mon Sep 17 00:00:00 2001 From: Samim Mirhosseini Date: Wed, 24 Jan 2018 11:42:50 -0500 Subject: [PATCH 18/23] adding drivelist to dep variable --- src/baker.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/baker.js b/src/baker.js index 5ae75911..2b097ca7 100755 --- a/src/baker.js +++ b/src/baker.js @@ -55,7 +55,8 @@ let dep = { ansible, configPath, bakerletsPath, - remotesPath + remotesPath, + drivelist }; // Internal dependencies From f666c8f3cd96cd9c7503914e3853abdf62ddaf65 Mon Sep 17 00:00:00 2001 From: Samim Mirhosseini Date: Wed, 24 Jan 2018 15:47:59 -0500 Subject: [PATCH 19/23] changing depdency check to warning --- src/lib/modules/validator.js | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/lib/modules/validator.js b/src/lib/modules/validator.js index bc679032..1db172c5 100644 --- a/src/lib/modules/validator.js +++ b/src/lib/modules/validator.js @@ -4,15 +4,20 @@ module.exports = function(dep) { let result = {}; result.validateDependencies = async function(){ - const { hasbin, Promise, fs, path, drivelist } = dep; + const { hasbin, Promise, fs, path, drivelist, print } = dep; let platform = process.platform; let dependencyNotFound = 'Dependencies not found. Make sure you have installed VirtualBox and Vagrant.'; if (platform == 'darwin' || platform === 'linux') { hasbin.all( ['vagrant', 'virtualbox'], hasDependencies => { if(hasDependencies) return true; - else throw dependencyNotFound;} - ); + else{ + // throw dependencyNotFound; + print.warning(dependencyNotFound, 1) + return true; + } + + }); } else { hasbin('vagrant', async function (hasVagrant) { @@ -25,7 +30,9 @@ module.exports = function(dep) { if(err) { fs.access(path.resolve(path.join(process.env.PROGRAMFILES, `/Oracle/VirtualBox`)), err => { if(err){ - throw dependencyNotFound; + // throw dependencyNotFound; + print.warning(dependencyNotFound, 1) + return true; } }); } @@ -35,7 +42,9 @@ module.exports = function(dep) { }); }) } else { - throw dependencyNotFound; + // throw dependencyNotFound; + print.warning(dependencyNotFound, 1) + return true; } }); } From bc67a610a3f54a56bc3640e657d9bb4d059f3520 Mon Sep 17 00:00:00 2001 From: Samim Mirhosseini Date: Mon, 29 Jan 2018 10:47:12 -0500 Subject: [PATCH 20/23] refactoring mongodb bakelet --- services/mongodb/mongodb3.6.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/services/mongodb/mongodb3.6.yml b/services/mongodb/mongodb3.6.yml index 42a37b2e..078c9f89 100644 --- a/services/mongodb/mongodb3.6.yml +++ b/services/mongodb/mongodb3.6.yml @@ -2,12 +2,18 @@ - hosts: localhost tasks: - - name: Added mongodb key + - name: add an apt key of MongoDB + become: yes apt_key: keyserver: keyserver.ubuntu.com - id: 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 + # url: https://www.mongodb.org/static/pgp/server-3.6.asc + id: 91FA4AD5 + state: present + + # - name: create list file for MongoDB + # command: echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.6 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.6.list - - name: Add mongodb repository + - name: create list file for MongoDB become: yes apt_repository: repo: deb https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.6 multiverse @@ -16,10 +22,12 @@ - name: Install mongodb become: yes apt: - pkg: mongodb-org + pkg: "{{item}}" state: latest update_cache: yes - cache_valid_time: 86400 + with_items: + - libssl1.0.0 + - mongodb-org-server - name: ensure mongodb is running and starts on boot service: name=mongod state=restarted enabled=true @@ -31,4 +39,4 @@ state: latest - name: Install the latest pymongo package - pip: name=pymongo state=latest use_mirrors=no + pip: name=pymongo state=latest use_mirrors=no \ No newline at end of file From 6d34e2e6dc01ee70d006eedf51620a056aa34e66 Mon Sep 17 00:00:00 2001 From: Samim Mirhosseini Date: Mon, 29 Jan 2018 11:19:49 -0500 Subject: [PATCH 21/23] fixing conflicts --- src/remotes/bakerlets-source/services/mongodb/mongodb3.6.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/remotes/bakerlets-source/services/mongodb/mongodb3.6.yml b/src/remotes/bakerlets-source/services/mongodb/mongodb3.6.yml index 37da63dd..d1186b48 100644 --- a/src/remotes/bakerlets-source/services/mongodb/mongodb3.6.yml +++ b/src/remotes/bakerlets-source/services/mongodb/mongodb3.6.yml @@ -32,8 +32,6 @@ - name: ensure mongodb is running and starts on boot service: name=mongod state=restarted enabled=true become: yes -<<<<<<< HEAD -======= - name: Installing Python-Pip apt: @@ -42,4 +40,3 @@ - name: Install the latest pymongo package pip: name=pymongo state=latest use_mirrors=no ->>>>>>> 1c8b11a5d9a90f2a250ed2fdd54bf7e6e6ca1f42 From 45aeca3d4388f1149c4d8148ed768a336753471b Mon Sep 17 00:00:00 2001 From: Samim Mirhosseini Date: Mon, 29 Jan 2018 20:41:33 -0500 Subject: [PATCH 22/23] adding env Bakelet --- env/env.yml.mustache | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 env/env.yml.mustache diff --git a/env/env.yml.mustache b/env/env.yml.mustache new file mode 100644 index 00000000..a9ce3894 --- /dev/null +++ b/env/env.yml.mustache @@ -0,0 +1,14 @@ +--- +- hosts: all + become: yes + + tasks: + {{#env}} + - name: Adding the path in the bashrc files + lineinfile: + dest: /home/{{=<% %>=}}{{ansible_user}}<%={{ }}=%>/.bashrc + line: 'export {{KEY}}={{VALUE}}' + insertafter: 'EOF' + state: present + + {{/env}} From 5a7d1b4228d1c7248c7ee2e3d16cc987373c4c77 Mon Sep 17 00:00:00 2001 From: Samim Mirhosseini Date: Mon, 29 Jan 2018 20:42:49 -0500 Subject: [PATCH 23/23] adding env Bakelet --- src/lib/bakerlets/env/env.js | 48 ++++++++++++++++++++++++++++++++++++ src/lib/bakerlets/resolve.js | 6 +++++ 2 files changed, 54 insertions(+) create mode 100644 src/lib/bakerlets/env/env.js diff --git a/src/lib/bakerlets/env/env.js b/src/lib/bakerlets/env/env.js new file mode 100644 index 00000000..f15cdc77 --- /dev/null +++ b/src/lib/bakerlets/env/env.js @@ -0,0 +1,48 @@ +const { commands, modules } = require('../../../baker'); +const baker = modules['baker']; +const ssh = modules['ssh']; + +const Bakerlet = require('../bakerlet'); +const path = require('path'); +const mustache = require('mustache'); +const fs = require('fs-extra'); + +class Env extends Bakerlet { + constructor(name, ansibleSSHConfig, version) { + super(ansibleSSHConfig); + + this.name = name; + this.version = version; + } + + async load(obj, variables) { + this.variables = variables; + this.envVars = {env: []}; + + obj.env.forEach( e => { + this.envVars.env.push({KEY: Object.keys(e)[0], VALUE: Object.values(e)[0]}) + }) + + let playbookTemplate = path.resolve( + this.remotesPath, + `bakerlets-source/env/env.yml.mustache` + ); + let playbookRendered = mustache.render(await fs.readFileAsync(playbookTemplate, 'utf8'), this.envVars); + + let cmd = `echo "${playbookRendered.replace(/"/g, '\\"')}" > /home/vagrant/baker/${this.name}/env.yml`; + await this.exec(cmd); + } + + async install() { + var cmd = `env.yml`; + await baker.runAnsiblePlaybook( + { name: this.name }, + cmd, + this.ansibleSSHConfig, + this.verbose, + this.variables + ); + } +} + +module.exports = Env; diff --git a/src/lib/bakerlets/resolve.js b/src/lib/bakerlets/resolve.js index 67a9c1f1..dc6ea38d 100644 --- a/src/lib/bakerlets/resolve.js +++ b/src/lib/bakerlets/resolve.js @@ -69,6 +69,12 @@ module.exports.resolveBakerlet = async function(bakerletsPath,remotesPath,doc,ba } } + if( doc.env ) + { + doc.env = [{env: doc.env}]; // fixing the format // TODO: it works ok, but probably too hacky + await resolve(doc.name, bakerScriptPath, remotesPath, path.join(bakerletsPath,"env"), doc.env[0], extra_vars, verbose); + } + if( doc.start ) { let dir = path.join(boxes, doc.name);