Skip to content

Commit

Permalink
Merge pull request #76 from juttle/adapter-end-to-end-tests
Browse files Browse the repository at this point in the history 
added adapter end to end tests
  • Loading branch information
@rlgomes
rlgomes committed Mar 11, 2016
2 parents 2919230 + 937d8ab commit 952be2e
Show file tree
Hide file tree
Showing 9 changed files with 465 additions and 95 deletions.
1 change: 1 addition & 0 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ before_install:
- curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
- chmod +x docker-compose
- sudo mv docker-compose /usr/local/bin
- docker build -t juttle/juttle-engine:latest .

script:
- gulp lint
Expand Down
5 changes: 3 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
"juttle-cloudwatch-adapter": "^0.3.0",
"juttle-elastic-adapter": "^0.5.0",
"juttle-gmail-adapter": "^0.5.0",
"juttle-graphite-adapter": "^0.4.0",
"juttle-graphite-adapter": "^0.4.2",
"juttle-influx-adapter": "^0.5.0",
"juttle-mysql-adapter": "^0.5.0",
"juttle-opentsdb-adapter": "^0.2.0",
Expand Down Expand Up @@ -70,6 +70,7 @@
"selenium-grid-status": "^0.2.0",
"selenium-webdriver": "^2.48.2",
"tmp": "0.0.28",
"underscore": "^1.8.3"
"underscore": "^1.8.3",
"uuid": "^2.0.1"
}
}
3 changes: 3 additions & 0 deletions test/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ To run the built in system tests simply run:
gulp test --sys
```

**NOTE:** be sure to run `docker build -t juttle-engine:local .` after each
code change if you intend on running the system tests locally.

The system tests rely on having docker to bring up the selenium containers and
other supporting [docker](https://www.docker.com/) containers. If you happen to
want to run the selenium tests locally by using your own available chrome
Expand Down
196 changes: 196 additions & 0 deletions test/adapters/adapters.spec.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
'use strict';

let Promise = require('bluebird');
let docker = require('../lib/docker');
let expect = require('chai').expect;
let fs = Promise.promisifyAll(require('fs'));
let logger = require('mocha-logger');
let path = require('path');
let retry = require('bluebird-retry');
let tmp = require('tmp');
let uuid = require('uuid');

function juttleBin(juttle) {
logger.log(`juttle -e "${juttle}"`);
return docker.exec('juttle-engine-local',
['/opt/juttle-engine/bin/juttle',
'-e',
juttle], { quiet: true });
}

describe('adapters', () => {

before(() => {
var tmpdir = tmp.dirSync().name;
return Promise.all([
docker.destroy('elasticsearch-adapter-test'),
docker.destroy('influxdb-adapter-test'),
docker.destroy('graphite-adapter-test'),
docker.destroy('juttle-engine-local')
])
.then(() => {
return docker.checkJuttleEngineLocalExists();
})
.then(() => {
return Promise.all([
docker.run({
name: 'elasticsearch-adapter-test',
image: 'elasticsearch:1.5.2',
detach: true
}),
docker.run({
name: 'influxdb-adapter-test',
image: 'tutum/influxdb:0.10',
detach: true
}),
docker.run({
name: 'graphite-adapter-test',
image: 'sitespeedio/graphite:0.9.14',
detach: true
})
]);
})
.then(() => {
var config = JSON.stringify({
adapters: {
elastic: {
address: 'elasticsearch',
port: 9200
},
influx: {
url: 'http://influxdb:8086'
},
graphite: {
carbon: {
host: 'graphite',
port: 2003
},
webapp: {
host: 'graphite',
port: 80,
username: 'guest',
password: 'guest'
}
}
}
});

var filename = path.join(tmpdir, '.juttle-config.json')
return fs.writeFileAsync(filename, config)
})
.then(() => {
return docker.run({
name: 'juttle-engine-local',
image: 'juttle/juttle-engine:latest',
ports: ['8080:8080'],
links: [
'elasticsearch-adapter-test:elasticsearch',
'influxdb-adapter-test:influxdb',
'graphite-adapter-test:graphite'
],
volumes: [`${tmpdir}:/tmp`],
workdir: '/tmp',
detach: true
});
})
.then(() => {
return Promise.all([
docker.waitForSuccess('elasticsearch-adapter-test',
['curl', 'http://localhost:9200']),
docker.waitForSuccess('influxdb-adapter-test',
['influx', '-execute', 'show databases']),
docker.waitForSuccess('graphite-adapter-test',
['curl', '-u', 'guest:guest', 'http://localhost:80'])
]);
});
});

after(() => {
return Promise.all([
docker.destroy('elasticsearch-adapter-test'),
docker.destroy('influxdb-adapter-test'),
docker.destroy('graphite-adapter-test'),
docker.destroy('juttle-engine-local')
]);
});

describe('juttle-elastic-adapter', () => {
it('can write data and read back data using the juttle CLI', () => {
var id = uuid.v1().slice(0, 8);

return juttleBin(`emit -limit 5 -from :2014-01-01: | put name="test-${id}",value=count() | write elastic`)
.then(() => {
return retry(() => {
return juttleBin(`read elastic -from :2014-01-01: name="test-${id}" | view text`)
.then((output) => {
var data = JSON.parse(output);
expect(data).to.deep.equal([
{ time: '2014-01-01T00:00:00.000Z', name: `test-${id}`, value: 1 },
{ time: '2014-01-01T00:00:01.000Z', name: `test-${id}`, value: 2 },
{ time: '2014-01-01T00:00:02.000Z', name: `test-${id}`, value: 3 },
{ time: '2014-01-01T00:00:03.000Z', name: `test-${id}`, value: 4 },
{ time: '2014-01-01T00:00:04.000Z', name: `test-${id}`, value: 5 }
]);
});
},{ timeout: 50000 } );
});
});
});

describe('juttle-influx-adapter', () => {
it('can write data and read back data using the juttle CLI', () => {
var id = uuid.v1().slice(0, 8);
var db = `testdb_${id}`;

return juttleBin(`read influx -db 'dummy' -raw 'CREATE DATABASE ${db}';`)
.then(() => {
return juttleBin(`emit -limit 5 -from :2014-01-01: | put name="test-${id}", value=count() | write influx -db '${db}'`);
})
.then(() => {
return retry(() => {
return juttleBin(`read influx -db '${db}' -from :2014-01-01: name="test-${id}" | view text`)
.then((output) => {
var data = JSON.parse(output);
expect(data).to.deep.equal([
{ time: '2014-01-01T00:00:00.000Z', name: `test-${id}`, value: 1 },
{ time: '2014-01-01T00:00:01.000Z', name: `test-${id}`, value: 2 },
{ time: '2014-01-01T00:00:02.000Z', name: `test-${id}`, value: 3 },
{ time: '2014-01-01T00:00:03.000Z', name: `test-${id}`, value: 4 },
{ time: '2014-01-01T00:00:04.000Z', name: `test-${id}`, value: 5 }
]);
});
}, { timeout: 5000 });
});
});
});

describe('juttle-graphite-adapter', () => {
it('can write data and read back data using the juttle CLI', () => {
var id = uuid.v1().slice(0, 8);

// default data retention for graphite server is:
//
// retentions = 5m:1d,15m:21d,30m:60d
//
// so we'll generate data from 30 minutes ago every 5 minutes and
// then read back the exact points
return juttleBin(`emit -limit 5 -every :5m: -from :30 minutes ago: | put name="test-${id}", value=count() | write graphite`)
.then(() => {
return retry(() => {
return juttleBin(`read graphite -from :60 minutes ago: name="test-${id}" | keep name, value | view text`)
.then((output) => {
var data = JSON.parse(output);
expect(data).to.deep.equal([
{ name: `test-${id}`, value: 1 },
{ name: `test-${id}`, value: 2 },
{ name: `test-${id}`, value: 3 },
{ name: `test-${id}`, value: 4 },
{ name: `test-${id}`, value: 5 }
]);
})
}, { timeout: 5000 });
});
});
});

});
113 changes: 61 additions & 52 deletions test/examples/examples.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,47 +3,40 @@
let _ = require('underscore');
let Promise = require('bluebird');
let chakram = require('chakram');
let docker = require('../lib/docker');
var expect = chakram.expect;
let execAsync = Promise.promisify(require('child_process').exec);
let fs = require('fs');
let http = require('http');
let logger = require('mocha-logger');
let path = require('path');
let cmd = require('../lib/cmd');
let retry = require('bluebird-retry');
let spawn = require('child_process').spawn;
let WebSocket = require('ws');

function spawnify(command, args, options) {
return new Promise((resolve, reject) => {
args = args || [];
logger.log('spawning "' + command + ' ' + args.join(' '));
options = _.extend( {
detached: true,
stdio: ['ignore', 'pipe', 'pipe']
}, options);

var spawned = spawn(command, args, options);
var stdout = '';
var stderr = '';

spawned.stdout.on('data', (data) => {
logger.log('STDOUT:', data);
stdout += data;
});
const YMLS = [
'dc-juttle-engine.yml',
'cadvisor-influx/dc-cadvisor-influx.yml',
'postgres-diskstats/dc-postgres.yml'
]

spawned.stderr.on('data', (data) => {
logger.log('STDERR:', data);
stderr += data;
});
function dockerCompose(ymls, command, commandArgs) {
commandArgs = commandArgs || [];
var args = [];

spawned.on('close', (code) => {
if (code === 0) {
resolve(stdout, stderr);
} else {
reject(Error('command: "' + command + ' ' + args.join(' ') +
'", failed with ' + code));
}
});
_.each(ymls, (yml) => {
args.push('-f');
args.push(yml);
});

args.push(command);
args = args.concat(commandArgs);

var env = Object.create(process.env);
// we need to set the PWD used by those yml files to . otherwise we inherit
// from the current process running at the root of the source code
env.PWD = '.';
return cmd.spawnAsync('docker-compose', args, {
cwd: 'examples',
env: env
});
}

Expand All @@ -56,38 +49,54 @@ describe('examples', () => {
let retryHTTPOptions = {
interval: 500,
timeout: 10000
}
};

before(() => {
return execAsync('docker run ubuntu route | grep default | awk \'{print $2}\'')
.then((stdout) => {
hostAddress = stdout.trim();
return spawnify('docker', ['build', '-q', '-t', 'juttle/juttle-engine:latest', '.']);
return docker.getHostAddress()
.then((address) => {
hostAddress = address;
return docker.checkJuttleEngineLocalExists();
})
.then(() => {
return dockerCompose(YMLS, 'stop');
})
.then(() => {
return spawnify('test/examples/start-example-containers.sh');
return dockerCompose(YMLS, 'rm', ['--force']);
})
.then(() => {
return retry(() => {
return new Promise((resolve, reject) => {
http.get(baseUrl, (resp) => {
if (resp.statusCode !== 200) {
reject(Error(`juttle-engine not responding on ${baseUrl}`));
} else {
resolve();
}
})
.on('error', (err) => {
reject(err);
return dockerCompose(YMLS, 'up', ['-d']);
})
.then(() => {
// wait for various underlying storages to be up and running
return Promise.all([
docker.waitForSuccess('examples_postgres_1',
['pgrep', 'postgres']),
docker.waitForSuccess('examples_influxdb_1',
['influx', '-execute', 'show databases']),
retry(() => {
return new Promise((resolve, reject) => {
http.get(baseUrl, (resp) => {
if (resp.statusCode !== 200) {
reject(Error(`juttle-engine not responding on ${baseUrl}`));
} else {
resolve();
}
})
.on('error', (err) => {
reject(err);
});
});
});
}, retryHTTPOptions);
}, retryHTTPOptions)
]);
});
});

after(() => {
// shutdown and remove containers
return spawnify('test/examples/stop-example-containers.sh');
return dockerCompose(YMLS, 'stop')
.then(() => {
return dockerCompose(YMLS, 'rm', ['--force']);
});
});

_.each({
Expand Down
Loading

0 comments on commit 952be2e

Please sign in to comment.