diff --git a/.travis.yml b/.travis.yml index 3450a18b..c585c2a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,15 +44,15 @@ script: - echo "Working copy disk usage"; du -chs . - grunt --version - grunt help - - grunt --quiet --timer + - grunt --timer - echo "Fully built disk usage"; du -chs . - - grunt install --db-url=sqlite:/`pwd`/.ht.sqlite --quiet + - grunt install --db-url=sqlite:/`pwd`/.ht.sqlite - grunt serve:test >/dev/null & - until curl -I -XGET -s $GDT_TEST_URL 2>/dev/null | egrep -q '^HTTP.*200'; do sleep 0.5; done - vendor/bin/drush --root=build/html en -y simpletest - grunt test - sleep 1; while (ps aux | grep '[b]ehat' > /dev/null); do sleep 1; done - for pid in `ps aux | grep drush | grep runserver | awk '{print $2}'`; do echo "Stopping drush pid $pid"; kill -SIGINT $pid; done; - - grunt package --quiet + - grunt package - mocha --timeout 30000 node_modules/grunt-drupal-tasks/test/build.js - mocha node_modules/grunt-drupal-tasks/test/library.js diff --git a/docs/60_PACKAGE.md b/docs/60_PACKAGE.md index f2ed4443..73322116 100644 --- a/docs/60_PACKAGE.md +++ b/docs/60_PACKAGE.md @@ -51,6 +51,35 @@ directory. Defaults to the docroot. packages directory. This can be overridden by calling grunt package with the `--name` parameter. +**archive**: Used to enable the `compress` option via the configuration instead +of the command line. Defaults to false. + +**rsync**: If set to true, rsync will be used instead of file copying for +improved performance. This automatically enables the `compress` option and will +write the archive to the package destination. + +**vmData**: This path is used as the intermediate generation directory for the +package when using the `rsync` option. By selecting a volume mounted in your +VM `/data` area, this can significantly improve performance. The archive is +still moved to the normal package destination when complete. Defaults to +`build/vm_data`. + +## Performance + +When running `grunt package` within a docker container, the default file +operations will be using NFS to update the files on your local disk, which will +be very slow. Using the `rsync` option along with creating the `build/vm_data` +intermediate mount point in your docker container can improve performance by +a factor of 20 or more. However, the downside is only the compressed archive of +your package will be written to your local disk. The full package folder will +only be accessible within your docker container. + +A sample docker-composer.yml entry to mount the vm_data looks like this: +``` + volumes: + - /data/PROJECTNAME/package:/var/www/build/vm_data +``` + ## Packaging for Acquia The `package` command has the flexibility to support many different use cases, @@ -65,6 +94,7 @@ Acquia repository with support for custom hooks and scripts. "packages": { "srcFiles": ["!sites/*/files/**", "!xmlrpc.php", "!modules/php/*"], "projFiles": ["README*", "bin/**", "hooks/**"], + "rsync": true, "dest": { "docroot": "docroot" } diff --git a/tasks/package.js b/tasks/package.js index d56ce90c..b105cefa 100644 --- a/tasks/package.js +++ b/tasks/package.js @@ -61,27 +61,99 @@ module.exports = function(grunt) { grunt.registerTask('package', 'Package the operational codebase for deployment. Use package:compress to create an archive.', function() { grunt.loadNpmTasks('grunt-contrib-copy'); + grunt.loadNpmTasks('grunt-shell'); var config = grunt.config.get('config.packages'); - var srcFiles = ['**', '!**/.gitkeep'].concat((config && config.srcFiles && config.srcFiles.length) ? config.srcFiles : '**'); + var srcFiles = (config && config.srcFiles && config.srcFiles.length) ? config.srcFiles : []; var projFiles = (config && config.projFiles && config.projFiles.length) ? config.projFiles : []; // Look for a package target spec, build destination path. var packageName = grunt.option('name') || config.name || 'package'; - var destPath = grunt.config.get('config.buildPaths.packages') + '/' + packageName; + + // Determine if we are using rsync and vmPath and tarball for performance. + var useRsync = grunt.config('config.packages.rsync') || false; + // Determine if we are creating a tarball archive. + var archive = grunt.config('config.packages.archive') || useRsync; + + // When using rsync, generate to a path that can be mounted in the VM. + var vmPath = grunt.config.get('config.packages.vmData') || 'build/vm_data'; + var destPath = vmPath + '/' + packageName; + var finalPath = grunt.config.get('config.buildPaths.packages') + '/' + packageName; + if (!useRsync) { + // If not using rsync, then directly generate to final path. + destPath = finalPath; + } + var tasks = []; + var cleanPaths = [destPath]; + if (destPath !== finalPath) { + cleanPaths.push(finalPath); + } grunt.option('package-dest', destPath); + grunt.config.set('clean.packages', cleanPaths); + tasks.push('clean:packages'); + + var excludePaths = ['bower_components', 'node_modules', '.gitkeep']; + + if (useRsync) { + // Pull negate conditions from srcFiles into excludePaths. + for (var srcIndex = 0; srcIndex < srcFiles.length; srcIndex++) { + var item = srcFiles[srcIndex]; + if (item.substr(0, 1) === '!') { + item = item.slice(1); + item = item.replace('**', '*'); + excludePaths.push(item); + } + } + + // Setup default rsync command and options. + var rsync = 'rsync -ahWL --no-l --stats --chmod=Du+rwx '; + for (var pathIndex = 0; pathIndex < excludePaths.length; pathIndex++) { + rsync = rsync + "--exclude " + excludePaths[pathIndex] + ' '; + } + + // Ensure destination exists and sent the rsync. + var srcPath = grunt.config('config.buildPaths.html'); + var destSrc = path.resolve(destPath, grunt.config.get('config.packages.dest.docroot') || ''); + var rsyncCommand = rsync + srcPath + '/ ' + destSrc + '/'; + + grunt.config.set('mkdir.package', { + options: { + create: [destSrc] + } + }); + + grunt.config('shell.rsync', { + command: rsyncCommand + }); + } else { + // Use slower copy if rsync is disabled in options. + srcFiles.unshift('**'); + // Add any additional excludePaths to srcFiles. + for (var i = 0; i < excludePaths.length; i++) { + srcFiles.push('!**/' + excludePaths[i]); + } + grunt.config('copy.source', { + files: [ + { + expand: true, + cwd: '<%= config.buildPaths.html %>', + src: srcFiles, + dest: path.resolve(destPath, grunt.config.get('config.packages.dest.docroot') || ''), + dot: true, + follow: true + } + ], + options: { + gruntLogHeader: false, + mode: true + } + }); + } + // Always copy any files specified in projFiles. grunt.config('copy.package', { files: [ - { - expand: true, - cwd: '<%= config.buildPaths.html %>', - src: srcFiles, - dest: path.resolve(destPath, grunt.config.get('config.packages.dest.docroot') || ''), - dot: true, - follow: true - }, { expand: true, src: projFiles, @@ -96,9 +168,12 @@ module.exports = function(grunt) { } }); - grunt.config.set('clean.packages', [destPath]); - - tasks.push('clean:packages'); + if (useRsync) { + tasks.push('mkdir:package'); + tasks.push('shell:rsync'); + } else { + tasks.push('copy:source'); + } tasks.push('copy:package'); // If the `composer.json` file is being packaged, rebuild composer dependencies without dev. @@ -117,25 +192,31 @@ module.exports = function(grunt) { tasks.push('composer:drupal-scaffold'); } - if (this.args[0] && this.args[0] === 'compress') { + if (archive || (this.args[0] && this.args[0] === 'compress')) { grunt.loadNpmTasks('grunt-contrib-compress'); grunt.config('compress.package', { options: { archive: destPath + '.tgz', - mode: 'tgz', - gruntLogHeader: false + mode: 'tgz' }, files: [ { expand: true, dot: true, - cwd: grunt.config.get('config.buildPaths.packages') + '/' + packageName, + cwd: destPath, src: ['**'] } ] }); tasks.push('compress:package'); + + if (destPath !== finalPath) { + grunt.config('shell.mvArchive', { + command: 'mv ' + destPath + '.tgz ' + finalPath + '.tgz' + }); + tasks.push('shell:mvArchive'); + } } grunt.task.run(tasks);