Add rsync option to package command and improve performance

.travis.yml

@@ -44,15 +44,15 @@ script:
   - echo "Working copy disk usage"; du -chs .
   - grunt --version
   - grunt help
-  - grunt --quiet --timer
+  - grunt --timer
   - echo "Fully built disk usage"; du -chs .
-  - grunt install --db-url=sqlite:/`pwd`/.ht.sqlite --quiet
+  - grunt install --db-url=sqlite:/`pwd`/.ht.sqlite
   - grunt serve:test >/dev/null &
   - until curl -I -XGET -s $GDT_TEST_URL 2>/dev/null | egrep -q '^HTTP.*200'; do sleep 0.5; done
   - vendor/bin/drush --root=build/html en -y simpletest
   - grunt test
   - sleep 1; while (ps aux | grep '[b]ehat' > /dev/null); do sleep 1; done
   - for pid in `ps aux | grep drush | grep runserver | awk '{print $2}'`; do echo "Stopping drush pid $pid"; kill -SIGINT $pid; done;
-  - grunt package --quiet
+  - grunt package
   - mocha --timeout 30000 node_modules/grunt-drupal-tasks/test/build.js
   - mocha node_modules/grunt-drupal-tasks/test/library.js

docs/60_PACKAGE.md

@@ -51,6 +51,35 @@ directory.  Defaults to the docroot.
 packages directory. This can be overridden by calling grunt package with the
 `--name` parameter.
 
+**archive**: Used to enable the `compress` option via the configuration instead
+of the command line. Defaults to false.
+
+**rsync**: If set to true, rsync will be used instead of file copying for
+improved performance. This automatically enables the `compress` option and will
+write the archive to the package destination.
+
+**vmData**: This path is used as the intermediate generation directory for the
+package when using the `rsync` option. By selecting a volume mounted in your
+VM `/data` area, this can significantly improve performance. The archive is
+still moved to the normal package destination when complete.  Defaults to
+`build/vm_data`.
+
+## Performance
+
+When running `grunt package` within a docker container, the default file
+operations will be using NFS to update the files on your local disk, which will
+be very slow. Using the `rsync` option along with creating the `build/vm_data`
+intermediate mount point in your docker container can improve performance by
+a factor of 20 or more. However, the downside is only the compressed archive of
+your package will be written to your local disk. The full package folder will
+only be accessible within your docker container.
+
+A sample docker-composer.yml entry to mount the vm_data looks like this:
+```
+  volumes:
+    - /data/PROJECTNAME/package:/var/www/build/vm_data
+```
+
 ## Packaging for Acquia
 
 The `package` command has the flexibility to support many different use cases,
@@ -65,6 +94,7 @@ Acquia repository with support for custom hooks and scripts.
   "packages": {
     "srcFiles": ["!sites/*/files/**", "!xmlrpc.php", "!modules/php/*"],
     "projFiles": ["README*", "bin/**", "hooks/**"],
+    "rsync": true,
     "dest": {
       "docroot": "docroot"
     }

tasks/package.js

@@ -61,27 +61,99 @@ module.exports = function(grunt) {
 
   grunt.registerTask('package', 'Package the operational codebase for deployment. Use package:compress to create an archive.', function() {
     grunt.loadNpmTasks('grunt-contrib-copy');
+    grunt.loadNpmTasks('grunt-shell');
 
     var config = grunt.config.get('config.packages');
-    var srcFiles = ['**', '!**/.gitkeep'].concat((config && config.srcFiles && config.srcFiles.length) ? config.srcFiles : '**');
+    var srcFiles = (config && config.srcFiles && config.srcFiles.length) ? config.srcFiles : [];
     var projFiles = (config && config.projFiles && config.projFiles.length) ? config.projFiles : [];
 
     // Look for a package target spec, build destination path.
     var packageName = grunt.option('name') || config.name || 'package';
-    var destPath = grunt.config.get('config.buildPaths.packages') + '/' + packageName;
+
+    // Determine if we are using rsync and vmPath and tarball for performance.
+    var useRsync = grunt.config('config.packages.rsync') || false;
+    // Determine if we are creating a tarball archive.
+    var archive = grunt.config('config.packages.archive') || useRsync;
+
+    // When using rsync, generate to a path that can be mounted in the VM.
+    var vmPath = grunt.config.get('config.packages.vmData') || 'build/vm_data';
+    var destPath = vmPath + '/' + packageName;
+    var finalPath = grunt.config.get('config.buildPaths.packages') + '/' + packageName;
+    if (!useRsync) {
+      // If not using rsync, then directly generate to final path.
+      destPath = finalPath;
+    }
+
     var tasks = [];
+    var cleanPaths = [destPath];
+    if (destPath !== finalPath) {
+      cleanPaths.push(finalPath);
+    }
     grunt.option('package-dest', destPath);
+    grunt.config.set('clean.packages', cleanPaths);
+    tasks.push('clean:packages');
+
+    var excludePaths = ['bower_components', 'node_modules', '.gitkeep'];
+
+    if (useRsync) {
+      // Pull negate conditions from srcFiles into excludePaths.
+      for (var srcIndex = 0; srcIndex < srcFiles.length; srcIndex++) {
+        var item = srcFiles[srcIndex];
+        if (item.substr(0, 1) === '!') {
+          item = item.slice(1);
+          item = item.replace('**', '*');
+          excludePaths.push(item);
+        }
+      }
+
+      // Setup default rsync command and options.
+      var rsync = 'rsync -ahWL --no-l --stats --chmod=Du+rwx ';
+      for (var pathIndex = 0; pathIndex < excludePaths.length; pathIndex++) {
+        rsync = rsync + "--exclude " + excludePaths[pathIndex] + ' ';
+      }
+
+      // Ensure destination exists and sent the rsync.
+      var srcPath = grunt.config('config.buildPaths.html');
+      var destSrc = path.resolve(destPath, grunt.config.get('config.packages.dest.docroot') || '');
+      var rsyncCommand = rsync + srcPath + '/ ' + destSrc + '/';
+
+      grunt.config.set('mkdir.package', {
+        options: {
+          create: [destSrc]
+        }
+      });
+
+      grunt.config('shell.rsync', {
+        command: rsyncCommand
+      });
+    } else {
+      // Use slower copy if rsync is disabled in options.
+      srcFiles.unshift('**');
+      // Add any additional excludePaths to srcFiles.
+      for (var i = 0; i < excludePaths.length; i++) {
+        srcFiles.push('!**/' + excludePaths[i]);
+      }
+      grunt.config('copy.source', {
+        files: [
+          {
+            expand: true,
+            cwd: '<%= config.buildPaths.html %>',
+            src: srcFiles,
+            dest: path.resolve(destPath, grunt.config.get('config.packages.dest.docroot') || ''),
+            dot: true,
+            follow: true
+          }
+        ],
+        options: {
+          gruntLogHeader: false,
+          mode: true
+        }
+      });
+    }
 
+    // Always copy any files specified in projFiles.
     grunt.config('copy.package', {
       files: [
-        {
-          expand: true,
-          cwd: '<%= config.buildPaths.html %>',
-          src: srcFiles,
-          dest: path.resolve(destPath, grunt.config.get('config.packages.dest.docroot') || ''),
-          dot: true,
-          follow: true
-        },
         {
           expand: true,
           src: projFiles,
@@ -96,9 +168,12 @@ module.exports = function(grunt) {
       }
     });
 
-    grunt.config.set('clean.packages', [destPath]);
-
-    tasks.push('clean:packages');
+    if (useRsync) {
+      tasks.push('mkdir:package');
+      tasks.push('shell:rsync');
+    } else {
+      tasks.push('copy:source');
+    }
     tasks.push('copy:package');
 
     // If the `composer.json` file is being packaged, rebuild composer dependencies without dev.
@@ -117,25 +192,31 @@ module.exports = function(grunt) {
       tasks.push('composer:drupal-scaffold');
     }
 
-    if (this.args[0] && this.args[0] === 'compress') {
+    if (archive || (this.args[0] && this.args[0] === 'compress')) {
       grunt.loadNpmTasks('grunt-contrib-compress');
       grunt.config('compress.package', {
         options: {
           archive: destPath + '.tgz',
-          mode: 'tgz',
-          gruntLogHeader: false
+          mode: 'tgz'
         },
         files: [
           {
             expand: true,
             dot: true,
-            cwd: grunt.config.get('config.buildPaths.packages') + '/' + packageName,
+            cwd: destPath,
             src: ['**']
           }
         ]
       });
 
       tasks.push('compress:package');
+
+      if (destPath !== finalPath) {
+        grunt.config('shell.mvArchive', {
+          command: 'mv ' + destPath + '.tgz ' + finalPath + '.tgz'
+        });
+        tasks.push('shell:mvArchive');
+      }
     }
 
     grunt.task.run(tasks);