Fix precaching to be consistent (#76)

* Fix precaching to be consistent sw-precache reads directly from disk to generate precache manifests, so we must wait for the streams to be complete before calling generateServiceWorker. Read the config file once outside of the stream, let sw-precache write to disk Clone config so that bundled and unbundled don't write into each other Fixes #75 * Add tests for service worker generation Also handle uncaught promises in build Remove console.log * Switch out tmp for temp
Polymer · May 10, 2016 · 176fac7 · 176fac7
1 parent a3a74a2
commit 176fac7
Show file tree

Hide file tree

Showing 8 changed files with 172 additions and 147 deletions.
diff --git a/package.json b/package.json
@@ -21,6 +21,7 @@
   "author": "The Polymer Project Authors",
   "license": "BSD-3-Clause",
   "dependencies": {
+    "clone": "^1.0.2",
     "command-line-args": "^2.1.6",
     "command-line-commands": "^0.1.2",
     "css-slam": "^1.1.0",
@@ -57,6 +58,7 @@
     "eslint": "^2.9.0",
     "gulp-mocha": "^2.2.0",
     "sinon": "^1.17.4",
+    "temp": "^0.8.3",
     "typescript": "^1.8.10",
     "typings": "^0.8.1",
     "vinyl-fs-fake": "^1.1.0",

diff --git a/src/build/build.ts b/src/build/build.ts
@@ -8,6 +8,7 @@
  * subject to an additional IP rights grant found at http://polymer.github.io/PATENTS.txt
  */
 
+import clone = require('clone');
 import * as fs from 'fs';
 import * as gulp from 'gulp';
 import * as gulpif from 'gulp-if';
@@ -24,7 +25,7 @@ import {Logger} from './logger';
 import {optimize, OptimizeOptions} from './optimize';
 import {waitForAll, compose, ForkedVinylStream} from './streams';
 import {StreamResolver} from './stream-resolver';
-import {SWPreCacheTransform} from './sw-precache';
+import {generateServiceWorker, parsePreCacheConfig, SWConfig} from './sw-precache';
 import {VulcanizeTransform} from './vulcanize';
 
 // non-ES compatible modules
@@ -40,13 +41,18 @@ export interface BuildOptions {
   swPrecacheConfig?: string;
 }
 
-process.on('uncaughtException', (err) => {
-  console.log(`Caught exception: ${err}`);
-  console.error(err.stack);
+process.on('uncaughtException', (error) => {
+  console.log(`Caught exception: ${error}`);
+  console.error(error.stack);
+});
+
+process.on('unhandledRejection', (error) => {
+  console.log(`Promise rejection: ${error}`);
+  console.error(error.stack);
 });
 
 export function build(options?: BuildOptions): Promise<any> {
-  return new Promise<any>((resolve, _) => {
+  return new Promise<any>((buildResolve, _) => {
     options = options || {};
     let root = process.cwd();
     let main = path.resolve(root, options.main || 'index.html');
@@ -108,50 +114,44 @@ export function build(options?: BuildOptions): Promise<any> {
         .pipe(depsProject.split)
         .pipe(optimize(optimizeOptions))
         .pipe(depsProject.rejoin)
-        .pipe(vfs.src(swPrecacheConfig, {
-          cwdbase: true,
-          allowEmpty: true,
-          passthrough: true
-        }));
 
     let allFiles = mergeStream(sourcesStream, depsStream);
 
     let serviceWorkerName = 'service-worker.js';
 
     let unbundledPhase = new ForkedVinylStream(allFiles)
-      .pipe(new SWPreCacheTransform({
-        root,
-        main,
-        buildRoot: 'build/unbundled',
-        serviceWorkerName,
-        configFileName: swPrecacheConfig
-      }))
       .pipe(vfs.dest('build/unbundled'))
 
-    // SWPreCacheTransform needs the deps from bundler after bundles are created
-    // therefore, give transform a promise that is resolved when bundler is done
-    let depsResolve: (value: string[]) => void;
-    let depsPromise = new Promise<string[]>((resolve) => {
-      depsResolve = resolve;
-    });
-
     let bundledPhase = new ForkedVinylStream(allFiles)
       .pipe(bundler.bundle)
-      .on('end', () => {
-        // give the entry points and shared bundle to SWPreCacheTransform
-        let depsList = Array.from(bundler.entrypointFiles.keys());
-        depsList.push(bundler.sharedBundleUrl);
-        depsResolve(depsList);
-      })
-      .pipe(new SWPreCacheTransform({
+      .pipe(vfs.dest('build/bundled'));
+
+    let genSW = (buildRoot: string, deps: string[], swConfig: SWConfig) => {
+      return generateServiceWorker({
         root,
         main,
-        buildRoot: 'build/bundled',
-        deps: depsPromise,
-        serviceWorkerName,
-        configFileName: swPrecacheConfig
-      }))
-      .pipe(vfs.dest('build/bundled'));
+        deps,
+        buildRoot,
+        swConfig: clone(swConfig),
+        serviceWorkerPath: path.join(root, buildRoot, serviceWorkerName)
+      });
+    };
+
+    waitForAll([unbundledPhase, bundledPhase]).then(() => {
+      let unbundledDeps = Array.from(bundler.streamResolver.requestedUrls);
+
+      let bundledDeps = Array.from(bundler.entrypointFiles.keys());
+      bundledDeps.push(bundler.sharedBundleUrl);
+
+      parsePreCacheConfig(swPrecacheConfig).then((swConfig) => {
+        Promise.all([
+          genSW('build/unbundled', unbundledDeps, swConfig),
+          genSW('build/bundled', bundledDeps, swConfig)
+        ]).then(() => {
+          buildResolve();
+        });
+      })
+    });
   });
 }
 

diff --git a/src/build/sw-precache.ts b/src/build/sw-precache.ts
@@ -10,14 +10,14 @@
 
 import * as fs from 'fs';
 import * as path from 'path';
-import {Transform} from 'stream';
+import {PassThrough} from 'stream';
 import File = require('vinyl');
 
 // non-ES compatible modules
 const swPrecache = require('sw-precache');
 const Module = require('module');
 
-interface SWConfig {
+export interface SWConfig {
   cacheId?: string,
   directoryIndex?: string;
   dynamicUrlToDependencies?: {
@@ -47,40 +47,48 @@ interface SWConfig {
   verbose?: boolean;
 }
 
-function generateServiceWorker(
-    root: string,
-    main: string,
-    deps: string[],
-    buildRoot: string,
-    swConfig?: SWConfig
-  ): Promise<string> {
-  swConfig = swConfig || <SWConfig>{};
+export function generateServiceWorker(options: generateServiceWorkerOptions)
+: Promise<void> {
+  let swConfig = options.swConfig || <SWConfig>{};
   // strip root prefix, so buildRoot prefix can be added safely
-  deps = deps.map((p) => {
-    if (p.startsWith(root)) {
-      return p.substring(root.length);
+  let deps = options.deps.map((p) => {
+    if (p.startsWith(options.root)) {
+      return p.substring(options.root.length);
     }
     return p;
   });
-  let mainHtml = main.substring(root.length);
+  let mainHtml = options.main.substring(options.root.length);
   let precacheFiles = new Set(swConfig.staticFileGlobs);
   deps.forEach((p) => precacheFiles.add(p));
   precacheFiles.add(mainHtml);
 
   let precacheList = Array.from(precacheFiles);
-  precacheList = precacheList.map((p) => path.join(buildRoot, p));
+  precacheList = precacheList.map((p) => path.join(options.buildRoot, p));
 
   // swPrecache will determine the right urls by stripping buildRoot
-  swConfig.stripPrefix = buildRoot;
+  swConfig.stripPrefix = options.buildRoot;
   // static files will be pre-cached
   swConfig.staticFileGlobs = precacheList;
 
-  console.log(`Generating service worker for ${buildRoot}`);
+  console.log(`Generating service worker for ${options.buildRoot}`);
 
-  return swPrecache.generate(swConfig);
+  return swPrecache.write(options.serviceWorkerPath, swConfig);
 }
 
-export interface SWPreCacheTransformOptions {
+export function parsePreCacheConfig(configFile: string): Promise<SWConfig> {
+  return new Promise<SWConfig>((resolve, reject) => {
+    try {
+      let config: SWConfig = require(configFile);
+      resolve(config);
+    } catch(e) {
+      console.error(`Could not load sw-precache config from ${configFile}`);
+      console.error(e);
+      resolve();
+    }
+  });
+}
+
+export interface generateServiceWorkerOptions {
   /**
    * folder containing files to be served by the service worker.
    */
@@ -94,100 +102,16 @@ export interface SWPreCacheTransformOptions {
    */
   buildRoot: string;
   /**
-   * Name of the output service worker file.
+   * File path of the output service worker file.
    */
-  serviceWorkerName: string;
+  serviceWorkerPath: string;
   /**
-   * Promise that returns the list of files to be cached by the service worker.
-   *
-   * If not given, all files streamed to SWPreCacheTransform will be put into
-   * the precache list, except for the file matching `configFileName`.
+   * List of files to be cached by the service worker,
+   * in addition to files found in `swConfig.staticFileGlobs`
    */
-  deps?: Promise<string[]>;
+  deps: string[];
   /**
-   * Existing config file to use as a base for the serivce worker generation.
-   *
-   * This file will not be copied into the output bundles.
+   * Existing config to use as a base for the serivce worker generation.
    */
-  configFileName?: string;
-}
-
-export class SWPreCacheTransform extends Transform {
-  swConfig: SWConfig;
-  options: SWPreCacheTransformOptions;
-  fileSet: Set<string>;
-  fullConfigFilePath: string;
-
-  constructor(options: SWPreCacheTransformOptions) {
-    super({objectMode: true});
-    this.options = options;
-    // if no given deps, collect all input files as deps
-    if (!options.deps) {
-      this.fileSet = new Set<string>();
-    }
-    if (options.configFileName) {
-      this.fullConfigFilePath = path.resolve(
-        options.root,
-        options.configFileName
-      );
-    }
-  }
-
-  _transform(file: File, encoding: string, callback: (error?, data?) => void): void {
-    if (file.path === this.fullConfigFilePath) {
-      try {
-        if (file.path.endsWith('js')) {
-          // `module._compile` is the heart of `require`
-          // http://fredkschott.com/post/2014/06/require-and-the-module-system/
-          let m = new Module(file.path);
-          m._compile(
-            file.contents.toString(),
-            file.path
-          );
-          this.swConfig = m.exports;
-        } else if (file.path.endsWith('json')) {
-          this.swConfig = JSON.parse(file.contents.toString());
-        }
-      } catch(e) {
-        let cfn = this.options.configFileName;
-        console.error(`Could not load service worker config from ${cfn}`);
-        console.error(e);
-      }
-      callback();
-    } else {
-      if (this.fileSet) {
-        this.fileSet.add(file.path);
-      }
-      callback(null, file);
-    }
-  }
-
-  _flush(callback: (error?) => void) {
-    let promise: Promise<string[]>;
-    if (this.fileSet) {
-      promise = Promise.resolve(Array.from(this.fileSet));
-    } else {
-      promise = this.options.deps;
-    }
-    promise.then((deps) => {
-      return generateServiceWorker(
-        this.options.root,
-        this.options.main,
-        deps,
-        this.options.buildRoot,
-        this.swConfig
-      );
-    })
-    .then((config) => {
-      let file = new File({
-        path: path.resolve(
-          this.options.root,
-          this.options.serviceWorkerName
-        ),
-        contents: new Buffer(config)
-      });
-      this.push(file);
-      callback();
-    });
-  }
+  swConfig?: SWConfig;
 }
diff --git a/test/build/precache/config.js b/test/build/precache/config.js
@@ -0,0 +1,3 @@
+module.exports = {
+  staticFileGlobs: ['*']
+}
diff --git a/test/build/precache/static/fizz.html b/test/build/precache/static/fizz.html
@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <title></title>
+  </head>
+  <body>
+
+  </body>
+</html>
diff --git a/test/build/precache/static/foo.js b/test/build/precache/static/foo.js
@@ -0,0 +1 @@
+var x = 3;