From 176fac7e678a76ee99eca4a6807c9411cb1fbc21 Mon Sep 17 00:00:00 2001 From: Daniel Freedman Date: Tue, 10 May 2016 16:07:15 -0700 Subject: [PATCH] Fix precaching to be consistent (#76) * Fix precaching to be consistent sw-precache reads directly from disk to generate precache manifests, so we must wait for the streams to be complete before calling generateServiceWorker. Read the config file once outside of the stream, let sw-precache write to disk Clone config so that bundled and unbundled don't write into each other Fixes #75 * Add tests for service worker generation Also handle uncaught promises in build Remove console.log * Switch out tmp for temp --- package.json | 2 + src/build/build.ts | 74 +++++++------- src/build/sw-precache.ts | 144 +++++++-------------------- test/build/precache/config.js | 3 + test/build/precache/static/fizz.html | 10 ++ test/build/precache/static/foo.js | 1 + test/build/precache_test.js | 84 ++++++++++++++++ typings.json | 1 + 8 files changed, 172 insertions(+), 147 deletions(-) create mode 100644 test/build/precache/config.js create mode 100644 test/build/precache/static/fizz.html create mode 100644 test/build/precache/static/foo.js create mode 100644 test/build/precache_test.js diff --git a/package.json b/package.json index 29aed245..41b74f2a 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "author": "The Polymer Project Authors", "license": "BSD-3-Clause", "dependencies": { + "clone": "^1.0.2", "command-line-args": "^2.1.6", "command-line-commands": "^0.1.2", "css-slam": "^1.1.0", @@ -57,6 +58,7 @@ "eslint": "^2.9.0", "gulp-mocha": "^2.2.0", "sinon": "^1.17.4", + "temp": "^0.8.3", "typescript": "^1.8.10", "typings": "^0.8.1", "vinyl-fs-fake": "^1.1.0", diff --git a/src/build/build.ts b/src/build/build.ts index 92e4f542..7131cdc1 100644 --- a/src/build/build.ts +++ b/src/build/build.ts @@ -8,6 +8,7 @@ * subject to an additional IP rights grant found at http://polymer.github.io/PATENTS.txt */ +import clone = require('clone'); import * as fs from 'fs'; import * as gulp from 'gulp'; import * as gulpif from 'gulp-if'; @@ -24,7 +25,7 @@ import {Logger} from './logger'; import {optimize, OptimizeOptions} from './optimize'; import {waitForAll, compose, ForkedVinylStream} from './streams'; import {StreamResolver} from './stream-resolver'; -import {SWPreCacheTransform} from './sw-precache'; +import {generateServiceWorker, parsePreCacheConfig, SWConfig} from './sw-precache'; import {VulcanizeTransform} from './vulcanize'; // non-ES compatible modules @@ -40,13 +41,18 @@ export interface BuildOptions { swPrecacheConfig?: string; } -process.on('uncaughtException', (err) => { - console.log(`Caught exception: ${err}`); - console.error(err.stack); +process.on('uncaughtException', (error) => { + console.log(`Caught exception: ${error}`); + console.error(error.stack); +}); + +process.on('unhandledRejection', (error) => { + console.log(`Promise rejection: ${error}`); + console.error(error.stack); }); export function build(options?: BuildOptions): Promise { - return new Promise((resolve, _) => { + return new Promise((buildResolve, _) => { options = options || {}; let root = process.cwd(); let main = path.resolve(root, options.main || 'index.html'); @@ -108,50 +114,44 @@ export function build(options?: BuildOptions): Promise { .pipe(depsProject.split) .pipe(optimize(optimizeOptions)) .pipe(depsProject.rejoin) - .pipe(vfs.src(swPrecacheConfig, { - cwdbase: true, - allowEmpty: true, - passthrough: true - })); let allFiles = mergeStream(sourcesStream, depsStream); let serviceWorkerName = 'service-worker.js'; let unbundledPhase = new ForkedVinylStream(allFiles) - .pipe(new SWPreCacheTransform({ - root, - main, - buildRoot: 'build/unbundled', - serviceWorkerName, - configFileName: swPrecacheConfig - })) .pipe(vfs.dest('build/unbundled')) - // SWPreCacheTransform needs the deps from bundler after bundles are created - // therefore, give transform a promise that is resolved when bundler is done - let depsResolve: (value: string[]) => void; - let depsPromise = new Promise((resolve) => { - depsResolve = resolve; - }); - let bundledPhase = new ForkedVinylStream(allFiles) .pipe(bundler.bundle) - .on('end', () => { - // give the entry points and shared bundle to SWPreCacheTransform - let depsList = Array.from(bundler.entrypointFiles.keys()); - depsList.push(bundler.sharedBundleUrl); - depsResolve(depsList); - }) - .pipe(new SWPreCacheTransform({ + .pipe(vfs.dest('build/bundled')); + + let genSW = (buildRoot: string, deps: string[], swConfig: SWConfig) => { + return generateServiceWorker({ root, main, - buildRoot: 'build/bundled', - deps: depsPromise, - serviceWorkerName, - configFileName: swPrecacheConfig - })) - .pipe(vfs.dest('build/bundled')); + deps, + buildRoot, + swConfig: clone(swConfig), + serviceWorkerPath: path.join(root, buildRoot, serviceWorkerName) + }); + }; + + waitForAll([unbundledPhase, bundledPhase]).then(() => { + let unbundledDeps = Array.from(bundler.streamResolver.requestedUrls); + + let bundledDeps = Array.from(bundler.entrypointFiles.keys()); + bundledDeps.push(bundler.sharedBundleUrl); + + parsePreCacheConfig(swPrecacheConfig).then((swConfig) => { + Promise.all([ + genSW('build/unbundled', unbundledDeps, swConfig), + genSW('build/bundled', bundledDeps, swConfig) + ]).then(() => { + buildResolve(); + }); + }) + }); }); } diff --git a/src/build/sw-precache.ts b/src/build/sw-precache.ts index 0c459ade..5f107ef5 100644 --- a/src/build/sw-precache.ts +++ b/src/build/sw-precache.ts @@ -10,14 +10,14 @@ import * as fs from 'fs'; import * as path from 'path'; -import {Transform} from 'stream'; +import {PassThrough} from 'stream'; import File = require('vinyl'); // non-ES compatible modules const swPrecache = require('sw-precache'); const Module = require('module'); -interface SWConfig { +export interface SWConfig { cacheId?: string, directoryIndex?: string; dynamicUrlToDependencies?: { @@ -47,40 +47,48 @@ interface SWConfig { verbose?: boolean; } -function generateServiceWorker( - root: string, - main: string, - deps: string[], - buildRoot: string, - swConfig?: SWConfig - ): Promise { - swConfig = swConfig || {}; +export function generateServiceWorker(options: generateServiceWorkerOptions) +: Promise { + let swConfig = options.swConfig || {}; // strip root prefix, so buildRoot prefix can be added safely - deps = deps.map((p) => { - if (p.startsWith(root)) { - return p.substring(root.length); + let deps = options.deps.map((p) => { + if (p.startsWith(options.root)) { + return p.substring(options.root.length); } return p; }); - let mainHtml = main.substring(root.length); + let mainHtml = options.main.substring(options.root.length); let precacheFiles = new Set(swConfig.staticFileGlobs); deps.forEach((p) => precacheFiles.add(p)); precacheFiles.add(mainHtml); let precacheList = Array.from(precacheFiles); - precacheList = precacheList.map((p) => path.join(buildRoot, p)); + precacheList = precacheList.map((p) => path.join(options.buildRoot, p)); // swPrecache will determine the right urls by stripping buildRoot - swConfig.stripPrefix = buildRoot; + swConfig.stripPrefix = options.buildRoot; // static files will be pre-cached swConfig.staticFileGlobs = precacheList; - console.log(`Generating service worker for ${buildRoot}`); + console.log(`Generating service worker for ${options.buildRoot}`); - return swPrecache.generate(swConfig); + return swPrecache.write(options.serviceWorkerPath, swConfig); } -export interface SWPreCacheTransformOptions { +export function parsePreCacheConfig(configFile: string): Promise { + return new Promise((resolve, reject) => { + try { + let config: SWConfig = require(configFile); + resolve(config); + } catch(e) { + console.error(`Could not load sw-precache config from ${configFile}`); + console.error(e); + resolve(); + } + }); +} + +export interface generateServiceWorkerOptions { /** * folder containing files to be served by the service worker. */ @@ -94,100 +102,16 @@ export interface SWPreCacheTransformOptions { */ buildRoot: string; /** - * Name of the output service worker file. + * File path of the output service worker file. */ - serviceWorkerName: string; + serviceWorkerPath: string; /** - * Promise that returns the list of files to be cached by the service worker. - * - * If not given, all files streamed to SWPreCacheTransform will be put into - * the precache list, except for the file matching `configFileName`. + * List of files to be cached by the service worker, + * in addition to files found in `swConfig.staticFileGlobs` */ - deps?: Promise; + deps: string[]; /** - * Existing config file to use as a base for the serivce worker generation. - * - * This file will not be copied into the output bundles. + * Existing config to use as a base for the serivce worker generation. */ - configFileName?: string; -} - -export class SWPreCacheTransform extends Transform { - swConfig: SWConfig; - options: SWPreCacheTransformOptions; - fileSet: Set; - fullConfigFilePath: string; - - constructor(options: SWPreCacheTransformOptions) { - super({objectMode: true}); - this.options = options; - // if no given deps, collect all input files as deps - if (!options.deps) { - this.fileSet = new Set(); - } - if (options.configFileName) { - this.fullConfigFilePath = path.resolve( - options.root, - options.configFileName - ); - } - } - - _transform(file: File, encoding: string, callback: (error?, data?) => void): void { - if (file.path === this.fullConfigFilePath) { - try { - if (file.path.endsWith('js')) { - // `module._compile` is the heart of `require` - // http://fredkschott.com/post/2014/06/require-and-the-module-system/ - let m = new Module(file.path); - m._compile( - file.contents.toString(), - file.path - ); - this.swConfig = m.exports; - } else if (file.path.endsWith('json')) { - this.swConfig = JSON.parse(file.contents.toString()); - } - } catch(e) { - let cfn = this.options.configFileName; - console.error(`Could not load service worker config from ${cfn}`); - console.error(e); - } - callback(); - } else { - if (this.fileSet) { - this.fileSet.add(file.path); - } - callback(null, file); - } - } - - _flush(callback: (error?) => void) { - let promise: Promise; - if (this.fileSet) { - promise = Promise.resolve(Array.from(this.fileSet)); - } else { - promise = this.options.deps; - } - promise.then((deps) => { - return generateServiceWorker( - this.options.root, - this.options.main, - deps, - this.options.buildRoot, - this.swConfig - ); - }) - .then((config) => { - let file = new File({ - path: path.resolve( - this.options.root, - this.options.serviceWorkerName - ), - contents: new Buffer(config) - }); - this.push(file); - callback(); - }); - } + swConfig?: SWConfig; } diff --git a/test/build/precache/config.js b/test/build/precache/config.js new file mode 100644 index 00000000..4da51f07 --- /dev/null +++ b/test/build/precache/config.js @@ -0,0 +1,3 @@ +module.exports = { + staticFileGlobs: ['*'] +} diff --git a/test/build/precache/static/fizz.html b/test/build/precache/static/fizz.html new file mode 100644 index 00000000..f540bc68 --- /dev/null +++ b/test/build/precache/static/fizz.html @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/test/build/precache/static/foo.js b/test/build/precache/static/foo.js new file mode 100644 index 00000000..48684502 --- /dev/null +++ b/test/build/precache/static/foo.js @@ -0,0 +1 @@ +var x = 3; diff --git a/test/build/precache_test.js b/test/build/precache_test.js new file mode 100644 index 00000000..d0891f06 --- /dev/null +++ b/test/build/precache_test.js @@ -0,0 +1,84 @@ +/** + * @license + * Copyright (c) 2016 The Polymer Project Authors. All rights reserved. + * This code may only be used under the BSD style license found at http://polymer.github.io/LICENSE.txt + * The complete set of authors may be found at http://polymer.github.io/AUTHORS.txt + * The complete set of contributors may be found at http://polymer.github.io/CONTRIBUTORS.txt + * Code distributed by Google as part of the polymer project is also + * subject to an additional IP rights grant found at http://polymer.github.io/PATENTS.txt + */ + +'use strict'; + +const assert = require('chai').assert; +const fs = require('fs'); +const path = require('path'); +const temp = require('temp').track(); +const vfs = require('vinyl-fs-fake'); + +const precache = require('../../lib/build/sw-precache'); + +suite('sw-precache', () => { + const configFile = path.resolve(__dirname, 'precache', 'config.js'); + suite('parsing', () => { + test('js file', (done) => { + precache.parsePreCacheConfig(configFile).then((config) => { + assert.ok(config); + assert.property(config, 'staticFileGlobs'); + done(); + }) + }); + }); + + suite('generation', () => { + let buildRoot; + setup((done) => { + temp.mkdir('polymer-cli', (err, dir) => { + if (err) { + return done(err); + } + buildRoot = dir; + vfs.src(path.join(__dirname, 'precache/static/*')) + .pipe(vfs.dest(dir)) + .on('finish', () => done()); + } + ); + }); + + teardown((done) => { + temp.cleanup(done) + }); + + test('without config', (done) => { + precache.generateServiceWorker({ + root: path.resolve(__dirname, 'precache/static'), + main: path.resolve(__dirname, 'precache/static/fizz.html'), + buildRoot, + deps: [], + serviceWorkerPath: path.join(buildRoot, 'service-worker.js') + }).then(() => { + let content = fs.readFileSync(path.join(buildRoot, 'service-worker.js'), 'utf-8'); + assert.include(content, '/fizz.html', 'main file should be present'); + done(); + }); + }); + + test('with config', (done) => { + precache.parsePreCacheConfig(configFile).then((config) => { + return precache.generateServiceWorker({ + root: path.resolve(__dirname, 'precache/static'), + main: path.resolve(__dirname, 'precache/static/fizz.html'), + buildRoot, + deps: [], + swConfig: config, + serviceWorkerPath: path.join(buildRoot, 'service-worker.js') + }); + }).then(() => { + let content = fs.readFileSync(path.join(buildRoot, 'service-worker.js'), 'utf-8'); + assert.include(content, '/fizz.html', 'main file should be present'); + assert.include(content, '/foo.js', 'staticFileGlobs should match foo.js'); + done(); + }); + }); + }) +}); diff --git a/typings.json b/typings.json index 7fbdd2a0..b48c57e7 100644 --- a/typings.json +++ b/typings.json @@ -1,5 +1,6 @@ { "ambientDependencies": { + "clone": "registry:dt/clone#0.1.11+20160317120654", "express": "registry:dt/express#4.0.0+20160317120654", "express-serve-static-core": "registry:dt/express-serve-static-core#0.0.0+20160322035842", "glob": "registry:dt/glob#5.0.10+20160317120654",