ci: update references before each build

Build automatically checks and updates references before running tsc. Add 2 new npm scripts: - `prepare:references`: updates the references, if required. - `test:references`: fails if references are out of sync. Signed-off-by: Paul Maréchal <[email protected]>
logicmoo · Feb 11, 2020 · 3fc9b7a · 3fc9b7a
1 parent 329fd08
commit 3fc9b7a
Show file tree

Hide file tree

Showing 4 changed files with 157 additions and 45 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -105,7 +105,7 @@ install:
   - THEIA_SKIP_NPM_PREPARE=1 yarn install --skip-integrity-check
   - npx electron-replace-ffmpeg # re-download library (in case it was cached)
   - npx electron-codecs-test # test library
-  - yarn prepare
+  - yarn prepare # actually build
   - scripts/check_git_status.sh
 script:
   - travis_retry yarn test:theia

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,10 @@
 ## v0.16.0
 
 - [core] added a new React-based dialog type `ReactDialog` [#6855](https://github.com/eclipse-theia/theia/pull/6855)
+- [repo] added 2 new npm scripts:
+  - `test:references`: fails if typescript references are out of sync.
+  - `prepare:references`: updates typescript references, if required.
+- [repo] the `prepare` script now updates typescript references.
 
 Breaking changes:
 

diff --git a/package.json b/package.json
@@ -42,8 +42,9 @@
   "scripts": {
     "preinstall": "node-gyp install",
     "postinstall": "node scripts/post-install.js",
-    "prepare": "yarn prepare:travis && yarn prepare:build && yarn prepare:hoisting && yarn download:plugins",
+    "prepare": "yarn prepare:travis && yarn prepare:references && yarn prepare:build && yarn prepare:hoisting && yarn download:plugins",
     "prepare:travis": "node scripts/prepare-travis.js",
+    "prepare:references": "node scripts/compile-references.js",
     "prepare:build": "yarn build && run lint && run build \"@theia/example-*\" --stream --parallel",
     "prepare:hoisting": "theia check:hoisted -s",
     "clean": "yarn lint:clean && node scripts/run-reverse-topo.js yarn clean",
@@ -53,7 +54,8 @@
     "lint:clean": "rimraf .eslintcache",
     "lint:oneshot": "node --max-old-space-size=4096 node_modules/eslint/bin/eslint.js --cache=true \"{dev-packages,packages,examples}/**/*.{ts,tsx}\"",
     "docs": "rimraf gh-pages/docs/next && typedoc --tsconfig configs/typedoc-tsconfig.json --options configs/typedoc.json",
-    "test": "yarn test:theia && yarn test:electron && yarn test:browser",
+    "test": "yarn test:references && yarn test:theia && yarn test:electron && yarn test:browser",
+    "test:references": "node scripts/compile-references --dry-run",
     "test:theia": "run test \"@theia/!(example-)*\" --stream --concurrency=1",
     "test:browser": "yarn rebuild:browser && run test \"@theia/example-browser\"",
     "test:electron": "yarn rebuild:electron && run test \"@theia/example-electron\"",

diff --git a/scripts/compile-references.js b/scripts/compile-references.js
@@ -1,3 +1,6 @@
+// @ts-check
+'use-strict';
+
 /********************************************************************************
  * Copyright (C) 2020 Ericsson and others.
  *
@@ -13,7 +16,6 @@
  *
  * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
  ********************************************************************************/
-'use-strict';
 
 /**
  * This script generates tsconfig references between our workspaces, it also
@@ -23,62 +25,93 @@
  * only when required, but it cannot infer workspaces by itself, it has to be
  * explicitly defined [1].
  *
+ * You can do a dry run using the cli flag `--dry-run`, the script will exit
+ * with a code different from zero if something needed to be updated.
+ *
  * [1]: https://www.typescriptlang.org/docs/handbook/project-references.html
  */
 
-// @ts-check
 
 const cp = require('child_process');
 const path = require('path').posix;
 const fs = require('fs');
 
-const CWD = path.join(__dirname, '..');
+const ROOT = path.join(__dirname, '..');
+
+const DRY_RUN = popFlag(process.argv, '--dry-run');
 
-const FORCE_REWRITE = Boolean(process.env['THEIA_REPO_FORCE_REWRITE']);
+const FORCE_REWRITE = popFlag(process.argv, '--force-rewrite') && !DRY_RUN;
 
 /** @type {{ [packageName: string]: YarnWorkspace }} */
 const YARN_WORKSPACES = JSON.parse(cp.execSync('yarn --silent workspaces info').toString());
 
+// Add the package name inside each package object.
+for (const [packageName, yarnWorkspace] of Object.entries(YARN_WORKSPACES)) {
+    yarnWorkspace.name = packageName;
+}
+
 /** @type {YarnWorkspace} */
 const THEIA_MONOREPO = {
+    name: '@theia/monorepo',
     workspaceDependencies: Object.keys(YARN_WORKSPACES),
-    location: '.',
+    location: ROOT,
 };
 
-// Configure all `compile.tsconfig.json` files of this monorepo
-for (const packageName of Object.keys(YARN_WORKSPACES)) {
-    const workspacePackage = YARN_WORKSPACES[packageName];
-    const tsconfigCompilePath = path.join(CWD, workspacePackage.location, 'compile.tsconfig.json');
-    const references = getTypescriptReferences(workspacePackage);
-    configureTypeScriptCompilation(tsconfigCompilePath, references);
-}
+try {
+    let rewriteRequired = false;
+    let result = false;
 
-{
-    const tsconfigCompilePath = path.join(CWD, 'configs', 'root-compilation.tsconfig.json');
-    const references = getTypescriptReferences({
-        workspaceDependencies: Object.keys(YARN_WORKSPACES),
-        location: path.join(CWD, 'configs'),
-    });
-    configureTypeScriptCompilation(tsconfigCompilePath, references);
+    // Configure all `compile.tsconfig.json` files of this monorepo
+    for (const packageName of Object.keys(YARN_WORKSPACES)) {
+        const workspacePackage = YARN_WORKSPACES[packageName];
+        const tsconfigCompilePath = path.join(ROOT, workspacePackage.location, 'compile.tsconfig.json');
+        const references = getTypescriptReferences(workspacePackage);
+        result = configureTypeScriptCompilation(workspacePackage, tsconfigCompilePath, references);
+        rewriteRequired = rewriteRequired || result;
+    }
+
+    // Configure our root compilation configuration, living inside `configs/root-compilation.tsconfig.json`.
+    const configsFolder = path.join(ROOT, 'configs');
+    const tsconfigCompilePath = path.join(configsFolder, 'root-compilation.tsconfig.json');
+    const references = getTypescriptReferences(THEIA_MONOREPO, configsFolder);
+    result = configureTypeScriptCompilation(THEIA_MONOREPO, tsconfigCompilePath, references);
+    rewriteRequired = rewriteRequired || result;
 
     // Configure the root `tsconfig.json` for code navigation using `tsserver`.
-    const tsconfigNavPath = path.join(CWD, 'tsconfig.json');
-    configureTypeScriptNavigation(tsconfigNavPath);
+    const tsconfigNavPath = path.join(ROOT, 'tsconfig.json');
+    result = configureTypeScriptNavigation(THEIA_MONOREPO, tsconfigNavPath);
+    rewriteRequired = rewriteRequired || result;
+
+    // CI will be able to tell if references got changed by looking at the exit code.
+    if (rewriteRequired) {
+        if (DRY_RUN) {
+            // Running a dry run usually only happens when a developer or CI runs the tests, so we only print the help then.
+            console.error('TypeScript references seem to be out of sync, run "yarn prepare:references" to fix.');
+            process.exitCode = 1;
+        } else {
+            console.warn('TypeScript references were out of sync and got updated.');
+        }
+    }
+
+} catch (error) {
+    console.error(error);
+    process.exitCode = 1;
 }
 
 /**
  * @param {YarnWorkspace} requestedPackage
+ * @param {string} [overrideLocation] affects how relative paths are computed.
  * @returns {string[]} project references for `requestedPackage`.
  */
-function getTypescriptReferences(requestedPackage) {
+function getTypescriptReferences(requestedPackage, overrideLocation) {
     const references = [];
     for (const dependency of requestedPackage.workspaceDependencies || []) {
         const depWorkspace = YARN_WORKSPACES[dependency];
         const depConfig = path.join(depWorkspace.location, 'compile.tsconfig.json');
         if (!fs.existsSync(depConfig)) {
             continue;
         }
-        const relativePath = path.relative(requestedPackage.location, depWorkspace.location);
+        const relativePath = path.relative(overrideLocation || requestedPackage.location, depWorkspace.location);
         references.push(relativePath);
     }
     return references;
@@ -88,22 +121,28 @@ function getTypescriptReferences(requestedPackage) {
  * Wires a given compilation tsconfig file according to the provided references.
  * This allows TypeScript to operate in build mode.
  *
+ * @param {YarnWorkspace} targetPackage for debug purpose.
  * @param {string} tsconfigPath path to the tsconfig file to edit.
  * @param {string[]} references list of paths to the related project roots.
+ * @returns {boolean} rewrite was needed.
  */
-function configureTypeScriptCompilation(tsconfigPath, references) {
+function configureTypeScriptCompilation(targetPackage, tsconfigPath, references) {
     if (!fs.existsSync(tsconfigPath)) {
-        return;
+        return false;
     }
-    let needRewrite = false;
     const tsconfigJson = readJsonFile(tsconfigPath);
+
+    let needRewrite = FORCE_REWRITE;
+
     if (!tsconfigJson.compilerOptions) {
         // Somehow no `compilerOptions` literal is defined.
         tsconfigJson.compilerOptions = {
             composite: true,
             rootDir: 'src',
             outDir: 'lib',
         };
+        needRewrite = true;
+
     } else if (!tsconfigJson.compilerOptions.composite) {
         // `compilerOptions` is missing the `composite` literal.
         tsconfigJson.compilerOptions = {
@@ -112,43 +151,83 @@ function configureTypeScriptCompilation(tsconfigPath, references) {
         };
         needRewrite = true;
     }
-    const currentReferences = new Set((tsconfigJson['references'] || []).map(reference => reference.path));
-    for (const reference of references) {
-        const tsconfigReference = path.join(reference, 'compile.tsconfig.json');
-        if (!currentReferences.has(tsconfigReference)) {
-            currentReferences.add(tsconfigReference);
+
+    /** @type {string[]} */
+    const tsconfigReferences = references
+        .map(reference => path.join(reference, 'compile.tsconfig.json'));
+
+    /** @type {string[]} */
+    const currentReferences = (tsconfigJson['references'] || [])
+        // We will work on a set of paths, easier to handle than objects.
+        .map(reference => reference.path)
+        // Remove any invalid reference (maybe outdated).
+        .filter((referenceRelativePath, index, self) => {
+            if (!tsconfigReferences.includes(referenceRelativePath)) {
+                // Found a reference that wasn't automatically computed, will remove.
+                console.warn(`error: ${targetPackage.name} untracked reference: ${referenceRelativePath}`);
+                needRewrite = true;
+                return false; // remove
+            }
+            if (self.indexOf(referenceRelativePath) !== index) {
+                // Remove duplicates.
+                console.error(`error: ${targetPackage.name} duplicate reference: ${referenceRelativePath}`);
+                needRewrite = true;
+                return false; // remove
+            }
+            const referencePath = path.join(path.dirname(tsconfigPath), referenceRelativePath);
+            try {
+                const referenceStat = fs.statSync(referencePath);
+                if (referenceStat.isDirectory() && fs.statSync(path.join(referencePath, 'tsconfig.json')).isFile()) {
+                    return true; // keep
+                } else if (referenceStat.isFile()) { // still could be something else than a tsconfig, but good enough.
+                    return true; // keep
+                }
+            } catch {
+                // passthrough
+            }
+            console.error(`error: ${targetPackage.name} invalid reference: ${referenceRelativePath}`);
+            needRewrite = true;
+            return false; // remove
+        });
+
+    for (const tsconfigReference of tsconfigReferences) {
+        if (!currentReferences.includes(tsconfigReference)) {
+            console.error(`error: ${targetPackage.name} missing reference: ${tsconfigReference}`);
+            currentReferences.push(tsconfigReference);
             needRewrite = true;
         }
     }
-    if (FORCE_REWRITE || needRewrite) {
-        tsconfigJson.references = [];
-        for (const reference of currentReferences) {
-            tsconfigJson.references.push({
-                path: reference,
-            });
-        }
+    if (!DRY_RUN && needRewrite) {
+        tsconfigJson.references = currentReferences.map(path => ({ path }));
         const content = JSON.stringify(tsconfigJson, undefined, 2);
         fs.writeFileSync(tsconfigPath, content + '\n');
+        console.warn(`info: ${tsconfigPath} updated.`);
     }
+    return needRewrite;
 }
 
 /**
  * Wire the root `tsconfig.json` to map scoped import to real location in the monorepo.
  * This setup is a shim for the TypeScript language server to provide cross-package navigation.
  * Compilation is done via `compile.tsconfig.json` files.
  *
+ * @param {YarnWorkspace} targetPackage for debug purpose.
  * @param {string} tsconfigPath
+ * @returns {boolean} rewrite was needed.
  */
-function configureTypeScriptNavigation(tsconfigPath) {
-    let needRewrite = false;
+function configureTypeScriptNavigation(targetPackage, tsconfigPath) {
     const tsconfigJson = readJsonFile(tsconfigPath);
+
+    let needRewrite = FORCE_REWRITE;
+
     if (typeof tsconfigJson.compilerOptions === 'undefined') {
         // Somehow no `compilerOptions` literal is defined.
         tsconfigJson.compilerOptions = {
             baseUrl: '.',
             paths: {},
         };
         needRewrite = true;
+
     } else if (typeof tsconfigJson.compilerOptions.paths === 'undefined') {
         // `compilerOptions` is missing the `paths` literal.
         tsconfigJson.compilerOptions = {
@@ -157,8 +236,10 @@ function configureTypeScriptNavigation(tsconfigPath) {
         };
         needRewrite = true;
     }
+
     /** @type {{ [prefix: string]: string[] }} */
     const currentPaths = tsconfigJson.compilerOptions.paths;
+
     for (const packageName of THEIA_MONOREPO.workspaceDependencies) {
         const depWorkspace = YARN_WORKSPACES[packageName];
 
@@ -169,25 +250,49 @@ function configureTypeScriptNavigation(tsconfigPath) {
 
         const depSrcPath = path.join(depWorkspace.location, 'src');
         const depConfigPath = path.join(depWorkspace.location, 'compile.tsconfig.json');
+
         if (fs.existsSync(depConfigPath) && fs.existsSync(depSrcPath)) {
             // If it is a TypeScript dependency, map `lib` imports to our local sources in `src`.
             const depConfigJson = readJsonFile(depConfigPath);
             originalImportPath = `${packageName}/${depConfigJson.compilerOptions.outDir}/*`;
             mappedFsPath = path.relative(THEIA_MONOREPO.location, path.join(depSrcPath, '*'));
+
         } else {
             // I don't really know what to do here, simply point to our local package root.
             originalImportPath = `${packageName}/*`;
             mappedFsPath = path.relative(THEIA_MONOREPO.location, path.join(depWorkspace.location, '*'));
         }
 
-        if (typeof currentPaths[originalImportPath] === 'undefined' || currentPaths[originalImportPath][0] !== mappedFsPath) {
+        /** @type {string[] | undefined} */
+        const currentFsPaths = currentPaths[originalImportPath];
+
+        if (typeof currentFsPaths === 'undefined' || currentFsPaths.length !== 1 || currentFsPaths[0] !== mappedFsPath) {
+            console.error(`error: ${targetPackage.name} invalid mapped path: ${JSON.stringify({ [originalImportPath]: currentFsPaths })}`);
             currentPaths[originalImportPath] = [mappedFsPath];
             needRewrite = true;
         }
     }
-    if (FORCE_REWRITE || needRewrite) {
+    if (!DRY_RUN && needRewrite) {
         const content = JSON.stringify(tsconfigJson, undefined, 2);
         fs.writeFileSync(tsconfigPath, content + '\n');
+        console.warn(`info: ${tsconfigPath} updated.`);
+    }
+    return needRewrite;
+}
+
+/**
+ *
+ * @param {string[]} argv
+ * @param {string} flag
+ * @returns {boolean}
+ */
+function popFlag(argv, flag) {
+    const flagIndex = argv.indexOf(flag)
+    if (flagIndex !== -1) {
+        argv.splice(flagIndex, 1);
+        return true;
+    } else {
+        return false;
     }
 }
 
@@ -206,6 +311,7 @@ function readJsonFile(filePath) {
 
 /**
  * @typedef YarnWorkspace
+ * @property {string} name
  * @property {string} location
  * @property {string[]} workspaceDependencies
  */