Contribution of a generator implementation collecting tracing informa…

…tion and composing a sourceMap

package.json

@@ -19,9 +19,11 @@
     "esbuild": "^0.15.8",
     "langium": "~2.0.0",
     "langium-arithmetics-dsl": "~2.0.0",
+    "js-base64": "^3.7.4",
     "monaco-editor-workers": "^0.34.2",
     "monaco-editor-wrapper": "^1.6.0",
     "shx": "^0.3.4",
+    "source-map": "^0.7.4",
     "vscode-languageserver": "~8.0.0",
     "concurrently": "^7.6.0",
     "http-server": "^14.1.1"

src/arithmeticsEditor/index.ts

@@ -9,7 +9,7 @@ import { expandToString } from 'langium';
 import { buildWorkerDefinition } from 'monaco-editor-workers';
 import { CodeEditorConfig, MonacoEditorLanguageClientWrapper, WorkerConfigOptions } from 'monaco-editor-wrapper';
 import { Diagnostic, DiagnosticSeverity, NotificationType } from 'vscode-languageserver/browser.js';
-import { generate } from '../generator/generator-with-nodes.js';
+import { generate } from '../generator/generator-with-tracing.js';
 
 type DocumentChange = { uri: string, content: string, diagnostics: Diagnostic[] };
 

src/generator/generator-with-tracing.ts

@@ -0,0 +1,150 @@
+/******************************************************************************
+ * Copyright 2023 TypeFox GmbH
+ * This program and the accompanying materials are made available under the
+ * terms of the MIT License, which is available in the project root.
+ ******************************************************************************/
+
+import { Base64 } from 'js-base64';
+import { AstNodeWithTextRegion, DefaultAstNodeLocator, DefaultJsonSerializer, DefaultNameProvider, expandToNode, expandTracedToNode, Generated, GeneratorNode, joinToNode, joinTracedToNode, NL, toStringAndTrace, traceToNode, TreeStreamImpl } from 'langium';
+import { Definition, Evaluation, Expression, isBinaryExpression, isDefinition, isFunctionCall, isNumberLiteral, Module, Statement } from 'langium-arithmetics-dsl/api';
+import { SourceMapGenerator, StartOfSourceMap } from 'source-map/lib/source-map-generator.js'; // importing directly from 'source-map' affects bundling, as the 'source-map-consumer' is then also loaded and refers to node.js builtins
+
+export function generate( {uri, content}: { uri: string, content: string | Module }): string {
+    const filename = uri.substring(uri.lastIndexOf('/'))
+
+    if (typeof content === 'string') {
+        content = deserializeModule(content);
+    }
+
+    let { text, trace } = toStringAndTrace(
+        generateModule(content)
+    );
+
+    const mapper: SourceMapGenerator = new SourceMapGenerator(<StartOfSourceMap>{ file: filename + '.js' });
+    const sourceDefinitionText = (content as AstNodeWithTextRegion).$sourceText ?? '';
+    mapper.setSourceContent(filename, sourceDefinitionText/*.replace(/DEF/ig, 'var')*/ ?? '<Source text not available>');
+
+    new TreeStreamImpl(trace, r => r.children ?? [], { includeRoot: true }).forEach(r => {
+        if (!r.sourceRegion
+            || !r.targetRegion
+            || r.children?.[0].targetRegion.offset === r.targetRegion.offset /* if the first child starts at the same position like this (potentially encompassing) region, skip this on and continue with the child(ren) */
+        ) {
+            return;
+        }
+
+        const sourceStart = r.sourceRegion.range?.start;
+        const targetStart = r.targetRegion.range?.start;
+
+        const sourceEnd = r.sourceRegion?.range?.end;
+        const sourceText = sourceEnd && sourceDefinitionText.length >= r.sourceRegion.end
+            ? sourceDefinitionText.substring(r.sourceRegion.offset, r.sourceRegion.end) : ''
+
+        sourceStart && targetStart && mapper.addMapping({
+            original:  { line: sourceStart.line + 1, column: sourceStart.character },
+            generated: { line: targetStart.line + 1, column: targetStart.character },
+            source: filename,
+            name: /^[A-Za-z_]$/.test(sourceText) ? sourceText.toLowerCase() : undefined
+        });
+
+        // const sourceEnd = r.sourceRegion?.range?.end;
+        // const sourceText = sourceEnd && sourceDefinitionText.length >= r.sourceRegion.end
+        //     ? sourceDefinitionText.substring(r.sourceRegion.offset, r.sourceRegion.end) : ''
+        const targetEnd = r.targetRegion?.range?.end;
+        const targetText = targetEnd && text.length >= r.targetRegion.end
+            ? text.substring(r.targetRegion.offset, r.targetRegion.end) : ''
+
+        sourceEnd && targetEnd && !r.children && sourceText && targetText
+                && !/\s/.test(sourceText) && !/\s/.test(targetText)
+                && mapper.addMapping({
+            original:  { line: sourceEnd.line + 1, column: sourceEnd.character },
+            generated: { line: targetEnd.line + 1, column: targetEnd.character},
+            source: filename
+        });
+    });
+
+    const sourceMap = mapper.toString();
+    if (sourceMap) {
+        text += `\n\n//# sourceMappingURL=data:application/json;charset=urf-8;base64,${Base64.encode(sourceMap)}`;
+    }
+    return text;
+}
+
+function generateModule(root: Module): GeneratorNode {
+    return expandToNode`
+        "use strict";
+        (() => {
+            ${generateModuleContent(root)}
+        })
+    `;
+}
+
+const lastComputableExpressionValueVarName = 'lastComputableExpressionValue';
+
+function generateModuleContent(module: Module): Generated {
+    return expandTracedToNode(module)`
+        let ${lastComputableExpressionValueVarName};
+        ${ joinTracedToNode(module, 'statements')(module.statements, generateStatement, { appendNewLineIfNotEmpty: true }) }
+
+        return ${lastComputableExpressionValueVarName};
+    `;
+}
+
+function generateStatement(stmt: Statement): Generated {
+    if (isDefinition(stmt))
+        return generateDefinition(stmt);
+    else
+        return generateEvaluation(stmt);
+}
+
+function generateDefinition(def: Definition): Generated {
+    return def.args && def.args.length ?
+        expandTracedToNode(def)`
+            const ${traceToNode(def, 'name')(def.name)} = (${joinTracedToNode(def, 'args')(def.args, arg => traceToNode(arg)(arg.name), { separator: ', '})}) => ${generateExpression(def.expr)};
+        ` : expandTracedToNode(def)`
+            ${traceToNode(def)('const')} ${traceToNode(def, 'name')(def.name)} = ${lastComputableExpressionValueVarName} = ${generateExpression(def.expr)};
+        `;
+}
+
+function generateEvaluation(evaln: Evaluation): Generated {
+    return expandTracedToNode(evaln)`
+        ${lastComputableExpressionValueVarName} = ${generateExpression(evaln.expression)};
+    `;
+}
+
+function generateExpression(expr: Expression): Generated {
+
+    if (isNumberLiteral(expr)) {
+        return traceToNode(expr, 'value')( expr.value.toString() );
+
+    } else if (isBinaryExpression(expr)) {
+        const leftAsIs  = isNumberLiteral(expr.left)  || isFunctionCall(expr.left);
+        const rightAsIs = isNumberLiteral(expr.right) || isFunctionCall(expr.right);
+        const left  = leftAsIs  ? generateExpression(expr.left)  : expandTracedToNode(expr, "left" )`(${generateExpression(expr.left )})`;
+        const right = rightAsIs ? generateExpression(expr.right) : expandTracedToNode(expr, "right")`(${generateExpression(expr.right)})`;
+        return expandTracedToNode(expr)`
+            ${left} ${traceToNode(expr, 'operator')(expr.operator)} ${right}
+        `;
+
+    } else {
+        return traceToNode(expr)(
+            parent => parent
+                .appendTraced(expr, 'func')(expr.func.ref?.name)
+                .appendTracedTemplateIf(!!expr.args.length, expr, 'args')`
+                    (
+                        ${joinToNode(expr.args, generateExpression, { separator: ', ' })}
+                    )
+                `
+        );
+    };
+}
+
+function deserializeModule(input: string): Module {
+    return new DefaultJsonSerializer({
+        workspace: {
+            AstNodeLocator: new DefaultAstNodeLocator()
+        },
+        references: {
+            NameProvider: new DefaultNameProvider()
+        }
+    } as any).deserialize(input) as Module;
+}

src/generator/source-map.d.ts

@@ -0,0 +1,71 @@
+/******************************************************************************
+ * Copyright 2023 TypeFox GmbH
+ * This program and the accompanying materials are made available under the
+ * terms of the MIT License, which is available in the project root.
+ ******************************************************************************/
+
+// We shouldn't import 'SourceMapGenerator' from the 'source-map' package directly
+//  as that also loads 'source-map-consumer.js' that transitively refers to node.js builtins.
+// That affects bundling: Although transitive depdendencies on node.js builtins can be marked as external,
+//  esbuild's tree shaking doesn't drop the 'SourceMapConsumer' and we just don't need it.
+//
+// Therefore, we just replicated the required type defs of 'SourceMapGenerator' and friends in here. ;-)
+
+declare module 'source-map/lib/source-map-generator.js' {
+// original license: BSD-3-Clause
+// original header:
+    // Type definitions for source-map 0.7
+    // Project: https://github.com/mozilla/source-map
+    // Definitions by: Morten Houston Ludvigsen <https://github.com/MortenHoustonLudvigsen>,
+    //                 Ron Buckton <https://github.com/rbuckton>,
+    //                 John Vilk <https://github.com/jvilk>
+    // Definitions: https://github.com/mozilla/source-map
+
+    export interface StartOfSourceMap {
+        file?: string;
+        sourceRoot?: string;
+        skipValidation?: boolean;
+    }
+
+    export interface Mapping {
+        generated: Position;
+        original: Position;
+        source: string;
+        name?: string;
+    }
+
+    export interface RawSourceMap {
+        version: number;
+        sources: string[];
+        names: string[];
+        sourceRoot?: string;
+        sourcesContent?: string[];
+        mappings: string;
+        file: string;
+    }
+
+    export class SourceMapGenerator {
+        constructor(startOfSourceMap?: StartOfSourceMap);
+
+        /**
+         * Add a single mapping from original source line and column to the generated
+         * source's line and column for this source map being created. The mapping
+         * object should have the following properties:
+         *
+         *   - generated: An object with the generated line and column positions.
+         *   - original: An object with the original line and column positions.
+         *   - source: The original source file (relative to the sourceRoot).
+         *   - name: An optional original token name for this mapping.
+         */
+        addMapping(mapping: Mapping): void;
+
+        /**
+         * Set the source content for a source file.
+         */
+        setSourceContent(sourceFile: string, sourceContent: string): void;
+
+        toString(): string;
+
+        toJSON(): RawSourceMap;
+    }
+}