diff --git a/package-lock.json b/package-lock.json index b1af126..407105f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,11 +12,13 @@ "concurrently": "^7.6.0", "esbuild": "^0.15.8", "http-server": "^14.1.1", + "js-base64": "^3.7.4", "langium": "~2.0.0", "langium-arithmetics-dsl": "~2.0.0", "monaco-editor-workers": "^0.34.2", "monaco-editor-wrapper": "^1.6.0", "shx": "^0.3.4", + "source-map": "^0.7.4", "typescript": "^5.0.0", "vscode-languageserver": "~8.0.0" } @@ -1007,6 +1009,12 @@ "node": ">=8" } }, + "node_modules/js-base64": { + "version": "3.7.5", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.5.tgz", + "integrity": "sha512-3MEt5DTINKqfScXKfJFrRbxkrnk2AxPWGBL/ycjz4dK8iqiSJ06UxD8jh8xuh6p10TX4t2+7FsBYVxxQbMg+qA==", + "dev": true + }, "node_modules/langium": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/langium/-/langium-2.0.2.tgz", @@ -1419,6 +1427,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/source-map": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", + "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, "node_modules/spawn-command": { "version": "0.0.2-1", "resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2-1.tgz", @@ -2343,6 +2360,12 @@ "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true }, + "js-base64": { + "version": "3.7.5", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.5.tgz", + "integrity": "sha512-3MEt5DTINKqfScXKfJFrRbxkrnk2AxPWGBL/ycjz4dK8iqiSJ06UxD8jh8xuh6p10TX4t2+7FsBYVxxQbMg+qA==", + "dev": true + }, "langium": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/langium/-/langium-2.0.2.tgz", @@ -2654,6 +2677,12 @@ "object-inspect": "^1.9.0" } }, + "source-map": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", + "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==", + "dev": true + }, "spawn-command": { "version": "0.0.2-1", "resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2-1.tgz", diff --git a/package.json b/package.json index 9dfbbbf..8cfd946 100644 --- a/package.json +++ b/package.json @@ -19,9 +19,11 @@ "esbuild": "^0.15.8", "langium": "~2.0.0", "langium-arithmetics-dsl": "~2.0.0", + "js-base64": "^3.7.4", "monaco-editor-workers": "^0.34.2", "monaco-editor-wrapper": "^1.6.0", "shx": "^0.3.4", + "source-map": "^0.7.4", "vscode-languageserver": "~8.0.0", "concurrently": "^7.6.0", "http-server": "^14.1.1" diff --git a/src/arithmeticsEditor/index.ts b/src/arithmeticsEditor/index.ts index caa5704..2b2b199 100644 --- a/src/arithmeticsEditor/index.ts +++ b/src/arithmeticsEditor/index.ts @@ -9,7 +9,7 @@ import { expandToString } from 'langium'; import { buildWorkerDefinition } from 'monaco-editor-workers'; import { CodeEditorConfig, MonacoEditorLanguageClientWrapper, WorkerConfigOptions } from 'monaco-editor-wrapper'; import { Diagnostic, DiagnosticSeverity, NotificationType } from 'vscode-languageserver/browser.js'; -import { generate } from '../generator/generator-with-nodes.js'; +import { generate } from '../generator/generator-with-tracing.js'; type DocumentChange = { uri: string, content: string, diagnostics: Diagnostic[] }; diff --git a/src/generator/generator-with-tracing.ts b/src/generator/generator-with-tracing.ts new file mode 100644 index 0000000..f196657 --- /dev/null +++ b/src/generator/generator-with-tracing.ts @@ -0,0 +1,153 @@ +/****************************************************************************** + * Copyright 2023 TypeFox GmbH + * This program and the accompanying materials are made available under the + * terms of the MIT License, which is available in the project root. + ******************************************************************************/ + +import { Base64 } from 'js-base64'; +import { AstNodeWithTextRegion, DefaultAstNodeLocator, DefaultJsonSerializer, DefaultNameProvider, expandToNode, expandTracedToNode, Generated, GeneratorNode, joinToNode, joinTracedToNode, NL, toStringAndTrace, traceToNode, TreeStreamImpl } from 'langium'; +import { Definition, Evaluation, Expression, isBinaryExpression, isDefinition, isFunctionCall, isNumberLiteral, Module, Statement } from 'langium-arithmetics-dsl/api'; +import { SourceMapGenerator, StartOfSourceMap } from 'source-map/lib/source-map-generator.js'; // importing directly from 'source-map' affects bundling, as the 'source-map-consumer' is then also loaded and refers to node.js builtins + +export function generate( {uri, content}: { uri: string, content: string | Module }): string { + const filename = uri.substring(uri.lastIndexOf('/')) + + if (typeof content === 'string') { + content = deserializeModule(content); + } + + let { text, trace } = toStringAndTrace( + generateModule(content) + ); + + const mapper: SourceMapGenerator = new SourceMapGenerator({ file: filename + '.js' }); + const sourceDefinitionText = (content as AstNodeWithTextRegion).$sourceText ?? ''; + mapper.setSourceContent(filename, sourceDefinitionText/*.replace(/DEF/ig, 'var')*/ ?? ''); + + new TreeStreamImpl(trace, r => r.children ?? [], { includeRoot: true }).forEach(r => { + if (!r.sourceRegion + || !r.targetRegion + || r.children?.[0].targetRegion.offset === r.targetRegion.offset /* if the first child starts at the same position like this (potentially encompassing) region, skip this on and continue with the child(ren) */ + ) { + return; + } + + const sourceStart = r.sourceRegion.range?.start; + const targetStart = r.targetRegion.range?.start; + + const sourceEnd = r.sourceRegion?.range?.end; + const sourceText = sourceEnd && sourceDefinitionText.length >= r.sourceRegion.end + ? sourceDefinitionText.substring(r.sourceRegion.offset, r.sourceRegion.end) : '' + + sourceStart && targetStart && mapper.addMapping({ + original: { line: sourceStart.line + 1, column: sourceStart.character }, + generated: { line: targetStart.line + 1, column: targetStart.character }, + source: filename, + name: /^[A-Za-z_]$/.test(sourceText) ? sourceText.toLowerCase() : undefined + }); + + // const sourceEnd = r.sourceRegion?.range?.end; + // const sourceText = sourceEnd && sourceDefinitionText.length >= r.sourceRegion.end + // ? sourceDefinitionText.substring(r.sourceRegion.offset, r.sourceRegion.end) : '' + const targetEnd = r.targetRegion?.range?.end; + const targetText = targetEnd && text.length >= r.targetRegion.end + ? text.substring(r.targetRegion.offset, r.targetRegion.end) : '' + + sourceEnd && targetEnd && !r.children && sourceText && targetText + && !/\s/.test(sourceText) && !/\s/.test(targetText) + && mapper.addMapping({ + original: { line: sourceEnd.line + 1, column: sourceEnd.character }, + generated: { line: targetEnd.line + 1, column: targetEnd.character}, + source: filename + }); + }); + + const sourceMap = mapper.toString(); + if (sourceMap) { + text += `\n\n//# sourceMappingURL=data:application/json;charset=urf-8;base64,${Base64.encode(sourceMap)}`; + } + return text; +} + +function generateModule(root: Module): GeneratorNode { + return expandToNode` + "use strict"; + (() => { + ${generateModuleContent(root)} + }) + `; +} + +const lastComputableExpressionValueVarName = 'lastComputableExpressionValue'; + +function generateModuleContent(module: Module): Generated { + return expandTracedToNode(module)` + let ${lastComputableExpressionValueVarName}; + ${ joinTracedToNode(module, 'statements')(module.statements, generateStatement, { appendNewLineIfNotEmpty: true }) } + + return ${lastComputableExpressionValueVarName}; + `; +} + +function generateStatement(stmt: Statement): Generated { + if (isDefinition(stmt)) + return generateDefinition(stmt); + else + return generateEvaluation(stmt); +} + +function generateDefinition(def: Definition): Generated { + return def.args && def.args.length ? + expandTracedToNode(def)` + const ${traceToNode(def, 'name')(def.name)} = (${joinTracedToNode(def, 'args')(def.args, arg => traceToNode(arg)(arg.name), { separator: ', '})}) => ${generateExpression(def.expr)}; + ` : expandTracedToNode(def)` + ${traceToNode(def)('const')} ${traceToNode(def, 'name')(def.name)} = ${lastComputableExpressionValueVarName} = ${generateExpression(def.expr)}; + `; +} + +function generateEvaluation(evaln: Evaluation): Generated { + return expandTracedToNode(evaln)` + ${lastComputableExpressionValueVarName} = ${generateExpression(evaln.expression)}; + `; +} + +function generateExpression(expr: Expression): Generated { + + if (isNumberLiteral(expr)) { + return traceToNode(expr, 'value')( expr.value.toString() ); + + } else if (isBinaryExpression(expr)) { + const leftAsIs = isNumberLiteral(expr.left) || isFunctionCall(expr.left); + const rightAsIs = isNumberLiteral(expr.right) || isFunctionCall(expr.right); + const left = leftAsIs ? generateExpression(expr.left) : expandTracedToNode(expr, "left" )`(${generateExpression(expr.left )})`; + const right = rightAsIs ? generateExpression(expr.right) : expandTracedToNode(expr, "right")`(${generateExpression(expr.right)})`; + return expandTracedToNode(expr)` + ${left} ${traceToNode(expr, 'operator')(expr.operator)} ${right} + `; + + } else { + return traceToNode(expr)( + parent => parent + .appendTraced(expr, 'func')(expr.func.ref?.name) + .appendTracedTemplateIf(!!expr.args.length, expr, 'args')` + ( + ${joinToNode(expr.args, generateExpression, { separator: ', ' })} + ) + ` + ); + }; +} + +function deserializeModule(input: string): Module { + return new DefaultJsonSerializer({ + workspace: { + AstNodeLocator: new DefaultAstNodeLocator() + }, + references: { + NameProvider: new DefaultNameProvider() + }, + documentation: { + CommentProvider: undefined + } + } as any).deserialize(input) as Module; +} \ No newline at end of file diff --git a/src/generator/source-map.d.ts b/src/generator/source-map.d.ts new file mode 100644 index 0000000..25f5fb0 --- /dev/null +++ b/src/generator/source-map.d.ts @@ -0,0 +1,71 @@ +/****************************************************************************** + * Copyright 2023 TypeFox GmbH + * This program and the accompanying materials are made available under the + * terms of the MIT License, which is available in the project root. + ******************************************************************************/ + +// We shouldn't import 'SourceMapGenerator' from the 'source-map' package directly +// as that also loads 'source-map-consumer.js' that transitively refers to node.js builtins. +// That affects bundling: Although transitive depdendencies on node.js builtins can be marked as external, +// esbuild's tree shaking doesn't drop the 'SourceMapConsumer' and we just don't need it. +// +// Therefore, we just replicated the required type defs of 'SourceMapGenerator' and friends in here. ;-) + +declare module 'source-map/lib/source-map-generator.js' { +// original license: BSD-3-Clause +// original header: + // Type definitions for source-map 0.7 + // Project: https://github.com/mozilla/source-map + // Definitions by: Morten Houston Ludvigsen , + // Ron Buckton , + // John Vilk + // Definitions: https://github.com/mozilla/source-map + + export interface StartOfSourceMap { + file?: string; + sourceRoot?: string; + skipValidation?: boolean; + } + + export interface Mapping { + generated: Position; + original: Position; + source: string; + name?: string; + } + + export interface RawSourceMap { + version: number; + sources: string[]; + names: string[]; + sourceRoot?: string; + sourcesContent?: string[]; + mappings: string; + file: string; + } + + export class SourceMapGenerator { + constructor(startOfSourceMap?: StartOfSourceMap); + + /** + * Add a single mapping from original source line and column to the generated + * source's line and column for this source map being created. The mapping + * object should have the following properties: + * + * - generated: An object with the generated line and column positions. + * - original: An object with the original line and column positions. + * - source: The original source file (relative to the sourceRoot). + * - name: An optional original token name for this mapping. + */ + addMapping(mapping: Mapping): void; + + /** + * Set the source content for a source file. + */ + setSourceContent(sourceFile: string, sourceContent: string): void; + + toString(): string; + + toJSON(): RawSourceMap; + } +}