From d0f2a5b9910cfe7958237a5cccbd90e8ddb9df52 Mon Sep 17 00:00:00 2001 From: George Lemon Date: Sat, 16 Mar 2024 09:37:03 +0200 Subject: [PATCH] add more `badIndentation` checkers | add placeholder & snippets support Signed-off-by: George Lemon --- src/tim/engine/ast.nim | 64 ++++++- src/tim/engine/compilers/html.nim | 161 +++++++++++------- src/tim/engine/compilers/tim.nim | 1 + src/tim/engine/logging.nim | 44 ++--- src/tim/engine/meta.nim | 39 ++++- src/tim/engine/parser.nim | 270 +++++++++++++++++++++--------- src/tim/engine/tokens.nim | 6 +- tim.nimble | 4 +- 8 files changed, 408 insertions(+), 181 deletions(-) diff --git a/src/tim/engine/ast.nim b/src/tim/engine/ast.nim index 3fb2856..4189d15 100755 --- a/src/tim/engine/ast.nim +++ b/src/tim/engine/ast.nim @@ -26,7 +26,7 @@ type ntLitBool = "bool" ntLitArray = "array" ntLitObject = "object" - ntLitFunction = "function" + ntFunction = "function" ntVariableDef = "Variable" ntAssignExpr = "Assignment" @@ -41,19 +41,23 @@ type ntBracketExpr = "BracketExpression" ntIndexRange = "IndexRange" ntConditionStmt = "ConditionStatement" + ntCaseStmt = "CaseExpression" ntLoopStmt = "LoopStmt" ntViewLoader = "ViewLoader" ntInclude = "Include" + ntImport = "Import" ntPlaceholder = "Placeholder" - + ntStream = "Stream" ntJavaScriptSnippet = "JavaScriptSnippet" ntYamlSnippet = "YAMLSnippet" ntJsonSnippet = "JsonSnippet" ntClientBlock = "ClientSideStatement" + ntStmtList = "StatementList" CommandType* = enum cmdEcho = "echo" cmdReturn = "return" + cmdDiscard = "discard" StorageType* = enum scopeStorage @@ -137,6 +141,10 @@ type ## a sequence of `elif` branches condElseBranch*: seq[Node] ## the body of an `else` branch + of ntCaseStmt: + caseExpr*: Node + caseBranch*: seq[ConditionBranch] + caseElse*: seq[Node] of ntLoopStmt: loopItem*: Node ## a node type of `ntIdent` or `ntIdentPair` @@ -156,6 +164,7 @@ type of ntLitBool: bVal*: bool of ntLitArray: + arrayType*: NodeType arrayItems*: seq[Node] ## a sequence of nodes representing an array of ntLitObject: @@ -189,7 +198,7 @@ type of ntIndexRange: rangeNodes*: array[2, Node] rangeLastIndex*: bool # from end to start using ^ circumflex accent - of ntLitFunction: + of ntFunction: fnIdent*: string ## function identifier name fnParams*: OrderedTable[string, FnParam] @@ -200,6 +209,7 @@ type ## the return type of a function ## if a function has no return type, then `ntUnknown` ## is used as default (void) + fnFwdDecl*, fnExport*: bool of ntJavaScriptSnippet, ntYamlSnippet, ntJsonSnippet: snippetCode*: string @@ -210,11 +220,14 @@ type of ntInclude: includes*: seq[string] ## a sequence of files to be included + of ntImport: + modules*: seq[string] + ## a sequence containing imported modules of ntPlaceholder: placeholderName*: string - ## the name of a placehodler - placeholderNodes*: seq[Node] - ## nodes mapped to a placehodler + ## placeholder target name + of ntStream: + streamContent*: JsonNode of ntClientBlock: clientTargetElement*: string ## an existing HTML selector to used @@ -227,6 +240,8 @@ type ## other statements (such `if`, `for`, `var`) are getting ## interpreted at compile-time (for static templates) or ## on the fly for templates marked as jit. + of ntStmtList: + stmtList*: seq[Node] else: discard meta*: Meta @@ -243,7 +258,7 @@ type Meta* = array[3, int] ScopeTable* = TableRef[string, Node] TimPartialsTable* = TableRef[string, (Ast, seq[cli.Row])] - Ast* = object + Ast* {.acyclic.} = ref object src*: string ## trace the source path nodes*: seq[Node] @@ -254,6 +269,13 @@ type const ntAssignableSet* = {ntLitString, ntLitInt, ntLitFloat, ntLitBool, ntLitObject, ntLitArray} +# proc add*(x: Node, y: Node) = +# if likely y != nil: +# case x.nt +# of ntStmtList: +# x.stmtList.add(y) +# else: discard + proc getInfixOp*(kind: TokenKind, isInfixInfix: bool): InfixOp = result = case kind: @@ -278,7 +300,7 @@ proc getInfixMathOp*(kind: TokenKind, isInfixInfix: bool): MathOp = case kind: of tkPlus: mPlus of tkMinus: mMinus - of tkMultiply: mMulti + of tkAsterisk: mMulti of tkDivide: mDiv of tkMod: mMod else: invalidCalcOp @@ -450,20 +472,39 @@ proc newString*(tk: TokenTuple): Node = result = newNode(ntLitString, tk) result.sVal = tk.value +proc newString*(v: string): Node = + ## Create a new string value node + result = newNode(ntLitString) + result.sVal = v + proc newInteger*(v: int, tk: TokenTuple): Node = result = newNode(ntLitInt, tk) result.iVal = v +proc newInteger*(v: int): Node = + result = newNode(ntLitInt) + result.iVal = v + proc newFloat*(v: float, tk: TokenTuple): Node = ## Create a new float value node result = newNode(ntLitFloat, tk) result.fVal = v +proc newFloat*(v: float): Node = + ## Create a new float value node + result = newNode(ntLitFloat) + result.fVal = v + proc newBool*(v: bool, tk: TokenTuple): Node = ## Create a new bool value Node result = newNode(ntLitBool, tk) result.bVal = v +proc newBool*(v: bool): Node = + ## Create a new bool value Node + result = newNode(ntLitBool) + result.bVal = v + proc newVariable*(varName: string, varValue: Node, meta: Meta): Node = ## Create a new variable definition Node result = newNode(ntVariableDef) @@ -485,7 +526,7 @@ proc newAssignment*(tk: TokenTuple, varValue: Node): Node = proc newFunction*(tk: TokenTuple, ident: string): Node = ## Create a new Function definition Node - result = newNode(ntLitFunction, tk) + result = newNode(ntFunction, tk) result.fnIdent = ident proc newCall*(tk: TokenTuple): Node = @@ -550,6 +591,11 @@ proc toTimNode*(x: JsonNode): Node = for v in x: result.arrayItems.add(toTimNode(v)) else: discard + +proc newStream*(node: JsonNode): Node = + ## Create a new Stream from `node` + Node(nt: ntStream, streamContent: node) + # proc toTimNode(): NimNode = # # https://github.com/nim-lang/Nim/blob/version-2-0/lib/pure/json.nim#L410 # case x.kind diff --git a/src/tim/engine/compilers/html.nim b/src/tim/engine/compilers/html.nim index 477756a..1408f72 100755 --- a/src/tim/engine/compilers/html.nim +++ b/src/tim/engine/compilers/html.nim @@ -27,6 +27,9 @@ type # jsComp: Table[string, JSCompiler] # todo # Forward Declaration +proc newCompiler*(ast: Ast, minify = true, indent = 2): HtmlCompiler +proc getHtml*(c: HtmlCompiler): string + proc walkNodes(c: var HtmlCompiler, nodes: seq[Node], scopetables: var seq[ScopeTable], parentNodeType: NodeType = ntUnknown, xel = newStringOfCap(0)): Node {.discardable.} @@ -35,9 +38,18 @@ proc typeCheck(c: var HtmlCompiler, x, node: Node): bool proc typeCheck(c: var HtmlCompiler, node: Node, expect: NodeType): bool proc mathInfixEvaluator(c: var HtmlCompiler, lhs, rhs: Node, op: MathOp, scopetables: var seq[ScopeTable]): Node proc dotEvaluator(c: var HtmlCompiler, node: Node, scopetables: var seq[ScopeTable]): Node -proc getValue(c: var HtmlCompiler, node: Node, scopetables: var seq[ScopeTable]): Node -proc fnCall(c: var HtmlCompiler, node: Node, scopetables: var seq[ScopeTable]): Node + +proc infixEvaluator(c: var HtmlCompiler, lhs, rhs: Node, + infixOp: InfixOp, scopetables: var seq[ScopeTable]): bool + +proc getValue(c: var HtmlCompiler, node: Node, + scopetables: var seq[ScopeTable]): Node + +proc fnCall(c: var HtmlCompiler, node: Node, + scopetables: var seq[ScopeTable]): Node + proc hasError*(c: HtmlCompiler): bool = c.hasErrors # or c.logger.errorLogs.len > 0 + proc bracketEvaluator(c: var HtmlCompiler, node: Node, scopetables: var seq[ScopeTable]): Node @@ -83,7 +95,7 @@ when not defined timStandalone: scopetables[^1][node.varName] = node else: c.globalScope[node.varName] = node - of ntLitFunction: + of ntFunction: if scopetables.len > 0: scopetables[^1][node.fnIdent] = node else: @@ -373,7 +385,7 @@ proc bracketEvaluator(c: var HtmlCompiler, node: Node, of scopeStorage: let index = c.getValue(node.bracketIndex, scopetables) if likely(index != nil): - return c.walkAccessorStorage(node.bracketLHS, index, scopetables) + result = c.walkAccessorStorage(node.bracketLHS, index, scopetables) proc writeDotExpr(c: var HtmlCompiler, node: Node, scopetables: var seq[ScopeTable]) = # Handle dot expressions @@ -392,6 +404,7 @@ proc evalCmd(c: var HtmlCompiler, node: Node, scopetables: var seq[ScopeTable]): print(val) of cmdReturn: return val + else: discard proc infixEvaluator(c: var HtmlCompiler, lhs, rhs: Node, infixOp: InfixOp, scopetables: var seq[ScopeTable]): bool = @@ -609,6 +622,11 @@ proc getValue(c: var HtmlCompiler, node: Node, result = c.dotEvaluator(node, scopetables) of ntBracketExpr: result = c.bracketEvaluator(node, scopetables) + if likely(result != nil): + case result.nt + of ntInfixExpr: + return c.getValue(result, scopetables) + else: discard of ntMathInfixExpr: # evaluate a math expression and returns its value result = c.mathInfixEvaluator(node.infixMathLeft, @@ -914,36 +932,30 @@ proc typeCheck(c: var HtmlCompiler, node: Node, expect: NodeType): bool = # # Compile Handlers # +proc checkArrayStorage(c: var HtmlCompiler, node: Node, scopetables: var seq[ScopeTable]): bool +proc checkObjectStorage(c: var HtmlCompiler, node: Node, scopetables: var seq[ScopeTable]): bool + proc checkObjectStorage(c: var HtmlCompiler, node: Node, scopetables: var seq[ScopeTable]): bool = - # Check object storage + # a simple checker and ast modified for object storages for k, v in node.objectItems.mpairs: case v.nt - of ntIdent: + of ntLitArray: + if unlikely(not c.checkArrayStorage(v, scopetables)): + return false + else: var valNode = c.getValue(v, scopetables) if likely(valNode != nil): - # todo something with safe var - # if v.identSafe: - # v = valNode - # case v.nt - # of ntLitString: - # v.sVal = xmltree.escape(v.sVal) - # else: discard - # else: v = valNode - else: return - else: discard + else: discard # todo error result = true proc checkArrayStorage(c: var HtmlCompiler, node: Node, scopetables: var seq[ScopeTable]): bool = - # Check array storage + # a simple checker and ast modified for array storages for v in node.arrayItems.mitems: - case v.nt - of ntIdent: - var valNode = c.getValue(v, scopetables) - if likely(valNode != nil): - v = valNode - else: return - else: discard + var valNode = c.getValue(v, scopetables) + if likely(valNode != nil): + v = valNode + else: discard # todo error result = true proc varExpr(c: var HtmlCompiler, node: Node, scopetables: var seq[ScopeTable]) = @@ -994,8 +1006,8 @@ proc fnCall(c: var HtmlCompiler, node: Node, # Handle function calls let some = c.getScope(node.callIdent, scopetables) if likely(some.scopeTable != nil): - newScope(scopetables) let fnNode = some.scopeTable[node.callIdent] + newScope(scopetables) if fnNode.fnParams.len > 0: # add params to the stack for k, p in fnNode.fnParams: @@ -1018,7 +1030,7 @@ proc fnCall(c: var HtmlCompiler, node: Node, else: if c.typeCheck(node.callArgs[i], p.pType): let someParam = c.getScope(k, scopetables) - echo node.callArgs[i] + # echo node.callArgs[i] someParam.scopeTable[k].varValue = node.callArgs[i] else: return inc i @@ -1133,17 +1145,32 @@ template htmlblock(x: Node, body) = add c.output, ">" c.stickytail = false -proc htmlElement(c: var HtmlCompiler, node: Node, scopetables: var seq[ScopeTable]) = +proc htmlElement(c: var HtmlCompiler, node: Node, + scopetables: var seq[ScopeTable]) = # Handle HTML element htmlblock node: c.walkNodes(node.nodes, scopetables, ntHtmlElement) -proc evaluatePartials(c: var HtmlCompiler, includes: seq[string], scopetables: var seq[ScopeTable]) = +proc evaluatePartials(c: var HtmlCompiler, + includes: seq[string], scopetables: var seq[ScopeTable]) = # Evaluate included partials for x in includes: if likely(c.ast.partials.hasKey(x)): c.walkNodes(c.ast.partials[x][0].nodes, scopetables) +proc evaluatePlaceholder(c: var HtmlCompiler, node: Node, + scopetables: var seq[ScopeTable]) = + ## Evaluate a placehodler + if c.engine.hasPlaceholder(node.placeholderName): + var i = 0 + for tree in c.engine.snippets(node.placeholderName): + let phc = newCompiler(tree) + if not phc.hasErrors: + add c.output, phc.getHtml() + else: + echo "ignore snippet" + c.engine.deleteSnippet(node.placeholderName, i) + inc i # # JS API # @@ -1171,55 +1198,56 @@ proc walkNodes(c: var HtmlCompiler, nodes: seq[Node], xel = newStringOfCap(0)): Node {.discardable.} = # Evaluate a seq[Node] nodes for i in 0..nodes.high: - case nodes[i].nt + let node = nodes[i] + case node.nt of ntHtmlElement: if not c.isClientSide: - c.htmlElement(nodes[i], scopetables) + c.htmlElement(node, scopetables) else: - c.createHtmlElement(nodes[i], scopetables, xel) + c.createHtmlElement(node, scopetables, xel) of ntIdent: - let x: Node = c.getValue(nodes[i], scopetables) + let x: Node = c.getValue(node, scopetables) if not c.isClientSide: - write x, true, nodes[i].identSafe + write x, true, node.identSafe else: add c.jsOutputCode, domInnerText % [xel, x.toString()] of ntDotExpr: - let x: Node = c.dotEvaluator(nodes[i], scopetables) + let x: Node = c.dotEvaluator(node, scopetables) if not c.isClientSide: write x, true, false else: add c.jsOutputCode, domInnerText % [xel, x.toString()] of ntVariableDef: - c.varExpr(nodes[i], scopetables) + c.varExpr(node, scopetables) of ntCommandStmt: - case nodes[i].cmdType + case node.cmdType of cmdReturn: - return c.evalCmd(nodes[i], scopetables) + return c.evalCmd(node, scopetables) else: - discard c.evalCmd(nodes[i], scopetables) + discard c.evalCmd(node, scopetables) of ntAssignExpr: - c.assignExpr(nodes[i], scopetables) + c.assignExpr(node, scopetables) of ntConditionStmt: - result = c.evalCondition(nodes[i], scopetables) + result = c.evalCondition(node, scopetables) if result != nil: return # a resulted ntCommandStmt Node of type cmdReturn of ntLoopStmt: - c.evalLoop(nodes[i], scopetables) + c.evalLoop(node, scopetables) of ntLitString, ntLitInt, ntLitFloat, ntLitBool: if not c.isClientSide: - write nodes[i], true, false + write node, true, false else: - add c.jsOutputCode, domInnerText % [xel, nodes[i].toString()] + add c.jsOutputCode, domInnerText % [xel, node.toString()] of ntMathInfixExpr: - let x: Node = c.mathInfixEvaluator(nodes[i].infixMathLeft, - nodes[i].infixMathRight, nodes[i].infixMathOp, scopetables) + let x: Node = c.mathInfixEvaluator(node.infixMathLeft, + node.infixMathRight, node.infixMathOp, scopetables) write x, true, false - of ntLitFunction: - c.fnDef(nodes[i], scopetables) + of ntFunction: + c.fnDef(node, scopetables) of ntInfixExpr: - case nodes[i].infixOp + case node.infixOp of AMP: - c.evalConcat(nodes[i], scopetables) + c.evalConcat(node, scopetables) else: discard # todo of ntViewLoader: c.head = c.output @@ -1227,25 +1255,27 @@ proc walkNodes(c: var HtmlCompiler, nodes: seq[Node], of ntCall: case parentNodeType of ntHtmlElement: - let returnNode = c.fnCall(nodes[i], scopetables) + let returnNode = c.fnCall(node, scopetables) if likely(returnNode != nil): write returnNode, true, false else: - discard c.fnCall(nodes[i], scopetables) + let x = c.fnCall(node, scopetables) + if unlikely x != nil: + compileErrorWithArgs(fnReturnMissingCommand, [node.callIdent, $(x.nt)]) of ntInclude: - c.evaluatePartials(nodes[i].includes, scopetables) + c.evaluatePartials(node.includes, scopetables) of ntJavaScriptSnippet: - add c.jsOutput, nodes[i].snippetCode + add c.jsOutput, node.snippetCode of ntJsonSnippet: try: add c.jsonOutput, - jsony.toJson(jsony.fromJson(nodes[i].snippetCode)) + jsony.toJson(jsony.fromJson(node.snippetCode)) except jsony.JsonError as e: - compileErrorWithArgs(internalError, nodes[i].meta, [e.msg]) + compileErrorWithArgs(internalError, node.meta, [e.msg]) of ntClientBlock: - c.jsTargetElement = nodes[i].clientTargetElement + c.jsTargetElement = node.clientTargetElement c.isClientSide = true - c.walkNodes(nodes[i].clientStmt, scopetables) + c.walkNodes(node.clientStmt, scopetables) add c.jsOutputCode, "}" add c.jsOutput, "document.addEventListener('DOMContentLoaded', function(){" @@ -1255,6 +1285,8 @@ proc walkNodes(c: var HtmlCompiler, nodes: seq[Node], setLen(c.jsTargetElement, 0) reset(c.jsCountEl) c.isClientSide = false + of ntPlaceholder: + c.evaluatePlaceholder(node, scopetables) else: discard # @@ -1262,20 +1294,22 @@ proc walkNodes(c: var HtmlCompiler, nodes: seq[Node], # when not defined timStandalone: - proc newCompiler*(engine: TimEngine, ast: Ast, tpl: TimTemplate, minify = true, - indent = 2, data: JsonNode = newJObject()): HtmlCompiler = + proc newCompiler*(engine: TimEngine, ast: Ast, + tpl: TimTemplate, minify = true, indent = 2, + data: JsonNode = newJObject()): HtmlCompiler = ## Create a new instance of `HtmlCompiler` assert indent in [2, 4] data["global"] = engine.getGlobalData() result = HtmlCompiler( - ast: ast, + engine: engine, tpl: tpl, start: true, tplType: tpl.getType, logger: Logger(filePath: tpl.getSourcePath()), data: data, - minify: minify + minify: minify, + ast: ast, ) if minify: setLen(result.nl, 0) var scopetables = newSeq[ScopeTable]() @@ -1287,12 +1321,13 @@ else: assert indent in [2, 4] result = HtmlCompiler( - ast: ast, + engine: engine, tpl: tpl, start: true, tplType: tpl.getType, logger: Logger(filePath: tpl.getSourcePath()), - minify: minify + minify: minify, + ast: ast, ) if minify: setLen(result.nl, 0) var scopetables = newSeq[ScopeTable]() diff --git a/src/tim/engine/compilers/tim.nim b/src/tim/engine/compilers/tim.nim index 5485461..e6870bf 100755 --- a/src/tim/engine/compilers/tim.nim +++ b/src/tim/engine/compilers/tim.nim @@ -5,6 +5,7 @@ type TimCompiler* = object of RootObj ast*: Ast tpl*: TimTemplate + engine*: TimEngine nl*: string = "\n" output*, jsOutput*, jsonOutput*, yamlOutput*, cssOutput*: string diff --git a/src/tim/engine/logging.nim b/src/tim/engine/logging.nim index 9adc2dd..85f604e 100755 --- a/src/tim/engine/logging.nim +++ b/src/tim/engine/logging.nim @@ -14,27 +14,29 @@ when compileOption("app", "console"): type Message* = enum - invalidIndentation = "Invalid indentation" - unexpectedToken = "Unexpected token $" - undeclaredVariable = "Undeclared variable $" - invalidAccessorStorage = "Invalid accessor storage $ for $" - varRedefine = "Attempt to redefine variable $" - varImmutable = "Attempt to reassign value to immutable constant $" - fnRedefine = "Attempt to redefine function $" - fnUndeclared = "Undeclared function $" - fnExtraArg = "Extra arguments given. Got $ expected $" - badIndentation = "Nestable statement requires indentation" - invalidContext = "Invalid $ in this context" - invalidViewLoader = "Invalid use of `@view` in this context. Use a layout instead" - duplicateViewLoader = "Duplicate `@view` loader" - typeMismatch = "Type mismatch. Got $ expected $" - duplicateAttribute = "Duplicate HTML attribute $" - duplicateField = "Duplicate field $" - undeclaredField = "Undeclared field $" - invalidIterator = "Invalid iterator" - indexDefect = "Index $ not in $" - importNotFound = "Cannot open file: $" - importCircularError = "Circular import detected: $" + invalidIndentation = "Invalid indentation [InvalidIndentation]" + unexpectedToken = "Unexpected token $ [UnexpectedToken]" + undeclaredVariable = "Undeclared variable $ [UndeclaredVariable]" + invalidAccessorStorage = "Invalid accessor storage $ for $ [InvalidAccessorStorage]" + varRedefine = "Attempt to redefine variable $ [VarRedefine]" + varImmutable = "Attempt to reassign value to immutable constant $ [VarImmutable]" + fnRedefine = "Attempt to redefine function $ [RedefineFunction]" + fnUndeclared = "Undeclared function $ [UndeclaredFunction]" + fnReturnMissingCommand = "Expression $ is of type $ and has to be used or discarded [UseOrDiscard]" + fnExtraArg = "Extra arguments given. Got $ expected $ [ExtraArgs]" + badIndentation = "Nestable statement requires indentation [BadIndentation]" + invalidContext = "Invalid $ in this context [InvalidContext]" + invalidViewLoader = "Invalid use of `@view` in this context. Use a layout instead [InvalidViewLoader]" + duplicateViewLoader = "Duplicate `@view` loader [DuplicateViewLoaded]" + typeMismatch = "Type mismatch. Got $ expected $ [TypeMismatch]" + duplicateAttribute = "Duplicate HTML attribute $ [DuplicateAttribute]" + duplicateField = "Duplicate field $ [DuplicateField]" + undeclaredField = "Undeclared field $ [UndeclaredField]" + invalidIterator = "Invalid iterator [InvalidIterator]" + indexDefect = "Index $ not in $ [IndexDefect]" + importNotFound = "Cannot open file: $ [ImportNotFound]" + importCircularError = "Circular import detected: $ [CircularImport]" + eof = "EOF reached before closing $ [EOF]" internalError = "$" Level* = enum diff --git a/src/tim/engine/meta.nim b/src/tim/engine/meta.nim index 9490c96..40938a3 100755 --- a/src/tim/engine/meta.nim +++ b/src/tim/engine/meta.nim @@ -40,10 +40,12 @@ type TimCallback* = proc() {.nimcall, gcsafe.} TimEngine* = ref object base, src, output: string - minify: bool + minify, htmlErrors: bool indentSize: int layouts, views, partials: TemplateTable = TemplateTable() errors*: seq[string] + placeholders: Table[string, seq[Ast]] + ## A table containing available placeholders when defined timStandalone: globals: Globals else: @@ -51,6 +53,30 @@ type TimError* = object of CatchableError +# +# Placeholders API +# +proc addPlaceholder*(engine: TimEngine, + k: string, snippetTree: Ast) = + if engine.placeholders.hasKey(k): + engine.placeholders[k].add(snippetTree) + else: + engine.placeholders[k] = @[snippetTree] + +proc hasPlaceholder*(engine: TimEngine, k: string): bool = + result = engine.placeholders.hasKey(k) + +iterator listPlaceholders*(engine: TimEngine): (string, seq[Ast]) = + for k, v in engine.placeholders.mpairs: + yield (k, v) + +iterator snippets*(engine: TimEngine, k: string): Ast = + for x in engine.placeholders[k]: + yield x + +proc deleteSnippet*(engine: TimEngine, k: string, i: int) = + engine.placeholders[k].del(i) + proc getPath*(engine: TimEngine, key: string, templateType: TimTemplateType): string = ## Get absolute path of `key` view, partial or layout var k: string @@ -241,10 +267,14 @@ proc getView*(engine: TimEngine, key: string): TimTemplate = result = engine.views[engine.getPath(key, ttView)] result.inUse = true +proc getTemplatePath*(engine: TimEngine, path: string): string = + path.replace(engine.base, "") + proc isUsed*(t: TimTemplate): bool = t.inUse +proc showHtmlErrors*(engine: TimEngine): bool = engine.htmlErrors -proc newTim*(src, output, basepath: string, - minify = true, indent = 2): TimEngine = +proc newTim*(src, output, basepath: string, minify = true, + indent = 2, showHtmlError = false): TimEngine = ## Initializes `TimEngine` engine var basepath = if basepath.fileExists: @@ -262,7 +292,8 @@ proc newTim*(src, output, basepath: string, output: normalizedPath(basepath / output), base: basepath, minify: minify, - indentSize: indent + indentSize: indent, + htmlErrors: showHtmlError ) for sourceDir in [ttLayout, ttView, ttPartial]: diff --git a/src/tim/engine/parser.nim b/src/tim/engine/parser.nim index 5db7399..a2eecff 100755 --- a/src/tim/engine/parser.nim +++ b/src/tim/engine/parser.nim @@ -9,8 +9,9 @@ import std/[macros, streams, lexbase, strutils, sequtils, re, tables] from std/os import `/` import ./meta, ./tokens, ./ast, ./logging -import pkg/kapsis/cli +# import ./stdlib +import pkg/kapsis/cli import pkg/importer type @@ -33,8 +34,6 @@ type ## Parser internals includes: Table[string, Meta] ## A table to store all `@include` statements - placeholders: Table[string, Node] - ## A table containing available placeholders tree: Ast ## The generated Abstract Syntax Tree @@ -43,7 +42,7 @@ type const tkCompSet = {tkEQ, tkNE, tkGT, tkGTE, tkLT, tkLTE, tkAmp, tkAndAnd} - tkMathSet = {tkPlus, tkMinus, tkMultiply, tkDivide} + tkMathSet = {tkPlus, tkMinus, tkAsterisk, tkDivide} tkAssignableSet = { tkString, tkBacktick, tkBool, tkFloat, tkIdentifier, tkInteger, tkIdentVar, tkIdentVarSafe, tkLC, tkLB @@ -57,21 +56,34 @@ const # # Forward Declaration # -proc getPrefixFn(p: var Parser, excludes, includes: set[TokenKind] = {}): PrefixFunction {.gcsafe.} -proc getInfixFn(p: var Parser, excludes, includes: set[TokenKind] = {}): InfixFunction {.gcsafe.} +proc getPrefixFn(p: var Parser, excludes, + includes: set[TokenKind] = {}): PrefixFunction {.gcsafe.} + +proc getInfixFn(p: var Parser, excludes, + includes: set[TokenKind] = {}): InfixFunction {.gcsafe.} proc parseInfix(p: var Parser, lhs: Node): Node {.gcsafe.} -proc getPrefixOrInfix(p: var Parser, includes, excludes: set[TokenKind] = {}, infix: Node = nil): Node {.gcsafe.} -proc parsePrefix(p: var Parser, excludes, includes: set[TokenKind] = {}): Node {.gcsafe.} -proc pAnoArray(p: var Parser, excludes, includes: set[TokenKind] = {}): Node {.gcsafe.} -proc pAnoObject(p: var Parser, excludes, includes: set[TokenKind] = {}): Node {.gcsafe.} +proc getPrefixOrInfix(p: var Parser, includes, + excludes: set[TokenKind] = {}, infix: Node = nil): Node {.gcsafe.} + +proc parsePrefix(p: var Parser, + excludes, includes: set[TokenKind] = {}): Node {.gcsafe.} + +proc pAnoArray(p: var Parser, excludes, + includes: set[TokenKind] = {}): Node {.gcsafe.} + +proc pAnoObject(p: var Parser, excludes, + includes: set[TokenKind] = {}): Node {.gcsafe.} + proc pAssignable(p: var Parser): Node {.gcsafe.} proc parseBracketExpr(p: var Parser, lhs: Node): Node {.gcsafe.} + proc parseDotExpr(p: var Parser, lhs: Node): Node {.gcsafe.} -proc pFunctionCall(p: var Parser, excludes, includes: set[TokenKind] = {}): Node {.gcsafe.} +proc pFunctionCall(p: var Parser, excludes, + includes: set[TokenKind] = {}): Node {.gcsafe.} proc parseMathExp(p: var Parser, lhs: Node): Node {.gcsafe.} proc parseCompExp(p: var Parser, lhs: Node): Node {.gcsafe.} @@ -82,6 +94,11 @@ template caseNotNil(x: Node, body): untyped = body else: return nil +template caseNotNil(x: Node, body, then): untyped = + if likely(x != nil): + body + else: then + # # Error API # @@ -190,13 +207,13 @@ proc getStorageType(p: var Parser): StorageType = proc getType(p: var Parser): NodeType = result = case p.curr.kind: - of tkLitArray: ntLitArray - of tkLitBool: ntLitBool - of tkLitFloat: ntLitFloat - of tkLitFunction: ntLitFunction + of tkLitString: ntLitString of tkLitInt: ntLitInt + of tkLitFloat: ntLitFloat + of tkLitBool: ntLitBool + of tkLitArray: ntLitArray of tkLitObject: ntLitObject - of tkLitString: ntLitString + of tkLitFunction: ntFunction else: ntUnknown # @@ -285,6 +302,7 @@ proc parseBracketExpr(p: var Parser, lhs: Node): Node {.gcsafe.} = result.bracketLHS = lhs caseNotNil index: if p.curr is tkRB: + walk p # tkRB result.bracketIndex = index while true: case p.curr.kind @@ -299,7 +317,7 @@ proc parseBracketExpr(p: var Parser, lhs: Node): Node {.gcsafe.} = else: break # todo handle infix expressions elif isRange: - walk p, 2 + walk p, 2 # tkDOT * 2 let lastIndex = if p.curr is tkCaret: walk p; true @@ -344,6 +362,8 @@ prefixHandle pIdent: else: discard if p.curr is tkDot and p.curr.line == result.meta[0]: result = p.parseDotExpr(result) + of tkTernary: + result = p.parseTernaryExpr(result) else: discard prefixHandle pIdentOrAssignment: @@ -410,7 +430,8 @@ template anyAttrIdent(): untyped = (p.curr is tkIdentifier and (p.curr.line == el.line or (p.curr.isChild(el) and p.next is tkAssign))) ) -proc parseAttributes(p: var Parser, attrs: var HtmlAttributes, el: TokenTuple) {.gcsafe.} = +proc parseAttributes(p: var Parser, + attrs: var HtmlAttributes, el: TokenTuple) {.gcsafe.} = # parse HTML element attributes while true: case p.curr.kind @@ -457,8 +478,10 @@ proc parseAttributes(p: var Parser, attrs: var HtmlAttributes, el: TokenTuple) { x = p.pFunctionCall() else: x = p.pIdent() - if likely(x != nil): + caseNotNil x: attrs[attrKey.value] = @[x] + do: + discard else: errorWithArgs(duplicateAttribute, attrKey, [attrKey.value]) else: break # errorWithArgs(invalidAttribute, p.prev, [p.prev.value]) @@ -490,6 +513,14 @@ prefixHandle pElement: of tkIdentifier: result.attrs = HtmlAttributes() p.parseAttributes(result.attrs, this) + of tkIdentVar, tkIdentVarSafe: + let x = p.pIdent() + caseNotNil x: + case x.nt + of ntConditionStmt: + echo x + else: + discard # todo of tkLP: discard # todo # let groupNode = p.pGroupExpr() @@ -516,7 +547,7 @@ prefixHandle pElement: else: discard if p.curr isnot tkIdentifier or p.isFnCall(): let valNode = p.getPrefixOrInfix() - if likely(valNode != nil): + caseNotNil valNode: add result.nodes, valNode of tkGT: # parse inline HTML tags @@ -568,13 +599,15 @@ proc parseCondBranch(p: var Parser, tk: TokenTuple): ConditionBranch {.gcsafe.} walk p # `if` or `elif` token result.expr = p.getPrefixOrInfix() if p.curr is tkColon: walk p # colon is optional - if likely(result.expr != nil): + caseNotNil result.expr: while p.curr.isChild(tk): let node = p.getPrefixOrInfix() - if likely(node != nil): + caseNotNil node: add result.body, node + do: return if unlikely(result.body.len == 0): error(badIndentation, p.curr) + do: return prefixHandle pCondition: # parse `if`, `elif`, `else` condition statements @@ -602,6 +635,38 @@ prefixHandle pCondition: if unlikely(result.condElseBranch.len == 0): return nil +proc parseStatement(p: var Parser, parent: (TokenTuple, Node), + excludes, includes: set[TokenKind]): Node {.gcsafe.} = + ## Parse a statement node + result = ast.newNode(ntStmtList) + while p.curr isnot tkEOF: + let tk = p.curr + let node = p.parsePrefix(excludes, includes) + caseNotNil node: + add result.stmtList, node + +prefixHandle pCase: + # parse a conditional `case` block + let tk = p.curr + result = ast.newNode(ntCaseStmt) + walk p + let caseExpr = p.getPrefixOrInfix() + caseNotNil caseExpr: + let firstof = p.curr + expectWalk tkOF + while p.curr is tkOF and (p.curr.isChild(tk) and p.curr.pos > firstof.pos): + let currOfToken = p.curr + walk p # tkOF + let caseValue = p.getPrefixOrInfix() + caseNotNil caseValue: + expectWalk tkColon + let caseBody = p.parseStatement((currOfToken, result), excludes, includes) + caseNotNil caseBody: + echo caseBody + # add result.caseBranch, caseBody + + # parse `else` branch + prefixHandle pFor: # parse `for` statement let tk = p.curr @@ -641,44 +706,49 @@ prefixHandle pFor: prefixHandle pAnoObject: # parse an anonymous object - let anno = ast.newNode(ntLitObject, p.curr) - anno.objectItems = newOrderedTable[string, Node]() - walk p # { - while p.curr.isIdent(anyIdent = true, anyStringKey = true) and p.next.kind == tkColon: - let fName = p.curr - if unlikely(p.curr is tkColon): - return nil - else: walk p, 2 - if likely(anno.objectItems.hasKey(fName.value) == false): - var item: Node - case p.curr.kind - of tkLB: - item = p.pAnoArray() - of tkLC: - item = p.pAnoObject() - else: - item = p.getPrefixOrInfix(includes = tkAssignableSet) - if likely(item != nil): - anno.objectItems[fName.value] = item - else: return - else: - errorWithArgs(duplicateField, fName, [fName.value]) - if p.curr is tkComma: - walk p # next k/v pair - if likely(p.curr is tkRC): - walk p - return anno + result = ast.newNode(ntLitObject, p.curr) + result.objectItems = newOrderedTable[string, Node]() + walk p # tkLC + while p.curr isnot tkRC and not p.hasErrors: + if p.curr is tkEOF: + errorWithArgs(eof, p.curr, [$tkRC]) + if p.curr.isIdent(anyIdent = true, anyStringKey = true) and + p.next is tkColon: + let k = p.curr + walk p, 2 # key and colon + if likely(not result.objectItems.hasKey(k.value)): + var v: Node + case p.curr.kind + of tkLB: + v = p.pAnoArray() + of tkLC: + v = p.pAnoObject() + else: + v = p.getPrefixOrInfix(includes = tkAssignableSet) + caseNotNil v: + result.objectItems[k.value] = v + if p.curr is tkComma: + walk p + else: + if p.curr.line == v.meta[0]: + result = nil + error(badIndentation, p.curr) + do: + result = nil + break + else: errorWithArgs(duplicateField, k, [k.value]) + walk p # tkRC prefixHandle pAnoArray: # parse an anonymous array let tk = p.curr walk p # [ var items: seq[Node] - while p.curr.kind != tkRB: + while p.curr.kind != tkRB and not p.hasErrors: var item = p.pAssignable() - if likely(item != nil): + caseNotNil item: add items, item - else: + do: if p.curr is tkLB: item = p.pAnoArray() caseNotNil item: @@ -687,9 +757,12 @@ prefixHandle pAnoArray: item = p.pAnoObject() caseNotNil item: add items, item - else: return # todo error + else: return nil # todo error if p.curr is tkComma: walk p + else: + if p.curr isnot tkRB and p.curr.line == item.meta[0]: + error(badIndentation, p.curr) expectWalk tkRB result = ast.newNode(ntLitArray, tk) result.arrayItems = items @@ -772,9 +845,8 @@ prefixHandle pClientSide: walk p while p.curr isnot tkEnd and p.curr.isChild(tk): let n: Node = p.getPrefixOrInfix() - if likely(n != nil): + caseNotNil n: add result.clientStmt, n - else: return nil expectWalk tkEnd prefixHandle pPlaceholder: @@ -789,7 +861,7 @@ template handleImplicitDefaultValue {.dirty.} = # handle implicit default value walk p let implNode = p.getPrefixOrInfix(includes = tkAssignableSet) - if likely(implNode != nil): + caseNotNil implNode: result.fnParams[pName.value].pImplVal = implNode prefixHandle pFunction: @@ -798,6 +870,8 @@ prefixHandle pFunction: expect tkIdentifier: # function identifier result = ast.newFunction(this, p.curr.value) walk p + if p.curr is tkAsterisk: + result.fnExport = true expectWalk tkLP while p.curr isnot tkRP: case p.curr.kind @@ -833,16 +907,19 @@ prefixHandle pFunction: # set a return type result.fnReturnType = p.getType walk p - expectWalk tkAssign # begin function body - while p.curr.isChild(this): - # todo disallow use of html inside a function - # todo cleanup parser code and make use of includes/excludes - let node = p.getPrefixOrInfix() - if likely(node != nil): - add result.fnBody, node - else: return nil - if unlikely(result.fnBody.len == 0): - error(badIndentation, p.curr) + if p.curr is tkAssign: + # begin function body + walk p + while p.curr.isChild(this): + # todo disallow use of html inside a function + # todo cleanup parser code and make use of includes/excludes + let node = p.getPrefixOrInfix() + caseNotNil node: + add result.fnBody, node + if unlikely(result.fnBody.len == 0): + error(badIndentation, p.curr) + else: + result.fnFwdDecl = true prefixHandle pFunctionCall: # parse a function call @@ -850,9 +927,8 @@ prefixHandle pFunctionCall: walk p, 2 # we know tkLP is next so we'll skip it while p.curr isnot tkRP: let argNode = p.getPrefixOrInfix(includes = tkAssignableSet) - if likely(argNode != nil): + caseNotNil argNode: add result.callArgs, argNode - else: return nil walk p # tkRP # @@ -864,7 +940,7 @@ proc parseCompExp(p: var Parser, lhs: Node): Node {.gcsafe.} = walk p let rhstk = p.curr let rhs = p.parsePrefix(includes = tkComparable) - if likely(rhs != nil): + caseNotNil rhs: result = ast.newNode(ntInfixExpr, rhstk) result.infixLeft = lhs result.infixOp = op @@ -888,7 +964,7 @@ proc parseCompExp(p: var Parser, lhs: Node): Node {.gcsafe.} = proc parseTernaryExpr(p: var Parser, lhs: Node): Node {.gcsafe.} = # parse an one line conditional using ternary operator - discard # todo + echo lhs proc parseMathExp(p: var Parser, lhs: Node): Node {.gcsafe.} = # parse math expressions with symbols (+, -, *, /) @@ -896,12 +972,12 @@ proc parseMathExp(p: var Parser, lhs: Node): Node {.gcsafe.} = walk p let rhstk = p.curr let rhs = p.parsePrefix(includes = tkComparable) - if likely(rhs != nil): + caseNotNil rhs: result = ast.newNode(ntMathInfixExpr, rhstk) result.infixMathOp = infixOp result.infixMathLeft = lhs case p.curr.kind - of tkMultiply, tkDivide: + of tkAsterisk, tkDivide: result.infixMathRight = p.parseMathExp(rhs) of tkPlus, tkMinus: result.infixMathRight = rhs @@ -943,6 +1019,7 @@ proc getPrefixFn(p: var Parser, excludes, includes: set[TokenKind] = {}): Prefix of tkBool: pBool of tkEchoCmd: pEchoCommand of tkIF: pCondition + of tkCase: pCase of tkFor: pFor of tkIdentifier: if p.next is tkLP and p.next.wsno == 0: @@ -961,7 +1038,8 @@ proc getPrefixFn(p: var Parser, excludes, includes: set[TokenKind] = {}): Prefix of tkPlaceholder: pPlaceholder else: nil -proc parsePrefix(p: var Parser, excludes, includes: set[TokenKind] = {}): Node {.gcsafe.} = +proc parsePrefix(p: var Parser, excludes, + includes: set[TokenKind] = {}): Node {.gcsafe.} = let prefixFn = p.getPrefixFn(excludes, includes) if likely(prefixFn != nil): return p.prefixFn(excludes, includes) @@ -972,11 +1050,10 @@ proc getPrefixOrInfix(p: var Parser, includes, let lhs = p.parsePrefix(excludes, includes) var infixNode: Node if p.curr.isInfix: - if likely(lhs != nil): + caseNotNil lhs: infixNode = p.parseInfix(lhs) - if likely(infixNode != nil): + caseNotNil infixNode: return infixNode - else: return result = lhs proc parseRoot(p: var Parser, excludes, includes: set[TokenKind] = {}): Node {.gcsafe.} = @@ -989,6 +1066,7 @@ proc parseRoot(p: var Parser, excludes, includes: set[TokenKind] = {}): Node {.g of tkIdentVar, tkIdentVarSafe: p.pIdentOrAssignment() of tkIF: p.pCondition() + of tkCase: p.pCase() of tkFor: p.pFor() of tkViewLoader: p.pViewLoader() of tkIdentifier: @@ -1053,8 +1131,9 @@ template startParse(path: string): untyped = reset(p.handle.tree) # reset incomplete tree break let node = p.handle.parseRoot() - if likely(node != nil): + caseNotNil node: add p.handle.tree.nodes, node + do: discard lexbase.close(p.handle.lex) if isMainParser: if p.handle.includes.len > 0 and not p.handle.hasErrors: @@ -1075,6 +1154,16 @@ template collectImporterErrors = meta[2], true, [err.fpath.replace(engine.getSourcePath(), "")]) p.handle.hasErrors = true +proc initSystemModule(p: var Parser) = + ## Make `std/system` available by default + let sysid = "std/system" + var sysNode = ast.newNode(ntImport) + sysNode.modules.add(sysid) + p.tree.nodes.add(sysNode) + # var L = initTicketLock() + # importer(sysid, p.dirPath, addr(p.stylesheets), + # addr L, true, std(sysid)[1]) + # # Public API # @@ -1086,11 +1175,14 @@ proc newParser*(engine: TimEngine, tpl: TimTemplate, engine.getSourcePath() / $(ttPartial), baseIsMain = true ) + p.handle.tree = Ast() p.handle.lex = newLexer(readFile(tpl.sources.src), allowMultilineStrings = true) p.handle.engine = engine p.handle.tpl = tpl p.handle.isMain = isMainParser p.handle.refreshAst = refreshAst + # initstdlib() + # p.initSystemModule() startParse(tpl.sources.src) if isMainParser: {.gcsafe.}: @@ -1101,13 +1193,27 @@ proc newParser*(engine: TimEngine, tpl: TimTemplate, collectImporterErrors() result = p.handle -proc newParser*(engine: TimEngine, id: string, code: string): Parser {.gcsafe.} = +proc newParser*(id: string, code: string): Parser {.gcsafe.} = ## A proc used to parse `timl` code on the fly ## `id` is used to identify the current `code - var p = newImport[Parser](id, engine.getSourcePath() / $(ttPartial), baseIsMain = true) - p.handle.lex = newLexer(code, allowMultilineStrings = true) - # startParse(id) - result = p.handle + # var p = newImport[Parser](id, engine.getSourcePath() / $(ttPartial), baseIsMain = true) + var p = Parser(tree: Ast(), lex: newLexer(code, allowMultilineStrings = true)) + p.curr = p.lex.getToken() + p.next = p.lex.getToken() + p.logger = Logger(filePath: id) + while p.curr isnot tkEOF: + if unlikely(p.lex.hasError): + p.logger.newError(internalError, p.curr.line, + p.curr.col, false, p.lex.getError) + if unlikely(p.hasErrors): + reset(p.tree) # reset incomplete tree + break + let node = p.parseRoot() + caseNotNil node: + add p.tree.nodes, node + do: discard + lexbase.close(p.lex) + result = p proc getAst*(p: Parser): Ast {.gcsafe.} = ## Returns the constructed AST diff --git a/src/tim/engine/tokens.nim b/src/tim/engine/tokens.nim index fb13217..abc716b 100755 --- a/src/tim/engine/tokens.nim +++ b/src/tim/engine/tokens.nim @@ -156,7 +156,7 @@ const toktokSettings = registerTokens toktokSettings: plus = '+' minus = '-' - multiply = '*' + asterisk = '*' divide = '/': doc = tokenize(handleDocBlock, '*') comment = tokenize(handleInlineComment, '/') @@ -186,11 +186,14 @@ registerTokens toktokSettings: pipe = '|': orOr = '|' backtick = tokenize(handleBackticks, '`') + `case` = "case" + `of` = "of" `if` = "if" `elif` = "elif" `else` = "else" `and` = "and" `for` = "for" + `while` = "while" `in` = "in" `or` = "or" `bool` = ["true", "false"] @@ -221,5 +224,6 @@ registerTokens toktokSettings: `const` = "const" returnCmd = "return" echoCmd = "echo" + discardCmd = "discard" identVar = tokenize(handleVar, '$') identVarSafe \ No newline at end of file diff --git a/tim.nimble b/tim.nimble index 52c9abe..ad47d4c 100755 --- a/tim.nimble +++ b/tim.nimble @@ -13,7 +13,7 @@ skipDirs = @["example", "editors"] # Dependencies requires "nim >= 2.0.0" -requires "toktok >= 0.1.3" +requires "toktok#head" requires "jsony" requires "https://github.com/openpeeps/importer" requires "watchout#head" @@ -21,6 +21,8 @@ requires "kapsis#head" requires "denim#head" requires "checksums" requires "flatty#head" +requires "nyml" +# requires "bro" requires "httpx", "websocketx" task node, "Build a NODE addon":