wip universal api - server-side rendering powered by zmq

Signed-off-by: George Lemon <[email protected]>

src/tim.nims

@@ -4,6 +4,7 @@
 
 when defined napibuild:
   --define:napiOrWasm
+  --define:watchoutBrowserSync
   --noMain:on
   --passC:"-I/usr/include/node -I/usr/local/include/node"
 

src/tim/app/jitCmd.nim

@@ -0,0 +1,32 @@
+import std/[osproc, os]
+import pkg/flatty
+import pkg/kapsis/[runtime, cli]
+
+import ../engine/[parser, ast]
+import ../engine/compilers/nimc
+import ../server/vm
+
+proc binCommand*(v: Values) =
+  ## Execute Just-in-Time compilation of the specifie
+  let
+    cachedPath = v.get("ast").getPath.path
+    cachedAst = readFile(cachedPath)
+    c = nimc.newCompiler(fromFlatty(cachedAst, Ast), true)
+  var
+    genFilepath = cachedPath.changeFileExt(".nim")
+    genFilepathTuple = genFilepath.splitFile()
+  # nim requires that module name starts with a letter
+  genFilepathTuple.name = "r_" & genFilepathTuple.name
+  genFilepath = genFilepathTuple.dir / genFilepathTuple.name & genFilepathTuple.ext
+  let dynlibPath = cachedPath.changeFileExt(".dylib")
+  writeFile(genFilepath, c.exportCode())
+  # if not dynlibPath.fileExists:
+  let status = execCmdEx("nim c --mm:arc -d:danger --opt:speed --app:lib --noMain -o:" & dynlibPath & " " & genFilePath)
+  if status.exitCode > 0:
+    return # nim compilation error  
+  removeFile(genFilepath)
+  var collection = DynamicTemplates()
+  let hashedName = cachedPath.splitFile.name
+  collection.load(hashedName)
+  echo collection.render(hashedName)
+  collection.unload(hashedName)

src/tim/app/liveCmd.nim

@@ -0,0 +1,23 @@
+import std/[os, strutils]
+import pkg/nyml
+import pkg/kapsis/[cli, runtime] 
+
+import ../server/[app, config]
+import ../engine/meta
+
+proc runCommand*(v: Values) =
+  ## Run a new Universal Tim Engine microservice
+  ## in the background using the `tcp` socket.
+  ## 
+  ## This feature is powered by ZeroMQ and makes Tim Engine
+  ## available from any programming language that implements `libzmq`.
+  ## 
+  ## More details about ZeroMQ check [https://github.com/zeromq](https://github.com/zeromq)
+  let path = absolutePath(v.get("config").getPath.path)
+  let config = fromYaml(path.readFile, TimConfig)
+  var timEngine = newTim(
+    config.source,
+    config.output,
+    path.parentDir
+  )
+  app.run(timEngine, config)

src/tim/app/srcCmd.nim

@@ -0,0 +1,77 @@
+import std/[os, strutils]
+import pkg/kapsis/[cli, runtime] 
+import ../engine/parser
+import ../engine/logging
+import ../engine/compilers/[html, nimc]
+
+proc srcCommand*(v: Values) = 
+  ## Transpiles a `.timl` file to a target source
+  let
+    fpath = v.get("timl").getStr
+    ext = v.get("ext").getStr
+    pretty = v.has("pretty")
+    # enableWatcher = v.has("w")
+  var
+    name: string
+    timlCode: string
+  if v.has"code":
+    timlCode = fpath
+  else:
+    name = fpath
+    timlCode = readFile(getCurrentDir() / fpath)
+  let p = parseSnippet(name, timlCode)
+  if likely(not p.hasErrors):
+    if ext == "html":
+      let c = html.newCompiler(parser.getAst(p), pretty == false)
+      if likely(not c.hasErrors):
+        display c.getHtml().strip
+        discard
+      else:
+        for err in c.logger.errors:
+          display err
+        displayInfo c.logger.filePath
+        quit(1)
+    elif ext == "nim":
+      let c = nimc.newCompiler(parser.getAst(p))
+      display c.exportCode()
+    else:
+      displayError("Unknown target `" & ext & "`")
+      quit(1)
+  else:
+    for err in p.logger.errors:
+      display(err)
+    displayInfo p.logger.filePath
+    quit(1)
+
+
+# import std/critbits
+
+# type
+#   ViewHandle* = proc(): string
+#   LayoutHandle* = proc(viewHtml: string): string
+#   ViewsTree* = CritBitTree[ViewHandle]
+#   LayoutsTree* = CritBitTree[LayoutHandle]
+
+# proc getIndex(): string =
+#   result = "view html"
+
+# var views = ViewsTree()
+# views["index"] = getIndex
+
+# proc getBaseLayout(viewHtml: string): string =
+#   result = "start layout"
+#   add result, viewHtml
+#   add result, "end layout"
+
+# var layouts = LayoutsTree()
+# layouts["base"] = getBaseLayout
+
+# template render*(viewName: string, layoutName = "base"): untyped =
+#   if likely(views.hasKey(viewName)):
+#     let viewHtml = views[viewName]()
+#     if likely(layouts.hasKey(layoutName)):
+#       layouts[layoutName](viewHtml)
+#     else: ""
+#   else: ""
+
+# echo render("index")

src/tim/server/app.nim

@@ -0,0 +1,211 @@
+import std/[os, strutils, sequtils, json, critbits]
+import pkg/[zmq, watchout, jsony]
+import pkg/kapsis/[cli]
+
+import ./config
+import ../engine/[meta, parser, logging]
+import ../engine/compilers/[html, nimc]
+
+type
+  CacheTable = CritBitTree[string]
+
+var Cache = CacheTable()
+const
+  address = "tcp://127.0.0.1:5559"
+  DOCKTYPE = "<!DOCKTYPE html>"
+  defaultLayout = "base"
+
+template displayErrors(l: Logger) =
+  for err in l.errors:
+    display(err)
+  display(l.filePath)
+
+proc transpileCode(engine: TimEngine, tpl: TimTemplate,
+    config: TimConfig, refreshAst = false) =
+  ## Transpile `tpl` TimTemplate to a specific target source
+  var p: Parser = engine.newParser(tpl, refreshAst = refreshAst)
+  if likely(not p.hasError):
+    case config.target
+    of tsHtml:
+      # When `HTML` is the preferred target source
+      # Tim Engine will run as a microservice app in background
+      # powered by Zero MQ. The pre-compiled templates are stored
+      # in a Cache table for when rendering is needed.
+      # echo engine.getTargetSourcePath(tpl, config.output, $config.target)
+      if tpl.jitEnabled():
+        # if marked as JIT will save the produced
+        # binary AST on disk for runtime computation
+        engine.writeAst(tpl, parser.getAst(p))
+      else:
+        # otherwise, compiles AST to static HTML
+        var c = html.newCompiler(engine, parser.getAst(p), tpl,
+          engine.isMinified, engine.getIndentSize)
+        if likely(not c.hasError):
+          case tpl.getType:
+            of ttView:
+              engine.writeHtml(tpl, c.getHtml)
+            of ttLayout:
+              engine.writeHtml(tpl, c.getHead)
+            else: discard
+        else:
+          c.logger.displayErrors()
+      # Cache[tpl.getHash()] = c.getHtml().strip
+    of tsNim:
+      let c = nimc.newCompiler(parser.getAst(p))
+      writeFile(engine.getTargetSourcePath(tpl, config.output, $config.target), c.exportCode())
+    else: discard
+  else: p.logger.displayErrors()
+
+proc resolveDependants(engine: TimEngine,
+    deps: seq[string], config: TimConfig) =
+  for path in deps:
+    let tpl = engine.getTemplateByPath(path)
+    case tpl.getType
+    of ttPartial:
+      echo tpl.getDeps.toSeq
+      engine.resolveDependants(tpl.getDeps.toSeq, config)
+    else:
+      engine.transpileCode(tpl, config, true)
+
+proc precompile(engine: TimEngine, config: TimConfig, globals: JsonNode) =
+  ## Pre-compiles available templates
+  engine.setGlobalData(globals)
+  proc notify(label, fname: string) =
+    echo label
+    echo indent(fname & "\n", 3)
+
+  # Callback `onFound`
+  proc onFound(file: watchout.File) =
+    # Runs when detecting a new template.
+    let tpl: TimTemplate =
+        engine.getTemplateByPath(file.getPath())
+    case tpl.getType
+    of ttView, ttLayout:
+      engine.transpileCode(tpl, config)
+    else: discard
+
+  # Callback `onChange`
+  proc onChange(file: watchout.File) =
+    # Runs when detecting changes
+    let tpl: TimTemplate = engine.getTemplateByPath(file.getPath())
+    notify("✨ Changes detected", file.getName())
+    case tpl.getType()
+    of ttView, ttLayout:
+      engine.transpileCode(tpl, config)
+    else:
+      engine.resolveDependants(tpl.getDeps.toSeq, config)
+
+  # Callback `onDelete`
+  proc onDelete(file: watchout.File) =
+    # Runs when deleting a file
+    notify("✨ Deleted", file.getName())
+    engine.clearTemplateByPath(file.getPath())
+
+  var watcher =
+    newWatchout(
+      @[engine.getSourcePath() / "*"],
+      onChange, onFound, onDelete,
+      recursive = true,
+      ext = @["timl"], delay = config.sync.delay,
+      browserSync =
+        WatchoutBrowserSync(
+          port: config.sync.port,
+          delay: config.sync.delay
+        )
+      )
+  # watch for file changes in a separate thread
+  watcher.start(config.target != tsHtml)
+
+proc jitCompiler(engine: TimEngine,
+    tpl: TimTemplate, data: JsonNode): HtmlCompiler =
+  ## Compiles `tpl` AST at runtime
+  html.newCompiler(
+    engine,
+    engine.readAst(tpl),
+    tpl,
+    engine.isMinified,
+    engine.getIndentSize,
+    data
+  )
+
+template layoutWrapper(getViewBlock) {.dirty.} =
+  result = DOCKTYPE
+  var layoutTail: string
+  var hasError: bool
+  if not layout.jitEnabled:
+    # when requested layout is pre-rendered
+    # will use the static HTML version from disk
+    add result, layout.getHtml()
+    getViewBlock
+    layoutTail = layout.getTail()
+  else:
+    var jitLayout = engine.jitCompiler(layout, data)
+    if likely(not jitLayout.hasError):
+      add result, jitLayout.getHead()
+      getViewBlock
+      layoutTail = jitLayout.getTail()
+    else:
+      hasError = true
+      jitLayout.logger.displayErrors()
+  add result, layoutTail
+
+proc render(engine: TimEngine, viewName, layoutName: string, local = newJObject()): string =
+  # Renders a `viewName`
+  if likely(engine.hasView(viewName)):
+    var
+      view: TimTemplate = engine.getView(viewName)
+      data: JsonNode = newJObject()
+    data["local"] = local
+    if likely(engine.hasLayout(layoutName)):
+      var layout: TimTemplate = engine.getLayout(layoutName)
+      if not view.jitEnabled:
+        # render a pre-compiled HTML
+        layoutWrapper:
+          add result, indent(view.getHtml(), layout.getViewIndent)
+      else:
+        # compile and render template at runtime
+        layoutWrapper:
+          var jitView = engine.jitCompiler(view, data)
+          if likely(not jitView.hasError):
+            add result, indent(jitView.getHtml(), layout.getViewIndent)
+          else:
+            jitView.logger.displayErrors()
+            hasError = true
+  else:
+    raise newException(TimError, "View not found")
+
+proc run*(engine: var TimEngine, config: TimConfig) =
+  config.output = normalizedPath(engine.getBasePath / config.output)
+  case config.target
+  of tsHtml:
+    display("Tim Engine is running at " & address)
+    var rep = listen(address, mode = REP)
+    var hasGlobalStorage: bool
+    defer: rep.close()
+    while true:
+      let req = rep.receiveAll()
+      try:
+        let command = req[0]
+        case command
+        of "render":
+          let local = req[3].fromJson
+          let output = engine.render(req[1], req[2], local)
+          rep.send(output)
+        of "global.storage": # runs once
+          if not hasGlobalStorage:
+            let globals = req[1].fromJson
+            engine.precompile(config, globals)
+            hasGlobalStorage = true
+            rep.send("")
+          else:
+            rep.send("")
+        else: discard # unknown command error ?
+      except TimError as e:
+        rep.send(e.msg)
+      sleep(10)
+  else:
+    discard existsOrCreateDir(config.output / "views")
+    discard existsOrCreateDir(config.output / "layouts")
+    discard existsOrCreateDir(config.output / "partials")
+    display("Tim Engine is running Source-to-Source")
+    engine.precompile(config, newJObject())