WIP: Component System

WIP: Component system

Most of the first draft is done. The major exception for the component info is the treecheck code for param blocks.
I also have started a bit of the support work that needs to be done, such as removing static lock checks (the dynamic ones need to be redone still).

.gitignore

@@ -5,3 +5,4 @@ tests/t[^.]*[^m]
 tags
 *.dSYM
 /con4m
+files/con4m/parse

files/con4m/components.nim

@@ -0,0 +1,134 @@
+import os, tables, types, nimutils, uri, lex
+
+# This has some cyclic dependencies, so we make sure C prototypes get
+# generated with local scope only; we then do not import those modules.
+
+proc parse(s: Stream, filename: string): Con4mNode {.importc, cdecl.}
+proc checkTree(node: Con4mNode, s: ConfigState) {.importc, cdecl.}
+
+var
+  defaultUrlStore = ""
+  componentInfo   = Table[string, ComponentInfo]
+  programRoot: ComponentInfo
+
+proc fullComponentSpec*(name, location: string): string =
+  if location == "":
+    if defaultUrlStore != "":
+      result = defaultUrlStore
+    else:
+      result = getAppDir().resolvePath()
+
+  elif location.startswith("https://"):
+    result = location
+
+  else:
+    result = location.resolvePath()
+
+proc setDefaultStoreUrl*(url: string) =
+  once:
+    defaultUrlStore = url
+
+proc getComponentReference*(name: string, location: string): ComponentInfo =
+  let key = fullComponentSpec(name, location)
+
+  if key notin componentInfo:
+    componentInfo[key] = ComponentInfo(url: key)
+
+  return componentInfo[key]
+
+proc fetchAttempt(url, extension: string): string =
+  var
+    finalExt = if extension.startswith("."): extension else "." & extension
+    uri      = parseUri(url & finalExt)
+    context  = newContext(verifyMode = CVerifyPeer)
+    client   = newHttpClient(sslContext = context, timeout = 1000)
+    response = client.safeRequest(url = uri, httpMethod = HttpGet)
+
+  if response.status[0] != '2':
+    return ""
+
+  return response.bodyStream.readAll()
+
+proc fetchComponent*(name, location: string, extension = ".c4m"):
+                   ComponentInfo =
+  ## This returns a parsed component, but does NOT go beyond that.  The
+  ## parse phase will NOT attempt to semantically validate a component,
+  ## will NOT go and fetch dependent comonents, and will NOT do cycle
+  ## checking at all. Use loadComponent below for those.
+  ##
+  ## This will raise an exception if anything goes wrong.
+
+  result = getComponentReference(name, location)
+
+  if result.url.startsWith("https://"):
+    if checkForUpdates or result.hash == "":
+      let
+        source = result.url.fetchAttempt(extension)
+        hash   = sha256(source)
+
+      if source == "":
+        if result.hash == "":
+          raise newException(IOError, "Could not retrieve needed source " &
+            "file: " & result.url)
+
+      elif hash == result.hash:
+        return
+
+      result.hash   = hash
+      result.source = source
+
+  else:
+    try:
+      let
+        source = result.url.readFile()
+        hash   = sha256(source)
+
+      if hash == result.hash:
+        return
+
+      result.hash   = hash
+      result.source = source
+    except:
+      if result.hash == "":
+        raise newException(IOError, "Could not retrieve needed source " &
+          "file: " & result.url)
+
+  let
+    stream          = newStringStream(result.source)
+    (valid, toks)   = stream.lex(result.url)
+
+  if not valid:
+    let msg = case tokens[^1].kind
+    of ErrorTok:         "Invalid character found"
+    of ErrorLongComment: "Unterminated comment"
+    of ErrorStringLit:   "Unterminated string"
+    of ErrorCharLit:     "Invalid char literal"
+    of ErrorOtherLit:    "Unterminated literal"
+    else:                "Unknown error" # Not be possible w/o a lex bug
+
+  result.entrypoint = tokens.parse(result.url)
+
+proc loadComponent*(s: ConfigState, component: ComponentInfo) =
+  let savedComponent = s.currentComponent
+
+  if not result.typed:
+    state.currentComponent = result
+    state.currentComponent.entrypoint.checkTree(state)
+    state.currentComponent.typed = true
+    for component in state.currentComponent.componentsUsed:
+      s.loadComponent(component)
+    state.currentComponent = savedComponent
+
+  # Now that any sub-components have loaded, we *could* do a check for
+  # cycles, but we are currently skipping it. It'll be runtime-only
+  # for now.
+
+proc loadComponent*(s: ConfigState, name, location: string, extension = ".c4m"):
+                  ComponentInfo =
+
+  if s.currentComponent == "":
+    # Main file; register the component.
+    registerComponent(name, location)
+
+  result = fetchComponent(name, location, extension)
+  s.loadComponent(result)

files/con4m/dollars.nim

@@ -205,6 +205,8 @@ proc `$`*(self: Con4mNode, i: int = 0): string =
   of NodeVarDecl:      fmtNt("VarDecl")
   of NodeExportDecl:   fmtNt("ExportDecl")
   of NodeVarSymNames:  fmtNt("VarSymNames")
+  of NodeUse:          fmtNt("Use")
+  of NodeParameter:    fmtNt("Parameter")
   of NodeOr, NodeAnd, NodeNe, NodeCmp, NodeGte, NodeLte, NodeGt,
      NodeLt, NodePlus, NodeMinus, NodeMod, NodeMul, NodeDiv:
     fmtNt($(self.token.get()))

files/con4m/eval.nim

@@ -6,7 +6,7 @@
 ## :Copyright: 2022
 
 import options, tables, strformat, unicode, nimutils, types, st, parse,
-       treecheck, typecheck, dollars, errmsg
+       treecheck, typecheck, dollars, errmsg, components
 
 when (NimMajor, NimMinor) >= (1, 7):
   {.warning[CastSizes]: off.}
@@ -200,6 +200,36 @@ template binaryOpWork(typeWeAreOping: typedesc,
 
   node.value = pack(ret)
 
+proc evalNode*(node: Con4mNode, s: ConfigState)
+
+proc evalComponent*(s: ConfigState, module: string, url: string = "") =
+  let
+    savedGlobals    = s.keptGlobals
+    savedScopes     = s.frames
+    savedComponent  = s.currentComponent
+    savedFuncOrigin = s.funcOrigin
+
+  s.currentComponent = getComponentReference(module, url)
+  s.funcOrigin       = false
+  s.keptGlobals     =  s.currentComponent.savedVars
+
+  var newFrame = RuntimeFrame()
+  s.frames     = @[newFrame]
+
+  for k, v in s.currentComponent.savedVars:
+    newFrame[k] = v.value
+
+  s.evalNode(s.currentComponent.entrypoint, s)
+
+  if len(s.frames) > 0:
+    for k, v in s.frames[0]:
+      s.currentComponent.savedVars[k] = v
+
+  s.funcOrigin       = savedFuncOrigin
+  s.currentComponent = savedComponent
+  s.frames           = savedScopes
+  s.keptGlobals      = savedGlobals
+
 proc evalNode*(node: Con4mNode, s: ConfigState) =
   ## This does the bulk of the work.  Typically, it will descend into
   ## the tree to evaluate, and then take whatever action is
@@ -208,8 +238,18 @@ proc evalNode*(node: Con4mNode, s: ConfigState) =
   # These are explicit just to make sure I don't end up w/ implementation
   # errors that baffle me.
   of NodeFuncDef, NodeType, NodeVarDecl, NodeExportDecl, NodeVarSymNames,
-       NodeCallbackLit:
+       NodeCallbackLit, NodeParameter:
     return # Nothing to do, everything was done in the check phase.
+  of NodeUse:
+    let
+      kids      = node.children
+      module    = kids[0].getTokenText()
+
+    if len(kids) == 1:
+      s.evalComponent(module)
+    else:
+      s.evalComponent(module, kids[1].getTokenText())
+
   of NodeReturn:
     if node.children.len() != 0:
       node.children[0].evalNode(s)
@@ -581,3 +621,10 @@ proc evalNode*(node: Con4mNode, s: ConfigState) =
         node.value = node.attrRef.value.get()
     else:
       node.value = s.runtimeVarLookup(node.getTokenText())
+
+proc evalComponent*(s: ConfigState, name, location: string,
+                    extension = ".c4m") =
+  let
+    component = s.loadComponent(name, location, extension)
+
+  component.entrypoint.evalNode(s)

files/con4m/lex.nim

@@ -580,6 +580,8 @@ proc lex*(s: Stream, filename: string = ""): (bool, seq[Con4mToken]) =
       of "else":           tok(TtElse)
       of "for":            tok(TtFor)
       of "from":           tok(TtFrom)
+      of "use":            tok(TtUse)
+      of "parameter":      tok(TtParameter)
       of "to":             tok(TtTo)
       of "break":          tok(TtBreak)
       of "continue":       tok(TtContinue)