Implement comments

promptware · Nov 12, 2024 · 29ce064 · 29ce064
1 parent 310d264
commit 29ce064
Show file tree

Hide file tree

Showing 12 changed files with 262 additions and 129 deletions.
diff --git a/README.md b/README.md
@@ -46,7 +46,7 @@ See [`python/examples/`](./python/examples/) for more.
 
 This is an early work-in-progress. Follow [me on twitter](https://x.com/klntsky) for updates
 
-- [ ] Finalize and specify the initial version of the syntax
+- [ ] Specify the initial version of the syntax
 - [ ] Implement a parser
   - [x] implement parse tree -> AST conversion
   - [ ] return error throwing to the parser
@@ -56,7 +56,10 @@ This is an early work-in-progress. Follow [me on twitter](https://x.com/klntsky)
   - [x] `[:if ... :then ... :else ...]`
   - [x] `[$ meta-prompt]`
   - [x] `[:use module :param1=value1]`
+  - [ ] `[:model model-id ...]` for dynamic model selection
   - [ ] `[# comments]`
+  - [ ] `[:status some-status]` - to show during prompt evaluation
+  - [ ] `[:call ffi-function :param1=foo :param2=bar]`
 - [ ] Implement an evaluator
   - [x] meta-prompting
   - [x] conditionals
@@ -72,12 +75,22 @@ This is an early work-in-progress. Follow [me on twitter](https://x.com/klntsky)
   - [ ] exceptions
     - [ ] throwing exceptions
     - [ ] recovering from exceptions
+- [ ] FFI
+  - [ ] syntax
+  - [ ] API
+  - [ ] standard library
 - Utils
   - [x] Unbound variable auto discovery to turn metaprompts into interfaces
 - [ ] Add function definitions
   - [ ] enable function scopes
 - [ ] Add a module system
+  - [x] syntax
+  - [x] runtime
+  - [ ] tests
 - [ ] Add a package system
+  - [ ] specify package format
+  - [ ] create a package registry
+  - [ ] package installer
 
 ## Notable sources of inspiration
 

diff --git a/grammar/MetaPrompt.g4 b/grammar/MetaPrompt.g4
@@ -6,6 +6,7 @@ expr: LB expr1 RB
     | text
     | RB
     | LB
+    | COMMENT_KW
     ;
 
 expr1
@@ -18,6 +19,7 @@ meta_body
     | IF_KW exprs THEN_KW exprs
     | USE parameters?
     | META_KW exprs
+    | COMMENT_KW exprs
     | VAR_NAME EQ_KW exprs
     | VAR_NAME
     ;
@@ -32,6 +34,7 @@ LB : '[';
 RB : ']';
 EQ_KW : '=' ;
 META_KW : '$' ;
+COMMENT_KW : '#' ;
 CHAR : . ;
 USE : ':use' WS+ [a-zA-Z0-9/_.-]+ WS*;
 fragment WS : ' '|'\n';

diff --git a/python/src/eval.py b/python/src/eval.py
@@ -2,6 +2,7 @@
 from env import Env
 from runtime import Runtime
 from typing import AsyncGenerator
+from loader import extract_variables
 
 async def _eval_exprs(exprs, runtime):
     """A helper for eval_ast"""
@@ -42,7 +43,20 @@ async def eval_ast(ast, runtime):
     elif ast["type"] == "use":
         parameters = ast["parameters"]
         module_name = ast["module_name"]
-        raise NotImplementedError("[:use ...] not implemented yet")
+        loaded_ast = runtime.load_module(module_name)
+        required_variables = extract_variables(loaded_ast)
+        for required in required_variables:
+            if required not in parameters:
+                raise ImportError(
+                    f"Module {module_name} requires {required} as a parameter, but it was not provided"
+                )
+        old_env = runtime.env
+        # TODO: persist some variables?
+        runtime.env = Env(parameters)
+        async for expr in eval_ast(loaded_ast, runtime):
+            yield expr
+        runtime.env = old_env
+
     elif ast["type"] == "assign":
         var_name = ast["name"]
         value = await _collect_exprs(ast['exprs'], runtime)

diff --git a/python/src/parse_metaprompt.py b/python/src/parse_metaprompt.py
@@ -46,6 +46,8 @@ def visitExpr(self, ctx: MetaPromptParser.ExprContext):
                 items.append({"type": "text", "text": "]"})
             if ctx.LB() is not None:
                 items.append({"type": "text", "text": "["})
+            if ctx.COMMENT_KW() is not None:
+                items.append({"type": "text", "text": "#"})
         return items
 
     def visitExpr1(self, ctx: MetaPromptParser.Expr1Context):
@@ -83,6 +85,12 @@ def visitMeta_body(self, ctx: MetaPromptParser.Meta_bodyContext):
                 "then": then_node,
                 "else": [],
             }
+        elif ctx.COMMENT_KW() is not None:
+            exprs = self.visit(exprs_list[0])
+            return {
+                "type": "comment",
+                "exprs": exprs
+            }
         elif ctx.USE() is not None:
             module_name = ctx.USE().getText().removeprefix(':use').strip()
             parameters = {}

diff --git a/python/src/parser/MetaPrompt.interp b/python/src/parser/MetaPrompt.interp
@@ -4,6 +4,7 @@ null
 ']'
 '='
 '$'
+'#'
 null
 null
 ':if'
@@ -17,6 +18,7 @@ LB
 RB
 EQ_KW
 META_KW
+COMMENT_KW
 CHAR
 USE
 IF_KW
@@ -35,4 +37,4 @@ text
 
 
 atn:
-[4, 1, 10, 73, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 1, 0, 1, 0, 1, 0, 1, 1, 5, 1, 19, 8, 1, 10, 1, 12, 1, 22, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 31, 8, 2, 1, 3, 1, 3, 3, 3, 35, 8, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 3, 4, 51, 8, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 3, 4, 59, 8, 4, 1, 5, 1, 5, 1, 5, 4, 5, 64, 8, 5, 11, 5, 12, 5, 65, 1, 6, 4, 6, 69, 8, 6, 11, 6, 12, 6, 70, 1, 6, 1, 20, 0, 7, 0, 2, 4, 6, 8, 10, 12, 0, 0, 78, 0, 14, 1, 0, 0, 0, 2, 20, 1, 0, 0, 0, 4, 30, 1, 0, 0, 0, 6, 34, 1, 0, 0, 0, 8, 58, 1, 0, 0, 0, 10, 63, 1, 0, 0, 0, 12, 68, 1, 0, 0, 0, 14, 15, 3, 2, 1, 0, 15, 16, 5, 0, 0, 1, 16, 1, 1, 0, 0, 0, 17, 19, 3, 4, 2, 0, 18, 17, 1, 0, 0, 0, 19, 22, 1, 0, 0, 0, 20, 21, 1, 0, 0, 0, 20, 18, 1, 0, 0, 0, 21, 3, 1, 0, 0, 0, 22, 20, 1, 0, 0, 0, 23, 24, 5, 1, 0, 0, 24, 25, 3, 6, 3, 0, 25, 26, 5, 2, 0, 0, 26, 31, 1, 0, 0, 0, 27, 31, 3, 12, 6, 0, 28, 31, 5, 2, 0, 0, 29, 31, 5, 1, 0, 0, 30, 23, 1, 0, 0, 0, 30, 27, 1, 0, 0, 0, 30, 28, 1, 0, 0, 0, 30, 29, 1, 0, 0, 0, 31, 5, 1, 0, 0, 0, 32, 35, 3, 8, 4, 0, 33, 35, 3, 2, 1, 0, 34, 32, 1, 0, 0, 0, 34, 33, 1, 0, 0, 0, 35, 7, 1, 0, 0, 0, 36, 37, 5, 7, 0, 0, 37, 38, 3, 2, 1, 0, 38, 39, 5, 8, 0, 0, 39, 40, 3, 2, 1, 0, 40, 41, 5, 9, 0, 0, 41, 42, 3, 2, 1, 0, 42, 59, 1, 0, 0, 0, 43, 44, 5, 7, 0, 0, 44, 45, 3, 2, 1, 0, 45, 46, 5, 8, 0, 0, 46, 47, 3, 2, 1, 0, 47, 59, 1, 0, 0, 0, 48, 50, 5, 6, 0, 0, 49, 51, 3, 10, 5, 0, 50, 49, 1, 0, 0, 0, 50, 51, 1, 0, 0, 0, 51, 59, 1, 0, 0, 0, 52, 53, 5, 4, 0, 0, 53, 59, 3, 2, 1, 0, 54, 55, 5, 10, 0, 0, 55, 56, 5, 3, 0, 0, 56, 59, 3, 2, 1, 0, 57, 59, 5, 10, 0, 0, 58, 36, 1, 0, 0, 0, 58, 43, 1, 0, 0, 0, 58, 48, 1, 0, 0, 0, 58, 52, 1, 0, 0, 0, 58, 54, 1, 0, 0, 0, 58, 57, 1, 0, 0, 0, 59, 9, 1, 0, 0, 0, 60, 61, 5, 10, 0, 0, 61, 62, 5, 3, 0, 0, 62, 64, 3, 2, 1, 0, 63, 60, 1, 0, 0, 0, 64, 65, 1, 0, 0, 0, 65, 63, 1, 0, 0, 0, 65, 66, 1, 0, 0, 0, 66, 11, 1, 0, 0, 0, 67, 69, 5, 5, 0, 0, 68, 67, 1, 0, 0, 0, 69, 70, 1, 0, 0, 0, 70, 68, 1, 0, 0, 0, 70, 71, 1, 0, 0, 0, 71, 13, 1, 0, 0, 0, 7, 20, 30, 34, 50, 58, 65, 70]
+[4, 1, 11, 76, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 1, 0, 1, 0, 1, 0, 1, 1, 5, 1, 19, 8, 1, 10, 1, 12, 1, 22, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 32, 8, 2, 1, 3, 1, 3, 3, 3, 36, 8, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 3, 4, 52, 8, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 3, 4, 62, 8, 4, 1, 5, 1, 5, 1, 5, 4, 5, 67, 8, 5, 11, 5, 12, 5, 68, 1, 6, 4, 6, 72, 8, 6, 11, 6, 12, 6, 73, 1, 6, 1, 20, 0, 7, 0, 2, 4, 6, 8, 10, 12, 0, 0, 83, 0, 14, 1, 0, 0, 0, 2, 20, 1, 0, 0, 0, 4, 31, 1, 0, 0, 0, 6, 35, 1, 0, 0, 0, 8, 61, 1, 0, 0, 0, 10, 66, 1, 0, 0, 0, 12, 71, 1, 0, 0, 0, 14, 15, 3, 2, 1, 0, 15, 16, 5, 0, 0, 1, 16, 1, 1, 0, 0, 0, 17, 19, 3, 4, 2, 0, 18, 17, 1, 0, 0, 0, 19, 22, 1, 0, 0, 0, 20, 21, 1, 0, 0, 0, 20, 18, 1, 0, 0, 0, 21, 3, 1, 0, 0, 0, 22, 20, 1, 0, 0, 0, 23, 24, 5, 1, 0, 0, 24, 25, 3, 6, 3, 0, 25, 26, 5, 2, 0, 0, 26, 32, 1, 0, 0, 0, 27, 32, 3, 12, 6, 0, 28, 32, 5, 2, 0, 0, 29, 32, 5, 1, 0, 0, 30, 32, 5, 5, 0, 0, 31, 23, 1, 0, 0, 0, 31, 27, 1, 0, 0, 0, 31, 28, 1, 0, 0, 0, 31, 29, 1, 0, 0, 0, 31, 30, 1, 0, 0, 0, 32, 5, 1, 0, 0, 0, 33, 36, 3, 8, 4, 0, 34, 36, 3, 2, 1, 0, 35, 33, 1, 0, 0, 0, 35, 34, 1, 0, 0, 0, 36, 7, 1, 0, 0, 0, 37, 38, 5, 8, 0, 0, 38, 39, 3, 2, 1, 0, 39, 40, 5, 9, 0, 0, 40, 41, 3, 2, 1, 0, 41, 42, 5, 10, 0, 0, 42, 43, 3, 2, 1, 0, 43, 62, 1, 0, 0, 0, 44, 45, 5, 8, 0, 0, 45, 46, 3, 2, 1, 0, 46, 47, 5, 9, 0, 0, 47, 48, 3, 2, 1, 0, 48, 62, 1, 0, 0, 0, 49, 51, 5, 7, 0, 0, 50, 52, 3, 10, 5, 0, 51, 50, 1, 0, 0, 0, 51, 52, 1, 0, 0, 0, 52, 62, 1, 0, 0, 0, 53, 54, 5, 4, 0, 0, 54, 62, 3, 2, 1, 0, 55, 56, 5, 5, 0, 0, 56, 62, 3, 2, 1, 0, 57, 58, 5, 11, 0, 0, 58, 59, 5, 3, 0, 0, 59, 62, 3, 2, 1, 0, 60, 62, 5, 11, 0, 0, 61, 37, 1, 0, 0, 0, 61, 44, 1, 0, 0, 0, 61, 49, 1, 0, 0, 0, 61, 53, 1, 0, 0, 0, 61, 55, 1, 0, 0, 0, 61, 57, 1, 0, 0, 0, 61, 60, 1, 0, 0, 0, 62, 9, 1, 0, 0, 0, 63, 64, 5, 11, 0, 0, 64, 65, 5, 3, 0, 0, 65, 67, 3, 2, 1, 0, 66, 63, 1, 0, 0, 0, 67, 68, 1, 0, 0, 0, 68, 66, 1, 0, 0, 0, 68, 69, 1, 0, 0, 0, 69, 11, 1, 0, 0, 0, 70, 72, 5, 6, 0, 0, 71, 70, 1, 0, 0, 0, 72, 73, 1, 0, 0, 0, 73, 71, 1, 0, 0, 0, 73, 74, 1, 0, 0, 0, 74, 13, 1, 0, 0, 0, 7, 20, 31, 35, 51, 61, 68, 73]
diff --git a/python/src/parser/MetaPrompt.tokens b/python/src/parser/MetaPrompt.tokens
@@ -2,16 +2,18 @@ LB=1
 RB=2
 EQ_KW=3
 META_KW=4
-CHAR=5
-USE=6
-IF_KW=7
-THEN_KW=8
-ELSE_KW=9
-VAR_NAME=10
+COMMENT_KW=5
+CHAR=6
+USE=7
+IF_KW=8
+THEN_KW=9
+ELSE_KW=10
+VAR_NAME=11
 '['=1
 ']'=2
 '='=3
 '$'=4
-':if'=7
-':then'=8
-':else'=9
+'#'=5
+':if'=8
+':then'=9
+':else'=10
diff --git a/python/src/parser/MetaPromptLexer.interp b/python/src/parser/MetaPromptLexer.interp
@@ -4,6 +4,7 @@ null
 ']'
 '='
 '$'
+'#'
 null
 null
 ':if'
@@ -17,6 +18,7 @@ LB
 RB
 EQ_KW
 META_KW
+COMMENT_KW
 CHAR
 USE
 IF_KW
@@ -29,6 +31,7 @@ LB
 RB
 EQ_KW
 META_KW
+COMMENT_KW
 CHAR
 USE
 WS
@@ -45,4 +48,4 @@ mode names:
 DEFAULT_MODE
 
 atn:
-[4, 0, 10, 80, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 1, 0, 1, 0, 1, 1, 1, 1, 1, 2, 1, 2, 1, 3, 1, 3, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 4, 5, 40, 8, 5, 11, 5, 12, 5, 41, 1, 5, 4, 5, 45, 8, 5, 11, 5, 12, 5, 46, 1, 5, 5, 5, 50, 8, 5, 10, 5, 12, 5, 53, 9, 5, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 5, 10, 76, 8, 10, 10, 10, 12, 10, 79, 9, 10, 0, 0, 11, 1, 1, 3, 2, 5, 3, 7, 4, 9, 5, 11, 6, 13, 0, 15, 7, 17, 8, 19, 9, 21, 10, 1, 0, 4, 4, 0, 45, 57, 65, 90, 95, 95, 97, 122, 2, 0, 10, 10, 32, 32, 2, 0, 65, 90, 97, 122, 3, 0, 48, 57, 65, 90, 97, 122, 82, 0, 1, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 1, 23, 1, 0, 0, 0, 3, 25, 1, 0, 0, 0, 5, 27, 1, 0, 0, 0, 7, 29, 1, 0, 0, 0, 9, 31, 1, 0, 0, 0, 11, 33, 1, 0, 0, 0, 13, 54, 1, 0, 0, 0, 15, 56, 1, 0, 0, 0, 17, 60, 1, 0, 0, 0, 19, 66, 1, 0, 0, 0, 21, 72, 1, 0, 0, 0, 23, 24, 5, 91, 0, 0, 24, 2, 1, 0, 0, 0, 25, 26, 5, 93, 0, 0, 26, 4, 1, 0, 0, 0, 27, 28, 5, 61, 0, 0, 28, 6, 1, 0, 0, 0, 29, 30, 5, 36, 0, 0, 30, 8, 1, 0, 0, 0, 31, 32, 9, 0, 0, 0, 32, 10, 1, 0, 0, 0, 33, 34, 5, 58, 0, 0, 34, 35, 5, 117, 0, 0, 35, 36, 5, 115, 0, 0, 36, 37, 5, 101, 0, 0, 37, 39, 1, 0, 0, 0, 38, 40, 3, 13, 6, 0, 39, 38, 1, 0, 0, 0, 40, 41, 1, 0, 0, 0, 41, 39, 1, 0, 0, 0, 41, 42, 1, 0, 0, 0, 42, 44, 1, 0, 0, 0, 43, 45, 7, 0, 0, 0, 44, 43, 1, 0, 0, 0, 45, 46, 1, 0, 0, 0, 46, 44, 1, 0, 0, 0, 46, 47, 1, 0, 0, 0, 47, 51, 1, 0, 0, 0, 48, 50, 3, 13, 6, 0, 49, 48, 1, 0, 0, 0, 50, 53, 1, 0, 0, 0, 51, 49, 1, 0, 0, 0, 51, 52, 1, 0, 0, 0, 52, 12, 1, 0, 0, 0, 53, 51, 1, 0, 0, 0, 54, 55, 7, 1, 0, 0, 55, 14, 1, 0, 0, 0, 56, 57, 5, 58, 0, 0, 57, 58, 5, 105, 0, 0, 58, 59, 5, 102, 0, 0, 59, 16, 1, 0, 0, 0, 60, 61, 5, 58, 0, 0, 61, 62, 5, 116, 0, 0, 62, 63, 5, 104, 0, 0, 63, 64, 5, 101, 0, 0, 64, 65, 5, 110, 0, 0, 65, 18, 1, 0, 0, 0, 66, 67, 5, 58, 0, 0, 67, 68, 5, 101, 0, 0, 68, 69, 5, 108, 0, 0, 69, 70, 5, 115, 0, 0, 70, 71, 5, 101, 0, 0, 71, 20, 1, 0, 0, 0, 72, 73, 5, 58, 0, 0, 73, 77, 7, 2, 0, 0, 74, 76, 7, 3, 0, 0, 75, 74, 1, 0, 0, 0, 76, 79, 1, 0, 0, 0, 77, 75, 1, 0, 0, 0, 77, 78, 1, 0, 0, 0, 78, 22, 1, 0, 0, 0, 79, 77, 1, 0, 0, 0, 5, 0, 41, 46, 51, 77, 0]
+[4, 0, 11, 84, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 1, 0, 1, 0, 1, 1, 1, 1, 1, 2, 1, 2, 1, 3, 1, 3, 1, 4, 1, 4, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 4, 6, 44, 8, 6, 11, 6, 12, 6, 45, 1, 6, 4, 6, 49, 8, 6, 11, 6, 12, 6, 50, 1, 6, 5, 6, 54, 8, 6, 10, 6, 12, 6, 57, 9, 6, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 5, 11, 80, 8, 11, 10, 11, 12, 11, 83, 9, 11, 0, 0, 12, 1, 1, 3, 2, 5, 3, 7, 4, 9, 5, 11, 6, 13, 7, 15, 0, 17, 8, 19, 9, 21, 10, 23, 11, 1, 0, 4, 4, 0, 45, 57, 65, 90, 95, 95, 97, 122, 2, 0, 10, 10, 32, 32, 2, 0, 65, 90, 97, 122, 3, 0, 48, 57, 65, 90, 97, 122, 86, 0, 1, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 1, 25, 1, 0, 0, 0, 3, 27, 1, 0, 0, 0, 5, 29, 1, 0, 0, 0, 7, 31, 1, 0, 0, 0, 9, 33, 1, 0, 0, 0, 11, 35, 1, 0, 0, 0, 13, 37, 1, 0, 0, 0, 15, 58, 1, 0, 0, 0, 17, 60, 1, 0, 0, 0, 19, 64, 1, 0, 0, 0, 21, 70, 1, 0, 0, 0, 23, 76, 1, 0, 0, 0, 25, 26, 5, 91, 0, 0, 26, 2, 1, 0, 0, 0, 27, 28, 5, 93, 0, 0, 28, 4, 1, 0, 0, 0, 29, 30, 5, 61, 0, 0, 30, 6, 1, 0, 0, 0, 31, 32, 5, 36, 0, 0, 32, 8, 1, 0, 0, 0, 33, 34, 5, 35, 0, 0, 34, 10, 1, 0, 0, 0, 35, 36, 9, 0, 0, 0, 36, 12, 1, 0, 0, 0, 37, 38, 5, 58, 0, 0, 38, 39, 5, 117, 0, 0, 39, 40, 5, 115, 0, 0, 40, 41, 5, 101, 0, 0, 41, 43, 1, 0, 0, 0, 42, 44, 3, 15, 7, 0, 43, 42, 1, 0, 0, 0, 44, 45, 1, 0, 0, 0, 45, 43, 1, 0, 0, 0, 45, 46, 1, 0, 0, 0, 46, 48, 1, 0, 0, 0, 47, 49, 7, 0, 0, 0, 48, 47, 1, 0, 0, 0, 49, 50, 1, 0, 0, 0, 50, 48, 1, 0, 0, 0, 50, 51, 1, 0, 0, 0, 51, 55, 1, 0, 0, 0, 52, 54, 3, 15, 7, 0, 53, 52, 1, 0, 0, 0, 54, 57, 1, 0, 0, 0, 55, 53, 1, 0, 0, 0, 55, 56, 1, 0, 0, 0, 56, 14, 1, 0, 0, 0, 57, 55, 1, 0, 0, 0, 58, 59, 7, 1, 0, 0, 59, 16, 1, 0, 0, 0, 60, 61, 5, 58, 0, 0, 61, 62, 5, 105, 0, 0, 62, 63, 5, 102, 0, 0, 63, 18, 1, 0, 0, 0, 64, 65, 5, 58, 0, 0, 65, 66, 5, 116, 0, 0, 66, 67, 5, 104, 0, 0, 67, 68, 5, 101, 0, 0, 68, 69, 5, 110, 0, 0, 69, 20, 1, 0, 0, 0, 70, 71, 5, 58, 0, 0, 71, 72, 5, 101, 0, 0, 72, 73, 5, 108, 0, 0, 73, 74, 5, 115, 0, 0, 74, 75, 5, 101, 0, 0, 75, 22, 1, 0, 0, 0, 76, 77, 5, 58, 0, 0, 77, 81, 7, 2, 0, 0, 78, 80, 7, 3, 0, 0, 79, 78, 1, 0, 0, 0, 80, 83, 1, 0, 0, 0, 81, 79, 1, 0, 0, 0, 81, 82, 1, 0, 0, 0, 82, 24, 1, 0, 0, 0, 83, 81, 1, 0, 0, 0, 5, 0, 45, 50, 55, 81, 0]
diff --git a/python/src/parser/MetaPromptLexer.py b/python/src/parser/MetaPromptLexer.py
@@ -10,33 +10,34 @@
 
 def serializedATN():
     return [
-        4,0,10,80,6,-1,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,
-        6,7,6,2,7,7,7,2,8,7,8,2,9,7,9,2,10,7,10,1,0,1,0,1,1,1,1,1,2,1,2,
-        1,3,1,3,1,4,1,4,1,5,1,5,1,5,1,5,1,5,1,5,4,5,40,8,5,11,5,12,5,41,
-        1,5,4,5,45,8,5,11,5,12,5,46,1,5,5,5,50,8,5,10,5,12,5,53,9,5,1,6,
-        1,6,1,7,1,7,1,7,1,7,1,8,1,8,1,8,1,8,1,8,1,8,1,9,1,9,1,9,1,9,1,9,
-        1,9,1,10,1,10,1,10,5,10,76,8,10,10,10,12,10,79,9,10,0,0,11,1,1,3,
-        2,5,3,7,4,9,5,11,6,13,0,15,7,17,8,19,9,21,10,1,0,4,4,0,45,57,65,
-        90,95,95,97,122,2,0,10,10,32,32,2,0,65,90,97,122,3,0,48,57,65,90,
-        97,122,82,0,1,1,0,0,0,0,3,1,0,0,0,0,5,1,0,0,0,0,7,1,0,0,0,0,9,1,
-        0,0,0,0,11,1,0,0,0,0,15,1,0,0,0,0,17,1,0,0,0,0,19,1,0,0,0,0,21,1,
-        0,0,0,1,23,1,0,0,0,3,25,1,0,0,0,5,27,1,0,0,0,7,29,1,0,0,0,9,31,1,
-        0,0,0,11,33,1,0,0,0,13,54,1,0,0,0,15,56,1,0,0,0,17,60,1,0,0,0,19,
-        66,1,0,0,0,21,72,1,0,0,0,23,24,5,91,0,0,24,2,1,0,0,0,25,26,5,93,
-        0,0,26,4,1,0,0,0,27,28,5,61,0,0,28,6,1,0,0,0,29,30,5,36,0,0,30,8,
-        1,0,0,0,31,32,9,0,0,0,32,10,1,0,0,0,33,34,5,58,0,0,34,35,5,117,0,
-        0,35,36,5,115,0,0,36,37,5,101,0,0,37,39,1,0,0,0,38,40,3,13,6,0,39,
-        38,1,0,0,0,40,41,1,0,0,0,41,39,1,0,0,0,41,42,1,0,0,0,42,44,1,0,0,
-        0,43,45,7,0,0,0,44,43,1,0,0,0,45,46,1,0,0,0,46,44,1,0,0,0,46,47,
-        1,0,0,0,47,51,1,0,0,0,48,50,3,13,6,0,49,48,1,0,0,0,50,53,1,0,0,0,
-        51,49,1,0,0,0,51,52,1,0,0,0,52,12,1,0,0,0,53,51,1,0,0,0,54,55,7,
-        1,0,0,55,14,1,0,0,0,56,57,5,58,0,0,57,58,5,105,0,0,58,59,5,102,0,
-        0,59,16,1,0,0,0,60,61,5,58,0,0,61,62,5,116,0,0,62,63,5,104,0,0,63,
-        64,5,101,0,0,64,65,5,110,0,0,65,18,1,0,0,0,66,67,5,58,0,0,67,68,
-        5,101,0,0,68,69,5,108,0,0,69,70,5,115,0,0,70,71,5,101,0,0,71,20,
-        1,0,0,0,72,73,5,58,0,0,73,77,7,2,0,0,74,76,7,3,0,0,75,74,1,0,0,0,
-        76,79,1,0,0,0,77,75,1,0,0,0,77,78,1,0,0,0,78,22,1,0,0,0,79,77,1,
-        0,0,0,5,0,41,46,51,77,0
+        4,0,11,84,6,-1,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,
+        6,7,6,2,7,7,7,2,8,7,8,2,9,7,9,2,10,7,10,2,11,7,11,1,0,1,0,1,1,1,
+        1,1,2,1,2,1,3,1,3,1,4,1,4,1,5,1,5,1,6,1,6,1,6,1,6,1,6,1,6,4,6,44,
+        8,6,11,6,12,6,45,1,6,4,6,49,8,6,11,6,12,6,50,1,6,5,6,54,8,6,10,6,
+        12,6,57,9,6,1,7,1,7,1,8,1,8,1,8,1,8,1,9,1,9,1,9,1,9,1,9,1,9,1,10,
+        1,10,1,10,1,10,1,10,1,10,1,11,1,11,1,11,5,11,80,8,11,10,11,12,11,
+        83,9,11,0,0,12,1,1,3,2,5,3,7,4,9,5,11,6,13,7,15,0,17,8,19,9,21,10,
+        23,11,1,0,4,4,0,45,57,65,90,95,95,97,122,2,0,10,10,32,32,2,0,65,
+        90,97,122,3,0,48,57,65,90,97,122,86,0,1,1,0,0,0,0,3,1,0,0,0,0,5,
+        1,0,0,0,0,7,1,0,0,0,0,9,1,0,0,0,0,11,1,0,0,0,0,13,1,0,0,0,0,17,1,
+        0,0,0,0,19,1,0,0,0,0,21,1,0,0,0,0,23,1,0,0,0,1,25,1,0,0,0,3,27,1,
+        0,0,0,5,29,1,0,0,0,7,31,1,0,0,0,9,33,1,0,0,0,11,35,1,0,0,0,13,37,
+        1,0,0,0,15,58,1,0,0,0,17,60,1,0,0,0,19,64,1,0,0,0,21,70,1,0,0,0,
+        23,76,1,0,0,0,25,26,5,91,0,0,26,2,1,0,0,0,27,28,5,93,0,0,28,4,1,
+        0,0,0,29,30,5,61,0,0,30,6,1,0,0,0,31,32,5,36,0,0,32,8,1,0,0,0,33,
+        34,5,35,0,0,34,10,1,0,0,0,35,36,9,0,0,0,36,12,1,0,0,0,37,38,5,58,
+        0,0,38,39,5,117,0,0,39,40,5,115,0,0,40,41,5,101,0,0,41,43,1,0,0,
+        0,42,44,3,15,7,0,43,42,1,0,0,0,44,45,1,0,0,0,45,43,1,0,0,0,45,46,
+        1,0,0,0,46,48,1,0,0,0,47,49,7,0,0,0,48,47,1,0,0,0,49,50,1,0,0,0,
+        50,48,1,0,0,0,50,51,1,0,0,0,51,55,1,0,0,0,52,54,3,15,7,0,53,52,1,
+        0,0,0,54,57,1,0,0,0,55,53,1,0,0,0,55,56,1,0,0,0,56,14,1,0,0,0,57,
+        55,1,0,0,0,58,59,7,1,0,0,59,16,1,0,0,0,60,61,5,58,0,0,61,62,5,105,
+        0,0,62,63,5,102,0,0,63,18,1,0,0,0,64,65,5,58,0,0,65,66,5,116,0,0,
+        66,67,5,104,0,0,67,68,5,101,0,0,68,69,5,110,0,0,69,20,1,0,0,0,70,
+        71,5,58,0,0,71,72,5,101,0,0,72,73,5,108,0,0,73,74,5,115,0,0,74,75,
+        5,101,0,0,75,22,1,0,0,0,76,77,5,58,0,0,77,81,7,2,0,0,78,80,7,3,0,
+        0,79,78,1,0,0,0,80,83,1,0,0,0,81,79,1,0,0,0,81,82,1,0,0,0,82,24,
+        1,0,0,0,83,81,1,0,0,0,5,0,45,50,55,81,0
     ]
 
 class MetaPromptLexer(Lexer):
@@ -49,26 +50,27 @@ class MetaPromptLexer(Lexer):
     RB = 2
     EQ_KW = 3
     META_KW = 4
-    CHAR = 5
-    USE = 6
-    IF_KW = 7
-    THEN_KW = 8
-    ELSE_KW = 9
-    VAR_NAME = 10
+    COMMENT_KW = 5
+    CHAR = 6
+    USE = 7
+    IF_KW = 8
+    THEN_KW = 9
+    ELSE_KW = 10
+    VAR_NAME = 11
 
     channelNames = [ u"DEFAULT_TOKEN_CHANNEL", u"HIDDEN" ]
 
     modeNames = [ "DEFAULT_MODE" ]
 
     literalNames = [ "<INVALID>",
-            "'['", "']'", "'='", "'$'", "':if'", "':then'", "':else'" ]
+            "'['", "']'", "'='", "'$'", "'#'", "':if'", "':then'", "':else'" ]
 
     symbolicNames = [ "<INVALID>",
-            "LB", "RB", "EQ_KW", "META_KW", "CHAR", "USE", "IF_KW", "THEN_KW", 
-            "ELSE_KW", "VAR_NAME" ]
+            "LB", "RB", "EQ_KW", "META_KW", "COMMENT_KW", "CHAR", "USE", 
+            "IF_KW", "THEN_KW", "ELSE_KW", "VAR_NAME" ]
 
-    ruleNames = [ "LB", "RB", "EQ_KW", "META_KW", "CHAR", "USE", "WS", "IF_KW", 
-                  "THEN_KW", "ELSE_KW", "VAR_NAME" ]
+    ruleNames = [ "LB", "RB", "EQ_KW", "META_KW", "COMMENT_KW", "CHAR", 
+                  "USE", "WS", "IF_KW", "THEN_KW", "ELSE_KW", "VAR_NAME" ]
 
     grammarFileName = "MetaPrompt.g4"
 

diff --git a/python/src/parser/MetaPromptLexer.tokens b/python/src/parser/MetaPromptLexer.tokens
@@ -2,16 +2,18 @@ LB=1
 RB=2
 EQ_KW=3
 META_KW=4
-CHAR=5
-USE=6
-IF_KW=7
-THEN_KW=8
-ELSE_KW=9
-VAR_NAME=10
+COMMENT_KW=5
+CHAR=6
+USE=7
+IF_KW=8
+THEN_KW=9
+ELSE_KW=10
+VAR_NAME=11
 '['=1
 ']'=2
 '='=3
 '$'=4
-':if'=7
-':then'=8
-':else'=9
+'#'=5
+':if'=8
+':then'=9
+':else'=10
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,6 +4,7 @@ null @@
     ']'
     '='
     '$'
+    '#'
     null
     null
     ':if'
@@ Expand All / @@ -17,6 +18,7 @@ LB @@
     RB
     EQ_KW
     META_KW
+    COMMENT_KW
     CHAR
     USE
     IF_KW
@@ Expand All / @@ -35,4 +37,4 @@ text @@
     atn:
-    [4, 1, 10, 73, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 1, 0, 1, 0, 1, 0, 1, 1, 5, 1, 19, 8, 1, 10, 1, 12, 1, 22, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 31, 8, 2, 1, 3, 1, 3, 3, 3, 35, 8, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 3, 4, 51, 8, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 3, 4, 59, 8, 4, 1, 5, 1, 5, 1, 5, 4, 5, 64, 8, 5, 11, 5, 12, 5, 65, 1, 6, 4, 6, 69, 8, 6, 11, 6, 12, 6, 70, 1, 6, 1, 20, 0, 7, 0, 2, 4, 6, 8, 10, 12, 0, 0, 78, 0, 14, 1, 0, 0, 0, 2, 20, 1, 0, 0, 0, 4, 30, 1, 0, 0, 0, 6, 34, 1, 0, 0, 0, 8, 58, 1, 0, 0, 0, 10, 63, 1, 0, 0, 0, 12, 68, 1, 0, 0, 0, 14, 15, 3, 2, 1, 0, 15, 16, 5, 0, 0, 1, 16, 1, 1, 0, 0, 0, 17, 19, 3, 4, 2, 0, 18, 17, 1, 0, 0, 0, 19, 22, 1, 0, 0, 0, 20, 21, 1, 0, 0, 0, 20, 18, 1, 0, 0, 0, 21, 3, 1, 0, 0, 0, 22, 20, 1, 0, 0, 0, 23, 24, 5, 1, 0, 0, 24, 25, 3, 6, 3, 0, 25, 26, 5, 2, 0, 0, 26, 31, 1, 0, 0, 0, 27, 31, 3, 12, 6, 0, 28, 31, 5, 2, 0, 0, 29, 31, 5, 1, 0, 0, 30, 23, 1, 0, 0, 0, 30, 27, 1, 0, 0, 0, 30, 28, 1, 0, 0, 0, 30, 29, 1, 0, 0, 0, 31, 5, 1, 0, 0, 0, 32, 35, 3, 8, 4, 0, 33, 35, 3, 2, 1, 0, 34, 32, 1, 0, 0, 0, 34, 33, 1, 0, 0, 0, 35, 7, 1, 0, 0, 0, 36, 37, 5, 7, 0, 0, 37, 38, 3, 2, 1, 0, 38, 39, 5, 8, 0, 0, 39, 40, 3, 2, 1, 0, 40, 41, 5, 9, 0, 0, 41, 42, 3, 2, 1, 0, 42, 59, 1, 0, 0, 0, 43, 44, 5, 7, 0, 0, 44, 45, 3, 2, 1, 0, 45, 46, 5, 8, 0, 0, 46, 47, 3, 2, 1, 0, 47, 59, 1, 0, 0, 0, 48, 50, 5, 6, 0, 0, 49, 51, 3, 10, 5, 0, 50, 49, 1, 0, 0, 0, 50, 51, 1, 0, 0, 0, 51, 59, 1, 0, 0, 0, 52, 53, 5, 4, 0, 0, 53, 59, 3, 2, 1, 0, 54, 55, 5, 10, 0, 0, 55, 56, 5, 3, 0, 0, 56, 59, 3, 2, 1, 0, 57, 59, 5, 10, 0, 0, 58, 36, 1, 0, 0, 0, 58, 43, 1, 0, 0, 0, 58, 48, 1, 0, 0, 0, 58, 52, 1, 0, 0, 0, 58, 54, 1, 0, 0, 0, 58, 57, 1, 0, 0, 0, 59, 9, 1, 0, 0, 0, 60, 61, 5, 10, 0, 0, 61, 62, 5, 3, 0, 0, 62, 64, 3, 2, 1, 0, 63, 60, 1, 0, 0, 0, 64, 65, 1, 0, 0, 0, 65, 63, 1, 0, 0, 0, 65, 66, 1, 0, 0, 0, 66, 11, 1, 0, 0, 0, 67, 69, 5, 5, 0, 0, 68, 67, 1, 0, 0, 0, 69, 70, 1, 0, 0, 0, 70, 68, 1, 0, 0, 0, 70, 71, 1, 0, 0, 0, 71, 13, 1, 0, 0, 0, 7, 20, 30, 34, 50, 58, 65, 70]
+    [4, 1, 11, 76, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 1, 0, 1, 0, 1, 0, 1, 1, 5, 1, 19, 8, 1, 10, 1, 12, 1, 22, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 32, 8, 2, 1, 3, 1, 3, 3, 3, 36, 8, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 3, 4, 52, 8, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 3, 4, 62, 8, 4, 1, 5, 1, 5, 1, 5, 4, 5, 67, 8, 5, 11, 5, 12, 5, 68, 1, 6, 4, 6, 72, 8, 6, 11, 6, 12, 6, 73, 1, 6, 1, 20, 0, 7, 0, 2, 4, 6, 8, 10, 12, 0, 0, 83, 0, 14, 1, 0, 0, 0, 2, 20, 1, 0, 0, 0, 4, 31, 1, 0, 0, 0, 6, 35, 1, 0, 0, 0, 8, 61, 1, 0, 0, 0, 10, 66, 1, 0, 0, 0, 12, 71, 1, 0, 0, 0, 14, 15, 3, 2, 1, 0, 15, 16, 5, 0, 0, 1, 16, 1, 1, 0, 0, 0, 17, 19, 3, 4, 2, 0, 18, 17, 1, 0, 0, 0, 19, 22, 1, 0, 0, 0, 20, 21, 1, 0, 0, 0, 20, 18, 1, 0, 0, 0, 21, 3, 1, 0, 0, 0, 22, 20, 1, 0, 0, 0, 23, 24, 5, 1, 0, 0, 24, 25, 3, 6, 3, 0, 25, 26, 5, 2, 0, 0, 26, 32, 1, 0, 0, 0, 27, 32, 3, 12, 6, 0, 28, 32, 5, 2, 0, 0, 29, 32, 5, 1, 0, 0, 30, 32, 5, 5, 0, 0, 31, 23, 1, 0, 0, 0, 31, 27, 1, 0, 0, 0, 31, 28, 1, 0, 0, 0, 31, 29, 1, 0, 0, 0, 31, 30, 1, 0, 0, 0, 32, 5, 1, 0, 0, 0, 33, 36, 3, 8, 4, 0, 34, 36, 3, 2, 1, 0, 35, 33, 1, 0, 0, 0, 35, 34, 1, 0, 0, 0, 36, 7, 1, 0, 0, 0, 37, 38, 5, 8, 0, 0, 38, 39, 3, 2, 1, 0, 39, 40, 5, 9, 0, 0, 40, 41, 3, 2, 1, 0, 41, 42, 5, 10, 0, 0, 42, 43, 3, 2, 1, 0, 43, 62, 1, 0, 0, 0, 44, 45, 5, 8, 0, 0, 45, 46, 3, 2, 1, 0, 46, 47, 5, 9, 0, 0, 47, 48, 3, 2, 1, 0, 48, 62, 1, 0, 0, 0, 49, 51, 5, 7, 0, 0, 50, 52, 3, 10, 5, 0, 51, 50, 1, 0, 0, 0, 51, 52, 1, 0, 0, 0, 52, 62, 1, 0, 0, 0, 53, 54, 5, 4, 0, 0, 54, 62, 3, 2, 1, 0, 55, 56, 5, 5, 0, 0, 56, 62, 3, 2, 1, 0, 57, 58, 5, 11, 0, 0, 58, 59, 5, 3, 0, 0, 59, 62, 3, 2, 1, 0, 60, 62, 5, 11, 0, 0, 61, 37, 1, 0, 0, 0, 61, 44, 1, 0, 0, 0, 61, 49, 1, 0, 0, 0, 61, 53, 1, 0, 0, 0, 61, 55, 1, 0, 0, 0, 61, 57, 1, 0, 0, 0, 61, 60, 1, 0, 0, 0, 62, 9, 1, 0, 0, 0, 63, 64, 5, 11, 0, 0, 64, 65, 5, 3, 0, 0, 65, 67, 3, 2, 1, 0, 66, 63, 1, 0, 0, 0, 67, 68, 1, 0, 0, 0, 68, 66, 1, 0, 0, 0, 68, 69, 1, 0, 0, 0, 69, 11, 1, 0, 0, 0, 70, 72, 5, 6, 0, 0, 71, 70, 1, 0, 0, 0, 72, 73, 1, 0, 0, 0, 73, 71, 1, 0, 0, 0, 73, 74, 1, 0, 0, 0, 74, 13, 1, 0, 0, 0, 7, 20, 31, 35, 51, 61, 68, 73]