Skip to content

Commit

Permalink
Improve the parser: match brackets more greedily
Browse files Browse the repository at this point in the history
  • Loading branch information
@klntsky
klntsky committed Nov 9, 2024
1 parent d6968ca commit d93c94d
Show file tree
Hide file tree
Showing 6 changed files with 109 additions and 77 deletions.
7 changes: 4 additions & 3 deletions grammar/MetaPrompt.g4
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
grammar MetaPrompt;

prompt: exprs EOF ;
exprs: expr* ;
expr: text? LB expr1
exprs: expr*? ;
expr: LB expr1 RB
| text
| RB
| LB
;

expr1
: meta_body RB
: meta_body
| exprs
;

Expand Down
5 changes: 4 additions & 1 deletion python/metaprompt/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,10 @@ async def eval_ast(ast, runtime):
retries += 1
if retries >= MAX_RETRIES:
raise ValueError(
"Failed to answer :if prompt: " + prompt + "\nOutput: " + prompt_result
"Failed to answer :if prompt: "
+ prompt
+ "\nOutput: "
+ prompt_result
)
if prompt_result == "true":
async for chunk in eval_ast(ast["then"], runtime):
Expand Down
10 changes: 7 additions & 3 deletions python/metaprompt/parse_metaprompt.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,20 @@ def visitExpr(self, ctx: MetaPromptParser.ExprContext):
items = []
if ctx.text() is not None:
items.append(self.visit(ctx.text()))
if ctx.LB() is not None:
if ctx.expr1() is not None:
expr1 = self.visit(ctx.expr1())
if expr1["type"] == "meta":
items.append(expr1["meta"])
elif expr1["type"] == "exprs":
items.append({"type": "text", "text": "["})
for child in expr1["exprs"]:
items.append(child)
if ctx.RB() is not None:
items.append({"type": "text", "text": "]"})
items.append({"type": "text", "text": "]"})
else:
if ctx.RB() is not None:
items.append({"type": "text", "text": "]"})
if ctx.LB() is not None:
items.append({"type": "text", "text": "["})
return items

def visitExpr1(self, ctx: MetaPromptParser.Expr1Context):
Expand Down
2 changes: 1 addition & 1 deletion python/metaprompt/parser/grammar/MetaPrompt.interp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,4 @@ text


atn:
[4, 1, 9, 62, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 1, 0, 1, 0, 1, 0, 1, 1, 5, 1, 17, 8, 1, 10, 1, 12, 1, 20, 9, 1, 1, 2, 3, 2, 23, 8, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 29, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 35, 8, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 3, 4, 55, 8, 4, 1, 5, 4, 5, 58, 8, 5, 11, 5, 12, 5, 59, 1, 5, 0, 0, 6, 0, 2, 4, 6, 8, 10, 0, 0, 65, 0, 12, 1, 0, 0, 0, 2, 18, 1, 0, 0, 0, 4, 28, 1, 0, 0, 0, 6, 34, 1, 0, 0, 0, 8, 54, 1, 0, 0, 0, 10, 57, 1, 0, 0, 0, 12, 13, 3, 2, 1, 0, 13, 14, 5, 0, 0, 1, 14, 1, 1, 0, 0, 0, 15, 17, 3, 4, 2, 0, 16, 15, 1, 0, 0, 0, 17, 20, 1, 0, 0, 0, 18, 16, 1, 0, 0, 0, 18, 19, 1, 0, 0, 0, 19, 3, 1, 0, 0, 0, 20, 18, 1, 0, 0, 0, 21, 23, 3, 10, 5, 0, 22, 21, 1, 0, 0, 0, 22, 23, 1, 0, 0, 0, 23, 24, 1, 0, 0, 0, 24, 25, 5, 1, 0, 0, 25, 29, 3, 6, 3, 0, 26, 29, 3, 10, 5, 0, 27, 29, 5, 2, 0, 0, 28, 22, 1, 0, 0, 0, 28, 26, 1, 0, 0, 0, 28, 27, 1, 0, 0, 0, 29, 5, 1, 0, 0, 0, 30, 31, 3, 8, 4, 0, 31, 32, 5, 2, 0, 0, 32, 35, 1, 0, 0, 0, 33, 35, 3, 2, 1, 0, 34, 30, 1, 0, 0, 0, 34, 33, 1, 0, 0, 0, 35, 7, 1, 0, 0, 0, 36, 37, 5, 6, 0, 0, 37, 38, 3, 2, 1, 0, 38, 39, 5, 7, 0, 0, 39, 40, 3, 2, 1, 0, 40, 41, 5, 8, 0, 0, 41, 42, 3, 2, 1, 0, 42, 55, 1, 0, 0, 0, 43, 44, 5, 6, 0, 0, 44, 45, 3, 2, 1, 0, 45, 46, 5, 7, 0, 0, 46, 47, 3, 2, 1, 0, 47, 55, 1, 0, 0, 0, 48, 49, 5, 4, 0, 0, 49, 55, 3, 2, 1, 0, 50, 51, 5, 9, 0, 0, 51, 52, 5, 3, 0, 0, 52, 55, 3, 2, 1, 0, 53, 55, 5, 9, 0, 0, 54, 36, 1, 0, 0, 0, 54, 43, 1, 0, 0, 0, 54, 48, 1, 0, 0, 0, 54, 50, 1, 0, 0, 0, 54, 53, 1, 0, 0, 0, 55, 9, 1, 0, 0, 0, 56, 58, 5, 5, 0, 0, 57, 56, 1, 0, 0, 0, 58, 59, 1, 0, 0, 0, 59, 57, 1, 0, 0, 0, 59, 60, 1, 0, 0, 0, 60, 11, 1, 0, 0, 0, 6, 18, 22, 28, 34, 54, 59]
[4, 1, 9, 60, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 1, 0, 1, 0, 1, 0, 1, 1, 5, 1, 17, 8, 1, 10, 1, 12, 1, 20, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 29, 8, 2, 1, 3, 1, 3, 3, 3, 33, 8, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 3, 4, 53, 8, 4, 1, 5, 4, 5, 56, 8, 5, 11, 5, 12, 5, 57, 1, 5, 1, 18, 0, 6, 0, 2, 4, 6, 8, 10, 0, 0, 63, 0, 12, 1, 0, 0, 0, 2, 18, 1, 0, 0, 0, 4, 28, 1, 0, 0, 0, 6, 32, 1, 0, 0, 0, 8, 52, 1, 0, 0, 0, 10, 55, 1, 0, 0, 0, 12, 13, 3, 2, 1, 0, 13, 14, 5, 0, 0, 1, 14, 1, 1, 0, 0, 0, 15, 17, 3, 4, 2, 0, 16, 15, 1, 0, 0, 0, 17, 20, 1, 0, 0, 0, 18, 19, 1, 0, 0, 0, 18, 16, 1, 0, 0, 0, 19, 3, 1, 0, 0, 0, 20, 18, 1, 0, 0, 0, 21, 22, 5, 1, 0, 0, 22, 23, 3, 6, 3, 0, 23, 24, 5, 2, 0, 0, 24, 29, 1, 0, 0, 0, 25, 29, 3, 10, 5, 0, 26, 29, 5, 2, 0, 0, 27, 29, 5, 1, 0, 0, 28, 21, 1, 0, 0, 0, 28, 25, 1, 0, 0, 0, 28, 26, 1, 0, 0, 0, 28, 27, 1, 0, 0, 0, 29, 5, 1, 0, 0, 0, 30, 33, 3, 8, 4, 0, 31, 33, 3, 2, 1, 0, 32, 30, 1, 0, 0, 0, 32, 31, 1, 0, 0, 0, 33, 7, 1, 0, 0, 0, 34, 35, 5, 6, 0, 0, 35, 36, 3, 2, 1, 0, 36, 37, 5, 7, 0, 0, 37, 38, 3, 2, 1, 0, 38, 39, 5, 8, 0, 0, 39, 40, 3, 2, 1, 0, 40, 53, 1, 0, 0, 0, 41, 42, 5, 6, 0, 0, 42, 43, 3, 2, 1, 0, 43, 44, 5, 7, 0, 0, 44, 45, 3, 2, 1, 0, 45, 53, 1, 0, 0, 0, 46, 47, 5, 4, 0, 0, 47, 53, 3, 2, 1, 0, 48, 49, 5, 9, 0, 0, 49, 50, 5, 3, 0, 0, 50, 53, 3, 2, 1, 0, 51, 53, 5, 9, 0, 0, 52, 34, 1, 0, 0, 0, 52, 41, 1, 0, 0, 0, 52, 46, 1, 0, 0, 0, 52, 48, 1, 0, 0, 0, 52, 51, 1, 0, 0, 0, 53, 9, 1, 0, 0, 0, 54, 56, 5, 5, 0, 0, 55, 54, 1, 0, 0, 0, 56, 57, 1, 0, 0, 0, 57, 55, 1, 0, 0, 0, 57, 58, 1, 0, 0, 0, 58, 11, 1, 0, 0, 0, 5, 18, 28, 32, 52, 57]
131 changes: 62 additions & 69 deletions python/metaprompt/parser/grammar/MetaPromptParser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,26 +10,25 @@

def serializedATN():
return [
4,1,9,62,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,1,0,1,0,
1,0,1,1,5,1,17,8,1,10,1,12,1,20,9,1,1,2,3,2,23,8,2,1,2,1,2,1,2,1,
2,3,2,29,8,2,1,3,1,3,1,3,1,3,3,3,35,8,3,1,4,1,4,1,4,1,4,1,4,1,4,
1,4,1,4,1,4,1,4,1,4,1,4,1,4,1,4,1,4,1,4,1,4,1,4,3,4,55,8,4,1,5,4,
5,58,8,5,11,5,12,5,59,1,5,0,0,6,0,2,4,6,8,10,0,0,65,0,12,1,0,0,0,
2,18,1,0,0,0,4,28,1,0,0,0,6,34,1,0,0,0,8,54,1,0,0,0,10,57,1,0,0,
0,12,13,3,2,1,0,13,14,5,0,0,1,14,1,1,0,0,0,15,17,3,4,2,0,16,15,1,
0,0,0,17,20,1,0,0,0,18,16,1,0,0,0,18,19,1,0,0,0,19,3,1,0,0,0,20,
18,1,0,0,0,21,23,3,10,5,0,22,21,1,0,0,0,22,23,1,0,0,0,23,24,1,0,
0,0,24,25,5,1,0,0,25,29,3,6,3,0,26,29,3,10,5,0,27,29,5,2,0,0,28,
22,1,0,0,0,28,26,1,0,0,0,28,27,1,0,0,0,29,5,1,0,0,0,30,31,3,8,4,
0,31,32,5,2,0,0,32,35,1,0,0,0,33,35,3,2,1,0,34,30,1,0,0,0,34,33,
1,0,0,0,35,7,1,0,0,0,36,37,5,6,0,0,37,38,3,2,1,0,38,39,5,7,0,0,39,
40,3,2,1,0,40,41,5,8,0,0,41,42,3,2,1,0,42,55,1,0,0,0,43,44,5,6,0,
0,44,45,3,2,1,0,45,46,5,7,0,0,46,47,3,2,1,0,47,55,1,0,0,0,48,49,
5,4,0,0,49,55,3,2,1,0,50,51,5,9,0,0,51,52,5,3,0,0,52,55,3,2,1,0,
53,55,5,9,0,0,54,36,1,0,0,0,54,43,1,0,0,0,54,48,1,0,0,0,54,50,1,
0,0,0,54,53,1,0,0,0,55,9,1,0,0,0,56,58,5,5,0,0,57,56,1,0,0,0,58,
59,1,0,0,0,59,57,1,0,0,0,59,60,1,0,0,0,60,11,1,0,0,0,6,18,22,28,
34,54,59
4,1,9,60,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,1,0,1,0,
1,0,1,1,5,1,17,8,1,10,1,12,1,20,9,1,1,2,1,2,1,2,1,2,1,2,1,2,1,2,
3,2,29,8,2,1,3,1,3,3,3,33,8,3,1,4,1,4,1,4,1,4,1,4,1,4,1,4,1,4,1,
4,1,4,1,4,1,4,1,4,1,4,1,4,1,4,1,4,1,4,3,4,53,8,4,1,5,4,5,56,8,5,
11,5,12,5,57,1,5,1,18,0,6,0,2,4,6,8,10,0,0,63,0,12,1,0,0,0,2,18,
1,0,0,0,4,28,1,0,0,0,6,32,1,0,0,0,8,52,1,0,0,0,10,55,1,0,0,0,12,
13,3,2,1,0,13,14,5,0,0,1,14,1,1,0,0,0,15,17,3,4,2,0,16,15,1,0,0,
0,17,20,1,0,0,0,18,19,1,0,0,0,18,16,1,0,0,0,19,3,1,0,0,0,20,18,1,
0,0,0,21,22,5,1,0,0,22,23,3,6,3,0,23,24,5,2,0,0,24,29,1,0,0,0,25,
29,3,10,5,0,26,29,5,2,0,0,27,29,5,1,0,0,28,21,1,0,0,0,28,25,1,0,
0,0,28,26,1,0,0,0,28,27,1,0,0,0,29,5,1,0,0,0,30,33,3,8,4,0,31,33,
3,2,1,0,32,30,1,0,0,0,32,31,1,0,0,0,33,7,1,0,0,0,34,35,5,6,0,0,35,
36,3,2,1,0,36,37,5,7,0,0,37,38,3,2,1,0,38,39,5,8,0,0,39,40,3,2,1,
0,40,53,1,0,0,0,41,42,5,6,0,0,42,43,3,2,1,0,43,44,5,7,0,0,44,45,
3,2,1,0,45,53,1,0,0,0,46,47,5,4,0,0,47,53,3,2,1,0,48,49,5,9,0,0,
49,50,5,3,0,0,50,53,3,2,1,0,51,53,5,9,0,0,52,34,1,0,0,0,52,41,1,
0,0,0,52,46,1,0,0,0,52,48,1,0,0,0,52,51,1,0,0,0,53,9,1,0,0,0,54,
56,5,5,0,0,55,54,1,0,0,0,56,57,1,0,0,0,57,55,1,0,0,0,57,58,1,0,0,
0,58,11,1,0,0,0,5,18,28,32,52,57
]

class MetaPromptParser ( Parser ):
Expand Down Expand Up @@ -173,8 +172,8 @@ def exprs(self):
self.state = 18
self._errHandler.sync(self)
_alt = self._interp.adaptivePredict(self._input,0,self._ctx)
while _alt!=2 and _alt!=ATN.INVALID_ALT_NUMBER:
if _alt==1:
while _alt!=1 and _alt!=ATN.INVALID_ALT_NUMBER:
if _alt==1+1:
self.state = 15
self.expr()
self.state = 20
Expand Down Expand Up @@ -204,13 +203,13 @@ def expr1(self):
return self.getTypedRuleContext(MetaPromptParser.Expr1Context,0)


def RB(self):
return self.getToken(MetaPromptParser.RB, 0)

def text(self):
return self.getTypedRuleContext(MetaPromptParser.TextContext,0)


def RB(self):
return self.getToken(MetaPromptParser.RB, 0)

def getRuleIndex(self):
return MetaPromptParser.RULE_expr

Expand All @@ -235,39 +234,38 @@ def expr(self):

localctx = MetaPromptParser.ExprContext(self, self._ctx, self.state)
self.enterRule(localctx, 4, self.RULE_expr)
self._la = 0 # Token type
try:
self.state = 28
self._errHandler.sync(self)
la_ = self._interp.adaptivePredict(self._input,2,self._ctx)
la_ = self._interp.adaptivePredict(self._input,1,self._ctx)
if la_ == 1:
self.enterOuterAlt(localctx, 1)
self.state = 22
self._errHandler.sync(self)
_la = self._input.LA(1)
if _la==5:
self.state = 21
self.text()


self.state = 24
self.state = 21
self.match(MetaPromptParser.LB)
self.state = 25
self.state = 22
self.expr1()
self.state = 23
self.match(MetaPromptParser.RB)
pass

elif la_ == 2:
self.enterOuterAlt(localctx, 2)
self.state = 26
self.state = 25
self.text()
pass

elif la_ == 3:
self.enterOuterAlt(localctx, 3)
self.state = 27
self.state = 26
self.match(MetaPromptParser.RB)
pass

elif la_ == 4:
self.enterOuterAlt(localctx, 4)
self.state = 27
self.match(MetaPromptParser.LB)
pass


except RecognitionException as re:
localctx.exception = re
Expand All @@ -289,9 +287,6 @@ def meta_body(self):
return self.getTypedRuleContext(MetaPromptParser.Meta_bodyContext,0)


def RB(self):
return self.getToken(MetaPromptParser.RB, 0)

def exprs(self):
return self.getTypedRuleContext(MetaPromptParser.ExprsContext,0)

Expand Down Expand Up @@ -321,19 +316,17 @@ def expr1(self):
localctx = MetaPromptParser.Expr1Context(self, self._ctx, self.state)
self.enterRule(localctx, 6, self.RULE_expr1)
try:
self.state = 34
self.state = 32
self._errHandler.sync(self)
token = self._input.LA(1)
if token in [4, 6, 9]:
self.enterOuterAlt(localctx, 1)
self.state = 30
self.meta_body()
self.state = 31
self.match(MetaPromptParser.RB)
pass
elif token in [-1, 1, 2, 5, 7, 8]:
elif token in [1, 2, 5]:
self.enterOuterAlt(localctx, 2)
self.state = 33
self.state = 31
self.exprs()
pass
else:
Expand Down Expand Up @@ -405,58 +398,58 @@ def meta_body(self):
localctx = MetaPromptParser.Meta_bodyContext(self, self._ctx, self.state)
self.enterRule(localctx, 8, self.RULE_meta_body)
try:
self.state = 54
self.state = 52
self._errHandler.sync(self)
la_ = self._interp.adaptivePredict(self._input,4,self._ctx)
la_ = self._interp.adaptivePredict(self._input,3,self._ctx)
if la_ == 1:
self.enterOuterAlt(localctx, 1)
self.state = 36
self.state = 34
self.match(MetaPromptParser.IF_KW)
self.state = 37
self.state = 35
self.exprs()
self.state = 38
self.state = 36
self.match(MetaPromptParser.THEN_KW)
self.state = 39
self.state = 37
self.exprs()
self.state = 40
self.state = 38
self.match(MetaPromptParser.ELSE_KW)
self.state = 41
self.state = 39
self.exprs()
pass

elif la_ == 2:
self.enterOuterAlt(localctx, 2)
self.state = 43
self.state = 41
self.match(MetaPromptParser.IF_KW)
self.state = 44
self.state = 42
self.exprs()
self.state = 45
self.state = 43
self.match(MetaPromptParser.THEN_KW)
self.state = 46
self.state = 44
self.exprs()
pass

elif la_ == 3:
self.enterOuterAlt(localctx, 3)
self.state = 48
self.state = 46
self.match(MetaPromptParser.META_KW)
self.state = 49
self.state = 47
self.exprs()
pass

elif la_ == 4:
self.enterOuterAlt(localctx, 4)
self.state = 50
self.state = 48
self.match(MetaPromptParser.VAR_NAME)
self.state = 51
self.state = 49
self.match(MetaPromptParser.EQ_KW)
self.state = 52
self.state = 50
self.exprs()
pass

elif la_ == 5:
self.enterOuterAlt(localctx, 5)
self.state = 53
self.state = 51
self.match(MetaPromptParser.VAR_NAME)
pass

Expand Down Expand Up @@ -509,19 +502,19 @@ def text(self):
self.enterRule(localctx, 10, self.RULE_text)
try:
self.enterOuterAlt(localctx, 1)
self.state = 57
self.state = 55
self._errHandler.sync(self)
_alt = 1
while _alt!=2 and _alt!=ATN.INVALID_ALT_NUMBER:
if _alt == 1:
self.state = 56
self.state = 54
self.match(MetaPromptParser.CHAR)

else:
raise NoViableAltException(self)
self.state = 59
self.state = 57
self._errHandler.sync(self)
_alt = self._interp.adaptivePredict(self._input,5,self._ctx)
_alt = self._interp.adaptivePredict(self._input,4,self._ctx)

except RecognitionException as re:
localctx.exception = re
Expand Down
31 changes: 31 additions & 0 deletions python/metaprompt/tests/test_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,3 +97,34 @@ def test_dummy_meta2():
def test_meta_dollar():
result = parse_metaprompt("[$ foo]")
assert result["exprs"] == [{'type': 'meta', 'exprs': [ {'type': 'text', 'text': " foo"}]}]


def test_meta_dollar2():
result = parse_metaprompt("[$ foo][$ foo]")
assert result["exprs"] == [
{'type': 'meta', 'exprs': [ {'type': 'text', 'text': " foo"}]},
{'type': 'meta', 'exprs': [ {'type': 'text', 'text': " foo"}]},
]


def test_meta_dollar2():
result = parse_metaprompt("[$ foo]]")
assert result["exprs"] == [
{'type': 'meta', 'exprs': [ {'type': 'text', 'text': " foo"}]},
t("]")
]


def test_assign():
result = parse_metaprompt("[:foo=bar]")
assert result["exprs"] == [{'type': 'assign', 'name': 'foo', 'exprs': [ {'type': 'text', 'text': "bar"}]}]


def test_assign_trailing_bracket():
result = parse_metaprompt("[:foo=bar]]")
assert result["exprs"] == [
{'type': 'assign', 'name': 'foo',
'exprs': [ {'type': 'text', 'text': "bar"}]
},
t(']')
]

0 comments on commit d93c94d

Please sign in to comment.