Extracted and generalized expression split.

cinje/inline/text.py

@@ -5,7 +5,7 @@
 from itertools import chain
 from pprint import pformat
 
-from ..util import pypy, iterate, chunk, Line, ensure_buffer
+from ..util import pypy, iterate, splitexpr, chunk, Line, ensure_buffer
 
 
 def gather(input):
@@ -116,18 +116,8 @@ def inner_chain():
 				continue
 
 			if token == 'format':
-				# We need to split the expression defining the format string from the values to pass when formatting.
-				# We want to allow any Python expression, so we'll need to piggyback on Python's own parser in order
-				# to exploit the currently available syntax.  Apologies, this is probably the scariest thing in here.
-				split = -1
-
-				try:
-					ast.parse(chunk_)
-				except SyntaxError as e:  # We expect this, and catch it.  It'll have exploded after the first expr.
-					split = chunk_.rfind(' ', 0, e.offset)
-
-				token = '_bless(' + chunk_[:split].rstrip() + ').format'
-				chunk_ = chunk_[split:].lstrip()
+				token, chunk_ = splitexpr(chunk_, 1)
+				token = '_bless(' + token + ').format'
 
 			yield Line(lineno, prefix + token + '(' + chunk_ + ')' + suffix, scope)
 

cinje/util.py

@@ -7,6 +7,7 @@
 # ## Imports
 
 import sys
+import ast
 
 from codecs import iterencode
 from inspect import isfunction, isclass
@@ -243,6 +244,43 @@ def xmlargs(_source=None, **values):
 	return bless(" " + ejoin(parts)) if parts else ''
 
 
+def splitexpr(text, limit=0):
+	"""Split a given line of text into constituent expressions."""
+
+	# This is rather nasty, so we've isolated it here.
+
+	parts = []
+
+	while text:
+		split = -1
+
+		try:
+			ast.parse(text)
+		except SyntaxError as e:  # We expect this, and catch it.  It'll have exploded after the first expr.
+			if pypy:
+				split = e.offset
+			else:
+				split = text.rfind(text[e.offset - 1] if text[e.offset - 1] in "'\"" else ' ', 0, e.offset - 1)
+
+		if split < 0:
+			parts.append(text)
+			break
+
+		chunk = text[:split].rstrip()
+
+		# Verify this is a good split.
+		ast.parse(chunk)  # We want this to explode if invalid.
+
+		parts.append(chunk)
+		text = text[split:].lstrip()
+
+		if limit and len(parts) == limit:
+			parts.append(text)
+			break
+
+	return parts
+
+
 def chunk(text, mapping={None: 'text', '${': '_escape', '#{': '_bless', '&{': '_args', '%{': 'format', '@{': '_json'}):
 	"""Chunkify and "tag" a block of text into plain text and code sections.
 

test/test_util/test_functions.py

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-from cinje.util import interruptable, iterate, xmlargs, chunk, ensure_buffer, Line, strip_tags
+from cinje.util import interruptable, iterate, xmlargs, splitexpr, chunk, ensure_buffer, Line, strip_tags
 
 # Note: ensure_buffer is tested indirectly via template conformance testing.
 
@@ -108,6 +108,20 @@ def test_defaults_overridden(self):
 			))
 
 
+class TestExpressionSplit(object):
+	def test_object_quote_single(self):
+		assert splitexpr("foo 'Hello world!'") == ['foo', "'Hello world!'"]
+
+	def test_object_quote_double(self):
+		assert splitexpr('bar "Farewell cruel world!"') == ['bar', '"Farewell cruel world!"']
+
+	def test_call_then_argspec(self):
+		assert splitexpr('baz(diz, "thing") 27, 42') == ['baz(diz, "thing")', '27, 42']
+
+	def test_partial_expression(self):
+		assert splitexpr('asdf 24 asdf"') == ['asdf', '24', 'asdf"']
+
+
 class TestChunker(object):
 	def _do(self, value):
 		token, kind, value = value