blob: 538da97984ce2e16d2385b60f9b077156abc9b91 [file] [log] [blame]
import unittest
from mako.lexer import Lexer
from mako import exceptions, util
from util import flatten_result, result_lines
from mako.template import Template
import re
from test import TemplateTest, template_base, skip_if, eq_, assert_raises_message
# create fake parsetree classes which are constructed
# exactly as the repr() of a real parsetree object.
# this allows us to use a Python construct as the source
# of a comparable repr(), which is also hit by the 2to3 tool.
def repr_arg(x):
if isinstance(x, dict):
return util.sorted_dict_repr(x)
else:
return repr(x)
from mako import parsetree
for cls in parsetree.__dict__.values():
if isinstance(cls, type) and \
issubclass(cls, parsetree.Node):
clsname = cls.__name__
exec ("""
class %s(object):
def __init__(self, *args):
self.args = args
def __repr__(self):
return "%%s(%%s)" %% (
self.__class__.__name__,
", ".join(repr_arg(x) for x in self.args)
)
""" % clsname) in locals()
# NOTE: most assertion expressions were generated, then formatted
# by PyTidy, hence the dense formatting.
class LexerTest(TemplateTest):
def _compare(self, node, expected):
eq_(repr(node), repr(expected))
def test_text_and_tag(self):
template = """
<b>Hello world</b>
<%def name="foo()">
this is a def.
</%def>
and some more text.
"""
node = Lexer(template).parse()
self._compare(node, TemplateNode({},
[Text(u'''\n<b>Hello world</b>\n ''', (1,
1)), DefTag(u'def', {u'name': u'foo()'}, (3, 9),
[Text(u'''\n this is a def.\n ''',
(3, 28))]),
Text(u'''\n \n and some more text.\n''',
(5, 16))]))
def test_unclosed_tag(self):
template = """
<%def name="foo()">
other text
"""
try:
nodes = Lexer(template).parse()
assert False
except exceptions.SyntaxException, e:
assert str(e) == "Unclosed tag: <%def> at line: 5 char: 9"
def test_onlyclosed_tag(self):
template = \
"""
<%def name="foo()">
foo
</%def>
</%namespace>
hi.
"""
self.assertRaises(exceptions.SyntaxException,
Lexer(template).parse)
def test_noexpr_allowed(self):
template = \
"""
<%namespace name="${foo}"/>
"""
self.assertRaises(exceptions.CompileException,
Lexer(template).parse)
def test_unmatched_tag(self):
template = \
"""
<%namespace name="bar">
<%def name="foo()">
foo
</%namespace>
</%def>
hi.
"""
self.assertRaises(exceptions.SyntaxException,
Lexer(template).parse)
def test_nonexistent_tag(self):
template = """
<%lala x="5"/>
"""
self.assertRaises(exceptions.CompileException,
Lexer(template).parse)
def test_wrongcase_tag(self):
template = \
"""
<%DEF name="foo()">
</%def>
"""
self.assertRaises(exceptions.CompileException,
Lexer(template).parse)
def test_percent_escape(self):
template = \
"""
%% some whatever.
%% more some whatever
% if foo:
% endif
"""
node = Lexer(template).parse()
self._compare(node, TemplateNode({}, [Text(u'''\n \n''',
(1, 1)), Text(u'''% some whatever.\n\n''', (3, 2)),
Text(u' %% more some whatever\n', (5, 2)),
ControlLine(u'if', u'if foo:', False, (6, 1)),
ControlLine(u'if', u'endif', True, (7, 1)),
Text(u' ', (8, 1))]))
def test_text_tag(self):
template = \
"""
## comment
% if foo:
hi
% endif
<%text>
# more code
% more code
<%illegal compionent>/></>
<%def name="laal()">def</%def>
</%text>
<%def name="foo()">this is foo</%def>
% if bar:
code
% endif
"""
node = Lexer(template).parse()
self._compare(node,
TemplateNode({}, [Text(u'\n', (1, 1)),
Comment(u'comment', (2, 1)),
ControlLine(u'if', u'if foo:', False, (3, 1)),
Text(u' hi\n', (4, 1)),
ControlLine(u'if', u'endif', True, (5, 1)),
Text(u' ', (6, 1)), TextTag(u'text', {},
(6, 9),
[Text(u'''\n # more code\n '''
'''\n % more code\n '''
'''<%illegal compionent>/></>\n '''
'''<%def name="laal()">def</%def>\n '''
''' \n \n ''',
(6, 16))]), Text(u'''
''', (14, 17)),
DefTag(u'def', {u'name': u'foo()'}, (16, 9),
[Text(u'this is foo', (16, 28))]),
Text(u'''\n \n''', (16, 46)),
ControlLine(u'if', u'if bar:', False, (18, 1)),
Text(u' code\n', (19, 1)),
ControlLine(u'if', u'endif', True, (20, 1)),
Text(u' ', (21, 1))]))
def test_def_syntax(self):
template = \
"""
<%def lala>
hi
</%def>
"""
self.assertRaises(exceptions.CompileException,
Lexer(template).parse)
def test_def_syntax_2(self):
template = \
"""
<%def name="lala">
hi
</%def>
"""
self.assertRaises(exceptions.CompileException,
Lexer(template).parse)
def test_whitespace_equals(self):
template = \
"""
<%def name = "adef()" >
adef
</%def>
"""
node = Lexer(template).parse()
self._compare(node, TemplateNode({}, [Text(u'\n ',
(1, 1)), DefTag(u'def', {u'name': u'adef()'}, (2,
13),
[Text(u'''\n adef\n ''',
(2, 36))]), Text(u'\n ', (4, 20))]))
def test_ns_tag_closed(self):
template = \
"""
<%self:go x="1" y="2" z="${'hi' + ' ' + 'there'}"/>
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({},
[Text(u'''
''', (1, 1)),
CallNamespaceTag(u'self:go', {u'x': u'1', u'y'
: u'2', u'z': u"${'hi' + ' ' + 'there'}"}, (3,
13), []), Text(u'\n ', (3, 64))]))
def test_ns_tag_empty(self):
template = \
"""
<%form:option value=""></%form:option>
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({}, [Text(u'\n ',
(1, 1)), CallNamespaceTag(u'form:option',
{u'value': u''}, (2, 13), []), Text(u'\n '
, (2, 51))]))
def test_ns_tag_open(self):
template = \
"""
<%self:go x="1" y="${process()}">
this is the body
</%self:go>
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({},
[Text(u'''
''', (1, 1)),
CallNamespaceTag(u'self:go', {u'x': u'1', u'y'
: u'${process()}'}, (3, 13),
[Text(u'''
this is the body
''',
(3, 46))]), Text(u'\n ', (5, 24))]))
def test_expr_in_attribute(self):
"""test some slightly trickier expressions.
you can still trip up the expression parsing, though, unless we
integrated really deeply somehow with AST."""
template = \
"""
<%call expr="foo>bar and 'lala' or 'hoho'"/>
<%call expr='foo<bar and hoho>lala and "x" + "y"'/>
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({}, [Text(u'\n ',
(1, 1)), CallTag(u'call', {u'expr'
: u"foo>bar and 'lala' or 'hoho'"}, (2, 13), []),
Text(u'\n ', (2, 57)), CallTag(u'call'
, {u'expr': u'foo<bar and hoho>lala and "x" + "y"'
}, (3, 13), []), Text(u'\n ', (3, 64))]))
def test_pagetag(self):
template = \
"""
<%page cached="True", args="a, b"/>
some template
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({}, [Text(u'\n ',
(1, 1)), PageTag(u'page', {u'args': u'a, b',
u'cached': u'True'}, (2, 13), []),
Text(u'''
some template
''',
(2, 48))]))
def test_nesting(self):
template = \
"""
<%namespace name="ns">
<%def name="lala(hi, there)">
<%call expr="something()"/>
</%def>
</%namespace>
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({},
[Text(u'''
''', (1, 1)),
NamespaceTag(u'namespace', {u'name': u'ns'}, (3,
9), [Text(u'\n ', (3, 31)),
DefTag(u'def', {u'name': u'lala(hi, there)'}, (4,
13), [Text(u'\n ', (4, 42)),
CallTag(u'call', {u'expr': u'something()'}, (5,
17), []), Text(u'\n ', (5, 44))]),
Text(u'\n ', (6, 20))]),
Text(u'''
''', (7, 22))]))
if util.py3k:
def test_code(self):
template = \
"""text
<%
print("hi")
for x in range(1,5):
print(x)
%>
more text
<%!
import foo
%>
"""
nodes = Lexer(template).parse()
self._compare(nodes,
TemplateNode({}, [
Text(u'text\n ', (1, 1)),
Code(u'\nprint("hi")\nfor x in range(1,5):\n '
'print(x)\n \n', False, (2, 5)),
Text(u'\nmore text\n ', (6, 7)),
Code(u'\nimport foo\n \n', True, (8, 5)),
Text(u'\n', (10, 7))])
)
else:
def test_code(self):
template = \
"""text
<%
print "hi"
for x in range(1,5):
print x
%>
more text
<%!
import foo
%>
"""
nodes = Lexer(template).parse()
self._compare(nodes,
TemplateNode({}, [
Text(u'text\n ', (1, 1)),
Code(u'\nprint "hi"\nfor x in range(1,5):\n '
'print x\n \n', False, (2, 5)),
Text(u'\nmore text\n ', (6, 7)),
Code(u'\nimport foo\n \n', True, (8, 5)),
Text(u'\n', (10, 7))])
)
def test_code_and_tags(self):
template = \
"""
<%namespace name="foo">
<%def name="x()">
this is x
</%def>
<%def name="y()">
this is y
</%def>
</%namespace>
<%
result = []
data = get_data()
for x in data:
result.append(x+7)
%>
result: <%call expr="foo.x(result)"/>
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({}, [Text(u'\n', (1, 1)),
NamespaceTag(u'namespace', {u'name': u'foo'}, (2,
1), [Text(u'\n ', (2, 24)), DefTag(u'def',
{u'name': u'x()'}, (3, 5),
[Text(u'''\n this is x\n ''', (3, 22))]),
Text(u'\n ', (5, 12)), DefTag(u'def', {u'name'
: u'y()'}, (6, 5),
[Text(u'''\n this is y\n ''', (6, 22))]),
Text(u'\n', (8, 12))]), Text(u'''\n\n''', (9, 14)),
Code(u'''\nresult = []\ndata = get_data()\n'''
'''for x in data:\n result.append(x+7)\n\n''',
False, (11, 1)), Text(u'''\n\n result: ''', (16,
3)), CallTag(u'call', {u'expr': u'foo.x(result)'
}, (18, 13), []), Text(u'\n', (18, 42))]))
def test_expression(self):
template = \
"""
this is some ${text} and this is ${textwith | escapes, moreescapes}
<%def name="hi()">
give me ${foo()} and ${bar()}
</%def>
${hi()}
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({},
[Text(u'\n this is some ', (1, 1)),
Expression(u'text', [], (2, 22)),
Text(u' and this is ', (2, 29)),
Expression(u'textwith ', ['escapes', 'moreescapes'
], (2, 42)), Text(u'\n ', (2, 76)),
DefTag(u'def', {u'name': u'hi()'}, (3, 9),
[Text(u'\n give me ', (3, 27)),
Expression(u'foo()', [], (4, 21)), Text(u' and ',
(4, 29)), Expression(u'bar()', [], (4, 34)),
Text(u'\n ', (4, 42))]), Text(u'\n '
, (5, 16)), Expression(u'hi()', [], (6, 9)),
Text(u'\n', (6, 16))]))
def test_tricky_expression(self):
template = """
${x and "|" or "hi"}
"""
nodes = Lexer(template).parse()
self._compare(
nodes,
TemplateNode({}, [
Text(u'\n \n ', (1, 1)),
Expression(u'x and "|" or "hi"', [], (3, 13)),
Text(u'\n ', (3, 33))
])
)
template = """
${hello + '''heres '{|}' text | | }''' | escape1}
"""
nodes = Lexer(template).parse()
self._compare(
nodes,
TemplateNode({}, [
Text(u'\n \n ', (1, 1)),
Expression(u"hello + '''heres '{|}' text | | }''' ",
['escape1'], (3, 13)),
Text(u'\n ', (3, 62))
])
)
def test_tricky_code(self):
if util.py3k:
template = """<% print('hi %>') %>"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({},
[Code(u"print('hi %>') \n", False, (1, 1))]))
else:
template = """<% print 'hi %>' %>"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({},
[Code(u"print 'hi %>' \n", False, (1, 1))]))
def test_tricky_code_2(self):
template = \
"""<%
# someone's comment
%>
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({},
[Code(u"""
# someone's comment
""",
False, (1, 1)), Text(u'\n ', (3, 11))]))
if util.py3k:
def test_tricky_code_3(self):
template = \
"""<%
print('hi')
# this is a comment
# another comment
x = 7 # someone's '''comment
print('''
there
''')
# someone else's comment
%> '''and now some text '''"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({},
[Code(u"""
print('hi')
# this is a comment
# another comment
x = 7 # someone's '''comment
print('''
there
''')
# someone else's comment
""",
False, (1, 1)),
Text(u" '''and now some text '''", (10,
11))]))
else:
def test_tricky_code_3(self):
template = \
"""<%
print 'hi'
# this is a comment
# another comment
x = 7 # someone's '''comment
print '''
there
'''
# someone else's comment
%> '''and now some text '''"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({},
[Code(u"""\nprint 'hi'\n# this is a comment\n"""
"""# another comment\nx = 7 """
"""# someone's '''comment\nprint '''\n """
"""there\n '''\n# someone else's """
"""comment\n \n""",
False, (1, 1)),
Text(u" '''and now some text '''", (10,11))]))
def test_control_lines(self):
template = \
"""
text text la la
% if foo():
mroe text la la blah blah
% endif
and osme more stuff
% for l in range(1,5):
tex tesl asdl l is ${l} kfmas d
% endfor
tetx text
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({},
[Text(u'''\ntext text la la\n''', (1, 1)),
ControlLine(u'if', u'if foo():', False, (3, 1)),
Text(u' mroe text la la blah blah\n', (4, 1)),
ControlLine(u'if', u'endif', True, (5, 1)),
Text(u'''\n and osme more stuff\n''', (6,
1)), ControlLine(u'for', u'for l in range(1,5):',
False, (8, 1)), Text(u' tex tesl asdl l is ',
(9, 1)), Expression(u'l', [], (9, 24)),
Text(u' kfmas d\n', (9, 28)), ControlLine(u'for',
u'endfor', True, (10, 1)),
Text(u''' tetx text\n \n''', (11, 1))]))
def test_control_lines_2(self):
template = \
"""% for file in requestattr['toc'].filenames:
x
% endfor
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({}, [ControlLine(u'for',
u"for file in requestattr['toc'].filenames:",
False, (1, 1)), Text(u' x\n', (2, 1)),
ControlLine(u'for', u'endfor', True, (3, 1))]))
def test_long_control_lines(self):
template = \
"""
% for file in \\
requestattr['toc'].filenames:
x
% endfor
"""
nodes = Lexer(template).parse()
self._compare(
nodes,
TemplateNode({}, [
Text(u'\n', (1, 1)),
ControlLine(u'for', u"for file in \\\n "
"requestattr['toc'].filenames:",
False, (2, 1)),
Text(u' x\n', (4, 1)),
ControlLine(u'for', u'endfor', True, (5, 1)),
Text(u' ', (6, 1))
])
)
def test_unmatched_control(self):
template = """
% if foo:
% for x in range(1,5):
% endif
"""
assert_raises_message(
exceptions.SyntaxException,
"Keyword 'endif' doesn't match keyword 'for' at line: 5 char: 1",
Lexer(template).parse
)
def test_unmatched_control_2(self):
template = """
% if foo:
% for x in range(1,5):
% endfor
"""
assert_raises_message(
exceptions.SyntaxException,
"Unterminated control keyword: 'if' at line: 3 char: 1",
Lexer(template).parse
)
def test_unmatched_control_3(self):
template = """
% if foo:
% for x in range(1,5):
% endlala
% endif
"""
assert_raises_message(
exceptions.SyntaxException,
"Keyword 'endlala' doesn't match keyword 'for' at line: 5 char: 1",
Lexer(template).parse
)
def test_ternary_control(self):
template = \
"""
% if x:
hi
% elif y+7==10:
there
% elif lala:
lala
% else:
hi
% endif
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({}, [Text(u'\n', (1, 1)),
ControlLine(u'if', u'if x:', False, (2, 1)),
Text(u' hi\n', (3, 1)),
ControlLine(u'elif', u'elif y+7==10:', False, (4,
1)), Text(u' there\n', (5, 1)),
ControlLine(u'elif', u'elif lala:', False, (6,
1)), Text(u' lala\n', (7, 1)),
ControlLine(u'else', u'else:', False, (8, 1)),
Text(u' hi\n', (9, 1)),
ControlLine(u'if', u'endif', True, (10, 1))]))
def test_integration(self):
template = \
"""<%namespace name="foo" file="somefile.html"/>
## inherit from foobar.html
<%inherit file="foobar.html"/>
<%def name="header()">
<div>header</div>
</%def>
<%def name="footer()">
<div> footer</div>
</%def>
<table>
% for j in data():
<tr>
% for x in j:
<td>Hello ${x| h}</td>
% endfor
</tr>
% endfor
</table>
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({}, [NamespaceTag(u'namespace'
, {u'file': u'somefile.html', u'name': u'foo'},
(1, 1), []), Text(u'\n', (1, 46)),
Comment(u'inherit from foobar.html', (2, 1)),
InheritTag(u'inherit', {u'file': u'foobar.html'},
(3, 1), []), Text(u'''\n\n''', (3, 31)),
DefTag(u'def', {u'name': u'header()'}, (5, 1),
[Text(u'''\n <div>header</div>\n''', (5,
23))]), Text(u'\n', (7, 8)), DefTag(u'def',
{u'name': u'footer()'}, (8, 1),
[Text(u'''\n <div> footer</div>\n''', (8,
23))]), Text(u'''\n\n<table>\n''', (10, 8)),
ControlLine(u'for', u'for j in data():', False,
(13, 1)), Text(u' <tr>\n', (14, 1)),
ControlLine(u'for', u'for x in j:', False, (15,
1)), Text(u' <td>Hello ', (16, 1)),
Expression(u'x', ['h'], (16, 23)), Text(u'</td>\n'
, (16, 30)), ControlLine(u'for', u'endfor', True,
(17, 1)), Text(u' </tr>\n', (18, 1)),
ControlLine(u'for', u'endfor', True, (19, 1)),
Text(u'</table>\n', (20, 1))]))
def test_comment_after_statement(self):
template = \
"""
% if x: #comment
hi
% else: #next
hi
% endif #end
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({}, [Text(u'\n', (1, 1)),
ControlLine(u'if', u'if x: #comment', False, (2,
1)), Text(u' hi\n', (3, 1)),
ControlLine(u'else', u'else: #next', False, (4,
1)), Text(u' hi\n', (5, 1)),
ControlLine(u'if', u'endif #end', True, (6, 1))]))
def test_crlf(self):
template = open(self._file_path("crlf.html"), 'rb').read()
nodes = Lexer(template).parse()
self._compare(
nodes,
TemplateNode({}, [
Text(u'<html>\r\n\r\n', (1, 1)),
PageTag(u'page', {
u'args': u"a=['foo',\n 'bar']"
}, (3, 1), []),
Text(u'\r\n\r\nlike the name says.\r\n\r\n', (4, 26)),
ControlLine(u'for', u'for x in [1,2,3]:', False, (8, 1)),
Text(u' ', (9, 1)),
Expression(u'x', [], (9, 9)),
ControlLine(u'for', u'endfor', True, (10, 1)),
Text(u'\r\n', (11, 1)),
Expression(u"trumpeter == 'Miles' and "
"trumpeter or \\\n 'Dizzy'",
[], (12, 1)),
Text(u'\r\n\r\n', (13, 15)),
DefTag(u'def', {u'name': u'hi()'}, (15, 1), [
Text(u'\r\n hi!\r\n', (15, 19))]),
Text(u'\r\n\r\n</html>\r\n', (17, 8))
])
)
assert flatten_result(Template(template).render()) \
== """<html> like the name says. 1 2 3 Dizzy </html>"""
def test_comments(self):
template = \
"""
<style>
#someselector
# other non comment stuff
</style>
## a comment
# also not a comment
## this is a comment
this is ## not a comment
<%doc> multiline
comment
</%doc>
hi
"""
nodes = Lexer(template).parse()
self._compare(nodes, TemplateNode({},
[Text(u'''\n<style>\n #someselector\n # '''
'''other non comment stuff\n</style>\n''',
(1, 1)), Comment(u'a comment', (6, 1)),
Text(u'''\n# also not a comment\n\n''', (7, 1)),
Comment(u'this is a comment', (10, 1)),
Text(u''' \nthis is ## not a comment\n\n''', (11,
1)), Comment(u''' multiline\ncomment\n''', (14,
1)), Text(u'''
hi
''', (16, 8))]))
def test_docs(self):
template = \
"""
<%doc>
this is a comment
</%doc>
<%def name="foo()">
<%doc>
this is the foo func
</%doc>
</%def>
"""
nodes = Lexer(template).parse()
self._compare(nodes,
TemplateNode({}, [Text(u'\n ', (1,
1)),
Comment(u'''\n this is a comment\n ''',
(2, 9)), Text(u'\n ', (4, 16)),
DefTag(u'def', {u'name': u'foo()'}, (5, 9),
[Text(u'\n ', (5, 28)),
Comment(u'''\n this is the foo func\n'''
''' ''',
(6, 13)), Text(u'\n ', (8, 20))]),
Text(u'\n ', (9, 16))]))
def test_preprocess(self):
def preproc(text):
return re.sub(r'(?<=\n)\s*#[^#]', '##', text)
template = \
"""
hi
# old style comment
# another comment
"""
nodes = Lexer(template, preprocessor=preproc).parse()
self._compare(nodes, TemplateNode({}, [Text(u'''\n hi\n''',
(1, 1)), Comment(u'old style comment', (3, 1)),
Comment(u'another comment', (4, 1))]))