template_parser: Add support for jinja2 whitespace markers.

Fixes #11516.

tools/lib/pretty_print.py

@@ -39,7 +39,9 @@ def pretty_print_html(html, num_spaces=4):
     for token in tokens:
 
         if token.kind in ('html_start', 'handlebars_start', 'handlebars_singleton',
-                          'html_singleton', 'django_start') and stack[-1]['tag'] != 'pre':
+                          'html_singleton', 'django_start',
+                          'jinja2_whitespace_stripped_type2_start',
+                          'jinja2_whitespace_stripped_start') and stack[-1]['tag'] != 'pre':
             # An HTML start tag should only cause a new indent if we
             # are on a new line.
             if (token.tag not in ('extends', 'include', 'else', 'elif') and
@@ -50,8 +52,12 @@ def pretty_print_html(html, num_spaces=4):
                 if is_block:
                     if (((token.kind == 'handlebars_start' and
                             stack[-1]['token_kind'] == 'handlebars_start') or
-                            (token.kind == 'django_start' and
+                            (token.kind in {'django_start',
-                             stack[-1]['token_kind'] == 'django_start')) and
+                                            'jinja2_whitespace_stripped_type2_start',
+                                            'jinja2_whitespace_stripped_start'} and
+                             stack[-1]['token_kind'] in {'django_start',
+                                                         'jinja2_whitespace_stripped_type2_start',
+                                                         'jinja2_whitespace_stripped_start'})) and
                             not stack[-1]['indenting']):
                         info = stack.pop()
                         info['depth'] = info['depth'] + 1
@@ -94,7 +100,8 @@ def pretty_print_html(html, num_spaces=4):
                     )
                 stack.append(info)
         elif (token.kind in ('html_end', 'handlebars_end', 'html_singleton_end',
-                             'django_end', 'handlebars_singleton_end') and
+                             'django_end', 'handlebars_singleton_end',
+                             'jinja2_whitespace_stripped_end') and
               (stack[-1]['tag'] != 'pre' or token.tag == 'pre')):
             info = stack.pop()
             if info['block']:

tools/lib/template_parser.py

@@ -87,6 +87,15 @@ def tokenize(text):
         # type: () -> bool
         return looking_at("{% end")
 
+    def looking_at_jinja2_end_whitespace_stripped():
+        # type: () -> bool
+        return looking_at("{%- end")
+
+    def looking_at_jinja2_start_whitespace_stripped_type2():
+        # type: () -> bool
+        # This function detects tag like {%- if foo -%}...{% endif %}
+        return looking_at("{%-") and not looking_at("{%- end")
+
     state = TokenizerState()
     tokens = []
 
@@ -139,10 +148,21 @@ def tokenize(text):
                 s = get_django_tag(text, state.i)
                 tag = s[3:-2].split()[0]
                 kind = 'django_start'
+
+                if s[-3] == '-':
+                    kind = 'jinja2_whitespace_stripped_start'
             elif looking_at_django_end():
                 s = get_django_tag(text, state.i)
                 tag = s[6:-3]
                 kind = 'django_end'
+            elif looking_at_jinja2_end_whitespace_stripped():
+                s = get_django_tag(text, state.i)
+                tag = s[7:-3]
+                kind = 'jinja2_whitespace_stripped_end'
+            elif looking_at_jinja2_start_whitespace_stripped_type2():
+                s = get_jinja2_whitespace_stripped_tag(text, state.i)
+                tag = s[3:-3].split()[0]
+                kind = 'jinja2_whitespace_stripped_type2_start'
             else:
                 advance(1)
                 continue
@@ -274,7 +294,8 @@ def validate(fn=None, text=None, check_indent=True):
         elif kind == 'handlebars_end':
             state.matcher(token)
 
-        elif kind == 'django_start':
+        elif kind in {'django_start',
+                      'jinja2_whitespace_stripped_type2_start'}:
             if is_django_block_tag(tag):
                 start_tag_matcher(token)
         elif kind == 'django_end':
@@ -341,6 +362,16 @@ def get_django_tag(text, i):
     s = text[i:end+2]
     return s
 
+def get_jinja2_whitespace_stripped_tag(text, i):
+    # type: (str, int) -> str
+    end = i + 3
+    while end < len(text) - 1 and text[end] != '%':
+        end += 1
+    if text[end-1] != '-' or text[end] != '%' or text[end+1] != '}':
+        raise TokenizationException('Tag missing "-%}"', text[i:end+2])
+    s = text[i:end+2]
+    return s
+
 def get_html_tag(text, i):
     # type: (str, int) -> str
     quote_count = 0

tools/tests/test_template_parser.py

@@ -191,6 +191,54 @@ class ParserTest(unittest.TestCase):
             '''
         validate(text=my_html)
 
+    def test_validate_jinja2_whitespace_markers(self) -> None:
+        my_html = '''
+        {% if foo -%}
+        this is foo
+        {%- endif %}
+        '''
+        validate(text=my_html)
+
+    def test_validate_mismatch_jinja2_whitespace_markers_1(self) -> None:
+        my_html = '''
+        {% if foo %}
+        this is foo
+        {%- endif %}
+        '''
+        self._assert_validate_error('Missing end tag', text=my_html)
+
+    def test_validate_mismatch_jinja2_whitespace_markers_2(self) -> None:
+        my_html = '''
+        {% if foo -%}
+        this is foo
+        {% endif %}
+        '''
+        self._assert_validate_error('No start tag', text=my_html)
+
+    def test_validate_jinja2_whitespace_type2_markers(self) -> None:
+        my_html = '''
+        {%- if foo -%}
+        this is foo
+        {% endif %}
+        '''
+        validate(text=my_html)
+
+    def test_validate_mismatch_jinja2_whitespace_type2_markers(self) -> None:
+        my_html = '''
+        {%- if foo -%}
+        this is foo
+        {%- endif %}
+        '''
+        self._assert_validate_error('Missing end tag', text=my_html)
+
+    def test_validate_incomplete_jinja2_whitespace_type2_markers(self) -> None:
+        my_html = '''
+        {%- if foo %}
+        this is foo
+        {% endif %}
+        '''
+        self._assert_validate_error('Tag missing "-%}" at Line 2 Col 9:"{%- if foo %}"', text=my_html)
+
     def test_tokenize(self) -> None:
         tag = '<meta whatever>bla'
         token = tokenize(tag)[0]
@@ -240,3 +288,18 @@ class ParserTest(unittest.TestCase):
         token = tokenize(tag)[0]
         self.assertEqual(token.kind, 'django_end')
         self.assertEqual(token.tag, 'if')
+
+        tag = '{% if foo -%}bla'
+        token = tokenize(tag)[0]
+        self.assertEqual(token.kind, 'jinja2_whitespace_stripped_start')
+        self.assertEqual(token.tag, 'if')
+
+        tag = '{%- endif %}bla'
+        token = tokenize(tag)[0]
+        self.assertEqual(token.kind, 'jinja2_whitespace_stripped_end')
+        self.assertEqual(token.tag, 'if')
+
+        tag = '{%- if foo -%}bla'
+        token = tokenize(tag)[0]
+        self.assertEqual(token.kind, 'jinja2_whitespace_stripped_type2_start')
+        self.assertEqual(token.tag, 'if')