template_parser: Fix Jinja2 tag matching with strip whitespace syntax.

In this commit, we basically match any kinda of jinja2 start tag,
    no matter its special kind (eg. jinja2_whitespace_stripped_start)
    to any kinda jinja2 end tag (eg. jinja2_whitespace_stripped_end)

    Idea is special operators like `-` do not change the meaning of
    inline tag and thus matching shouldn't depend upon this.

tools/lib/template_parser.py

@@ -160,7 +160,7 @@ def tokenize(text):
                 tag = s[7:-3]
                 kind = 'jinja2_whitespace_stripped_end'
             elif looking_at_jinja2_start_whitespace_stripped_type2():
-                s = get_jinja2_whitespace_stripped_tag(text, state.i)
+                s = get_django_tag(text, state.i, stripped=True)
                 tag = s[3:-3].split()[0]
                 kind = 'jinja2_whitespace_stripped_type2_start'
             else:
@@ -175,7 +175,7 @@ def tokenize(text):
         token = Token(
             kind=kind,
             s=s,
-            tag=tag,
+            tag=tag.strip(),
             line=state.line,
             col=state.col,
             line_span=line_span
@@ -295,11 +295,11 @@ def validate(fn=None, text=None, check_indent=True):
         elif kind == 'handlebars_end':
             state.matcher(token)
 
-        elif kind in {'django_start',
+        elif kind in {'django_start', 'jinja2_whitespace_stripped_start',
                       'jinja2_whitespace_stripped_type2_start'}:
             if is_django_block_tag(tag):
                 start_tag_matcher(token)
-        elif kind == 'django_end':
+        elif kind in {'django_end', 'jinja2_whitespace_stripped_end'}:
             state.matcher(token)
 
     if state.depth != 0:
@@ -354,9 +354,11 @@ def get_handlebars_tag(text, i):
     s = text[i:end+2]
     return s
 
-def get_django_tag(text, i):
+def get_django_tag(text, i, stripped=False):
-    # type: (str, int) -> str
+    # type: (str, int, bool) -> str
     end = i + 2
+    if stripped:
+        end += 1
     while end < len(text) - 1 and text[end] != '%':
         end += 1
     if text[end] != '%' or text[end+1] != '}':
@@ -364,16 +366,6 @@ def get_django_tag(text, i):
     s = text[i:end+2]
     return s
 
-def get_jinja2_whitespace_stripped_tag(text, i):
-    # type: (str, int) -> str
-    end = i + 3
-    while end < len(text) - 1 and text[end] != '%':
-        end += 1
-    if text[end-1] != '-' or text[end] != '%' or text[end+1] != '}':
-        raise TokenizationException('Tag missing "-%}"', text[i:end+2])
-    s = text[i:end+2]
-    return s
-
 def get_html_tag(text, i):
     # type: (str, int) -> str
     quote_count = 0

tools/tests/test_template_parser.py

@@ -191,30 +191,46 @@ class ParserTest(unittest.TestCase):
             '''
         validate(text=my_html)
 
-    def test_validate_jinja2_whitespace_markers(self) -> None:
+    def test_validate_jinja2_whitespace_markers_1(self) -> None:
         my_html = '''
         {% if foo -%}
         this is foo
+        {% endif %}
+        '''
+        validate(text=my_html)
+
+    def test_validate_jinja2_whitespace_markers_2(self) -> None:
+        my_html = '''
+        {% if foo %}
+        this is foo
         {%- endif %}
         '''
         validate(text=my_html)
 
+    def test_validate_jinja2_whitespace_markers_3(self) -> None:
+        my_html = '''
+        {% if foo %}
+        this is foo
+        {% endif -%}
+        '''
+        validate(text=my_html)
+
+    def test_validate_jinja2_whitespace_markers_4(self) -> None:
+        my_html = '''
+        {%- if foo %}
+        this is foo
+        {% endif %}
+        '''
+        validate(text=my_html)
+
     def test_validate_mismatch_jinja2_whitespace_markers_1(self) -> None:
         my_html = '''
         {% if foo %}
         this is foo
-        {%- endif %}
+        {%- if bar %}
         '''
         self._assert_validate_error('Missing end tag', text=my_html)
 
-    def test_validate_mismatch_jinja2_whitespace_markers_2(self) -> None:
-        my_html = '''
-        {% if foo -%}
-        this is foo
-        {% endif %}
-        '''
-        self._assert_validate_error('No start tag', text=my_html)
-
     def test_validate_jinja2_whitespace_type2_markers(self) -> None:
         my_html = '''
         {%- if foo -%}
@@ -223,22 +239,6 @@ class ParserTest(unittest.TestCase):
         '''
         validate(text=my_html)
 
-    def test_validate_mismatch_jinja2_whitespace_type2_markers(self) -> None:
-        my_html = '''
-        {%- if foo -%}
-        this is foo
-        {%- endif %}
-        '''
-        self._assert_validate_error('Missing end tag', text=my_html)
-
-    def test_validate_incomplete_jinja2_whitespace_type2_markers(self) -> None:
-        my_html = '''
-        {%- if foo %}
-        this is foo
-        {% endif %}
-        '''
-        self._assert_validate_error('Tag missing "-%}" at Line 2 Col 9:"{%- if foo %}"', text=my_html)
-
     def test_tokenize(self) -> None:
         tag = '<meta whatever>bla'
         token = tokenize(tag)[0]