From d0492b103b2c1db6c0314cb82bc28a84959d4b02 Mon Sep 17 00:00:00 2001
From: Aditya Bansal <adi.bansal241996@gmail.com>
Date: Fri, 17 Jan 2020 01:42:02 +0530
Subject: [PATCH] template_parser: Fix Jinja2 tag matching with strip
 whitespace syntax.

    In this commit, we basically match any kinda of jinja2 start tag,
    no matter its special kind (eg. jinja2_whitespace_stripped_start)
    to any kinda jinja2 end tag (eg. jinja2_whitespace_stripped_end)

    Idea is special operators like `-` do not change the meaning of
    inline tag and thus matching shouldn't depend upon this.
---
 tools/lib/template_parser.py        | 24 +++++--------
 tools/tests/test_template_parser.py | 52 ++++++++++++++---------------
 2 files changed, 34 insertions(+), 42 deletions(-)

diff --git a/tools/lib/template_parser.py b/tools/lib/template_parser.py
index 485291c955..2716d5bfa7 100644
--- a/tools/lib/template_parser.py
+++ b/tools/lib/template_parser.py
@@ -160,7 +160,7 @@ def tokenize(text):
                 tag = s[7:-3]
                 kind = 'jinja2_whitespace_stripped_end'
             elif looking_at_jinja2_start_whitespace_stripped_type2():
-                s = get_jinja2_whitespace_stripped_tag(text, state.i)
+                s = get_django_tag(text, state.i, stripped=True)
                 tag = s[3:-3].split()[0]
                 kind = 'jinja2_whitespace_stripped_type2_start'
             else:
@@ -175,7 +175,7 @@ def tokenize(text):
         token = Token(
             kind=kind,
             s=s,
-            tag=tag,
+            tag=tag.strip(),
             line=state.line,
             col=state.col,
             line_span=line_span
@@ -295,11 +295,11 @@ def validate(fn=None, text=None, check_indent=True):
         elif kind == 'handlebars_end':
             state.matcher(token)
 
-        elif kind in {'django_start',
+        elif kind in {'django_start', 'jinja2_whitespace_stripped_start',
                       'jinja2_whitespace_stripped_type2_start'}:
             if is_django_block_tag(tag):
                 start_tag_matcher(token)
-        elif kind == 'django_end':
+        elif kind in {'django_end', 'jinja2_whitespace_stripped_end'}:
             state.matcher(token)
 
     if state.depth != 0:
@@ -354,9 +354,11 @@ def get_handlebars_tag(text, i):
     s = text[i:end+2]
     return s
 
-def get_django_tag(text, i):
-    # type: (str, int) -> str
+def get_django_tag(text, i, stripped=False):
+    # type: (str, int, bool) -> str
     end = i + 2
+    if stripped:
+        end += 1
     while end < len(text) - 1 and text[end] != '%':
         end += 1
     if text[end] != '%' or text[end+1] != '}':
@@ -364,16 +366,6 @@ def get_django_tag(text, i):
     s = text[i:end+2]
     return s
 
-def get_jinja2_whitespace_stripped_tag(text, i):
-    # type: (str, int) -> str
-    end = i + 3
-    while end < len(text) - 1 and text[end] != '%':
-        end += 1
-    if text[end-1] != '-' or text[end] != '%' or text[end+1] != '}':
-        raise TokenizationException('Tag missing "-%}"', text[i:end+2])
-    s = text[i:end+2]
-    return s
-
 def get_html_tag(text, i):
     # type: (str, int) -> str
     quote_count = 0
diff --git a/tools/tests/test_template_parser.py b/tools/tests/test_template_parser.py
index 9bec993224..894058251f 100644
--- a/tools/tests/test_template_parser.py
+++ b/tools/tests/test_template_parser.py
@@ -191,30 +191,46 @@ class ParserTest(unittest.TestCase):
             '''
         validate(text=my_html)
 
-    def test_validate_jinja2_whitespace_markers(self) -> None:
+    def test_validate_jinja2_whitespace_markers_1(self) -> None:
         my_html = '''
         {% if foo -%}
         this is foo
+        {% endif %}
+        '''
+        validate(text=my_html)
+
+    def test_validate_jinja2_whitespace_markers_2(self) -> None:
+        my_html = '''
+        {% if foo %}
+        this is foo
         {%- endif %}
         '''
         validate(text=my_html)
 
+    def test_validate_jinja2_whitespace_markers_3(self) -> None:
+        my_html = '''
+        {% if foo %}
+        this is foo
+        {% endif -%}
+        '''
+        validate(text=my_html)
+
+    def test_validate_jinja2_whitespace_markers_4(self) -> None:
+        my_html = '''
+        {%- if foo %}
+        this is foo
+        {% endif %}
+        '''
+        validate(text=my_html)
+
     def test_validate_mismatch_jinja2_whitespace_markers_1(self) -> None:
         my_html = '''
         {% if foo %}
         this is foo
-        {%- endif %}
+        {%- if bar %}
         '''
         self._assert_validate_error('Missing end tag', text=my_html)
 
-    def test_validate_mismatch_jinja2_whitespace_markers_2(self) -> None:
-        my_html = '''
-        {% if foo -%}
-        this is foo
-        {% endif %}
-        '''
-        self._assert_validate_error('No start tag', text=my_html)
-
     def test_validate_jinja2_whitespace_type2_markers(self) -> None:
         my_html = '''
         {%- if foo -%}
@@ -223,22 +239,6 @@ class ParserTest(unittest.TestCase):
         '''
         validate(text=my_html)
 
-    def test_validate_mismatch_jinja2_whitespace_type2_markers(self) -> None:
-        my_html = '''
-        {%- if foo -%}
-        this is foo
-        {%- endif %}
-        '''
-        self._assert_validate_error('Missing end tag', text=my_html)
-
-    def test_validate_incomplete_jinja2_whitespace_type2_markers(self) -> None:
-        my_html = '''
-        {%- if foo %}
-        this is foo
-        {% endif %}
-        '''
-        self._assert_validate_error('Tag missing "-%}" at Line 2 Col 9:"{%- if foo %}"', text=my_html)
-
     def test_tokenize(self) -> None:
         tag = '<meta whatever>bla'
         token = tokenize(tag)[0]