Add test coverage for parsers in tools/lib.

Now, `tools/test-all` calls a new program called `tools/tests-tools`
that runs unit tests in `test_css_parser.py` and 'test_template_parser.py`.

This puts 100% line coverage on tools/lib/css_parser.py.
This puts about 50% line coverage on tools/lib/template_parser.py.

tools/test-all

@@ -15,6 +15,7 @@ function run {
 }
 
 run ./tools/clean-repo
+run ./tools/test-tools
 run ./tools/lint-all
 run ./tools/test-migrations
 run ./tools/test-js-with-node

tools/test-tools

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import optparse
+import os
+import sys
+import unittest
+
+if __name__ == '__main__':
+    parser = optparse.OptionParser()
+    parser.add_option('--coverage', dest='coverage',
+                      action="store_true",
+                      default=False, help='Compute test coverage.')
+    (options, _) = parser.parse_args()
+
+    def dir_join(dir1, dir2):
+        # type: (str, str) -> str
+        return os.path.abspath(os.path.join(dir1, dir2))
+
+    tools_dir = os.path.dirname(os.path.abspath(__file__))
+    root_dir = dir_join(tools_dir, '..')
+    tools_test_dir = dir_join(tools_dir, 'tests')
+
+    sys.path.insert(0, root_dir)
+
+    loader = unittest.TestLoader() # type: ignore # https://github.com/python/typeshed/issues/372
+
+    if options.coverage:
+        import coverage
+        cov = coverage.Coverage(omit="*/zulip-venv-cache/*")
+        cov.start()
+
+    suite = loader.discover(start_dir=tools_test_dir, top_level_dir=root_dir)
+    runner = unittest.TextTestRunner(verbosity=2)
+    result = runner.run(suite) # type: ignore # https://github.com/python/typeshed/issues/372
+    if result.errors or result.failures:
+        raise Exception('Test failed!')
+
+    if options.coverage:
+        cov.stop()
+        cov.save()
+        cov.html_report(directory='var/tools_coverage')
+        print("HTML report saved to var/tools_coverage")
+
+    print('SUCCESS')

tools/tests/test_css_parser.py

@@ -0,0 +1,169 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from typing import cast
+
+import sys
+import unittest
+
+try:
+    from tools.lib.css_parser import (
+        CssParserException,
+        CssSection,
+        parse,
+    )
+except ImportError:
+    print('ERROR!!! You need to run this via tools/test-tools.')
+    sys.exit(1)
+
+class ParserTestHappyPath(unittest.TestCase):
+    def test_basic_parse(self):
+        # type: () -> None
+        my_selector = 'li.foo'
+        my_block = '''{
+                color: red;
+            }'''
+        my_css = my_selector + ' ' + my_block
+        res = parse(my_css)
+        self.assertEqual(res.text(), my_css)
+        section = cast(CssSection, res.sections[0])
+        block = section.declaration_block
+        self.assertEqual(block.text().strip(), my_block)
+        declaration = block.declarations[0]
+        self.assertEqual(declaration.css_property, 'color')
+        self.assertEqual(declaration.css_value.text().strip(), 'red')
+
+    def test_same_line_comment(self):
+        # type: () -> None
+        my_css = '''
+            li.hide {
+                display: none; /* comment here */
+                /* Not to be confused
+                   with this comment */
+                color: green;
+            }'''
+        res = parse(my_css)
+        section = cast(CssSection, res.sections[0])
+        block = section.declaration_block
+        declaration = block.declarations[0]
+        self.assertIn('/* comment here */', declaration.text())
+
+    def test_multi_line_selector(self):
+        # type: () -> None
+        my_css = '''
+            h1,
+            h2,
+            h3 {
+                top: 0
+            }'''
+        res = parse(my_css)
+        section = res.sections[0]
+        selectors = section.selector_list.selectors
+        self.assertEqual(len(selectors), 3)
+
+    def test_comment_at_end(self):
+        # type: () -> None
+        '''
+        This test verifies the current behavior, which is to
+        attach comments to the preceding rule, but we should
+        probably change it so the comments gets attached to
+        the next block, if possible.
+        '''
+        my_css = '''
+            p {
+                color: black;
+            }
+
+            /* comment at the end of the text */
+            '''
+        res = parse(my_css)
+        self.assertEqual(len(res.sections), 1)
+        section = res.sections[0]
+        self.assertIn('comment at the end', section.post_fluff)
+
+    def test_media_block(self):
+        # type: () -> None
+        my_css = '''
+            @media (max-width: 300px) {
+                h5 {
+                    margin: 0;
+                }
+            }'''
+        res = parse(my_css)
+        self.assertEqual(len(res.sections), 1)
+        self.assertEqual(res.text(), my_css)
+
+class ParserTestSadPath(unittest.TestCase):
+    '''
+    Use this class for tests that verify the parser will
+    appropriately choke on malformed CSS.
+
+    We prevent some things that are technically legal
+    in CSS, like having comments in the middle of list
+    of selectors.  Some of this is just for expediency;
+    some of this is to enforce consistent formatting.
+    '''
+
+    def _assert_error(self, my_css, error):
+        # See https://github.com/python/typeshed/issues/372
+        # for why we have to ingore types here.
+        with self.assertRaisesRegexp(CssParserException, error): # type: ignore
+            parse(my_css)
+
+    def test_unexpected_end_brace(self):
+        # type: () -> None
+        my_css = '''
+            @media (max-width: 975px) {
+                body {
+                    color: red;
+                }
+            }} /* whoops */'''
+        error = 'unexpected }'
+        self._assert_error(my_css, error)
+
+    def test_empty_section(self):
+        # type: () -> None
+        my_css = '''
+
+            /* nothing to see here, move along */
+            '''
+        error = 'unexpected empty section'
+        self._assert_error(my_css, error)
+
+    def test_missing_colon(self):
+        # type: () -> None
+        my_css = '''
+            .hide
+            {
+                display none /* no colon here */
+            }'''
+        error = 'We expect a colon here'
+        self._assert_error(my_css, error)
+
+    def test_unclosed_comment(self):
+        # type: () -> None
+        my_css = ''' /* comment with no end'''
+        error = 'unclosed comment'
+        self._assert_error(my_css, error)
+
+    def test_missing_selectors(self):
+        # type: () -> None
+        my_css = '''
+            /* no selectors here */
+            {
+                bottom: 0
+            }'''
+        error = 'Missing selector'
+        self._assert_error(my_css, error)
+
+    def test_disallow_comments_in_selectors(self):
+        # type: () -> None
+        my_css = '''
+            h1,
+            h2, /* comment here not allowed by Zulip */
+            h3 {
+                top: 0
+            }'''
+        error = 'Comments in selector section are not allowed'
+        self._assert_error(my_css, error)
+

tools/tests/test_template_parser.py

@@ -0,0 +1,34 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+import sys
+import unittest
+
+try:
+    from tools.lib.template_parser import (
+        is_django_block_tag,
+        validate,
+    )
+except ImportError:
+    print('ERROR!!! You need to run this via tools/test-tools.')
+    sys.exit(1)
+
+class ParserTest(unittest.TestCase):
+    def test_is_django_block_tag(self):
+        # type: () -> None
+        self.assertTrue(is_django_block_tag('block'))
+        self.assertFalse(is_django_block_tag('not a django tag'))
+
+    def test_validate_vanilla_html(self):
+        # type: () -> None
+        '''
+        Verify that validate() does not raise errors for
+        well-formed HTML.
+        '''
+        my_html = '''
+            <table>
+                <tr>
+                <td>foo</td>
+                </tr>
+            </table>'''
+        validate(text=my_html)