zerver: Remove simplejson dependency.

Modified by tabbott to put the third-party code in a new file.

Fixes #6970.

docs/THIRDPARTY

@@ -182,6 +182,10 @@ Files: zerver/lib/decorator.py zerver/management/commands/runtornado.py scripts/
 Copyright: Django Software Foundation and individual contributors
 License: BSD-3-Clause
 
+Files: zerver/lib/json_encoder_for_html.py zerver/tests/test_json_encoder_for_html.py
+Copyright: 2006 Bob Ippolito
+License: MIT or Academic Free License v. 2.1
+
 License: Apache-2.0
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

requirements/common.in

@@ -119,9 +119,6 @@ redis==2.10.6
 requests_oauthlib==0.8.0
 rsa==3.4.2
 
-# Needed for its HTML encoder for page_params
-simplejson==3.11.1
-
 # Needed for Python 2+3 compatibility
 six==1.11.0
 smmap==0.9.0
@@ -184,6 +181,3 @@ social-auth-app-django==1.2.0
 
 # Needed for messages' rendered content parsing in push notifications.
 lxml==4.1.0
-
-# One occurrence left in home.py
-simplejson==3.11.1

requirements/dev.txt

@@ -134,7 +134,6 @@ scrapy==1.4.0
 service-identity==17.0.0  # via scrapy
 sh==1.11                  # via gitlint
 simplegeneric==0.8.1      # via ipython
-simplejson==3.11.1
 six==1.11.0
 smmap==0.9.0
 snakeviz==0.4.2

requirements/prod.txt

@@ -92,7 +92,6 @@ requests-oauthlib==0.8.0
 requests==2.18.4          # via premailer, pyoembed, python-gcm, python-twitter, requests-oauthlib, social-auth-core
 rsa==3.4.2
 simplegeneric==0.8.1      # via ipython
-simplejson==3.11.1
 six==1.11.0
 smmap==0.9.0
 social-auth-app-django==1.2.0

zerver/lib/json_encoder_for_html.py

@@ -0,0 +1,28 @@
+import json
+
+# Taken from
+# https://github.com/simplejson/simplejson/blob/8edc82afcf6f7512b05fba32baa536fe756bd273/simplejson/encoder.py#L378-L402
+# License: MIT
+class JSONEncoderForHTML(json.JSONEncoder):
+    """An encoder that produces JSON safe to embed in HTML.
+    To embed JSON content in, say, a script tag on a web page, the
+    characters &, < and > should be escaped. They cannot be escaped
+    with the usual entities (e.g. &amp;) because they are not expanded
+    within <script> tags.
+    """
+
+    def encode(self, o):
+        # type: (Dict[str, Any]) -> str
+        # Override JSONEncoder.encode because it has hacks for
+        # performance that make things more complicated.
+        chunks = self.iterencode(o, True)
+        return ''.join(chunks)
+
+    def iterencode(self, o, _one_shot=False):
+        # type: (Dict[str, Any], Optional[bool]) -> Iterator[str]
+        chunks = super().iterencode(o, _one_shot)
+        for chunk in chunks:
+            chunk = chunk.replace('&', '\\u0026')
+            chunk = chunk.replace('<', '\\u003c')
+            chunk = chunk.replace('>', '\\u003e')
+            yield chunk

zerver/tests/test_json_encoder_for_html.py

@@ -0,0 +1,32 @@
+import json
+from zerver.lib.json_encoder_for_html import JSONEncoderForHTML
+from zerver.lib.test_classes import ZulipTestCase
+
+class TestJSONEncoder(ZulipTestCase):
+    # Test EncoderForHTML
+    # Taken from
+    # https://github.com/simplejson/simplejson/blob/8edc82afcf6f7512b05fba32baa536fe756bd273/simplejson/tests/test_encode_for_html.py
+    # License: MIT
+    decoder = json.JSONDecoder()
+    encoder = JSONEncoderForHTML()
+
+    def test_basic_encode(self) -> None:
+        self.assertEqual(r'"\u0026"', self.encoder.encode('&'))
+        self.assertEqual(r'"\u003c"', self.encoder.encode('<'))
+        self.assertEqual(r'"\u003e"', self.encoder.encode('>'))
+
+    def test_basic_roundtrip(self) -> None:
+        for char in '&<>':
+            self.assertEqual(
+                char, self.decoder.decode(
+                    self.encoder.encode(char)))
+
+    def test_prevent_script_breakout(self) -> None:
+        bad_string = '</script><script>alert("gotcha")</script>'
+        self.assertEqual(
+            r'"\u003c/script\u003e\u003cscript\u003e'
+            r'alert(\"gotcha\")\u003c/script\u003e"',
+            self.encoder.encode(bad_string))
+        self.assertEqual(
+            bad_string, self.decoder.decode(
+                self.encoder.encode(bad_string)))

zerver/views/home.py

@@ -1,4 +1,4 @@
-from typing import Any, List, Dict, Optional, Text
+from typing import Any, List, Dict, Optional, Text, Iterator
 
 from django.conf import settings
 from django.core.urlresolvers import reverse
@@ -22,6 +22,7 @@ from zerver.lib.actions import update_user_presence, do_change_tos_version, \
 from zerver.lib.avatar import avatar_url
 from zerver.lib.i18n import get_language_list, get_language_name, \
     get_language_list_for_templates
+from zerver.lib.json_encoder_for_html import JSONEncoderForHTML
 from zerver.lib.push_notifications import num_push_devices_for_user
 from zerver.lib.streams import access_stream_by_name
 from zerver.lib.subdomains import get_subdomain
@@ -32,7 +33,6 @@ import datetime
 import logging
 import os
 import re
-import simplejson
 import time
 
 @zulip_login_required
@@ -227,7 +227,7 @@ def home_real(request):
     request._log_data['extra'] = "[%s]" % (register_ret["queue_id"],)
     response = render(request, 'zerver/index.html',
                       context={'user_profile': user_profile,
-                               'page_params': simplejson.encoder.JSONEncoderForHTML().encode(page_params),
+                               'page_params': JSONEncoderForHTML().encode(page_params),
                                'nofontface': is_buggy_ua(request.META.get("HTTP_USER_AGENT", "Unspecified")),
                                'avatar_url': avatar_url(user_profile),
                                'show_debug':