build_pygments_data: Reprioritize and categorize pygment language codes.

We now organize the pygment language codes into meaningful categories
- default, custom and aliases.

Further the `lang.json` list now contains a dataset extracted from the
"language" section of https://insights.stackoverflow.com/survey/2020
and is prioritized based on current language trends.

frontend_tests/node_tests/typeahead_helper.js

@@ -253,12 +253,12 @@ test("sort_streams", (override) => {
 test("sort_languages", () => {
     Object.assign(pygments_data, {
         langs: {
-            python: {priority: 40},
+            python: {priority: 26},
-            javscript: {priority: 50},
+            javscript: {priority: 27},
-            php: {priority: 38},
+            php: {priority: 16},
-            pascal: {priority: 29},
+            pascal: {priority: 15},
-            perl: {priority: 22},
+            perl: {priority: 3},
-            css: {priority: 0},
+            css: {priority: 21},
         },
     });
 
@@ -269,7 +269,7 @@ test("sort_languages", () => {
     assert.deepEqual(test_langs, ["python", "php", "pascal", "perl", "javascript"]);
 
     // Test if popularity between two languages are the same
-    pygments_data.langs.php = {priority: 40};
+    pygments_data.langs.php = {priority: 26};
     test_langs = ["pascal", "perl", "php", "python", "javascript"];
     test_langs = th.sort_languages(test_langs, "p");
 
@@ -291,7 +291,7 @@ test("sort_languages", () => {
     test_langs = th.sort_languages(test_langs, "java");
     assert.deepEqual(test_langs, ["java", "javascript", "js", "j"]);
     test_langs = th.sort_languages(test_langs, "j");
-    assert.deepEqual(test_langs, ["j", "javascript", "java", "js"]);
+    assert.deepEqual(test_langs, ["j", "javascript", "js", "java"]);
 });
 
 function get_typeahead_result(query, current_stream, current_topic) {

tools/setup/build_pygments_data

@@ -5,11 +5,15 @@ import os
 from pygments.lexers import get_all_lexers
 
 ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../")
+# The current priorities data set is based on StackOverflow's 2020 survey.
 DATA_PATH = os.path.join(ZULIP_PATH, "tools", "setup", "lang.json")
 OUT_PATH = os.path.join(ZULIP_PATH, "static", "generated", "pygments_data.json")
 
 with open(DATA_PATH) as f:
-    priorities = json.load(f)
+    pygments_data = json.load(f)
+    priorities = dict(
+        **pygments_data["default"], **pygments_data["custom"], **pygments_data["aliases"]
+    )
 
 lexers = get_all_lexers()
 langs = {

tools/setup/lang.json

@@ -1,59 +1,44 @@
 {
-    "abap": 27,
+    "default": {
-    "ada": 25,
+        "javascript": 27,
-    "awk": 1,
+        "python": 26,
-    "bash": 7,
+        "java": 25,
-    "c": 49,
+        "go": 24,
-    "c#": 47,
+        "rust": 23,
-    "c++": 48,
+        "html": 22,
-    "cobol": 26,
+        "css": 21,
-    "cpp": 48,
+        "sql": 20,
-    "csharp": 47,
+        "bash": 19,
-    "css": 48,
+        "powershell": 18,
-    "d": 29,
+        "c#": 17,
-    "dart": 28,
+        "php": 16,
-    "delphi": 42,
+        "typescript": 15,
-    "erlang": 10,
+        "c++": 14,
-    "fsharp": 19,
+        "c": 13,
-    "go": 34,
+        "kotlin": 12,
-    "groovy": 13,
+        "ruby": 11,
-    "haskell": 15,
+        "asm": 10,
-    "html": 30,
+        "vb.net": 9,
-    "java": 50,
+        "swift": 8,
-    "javascript": 51,
+        "r": 7,
-    "js": 43,
+        "objective-c": 6,
-    "julia": 4,
+        "dart": 5,
-    "latex": 40,
+        "scala": 4,
-    "lisp": 18,
+        "perl": 3,
-    "lua": 22,
+        "haskell": 2,
-    "mask": 2,
+        "julia": 1
-    "math": 50,
+    },
-    "matlab": 33,
+    "custom": {
-    "mql": 9,
+        "latex": 10,
-    "mql4": 9,
+        "math": 5,
-    "objective-c": 35,
+        "quote": 5,
-    "objectivec": 35,
+        "spoiler": 5
-    "objectpascal": 42,
+    },
-    "pascal": 42,
+    "aliases": {
-    "perl": 40,
+        "js": 27,
-    "php": 44,
+        "csharp": 17,
-    "pl": 40,
+        "cpp": 14,
-    "prolog": 16,
+        "tex": 10,
-    "python": 46,
+        "vbnet": 9
-    "quote": 50,
+    }
-    "r": 37,
-    "rb": 39,
-    "ruby": 39,
-    "rust": 8,
-    "sas": 30,
-    "scala": 21,
-    "scheme": 14,
-    "sql": 32,
-    "spoiler": 50,
-    "swift": 41,
-    "tex": 40,
-    "text": 1,
-    "vb.net": 45,
-    "vbnet": 45,
-    "xml": 1
 }

version.py

@@ -48,4 +48,4 @@ API_FEATURE_LEVEL = 71
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = "147.1"
+PROVISION_VERSION = "148.0"