semgrep: Upgrade semgrep to 0.17.0.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

requirements/dev.in

@@ -69,3 +69,6 @@ cairosvg
 
 # Needed for tools/check-thirdparty
 python-debian
+
+# Pattern-based lint tool
+semgrep

requirements/dev.txt

@@ -42,7 +42,7 @@ arrow==0.15.5 \
 attrs==19.3.0 \
     --hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \
     --hash=sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72 \
-    # via automat, jsonschema, openapi-core, service-identity, twisted
+    # via automat, jsonschema, openapi-core, semgrep, service-identity, twisted
 automat==20.2.0 \
     --hash=sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33 \
     --hash=sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111 \
@@ -169,6 +169,10 @@ click==7.0 \
     --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
     --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
     # via gitlint, pip-tools
+colorama==0.4.3 \
+    --hash=sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff \
+    --hash=sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1 \
+    # via semgrep
 commonmark==0.9.1 \
     --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \
     --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 \
@@ -644,7 +648,7 @@ openapi-spec-validator==0.2.9 \
 packaging==20.4 \
     --hash=sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8 \
     --hash=sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181 \
-    # via sphinx
+    # via semgrep, sphinx
 parse==1.16.0 \
     --hash=sha256:cd89e57aed38dcf3e0ff8253f53121a3b23e6181758993323658bffc048a5c19 \
     # via openapi-core
@@ -905,7 +909,7 @@ requests-oauthlib==1.3.0 \
 requests[security]==2.24.0 \
     --hash=sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b \
     --hash=sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898 \
-    # via -r requirements/common.in, docker, hypchat, matrix-client, moto, premailer, pyoembed, python-digitalocean, python-gcm, python-twitter, requests-oauthlib, responses, social-auth-core, sphinx, stripe, twilio, zulip
+    # via -r requirements/common.in, docker, hypchat, matrix-client, moto, premailer, pyoembed, python-digitalocean, python-gcm, python-twitter, requests-oauthlib, responses, semgrep, social-auth-core, sphinx, stripe, twilio, zulip
 responses==0.10.15 \
     --hash=sha256:7bb697a5fedeb41d81e8b87f152d453d5cab42dcd1691b6a7d6097e94d33f373 \
     --hash=sha256:af94d28cdfb48ded0ad82a5216616631543650f440334a693479b8991a6594a2 \
@@ -914,6 +918,31 @@ rsa==4.6 \
     --hash=sha256:109ea5a66744dd859bf16fe904b8d8b627adafb9408753161e766a92e7d681fa \
     --hash=sha256:6166864e23d6b5195a5cfed6cd9fed0fe774e226d8f854fcb23b7bbef0350233 \
     # via python-jose
+ruamel.yaml.clib==0.2.0 \
+    --hash=sha256:1e77424825caba5553bbade750cec2277ef130647d685c2b38f68bc03453bac6 \
+    --hash=sha256:392b7c371312abf27fb549ec2d5e0092f7ef6e6c9f767bfb13e83cb903aca0fd \
+    --hash=sha256:4d55386129291b96483edcb93b381470f7cd69f97585829b048a3d758d31210a \
+    --hash=sha256:550168c02d8de52ee58c3d8a8193d5a8a9491a5e7b2462d27ac5bf63717574c9 \
+    --hash=sha256:57933a6986a3036257ad7bf283529e7c19c2810ff24c86f4a0cfeb49d2099919 \
+    --hash=sha256:615b0396a7fad02d1f9a0dcf9f01202bf9caefee6265198f252c865f4227fcc6 \
+    --hash=sha256:77556a7aa190be9a2bd83b7ee075d3df5f3c5016d395613671487e79b082d784 \
+    --hash=sha256:7aee724e1ff424757b5bd8f6c5bbdb033a570b2b4683b17ace4dbe61a99a657b \
+    --hash=sha256:8073c8b92b06b572e4057b583c3d01674ceaf32167801fe545a087d7a1e8bf52 \
+    --hash=sha256:9c6d040d0396c28d3eaaa6cb20152cb3b2f15adf35a0304f4f40a3cf9f1d2448 \
+    --hash=sha256:a0ff786d2a7dbe55f9544b3f6ebbcc495d7e730df92a08434604f6f470b899c5 \
+    --hash=sha256:b1b7fcee6aedcdc7e62c3a73f238b3d080c7ba6650cd808bce8d7761ec484070 \
+    --hash=sha256:b66832ea8077d9b3f6e311c4a53d06273db5dc2db6e8a908550f3c14d67e718c \
+    --hash=sha256:be018933c2f4ee7de55e7bd7d0d801b3dfb09d21dad0cce8a97995fd3e44be30 \
+    --hash=sha256:d0d3ac228c9bbab08134b4004d748cf9f8743504875b3603b3afbb97e3472947 \
+    --hash=sha256:d10e9dd744cf85c219bf747c75194b624cc7a94f0c80ead624b06bfa9f61d3bc \
+    --hash=sha256:ea4362548ee0cbc266949d8a441238d9ad3600ca9910c3fe4e82ee3a50706973 \
+    --hash=sha256:ed5b3698a2bb241b7f5cbbe277eaa7fe48b07a58784fba4f75224fd066d253ad \
+    --hash=sha256:f9dcc1ae73f36e8059589b601e8e4776b9976effd76c21ad6a855a74318efd6e \
+    # via ruamel.yaml
+ruamel.yaml==0.16.10 \
+    --hash=sha256:0962fd7999e064c4865f96fb1e23079075f4a2a14849bcdc5cdba53a24f9759b \
+    --hash=sha256:099c644a778bf72ffa00524f78dd0b6476bca94a1da344130f4bf3381ce5b954 \
+    # via semgrep
 s3transfer==0.3.3 \
     --hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \
     --hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db \
@@ -922,6 +951,11 @@ scrapy==2.2.1 \
     --hash=sha256:6a09beb5190bfdee2d72cf261822eae5d92fe8a86ac9ee1f55fc44b4864ca583 \
     --hash=sha256:d9d898739f199bd9f9e2258770d5bfeeb754b6ed4eb84a41c04fd52e9649266d \
     # via -r requirements/dev.in
+semgrep==0.17.0 \
+    --hash=sha256:16d2a84e171f88e170032f2fbe6f9577fe1d642d4b7177dd4ab32e24aea6ff0c \
+    --hash=sha256:9ecc5c1d1321e9780aafcb04fb4a9882b27ba860dd713ef00b2b19b6bc21d86a \
+    --hash=sha256:c61cbb7104833ce6618353ed5915da559f1a409139a97205767b62c18c13124a \
+    # via -r requirements/dev.in
 sentry-sdk==0.16.2 \
     --hash=sha256:2de15b13836fa3522815a933bd9c887c77f4868071043349f94f1b896c1bcfb8 \
     --hash=sha256:38bb09d0277117f76507c8728d9a5156f09a47ac5175bb8072513859d19a593b \
@@ -1061,6 +1095,10 @@ tornado==4.5.3 \
     --hash=sha256:ab587996fe6fb9ce65abfda440f9b61e4f9f2cf921967723540679176915e4c3 \
     --hash=sha256:b36298e9f63f18cad97378db2222c0e0ca6a55f6304e605515e05a25483ed51a \
     # via -r requirements/common.in, snakeviz
+tqdm==4.48.0 \
+    --hash=sha256:6baa75a88582b1db6d34ce4690da5501d2a1cb65c34664840a456b2c9f794d29 \
+    --hash=sha256:fcb7cb5b729b60a27f300b15c1ffd4744f080fb483b88f31dc8654b082cc8ea5 \
+    # via semgrep
 traitlets==4.3.3 \
     --hash=sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44 \
     --hash=sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7 \

tools/lib/provision.py

@@ -391,9 +391,6 @@ def main(options: argparse.Namespace) -> "NoReturn":
     # Install shellcheck.
     run_as_root(["tools/setup/install-shellcheck"])
 
-    # Install semgrep.
-    run_as_root(["tools/setup/install-semgrep"])
-
     setup_venvs.main()
 
     run_as_root(["cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])

tools/setup/install-semgrep

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-version=0.14.0
-tarball=semgrep-v$version-ubuntu-16.04.tgz
-sha256=8b9437af0540ed9664904f9603d9d6ad011dad46433cba74e524c7753c7732c9
-tarball_url=https://github.com/returntocorp/semgrep/releases/download/v$version/$tarball
-
-check_version () {
-    out="$(semgrep --version 2>/dev/null)" && [ "$out" = "$version" ]
-}
-
-if ! check_version; then
-    tmpdir="$(mktemp -d)"
-    trap 'rm -r "$tmpdir"' EXIT
-    cd "$tmpdir"
-    wget -nv "$tarball_url"
-    sha256sum -c <<< "$sha256  $tarball"
-    tar -xzf "$tarball" -C /usr/local/lib/ semgrep-files/
-    ln -sf /usr/local/lib/semgrep-files/semgrep /usr/local/bin/semgrep
-    ln -sf /usr/local/lib/semgrep-files/semgrep-core /usr/local/bin/semgrep-core
-
-    # Clean old files from sgrep 0.4.9b5.
-    rm -rf /usr/local/lib/sgrep-lint-files /usr/local/bin/sgrep-lint /usr/local/bin/sgrep
-
-    check_version
-fi

version.py

@@ -44,4 +44,4 @@ API_FEATURE_LEVEL = 27
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = '94.1'
+PROVISION_VERSION = '94.2'