diff --git a/requirements/dev.in b/requirements/dev.in index f96abf5be6..f45c4f57d7 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -69,3 +69,6 @@ cairosvg # Needed for tools/check-thirdparty python-debian + +# Pattern-based lint tool +semgrep diff --git a/requirements/dev.txt b/requirements/dev.txt index ccbf7d949c..1ca8501ceb 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -42,7 +42,7 @@ arrow==0.15.5 \ attrs==19.3.0 \ --hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \ --hash=sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72 \ - # via automat, jsonschema, openapi-core, service-identity, twisted + # via automat, jsonschema, openapi-core, semgrep, service-identity, twisted automat==20.2.0 \ --hash=sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33 \ --hash=sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111 \ @@ -169,6 +169,10 @@ click==7.0 \ --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \ --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \ # via gitlint, pip-tools +colorama==0.4.3 \ + --hash=sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff \ + --hash=sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1 \ + # via semgrep commonmark==0.9.1 \ --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 \ @@ -644,7 +648,7 @@ openapi-spec-validator==0.2.9 \ packaging==20.4 \ --hash=sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8 \ --hash=sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181 \ - # via sphinx + # via semgrep, sphinx parse==1.16.0 \ --hash=sha256:cd89e57aed38dcf3e0ff8253f53121a3b23e6181758993323658bffc048a5c19 \ # via openapi-core @@ -905,7 +909,7 @@ requests-oauthlib==1.3.0 \ requests[security]==2.24.0 \ --hash=sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b \ --hash=sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898 \ - # via -r requirements/common.in, docker, hypchat, matrix-client, moto, premailer, pyoembed, python-digitalocean, python-gcm, python-twitter, requests-oauthlib, responses, social-auth-core, sphinx, stripe, twilio, zulip + # via -r requirements/common.in, docker, hypchat, matrix-client, moto, premailer, pyoembed, python-digitalocean, python-gcm, python-twitter, requests-oauthlib, responses, semgrep, social-auth-core, sphinx, stripe, twilio, zulip responses==0.10.15 \ --hash=sha256:7bb697a5fedeb41d81e8b87f152d453d5cab42dcd1691b6a7d6097e94d33f373 \ --hash=sha256:af94d28cdfb48ded0ad82a5216616631543650f440334a693479b8991a6594a2 \ @@ -914,6 +918,31 @@ rsa==4.6 \ --hash=sha256:109ea5a66744dd859bf16fe904b8d8b627adafb9408753161e766a92e7d681fa \ --hash=sha256:6166864e23d6b5195a5cfed6cd9fed0fe774e226d8f854fcb23b7bbef0350233 \ # via python-jose +ruamel.yaml.clib==0.2.0 \ + --hash=sha256:1e77424825caba5553bbade750cec2277ef130647d685c2b38f68bc03453bac6 \ + --hash=sha256:392b7c371312abf27fb549ec2d5e0092f7ef6e6c9f767bfb13e83cb903aca0fd \ + --hash=sha256:4d55386129291b96483edcb93b381470f7cd69f97585829b048a3d758d31210a \ + --hash=sha256:550168c02d8de52ee58c3d8a8193d5a8a9491a5e7b2462d27ac5bf63717574c9 \ + --hash=sha256:57933a6986a3036257ad7bf283529e7c19c2810ff24c86f4a0cfeb49d2099919 \ + --hash=sha256:615b0396a7fad02d1f9a0dcf9f01202bf9caefee6265198f252c865f4227fcc6 \ + --hash=sha256:77556a7aa190be9a2bd83b7ee075d3df5f3c5016d395613671487e79b082d784 \ + --hash=sha256:7aee724e1ff424757b5bd8f6c5bbdb033a570b2b4683b17ace4dbe61a99a657b \ + --hash=sha256:8073c8b92b06b572e4057b583c3d01674ceaf32167801fe545a087d7a1e8bf52 \ + --hash=sha256:9c6d040d0396c28d3eaaa6cb20152cb3b2f15adf35a0304f4f40a3cf9f1d2448 \ + --hash=sha256:a0ff786d2a7dbe55f9544b3f6ebbcc495d7e730df92a08434604f6f470b899c5 \ + --hash=sha256:b1b7fcee6aedcdc7e62c3a73f238b3d080c7ba6650cd808bce8d7761ec484070 \ + --hash=sha256:b66832ea8077d9b3f6e311c4a53d06273db5dc2db6e8a908550f3c14d67e718c \ + --hash=sha256:be018933c2f4ee7de55e7bd7d0d801b3dfb09d21dad0cce8a97995fd3e44be30 \ + --hash=sha256:d0d3ac228c9bbab08134b4004d748cf9f8743504875b3603b3afbb97e3472947 \ + --hash=sha256:d10e9dd744cf85c219bf747c75194b624cc7a94f0c80ead624b06bfa9f61d3bc \ + --hash=sha256:ea4362548ee0cbc266949d8a441238d9ad3600ca9910c3fe4e82ee3a50706973 \ + --hash=sha256:ed5b3698a2bb241b7f5cbbe277eaa7fe48b07a58784fba4f75224fd066d253ad \ + --hash=sha256:f9dcc1ae73f36e8059589b601e8e4776b9976effd76c21ad6a855a74318efd6e \ + # via ruamel.yaml +ruamel.yaml==0.16.10 \ + --hash=sha256:0962fd7999e064c4865f96fb1e23079075f4a2a14849bcdc5cdba53a24f9759b \ + --hash=sha256:099c644a778bf72ffa00524f78dd0b6476bca94a1da344130f4bf3381ce5b954 \ + # via semgrep s3transfer==0.3.3 \ --hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \ --hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db \ @@ -922,6 +951,11 @@ scrapy==2.2.1 \ --hash=sha256:6a09beb5190bfdee2d72cf261822eae5d92fe8a86ac9ee1f55fc44b4864ca583 \ --hash=sha256:d9d898739f199bd9f9e2258770d5bfeeb754b6ed4eb84a41c04fd52e9649266d \ # via -r requirements/dev.in +semgrep==0.17.0 \ + --hash=sha256:16d2a84e171f88e170032f2fbe6f9577fe1d642d4b7177dd4ab32e24aea6ff0c \ + --hash=sha256:9ecc5c1d1321e9780aafcb04fb4a9882b27ba860dd713ef00b2b19b6bc21d86a \ + --hash=sha256:c61cbb7104833ce6618353ed5915da559f1a409139a97205767b62c18c13124a \ + # via -r requirements/dev.in sentry-sdk==0.16.2 \ --hash=sha256:2de15b13836fa3522815a933bd9c887c77f4868071043349f94f1b896c1bcfb8 \ --hash=sha256:38bb09d0277117f76507c8728d9a5156f09a47ac5175bb8072513859d19a593b \ @@ -1061,6 +1095,10 @@ tornado==4.5.3 \ --hash=sha256:ab587996fe6fb9ce65abfda440f9b61e4f9f2cf921967723540679176915e4c3 \ --hash=sha256:b36298e9f63f18cad97378db2222c0e0ca6a55f6304e605515e05a25483ed51a \ # via -r requirements/common.in, snakeviz +tqdm==4.48.0 \ + --hash=sha256:6baa75a88582b1db6d34ce4690da5501d2a1cb65c34664840a456b2c9f794d29 \ + --hash=sha256:fcb7cb5b729b60a27f300b15c1ffd4744f080fb483b88f31dc8654b082cc8ea5 \ + # via semgrep traitlets==4.3.3 \ --hash=sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44 \ --hash=sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7 \ diff --git a/tools/lib/provision.py b/tools/lib/provision.py index 4111ec6329..40f3610297 100755 --- a/tools/lib/provision.py +++ b/tools/lib/provision.py @@ -391,9 +391,6 @@ def main(options: argparse.Namespace) -> "NoReturn": # Install shellcheck. run_as_root(["tools/setup/install-shellcheck"]) - # Install semgrep. - run_as_root(["tools/setup/install-semgrep"]) - setup_venvs.main() run_as_root(["cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH]) diff --git a/tools/setup/install-semgrep b/tools/setup/install-semgrep deleted file mode 100755 index 55e0ca10b6..0000000000 --- a/tools/setup/install-semgrep +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -set -e - -version=0.14.0 -tarball=semgrep-v$version-ubuntu-16.04.tgz -sha256=8b9437af0540ed9664904f9603d9d6ad011dad46433cba74e524c7753c7732c9 -tarball_url=https://github.com/returntocorp/semgrep/releases/download/v$version/$tarball - -check_version () { - out="$(semgrep --version 2>/dev/null)" && [ "$out" = "$version" ] -} - -if ! check_version; then - tmpdir="$(mktemp -d)" - trap 'rm -r "$tmpdir"' EXIT - cd "$tmpdir" - wget -nv "$tarball_url" - sha256sum -c <<< "$sha256 $tarball" - tar -xzf "$tarball" -C /usr/local/lib/ semgrep-files/ - ln -sf /usr/local/lib/semgrep-files/semgrep /usr/local/bin/semgrep - ln -sf /usr/local/lib/semgrep-files/semgrep-core /usr/local/bin/semgrep-core - - # Clean old files from sgrep 0.4.9b5. - rm -rf /usr/local/lib/sgrep-lint-files /usr/local/bin/sgrep-lint /usr/local/bin/sgrep - - check_version -fi diff --git a/version.py b/version.py index 439a0f1f43..d83b1587d4 100644 --- a/version.py +++ b/version.py @@ -44,4 +44,4 @@ API_FEATURE_LEVEL = 27 # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = '94.1' +PROVISION_VERSION = '94.2'