From e75f0c1ee4267ece88d37c672be914788ac1d592 Mon Sep 17 00:00:00 2001 From: Harshit Bansal Date: Fri, 17 Nov 2017 01:41:06 +0000 Subject: [PATCH] requirements: Rename requirements files. This commit renames various source requirements files like `dev.txt`, `mypy.txt` etc to `dev.in`, `mypy.in` etc and various locked requirements files like `dev_lock.txt`, `mypy_lock.txt` etc to `dev.txt`, `mypy.txt` etc. This will help in emphasizing to the user that *.in are actually input to `update-locked-requirements` tool which should be run after updating any of these. --- docs/development/setup-advanced.md | 2 +- docs/requirements.readthedocs.txt | 2 +- docs/subsystems/dependencies.md | 38 ++--- docs/subsystems/documentation.md | 3 +- requirements/README.md | 4 +- requirements/{common.txt => common.in} | 4 +- requirements/dev.in | 49 ++++++ requirements/dev.txt | 207 +++++++++++++++++++------ requirements/dev_lock.txt | 165 -------------------- requirements/docs.in | 26 ++++ requirements/docs.txt | 46 +++--- requirements/docs_lock.txt | 30 ---- requirements/mypy.in | 5 + requirements/mypy.txt | 15 +- requirements/mypy_lock.txt | 12 -- requirements/prod.in | 6 + requirements/prod.txt | 116 +++++++++++++- requirements/prod_lock.txt | 109 ------------- requirements/thumbor.in | 12 ++ requirements/thumbor.txt | 37 ++++- requirements/thumbor_lock.txt | 33 ---- scripts/lib/create-production-venv | 4 +- scripts/lib/create-thumbor-venv | 2 +- tools/diagnose | 2 +- tools/setup/setup_venvs.py | 2 +- tools/test-locked-requirements | 8 +- tools/update-locked-requirements | 10 +- 27 files changed, 476 insertions(+), 473 deletions(-) rename requirements/{common.txt => common.in} (97%) create mode 100644 requirements/dev.in delete mode 100644 requirements/dev_lock.txt create mode 100644 requirements/docs.in delete mode 100644 requirements/docs_lock.txt create mode 100644 requirements/mypy.in delete mode 100644 requirements/mypy_lock.txt create mode 100644 requirements/prod.in delete mode 100644 requirements/prod_lock.txt create mode 100644 requirements/thumbor.in delete mode 100644 requirements/thumbor_lock.txt diff --git a/docs/development/setup-advanced.md b/docs/development/setup-advanced.md index c9ffa1d7ca..b3aee42e7e 100644 --- a/docs/development/setup-advanced.md +++ b/docs/development/setup-advanced.md @@ -318,7 +318,7 @@ sudo virtualenv /srv/zulip-py3-venv -p python3 # Create a python3 virtualenv sudo chown -R `whoami`:`whoami` /srv/zulip-py3-venv source /srv/zulip-py3-venv/bin/activate # Activate python3 virtualenv pip install --upgrade pip # upgrade pip itself because older versions have known issues -pip install --no-deps -r requirements/dev_lock.txt # install python packages required for development +pip install --no-deps -r requirements/dev.txt # install python packages required for development ``` Now run these commands: diff --git a/docs/requirements.readthedocs.txt b/docs/requirements.readthedocs.txt index 52895048ed..e287f2474b 120000 --- a/docs/requirements.readthedocs.txt +++ b/docs/requirements.readthedocs.txt @@ -1 +1 @@ -../requirements/docs_lock.txt \ No newline at end of file +../requirements/docs.txt \ No newline at end of file diff --git a/docs/subsystems/dependencies.md b/docs/subsystems/dependencies.md index c2fd139562..27d3139e24 100644 --- a/docs/subsystems/dependencies.md +++ b/docs/subsystems/dependencies.md @@ -117,8 +117,8 @@ highlighting. The system is largely managed by the code in versions in a `requirements.txt` file to declare what we're using. Since we have a few different installation targets, we maintain several `requirements.txt` format files in the `requirements/` - directory (e.g. `dev.txt` for development, `prod.txt` for - production, `docs.txt` for ReadTheDocs, `common.txt` for the vast + directory (e.g. `dev.in` for development, `prod.in` for + production, `docs.in` for ReadTheDocs, `common.in` for the vast majority of packages common to prod and development, etc.). We use `pip install --no-deps` to ensure we only install the packages we explicitly declare as dependencies. @@ -133,6 +133,23 @@ highlighting. The system is largely managed by the code in effect is that it's easy to debug problems caused by dependency upgrades, since we're always doing those upgrades with an explicit commit updating the `requirements/` directory. +* **Pinning versions of indirect dependencies**. We "pin" or "lock" + the versions of our indirect dependencies files with + `tools/update-locked-requirements` (powered by `pip-compile`). What + this means is that we have some "source" requirements files, like + `requirements/common.in`, that declare the packages that Zulip + depends on directly. Those packages have their own recursive + dependencies. When adding or removing a dependency from Zulip, one + simply edits the appropriate "source" requirements files, and then + runs `tools/update-locked-requirements`. That tool will use `pip + compile` to generate the locked requirements files like `prod.txt`, + `dev.txt` etc files that explicitly declare versions of all of + Zulip's recursive dependencies. For indirect dependencies (i.e. + dependencies not explicitly declared in the source requirements files), + it provides helpful comments explaining which direct dependency (or + dependencies) needed that indirect dependency. The process for + using this system is documented in more detail in + `requirements/README.md`. * **Caching of virtualenvs and packages**. To make updating the dependencies of a Zulip installation efficient, we maintain a cache of virtualenvs named by the hash of the relevant `requirements.txt` @@ -153,23 +170,6 @@ highlighting. The system is largely managed by the code in production deployment directory under `/home/zulip/deployments/`. This helps ensure that a Zulip installation doesn't leak large amounts of disk over time. -* **Pinning versions of indirect dependencies**. We "pin" or "lock" - the versions of our indirect dependencies files with - `tools/update-locked-requirements` (powered by `pip-compile`). What - this means is that we have some "source" requirements files, like - `requirements/common.txt`, that declare the packages that Zulip - depends on directly. Those packages have their own recursive - dependencies. When adding or removing a dependency from Zulip, one - simply edits the appropriate "source" requirements files, and then - runs `tools/update-locked-requirements`. That tool will use `pip - compile` to generate the `prod_lock.txt` and `dev_lock.txt` files - that explicitly declare versions of all of Zulip's recursive - dependencies. For indirect dependencies (i.e. dependencies not - explicitly declared in the source requirements files), it provides - helpful comments explaining which direct dependency (or - dependencies) needed that indirect dependency. The process for - using this system is documented in more detail in - `requirements/README.md`. * **Scripts**. Often, we want a script running in production to use the Zulip virtualenv. To make that work without a lot of duplicated code, we have a helpful library, diff --git a/docs/subsystems/documentation.md b/docs/subsystems/documentation.md index be5df6fd23..4e20274e70 100644 --- a/docs/subsystems/documentation.md +++ b/docs/subsystems/documentation.md @@ -58,7 +58,8 @@ Markdown, though that won't be as faithful as the `make html` approach. When editing dependencies for the Zulip documentation, you should edit -`requirements/docs.txt` (which is used by ReadTheDocs to build the +`requirements/docs.in` and then run `tools/update-locked-requirements` +which updates docs.txt file (which is used by ReadTheDocs to build the Zulip developer documentation, without installing all of Zulip's dependencies). diff --git a/requirements/README.md b/requirements/README.md index 7d557703ad..b7bf5c246d 100644 --- a/requirements/README.md +++ b/requirements/README.md @@ -14,8 +14,8 @@ scripts directly. The rest are implicit dependencies. common and dev are locked. Steps to update a lock file, e.g. to update ipython from 5.3.0 to 6.0.0 in -common.txt and propagate it to dev_lock.txt and prod_lock.txt: -0. Replace `ipython==5.4.1` with `ipython==6.0.0` in common.txt +common.in and propagate it to dev.txt and prod.txt: +0. Replace `ipython==5.4.1` with `ipython==6.0.0` in common.in 1. Run './tools/update-locked-requirements' 2. Increase `PROVISION_VERSION` in `version.py`. 3. Run `./tools/provision` to install the new deps and test them diff --git a/requirements/common.txt b/requirements/common.in similarity index 97% rename from requirements/common.txt rename to requirements/common.in index fef2c88197..d6e1883789 100644 --- a/requirements/common.txt +++ b/requirements/common.in @@ -1,6 +1,6 @@ # After editing this file, you MUST afterward run -# /tools/update-locked-requirements to update requirements/dev_lock.txt -# and requirements/prod_lock.txt. +# /tools/update-locked-requirements to update requirements/dev.txt +# and requirements/prod.txt. # See requirements/README.md for more detail. # Django itself Django==1.11.6 diff --git a/requirements/dev.in b/requirements/dev.in new file mode 100644 index 0000000000..10a3eefb9b --- /dev/null +++ b/requirements/dev.in @@ -0,0 +1,49 @@ +# After editing this file, you MUST afterward run +# /tools/update-locked-requirements to update requirements/dev.txt. +# See requirements/README.md for more detail. +-r common.in +-r docs.in + +# moto s3 mock +moto==1.1.24 + +# Needed for running tools/run-dev.py +Twisted==17.9.0 + +# Needed for documentation links test +Scrapy==1.4.0 + +# Needed to compute test coverage +coverage==4.4.1 + +# fake for LDAP testing +fakeldap==0.6.1 + +# For sorting imports +isort==4.2.15 + +# for pep8 linter +pycodestyle==2.3.1 + +# Needed to run pyflakes linter +pyflakes==1.6.0 + +# Needed to run tests in parallel +tblib==1.3.2 + +# Needed to lint Git commit messages +gitlint==0.8.2 + +# Needed for visualising cprofile reports +snakeviz==0.4.2 + +# Needed to sync translations from transifex +transifex-client==0.12.4 + +# Needed for creating digital ocean droplets +python-digitalocean==1.12 + +# Needed for updating the locked pip dependencies +pip-tools==1.10.1 + +-r mypy.in diff --git a/requirements/dev.txt b/requirements/dev.txt index b50763c46a..e29828666f 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,49 +1,166 @@ -# After editing this file, you MUST afterward run -# /tools/update-locked-requirements to update requirements/dev_lock.txt. -# See requirements/README.md for more detail. --r common.txt --r docs.txt - -# moto s3 mock -moto==1.1.24 - -# Needed for running tools/run-dev.py -Twisted==17.9.0 - -# Needed for documentation links test -Scrapy==1.4.0 - -# Needed to compute test coverage +# +# This file is GENERATED. Don't edit directly. +# +# To update, edit the non-"lock" files in requirements/*.txt, then: +# +# tools/update-locked-requirements +# +# For details, see requirements/README.md . +# +git+https://github.com/zulip/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.10.zulip1 +git+https://github.com/zulip/ultrajson@70ac02bec#egg=ujson==1.35+git +git+https://github.com/zulip/virtualenv-clone.git@44e831da39ffb6b9bb5c7d103d98babccdca0456#egg=virtualenv-clone==0.2.6.zulip1 +git+https://github.com/zulip/python-zulip-api.git@0.3.7#egg=zulip==0.3.7_git&subdirectory=zulip +git+https://github.com/zulip/python-zulip-api.git@0.3.7#egg=zulip_bots==0.3.7+git&subdirectory=zulip_bots +alabaster==0.7.10 +apns2==0.3.0 +argon2-cffi==16.3.0 +arrow==0.10.0 # via gitlint +asn1crypto==0.23.0 # via cryptography +attrs==17.3.0 # via automat, service-identity +automat==0.6.0 # via twisted +aws-xray-sdk==0.94 # via moto +babel==2.5.1 +backports-abc==0.5 +backports.ssl-match-hostname==3.5.0.1 +beautifulsoup4==4.6.0 +boto3==1.4.7 # via moto +boto==2.48.0 +botocore==1.7.46 # via boto3, moto, s3transfer +cchardet==2.1.1 +certifi==2017.7.27.1 +cffi==1.11.2 +chardet==3.0.4 +click==6.6 # via gitlint, pip-tools +commonmark==0.5.4 +constantly==15.1.0 # via twisted +cookies==2.2.1 # via moto coverage==4.4.1 - -# fake for LDAP testing +cryptography==2.1.2 +cssselect==1.0.1 # via parsel, premailer, scrapy +cssutils==1.0.2 # via premailer +decorator==4.1.2 # via ipython, traitlets +defusedxml==0.5.0 +django-auth-ldap==1.2.16 +django-bitfield==1.9.3 +django-pipeline==1.6.13 +django-statsd-mozilla==0.4.0 +django-webpack-loader==0.5.0 +django==1.11.6 +docker-pycreds==0.2.1 # via docker +docker==2.6.1 # via moto +docopt==0.6.2 +docutils==0.14 fakeldap==0.6.1 - -# For sorting imports -isort==4.2.15 - -# for pep8 linter -pycodestyle==2.3.1 - -# Needed to run pyflakes linter -pyflakes==1.6.0 - -# Needed to run tests in parallel -tblib==1.3.2 - -# Needed to lint Git commit messages +first==2.0.1 # via pip-tools +gitdb==0.6.4 gitlint==0.8.2 - -# Needed for visualising cprofile reports -snakeviz==0.4.2 - -# Needed to sync translations from transifex -transifex-client==0.12.4 - -# Needed for creating digital ocean droplets -python-digitalocean==1.12 - -# Needed for updating the locked pip dependencies +google-api-python-client==1.6.4 +h2==2.6.2 # via hyper +hpack==3.0.0 # via h2 +html2text==2017.10.4 +httplib2==0.10.3 +hyper==0.7.0 # via apns2 +hyperframe==3.2.0 # via h2, hyper +hyperlink==17.3.1 # via twisted +idna==2.6 # via cryptography, requests +ijson==2.3 +imagesize==0.7.1 +incremental==17.5.0 # via twisted +ipython-genutils==0.2.0 # via traitlets +ipython==6.2.1 +isort==4.2.15 +jedi==0.11.0 # via ipython +jinja2==2.9.6 +jmespath==0.9.3 # via boto3, botocore +jsonpickle==0.9.5 # via aws-xray-sdk +lxml==4.1.0 +markdown-include==0.5.1 +markdown==2.6.9 +markupsafe==1.0 +mock==2.0.0 +moto==1.1.24 +mypy==0.540 +mypy_extensions==0.3.0 +ndg-httpsclient==0.4.3 +oauth2client==4.1.2 +oauthlib==2.0.6 +olefile==0.44 # via pillow +ordereddict==1.1 # via gitlint +parsel==1.2.0 # via scrapy +parso==0.1.0 # via jedi +pbr==3.1.1 # via mock +pexpect==4.3.0 # via ipython +pickleshare==0.7.4 # via ipython +pika==0.11.0 +pillow==4.3.0 pip-tools==1.10.1 - --r mypy.txt +polib==1.0.8 +premailer==3.1.1 +prompt-toolkit==1.0.15 # via ipython +psycopg2==2.7.3.2 +ptyprocess==0.5.2 # via pexpect +py3dns==3.1.0 +pyaml==17.10.0 # via moto +pyasn1-modules==0.1.5 +pyasn1==0.3.7 +pycodestyle==2.3.1 +pycparser==2.18 # via cffi +pycrypto==2.6.1 +pydispatcher==2.0.5 # via scrapy +pyflakes==1.6.0 +pygments==2.2.0 +pyjwt==1.5.3 +pyldap==2.4.37 +pylibmc==1.5.2 +pyoembed==0.1.2 +pyopenssl==17.3.0 # via ndg-httpsclient, scrapy, service-identity +python-dateutil==2.6.1 +python-digitalocean==1.12 +python-gcm==0.4 +python-twitter==3.3 +python3-openid==3.1.0 # via social-auth-core +pytz==2017.2 +pyyaml==3.12 # via pyaml +queuelib==1.4.2 # via scrapy +recommonmark==0.4.0 +redis==2.10.6 +regex==2017.11.9 +requests-oauthlib==0.8.0 +requests==2.18.4 # via aws-xray-sdk, docker, moto, premailer, pyoembed, python-digitalocean, python-gcm, python-twitter, requests-oauthlib, social-auth-core, sphinx +rsa==3.4.2 +s3transfer==0.1.11 # via boto3 +scrapy==1.4.0 +service-identity==17.0.0 # via scrapy +sh==1.11 # via gitlint +simplegeneric==0.8.1 # via ipython +simplejson==3.11.1 +six==1.11.0 +smmap==0.9.0 +snakeviz==0.4.2 +snowballstemmer==1.2.1 +social-auth-app-django==1.2.0 +social-auth-core==1.5.0 # via social-auth-app-django +sockjs-tornado==1.0.3 +sourcemap==0.2.1 +sphinx-rtd-theme==0.2.4 +sphinx==1.6.5 +sphinxcontrib-websupport==1.0.1 # via sphinx +sqlalchemy==1.1.14 +statsd==3.2.1 # via django-statsd-mozilla +tblib==1.3.2 +tornado==4.5.2 +traitlets==4.3.2 # via ipython +transifex-client==0.12.4 +twisted==17.9.0 +typed-ast==1.1.0 # via mypy +typing==3.6.2 +uritemplate==3.0.0 +urllib3==1.22 # via requests, transifex-client +w3lib==1.18.0 # via parsel, scrapy +wcwidth==0.1.7 # via prompt-toolkit +websocket-client==0.44.0 # via docker +werkzeug==0.12.2 # via moto +wrapt==1.10.11 # via aws-xray-sdk +xmltodict==0.11.0 # via moto +zope.interface==4.4.3 # via twisted diff --git a/requirements/dev_lock.txt b/requirements/dev_lock.txt deleted file mode 100644 index ff7f669dc9..0000000000 --- a/requirements/dev_lock.txt +++ /dev/null @@ -1,165 +0,0 @@ -# -# This file is GENERATED. Don't edit directly. -# -# To update, edit the non-"lock" files in requirements/*.txt, then: -# -# tools/update-locked-requirements -# -# For details, see requirements/README.md . -# -git+https://github.com/zulip/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.10.zulip1 -git+https://github.com/zulip/ultrajson@70ac02bec#egg=ujson==1.35+git -git+https://github.com/zulip/virtualenv-clone.git@44e831da39ffb6b9bb5c7d103d98babccdca0456#egg=virtualenv-clone==0.2.6.zulip1 -git+https://github.com/zulip/python-zulip-api.git@0.3.7#egg=zulip==0.3.7_git&subdirectory=zulip -git+https://github.com/zulip/python-zulip-api.git@0.3.7#egg=zulip_bots==0.3.7+git&subdirectory=zulip_bots -alabaster==0.7.10 -apns2==0.3.0 -argon2-cffi==16.3.0 -arrow==0.10.0 # via gitlint -asn1crypto==0.22.0 # via cryptography -attrs==17.2.0 # via automat, service-identity -automat==0.6.0 # via twisted -aws-xray-sdk==0.93 # via moto -babel==2.5.1 -backports-abc==0.5 -backports.ssl-match-hostname==3.5.0.1 -beautifulsoup4==4.6.0 -boto3==1.4.7 # via moto -boto==2.48.0 -botocore==1.7.36 # via boto3, moto, s3transfer -cchardet==2.1.1 -certifi==2017.7.27.1 -cffi==1.11.2 -chardet==3.0.4 -click==6.6 # via gitlint, pip-tools -commonmark==0.5.4 -constantly==15.1.0 # via twisted -cookies==2.2.1 # via moto -coverage==4.4.1 -cryptography==2.1.2 -cssselect==1.0.1 # via parsel, premailer, scrapy -cssutils==1.0.2 # via premailer -decorator==4.1.2 # via ipython, traitlets -defusedxml==0.5.0 -django-auth-ldap==1.2.16 -django-bitfield==1.9.3 -django-pipeline==1.6.13 -django-statsd-mozilla==0.4.0 -django-webpack-loader==0.5.0 -django==1.11.6 -docker-pycreds==0.2.1 # via docker -docker==2.5.1 # via moto -docopt==0.6.2 -docutils==0.14 -fakeldap==0.6.1 -first==2.0.1 # via pip-tools -gitdb==0.6.4 -gitlint==0.8.2 -google-api-python-client==1.6.4 -h2==2.6.2 # via hyper -hpack==3.0.0 # via h2 -html2text==2017.10.4 -httplib2==0.10.3 -hyper==0.7.0 # via apns2 -hyperframe==3.2.0 # via h2, hyper -hyperlink==17.3.1 # via twisted -idna==2.6 # via cryptography, requests -ijson==2.3 -imagesize==0.7.1 -incremental==17.5.0 # via twisted -ipython-genutils==0.2.0 # via traitlets -ipython==6.2.1 -isort==4.2.15 -jedi==0.10.2 # via ipython -jinja2==2.9.6 -jmespath==0.9.3 # via boto3, botocore -jsonpickle==0.9.5 # via aws-xray-sdk -lxml==4.1.0 -markdown-include==0.5.1 -markdown==2.6.9 -markupsafe==1.0 -mock==2.0.0 -moto==1.1.24 -mypy==0.540 -mypy_extensions==0.3.0 -ndg-httpsclient==0.4.3 -oauth2client==4.1.2 -oauthlib==2.0.6 -olefile==0.44 # via pillow -ordereddict==1.1 # via gitlint -parsel==1.2.0 # via scrapy -pbr==3.1.1 # via mock -pexpect==4.2.1 # via ipython -pickleshare==0.7.4 # via ipython -pika==0.11.0 -pillow==4.3.0 -pip-tools==1.10.1 -polib==1.0.8 -premailer==3.1.1 -prompt-toolkit==1.0.15 # via ipython -psycopg2==2.7.3.2 -ptyprocess==0.5.2 # via pexpect -py3dns==3.1.0 -pyaml==17.10.0 # via moto -pyasn1-modules==0.1.5 -pyasn1==0.3.7 -pycodestyle==2.3.1 -pycparser==2.18 # via cffi -pycrypto==2.6.1 -pydispatcher==2.0.5 # via scrapy -pyflakes==1.6.0 -pygments==2.2.0 -pyjwt==1.5.3 -pyldap==2.4.37 -pylibmc==1.5.2 -pyoembed==0.1.2 -pyopenssl==17.0.0 # via ndg-httpsclient, scrapy, service-identity -python-dateutil==2.6.1 -python-digitalocean==1.12 -python-gcm==0.4 -python-twitter==3.3 -python3-openid==3.1.0 # via social-auth-core -pytz==2017.2 -pyyaml==3.12 # via pyaml -queuelib==1.4.2 # via scrapy -recommonmark==0.4.0 -redis==2.10.6 -regex==2017.7.28 -requests-oauthlib==0.8.0 -requests==2.18.4 # via aws-xray-sdk, docker, moto, premailer, pyoembed, python-digitalocean, python-gcm, python-twitter, requests-oauthlib, social-auth-core, sphinx -rsa==3.4.2 -s3transfer==0.1.11 # via boto3 -scrapy==1.4.0 -service-identity==17.0.0 # via scrapy -sh==1.11 # via gitlint -simplegeneric==0.8.1 # via ipython -simplejson==3.11.1 -six==1.11.0 -smmap==0.9.0 -snakeviz==0.4.2 -snowballstemmer==1.2.1 -social-auth-app-django==1.2.0 -social-auth-core==1.4.0 # via social-auth-app-django -sockjs-tornado==1.0.3 -sourcemap==0.2.1 -sphinx-rtd-theme==0.2.4 -sphinx==1.6.5 -sphinxcontrib-websupport==1.0.1 # via sphinx -sqlalchemy==1.1.14 -statsd==2.1.2 # via django-statsd-mozilla -tblib==1.3.2 -tornado==4.5.2 -traitlets==4.3.2 # via ipython -transifex-client==0.12.4 -twisted==17.9.0 -typed-ast==1.1.0 # via mypy -typing==3.6.2 -uritemplate==3.0.0 -urllib3==1.22 # via requests, transifex-client -w3lib==1.18.0 # via parsel, scrapy -wcwidth==0.1.7 # via prompt-toolkit -websocket-client==0.44.0 # via docker -werkzeug==0.12.2 # via moto -wrapt==1.10.11 # via aws-xray-sdk -xmltodict==0.11.0 # via moto -zope.interface==4.4.2 # via twisted diff --git a/requirements/docs.in b/requirements/docs.in new file mode 100644 index 0000000000..4241cc2fc5 --- /dev/null +++ b/requirements/docs.in @@ -0,0 +1,26 @@ +# This is used by ReadTheDocs to install dependencies, so it's +# valuable to keep this file as just the dependencies for our +# documentation. + +# After editing this file, you MUST afterward run +# /tools/update-locked-requirements to update requirements/dev.txt +# and requirements/docs.txt. +# See requirements/README.md for more detail. + +# Needed to build RTD docs +sphinx==1.6.5 +sphinx-rtd-theme==0.2.4 + +# Needed to build markdown docs +recommonmark==0.4.0 + +# Dependencies of Sphinx +alabaster==0.7.10 +babel==2.5.1 +# Upgrading to the latest version of CommonMark breaks the +# compatibility with recommonmark. See +# https://github.com/rtfd/recommonmark/issues/24 +CommonMark==0.5.4 +docutils==0.14 +imagesize==0.7.1 +snowballstemmer==1.2.1 diff --git a/requirements/docs.txt b/requirements/docs.txt index 01826c8bd8..3a14721a22 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,26 +1,30 @@ -# This is used by ReadTheDocs to install dependencies, so it's -# valuable to keep this file as just the dependencies for our -# documentation. - -# After editing this file, you MUST afterward run -# /tools/update-locked-requirements to update requirements/dev_lock.txt -# and requirements/docs_lock.txt. -# See requirements/README.md for more detail. - -# Needed to build RTD docs -sphinx==1.6.5 -sphinx-rtd-theme==0.2.4 - -# Needed to build markdown docs -recommonmark==0.4.0 - -# Dependencies of Sphinx +# +# This file is GENERATED. Don't edit directly. +# +# To update, edit the non-"lock" files in requirements/*.txt, then: +# +# tools/update-locked-requirements +# +# For details, see requirements/README.md . +# alabaster==0.7.10 babel==2.5.1 -# Upgrading to the latest version of CommonMark breaks the -# compatibility with recommonmark. See -# https://github.com/rtfd/recommonmark/issues/24 -CommonMark==0.5.4 +certifi==2017.11.5 # via requests +chardet==3.0.4 # via requests +commonmark==0.5.4 docutils==0.14 +idna==2.6 # via requests imagesize==0.7.1 +jinja2==2.10 # via sphinx +markupsafe==1.0 # via jinja2 +pygments==2.2.0 # via sphinx +pytz==2017.3 # via babel +recommonmark==0.4.0 +requests==2.18.4 # via sphinx +six==1.11.0 # via sphinx snowballstemmer==1.2.1 +sphinx-rtd-theme==0.2.4 +sphinx==1.6.5 +sphinxcontrib-websupport==1.0.1 # via sphinx +typing==3.6.2 # via sphinx +urllib3==1.22 # via requests diff --git a/requirements/docs_lock.txt b/requirements/docs_lock.txt deleted file mode 100644 index 3a14721a22..0000000000 --- a/requirements/docs_lock.txt +++ /dev/null @@ -1,30 +0,0 @@ -# -# This file is GENERATED. Don't edit directly. -# -# To update, edit the non-"lock" files in requirements/*.txt, then: -# -# tools/update-locked-requirements -# -# For details, see requirements/README.md . -# -alabaster==0.7.10 -babel==2.5.1 -certifi==2017.11.5 # via requests -chardet==3.0.4 # via requests -commonmark==0.5.4 -docutils==0.14 -idna==2.6 # via requests -imagesize==0.7.1 -jinja2==2.10 # via sphinx -markupsafe==1.0 # via jinja2 -pygments==2.2.0 # via sphinx -pytz==2017.3 # via babel -recommonmark==0.4.0 -requests==2.18.4 # via sphinx -six==1.11.0 # via sphinx -snowballstemmer==1.2.1 -sphinx-rtd-theme==0.2.4 -sphinx==1.6.5 -sphinxcontrib-websupport==1.0.1 # via sphinx -typing==3.6.2 # via sphinx -urllib3==1.22 # via requests diff --git a/requirements/mypy.in b/requirements/mypy.in new file mode 100644 index 0000000000..b9a1abd3f8 --- /dev/null +++ b/requirements/mypy.in @@ -0,0 +1,5 @@ +# After editing this file, you MUST afterward run +# /tools/update-locked-requirements to update requirements/dev.txt +# and requirements/mypy.txt. +# See requirements/README.md for more detail. +mypy==0.540 diff --git a/requirements/mypy.txt b/requirements/mypy.txt index b4c4b01f2e..54b59e260e 100644 --- a/requirements/mypy.txt +++ b/requirements/mypy.txt @@ -1,5 +1,12 @@ -# After editing this file, you MUST afterward run -# /tools/update-locked-requirements to update requirements/dev_lock.txt -and requirements/mypy_lock.txt. -# See requirements/README.md for more detail. +# +# This file is GENERATED. Don't edit directly. +# +# To update, edit the non-"lock" files in requirements/*.txt, then: +# +# tools/update-locked-requirements +# +# For details, see requirements/README.md . +# mypy==0.540 +typed-ast==1.1.0 # via mypy +typing==3.6.2 # via mypy diff --git a/requirements/mypy_lock.txt b/requirements/mypy_lock.txt deleted file mode 100644 index 54b59e260e..0000000000 --- a/requirements/mypy_lock.txt +++ /dev/null @@ -1,12 +0,0 @@ -# -# This file is GENERATED. Don't edit directly. -# -# To update, edit the non-"lock" files in requirements/*.txt, then: -# -# tools/update-locked-requirements -# -# For details, see requirements/README.md . -# -mypy==0.540 -typed-ast==1.1.0 # via mypy -typing==3.6.2 # via mypy diff --git a/requirements/prod.in b/requirements/prod.in new file mode 100644 index 0000000000..588f3e0e36 --- /dev/null +++ b/requirements/prod.in @@ -0,0 +1,6 @@ +# After editing this file, you MUST afterward run +# /tools/update-locked-requirements to update requirements/prod.txt. +# See requirements/README.md for more detail. +-r common.in +# Used for running the Zulip production Django server +uWSGI==2.0.14 diff --git a/requirements/prod.txt b/requirements/prod.txt index 6e7f97fae7..964d53edea 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -1,6 +1,110 @@ -# After editing this file, you MUST afterward run -# /tools/update-locked-requirements to update requirements/prod_lock.txt. -# See requirements/README.md for more detail. --r common.txt -# Used for running the Zulip production Django server -uWSGI==2.0.14 +# +# This file is GENERATED. Don't edit directly. +# +# To update, edit the non-"lock" files in requirements/*.txt, then: +# +# tools/update-locked-requirements +# +# For details, see requirements/README.md . +# +git+https://github.com/zulip/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.10.zulip1 +git+https://github.com/zulip/ultrajson@70ac02bec#egg=ujson==1.35+git +git+https://github.com/zulip/virtualenv-clone.git@44e831da39ffb6b9bb5c7d103d98babccdca0456#egg=virtualenv-clone==0.2.6.zulip1 +git+https://github.com/zulip/python-zulip-api.git@0.3.7#egg=zulip==0.3.7_git&subdirectory=zulip +git+https://github.com/zulip/python-zulip-api.git@0.3.7#egg=zulip_bots==0.3.7+git&subdirectory=zulip_bots +apns2==0.3.0 +argon2-cffi==16.3.0 +asn1crypto==0.23.0 # via cryptography +backports-abc==0.5 +backports.ssl-match-hostname==3.5.0.1 +beautifulsoup4==4.6.0 +boto==2.48.0 +cchardet==2.1.1 +certifi==2017.7.27.1 +cffi==1.11.2 +chardet==3.0.4 +cryptography==2.1.2 +cssselect==1.0.1 # via premailer +cssutils==1.0.2 # via premailer +decorator==4.1.2 # via ipython, traitlets +defusedxml==0.5.0 +django-auth-ldap==1.2.16 +django-bitfield==1.9.3 +django-pipeline==1.6.13 +django-statsd-mozilla==0.4.0 +django-webpack-loader==0.5.0 +django==1.11.6 +docopt==0.6.2 +gitdb==0.6.4 +google-api-python-client==1.6.4 +h2==2.6.2 # via hyper +hpack==3.0.0 # via h2 +html2text==2017.10.4 +httplib2==0.10.3 +hyper==0.7.0 # via apns2 +hyperframe==3.2.0 # via h2, hyper +idna==2.6 # via cryptography, requests +ijson==2.3 +ipython-genutils==0.2.0 # via traitlets +ipython==6.2.1 +jedi==0.11.0 # via ipython +jinja2==2.9.6 +lxml==4.1.0 +markdown-include==0.5.1 +markdown==2.6.9 +markupsafe==1.0 +mock==2.0.0 +mypy_extensions==0.3.0 +ndg-httpsclient==0.4.3 +oauth2client==4.1.2 +oauthlib==2.0.6 +olefile==0.44 # via pillow +parso==0.1.0 # via jedi +pbr==3.1.1 # via mock +pexpect==4.3.0 # via ipython +pickleshare==0.7.4 # via ipython +pika==0.11.0 +pillow==4.3.0 +polib==1.0.8 +premailer==3.1.1 +prompt-toolkit==1.0.15 # via ipython +psycopg2==2.7.3.2 +ptyprocess==0.5.2 # via pexpect +py3dns==3.1.0 +pyasn1-modules==0.1.5 +pyasn1==0.3.7 +pycparser==2.18 # via cffi +pycrypto==2.6.1 +pygments==2.2.0 +pyjwt==1.5.3 +pyldap==2.4.37 +pylibmc==1.5.2 +pyoembed==0.1.2 +pyopenssl==17.3.0 # via ndg-httpsclient +python-dateutil==2.6.1 +python-gcm==0.4 +python-twitter==3.3 +python3-openid==3.1.0 # via social-auth-core +pytz==2017.2 +redis==2.10.6 +regex==2017.11.9 +requests-oauthlib==0.8.0 +requests==2.18.4 # via premailer, pyoembed, python-gcm, python-twitter, requests-oauthlib, social-auth-core +rsa==3.4.2 +simplegeneric==0.8.1 # via ipython +simplejson==3.11.1 +six==1.11.0 +smmap==0.9.0 +social-auth-app-django==1.2.0 +social-auth-core==1.5.0 # via social-auth-app-django +sockjs-tornado==1.0.3 +sourcemap==0.2.1 +sqlalchemy==1.1.14 +statsd==3.2.1 # via django-statsd-mozilla +tornado==4.5.2 +traitlets==4.3.2 # via ipython +typing==3.6.2 +uritemplate==3.0.0 +urllib3==1.22 # via requests +uwsgi==2.0.14 +wcwidth==0.1.7 # via prompt-toolkit diff --git a/requirements/prod_lock.txt b/requirements/prod_lock.txt deleted file mode 100644 index 28268d26a6..0000000000 --- a/requirements/prod_lock.txt +++ /dev/null @@ -1,109 +0,0 @@ -# -# This file is GENERATED. Don't edit directly. -# -# To update, edit the non-"lock" files in requirements/*.txt, then: -# -# tools/update-locked-requirements -# -# For details, see requirements/README.md . -# -git+https://github.com/zulip/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.10.zulip1 -git+https://github.com/zulip/ultrajson@70ac02bec#egg=ujson==1.35+git -git+https://github.com/zulip/virtualenv-clone.git@44e831da39ffb6b9bb5c7d103d98babccdca0456#egg=virtualenv-clone==0.2.6.zulip1 -git+https://github.com/zulip/python-zulip-api.git@0.3.7#egg=zulip==0.3.7_git&subdirectory=zulip -git+https://github.com/zulip/python-zulip-api.git@0.3.7#egg=zulip_bots==0.3.7+git&subdirectory=zulip_bots -apns2==0.3.0 -argon2-cffi==16.3.0 -asn1crypto==0.22.0 # via cryptography -backports-abc==0.5 -backports.ssl-match-hostname==3.5.0.1 -beautifulsoup4==4.6.0 -boto==2.48.0 -cchardet==2.1.1 -certifi==2017.7.27.1 -cffi==1.11.2 -chardet==3.0.4 -cryptography==2.1.2 -cssselect==1.0.1 # via premailer -cssutils==1.0.2 # via premailer -decorator==4.1.2 # via ipython, traitlets -defusedxml==0.5.0 -django-auth-ldap==1.2.16 -django-bitfield==1.9.3 -django-pipeline==1.6.13 -django-statsd-mozilla==0.4.0 -django-webpack-loader==0.5.0 -django==1.11.6 -docopt==0.6.2 -gitdb==0.6.4 -google-api-python-client==1.6.4 -h2==2.6.2 # via hyper -hpack==3.0.0 # via h2 -html2text==2017.10.4 -httplib2==0.10.3 -hyper==0.7.0 # via apns2 -hyperframe==3.2.0 # via h2, hyper -idna==2.6 # via cryptography, requests -ijson==2.3 -ipython-genutils==0.2.0 # via traitlets -ipython==6.2.1 -jedi==0.10.2 # via ipython -jinja2==2.9.6 -lxml==4.1.0 -markdown-include==0.5.1 -markdown==2.6.9 -markupsafe==1.0 -mock==2.0.0 -mypy_extensions==0.3.0 -ndg-httpsclient==0.4.3 -oauth2client==4.1.2 -oauthlib==2.0.6 -olefile==0.44 # via pillow -pbr==3.1.1 # via mock -pexpect==4.2.1 # via ipython -pickleshare==0.7.4 # via ipython -pika==0.11.0 -pillow==4.3.0 -polib==1.0.8 -premailer==3.1.1 -prompt-toolkit==1.0.15 # via ipython -psycopg2==2.7.3.2 -ptyprocess==0.5.2 # via pexpect -py3dns==3.1.0 -pyasn1-modules==0.1.5 -pyasn1==0.3.7 -pycparser==2.18 # via cffi -pycrypto==2.6.1 -pygments==2.2.0 -pyjwt==1.5.3 -pyldap==2.4.37 -pylibmc==1.5.2 -pyoembed==0.1.2 -pyopenssl==17.2.0 # via ndg-httpsclient -python-dateutil==2.6.1 -python-gcm==0.4 -python-twitter==3.3 -python3-openid==3.1.0 # via social-auth-core -pytz==2017.2 -redis==2.10.6 -regex==2017.7.28 -requests-oauthlib==0.8.0 -requests==2.18.4 # via premailer, pyoembed, python-gcm, python-twitter, requests-oauthlib, social-auth-core -rsa==3.4.2 -simplegeneric==0.8.1 # via ipython -simplejson==3.11.1 -six==1.11.0 -smmap==0.9.0 -social-auth-app-django==1.2.0 -social-auth-core==1.4.0 # via social-auth-app-django -sockjs-tornado==1.0.3 -sourcemap==0.2.1 -sqlalchemy==1.1.14 -statsd==2.1.2 # via django-statsd-mozilla -tornado==4.5.2 -traitlets==4.3.2 # via ipython -typing==3.6.2 -uritemplate==3.0.0 -urllib3==1.22 # via requests -uwsgi==2.0.14 -wcwidth==0.1.7 # via prompt-toolkit diff --git a/requirements/thumbor.in b/requirements/thumbor.in new file mode 100644 index 0000000000..2f2658dd05 --- /dev/null +++ b/requirements/thumbor.in @@ -0,0 +1,12 @@ +tc-aws==6.0.3 +thumbor==6.3.2 +typing==3.6.2 + +# Below dependencies should ideally have been added to thumbor.txt by the +# use of tools/update-locked-requirements but for some reason `pip-compile` +# doesn't do so. An issue relating to this is +# https://github.com/thumbor/thumbor/issues/1006 +backports-abc==0.5 +backports.ssl-match-hostname==3.5.0.1 +certifi==2016.9.26 +singledispatch==3.4.0.3 diff --git a/requirements/thumbor.txt b/requirements/thumbor.txt index 8d05f7c5f2..99d951e8da 100644 --- a/requirements/thumbor.txt +++ b/requirements/thumbor.txt @@ -1,12 +1,33 @@ -tc-aws==6.0.3 -thumbor==6.3.2 -typing==3.6.2 - -# Below dependencies should ideally have been added to thumbor_lock.txt by the -# use of tools/update-locked-requirements but for some reason `pip-compile` -# doesn't do so. An issue relating to this is -# https://github.com/thumbor/thumbor/issues/1006 +# +# This file is GENERATED. Don't edit directly. +# +# To update, edit the non-"lock" files in requirements/*.txt, then: +# +# tools/update-locked-requirements +# +# For details, see requirements/README.md . +# +argparse==1.4.0 # via thumbor backports-abc==0.5 backports.ssl-match-hostname==3.5.0.1 +botocore==1.5.95 # via tornado-botocore certifi==2016.9.26 +derpconf==0.8.1 # via thumbor +docutils==0.14 # via botocore +futures==3.1.1 # via thumbor +jmespath==0.9.3 # via botocore +libthumbor==1.3.2 # via thumbor +pexif==0.15 # via thumbor +pillow==3.4.2 # via thumbor +pycrypto==2.6.1 # via thumbor +pycurl==7.43.0 # via thumbor, tornado-botocore +python-dateutil==2.6.1 # via botocore, tc-aws +pytz==2017.3 # via thumbor singledispatch==3.4.0.3 +six==1.11.0 # via derpconf, libthumbor, python-dateutil, singledispatch +statsd==3.2.1 # via thumbor +tc-aws==6.0.3 +thumbor==6.3.2 +tornado-botocore==1.2.0 # via tc-aws +tornado==4.5.2 # via thumbor, tornado-botocore +typing==3.6.2 diff --git a/requirements/thumbor_lock.txt b/requirements/thumbor_lock.txt deleted file mode 100644 index 5ee3238f3c..0000000000 --- a/requirements/thumbor_lock.txt +++ /dev/null @@ -1,33 +0,0 @@ -# -# This file is GENERATED. Don't edit directly. -# -# To update, edit the non-"lock" files in requirements/*.txt, then: -# -# tools/update-locked-requirements -# -# For details, see requirements/README.md . -# -argparse==1.4.0 # via thumbor -backports-abc==0.5 -backports.ssl-match-hostname==3.5.0.1 -botocore==1.2.0 # via tornado-botocore -certifi==2016.9.26 -derpconf==0.8.1 # via thumbor -docutils==0.12 # via botocore -futures==3.1.1 # via thumbor -jmespath==0.7.1 # via botocore -libthumbor==1.3.2 # via thumbor -pexif==0.15 # via thumbor -pillow==3.4.2 # via thumbor -pycrypto==2.6.1 # via thumbor -pycurl==7.43.0 # via thumbor -python-dateutil==2.6.0 # via botocore, tc-aws -pytz==2017.3 # via thumbor -singledispatch==3.4.0.3 -six==1.10.0 # via derpconf, libthumbor, python-dateutil, singledispatch -statsd==3.2.1 # via thumbor -tc-aws==6.0.3 -thumbor==6.3.2 -tornado-botocore==1.0.2 # via tc-aws -tornado==4.5.2 # via thumbor, tornado-botocore -typing==3.6.2 diff --git a/scripts/lib/create-production-venv b/scripts/lib/create-production-venv index 08daee349a..dc8da9adc8 100755 --- a/scripts/lib/create-production-venv +++ b/scripts/lib/create-production-venv @@ -21,13 +21,13 @@ run(["apt-get", "-y", "install"] + VENV_DEPENDENCIES) python_version = sys.version_info[0] # Set the current working directory to the Zulip checkout, so the api/ -# relative path in requirements/common.txt works. +# relative path in requirements/common.in works. os.chdir(ZULIP_PATH) venv_name = "zulip-venv" if sys.version_info[0] == 2 else "zulip-py3-venv" cached_venv_path = setup_virtualenv( os.path.join(args.deploy_path, venv_name), - os.path.join(ZULIP_PATH, "requirements", "prod_lock.txt"), + os.path.join(ZULIP_PATH, "requirements", "prod.txt"), virtualenv_args=['-p', 'python{}'.format(python_version)]) current_venv_path = os.path.join(args.deploy_path, 'zulip-current-venv') diff --git a/scripts/lib/create-thumbor-venv b/scripts/lib/create-thumbor-venv index 1f823a6162..d52305cd25 100755 --- a/scripts/lib/create-thumbor-venv +++ b/scripts/lib/create-thumbor-venv @@ -21,5 +21,5 @@ run(["apt-get", "-y", "install"] + THUMBOR_VENV_DEPENDENCIES) venv_name = "zulip-thumbor-venv" cached_venv_path = setup_virtualenv( os.path.join(args.deploy_path, venv_name), - os.path.join(ZULIP_PATH, "requirements", "thumbor_lock.txt"), + os.path.join(ZULIP_PATH, "requirements", "thumbor.txt"), virtualenv_args=['-p', 'python']) diff --git a/tools/diagnose b/tools/diagnose index cfa67fb8bc..dc3f0444b1 100755 --- a/tools/diagnose +++ b/tools/diagnose @@ -107,7 +107,7 @@ def check_venv(): # type: () -> bool path = os.path.join(ROOT_DIR, 'scripts', 'lib', 'hash_reqs.py') cache_dir = '/srv/zulip-venv-cache/' - for fn in ['dev_lock.txt']: + for fn in ['dev.txt']: requirements_file = os.path.join(ROOT_DIR, "requirements", fn) output = subprocess.check_output([path, requirements_file], universal_newlines=True) sha1sum = output.split()[0] diff --git a/tools/setup/setup_venvs.py b/tools/setup/setup_venvs.py index 5cdd6b50bc..6c6389ae4e 100755 --- a/tools/setup/setup_venvs.py +++ b/tools/setup/setup_venvs.py @@ -13,7 +13,7 @@ from scripts.lib.zulip_tools import run, subprocess_text_output OLD_VENV_PATH = "/srv/zulip-venv" VENV_PATH = "/srv/zulip-py3-venv" -DEV_REQS_FILE = os.path.join(ZULIP_PATH, "requirements", "dev_lock.txt") +DEV_REQS_FILE = os.path.join(ZULIP_PATH, "requirements", "dev.txt") def main(is_travis=False): # type: (bool) -> None diff --git a/tools/test-locked-requirements b/tools/test-locked-requirements index 077f532c20..33c71adc85 100755 --- a/tools/test-locked-requirements +++ b/tools/test-locked-requirements @@ -12,10 +12,10 @@ ZULIP_PATH = os.path.dirname(TOOLS_DIR) sys.path.append(ZULIP_PATH) from scripts.lib.zulip_tools import run -PROD_LOCK_FILE = os.path.join(ZULIP_PATH, 'requirements', 'prod_lock.txt') -DEV_LOCK_FILE = os.path.join(ZULIP_PATH, 'requirements', 'dev_lock.txt') -TEST_PROD_LOCK_FILE = '/var/tmp/test_prod_lock.txt' -TEST_DEV_LOCK_FILE = '/var/tmp/test_dev_lock.txt' +PROD_LOCK_FILE = os.path.join(ZULIP_PATH, 'requirements', 'prod.txt') +DEV_LOCK_FILE = os.path.join(ZULIP_PATH, 'requirements', 'dev.txt') +TEST_PROD_LOCK_FILE = '/var/tmp/test_prod.txt' +TEST_DEV_LOCK_FILE = '/var/tmp/test_dev.txt' def print_diff(path_file1, path_file2): # type: (Text, Text) -> None diff --git a/tools/update-locked-requirements b/tools/update-locked-requirements index 064255aa31..e0eabaef55 100755 --- a/tools/update-locked-requirements +++ b/tools/update-locked-requirements @@ -63,8 +63,8 @@ if [ $# -gt 0 ]; then done fi -compile_requirements requirements/prod.txt $OUTPUT_BASE_DIR/prod_lock.txt -compile_requirements requirements/dev.txt $OUTPUT_BASE_DIR/dev_lock.txt -compile_requirements requirements/mypy.txt $OUTPUT_BASE_DIR/mypy_lock.txt -compile_requirements requirements/docs.txt $OUTPUT_BASE_DIR/docs_lock.txt -compile_requirements requirements/thumbor.txt $OUTPUT_BASE_DIR/thumbor_lock.txt py2 +compile_requirements requirements/prod.in $OUTPUT_BASE_DIR/prod.txt +compile_requirements requirements/dev.in $OUTPUT_BASE_DIR/dev.txt +compile_requirements requirements/mypy.in $OUTPUT_BASE_DIR/mypy.txt +compile_requirements requirements/docs.in $OUTPUT_BASE_DIR/docs.txt +compile_requirements requirements/thumbor.in $OUTPUT_BASE_DIR/thumbor.txt py2