mirror of https://github.com/zulip/zulip.git
124 lines
4.5 KiB
Python
Executable File
124 lines
4.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import difflib
|
|
import filecmp
|
|
import glob
|
|
import hashlib
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import ujson
|
|
|
|
from typing import Optional, List
|
|
|
|
TOOLS_DIR = os.path.abspath(os.path.dirname(__file__))
|
|
ZULIP_PATH = os.path.dirname(TOOLS_DIR)
|
|
REQS_DIR = os.path.join(ZULIP_PATH, 'requirements')
|
|
CACHE_DIR = os.path.join(ZULIP_PATH, 'var', 'tmp')
|
|
if 'TRAVIS' in os.environ:
|
|
CACHE_DIR = os.path.join(os.environ['HOME'], 'misc')
|
|
CACHE_FILE = os.path.join(CACHE_DIR, 'requirements_hashes')
|
|
|
|
def print_diff(path_file1, path_file2):
|
|
# type: (str, str) -> None
|
|
with open(path_file1) as file1:
|
|
with open(path_file2) as file2:
|
|
diff = difflib.unified_diff(
|
|
file1.readlines(),
|
|
file2.readlines(),
|
|
fromfile=path_file1,
|
|
tofile=path_file2,
|
|
)
|
|
sys.stdout.writelines(diff)
|
|
|
|
def test_locked_requirements(tmp_dir):
|
|
# type: (str) -> bool
|
|
# `pip-compile` tries to avoid unnecessarily updating recursive dependencies
|
|
# if lock files are present already. If we don't copy these files to the tmp
|
|
# dir then recursive dependencies will get updated to their latest version
|
|
# without any change in the input requirements file and the test will not pass.
|
|
for locked_file in glob.glob(os.path.join(REQS_DIR, "*.txt")):
|
|
fn = os.path.basename(locked_file)
|
|
locked_file = os.path.join(REQS_DIR, fn)
|
|
test_locked_file = os.path.join(tmp_dir, fn)
|
|
shutil.copyfile(locked_file, test_locked_file)
|
|
subprocess.check_call([os.path.join(TOOLS_DIR, 'update-locked-requirements'), '--output-dir', tmp_dir])
|
|
|
|
same = True
|
|
for test_locked_file in glob.glob(os.path.join(tmp_dir, "*.txt")):
|
|
fn = os.path.basename(test_locked_file)
|
|
locked_file = os.path.join(REQS_DIR, fn)
|
|
same = same and filecmp.cmp(test_locked_file, locked_file, shallow=False)
|
|
|
|
return same
|
|
|
|
def get_requirements_hash(tmp_dir, use_test_lock_files=False):
|
|
# type: (str, Optional[bool]) -> str
|
|
sha1 = hashlib.sha1()
|
|
reqs_files = sorted(glob.glob(os.path.join(REQS_DIR, "*.in")))
|
|
lock_files_path = REQS_DIR
|
|
if use_test_lock_files:
|
|
lock_files_path = tmp_dir
|
|
reqs_files.extend(sorted(glob.glob(os.path.join(lock_files_path, "*.txt"))))
|
|
for file_path in reqs_files:
|
|
with open(file_path) as fp:
|
|
sha1.update(fp.read().encode("utf-8"))
|
|
return sha1.hexdigest()
|
|
|
|
def may_be_setup_cache():
|
|
# type: () -> None
|
|
os.makedirs(CACHE_DIR, exist_ok=True)
|
|
if not os.path.exists(CACHE_FILE):
|
|
with open(CACHE_FILE, 'w') as fp:
|
|
ujson.dump([], fp)
|
|
|
|
def load_cache():
|
|
# type: () -> List[str]
|
|
with open(CACHE_FILE) as fp:
|
|
hash_list = ujson.load(fp)
|
|
return hash_list
|
|
|
|
def update_cache(hash_list):
|
|
# type: (List[str]) -> None
|
|
# We store last 100 hash entries. Aggressive caching is
|
|
# not a problem as it is cheap to do.
|
|
if len(hash_list) > 100:
|
|
hash_list = hash_list[-100:]
|
|
with open(CACHE_FILE, 'w') as fp:
|
|
ujson.dump(hash_list, fp)
|
|
|
|
def main():
|
|
# type: () -> None
|
|
may_be_setup_cache()
|
|
hash_list = load_cache()
|
|
tmp = tempfile.TemporaryDirectory()
|
|
tmp_dir = tmp.name
|
|
curr_hash = get_requirements_hash(tmp_dir)
|
|
|
|
if curr_hash in hash_list:
|
|
# We have already checked this set of requirements and they
|
|
# were consistent so no need to check again.
|
|
return
|
|
|
|
requirements_are_consistent = test_locked_requirements(tmp_dir)
|
|
|
|
# Cache the hash so that we need not to run the `update_locked_requirements`
|
|
# tool again for checking this set of requirements.
|
|
valid_hash = get_requirements_hash(tmp_dir, use_test_lock_files=True)
|
|
update_cache([h for h in hash_list if h != valid_hash] + [valid_hash])
|
|
if not requirements_are_consistent:
|
|
for test_locked_file in glob.glob(os.path.join(tmp_dir, "*.txt")):
|
|
fn = os.path.basename(test_locked_file)
|
|
locked_file = os.path.join(REQS_DIR, fn)
|
|
print_diff(locked_file, test_locked_file)
|
|
# Flush the output to ensure we print the error at the end.
|
|
sys.stdout.flush()
|
|
raise Exception("It looks like you have updated some python dependencies but haven't "
|
|
"updated locked requirements files. Please update them by running "
|
|
"`tools/update-locked-requirements`. For more information please "
|
|
"refer to `requirements/README.md`.")
|
|
|
|
if __name__ == '__main__':
|
|
main()
|