2016-08-18 13:51:25 +02:00
|
|
|
import hashlib
|
2018-12-07 23:43:52 +01:00
|
|
|
import json
|
2020-06-11 00:54:34 +02:00
|
|
|
import os
|
2019-08-13 06:56:46 +02:00
|
|
|
import shutil
|
2020-10-30 01:36:18 +01:00
|
|
|
import subprocess
|
2021-05-28 03:51:42 +02:00
|
|
|
from typing import Dict, List, Optional
|
2016-08-18 13:51:25 +02:00
|
|
|
|
2020-10-30 01:36:18 +01:00
|
|
|
from scripts.lib.zulip_tools import run
|
2016-08-18 13:51:25 +02:00
|
|
|
|
2017-09-22 08:15:01 +02:00
|
|
|
ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
2017-07-27 23:22:52 +02:00
|
|
|
ZULIP_SRV_PATH = "/srv"
|
2016-08-18 13:51:25 +02:00
|
|
|
|
2021-02-12 08:20:45 +01:00
|
|
|
NODE_MODULES_CACHE_PATH = os.path.join(ZULIP_SRV_PATH, "zulip-npm-cache")
|
|
|
|
|
YARN_BIN = os.path.join(ZULIP_SRV_PATH, "zulip-yarn/bin/yarn")
|
|
|
|
|
YARN_PACKAGE_JSON = os.path.join(ZULIP_SRV_PATH, "zulip-yarn/package.json")
|
2017-07-27 23:22:52 +02:00
|
|
|
|
2017-07-28 00:18:23 +02:00
|
|
|
DEFAULT_PRODUCTION = False
|
|
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
|
python: Convert function type annotations to Python 3 style.
Generated by com2ann (slightly patched to avoid also converting
assignment type annotations, which require Python 3.6), followed by
some manual whitespace adjustment, and six fixes for runtime issues:
- def __init__(self, token: Token, parent: Optional[Node]) -> None:
+ def __init__(self, token: Token, parent: "Optional[Node]") -> None:
-def main(options: argparse.Namespace) -> NoReturn:
+def main(options: argparse.Namespace) -> "NoReturn":
-def fetch_request(url: str, callback: Any, **kwargs: Any) -> Generator[Callable[..., Any], Any, None]:
+def fetch_request(url: str, callback: Any, **kwargs: Any) -> "Generator[Callable[..., Any], Any, None]":
-def assert_server_running(server: subprocess.Popen[bytes], log_file: Optional[str]) -> None:
+def assert_server_running(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> None:
-def server_is_up(server: subprocess.Popen[bytes], log_file: Optional[str]) -> bool:
+def server_is_up(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> bool:
- method_kwarg_pairs: List[FuncKwargPair],
+ method_kwarg_pairs: "List[FuncKwargPair]",
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-19 03:48:37 +02:00
|
|
|
def get_yarn_args(production: bool) -> List[str]:
|
2017-07-28 00:18:23 +02:00
|
|
|
if production:
|
|
|
|
|
yarn_args = ["--prod"]
|
|
|
|
|
else:
|
|
|
|
|
yarn_args = []
|
|
|
|
|
return yarn_args
|
|
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
|
python: Convert function type annotations to Python 3 style.
Generated by com2ann (slightly patched to avoid also converting
assignment type annotations, which require Python 3.6), followed by
some manual whitespace adjustment, and six fixes for runtime issues:
- def __init__(self, token: Token, parent: Optional[Node]) -> None:
+ def __init__(self, token: Token, parent: "Optional[Node]") -> None:
-def main(options: argparse.Namespace) -> NoReturn:
+def main(options: argparse.Namespace) -> "NoReturn":
-def fetch_request(url: str, callback: Any, **kwargs: Any) -> Generator[Callable[..., Any], Any, None]:
+def fetch_request(url: str, callback: Any, **kwargs: Any) -> "Generator[Callable[..., Any], Any, None]":
-def assert_server_running(server: subprocess.Popen[bytes], log_file: Optional[str]) -> None:
+def assert_server_running(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> None:
-def server_is_up(server: subprocess.Popen[bytes], log_file: Optional[str]) -> bool:
+def server_is_up(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> bool:
- method_kwarg_pairs: List[FuncKwargPair],
+ method_kwarg_pairs: "List[FuncKwargPair]",
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-19 03:48:37 +02:00
|
|
|
def generate_sha1sum_node_modules(
|
2021-02-12 08:19:30 +01:00
|
|
|
setup_dir: Optional[str] = None,
|
|
|
|
|
production: bool = DEFAULT_PRODUCTION,
|
python: Convert function type annotations to Python 3 style.
Generated by com2ann (slightly patched to avoid also converting
assignment type annotations, which require Python 3.6), followed by
some manual whitespace adjustment, and six fixes for runtime issues:
- def __init__(self, token: Token, parent: Optional[Node]) -> None:
+ def __init__(self, token: Token, parent: "Optional[Node]") -> None:
-def main(options: argparse.Namespace) -> NoReturn:
+def main(options: argparse.Namespace) -> "NoReturn":
-def fetch_request(url: str, callback: Any, **kwargs: Any) -> Generator[Callable[..., Any], Any, None]:
+def fetch_request(url: str, callback: Any, **kwargs: Any) -> "Generator[Callable[..., Any], Any, None]":
-def assert_server_running(server: subprocess.Popen[bytes], log_file: Optional[str]) -> None:
+def assert_server_running(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> None:
-def server_is_up(server: subprocess.Popen[bytes], log_file: Optional[str]) -> bool:
+def server_is_up(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> bool:
- method_kwarg_pairs: List[FuncKwargPair],
+ method_kwarg_pairs: "List[FuncKwargPair]",
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-19 03:48:37 +02:00
|
|
|
) -> str:
|
2017-08-19 11:06:45 +02:00
|
|
|
if setup_dir is None:
|
|
|
|
|
setup_dir = os.path.realpath(os.getcwd())
|
2021-02-12 08:20:45 +01:00
|
|
|
PACKAGE_JSON_FILE_PATH = os.path.join(setup_dir, "package.json")
|
|
|
|
|
YARN_LOCK_FILE_PATH = os.path.join(setup_dir, "yarn.lock")
|
2021-05-28 03:51:42 +02:00
|
|
|
data: Dict[str, object] = {}
|
2021-08-02 23:19:49 +02:00
|
|
|
with open(PACKAGE_JSON_FILE_PATH) as f:
|
2021-05-28 03:51:42 +02:00
|
|
|
data[PACKAGE_JSON_FILE_PATH] = f.read().strip()
|
2017-09-26 01:52:27 +02:00
|
|
|
if os.path.exists(YARN_LOCK_FILE_PATH):
|
|
|
|
|
# For backwards compatibility, we can't assume yarn.lock exists
|
2021-08-02 23:19:49 +02:00
|
|
|
with open(YARN_LOCK_FILE_PATH) as f:
|
2021-05-28 03:51:42 +02:00
|
|
|
data[YARN_LOCK_FILE_PATH] = f.read().strip()
|
2020-04-09 21:51:58 +02:00
|
|
|
with open(YARN_PACKAGE_JSON) as f:
|
2021-05-28 03:51:42 +02:00
|
|
|
data["yarn-package-version"] = json.load(f)["version"]
|
2022-01-22 07:52:54 +01:00
|
|
|
data["node-version"] = subprocess.check_output(["node", "--version"], text=True).strip()
|
2021-05-28 03:51:42 +02:00
|
|
|
data["yarn-args"] = get_yarn_args(production=production)
|
|
|
|
|
|
|
|
|
|
sha1sum = hashlib.sha1()
|
2021-08-02 23:20:39 +02:00
|
|
|
sha1sum.update(json.dumps(data, sort_keys=True).encode())
|
2017-06-17 20:05:41 +02:00
|
|
|
return sha1sum.hexdigest()
|
|
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
|
python: Convert function type annotations to Python 3 style.
Generated by com2ann (slightly patched to avoid also converting
assignment type annotations, which require Python 3.6), followed by
some manual whitespace adjustment, and six fixes for runtime issues:
- def __init__(self, token: Token, parent: Optional[Node]) -> None:
+ def __init__(self, token: Token, parent: "Optional[Node]") -> None:
-def main(options: argparse.Namespace) -> NoReturn:
+def main(options: argparse.Namespace) -> "NoReturn":
-def fetch_request(url: str, callback: Any, **kwargs: Any) -> Generator[Callable[..., Any], Any, None]:
+def fetch_request(url: str, callback: Any, **kwargs: Any) -> "Generator[Callable[..., Any], Any, None]":
-def assert_server_running(server: subprocess.Popen[bytes], log_file: Optional[str]) -> None:
+def assert_server_running(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> None:
-def server_is_up(server: subprocess.Popen[bytes], log_file: Optional[str]) -> bool:
+def server_is_up(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> bool:
- method_kwarg_pairs: List[FuncKwargPair],
+ method_kwarg_pairs: "List[FuncKwargPair]",
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-19 03:48:37 +02:00
|
|
|
def setup_node_modules(
|
|
|
|
|
production: bool = DEFAULT_PRODUCTION,
|
|
|
|
|
prefer_offline: bool = False,
|
|
|
|
|
) -> None:
|
2017-07-28 00:18:23 +02:00
|
|
|
yarn_args = get_yarn_args(production=production)
|
2017-07-27 23:22:52 +02:00
|
|
|
if prefer_offline:
|
|
|
|
|
yarn_args.append("--prefer-offline")
|
2017-07-28 00:18:23 +02:00
|
|
|
sha1sum = generate_sha1sum_node_modules(production=production)
|
2017-07-22 01:21:01 +02:00
|
|
|
target_path = os.path.join(NODE_MODULES_CACHE_PATH, sha1sum)
|
2021-02-12 08:20:45 +01:00
|
|
|
cached_node_modules = os.path.join(target_path, "node_modules")
|
|
|
|
|
success_stamp = os.path.join(target_path, ".success-stamp")
|
2016-08-18 13:51:25 +02:00
|
|
|
# Check if a cached version already exists
|
|
|
|
|
if not os.path.exists(success_stamp):
|
2021-02-12 08:19:30 +01:00
|
|
|
do_yarn_install(target_path, yarn_args, success_stamp)
|
2016-08-18 13:51:25 +02:00
|
|
|
|
2021-08-02 23:36:06 +02:00
|
|
|
print(f"Using cached node modules from {cached_node_modules}")
|
2021-02-12 08:20:45 +01:00
|
|
|
if os.path.islink("node_modules"):
|
|
|
|
|
os.remove("node_modules")
|
|
|
|
|
elif os.path.isdir("node_modules"):
|
|
|
|
|
shutil.rmtree("node_modules")
|
|
|
|
|
os.symlink(cached_node_modules, "node_modules")
|
2016-08-18 13:51:25 +02:00
|
|
|
|
2021-02-12 08:19:30 +01:00
|
|
|
|
python: Convert function type annotations to Python 3 style.
Generated by com2ann (slightly patched to avoid also converting
assignment type annotations, which require Python 3.6), followed by
some manual whitespace adjustment, and six fixes for runtime issues:
- def __init__(self, token: Token, parent: Optional[Node]) -> None:
+ def __init__(self, token: Token, parent: "Optional[Node]") -> None:
-def main(options: argparse.Namespace) -> NoReturn:
+def main(options: argparse.Namespace) -> "NoReturn":
-def fetch_request(url: str, callback: Any, **kwargs: Any) -> Generator[Callable[..., Any], Any, None]:
+def fetch_request(url: str, callback: Any, **kwargs: Any) -> "Generator[Callable[..., Any], Any, None]":
-def assert_server_running(server: subprocess.Popen[bytes], log_file: Optional[str]) -> None:
+def assert_server_running(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> None:
-def server_is_up(server: subprocess.Popen[bytes], log_file: Optional[str]) -> bool:
+def server_is_up(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> bool:
- method_kwarg_pairs: List[FuncKwargPair],
+ method_kwarg_pairs: "List[FuncKwargPair]",
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-19 03:48:37 +02:00
|
|
|
def do_yarn_install(
|
|
|
|
|
target_path: str,
|
|
|
|
|
yarn_args: List[str],
|
python: Use trailing commas consistently.
Automatically generated by the following script, based on the output
of lint with flake8-comma:
import re
import sys
last_filename = None
last_row = None
lines = []
for msg in sys.stdin:
m = re.match(
r"\x1b\[35mflake8 \|\x1b\[0m \x1b\[1;31m(.+):(\d+):(\d+): (\w+)", msg
)
if m:
filename, row_str, col_str, err = m.groups()
row, col = int(row_str), int(col_str)
if filename == last_filename:
assert last_row != row
else:
if last_filename is not None:
with open(last_filename, "w") as f:
f.writelines(lines)
with open(filename) as f:
lines = f.readlines()
last_filename = filename
last_row = row
line = lines[row - 1]
if err in ["C812", "C815"]:
lines[row - 1] = line[: col - 1] + "," + line[col - 1 :]
elif err in ["C819"]:
assert line[col - 2] == ","
lines[row - 1] = line[: col - 2] + line[col - 1 :].lstrip(" ")
if last_filename is not None:
with open(last_filename, "w") as f:
f.writelines(lines)
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-10 05:23:40 +02:00
|
|
|
success_stamp: str,
|
python: Convert function type annotations to Python 3 style.
Generated by com2ann (slightly patched to avoid also converting
assignment type annotations, which require Python 3.6), followed by
some manual whitespace adjustment, and six fixes for runtime issues:
- def __init__(self, token: Token, parent: Optional[Node]) -> None:
+ def __init__(self, token: Token, parent: "Optional[Node]") -> None:
-def main(options: argparse.Namespace) -> NoReturn:
+def main(options: argparse.Namespace) -> "NoReturn":
-def fetch_request(url: str, callback: Any, **kwargs: Any) -> Generator[Callable[..., Any], Any, None]:
+def fetch_request(url: str, callback: Any, **kwargs: Any) -> "Generator[Callable[..., Any], Any, None]":
-def assert_server_running(server: subprocess.Popen[bytes], log_file: Optional[str]) -> None:
+def assert_server_running(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> None:
-def server_is_up(server: subprocess.Popen[bytes], log_file: Optional[str]) -> bool:
+def server_is_up(server: "subprocess.Popen[bytes]", log_file: Optional[str]) -> bool:
- method_kwarg_pairs: List[FuncKwargPair],
+ method_kwarg_pairs: "List[FuncKwargPair]",
Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-19 03:48:37 +02:00
|
|
|
) -> None:
|
2019-08-13 06:56:46 +02:00
|
|
|
os.makedirs(target_path, exist_ok=True)
|
2021-02-12 08:20:45 +01:00
|
|
|
shutil.copy("package.json", target_path)
|
2019-08-13 06:56:46 +02:00
|
|
|
shutil.copy("yarn.lock", target_path)
|
2019-08-28 00:32:07 +02:00
|
|
|
shutil.copy(".yarnrc", target_path)
|
2021-02-12 08:20:45 +01:00
|
|
|
cached_node_modules = os.path.join(target_path, "node_modules")
|
2019-08-13 07:04:28 +02:00
|
|
|
print("Cached version not found! Installing node modules.")
|
2017-07-27 23:22:52 +02:00
|
|
|
|
2019-08-13 07:04:28 +02:00
|
|
|
# Copy the existing node_modules to speed up install
|
2019-10-29 00:49:23 +01:00
|
|
|
if os.path.exists("node_modules") and not os.path.exists(cached_node_modules):
|
2019-10-06 21:53:15 +02:00
|
|
|
shutil.copytree("node_modules/", cached_node_modules, symlinks=True)
|
2021-02-12 08:20:45 +01:00
|
|
|
if os.environ.get("CUSTOM_CA_CERTIFICATES"):
|
|
|
|
|
run([YARN_BIN, "config", "set", "cafile", os.environ["CUSTOM_CA_CERTIFICATES"]])
|
2021-02-12 08:19:30 +01:00
|
|
|
run(
|
|
|
|
|
[YARN_BIN, "install", "--non-interactive", "--frozen-lockfile", *yarn_args], cwd=target_path
|
|
|
|
|
)
|
2021-02-12 08:20:45 +01:00
|
|
|
with open(success_stamp, "w"):
|
2019-08-13 06:56:46 +02:00
|
|
|
pass
|
