Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Annotate zephyr_mirror_backend.py. 

With some tweaks by tabbott to update tuple->Tuple and update comments.
This commit is contained in:
anirudhjain75 2016-12-07 00:33:23 +05:30 committed by Tim Abbott
parent 83ca1ab149
commit 4bb6735e77
2 changed files with 52 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
bots/zephyr_mirror_backend.py
View File

@ -21,7 +21,8 @@
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from __future__ import absolute_import
from typing import Any, List
from typing import IO, Any, Text, Union, Set, Tuple
from types import FrameType
import sys
from six.moves import map
@ -52,6 +53,7 @@ CURRENT_STATE = States.Startup
logger = None # type: logging.Logger
def to_zulip_username(zephyr_username):
# type: (str) -> str
if "@" in zephyr_username:
(user, realm) = zephyr_username.split("@")
else:
@ -64,6 +66,7 @@ def to_zulip_username(zephyr_username):
return user.lower() + "|" + realm.upper() + "@mit.edu"
def to_zephyr_username(zulip_username):
# type: (str) -> str
(user, realm) = zulip_username.split("@")
if "|" not in user:
# Hack to make ctl's fake username setup work :)
@ -86,6 +89,7 @@ def to_zephyr_username(zulip_username):
# or (3) the first word of the next line is longer than this entire
# line.
def different_paragraph(line, next_line):
# type: (str, str) -> bool
words = next_line.split()
return (len(line + " " + words[0]) < len(next_line) * 0.8 or
len(line + " " + words[0]) < 50 or
@ -94,6 +98,7 @@ def different_paragraph(line, next_line):
# Linewrapping algorithm based on:
# http://gcbenison.wordpress.com/2011/07/03/a-program-to-intelligently-remove-carriage-returns-so-you-can-paste-text-without-having-it-look-awful/ #ignorelongline
def unwrap_lines(body):
# type: (str) -> str
lines = body.split("\n")
result = ""
previous_line = lines[0]
@ -117,6 +122,7 @@ def unwrap_lines(body):
return result
def send_zulip(zeph):
# type: (Dict[str, str]) -> Dict[str, str]
message = {}
if options.forward_class_messages:
message["forged"] = "yes"
@ -149,6 +155,7 @@ def send_zulip(zeph):
return zulip_client.send_message(message)
def send_error_zulip(error_msg):
# type: (str) -> None
message = {"type": "private",
"sender": zulip_account_email,
"to": zulip_account_email,
@ -158,6 +165,7 @@ def send_error_zulip(error_msg):
current_zephyr_subs = set()
def zephyr_bulk_subscribe(subs):
# type: (List[Tuple[str, str, str]]) -> None
try:
zephyr._z.subAll(subs)
except IOError:
@ -193,6 +201,7 @@ def zephyr_bulk_subscribe(subs):
current_zephyr_subs.add(cls)
def update_subscriptions():
# type: () -> None
try:
f = open(options.stream_file_path, "r")
public_streams = simplejson.loads(f.read())
@ -216,6 +225,7 @@ def update_subscriptions():
zephyr_bulk_subscribe(list(classes_to_subscribe))
def maybe_kill_child():
# type: () -> None
try:
if child_pid is not None:
os.kill(child_pid, signal.SIGTERM)
@ -224,6 +234,7 @@ def maybe_kill_child():
logger.exception("")
def maybe_restart_mirroring_script():
# type: () -> None
if os.stat(os.path.join(options.root_path, "stamps", "restart_stamp")).st_mtime > start_time or \
((options.user == "tabbott" or options.user == "tabbott/extra") and
os.stat(os.path.join(options.root_path, "stamps", "tabbott_stamp")).st_mtime > start_time):
@ -243,6 +254,7 @@ def maybe_restart_mirroring_script():
time.sleep(1)
def process_loop(log):
# type: (IO) -> None
restart_check_count = 0
last_check_time = time.time()
while True:
@ -282,6 +294,7 @@ def process_loop(log):
logger.exception("Error updating subscriptions from Zulip:")
def parse_zephyr_body(zephyr_data):
# type: (str) -> Tuple[str, str]
try:
(zsig, body) = zephyr_data.split("\x00", 1)
except ValueError:
@ -289,6 +302,7 @@ def parse_zephyr_body(zephyr_data):
return (zsig, body)
def parse_crypt_table(zephyr_class, instance):
# type: (Text, str) -> str
try:
crypt_table = open(os.path.join(os.environ["HOME"], ".crypt-table"))
except IOError:
@ -310,6 +324,7 @@ def parse_crypt_table(zephyr_class, instance):
return None
def decrypt_zephyr(zephyr_class, instance, body):
# type: (Text, str, str) -> str
keypath = parse_crypt_table(zephyr_class, instance)
if keypath is None:
# We can't decrypt it, so we just return the original body
@ -340,6 +355,7 @@ def decrypt_zephyr(zephyr_class, instance, body):
return decrypted
def process_notice(notice, log):
# type: (zulip, IO) -> None
(zsig, body) = parse_zephyr_body(notice.message)
is_personal = False
is_huddle = False
@ -438,6 +454,8 @@ def process_notice(notice, log):
os._exit(0)
def decode_unicode_byte_strings(zeph):
# type: (Dict[str, Any]) -> Dict[str, str]
# 'Any' can be of any type of text that is converted to str.
for field in zeph.keys():
if isinstance(zeph[field], str):
try:
@ -448,11 +466,13 @@ def decode_unicode_byte_strings(zeph):
return zeph
def quit_failed_initialization(message):
# type: (str) -> str
logger.error(message)
maybe_kill_child()
sys.exit(1)
def zephyr_init_autoretry():
# type: () -> None
backoff = zulip.RandomExponentialBackoff()
while backoff.keep_going():
try:
@ -468,6 +488,7 @@ def zephyr_init_autoretry():
quit_failed_initialization("Could not initialize Zephyr library, quitting!")
def zephyr_load_session_autoretry(session_path):
# type: (str) -> None
backoff = zulip.RandomExponentialBackoff()
while backoff.keep_going():
try:
@ -483,6 +504,7 @@ def zephyr_load_session_autoretry(session_path):
quit_failed_initialization("Could not load saved Zephyr session, quitting!")
def zephyr_subscribe_autoretry(sub):
# type: (Tuple[str, str, str]) -> None
backoff = zulip.RandomExponentialBackoff()
while backoff.keep_going():
try:
@ -498,6 +520,7 @@ def zephyr_subscribe_autoretry(sub):
quit_failed_initialization("Could not subscribe to personals, quitting!")
def zephyr_to_zulip(options):
# type: (Any) -> None
if options.use_sessions and os.path.exists(options.session_path):
logger.info("Loading old session")
zephyr_load_session_autoretry(options.session_path)
@ -549,6 +572,7 @@ def zephyr_to_zulip(options):
process_loop(None)
def send_zephyr(zwrite_args, content):
# type: (list, str) -> Tuple[int, str]
p = subprocess.Popen(zwrite_args, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate(input=content.encode("utf-8"))
@ -565,12 +589,15 @@ def send_zephyr(zwrite_args, content):
return (p.returncode, stderr)
def send_authed_zephyr(zwrite_args, content):
# type: (list[str], str) -> Tuple[int, str]
return send_zephyr(zwrite_args, content)
def send_unauthed_zephyr(zwrite_args, content):
# type: (list[str], str) -> Tuple[int, str]
return send_zephyr(zwrite_args + ["-d"], content)
def zcrypt_encrypt_content(zephyr_class, instance, content):
# type: (str, str, str) -> str
keypath = parse_crypt_table(zephyr_class, instance)
if keypath is None:
return None
@ -596,13 +623,15 @@ def zcrypt_encrypt_content(zephyr_class, instance, content):
return encrypted
def forward_to_zephyr(message):
# type: (Dict[str, Any]) -> None
# 'Any' can be of any type of text
support_heading = "Hi there! This is an automated message from Zulip."
support_closing = """If you have any questions, please be in touch through the \
Feedback button or at support@zulipchat.com."""
wrapper = textwrap.TextWrapper(break_long_words=False, break_on_hyphens=False)
wrapped_content = "\n".join("\n".join(wrapper.wrap(line))
for line in message["content"].replace("@", "@@").split("\n"))
for line in message["content"].replace("@", "@@").split("\n"))
zwrite_args = ["zwrite", "-n", "-s", message["sender_full_name"],
"-F", "Zephyr error: See http://zephyr.1ts.org/wiki/df",
@ -656,7 +685,7 @@ Feedback button or at support@zulipchat.com."""
if message.get("invite_only_stream"):
result = zcrypt_encrypt_content(zephyr_class, instance, wrapped_content)
if result is None:
return send_error_zulip("""%s
send_error_zulip("""%s
Your Zulip-Zephyr mirror bot was unable to forward that last message \
from Zulip to Zephyr because you were sending to a zcrypted Zephyr \
@ -665,6 +694,7 @@ key (perhaps because your AFS tokens expired). That means that while \
Zulip users (like you) received it, Zephyr users did not.
%s""" % (support_heading, support_closing))
return
# Proceed with sending a zcrypted message
wrapped_content = result
@ -679,7 +709,7 @@ Zulip users (like you) received it, Zephyr users did not.
if code == 0 and stderr == "":
return
elif code == 0:
return send_error_zulip("""%s
send_error_zulip("""%s
Your last message was successfully mirrored to zephyr, but zwrite \
returned the following warning:
@ -687,6 +717,7 @@ returned the following warning:
%s
%s""" % (support_heading, stderr, support_closing))
return
elif code != 0 and (stderr.startswith("zwrite: Ticket expired while sending notice to ") or
stderr.startswith("zwrite: No credentials cache found while sending notice to ")):
# Retry sending the message unauthenticated; if that works,
@ -695,7 +726,7 @@ returned the following warning:
if code == 0:
if options.ignore_expired_tickets:
return
return send_error_zulip("""%s
send_error_zulip("""%s
Your last message was forwarded from Zulip to Zephyr unauthenticated, \
because your Kerberos tickets have expired. It was sent successfully, \
@ -704,11 +735,12 @@ are running the Zulip-Zephyr mirroring bot, so we can send \
authenticated Zephyr messages for you again.
%s""" % (support_heading, support_closing))
return
# zwrite failed and it wasn't because of expired tickets: This is
# probably because the recipient isn't subscribed to personals,
# but regardless, we should just notify the user.
return send_error_zulip("""%s
send_error_zulip("""%s
Your Zulip-Zephyr mirror bot was unable to forward that last message \
from Zulip to Zephyr. That means that while Zulip users (like you) \
@ -717,8 +749,11 @@ received it, Zephyr users did not. The error message from zwrite was:
%s
%s""" % (support_heading, stderr, support_closing))
return
def maybe_forward_to_zephyr(message):
# type: (Dict[str, Any]) -> None
# The key string can be used to direct any type of text.
if (message["sender_email"] == zulip_account_email):
if not ((message["type"] == "stream") or
(message["type"] == "private" and
@ -740,6 +775,7 @@ def maybe_forward_to_zephyr(message):
logger.exception("Error forwarding message:")
def zulip_to_zephyr(options):
# type: (int) -> None
# Sync messages from zulip to zephyr
logger.info("Starting syncing messages.")
while True:
@ -750,6 +786,7 @@ def zulip_to_zephyr(options):
time.sleep(1)
def subscribed_to_mail_messages():
# type: () -> bool
# In case we have lost our AFS tokens and those won't be able to
# parse the Zephyr subs file, first try reading in result of this
# query from the environment so we can avoid the filesystem read.
@ -764,6 +801,7 @@ def subscribed_to_mail_messages():
return False
def add_zulip_subscriptions(verbose):
# type: (bool) -> None
zephyr_subscriptions = set()
skipped = set()
for (cls, instance, recipient) in parse_zephyr_subs(verbose=verbose):
@ -850,9 +888,11 @@ web interface.
""")) + "\n")
def valid_stream_name(name):
# type: (str) -> bool
return name != ""
def parse_zephyr_subs(verbose=False):
# type: (bool) -> Union[List, Tuple, Set[Tuple[str, str, str]]]
zephyr_subscriptions = set()
subs_file = os.path.join(os.environ["HOME"], ".zephyr.subs")
if not os.path.exists(subs_file):
@ -907,6 +947,7 @@ def open_logger():
return logger
def configure_logger(logger, direction_name):
# type: (logging.Logger, str) -> None
if direction_name is None:
log_format = "%(message)s"
else:
@ -921,6 +962,7 @@ def configure_logger(logger, direction_name):
handler.setFormatter(formatter)
def parse_args():
# type: () -> Tuple
parser = optparse.OptionParser()
parser.add_option('--forward-class-messages',
default=False,
@ -1004,6 +1046,7 @@ def parse_args():
return parser.parse_args()
def die_gracefully(signal, frame):
# type: (int, FrameType) -> None
if CURRENT_STATE == States.ZulipToZephyr or CURRENT_STATE == States.ChildSending:
# this is a child process, so we want os._exit (no clean-up necessary)
os._exit(1)
@ -1029,6 +1072,9 @@ if __name__ == "__main__":
signal.signal(signal.SIGINT, die_gracefully)
# The properties available on 'options' are dynamically
# determined, so we have to treat it as an Any for type
# annotations.
(options, args) = parse_args() # type: Any, List[str]
logger = open_logger()

1
tools/run-mypy
View File

@ -37,7 +37,6 @@ api/integrations/svn/post-commit
api/integrations/twitter/twitter-bot
api/integrations/twitter/twitter-search-bot
api/zulip/__init__.py
bots/zephyr_mirror_backend.py
tools/deprecated/inject-messages/inject-messages
zproject/settings.py
zproject/test_settings.py