zulip/tools/review

#!/usr/bin/env python
# encoding: utf-8
#
# Copyright (C) 2010 Ksplice, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Author: Greg Price <price@ksplice.com>

from __future__ import unicode_literals

## CC_EMAIL: All review requests will be CC'd here.
CC_EMAIL = 'code-review@zulip.com'

## DOMAIN: Reviewers without an '@' will be assumed to be localparts here.
DOMAIN = 'zulip.com'

## TEST_SCRIPT: Test file to run; path relative to root of the working tree, or absolute.
TEST_SCRIPT = "tools/test-all"

##### END CONFIG #####
## But you might want to change behavior below.

import sys
import os
import optparse
import posixpath
import subprocess
import tempfile
import git
import shlex
import codecs
import fcntl
from StringIO import StringIO
from email.message import Message
from email.header import Header
from shutil import rmtree, copyfile, copytree
from time import sleep
from signal import SIGINT, SIGTERM
from errno import EAGAIN

sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../api'))
import zulip

usage = """
%%prog -r <reviewer> [-r <another-reviewer>] [-T  <alt tmp dir>] [-s <summary>] [-m <message>] [options] {<since>|<revision-range>}

Send a patch series for review, after optionally successfully running test-all (if you specify '-T').
If requested, tests will run in /tmp, and thus you can continue to work in your own working tree.
Sends mail to the reviewer, CC %s, identifying the commits to be reviewed.

Name a range of commits, or name a single commit (e.g., 'HEAD^' or
'origin/master') to identify all commits on HEAD since that commit.

If you set REVIEW_USE_SENDMAIL to be nonempty, /usr/bin/sendmail will
be used to mail the review request. Otherwise, msmtp will be used by
default.
""".strip() % CC_EMAIL


def parse_options(args):
  parser = optparse.OptionParser(usage)
  parser.add_option('--first-parent', action='store_true', dest='first_parent',
                    help='follow first parents only')
  parser.add_option('-r', '--reviewer', type='string', dest='reviewers', action="append",
                    help='the person you are asking to do the review')
  parser.add_option('--stdout', action='store_true', dest='stdout',
                    help='send to standard output rather than send mail')
  parser.add_option('--format', type='choice', dest='format',
                    choices=['oneline', 'message', 'patch'],
                    help="'patch' (default for one commit), 'message' (default for more), or 'oneline'")
  parser.add_option('-s', '--summary', type='string', dest='summary',
                    help='summary for subject line')
  parser.add_option('-m', '--message', type='string', dest='message',
                    help='message for body of email')
  parser.add_option('-t', '--testing', type='string', dest='testing',
                    help='extent and methods of testing employed')
  parser.add_option('-e', '--edit', action='store_true', dest='edit',
                    help='spawn $EDITOR and edit review request')
  parser.add_option('-T', '--run-tests', action='store_true', dest='run_tests',
                    help='run test before sending the review')
  parser.add_option('--testing-tmp-directory', type='string', dest="tmp_dir", default="/tmp",
                    help="specify different temp directory for testing (default /tmp)")

  options, args = parser.parse_args(args)
  if not options.reviewers:
    parser.error('reviewer required')
  reviewers_fixed = []
  for reviewer in options.reviewers:
    if '@' not in reviewer:
      reviewers_fixed.append(reviewer + '@' + DOMAIN)
    else:
      reviewers_fixed.append(reviewer)
  options.reviewers = reviewers_fixed
  if len(args) < 2:
    parser.error('must specify revision(s) to be reviewed')
  return options, args


def get_default_remote(repo):
  try:
    return repo.git.config('--get', 'remotes.default')
  except git.exc.GitCommandError:
    try:
      branch = repo.active_branch
    except TypeError:
      return 'origin'
    try:
      return repo.git.config('--get', ('branch.%s.remote' % unicode(str(branch), "utf-8")).encode("utf-8"))
    except git.exc.GitCommandError:
      return 'origin'


def get_reponame(repo):
  remote = get_default_remote(repo)

  try:
    url = repo.git.config('--get', 'remote.%s.url' % remote)
  except git.exc.GitCommandError:
    url = repo.wd

  name = posixpath.basename(posixpath.normpath(url.split(':', 1)[-1]))
  if name.endswith('.git'):
    name = name[:-len('.git')]
  return name


def parse_revs(repo, opts, args):
  args = repo.git.rev_parse(*args).splitlines()
  if len(args) == 1:
      args = ['^' + args[0].lstrip('^'), 'HEAD']
  if opts.first_parent:
      args[:0] = ['--first-parent']
  return [repo.commit(c) for c in repo.git.rev_list('--reverse', *args).split()]


def get_current_user(repo):
  ident = unicode(repo.git.var('GIT_AUTHOR_IDENT'), 'utf-8', 'replace')
  return ident[:ident.rindex('>') + 1]

def get_current_user_email(repo):
  me = get_current_user(repo)
  return me[me.index("<") + 1:me.index('>')]

def make_header(repo, opts, revs):
  me = get_current_user(repo)
  reponame = get_reponame(repo)

  remote = get_default_remote(repo)
  (sha, name) = repo.git.name_rev(revs[-1].hexsha,
                                  refs='refs/remotes/%s/*' % (remote,),
                                  always=True).split()
  name = unicode(name, "utf-8")
  prefix = 'remotes/' + remote + "/"
  if name.startswith(prefix):
      name = name[len(prefix):]
      tip_name = '%s (%s)' % (name, revs[-1].hexsha[:7])
  else:
    (_, local_name) = repo.git.name_rev(revs[-1].hexsha,
                                        refs='refs/heads/*',
                                        always=True).split()
    local_name = unicode(local_name, "utf-8")
    if local_name == "undefined":
      print >>sys.stderr, "ERROR: Can't find this commit in remote or identify local branch!"
      sys.exit(1)

    email_basename = get_current_user_email(repo).split("@")[0]
    if local_name.startswith("%s-" % (email_basename,)):
      # Try to push the local branch to the remote automatically
      try:
        print "Attempting to push %s to remote %s" % (local_name, remote)
        repo.git.push(remote, local_name)
        tip_name = '%s (%s)' % (local_name, revs[-1].hexsha[:7])
      except git.GitCommandError, e:
        print >>sys.stderr, "ERROR: Couldn't push %s to remote" % (local_name,)
        print >>sys.stderr, e
        sys.exit(1)
    else:
      print >>sys.stderr, "ERROR: Can't find this commit in remote -- did you push %s?" % (local_name)
      sys.exit(1)

  objective_summary = '%d commit(s) to %s' % (len(revs), tip_name)
  summary = ('%s (%s)' % (opts.summary, objective_summary) if opts.summary
             else objective_summary)

  return [('From', Header(me.encode("utf-8"), "utf-8")),
          ('To', Header(', '.join(opts.reviewers), "utf-8")),
          ('Cc', Header(CC_EMAIL, "utf-8")),
          ('Subject', Header('%s review: %s' % (
              reponame, summary), "utf-8"))]


def write_template(target, repo, opts):
  me = get_current_user(repo)

  print >>target, 'Dear %s,' % ", ".join(opts.reviewers)
  print >>target
  print >>target, 'At your convenience, please review the following commits.'
  print >>target
  if opts.message:
    print >>target, opts.message
    print >>target
  print >>target, 'Testing:'
  if opts.testing:
    print >>target, opts.testing
  else:
    print >>target, '(No formal testing done, or none specified.)'
  print >>target
  print >>target, 'Thanks,'
  print >>target, me


def write_commitmsg(target, repo, opts, revs):

  if opts.format == 'oneline':
    for r in revs:
      print >>target, unicode(repo.git.log('-n1', '--oneline', r), 'utf-8', 'replace')
  elif opts.format == 'message' or opts.format is None and len(revs) > 1:
    for r in revs:
      if opts.first_parent:
        print >>target, unicode(repo.git.log('-n1', r), 'utf-8', 'replace')
        print >>target, unicode(repo.git.diff('--stat', str(r)+'^', r), 'utf-8', 'replace')
      else:
        print >>target, unicode(repo.git.log('-n1', '--stat', r), 'utf-8', 'replace')
      print >>target
  elif opts.format == 'patch' or opts.format is None and len(revs) == 1:
    for r in revs:
      if opts.first_parent:
        print >>target, unicode(repo.git.log('-n1', r), 'utf-8', 'replace')
        print >>target, unicode(repo.git.diff('--stat', '-p', str(r)+'^', r), 'utf-8', 'replace')
      else:
        print >>target, unicode(repo.git.log('-n1', '--stat', '-p', r), 'utf-8', 'replace')
      print >>target
  else:
    raise Exception("Bad format option.")


def edit(repo, opts, revs, headers):
  template = StringIO()
  commitmsg = StringIO()

  write_template(template, repo, opts)
  write_commitmsg(commitmsg, repo, opts, revs)

  temp = codecs.getwriter('utf-8')(tempfile.NamedTemporaryFile(prefix="review-"))

  # Prepare editable buffer.

  print >>temp, """# This is an editable review request.  All lines beginning with # will
# be ignored.  To abort the commit, remove all lines from this buffer."""
  print >>temp, "#"
  for (key, value) in headers:
    print >>temp, u"# %s: %s" % (key, value)
  print >>temp
  print >>temp, template.getvalue()
  for line in commitmsg.getvalue().splitlines():
    print >>temp, "# " + line
  temp.flush()

  # Open EDITOR to edit buffer.

  editor = os.getenv('EDITOR','emacs')
  subprocess.check_call(shlex.split(editor) + [temp.name])

  # Check if buffer is empty, and if so abort.

  if (os.path.getsize(temp.name) == 0):
    print >>sys.stderr, "Aborting due to empty buffer."
    sys.exit(2)

  # Reopen temp file, slurp it in, and reconstruct mail.

  final = codecs.open(temp.name, 'r', 'utf-8')
  msg = Message()
  for (key, value) in headers:
    msg[key] = value
  msg.set_payload(
    ("".join(line for line in final if not line.startswith("#")).strip() +
     "\n\n" + commitmsg.getvalue()).encode('utf-8'),
    'utf-8')

  # Clean up.

  temp.close()
  final.close()
  try:
    os.unlink(temp.name)
  except OSError:
    pass
  return msg

def clone_to_tmp(repo, tmp_path, test_existence_only=False):
  print "Cloning into", tmp_path
  assert not os.path.exists(tmp_path), \
    'Path ' + tmp_path + ' exists. Either another test is running there ' \
    'or it was left there so that you could view the testing log. If you are sure ' \
    'another test is not running there, remove this directory before proceeding.'
  if test_existence_only:
      return
  clone = repo.clone(tmp_path)
  clone.git.checkout(repo.commit().hexsha)
  return clone

def cleanup_tmp(cloned_repo):
  print "Cleaning up..."
  assert os.path.exists(cloned_repo.working_tree_dir), "could not find " + cloned_repo.working_tree_dir
  rmtree(cloned_repo.working_tree_dir)
  return

def run_test_script(repo, tmp_path):
  # Copy over relevant files to $tmp_dir/humbug_test.git
  working_dir = os.getcwd()
  clone = clone_to_tmp(repo, tmp_path)
  for filename in ['zerver/fixtures/available-migrations',
                   'zerver/fixtures/migration-status',
                   'zerver/fixtures/messages.json',
                   'event_queues.pickle']:
    if os.path.isfile(filename):
      copyfile(filename, clone.working_tree_dir + "/" + filename)
  for directory in ['event_log', 'stats']:
    if os.path.isdir(directory):
      copytree(directory, clone.working_tree_dir + "/" + directory)

  # Go to $tmp_path and run TEST_SCRIPT
  os.chdir(clone.working_tree_dir)
  os.environ["PWD"] = os.getcwd()
  log = open("test.log", 'w')
  log_path = os.getcwd() + "/" + log.name
  print "Testing [log=%s] ..." % (log_path,)
  try:
    proc = subprocess.Popen(TEST_SCRIPT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=sys.stdin)
    # Set the proc.stdout.read() to be non-blocking
    fcntl.fcntl(proc.stdout, fcntl.F_SETFL, os.O_NONBLOCK)
    while proc.poll() is None:
      try:
        buf = proc.stdout.read(1024)
        if buf != "":
          sys.stdout.write(buf)
          sys.stdout.flush()
          log.write(buf)
          log.flush()
      except IOError as e:
        #ignore EAGAIN errors caused by non-blocking prod.stdout
        if e.errno != EAGAIN:
          raise e
        else:
          continue
      sleep(.5)
  except KeyboardInterrupt as e:
    print "\nkilling tests..."
    print "You may find the testing log in", log_path
    proc.send_signal(SIGINT)
    sleep(2)
    proc.send_signal(SIGTERM)
    proc.wait()
    buf = proc.stdout.read()
    log.write(buf)
    sys.stdout.write(buf)
    raise e
  buf = proc.stdout.read()
  sys.stdout.write(buf)
  log.write(buf)
  if proc.returncode!=0:
    print TEST_SCRIPT, "exited with status code", proc.returncode
    print "You may find the testing log in", log_path
  else:
    os.chdir(working_dir)
    os.environ["PWD"] = os.getcwd()
    cleanup_tmp(clone)
  return proc.returncode

def main(args):
  opts, args = parse_options(args)
  repo = git.Repo()
  revs = parse_revs(repo, opts, args[1:])
  if not revs:
    print >>sys.stderr, '%s: no revisions specified' % os.path.basename(args[0])
    return 2

  if not opts.stdout:
    client = zulip.Client(verbose=True)

    if 'staging' not in client.base_url:
      print '''

        HEY!  You have to point your .zuliprc to staging.
        ABORTING

        '''
      sys.exit(1)

  # Attempt to catch permissions/invalid paths early for tmp_dir
  if opts.run_tests:
    tmp_path = opts.tmp_dir + "/humbug_test.git"
    clone_to_tmp(repo, tmp_path, test_existence_only=True)
    os.mkdir(tmp_path)
    os.rmdir(tmp_path)

  # Also pushes the branch to your remote
  headers = make_header(repo, opts, revs)
  if opts.edit:
    msg = edit(repo, opts, revs, headers)

  else:
    # Just build the message.
    msg = Message()
    for (key, value) in headers:
      msg[key] = value

    template = StringIO()
    commitmsg = StringIO()

    write_template(template, repo, opts)
    write_commitmsg(commitmsg, repo, opts, revs)
    msg.set_payload(
      (template.getvalue() + "\n" + commitmsg.getvalue()).encode('utf-8'),
      'utf-8')

  # Run TEST_SCRIPT on HEAD, if desired.
  if opts.run_tests:
    testing_status_code = None
    try:
      testing_status_code = run_test_script(repo, tmp_path)
      if testing_status_code != 0:
        sys.exit(testing_status_code)
    finally:
      if testing_status_code != 0:
        commit_message_path = opts.tmp_dir + "/commit.msg"
        open(commit_message_path, 'w').write(msg.get_payload(decode=True))
        print "You may find your commit message at", commit_message_path

  print "Processing review message..."
  # Send or print the message, as appropriate.
  if opts.stdout:
    for (key, value) in msg.items():
      print >>sys.stdout, u"%s: %s" % (key, value)
    print >>sys.stdout
    print >>sys.stdout, msg.get_payload(decode=True),
  else:
    reviewer_usernames = [x.split("@")[0] for x in opts.reviewers]

    content = ("Sent a review request to %s for\n%s" %
        (' '.join('@'+x for x in reviewer_usernames),
         msg.get('Subject', '<unknown>')))
    subject = "Reviews for " + ", ".join(reviewer_usernames)

    client.send_message({'type': "stream",
                         'subject': subject[:60],
                         'to': "review",
                         'content': content})

    if os.environ.get('REVIEW_USE_SENDMAIL', ''):
      command = ['/usr/sbin/sendmail', '-bm', '-t']
    else:
      command = ['msmtp', '-t']
    subprocess.Popen(command,
                     stdin=subprocess.PIPE).communicate(msg.as_string())

if __name__ == '__main__':
  sys.exit(main(sys.argv))