ipatool

#!/usr/bin/python3

"""Tool helping the IPA project processes

Usage:
  ipatool --help
  ipatool [options] [-v...] sample-config
  ipatool [options] [-v...] push [--branch=BRANCH...] [--reviewer=NAME...] [--] [PATCH ...]
  ipatool [options] [-v...] start-review [-f] [--am] [--ticket=NUMBER...] [--] [PATCH ...]
  ipatool [options] [-v...] am [--] [PATCH ...]

Common Options:
  -h, --help           Display this help and exit
  -v, --verbose        Increase verbosity
  --config FILE        Configuration file [default: ~/.ipa/toolconf.yaml]
  --no-reviewer        Do not add a Reviewed-By: line
  -n, --dry-run        Do not push
  --no-trac            Do not contact Trac
  --no-fetch           Do not synchronize before pushing
  --color=(auto|always|never)  Colorize output [default: auto]
  PATCH                Patch to push, or directory with *.patch files

Configuration is specified in the file given by --config.
Run ipatool sample-config to see what needs to go in it.

ipatool push:
  The given patches are applied on top of the given upstream branches,
  and pushed upstream.
  A "Reviewed-By" line with the name(s) given by --reviewer is added
  to all patches unless --no-reviewer is given).
  If neither --reviewer nor --no-reviewer is given, a reviewer is looked up in
  the "Reviewed by" field of the Trac ticket(s). If that does not yield one
  reviewer, the pushpatches command fails.
  If the reviewer name is not in the
  form "Name Last <mail@address.example>", it is looked up in the contributors
  as listed in `git shortlog -se`.

  -b, --branch=BRANCH  Branch to push to (detected from ticket if not given here)
  -r, --reviewer=NAME  Reviewer(s) of the patches

ipatool start-review:
  Sets yourself as the reviewer for given tickets, and also adds you to CC.
  Tickets may be mentioned by number with the --ticket option,
  and are searched in any patches given on the command line.
  If no tickets and no patches are specified, all patches from the configured
  patchdir are searched.

  -t, --ticket NUMBER  Number of ticket to update
  -f, --force          Force setting reviewer even if already set
  --am                 Also apply the patches as in `ipatool am`

ipatool am:
  Applies the given patches to the working tree, using a configured command.
  Uses the configured patchdir if no patches are given.
"""

SAMPLE_CONFIG = """
# Local "clean" repository
clean-repo-path: ~/dev/freeipa-clean
remote: origin

# Default directory where patches to push are stored
patchdir: ~/patches/to-apply

# URLs to use in reporrts & messages
ticket-url: https://fedorahosted.org/freeipa/ticket/
commit-url: https://fedorahosted.org/freeipa/changeset/
bugzilla-bug-url: https://bugzilla.redhat.com/show_bug.cgi?id=

# Trac XML login details
trac-protocol: https
trac-host: fedorahosted.org
trac-path: freeipa
trac-username: pviktori
trac-password: "!@#$%^&*"

# Mapping of Trac logins to Git-style author lines
trac-username-map:
    abbra: Alexander Bokovoy <abokovoy@redhat.com>

# Command to run "git am" on the development tree (as argv list)
am-command: ["ssh", "ipa-devel-vm.local", "cd ~/freeipa/ ; git am -3"]

# Currently unused :(
browser: firefox
"""

import glob
import sys
import os
import subprocess
import re
import collections
import xmlrpc.client
import pprint

import yaml       # yum install python3-PyYAML
import blessings  # yum install python3-blessings
import unidecode  # yum install python3-unidecode
import docopt     # yum install python3-docopt

MILESTONES = {
    r"^FreeIPA 3\.3\..*": ['master', 'ipa-4-1', 'ipa-4-0', 'ipa-3-3'],
    r"^FreeIPA 4\.0.*": ['master', 'ipa-4-1', 'ipa-4-0'],
    r"^FreeIPA 4\.1.*": ['master', 'ipa-4-1'],
    r"^FreeIPA 4\.2.*": ['master', 'ipa-4-2'],
    r"^FreeIPA 4\.3.*": ['master'],
}

COLOR_OPT_MAP = {'auto': False, 'always': True, 'never': None}

SUBJECT_RE = re.compile('^Subject:( *\[PATCH( [^]*])?\])*(?P<subj>.*)')

SubprocessResult = collections.namedtuple(
    'SubprocessResult', 'stdout stderr returncode')
TracTicketData = collections.namedtuple(
    'TracTicketData', 'id time_created time_changed attributes')


def cleanpath(path):
    """Return absolute path with leading ~ expanded"""
    path = os.path.expanduser(path)
    path = os.path.abspath(path)
    return path


def shellquote(arg):
    """Quote an argument for the shell"""
    if re.match('^[-_.:/=a-zA-Z0-9]*$', arg):
        return arg
    else:
        return "'%s'" % arg.replace("'", r"'\''")


class reify(object):
    # https://github.com/Pylons/pyramid/blob/1.4-branch/pyramid/decorator.py
    def __init__(self, wrapped):
        self.wrapped = wrapped
        self.__doc__ = getattr(wrapped, '__doc__', None)

    def __get__(self, inst, objtype=None):
        if inst is None:
            return self
        val = self.wrapped(inst)
        setattr(inst, self.wrapped.__name__, val)
        return val


class Patch(object):
    """Represents a sanitized patch

    - Removes ">" from From lines in the metadata/message
    - Adds a Reviewed-By tag

    Attributes:
    * subject - name of the patch
    * lines - iterator of lines with the patch
    * ticket_numbers - numbers of referenced Trac tickets
    """
    def __init__(self, config, filename):
        if filename:
            self.filename = filename
            with open(filename) as file:
                lines = list(file)
        else:
            self.filename = '(patch)'
        assert lines
        lines_iter = iter(lines)
        self.head_lines = []
        self.patch_lines = []
        in_subject = False
        self.subject = ''
        for line in lines_iter:
            if not line.startswith(' '):
                in_subject = False
            if in_subject:
                self.subject += line.rstrip()
            match = SUBJECT_RE.match(line)
            if match:
                self.subject = match.group('subj').strip()
                in_subject = True

            if line.startswith('>From'):
                self.head_lines.append(line[1:])
            elif any([
                    line == '---\n',
                    line.startswith('diff -'),
                    line.startswith('Index: ')]):
                self.patch_lines.append(line)
                break
            else:
                self.head_lines.append(line)
        self.patch_lines.extend(lines_iter)

        self.ticket_numbers = []
        for line in self.lines:
            regex = '%s(\d*)' % re.escape(config['ticket-url'])
            for match in re.finditer(regex, line):
                self.ticket_numbers.append(int(match.group(1)))

    def add_reviewer(self, reviewer):
        if not re.match('^[-_a-zA-Z0-9]+: .*$', self.head_lines[-1]):
            self.head_lines.append('\n')
        self.head_lines.append('Reviewed-By: %s\n' % reviewer)

    @property
    def lines(self):
        yield from self.head_lines
        yield from self.patch_lines


class Ticket(object):
    """Trac ticket with lazily fetched information"""
    def __init__(self, trac, number):
        self.trac = trac
        self.number = number

    @reify
    def data(self):
        print('Retrieving ticket %s' % self.number)
        data = TracTicketData(*self.trac.ticket.get(self.number))
        return data

    @property
    def attributes(self):
        return self.data.attributes


class Context(object):
    """Holds options, configuration, and helpers for a tool"""
    def __init__(self, options):
        self.options = options
        with open(os.path.expanduser(options['--config'])) as conf_file:
            self.config = yaml.safe_load(conf_file)
        self.term = blessings.Terminal(
            force_styling=COLOR_OPT_MAP[options['--color']])
        self.verbosity = self.options['--verbose']
        if self.verbosity:
            print('Options:')
            pprint.pprint(self.options)
            print('Config:')
            pprint.pprint(self.config)
        if self.options['--no-trac']:
            self.trac = None
        else:
            url = ('{c[trac-protocol]}://{c[trac-host]}/{c[trac-path]}'
                   '/xmlrpc').format(c=self.config)
            self.trac = xmlrpc.client.ServerProxy(url)

        self.color_arg = self.options['--color']
        if self.color_arg == 'auto':
            if self.term.is_a_tty:
                self.color_arg = 'always'
            else:
                self.color_arg = 'never'

        # ipatool sets GIT_COMMITTER_DATE to a fixed value, so
        # commits to parallel branches hopefully end up identical.
        # Getting the current date with timezone info is a pain in Python
        # (needs extra dependencies), so just run date
        self.isodate_now = ''
        date_result = self.runprocess(['date', '-Iseconds'],
                                      env={'GIT_COMMIT_DATE': ''})
        self.isodate_now = date_result.stdout.strip()

    def trac_login(self):
        url = ('{c[trac-protocol]}://{c[trac-username]}:{c[trac-password]}'
               '@{c[trac-host]}/{c[trac-path]}'
               '/login/xmlrpc').format(c=self.config)
        self.trac = xmlrpc.client.ServerProxy(url)

    commands = {}
    @classmethod
    def command(cls, name):
        def decorator(func):
            cls.commands[name] = func
            return func
        return decorator

    def run(self):
        for name, func in self.commands.items():
            if self.options[name]:
                return func(self)
        else:
            print('Regstered commands: %s' % ', '.join(self.commands))
            self.die('Internal error: No command found')

    def die(self, message):
        print(self.term.red(message))
        exit(1)

    def get_patches(self):
        paths = self.options['PATCH'] or [self.config['patchdir']]
        for path in paths:
            path = cleanpath(path)
            if os.path.isdir(path):
                filenames = glob.glob(os.path.join(path, '*.patch'))
                for filename in sorted(filenames):
                    yield Patch(self.config, filename)
            else:
                yield Patch(self.config, path)

    def runprocess(self, argv, check_stdout=None, check_stderr=None,
                   check_returncode=0, stdin_string='', fail_message=None,
                   timeout=5, verbosity=None, env=None):
        """Run a command in a subprocess, check & return result"""
        if env is None:
            env = os.environ
        env.setdefault('GIT_COMMITTER_DATE', self.isodate_now)
        argv_repr = ' '.join(shellquote(a) for a in argv)
        if verbosity is None:
            verbosity = self.verbosity
        if verbosity:
            print(self.term.blue(argv_repr))
        if verbosity > 2:
            print(self.term.yellow(stdin_string.rstrip()))
        PIPE = subprocess.PIPE
        proc = subprocess.Popen(argv, stdout=PIPE, stderr=PIPE, stdin=PIPE,
                                env=env)
        try:
            stdout, stderr = proc.communicate(stdin_string.encode('utf-8'),
                                              timeout=timeout)
            timeout_expired = False
        except subprocess.TimeoutExpired:
            proc.kill()
            stdout = stderr = b''
            timeout_expired = True
        stdout = stdout.decode('utf-8')
        stderr = stderr.decode('utf-8')
        returncode = proc.returncode
        failed = any([
            timeout_expired,
            (check_stdout is not None and check_stdout != stdout),
            (check_stderr is not None and check_stderr != stderr),
            (check_returncode is not None and check_returncode != returncode),
        ])
        if failed and not verbosity:
            print(self.term.blue(argv_repr))
        if failed or verbosity >= 2:
            if stdout:
                print(stdout.rstrip())
            if stderr:
                print(self.term.yellow(stderr.rstrip()))
            print('→ %s' % self.term.blue(str(proc.returncode)))
        if failed:
            if timeout_expired:
                self.die('Command timeout expired')
            elif fail_message:
                self.die(fail_message)
            else:
                self.die('Command failed')
        return SubprocessResult(stdout, stderr, returncode)

@Context.command('sample-config')
def sample_config_command(ctx):
    print(ctx.term.cyan('Copy the following to %s, and modify to taste:' %
                        ctx.options['--config']),
          file=sys.stderr)
    print(ctx.term.cyan('---8<---'.ljust(70, '-')), file=sys.stderr)
    print(SAMPLE_CONFIG.strip())
    print(ctx.term.cyan('--->8---'.rjust(70, '-')), file=sys.stderr)

def ensure_clean_repo(ctx):
    """Make sure the working tree matches the git index"""
    ctx.runprocess(['git', 'status', '--porcelain'],
                   check_stdout='',
                   check_stderr='',
                   fail_message='Repository %s not clean' % os.getcwd())

def get_rewiewers(ctx, tickets):
    """Get name & address of reviewers, or empty list for --no-reviewer

    Raises if a suitable reviewer is not found
    """
    if ctx.options['--no-reviewer']:
        return []
    reviewers = ctx.options['--reviewer']
    if ctx.trac and not reviewers:
        reviewers = set()
        for ticket in tickets:
            if ticket.attributes.get('reviewer'):
                reviewers.add(ticket.attributes['reviewer'])
        if len(reviewers) > 1:
            print('Reviewers found: %s' % ', '.join(reviewers))
            ctx.die('Too many reviewers found in ticket(s), '
                        'specify --reviewer explicitly')
        if not reviewers:
            ctx.die('No reviewer found in ticket(s), '
                        'specify --reviewer explicitly')
        username_map = ctx.config.get('trac-username-map', {})
        reviewers = [username_map.get(r, r) for r in reviewers]
    if not reviewers:
        ctx.die('No reviewer found, please specify --reviewer')
    return [normalize_reviewer(ctx, r) for r in reviewers]

def normalize_reviewer(ctx, reviewer):
    """Expand a partial reviewer name to a full name + address

    Uses the list of contributors from git's mailmap
    """
    name_re = re.compile(r'^\w+ [^<]+ <.*@.*\..*>$')
    if name_re.match(reviewer):
        return reviewer
    rbranch = '%s/master' % ctx.config['remote']
    cmd = ['git',
           '-c', 'mailmap.blob=origin/master:.mailmap',
           'shortlog', '-sen', rbranch]
    names = ctx.runprocess(cmd).stdout.splitlines()
    names = (name.split('\t', 1)[-1] for name in names)
    names = (name for name in names if name_re.match(name))
    names = [name for name in names if reviewer.lower() in name.lower()]
    if not names:
        ctx.die('Reviewer %s not found' % reviewer)
    elif len(names) > 1:
        print(ctx.term.red('Reviewer %s could be:' % reviewer))
        for name in names:
            print('- %s' % name)
        ctx.die('Multiple matches found for reviewer')
    else:
        name = unidecode.unidecode(names[0])
        return name

def apply_patches(ctx, patches, branch):
    """Apply patches to the given branch

    Checks out the branch
    """
    ctx.runprocess(['git', 'checkout',
                    '%s/%s' % (ctx.config['remote'], branch)])
    for patch in patches:
        print('Aplying to %s: %s' % (branch, patch.subject))
        ctx.runprocess(['git', 'am'], stdin_string=''.join(patch.lines))
    sha1 = ctx.runprocess(['git', 'rev-parse', 'HEAD']).stdout.strip()
    if ctx.verbosity:
        print('Resulting hash: %s' % sha1)
    return sha1

def print_push_info(ctx, patches, sha1s, ticket_numbers, tickets):
    """Print lots of info about the to-be-pushed commits"""
    remote = ctx.config['remote']
    branches = sha1s.keys()

    trac_log = []
    bugzilla_log = ['Fixed upstream']
    for branch in branches:
        trac_log.append('%s:' % branch)
        bugzilla_log.append('%s:' % branch)
        log_result = ctx.runprocess(
            ['git', 'log', '--graph', '--oneline', '--abbrev=99',
             '--color=%s' % ctx.color_arg,
                '%s/%s..%s' % (remote, branch, sha1s[branch])])
        trac_log.extend(
            line.rstrip()
            for line in reversed(log_result.stdout.splitlines()))

        log_result = ctx.runprocess(
            ['git', 'log', '--pretty=format:%H',
             '%s/%s..%s' % (remote, branch, sha1s[branch])])
        bugzilla_log.extend(
            ctx.config['commit-url'] + line.strip()
            for line in reversed(log_result.stdout.splitlines()))

    bugzilla_urls = []
    bugzilla_re = re.compile('(%s\d+)' %
                                re.escape(ctx.config['bugzilla-bug-url']))
    for ticket in tickets:
        for match in bugzilla_re.finditer(ticket.attributes['rhbz']):
            bugzilla_urls.append(match.group(0))

    for branch in branches:
        print(ctx.term.cyan('=== Diffstat for %s ===' % branch))
        log_result = ctx.runprocess(
            ['git', 'diff', '--stat', '--color=%s' % ctx.color_arg,
             '%s/%s..%s' % (remote, branch, sha1s[branch])],
            verbosity=2)
        print(ctx.term.cyan('=== Log for %s ===' % branch))
        log_result = ctx.runprocess(
            ['git', 'log', '--reverse', '--color=%s' % ctx.color_arg,
             '%s/%s..%s' % (remote, branch, sha1s[branch])],
            verbosity=2)

    print(ctx.term.cyan('=== Patches pushed ==='))
    for patch in patches:
        print(patch.filename)

    print(ctx.term.cyan('=== Mail summary ==='))
    if len(branches) == 1:
        print('Pushed to ', end='')
    else:
        print('Pushed to:')
    for branch in branches:
        print('%s: %s' % (branch, sha1s[branch]))

    print(ctx.term.cyan('=== Trac comment ==='))
    print('\n'.join(trac_log))

    print(ctx.term.cyan('=== Bugzilla comment ==='))
    print('\n'.join(bugzilla_log))

    if ticket_numbers:
        print(ctx.term.cyan('=== Tickets fixed ==='))
        for number in sorted(ticket_numbers):
            print('%s%s' % (ctx.config['ticket-url'], number))

    if bugzilla_urls:
        print(ctx.term.cyan('=== Bugzillas fixed ==='))
        print('\n'.join(bugzilla_urls))

    print(ctx.term.cyan('=== Ready to push ==='))

@Context.command('push')
def push_command(ctx):
    patches = list(ctx.get_patches())
    if not patches:
        ctx.die('No patches to push')

    os.chdir(cleanpath(ctx.config['clean-repo-path']))
    ensure_clean_repo(ctx)

    ticket_numbers = set()
    for patch in patches:
        ticket_numbers.update(patch.ticket_numbers)
    if ctx.trac:
        tickets = [Ticket(ctx.trac, n) for n in ticket_numbers]
    else:
        tickets = []

    reviewers = get_rewiewers(ctx, tickets)
    if reviewers:
        for reviewer in reviewers:
            print('Reviewer: %s' % reviewer)
            for patch in patches:
                patch.add_reviewer(reviewer)
    else:
        print('No reviewer')

    branches = ctx.options['--branch']
    if not branches:
        if not tickets:
            if ctx.trac:
                ctx.die('No branches specified and no tickets found')
            else:
                ctx.die('No branches specified and trac disabled')
        if ctx.verbosity:
            print('Divining branches from tickets: %s' %
                    ', '.join(str(t.number) for t in tickets))
        milestones = set(t.attributes['milestone'] for t in tickets)
        if not milestones:
            ctx.die('No milestones found in tickets')
        elif len(milestones) > 1:
            ctx.die('Tickets belong to disparate milestones; '
                        'fix them in Trac or specify branches explicitly')
        [milestone] = milestones
        for template, templ_branches in MILESTONES.items():
            if re.match(template, milestone):
                branches = templ_branches
                break
        else:
            ctx.die('No branches correspond to `%s`. ' % milestone +
                        'Update MILESTONES in the ipatool script.')
    print('Will apply %s patches to: %s' %
            (len(patches), ', '.join(branches)))

    remote = ctx.config['remote']

    if not ctx.options['--no-fetch']:
        print('Fetching...')
        ctx.runprocess(['git', 'fetch', remote], timeout=60)

    rev_parse = ctx.runprocess(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
    old_branch = rev_parse.stdout.strip()
    if ctx.verbosity:
        print('Old branch: %s' % old_branch)
    try:
        sha1s = collections.OrderedDict()
        for branch in branches:
            sha1s[branch] = apply_patches(ctx, patches, branch)

        push_args = ['%s:%s' % (sha1, branch)
                        for branch, sha1 in sha1s.items()]
        print('Trying push...')
        ctx.runprocess(['git', 'push', '--dry-run', remote] + push_args,
                       timeout=60, verbosity=2)

        print('Generating info...')
        print_push_info(ctx, patches, sha1s, ticket_numbers, tickets)

        if ctx.options['--dry-run']:
            print('Exiting, --dry-run specified')
        else:
            while True:
                print('(k will start `gitk`)')
                branchesrepr = ', '.join(branches)
                response = input('Push to %s? [y/n/k] ' % branchesrepr)
                if response.lower() == 'n':
                    break
                elif response.lower() == 'k':
                    ctx.runprocess(['gitk'] +
                                    branches +
                                    list(sha1s.values()),
                                   timeout=None)
                elif response.lower() == 'y':
                    print('Pushing')
                    ctx.runprocess(['git', 'push', remote] + push_args,
                                   timeout=60, verbosity=2)
                    break

    finally:
        print('Cleaning up')
        ctx.runprocess(['git', 'am', '--abort'], check_returncode=None)
        ctx.runprocess(['git', 'reset', '--hard'], check_returncode=None)
        ctx.runprocess(['git', 'checkout', old_branch], check_returncode=None)
        ctx.runprocess(['git', 'clean', '-fxd'], check_returncode=None)


@Context.command('start-review')
def start_review_command(ctx):
    ticket_numbers = set(ctx.options['--ticket'])
    if ctx.options['PATCH']:
        patches = ctx.get_patches()
    if not ticket_numbers:
        print(ctx.term.yellow('Using patches from %s' % ctx.config['patchdir']))
        patches = ctx.get_patches()
    else:
        patches = ()
    for patch in patches:
        ticket_numbers.update(patch.ticket_numbers)
    ticket_numbers = sorted(ticket_numbers)
    if ctx.verbosity:
        print('Tickets selected: %s' % ticket_numbers)
    ctx.trac_login()
    tickets = [Ticket(ctx.trac, int(n)) for n in ticket_numbers]
    if not tickets:
        ctx.die('No tickets selected')
    if not ctx.trac:
        ctx.die('Cannot work with --no-trac')
    [ticket.attributes for ticket in tickets]

    existing_reviewers = []
    for ticket in tickets:
        print(ctx.term.blue('Ticket #%s' % ticket.number))
        print('- summary:', ticket.attributes['summary'])
        reviewer = ticket.attributes.get('reviewer')
        print('- reviewer:', ctx.term.yellow(str(reviewer or '')) or 'none')
        if reviewer:
            existing_reviewers.append(reviewer)
        print('- cc:', ticket.attributes['cc'])
    if existing_reviewers:
        if ctx.options['--force']:
            print(ctx.term.yellow('Existing reviewer(s) found'))
        else:
            ctx.die("Existing reviewer(s) found; "
                    "won't overwrite without --force")

    if ctx.options['--dry-run']:
        ctx.die('Exiting, --dry-run specified')
    else:
        while True:
            response = input('Start review on these tickets? [y/n] ')
            if response.lower() == 'n':
                return
            elif response.lower() == 'y':
                break

    for ticket in tickets:
        print('Starting review: #%s' % ticket.number)
        old_cc = ticket.attributes.get('cc')
        if old_cc:
            new_cc = '%s, %s' % (old_cc, ctx.config['trac-username'])
        else:
            new_cc = ctx.config['trac-username']
        ctx.trac.ticket.update(ticket.number,
                               'Starting review',
                               {
                                    'reviewer': ctx.config['trac-username'],
                                    'cc': new_cc,
                               },
                               True)
        print(ctx.term.green('OK'))

    if ctx.options['--am']:
        print('Applying patches to worktree...')
        am_patches(ctx, patches)


def am_patches(ctx, patches):
    for patch in patches:
        print('Applying patch:', patch.filename)
        ctx.runprocess(ctx.config['am-command'],
                       stdin_string=''.join(patch.lines),
                       timeout=60, verbosity=2)


@Context.command('am')
def am_command(ctx):
    patches = ctx.get_patches()
    am_patches(ctx, patches)


if __name__ == '__main__':
    Context(docopt.docopt(__doc__)).run()