path: root/Tools/scripts/patchtool.py

#!/usr/local/bin/python
#
# patchtool.py - a tool to automate common operation with patchfiles in the
# FreeBSD Ports Collection.
#
# ----------------------------------------------------------------------------
# "THE BEER-WARE LICENSE" (Revision 42, (c) Poul-Henning Kamp):
# Maxim Sobolev <sobomax@FreeBSD.org> wrote this file.  As long as you retain
# this notice you can do whatever you want with this stuff. If we meet some
# day, and you think this stuff is worth it, you can buy me a beer in return.
#
# Maxim Sobolev
# ----------------------------------------------------------------------------
#
# $FreeBSD$
#
# MAINTAINER= sobomax@FreeBSD.org <- any unapproved commits to this file are
#                    highly discouraged!!!
#

import os, os.path, popen2, sys, getopt, glob, errno, types

# Some global variables used as constants
True = 1
False = 0


# Tweakable global variables. User is able to override any of these by setting
# appropriate environment variable prefixed by `PT_', eg:
# $ export PT_CVS_ID="FooOS"
# $ export PT_DIFF_CMD="/usr/local/bin/mydiff"
# will force script to use "FooOS" as a CVS_ID instead of "FreeBSD" and
# "/usr/local/bin/mydiff" as a command to generate diffs.
class Vars:
    CVS_ID = 'FreeBSD'

    DIFF_ARGS = '-du'
    DIFF_SUFX = '.orig'
    PATCH_PREFIX = 'patch-'
    RCSDIFF_SUFX = ',v'

    CD_CMD = 'cd'
    DIFF_CMD = '/usr/bin/diff'
    MAKE_CMD = '/usr/bin/make'
    PRINTF_CMD = '/usr/bin/printf'
    RCSDIFF_CMD = '/usr/bin/rcsdiff'

    DEFAULT_MAKEFILE = 'Makefile'
    DEV_NULL = '/dev/null'
    ETC_MAKE_CONF = '/etc/make.conf'
    
    SLASH_REPL_SYMBOL = '::'    # The sysmbol to replace '/' when auto-generating
                            # patchnames


#
# Check if the supplied patch refers to a port's directory.
#
def isportdir(path, soft = False):
    REQ_FILES = ('Makefile', 'pkg-comment', 'pkg-descr', 'pkg-plist', \
      'distinfo')
    if not os.path.isdir(path) and soft != True:
        raise IOError(errno.ENOENT, path)
        # Not reached #

    try:
        content = os.listdir(path)
    except OSError:
        return False

    for file in REQ_FILES:
        if file not in content:
            return False
    return True


#
# Traverse directory tree up from the path pointed by argument and return if
# root directory of a port is found.
#
def locateportdir(path, wrkdirprefix= '', strict = False):
    # Flag to relax error checking in isportdir() function. It required when
    # WRKDIRPREFIX is defined.
    softisport = False

    path = os.path.abspath(path)

    if wrkdirprefix != '':
        wrkdirprefix= os.path.abspath(wrkdirprefix)
        commonprefix = os.path.commonprefix((path, wrkdirprefix))
        if commonprefix != wrkdirprefix:
            return ''
        path = path[len(wrkdirprefix):]
        softisport = True

    while path != '/':
        if isportdir(path, softisport) == True:
            return path
        path = os.path.abspath(os.path.join(path, '..'))

    if strict == True:
        raise LocatePDirError(path)
        # Not reached #
    else:
        return ''


#
# Get value of a make(1) variable called varname. Optionally maintain a cache
# for resolved varname:makepath pairs to speed-up operation if the same variable
# from the exactly same file is requested repeatedly (invocation of make(1) is
# very expensive operation...)
#
def querymakevar(varname, path = 'Makefile', strict = False, cache = {}):
    path = os.path.abspath(path)

    if cache.has_key((varname, path)) == 1:
        return cache[(varname, path)]

    origpath = path
    if os.path.isdir(path):
        path = os.path.join(path, Vars.DEFAULT_MAKEFILE)
    if not os.path.isfile(path):
        raise IOError(errno.ENOENT, path)
        # Not reached #

    dir = os.path.dirname(path)
    CMDLINE = '%s %s && %s -f %s -V %s' % (Vars.CD_CMD, dir, Vars.MAKE_CMD, \
      path, varname)
    pipe = popen2.popen3(CMDLINE)
    retval = ''
    for line in pipe[0].readlines():
        retval = retval + line.strip() + ' '
    for fd in pipe:
        fd.close()
    retval = retval[:-1]
    if strict == True and retval.strip() == '':
        raise MakeVarError(path, varname)
        # Not reached #

    cache[(varname, origpath)] = retval
    return retval


#
# Get a path of `path'  relatively to wrksrc. For example:
# path:     /foo/bar
# wrksrc:   /foo/bar/baz/somefile.c
# getrelpath:   baz/somefile.c
# Most of the code here is to handle cases when ../ operation is required to
# reach wrksrc from path, for example:
# path:     /foo/bar
# wrksrc:   /foo/baz/somefile.c
# getrelpath:   ../baz/somefile.c
#
def getrelpath(path, wrksrc):
    path = os.path.abspath(path)
    wrksrc = os.path.abspath(wrksrc) + '/'
    commonpart = os.path.commonprefix((path, wrksrc))
    while commonpart[-1:] != '/':
        commonpart = commonpart[:-1]
    path = path[len(commonpart):]
    wrksrc = wrksrc[len(commonpart):]
    adjust = ''
    while os.path.normpath(os.path.join(wrksrc, adjust)) != '.':
        adjust = os.path.join(adjust, '..')
    relpath = os.path.join(adjust, path)
    return relpath


#
# Generare a diff between saved and current versions of the file pointed by the
# wrksrc+path. Apply heuristics to locate saved version of the file in question
# and if it fails assume that file is new, so /dev/null is to be used as
# original file. Optionally save generated patch into `outfile' instead of
# dumping it to stdout. Generated patches automatically being tagged with
# "FreeBSD" cvs id.
#
def gendiff(path, wrksrc, outfile = ''):
    IDGEN_CMD = '%s "\\n\\$%s\\$\\n\\n"' % (Vars.PRINTF_CMD, Vars.CVS_ID)

    fullpath = os.path.join(wrksrc, path)
    if not os.path.isfile(fullpath):
        raise IOError(errno.ENOENT, fullpath)
        # Not reached #

    cmdline = ''
    if os.path.isfile(fullpath + Vars.DIFF_SUFX):       # Normal diff
        cmdline = '%s %s %s%s %s' % (Vars.DIFF_CMD, Vars.DIFF_ARGS, path, \
          Vars.DIFF_SUFX, path)
    elif os.path.isfile(fullpath + Vars.RCSDIFF_SUFX):  # RCS diff
        cmdline = '%s %s %s' % (Vars.RCSDIFF_CMD, Vars.DIFF_ARGS, path)
    else:                                               # New file
        cmdline = '%s %s %s %s' % (Vars.DIFF_CMD, Vars.DIFF_ARGS, \
          Vars.DEV_NULL, path)

    if outfile != '':
        cmdline = '( %s && %s ) 2>%s' % (IDGEN_CMD, cmdline, Vars.DEV_NULL)

    savedir = os.getcwd()
    os.chdir(wrksrc)
    pipe = popen2.Popen3(cmdline)
    outbuf = pipe.fromchild.readlines()
    for stream in (pipe.fromchild, pipe.tochild):
        stream.close()
    exitval = os.WEXITSTATUS(pipe.wait())
    if exitval == 0:    # No differences were found
        retval = False
        retmsg = 'no differencies found between original and current ' \
              'version of "%s"' % fullpath
    elif exitval == 1:  # Some differences  were  found
        if (outfile != ''):
            open(outfile, 'w').writelines(outbuf)
        else:
            sys.stdout.writelines(outbuf)
        retval = True
        retmsg = ''
    else:               # Error occured
        raise ECmdError('"%s"' % cmdline, \
          'external command returned non-zero error code')
        # Not reached #

    os.chdir(savedir)
    return (retval, retmsg)


#
# Automatically generate a name for a patch based on its path relative to
# wrksrc. Use simple scheme to ensute 1-to-1 mapping between path and
# patchname - replace all '_' with '__' and all '/' with '_'.
# 
def makepatchname(path, patchdir = ''):
    SRS = Vars.SLASH_REPL_SYMBOL
    retval = Vars.PATCH_PREFIX + \
      path.replace(SRS, SRS + SRS).replace('/', SRS)
    retval = os.path.join(patchdir, retval)
    return retval


#
# Write a specified message to stderr.
#
def write_msg(message):
    if type(message) == types.StringType:
        message = message,
    sys.stderr.writelines(message)


#
# Print specified message to stdout and ask user [y/N]?. Optionally allow
# specify default answer, i.e. return value if user typed only <cr>
#
def query_yn(message, default = False):
    while True:
        if default == True:
            yn = 'Y/n'
        elif default == False:
            yn = 'y/N'
        else:
            yn = 'Y/N'

        reply = raw_input('%s [%s]: ' % (message, yn))

        if reply == 'y' or reply == 'Y':
            return True
        elif reply == 'n' or reply == 'N':
            return False
        elif reply == '' and default in (True, False):
            return default
        print 'Wrong answer "%s", please try again' % reply


#
# Print optional message and usage information and exit with specified exit
# code.
#
def usage(code, msg = ''):
    myname = os.path.basename(sys.argv[0])
    write_msg((str(msg), """
Usage: %s [-afi] file ...
       %s -u [-i] [patchfile|patchdir ...]
""" % (myname, myname)))
    sys.exit(code)


#
# Simple custom exception
#
class MyError:
    msg = 'error'

    def __init__(self, file, msg=''):
        self.file = file
        if msg != '':
            self.msg = msg

    def __str__(self):
            return '%s: %s' % (self.file, self.msg)


#
# Error parsing patchfile
#
class PatchError(MyError):
    msg = 'corrupt patchfile, or not patchfile at all'


#
# Error executing external command
#
class ECmdError(MyError):
    pass


#
# Error getting value of makefile variable
#
class MakeVarError(MyError):
    def __init__(self, file, makevar, msg=''):
        self.file = file
        if msg != '':
            self.msg = msg
        else:
            self.msg = 'can\'t get %s value' % makevar


#
# Error locating portdir
#
class LocatePDirError(MyError):
    msg = 'can\'t locate portdir'


class Patch:
    fullpath = ''
    minus3file = ''
    plus3file = ''
    wrksrc = ''
    patchmtime = 0
    targetmtime = 0

    def __init__(self, path, wrksrc):
        MINUS3_DELIM = '--- '
        PLUS3_DELIM = '+++ '

        path = os.path.abspath(path)
        if not os.path.isfile(path):
            raise IOError(errno.ENOENT, path)
            # Not reached #

        self.fullpath = path
        file = open(path)

        for line in file.readlines():
            if self.minus3file == '':
                if line[:len(MINUS3_DELIM)] == MINUS3_DELIM:
                    lineparts = line.split()
                    try:
                        self.minus3file = lineparts[1]
                    except IndexError:
                        raise PatchError(path)
                        # Not reached #
                    continue
            elif line[:len(PLUS3_DELIM)] == PLUS3_DELIM:
                lineparts = line.split()
                try:
                    self.plus3file = lineparts[1]
                except IndexError:
                    raise PatchError(path)
                    # Not reached #
                break

        file.close()

        if self.minus3file == '' or self.plus3file == '':
            raise PatchError(path)
            # Not reached #

        self.wrksrc = os.path.abspath(wrksrc)
        self.patchmtime = os.path.getmtime(self.fullpath)
        plus3file = os.path.join(self.wrksrc, self.plus3file)
        if os.path.isfile(plus3file):
            self.targetmtime = os.path.getmtime(plus3file)
        else:
            self.targetmtime = 0

    def update(self, patch_cookiemtime = 0, ignoremtime = False):
        targetfile = os.path.join(self.wrksrc, self.plus3file)
        if not os.path.isfile(targetfile):
            raise IOError(errno.ENOENT, targetfile)
            # Not reached #

        patchdir = os.path.dirname(self.fullpath)
        if not os.path.isdir(patchdir):
            os.mkdir(patchdir)

        if ignoremtime == True or self.patchmtime == 0 or \
          self.targetmtime == 0 or \
          (self.patchmtime < self.targetmtime and \
          patch_cookiemtime < self.targetmtime):
            retval = gendiff(self.plus3file, self.wrksrc, self.fullpath)
            if retval[0] == True:
                self.patchmtime = os.path.getmtime(self.fullpath)
        else:
            retval = (False, 'patch is already up to date')
        return retval


class NewPatch(Patch):
    def __init__(self, patchdir, wrksrc, relpath):
        self.fullpath = makepatchname(relpath, os.path.abspath(patchdir))
        self.wrksrc = os.path.abspath(wrksrc)
        self.plus3file = relpath
        self.minus3file = relpath
        self.patchmtime = 0
        plus3file = os.path.join(self.wrksrc, self.plus3file)
        if os.path.isfile(plus3file):
            self.targetmtime = os.path.getmtime(plus3file)
        else:
            self.targetmtime = 0


class PatchesCollection:
    patches = {}

    def __init__(self):
        self.patches = {}
        pass

    def adddir(self, patchdir, wrksrc):
        if not os.path.isdir(patchdir):
            raise IOError(errno.ENOENT, patchdir)
            # Not reached #

        for file in glob.glob(os.path.join(patchdir, Vars.PATCH_PREFIX + '*')):
            self.addpatchfile(file, wrksrc)

    def addpatchfile(self, path, wrksrc):
        path = os.path.abspath(path)
        if not self.patches.has_key(path):
            self.addpatchobj(Patch(path, wrksrc))

    def addpatchobj(self, patchobj):
        self.patches[patchobj.fullpath] = patchobj

    def lookupbyname(self, path):
        path = os.path.abspath(path)
        if self.patches.has_key(path):
            return self.patches[path]
        return None

    def lookupbytarget(self, wrksrc, relpath):
        wrksrc = os.path.abspath(wrksrc)
        for patch in self.patches.values():
            if wrksrc == patch.wrksrc and relpath == patch.plus3file:
                return patch
        return None

    def getpatchobjs(self):
        return self.patches.values()


#
# Resolve all symbolic links in the given path to a file
#
def truepath(path):
    if not os.path.isfile(path):
        raise IOError(errno.ENOENT, path)

    result = ''
    while len(path) > 0:
        path, lastcomp = os.path.split(path)
        if len(lastcomp) == 0:
            lastcomp = path
            path = ''
        result = os.path.join(lastcomp, result)
        if len(path) == 0:
            break
        if os.path.islink(path):
            linkto = os.path.normpath(os.readlink(path))
            if linkto[0] != '/':
                path = os.path.join(path, linkto)
            else:
                path = linkto
    return result[:-1]


def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'afui')
    except getopt.GetoptError, msg:
        usage(2, msg)

    automatic = False
    force = False
    mode = generate
    ignoremtime = False

    for o, a in opts:
        if o == '-a':
            automatic = True
        elif o == '-f':
            force = True
        elif o == '-u':
            mode = update
        elif o == '-i':
            ignoremtime = True
        else:
            usage(2)

    # Allow user to override internal constants
    for varname in dir(Vars):
        if varname[:2] == '__' and varname[-2:] == '__':
            continue
        try:
            value = os.environ['PT_' + varname]
            setattr(Vars, varname, value)
        except KeyError:
            pass

    mode(args, automatic, force, ignoremtime)

    sys.exit(0)


#
# Display a diff or generate patchfile for the files pointed out by args.
#
def generate(args, automatic, force, ignoremtime):
    if len(args) == 0:
        usage(2, "ERROR: no input files specified")

    patches = PatchesCollection()

    for filepath in args:
        if not os.path.isfile(filepath):
            raise IOError(errno.ENOENT, filepath)
            # Not reached #

        filepath = truepath(filepath)

        wrkdirprefix = querymakevar('WRKDIRPREFIX', Vars.ETC_MAKE_CONF, False)
        portdir = locateportdir(os.path.dirname(filepath), wrkdirprefix, True)
        wrksrc = querymakevar('WRKSRC', portdir, True)

        relpath = getrelpath(filepath, wrksrc)

        if automatic == True:
            patchdir = querymakevar('PATCHDIR', portdir, True)

            if os.path.isdir(patchdir):
                patches.adddir(patchdir, wrksrc)

            patchobj = patches.lookupbytarget(wrksrc, relpath)
            if patchobj == None:
                patchobj = NewPatch(patchdir, wrksrc, relpath)
                patches.addpatchobj(patchobj)

            if not force and os.path.exists(patchobj.fullpath) and \
              os.path.getsize(patchobj.fullpath) > 0:
                try:
                    retval = query_yn('Target patchfile "%s" already ' \
                      'exists, do you want to  replace it?' % \
                      os.path.basename(patchobj.fullpath))
                except KeyboardInterrupt:
                    sys.exit('\nAction aborted')
                    # Not reached #
                if retval == False:
                    continue

            write_msg('Generating patchfile: %s...' % \
              os.path.basename(patchobj.fullpath))

            try:
                retval = None
                retval = patchobj.update(ignoremtime = ignoremtime)
            finally:
                # Following tricky magic intended to let us append \n even if
                # we are going to die due to unhandled exception
                if retval == None:
                    write_msg('OUCH!\n')

            if retval[0] == False:
                write_msg('skipped (%s)\n' % retval[1])
            else:
                write_msg('ok\n')

        else:   # automatic != True
            retval = gendiff(relpath, wrksrc)
            if retval[0] == False:
                write_msg('WARNING: %s\n' % retval[1])


#
# Atomatically update all patches pointed by args (may be individual
# patchfiles, patchdirs or any directories in a portdirs). If directory argument
# is encountered, all patches that belong to the port are updated. If no
# arguments are supplied - current directory is assumed.
#
# The procedure homours last modification times of the patchfile, file from
# which diff to be generated and `EXTRACT_COOKIE' file (usually
# ${WRKDIR}/.extract_cookie) to update only those patches that are really need
# to be updated.
#
def update(args, automatic, force, ignoremtime):
    if len(args) == 0:
        args = './',

    for path in args:
        if not os.path.exists(path):
            raise IOError(errno.ENOENT, path)
            # Not reached #

        patches = PatchesCollection()

        if os.path.isdir(path):
            for wrkdirprefix in (querymakevar('WRKDIRPREFIX', \
              Vars.ETC_MAKE_CONF, False), ''):
                portdir = locateportdir(path, wrkdirprefix, False)
                if portdir != '':
                    break
            if portdir == '':
                raise LocatePDirError(os.path.abspath(path))
                # Not reached #

            wrksrc = querymakevar('WRKSRC', portdir, True)
            patchdir = querymakevar('PATCHDIR', portdir, True)

            if os.path.isdir(patchdir):
                patches.adddir(patchdir, wrksrc)
            else:
                continue

        elif os.path.isfile(path):
            portdir = locateportdir(os.path.dirname(path), '' , True)
            wrksrc = querymakevar('WRKSRC', portdir, True)
            patches.addpatchfile(path, wrksrc)

        patch_cookie = querymakevar('PATCH_COOKIE', portdir, True)
        if os.path.isfile(patch_cookie):
            patch_cookiemtime = os.path.getmtime(patch_cookie)
        else:
            patch_cookiemtime = 0

        for patchobj in patches.getpatchobjs():
            write_msg('Updating patchfile: %s...' % \
              os.path.basename(patchobj.fullpath))

            try:
                retval = None
                retval = patchobj.update(patch_cookiemtime, \
                  ignoremtime)
            finally:
                if retval == None:
                    write_msg('OUCH!\n')

            if retval[0] == False:
                write_msg('skipped (%s)\n' % retval[1])
            else:
                write_msg('ok\n')


if __name__ == '__main__':
    try:
        main()
    except (PatchError, ECmdError, MakeVarError, LocatePDirError), msg:
        sys.exit('ERROR: ' + str(msg))
    except IOError, (code, msg):
        sys.exit('ERROR: %s: %s' % (str(msg), os.strerror(code)))