diff options
| -rw-r--r-- | Tools/scripts/README.patchtool | 136 | ||||
| -rwxr-xr-x | Tools/scripts/patchtool.py | 639 |
2 files changed, 775 insertions, 0 deletions
diff --git a/Tools/scripts/README.patchtool b/Tools/scripts/README.patchtool new file mode 100644 index 000000000000..20388c2fde23 --- /dev/null +++ b/Tools/scripts/README.patchtool @@ -0,0 +1,136 @@ +INTRODUCTION +------------ +Patchtool is a tool aimed to simplify common operations with patchfiles. It +was designed using real world's experience with the subject and expected to be +very handy for an active porter. + + +MODES OF OPERATION +------------------ +The tool has the following two basic modes of operation: +o generation/update of patchfiles. In this mode user provides list of working + files for which patchfiles are to be generated and the tool generates or + updates corresponding patches automatically guessing if there is an original + version (.orig file, rcs(1)) or it is a completely new file; + +o automatic update of the existing patchfiles. In this mode user provides a + list of patchfiles or directories containing patchfiles and the tool + re-generates specified patches. In this mode the tool tries hard to guess + whether the patchfile in question is already up to date or not and updates + only those patchfiles, which are found to be outdated. + +The following options are supported: + -a -- automatically save resulting diff into a patchfile; + -f -- don't ask any question if target patchfile already exists and is to be + replaced; + -u -- run tool in the "update" mode (see above); + -i -- perform requiested operation (generate or update) even if the + patchfile seems to be up-to-date based on last modification time of + all files involved. + +The tool supports dozen environment variables that can be used to override +default settings. You can find complete list at the top of patchtool.py, +following are the most useful ones: +PT_CVS_ID -- CVS id to be added at the top of each patchfile generated + ("FreeBSD"); +PT_DIFF_ARGS -- diff(1) args used to generate patchfile ("-du"); +PT_DIFF_SUFX -- suffix used to save original version of the file (".orig"); + + +KNOWN BUGS AND LIMITATIONS +-------------------------- +o It is assumed that each patchfile contains exactly one diff, so the tool + may remove useful diffs when there are several diffs merged into one + patchfile. Actually I don't think that it is a bug, because Porter's + Handbook clearly demands to follow a "one diff - one patchfile" rule. + Perhaps portlint(1) should be extended to warn about such abuses; + +o only '+++' supported as the prefix for the name of target file in the + patchfile. Neither '***' nor 'Index:' are not recognised/supported yet; + +o please keep in mind that when you are trying to do automatic update and + some of the patches are for auto-generated/mangled files (e.g. configure + script in the situation when USE_LIBTOOL is used) then you would end up + with some of patchfiles containing bogus hunks; + +o by default the tool tries to recognise saved original version on the file + first by probing file with '.orig' suffix added and if it fails then by + probing file with ',v' suffix added. If you use other suffix to save + original version, for example '~', then set PT_DIFF_SUFX environment + variable to match your settings, otherwise the tool will not function + properly. + + +REPORTING BUGS AND PROPOSING ENHANCEMENTS +----------------------------------------- +The author of the tool is Maxim Sobolev <sobomax@FreeBSD.org>. Any bug +reports, patches, proposals or suggestions are encouraged. You can do it +either contacting author directly at the e-mail above or submitting a FreeBSD +problem report. + + +EXAMPLES +-------- +Following are several sample sessions which show common usage patterns for +this tool. + +1. Generation mode (usefull when creating new port). +$ cd /somewhere/foo ; make +[compilation blows with error in src/bar/baz.c] +$ cd work/foo-1.0/src/bar +[digged here and there and finally have found a solution] +$ vi baz.c +[fixing it] +$ patchtool baz.c +[reading diff] +$ patchtool -a baz.c +Generating patchfile: patch-src_bar_baz.c...ok +$ cd ../../../../ +$ make +[works as expected, wow!] +$ make install clean +$ send-pr +[...] + +2. Generation mode when target patchfile already exists (Minor port update). +$ cd /somewhere/foo ; make +[...] +1 out of 4 hunks failed--saving rejects to Makefile.rej +>> Patch patch-aa failed to apply cleanly. +*** Error code 1 +$ cd work/foo-1.0 +[examining Makefile.rej] +$ vi Makefile +[merging changes by hand] +$ patchtool Makefile +[reading diff] +$ patchtool -a Makefile +Target patchfile "patch-aa" already exists, do you want to replace it? [y/N]: y +Generating patchfile: patch-aa...ok +$ cd ../../ +$ make clean +$ make install clean +$ send-pr +[...] + +3. "Gross" update mode (Major update, when several existing patches do not apply + cleanly). +$ cd /somewhere/foo +$ vi Makefile +[increase PORTVERSION] +$ make fetch makesum patch +[several patches are failing to apply cleanly] +$ cd work/foo-1.0 +[doing merging work, finally all changes are merged] +$ cd ../../ ; make all install +[compile and works like a charm] +$ pwd +/somewhere/foo +$ patchtool -u ./ +Updating patchfile: patch-aa +Updating patchfile: patch-as +Updating patchfile: patch-foo.c +Updating patchfile: patch-foo_bar.c +$ make clean +$ send-pr +[...] diff --git a/Tools/scripts/patchtool.py b/Tools/scripts/patchtool.py new file mode 100755 index 000000000000..08f8f528851d --- /dev/null +++ b/Tools/scripts/patchtool.py @@ -0,0 +1,639 @@ +#!/usr/local/bin/python +# +# patchtool.py - a tool to automate common operation with patchfiles in the +# FreeBSD Ports Collection. +# +# ---------------------------------------------------------------------------- +# "THE BEER-WARE LICENSE" (Revision 42, (c) Poul-Henning Kamp): +# Maxim Sobolev <sobomax@FreeBSD.org> wrote this file. As long as you retain +# this notice you can do whatever you want with this stuff. If we meet some +# day, and you think this stuff is worth it, you can buy me a beer in return. +# +# Maxim Sobolev +# ---------------------------------------------------------------------------- +# +# $FreeBSD$ +# +# MAINTAINER= sobomax@FreeBSD.org <- any unapproved commits to this file are +# highly discouraged!!! +# + +import os, os.path, popen2, sys, getopt, glob, errno, types + +# Some global variables used as constants +True = 1 +False = 0 + + +# Tweakable global variables. User is able to override any of these by setting +# appropriate environment variable prefixed by `PT_', eg: +# $ export PT_CVS_ID="FooOS" +# $ export PT_DIFF_CMD="/usr/local/bin/mydiff" +# will force script to use "FooOS" as a CVS_ID instead of "FreeBSD" and +# "/usr/local/bin/mydiff" as a command to generate diffs. +class Vars: + CVS_ID = 'FreeBSD' + + DIFF_ARGS = '-du' + DIFF_SUFX = '.orig' + PATCH_PREFIX = 'patch-' + RCSDIFF_SUFX = ',v' + + CD_CMD = 'cd' + DIFF_CMD = '/usr/bin/diff' + MAKE_CMD = '/usr/bin/make' + PRINTF_CMD = '/usr/bin/printf' + RCSDIFF_CMD = '/usr/bin/rcsdiff' + + DEFAULT_MAKEFILE = 'Makefile' + DEV_NULL = '/dev/null' + ETC_MAKE_CONF = '/etc/make.conf' + + SLASH_REPL_SYMBOL = '_' # The sysmbol to replace '/' when auto-generating + # patchnames + + +# +# Check if the supplied patch refers to a port's directory. +# +def isportdir(path, soft = False): + REQ_FILES = ('Makefile', 'pkg-comment', 'pkg-descr', 'pkg-plist', \ + 'distinfo') + if not os.path.isdir(path) and soft != True: + raise IOError(errno.ENOENT, path) + # Not reached # + + try: + content = os.listdir(path) + except OSError: + return False + + for file in REQ_FILES: + if file not in content: + return False + return True + + +# +# Traverse directory tree up from the path pointed by argument and return if +# root directory of a port is found. +# +def locateportdir(path, wrkdirprefix= '', strict = False): + # Flag to relax error checking in isportdir() function. It required when + # WRKDIRPREFIX is defined. + softisport = False + + path = os.path.abspath(path) + + if wrkdirprefix != '': + wrkdirprefix= os.path.abspath(wrkdirprefix) + commonprefix = os.path.commonprefix((path, wrkdirprefix)) + if commonprefix != wrkdirprefix: + return '' + path = path[len(wrkdirprefix):] + softisport = True + + while path != '/': + if isportdir(path, softisport) == True: + return path + path = os.path.abspath(os.path.join(path, '..')) + + if strict == True: + raise LocatePDirError(path) + # Not reached # + else: + return '' + + +# +# Get value of a make(1) variable called varname. Optionally maintain a cache +# for resolved varname:makepath pairs to speed-up operation if the same variable +# from the exactly same file is requiested repeatedly (invocation of make(1) is +# very expensive operation...) +# +def querymakevar(varname, path = 'Makefile', strict = False, cache = {}): + path = os.path.abspath(path) + + if cache.has_key((varname, path)) == 1: + return cache[(varname, path)] + + origpath = path + if os.path.isdir(path): + path = os.path.join(path, Vars.DEFAULT_MAKEFILE) + if not os.path.isfile(path): + raise IOError(errno.ENOENT, path) + # Not reached # + + dir = os.path.dirname(path) + CMDLINE = '%s %s && %s -f %s -V %s' % (Vars.CD_CMD, dir, Vars.MAKE_CMD, \ + path, varname) + pipe = popen2.popen3(CMDLINE) + retval = '' + for line in pipe[0].readlines(): + retval = retval + line.strip() + ' ' + for fd in pipe: + fd.close() + retval = retval[:-1] + if strict == True and retval.strip() == '': + raise MakeVarError(path, varname) + # Not reached # + + cache[(varname, origpath)] = retval + return retval + + +# +# Get a path of `path' relatively to wrksrc. For example: +# path: /foo/bar +# wrksrc: /foo/bar/baz/somefile.c +# getrelpath: baz/somefile.c +# Most of the code here is to handle cases when ../ operation is required to +# reach wrksrc from path, for example: +# path: /foo/bar +# wrksrc: /foo/baz/somefile.c +# getrelpath: ../baz/somefile.c +# +def getrelpath(path, wrksrc): + path = os.path.abspath(path) + wrksrc = os.path.abspath(wrksrc) + commonpart = os.path.commonprefix((path, wrksrc)) + path = path[len(commonpart):] + wrksrc = wrksrc[len(commonpart):] + if wrksrc == '': + path = path[1:] + adjust = '' + while os.path.normpath(os.path.join(wrksrc, adjust)) != '.': + adjust = os.path.join(adjust, '..') + relpath = os.path.join(adjust, path) + return relpath + + +# +# Generare a diff between saved and current versions of the file pointed by the +# wrksrc+path. Apply heuristics to locate saved version of the file in question +# and if it fails assume that file is new, so /dev/null is to be used as +# original file. Optionally save generated patch into `outfile' instead of +# dumping it to stdout. Generated patches automatically being tagged with +# "FreeBSD" cvs id. +# +def gendiff(path, wrksrc, outfile = ''): + IDGEN_CMD = '%s "\\n\\$%s\\$\\n\\n"' % (Vars.PRINTF_CMD, Vars.CVS_ID) + + fullpath = os.path.join(wrksrc, path) + if not os.path.isfile(fullpath): + raise IOError(errno.ENOENT, fullpath) + # Not reached # + + cmdline = '' + if os.path.isfile(fullpath + Vars.DIFF_SUFX): # Normal diff + cmdline = '%s %s %s%s %s' % (Vars.DIFF_CMD, Vars.DIFF_ARGS, path, \ + Vars.DIFF_SUFX, path) + elif os.path.isfile(fullpath + Vars.RCSDIFF_SUFX): # RCS diff + cmdline = '%s %s %s' % (Vars.RCSDIFF_CMD, Vars.DIFF_ARGS, path) + else: # New file + cmdline = '%s %s %s %s' % (Vars.DIFF_CMD, Vars.DIFF_ARGS, \ + Vars.DEV_NULL, path) + + if outfile != '': + cmdline = '( %s && %s ) 1>%s 2>%s' % (IDGEN_CMD, cmdline, outfile, \ + Vars.DEV_NULL) + savedir = os.getcwd() + os.chdir(wrksrc) + exitstat = os.system(cmdline) + if os.WIFEXITED(exitstat): + exitval = os.WEXITSTATUS(exitstat) + if exitval == 0: # No differences were found + if outfile != '': + os.unlink(outfile) + retval = False + retmsg = 'no differencies found between original and current ' \ + 'version of "%s"' % fullpath + elif exitval == 1: # Some differences were found + retval = True + retmsg = '' + else: # Error occured + raise ECmdError('"%s"' % cmdline, \ + 'external command returned non-zero error code') + # Not reached # + + os.chdir(savedir) + return (retval, retmsg) + + +# +# Automatically generate a name for a patch based on its path relative to +# wrksrc. Use simple scheme to ensute 1-to-1 mapping between path and +# patchname - replace all '_' with '__' and all '/' with '_'. +# +def makepatchname(path, patchdir = ''): + SRS = Vars.SLASH_REPL_SYMBOL + retval = Vars.PATCH_PREFIX + \ + path.replace(SRS, SRS + SRS).replace('/', SRS) + retval = os.path.join(patchdir, retval) + return retval + + +# +# Write a specified message to stderr. +# +def write_msg(message): + if type(message) == types.StringType: + message = message, + sys.stderr.writelines(message) + + +# +# Print specified message to stdout and ask user [y/N]?. Optionally allow +# specify default answer, i.e. return value if user typed only <cr> +# +def query_yn(message, default = False): + while True: + if default == True: + yn = 'Y/n' + elif default == False: + yn = 'y/N' + else: + yn = 'Y/N' + + reply = raw_input('%s [%s]: ' % (message, yn)) + + if reply == 'y' or reply == 'Y': + return True + elif reply == 'n' or reply == 'N': + return False + elif reply == '' and default in (True, False): + return default + print 'Wrong answer "%s", please try again' % reply + + +# +# Print optional message and usage information and exit with specified exit +# code. +# +def usage(code, msg = ''): + myname = os.path.basename(sys.argv[0]) + write_msg((str(msg), """ +Usage: %s [-afi] file ... + %s -u [-i] [patchfile|patchdir ...] +""" % (myname, myname))) + sys.exit(code) + + +# +# Simple custom exception +# +class MyError: + msg = 'error' + + def __init__(self, file, msg=''): + self.file = file + if msg != '': + self.msg = msg + + def __str__(self): + return '%s: %s' % (self.file, self.msg) + + +# +# Error parsing patchfile +# +class PatchError(MyError): + msg = 'corrupt patchfile, or not patchfile at all' + + +# +# Error executing external command +# +class ECmdError(MyError): + pass + + +# +# Error getting value of makefile variable +# +class MakeVarError(MyError): + def __init__(self, file, makevar, msg=''): + self.file = file + if msg != '': + self.msg = msg + else: + self.msg = 'can\'t get %s value' % makevar + + +# +# Error locating portdir +# +class LocatePDirError(MyError): + msg = 'can\'t locate portdir' + + +class Patch: + fullpath = '' + minus3file = '' + plus3file = '' + wrksrc = '' + patchmtime = 0 + targetmtime = 0 + + def __init__(self, path, wrksrc): + MINUS3_DELIM = '--- ' + PLUS3_DELIM = '+++ ' + + path = os.path.abspath(path) + if not os.path.isfile(path): + raise IOError(errno.ENOENT, path) + # Not reached # + + self.fullpath = path + file = open(path) + + for line in file.readlines(): + if self.minus3file == '': + if line[:len(MINUS3_DELIM)] == MINUS3_DELIM: + lineparts = line.split() + try: + self.minus3file = lineparts[1] + except IndexError: + raise PatchError(path) + # Not reached # + continue + elif line[:len(PLUS3_DELIM)] == PLUS3_DELIM: + lineparts = line.split() + try: + self.plus3file = lineparts[1] + except IndexError: + raise PatchError(path) + # Not reached # + break + + file.close() + + if self.minus3file == '' or self.plus3file == '': + raise PatchError(path) + # Not reached # + + self.wrksrc = os.path.abspath(wrksrc) + self.patchmtime = os.path.getmtime(self.fullpath) + plus3file = os.path.join(self.wrksrc, self.plus3file) + if os.path.isfile(plus3file): + self.targetmtime = os.path.getmtime(plus3file) + else: + self.targetmtime = 0 + + def update(self, patch_cookiemtime = 0, ignoremtime = False): + targetfile = os.path.join(self.wrksrc, self.plus3file) + if not os.path.isfile(targetfile): + raise IOError(errno.ENOENT, targetfile) + # Not reached # + + patchdir = os.path.dirname(self.fullpath) + if not os.path.isdir(patchdir): + os.mkdir(patchdir) + + if ignoremtime == True or self.patchmtime == 0 or \ + self.targetmtime == 0 or \ + (self.patchmtime < self.targetmtime and \ + patch_cookiemtime < self.targetmtime): + retval = gendiff(self.plus3file, self.wrksrc, self.fullpath) + if retval[0] == True: + self.patchmtime = os.path.getmtime(self.fullpath) + else: + retval = (False, 'patch is already up to date') + return retval + + +class NewPatch(Patch): + def __init__(self, patchdir, wrksrc, relpath): + self.fullpath = makepatchname(relpath, os.path.abspath(patchdir)) + self.wrksrc = os.path.abspath(wrksrc) + self.plus3file = relpath + self.minus3file = relpath + self.patchmtime = 0 + plus3file = os.path.join(self.wrksrc, self.plus3file) + if os.path.isfile(plus3file): + self.targetmtime = os.path.getmtime(plus3file) + else: + self.targetmtime = 0 + + +class PatchesCollection: + patches = {} + + def __init__(self): + self.patches = {} + pass + + def adddir(self, patchdir, wrksrc): + if not os.path.isdir(patchdir): + raise IOError(errno.ENOENT, patchdir) + # Not reached # + + for file in glob.glob(os.path.join(patchdir, Vars.PATCH_PREFIX + '*')): + self.addpatchfile(file, wrksrc) + + def addpatchfile(self, path, wrksrc): + path = os.path.abspath(path) + if not self.patches.has_key(path): + self.addpatchobj(Patch(path, wrksrc)) + + def addpatchobj(self, patchobj): + self.patches[patchobj.fullpath] = patchobj + + def lookupbyname(self, path): + path = os.path.abspath(path) + if self.patches.has_key(path): + return self.patches[path] + return None + + def lookupbytarget(self, wrksrc, relpath): + wrksrc = os.path.abspath(wrksrc) + for patch in self.patches.values(): + if wrksrc == patch.wrksrc and relpath == patch.plus3file: + return patch + return None + + def getpatchobjs(self): + return self.patches.values() + + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], 'afui') + except getopt.GetoptError, msg: + usage(2, msg) + + automatic = False + force = False + mode = generate + ignoremtime = False + + for o, a in opts: + if o == '-a': + automatic = True + elif o == '-f': + force = True + elif o == '-u': + mode = update + elif o == '-i': + ignoremtime = True + else: + usage(2) + + # Allow user to override internal constants + for varname in dir(Vars): + if varname[:2] == '__' and varname[-2:] == '__': + continue + try: + value = os.environ['PT_' + varname] + setattr(Vars, varname, value) + except KeyError: + pass + + mode(args, automatic, force, ignoremtime) + + sys.exit(0) + + +# +# Display a diff or generate patchfile for the files pointed out by args. +# +def generate(args, automatic, force, ignoremtime): + if len(args) == 0: + usage(2, "ERROR: no input files specified") + + patches = PatchesCollection() + + for filepath in args: + if not os.path.isfile(filepath): + raise IOError(errno.ENOENT, filepath) + # Not reached # + + wrkdirprefix = querymakevar('WRKDIRPREFIX', Vars.ETC_MAKE_CONF, False) + portdir = locateportdir(os.path.dirname(filepath), wrkdirprefix, True) + wrksrc = querymakevar('WRKSRC', portdir, True) + + relpath = getrelpath(filepath, wrksrc) + + if automatic == True: + patchdir = querymakevar('PATCHDIR', portdir, True) + + if os.path.isdir(patchdir): + patches.adddir(patchdir, wrksrc) + + patchobj = patches.lookupbytarget(wrksrc, relpath) + if patchobj == None: + patchobj = NewPatch(patchdir, wrksrc, relpath) + patches.addpatchobj(patchobj) + + if not force and os.path.exists(patchobj.fullpath) and \ + os.path.getsize(patchobj.fullpath) > 0: + try: + retval = query_yn('Target patchfile "%s" already ' \ + 'exists, do you want to replace it?' % \ + os.path.basename(patchobj.fullpath)) + except KeyboardInterrupt: + sys.exit('\nAction aborted') + # Not reached # + if retval == False: + continue + + write_msg('Generating patchfile: %s...' % \ + os.path.basename(patchobj.fullpath)) + + try: + retval = None + retval = patchobj.update(ignoremtime = ignoremtime) + finally: + # Following tricky magic intended to let us append \n even if + # we are going to die due to unhandled exception + if retval == None: + write_msg('OUCH!\n') + + if retval[0] == False: + write_msg('skipped (%s)\n' % retval[1]) + else: + write_msg('ok\n') + + else: # automatic != True + retval = gendiff(relpath, wrksrc) + if retval[0] == False: + write_msg('WARNING: %s\n' % retval[1]) + + +# +# Atomatically update all patches pointed by args (may be individual +# patchfiles, patchdirs or any directories in a portdirs). If directory argument +# is encountered, all patches that belong to the port are updated. If no +# arguments are supplied - current directory is assumed. +# +# The procedure homours last modification times of the patchfile, file from +# which diff to be generated and `EXTRACT_COOKIE' file (usually +# ${WRKDIR}/.extract_cookie) to update only those patches that are really need +# to be updated. +# +def update(args, automatic, force, ignoremtime): + if len(args) == 0: + args = './', + + for path in args: + if not os.path.exists(path): + raise IOError(errno.ENOENT, path) + # Not reached # + + patches = PatchesCollection() + + if os.path.isdir(path): + for wrkdirprefix in (querymakevar('WRKDIRPREFIX', \ + Vars.ETC_MAKE_CONF, False), ''): + portdir = locateportdir(path, wrkdirprefix, False) + if portdir != '': + break + if portdir == '': + raise LocatePDirError(os.path.abspath(path)) + # Not reached # + + wrksrc = querymakevar('WRKSRC', portdir, True) + patchdir = querymakevar('PATCHDIR', portdir, True) + + if os.path.isdir(patchdir): + patches.adddir(patchdir, wrksrc) + else: + continue + + elif os.path.isfile(path): + portdir = locateportdir(os.path.dirname(path), '' , True) + wrksrc = querymakevar('WRKSRC', portdir, True) + patches.addpatchfile(path, wrksrc) + + patch_cookie = querymakevar('PATCH_COOKIE', portdir, True) + if os.path.isfile(patch_cookie): + patch_cookiemtime = os.path.getmtime(patch_cookie) + else: + patch_cookiemtime = 0 + + for patchobj in patches.getpatchobjs(): + write_msg('Updating patchfile: %s...' % \ + os.path.basename(patchobj.fullpath)) + + try: + retval = None + retval = patchobj.update(patch_cookiemtime, \ + ignoremtime) + finally: + if retval == None: + write_msg('OUCH!\n') + + if retval[0] == False: + write_msg('skipped (%s)\n' % retval[1]) + else: + write_msg('ok\n') + + +if __name__ == '__main__': + try: + main() + except (PatchError, ECmdError, MakeVarError, LocatePDirError), msg: + sys.exit('ERROR: ' + str(msg)) + except IOError, (code, msg): + sys.exit('ERROR: %s: %s' % (str(msg), os.strerror(code))) + |