From 31fb70c70d5500435a538d2c8404d9e81b32a0a6 Mon Sep 17 00:00:00 2001 From: Satoshi Asami Date: Tue, 29 Aug 2000 08:51:57 +0000 Subject: Exit with error status in ${portbuild}/status when an error occurs. Make INDEX build failure a fatal error. Copy INDEX to INDEX-${branch} for later perusal. There is no -a flag in OpenSSH's scp. Use sed to remove /usr/ports and stuff from restricted.sh and cdrom.sh, so they can be run without further modification using relative directories. Take more options, namely -norestr (don't build RESTRICTED ports), -noplistcheck (turn off "extra files check"), and -nodummy (use open-motif instead of Motif-dummy and XFree86-4-* instead of XFree86-3). -nodummy will become the default soon. Also a new flag -nocvs will prevent cvs update on ${branch}/src only (while -nocvsup will prevent cvsup and cvs update on usr/ports). -nocvs is to be used when builds for more than one branch shares a same cvsup session. Write out time cvsup is done so it can be used by log processing scripts. Handle dummy ports by using a more generic approach -- list the directories they reside in (e.g., x11-toolkits/Motif-dummy) and generate the package name from there, instead of hardcoding them package names everywhere. Hopefully this will go away when we get rid of the special case handling for Motif and XFree86. Add something called "quickports" -- these are packages with very long dependency lists that always seem to throttle the parallel build near the end. By spelling them out explicitly, try to make make build dependencies for them earlier while there are a lot of other stuff to do. (Our make doesn't do anything with the target list, but maybe I can fix that someday.) Copy bindist.tar to clients before the second run of package build too. It can be useful to "sneak" in fixes if something minorly wrong is discovered in the bindist during the first run. Since there is a md5 checking, this copying will not be done unless the files are actually different. When -norestr is set, delete restricted packages and distfiles using restricted.sh immediately after completion of package build. Generate CHECKSUM.MD5 after packages are built. Copy distfiles over to the ftp site after everything is done. This is run in the background on bak/distfiles so the next package build can immediately start. --- Tools/portbuild/scripts/dopackages | 215 +++++++++++++++++++++++++++++-------- 1 file changed, 170 insertions(+), 45 deletions(-) (limited to 'Tools/portbuild') diff --git a/Tools/portbuild/scripts/dopackages b/Tools/portbuild/scripts/dopackages index 32acc281683d..bcc4a5f4d7b2 100755 --- a/Tools/portbuild/scripts/dopackages +++ b/Tools/portbuild/scripts/dopackages @@ -7,11 +7,19 @@ user=asami # packages for dependencies only dummyports="x11/XFree86 x11-toolkits/Motif-dummy" -date=$(date '+%Y%m%d') +# packages with very long dependency lists -- try to start building these first +quickports="x11/kde11 x11/kde11-i18n x11/gnome" + +status=${pb}/status + +errorexit () { + echo "$1" > ${status} + exit $1 +} usage () { - echo "usage: [-nobuild] [-noindex] [-noduds] [-nocvsup] branch" - exit 1 + echo "usage: [-nobuild] [-noindex] [-noduds] [-nocvsup] [-nocvs] [-norestr] [-noplistcheck] [-nodummy] branch date" + errorexit 1 } # usage: makeindex pb scripts branch user @@ -26,18 +34,11 @@ makeindex () { echo "generating index" echo "================================================" echo "index generation started at $(date)" - ${scripts}/makeindex ${branch} || exit 1 + ${scripts}/makeindex ${branch} || errorexit 1 echo "index generation ended at $(date)" - if grep -q non-existent INDEX; then - echo "errors in INDEX:" - grep non-existent INDEX - exit 1 -# grep -C non-existent INDEX -# grep -v non-existent INDEX > INDEX.tmp -# mv -f INDEX.tmp INDEX - fi echo $(wc -l INDEX.tmp +# mv -f INDEX.tmp INDEX + fi if ! awk -F '|' '{if (NF != 10) { error=1; printf("line %d: %s\n", NR, $0)}} END {if (error == 1) exit(1)}' INDEX; then echo "error in INDEX" - exit 1 + errorexit 1 fi } @@ -64,7 +73,7 @@ makeduds () { echo "duds generation started at $(date)" if ! ${scripts}/makeduds ${branch}; then echo "error(s) detected, exiting script at $(date)" - exit 1 + errorexit 1 fi echo "duds generation ended at $(date)" echo $(wc -l < ${pb}/${branch}/duds) "items in duds" @@ -83,7 +92,8 @@ setupnode () { md5=$6 echo "setting up of $node started at $(date)" - scp -p -a ${scripts}/setupnode ${node}:${scripts} + ssh -n ${node} mkdir -p ${scripts} + scp -p ${scripts}/setupnode ${node}:${scripts} ssh -n ${node} ${scripts}/setupnode ${me} ${pb} ${branch} ${md5} echo "setting up of $node ended at $(date)" } @@ -99,7 +109,10 @@ restrictedlist () { echo "creating restricted list" echo "================================================" echo "restricted list generation started at $(date)" - make ECHO_MSG=/usr/bin/true clean-restricted-list > ${pb}/${branch}/restricted.sh + make ECHO_MSG=/usr/bin/true clean-restricted-list \ + | sed -e "s./usr/ports/distfiles/./distfiles/.g" \ + -e "s./usr/ports/./${branch}/.g" \ + > ${pb}/${branch}/restricted.sh echo "restricted list generation ended at $(date)" echo $(grep -c '^#' ${pb}/${branch}/restricted.sh) "ports in ${pb}/${branch}/restricted.sh" } @@ -115,7 +128,10 @@ cdromlist () { echo "creating cdrom list" echo "================================================" echo "cdrom list generation started at $(date)" - make ECHO_MSG=/usr/bin/true clean-for-cdrom-list > ${pb}/${branch}/cdrom.sh + make ECHO_MSG=/usr/bin/true clean-for-cdrom-list \ + | sed -e "s./usr/ports/distfiles/./distfiles/.g" \ + -e "s./usr/ports/./${branch}/.g" \ + > ${pb}/${branch}/cdrom.sh echo "cdrom list generation ended at $(date)" echo $(grep -c '^#' ${pb}/${branch}/cdrom.sh) "ports in ${pb}/${branch}/cdrom.sh" } @@ -142,9 +158,13 @@ nobuild=0 noindex=0 noduds=0 nocvsup=0 +nocvs=0 +norestr=0 +noplistcheck=0 +nodummy=0 # optional arguments -while [ $# -gt 1 ]; do +while [ $# -gt 2 ]; do case "x$1" in x-nobuild) nobuild=1 @@ -158,6 +178,18 @@ while [ $# -gt 1 ]; do x-nocvsup) nocvsup=1 ;; + x-nocvs) + nocvs=1 + ;; + x-norestr) + norestr=1 + ;; + x-noplistcheck) + noplistcheck=1 + ;; + x-nodummy) + nodummy=1 + ;; *) usage ;; @@ -165,18 +197,34 @@ while [ $# -gt 1 ]; do shift done -# mandatory argument +# mandatory arguments branch=$1 -if [ "x$branch" != x3 -a "x$branch" != x4 ]; then +date=$2 +if [ "x$branch" != x3 -a "x$branch" != x4 -a "x$branch" != x5 ]; then usage fi +if [ "$norestr" = 1 ]; then + export NO_RESTRICTED=t +fi + +if [ "$noplistcheck" = 1 ]; then + export NOPLISTCHECK=t +fi + +if [ "$nodummy" = 1 ]; then + export NODUMMY=t + export MOTIF_OPEN=t + export XFREE86_VERSION=4 +fi + cd ${pb}/usr/ports if [ "$nocvsup" = 0 ]; then echo "================================================" echo "running cvsup" echo "================================================" su ${user} -c 'cvsup -g -L 0 /etc/supfile.cvsup' + date > ${pb}/cvsdone echo "================================================" echo "running cvs update on /usr/ports" echo "================================================" @@ -192,7 +240,15 @@ if [ "$nocvsup" = 0 ]; then su ${user} -c 'cvs -q update -d -P' fi -# these two not in background to check return status +if [ "$nocvs" = 0 ]; then + echo "================================================" + echo "running cvs update on /${branch}/src" + echo "================================================" + cd ${pb}/${branch}/src + su ${user} -c 'cvs -q update -d -P' +fi + +# this one not in background to check return status if [ "$noduds" = 0 ]; then makeduds ${pb} ${scripts} ${branch} @@ -223,38 +279,71 @@ checkindex ${pb} echo "all preparation ended at $(date)" cd ${pb}/usr/ports -make parallel > ../../${branch}/Makefile +make parallel branch=${branch} > ../../${branch}/Makefile + +cd ${pb}/usr/ports + +if [ "$nodummy" = 0 ]; then + dummypkgs="" + for i in ${dummyports}; do + if [ -d $i ]; then + dummypkgs="${dummypkgs} $(cd $i; make package-name).tgz" + else + echo "dummy port directory \"$i\" does not exist -- skipping" + fi + done +fi + +quickpkgs="" +for i in ${quickports}; do + if [ -d $i ]; then + quickpkgs="${quickpkgs} $(cd $i; make package-name).tgz" + else + echo "quick port directory \"$i\" does not exist -- skipping" + fi +done cd ${pb} if [ "$nobuild" = 0 ]; then - rm -rf bak/distfiles - mv -f distfiles bak +### rm -rf bak/distfiles +### mv -f distfiles bak mkdir distfiles chown -R ${user} distfiles cd ${pb}/${branch}/bak rm -rf errors logs packages old-errors cd ${pb}/${branch} mv -f errors logs packages old-errors make.* bak + cd bak/packages/All + for i in *.tgz; do + cp /dev/null $i + done + cd ${pb}/${branch} mkdir -p packages/All mkdir -p ${pb}/archive/errorlogs - rm -rf ${pb}/archive/errorlogs/errors.${branch}.${date} - mkdir -p ${pb}/archive/errorlogs/errors.${branch}.${date} - ln -sf ${pb}/archive/errorlogs/errors.${branch}.${date} ${pb}/${branch}/errors - rm -rf ${pb}/archive/errorlogs/alllogs.${branch}.${date} - mkdir -p ${pb}/archive/errorlogs/alllogs.${branch}.${date} - ln -sf ${pb}/archive/errorlogs/alllogs.${branch}.${date} ${pb}/${branch}/logs - chown -RL ${user} errors logs packages - echo "error logs in ${pb}/archive/errorlogs/errors.${branch}.${date}" - for dir in ${dummyports}; do - cp -p tarballs/$(cd ${pb}/usr/ports/$dir; make package-name).tgz packages/All - done - count=$(awk '{sum+=$2}END{print sum+NR/2}' ${pb}/mlist) + rm -rf ${pb}/archive/errorlogs/e.${branch}.${date} + mkdir -p ${pb}/archive/errorlogs/e.${branch}.${date} + ln -sf ${pb}/archive/errorlogs/e.${branch}.${date} ${pb}/${branch}/errors + rm -rf ${pb}/archive/errorlogs/a.${branch}.${date} + mkdir -p ${pb}/archive/errorlogs/a.${branch}.${date} + ln -sf ${pb}/archive/errorlogs/a.${branch}.${date} ${pb}/${branch}/logs + cp -p ${pb}/cvsdone ${pb}/archive/errorlogs/e.${branch}.${date}/cvsdone + cp -p ${pb}/cvsdone ${pb}/archive/errorlogs/a.${branch}.${date}/cvsdone + cp -p ${pb}/usr/ports/INDEX-${branch} ${pb}/archive/errorlogs/e.${branch}.${date}/INDEX + cp -p ${pb}/usr/ports/INDEX-${branch} ${pb}/archive/errorlogs/a.${branch}.${date}/INDEX + chown -RL ${user} errors logs packages + echo "error logs in ${pb}/archive/errorlogs/e.${branch}.${date}" + if [ "$nodummy" = 0 ]; then + for dir in ${dummyports}; do + cp -p tarballs/$(cd ${pb}/usr/ports/$dir; make package-name).tgz packages/All + done + fi + count=$(awk '{sum+=$2}END{print sum+NR/2}' ${pb}/mlist | sed -e 's/\..*$//') cd ${pb}/${branch}/packages/All echo "================================================" echo "building packages (phase 1)" echo "================================================" echo "started at $(date)" - make -k -j$count -f ../../Makefile > ../../make.0 2>&1 ../../make.0 2>&1 ../../make.1 2>&1 ../../make.1 2>&1 \ + if [ "$nodummy" = 0 ]; then + rm ${dummypkgs} + fi + ${pb}/scripts/chopindex ${pb}/usr/ports/INDEX-${branch} ${pb}/${branch}/packages > \ ${pb}/${branch}/packages/INDEX echo $(ls -1 ${pb}/${branch}/packages/All | wc -l) "packages built" echo $(cat ${pb}/${branch}/packages/INDEX | wc -l) "lines in INDEX" - md5 *.tgz > CHECKSUM.MD5 echo $(echo $(du -sk ${pb}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages" echo $(echo $(du -sk ${pb}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles" + if [ "$norestr" = 0 ]; then + echo "deleting restricted ports" + sh ${pb}/${branch}/restricted.sh + ${pb}/scripts/chopindex ${pb}/usr/ports/INDEX ${pb}/${branch}/packages > \ + ${pb}/${branch}/packages/INDEX + echo $(ls -1 ${pb}/${branch}/packages/All | wc -l) "packages built" + echo $(cat ${pb}/${branch}/packages/INDEX | wc -l) "lines in INDEX" + echo $(echo $(du -sk ${pb}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages" + echo $(echo $(du -sk ${pb}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles" + fi + echo "generating CHECKSUM.MD5" + md5 *.tgz > CHECKSUM.MD5 cd ${pb}/${branch} if grep -q 'ptimeout: killing' make.1; then echo "The following port(s) timed out:" @@ -302,7 +418,7 @@ if [ "$nobuild" = 0 ]; then echo "================================================" cd ${pb}/${branch}/errors for i in *.log; do - if [ ! -f ../bak/errors/$i ]; then + if [ ! -f ${pb}/${branch}/bak/errors/$i ]; then echo -n " $(basename $i .log)" fi done @@ -312,7 +428,7 @@ if [ "$nobuild" = 0 ]; then echo "================================================" cd ${pb}/${branch}/bak/packages/All for i in *.tgz; do - if [ ! -f ../../../packages/All/$i ]; then + if [ ! -f ${pb}/${branch}/packages/All/$i ]; then echo -n " $(basename $i .tgz)" fi done @@ -322,7 +438,7 @@ if [ "$nobuild" = 0 ]; then echo "================================================" cd ${pb}/${branch}/bak/errors for i in *.log; do - if [ ! -f ../../errors/$i ]; then + if [ ! -f ${pb}/${branch}/errors/$i ]; then echo -n " $(basename $i .log)" fi done @@ -332,7 +448,7 @@ if [ "$nobuild" = 0 ]; then echo "================================================" cd ${pb}/${branch}/packages/All for i in *.tgz; do - if [ ! -f ../../bak/packages/All/$i ]; then + if [ ! -f ${pb}/${branch}/bak/packages/All/$i ]; then echo -n " $(basename $i .tgz)" fi done @@ -343,6 +459,15 @@ if [ "$nobuild" = 0 ]; then echo "started at $(date)" cd ${pb}/usr tar --exclude CVS -czf ${pb}/${branch}/tarballs/ports.tar.gz ports + echo "ended at $(date)" + echo "================================================" + echo "copying distfiles" + echo "================================================" + echo "started at $(date)" + cd ${pb} + rm -rf bak/distfiles + mv -f distfiles bak + su ${user} -c ${scripts}/cpdistfiles > ${pb}/cpdistfiles.log 2>&1