path: root/Tools/portbuild/scripts/dopackages

#!/bin/sh

# configurable variables
pb=/a/asami/portbuild
user=asami

# packages for dependencies only
dummyports="x11/XFree86 x11-toolkits/Motif-dummy"

# packages with very long dependency lists -- try to start building these first
quickports="x11/kde11 x11/kde11-i18n x11/gnome"

status=${pb}/status

errorexit () {
  echo "$1" > ${status}
  exit $1
}

usage () {
  echo "usage: [-nobuild] [-noindex] [-noduds] [-nocvsup] [-nocvs] [-norestr] [-noplistcheck] [-nodummy] branch date"
  errorexit 1
}

# usage: makeindex pb scripts branch user
makeindex () {
  pb=$1
  scripts=$2
  branch=$3
  user=$4

  cd ${pb}/usr/ports
  echo "================================================"
  echo "generating index"
  echo "================================================"
  echo "index generation started at $(date)"
  ${scripts}/makeindex ${branch} || errorexit 1
  echo "index generation ended at $(date)"
  echo $(wc -l <INDEX) "lines in INDEX"
  chown ${user} INDEX
  cp -p INDEX INDEX-${branch}
}

# usage: checkindex pb
checkindex () {
  pb=$1

  cd ${pb}/usr/ports
  if grep -q non-existent INDEX; then
    echo "errors in INDEX:"
    grep -n non-existent INDEX
    errorexit 1
#    grep -C non-existent INDEX
#    grep -v non-existent INDEX > INDEX.tmp
#    mv -f INDEX.tmp INDEX
  fi
  if ! awk -F '|' '{if (NF != 10) { error=1; printf("line %d: %s\n", NR, $0)}} END {if (error == 1) exit(1)}' INDEX; then
    echo "error in INDEX"
    errorexit 1
  fi
}

# usage: makeduds pb scripts branch
makeduds () {
  pb=$1
  scripts=$2
  branch=$3

  cd ${pb}/usr/ports
  echo "================================================"
  echo "generating duds"
  echo "================================================"
  echo "duds generation started at $(date)"
  if ! ${scripts}/makeduds ${branch}; then
    echo "error(s) detected, exiting script at $(date)"
    errorexit 1
  fi
  echo "duds generation ended at $(date)"
  echo $(wc -l < ${pb}/${branch}/duds) "items in duds"
  echo "duds diff:"
  diff ${pb}/${branch}/duds.old ${pb}/${branch}/duds
  cp -p ${pb}/${branch}/duds ${pb}/${branch}/duds.old
}

# usage: setupnode pb scripts branch me node md5
setupnode () {
  pb=$1
  scripts=$2
  branch=$3
  me=$4
  node=$5
  md5=$6

  echo "setting up of $node started at $(date)"
  ssh -n ${node} mkdir -p ${scripts}
  scp -p ${scripts}/setupnode ${node}:${scripts}
  ssh -n ${node} ${scripts}/setupnode ${me} ${pb} ${branch} ${md5}
  echo "setting up of $node ended at $(date)"
}

# usage: restrictedlist pb scripts branch
restrictedlist () {
  pb=$1
  scripts=$2
  branch=$3

  cd ${pb}/usr/ports
  echo "================================================"
  echo "creating restricted list"
  echo "================================================"
  echo "restricted list generation started at $(date)"
  make ECHO_MSG=/usr/bin/true clean-restricted-list \
    | sed -e "s./usr/ports/distfiles/./distfiles/.g" \
          -e "s./usr/ports/./${branch}/.g" \
      > ${pb}/${branch}/restricted.sh
  echo "restricted list generation ended at $(date)"
  echo $(grep -c '^#' ${pb}/${branch}/restricted.sh) "ports in ${pb}/${branch}/restricted.sh"
}
  
# usage: cdromlist pb scripts branch
cdromlist () {
  pb=$1
  scripts=$2
  branch=$3

  cd ${pb}/usr/ports
  echo "================================================"
  echo "creating cdrom list"
  echo "================================================"
  echo "cdrom list generation started at $(date)"
  make ECHO_MSG=/usr/bin/true clean-for-cdrom-list \
    | sed -e "s./usr/ports/distfiles/./distfiles/.g" \
          -e "s./usr/ports/./${branch}/.g" \
      > ${pb}/${branch}/cdrom.sh
  echo "cdrom list generation ended at $(date)"
  echo $(grep -c '^#' ${pb}/${branch}/cdrom.sh) "ports in ${pb}/${branch}/cdrom.sh"
}

scripts=${pb}/scripts

umask 002
export PORTSDIR=${pb}/usr/ports

me=$(hostname -s)

echo "Subject: $me package building logs"
echo
echo "Called with arguments: "${1+"$@"}
echo "Started at $(date)"

PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/X11R6/bin

if [ $# = 0 ]; then
  usage
fi

nobuild=0
noindex=0
noduds=0
nocvsup=0
nocvs=0
norestr=0
noplistcheck=0
nodummy=0

# optional arguments
while [ $# -gt 2 ]; do
  case "x$1" in
    x-nobuild)
      nobuild=1
      ;;
    x-noindex)
      noindex=1
      ;;
    x-noduds)
      noduds=1
      ;;
    x-nocvsup)
      nocvsup=1
      ;;
    x-nocvs)
      nocvs=1
      ;;
    x-norestr)
      norestr=1
      ;;
    x-noplistcheck)
      noplistcheck=1
      ;;
    x-nodummy)
      nodummy=1
      ;;
    *)
      usage
      ;;
  esac
  shift
done

# mandatory arguments
branch=$1
date=$2
if [ "x$branch" != x3 -a "x$branch" != x4 -a "x$branch" != x5 ]; then
  usage
fi

if [ "$norestr" = 1 ]; then
  export NO_RESTRICTED=t
fi

if [ "$noplistcheck" = 1 ]; then
  export NOPLISTCHECK=t
fi

if [ "$nodummy" = 1 ]; then
  export NODUMMY=t
  export MOTIF_OPEN=t
  export XFREE86_VERSION=4
fi

cd ${pb}/usr/ports
if [ "$nocvsup" = 0 ]; then
  echo "================================================"
  echo "running cvsup"
  echo "================================================"
  su ${user} -c 'cvsup -g -L 0 /etc/supfile.cvsup'
  date > ${pb}/cvsdone
  echo "================================================"
  echo "running cvs update on /usr/ports"
  echo "================================================"
  su ${user} -c 'cvs -q update -d -P'
  echo "================================================"
  echo "running make checksubdirs"
  echo "================================================"
  make checksubdirs 
  echo "================================================"
  echo "running cvs update on /usr/opt/doc"
  echo "================================================"
  cd ${pb}/usr/opt/doc
  su ${user} -c 'cvs -q update -d -P'
fi

if [ "$nocvs" = 0 ]; then
  echo "================================================"
  echo "running cvs update on /${branch}/src"
  echo "================================================"
  cd ${pb}/${branch}/src
  su ${user} -c 'cvs -q update -d -P'
fi

# this one not in background to check return status

if [ "$noduds" = 0 ]; then
  makeduds ${pb} ${scripts} ${branch}
fi

if [ "$noindex" = 0 ]; then
  makeindex ${pb} ${scripts} ${branch} ${user} &
fi

md5=$(/sbin/md5 ${pb}/${branch}/tarballs/bindist.tar | awk '{print $4}')
echo "================================================"
echo "setting up nodes"
echo "================================================"
for node in $(awk '{print $1}' ${pb}/mlist); do
  setupnode ${pb} ${scripts} ${branch} ${me} ${node} ${md5} &
  sleep 2
done

restrictedlist ${pb} ${scripts} ${branch} &
sleep 2

cdromlist ${pb} ${scripts} ${branch} &

wait

checkindex ${pb}

echo "all preparation ended at $(date)"

cd ${pb}/usr/ports
make parallel branch=${branch} > ../../${branch}/Makefile

cd ${pb}/usr/ports

if [ "$nodummy" = 0 ]; then
  dummypkgs=""
  for i in ${dummyports}; do
    if [ -d $i ]; then
      dummypkgs="${dummypkgs} $(cd $i; make package-name).tgz"
    else
      echo "dummy port directory \"$i\" does not exist -- skipping"
    fi
  done
fi

quickpkgs=""
for i in ${quickports}; do
  if [ -d $i ]; then
    quickpkgs="${quickpkgs} $(cd $i; make package-name).tgz"
  else
    echo "quick port directory \"$i\" does not exist -- skipping"
  fi
done

cd ${pb}
if [ "$nobuild" = 0 ]; then
###  rm -rf bak/distfiles
###  mv -f distfiles bak
  mkdir -p distfiles
  chown -R ${user} distfiles
  cd ${pb}/${branch}/bak
  rm -rf errors logs packages old-errors
  cd ${pb}/${branch}
  mv -f errors logs packages old-errors make.* bak
###delete old packages -- use when diskspace is scarce
#  if cd bak/packages/All; then
#    for i in *.tgz; do
#      cp /dev/null $i
#    done
#  fi
  cd ${pb}/${branch}
  mkdir -p packages/All
  mkdir -p ${pb}/archive/errorlogs
  rm -rf ${pb}/archive/errorlogs/e.${branch}.${date}
  mkdir -p ${pb}/archive/errorlogs/e.${branch}.${date}
  ln -sf ${pb}/archive/errorlogs/e.${branch}.${date} ${pb}/${branch}/errors
  rm -rf ${pb}/archive/errorlogs/a.${branch}.${date}
  mkdir -p ${pb}/archive/errorlogs/a.${branch}.${date}
  ln -sf ${pb}/archive/errorlogs/a.${branch}.${date} ${pb}/${branch}/logs
  cp -p ${pb}/cvsdone ${pb}/archive/errorlogs/e.${branch}.${date}/cvsdone
  cp -p ${pb}/cvsdone ${pb}/archive/errorlogs/a.${branch}.${date}/cvsdone
  cp -p ${pb}/usr/ports/INDEX-${branch} ${pb}/archive/errorlogs/e.${branch}.${date}/INDEX
  cp -p ${pb}/usr/ports/INDEX-${branch} ${pb}/archive/errorlogs/a.${branch}.${date}/INDEX
  chown -RL ${user} errors logs packages
  echo "error logs in ${pb}/archive/errorlogs/e.${branch}.${date}"
  if [ "$nodummy" = 0 ]; then
    for dir in ${dummyports}; do
      cp -p tarballs/$(cd ${pb}/usr/ports/$dir; make package-name).tgz packages/All
    done
  fi

  count=$(awk '{sum+=$2}END{print sum+NR/2}' ${pb}/mlist | sed -e 's/\..*$//')
  cd ${pb}/${branch}/packages/All
  ln -sf ../../Makefile .
  echo "================================================"
  echo "building packages (phase 1)"
  echo "================================================"
  echo "started at $(date)"
  make -k -j$count ${quickpkgs} all > ../../make.0 2>&1 </dev/null
  echo "ended at $(date)"
  echo $(echo $(ls -1 ${pb}/${branch}/packages/All | wc -l) - 2 | bc) "packages built"
  echo $(echo $(du -sk ${pb}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
  echo $(echo $(du -sk ${pb}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles"
  cd ${pb}/${branch}
  if grep -qE '(ptimeout|pnohang): killing' make.0; then
    echo "The following port(s) timed out:"
    grep -E '(ptimeout|pnohang): killing' make.0 | sed -e 's/^.*ptimeout:/ptimeout:/' -e 's/^.*pnohang:/pnohang:/' 
  fi
  cp -rp errors old-errors
  cd ${pb}/${branch}/old-errors
  ${pb}/scripts/processlogs

  md5=$(/sbin/md5 ${pb}/${branch}/tarballs/bindist.tar | awk '{print $4}')
  echo "================================================"
  echo "setting up nodes"
  echo "================================================"
  for node in $(awk '{print $1}' ${pb}/mlist); do
    setupnode ${pb} ${scripts} ${branch} ${me} ${node} ${md5} &
    sleep 2
  done

  wait

  echo "setting up of nodes ended at $(date)"

  count=$(awk '{sum+=$2}END{print sum+NR/2}' ${pb}/mlist | sed -e 's/\..*$//')
  cd ${pb}/${branch}/packages/All
  echo "================================================"
  echo "building packages (phase 2)"
  echo "================================================"
  echo "started at $(date)"
  make -k -j$count ${quickpkgs} all > ../../make.1 2>&1 </dev/null
  echo "ended at $(date)"
  rm Makefile
  if [ "$nodummy" = 0 ]; then
    rm ${dummypkgs}
  fi
  ${pb}/scripts/chopindex ${pb}/usr/ports/INDEX-${branch} ${pb}/${branch}/packages > \
    ${pb}/${branch}/packages/INDEX
  echo $(ls -1 ${pb}/${branch}/packages/All | wc -l) "packages built"
  echo $(cat ${pb}/${branch}/packages/INDEX | wc -l) "lines in INDEX"
  echo $(echo $(du -sk ${pb}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
  echo $(echo $(du -sk ${pb}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles"
  if [ "$norestr" = 0 ]; then
    echo "deleting restricted ports"
    sh ${pb}/${branch}/restricted.sh
    ${pb}/scripts/chopindex ${pb}/usr/ports/INDEX ${pb}/${branch}/packages > \
      ${pb}/${branch}/packages/INDEX
    echo $(ls -1 ${pb}/${branch}/packages/All | wc -l) "packages built"
    echo $(cat ${pb}/${branch}/packages/INDEX | wc -l) "lines in INDEX"
    echo $(echo $(du -sk ${pb}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
    echo $(echo $(du -sk ${pb}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles"
  fi
  rm -rf ${pb}/${branch}/bad
  mkdir -p ${pb}/${branch}/bad
  echo "checking packages"
  for i in *.tgz; do
    if ! gzip -t $i; then
      echo "Warning: package $i is bad, moving to ${pb}/${branch}/bad"
      # the latest link will be left behind...
      mv $i ${pb}/${branch}/bad
      rm ../*/$i
    fi
  done
  echo "generating CHECKSUM.MD5"
  md5 *.tgz > CHECKSUM.MD5
  cd ${pb}/${branch}
  if grep -qE '(ptimeout|pnohang): killing' make.1; then
    echo "The following port(s) timed out:"
    grep -E '(ptimeout|pnohang): killing' make.1 | sed -e 's/^.*ptimeout:/ptimeout:/' -e 's/^.*pnohang:/pnohang:/'
  fi
  cd ${pb}/${branch}/old-errors
  new=""
  for i in *.log; do
    if [ ! -f ../errors/$i ]; then
      new="$new $(basename $i .log)"
    fi
  done
  if [ "x$new" != "x" ]; then
    echo "The following port(s) didn't build the first time around: $new"
  fi
  echo "================================================"
  echo "new failures"
  echo "================================================"
  cd ${pb}/${branch}/errors
  for i in *.log; do
    if [ ! -f ${pb}/${branch}/bak/errors/$i ]; then
      echo -n " $(basename $i .log)"
    fi
  done
  if cd ${pb}/${branch}/bak/packages/All; then
    echo
    echo "================================================"
    echo "old packages"
    echo "================================================"
    for i in *.tgz; do
      if [ ! -f ${pb}/${branch}/packages/All/$i ]; then
        echo -n " $(basename $i .tgz)"
      fi
    done
  fi
  if cd ${pb}/${branch}/bak/errors; then
    echo
    echo "================================================"
    echo "old failures"
    echo "================================================"
    for i in *.log; do
      if [ ! -f ${pb}/${branch}/errors/$i ]; then
        echo -n " $(basename $i .log)"
      fi
    done
  fi
  echo
  echo "================================================"
  echo "new packages"
  echo "================================================"
  cd ${pb}/${branch}/packages/All
  for i in *.tgz; do
    if [ ! -f ${pb}/${branch}/bak/packages/All/$i ]; then
      echo -n " $(basename $i .tgz)"
    fi
  done
  echo
  echo "================================================"
  echo "archiving /usr/ports"
  echo "================================================"
  echo "started at $(date)"
  cd ${pb}/usr
  tar --exclude CVS -czf ${pb}/${branch}/tarballs/ports.tar.gz ports
  echo "ended at $(date)"
  echo "================================================"
  echo "copying distfiles"
  echo "================================================"
  echo "started at $(date)"
  cd ${pb}
  rm -rf bak/distfiles
  mv -f distfiles bak
  su ${user} -c ${scripts}/cpdistfiles > ${pb}/cpdistfiles.log 2>&1 </dev/null &
  # if norestr flag is set, assume it's for ftp and copy the packages over
  if [ "$norestr" = 1 ]; then
    echo "ended at $(date)"
    echo "================================================"
    echo "copying packages"
    echo "================================================"
    su ${user} -c '${scripts}/docppackages ${branch}'
  fi
fi
echo "================================================"
echo "all done at $(date)"
echo "================================================"