Minix/pkgsrc-ng
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Files
2016Q3
pkgsrc-ng/minix/pbulk-ng.sh
Lionel Sambuc 5f9af68f96 Step 0: minix category 
This contains MINIX specific packages, configuration and (PKGSRC) build
scripts.
2016-11-29 23:17:20 +01:00

434 lines
12 KiB
Bash
Executable File
Raw Permalink Blame History

#!/bin/sh
# This script tries to optimise time building for jailed pbulk builds
# at the expense of disk space.
#
# A full (all packages) build requires currently about:
# - 1.0GB pkgsrc itself
# - 0.3GB chroot-minix
# - 1.3GB chroot-bootstrap
# - 1.3GB chroot-pbulk (w/o distfiles)
# - 24.0GB distfiles (sources of the packages)
# - 20.0GB during build, to store temporary files and produced packages
# ==========
# ~48.0GB total
#
# I (LSC) recommend a partition of at least 100GB for a full build.
#
# this will create the following hierarchy (assuming defaults) :
# /usr/bbuild
# /usr/bbuild/chroot-minix Pristine minix rootfs
# /usr/bbuild/chroot-bootstrap Pristine minix + bootstrapped pbulk tools
# /usr/bbuild/chroot-pbulk Root system with build pkgsrc.
#
# Upon successful completion the following will be copied into:
# /usr/bbuild/save/YYYYMMDD-HHmmss.logs pbulk-log files
# /usr/bbuild/save/YYYYMMDD-HHmmss.packages generated packages
# /usr/bbuild/save/YYYYMMDD-HHmmss.disfiles fetched distfiles
#
# Exit at the first error
set -e
# Some useful constant
YES="yes"
NO="no"
# Defaults
: ${TOPDIR=/usr/bbuild}
: ${SAVEDIR=/usr/bbuild/save}
: ${TOOLDIR=/usr/bbuild/tooldir.$(uname -p)}
: ${OBJDIR=/usr/bbuild/obj.$(uname -p)}
: ${MINIXSRCDIR=/usr/src}
: ${PKGSRCDIR=/usr/pkgsrc}
: ${DISTFILESDIR=/usr/pkgsrc/distfiles}
: ${USETOOLS=yes}
: ${ROOT_MINIX=${TOPDIR}/chroot-minix}
: ${ROOT_BOOTSTRAP=${TOPDIR}/chroot-bootstrap}
: ${ROOT_PBULK=${TOPDIR}/chroot-pbulk}
# By default copy the local pkgsrc repository
: ${PKGSRC_COPY=${YES}}
: ${PKGSRC_REMOTE=pkgsrc}
: ${PKGSRC_URL=git://git.minix3.org/pkgsrc.git}
: ${PKGSRC_BRANCH=minix-master}
# Destination pkgsrc git, absolute path, has to contain ROOT_BOOTSTRAP!
: ${PKGSRC_GIT=${ROOT_BOOTSTRAP}/usr/pkgsrc/.git}
# By default re-use root FS if available
: ${BUILD_ROOT_BOOTSTRAP=${NO}}
: ${BUILD_ROOT_MINIX=${NO}}
: ${BUILD_ROOT_PBULK=${NO}}
# By default copy to a safe place the generated packages, distfiles and logs
: ${SAVE_PACKAGES=${YES}}
: ${SAVE_DISTFILES=${YES}}
: ${SAVE_LOGS=${YES}}
# Use tools through variables, ease the debug process
: ${DRY_RUN=${NO}}
# Some private variables which may used from within the chroots
: ${CMD_BOOTSTRAP=./bootstrap/bootstrap}
: ${CMD_BOOTSTRAP_CLEANUP=./bootstrap/cleanup}
# This cannot be changed without updating pbulk.conf
: ${BOOTSTRAP_PREFIX=/usr/pbulk}
: ${PKGSRC_PREFIX=/usr/pkgsrc}
# Generate a clean PATH for the jails.
CHROOT_PATH=""
for d in ${BOOTSTRAP_PREFIX} /usr/pkg /usr/X11R7 /usr ''
do
CHROOT_PATH=${CHROOT_PATH}:${d}/bin:${d}/sbin
done
CHROOT_PATH=${CHROOT_PATH}:/usr/games
LD_CHROOT_PATH=/usr/pkg/lib:/usr/X11R7/lib:/usr/lib:/lib
if [ ! -d $MINIXSRCDIR ]
then
echo Please install the minix sources in $MINIXSRCDIR.
exit 1
fi
#============================================================================
if [ ${DRY_RUN} = ${YES} ]
then
RM='echo ##: rm '
MV='echo ##: mv '
CP='echo ##: cp '
CD='echo ##: cd '
LN='echo ##: ln '
SED='echo ##: sed '
CHROOT='echo ##: chroot '
MKDIR='echo ##: mkdir '
TAR='echo ##: tar '
EXPORT='echo ##: export '
PKG_ADD='echo ##: pkg_add '
SYNCTREE='echo ##: synctree '
GIT='echo ##: git '
BMAKE='echo ##: bmake '
CMD_RELEASE="echo ##: ${CMD_RELEASE} "
CMD_BOOTSTRAP="echo ##: ${CMD_BOOTSTRAP} "
CMD_BOOTSTRAP_CLEANUP="echo ##: ${CMD_BOOTSTRAP_CLEANUP} "
CMD_RESET_ERRORS="echo ##: test ! -f /usr/pbulk-logs/meta/error || rm /usr/pbulk-logs/meta/error "
CMD_BULKBUILD="echo ##: bulkbuild "
CMD_BULKBUILD_RESTART="echo ##: bulkbuild-restart "
DIRNAME='echo _dirname_ '
# Kind of an exception, but as it used to collect
# all the output of a phase, we want it to be echoed,
# instead of saved in a log file
TEE="cat - "
else
RM='rm '
MV='mv '
CP='cp '
CD='cd '
LN='ln '
SED='sed '
DIRNAME='dirname '
CHROOT='chroot '
MKDIR='mkdir '
TAR='tar '
EXPORT='export '
PKG_ADD='pkg_add '
SYNCTREE='synctree '
GIT='git '
BMAKE='bmake '
TEE='tee '
CMD_RESET_ERRORS="test ! -f /usr/pbulk-logs/meta/error || rm /usr/pbulk-logs/meta/error "
CMD_BULKBUILD="bulkbuild "
CMD_BULKBUILD_RESTART="bulkbuild-restart "
fi
# Check at which step which should start :
[ ! -d "${ROOT_MINIX}" ] && BUILD_ROOT_MINIX=${YES}
[ ! -d "${ROOT_BOOTSTRAP}" ] && BUILD_ROOT_BOOTSTRAP=${YES}
[ ! -d "${ROOT_PBULK}" ] && BUILD_ROOT_PBULK=${YES}
# Ensure that all the steps following the first to be generated
# are also re-generated.
[ ${BUILD_ROOT_MINIX} = ${YES} ] && BUILD_ROOT_BOOTSTRAP=${YES}
[ ${BUILD_ROOT_BOOTSTRAP} = ${YES} ] && BUILD_ROOT_PBULK=${YES}
#============================================================================
build_minix() {
echo ":-> Building minix chroot in ${ROOT_MINIX}"
(
exec 2>&1
set -e
echo ":--> Building minix sources [${BUILD_START}]"
${CD} ${MINIXSRCDIR}
HOST_CC=clang HOST_CXX=clang++ ./build.sh \
-m i386 \
-O ${OBJDIR} \
-T ${TOOLDIR} \
-D ${ROOT_MINIX} \
-V SLOPPY_FLIST=yes \
-V MKX11=yes \
-V MKUPDATE=yes \
-V MKLLVM=yes \
-V MKLIBCXX=yes \
-V MKGCCCMDS=no \
-V MKLIBSTDCXX=no \
-V MKKYUA=no \
-V MKATF=no \
distribution
echo ":--> Copying config files"
for f in hosts resolv.conf
do
[ -f /etc/${f} ] && ${CP} /etc/${f} ${ROOT_MINIX}/etc/${f}
done
) | ${TEE} ${TOPDIR}/1-build_minix.log
echo ":-> Building minix chroot done"
return 0
}
build_bootstrap() {
echo ":-> Building bootstrapped chroot"
(
exec 2>&1
set -e
echo ":--> Initializing chroot in ${ROOT_BOOTSTRAP} [${BUILD_START}]"
if [ ${PKGSRC_COPY} = ${YES} ]
then
echo ":--> Copying from ${PKGSRCDIR}"
# Copy and use our local pkgsrc repository as it is
${MKDIR} -p ${ROOT_BOOTSTRAP}${PKGSRC_PREFIX}
${SYNCTREE} -f ${PKGSRCDIR} ${ROOT_BOOTSTRAP}${PKGSRC_PREFIX} >/dev/null
else
echo ":--> Cloning from ${PKGSRC_URL}/${PKGSRC_BRANCH}"
# Copy our own pkgsrc repository there so the new
# repository does not have to retrieve objects we
# already have locally.
${MKDIR} -p ${PKGSRC_GIT}
${SYNCTREE} -f ${PKGSRCDIR}/.git ${PKGSRC_GIT} >/dev/null
${GIT} --git-dir ${PKGSRC_GIT} remote rm ${PKGSRC_REMOTE}
${GIT} --git-dir ${PKGSRC_GIT} remote add ${PKGSRC_REMOTE} ${PKGSRC_URL}
${GIT} --git-dir ${PKGSRC_GIT} fetch ${PKGSRC_REMOTE}
${GIT} --git-dir ${PKGSRC_GIT} checkout -f ${PKGSRC_REMOTE}/${PKGSRC_BRANCH}
fi
# Bonus distfiles
echo ":--> Copying prefetched distfiles from ${DISTFILESDIR}"
${MKDIR} -p ${ROOT_BOOTSTRAP}${PKGSRC_PREFIX}/distfiles
${SYNCTREE} -f ${DISTFILESDIR} ${ROOT_BOOTSTRAP}${PKGSRC_PREFIX}/distfiles >/dev/null
# Ensure that the package directoy is clean and exists
${RM} -rf ${ROOT_BOOTSTRAP}${PKGSRC_PREFIX}/packages/$(uname -r)/
${MKDIR} -p ${ROOT_BOOTSTRAP}${PKGSRC_PREFIX}/packages/$(uname -r)/$(uname -p)/All
echo ":--> Bootstrapping pbulk"
${CHROOT} ${ROOT_BOOTSTRAP} sh -c \
"(
set -e
${EXPORT} PATH=${CHROOT_PATH}
${EXPORT} LD_LIBRARY_PATH=${LD_CHROOT_PATH}
${CD} ${PKGSRC_PREFIX}
# First stage, PBULK bootstrap & installation
# Trim the .ifdef BSD_PKG_MK and .endif lines to make a 'fragment'
# and adapt a few path to the ones expected for pbulk
${SED} \
-e '/.*BSD_PKG_MK/d' \
-e 's@VARBASE?=.*@VARBASE= '${BOOTSTRAP_PREFIX}'/var@' \
-e 's@PKG_DBDIR?=.*@PKG_DBDIR= '${BOOTSTRAP_PREFIX}'/pkgdb@' \
-e 's@WRKOBJDIR?=.*@WRKOBJDIR= '${BOOTSTRAP_PREFIX}'/work@' \
./minix/mk.conf.minix \
> ./minix/mk.conf.minix.pbulk.frag
echo ':--> Building pbulk kit'
${CMD_BOOTSTRAP} \
--prefix=${BOOTSTRAP_PREFIX} \
--varbase=${BOOTSTRAP_PREFIX}/var \
--pkgdbdir=${BOOTSTRAP_PREFIX}/pkgdb \
--workdir=${BOOTSTRAP_PREFIX}/work \
--mk-fragment=./minix/mk.conf.minix.pbulk.frag
# Install pbulk into /usr/pbulk
echo ':--> Building and installing pbulk'
${BMAKE} -C ./devel/pth package-install
${BMAKE} -C ./pkgtools/pbulk package-install
${SED} -e 's/OP_SYS_VER/'$(uname -r)'/g' ./minix/pbulk.conf > ${BOOTSTRAP_PREFIX}/etc/pbulk.conf
# First stage: done
echo ':--> Bootstrap cleanup'
${CMD_BOOTSTRAP_CLEANUP}
# Second stage, pkgsrc bootstrap & installation
# Trim the .ifdef BSD_PKG_MK and .endif lines to make a 'fragment'
${SED} -e '/.*BSD_PKG_MK/d' \
./minix/mk.conf.minix \
> ./minix/mk.conf.minix.frag
echo ':--> Building binary pkgsrc kit'
${CMD_BOOTSTRAP} \
--varbase=/usr/pkg/var \
--pkgdbdir=/usr/pkg/var/db/pkg \
--mk-fragment=./minix/mk.conf.minix.frag \
--workdir=${PKGSRC_PREFIX}/work \
--gzip-binary-kit=${BOOTSTRAP_PREFIX}/bootstrap.tar.gz
${RM} -rf ./packages/$(uname -r)/
${MKDIR} -p ./packages/$(uname -r)/$(uname -p)/All
# Use the same mk.conf that our users instead of the hybrid
# auto-generated mk.conf from bootstrap.
${TAR} -C /tmp -xzf ${BOOTSTRAP_PREFIX}/bootstrap.tar.gz
${CP} ./minix/mk.conf.minix /tmp/usr/pkg/etc/mk.conf
${TAR} -C /tmp -hzcf ${BOOTSTRAP_PREFIX}/bootstrap.tar.gz usr
${RM} -rf /tmp/usr
# Second stage: done
)"
echo ":--> Bootstrapping pbulk done"
) | ${TEE} ${TOPDIR}/2-build_bootstrap.log
echo ":-> Building bootstrapped chroot done"
return 0
}
pbulk_start() {
echo ":-> Building packages from scratch"
(
exec 2>&1
set -e
${CHROOT} ${ROOT_PBULK} sh -c \
"(
set -e
${EXPORT} PATH=${CHROOT_PATH}
${EXPORT} LD_LIBRARY_PATH=${LD_CHROOT_PATH}
${CD} ${PKGSRC_PREFIX}
echo ':--> Starting build ['${BUILD_START}']'
${CMD_BULKBUILD}
)"
) | ${TEE} ${TOPDIR}/3-pbulk.log
echo ":-> Building packages from scratch done"
return 0
}
pbulk_restart() {
echo ":-> Building packages from previous build"
(
exec 2>&1
set -e
${CHROOT} ${ROOT_PBULK} sh -c \
"(
set -e
${EXPORT} PATH=${CHROOT_PATH}
${EXPORT} LD_LIBRARY_PATH=${LD_CHROOT_PATH}
${CD} ${PKGSRC_PREFIX}
echo ':--> Resetting error file'
${CMD_RESET_ERRORS}
echo ':--> Restarting build ['${BUILD_START}']'
${CMD_BULKBUILD_RESTART}
)"
) | ${TEE} ${TOPDIR}/3-pbulk.log
echo ":-> Building packages from previous build done"
}
#============================================================================
# Initializations are done, start applying the requested actions on the system
BUILD_START=$(date)
echo -e "\n:: pbulk started on ${BUILD_START}"
if [ ${BUILD_ROOT_MINIX} = ${YES} ]
then
echo -e "\n:> Generating minix root fs."
${RM} -rf ${ROOT_MINIX}
# Ensure presence of destination directory
${MKDIR} -p ${ROOT_MINIX}
build_minix
fi
if [ ${BUILD_ROOT_BOOTSTRAP} = ${YES} ]
then
echo -e "\n:> Bootstrapping pkgsrc."
# Ensure the new chroot is clean.
${MKDIR} -p ${ROOT_BOOTSTRAP}
${SYNCTREE} -f ${ROOT_MINIX} ${ROOT_BOOTSTRAP} >/dev/null
build_bootstrap
fi
if [ ${BUILD_ROOT_PBULK} = ${YES} ]
then
echo -e "\n:> Initializing pbulk root."
# Ensure the new chroot is clean.
${MKDIR} -p ${ROOT_PBULK}
${SYNCTREE} -f ${ROOT_BOOTSTRAP} ${ROOT_PBULK} >/dev/null
echo -e "\n:> Building packages from scratch."
pbulk_start
else
# We want to re-use a previous pbulk.
# Just make sure that any modification within the pkgsrc tree is visible
# find param to make synctree keep files in packages and distfiles
# Also requires regenerating the bootstrap tarball, as well as updating
# ROOT_PBULK/usr/pbulk/etc/pbulk.conf
#${SYNCTREE} -f ${PKGSRCDIR} ${ROOT_PBULK}${PKGSRC_PREFIX} >/dev/null
echo -e "\n:> Restarting build of packages."
pbulk_restart
fi
_build_end=$(date '+%Y%m%d%H%M.%S')
BUILD_END=$(date -j ${_build_end})
# We have to do this here, otherwise the date field would be empty
: ${TIMESTAMP=$(date -j '+%Y%m%d-%H%M%S' ${_build_end})}
: ${ROOT_LOGS=${SAVEDIR}/${TIMESTAMP}.logs}
: ${ROOT_DISTFILES=${SAVEDIR}/${TIMESTAMP}.distfiles}
: ${ROOT_PACKAGES=${SAVEDIR}/${TIMESTAMP}.packages}
if [ ${SAVE_LOGS} = ${YES} ]
then
${MKDIR} -p ${ROOT_LOGS}
${CP} -pfr ${TOPDIR}/1-build_minix.log ${ROOT_LOGS}/
${CP} -pfr ${TOPDIR}/2-build_bootstrap.log ${ROOT_LOGS}/
${CP} -pfr ${TOPDIR}/3-pbulk.log ${ROOT_LOGS}/
${SYNCTREE} -uf ${ROOT_PBULK}/usr/pbulk-logs ${ROOT_LOGS}/pbulk-logs
fi
if [ ${SAVE_DISTFILES} = ${YES} ]
then
${SYNCTREE} -uf ${ROOT_PBULK}${PKGSRC_PREFIX}/distfiles ${ROOT_DISTFILES}
fi
if [ ${SAVE_PACKAGES} = ${YES} ]
then
${SYNCTREE} -uf ${ROOT_PBULK}${PKGSRC_PREFIX}/packages ${ROOT_PACKAGES}
fi
echo -e "\n:: pbulk finished:"
echo ":> started on : ${BUILD_START}"
echo ":> finished on : ${BUILD_END}"
echo ":> Build logs : ${ROOT_LOGS}"
echo ":> Distfiles : ${ROOT_DISTFILES}"
echo ":> Packages : ${ROOT_PACKAGES}"
Reference in New Issue View Git Blame Copy Permalink