pkgsrc-ng/minix/pbulk-ng.sh

#!/bin/sh

# This script tries to optimise time building for jailed pbulk builds
# at the expense of disk space.
#
# A full (all packages) build requires currently about:
#  -   1.0GB	pkgsrc itself
#  -   0.3GB	chroot-minix
#  -   1.3GB	chroot-bootstrap
#  -   1.3GB	chroot-pbulk (w/o distfiles)
#  -  24.0GB	distfiles (sources of the packages)
#  -  20.0GB    during build, to store temporary files and produced packages
# ==========
#    ~48.0GB	total
#
# I (LSC) recommend a partition of at least 100GB for a full build.
#
# this will create the following hierarchy (assuming defaults) :
# /usr/bbuild
# /usr/bbuild/chroot-minix	Pristine minix rootfs
# /usr/bbuild/chroot-bootstrap	Pristine minix + bootstrapped pbulk tools
# /usr/bbuild/chroot-pbulk	Root system with build pkgsrc.
#
# Upon successful completion the following will be copied into:
# /usr/bbuild/save/YYYYMMDD-HHmmss.logs     pbulk-log files
# /usr/bbuild/save/YYYYMMDD-HHmmss.packages generated packages
# /usr/bbuild/save/YYYYMMDD-HHmmss.disfiles fetched distfiles
#

# Exit at the first error
set -e

# Some useful constant
YES="yes"
NO="no"

# Defaults
: ${TOPDIR=/usr/bbuild}
: ${SAVEDIR=/usr/bbuild/save}
: ${TOOLDIR=/usr/bbuild/tooldir.$(uname -p)}
: ${MINIXSRCDIR=/usr/src}
: ${PKGSRCDIR=/usr/pkgsrc}
: ${DISTFILESDIR=/usr/pkgsrc/distfiles}
: ${USETOOLS=no}

: ${ROOT_MINIX=${TOPDIR}/chroot-minix}
: ${ROOT_BOOTSTRAP=${TOPDIR}/chroot-bootstrap}
: ${ROOT_PBULK=${TOPDIR}/chroot-pbulk}

# By default copy the local pkgsrc repository
: ${PKGSRC_COPY=${YES}}
: ${PKGSRC_REMOTE=pkgsrc}
: ${PKGSRC_URL=git://git.minix3.org/pkgsrc.git}
: ${PKGSRC_BRANCH=minix-master}
# Destination pkgsrc git, absolute path, has to contain ROOT_BOOTSTRAP!
: ${PKGSRC_GIT=${ROOT_BOOTSTRAP}/usr/pkgsrc/.git}

# By default re-use root FS if available
: ${BUILD_ROOT_BOOTSTRAP=${NO}}
: ${BUILD_ROOT_MINIX=${NO}}
: ${BUILD_ROOT_PBULK=${NO}}

# By default copy to a safe place the generated packages, distfiles and logs
: ${SAVE_PACKAGES=${YES}}
: ${SAVE_DISTFILES=${YES}}
: ${SAVE_LOGS=${YES}}

# Use tools through variables, ease the debug process
: ${DRY_RUN=${NO}}

# Some private variables which may used from within the chroots
: ${CMD_BOOTSTRAP=./bootstrap/bootstrap}
: ${CMD_BOOTSTRAP_CLEANUP=./bootstrap/cleanup}
#   This cannot be changed without updating pbulk.conf
: ${BOOTSTRAP_PREFIX=/usr/pbulk}
: ${PKGSRC_PREFIX=/usr/pkgsrc}

# Generate a clean PATH for the jails.
CHROOT_PATH=""
for d in ${BOOTSTRAP_PREFIX} /usr/pkg /usr/X11R7 /usr ''
do
	CHROOT_PATH=${CHROOT_PATH}:${d}/bin:${d}/sbin
done
CHROOT_PATH=${CHROOT_PATH}:/usr/games
LD_CHROOT_PATH=/usr/pkg/lib:/usr/X11R7/lib:/usr/lib:/lib

if [ ! -d $MINIXSRCDIR ]
then
	echo Please install the minix sources in $MINIXSRCDIR.
	exit 1
fi

#============================================================================
if [ ${DRY_RUN} = ${YES} ]
then
	RM='echo ##: rm '
	MV='echo ##: mv '
	CP='echo ##: cp '
	CD='echo ##: cd '
	LN='echo ##: ln '
	SED='echo ##: sed '
	CHROOT='echo ##: chroot '
	MKDIR='echo ##: mkdir '
	TAR='echo ##: tar '
	EXPORT='echo ##: export '
	PKG_ADD='echo ##: pkg_add '
	SYNCTREE='echo ##: synctree '
	GIT='echo ##: git '
	BMAKE='echo ##: bmake '
	CMD_RELEASE="echo ##: ${CMD_RELEASE} "
	CMD_BOOTSTRAP="echo ##: ${CMD_BOOTSTRAP} "
	CMD_BOOTSTRAP_CLEANUP="echo ##: ${CMD_BOOTSTRAP_CLEANUP} "
	CMD_RESET_ERRORS="echo ##: test ! -f /usr/pbulk-logs/meta/error || rm /usr/pbulk-logs/meta/error "
	CMD_BULKBUILD="echo ##: bulkbuild "
	CMD_BULKBUILD_RESTART="echo ##: bulkbuild-restart "

	DIRNAME='echo _dirname_ '
	# Kind of an exception, but as it used to collect
	# all the output of a phase, we want it to be echoed,
	# instead of saved in a log file
	TEE="cat - "
else
	RM='rm '
	MV='mv '
	CP='cp '
	CD='cd '
	LN='ln '
	SED='sed '
	DIRNAME='dirname '
	CHROOT='chroot '
	MKDIR='mkdir '
	TAR='tar '
	EXPORT='export '
	PKG_ADD='pkg_add '
	SYNCTREE='synctree '
	GIT='git '
	BMAKE='bmake '
	TEE='tee '
	CMD_RESET_ERRORS="test ! -f /usr/pbulk-logs/meta/error || rm /usr/pbulk-logs/meta/error "
	CMD_BULKBUILD="bulkbuild "
	CMD_BULKBUILD_RESTART="bulkbuild-restart "
fi

# Check at which step which should start :
[ ! -d "${ROOT_MINIX}" ] && BUILD_ROOT_MINIX=${YES}
[ ! -d "${ROOT_BOOTSTRAP}" ] && BUILD_ROOT_BOOTSTRAP=${YES}
[ ! -d "${ROOT_PBULK}" ] && BUILD_ROOT_PBULK=${YES}

# Ensure that all the steps following the first to be generated
# are also re-generated.
[ ${BUILD_ROOT_MINIX} = ${YES} ] && BUILD_ROOT_BOOTSTRAP=${YES}
[ ${BUILD_ROOT_BOOTSTRAP} = ${YES} ] && BUILD_ROOT_PBULK=${YES}

#============================================================================
build_minix() {
	echo ":-> Building minix chroot in ${ROOT_MINIX}"
	(
		exec 2>&1
		set +e

		echo ":--> Building minix sources [${BUILD_START}]"

		${CD} ${MINIXSRCDIR}

		# Make fetch scripts executable
		find . -name fetch.sh -exec chmod 755 {} \;

		# Detect the default toolchain, and install the same in the
		# jail. Having cc point to gcc while clang is installed, is
		# unsupported. Either this is a gcc-only system, or cc points
		# to clang, as it is the default compiler.
		which clang > /dev/null; _clang=$?
		which gcc > /dev/null; _gcc=$?
		which cc > /dev/null; _cc=$?

		if [ ! ${_gcc} = 0 -a ! ${_clang} = 0 ]; then
			echo "No toolchain detected, please install clang or gcc."
			exit 1
		fi

		if [ ${_cc} = 0 -a ${_clang} = 0 -a $(cc --version | grep -q clang) ]
		then
			echo "It is unsupported to have cc set to someting else than clang,"
		        echo "while clang is also installed on the system."
			echo
			echo "Please fix your cc symlink, or uninstall clang."

			exit 1
		fi

		if [ ${_clang} = 0 ];
		then
			toolchain=" MKLLVM=yes MKLIBCXX=yes MKGCCCMDS=no MKLIBSTDCXX=no"
		fi

		# If both gcc and clang are installed, we still prefer clang.
		if [ ${_gcc} = 0 -a ! ${_clang} = 0 ];
		then
			toolchain=" MKLLVM=no MKLIBCXX=no MKGCCCMDS=yes MKLIBSTDCXX=yes"

			# Download Binutils, gmake & texinfo
			./external/lgpl3/mpfr/fetch.sh
			./external/lgpl3/gmp/fetch.sh
			./external/lgpl3/mpc/fetch.sh
			./external/gpl3/gcc/fetch.sh
		fi

		# Download Binutils, gmake & texinfo
		./gnu/dist/fetch.sh
		./external/gpl3/binutils/fetch.sh

		export MAKEOBJDIR='${.CURDIR:C,^/mnt/src,/mnt/obj.pbulk.386,}'
		export MKOBJDIRS=yes
		export MKBINUTILS=yes

		make build etcforce \
			USETOOLS=${USETOOLS} \
			TOOLDIR=${TOOLDIR} \
			DESTDIR=${ROOT_MINIX} \
			SLOPPY_FLIST=yes \
			MKX11=yes \
			MKUPDATE=yes \
			${toolchain} \
			MKKYUA=no \
			MKATF=no

		echo ":--> Copying config files"
		for f in hosts resolv.conf
		do
			[ -f /etc/${f} ] && ${CP} /etc/${f} ${ROOT_MINIX}/etc/${f}
		done
	) | ${TEE} ${TOPDIR}/1-build_minix.log
	echo ":-> Building minix chroot done"

	return 0
}

build_bootstrap() {
	echo ":-> Building bootstrapped chroot"
	(
		exec 2>&1
		set -e
		echo ":--> Initializing chroot in ${ROOT_BOOTSTRAP} [${BUILD_START}]"

		if [ ${PKGSRC_COPY} = ${YES} ]
		then
			echo ":--> Copying from ${PKGSRCDIR}"
			# Copy and use our local pkgsrc repository as it is
			${MKDIR} -p ${ROOT_BOOTSTRAP}${PKGSRC_PREFIX}
			${SYNCTREE} -f ${PKGSRCDIR} ${ROOT_BOOTSTRAP}${PKGSRC_PREFIX} >/dev/null
		else
			echo ":--> Cloning from ${PKGSRC_URL}/${PKGSRC_BRANCH}"
			# Copy our own pkgsrc repository there so the new
			# repository does not have to retrieve objects we
			# already have locally.
			${MKDIR} -p ${PKGSRC_GIT}
			${SYNCTREE} -f ${PKGSRCDIR}/.git ${PKGSRC_GIT} >/dev/null
			${GIT} --git-dir ${PKGSRC_GIT} remote rm ${PKGSRC_REMOTE}
			${GIT} --git-dir ${PKGSRC_GIT} remote add ${PKGSRC_REMOTE} ${PKGSRC_URL} 
			${GIT} --git-dir ${PKGSRC_GIT} fetch ${PKGSRC_REMOTE}
			${GIT} --git-dir ${PKGSRC_GIT} checkout -f ${PKGSRC_REMOTE}/${PKGSRC_BRANCH}
		fi

		# Bonus distfiles	
		echo ":--> Copying prefetched distfiles from ${DISTFILESDIR}"
		${MKDIR} -p ${ROOT_BOOTSTRAP}${PKGSRC_PREFIX}/distfiles
		${SYNCTREE} -f ${DISTFILESDIR} ${ROOT_BOOTSTRAP}${PKGSRC_PREFIX}/distfiles >/dev/null

		# Ensure that the package directoy is clean and exists
		${RM} -rf ${ROOT_BOOTSTRAP}${PKGSRC_PREFIX}/packages/$(uname -r)/
		${MKDIR} -p ${ROOT_BOOTSTRAP}${PKGSRC_PREFIX}/packages/$(uname -r)/$(uname -p)/All

		echo ":--> Bootstrapping pbulk"
		${CHROOT} ${ROOT_BOOTSTRAP} sh -c \
		"(
			set -e
			${EXPORT} PATH=${CHROOT_PATH}
			${EXPORT} LD_LIBRARY_PATH=${LD_CHROOT_PATH}

			${CD} ${PKGSRC_PREFIX}

			# First stage, PBULK bootstrap & installation
			
			# Trim the .ifdef BSD_PKG_MK and .endif lines to make a 'fragment'
			# and adapt a few path to the ones expected for pbulk
			${SED} \
				-e '/.*BSD_PKG_MK/d' \
				-e 's@VARBASE?=.*@VARBASE=	'${BOOTSTRAP_PREFIX}'/var@' \
				-e 's@PKG_DBDIR?=.*@PKG_DBDIR=	'${BOOTSTRAP_PREFIX}'/pkgdb@' \
				-e 's@WRKOBJDIR?=.*@WRKOBJDIR=	'${BOOTSTRAP_PREFIX}'/work@' \
				./minix/mk.conf.minix \
				> ./minix/mk.conf.minix.pbulk.frag

			echo ':--> Building pbulk kit'
			${CMD_BOOTSTRAP} \
				--prefix=${BOOTSTRAP_PREFIX} \
				--varbase=${BOOTSTRAP_PREFIX}/var \
				--pkgdbdir=${BOOTSTRAP_PREFIX}/pkgdb \
				--workdir=${BOOTSTRAP_PREFIX}/work \
				--mk-fragment=./minix/mk.conf.minix.pbulk.frag

			# Install pbulk into /usr/pbulk
			echo ':--> Building and installing pbulk'
			${BMAKE} -C ./devel/pth package-install
			${BMAKE} -C ./pkgtools/pbulk package-install
			${SED} -e 's/OP_SYS_VER/'$(uname -r)'/g' ./minix/pbulk.conf > ${BOOTSTRAP_PREFIX}/etc/pbulk.conf

			# First stage: done

			echo ':--> Bootstrap cleanup'
			${CMD_BOOTSTRAP_CLEANUP}

			# Second stage, pkgsrc bootstrap & installation

			# Trim the .ifdef BSD_PKG_MK and .endif lines to make a 'fragment'
			${SED} -e '/.*BSD_PKG_MK/d' \
				./minix/mk.conf.minix \
				> ./minix/mk.conf.minix.frag

			echo ':--> Building binary pkgsrc kit'
                 	${CMD_BOOTSTRAP} \
				--varbase=/usr/pkg/var \
				--pkgdbdir=/usr/pkg/var/db/pkg \
				--mk-fragment=./minix/mk.conf.minix.frag \
				--workdir=${PKGSRC_PREFIX}/work \
				--gzip-binary-kit=${BOOTSTRAP_PREFIX}/bootstrap.tar.gz

			${RM} -rf ./packages/$(uname -r)/
			${MKDIR} -p ./packages/$(uname -r)/$(uname -p)/All

			# Use the same mk.conf that our users instead of the hybrid 
			# auto-generated mk.conf from bootstrap.
			${TAR} -C /tmp -xzf ${BOOTSTRAP_PREFIX}/bootstrap.tar.gz
			${CP} ./minix/mk.conf.minix /tmp/usr/pkg/etc/mk.conf
			${TAR} -C /tmp -hzcf ${BOOTSTRAP_PREFIX}/bootstrap.tar.gz usr
			${RM} -rf /tmp/usr

			# Second stage: done
		)"

		echo ":--> Bootstrapping pbulk done"
		
	) | ${TEE} ${TOPDIR}/2-build_bootstrap.log
	echo ":-> Building bootstrapped chroot done"

	return 0
}

pbulk_start() {
	echo ":-> Building packages from scratch"
	(
		exec 2>&1
		set -e

		${CHROOT} ${ROOT_PBULK} sh -c \
		"(
			set -e
			${EXPORT} PATH=${CHROOT_PATH}
			${EXPORT} LD_LIBRARY_PATH=${LD_CHROOT_PATH}

			${CD} ${PKGSRC_PREFIX}

			echo ':--> Starting build ['${BUILD_START}']'
			${CMD_BULKBUILD}
		)"
	) | ${TEE} ${TOPDIR}/3-pbulk.log
	echo ":-> Building packages from scratch done"

	return 0
}

pbulk_restart() {
	echo ":-> Building packages from previous build"
	(
		exec 2>&1
		set -e

		${CHROOT} ${ROOT_PBULK} sh -c \
		"(
			set -e
			${EXPORT} PATH=${CHROOT_PATH}
			${EXPORT} LD_LIBRARY_PATH=${LD_CHROOT_PATH}

			${CD} ${PKGSRC_PREFIX}

			echo ':--> Resetting error file'
			${CMD_RESET_ERRORS}

			echo ':--> Restarting build ['${BUILD_START}']'
			${CMD_BULKBUILD_RESTART}
		)"
	) | ${TEE} ${TOPDIR}/3-pbulk.log

	echo ":-> Building packages from previous build done"
}

#============================================================================
# Initializations are done, start applying the requested actions on the system

BUILD_START=$(date)
echo -e "\n:: pbulk started on ${BUILD_START}"

if [ ${BUILD_ROOT_MINIX} = ${YES} ]
then
	echo -e "\n:> Generating minix root fs."
	${RM} -rf ${ROOT_MINIX}

	# Ensure presence of destination directory
	${MKDIR} -p ${ROOT_MINIX}

	build_minix
fi

if [ ${BUILD_ROOT_BOOTSTRAP} = ${YES} ]
then
	echo -e "\n:> Bootstrapping pkgsrc."
	# Ensure the new chroot is clean.
	${MKDIR} -p ${ROOT_BOOTSTRAP}
	${SYNCTREE} -f ${ROOT_MINIX} ${ROOT_BOOTSTRAP} >/dev/null

	build_bootstrap
fi

if [ ${BUILD_ROOT_PBULK} = ${YES} ]
then
	echo -e "\n:> Initializing pbulk root."
	# Ensure the new chroot is clean.
	${MKDIR} -p ${ROOT_PBULK}
	${SYNCTREE} -f ${ROOT_BOOTSTRAP} ${ROOT_PBULK} >/dev/null

	echo -e "\n:> Building packages from scratch."
	pbulk_start
else
	# We want to re-use a previous pbulk.
	# Just make sure that any modification within the pkgsrc tree is visible
	# find param to make synctree keep files in packages and distfiles
	# Also requires regenerating the bootstrap tarball, as well as updating
	# ROOT_PBULK/usr/pbulk/etc/pbulk.conf
	#${SYNCTREE} -f ${PKGSRCDIR} ${ROOT_PBULK}${PKGSRC_PREFIX} >/dev/null

	echo -e "\n:> Restarting build of packages."
	pbulk_restart
fi

_build_end=$(date '+%Y%m%d%H%M.%S')
BUILD_END=$(date -j ${_build_end})

# We have to do this here, otherwise the date field would be empty
: ${TIMESTAMP=$(date -j '+%Y%m%d-%H%M%S' ${_build_end})}
: ${ROOT_LOGS=${SAVEDIR}/${TIMESTAMP}.logs}
: ${ROOT_DISTFILES=${SAVEDIR}/${TIMESTAMP}.distfiles}
: ${ROOT_PACKAGES=${SAVEDIR}/${TIMESTAMP}.packages}

if [ ${SAVE_LOGS} = ${YES} ]
then
	${MKDIR} -p ${ROOT_LOGS}
	${CP} -pfr ${TOPDIR}/1-build_minix.log ${ROOT_LOGS}/
	${CP} -pfr ${TOPDIR}/2-build_bootstrap.log ${ROOT_LOGS}/
	${CP} -pfr ${TOPDIR}/3-pbulk.log ${ROOT_LOGS}/
	${SYNCTREE} -uf ${ROOT_PBULK}/usr/pbulk-logs ${ROOT_LOGS}/pbulk-logs
fi

if [ ${SAVE_DISTFILES} = ${YES} ]
then
	${SYNCTREE} -uf ${ROOT_PBULK}${PKGSRC_PREFIX}/distfiles ${ROOT_DISTFILES}
fi

if [ ${SAVE_PACKAGES} = ${YES} ]
then
	${SYNCTREE} -uf ${ROOT_PBULK}${PKGSRC_PREFIX}/packages ${ROOT_PACKAGES}
fi

echo -e "\n:: pbulk finished:"
echo ":>   started on  : ${BUILD_START}"
echo ":>   finished on : ${BUILD_END}"
echo ":> Build logs    : ${ROOT_LOGS}"
echo ":> Distfiles     : ${ROOT_DISTFILES}"
echo ":> Packages      : ${ROOT_PACKAGES}"