Minix/pkgsrc-ng
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Import of pkgsrc-2013Q2

Browse Source
This commit is contained in:
Lionel Sambuc
2013-09-26 17:14:40 +02:00
commit 785076ae39
74991 changed files with 4380255 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkgtools/pbulk/DESCR Normal file
View File

@@ -0,0 +1,4 @@
pbulk is the modular bulk build framework for pkgsrc.
This package contains the front end bulk build and the script logic
for full and limited bulk builds.

7
pkgtools/pbulk/MESSAGE Normal file
View File

@@ -0,0 +1,7 @@
===========================================================================
$NetBSD: MESSAGE,v 1.2 2008/02/13 21:04:10 tnn Exp $
If you are updating pbulk from earlier versions, please don't forget
to update the configuration file. Otherwise unexpected results can
occur.
===========================================================================

55
pkgtools/pbulk/Makefile Normal file
View File

@@ -0,0 +1,55 @@
# $NetBSD: Makefile,v 1.66 2012/11/23 12:13:34 joerg Exp $
DISTNAME= pbulk-0.51
COMMENT= Modular bulk build framework
.include "../../pkgtools/pbulk/Makefile.common"
USE_TOOLS+= awk:run bzip2:run digest:run gzip:run make:run \
mail:run sed:run tar:run
DEPENDS+= rsync-[0-9]*:../../net/rsync
DEPENDS+= pbulk-base>=0.38:../../pkgtools/pbulk-base
.include "../../mk/bsd.prefs.mk"
.if ${OPSYS} == "NetBSD" && ${MAKE} == "/usr/bin/make"
TARGET_MAKE= ${MAKE}
.else
TARGET_MAKE= $${prefix}/bin/bmake
.endif
NEATO= ${PREFIX}/bin/neato
SUBST_CLASSES+= tools
SUBST_STAGE.tools= post-patch
SUBST_MESSAGE.tools= Fixing references to tools
SUBST_FILES.tools= pbulk.conf scripts/build scripts/build-client-start \
scripts/bulkbuild scripts/bulkbuild-rebuild scripts/bulkbuild-restart \
scripts/client-clean \
scripts/pkg-build scripts/pkg-up-to-date scripts/pre-build \
scripts/report scripts/scan scripts/scan-client-start scripts/upload \
scripts/compute-packages.awk scripts/create-broken-graph.awk \
scripts/create-report-html.awk scripts/create-report-txt.awk \
scripts/create-report.awk
SUBST_VARS.tools= AWK BZIP2 CHOWN DIGEST GZIP_CMD ID MAIL_CMD NEATO \
PBULK_CONFIG PBULK_CONFIG_VERSION PKG_ADD_CMD \
PKG_DELETE_CMD PKG_INFO_CMD PREFIX SED SH TAR \
TARGET_MAKE
CONF_FILES+= share/examples/pbulk/pbulk.conf ${PKG_SYSCONFDIR}/pbulk.conf
PBULK_CONFIG= ${PKG_SYSCONFDIR}/pbulk.conf
PBULK_CONFIG_VERSION= 0.51
INSTALLATION_DIRS= bin libexec/pbulk share/examples/pbulk
USE_BSD_MAKEFILE= yes
BUILD_DIRS= scripts
do-extract:
${CP} -r ${FILESDIR}/pbulk ${WRKDIR}
post-install:
${INSTALL_DATA} ${WRKSRC}/pbulk.conf ${DESTDIR}${PREFIX}/share/examples/pbulk/pbulk.conf
.include "../../mk/bsd.pkg.mk"

12
pkgtools/pbulk/Makefile.common Normal file
View File

@@ -0,0 +1,12 @@
# $NetBSD: Makefile.common,v 1.2 2011/11/27 19:53:30 joerg Exp $
CATEGORIES= pkgtools
MASTER_SITES= # empty
DISTFILES= # empty
MAINTAINER= joerg@NetBSD.org
HOMEPAGE= ftp://ftp.NetBSD.org/pub/NetBSD/packages/pkgsrc/doc/pkgsrc.html
FILESDIR= ${.CURDIR}/../../pkgtools/pbulk/files
WRKSRC= ${WRKDIR}/pbulk

20
pkgtools/pbulk/PLIST Normal file
View File

@@ -0,0 +1,20 @@
@comment $NetBSD: PLIST,v 1.5 2009/06/14 18:11:02 joerg Exp $
bin/bulkbuild
bin/bulkbuild-rebuild
bin/bulkbuild-restart
libexec/pbulk/build
libexec/pbulk/build-client-start
libexec/pbulk/client-clean
libexec/pbulk/compute-packages
libexec/pbulk/create-broken-graph
libexec/pbulk/create-report
libexec/pbulk/create-report-html
libexec/pbulk/create-report-txt
libexec/pbulk/pkg-build
libexec/pbulk/pkg-up-to-date
libexec/pbulk/pre-build
libexec/pbulk/report
libexec/pbulk/scan
libexec/pbulk/scan-client-start
libexec/pbulk/upload
share/examples/pbulk/pbulk.conf

5
pkgtools/pbulk/files/pbulk/Makefile Normal file
View File

@@ -0,0 +1,5 @@
# $NetBSD: Makefile,v 1.2 2011/11/27 19:53:30 joerg Exp $
SUBDIR= lib .WAIT presolve pscan pbuild scripts
.include <bsd.subdir.mk>

13
pkgtools/pbulk/files/pbulk/Makefile.inc Normal file
View File

@@ -0,0 +1,13 @@
# $NetBSD: Makefile.inc,v 1.4 2009/01/31 23:25:38 joerg Exp $
BINDIR?= ${PREFIX}/bin
WARNS= 4
.if !defined(LIBPBULK_ONLY)
CPPFLAGS+= -I${.CURDIR}/../lib
DPADD+= ${.OBJDIR}/../lib/libpbulk.a
LDADD+= -L${.OBJDIR}/../lib -lpbulk
.endif
LDADD+= -lnbcompat

11
pkgtools/pbulk/files/pbulk/lib/Makefile Normal file
View File

@@ -0,0 +1,11 @@
# $NetBSD: Makefile,v 1.1.1.1 2007/06/19 19:49:59 joerg Exp $
LIB= pbulk
MKPRIVATELIB= yes
MKLINKLIB= no
MKPIC= no
SRCS= alloc.c atomic.c event.c exec.c match.c read_child.c read_file.c netaddr.c
LIBPBULK_ONLY= # defined
.include <bsd.lib.mk>

117
pkgtools/pbulk/files/pbulk/lib/alloc.c Normal file
View File

@@ -0,0 +1,117 @@
/* $NetBSD: alloc.c,v 1.3 2012/11/23 12:13:35 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <nbcompat/err.h>
#include <stdarg.h>
#include <nbcompat/stdio.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/string.h>
#include "pbulk.h"
char *
xasprintf(const char *fmt, ...)
{
va_list ap;
char *buf;
va_start(ap, fmt);
if (vasprintf(&buf, fmt, ap) == -1)
err(1, "asprintf failed");
va_end(ap);
return buf;
}
void *
xmalloc(size_t len)
{
void *ptr;
if ((ptr = malloc(len)) == NULL)
err(1, "malloc failed");
return ptr;
}
void *
xrealloc(void *buf, size_t len)
{
void *ptr;
if ((ptr = realloc(buf, len)) == NULL)
err(1, "realloc failed");
return ptr;
}
char *
xstrdup(const char *str)
{
char *buf;
if ((buf = strdup(str)) == NULL)
err(1, "strdup failed");
return buf;
}
char *
xstrndup(const char *str, size_t len)
{
char *buf;
buf = xmalloc(len + 1);
strncpy(buf, str, len);
buf[len] = '\0';
return buf;
}
size_t
djb_hash(const char *s)
{
size_t h = 5381;
while (*s)
h = h * 33 + (size_t)(unsigned char)*s++;
return h;
}
size_t
djb_hash2(const char *s, const char *e)
{
size_t h = 5381;
while (*s && s < e)
h = h * 33 + (size_t)(unsigned char)*s++;
return h;
}

85
pkgtools/pbulk/files/pbulk/lib/atomic.c Normal file
View File

@@ -0,0 +1,85 @@
/* $NetBSD: atomic.c,v 1.2 2007/06/25 21:38:43 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <nbcompat/stat.h>
#include <nbcompat/err.h>
#include <fcntl.h>
#include <nbcompat/limits.h>
#include <nbcompat/string.h>
#include <nbcompat/unistd.h>
#include "pbulk.h"
ssize_t
atomic_read(int fd, void *real_buf, size_t bytes)
{
char *buf = real_buf;
ssize_t len;
ssize_t ret;
len = 0;
do {
ret = read(fd, buf, bytes);
if (ret == -1)
return ret;
buf += ret;
len += ret;
bytes -= ret;
} while (ret != 0 && bytes != 0);
return len;
}
ssize_t
atomic_write(int fd, const void *real_buf, size_t bytes)
{
const char *buf = real_buf;
ssize_t len;
ssize_t ret;
len = 0;
do {
ret = write(fd, buf, bytes);
if (ret == -1)
return ret;
buf += ret;
len += ret;
bytes -= ret;
} while (ret != 0 && bytes != 0);
return len;
}

377
pkgtools/pbulk/files/pbulk/lib/event.c Normal file
View File

@@ -0,0 +1,377 @@
/* $NetBSD: event.c,v 1.6 2009/08/23 18:02:04 joerg Exp $ */
/*-
* Copyright (c) 2007, 2009 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <nbcompat/types.h>
#include <sys/ioctl.h>
#ifdef __sun
#include <sys/filio.h>
#endif
#if HAVE_POLL_H
#include <poll.h>
#endif
#if HAVE_SYS_POLL_H
#include <sys/poll.h>
#endif
#include <nbcompat/time.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/unistd.h>
#include <signal.h>
#include "pbulk.h"
struct deferred_read_arg {
void *cb_arg;
void (*cb_ok)(void *);
void (*cb_error)(void *);
struct event ev;
int fd;
char *buf;
size_t remaining;
};
struct deferred_write_arg {
void *cb_arg;
void (*cb_ok)(void *);
void (*cb_error)(void *);
struct event ev;
int fd;
const char *buf;
size_t remaining;
};
static void
deferred_read_handler(int fd, void *arg)
{
struct deferred_read_arg *data = arg;
ssize_t received;
received = read(data->fd, data->buf, data->remaining);
if (received == -1 || received == 0) {
(*data->cb_error)(data->cb_arg);
free(data);
return;
}
data->buf += received;
data->remaining -= received;
if (data->remaining == 0) {
(*data->cb_ok)(data->cb_arg);
free(data);
return;
}
event_add(&data->ev, data->fd, 0, 0, deferred_read_handler, data);
}
void
deferred_read(int fd, void *buf, size_t buf_len, void *arg,
void (*cb_ok)(void *), void (*cb_error)(void *))
{
struct deferred_read_arg *data;
data = xmalloc(sizeof(*data));
data->cb_arg = arg;
data->cb_ok = cb_ok;
data->cb_error = cb_error;
data->fd = fd;
data->buf = buf;
data->remaining = buf_len;
event_add(&data->ev, data->fd, 0, 0, deferred_read_handler, data);
}
static void
deferred_write_handler(int fd, void *arg)
{
struct deferred_write_arg *data = arg;
ssize_t sent;
sent = write(data->fd, data->buf, data->remaining);
if (sent == -1 || sent == 0) {
(*data->cb_error)(data->cb_arg);
free(data);
return;
}
data->buf += sent;
data->remaining -= sent;
if (data->remaining == 0) {
(*data->cb_ok)(data->cb_arg);
free(data);
return;
}
event_add(&data->ev, data->fd, 1, 0, deferred_write_handler, data);
}
void
deferred_write(int fd, const void *buf, size_t buf_len, void *arg,
void (*cb_ok)(void *), void (*cb_error)(void *))
{
struct deferred_write_arg *data;
data = xmalloc(sizeof(*data));
data->cb_arg = arg;
data->cb_ok = cb_ok;
data->cb_error = cb_error;
data->fd = fd;
data->buf = buf;
data->remaining = buf_len;
event_add(&data->ev, data->fd, 1, 0, deferred_write_handler, data);
}
int
set_nonblocking(int fd)
{
int ioctl_arg;
ioctl_arg = 1;
if (ioctl(fd, FIONBIO, &ioctl_arg) == -1)
return -1;
return 0;
}
static size_t active_events;
static LIST_HEAD(, event) all_events;
static LIST_HEAD(, signal_event) all_signals;
static struct pollfd *poll_list;
static size_t poll_allocated;
static int signal_pipe[2];
static struct event signal_event;
static void signal_read_handler(int, void *);
void
event_init(void)
{
LIST_INIT(&all_events);
LIST_INIT(&all_signals);
if (pipe(signal_pipe))
err(1, "can't create signal pipe");
event_add(&signal_event, signal_pipe[0], 0, 1, signal_read_handler,
NULL);
set_nonblocking(signal_pipe[0]);
set_nonblocking(signal_pipe[1]);
}
void
event_add(struct event *ev, int fd, int do_write, int is_persistent,
void (*handler)(int, void *), void *arg)
{
ev->ev_fd = fd;
ev->ev_write = do_write;
ev->ev_persistent = is_persistent;
ev->ev_handler = handler;
ev->ev_arg = arg;
LIST_INSERT_HEAD(&all_events, ev, ev_link);
++active_events;
}
void
event_del(struct event *ev)
{
LIST_REMOVE(ev, ev_link);
--active_events;
}
static void
mask_all(void)
{
sigset_t mask;
sigfillset(&mask);
sigprocmask(SIG_SETMASK, &mask, NULL);
}
static void
unmask_all(void)
{
sigset_t mask;
sigemptyset(&mask);
sigprocmask(SIG_SETMASK, &mask, NULL);
}
static void
signal_read_handler(int fd, void *arg)
{
struct signal_event *sig, *next;
int got_something;
char buf[256];
for (;;) {
got_something = 0;
while (read(signal_pipe[0], buf, sizeof(buf)) > 0)
got_something = 1;
if (got_something == 0)
break;
for (sig = LIST_FIRST(&all_signals);
sig && (next = LIST_NEXT(sig, sig_link), 1);
sig = next) {
if (sig->sig_received) {
sig->sig_received = 0;
(*sig->sig_handler)(sig);
}
}
}
}
static void
signal_handler(int sig_id)
{
struct signal_event *sig;
char buf = 0;
int old_errno;
LIST_FOREACH(sig, &all_signals, sig_link) {
if (sig->sig_id == sig_id)
sig->sig_received = 1;
}
old_errno = errno;
write(signal_pipe[1], &buf, 1);
errno = old_errno;
}
void
signal_add(struct signal_event *sig, int sigtype,
void (*handler)(struct signal_event *))
{
sig->sig_id = sigtype;
sig->sig_received = 0;
sig->sig_handler = handler;
mask_all();
LIST_INSERT_HEAD(&all_signals, sig, sig_link);
signal(sigtype, signal_handler);
unmask_all();
}
void
signal_del(struct signal_event *sig)
{
mask_all();
LIST_REMOVE(sig, sig_link);
/* XXX Unset signal handler */
unmask_all();
}
static struct timeval exit_time;
void
event_dispatch(void)
{
struct timeval now;
struct event *ev, *next;
struct pollfd *iter, *last_iter;
int ret, timeout;
loop:
if (active_events > poll_allocated) {
if (poll_allocated == 0)
poll_allocated = 512;
while (active_events > poll_allocated)
poll_allocated <<= 1;
poll_list = xrealloc(poll_list,
sizeof(struct pollfd) * poll_allocated);
}
iter = poll_list;
LIST_FOREACH(ev, &all_events, ev_link) {
iter->fd = ev->ev_fd;
if (ev->ev_write)
iter->events = POLLOUT;
else
iter->events = POLLIN;
++iter;
}
last_iter = iter;
if (exit_time.tv_sec || exit_time.tv_usec) {
gettimeofday(&now, NULL);
timeout = (exit_time.tv_sec - now.tv_sec) * 1000;
timeout += (exit_time.tv_usec - now.tv_usec + 999) / 1000;
if (timeout < 0)
timeout = 0;
} else
timeout = -1;
ret = poll(poll_list, active_events, timeout);
if (ret < 0 && errno != EINTR)
return;
if (ret == 0 && timeout == 0)
return;
if (ret > 0) {
iter = poll_list;
for (ev = LIST_FIRST(&all_events);
iter < last_iter && ev && (next = LIST_NEXT(ev, ev_link), 1);
ev = next, ++iter) {
if (iter->revents) {
if (!ev->ev_persistent) {
--active_events;
LIST_REMOVE(ev, ev_link);
}
(*ev->ev_handler)(ev->ev_fd, ev->ev_arg);
--ret;
}
}
}
goto loop;
}
void
event_loopexit(struct timeval *tv)
{
gettimeofday(&exit_time, NULL);
exit_time.tv_sec += tv->tv_sec;
exit_time.tv_usec += tv->tv_usec;
while (exit_time.tv_usec >= 1000000) {
exit_time.tv_usec -= 1000000;
++exit_time.tv_sec;
}
while (exit_time.tv_usec < 0) {
exit_time.tv_usec += 1000000;
--exit_time.tv_sec;
}
}

86
pkgtools/pbulk/files/pbulk/lib/exec.c Normal file
View File

@@ -0,0 +1,86 @@
/* $NetBSD: exec.c,v 1.2 2007/06/25 21:38:44 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <nbcompat/err.h>
#ifdef HAVE_INTTYPES_H
#include <inttypes.h>
#endif
#include <nbcompat/string.h>
#include <nbcompat/unistd.h>
#include "pbulk.h"
#define UNCONST(x) ((void *)(uintptr_t)(x))
pid_t
fork_chdir_exec(const char *dir, const char *cmd, const char * const *argv, int *out)
{
static const char chdir_err[] = "Cannot change directory to \"";
static const char dup2_err[] = "Cannot reassign stdout of child";
pid_t child;
int output_pipe[2];
if (pipe(output_pipe) == -1)
err(1, "Cannot create pipe for output");
if ((child = vfork()) != 0) {
if (child == -1) {
(void)close(output_pipe[0]);
(void)close(output_pipe[1]);
} else {
*out = output_pipe[0];
(void)close(output_pipe[1]);
}
return child;
}
(void)close(output_pipe[0]);
if (chdir(dir) == -1) {
(void)write(STDERR_FILENO, chdir_err, sizeof(chdir_err) - 1);
(void)write(STDERR_FILENO, dir, strlen(dir));
(void)write(STDERR_FILENO, "\".\n", 3);
_exit(1);
}
if (output_pipe[1] != STDOUT_FILENO) {
if (dup2(output_pipe[1], STDOUT_FILENO) == -1) {
(void)write(STDERR_FILENO, dup2_err, sizeof(dup2_err) - 1);
_exit(1);
}
(void)close(output_pipe[1]);
}
(void)execvp(cmd, UNCONST(argv));
_exit(1);
/* NOTREACHED */
}

514
pkgtools/pbulk/files/pbulk/lib/match.c Normal file
View File

@@ -0,0 +1,514 @@
/* $NetBSD: match.c,v 1.5 2012/03/19 12:17:15 joerg Exp $ */
/*
* Copyright © 2002 Alistair G. Crooks. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote
* products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <nbcompat/ctype.h>
#include <nbcompat/err.h>
#include <nbcompat/fnmatch.h>
#include <nbcompat/limits.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/stdio.h>
#include <nbcompat/string.h>
#include "pbulk.h"
static int dewey_cmp(const char *, int, const char *);
static int dewey_match(const char *, const char *);
static int dewey_mktest(int *, const char *);
enum {
MaxPathSize = PATH_MAX
};
enum {
DEWEY_LT,
DEWEY_LE,
DEWEY_EQ,
DEWEY_GE,
DEWEY_GT,
DEWEY_NE
};
const char *
pkg_order(const char *match1, const char *match2)
{
const char *v1, *v2;
v1 = strrchr(match1, '-');
v2 = strrchr(match2, '-');
if (v1 == NULL || v2 == NULL)
errx(1, "Internal error");
++v1;
++v2;
if (dewey_cmp(v1, DEWEY_GT, v2))
return match1;
else if (dewey_cmp(v2, DEWEY_GT, v1))
return match2;
else if (strcmp(match1, match2) > 0)
return match1;
else
return match2;
}
#ifndef MIN
#define MIN(a,b) (((a) < (b)) ? (a) : (b))
#endif
#ifndef MAX
#define MAX(a,b) (((a) > (b)) ? (a) : (b))
#endif
#define NEWARRAY(type,ptr,size,where,action) do { \
if ((ptr = (type *) calloc(sizeof(type), (unsigned)(size))) == NULL) { \
warn("%s: can't allocate %lu bytes", where, \
(unsigned long)(size * sizeof(type))); \
action; \
} \
} while( /* CONSTCOND */ 0)
#define RENEW(type,ptr,size,where,action) do { \
type *newptr; \
if ((newptr = (type *) realloc(ptr, sizeof(type) * (size))) == NULL) { \
warn("%s: can't realloc %lu bytes", where, \
(unsigned long)(size * sizeof(type))); \
action; \
} \
ptr = newptr; \
} while( /* CONSTCOND */ 0)
#define NEW(type, ptr, where, action) NEWARRAY(type, ptr, 1, where, action)
#define FREE(ptr) (void) free(ptr)
#define ALLOC(type, v, size, c, init, where, action) do { \
if (size == 0) { \
size = init; \
NEWARRAY(type, v, size, where ": new", action); \
} else if (c == size) { \
size *= 2; \
RENEW(type, v, size, where ": renew", action); \
} \
} while( /* CONSTCOND */ 0)
#define PKG_PATTERN_MAX 1024
typedef int (*matchfn) (const char *, void *);
/* do not modify these values, or things will NOT work */
enum {
Alpha = -3,
Beta = -2,
RC = -1,
Dot = 0,
Patch = 1
};
/* this struct defines a version number */
typedef struct arr_t {
unsigned c; /* # of version numbers */
unsigned size; /* size of array */
int *v; /* array of decimal numbers */
int netbsd; /* any "nb" suffix */
} arr_t;
/* this struct describes a test */
typedef struct test_t {
const char *s; /* string representation */
unsigned len; /* length of string */
int t; /* enumerated type of test */
} test_t;
/* the tests that are recognised. */
static const test_t tests[] = {
{ "<=", 2, DEWEY_LE },
{ "<", 1, DEWEY_LT },
{ ">=", 2, DEWEY_GE },
{ ">", 1, DEWEY_GT },
{ "==", 2, DEWEY_EQ },
{ "!=", 2, DEWEY_NE },
{ NULL, 0, 0 }
};
static const test_t modifiers[] = {
{ "alpha", 5, Alpha },
{ "beta", 4, Beta },
{ "pre", 3, RC },
{ "rc", 2, RC },
{ "pl", 2, Dot },
{ "_", 1, Dot },
{ ".", 1, Dot },
{ NULL, 0, 0 }
};
/* locate the test in the tests array */
static int
dewey_mktest(int *op, const char *test)
{
const test_t *tp;
for (tp = tests ; tp->s ; tp++) {
if (strncasecmp(test, tp->s, tp->len) == 0) {
*op = tp->t;
return tp->len;
}
}
return -1;
}
/*
* make a component of a version number.
* '.' encodes as Dot which is '0'
* '_' encodes as 'patch level', or 'Dot', which is 0.
* 'pl' encodes as 'patch level', or 'Dot', which is 0.
* 'alpha' encodes as 'alpha version', or Alpha, which is -3.
* 'beta' encodes as 'beta version', or Beta, which is -2.
* 'rc' encodes as 'release candidate', or RC, which is -1.
* 'nb' encodes as 'netbsd version', which is used after all other tests
*/
static int
mkcomponent(arr_t *ap, const char *num)
{
static const char alphas[] = "abcdefghijklmnopqrstuvwxyz";
const test_t *modp;
int n;
const char *cp;
if (*num == 0) {
return 0;
}
ALLOC(int, ap->v, ap->size, ap->c, 62, "mkver", exit(EXIT_FAILURE));
if (isdigit((unsigned char)*num)) {
for (cp = num, n = 0 ; isdigit((unsigned char)*num) ; num++) {
n = (n * 10) + (*num - '0');
}
ap->v[ap->c++] = n;
return (int)(num - cp);
}
for (modp = modifiers ; modp->s ; modp++) {
if (strncasecmp(num, modp->s, modp->len) == 0) {
ap->v[ap->c++] = modp->t;
return modp->len;
}
}
if (strncasecmp(num, "nb", 2) == 0) {
for (cp = num, num += 2, n = 0 ; isdigit((unsigned char)*num) ; num++) {
n = (n * 10) + (*num - '0');
}
ap->netbsd = n;
return (int)(num - cp);
}
if (isalpha((unsigned char)*num)) {
ap->v[ap->c++] = Dot;
cp = strchr(alphas, tolower((unsigned char)*num));
ALLOC(int, ap->v, ap->size, ap->c, 62, "mkver", exit(EXIT_FAILURE));
ap->v[ap->c++] = (int)(cp - alphas) + 1;
return 1;
}
return 1;
}
/* make a version number string into an array of comparable 64bit ints */
static int
mkversion(arr_t *ap, const char *num)
{
(void) memset(ap, 0, sizeof(arr_t));
while (*num) {
num += mkcomponent(ap, num);
}
return 1;
}
#define DIGIT(v, c, n) (((n) < (c)) ? v[n] : 0)
/* compare the result against the test we were expecting */
static int
result(int cmp, int tst)
{
switch(tst) {
case DEWEY_LT:
return cmp < 0;
case DEWEY_LE:
return cmp <= 0;
case DEWEY_GT:
return cmp > 0;
case DEWEY_GE:
return cmp >= 0;
case DEWEY_EQ:
return cmp == 0;
case DEWEY_NE:
return cmp != 0;
default:
return 0;
}
}
/* do the test on the 2 vectors */
static int
vtest(arr_t *lhs, int tst, arr_t *rhs)
{
unsigned int c, i;
int cmp;
for (i = 0, c = MAX(lhs->c, rhs->c) ; i < c ; i++) {
if ((cmp = DIGIT(lhs->v, lhs->c, i) - DIGIT(rhs->v, rhs->c, i)) != 0) {
return result(cmp, tst);
}
}
return result(lhs->netbsd - rhs->netbsd, tst);
}
/*
* Compare two dewey decimal numbers
*/
static int
dewey_cmp(const char *lhs, int op, const char *rhs)
{
arr_t right;
arr_t left;
(void) memset(&left, 0, sizeof(left));
if (!mkversion(&left, lhs)) {
return 0;
}
(void) memset(&right, 0, sizeof(right));
if (!mkversion(&right, rhs)) {
return 0;
}
return vtest(&left, op, &right);
}
/*
* Perform dewey match on "pkg" against "pattern".
* Return 1 on match, 0 on non-match, -1 on error.
*/
static int
dewey_match(const char *pattern, const char *pkg)
{
const char *version;
const char *sep, *sep2;
int op, op2;
int n;
/* compare names */
if ((version=strrchr(pkg, '-')) == NULL) {
return 0;
}
if ((sep = strpbrk(pattern, "<>")) == NULL)
return -1;
/* compare name lengths */
if ((sep-pattern != version-pkg) ||
strncmp(pkg, pattern, (size_t)(version-pkg)) != 0)
return 0;
version++;
/* extract comparison operator */
if ((n = dewey_mktest(&op, sep)) < 0) {
return 0;
}
/* skip operator */
sep += n;
/* if greater than, look for less than */
sep2 = NULL;
if (op == DEWEY_GT || op == DEWEY_GE) {
if ((sep2 = strchr(sep, '<')) != NULL) {
if ((n = dewey_mktest(&op2, sep2)) < 0) {
return 0;
}
/* compare upper limit */
if (!dewey_cmp(version, op2, sep2+n))
return 0;
}
}
/* compare only pattern / lower limit */
if (sep2) {
char ver[PKG_PATTERN_MAX];
strlcpy(ver, sep, MIN((ssize_t)sizeof(ver), sep2-sep+1));
if (dewey_cmp(version, op, ver))
return 1;
}
else {
if (dewey_cmp(version, op, sep))
return 1;
}
return 0;
}
/*
* FreeBSD install - a package for the installation and maintainance
* of non-core utilities.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* Jordan K. Hubbard
* 18 July 1993
*
* Miscellaneous string utilities.
*
*/
/*
* Perform alternate match on "pkg" against "pattern",
* calling pmatch (recursively) to resolve any other patterns.
* Return 1 on match, 0 otherwise
*/
static int
alternate_match(const char *pattern, const char *pkg)
{
char *sep;
char buf[MaxPathSize];
char *last;
char *alt;
char *cp;
int cnt;
int found;
if ((sep = strchr(pattern, '{')) == (char *) NULL) {
errx(EXIT_FAILURE, "alternate_match(): '{' expected in `%s'", pattern);
}
(void) strncpy(buf, pattern, (size_t) (sep - pattern));
alt = &buf[sep - pattern];
last = (char *) NULL;
for (cnt = 0, cp = sep; *cp && last == (char *) NULL; cp++) {
if (*cp == '{') {
cnt++;
} else if (*cp == '}' && --cnt == 0 && last == (char *) NULL) {
last = cp + 1;
}
}
if (cnt != 0) {
errx(EXIT_FAILURE, "Malformed alternate `%s'", pattern);
}
for (found = 0, cp = sep + 1; *sep != '}'; cp = sep + 1) {
for (cnt = 0, sep = cp; cnt > 0 || (cnt == 0 && *sep != '}' && *sep != ','); sep++) {
if (*sep == '{') {
cnt++;
} else if (*sep == '}') {
cnt--;
}
}
(void) snprintf(alt, sizeof(buf) - (alt - buf), "%.*s%s", (int) (sep - cp), cp, last);
if (pkg_match(buf, pkg) == 1) {
found = 1;
}
}
return found;
}
/*
* Perform glob match on "pkg" against "pattern".
* Return 1 on match, 0 otherwise
*/
static int
glob_match(const char *pattern, const char *pkg)
{
return fnmatch(pattern, pkg, FNM_PERIOD) == 0;
}
/*
* Perform simple match on "pkg" against "pattern".
* Return 1 on match, 0 otherwise
*/
static int
simple_match(const char *pattern, const char *pkg)
{
return strcmp(pattern, pkg) == 0;
}
/*
* Performs a fast check if pattern can ever match pkg.
* Returns 1 if a match is possible and 0 otherwise.
*/
static int
quick_pkg_match(const char *pattern, const char *pkg)
{
#define simple(x) (isalnum((unsigned char)(x)) || (x) == '-')
if (!simple(pattern[0]))
return 1;
if (pattern[0] != pkg[0])
return 0;
if (!simple(pattern[1]))
return 1;
if (pattern[1] != pkg[1])
return 0;
return 1;
#undef simple
}
/*
* Match pkg against pattern, return 1 if matching, 0 else
*/
int
pkg_match(const char *pattern, const char *pkg)
{
if (quick_pkg_match(pattern, pkg) == 0)
return 0;
if (strchr(pattern, '{') != (char *) NULL) {
/* emulate csh-type alternates */
return alternate_match(pattern, pkg);
}
if (strpbrk(pattern, "<>") != (char *) NULL) {
int ret;
/* perform relational dewey match on version number */
ret = dewey_match(pattern, pkg);
if (ret < 0)
errx(EXIT_FAILURE, "dewey_match returned error");
return ret;
}
if (strpbrk(pattern, "*?[]") != (char *) NULL) {
/* glob match */
return glob_match(pattern, pkg);
}
/* no alternate, dewey or glob match -> simple compare */
return simple_match(pattern, pkg);
}

79
pkgtools/pbulk/files/pbulk/lib/netaddr.c Normal file
View File

@@ -0,0 +1,79 @@
/* $NetBSD: netaddr.c,v 1.7 2008/02/21 14:40:43 tnn Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <errno.h>
#include <netinet/in.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/string.h>
#include "pbulk.h"
int
parse_sockaddr_in(const char *str, struct sockaddr_in *addr)
{
const char *port_sep;
char *port_end;
struct in_addr in;
unsigned long tmp;
if ((port_sep = strrchr(str, ':')) != NULL) {
char *addr_part = strdup(str);
addr_part[port_sep - str] = '\0';
if (inet_aton(addr_part, &in) == 0) {
free(addr_part);
return -1;
}
free(addr_part);
str = port_sep + 1;
} else {
memset(&in, 0, sizeof(in));
}
errno = 0;
tmp = strtoul(str, &port_end, 10);
if (*str == '\0' || *port_end != '\0' || errno != 0 || tmp > 0xfffful)
return -1;
addr->sin_port = htons((in_port_t)tmp);
addr->sin_addr = in;
#if !defined(__sun) && !defined(__hpux) && !defined(__INTERIX) && \
!defined(__digital__) && !defined(__linux) && !defined(__sgi)
addr->sin_len = sizeof(*addr);
#endif
addr->sin_family = AF_INET;
return 0;
}

96
pkgtools/pbulk/files/pbulk/lib/pbulk.h Normal file
View File

@@ -0,0 +1,96 @@
/* $NetBSD: pbulk.h,v 1.4 2012/11/23 12:13:35 joerg Exp $ */
/*-
* Copyright (c) 2007, 2009 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <netinet/in.h>
#include <nbcompat/queue.h>
#include <nbcompat/unistd.h>
struct event {
LIST_ENTRY(event) ev_link;
int ev_fd;
int ev_write;
int ev_persistent;
void (*ev_handler)(int, void *);
void *ev_arg;
};
struct signal_event {
LIST_ENTRY(signal_event) sig_link;
int sig_id;
volatile int sig_received;
void (*sig_handler)(struct signal_event *);
};
#if defined(__GNUC__) && __GNUC__ >= 2
char *xasprintf(const char *, ...)
__attribute__((__format__(__printf__, 1, 2)));
#else
char *xasprintf(const char *, ...);
#endif
void event_init(void);
void event_add(struct event *, int, int, int,
void (*)(int, void *), void *);
void event_del(struct event *);
void signal_add(struct signal_event *, int,
void (*)(struct signal_event *));
void signal_del(struct signal_event *);
void event_dispatch(void);
void event_loopexit(struct timeval *tv);
int set_nonblocking(int);
void deferred_read(int fd, void *, size_t, void *,
void (*)(void *), void (*)(void *));
void deferred_write(int fd, const void *, size_t,
void *, void (*)(void *), void (*)(void *));
ssize_t atomic_read(int, void *, size_t);
ssize_t atomic_write(int, const void *, size_t);
int parse_sockaddr_in(const char *, struct sockaddr_in *);
pid_t fork_chdir_exec(const char *, const char *,
const char * const *, int *);
char *read_from_child(const char *, const char *,
const char * const *);
char *read_from_file(int fd);
void *xmalloc(size_t);
void *xrealloc(void *, size_t);
char *xstrdup(const char *);
char *xstrndup(const char *, size_t);
int pkg_match(const char *, const char *);
const char *pkg_order(const char *, const char *);
size_t djb_hash(const char *);
size_t djb_hash2(const char *, const char *);

85
pkgtools/pbulk/files/pbulk/lib/read_child.c Normal file
View File

@@ -0,0 +1,85 @@
/* $NetBSD: read_child.c,v 1.4 2013/04/14 17:03:30 dholland Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <sys/wait.h>
#include <nbcompat/err.h>
#include <signal.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/string.h>
#include "pbulk.h"
char *
read_from_child(const char *dir, const char *cmd, const char * const *argv)
{
char *buf;
size_t buf_len, cur_len;
ssize_t bytes_read;
int fd, status;
pid_t child;
if ((child = fork_chdir_exec(dir, cmd, argv, &fd)) == -1)
return NULL;
cur_len = 0;
buf_len = 4096;
buf = xmalloc(buf_len + 1);
while ((bytes_read = read(fd, buf + cur_len, buf_len - cur_len)) > 0) {
cur_len += bytes_read;
if (cur_len * 2 < buf_len)
continue;
buf_len *= 2;
buf = xrealloc(buf, buf_len + 1);
}
if (bytes_read == -1) {
warn("read failed");
(void)close(fd);
(void)kill(child, SIGTERM);
(void)waitpid(child, &status, 0);
exit(1);
}
(void)close(fd);
(void)waitpid(child, &status, 0);
if (status != 0 || memchr(buf, 0, cur_len) != NULL) {
free(buf);
return NULL;
}
buf[cur_len] = '\0';
return buf;
}

73
pkgtools/pbulk/files/pbulk/lib/read_file.c Normal file
View File

@@ -0,0 +1,73 @@
/* $NetBSD: read_file.c,v 1.4 2008/01/30 21:52:09 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <nbcompat/stat.h>
#include <nbcompat/err.h>
#include <fcntl.h>
#include <nbcompat/limits.h>
#include <nbcompat/string.h>
#include <nbcompat/unistd.h>
#include "pbulk.h"
char *
read_from_file(int fd)
{
struct stat sb;
size_t input_len;
ssize_t bytes_read;
char *input;
if (fstat(fd, &sb) == -1)
err(1, "Cannot stat input");
if ((sb.st_mode & S_IFMT) != S_IFREG)
errx(1, "Input is not regular file");
if (sb.st_size > SSIZE_MAX - 1)
errx(1, "Input too large");
input_len = (size_t)sb.st_size;
input = xmalloc(input_len + 1);
if ((bytes_read = read(fd, input, input_len)) == -1)
err(1, "Failed to read input");
if (bytes_read != sb.st_size)
errx(1, "Unexpected short read");
input[input_len] = '\0';
if (strlen(input) != input_len)
errx(1, "Invalid input (NUL character found)");
return input;
}

6
pkgtools/pbulk/files/pbulk/pbuild/Makefile Normal file
View File

@@ -0,0 +1,6 @@
# $NetBSD: Makefile,v 1.1.1.1 2007/06/19 19:49:56 joerg Exp $
PROG= pbulk-build
SRCS= pbuild.c jobs.c client.c master.c stat.c
.include <bsd.prog.mk>

120
pkgtools/pbulk/files/pbulk/pbuild/client.c Normal file
View File

@@ -0,0 +1,120 @@
/* $NetBSD: client.c,v 1.4 2012/01/19 18:53:32 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <sys/socket.h>
#include <nbcompat/err.h>
#include <errno.h>
#ifdef HAVE_INTTYPES_H
#include <inttypes.h>
#endif
#include <nbcompat/stdio.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/string.h>
#include <nbcompat/unistd.h>
#include <arpa/inet.h>
#include "pbulk.h"
#include "pbuild.h"
void
client_mode(const char *client_port)
{
struct sockaddr_in dst;
uint32_t build_info_len;
ssize_t recv_bytes, sent_bytes;
char *build_info;
int fd;
if (parse_sockaddr_in(client_port, &dst))
errx(1, "Could not parse addr/port");
fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (fd == -1)
err(1, "Could not create socket");
if (connect(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
err(1, "Could not connect socket");
loop:
sent_bytes = atomic_write(fd, "G", 1);
if (sent_bytes == -1)
err(1, "Could not write to socket");
if (sent_bytes == 0)
exit(0);
if (sent_bytes != 1)
errx(1, "Premature end of stream while writing to socket");
recv_bytes = atomic_read(fd, &build_info_len, 4);
if (recv_bytes == 0 || (recv_bytes == -1 && errno == ECONNRESET))
exit(0);
if (recv_bytes == -1)
err(1, "Could not read from socket");
if (recv_bytes != 4)
errx(1, "Premature end while reading build info from socket");
build_info_len = ntohl(build_info_len);
if (build_info_len < 10 || build_info_len > 0xffffff)
errx(1, "Invalid build info length from master");
build_info = xmalloc(build_info_len + 1);
build_info[build_info_len] = '\0';
recv_bytes = atomic_read(fd, build_info, build_info_len);
if (recv_bytes == -1)
err(1, "Could not read from socket");
if ((uint32_t)recv_bytes != build_info_len || strlen(build_info) != build_info_len)
errx(1, "Premature end of stream while reading path from socket");
if (verbosity > 0) {
const char *begin, *end;
if (strncmp(build_info, "PKGNAME=", 8) != 0)
err(1, "Inconsistent build info from server");
begin = build_info + 8;
if ((end = strchr(begin, '\n')) == NULL)
err(1, "Inconsistent build info from server");
printf("Building package %.*s\n", (int)(end - begin), begin);
fflush(stdout);
}
if (build_package(build_info, build_info_len) == 0)
sent_bytes = atomic_write(fd, "D", 1);
else
sent_bytes = atomic_write(fd, "F", 1);
if (sent_bytes == -1)
err(1, "Could not write to socket");
if (sent_bytes != 1)
errx(1, "Premature end of stream while writing to socket");
free(build_info);
goto loop;
}

562
pkgtools/pbulk/files/pbulk/pbuild/jobs.c Normal file
View File

@@ -0,0 +1,562 @@
/* $NetBSD: jobs.c,v 1.14 2012/11/23 12:13:35 joerg Exp $ */
/*-
* Copyright (c) 2007, 2009, 2011 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <nbcompat/stat.h>
#include <nbcompat/err.h>
#include <fcntl.h>
#include <limits.h>
#include <stdarg.h>
#include <nbcompat/stdio.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/string.h>
#include <nbcompat/time.h>
#include <nbcompat/unistd.h>
#include "pbulk.h"
#include "pbuild.h"
static int post_initial;
static int log_success;
static int log_failed;
SLIST_HEAD(buildhash, build_job);
static void build_tree(void);
static void mark_initial(void);
static struct buildhash *get_hash_chain(const char *, size_t);
static void hash_entries(void);
static void add_to_build_list(struct build_job *);
static struct build_job *jobs;
static size_t allocated_jobs, len_jobs;
static char *scan_output_content;
static TAILQ_HEAD(, build_job) buildable_jobs;
static void
ts_printf(const char *fmt, ...)
{
struct tm *tm;
time_t now;
va_list ap;
char buf[512];
struct build_stat st;
if (verbosity >= 2) {
now = time(NULL);
tm = localtime(&now);
if (strftime(buf, sizeof(buf), "%F %R", tm) == 0)
errx(1, "Formatted time doesn't fit into buffer");
(void)printf("%s ", buf);
}
if (verbosity >= 1) {
build_stats(&st);
printf("[%lu/%lu] ", (unsigned long)len_jobs - st.open_jobs,
(unsigned long)len_jobs);
}
va_start(ap, fmt);
(void)vprintf(fmt, ap);
va_end(ap);
(void)fflush(stdout);
}
static const char *
find_content(struct build_job *job, const char *prefix)
{
size_t len = strlen(prefix);
const char *line;
for (line = job->begin; line != job->end; line = strchr(line, '\n') + 1) {
if (strncmp(line, prefix, len) == 0)
return line + len;
}
return NULL;
}
static int
pkgname_dup(struct build_job *job, const char *line)
{
const char *pkgname;
char *pkgname_end;
size_t pkgname_len;
if (strncmp(line, "PKGNAME=", 8) != 0)
return 0;
pkgname = line + 8;
pkgname_end = strchr(pkgname, '\n');
pkgname_len = pkgname_end - pkgname;
if (pkgname_end == NULL || pkgname_len < 4 ||
strcspn(pkgname, " \t\n") != pkgname_len)
return 0;
job->pkgname = xstrndup(pkgname, pkgname_len);
job->pkgname_len = pkgname_len;
return 1;
}
static const char *
pbulk_item_end(const char *line)
{
const char *line_end;
do {
line_end = strchr(line, '\n');
if (line_end == NULL)
return NULL;
line = line_end + 1;
if (strncmp(line, "PKGNAME=", 8) == 0)
return line;
} while (*line != '\0');
return line;
}
SLIST_HEAD(depth_tree_head, build_job);
static int
compute_tree_depth_rec(struct build_job *job, struct build_job *root,
struct depth_tree_head *head, int *count)
{
struct dependency_list *dep_iter;
struct build_job *job_iter;
if (job == root && *count != 0) {
fprintf(stderr, "Cyclic dependency for package:\n%s\n", job->pkgname);
return -1;
}
SLIST_FOREACH(job_iter, head, depth_tree_link) {
if (job_iter == job)
return 0;
}
SLIST_INSERT_HEAD(head, job, depth_tree_link);
*count = *count + 1;
SLIST_FOREACH(dep_iter, &job->depending_pkgs, depends_link) {
if (compute_tree_depth_rec(dep_iter->dependency, root,
head, count)) {
fprintf(stderr, "%s\n", job->pkgname);
return -1;
}
}
return 0;
}
static void
compute_tree_depth(struct build_job *job)
{
struct depth_tree_head head;
SLIST_INIT(&head);
job->pkg_depth = 0;
if (compute_tree_depth_rec(job, job, &head, &job->pkg_depth))
exit(1);
}
void
init_jobs(const char *scan_output, const char *success_file, const char *error_file)
{
char *input;
const char *input_iter;
int fd;
size_t i;
TAILQ_INIT(&buildable_jobs);
if ((fd = open(scan_output, O_RDONLY, 0)) == -1)
err(1, "Cannot open input");
log_success = open(success_file, O_RDWR | O_CREAT | O_APPEND, 0666);
if (log_success == -1)
err(1, "Cannot open log file for successful builds");
log_failed = open(error_file, O_RDWR | O_CREAT | O_APPEND, 0666);
if (log_failed == -1)
err(1, "Cannot open log file for failed builds");
input = read_from_file(fd);
(void)close(fd);
if (allocated_jobs == 0) {
allocated_jobs = 1024;
jobs = xmalloc(allocated_jobs * sizeof(*jobs));
}
input_iter = input;
while (pkgname_dup(&jobs[len_jobs], input_iter)) {
jobs[len_jobs].begin = input_iter;
jobs[len_jobs].end = pbulk_item_end(input_iter);
jobs[len_jobs].state = JOB_INIT;
jobs[len_jobs].open_depends = 0;
SLIST_INIT(&jobs[len_jobs].depending_pkgs);
if (jobs[len_jobs].end == NULL)
errx(1, "Invalid input");
input_iter = jobs[len_jobs].end;
++len_jobs;
if (len_jobs == allocated_jobs) {
allocated_jobs *= 2;
jobs = xrealloc(jobs, allocated_jobs * sizeof(*jobs));
}
}
if (*input_iter != '\0')
errx(1, "Invalid input");
scan_output_content = input;
hash_entries();
build_tree();
for (i = 0; i < len_jobs; ++i)
compute_tree_depth(&jobs[i]);
mark_initial();
for (i = 0; i < len_jobs; ++i) {
if (jobs[i].state == JOB_INIT)
process_job(&jobs[i], JOB_OPEN, 0);
}
}
static void
mark_initial_state(int fd, enum job_state state, const char *type)
{
char *input;
const char *input_iter;
struct build_job *iter;
size_t len;
input = read_from_file(fd);
input_iter = input;
while (*input_iter != '\0') {
len = strcspn(input_iter, "\n");
SLIST_FOREACH(iter, get_hash_chain(input_iter, len), hash_link) {
if (iter->pkgname_len == len &&
strncmp(iter->pkgname, input_iter, len) == 0)
break;
}
if (iter == NULL)
errx(1, "Package from %s build log doesn't exist: %.*s", type, (int)len, input_iter);
process_job(iter, state, 0);
input_iter = strchr(input_iter, '\n');
if (input_iter == NULL)
errx(1, "Invalid input");
++input_iter;
}
free(input);
}
static void
mark_initial(void)
{
const char *line;
size_t i;
for (i = 0; i < len_jobs; ++i) {
if ((line = find_content(&jobs[i], "PKG_SKIP_REASON=")) == NULL)
errx(1, "Invalid scan output");
if (*line != '\n') {
process_job(&jobs[i], JOB_PREFAILED, 0);
continue;
}
if ((line = find_content(&jobs[i], "PKG_FAIL_REASON=")) == NULL)
errx(1, "Invalid scan output");
if (*line != '\n') {
process_job(&jobs[i], JOB_PREFAILED, 0);
continue;
}
}
mark_initial_state(log_success, JOB_DONE, "successful");
mark_initial_state(log_failed, JOB_FAILED, "failing");
if (verbosity >= 1)
printf("Initialisation complete.\n");
post_initial = 1;
}
static void
add_to_build_list(struct build_job *job)
{
struct build_job *iter;
TAILQ_FOREACH(iter, &buildable_jobs, build_link) {
if (iter->pkg_depth < job->pkg_depth) {
TAILQ_INSERT_BEFORE(iter, job, build_link);
return;
}
}
TAILQ_INSERT_TAIL(&buildable_jobs, job, build_link);
}
static void
build_tree(void)
{
size_t i, len;
struct build_job *iter, *job;
struct dependency_list *dep;
const char *depends;
for (i = 0; i < len_jobs; ++i) {
job = &jobs[i];
if ((depends = find_content(job, "DEPENDS=")) == NULL)
continue;
depends += strspn(depends, " \t");
while ((len = strcspn(depends, " \t\n")) != 0) {
SLIST_FOREACH(iter, get_hash_chain(depends, len), hash_link) {
if (iter->pkgname_len == len &&
strncmp(iter->pkgname, depends, len) == 0)
break;
}
if (iter == NULL)
errx(1, "Dependency `%.*s' doesn't exist", (int)strcspn(depends, " \t\n"), depends);
dep = xmalloc(sizeof(*dep));
dep->dependency = job;
SLIST_INSERT_HEAD(&iter->depending_pkgs, dep, depends_link);
++job->open_depends;
depends += len;
depends += strspn(depends, " \t");
}
}
}
struct build_job *
get_job(void)
{
struct build_job *job;
if ((job = TAILQ_FIRST(&buildable_jobs)) != NULL) {
TAILQ_REMOVE(&buildable_jobs, job, build_link);
process_job(job, JOB_IN_PROCESSING, 0);
}
return job;
}
static void
recursive_mark_broken(struct build_job *job, enum job_state state)
{
struct dependency_list *iter;
SLIST_FOREACH(iter, &job->depending_pkgs, depends_link) {
if (iter->dependency->state == JOB_OPEN ||
iter->dependency->state == JOB_INIT)
process_job(iter->dependency, state, 0);
}
}
/**
* Changes the state of the ''job'' to ''state'' and runs some code
* depending on the new state. If ''log_state'' is non-zero, the package
* name is written to either the "error" or the "success" log, depending
* on the ''state''.
*/
void
process_job(struct build_job *job, enum job_state state, int log_state)
{
struct dependency_list *iter;
char *buf;
job->state = state;
switch (state) {
case JOB_INIT:
errx(1, "Programming error");
case JOB_DONE:
SLIST_FOREACH(iter, &job->depending_pkgs, depends_link) {
--iter->dependency->open_depends;
if (iter->dependency->open_depends == 0 &&
iter->dependency->state == JOB_OPEN)
add_to_build_list(iter->dependency);
}
if (log_state) {
buf = xasprintf("%s\n", job->pkgname);
if (write(log_success, buf, strlen(buf)) == -1)
err(1, "Cannot log successful build");
free(buf);
}
if (verbosity >= 1 && post_initial)
ts_printf("Successfully built %s\n", job->pkgname);
break;
case JOB_FAILED:
if (log_state) {
buf = xasprintf("%s\n", job->pkgname);
if (write(log_failed, buf, strlen(buf)) == -1)
err(1, "Cannot log failed build");
free(buf);
}
if (verbosity >= 1 && post_initial)
ts_printf("Failed to build %s\n", job->pkgname);
/* FALLTHROUGH */
case JOB_INDIRECT_FAILED:
recursive_mark_broken(job, JOB_INDIRECT_FAILED);
break;
case JOB_PREFAILED:
case JOB_INDIRECT_PREFAILED:
/* FALLTHROUGH */
recursive_mark_broken(job, JOB_INDIRECT_PREFAILED);
break;
case JOB_IN_PROCESSING:
if (verbosity >= 1)
ts_printf("Starting build of %s\n", job->pkgname);
break;
case JOB_OPEN:
if (job->open_depends == 0)
add_to_build_list(job);
break;
}
}
void
build_stats(struct build_stat *st)
{
size_t i;
st->open_jobs = 0;
st->in_processing = 0;
st->failed = 0;
st->prefailed = 0;
st->indirect_failed = 0;
st->indirect_prefailed = 0;
st->done = 0;
for (i = 0; i < len_jobs; ++i) {
switch (jobs[i].state) {
case JOB_DONE:
++st->done;
break;
case JOB_FAILED:
++st->failed;
break;
case JOB_PREFAILED:
++st->prefailed;
break;
case JOB_INDIRECT_FAILED:
++st->indirect_failed;
break;
case JOB_INDIRECT_PREFAILED:
++st->indirect_prefailed;
break;
case JOB_OPEN:
++st->open_jobs;
break;
case JOB_IN_PROCESSING:
++st->in_processing;
break;
case JOB_INIT:
break;
}
}
}
void
finish_build(const char *report_file)
{
FILE *report;
size_t i;
const char *status;
if ((report = fopen(report_file, "w")) == NULL)
err(1, "Cannot open report file");
for (i = 0; i < len_jobs; ++i) {
switch (jobs[i].state) {
case JOB_DONE:
status = "done";
break;
case JOB_FAILED:
status = "failed";
break;
case JOB_PREFAILED:
status = "prefailed";
break;
case JOB_INDIRECT_FAILED:
status = "indirect-failed";
break;
case JOB_INDIRECT_PREFAILED:
status = "indirect-prefailed";
break;
case JOB_OPEN:
status = "open";
break;
case JOB_IN_PROCESSING:
status = "in_processing";
break;
default:
errx(1, "internal error");
}
fprintf(report, "%s|%s||%d\n", jobs[i].pkgname, status,
jobs[i].pkg_depth);
}
}
#define HASH_SIZE 4096
static size_t
hash_item(const char *s, size_t len)
{
return djb_hash2(s, s + len) % HASH_SIZE;
}
static struct buildhash hash_table[HASH_SIZE];
static void
hash_entries(void)
{
size_t i, hash;
for (i = 0; i < HASH_SIZE; ++i)
SLIST_INIT(&hash_table[i]);
for (i = 0; i < len_jobs; ++i) {
hash = hash_item(jobs[i].pkgname, jobs[i].pkgname_len);
SLIST_INSERT_HEAD(&hash_table[hash], &jobs[i], hash_link);
}
}
static struct buildhash *
get_hash_chain(const char *pkgname, size_t len)
{
return &hash_table[hash_item(pkgname, len)];
}

335
pkgtools/pbulk/files/pbulk/pbuild/master.c Normal file
View File

@@ -0,0 +1,335 @@
/* $NetBSD: master.c,v 1.9 2013/01/14 14:33:28 jperkin Exp $ */
/*-
* Copyright (c) 2007, 2009 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <nbcompat/types.h>
#include <sys/ioctl.h>
#include <nbcompat/queue.h>
#include <sys/socket.h>
#include <nbcompat/time.h>
#include <sys/wait.h>
#include <nbcompat/err.h>
#include <fcntl.h>
#include <signal.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/stdio.h>
#include <nbcompat/string.h>
#include <arpa/inet.h>
#include "pbulk.h"
#include "pbuild.h"
static int clients_started;
static LIST_HEAD(, build_peer) active_peers, inactive_peers, unassigned_peers;
static struct event listen_event;
static int listen_event_socket;
static struct signal_event child_event;
static pid_t child_pid;
struct build_peer {
LIST_ENTRY(build_peer) peer_link;
struct build_job *job;
int fd;
char tmp_buf[4];
char *buf;
};
static void assign_job(void *);
static void recv_command(struct build_peer *);
static void
kill_peer(void *arg)
{
struct build_peer *peer = arg;
(void)close(peer->fd);
LIST_REMOVE(peer, peer_link);
if (peer->job != NULL)
process_job(peer->job, JOB_OPEN, 1);
free(peer->buf);
free(peer);
peer = LIST_FIRST(&unassigned_peers);
if (peer != NULL)
assign_job(peer);
}
static void
finish_job(void *arg)
{
struct build_peer *peer = arg;
LIST_REMOVE(peer, peer_link);
if (peer->tmp_buf[0] == 'D')
process_job(peer->job, JOB_DONE, 1);
else if (peer->tmp_buf[0] == 'F')
process_job(peer->job, JOB_FAILED, 1);
else
kill_peer(peer);
peer->job = NULL;
recv_command(peer);
peer = LIST_FIRST(&unassigned_peers);
if (peer != NULL)
assign_job(peer);
}
static void
recv_status(void *arg)
{
struct build_peer *peer = arg;
deferred_read(peer->fd, peer->tmp_buf, 1, peer, finish_job,
kill_peer);
}
static void
send_build_info(void *arg)
{
struct build_peer *peer = arg;
deferred_write(peer->fd, peer->job->begin, peer->job->end - peer->job->begin, peer, recv_status,
kill_peer);
}
static void
sent_build_stats(void *arg)
{
struct build_peer *peer = arg;
free(peer->buf);
peer->buf = NULL;
assign_job(peer);
}
static void
send_build_stats(struct build_peer *peer)
{
struct build_stat st;
uint32_t tmp;
build_stats(&st);
peer->buf = xmalloc(7 * 4);
tmp = htonl(st.open_jobs);
(void)memcpy(peer->buf, &tmp, 4);
tmp = htonl(st.in_processing);
(void)memcpy(peer->buf + 4, &tmp, 4);
tmp = htonl(st.failed);
(void)memcpy(peer->buf + 8, &tmp, 4);
tmp = htonl(st.prefailed);
(void)memcpy(peer->buf + 12, &tmp, 4);
tmp = htonl(st.indirect_failed);
(void)memcpy(peer->buf + 16, &tmp, 4);
tmp = htonl(st.indirect_prefailed);
(void)memcpy(peer->buf + 20, &tmp, 4);
tmp = htonl(st.done);
(void)memcpy(peer->buf + 24, &tmp, 4);
deferred_write(peer->fd, peer->buf, 7 * 4, peer, sent_build_stats, kill_peer);
}
static void
shutdown_master(void)
{
struct timeval tv;
struct build_peer *peer;
event_del(&listen_event);
(void)close(listen_event_socket);
LIST_FOREACH(peer, &inactive_peers, peer_link)
(void)shutdown(peer->fd, SHUT_RDWR);
tv.tv_sec = 1;
tv.tv_usec = 0;
event_loopexit(&tv);
}
static void
assign_job(void *arg)
{
struct build_peer *peer = arg;
size_t build_info_len;
uint32_t net_build_info_len;
if (peer->tmp_buf[0] == 'S') {
send_build_stats(peer);
return;
}
if (peer->tmp_buf[0] != 'G') {
kill_peer(peer);
return;
}
LIST_REMOVE(peer, peer_link);
peer->job = clients_started ? get_job() : NULL;
if (peer->job == NULL) {
LIST_INSERT_HEAD(&unassigned_peers, peer, peer_link);
if (LIST_EMPTY(&active_peers) && clients_started)
shutdown_master();
return;
}
LIST_INSERT_HEAD(&active_peers, peer, peer_link);
build_info_len = peer->job->end - peer->job->begin;
if (build_info_len > 0xffffff)
errx(1, "Build info too long");
net_build_info_len = htonl(build_info_len);
(void)memcpy(peer->tmp_buf, &net_build_info_len, 4);
deferred_write(peer->fd, peer->tmp_buf, 4, peer, send_build_info,
kill_peer);
peer = LIST_FIRST(&unassigned_peers);
if (peer != NULL)
assign_job(peer);
}
static void
recv_command(struct build_peer *peer)
{
LIST_INSERT_HEAD(&inactive_peers, peer, peer_link);
deferred_read(peer->fd, peer->tmp_buf, 1, peer, assign_job,
kill_peer);
}
static void
listen_handler(int sock, void *arg)
{
struct build_peer *peer;
struct sockaddr_in src;
socklen_t src_len;
int fd;
src_len = sizeof(src);
if ((fd = accept(sock, (struct sockaddr *)&src, &src_len)) == -1) {
warn("Could not accept connection");
return;
}
if (set_nonblocking(fd) == -1) {
(void)close(fd);
warn("Could not set non-blocking IO");
return;
}
peer = xmalloc(sizeof(*peer));
peer->fd = fd;
peer->buf = NULL;
peer->job = NULL;
recv_command(peer);
}
static void
child_handler(struct signal_event *ev)
{
struct build_peer *peer;
int status;
if (waitpid(child_pid, &status, WNOHANG) == -1) {
if (errno == ECHILD)
return;
err(1, "Could not wait for child");
}
if (status != 0)
err(1, "Start script failed");
clients_started = 1;
signal_del(ev);
if ((peer = LIST_FIRST(&inactive_peers)) != NULL) {
LIST_REMOVE(peer, peer_link);
assign_job(peer);
}
}
void
master_mode(const char *master_port, const char *start_script)
{
struct sockaddr_in dst;
int fd;
LIST_INIT(&active_peers);
LIST_INIT(&inactive_peers);
LIST_INIT(&unassigned_peers);
event_init();
if (parse_sockaddr_in(master_port, &dst))
errx(1, "Could not parse addr/port");
fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (fd == -1)
err(1, "Could not create socket");
if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1)
err(1, "Could not set close-on-exec flag");
if (bind(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
err(1, "Could not bind socket");
if (listen(fd, 5) == -1)
err(1, "Could not listen on socket");
event_add(&listen_event, fd, 0, 1, listen_handler, NULL);
listen_event_socket = fd;
if (start_script) {
signal_add(&child_event, SIGCHLD, child_handler);
if ((child_pid = vfork()) == 0) {
execlp(start_script, start_script, (char *)NULL);
_exit(255);
}
if (child_pid == -1)
err(1, "Could not fork start script");
} else {
clients_started = 1;
}
event_dispatch();
(void)close(fd);
}

246
pkgtools/pbulk/files/pbulk/pbuild/pbuild.c Normal file
View File

@@ -0,0 +1,246 @@
/* $NetBSD: pbuild.c,v 1.6 2013/01/14 14:33:28 jperkin Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <sys/wait.h>
#include <nbcompat/err.h>
#include <errno.h>
#include <signal.h>
#include <nbcompat/stdio.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/string.h>
#include <nbcompat/time.h>
#include "pbulk.h"
#include "pbuild.h"
int verbosity;
static const char *build_path;
static const char *build_cmd;
static void standalone_mode(void);
static void
usage(void)
{
(void)fprintf(stderr, "usage: pbulk-build -s <master>\n");
(void)fprintf(stderr, "usage: pbulk-build [ -v ] -c <master> -b <build script>\n");
(void)fprintf(stderr, "usage: pbulk-build [ -v ] [ -I <start> ] [ -r <report> ] -m <port> <tree scan> <success file> <error file>\n");
(void)fprintf(stderr, "usage: pbulk-build [ -v ] [ -r <report> ] -b <build script> <tree scan> <success file> <error file>\n");
exit(1);
}
int
main(int argc, char **argv)
{
const char *stat_port;
const char *client_port;
const char *master_port;
const char *report_file;
const char *start_script;
struct sigaction sa;
int ch, modes;
setprogname("pbulk-build");
client_port = NULL;
master_port = NULL;
stat_port = NULL;
start_script = NULL;
report_file = NULL;
modes = 0;
while ((ch = getopt(argc, argv, "I:b:c:m:r:s:v")) != -1) {
switch (ch) {
case 'I':
start_script = optarg;
break;
case 'b':
build_path = optarg;
break;
case 'c':
client_port = optarg;
++modes;
break;
case 'm':
master_port = optarg;
++modes;
break;
case 'r':
report_file = optarg;
break;
case 's':
stat_port = optarg;
++modes;
break;
case 'v':
++verbosity;
break;
default:
usage();
}
}
argc -= optind;
argv += optind;
if (modes > 1) {
warnx("Only one of client mode, statistic mode or master mode can be active");
usage();
}
#if !defined(__INTERIX)
sa.sa_sigaction = NULL;
#endif
sa.sa_handler = SIG_IGN;
sa.sa_flags = 0;
(void)sigemptyset(&sa.sa_mask);
(void)sigaction(SIGPIPE, (struct sigaction *)&sa, NULL);
if (build_path == NULL && (modes == 0 || client_port != NULL))
usage();
if (stat_port != NULL) {
if (argc != 0)
usage();
stat_mode(stat_port);
}
if (master_port == NULL) {
build_cmd = strrchr(build_path, '/');
if (build_cmd == NULL)
build_cmd = build_path;
else
++build_cmd;
}
if (client_port != NULL) {
if (argc != 0)
usage();
client_mode(client_port);
}
if (argc != 3)
usage();
if (verbosity >= 2)
tzset();
init_jobs(argv[0], argv[1], argv[2]);
if (master_port != NULL)
master_mode(master_port, start_script);
else
standalone_mode();
if (report_file)
finish_build(report_file);
return 0;
}
int
build_package(const char *build_info, size_t len)
{
struct sigaction sa;
int input[2];
pid_t child;
if (pipe(input) == -1)
err(1, "Failed to create pipe");
child = vfork();
if (child == -1)
err(1, "Failed to create child");
if (child != 0) {
ssize_t bytes_written;
const char *begin;
size_t bytes_left;
int ret;
(void)close(input[0]);
begin = build_info;
bytes_left = len;
while (bytes_left > 0) {
bytes_written = write(input[1], begin, bytes_left);
if (bytes_written == -1 && errno == EPIPE)
break;
if (bytes_written <= 0) {
(void)close(input[1]);
(void)kill(child, SIGTERM);
(void)waitpid(child, &ret, 0);
return 1;
}
bytes_left -= bytes_written;
begin += bytes_written;
}
(void)close(input[1]);
(void)waitpid(child, &ret, 0);
return ret;
}
/* Reset SIGPIPE handling for child */
#if !defined(__INTERIX)
sa.sa_sigaction = NULL;
#endif
sa.sa_handler = SIG_DFL;
sa.sa_flags = 0;
(void)sigemptyset(&sa.sa_mask);
(void)sigaction(SIGPIPE, (struct sigaction *)&sa, NULL);
(void)close(input[1]);
if (dup2(input[0], 0) == -1) {
const char err_msg[] = "dup failed for stdin\n";
(void)write(STDERR_FILENO, err_msg, sizeof(err_msg) - 1);
_exit(255);
}
(void)execl(build_path, build_cmd, (char *)NULL);
_exit(255);
}
static void
standalone_mode(void)
{
struct build_job *job;
while ((job = get_job()) != NULL) {
if (build_package(job->begin, job->end - job->begin) == 0)
process_job(job, JOB_DONE, 1);
else
process_job(job, JOB_FAILED, 1);
}
}

125
pkgtools/pbulk/files/pbulk/pbuild/pbuild.h Normal file
View File

@@ -0,0 +1,125 @@
/* $NetBSD: pbuild.h,v 1.6 2011/11/27 19:53:30 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat/queue.h>
#ifdef HAVE_INTTYPES_H
#include <inttypes.h>
#endif
/**
* At the very beginning of a pbulk, each job is in state JOB_INIT.
*
* (jobs.c, init_jobs:) If scanning the package failed or the package is
* not available for this configuration, its state is changed to
* JOB_PREFAILED. Packages that are listed in the "error" log are marked
* as JOB_FAILED, those in the the "success" log are marked as JOB_DONE.
* The remaining jobs are marked as JOB_OPEN.
*
* The packages that are ready to be built are those that have the state
* JOB_OPEN and no pending dependencies. At most one job can be in the
* state JOB_IN_PROCESSING. After trying to build a job, its state is
* set to either JOB_DONE or to JOB_FAILED. In the latter case, all the
* jobs that depend on this one are marked as JOB_INDIRECT_FAILED.
*/
enum job_state {
JOB_INIT,
JOB_OPEN,
JOB_IN_PROCESSING,
JOB_FAILED,
JOB_PREFAILED,
JOB_INDIRECT_FAILED,
JOB_INDIRECT_PREFAILED,
JOB_DONE
};
struct build_stat {
size_t open_jobs;
size_t in_processing;
size_t failed;
size_t prefailed;
size_t indirect_failed;
size_t indirect_prefailed;
size_t done;
};
struct build_job;
struct dependency_list {
struct build_job *dependency;
SLIST_ENTRY(dependency_list) depends_link;
};
struct build_job {
/** The package name, including the version number. */
char *pkgname;
size_t pkgname_len;
/**
* Pointers into the output from pbulk-resolve. The lines
* between these two pointers describe additional properties
* of the job, such as the PKGPATH in which to build the
* package. The information can be accessed with the
* find_content function.
*/
const char *begin;
const char *end;
enum job_state state;
int pkg_depth;
/**
* The number of direct dependencies that must be built before
* this package can be tried.
*/
size_t open_depends;
/** The packages that depend on this package. */
SLIST_HEAD(, dependency_list) depending_pkgs;
TAILQ_ENTRY(build_job) build_link;
SLIST_ENTRY(build_job) hash_link;
SLIST_ENTRY(build_job) depth_tree_link;
};
extern int verbosity;
void init_jobs(const char *, const char *, const char *);
struct build_job *get_job(void);
void process_job(struct build_job *, enum job_state, int);
int build_package(const char *, size_t);
void finish_build(const char *);
void build_stats(struct build_stat *);
void client_mode(const char *);
void master_mode(const char *, const char *);
void stat_mode(const char *);

132
pkgtools/pbulk/files/pbulk/pbuild/pbulk-build.1 Normal file
View File

@@ -0,0 +1,132 @@
.\" $NetBSD: pbulk-build.1,v 1.1.1.1 2007/06/19 19:49:55 joerg Exp $
.\"
.\" Copyright (c) 2007 Thomas Klausner and Joerg Sonnenberger.
.\" All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\" notice, this list of conditions and the following disclaimer.
.\" 2. Redistributions in binary form must reproduce the above copyright
.\" notice, this list of conditions and the following disclaimer in the
.\" documentation and/or other materials provided with the distribution.
.\"
.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
.\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
.\" POSSIBILITY OF SUCH DAMAGE.
.\"
.Dd June 11, 2007
.Dt PBULK-BUILD 1
.Os
.Sh NAME
.Nm pbulk-build
.Nd build all packages specified in input file
.Sh SYNOPSIS
.Nm
.Fl s Oo Ar ip: Oc Ns Ar port
.Nm
.Op Fl v
.Fl c Oo Ar ip: Oc Ns Ar port
.Fl b Ar build_script
.Nm
.Op Fl I Ar start_script
.Op Fl r Ar report_file
.Op Fl v
.Fl m Oo Ar ip: Oc Ns Ar port
.Ar input success error
.Nm
.Op Fl r Ar report_file
.Op Fl v
.Fl b Ar build_script
.Ar input success error
.Sh DESCRIPTION
.Nm
builds all packages specified in an input file.
.Pp
Supported options are:
.Bl -tag -width 15n -offset indent
.It Fl b Ar build_script
Use
.Ar build_script
for building the packages.
See
.Sx BUILD SCRIPT FORMAT
for details.
.It Fl c Oo Ar ip: Oc Ns Ar port
Obtain jobs from master running on the given
.Ar port
(or
.Ar ip:port ) .
If used with
.Fl v ,
print the name of the package to build to stdout.
.It Fl I Ar start_script
Run
.Ar start_script
after opening the socket and wait for completion before entering build loop.
.It Fl m Oo Ar ip: Oc Ns Ar port
Enter master mode.
The master binds to
.Ar port
(or
.Ar ip:port )
and waits for clients to connect and build individual packages.
.It Fl s Oo Ar ip: Oc Ns Ar port
Query the master running on the given
.Ar port
(or
.Ar ip:port )
for the current number of successful, open, and failed builds.
.It Fl r Ar report_file
Write name of each package,
the result of its build,
whether the package belongs to the restricted subset
and the size of the subtree
to
.Ar report_file
at the end of the build.
.It Fl v
Be more verbose.
Details depend on the other flags used with it.
.El
In normal mode (neither
.Fl c ,
.Fl m ,
nor
.Fl s
specified) and master mode
.Pq Fl m ,
.Nm
reads the resolved tree scan from
.Ar input .
It then writes successful builds to
.Ar success
and failing builds to
.Ar error .
If either
.Ar success
or
.Ar error
exists at start-up, they are read and the build continues where
they left off.
If
.Fl v
is specified once,
.Nm
prints the start of each build and the result.
If
.Fl v
is specified twice, each begin and end message is prefixed with
the current time.
.Ss BUILD SCRIPT FORMAT
XXX: to be documented
.\" XXX: .Sh EXIT STATUS

120
pkgtools/pbulk/files/pbulk/pbuild/stat.c Normal file
View File

@@ -0,0 +1,120 @@
/* $NetBSD: stat.c,v 1.3 2008/01/15 22:14:30 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <sys/socket.h>
#include <nbcompat/err.h>
#include <errno.h>
#ifdef HAVE_INTTYPES_H
#include <inttypes.h>
#endif
#include <nbcompat/stdlib.h>
#include <nbcompat/stdio.h>
#include <nbcompat/string.h>
#include <nbcompat/unistd.h>
#include <arpa/inet.h>
#include "pbulk.h"
#include "pbuild.h"
void
stat_mode(const char *client_port)
{
struct sockaddr_in dst;
ssize_t recv_bytes, sent_bytes;
char buf[7 * 4];
struct build_stat st;
uint32_t tmp;
int fd;
if (parse_sockaddr_in(client_port, &dst))
errx(1, "Could not parse addr/port");
fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (fd == -1)
err(1, "Could not create socket");
if (connect(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
err(1, "Could not connect socket");
sent_bytes = write(fd, "S", 1);
if (sent_bytes == -1)
err(1, "Could not write to socket");
if (sent_bytes == 0)
exit(0);
if (sent_bytes != 1)
errx(1, "Premature end of stream while writing to socket");
recv_bytes = atomic_read(fd, &buf, 7 * 4);
if (recv_bytes == 0 || (recv_bytes == -1 && errno == ECONNRESET))
exit(0);
if (recv_bytes == -1)
err(1, "Could not read from socket");
if (recv_bytes != 7 * 4)
errx(1, "Premature end while reading statistics from socket");
(void)memcpy(&tmp, buf, 4);
st.open_jobs = ntohl(tmp);
(void)memcpy(&tmp, buf + 4, 4);
st.in_processing = ntohl(tmp);
(void)memcpy(&tmp, buf + 8, 4);
st.failed = ntohl(tmp);
(void)memcpy(&tmp, buf + 12, 4);
st.prefailed = ntohl(tmp);
(void)memcpy(&tmp, buf + 16, 4);
st.indirect_failed = ntohl(tmp);
(void)memcpy(&tmp, buf + 20, 4);
st.indirect_prefailed = ntohl(tmp);
(void)memcpy(&tmp, buf + 24, 4);
st.done = ntohl(tmp);
(void)printf("Jobs not yet processed: %lu\n",
(unsigned long)st.open_jobs);
(void)printf("Jobs currently in processing: %lu\n",
(unsigned long)st.in_processing);
(void)printf("Successful builds: %lu\n",
(unsigned long)st.done);
(void)printf("Failing builds: %lu\n",
(unsigned long)(st.failed + st.prefailed + st.indirect_failed + st.indirect_prefailed));
(void)printf(" Directly broken: %lu\n",
(unsigned long)st.failed);
(void)printf(" Broken due to a broken dependency: %lu\n",
(unsigned long)st.indirect_failed);
(void)printf(" Not build as explicitly marked broken: %lu\n",
(unsigned long)st.prefailed);
(void)printf(" Broken due to an explicitly broken dependency: %lu\n",
(unsigned long)st.indirect_prefailed);
exit(0);
}

129
pkgtools/pbulk/files/pbulk/pbulk.conf Normal file
View File

@@ -0,0 +1,129 @@
# $NetBSD: pbulk.conf,v 1.19 2013/01/07 14:15:35 jperkin Exp $
# Version of the configuration file. This is bumped whenever the default
# config changes to notify the administrator about updates.
#
config_version=@PBULK_CONFIG_VERSION@
# The URL where the build report will be made available. This is only
# used in the .txt version of the report.
#
base_url=http://www.pkgsrc-box.org/reports/current/DragonFly-1.8
# If yes, keep the last scan results in ${bulklog}.old and try to reuse them.
reuse_scan_results=no
# The pbulk framework can use multiple machines to build the packages.
# On a single-processor, non-distributed build, you may want to say "no"
# here.
#
master_mode=yes
master_ip=192.168.75.10
scan_clients="192.168.75.21 192.168.75.22 192.168.75.23 192.168.75.24"
build_clients="192.168.75.21 192.168.75.22 192.168.75.23 192.168.75.24"
master_port_scan=${master_ip}:2001
master_port_build=${master_ip}:2002
# Some flags for the programs that publish the binary packages and the
# build report. If you want to disable the publishing at all, see below.
#
pkg_rsync_args="-av --delete-excluded -e ssh"
pkg_rsync_target="pkgsrc@192.168.75.1:/public/packages/current/DragonFly-1.8"
report_rsync_args="-avz --delete-excluded -e ssh"
report_rsync_target="pkgsrc@192.168.75.1:/public/reports/current/DragonFly-1.8"
report_subject_prefix="pkgsrc"
report_recipients="pkgsrc-bulk@netbsd.org"
# An archive containing the pkgsrc binary tree after bootstrapping.
#
bootstrapkit=/usr/pkgsrc/bootstrap/bootstrap.tar.gz
# Optionally build a subset of the available packages and their dependencies.
# The file is a newline separated list of package locations (e.g. lang/perl5).
#limited_list=/limited_list
# Optionally ignore unresolvable dependencies for a full build (e.g.
# a build without limited_list set). Default behavior is to bail out.
#
ignore_missing_dependencies=no
# If yes, consider a package up-to-date, if the dependency list matches
# the existing binary package and the recorded RCS IDs match the pkgsrc
# tree. Otherwise, additionally require that the package is not older
# than any of the dependencies.
#
skip_age_check=no
report_graph_script_limit=512
# Account used for user-destdir builds. This account should have
# no special permissions.
#
unprivileged_user=pbulk
# Variables used for the optional cross-compiling of packages.
#
cross_compile=no
target_arch=vax
target_destdir=/usr/src/destdir.${target_arch}
# The directories where the various files are created.
#
bulklog=/bulklog
packages=/packages
prefix=/usr/pkg
pkgsrc=/usr/pkgsrc
pkgdb=/var/db/pkg
varbase=/var
# The following programs must NOT be inside ${prefix}
pkg_info=@PKG_INFO_CMD@
pkg_add=@PKG_ADD_CMD@
pkg_delete=@PKG_DELETE_CMD@
# The tools that are used for building the packages. If you do not want
# to publish anything at all, set rsync=: and mail=:.
#
bzip2=@BZIP2@
digest=@DIGEST@
gzip="@GZIP_CMD@"
# On non-NetBSD, this should usually point at the bmake in ${prefix}, not the
# make used to build pbulk itself.
make=@TARGET_MAKE@
mail=@MAIL_CMD@
neato=@NEATO@
rsync=@PREFIX@/bin/rsync
sed=@SED@
tar=@TAR@
loc=${bulklog}/meta
pbuild=@PREFIX@/bin/pbulk-build
presolve=@PREFIX@/bin/pbulk-resolve
pscan=@PREFIX@/bin/pbulk-scan
# When a package build fails, it is often necessary to have a look at
# the working directory or the installed files. When these options are
# set to "yes", they will be archived in the log directory.
#
keep_wrkdir=no
keep_prefix=no
pkg_up_to_date_script=@PREFIX@/libexec/pbulk/pkg-up-to-date
pbuild_script=@PREFIX@/libexec/pbulk/pkg-build
pbuild_start_script=@PREFIX@/libexec/pbulk/build-client-start
pscan_prepare=@PREFIX@/libexec/pbulk/client-clean
pscan_start_script=@PREFIX@/libexec/pbulk/scan-client-start
report_script=@PREFIX@/libexec/pbulk/create-report
report_html_script=@PREFIX@/libexec/pbulk/create-report-html
report_txt_script=@PREFIX@/libexec/pbulk/create-report-txt
report_graph_script=@PREFIX@/libexec/pbulk/create-broken-graph
packages_script=@PREFIX@/libexec/pbulk/compute-packages
script_phase_pre_build=@PREFIX@/libexec/pbulk/pre-build
script_phase_build=@PREFIX@/libexec/pbulk/build
script_phase_report=@PREFIX@/libexec/pbulk/report
script_phase_scan=@PREFIX@/libexec/pbulk/scan
script_phase_upload=@PREFIX@/libexec/pbulk/upload

6
pkgtools/pbulk/files/pbulk/presolve/Makefile Normal file
View File

@@ -0,0 +1,6 @@
# $NetBSD: Makefile,v 1.2 2009/01/31 23:25:38 joerg Exp $
PROG= pbulk-resolve
SRCS= presolve.c
.include <bsd.prog.mk>

77
pkgtools/pbulk/files/pbulk/presolve/pbulk-resolve.1 Normal file
View File

@@ -0,0 +1,77 @@
.\" $NetBSD: pbulk-resolve.1,v 1.1.1.1 2007/06/19 19:49:57 joerg Exp $
.\"
.\" Copyright (c) 2007 Thomas Klausner and Joerg Sonnenberger.
.\" All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\" notice, this list of conditions and the following disclaimer.
.\" 2. Redistributions in binary form must reproduce the above copyright
.\" notice, this list of conditions and the following disclaimer in the
.\" documentation and/or other materials provided with the distribution.
.\"
.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
.\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
.\" POSSIBILITY OF SUCH DAMAGE.
.\"
.Dd June 12, 2007
.Dt PBULK-RESOLVE 1
.Os
.Sh NAME
.Nm pbulk-resolve
.Nd resolves dependencies in
.Xr pbulk-scan 1
output
.Sh SYNOPSIS
.Nm
.Op Fl v
.Op Fl i Ar missing
.Ar input Op ...
.Sh DESCRIPTION
.Nm
resolves dependencies based on the output of
.Xr pbulk-scan 1
saved in
.Ar input .
.Pp
Supported options are:
.Bl -tag -offset indent
.It Fl i Ar missing
Enter incremental mode.
For incremental mode, the package in
.Ar input
and the packages used to fulfill dependencies from the other input
files are written to standard output including the resolved
dependencies.
The location of each unresolved dependency is written to
.Ar missing .
In normal mode, unresolvable dependencies are printed and the
program exits with an error.
.It Fl v
If
.Fl v
is specified once and incremental mode is not active, a warning is
printed whenever the best matching package has a different location
than recorded in the dependency.
If
.Fl v
is specified twice, additionally a warning is printed whenever two
different packages can be used to fulfill a dependency.
.El
.Sh EXIT STATUS
.Nm
exits with return value 1 if an error occurred, or 0 if all
dependencies have been resolved successfully.
In incremental mode,
.Ar missing
contains all correct, but unresolvable dependencies.

437
pkgtools/pbulk/files/pbulk/presolve/presolve.c Normal file
View File

@@ -0,0 +1,437 @@
/* $NetBSD: presolve.c,v 1.4 2008/09/16 18:21:30 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <nbcompat/queue.h>
#include <sys/stat.h>
#include <nbcompat/ctype.h>
#include <nbcompat/err.h>
#include <fcntl.h>
#include <nbcompat/limits.h>
#include <nbcompat/stdio.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/string.h>
#include <nbcompat/unistd.h>
#include "pbulk.h"
static int partial, verbosity;
static FILE *incremental = NULL;
static void
usage(void)
{
(void)fprintf(stderr, "usage: pbulk-resolve [ -pv ] [ -i <missing> ] <pscan output> [ ... ]\n");
exit(1);
}
SLIST_HEAD(pkg_entry_hash, pkg_entry);
struct pkg_entry {
char *pkgname;
char *depends;
char *pkglocation;
int active;
int broken; /* Entry has missing dependencies */
const char *begin;
const char *end;
SLIST_ENTRY(pkg_entry) hash_link;
} *pkgs;
size_t len_pkgs, allocated_pkgs;
static char *pkgname_dup(const char *);
static const char *pbulk_item_end(const char *);
static void read_entries(const char *, int);
static int resolve_entry(struct pkg_entry *);
static void write_entries(void);
static void hash_entries(void);
static struct pkg_entry_hash *get_hash_chain(const char *);
int
main(int argc, char **argv)
{
size_t i;
int ch, ret;
setprogname("pbulk-resolve");
while ((ch = getopt(argc, argv, "i:pv")) != -1) {
switch (ch) {
case 'i':
if (incremental != NULL)
(void)fclose(incremental);
if ((incremental = fopen(optarg, "w")) == NULL)
err(1, "Cannot open output file");
break;
case 'p':
++partial;
break;
case 'v':
++verbosity;
break;
default:
usage();
}
}
argc -= optind;
argv += optind;
if (argc == 0 || (incremental == NULL && argc > 1))
usage();
if (partial && incremental != NULL)
usage();
read_entries(argv[0], 1);
while (--argc > 0)
read_entries(*++argv, 0);
hash_entries();
ret = 0;
for (i = 0; i < len_pkgs; ++i) {
if (resolve_entry(&pkgs[i]))
ret = 1;
}
if (ret == 0)
write_entries();
return ret;
}
static const char *
find_content(struct pkg_entry *pkg, const char *prefix)
{
size_t len = strlen(prefix);
const char *line;
for (line = pkg->begin; line != pkg->end; line = strchr(line, '\n') + 1) {
if (strncmp(line, prefix, len) == 0)
return line + len;
}
return NULL;
}
static void
log_multi_match(const char *pattern, const char *pkgname, const char *match)
{
if (verbosity < 2)
return;
warnx("Multiple matches for dependency %s of package %s: %s", pattern, pkgname, match);
}
static void
find_match_iter(const char *pattern, const char *pkgname, struct pkg_entry **best, struct pkg_entry *cur, size_t *matches)
{
if (pkg_match(pattern, cur->pkgname) == 0)
return;
if (*matches == 0) {
*best = cur;
++(*matches);
return;
}
if (*matches == 1)
log_multi_match(pattern, pkgname, (*best)->pkgname);
log_multi_match(pattern, pkgname, (*best)->pkgname);
if (pkg_order((*best)->pkgname, cur->pkgname) == cur->pkgname)
*best = cur;
++(*matches);
}
static void
validate_best_match(struct pkg_entry *match, const char *location,
const char *pattern, const char *pkgname)
{
if (verbosity < 1 || incremental != NULL)
return;
if (strcmp(match->pkglocation, location) != 0) {
warnx("Best matching %s differs from location %s for dependency %s of package %s",
match->pkgname, location, pattern, pkgname);
}
}
static struct pkg_entry *
find_match(const char *pkgname, const char *pattern, const char *location)
{
size_t matches;
struct pkg_entry *best;
best = NULL;
matches = 0;
if ((isalnum((unsigned char)pattern[0]) || pattern[0] == '-') &&
(isalnum((unsigned char)pattern[1]) || pattern[1] == '-') &&
(isalnum((unsigned char)pattern[2]) || pattern[2] == '-') &&
(isalnum((unsigned char)pattern[3]) || pattern[3] == '-') &&
strchr(pattern, '{') == NULL) {
struct pkg_entry *iter;
SLIST_FOREACH(iter, get_hash_chain(pattern), hash_link)
find_match_iter(pattern, pkgname, &best, iter, &matches);
} else {
size_t i;
for (i = 0; i < len_pkgs; ++i)
find_match_iter(pattern, pkgname, &best, &pkgs[i], &matches);
}
if (matches == 0) {
if (incremental != NULL)
(void)fprintf(incremental, "%s\n", location);
else
warnx("No match found for dependency %s of package %s", pattern, pkgname);
return NULL;
}
validate_best_match(best, location, pattern, pkgname);
return best;
}
static int
resolve_entry(struct pkg_entry *pkg)
{
const char *line, *pattern_begin, *pattern_end, *location_begin;
char *pattern, *location, *old_depends;
struct pkg_entry *best_match;
struct pkg_entry **depends_list;
size_t i;
int ret;
if (pkg->active == 0)
return 0;
if (find_content(pkg, "DEPENDS=") != NULL)
return 0;
ret = 0;
if ((line = find_content(pkg, "ALL_DEPENDS=")) == NULL)
errx(1, "No ALL_DEPENDS line for %s", pkg->pkgname);
depends_list = xmalloc(sizeof(struct pkg_entry *));
depends_list[0] = NULL;
line += strspn(line, " \t");
while (*line != '\n') {
pattern_begin = line;
pattern_end = location_begin = strchr(pattern_begin, ':');
if (location_begin == NULL ||
strncmp(location_begin, ":../../", 7) != 0) {
free(depends_list);
warnx("Incorrect dependency for %s, skipping", pkg->pkgname);
return 1;
}
location_begin += 7;
line = location_begin + strcspn(location_begin, " \t\n");
pattern = xstrndup(pattern_begin, pattern_end - pattern_begin);
location = xstrndup(location_begin, line - location_begin);
line += strspn(line, " \t");
for (i = 0; depends_list[i] != NULL; ++i) {
if (pkg_match(pattern, depends_list[i]->pkgname))
break;
}
if (depends_list[i] != NULL) {
/*
* If we already have a match, assume that it is most
* likely stricter than this one. The best match check
* was therefore already done.
*/
if (strcmp(depends_list[i]->pkglocation, location) != 0)
validate_best_match(depends_list[i], location, pattern, pkg->pkgname);
best_match = NULL; /* XXX For GCC */
} else
best_match = find_match(pkg->pkgname, pattern, location);
free(pattern);
free(location);
if (depends_list[i] != NULL)
continue;
if (best_match == NULL) {
ret = 1;
continue;
}
depends_list = xrealloc(depends_list, (i + 2) * sizeof(struct pkg_entry *));
depends_list[i] = best_match;
depends_list[i + 1] = NULL;
if (pkg->depends == NULL)
pkg->depends = xstrdup(best_match->pkgname);
else {
old_depends = pkg->depends;
pkg->depends = xasprintf("%s %s", old_depends, best_match->pkgname);
free(old_depends);
}
best_match->active = 1;
}
free(depends_list);
if (ret == 1) {
free(pkg->depends);
pkg->depends = NULL;
if (incremental != NULL || partial)
return 0;
else
return 1;
}
return 0;
}
static char *
pkgname_dup(const char *line)
{
const char *pkgname;
char *pkgname_end;
size_t pkgname_len;
if (strncmp(line, "PKGNAME=", 8) != 0)
return NULL;
pkgname = line + 8;
pkgname_end = strchr(pkgname, '\n');
pkgname_len = pkgname_end - pkgname;
if (pkgname_end == NULL || pkgname_len < 4 ||
strcspn(pkgname, " \t\n") != pkgname_len)
return NULL;
return xstrndup(pkgname, pkgname_len);
}
static const char *
pbulk_item_end(const char *line)
{
const char *line_end;
do {
line_end = strchr(line, '\n');
if (line_end == NULL)
return NULL;
line = line_end + 1;
if (strncmp(line, "PKGNAME=", 8) == 0)
return line;
} while (*line != '\0');
return line;
}
static void
read_entries(const char *input_file, int def_active)
{
char *input;
const char *input_iter;
const char *location_line, *location_line_end;
int fd;
if ((fd = open(input_file, O_RDONLY, 0)) == -1)
err(1, "Cannot open input");
input = read_from_file(fd);
(void)close(fd);
if (allocated_pkgs == 0) {
allocated_pkgs = 1024;
pkgs = xmalloc(allocated_pkgs * sizeof(*pkgs));
}
input_iter = input;
while ((pkgs[len_pkgs].pkgname = pkgname_dup(input_iter)) != NULL) {
pkgs[len_pkgs].active = def_active;
pkgs[len_pkgs].begin = input_iter;
pkgs[len_pkgs].end = pbulk_item_end(input_iter);
pkgs[len_pkgs].depends = NULL;
if (pkgs[len_pkgs].end == NULL)
errx(1, "Invalid input");
input_iter = pkgs[len_pkgs].end;
location_line = find_content(&pkgs[len_pkgs], "PKG_LOCATION=");
if (location_line == NULL)
errx(1, "Invalid input");
location_line_end = strchr(location_line, '\n');
if (location_line_end == NULL)
errx(1, "Invalid input");
pkgs[len_pkgs].pkglocation = xstrndup(location_line, location_line_end - location_line);
++len_pkgs;
if (len_pkgs == allocated_pkgs) {
allocated_pkgs *= 2;
pkgs = xrealloc(pkgs, allocated_pkgs * sizeof(*pkgs));
}
}
if (*input_iter != '\0')
errx(1, "Invalid input");
}
static void
write_entries(void)
{
size_t i;
for (i = 0; i < len_pkgs; ++i) {
if (pkgs[i].active == 0)
continue;
(void)fwrite(pkgs[i].begin, 1, pkgs[i].end - pkgs[i].begin, stdout);
if (pkgs[i].depends != NULL)
(void)printf("DEPENDS=%s\n", pkgs[i].depends);
}
}
#define HASH_SIZE 4096
#define HASH_ITEM(x) (((unsigned char)(x)[0] + (unsigned char)(x)[1] * 257 + (unsigned char)(x)[1] * 65537) & (HASH_SIZE - 1))
static struct pkg_entry_hash hash_table[HASH_SIZE];
static void
hash_entries(void)
{
size_t i, hash;
for (i = 0; i < HASH_SIZE; ++i)
SLIST_INIT(&hash_table[i]);
for (i = 0; i < len_pkgs; ++i) {
hash = HASH_ITEM(pkgs[i].pkgname);
SLIST_INSERT_HEAD(&hash_table[hash], &pkgs[i], hash_link);
}
}
static struct pkg_entry_hash *
get_hash_chain(const char *pkgname)
{
return &hash_table[HASH_ITEM(pkgname)];
}

6
pkgtools/pbulk/files/pbulk/pscan/Makefile Normal file
View File

@@ -0,0 +1,6 @@
# $NetBSD: Makefile,v 1.1.1.1 2007/06/19 19:49:57 joerg Exp $
PROG= pbulk-scan
SRCS= pscan.c jobs.c master.c client.c
.include <bsd.prog.mk>

124
pkgtools/pbulk/files/pbulk/pscan/client.c Normal file
View File

@@ -0,0 +1,124 @@
/* $NetBSD: client.c,v 1.3 2009/03/06 15:21:17 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <sys/socket.h>
#include <nbcompat/err.h>
#ifdef HAVE_INTTYPES_H
#include <inttypes.h>
#endif
#include <nbcompat/stdio.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/string.h>
#include <nbcompat/unistd.h>
#include <arpa/inet.h>
#include "pbulk.h"
#include "pscan.h"
void
client_mode(const char *client_port)
{
struct sockaddr_in dst;
uint16_t path_len;
uint32_t net_output_len;
ssize_t recv_bytes, sent_bytes;
size_t output_len;
char *path, *output;
int fd;
if (parse_sockaddr_in(client_port, &dst))
errx(1, "Could not parse addr/port");
fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (fd == -1)
err(1, "Could not create socket");
if (connect(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
err(1, "Could not connect socket");
loop:
recv_bytes = atomic_read(fd, &path_len, 2);
if (recv_bytes == -1)
err(1, "Could not read from socket");
if (recv_bytes == 0)
exit(0);
if (recv_bytes != 2)
errx(1, "Premature end while reading path length from socket");
path_len = ntohs(path_len);
if (path_len < 3)
errx(1, "Invalid path length from master");
path = xmalloc(path_len + 1);
path[path_len] = '\0';
recv_bytes = atomic_read(fd, path, path_len);
if (recv_bytes == -1)
err(1, "Could not read from socket");
if (recv_bytes != path_len || strlen(path) != path_len)
errx(1, "Premature end of stream while reading path from socket");
if (path[0] == '/' ||
strchr(path, '/') == NULL ||
strchr(path, '/') != strrchr(path, '/') ||
memcmp(path, "../", 3) == 0 ||
memcmp(path + path_len - 3, "/..", 3) == 0)
errx(1, "Invalid path from master");
if (verbosity >= 1) {
(void)printf("Scanning %s\n", path);
(void)fflush(stdout);
}
output = scan_pkglocation(path);
free(path);
if (output != NULL)
output_len = strlen(output);
else
output_len = 0;
if (output_len > 0xfffffffful)
errx(1, "Output too large");
net_output_len = htonl((uint32_t)output_len);
sent_bytes = write(fd, &net_output_len, 4);
if (sent_bytes == -1)
err(1, "Could not write to socket");
if (sent_bytes != 4)
errx(1, "Premature end of stream while writing to socket");
sent_bytes = write(fd, output, output_len);
if (sent_bytes == -1)
err(1, "Could not write to socket");
if ((size_t)sent_bytes != output_len)
errx(1, "Premature end of stream while writing to socket");
free(output);
goto loop;
}

498
pkgtools/pbulk/files/pbulk/pscan/jobs.c Normal file
View File

@@ -0,0 +1,498 @@
/* $NetBSD: jobs.c,v 1.7 2012/11/30 16:22:49 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <sys/uio.h>
#include <nbcompat/err.h>
#include <fcntl.h>
#include <nbcompat/stdio.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/string.h>
#include <nbcompat/unistd.h>
#include "pbulk.h"
#include "pscan.h"
#define UNCONST(x) ((void *)(uintptr_t)(x))
#define STAT_HASH_SIZE 131072
#define HASH_SIZE 32768
struct stat_cache {
char *path;
time_t mtime;
SLIST_ENTRY(stat_cache) hash_link;
};
SLIST_HEAD(stat_cache_hash, stat_cache);
static struct stat_cache_hash stat_hash_table[STAT_HASH_SIZE];
static time_t
stat_path(const char *path)
{
size_t val = djb_hash(path) % STAT_HASH_SIZE;
struct stat_cache_hash *h = &stat_hash_table[val];
struct stat_cache *e;
struct stat sb;
SLIST_FOREACH(e, h, hash_link) {
if (strcmp(path, e->path) == 0)
return e->mtime;
}
e = xmalloc(sizeof(*e));
e->path = xstrdup(path);
if (stat(path, &sb) == -1)
e->mtime = -1;
else
e->mtime = sb.st_mtime;
SLIST_INSERT_HEAD(h, e, hash_link);
return e->mtime;
}
struct scan_entry {
char *location;
char *data;
char *scan_depends;
SLIST_ENTRY(scan_entry) hash_link;
};
SLIST_HEAD(scan_entry_hash, scan_entry);
static struct scan_entry_hash hash_table[HASH_SIZE];
static time_t scan_mtime;
static size_t
hash_entry(const char *path)
{
return djb_hash(path) % HASH_SIZE;
}
static size_t
hash_entry2(const char *path, const char *path_end)
{
return djb_hash2(path, path_end) % HASH_SIZE;
}
static void
add_entry(const char *l_start, const char *l_end,
const char *s_start, const char *s_end,
const char *d_start, const char *d_end)
{
struct scan_entry *e;
struct scan_entry_hash *h;
if (l_start == l_end)
errx(1, "Location entry missing");
h = &hash_table[hash_entry2(l_start, l_end)];
SLIST_FOREACH(e, h, hash_link) {
if (strncmp(e->location, l_start, l_end - l_start) == 0 &&
e->location[l_end - l_start] == '\0') {
size_t l1, l2, l3;
l1 = strlen(e->data);
l2 = d_end - d_start;
l_start -= 13;
++l_end;
l3 = l_start - d_start;
e->data = xrealloc(e->data, l1 + l2 + 1);
memcpy(e->data + l1, d_start, l3);
memcpy(e->data + l1 + l3, l_end, d_end - l_end);
e->data[l1 + l3 + d_end - l_end] = '\0';
return;
}
}
e = xmalloc(sizeof(*e));
e->location = xstrndup(l_start, l_end - l_start);
e->data = xmalloc(d_end - d_start + 1);
l_start -= 13;
++l_end;
memcpy(e->data, d_start, l_start - d_start);
memcpy(e->data + (l_start - d_start), l_end, d_end - l_end);
e->data[l_start - d_start + d_end - l_end] = '\0';
if (s_start != s_end)
e->scan_depends = xstrndup(s_start, s_end - s_start);
else
e->scan_depends = NULL;
SLIST_INSERT_HEAD(h, e, hash_link);
}
void
read_old_scan(const char *path)
{
size_t i;
int fd;
char *buf;
struct stat sb;
const char *entry_start;
const char *l_start, *l_end;
const char *s_start, *s_end;
const char *line, *eol;
for (i = 0; i < HASH_SIZE; ++i)
SLIST_INIT(&hash_table[i]);
if (path == NULL)
return;
if ((fd = open(path, O_RDONLY)) == -1)
return;
if (fstat(fd, &sb) == -1) {
close(fd);
return;
}
scan_mtime = sb.st_mtime;
buf = read_from_file(fd);
entry_start = buf;
l_start = l_end = NULL;
entry_start = buf;
s_start = s_end = NULL;
for (line = buf; *line; line = eol) {
eol = strchr(line, '\n');
if (eol == NULL)
errx(1, "Incomplete old scan");
++eol;
if (strncmp(line, "PKGNAME=", 8) == 0) {
if (line == buf)
continue;
add_entry(l_start, l_end, s_start, s_end,
entry_start, line);
l_start = l_end = NULL;
entry_start = line;
s_start = s_end = NULL;
} else if (strncmp(line, "PKG_LOCATION=", 13) == 0) {
l_start = line + 13;
l_end = eol - 1;
} else if (strncmp(line, "SCAN_DEPENDS=", 13) == 0) {
s_start = line + 13;
s_end = eol - 1;
}
}
if (entry_start != line)
add_entry(l_start, l_end, s_start, s_end,
entry_start, line);
}
static struct scan_entry *
find_old_scan(const char *location)
{
struct scan_entry *e;
char *dep, *dep2, *path, *fullpath;
int is_current;
time_t mtime;
e = SLIST_FIRST(&hash_table[hash_entry(location)]);
while (e) {
if (strcmp(e->location, location) == 0)
break;
e = SLIST_NEXT(e, hash_link);
}
if (e == NULL)
return NULL;
if (e->scan_depends == NULL)
return e;
is_current = 1;
dep2 = dep = xstrdup(e->scan_depends);
while ((path = strtok(dep, " ")) != NULL) {
dep = NULL;
if (*path == '\0')
continue;
if (*path == '/') {
mtime = stat_path(path);
if (mtime == -1 || mtime >= scan_mtime) {
is_current = 0;
break;
}
continue;
}
if (strncmp("../../", path, 6) == 0) {
const char *s1 = strrchr(location, '/');
const char *s2 = strchr(location, '/');
if (s1 == s2)
fullpath = xasprintf("%s/%s", pkgsrc_tree,
path + 6);
else
fullpath = xasprintf("%s/%s/%s", pkgsrc_tree,
location, path);
} else {
fullpath = xasprintf("%s/%s/%s", pkgsrc_tree,
location, path);
}
mtime = stat_path(fullpath);
if (mtime == -1 || mtime >= scan_mtime) {
is_current = 0;
break;
}
}
free(dep2);
return is_current ? e : NULL;
}
static struct scan_job *jobs;
static size_t len_jobs, allocated_jobs, first_undone_job, done_jobs;
struct pkgname_hash {
struct pkgname_hash *next;
char *pkgname;
};
void
add_job(const char *cat, size_t cat_len, const char *dir, size_t dir_len)
{
char *location;
location = xasprintf("%.*s/%.*s", (int)cat_len, cat, (int)dir_len, dir);
add_job_full(location);
free(location);
}
void
add_job_full(const char *location)
{
if (len_jobs == allocated_jobs) {
if (allocated_jobs == 0) {
allocated_jobs = 1024;
jobs = xmalloc(sizeof(*jobs) * allocated_jobs);
} else {
allocated_jobs *= 2;
jobs = xrealloc(jobs, sizeof(*jobs) * allocated_jobs);
}
}
jobs[len_jobs].pkg_location = xstrdup(location);
jobs[len_jobs].scan_output = NULL;
jobs[len_jobs].state = JOB_OPEN;
++len_jobs;
}
struct scan_job *
get_job(void)
{
size_t i;
struct scan_entry *e;
struct scan_job * job;
for (i = first_undone_job; i < len_jobs; ++i) {
job = &jobs[i];
if (job->state != JOB_OPEN)
continue;
e = find_old_scan(job->pkg_location);
if (e == NULL) {
job->state = JOB_IN_PROCESSING;
return job;
}
job->scan_output = xstrdup(e->data);
process_job(job, JOB_DONE);
i = first_undone_job - 1;
}
return NULL;
}
void
process_job(struct scan_job *job, enum job_state state)
{
job->state = state;
for (; first_undone_job < len_jobs; ++first_undone_job) {
if (jobs[first_undone_job].state != JOB_DONE)
break;
}
if (state == JOB_DONE) {
++done_jobs;
if (verbosity >= 1) {
if (done_jobs % 50)
(void)putchar('.');
else
(void)printf(". %lu/%lu\n",
(unsigned long)done_jobs,
(unsigned long)len_jobs);
(void)fflush(stdout);
}
}
}
static char *
pkgname_dup(const char *line)
{
const char *pkgname;
char *pkgname_end;
size_t pkgname_len;
if (strncmp(line, "PKGNAME=", 8) != 0)
return NULL;
pkgname = line + 8;
pkgname_end = strchr(pkgname, '\n');
pkgname_len = pkgname_end - pkgname;
if (pkgname_end == NULL || pkgname_len < 2 ||
strcspn(pkgname, " \t\n") != pkgname_len)
return NULL;
return xstrndup(pkgname, pkgname_len);
}
#define HASH_ITEM(x) (djb_hash(x) % HASH_SIZE)
static struct pkgname_hash *pkgname_hash[HASH_SIZE];
static int
pkgname_in_hash(const char *pkgname)
{
struct pkgname_hash *iter;
for (iter = pkgname_hash[HASH_ITEM(pkgname)]; iter != NULL;
iter = iter->next) {
if (strcmp(iter->pkgname, pkgname) == 0)
return 1;
}
return 0;
}
static const char *
pbulk_item_end(const char *line)
{
const char *line_end;
do {
line_end = strchr(line, '\n');
if (line_end == NULL)
return NULL;
line = line_end + 1;
if (strncmp(line, "PKGNAME=", 8) == 0)
return line;
} while (*line != '\0');
return line;
}
static void
write_single_job(int fd, const char *begin_entry, struct scan_job *job)
{
struct pkgname_hash *entry;
const char *end_entry, *end_line;
char *pkgname;
int skip_current;
struct iovec output[5];
const int iovcnt = 5;
ssize_t expected;
for (; begin_entry != NULL && begin_entry[0] != '\0';) {
pkgname = pkgname_dup(begin_entry);
if (pkgname == NULL) {
warnx("pbulk-index output not recognized for %s",
job->pkg_location);
break;
}
if (pkgname_in_hash(pkgname)) {
warnx("Duplicate package: %s", pkgname);
skip_current = 1;
} else {
skip_current = 0;
}
end_entry = pbulk_item_end(begin_entry);
if (end_entry == NULL) {
free(pkgname);
warnx("pbulk-index output not recognized for %s",
job->pkg_location);
break;
}
if (skip_current == 0) {
end_line = strchr(begin_entry, '\n');
if (end_line == NULL || end_line > end_entry)
errx(254, "internal error");
output[0].iov_base = UNCONST(begin_entry);
expected = output[0].iov_len =
end_line + 1 - begin_entry;
output[1].iov_base = UNCONST("PKG_LOCATION=");
output[1].iov_len = strlen(output[1].iov_base);
expected += output[1].iov_len;
output[2].iov_base = job->pkg_location;
output[2].iov_len = strlen(output[2].iov_base);
expected += output[2].iov_len;
output[3].iov_base = UNCONST("\n");
output[3].iov_len = 1;
expected += output[3].iov_len;
output[4].iov_base = UNCONST(end_line + 1);
output[4].iov_len = end_entry - end_line - 1;
expected += output[4].iov_len;
if (writev(fd, output, iovcnt) != expected)
err(1, "writev failed");
entry = xmalloc(sizeof(*entry));
entry->next = pkgname_hash[HASH_ITEM(pkgname)];
entry->pkgname = pkgname;
pkgname_hash[HASH_ITEM(pkgname)] = entry;
} else {
free(pkgname);
}
begin_entry = end_entry;
}
}
void
write_jobs(const char *output_file)
{
size_t i;
int fd;
if (verbosity >= 1) {
if (done_jobs % 50)
(void)printf(" %lu/%lu\n", (unsigned long)done_jobs,
(unsigned long)len_jobs);
(void)fflush(stdout);
}
fd = open(output_file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
for (i = 0; i < len_jobs; ++i) {
if (jobs[i].state != JOB_DONE) {
warnx("%s was not processed", jobs[i].pkg_location);
continue;
}
if (jobs[i].scan_output == NULL) {
warnx("Scan failed for %s", jobs[i].pkg_location);
continue;
}
write_single_job(fd, jobs[i].scan_output, jobs + i);
}
}

286
pkgtools/pbulk/files/pbulk/pscan/master.c Normal file
View File

@@ -0,0 +1,286 @@
/* $NetBSD: master.c,v 1.8 2013/01/14 14:33:28 jperkin Exp $ */
/*-
* Copyright (c) 2007, 2009 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <nbcompat/types.h>
#include <nbcompat/queue.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <nbcompat/time.h>
#include <sys/wait.h>
#include <nbcompat/err.h>
#include <signal.h>
#include <fcntl.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/stdio.h>
#include <nbcompat/string.h>
#include <arpa/inet.h>
#include "pbulk.h"
#include "pscan.h"
static int clients_started;
static LIST_HEAD(, scan_peer) active_peers, inactive_peers;
static struct event listen_event;
static int listen_event_socket;
static struct signal_event child_event;
static pid_t child_pid;
struct scan_peer {
LIST_ENTRY(scan_peer) peer_link;
struct scan_job *job;
int fd;
char tmp_buf[4];
size_t output_len;
};
static void assign_job(struct scan_peer *);
static void
kill_peer(void *arg)
{
struct scan_peer *peer = arg;
(void)close(peer->fd);
LIST_REMOVE(peer, peer_link);
free(peer->job->scan_output);
peer->job->scan_output = NULL;
process_job(peer->job, JOB_OPEN);
free(peer);
peer = LIST_FIRST(&inactive_peers);
if (peer == NULL)
return;
LIST_REMOVE(peer, peer_link);
assign_job(peer);
}
static void
finish_job(void *arg)
{
struct scan_peer *peer = arg;
if (strlen(peer->job->scan_output) != peer->output_len) {
warnx("Invalid output len received from peer");
kill_peer(peer);
return;
}
LIST_REMOVE(peer, peer_link);
process_job(peer->job, JOB_DONE);
assign_job(peer);
}
static void
recv_output(void *arg)
{
struct scan_peer *peer = arg;
uint32_t output_len;
(void)memcpy(&output_len, peer->tmp_buf, 4);
output_len = ntohl(output_len);
if (output_len == 0) {
LIST_REMOVE(peer, peer_link);
process_job(peer->job, JOB_DONE);
assign_job(peer);
return;
}
if (output_len == 0xffffff) {
warnx("Invalid output len received from peer");
kill_peer(peer);
return;
}
peer->job->scan_output = xmalloc(output_len + 1);
peer->job->scan_output[output_len] = '\0';
peer->output_len = output_len;
deferred_read(peer->fd, peer->job->scan_output, output_len, peer, finish_job, kill_peer);
}
static void
recv_output_len(void *arg)
{
struct scan_peer *peer = arg;
deferred_read(peer->fd, peer->tmp_buf, 4, peer, recv_output, kill_peer);
}
static void
send_job_path(void *arg)
{
struct scan_peer *peer = arg;
deferred_write(peer->fd, peer->job->pkg_location,
strlen(peer->job->pkg_location), peer, recv_output_len,
kill_peer);
}
static void
shutdown_master(void)
{
struct timeval tv;
struct scan_peer *peer;
event_del(&listen_event);
(void)close(listen_event_socket);
LIST_FOREACH(peer, &inactive_peers, peer_link)
(void)shutdown(peer->fd, SHUT_RDWR);
tv.tv_sec = 1;
tv.tv_usec = 0;
event_loopexit(&tv);
}
static void
assign_job(struct scan_peer *peer)
{
size_t job_len;
uint16_t net_job_len;
peer->job = clients_started ? get_job() : NULL;
if (peer->job == NULL) {
LIST_INSERT_HEAD(&inactive_peers, peer, peer_link);
if (LIST_EMPTY(&active_peers) && clients_started)
shutdown_master();
return;
}
LIST_INSERT_HEAD(&active_peers, peer, peer_link);
peer->job->scan_output = NULL;
job_len = strlen(peer->job->pkg_location);
if (job_len > 0xffff)
errx(1, "Location inside pkgsrc tree too long");
net_job_len = htons(job_len);
(void)memcpy(peer->tmp_buf, &net_job_len, 2);
deferred_write(peer->fd, peer->tmp_buf, 2, peer, send_job_path,
kill_peer);
}
static void
listen_handler(int sock, void *arg)
{
struct scan_peer *peer;
struct sockaddr_in src;
socklen_t src_len;
int fd;
src_len = sizeof(src);
if ((fd = accept(sock, (struct sockaddr *)&src, &src_len)) == -1) {
warn("Could not accept connection");
return;
}
if (set_nonblocking(fd) == -1) {
(void)close(fd);
warn("Could not set non-blocking IO");
return;
}
peer = xmalloc(sizeof(*peer));
peer->fd = fd;
assign_job(peer);
}
static void
child_handler(struct signal_event *ev)
{
struct scan_peer *peer;
int status;
if (waitpid(child_pid, &status, WNOHANG) == -1) {
if (errno == ECHILD)
return;
err(1, "Could not wait for child");
}
if (status != 0)
err(1, "Start script failed");
clients_started = 1;
signal_del(ev);
while ((peer = LIST_FIRST(&inactive_peers)) != NULL) {
LIST_REMOVE(peer, peer_link);
assign_job(peer);
if (peer-> job == NULL)
break;
}
}
void
master_mode(const char *master_port, const char *start_script)
{
struct sockaddr_in dst;
int fd;
LIST_INIT(&active_peers);
LIST_INIT(&inactive_peers);
event_init();
if (parse_sockaddr_in(master_port, &dst))
errx(1, "Could not parse addr/port");
fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (fd == -1)
err(1, "Could not create socket");
if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1)
err(1, "Could not set close-on-exec flag");
if (bind(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
err(1, "Could not bind socket");
if (listen(fd, 5) == -1)
err(1, "Could not listen on socket");
event_add(&listen_event, fd, 0, 1, listen_handler, NULL);
listen_event_socket = fd;
if (start_script) {
signal_add(&child_event, SIGCHLD, child_handler);
if ((child_pid = vfork()) == 0) {
execlp(start_script, start_script, (char *)NULL);
_exit(255);
}
if (child_pid == -1)
err(1, "Could not fork start script");
} else {
clients_started = 1;
}
event_dispatch();
(void)close(fd);
}

95
pkgtools/pbulk/files/pbulk/pscan/pbulk-scan.1 Normal file
View File

@@ -0,0 +1,95 @@
.\" $NetBSD: pbulk-scan.1,v 1.1.1.1 2007/06/19 19:49:57 joerg Exp $
.\"
.\" Copyright (c) 2007 Thomas Klausner and Joerg Sonnenberger.
.\" All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\" notice, this list of conditions and the following disclaimer.
.\" 2. Redistributions in binary form must reproduce the above copyright
.\" notice, this list of conditions and the following disclaimer in the
.\" documentation and/or other materials provided with the distribution.
.\"
.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
.\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
.\" POSSIBILITY OF SUCH DAMAGE.
.\"
.Dd June 11, 2007
.Dt PBULK-SCAN 1
.Os
.Sh NAME
.Nm pbulk-scan
.Nd extracts information for pbulk from a pkgsrc tree
.Sh SYNOPSIS
.Nm
.Op Fl v
.Fl c Oo Ar ip: Oc Ns Ar port
.Fl M Ar make
.Ar pkgsrc
.Nm
.Op Fl lv
.Op Fl I Ar start_script
.Op Fl m Oo Ar ip: Oc Ns Ar port
.Fl M Ar make
.Ar pkgsrc output
.Sh DESCRIPTION
.Nm
extracts information for a pbulk bulk build from a pkgsrc tree.
.Pp
Supported options are:
.Bl -tag -width 15n -offset indent
.It Fl c Oo Ar ip: Oc Ns Ar port
Connect to pbulk bulk build master process on
.Ar port
(or
.Ar ip:port ) .
.It Fl I Ar start_script
Run
.Ar start_script
after opening the socket and wait for completion before entering scan loop.
.It Fl l
Read the list of locations to scan from stdin, one line per location.
Otherwise the list of locations is built from the
.Va SUBDIRS
variable in the top-level Makefile and the
.Va SUBDIRS
variables in the category Makefiles.
.It Fl M Ar make
Use
.Ar make
to extract the data.
Usually
.Dq make
or
.Dq bmake .
.It Fl m Oo Ar ip: Oc Ns Ar port
Enter master mode.
In this mode,
.Nm
waits for connections on
.Ar port
(or
.Ar ip:port ) .
.It Fl v
Log each location to be scanned or other progress to stdout.
.El
.Ar pkgsrc
is the path to the pkgsrc checkout and
.Ar output
is the output file.
.Pp
A warning is printed to stderr if the same package name occurs
more than once in the scan output.
.Sh EXIT STATUS
.Nm
exits with return value 1 on internal errors.

257
pkgtools/pbulk/files/pbulk/pscan/pscan.c Normal file
View File

@@ -0,0 +1,257 @@
/* $NetBSD: pscan.c,v 1.8 2012/11/23 12:13:35 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <nbcompat.h>
#include <sys/stat.h>
#include <sys/uio.h>
#include <nbcompat/err.h>
#ifdef HAVE_INTTYPES_H
#include <inttypes.h>
#endif
#include <fcntl.h>
#include <nbcompat/limits.h>
#include <signal.h>
#include <nbcompat/stdio.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/string.h>
#include <nbcompat/unistd.h>
#include "pbulk.h"
#include "pscan.h"
int verbosity;
static const char *bmake_path;
static const char *output_file;
const char *pkgsrc_tree;
static void find_full_tree(void);
static void read_limited_list(void);
static void standalone_mode(void);
static void
usage(void)
{
(void)fprintf(stderr, "usage: pbulk-scan -c <master> [ -v ] -M <make> <pkgsrc tree>\n");
(void)fprintf(stderr, "usage: pbulk-scan [ -I <start> ] [ -L <old scan> ] [ -l ] [ -v ]\n"
" [ -m <port> ] -M <make> <pksgrc tree> <output file>\n");
exit(1);
}
int
main(int argc, char **argv)
{
const char *client_port = NULL, *last_scan = NULL, *master_port = NULL;
const char *start_script = NULL;
int ch, limited_scan;
struct sigaction sa;
setprogname("pbulk-scan");
limited_scan = 0;
while ((ch = getopt(argc, argv, "I:M:L:lc:m:v")) != -1) {
switch (ch) {
case 'I':
start_script = optarg;
break;
case 'c':
client_port = optarg;
break;
case 'L':
last_scan = optarg;
break;
case 'l':
limited_scan = 1;
break;
case 'm':
master_port = optarg;
break;
case 'M':
bmake_path = optarg;
break;
case 'v':
++verbosity;
break;
default:
usage();
}
}
#if !defined(__INTERIX)
sa.sa_sigaction = NULL;
#endif
sa.sa_handler = SIG_IGN;
sa.sa_flags = 0;
(void)sigemptyset(&sa.sa_mask);
(void)sigaction(SIGPIPE, (struct sigaction *)&sa, NULL);
argc -= optind;
argv += optind;
if (bmake_path == NULL)
usage();
if (client_port != NULL && master_port != NULL) {
warnx("Only client mode or master mode can be active");
usage();
}
if (client_port == NULL)
read_old_scan(last_scan);
if (client_port) {
if (limited_scan != 0 || argc != 1)
usage();
pkgsrc_tree = argv[0];
client_mode(client_port);
}
if (argc != 2)
usage();
pkgsrc_tree = argv[0];
output_file = argv[1];
if (limited_scan == 0)
find_full_tree();
else
read_limited_list();
if (master_port != NULL)
master_mode(master_port, start_script);
else
standalone_mode();
write_jobs(output_file);
return 0;
}
char *
scan_pkglocation(const char *pkg_location)
{
const char * extract_pbulk_index[] = {
bmake_path,
"pbulk-index",
NULL
};
char *path, *buf;
path = xasprintf("%s/%s", pkgsrc_tree, pkg_location);
buf = read_from_child(path, bmake_path, extract_pbulk_index);
free(path);
return buf;
}
static void
read_limited_list(void)
{
char location[PATH_MAX], *eos;
while (fgets(location, PATH_MAX, stdin) != NULL) {
eos = strchr(location, '\n');
if (eos == NULL)
err(1, "Incomplete or too long input line");
if (location == eos)
continue;
if (*location == '#')
continue;
*eos = '\0';
add_job_full(location);
}
}
static void
find_full_tree(void)
{
const char * extract_subdir[] = {
bmake_path,
"show-subdir-var",
"VARNAME=SUBDIR",
NULL
};
char *cat_path;
char *buf, *buf_orig, *cat, *cat_orig;
size_t buf_len, cat_len;
buf = read_from_child(pkgsrc_tree, bmake_path, extract_subdir);
if (buf == NULL)
err(1, "Cannot extract categories");
cat = cat_orig = buf;
for (;;) {
cat += strspn(cat, " \t\n");
cat_len = strcspn(cat, " \t\n");
if (cat_len == 0)
break;
cat_path = xasprintf("%s/%.*s", pkgsrc_tree, (int)cat_len, cat);
buf_orig = buf = read_from_child(cat_path, bmake_path, extract_subdir);
free(cat_path);
if (buf == NULL) {
warnx("Cannot extract subdirectories for %.*s", (int)cat_len, cat);
cat += cat_len;
continue;
}
for (;;) {
buf += strspn(buf, " \t\n");
buf_len = strcspn(buf, " \t\n");
if (buf_len == 0)
break;
add_job(cat, cat_len, buf, buf_len);
buf += buf_len;
}
free(buf_orig);
cat += cat_len;
}
free(cat_orig);
}
static void
standalone_mode(void)
{
struct scan_job *job;
while ((job = get_job()) != NULL) {
job->scan_output = scan_pkglocation(job->pkg_location);
process_job(job, JOB_DONE);
}
}

64
pkgtools/pbulk/files/pbulk/pscan/pscan.h Normal file
View File

@@ -0,0 +1,64 @@
/* $NetBSD: pscan.h,v 1.3 2012/11/23 12:13:35 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
* All rights reserved.
*
* This code was developed as part of Google's Summer of Code 2007 program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifdef HAVE_INTTYPES_H
#include <inttypes.h>
#endif
enum job_state {
JOB_OPEN,
JOB_IN_PROCESSING,
JOB_DONE
};
struct scan_job {
char *pkg_location;
char *scan_output;
enum job_state state;
};
extern int verbosity;
extern const char *pkgsrc_tree;
char *scan_pkglocation(const char *);
void client_mode(const char *);
void master_mode(const char *, const char *);
void add_job(const char *, size_t, const char *, size_t);
void add_job_full(const char *);
struct scan_job *get_job(void);
void process_job(struct scan_job *, enum job_state);
void write_jobs(const char *);
void read_old_scan(const char *);

16
pkgtools/pbulk/files/pbulk/scripts/Makefile Normal file
View File

@@ -0,0 +1,16 @@
# $NetBSD: Makefile,v 1.3 2008/04/04 17:58:59 joerg Exp $
SCRIPTS= build build-client-start bulkbuild bulkbuild-rebuild \
bulkbuild-restart \
client-clean pkg-build pkg-up-to-date pre-build report \
scan scan-client-start upload \
compute-packages.awk create-broken-graph.awk \
create-report-html.awk create-report-txt.awk \
create-report.awk
SCRIPTSDIR= ${PREFIX}/libexec/pbulk
SCRIPTSDIR_bulkbuild= ${PREFIX}/bin
SCRIPTSDIR_bulkbuild-rebuild= ${PREFIX}/bin
SCRIPTSDIR_bulkbuild-restart= ${PREFIX}/bin
.include <bsd.prog.mk>

73
pkgtools/pbulk/files/pbulk/scripts/build Executable file
View File

@@ -0,0 +1,73 @@
#!@SH@
# $NetBSD: build,v 1.5 2008/09/16 18:21:30 joerg Exp $
#
# Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
# All rights reserved.
#
# This code was developed as part of Google's Summer of Code 2007 program.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
. @PBULK_CONFIG@
set -e
if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
echo "Your configuration has version ${config_version}."
echo "This version of pbulk expects version @PBULK_CONFIG_VERSION@."
exit 1
fi
echo "Building..."
case "${master_mode}" in
[nN][oO])
${pbuild} -r ${loc}/pbuild -v -b ${pbuild_script} ${loc}/presolve ${loc}/success ${loc}/error
;;
[yY][eE][sS])
${pbuild} -r ${loc}/pbuild -I ${pbuild_start_script} -m ${master_port_build} -v ${loc}/presolve ${loc}/success ${loc}/error
;;
*)
echo "master_mode must be either yes or no."
exit 1
;;
esac
date '+BUILD_END_ISO=%Y-%m-%d %H:%M' >> ${loc}/status
echo "Building pkg_summary..."
cd ${packages}/All
sed 's/$/.tgz/' < ${loc}/success | sort | xargs ${pkg_info} -X | ${gzip} -c > pkg_summary.gz
${gzip} -dc < pkg_summary.gz | ${bzip2} -c > pkg_summary.bz2
if [ "${checksum_packages}" != "no" ] && \
[ "${checksum_packages}" != "NO" ]; then
echo "Building SHA512..."
cd ${packages}
{
echo "All/pkg_summary.bz2"
echo "All/pkg_summary.gz"
sed 's|^\(.*\)$|All/\1.tgz|' < ${loc}/success
} | sort | xargs ${digest} SHA512 | ${bzip2} -c > SHA512.bz2
fi

16
pkgtools/pbulk/files/pbulk/scripts/build-client-start Executable file
View File

@@ -0,0 +1,16 @@
#!@SH@
# $NetBSD: build-client-start,v 1.2 2008/09/16 18:21:30 joerg Exp $
. @PBULK_CONFIG@
set -e
if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
echo "Your configuration has version ${config_version}."
echo "This version of pbulk expects version @PBULK_CONFIG_VERSION@."
exit 1
fi
for client in ${build_clients}; do
ssh $client "${pbuild} -c ${master_port_build} -b ${pbuild_script}" &
done

12
pkgtools/pbulk/files/pbulk/scripts/bulkbuild Executable file
View File

@@ -0,0 +1,12 @@
#!@SH@
# $NetBSD: bulkbuild,v 1.1.1.1 2007/06/19 19:49:59 joerg Exp $
. @PBULK_CONFIG@
set -e
${script_phase_pre_build}
${script_phase_scan}
${script_phase_build}
${script_phase_report}
${script_phase_upload}

43
pkgtools/pbulk/files/pbulk/scripts/bulkbuild-rebuild Executable file
View File

@@ -0,0 +1,43 @@
#!@SH@
# $NetBSD: bulkbuild-rebuild,v 1.3 2008/09/16 18:21:30 joerg Exp $
. @PBULK_CONFIG@
set -e
if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
echo "Your configuration has version ${config_version}."
echo "This version of pbulk expects version @PBULK_CONFIG_VERSION@."
exit 1
fi
check_pkg() {
if @AWK@ '{ if ($0 == "PKGNAME='$1'") exit 1 }' < ${loc}/presolve; then
echo "Package $1 not found, aborting."
exit 1
fi
}
pkg_failed() {
echo "failed"
exit 1
}
build_pkg() {
printf "Building package %s... " "$1"
@AWK@ '{ if ($0 == "PKGNAME='$1'") p=1; \
else if ($0 ~ "^PKGNAME=") p=0; \
if (p) print $0 }' < ${loc}/presolve | \
${pbuild_script} || pkg_failed "$1"
echo "done"
}
if [ $# = 0 ]; then
echo "Usage: $0 package ..."
fi
while [ $# != 0 ]; do
check_pkg "$1"
build_pkg "$1"
shift
done

42
pkgtools/pbulk/files/pbulk/scripts/bulkbuild-restart Executable file
View File

@@ -0,0 +1,42 @@
#!@SH@
# $NetBSD: bulkbuild-restart,v 1.3 2008/10/08 16:43:34 joerg Exp $
. @PBULK_CONFIG@
set -e
if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
echo "Your configuration has version ${config_version}."
echo "This version of pbulk expects version @PBULK_CONFIG_VERSION@."
exit 1
fi
check_list() {
@AWK@ -v presolve=${loc}/presolve 'BEGIN {
while (getline < presolve) {
if ($0 ~ "^PKGNAME=") {
cur = substr($0, 9)
pkg[cur] = 1
}
}
}
{ if (!pkg[$0]) exit(1) }' "$1" || return 1
return 0
}
if [ ! -e ${loc}/success -o ! -e ${loc}/error ]; then
restart_build=no
elif ! check_list ${loc}/success; then
restart_build=no
elif ! check_list ${loc}/error; then
restart_build=no
else
restart_build=yes
fi
if [ "$restart_build" = "no" ]; then
${script_phase_pre_build}
${script_phase_scan}
fi
${script_phase_build}
${script_phase_report}
${script_phase_upload}

34
pkgtools/pbulk/files/pbulk/scripts/client-clean Executable file
View File

@@ -0,0 +1,34 @@
#!@SH@
# $NetBSD: client-clean,v 1.9 2008/09/16 18:21:30 joerg Exp $
. @PBULK_CONFIG@
set -e
if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
echo "Your configuration has version ${config_version}."
echo "This version of pbulk expects version @PBULK_CONFIG_VERSION@."
exit 1
fi
if [ "$cross_compile" != "no" ]; then
cur_destdir=${target_destdir}
else
cur_destdir=
fi
# Clean build system first
rm -rf ${cur_destdir}${prefix} ${cur_destdir}${pkgdb} 2> /dev/null || true
rm -rf ${cur_destdir}${varbase}/qmail 2> /dev/null || true
# Note: It is not an error if ${prefix} still exists at this point,
# because it might be a mount point.
# Install fresh bootstrap state
if [ -f "${bootstrapkit}" ]; then
${tar} -xzf ${bootstrapkit} -C ${cur_destdir}/
elif [ -d "${bootstrapkit}" ]; then
[ -d ${prefix} ] || mkdir ${cur_destdir}${prefix}
cp -R ${bootstrapkit}/* ${cur_destdir}${prefix}
else
: "Do nothing"
fi

97
pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk Executable file
View File

@@ -0,0 +1,97 @@
#!@AWK@ -f
# $NetBSD: compute-packages.awk,v 1.4 2012/05/20 20:12:02 joerg Exp $
#
# Copyright (c) 2007, 2012 Joerg Sonnenberger <joerg@NetBSD.org>.
# All rights reserved.
#
# This code was developed as part of Google's Summer of Code 2007 program.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
function mark_restricted(PKG, dep, depend_list) {
if (PKG in restricted)
return
restricted[PKG] = 1
split(reverse_depends[PKG], depend_list, "[ \t]+")
for (dep in depend_list)
mark_restricted(depend_list[dep])
}
BEGIN {
meta_dir = ARGV[1]
success_file = meta_dir "/success"
presolve_file = meta_dir "/presolve"
while ((getline < presolve_file) > 0) {
if ($0 ~ "^PKGNAME=") {
cur = substr($0, 9)
pkgs[cur] = cur
}
if ($0 ~ "^CATEGORIES=")
categories[cur] = substr($0, 12)
if ($0 ~ "^BUILD_STATUS=")
status[cur] = substr($0, 14)
if ($0 ~ "^NO_BIN_ON_FTP=.")
initial_restricted[cur] = 1
if ($0 ~ "^DEPENDS=")
depends[cur] = substr($0, 9)
}
close(presolve_file)
for (pkg in depends) {
split(depends[pkg], depend_list, "[ \t]+")
for (dep in depend_list) {
cur_dep = depend_list[dep]
reverse_depends[cur_dep] = pkg " " reverse_depends[cur_dep]
}
}
for (pkg in initial_restricted)
mark_restricted(pkg)
while ((getline pkg < success_file) > 0) {
# skip restricted packages
if (pkg in restricted)
continue;
# build category/file list
split(categories[pkg], cats, "[ \t]+")
cats[0] = "All"
for (cat_idx in cats) {
cat = cats[cat_idx]
if (!(cat in printed_cats)) {
print "+ " cat "/"
printed_cats[cat] = cat
}
print "+ " cat "/" pkg ".tgz"
}
}
close(success_file)
}

91
pkgtools/pbulk/files/pbulk/scripts/create-broken-graph.awk Executable file
View File

@@ -0,0 +1,91 @@
#!@AWK@ -f
# $NetBSD: create-broken-graph.awk,v 1.2 2007/06/29 22:43:26 joerg Exp $
#
# Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
# All rights reserved.
#
# This code was developed as part of Google's Summer of Code 2007 program.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
BEGIN {
meta_dir = ARGV[1]
report_file = meta_dir "/report"
graph_file = meta_dir "/report.dot"
while ((getline < report_file) > 0) {
if ($0 ~ "^PKGNAME=")
cur = substr($0, 9)
else if ($0 ~ "^DEPENDS=")
depends[cur] = substr($0, 9)
else if ($0 ~ "^PKG_DEPTH=")
depth[cur] = substr($0, 11) - 1
else if ($0 ~ "^BUILD_STATUS=")
status[cur] = substr($0, 14)
}
close(report_file)
print "digraph \"Broken packages\" {" > graph_file
for (pkg in depends) {
split(depends[pkg], depend_list, "[ \t]+")
for (dep in depend_list) {
cur_pkg = depend_list[dep]
if (status[cur_pkg] != "failed" &&
status[cur_pkg] != "prefailed" &&
status[cur_pkg] != "indirect-failed" &&
status[cur_pkg] != "indirect-prefailed")
continue;
if (status[cur_pkg] == "failed")
color = "red"
else
color = "lightgray"
printf("\"%s\" -> \"%s\" [ color = \"%s\" ];\n", cur_pkg, pkg, color) > graph_file
drawn_pkgs[cur_pkg] = 1
drawn_pkgs[pkg] = 1
}
}
for (pkg in status) {
if (status[pkg] != "failed" &&
status[pkg] != "prefailed" &&
status[pkg] != "indirect-failed" &&
status[pkg] != "indirect-prefailed")
continue;
if (depth[pkg] == 0 && !(pkg in drawn_pkgs))
continue;
if (status[pkg] == "failed")
color = "red"
else if (status[pkg] == "prefailed")
color = "gray2"
else if (status[pkg] == "indirect-failed")
color = "orange"
else if (status[pkg] == "indirect-prefailed")
color = "lightgray"
printf("\"%s\" [ color = \"%s\", fontcolor = \"%s\", label=\"%s\" ];\n", pkg, color, color, pkg) > graph_file
}
print "}" > graph_file
}

346
pkgtools/pbulk/files/pbulk/scripts/create-report-html.awk Executable file
View File

@@ -0,0 +1,346 @@
#!@AWK@ -f
# $NetBSD: create-report-html.awk,v 1.14 2010/03/11 16:31:43 tnn Exp $
#
# Copyright (c) 2007, 2008 Joerg Sonnenberger <joerg@NetBSD.org>.
# All rights reserved.
#
# This code was developed as part of Google's Summer of Code 2007 program.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
function sort(ARRAY, INDICES, OPTIONS, i, idx, sort_cmd) {
sort_cmd = "sort " OPTIONS " > " tmp_sort
for (idx in ARRAY)
print idx | sort_cmd
close(sort_cmd)
i = 0
while ((getline < tmp_sort) > 0) {
INDICES[i] = $0
++i
}
close(tmp_sort)
system("rm " tmp_sort)
}
function print_failed_log_line(PKGNAME, PHASE, VAR) {
if (VAR == 1)
printf("<td><a href=\"../%s/%s.log\">%s</a></td>\n",
PKGNAME, PHASE, PHASE) > html_report
else
print "<td>&nbsp;</td>" > html_report
}
function print_pre_fail_reason(PKGNAME, chars, in_quote, in_sep, i) {
split(pre_fail_reason[PKGNAME], chars, "")
in_quote = 0
in_sep = 0
for (i = 1; i < length(pre_fail_reason[PKGNAME]); ++i) {
if (chars[i] == "\"") {
in_quote = 1 - in_quote
continue
}
if (in_quote == 0 && chars[i] == " ") {
if (!in_sep)
printf "<br />" > html_report
in_sep = 1
continue
}
in_sep = 0
if (chars[i] == "\\") {
++i
if (chars[i] == "n")
printf " " > html_report
else if (chars[i] == "t")
printf " " > html_report
else if (chars[i] == "<")
printf "&lt;" > html_report
else if (chars[i] == ">")
printf "&gt;" > html_report
else if (chars[i] == "&")
printf "&amp;" > html_report
else
printf "%s", chars[i] > html_report
continue
}
printf "%s", chars[i] > html_report
}
printf "\n" > html_report
}
function print_failed(PKGNAME, cmd, has_pre_clean, has_depends,
has_checksum,has_configure, has_build, has_install, has_package,
has_clean, has_deinstall, last_error) {
if (status[PKGNAME] == "failed") {
cmd = "ls " log_dir "/" PKGNAME " 2>/dev/null"
while ((cmd | getline) > 0) {
if ($0 == "pre-clean.log")
has_pre_clean = 1
else if ($0 == "depends.log")
has_depends = 1
else if ($0 == "checksum.log")
has_checksum = 1
else if ($0 == "configure.log")
has_configure = 1
else if ($0 == "build.log")
has_build = 1
else if ($0 == "install.log")
has_install = 1
else if ($0 == "package.log")
has_package = 1
else if ($0 == "clean.log")
has_clean = 1
else if ($0 == "deinstall.log")
has_deinstall = 1
}
close(cmd)
if (has_deinstall)
last_error = "deinstall.log"
else if (has_clean)
last_error = "clean.log"
else if (has_package)
last_error = "package.log"
else if (has_install)
last_error = "install.log"
else if (has_build)
last_error = "build.log"
else if (has_configure)
last_error = "configure.log"
else if (has_checksum)
last_error = "checksum.log"
else if (has_depends)
last_error = "depends.log"
else if (has_pre_clean)
last_error = "pre-clean.log"
}
print "<tr class=\"" status[PKGNAME] "\">" > html_report
if (last_error)
print "<td><a href=\"../" PKGNAME "/" last_error "\"> " location[PKGNAME] "</a></td>" > html_report
else
print "<td>" location[PKGNAME] "</td>" > html_report
print "<td>" PKGNAME "</td>" > html_report
if (depth[PKGNAME] == 0)
print "<td>&nbsp;</td>" > html_report
else
print "<td>" depth[PKGNAME] "</td>" > html_report
print "<td>" maintainer[PKGNAME] "</td>" > html_report
print "<td>" status[PKGNAME] "</td>" > html_report
print_failed_log_line(PKGNAME, "pre-clean", has_pre_clean)
print_failed_log_line(PKGNAME, "depends", has_depends)
print_failed_log_line(PKGNAME, "checksum", has_checksum)
print_failed_log_line(PKGNAME, "configure", has_configure)
print_failed_log_line(PKGNAME, "build", has_build)
print_failed_log_line(PKGNAME, "install", has_install)
print_failed_log_line(PKGNAME, "package", has_package)
print_failed_log_line(PKGNAME, "clean", has_clean)
print_failed_log_line(PKGNAME, "deinstall", has_deinstall)
print "</tr>" > html_report
if (status[PKGNAME] == "indirect-failed") {
print "<tr class=\"" status[PKGNAME] "\">" > html_report
printf "<td>&nbsp;</td>" > html_report
printf "<td colspan=\"13\"> Failed: " > html_report
printf "%s", failed_pkgs[PKGNAME] > html_report
print "</td>" > html_report
print "</tr>" > html_report
}
if (status[PKGNAME] == "prefailed") {
print "<tr class=\"" status[PKGNAME] "\">" > html_report
printf "<td>&nbsp;</td>" > html_report
printf "<td colspan=\"13\">" > html_report
print_pre_fail_reason(PKGNAME)
print "</td>" > html_report
print "</tr>" > html_report
}
}
function compute_direct_failure(CUR, dep, cur_dep, cur_depends, found, i) {
if (failed_pkgs[CUR] != "")
return
split(depends[CUR], cur_depends, "[ \t]+")
for (dep in cur_depends) {
cur_dep = cur_depends[dep]
if (status[cur_dep] == "failed") {
failed_pkgs[CUR] = failed_pkgs[CUR] " " cur_dep
} else if (status[cur_dep] == "indirect-failed") {
compute_direct_failure(cur_dep)
failed_pkgs[CUR] = failed_pkgs[cur_dep] " " failed_pkgs[CUR]
}
}
split(failed_pkgs[CUR], cur_depends, "[ \t]+")
failed_pkgs[CUR] = ""
for (dep in cur_depends) {
cur_dep = cur_depends[dep]
found[cur_dep] = 1
}
i = 0
for (dep in found) {
if (++i == 10) {
failed_pkgs[CUR] = failed_pkgs[CUR] " ..."
break;
}
failed_pkgs[CUR] = failed_pkgs[CUR] " " dep
}
}
BEGIN {
meta_dir = ARGV[1]
log_dir = ARGV[2]
report_file = meta_dir "/report"
html_report = meta_dir "/report.html"
status_file = meta_dir "/status"
tmp_sort = meta_dir "/tmp_sort"
pkgs_done = 0
pkgs_failed = 0
pkgs_prefailed = 0
pkgs_indirect_failed = 0
pkgs_indirect_prefailed = 0
while ((getline < status_file) > 0) {
if ($0 ~ "^PLATFORM=")
pkgsrc_platform = substr($0, 10)
else if ($0 ~ "^COMPILER=")
pkgsrc_compiler = substr($0, 10)
else if ($0 ~ "^BUILD_START_ISO=")
pkgsrc_build_start_iso = substr($0, 17)
else if ($0 ~ "^BUILD_END_ISO=")
pkgsrc_build_end_iso = substr($0, 15)
}
close(status_file)
while ((getline < report_file) > 0) {
if ($0 ~ "^PKGNAME=")
cur = substr($0, 9)
else if ($0 ~ "^MAINTAINER=")
maintainer[cur] = substr($0, 12)
else if ($0 ~ "^PKG_LOCATION=")
location[cur] = substr($0, 14)
else if ($0 ~ "^PKG_DEPTH=")
depth[cur] = substr($0, 11) - 1
else if ($0 ~ "^BUILD_STATUS=")
status[cur] = substr($0, 14)
else if ($0 ~ "^PKG_FAIL_REASON=")
pre_fail_reason[cur] = substr($0, 17)
else if ($0 ~ "^DEPENDS=")
depends[cur] = substr($0, 9)
}
close(report_file)
for (pkg in status) {
if (status[pkg] == "done")
++pkgs_done
else if (status[pkg] == "failed")
++pkgs_failed
else if (status[pkg] == "prefailed")
++pkgs_prefailed
else if (status[pkg] == "indirect-failed") {
compute_direct_failure(pkg)
++pkgs_indirect_failed
} else if (status[pkg] == "indirect-prefailed")
++pkgs_indirect_prefailed
}
print "<html>" > html_report
print " <head>" > html_report
printf(" <title> pkgsrc bulk build for %s from %s</title>\n",
pkgsrc_platform, pkgsrc_build_start_iso) > html_report
print " </head>" > html_report
print " <body>" > html_report
printf(" <h1> pkgsrc bulk build for %s</h1>\n", pkgsrc_platform) > html_report
printf(" <h2> Build start: %s</h2>\n", pkgsrc_build_start_iso) > html_report
printf(" <h2> Build end: %s</h2>\n", pkgsrc_build_end_iso) > html_report
print " <hr />" > html_report
all_pkgs = pkgs_done + pkgs_failed + pkgs_prefailed + pkgs_indirect_failed + pkgs_indirect_prefailed
print " <table>" > html_report
printf(" <tr><td>Total number of packages:</td><td>%d</td></tr>\n", all_pkgs) > html_report
printf(" <tr><td>Successfully built:</td><td>%d</td></tr>\n", pkgs_done) > html_report
printf(" <tr><td>Failed build:</td><td>%d</td></tr>\n", pkgs_failed) > html_report
printf(" <tr><td>Depending on failed package:</td><td>%d</td></tr>\n", pkgs_indirect_failed) > html_report
printf(" <tr><td>Explicitly broken or masked:</td><td>%d</td></tr>\n", pkgs_prefailed) > html_report
printf(" <tr><td>Depending on masked package:</td><td>%d</td></tr>\n", pkgs_indirect_prefailed) > html_report
print " </table>" > html_report
print " <hr />" > html_report
has_top_count = 0
for (pkg in status) {
if (depth[pkg] == 0 || status[pkg] != "failed")
continue
top_count[depth[pkg] " " pkg] = pkg
has_top_count = 1
}
if (has_top_count) {
sort(top_count, sorted_top_count, "-rn")
print " <h2>Packages causing the most breakage</h2>" > html_report
print " <table>" > html_report
print " <tr>" > html_report
print " <th> Location </th>" > html_report
print " <th> Package </th>" > html_report
print " <th> Breaks </th>" > html_report
print " <th> Maintainer </th>" > html_report
print " <th> Status </th>" > html_report
print " <th colspan=\"9\"> Build log </th>" > html_report
print " </tr>" > html_report
for (i = 0; i < 10 && sorted_top_count[i] != ""; ++i) {
pkg = top_count[sorted_top_count[i]]
print_failed(pkg)
}
print " </table>" > html_report
print " <hr />" > html_report
}
print " <h2> All unsuccessful builds </h2>" > html_report
print " <table>" > html_report
print " <tr>" > html_report
print " <th> Location </th>" > html_report
print " <th> Package </th>" > html_report
print " <th> Breaks </th>" > html_report
print " <th> Maintainer </th>" > html_report
print " <th> Status </th>" > html_report
print " <th colspan=\"9\"> Build log </th>" > html_report
print " </tr>" > html_report
for (pkg in status)
loc_pkg_array[location[pkg] " " pkg] = pkg
sort(loc_pkg_array, sorted_loc_pkg_array, "")
for (i = 0; sorted_loc_pkg_array[i] != ""; ++i) {
pkg = loc_pkg_array[sorted_loc_pkg_array[i]]
if (status[pkg] == "done")
continue
print_failed(pkg)
}
print " </table>" > html_report
print " </body>" > html_report
print "</html>" > html_report
}

195
pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk Executable file
View File

@@ -0,0 +1,195 @@
#!@AWK@ -f
# $NetBSD: create-report-txt.awk,v 1.8 2008/03/01 19:04:37 rillig Exp $
#
# Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
# All rights reserved.
#
# This code was developed as part of Google's Summer of Code 2007 program.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
function sort(ARRAY, INDICES, OPTIONS, i, idx, sort_cmd) {
sort_cmd = "sort " OPTIONS " > " tmp_sort
for (idx in ARRAY)
print idx | sort_cmd
close(sort_cmd)
i = 0
while ((getline < tmp_sort) > 0) {
INDICES[i] = $0
++i
}
close(tmp_sort)
system("rm " tmp_sort)
}
BEGIN {
meta_dir = ARGV[1]
report_file = meta_dir "/report"
txt_report = meta_dir "/report.txt"
html_report = meta_dir "/report.html"
status_file = meta_dir "/status"
tmp_sort = meta_dir "/tmp_sort"
pkgs_done = 0
pkgs_failed = 0
pkgs_prefailed = 0
pkgs_indirect_failed = 0
pkgs_indirect_prefailed = 0
while ((getline < status_file) > 0) {
if ($0 ~ "^PLATFORM=")
pkgsrc_platform = substr($0, 10)
else if ($0 ~ "^COMPILER=")
pkgsrc_compiler = substr($0, 10)
else if ($0 ~ "^BUILD_START_ISO=")
pkgsrc_build_start_iso = substr($0, 17)
else if ($0 ~ "^BUILD_START_DIR=")
pkgsrc_build_start_dir = substr($0, 17)
else if ($0 ~ "^BUILD_END_ISO=")
pkgsrc_build_end_iso = substr($0, 15)
else if ($0 ~ "^BASE_URL=")
pkgsrc_base_url = substr($0, 10)
}
close(status_file)
while ((getline < report_file) > 0) {
if ($0 ~ "^PKGNAME=")
cur = substr($0, 9)
else if ($0 ~ "^MAINTAINER=")
maintainer[cur] = substr($0, 12)
else if ($0 ~ "^PKG_LOCATION=")
location[cur] = substr($0, 14)
else if ($0 ~ "^PKG_DEPTH=")
depth[cur] = substr($0, 11) - 1
else if ($0 ~ "^BUILD_STATUS=") {
status[cur] = substr($0, 14)
}
}
close(report_file)
for (pkg in status) {
loc = location[pkg]
if (status[pkg] == "failed") {
broken_location[loc] += depth[pkg]
pkg_location[loc] = pkg
if (location_status[loc] == "")
location_status[loc] = "failed"
else if (location_status[loc] == "ignore")
location_status[loc] = "mixed"
} else {
if (location_status[loc] == "failed")
location_status[loc] = "mixed"
else if (location_status[loc] == "")
location_status[loc] = "ignore"
}
if (status[pkg] == "done")
++pkgs_done
else if (status[pkg] == "failed")
++pkgs_failed
else if (status[pkg] == "prefailed")
++pkgs_prefailed
else if (status[pkg] == "indirect-failed")
++pkgs_indirect_failed
else if (status[pkg] == "indirect-prefailed")
++pkgs_indirect_prefailed
}
print "pkgsrc bulk build report" > txt_report
print "========================" > txt_report
print "" > txt_report
print pkgsrc_platform > txt_report
print "Compiler: " pkgsrc_compiler > txt_report
print "" > txt_report
print "Build start: " pkgsrc_build_start_iso > txt_report
print "Build end: " pkgsrc_build_end_iso > txt_report
print "" > txt_report
report_base_url = pkgsrc_base_url "/" pkgsrc_build_start_dir
print "Full report: " report_base_url "/meta/report.html" > txt_report
print "Machine readable version: " report_base_url "/meta/report.bz2" > txt_report
print "" > txt_report
all_pkgs = pkgs_done + pkgs_failed + pkgs_prefailed + pkgs_indirect_failed + pkgs_indirect_prefailed
printf "Total number of packages: %5d\n", all_pkgs > txt_report
printf " Successfully built: %5d\n", pkgs_done > txt_report
printf " Failed to build: %5d\n", pkgs_failed > txt_report
printf " Depending on failed package: %5d\n", pkgs_indirect_failed > txt_report
printf " Explicitly broken or masked: %5d\n", pkgs_prefailed > txt_report
printf " Depending on masked package: %5d\n", pkgs_indirect_prefailed > txt_report
print "" > txt_report
has_top_count = 0
for (loc in location_status) {
if (broken_location[loc] == "" || broken_location[loc] == 0)
continue
top_count[broken_location[loc] " " loc] = loc
has_top_count = 1
}
if (has_top_count) {
sort(top_count, sorted_top_count, "-rn")
print "Packages breaking the most other packages" > txt_report
print "" > txt_report
print "Package Breaks Maintainer" > txt_report
print "-------------------------------------------------------------------------" > txt_report
for (i = 0; i < 10 && sorted_top_count[i] != ""; ++i) {
loc = top_count[sorted_top_count[i]]
printf "%- 37s % 6d %s\n", loc, broken_location[loc],
maintainer[pkg_location[loc]] > txt_report
}
print "" > txt_report
}
print "Build failures" > txt_report
print "" > txt_report
print "Package Breaks Maintainer" > txt_report
print "-------------------------------------------------------------------------" > txt_report
sort(location_status, sorted_loc, "")
for (i = 0; sorted_loc[i] != ""; ++i) {
loc = sorted_loc[i]
if (location_status[loc] == "ignore")
continue
if (broken_location[loc] == 0)
printf "%- 44s %s\n", loc,
maintainer[pkg_location[loc]] > txt_report
else
printf "%- 37s % 6d %s\n", loc, broken_location[loc],
maintainer[pkg_location[loc]] > txt_report
if (location_status[loc] != "mixed")
continue
for (p in status) {
if (location[p] != loc || status[p] != "failed")
continue
if (depth[p] == 0)
printf " %- 40s %s\n", p, maintainer[p] > txt_report
else
printf " %- 33s % 6d %s\n", p, depth[p], maintainer[p] > txt_report
}
}
close(txt_report)
}

89
pkgtools/pbulk/files/pbulk/scripts/create-report.awk Executable file
View File

@@ -0,0 +1,89 @@
#!@AWK@ -f
# $NetBSD: create-report.awk,v 1.4 2008/09/14 18:59:02 joerg Exp $
#
# Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
# All rights reserved.
#
# This code was developed as part of Google's Summer of Code 2007 program.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
BEGIN {
meta_dir = ARGV[1]
pbuild_file = meta_dir "/pbuild"
presolve_file = meta_dir "/presolve"
full_pbuild_file = meta_dir "/report"
FS = "|"
while ((getline < pbuild_file) > 0) {
status[$1] = $2
restricted[$1] = $3
depth[$1] = $4
}
close(pbuild_file)
FS = ""
while ((getline < presolve_file) > 0) {
if ($0 ~ "^PKGNAME=") {
cur = substr($0, 9)
pkg[cur] = $0
} else {
pkg[cur] = pkg[cur] "\n" $0
}
if ($0 ~ "^MAINTAINER=")
maintainer[cur] = substr($0, 12)
if ($0 ~ "^PKG_LOCATION=") {
loc = substr($0, 14)
location[cur] = loc
if (status[cur] == "failed") {
broken_location[loc] += depth[cur]
pkg_location[loc] = cur
if (location_status[loc] == "")
location_status[loc] = "failed"
else if (location_status[loc] == "ignore")
location_status[loc] = "mixed"
} else {
if (location_status[loc] == "failed")
location_status[loc] = "mixed"
else if (location_status[loc] == "")
location_status[loc] = "ignore"
}
}
}
close(presolve_file)
printf "" > full_pbuild_file
for (p in pkg) {
print pkg[p] > full_pbuild_file
print "PKG_DEPTH=" depth[p] > full_pbuild_file
print "BUILD_STATUS=" status[p] > full_pbuild_file
}
close(full_pbuild_file)
}

218
pkgtools/pbulk/files/pbulk/scripts/pkg-build Executable file
View File

@@ -0,0 +1,218 @@
#!@SH@
# $NetBSD: pkg-build,v 1.25 2012/11/23 12:13:35 joerg Exp $
#
# Copyright (c) 2007, 2008 Joerg Sonnenberger <joerg@NetBSD.org>.
# All rights reserved.
#
# This code was developed as part of Google's Summer of Code 2007 program.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
. @PBULK_CONFIG@
if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
echo "Your configuration has version ${config_version}."
echo "This version of pbulk expects version @PBULK_CONFIG_VERSION@."
exit 1
fi
cleanup() {
if [ "$keep_wrkdir" = yes ]; then
${make} pbulk-save-wrkdir INTO=${bulklog}/${pkgname}/wrkdir.tar.gz TAR="${tar}"
fi
if [ "$keep_prefix" = yes ] && [ -f ${bulklog}/${pkgname}/install.log ]; then
if [ "$cross_compile" != "no" ]; then
cur_destdir=${target_destdir}
else
cur_destdir=
fi
${tar} -czf ${bulklog}/${pkgname}/prefix.tar.gz ${cur_destdir}${prefix}
fi
${make} clean > /dev/null 2>&1 || true
exit 1
}
run_direct() { "$@"; }
run_su() { su ${unprivileged_user} -c '"$@"' make "$@"; }
run_make() {
local run_cmd
run_cmd=$1
shift
${run_cmd} ${make} $1 \
BATCH=1 \
DEPENDS_TARGET=/nonexistent \
${MAKE_FLAGS} \
WRKLOG=${bulklog}/${pkgname}/work.log
}
run_usergroup() {
[ "${use_destdir}" != no ] || return 0
case "${usergroup_phase}" in
*configure)
[ "$1" != "configure" ] || ${make} create-usergroup clean
;;
*build)
[ "$1" != "build" ] || ${make} create-usergroup
;;
pre-install)
[ "$1" != "install" ] || ${make} create-usergroup
;;
esac
}
while read build_info_line; do
case "${build_info_line}" in
PKGNAME=*)
pkgname=${build_info_line#PKGNAME=}
;;
PKG_LOCATION=*)
pkgdir=${build_info_line#PKG_LOCATION=}
;;
DEPENDS=*)
dependencies=${build_info_line#DEPENDS=}
;;
MULTI_VERSION=*)
MAKE_FLAGS=${build_info_line#MULTI_VERSION=}
;;
USE_DESTDIR=*)
use_destdir=${build_info_line#USE_DESTDIR=}
;;
BOOTSTRAP_PKG=*)
is_bootstrap=${build_info_line#BOOTSTRAP_PKG=}
;;
USERGROUP_PHASE=*)
usergroup_phase=${build_info_line#USERGROUP_PHASE=}
esac
done
case "$use_destdir" in
user-destdir)
run_build=run_su
run_install=run_su
;;
destdir)
run_build=run_su
run_install=run_direct
;;
*)
run_build=run_direct
run_install=run_direct
;;
esac
if [ `@ID@ -u` -ne 0 ]; then
run_build=run_direct
run_install=run_direct
fi
${pkg_up_to_date_script} ${pkgname} ${dependencies} && exit 0
set -e
@PREFIX@/libexec/pbulk/client-clean
# Create the output directory and clean it up
mkdir -p ${bulklog}/${pkgname}
rm -f ${bulklog}/${pkgname}/*
if [ "$use_destdir" = "destdir" -o "$use_destdir" = "user-destdir" ]; then
touch ${bulklog}/${pkgname}/work.log
@CHOWN@ ${unprivileged_user} ${bulklog}/${pkgname}/work.log
fi
pkg_add_normal() {
PKG_PATH=${packages}/All ${pkg_add} -K ${cur_pkgdb} "$@"
}
pkg_add_cross() {
PKG_PATH=${packages}/All ${pkg_add} -K ${cur_pkgdb} \
-m ${target_arch} -I -p ${target_destdir}${prefix} "$@"
echo "Fixing recorded cwd..."
${pkg_info} -K ${target_destdir}${pkgdb} | while read pkg junk; do
${sed} -e 's|@cwd '"${target_destdir}"'|@cwd |' \
${cur_pkgdb}/"${pkg}"/+CONTENTS > \
${cur_pkgdb}/"${pkg}"/+CONTENTS.tmp
mv ${cur_pkgdb}/"${pkg}"/+CONTENTS.tmp \
${cur_pkgdb}/"${pkg}"/+CONTENTS
done
}
if [ "$cross_compile" != "no" ]; then
pkg_add_cmd=pkg_add_cross
cur_pkgdb="${target_destdir}${pkgdb}"
else
pkg_add_cmd=pkg_add_normal
cur_pkgdb="${pkgdb}"
fi
# Go to target directory
cd ${pkgsrc}/${pkgdir}
# Clean build area, just in case
${make} clean > ${bulklog}/${pkgname}/pre-clean.log 2>&1
# Install all dependencies the package said it would need
if [ ! -z "$dependencies" ]; then
${pkg_add_cmd} $dependencies > ${bulklog}/${pkgname}/depends.log 2>&1
fi
# Build package, create a separate log file for each major phase
run_make run_direct checksum > ${bulklog}/${pkgname}/checksum.log 2>&1 || cleanup
run_usergroup configure > ${bulklog}/${pkgname}/configure.log 2>&1 || cleanup
run_make ${run_build} configure >> ${bulklog}/${pkgname}/configure.log 2>&1 || cleanup
run_usergroup build> ${bulklog}/${pkgname}/build.log 2>&1 || cleanup
run_make ${run_build} all >> ${bulklog}/${pkgname}/build.log 2>&1 || cleanup
run_usergroup install > ${bulklog}/${pkgname}/install.log 2>&1 || cleanup
run_make ${run_install} stage-install >> ${bulklog}/${pkgname}/install.log 2>&1 || cleanup
run_make run_direct package > ${bulklog}/${pkgname}/package.log 2>&1 || cleanup
# When using DESTDIR build, add the package once to test install rules.
# This is not done for potential bootstrap packages as they might already
# be installed.
if [ "${use_destdir}" != "no" ] && \
[ -z "${is_bootstrap}" ]; then
if ! ${pkg_add_cmd} ${pkgname} \
>> ${bulklog}/${pkgname}/package.log 2>&1; then
run_make run_direct package-clean
cleanup
fi
fi
# Clean build area
${make} clean > ${bulklog}/${pkgname}/clean.log 2>&1
# Test uninstall rules. This is not for cross-compiling as the install script
# is not run in that case anyway. This is also not done for packages marked as
# part of the bootstrap, those have the preserve flag set.
if [ "$cross_compile" = "no" ] && \
[ -z "${is_bootstrap}" ]; then
${pkg_delete} -K ${cur_pkgdb} ${pkgname} > ${bulklog}/${pkgname}/deinstall.log 2>&1
fi
# Comment the following out if you want to test all deinstall scripts.
# This is quite expensive and mostly redundant, so it is disabled by default.
#${pkg_delete} -r \* > /dev/null 2>&1 || true
# Cleanup build logs on success
rm -R ${bulklog}/${pkgname}

77
pkgtools/pbulk/files/pbulk/scripts/pkg-up-to-date Executable file
View File

@@ -0,0 +1,77 @@
#!@SH@
# $NetBSD: pkg-up-to-date,v 1.9 2010/02/24 22:51:37 joerg Exp $
#
# Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
# All rights reserved.
#
# This code was developed as part of Google's Summer of Code 2007 program.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
. @PBULK_CONFIG@
set -e
if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
echo "Your configuration has version ${config_version}."
echo "This version of pbulk expects version @PBULK_CONFIG_VERSION@."
exit 1
fi
pkg="${packages}/All/$1.tgz"
[ -f ${pkg} ]
${pkg_info} -qb ${pkg} | sed 's/:/ /' | while read file file_id; do
[ -z "$file" ] && continue
[ -e "${pkgsrc}/${file}" ]
id=`@SED@ -e '/[$]NetBSD/!d' -e 's/^.*\([$]NetBSD[^$]*[$]\).*$/\1/;q' ${pkgsrc}/${file}`
[ "$id" = "$file_id" ]
done
# TODO: compare build options
# Remove current package, so that only dependencies are in $* now.
shift
${pkg_info} -qN ${pkg} | while read dep; do
# pkg_info prints a trailing newline, ignore that
[ -z "${dep}" ] && continue
found=0
for dep2 in $*; do
if [ $dep = $dep2 ]; then
found=1
break
fi
done
[ $found = 1 ]
case "$skip_age_check" in
[Yy][Ee][Ss])
;;
*)
[ "${packages}/All/${dep}.tgz" -ot "${pkg}" ]
;;
esac
done

93
pkgtools/pbulk/files/pbulk/scripts/pre-build Executable file
View File

@@ -0,0 +1,93 @@
#!@SH@
# $NetBSD: pre-build,v 1.11 2012/11/23 12:13:35 joerg Exp $
#
# Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
# All rights reserved.
#
# This code was developed as part of Google's Summer of Code 2007 program.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
. @PBULK_CONFIG@
set -e
if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
echo "Your configuration has version ${config_version}."
echo "This version of pbulk expects version @PBULK_CONFIG_VERSION@."
exit 1
fi
if [ "$cross_compile" != "no" ]; then
if [ -z "${target_destdir}" ]; then
echo "target_destdir must be set for cross-compiling."
exit 1
fi
if [ ! -d "${target_destdir}" ]; then
echo "target_destdir must exist for cross-compiling."
exit 1
fi
if [ `cd "${target_destdir}" && pwd` = "/" ]; then
echo "target_destdir must not be / for cross-compiling."
exit 1
fi
if [ -z "${target_arch}" ]; then
echo "target_arch must be specified for cross-compiling."
exit 1
fi
fi
if [ -d "${bulklog}/meta" ]; then
echo "Warning: All log files of the previous pbulk run will be"
echo "removed in 5 seconds. If you want to abort, press Ctrl-C."
sleep 5
if [ "${reuse_scan_results}" = yes -a -f "${bulklog}/meta/pscan" ]; then
echo "Reusing old scan results"
rm -rf "${bulklog}.old"
mv "${bulklog}" "${bulklog}.old"
else
echo "Removing old scan results"
rm -rf "${bulklog}"/* || true
fi
else
rm -rf "${bulklog}"/* || true
fi
mkdir -p "${bulklog}" "${loc}"
@PREFIX@/libexec/pbulk/client-clean
# Log common settings...
opsys=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=OPSYS`
opver=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=OS_VERSION`
platform=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=MACHINE_ARCH`
compiler=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=PKGSRC_COMPILER`
echo "PLATFORM=${opsys} ${opver}/${platform}" > ${loc}/status
echo "COMPILER=${compiler}" >> ${loc}/status
date '+%Y-%m-%d %H:%M|%Y%m%d.%H%M' | sed 's/^\(.*\)|\(.*\)$/BUILD_START_ISO=\1\
BUILD_START_DIR=\2/' >> ${loc}/status
echo "BASE_URL=${base_url}" >> ${loc}/status

110
pkgtools/pbulk/files/pbulk/scripts/report Executable file
View File

@@ -0,0 +1,110 @@
#!@SH@
# $NetBSD: report,v 1.8 2008/09/16 18:21:30 joerg Exp $
#
# Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
# All rights reserved.
#
# This code was developed as part of Google's Summer of Code 2007 program.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
. @PBULK_CONFIG@
set -e
if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
echo "Your configuration has version ${config_version}."
echo "This version of pbulk expects version @PBULK_CONFIG_VERSION@."
exit 1
fi
echo "Build reports..."
${report_script} ${loc}
${bzip2} -zc ${loc}/report > ${loc}/report.bz2
${report_html_script} ${loc} ${bulklog}
${report_txt_script} ${loc}
${report_graph_script} ${loc}
if [ -x "${neato}" ] && \
[ "`grep -- '->' < ${loc}/report.dot | wc -l`" -lt ${report_graph_script_limit} ]; then
${neato} -Tsvg -Goverlap=ortho -Gsplines=true \
-o ${loc}/report.svg ${loc}/report.dot
else
rm -f ${loc}/report.svg
fi
while read line; do
case "${line}" in
BUILD_START_ISO=*)
build_start_iso=${line#BUILD_START_ISO=}
;;
BUILD_START_DIR=*)
build_start_dir=${line#BUILD_START_DIR=}
;;
PLATFORM=*)
platform=${line#PLATFORM=}
esac
done < ${loc}/status
if [ -z "${build_start_iso}" ] || [ -z "${build_start_dir}" ]; then
echo "Could not find start time of build."
exit 1
fi
echo "Sending report mail..."
cat ${loc}/report.txt | ${mail} -s "${report_subject_prefix} ${platform} ${build_start_iso}" ${report_recipients}
cd ${bulklog}
echo "Uploading report..."
{
echo "meta/report.bz2"
echo "meta/report.txt"
echo "meta/report.html"
echo "meta/report.dot"
echo "meta/report.svg"
echo "meta/status"
while read pkg; do
echo "${pkg}/pre-clean.log"
echo "${pkg}/depends.log"
echo "${pkg}/checksum.log"
echo "${pkg}/configure.log"
echo "${pkg}/build.log"
echo "${pkg}/install.log"
echo "${pkg}/package.log"
echo "${pkg}/clean.log"
echo "${pkg}/deinstall.log"
echo "${pkg}/work.log"
done < ${loc}/error
} | {
echo "+ meta/"
while read pkg; do
echo "+ ${pkg}/"
done < ${loc}/error
while read file; do
[ -f "$file" ] && echo "+ $file"
done
echo "- *"
} | ${rsync} --exclude-from=- ${report_rsync_args} . ${report_rsync_target}/${build_start_dir}

115
pkgtools/pbulk/files/pbulk/scripts/scan Executable file
View File

@@ -0,0 +1,115 @@
#!@SH@
# $NetBSD: scan,v 1.8 2012/11/23 12:13:35 joerg Exp $
#
# Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
# All rights reserved.
#
# This code was developed as part of Google's Summer of Code 2007 program.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
. @PBULK_CONFIG@
set -e
if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
echo "Your configuration has version ${config_version}."
echo "This version of pbulk expects version @PBULK_CONFIG_VERSION@."
exit 1
fi
if [ "${reuse_scan_results}" = yes -a -f "${bulklog}.old/meta/pscan" ]; then
echo "Using old scan results from ${bulklog}.old/meta/pscan"
extra_pscan_args="-L ${bulklog}.old/meta/pscan"
else
extra_pscan_args=""
fi
if [ -z "${limited_list}" ]; then
echo "Scanning..."
case "${master_mode}" in
[nN][oO])
${pscan} -v -M ${make} ${extra_pscan_args} ${pkgsrc} ${loc}/pscan 2>> ${loc}/pscan.stderr
;;
[yY][eE][sS])
${pscan} -v -I ${pscan_start_script} -m ${master_port_scan} -M ${make} ${extra_pscan_args} ${pkgsrc} ${loc}/pscan 2>> ${loc}/pscan.stderr
;;
*)
echo "master_mode must be either yes or no."
exit 1
;;
esac
echo "Resolving..."
if ! ${presolve} -v ${loc}/pscan > ${loc}/presolve 2> ${loc}/presolve-err.log; then
echo "Global dependency resolution failed, check ${loc}/presolve-err.log for details" >&2
case "${ignore_missing_dependencies}" in
[nN][oO])
exit 1
;;
[yY][eE][sS])
${presolve} -p -v ${loc}/pscan > ${loc}/presolve 2> /dev/null
;;
*)
echo "Unsupported value for ignore_missing_dependencies."
;;
esac
fi
else
initial=1
mkdir -p ${loc}
grep -v '^#' "${limited_list}" > ${loc}/missing
while [ -s ${loc}/missing ]; do
sort -u ${loc}/missing > ${loc}/missing.s
echo "Scanning..."
case "${master_mode}" in
[nN][oO])
${pscan} -v -l -M ${make} ${pkgsrc} ${loc}/pscan < ${loc}/missing.s
;;
[yY][eE][sS])
${pscan} -v -l -I ${pscan_start_script} -m ${master_port_scan} -M ${make} ${pkgsrc} ${loc}/pscan < ${loc}/missing.s
;;
*)
echo "master_mode must be either yes or no."
exit 1
;;
esac
echo "Resolving..."
if [ "$initial" = 1 ]; then
${presolve} -i ${loc}/missing -v ${loc}/pscan > ${loc}/presolve
initial=0
else
${presolve} -i ${loc}/missing -v ${loc}/presolve ${loc}/pscan > ${loc}/presolve.new
cmp -s ${loc}/presolve.new ${loc}/presolve && break
mv ${loc}/presolve.new ${loc}/presolve
fi
done
if [ -s ${loc}/missing ]; then
echo "Unresolvable dependencies found, exiting:"
cat ${loc}/missing
exit 1
fi
fi

22
pkgtools/pbulk/files/pbulk/scripts/scan-client-start Executable file
View File

@@ -0,0 +1,22 @@
#!@SH@
# $NetBSD: scan-client-start,v 1.3 2012/11/23 12:13:35 joerg Exp $
. @PBULK_CONFIG@
set -e
if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
echo "Your configuration has version ${config_version}."
echo "This version of pbulk expects version @PBULK_CONFIG_VERSION@."
exit 1
fi
if [ -f "${bulklog}.old/meta/pscan" ]; then
extra_pscan_args="-L ${bulklog}.old/meta/pscan"
else
extra_pscan_args=""
fi
for client in ${scan_clients}; do
ssh $client "${pscan_prepare} && ${pscan} -c ${master_port_scan} -M ${make} ${extra_pscan_args} ${pkgsrc}" &
done

53
pkgtools/pbulk/files/pbulk/scripts/upload Executable file
View File

@@ -0,0 +1,53 @@
#!@SH@
# $NetBSD: upload,v 1.3 2008/09/16 18:21:30 joerg Exp $
#
# Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
# All rights reserved.
#
# This code was developed as part of Google's Summer of Code 2007 program.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
. @PBULK_CONFIG@
set -e
if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
echo "Your configuration has version ${config_version}."
echo "This version of pbulk expects version @PBULK_CONFIG_VERSION@."
exit 1
fi
echo "Uploading packages..."
cd ${packages}
{
[ "${checksum_packages}" != "no" ] && \
[ "${checksum_packages}" != "NO" ] && \
echo "+ SHA512.bz2"
echo "+ All/pkg_summary.bz2"
echo "+ All/pkg_summary.gz"
${packages_script} ${loc}
echo "- *"
} | sort | ${rsync} --exclude-from=- ${pkg_rsync_args} . ${pkg_rsync_target}