Import of pkgsrc-2016Q3

pkgtools/pbulk/Makefile

@@ -1,6 +1,6 @@
-# $NetBSD: Makefile,v 1.73 2015/09/13 11:45:36 joerg Exp $
+# $NetBSD: Makefile,v 1.77 2016/05/09 00:08:21 joerg Exp $
 
-PKGNAME=	pbulk-0.57
+PKGNAME=	pbulk-0.61
 COMMENT=	Modular bulk build framework
 
 .include "../../pkgtools/pbulk/Makefile.common"
@@ -39,7 +39,7 @@ SUBST_VARS.tools=	AWK BZIP2 CHOWN DIGEST GZIP_CMD ID MAIL_CMD NEATO \
 CONF_FILES+=	share/examples/pbulk/pbulk.conf ${PKG_SYSCONFDIR}/pbulk.conf
 
 PBULK_CONFIG=		${PKG_SYSCONFDIR}/pbulk.conf
-PBULK_CONFIG_VERSION=	0.53
+PBULK_CONFIG_VERSION=	0.54
 
 INSTALLATION_DIRS=	bin libexec/pbulk share/examples/pbulk
 USE_BSD_MAKEFILE=	yes

pkgtools/pbulk/files/pbulk/lib/netaddr.c

@@ -1,4 +1,4 @@
-/* $NetBSD: netaddr.c,v 1.8 2014/03/14 09:45:31 jperkin Exp $ */
+/* $NetBSD: netaddr.c,v 1.11 2015/12/07 16:52:39 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -33,48 +33,180 @@
 
 #include <nbcompat.h>
 
-#include <sys/socket.h>
 #include <arpa/inet.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <netinet/in.h>
+#include <nbcompat/netdb.h>
 #include <nbcompat/stdlib.h>
 #include <nbcompat/string.h>
 
 #include "pbulk.h"
 
-int
-parse_sockaddr_in(const char *str, struct sockaddr_in *addr)
+struct listener {
+	LIST_ENTRY(listener) ls_link;
+	struct event ls_event;
+	int ls_fd;
+};
+
+static LIST_HEAD(, listener) all_listener = LIST_HEAD_INITIALIZER(&all_listener);
+
+void
+shutdown_listeners(void)
 {
-	const char *port_sep;
-	char *port_end;
-	struct in_addr in;
-	unsigned long tmp;
-
-	if ((port_sep = strrchr(str, ':')) != NULL) {
-		char *addr_part = strdup(str);
-
-		addr_part[port_sep - str] = '\0';
-		if (inet_aton(addr_part, &in) == 0) {
-			free(addr_part);
-			return -1;
-		}
-		free(addr_part);
-		str = port_sep + 1;
-	} else {
-		memset(&in, 0, sizeof(in));
+	struct listener *ls;
+	while ((ls = LIST_FIRST(&all_listener)) != NULL) {
+		LIST_REMOVE(ls, ls_link);
+		event_del(&ls->ls_event);
+		close(ls->ls_fd);
+		free(ls);
 	}
+}
 
-	errno = 0;
-	tmp = strtoul(str, &port_end, 10);
-	if (*str == '\0' || *port_end != '\0' || errno != 0 || tmp > 0xfffful)
+static int
+split_netaddr(const char *str, char **host, char **port, int *numeric)
+{
+	const char *sep = strrchr(str, ':');
+	if (sep == NULL) {
+		*numeric = 0;
+		*host = NULL;
+		*port = xstrdup(str);
+		return 0;
+	}
+	if (sep[1] == '\0') {
+		warnx("invalid network address with empty port: %s", str);
 		return -1;
-	memset(addr, 0, sizeof(struct sockaddr_in));
-	addr->sin_port = htons((in_port_t)tmp);
-	addr->sin_addr = in;
-#if !defined(__sun) && !defined(__hpux) && !defined(__INTERIX) && \
-    !defined(__digital__) && !defined(__linux) && !defined(__sgi)
-	addr->sin_len = sizeof(*addr);
-#endif
-	addr->sin_family = AF_INET;
+	}
+	if (sep == str) {
+		/* Consider ":port" as equivalent to just "port". */
+		*numeric = 0;
+		*host = NULL;
+		*port = xstrdup(sep + 1);
+		return 0;
+	}
+	if (str[0] == '[' && sep[-1] == ']') {
+		/* Recognize URL-style numeric IPv6 addresses in []. */
+		*numeric = 1;
+		*host = xstrndup(str + 1, sep - str - 2);
+		*port = xstrdup(sep + 1);
+		return 0;
+	}
+	if (memchr(str, ':', sep - str) != NULL) {
+		warnx("invalid network with colon in host name: %s", str);
+		return -1;
+	}
+	*numeric = 0;
+	*host = xstrndup(str, sep - str);
+	*port = xstrdup(sep + 1);
 	return 0;
 }
+
+static struct addrinfo *
+prepare_getaddrinfo(const char *netaddr, int passive)
+{
+	struct addrinfo hints, *result;
+	char *host, *port;
+	int numeric, rv;
+
+	if (split_netaddr(netaddr, &host, &port, &numeric))
+		return NULL;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_protocol = IPPROTO_TCP;
+	if (passive)
+		hints.ai_flags |= AI_PASSIVE;
+	if (numeric)
+		hints.ai_flags |= AI_NUMERICHOST;
+	rv = getaddrinfo(host, port, &hints, &result);
+	free(host);
+	free(port);
+	if (rv != 0) {
+		if (rv == EAI_SYSTEM)
+			warn("getaddrinfo failed");
+		else
+			warnx("getaddrinfo failed: %s", gai_strerror(rv));
+		return NULL;
+	}
+	return result;
+}
+
+int
+connect_sockaddr(const char *netaddr)
+{
+	struct addrinfo *result, *res;
+	int s;
+
+	if ((result = prepare_getaddrinfo(netaddr, 0)) == NULL)
+		return -1;
+
+	s = -1;
+	for (res = result; res != NULL; res = res->ai_next) {
+		s = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
+		if (s == -1)
+			continue;
+		if (connect(s, res->ai_addr, res->ai_addrlen) != -1)
+			break;
+		close(s);
+		s = -1;
+	}
+	freeaddrinfo(result);
+	return s;
+}
+
+static int
+bind_and_listen(struct addrinfo *res, void (*cb)(int, void *))
+{
+	static const int one = 1;
+	struct listener *ls;
+	int s;
+
+	s = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
+	if (s == -1)
+		return 0;
+	if (fcntl(s, F_SETFD, FD_CLOEXEC) == -1) {
+		close(s);
+		return 0;
+	}
+#ifdef IPV6_V6ONLY
+	/*
+	 * Disable mapped IPv4, which is still the default at least on Linux.
+	 * It doesn't matter if an error occurs, but success allows binding
+	 * to both IPv4 and IPv6 wild card addresses. Otherwise,
+	 * the second bind would fail.
+	 */
+	if (res->ai_family == AF_INET6)
+		setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &one, sizeof(one));
+#endif
+	if (bind(s, res->ai_addr, res->ai_addrlen) == -1) {
+		close(s);
+		return 0;
+	}
+	if (listen(s, 5) == -1) {
+		close(s);
+		return 0;
+	}
+	ls = xmalloc(sizeof(*ls));
+	ls->ls_fd = s;
+	LIST_INSERT_HEAD(&all_listener, ls, ls_link);
+	event_add(&ls->ls_event, s, 0, 1, cb, NULL);
+	return 1;
+}
+
+int
+listen_sockaddr(const char *netaddr, void (*cb)(int, void *))
+{
+	struct addrinfo *result, *res;
+	int got_address;
+
+	if ((result = prepare_getaddrinfo(netaddr, 0)) == NULL)
+		return -1;
+
+	got_address = 0;
+	for (res = result; res != NULL; res = res->ai_next)
+		got_address |= bind_and_listen(res, cb);
+
+	freeaddrinfo(result);
+	return got_address ? 0 : -1;
+}

pkgtools/pbulk/files/pbulk/lib/pbulk.h

@@ -1,4 +1,4 @@
-/* $NetBSD: pbulk.h,v 1.5 2014/01/15 13:52:10 joerg Exp $ */
+/* $NetBSD: pbulk.h,v 1.6 2015/12/07 16:52:39 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007, 2009 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -75,8 +75,9 @@ void		 deferred_write(int fd, const void *, size_t,
 ssize_t		 atomic_read(int, void *, size_t);
 ssize_t		 atomic_write(int, const void *, size_t);
 
-
-int		 parse_sockaddr_in(const char *, struct sockaddr_in *);
+int		 connect_sockaddr(const char *);
+int		 listen_sockaddr(const char *, void (*)(int, void *));
+void		 shutdown_listeners(void);
 pid_t		 fork_chdir_exec(const char *, const char *,
 				 const char * const *, int *);
 char		*read_from_child(const char *, const char *,

pkgtools/pbulk/files/pbulk/pbuild/client.c

@@ -1,4 +1,4 @@
-/* $NetBSD: client.c,v 1.4 2012/01/19 18:53:32 joerg Exp $ */
+/* $NetBSD: client.c,v 1.6 2015/12/07 16:52:39 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -52,20 +52,13 @@
 void
 client_mode(const char *client_port)
 {
-	struct sockaddr_in dst;
 	uint32_t build_info_len;
 	ssize_t recv_bytes, sent_bytes;
 	char *build_info;
 	int fd;
 
-	if (parse_sockaddr_in(client_port, &dst))
-		errx(1, "Could not parse addr/port");
-
-	fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
-	if (fd == -1)
-		err(1, "Could not create socket");	
-	if (connect(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
-		err(1, "Could not connect socket");
+	if ((fd = connect_sockaddr(client_port)) == -1)
+		err(1, "Could not creation connection to %s", client_port);
 
 loop:
 	sent_bytes = atomic_write(fd, "G", 1);
@@ -84,6 +77,8 @@ loop:
 	if (recv_bytes != 4)
 		errx(1, "Premature end while reading build info from socket");
 	build_info_len = ntohl(build_info_len);
+	if (build_info_len == 0)
+		exit(0);
 	if (build_info_len < 10 || build_info_len > 0xffffff)
 		errx(1, "Invalid build info length from master");
 

pkgtools/pbulk/files/pbulk/pbuild/jobs.c

@@ -1,4 +1,4 @@
-/* $NetBSD: jobs.c,v 1.15 2014/01/15 13:52:10 joerg Exp $ */
+/* $NetBSD: jobs.c,v 1.16 2015/11/03 19:06:47 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007, 2009, 2011 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -364,6 +364,12 @@ build_tree(void)
 	}
 }
 
+int
+has_job(void)
+{
+	return !TAILQ_EMPTY(&buildable_jobs);
+}
+
 struct build_job *
 get_job(void)
 {

pkgtools/pbulk/files/pbulk/pbuild/master.c

@@ -1,4 +1,4 @@
-/* $NetBSD: master.c,v 1.9 2013/01/14 14:33:28 jperkin Exp $ */
+/* $NetBSD: master.c,v 1.11 2015/12/07 16:52:39 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007, 2009 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -53,8 +53,6 @@
 
 static int clients_started;
 static LIST_HEAD(, build_peer) active_peers, inactive_peers, unassigned_peers;
-static struct event listen_event;
-static int listen_event_socket;
 static struct signal_event child_event;
 static pid_t child_pid;
 
@@ -71,6 +69,11 @@ struct build_peer {
 static void	assign_job(void *);
 static void	recv_command(struct build_peer *);
 
+static void
+do_nothing(void *arg)
+{
+}
+
 static void
 kill_peer(void *arg)
 {
@@ -177,10 +180,16 @@ shutdown_master(void)
 	struct timeval tv;
 	struct build_peer *peer;
 
-	event_del(&listen_event);
-	(void)close(listen_event_socket);
-	LIST_FOREACH(peer, &inactive_peers, peer_link)
-		(void)shutdown(peer->fd, SHUT_RDWR);
+	shutdown_listeners();
+
+	LIST_FOREACH(peer, &inactive_peers, peer_link) {
+		uint32_t net_build_info_len = htonl(0);
+		(void)memcpy(peer->tmp_buf, &net_build_info_len, 4);
+		deferred_write(peer->fd, peer->tmp_buf, 4, peer, do_nothing,
+		    kill_peer);
+	}
+
+	/* Give clients a second to close connections to prevent TIME_WAIT. */
 	tv.tv_sec = 1;
 	tv.tv_usec = 0;
 	event_loopexit(&tv);
@@ -243,12 +252,9 @@ static void
 listen_handler(int sock, void *arg)
 {
 	struct build_peer *peer;
-	struct sockaddr_in src;
-	socklen_t src_len;
 	int fd;
 
-	src_len = sizeof(src);
-	if ((fd = accept(sock, (struct sockaddr *)&src, &src_len)) == -1) {
+	if ((fd = accept(sock, NULL, 0)) == -1) {
 		warn("Could not accept connection");
 		return;
 	}
@@ -291,30 +297,14 @@ child_handler(struct signal_event *ev)
 void
 master_mode(const char *master_port, const char *start_script)
 {
-	struct sockaddr_in dst;
-	int fd;
-
 	LIST_INIT(&active_peers);
 	LIST_INIT(&inactive_peers);
 	LIST_INIT(&unassigned_peers);
 
 	event_init();
 
-	if (parse_sockaddr_in(master_port, &dst))
-		errx(1, "Could not parse addr/port");
-
-	fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
-	if (fd == -1)
-		err(1, "Could not create socket");
-	if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1)
-		err(1, "Could not set close-on-exec flag");
-	if (bind(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
-		err(1, "Could not bind socket");
-	if (listen(fd, 5) == -1)
-		err(1, "Could not listen on socket");
-
-	event_add(&listen_event, fd, 0, 1, listen_handler, NULL);
-	listen_event_socket = fd;
+	if (listen_sockaddr(master_port, listen_handler))
+		errx(1, "Could not create listen socket for %s", master_port);
 
 	if (start_script) {
 		signal_add(&child_event, SIGCHLD, child_handler);
@@ -330,6 +320,4 @@ master_mode(const char *master_port, const char *start_script)
 	}
 
 	event_dispatch();
-
-	(void)close(fd);
 }

pkgtools/pbulk/files/pbulk/pbuild/pbuild.c

@@ -1,4 +1,4 @@
-/* $NetBSD: pbuild.c,v 1.6 2013/01/14 14:33:28 jperkin Exp $ */
+/* $NetBSD: pbuild.c,v 1.7 2015/11/03 19:06:47 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -150,6 +150,7 @@ main(int argc, char **argv)
 		if (argc != 0)
 			usage();
 		client_mode(client_port);
+		/* UNREACHABLE */
 	}
 
 	if (argc != 3)
@@ -160,10 +161,12 @@ main(int argc, char **argv)
 
 	init_jobs(argv[0], argv[1], argv[2]);
 
-	if (master_port != NULL)
-		master_mode(master_port, start_script);
-	else
-		standalone_mode();
+	if (has_job()) {
+		if (master_port != NULL)
+			master_mode(master_port, start_script);
+		else
+			standalone_mode();
+	}
 
 	if (report_file)
 		finish_build(report_file);

pkgtools/pbulk/files/pbulk/pbuild/pbuild.h

@@ -1,4 +1,4 @@
-/* $NetBSD: pbuild.h,v 1.6 2011/11/27 19:53:30 joerg Exp $ */
+/* $NetBSD: pbuild.h,v 1.7 2015/11/03 19:06:47 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -114,6 +114,7 @@ struct build_job {
 extern int		 verbosity;
 
 void			 init_jobs(const char *, const char *, const char *);
+int			 has_job(void);
 struct build_job	*get_job(void);
 void			 process_job(struct build_job *, enum job_state, int);
 int			 build_package(const char *, size_t);

pkgtools/pbulk/files/pbulk/pbuild/pbulk-build.1

@@ -1,4 +1,4 @@
-.\" $NetBSD: pbulk-build.1,v 1.1.1.1 2007/06/19 19:49:55 joerg Exp $
+.\" $NetBSD: pbulk-build.1,v 1.2 2015/12/07 16:52:39 joerg Exp $
 .\"
 .\" Copyright (c) 2007 Thomas Klausner and Joerg Sonnenberger.
 .\" All rights reserved.
@@ -32,16 +32,16 @@
 .Nd build all packages specified in input file
 .Sh SYNOPSIS
 .Nm
-.Fl s Oo Ar ip: Oc Ns Ar port
+.Fl s Oo Ar host: Oc Ns Ar port
 .Nm
 .Op Fl v
-.Fl c Oo Ar ip: Oc Ns Ar port
+.Fl c Oo Ar host: Oc Ns Ar port
 .Fl b Ar build_script
 .Nm
 .Op Fl I Ar start_script
 .Op Fl r Ar report_file
 .Op Fl v
-.Fl m Oo Ar ip: Oc Ns Ar port
+.Fl m Oo Ar host: Oc Ns Ar port
 .Ar input success error
 .Nm
 .Op Fl r Ar report_file
@@ -61,11 +61,11 @@ for building the packages.
 See
 .Sx BUILD SCRIPT FORMAT
 for details.
-.It Fl c Oo Ar ip: Oc Ns Ar port
+.It Fl c Oo Ar host: Oc Ns Ar port
 Obtain jobs from master running on the given
 .Ar port
 (or
-.Ar ip:port ) .
+.Ar host:port ) .
 If used with
 .Fl v ,
 print the name of the package to build to stdout.
@@ -73,18 +73,18 @@ print the name of the package to build to stdout.
 Run
 .Ar start_script
 after opening the socket and wait for completion before entering build loop.
-.It Fl m Oo Ar ip: Oc Ns Ar port
+.It Fl m Oo Ar host : Oc Ns Ar port
 Enter master mode.
 The master binds to
 .Ar port
 (or
-.Ar ip:port )
+.Ar host:port )
 and waits for clients to connect and build individual packages.
-.It Fl s Oo Ar ip: Oc Ns Ar port
+.It Fl s Oo Ar host: Oc Ns Ar port
 Query the master running on the given
 .Ar port
 (or
-.Ar ip:port )
+.Ar host:port )
 for the current number of successful, open, and failed builds.
 .It Fl r Ar report_file
 Write name of each package,

pkgtools/pbulk/files/pbulk/pbuild/stat.c

@@ -1,4 +1,4 @@
-/* $NetBSD: stat.c,v 1.3 2008/01/15 22:14:30 joerg Exp $ */
+/* $NetBSD: stat.c,v 1.4 2015/12/07 16:52:39 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -52,21 +52,14 @@
 void
 stat_mode(const char *client_port)
 {
-	struct sockaddr_in dst;
 	ssize_t recv_bytes, sent_bytes;
 	char buf[7 * 4];
 	struct build_stat st;
 	uint32_t tmp;
 	int fd;
 
-	if (parse_sockaddr_in(client_port, &dst))
-		errx(1, "Could not parse addr/port");
-
-	fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
-	if (fd == -1)
-		err(1, "Could not create socket");	
-	if (connect(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
-		err(1, "Could not connect socket");
+	if ((fd = connect_sockaddr(client_port)) == -1)
+		err(1, "Could not creation connection to %s", client_port);
 
 	sent_bytes = write(fd, "S", 1);
 	if (sent_bytes == -1)

pkgtools/pbulk/files/pbulk/pscan/client.c

@@ -1,4 +1,4 @@
-/* $NetBSD: client.c,v 1.3 2009/03/06 15:21:17 joerg Exp $ */
+/* $NetBSD: client.c,v 1.6 2016/06/20 17:54:43 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -48,10 +48,105 @@
 #include "pbulk.h"
 #include "pscan.h"
 
-void
-client_mode(const char *client_port)
+static void
+find_full_tree_client(int fd, const char *bmake_path)
+{
+	const char * extract_subdir[] = {
+		bmake_path,
+		"show-subdir-var",
+		"VARNAME=SUBDIR",
+		NULL
+	};
+	char **pkgs;
+	size_t i, allocated_pkgs, len_pkgs, len_pkgs_data;
+	char *cat_path;
+	char *buf, *buf_orig, *cat, *cat_orig;
+	size_t buf_len, cat_len;
+	uint32_t net_len_pkgs_data;
+	ssize_t sent_bytes;
+
+	buf = read_from_child(pkgsrc_tree, bmake_path, extract_subdir);
+
+	if (buf == NULL)
+		err(1, "Cannot extract categories");
+
+	cat = cat_orig = buf;
+	allocated_pkgs = len_pkgs = 0;
+	len_pkgs_data = 0;
+	pkgs = NULL;
+	for (;;) {
+		cat += strspn(cat, " \t\n");
+		cat_len = strcspn(cat, " \t\n");
+		if (cat_len == 0)
+			break;
+
+		cat_path = xasprintf("%s/%.*s", pkgsrc_tree, (int)cat_len, cat);
+		buf_orig = buf = read_from_child(cat_path, bmake_path, extract_subdir);
+		free(cat_path);
+		if (buf == NULL) {
+			warnx("Cannot extract subdirectories for %.*s", (int)cat_len, cat);
+			cat += cat_len;
+			continue;
+		}
+
+		for (;;) {
+			buf += strspn(buf, " \t\n");
+			buf_len = strcspn(buf, " \t\n");
+			if (buf_len == 0)
+				break;
+			if (len_pkgs == allocated_pkgs) {
+				if (allocated_pkgs == 0) {
+					allocated_pkgs = 1024;
+					pkgs = xmalloc(sizeof(*pkgs) *
+					               allocated_pkgs);
+				} else {
+					allocated_pkgs *= 2;
+					pkgs = xrealloc(pkgs,
+					    sizeof(*pkgs) * allocated_pkgs);
+				}
+			}
+			pkgs[len_pkgs] = xasprintf("%.*s/%.*s", (int)cat_len,
+			    cat, (int)buf_len, buf);
+			len_pkgs_data += strlen(pkgs[len_pkgs]) + 1;
+			++len_pkgs;
+			buf += buf_len;
+		}
+		free(buf_orig);
+
+		cat += cat_len;
+	}
+
+	free(cat_orig);
+
+	if (len_pkgs_data > 0xfffffffful)
+		errx(1, "Directory list too large");
+	net_len_pkgs_data = ntohl((uint32_t)len_pkgs_data);
+
+	sent_bytes = atomic_write(fd, &net_len_pkgs_data, 4);
+	if (sent_bytes == -1)
+		err(1, "Could not write to socket");
+	if (sent_bytes != 4)
+		errx(1, "Premature end of stream while writing to socket");
+	for (i = 0; i < len_pkgs; ++i) {
+		size_t l = strlen(pkgs[i]);
+		sent_bytes = atomic_write(fd, pkgs[i], l);
+		if (sent_bytes == -1)
+			err(1, "Could not write to socket");
+		if ((size_t)sent_bytes != l)
+			errx(1, "Premature end of stream while writing to socket");
+		sent_bytes = atomic_write(fd, "\n", 1);
+		if (sent_bytes == -1)
+			err(1, "Could not write to socket");
+		if (sent_bytes != 1)
+			errx(1, "Premature end of stream while writing to socket");
+		free(pkgs[i]);
+	}
+	free(pkgs);
+}
+
+void
+client_mode(const char *client_port, const char *bmake_path)
 {
-	struct sockaddr_in dst;
 	uint16_t path_len;
 	uint32_t net_output_len;
 	ssize_t recv_bytes, sent_bytes;
@@ -59,14 +154,8 @@ client_mode(const char *client_port)
 	char *path, *output;
 	int fd;
 
-	if (parse_sockaddr_in(client_port, &dst))
-		errx(1, "Could not parse addr/port");
-
-	fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
-	if (fd == -1)
-		err(1, "Could not create socket");	
-	if (connect(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
-		err(1, "Could not connect socket");
+	if ((fd = connect_sockaddr(client_port)) == -1)
+		err(1, "Could not creation connection to %s", client_port);
 
 loop:
 	recv_bytes = atomic_read(fd, &path_len, 2);
@@ -77,6 +166,8 @@ loop:
 	if (recv_bytes != 2)
 		errx(1, "Premature end while reading path length from socket");
 	path_len = ntohs(path_len);
+	if (path_len == 0)
+		exit(0);
 	if (path_len < 3)
 		errx(1, "Invalid path length from master");
 
@@ -87,6 +178,13 @@ loop:
 		err(1, "Could not read from socket");
 	if (recv_bytes != path_len || strlen(path) != path_len)
 		errx(1, "Premature end of stream while reading path from socket");
+
+	if (strcmp(path, "find_full_tree") == 0) {
+		free(path);
+		find_full_tree_client(fd, bmake_path);
+		goto loop;
+	}
+
 	if (path[0] == '/' ||
 	    strchr(path, '/') == NULL ||
 	    strchr(path, '/') != strrchr(path, '/') ||
@@ -109,12 +207,12 @@ loop:
 		errx(1, "Output too large");
 	net_output_len = htonl((uint32_t)output_len);
 
-	sent_bytes = write(fd, &net_output_len, 4);
+	sent_bytes = atomic_write(fd, &net_output_len, 4);
 	if (sent_bytes == -1)
 		err(1, "Could not write to socket");
 	if (sent_bytes != 4)
 		errx(1, "Premature end of stream while writing to socket");
-	sent_bytes = write(fd, output, output_len);
+	sent_bytes = atomic_write(fd, output, output_len);
 	if (sent_bytes == -1)
 		err(1, "Could not write to socket");
 	if ((size_t)sent_bytes != output_len)

pkgtools/pbulk/files/pbulk/pscan/jobs.c

@@ -1,4 +1,4 @@
-/* $NetBSD: jobs.c,v 1.7 2012/11/30 16:22:49 joerg Exp $ */
+/* $NetBSD: jobs.c,v 1.9 2016/06/20 17:54:43 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -296,6 +296,28 @@ add_job_full(const char *location)
 	++len_jobs;
 }
 
+int
+has_job(void)
+{
+	size_t i;
+	struct scan_entry *e;
+	struct scan_job * job;
+
+	for (i = first_undone_job; i < len_jobs; ++i) {
+		job = &jobs[i];
+		if (job->state != JOB_OPEN)
+			continue;
+		e = find_old_scan(job->pkg_location);
+		if (e == NULL)
+			return 1;
+		job->scan_output = xstrdup(e->data);
+		process_job(job, JOB_DONE);
+		i = first_undone_job - 1;
+	}
+
+	return 0;
+}
+
 struct scan_job *
 get_job(void)
 {
@@ -320,11 +342,32 @@ get_job(void)
 	return NULL;
 }
 
+static void
+parse_full_tree(char *data) {
+	char *eol;
+
+	while (*data) {
+		eol = strchr(data, '\n');
+		if (eol == NULL)
+			err(1, "Incomplete line in full tree list");
+		if (data == eol)
+			continue;
+		*eol = '\0';
+		add_job_full(data);
+		data = eol + 1;
+	}
+}
+
 void
 process_job(struct scan_job *job, enum job_state state)
 {
 	job->state = state;
 
+	if (state == JOB_DONE &&
+	    strcmp(job->pkg_location, "find_full_tree") == 0) {
+		parse_full_tree(job->scan_output);
+	}
+
 	for (; first_undone_job < len_jobs; ++first_undone_job) {
 		if (jobs[first_undone_job].state != JOB_DONE)
 			break;
@@ -489,6 +532,8 @@ write_jobs(const char *output_file)
 			warnx("%s was not processed", jobs[i].pkg_location);
 			continue;
 		}
+		if (strcmp(jobs[i].pkg_location, "find_full_tree") == 0)
+			continue;
 		if (jobs[i].scan_output == NULL) {
 			warnx("Scan failed for %s", jobs[i].pkg_location);
 			continue;

pkgtools/pbulk/files/pbulk/pscan/master.c

@@ -1,4 +1,4 @@
-/* $NetBSD: master.c,v 1.8 2013/01/14 14:33:28 jperkin Exp $ */
+/* $NetBSD: master.c,v 1.12 2016/06/23 15:07:39 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007, 2009 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -53,8 +53,6 @@
 
 static int clients_started;
 static LIST_HEAD(, scan_peer) active_peers, inactive_peers;
-static struct event listen_event;
-static int listen_event_socket;
 static struct signal_event child_event;
 static pid_t child_pid;
 
@@ -70,6 +68,12 @@ struct scan_peer {
 };
 
 static void	assign_job(struct scan_peer *);
+static void	reassign_jobs(void);
+
+static void
+do_nothing(void *arg)
+{
+}
 
 static void
 kill_peer(void *arg)
@@ -103,6 +107,7 @@ finish_job(void *arg)
 	LIST_REMOVE(peer, peer_link);
 	process_job(peer->job, JOB_DONE);
 	assign_job(peer);
+	reassign_jobs();
 }
 
 static void
@@ -154,10 +159,17 @@ shutdown_master(void)
 	struct timeval tv;
 	struct scan_peer *peer;
 
-	event_del(&listen_event);
-	(void)close(listen_event_socket);
-	LIST_FOREACH(peer, &inactive_peers, peer_link)
-		(void)shutdown(peer->fd, SHUT_RDWR);
+	shutdown_listeners();
+
+	LIST_FOREACH(peer, &inactive_peers, peer_link) {
+		uint16_t net_job_len = htons(0);
+		(void)memcpy(peer->tmp_buf, &net_job_len, 2);
+
+		deferred_write(peer->fd, peer->tmp_buf, 2, peer, do_nothing,
+		    kill_peer);
+	}
+
+	/* Give clients a second to close connections to prevent TIME_WAIT. */
 	tv.tv_sec = 1;
 	tv.tv_usec = 0;
 	event_loopexit(&tv);
@@ -191,16 +203,28 @@ assign_job(struct scan_peer *peer)
 	    kill_peer);
 }
 
+static void
+reassign_jobs(void)
+{
+	struct scan_peer *peer;
+
+	while ((peer = LIST_FIRST(&inactive_peers)) != NULL) {
+		if (!has_job())
+			break;
+		LIST_REMOVE(peer, peer_link);
+		assign_job(peer);
+		if (peer-> job == NULL)
+			break;
+	}
+}
+
 static void
 listen_handler(int sock, void *arg)
 {
 	struct scan_peer *peer;
-	struct sockaddr_in src;
-	socklen_t src_len;
 	int fd;
 
-	src_len = sizeof(src);
-	if ((fd = accept(sock, (struct sockaddr *)&src, &src_len)) == -1) {
+	if ((fd = accept(sock, NULL, 0)) == -1) {
 		warn("Could not accept connection");
 		return;
 	}
@@ -218,7 +242,6 @@ listen_handler(int sock, void *arg)
 static void
 child_handler(struct signal_event *ev)
 {
-	struct scan_peer *peer;
 	int status;
 
 	if (waitpid(child_pid, &status, WNOHANG) == -1) {
@@ -231,41 +254,19 @@ child_handler(struct signal_event *ev)
 
 	clients_started = 1;
 	signal_del(ev);
-
-	while ((peer = LIST_FIRST(&inactive_peers)) != NULL) {
-		LIST_REMOVE(peer, peer_link);
-		assign_job(peer);
-		if (peer-> job == NULL)
-			break;
-	}
+	reassign_jobs();
 }
 
 void
 master_mode(const char *master_port, const char *start_script)
 {
-	struct sockaddr_in dst;
-	int fd;
-
 	LIST_INIT(&active_peers);
 	LIST_INIT(&inactive_peers);
 
 	event_init();
 
-	if (parse_sockaddr_in(master_port, &dst))
-		errx(1, "Could not parse addr/port");
-
-	fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
-	if (fd == -1)
-		err(1, "Could not create socket");	
-	if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1)
-		err(1, "Could not set close-on-exec flag");
-	if (bind(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
-		err(1, "Could not bind socket");
-	if (listen(fd, 5) == -1)
-		err(1, "Could not listen on socket");
-
-	event_add(&listen_event, fd, 0, 1, listen_handler, NULL);
-	listen_event_socket = fd;
+	if (listen_sockaddr(master_port, listen_handler))
+		errx(1, "Could not create listen socket for %s", master_port);
 
 	if (start_script) {
 		signal_add(&child_event, SIGCHLD, child_handler);
@@ -281,6 +282,4 @@ master_mode(const char *master_port, const char *start_script)
 	}
 
 	event_dispatch();
-
-	(void)close(fd);
 }

pkgtools/pbulk/files/pbulk/pscan/pbulk-scan.1

@@ -1,4 +1,4 @@
-.\" $NetBSD: pbulk-scan.1,v 1.1.1.1 2007/06/19 19:49:57 joerg Exp $
+.\" $NetBSD: pbulk-scan.1,v 1.2 2015/12/07 16:52:40 joerg Exp $
 .\"
 .\" Copyright (c) 2007 Thomas Klausner and Joerg Sonnenberger.
 .\" All rights reserved.
@@ -33,13 +33,13 @@
 .Sh SYNOPSIS
 .Nm
 .Op Fl v
-.Fl c Oo Ar ip: Oc Ns Ar port
+.Fl c Oo Ar host: Oc Ns Ar port
 .Fl M Ar make
 .Ar pkgsrc
 .Nm
 .Op Fl lv
 .Op Fl I Ar start_script
-.Op Fl m Oo Ar ip: Oc Ns Ar port
+.Op Fl m Oo Ar host: Oc Ns Ar port
 .Fl M Ar make
 .Ar pkgsrc output
 .Sh DESCRIPTION
@@ -48,11 +48,11 @@ extracts information for a pbulk bulk build from a pkgsrc tree.
 .Pp
 Supported options are:
 .Bl -tag -width 15n -offset indent
-.It Fl c Oo Ar ip: Oc Ns Ar port
+.It Fl c Oo Ar host: Oc Ns Ar port
 Connect to pbulk bulk build master process on
 .Ar port
 (or
-.Ar ip:port ) .
+.Ar host:port ) .
 .It Fl I Ar start_script
 Run
 .Ar start_script
@@ -72,14 +72,14 @@ Usually
 .Dq make
 or
 .Dq bmake .
-.It Fl m Oo Ar ip: Oc Ns Ar port
+.It Fl m Oo Ar host: Oc Ns Ar port
 Enter master mode.
 In this mode,
 .Nm
 waits for connections on
 .Ar port
 (or
-.Ar ip:port ) .
+.Ar host:port ) .
 .It Fl v
 Log each location to be scanned or other progress to stdout.
 .El

pkgtools/pbulk/files/pbulk/pscan/pscan.c

@@ -1,4 +1,4 @@
-/* $NetBSD: pscan.c,v 1.8 2012/11/23 12:13:35 joerg Exp $ */
+/* $NetBSD: pscan.c,v 1.10 2016/06/20 17:54:43 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -136,7 +136,7 @@ main(int argc, char **argv)
 			usage();
 		pkgsrc_tree = argv[0];
 
-		client_mode(client_port);
+		client_mode(client_port, bmake_path);
 	}
 
 	if (argc != 2)
@@ -145,15 +145,20 @@ main(int argc, char **argv)
 	pkgsrc_tree = argv[0];
 	output_file = argv[1];
 
-	if (limited_scan == 0)
-		find_full_tree();
-	else
+	if (limited_scan != 0)
 		read_limited_list();
 
-	if (master_port != NULL)
-		master_mode(master_port, start_script);
-	else
-		standalone_mode();
+	if (master_port != NULL) {
+		if (limited_scan == 0)
+			add_job_full("find_full_tree");
+		if (has_job())
+			master_mode(master_port, start_script);
+	} else {
+		if (limited_scan == 0)
+			find_full_tree();
+		if (has_job())
+			standalone_mode();
+	}
 
 	write_jobs(output_file);
 

pkgtools/pbulk/files/pbulk/pscan/pscan.h

@@ -1,4 +1,4 @@
-/* $NetBSD: pscan.h,v 1.3 2012/11/23 12:13:35 joerg Exp $ */
+/* $NetBSD: pscan.h,v 1.5 2016/06/20 17:54:43 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -52,7 +52,7 @@ extern const char *pkgsrc_tree;
 
 char		*scan_pkglocation(const char *);
 
-void		 client_mode(const char *);
+void		 client_mode(const char *, const char *);
 void		 master_mode(const char *, const char *);
 
 void		 add_job(const char *, size_t, const char *, size_t);
@@ -60,5 +60,6 @@ void		 add_job_full(const char *);
 struct scan_job	*get_job(void);
 void		 process_job(struct scan_job *, enum job_state);
 void		 write_jobs(const char *);
+int		 has_job(void);
 
-void		read_old_scan(const char *);
+void		 read_old_scan(const char *);

pkgtools/pbulk/files/pbulk/scripts/pkg-build

@@ -1,5 +1,5 @@
 #!@SH@
-# $NetBSD: pkg-build,v 1.27 2015/09/13 11:45:37 joerg Exp $
+# $NetBSD: pkg-build,v 1.30 2016/05/09 00:08:21 joerg Exp $
 #
 # Copyright (c) 2007, 2008 Joerg Sonnenberger <joerg@NetBSD.org>.
 # All rights reserved.
@@ -39,11 +39,8 @@ if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
 fi
 
 default_sync_package() {
+	mkdir -p ${packages}/All
 	cp "$1" "${packages}/All/${pkgname}${pkg_sufx}"
-	for cat in $3; do
-		mkdir -p ${packages}/$cat
-		ln -fs ../All/${pkgname}${pkg_sufx} ${packages}/$cat/${pkgname}${pkg_sufx}
-	done
 }
 
 cleanup() {
@@ -78,7 +75,6 @@ run_make() {
 }
 
 run_usergroup() {
-	[ "${use_destdir}" != no ] || return 0
 	case "${usergroup_phase}" in
 	*configure)
 		[ "$1" != "configure" ] || ${make} create-usergroup clean
@@ -125,20 +121,13 @@ if [ "$use_unprivileged_checksum" != "no" ]; then
 else
 	run_checksum=run_direct
 fi
-case "$use_destdir" in
-user-destdir)
+if [ "$use_destdir" = user-destdir ]; then
 	run_build=run_su
 	run_install=run_su
-	;;
-destdir)
+else
 	run_build=run_su
 	run_install=run_direct
-	;;
-*)
-	run_build=run_direct
-	run_install=run_direct
-	;;
-esac
+fi
 
 if [ `@ID@ -u` -ne 0 ]; then
 	run_checksum=run_direct
@@ -156,10 +145,8 @@ set -e
 mkdir -p ${bulklog}/${pkgname}
 rm -f ${bulklog}/${pkgname}/*
 
-if [ "$use_destdir" = "destdir" -o "$use_destdir" = "user-destdir" ]; then
-	touch ${bulklog}/${pkgname}/work.log
-	@CHOWN@ ${unprivileged_user} ${bulklog}/${pkgname}/work.log
-fi
+touch ${bulklog}/${pkgname}/work.log
+@CHOWN@ ${unprivileged_user} ${bulklog}/${pkgname}/work.log
 
 pkg_add_normal() {
 	PKG_PATH=${packages}/All ${pkg_add} -K ${cur_pkgdb} "$@"
@@ -206,11 +193,9 @@ run_make run_direct stage-package-create > ${bulklog}/${pkgname}/package.log 2>&
 
 pkgfile=$(run_make run_direct show-var VARNAME=STAGE_PKGFILE)
 
-# When using DESTDIR build, add the package once to test install rules.
-# This is not done for potential bootstrap packages as they might already
-# be installed.
-if [ "${use_destdir}" != "no" ] && \
-   [ -z "${is_bootstrap}" ]; then
+# Add the package once to test install rules.  This is not done for
+# potential bootstrap packages as they might already be installed.
+if [ -z "${is_bootstrap}" ]; then
 	if ! ${pkg_add_cmd} ${pkgfile} \
 	    >> ${bulklog}/${pkgname}/package.log 2>&1; then
 		cleanup