use netlink uevents to wait for block device

In normal case, netlink uevents are consumed by hyper_loop() event
handler. In the case of failing to find block devices, another epoll
is set up to check for specific device.

Signed-off-by: Peng Tao <bergwolf@gmail.com>

Author: bergwolf

src/Makefile.am

@@ -1,7 +1,7 @@
 COMMIT=`git describe --dirty --always --tags 2> /dev/null || true`
 AM_CFLAGS = -Wall -Werror -DVERSIONCOMMIT=\""$(VERSION), commit: $(COMMIT)"\"
 bin_PROGRAMS=hyperstart
-hyperstart_SOURCES=init.c jsmn.c net.c util.c parse.c parson.c container.c exec.c event.c portmapping.c
+hyperstart_SOURCES=init.c jsmn.c net.c util.c parse.c parson.c container.c exec.c event.c portmapping.c netlink.c
 if HAVE_VSOCK
 hyperstart_SOURCES+=vsock.c
 endif

src/container.c

@@ -20,6 +20,7 @@
 #include "hyper.h"
 #include "parse.h"
 #include "syscall.h"
+#include "netlink.h"
 
 static int container_populate_volume(char *src, char *dest)
 {
@@ -134,7 +135,7 @@ static int container_setup_volume(struct hyper_container *container)
 			if (!strcmp(vol->fstype, "xfs"))
 				options = "nouuid";
 
-			if (hyper_mount_blockdev(dev, path, vol->fstype, options) < 0) {
+			if (mount(dev, path, vol->fstype, 0, options) < 0) {
 				perror("mount volume device failed");
 				return -1;
 			}
@@ -528,6 +529,7 @@ struct hyper_container_arg {
 	struct hyper_pod	*pod;
 	int			mntns_referenced_efd;
 	int			container_inited_efd;
+	int			container_root_dev_efd;
 };
 
 static int hyper_setup_container_rootfs(void *data)
@@ -551,11 +553,6 @@ static int hyper_setup_container_rootfs(void *data)
 	/* To create files/directories accessible for all users. */
 	umask(0);
 
-	if (container->fstype && hyper_rescan_scsi() < 0) {
-		fprintf(stdout, "rescan scsi failed\n");
-		goto fail;
-	}
-
 	if (mount("", "/", NULL, MS_SLAVE|MS_REC, NULL) < 0) {
 		perror("mount SLAVE failed");
 		goto fail;
@@ -576,19 +573,24 @@ static int hyper_setup_container_rootfs(void *data)
 		char dev[128];
 		char *options = NULL;
 
+		/* wait for rootfs ready message */
+		if (hyper_eventfd_recv(arg->container_root_dev_efd) < 0) {
+			fprintf(stderr, "wait for /proc/self/ns/mnt opened failed\n");
+			goto fail;
+		}
+
 		if (container->scsiaddr) {
 			free(container->image);
 			container->image = NULL;
 			hyper_find_sd(container->scsiaddr, &container->image);
 		}
-
 		sprintf(dev, "/dev/%s", container->image);
 		fprintf(stdout, "device %s\n", dev);
 
 		if (!strncmp(container->fstype, "xfs", strlen("xfs")))
 			options = "nouuid";
 
-		if (hyper_mount_blockdev(dev, root, container->fstype, options) < 0) {
+		if (mount(dev, root, container->fstype, 0, options) < 0) {
 			perror("mount device failed");
 			goto fail;
 		}
@@ -717,6 +719,36 @@ static void hyper_cleanup_pty(struct hyper_container *c)
 		perror("clean up container pty failed");
 }
 
+int container_prepare_rootfs_dev(struct hyper_container *container, struct hyper_pod *pod)
+{
+	char dev[512];
+
+	if (container->fstype == NULL)
+		return 0;
+
+	if (hyper_rescan_scsi() < 0) {
+		fprintf(stderr, "failed to issue scsi rescan\n");
+		return -1;
+	}
+
+	if (container->scsiaddr) {
+		free(container->image);
+		container->image = NULL;
+		hyper_find_sd(container->scsiaddr, &container->image);
+	}
+
+	if (container->image) {
+		sprintf(dev, "/dev/%s", container->image);
+		if (access(dev, R_OK) == 0)
+			return 0;
+		sprintf(dev, "/block/%s", container->image);
+	} else {
+		sprintf(dev, "/0:0:%s/block/", container->scsiaddr);
+	}
+
+	return hyper_netlink_wait_dev(pod->ueventfd, dev);
+}
+
 int hyper_setup_container(struct hyper_container *container, struct hyper_pod *pod)
 {
 	int stacksize = getpagesize() * 42;
@@ -725,6 +757,7 @@ int hyper_setup_container(struct hyper_container *container, struct hyper_pod *p
 		.pod	= pod,
 		.mntns_referenced_efd	= -1,
 		.container_inited_efd 	= -1,
+		.container_root_dev_efd = -1,
 	};
 	int flags = CLONE_NEWNS | SIGCHLD;
 	char path[128];
@@ -735,7 +768,9 @@ int hyper_setup_container(struct hyper_container *container, struct hyper_pod *p
 
 	arg.mntns_referenced_efd = eventfd(0, EFD_CLOEXEC);
 	arg.container_inited_efd = eventfd(0, EFD_CLOEXEC);
-	if (arg.mntns_referenced_efd < 0 || arg.container_inited_efd < 0) {
+	arg.container_root_dev_efd = eventfd(0, EFD_CLOEXEC);
+	if (arg.mntns_referenced_efd < 0 || arg.container_inited_efd < 0 ||
+	    arg.container_root_dev_efd < 0) {
 		perror("create eventfd between pod init execcmd failed");
 		goto fail;
 	}
@@ -772,6 +807,13 @@ int hyper_setup_container(struct hyper_container *container, struct hyper_pod *p
 	fprintf(stdout, "hyper send mntns referenced event: normal\n");
 	hyper_eventfd_send(arg.mntns_referenced_efd, HYPER_EVENTFD_NORMAL);
 
+	if (container_prepare_rootfs_dev(container, pod) < 0) {
+		fprintf(stderr, "fail to prepare container rootfs dev\n");
+		goto fail;
+	}
+	fprintf(stdout, "hyper send root dev ready event: normal\n");
+	hyper_eventfd_send(arg.container_root_dev_efd, HYPER_EVENTFD_NORMAL);
+
 	/* wait for ready message */
 	if (hyper_eventfd_recv(arg.container_inited_efd) < 0) {
 		fprintf(stderr, "wait for setup container rootfs failed\n");
@@ -780,12 +822,14 @@ int hyper_setup_container(struct hyper_container *container, struct hyper_pod *p
 
 	close(arg.mntns_referenced_efd);
 	close(arg.container_inited_efd);
+	close(arg.container_root_dev_efd);
 	return 0;
 fail:
 	close(container->ns);
 	container->ns = -1;
 	close(arg.mntns_referenced_efd);
 	close(arg.container_inited_efd);
+	close(arg.container_root_dev_efd);
 	return -1;
 }
 

src/hyper.h

@@ -39,6 +39,7 @@ struct hyper_pod {
 	uint32_t		remains;
 	int			req_destroy;
 	int			efd;
+	int			ueventfd;
 };
 
 struct portmapping_white_list {
@@ -65,6 +66,7 @@ struct hyper_epoll {
 	struct hyper_event	tty;
 	struct hyper_event	vsock_ctl_listener;
 	struct hyper_event	vsock_msg_listener;
+	struct hyper_event	dev;
 };
 
 static inline int hyper_symlink(char *oldpath, char *newpath)

src/init.c

@@ -32,6 +32,7 @@
 #include "container.h"
 #include "syscall.h"
 #include "vsock.h"
+#include "netlink.h"
 
 static struct hyper_pod global_pod = {
 	.containers	=	LIST_HEAD_INIT(global_pod.containers),
@@ -1443,6 +1444,11 @@ static int hyper_loop(void)
 		}
 	}
 
+	if (hyper_setup_netlink_listener(&hyper_epoll.dev) < 0 ||
+	    hyper_add_event(hyper_epoll.efd, &hyper_epoll.dev, EPOLLIN))
+		return -1;
+	pod->ueventfd = hyper_epoll.dev.fd;
+
 	events = calloc(MAXEVENTS, sizeof(*events));
 
 	while (1) {

src/netlink.c

@@ -0,0 +1,104 @@
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <asm/types.h>
+#include <sys/socket.h>
+#include <linux/netlink.h>
+
+#include "event.h"
+
+static int hyper_netlink_expect_dev(int fd, const char *dev)
+{
+	int len;
+	char buf[1024] = {0};
+
+	do {
+		len = recv(fd, buf, sizeof(buf), 0);
+		if (len < 0)
+			break;
+		buf[len] = '\0';
+		if (dev && strncmp(buf, "add@", strlen("add@")) == 0) {
+			fprintf(stdout, "netlink add, expect %s, got %s\n", dev, buf);
+			if (strstr(buf, dev) != NULL)
+				return 1;
+		}
+	} while (len > 0);
+
+	return 0;
+}
+
+static int hyper_ctlfd_read(struct hyper_event *e, int efd, int events)
+{
+	return hyper_netlink_expect_dev(e->fd, NULL);
+}
+
+static struct hyper_event_ops hyper_devfd_ops = {
+	.read		= hyper_ctlfd_read,
+};
+
+int hyper_setup_netlink_listener(struct hyper_event *e)
+{
+	int fd;
+	struct sockaddr_nl sa;
+
+	memset(&sa, 0, sizeof(sa));
+	sa.nl_family = AF_NETLINK;
+	sa.nl_groups = 0xffffffff;
+	fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_KOBJECT_UEVENT);
+	if (fd < 0) {
+		perror("failed to create netlink socket");
+		return -1;
+	}
+	if (bind(fd, (struct sockaddr *) &sa, sizeof(sa)) < 0) {
+		perror("failed to bind netlink socket");
+		close(fd);
+		return -1;
+	}
+	e->fd = fd;
+
+	if (hyper_init_event(e, &hyper_devfd_ops, NULL) < 0) {
+		hyper_reset_event(e);
+		return -1;
+	}
+
+	return 0;
+}
+
+int hyper_netlink_wait_dev(int fd, const char *dev)
+{
+	struct epoll_event event = {
+		.events	= EPOLLIN,
+	};
+	int efd, n;
+
+	efd = epoll_create1(EPOLL_CLOEXEC);
+	if (efd < 0) {
+		perror("failed to create event poll fd");
+		return -1;
+	}
+
+	if (epoll_ctl(efd, EPOLL_CTL_ADD, fd, &event) < 0) {
+		perror("failed to add fd to epoll");
+		goto fail;
+	}
+
+	/* SIGCHLD is blocked by hyper_loop() */
+	while (1) {
+		n = epoll_wait(efd, &event, 1, 2000);
+		if (n < 0) {
+			perror("fail to wait netlink event");
+			goto fail;
+		} else if (n == 0) {
+			fprintf(stderr, "timeout waiting for device %s\n", dev);
+			goto fail;
+		}
+		if (hyper_netlink_expect_dev(fd, dev) > 0)
+			break;
+	}
+
+	close(efd);
+	return 0;
+fail:
+	close(efd);
+	return -1;
+}

src/netlink.h

@@ -0,0 +1,6 @@
+#ifndef _NETLINK_H_
+#define _NETLINK_H_
+
+int hyper_setup_netlink_listener(struct hyper_event *e);
+int hyper_netlink_wait_dev(int fd, const char *dev);
+#endif

src/util.c

@@ -102,22 +102,15 @@ int hyper_find_sd(char *addr, char **dev)
 	struct dirent **list;
 	struct dirent *dir;
 	char path[512];
-	int i, num, retry = 5;
+	int i, num;
 
 	sprintf(path, "/sys/class/scsi_disk/0:0:%s/device/block/", addr);
 	fprintf(stdout, "orig dev %s, scan path %s\n", *dev, path);
 
-	for (i = 0;; i++) {
-		num = scandir(path, &list, NULL, NULL);
-		if (num < 0) {
-			if (errno != ENOENT || i >= retry) {
-				perror("scan path failed");
-				return -1;
-			}
-			usleep(20000);
-			continue;
-		}
-		break;
+	num = scandir(path, &list, NULL, NULL);
+	if (num < 0) {
+		perror("scan path failed");
+		return -1;
 	}
 
 	for (i = 0; i < num; i++) {