Merge pull request #313 from bergwolf/netlink

use netlink uevent to get device ready message

Author: gao-feng

src/Makefile.am

@@ -1,7 +1,7 @@
 COMMIT=`git describe --dirty --always --tags 2> /dev/null || true`
 AM_CFLAGS = -Wall -Werror -DVERSIONCOMMIT=\""$(VERSION), commit: $(COMMIT)"\"
 bin_PROGRAMS=hyperstart
-hyperstart_SOURCES=init.c jsmn.c net.c util.c parse.c parson.c container.c exec.c event.c portmapping.c
+hyperstart_SOURCES=init.c jsmn.c net.c util.c parse.c parson.c container.c exec.c event.c portmapping.c netlink.c
 if HAVE_VSOCK
 hyperstart_SOURCES+=vsock.c
 endif

src/container.c

@@ -20,6 +20,7 @@
 #include "hyper.h"
 #include "parse.h"
 #include "syscall.h"
+#include "netlink.h"
 
 static int container_populate_volume(char *src, char *dest)
 {
@@ -134,7 +135,7 @@ static int container_setup_volume(struct hyper_container *container)
 			if (!strcmp(vol->fstype, "xfs"))
 				options = "nouuid";
 
-			if (hyper_mount_blockdev(dev, path, vol->fstype, options) < 0) {
+			if (mount(dev, path, vol->fstype, 0, options) < 0) {
 				perror("mount volume device failed");
 				return -1;
 			}
@@ -528,6 +529,7 @@ struct hyper_container_arg {
 	struct hyper_pod	*pod;
 	int			mntns_referenced_efd;
 	int			container_inited_efd;
+	int			container_root_dev_efd;
 };
 
 static int hyper_setup_container_rootfs(void *data)
@@ -551,11 +553,6 @@ static int hyper_setup_container_rootfs(void *data)
 	/* To create files/directories accessible for all users. */
 	umask(0);
 
-	if (container->fstype && hyper_rescan_scsi() < 0) {
-		fprintf(stdout, "rescan scsi failed\n");
-		goto fail;
-	}
-
 	if (mount("", "/", NULL, MS_SLAVE|MS_REC, NULL) < 0) {
 		perror("mount SLAVE failed");
 		goto fail;
@@ -576,19 +573,24 @@ static int hyper_setup_container_rootfs(void *data)
 		char dev[128];
 		char *options = NULL;
 
+		/* wait for rootfs ready message */
+		if (hyper_eventfd_recv(arg->container_root_dev_efd) < 0) {
+			fprintf(stderr, "wait for /proc/self/ns/mnt opened failed\n");
+			goto fail;
+		}
+
 		if (container->scsiaddr) {
 			free(container->image);
 			container->image = NULL;
 			hyper_find_sd(container->scsiaddr, &container->image);
 		}
-
 		sprintf(dev, "/dev/%s", container->image);
 		fprintf(stdout, "device %s\n", dev);
 
 		if (!strncmp(container->fstype, "xfs", strlen("xfs")))
 			options = "nouuid";
 
-		if (hyper_mount_blockdev(dev, root, container->fstype, options) < 0) {
+		if (mount(dev, root, container->fstype, 0, options) < 0) {
 			perror("mount device failed");
 			goto fail;
 		}
@@ -717,6 +719,36 @@ static void hyper_cleanup_pty(struct hyper_container *c)
 		perror("clean up container pty failed");
 }
 
+int container_prepare_rootfs_dev(struct hyper_container *container, struct hyper_pod *pod)
+{
+	char dev[512];
+
+	if (container->fstype == NULL)
+		return 0;
+
+	if (hyper_rescan_scsi() < 0) {
+		fprintf(stderr, "failed to issue scsi rescan\n");
+		return -1;
+	}
+
+	if (container->scsiaddr) {
+		free(container->image);
+		container->image = NULL;
+		hyper_find_sd(container->scsiaddr, &container->image);
+	}
+
+	if (container->image) {
+		sprintf(dev, "/dev/%s", container->image);
+		if (access(dev, R_OK) == 0)
+			return 0;
+		sprintf(dev, "/block/%s", container->image);
+	} else {
+		sprintf(dev, "/0:0:%s/block/", container->scsiaddr);
+	}
+
+	return hyper_netlink_wait_dev(pod->ueventfd, dev);
+}
+
 int hyper_setup_container(struct hyper_container *container, struct hyper_pod *pod)
 {
 	int stacksize = getpagesize() * 42;
@@ -725,6 +757,7 @@ int hyper_setup_container(struct hyper_container *container, struct hyper_pod *p
 		.pod	= pod,
 		.mntns_referenced_efd	= -1,
 		.container_inited_efd 	= -1,
+		.container_root_dev_efd = -1,
 	};
 	int flags = CLONE_NEWNS | SIGCHLD;
 	char path[128];
@@ -735,7 +768,9 @@ int hyper_setup_container(struct hyper_container *container, struct hyper_pod *p
 
 	arg.mntns_referenced_efd = eventfd(0, EFD_CLOEXEC);
 	arg.container_inited_efd = eventfd(0, EFD_CLOEXEC);
-	if (arg.mntns_referenced_efd < 0 || arg.container_inited_efd < 0) {
+	arg.container_root_dev_efd = eventfd(0, EFD_CLOEXEC);
+	if (arg.mntns_referenced_efd < 0 || arg.container_inited_efd < 0 ||
+	    arg.container_root_dev_efd < 0) {
 		perror("create eventfd between pod init execcmd failed");
 		goto fail;
 	}
@@ -772,6 +807,13 @@ int hyper_setup_container(struct hyper_container *container, struct hyper_pod *p
 	fprintf(stdout, "hyper send mntns referenced event: normal\n");
 	hyper_eventfd_send(arg.mntns_referenced_efd, HYPER_EVENTFD_NORMAL);
 
+	if (container_prepare_rootfs_dev(container, pod) < 0) {
+		fprintf(stderr, "fail to prepare container rootfs dev\n");
+		goto fail;
+	}
+	fprintf(stdout, "hyper send root dev ready event: normal\n");
+	hyper_eventfd_send(arg.container_root_dev_efd, HYPER_EVENTFD_NORMAL);
+
 	/* wait for ready message */
 	if (hyper_eventfd_recv(arg.container_inited_efd) < 0) {
 		fprintf(stderr, "wait for setup container rootfs failed\n");
@@ -780,12 +822,14 @@ int hyper_setup_container(struct hyper_container *container, struct hyper_pod *p
 
 	close(arg.mntns_referenced_efd);
 	close(arg.container_inited_efd);
+	close(arg.container_root_dev_efd);
 	return 0;
 fail:
 	close(container->ns);
 	container->ns = -1;
 	close(arg.mntns_referenced_efd);
 	close(arg.container_inited_efd);
+	close(arg.container_root_dev_efd);
 	return -1;
 }
 

src/hyper.h

@@ -39,6 +39,7 @@ struct hyper_pod {
 	uint32_t		remains;
 	int			req_destroy;
 	int			efd;
+	int			ueventfd;
 };
 
 struct portmapping_white_list {
@@ -65,6 +66,7 @@ struct hyper_epoll {
 	struct hyper_event	tty;
 	struct hyper_event	vsock_ctl_listener;
 	struct hyper_event	vsock_msg_listener;
+	struct hyper_event	dev;
 };
 
 static inline int hyper_symlink(char *oldpath, char *newpath)

src/init.c

@@ -32,6 +32,7 @@
 #include "container.h"
 #include "syscall.h"
 #include "vsock.h"
+#include "netlink.h"
 
 static struct hyper_pod global_pod = {
 	.containers	=	LIST_HEAD_INIT(global_pod.containers),
@@ -1191,10 +1192,10 @@ static int hyper_ctlmsg_handle(struct hyper_event *he, uint32_t len)
 		hyper_cmd_online_cpu_mem();
 		break;
 	case SETUPINTERFACE:
-		ret = hyper_cmd_setup_interface((char *)buf->data + 8, len - 8);
+		ret = hyper_cmd_setup_interface((char *)buf->data + 8, len - 8, pod);
 		break;
 	case SETUPROUTE:
-		ret = hyper_cmd_setup_route((char *)buf->data + 8, len - 8);
+		ret = hyper_cmd_setup_route((char *)buf->data + 8, len - 8, pod);
 		break;
 	case SIGNALPROCESS:
 		ret = hyper_signal_process(pod, (char *)buf->data + 8, len - 8);
@@ -1443,6 +1444,11 @@ static int hyper_loop(void)
 		}
 	}
 
+	if (hyper_setup_netlink_listener(&hyper_epoll.dev) < 0 ||
+	    hyper_add_event(hyper_epoll.efd, &hyper_epoll.dev, EPOLLIN))
+		return -1;
+	pod->ueventfd = hyper_epoll.dev.fd;
+
 	events = calloc(MAXEVENTS, sizeof(*events));
 
 	while (1) {

src/net.c

@@ -13,6 +13,7 @@
 #include "hyper.h"
 #include "util.h"
 #include "parse.h"
+#include "netlink.h"
 #include "../config.h"
 
 void hyper_set_be32(uint8_t *buf, uint32_t val)
@@ -131,16 +132,19 @@ static int addattr_l(struct nlmsghdr *n, int maxlen, int type, void *data, int a
 	return 0;
 }
 
-static int hyper_get_ifindex(char *nic)
+static int hyper_get_ifindex(char *nic, struct hyper_pod *pod)
 {
 	int fd, ifindex = -1;
 	char path[512], buf[8];
 
 	sprintf(path, "/sys/class/net/%s/ifindex", nic);
 	fprintf(stdout, "net device sys path is %s\n", path);
 
-	while (access(path, R_OK) < 0) {
-		sched_yield();
+	if (access(path, R_OK) < 0) {
+		sprintf(path, "/net/%s/", nic);
+		if (hyper_netlink_wait_dev(pod->ueventfd, path) < 0)
+			return -1;
+		sprintf(path, "/sys/class/net/%s/ifindex", nic);
 	}
 
 	fd = open(path, O_RDONLY);
@@ -291,7 +295,8 @@ static int get_netmask(unsigned *val, const char *addr)
 }
 
 static int hyper_setup_route(struct rtnl_handle *rth,
-			   struct hyper_route *rt)
+			     struct hyper_route *rt,
+			     struct hyper_pod *pod)
 {
 	uint32_t data;
 	struct {
@@ -330,7 +335,7 @@ static int hyper_setup_route(struct rtnl_handle *rth,
 	}
 
 	if (rt->device) {
-		int ifindex = hyper_get_ifindex(rt->device);
+		int ifindex = hyper_get_ifindex(rt->device, pod);
 		if (ifindex < 0) {
 			fprintf(stderr, "failed to get the ifindix of %s\n", rt->device);
 			return -1;
@@ -444,7 +449,8 @@ static int hyper_set_interface_mtu(struct rtnl_handle *rth,
 }
 
 static int hyper_setup_interface(struct rtnl_handle *rth,
-			       struct hyper_interface *iface)
+			         struct hyper_interface *iface,
+				 struct hyper_pod *pod)
 {
 	uint8_t data[4];
 	unsigned mask;
@@ -467,7 +473,7 @@ static int hyper_setup_interface(struct rtnl_handle *rth,
 	req.n.nlmsg_type = RTM_NEWADDR;
 	req.ifa.ifa_family = AF_INET;
 
-	ifindex = hyper_get_ifindex(iface->device);
+	ifindex = hyper_get_ifindex(iface->device, pod);
 	if (ifindex < 0) {
 		fprintf(stderr, "failed to get the ifindix of %s\n", iface->device);
 		return -1;
@@ -557,7 +563,7 @@ int hyper_setup_network(struct hyper_pod *pod)
 	for (i = 0; i < pod->i_num; i++) {
 		iface = &pod->iface[i];
 
-		ret = hyper_setup_interface(&rth, iface);
+		ret = hyper_setup_interface(&rth, iface, pod);
 		if (ret < 0) {
 			fprintf(stderr, "link up device %s failed\n", iface->device);
 			goto out;
@@ -573,7 +579,7 @@ int hyper_setup_network(struct hyper_pod *pod)
 	for (i = 0; i < pod->r_num; i++) {
 		rt = &pod->rt[i];
 
-		ret = hyper_setup_route(&rth, rt);
+		ret = hyper_setup_route(&rth, rt, pod);
 		if (ret < 0) {
 			fprintf(stderr, "setup route failed\n");
 			goto out;
@@ -585,7 +591,7 @@ int hyper_setup_network(struct hyper_pod *pod)
 	return ret;
 }
 
-int hyper_cmd_setup_interface(char *json, int length)
+int hyper_cmd_setup_interface(char *json, int length, struct hyper_pod *pod)
 {
 	int ret = -1;
 	struct hyper_interface *iface;
@@ -603,7 +609,7 @@ int hyper_cmd_setup_interface(char *json, int length)
 		fprintf(stderr, "parse interface failed\n");
 		goto out;
 	}
-	ret = hyper_setup_interface(&rth, iface);
+	ret = hyper_setup_interface(&rth, iface, pod);
 	if (ret < 0) {
 		fprintf(stderr, "link up device %s failed\n", iface->device);
 		goto out1;
@@ -617,7 +623,8 @@ int hyper_cmd_setup_interface(char *json, int length)
 	return ret;
 }
 
-int hyper_cmd_setup_route(char *json, int length) {
+int hyper_cmd_setup_route(char *json, int length, struct hyper_pod *pod)
+{
 	struct hyper_route *rts = NULL;
 	int i, ret = -1;
 	uint32_t r_num;
@@ -632,7 +639,7 @@ int hyper_cmd_setup_route(char *json, int length) {
 	}
 
 	for (i = 0; i < r_num; i++) {
-		ret = hyper_setup_route(&rth, &rts[i]);
+		ret = hyper_setup_route(&rth, &rts[i], pod);
 		if (ret < 0) {
 			fprintf(stderr, "setup route failed\n");
 			goto out;

src/net.h

@@ -46,8 +46,8 @@ uint32_t hyper_get_be32(uint8_t *buf);
 void hyper_set_be64(uint8_t *buf, uint64_t val);
 uint64_t hyper_get_be64(uint8_t *buf);
 int hyper_setup_network(struct hyper_pod *pod);
-int hyper_cmd_setup_interface(char *json, int length);
-int hyper_cmd_setup_route(char *json, int length);
+int hyper_cmd_setup_interface(char *json, int length, struct hyper_pod *pod);
+int hyper_cmd_setup_route(char *json, int length, struct hyper_pod *pod);
 int hyper_setup_dns(struct hyper_pod *pod);
 int hyper_setup_hostname(struct hyper_pod *pod);
 int hyper_send_data_block(int fd, uint8_t *data, uint32_t len);