nbd: wait for udev to report nbd block device ready

This makes sure "casync mkdev" will not notify readiness until the NBD
block device it allocated has been probed by udev, and thus is
guaranteed to be fully initialized, probed and registered.

Replaces: #114

Author: poettering

meson.build

@@ -114,7 +114,6 @@ libzstd = dependency('libzstd',
 
 libacl = cc.find_library('acl')
 
-
 if get_option('fuse')
         libfuse = dependency('fuse',
                              version : '>= 2.6')
@@ -130,6 +129,13 @@ else
 endif
 conf.set10('HAVE_SELINUX', get_option('selinux'))
 
+if get_option('udev')
+        libudev = dependency('libudev')
+else
+        libudev = []
+endif
+conf.set10('HAVE_UDEV', get_option('udev'))
+
 threads = dependency('threads')
 math = cc.find_library('m')
 
@@ -159,6 +165,7 @@ casync = executable(
                 libfuse,
                 liblzma,
                 libselinux,
+                libudev,
                 libz,
                 libzstd,
                 math,

meson_options.txt

@@ -5,5 +5,7 @@ option('fuse', type : 'boolean', value : true,
        description : 'build the FUSE integration (requires fuse-devel)')
 option('selinux', type : 'boolean', value : true,
        description : 'build the SELinux backend (requires libselinux-devel)')
+option('udev', type : 'boolean', value : true,
+       description : 'build the libudev integration (requires libudev-devel)')
 option('man', type : 'boolean', value : true,
        description : 'build and install man pages (requires spinx-build')

src/canbd.c

@@ -35,6 +35,8 @@ struct CaBlockDevice {
         int friendly_name_fd;
         char *friendly_name;
         char *friendly_name_path;
+
+        dev_t devnum;
 };
 
 CaBlockDevice *ca_block_device_new(void) {
@@ -270,6 +272,7 @@ static int ca_block_device_establish_friendly_name(CaBlockDevice *d) {
 int ca_block_device_open(CaBlockDevice *d) {
         static const int one = 1;
         bool free_device_path = false;
+        struct stat st;
         int r;
 
         if (!d)
@@ -295,6 +298,16 @@ int ca_block_device_open(CaBlockDevice *d) {
                 if (d->device_fd < 0)
                         return -errno;
 
+                if (fstat(d->device_fd, &st) < 0) {
+                        r = -errno;
+                        goto fail;
+                }
+
+                if (!S_ISBLK(st.st_mode)) {
+                        r = -ENOTBLK;
+                        goto fail;
+                }
+
                 if (ioctl(d->device_fd, NBD_SET_SOCK, d->socket_fd[1]) < 0) {
                         r = -errno;
                         goto fail;
@@ -320,6 +333,16 @@ int ca_block_device_open(CaBlockDevice *d) {
                                 goto fail;
                         }
 
+                        if (fstat(d->device_fd, &st) < 0) {
+                                r = -errno;
+                                goto fail;
+                        }
+
+                        if (!S_ISBLK(st.st_mode)) {
+                                r = -ENOTBLK;
+                                goto fail;
+                        }
+
                         if (ioctl(d->device_fd, NBD_SET_SOCK, d->socket_fd[1]) >= 0) {
                                 d->device_path = path;
                                 free_device_path = true;
@@ -338,6 +361,8 @@ int ca_block_device_open(CaBlockDevice *d) {
                 }
         }
 
+        d->devnum = st.st_rdev;
+
         r = ca_block_device_establish_friendly_name(d);
         if (r < 0)
                 goto fail;
@@ -489,6 +514,17 @@ int ca_block_device_put_data(CaBlockDevice *d, uint64_t offset, const void *data
         return 0;
 }
 
+int ca_block_device_get_poll_fd(CaBlockDevice *d) {
+        if (!d)
+                return -EINVAL;
+        if (d->device_fd < 0)
+                return -EUNATCH;
+        if (d->socket_fd[0] < 0)
+                return -EUNATCH;
+
+        return d->socket_fd[0];
+}
+
 int ca_block_device_poll(CaBlockDevice *d, uint64_t timeout_nsec, const sigset_t *ss) {
         struct pollfd pollfd;
         int r;
@@ -556,6 +592,21 @@ int ca_block_device_set_path(CaBlockDevice *d, const char *node) {
         return 1;
 }
 
+
+int ca_block_device_get_devnum(CaBlockDevice *d, dev_t *ret) {
+        if (!d)
+                return -EINVAL;
+
+        if (d->device_fd < 0)
+                return -EUNATCH;
+
+        if (d->devnum == 0)
+                return -EUNATCH;
+
+        *ret = d->devnum;
+        return 0;
+}
+
 int ca_block_device_test_nbd(const char *name) {
         unsigned u;
         size_t n;

src/canbd.h

@@ -38,6 +38,9 @@ int ca_block_device_get_path(CaBlockDevice *d, const char **ret);
 
 int ca_block_device_set_friendly_name(CaBlockDevice *d, const char *name);
 
+int ca_block_device_get_devnum(CaBlockDevice *d, dev_t *ret);
+int ca_block_device_get_poll_fd(CaBlockDevice *d);
+
 int ca_block_device_test_nbd(const char *name);
 
 #endif

src/casync-tool.c

@@ -4,6 +4,7 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <getopt.h>
+#include <poll.h>
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -27,6 +28,11 @@
 #include "signal-handler.h"
 #include "util.h"
 
+#ifdef HAVE_UDEV
+#include <libudev.h>
+#include "udev-util.h"
+#endif
+
 static enum arg_what {
         WHAT_ARCHIVE,
         WHAT_ARCHIVE_INDEX,
@@ -2556,15 +2562,24 @@ static int verb_mkdev(int argc, char *argv[]) {
                 MKDEV_BLOB_INDEX,
                 _MKDEV_OPERATION_INVALID = -1,
         } MkDevOperation;
+
         MkDevOperation operation = _MKDEV_OPERATION_INVALID;
-        _cleanup_(realloc_buffer_free) ReallocBuffer buffer = {};
         _cleanup_(ca_block_device_unrefp) CaBlockDevice *nbd = NULL;
-        const char *path = NULL, *name = NULL;
-        bool make_symlink = false, rm_symlink = false;
-        int r;
+        _cleanup_(realloc_buffer_free) ReallocBuffer buffer = {};
         _cleanup_(safe_close_nonstdp) int input_fd = -1;
-        _cleanup_free_ char *input = NULL;
+        bool make_symlink = false, rm_symlink = false;
         _cleanup_(ca_sync_unrefp) CaSync *s = NULL;
+        const char *path = NULL, *name = NULL;
+        _cleanup_free_ char *input = NULL;
+        bool initialized, sent_ready = false;
+        dev_t devnum;
+        int r;
+
+#if HAVE_UDEV
+        _cleanup_(udev_monitor_unrefp) struct udev_monitor *monitor = NULL;
+        _cleanup_(udev_device_unrefp) struct udev_device *d = NULL;
+        _cleanup_(udev_unrefp) struct udev *udev = NULL;
+#endif
 
         if (argc > 3) {
                 log_error("An blob path/URL expected, possibly followed by a device or symlink name.");
@@ -2762,6 +2777,49 @@ static int verb_mkdev(int argc, char *argv[]) {
                 goto finish;
         }
 
+        r = ca_block_device_get_devnum(nbd, &devnum);
+        if (r < 0) {
+                log_error_errno(r, "Failed to get device ID: %m");
+                goto finish;
+        }
+
+#if HAVE_UDEV
+        udev = udev_new();
+        if (!udev) {
+                r = log_error_errno(errno, "Failed to allocate udev context: %m");
+                goto finish;
+        }
+
+        monitor = udev_monitor_new_from_netlink(udev, "udev");
+        if (!monitor) {
+                r = log_error_errno(errno, "Failed to acquire udev monitor: %m");
+                goto finish;
+        }
+
+        r = udev_monitor_filter_add_match_subsystem_devtype(monitor, "block", NULL);
+        if (r < 0) {
+                log_error_errno(r, "Failed to addd udev match: %m");
+                goto finish;
+        }
+
+        r = udev_monitor_enable_receiving(monitor);
+        if (r < 0) {
+                log_error_errno(r, "Failed to start udev monitor: %m");
+                goto finish;
+        }
+
+        d = udev_device_new_from_devnum(udev, 'b', devnum);
+        if (!d) {
+                r = log_error_errno(errno, "Failed to get NBD udev device: %m");
+                goto finish;
+        }
+
+        initialized = udev_device_get_is_initialized(d) != 0;
+        d = udev_device_unref(d);
+#else
+        initialized = true;
+#endif
+
         if (make_symlink) {
                 if (symlink(path, name) < 0) {
                         r = log_error_errno(errno, "Failed to create symlink %s → %s: %m", name, path);
@@ -2773,8 +2831,6 @@ static int verb_mkdev(int argc, char *argv[]) {
 
         printf("Attached: %s\n", name ?: path);
 
-        (void) send_notify("READY=1");
-
         for (;;) {
                 uint64_t req_offset = 0, req_size = 0;
 
@@ -2783,6 +2839,35 @@ static int verb_mkdev(int argc, char *argv[]) {
                         goto finish;
                 }
 
+#if HAVE_UDEV
+                if (!initialized) {
+                        _cleanup_(udev_device_unrefp) struct udev_device *t = NULL;
+
+                        t = udev_monitor_receive_device(monitor);
+                        if (t && udev_device_get_devnum(t) == devnum)
+                                initialized = true;
+                }
+#endif
+
+                if (initialized && !sent_ready) {
+                        _cleanup_free_ char *t = NULL;
+
+#if HAVE_UDEV
+                        monitor = udev_monitor_unref(monitor);
+                        udev = udev_unref(udev);
+#endif
+
+                        t = strjoin("READY=1\n"
+                                    "DEVICE=", path, "\n");
+                        if (!t) {
+                                r = log_oom();
+                                goto finish;
+                        }
+
+                        (void) send_notify(t);
+                        sent_ready = true;
+                }
+
                 r = ca_block_device_step(nbd);
                 if (r < 0) {
                         log_error_errno(r, "Failed to read NBD request: %m");
@@ -2794,15 +2879,43 @@ static int verb_mkdev(int argc, char *argv[]) {
 
                 if (r == CA_BLOCK_DEVICE_POLL) {
                         sigset_t ss;
+                        int nbd_poll_fd, udev_fd;
+
+                        nbd_poll_fd = ca_block_device_get_poll_fd(nbd);
+                        if (nbd_poll_fd < 0) {
+                                r = log_error_errno(nbd_poll_fd, "Failed to acquire NBD poll file descriptor: %m");
+                                goto finish;
+                        }
+
+
+#if HAVE_UDEV
+                        if (monitor) {
+                                udev_fd = udev_monitor_get_fd(monitor);
+                                if (udev_fd < 0) {
+                                        r = log_error_errno(udev_fd, "Failed to acquire udev monitor fd: %m");
+                                        goto finish;
+                                }
+                        } else
+#endif
+                                udev_fd = -1;
 
                         block_exit_handler(SIG_BLOCK, &ss);
 
                         if (quit)
                                 r = -ESHUTDOWN;
                         else {
-                                r = ca_block_device_poll(nbd, UINT64_MAX, &ss);
-                                if ((r == -EINTR || r >= 0) && quit)
+                                struct pollfd p[2] = {
+                                        [0] = { .fd = nbd_poll_fd, .events = POLLIN },
+                                        [1] = { .fd = udev_fd, .events = POLLIN },
+                                };
+
+                                r = ppoll(p, udev_fd < 0 ? 1 : 2, NULL, &ss);
+                                if ((r >= 0 || errno == EINTR) && quit)
                                         r = -ESHUTDOWN;
+                                else if (r < 0)
+                                        r = -errno;
+                                else
+                                        r = 0;
                         }
 
                         block_exit_handler(SIG_UNBLOCK, NULL);

src/udev-util.h

@@ -0,0 +1,14 @@
+#ifndef fooudevutilhfoo
+#define fooudevutilhfoo
+
+/* SPDX-License-Identifier: LGPL-2.1+ */
+
+#include "util.h"
+
+#include <libudev.h>
+
+DEFINE_TRIVIAL_CLEANUP_FUNC(struct udev*, udev_unref);
+DEFINE_TRIVIAL_CLEANUP_FUNC(struct udev_device*, udev_device_unref);
+DEFINE_TRIVIAL_CLEANUP_FUNC(struct udev_monitor*, udev_monitor_unref);
+
+#endif

test/semaphore-run

@@ -1,7 +1,7 @@
 #!/bin/sh
 # SPDX-License-Identifier: LGPL-2.1+
 
-DEVPKGS="pkg-config liblzma-dev libcurl4-openssl-dev libssl-dev libacl1-dev libfuse-dev zlib1g-dev libzstd-dev"
+DEVPKGS="pkg-config liblzma-dev libcurl4-openssl-dev libssl-dev libacl1-dev libfuse-dev zlib1g-dev libzstd-dev libudev-dev"
 
 echo
 echo "============= Installing amd64 build dependencies =============="