If we cannot write zeroes efficiently, consider the command length when allocating the buffer instead of allocating MAX_REQUEST_SIZE. Signed-off-by: Nir Soffer <nsoffer(a)redhat.com&gt; --- copy/multi-thread-copying.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/copy/multi-thread-copying.c b/copy/multi-thread-copying.c index f86621b..b1cc9a5 100644 --- a/copy/multi-thread-copying.c +++ b/copy/multi-thread-copying.c @@ -501,6 +501,7 @@ static void fill_dst_range_with_zeroes (struct command *command) { char *data; + size_t data_size; if (destination_is_zero) goto free_and_return; @@ -517,7 +518,8 @@ fill_dst_range_with_zeroes (struct command *command) /* Fall back to loop writing zeroes. This is going to be slow * anyway, so do it synchronously. XXX */ - data = calloc (1, MAX_REQUEST_SIZE); + data_size = MIN (MAX_REQUEST_SIZE, command->slice.len); + data = calloc (1, data_size); if (!data) { perror ("calloc"); exit (EXIT_FAILURE); @@ -525,8 +527,8 @@ fill_dst_range_with_zeroes (struct command *command) while (command->slice.len > 0) { size_t len = command->slice.len; - if (len > MAX_REQUEST_SIZE) - len = MAX_REQUEST_SIZE; + if (len > data_size) + len = data_size; dst->ops->synch_write (dst, data, len, command->offset); command->slice.len -= len; -- 2.26.2

Add example for copying an image between nbd servers using libev event loop. Currently supports only dumb copying without using extents or trying to detect zeroes. The main motivation for adding this example is testing the efficiency of the home-brew event loop in nbdcopy. Testing this example shows similar performance compared with qemu-img convert. nbdcopy performs worse, but tweaking the request size shows similar performance using more cpu time. I tested this only with nbdkit memory plugin, using: nbdkit -f -r pattern size=1G -U /tmp/src.sock nbdkit -f memory size=1g -U /tmp/dst.sock I used hyperfine to run all benchmarks using --warmup=3 and --run=10. Benchmark #1: ./copy-libev nbd+unix:///?socket=/tmp/src.sock \ nbd+unix:///?socket=/tmp/dst.sock   Time (mean ± σ):     552.9 ms ±  47.4 ms    [User: 76.4 ms, System: 456.3 ms]   Range (min … max):   533.8 ms … 687.6 ms    10 runs qemu-img shows same performance, using slightly less cpu time: Benchmark #2: qemu-img convert -n -W nbd+unix:///?socket=/tmp/src.sock \ nbd+unix:///?socket=/tmp/dst.sock   Time (mean ± σ):     554.6 ms ±  42.4 ms    [User: 69.1 ms, System: 456.6 ms]   Range (min … max):   535.5 ms … 674.9 ms    10 runs nbdcopy is 78% slower, and uses 290% more cpu time: Benchmark #3: .nbdcopy --flush nbd+unix:///?socket=/tmp/src.sock \ nbd+unix:///?socket=/tmp/dst.sock   Time (mean ± σ):     935.8 ms ±  37.8 ms    [User: 206.4 ms, System: 1340.8 ms]   Range (min … max):   890.5 ms … 1017.6 ms    10 runs Disabling extents and sparse does not make a difference, but changing the request size show similar performance: Benchmark #4: ./nbdcopy --flush --no-extents --sparse=0 --request-size=1048576 \ nbd+unix:///?socket=/tmp/src.sock nbd+unix:///?socket=/tmp/dst.sock   Time (mean ± σ):     594.5 ms ±  39.2 ms    [User: 250.0 ms, System: 1197.7 ms]   Range (min … max):   578.2 ms … 705.8 ms    10 runs Decreasing number of requests is little faster and use less cpu time, but nbdcopy is still 5% slower and uses 240% more cpu time. Benchmark #5: ./nbdcopy --flush --no-extents --sparse=0 --request-size=1048576 --requests=16 \ nbd+unix:///?socket=/tmp/src.sock nbd+unix:///?socket=/tmp/dst.sock   Time (mean ± σ):     583.0 ms ±  30.7 ms    [User: 243.9 ms, System: 1051.5 ms]   Range (min … max):   566.6 ms … 658.3 ms    10 runs Signed-off-by: Nir Soffer <nsoffer(a)redhat.com&gt; --- .gitignore | 1 + configure.ac | 19 +++ examples/Makefile.am | 22 +++ examples/copy-libev.c | 304 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 346 insertions(+) create mode 100644 examples/copy-libev.c diff --git a/.gitignore b/.gitignore index 4935b81..f4ce15b 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,7 @@ Makefile.in /examples/server-flags /examples/strict-structured-reads /examples/threaded-reads-and-writes +/examples/copy-libev /fuse/nbdfuse /fuse/nbdfuse.1 /fuzzing/libnbd-fuzz-wrapper diff --git a/configure.ac b/configure.ac index 6cf563a..6d9dbbe 100644 --- a/configure.ac +++ b/configure.ac @@ -223,6 +223,25 @@ PKG_CHECK_MODULES([GLIB], [glib-2.0], [ ]) AM_CONDITIONAL([HAVE_GLIB], [test "x$GLIB_LIBS" != "x"]) +dnl libev support for examples that interoperate with libev event loop. +PKG_CHECK_MODULES([LIBEV], [libev], [ + AC_SUBST([LIBEV_CFLAGS]) + AC_SUBST([LIBEV_LIBS]) +],[ + dnl no pkg-config for libev, searching manually: + AC_CHECK_HEADERS([ev.h], [ + AC_CHECK_LIB([ev], [ev_time], [ + AC_SUBST([LIBEV_LIBS], ["-lev"]) + ], + [ + AC_MSG_WARN([libev not found, some examples will not be compiled]) + ]) + ],[ + AC_MSG_WARN([ev.h not found, some examples will not be compiled]) + ]) +]) +AM_CONDITIONAL([HAVE_LIBEV], [test "x$LIBEV_LIBS" != "x"]) + dnl FUSE is optional to build the FUSE module. AC_ARG_ENABLE([fuse], AS_HELP_STRING([--disable-fuse], [disable FUSE (guestmount) support]), diff --git a/examples/Makefile.am b/examples/Makefile.am index b99cac1..a8286a3 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -39,6 +39,11 @@ noinst_PROGRAMS += \ glib-main-loop endif +if HAVE_LIBEV +noinst_PROGRAMS += \ + copy-libev +endif + aio_connect_read_SOURCES = \ aio-connect-read.c \ $(NULL) @@ -213,3 +218,20 @@ glib_main_loop_LDADD = \ $(GLIB_LIBS) \ $(NULL) endif + +if HAVE_LIBEV +copy_libev_SOURCES = \ + copy-libev.c \ + $(NULL) +copy_libev_CPPFLAGS = \ + -I$(top_srcdir)/include \ + $(NULL) +copy_libev_CFLAGS = \ + $(WARNINGS_CFLAGS) \ + $(LIBEV_CFLAGS) \ + $(NULL) +copy_libev_LDADD = \ + $(top_builddir)/lib/libnbd.la \ + $(LIBEV_LIBS) \ + $(NULL) +endif diff --git a/examples/copy-libev.c b/examples/copy-libev.c new file mode 100644 index 0000000..034711a --- /dev/null +++ b/examples/copy-libev.c @@ -0,0 +1,304 @@ +/* This example shows you how to make libnbd interoperate with the + * libev event loop. For more information about libvev see: + * + * http://pod.tst.eu/http://cvs.schmorp.de/libev/ev.pod + * + * To build it you need the libev-devel pacakge. + * + * To run it: + * + * nbdkit -r pattern size=1G -U /tmp/src.sock + * nbdkit memory size=1g -U /tmp/dst.sock + * ./copy-ev nbd+unix:///?socket=/tmp/src.sock nbd+unix:///?socket=/tmp/dst.sock + * + * To debug it: + * + * LIBNBD_DEBUG=1 ./copy-ev ... + */ + +#include <assert.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> + +#include <libnbd.h> + +#include <ev.h> + +/* These values depend on the enviroment tested. + * + * For shared storage using direct I/O: + * + * MAX_REQUESTS 16 + * REQUEST_SIZE (1024 * 1024) + * + * For nbdkit memory plugin: + * + * MAX_REQUESTS 8 + * REQUEST_SIZE (128 * 1024) + */ +#define MAX_REQUESTS 16 +#define REQUEST_SIZE (1024 * 1024) + +#define MIN(a,b) (a) < (b) ? (a) : (b) + +#define DEBUG(fmt, ...) \ + do { \ + if (debug) \ + fprintf (stderr, "copy-libev: " fmt "\n", ## __VA_ARGS__); \ + } while (0) + +struct connection { + ev_io watcher; + struct nbd_handle *nbd; +}; + +struct request { + int64_t offset; + size_t length; + unsigned char *data; +}; + +static struct ev_loop *loop; +static ev_prepare prepare; +static struct connection src; +static struct connection dst; +static struct request requests[MAX_REQUESTS]; +static int64_t size; +static int64_t offset; +static int64_t written; +static bool debug; + +static void start_read(struct request *r); +static int read_completed(void *user_data, int *error); +static int write_completed(void *user_data, int *error); + +static inline int +get_fd(struct connection *c) +{ + return nbd_aio_get_fd (c->nbd); +} + +static inline int +get_events(struct connection *c) +{ + int events = 0; + unsigned dir = nbd_aio_get_direction (c->nbd); + + if (dir & LIBNBD_AIO_DIRECTION_WRITE) + events |= EV_WRITE; + + if (dir & LIBNBD_AIO_DIRECTION_READ) + events |= EV_READ; + + return events; +} + +static void +start_read(struct request *r) +{ + int64_t cookie; + + assert (offset < size); + + r->length = MIN (REQUEST_SIZE, size - offset); + r->offset = offset; + + DEBUG ("start read offset=%ld len=%ld", r->offset, r->length); + + cookie = nbd_aio_pread ( + src.nbd, r->data, r->length, r->offset, + (nbd_completion_callback) { .callback=read_completed, + .user_data=r }, + 0); + if (cookie == -1) { + fprintf (stderr, "start_read: %s", nbd_get_error ()); + exit (EXIT_FAILURE); + } + + offset += r->length; +} + +static int +read_completed (void *user_data, int *error) +{ + struct request *r = (struct request *)user_data; + int64_t cookie; + + DEBUG ("read completed, starting write offset=%ld len=%ld", + r->offset, r->length); + + cookie = nbd_aio_pwrite ( + dst.nbd, r->data, r->length, r->offset, + (nbd_completion_callback) { .callback=write_completed, + .user_data=r }, + 0); + if (cookie == -1) { + fprintf (stderr, "read_completed: %s", nbd_get_error ()); + exit (EXIT_FAILURE); + } + + return 1; +} + +static int +write_completed (void *user_data, int *error) +{ + struct request *r = (struct request *)user_data; + + written += r->length; + + DEBUG ("write completed offset=%ld len=%ld", r->offset, r->length); + + if (written == size) { + /* The last write completed. Stop all watchers and break out + * from the event loop. + */ + ev_io_stop (loop, &src.watcher); + ev_io_stop (loop, &dst.watcher); + ev_prepare_stop (loop, &prepare); + ev_break (loop, EVBREAK_ALL); + } + + /* If we have data to read, start a new read. */ + if (offset < size) + start_read(r); + + return 1; +} + +/* Notify libnbd about io events. */ +static void +io_cb (struct ev_loop *loop, ev_io *w, int revents) +{ + struct connection *c = (struct connection *)w; + + if (revents & EV_WRITE) + nbd_aio_notify_write (c->nbd); + + if (revents & EV_READ) + nbd_aio_notify_read (c->nbd); +} + +static inline void +update_watcher (struct connection *c) +{ + int events = get_events(c); + + if (events != c->watcher.events) { + ev_io_stop (loop, &c->watcher); + ev_io_set (&c->watcher, get_fd (c), events); + ev_io_start (loop, &c->watcher); + } +} + +/* Update watchers events based on libnbd handle state. */ +static void +prepare_cb (struct ev_loop *loop, ev_prepare *w, int revents) +{ + update_watcher (&src); + update_watcher (&dst); +} + +int +main (int argc, char *argv[]) +{ + int i; + + loop = EV_DEFAULT; + + if (argc != 3) { + fprintf (stderr, "Usage: copy-ev src-uri dst-uri\n"); + exit (EXIT_FAILURE); + } + + src.nbd = nbd_create (); + if (src.nbd == NULL) { + fprintf (stderr, "nbd_create: %s\n", nbd_get_error ()); + exit (EXIT_FAILURE); + } + + + dst.nbd = nbd_create (); + if (dst.nbd == NULL) { + fprintf (stderr, "nbd_create: %s\n", nbd_get_error ()); + exit (EXIT_FAILURE); + } + + debug = nbd_get_debug (src.nbd); + + /* Connecting is fast, so use the syncronous API. */ + + if (nbd_connect_uri (src.nbd, argv[1])) { + fprintf (stderr, "nbd_connect_uri: %s\n", nbd_get_error ()); + exit (EXIT_FAILURE); + } + + if (nbd_connect_uri (dst.nbd, argv[2])) { + fprintf (stderr, "nbd_connect_uri: %s\n", nbd_get_error ()); + exit (EXIT_FAILURE); + } + + size = nbd_get_size (src.nbd); + + if (size > nbd_get_size (dst.nbd)) { + fprintf (stderr, "destinatio is not large enough\n"); + exit (EXIT_FAILURE); + } + + /* Start the copy "loop". When request completes, it starts the + * next request, until entire image was copied. */ + + for (i = 0; i < MAX_REQUESTS && offset < size; i++) { + struct request *r = &requests[i]; + + r->data = malloc (REQUEST_SIZE); + if (r->data == NULL) { + perror ("malloc"); + exit (EXIT_FAILURE); + } + + start_read(r); + } + + /* Start watching events on src and dst handles. */ + + ev_io_init (&src.watcher, io_cb, get_fd (&src), get_events (&src)); + ev_io_start (loop, &src.watcher); + + ev_io_init (&dst.watcher, io_cb, get_fd (&dst), get_events (&dst)); + ev_io_start (loop, &dst.watcher); + + /* Register a prepare watcher for updating src and dst events once + * before the event loop waits for new events. + */ + + ev_prepare_init (&prepare, prepare_cb); + ev_prepare_start (loop, &prepare); + + /* Run the event loop. The call will return when entire image was + * copied. + */ + + ev_run (loop, 0); + + /* Copy completed - flush data to storage. */ + + DEBUG("flush"); + if (nbd_flush (dst.nbd, 0)) { + fprintf (stderr, "Cannot flush: %s", nbd_get_error ()); + exit (EXIT_FAILURE); + } + + /* We don't care about errors here since data was flushed. */ + + nbd_shutdown (dst.nbd, 0); + nbd_close (dst.nbd); + + nbd_shutdown (src.nbd, 0); + nbd_close (src.nbd); + + /* We can free requests data here, but it is not really needed. */ + + return 0; +} -- 2.26.2