[nbdkit PATCH v2] Introduce cacheextents filter
by Martin Kletzander
This filter caches the last result of the extents() call and offers a nice
speed-up for clients that only support req_on=1 in combination with plugins like
vddk, which has no overhead for returning information for multiple extents in
one call, but that call is very time-consuming.
Quick test showed that on a fast connection and a sparsely allocated 16G disk
with a OS installed `qemu-img map` runs 16s instead of 33s (out of which it
takes 5s to the first extents request). For 100G disk with no data on it, that
is one hole extent spanning the whole disk (where there is no space for
improvement) this does not add any noticeable overhead.
Signed-off-by: Martin Kletzander <mkletzan(a)redhat.com>
---
v2:
- Errors are set in _add and _fill functions
- Return values are checked for only -1
- Added a test
- Indentation fixed
- Fixed access before lock
configure.ac | 2 +
filters/cacheextents/Makefile.am | 64 ++++++
filters/cacheextents/cacheextents.c | 196 ++++++++++++++++++
.../nbdkit-cacheextents-filter.pod | 47 +++++
tests/Makefile.am | 5 +-
tests/test-cacheextents.sh | 56 +++++
6 files changed, 369 insertions(+), 1 deletion(-)
create mode 100644 filters/cacheextents/Makefile.am
create mode 100644 filters/cacheextents/cacheextents.c
create mode 100644 filters/cacheextents/nbdkit-cacheextents-filter.pod
create mode 100755 tests/test-cacheextents.sh
diff --git a/configure.ac b/configure.ac
index 58031f3c6d86..8db5a4cac3c5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -825,6 +825,7 @@ plugins="$(echo $(printf %s\\n $lang_plugins $non_lang_plugins | sort -u))"
filters="\
blocksize \
cache \
+ cacheextents \
cow \
delay \
error \
@@ -900,6 +901,7 @@ AC_CONFIG_FILES([Makefile
filters/Makefile
filters/blocksize/Makefile
filters/cache/Makefile
+ filters/cacheextents/Makefile
filters/cow/Makefile
filters/delay/Makefile
filters/error/Makefile
diff --git a/filters/cacheextents/Makefile.am b/filters/cacheextents/Makefile.am
new file mode 100644
index 000000000000..9e5b59c09c59
--- /dev/null
+++ b/filters/cacheextents/Makefile.am
@@ -0,0 +1,64 @@
+# nbdkit
+# Copyright (C) 2019 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+include $(top_srcdir)/common-rules.mk
+
+EXTRA_DIST = nbdkit-cacheextents-filter.pod
+
+filter_LTLIBRARIES = nbdkit-cacheextents-filter.la
+
+nbdkit_cacheextents_filter_la_SOURCES = \
+ cacheextents.c \
+ $(top_srcdir)/include/nbdkit-filter.h
+
+nbdkit_cacheextents_filter_la_CPPFLAGS = \
+ -I$(top_srcdir)/include \
+ -I$(top_srcdir)/common/include \
+ -I$(top_srcdir)/common/utils
+nbdkit_cacheextents_filter_la_CFLAGS = \
+ $(WARNINGS_CFLAGS)
+nbdkit_cacheextents_filter_la_LDFLAGS = \
+ -module -avoid-version -shared \
+ -Wl,--version-script=$(top_srcdir)/filters/filters.syms
+nbdkit_cacheextents_filter_la_LIBADD = \
+ $(top_builddir)/common/utils/libutils.la
+
+if HAVE_POD
+
+man_MANS = nbdkit-cacheextents-filter.1
+CLEANFILES += $(man_MANS)
+
+nbdkit-cacheextents-filter.1: nbdkit-cacheextents-filter.pod
+ $(PODWRAPPER) --section=1 --man $@ \
+ --html $(top_builddir)/html/$@.html \
+ $<
+
+endif HAVE_POD
diff --git a/filters/cacheextents/cacheextents.c b/filters/cacheextents/cacheextents.c
new file mode 100644
index 000000000000..01c0328aa424
--- /dev/null
+++ b/filters/cacheextents/cacheextents.c
@@ -0,0 +1,196 @@
+/* nbdkit
+ * Copyright (C) 2019 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <inttypes.h>
+
+#include <pthread.h>
+
+#include <nbdkit-filter.h>
+
+#include "cleanup.h"
+
+#define THREAD_MODEL NBDKIT_THREAD_MODEL_PARALLEL
+
+/* This lock protects the global state. */
+static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
+
+/* The real size of the underlying plugin. */
+static uint64_t size;
+
+/* Cached extents from the last extents () call and its start and end for the
+ sake of simplicity. */
+struct nbdkit_extents *cache_extents;
+static uint64_t cache_start;
+static uint64_t cache_end;
+
+static void
+cacheextents_unload (void)
+{
+ nbdkit_extents_free (cache_extents);
+}
+
+static int
+cacheextents_add (struct nbdkit_extents *extents, int *err)
+{
+ size_t i = 0;
+
+ for (i = 0; i < nbdkit_extents_count (cache_extents); i++) {
+ struct nbdkit_extent ex = nbdkit_get_extent (cache_extents, i);
+ if (nbdkit_add_extent (extents, ex.offset, ex.length, ex.type) == -1) {
+ *err = errno;
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+static int
+cacheextents_fill (struct nbdkit_extents *extents, int *err)
+{
+ size_t i = 0;
+ size_t count = nbdkit_extents_count (extents);
+ struct nbdkit_extent first = nbdkit_get_extent (extents, 0);
+ struct nbdkit_extent last = nbdkit_get_extent (extents, count - 1);
+
+ nbdkit_extents_free (cache_extents);
+ cache_start = first.offset;
+ cache_end = last.offset + last.length;
+ cache_extents = nbdkit_extents_new (cache_start, cache_end);
+
+ for (i = 0; i < count; i++) {
+ struct nbdkit_extent ex = nbdkit_get_extent (extents, i);
+ nbdkit_debug ("cacheextents: updating cache with"
+ ": offset=%" PRIu64
+ ": length=%" PRIu64
+ "; type=%x",
+ ex.offset, ex.length, ex.type);
+ if (nbdkit_add_extent (cache_extents, ex.offset, ex.length, ex.type) == -1) {
+ *err = errno;
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+static int
+cacheextents_extents (struct nbdkit_next_ops *next_ops, void *nxdata,
+ void *handle, uint32_t count, uint64_t offset, uint32_t flags,
+ struct nbdkit_extents *extents,
+ int *err)
+{
+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
+
+ nbdkit_debug ("cacheextents:"
+ " cache_start=%" PRIu64
+ " cache_end=%" PRIu64
+ " cache_extents=%p",
+ cache_start, cache_end, cache_extents);
+
+ if (cache_extents &&
+ offset >= cache_start && offset < cache_end) {
+ nbdkit_debug ("cacheextents: returning from cache");
+ return cacheextents_add (extents, err);
+ }
+
+ nbdkit_debug ("cacheextents: cache miss");
+ int r = next_ops->extents (nxdata, count, offset, flags, extents, err);
+ if (r == -1)
+ return r;
+
+ return cacheextents_fill (extents, err);
+}
+
+/* Any changes to the data need to clean the cache.
+ *
+ * Similarly to readahead filter this could be more intelligent, but there would
+ * be very little benefit.
+ */
+
+static void
+kill_cacheextents (void)
+{
+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
+ nbdkit_extents_free (cache_extents);
+ cache_extents = NULL;
+}
+
+static int
+cacheextents_pwrite (struct nbdkit_next_ops *next_ops, void *nxdata,
+ void *handle,
+ const void *buf, uint32_t count, uint64_t offset,
+ uint32_t flags, int *err)
+{
+ kill_cacheextents ();
+ return next_ops->pwrite (nxdata, buf, count, offset, flags, err);
+}
+
+static int
+cacheextents_trim (struct nbdkit_next_ops *next_ops, void *nxdata,
+ void *handle,
+ uint32_t count, uint64_t offset, uint32_t flags,
+ int *err)
+{
+ kill_cacheextents ();
+ return next_ops->trim (nxdata, count, offset, flags, err);
+}
+
+static int
+cacheextents_zero (struct nbdkit_next_ops *next_ops, void *nxdata,
+ void *handle,
+ uint32_t count, uint64_t offset, uint32_t flags,
+ int *err)
+{
+ kill_cacheextents ();
+ return next_ops->zero (nxdata, count, offset, flags, err);
+}
+
+static struct nbdkit_filter filter = {
+ .name = "cacheextents",
+ .longname = "nbdkit cacheextents filter",
+ .version = PACKAGE_VERSION,
+ .unload = cacheextents_unload,
+ .pwrite = cacheextents_pwrite,
+ .trim = cacheextents_trim,
+ .zero = cacheextents_zero,
+ .extents = cacheextents_extents,
+};
+
+NBDKIT_REGISTER_FILTER (filter)
diff --git a/filters/cacheextents/nbdkit-cacheextents-filter.pod b/filters/cacheextents/nbdkit-cacheextents-filter.pod
new file mode 100644
index 000000000000..88f3173b9375
--- /dev/null
+++ b/filters/cacheextents/nbdkit-cacheextents-filter.pod
@@ -0,0 +1,47 @@
+=head1 NAME
+
+nbdkit-cacheextents-filter - cache extents
+
+=head1 SYNOPSIS
+
+ nbdkit --filter=cacheextents plugin
+
+=head1 DESCRIPTION
+
+C<nbdkit-cacheextents-filter> is a filter that caches the result of last
+extents() call.
+
+A common use for this filter is to accelerate returning extents data for
+clients which ask for extents with the REQ_ONE flag set (like
+S<C<qemu-img convert>>) and is especially useful in combination with
+plugins that report multiple extents in one call, but with high latency
+for each of those calls (like L<nbdkit-vddk-plugin(1)>). For example:
+
+ nbdkit -U - --filter=cacheextents --run 'qemu-img map $nbd' vddk ...
+
+For files with big extents (when it is unlikely for one extents() call
+to return multiple different extents) this does not slow down the
+access.
+
+=head1 PARAMETERS
+
+There are no parameters specific to nbdkit-cacheextents-filter. Any
+parameters are passed through to and processed by the underlying
+plugin in the normal way.
+
+=head1 SEE ALSO
+
+L<nbdkit(1)>,
+L<nbdkit-cache-filter(1)>,
+L<nbdkit-readahead-filter(1)>,
+L<nbdkit-vddk-plugin(1)>,
+L<nbdkit-filter(3)>,
+L<qemu-img(1)>.
+
+=head1 AUTHORS
+
+Martin Kletzander
+
+=head1 COPYRIGHT
+
+Copyright (C) 2019 Red Hat Inc.
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 4148793c7297..a2bbaeac5d45 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -47,6 +47,7 @@ EXTRA_DIST = \
test-cache.sh \
test-cache-max-size.sh \
test-cache-on-read.sh \
+ test-cacheextents.sh \
test-captive.sh \
test-cow.sh \
test-cxx.sh \
@@ -365,7 +366,9 @@ test_oldstyle_LDADD = libtest.la $(LIBGUESTFS_LIBS)
endif HAVE_LIBGUESTFS
# Test export flags.
-TESTS += test-eflags.sh
+TESTS += \
+ test-cacheextents.sh \
+ test-eflags.sh
# common disk image shared with several tests
if HAVE_GUESTFISH
diff --git a/tests/test-cacheextents.sh b/tests/test-cacheextents.sh
new file mode 100755
index 000000000000..fd0bd1c1bc48
--- /dev/null
+++ b/tests/test-cacheextents.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2019 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+source ./functions.sh
+set -x
+set -e
+
+requires diff --version
+requires grep --version
+requires qemu-img --version
+
+nbdkit \
+ -U - \
+ --filter=cacheextents \
+ --filter=log \
+ --run 'qemu-img map --output=json $nbd' \
+ data data="0 @0x100000 1" size=2M \
+ logfile=test-cacheextents-access.log >test-cacheextents-actual.log
+
+
+nbdkit \
+ -U - \
+ --run 'qemu-img map --output=json $nbd' \
+ data data="0 @0x100000 1" size=2M >test-cacheextents-expected.log
+
+diff -q test-cacheextents-actual.log test-cacheextents-expected.log
+test $(grep -c "connection=1 Extents" test-cacheextents-access.log) -eq 1
--
2.21.0
5 years
[nbdkit PATCH 0/2] More on .thread_model
by Eric Blake
Rich pointed out that making thread_model dynamic even for plugins
makes some sense, so here is the code for doing it for 'sh'.
I'm less confident on how to do it for OCaml and Rust (not to mention
that those allow the plugin to already compile in their model, rather
than the language binding glue forcing a model). The other languages
(lua, perl, python, ruby) still need to be adjusted to use the v2 API,
although they could support this one even without implementing v2.
Eric Blake (2):
plugins: Add .thread_model callback
sh: Implement .thread_model callback
docs/nbdkit-plugin.pod | 33 ++++++++++++++++++---
plugins/sh/nbdkit-sh-plugin.pod | 15 ++++++++++
include/nbdkit-plugin.h | 2 ++
plugins/sh/sh.c | 51 +++++++++++++++++++++++++++++++--
server/plugins.c | 12 +++++++-
tests/test-eflags.sh | 49 ++++++++++++++++++++-----------
6 files changed, 139 insertions(+), 23 deletions(-)
--
2.20.1
5 years
[PATCH v5 0/7] v2v: switch to ocaml-libvirt
by Pino Toscano
Hi,
this series switches virt-2v to ocaml-libvirt, embedding the latest
version of it from git. This way, it is possible to improve the way
v2v connects to libvirt for both input, and output modules, and
interacts with libvirt (e.g. no more virsh calls needed in virt-v2v).
As side effect, virt-v2v now requires libvirt, as keeping it optional
would create too much burden.
I could not test all the libvirt input modes (like VDDK, and Xen), but
VMware and libvirtxml work fine as before.
Changes from v4:
- rebase on master
- improve libvirt.README following Rich's recommendation
- rename lazy_conn to libvirt_comm
Changes from v3:
- rebase on master
- update libvirt-ocaml from its master branch
- use $(srcdir) in common/mllibvirt/Makefile.am
- parse_libvirt_xml: open a libvirt connection only when needed (and
only once)
Changes from v2:
- rebase on master
Changes from v1:
- rebase on master
- update ocaml-libvirt from libvirt-ocaml.git on libvirt.org, and adjust
the code to it
- pass again the URI to input_libvirt_vddk, so an error message is
preserved
Pino Toscano (7):
v2v: require libvirt
common: Bundle the libvirt-ocaml library for use by virt-v2v
v2v: switch to ocaml-libvirt
v2v: -o libvirt: use a Lazy for the connection
v2v: -o libvirt: switch away from virsh
v2v: test-harness: stop using the external ocaml-libvirt
build: stop looking for ocaml-libvirt
.gitignore | 2 +
Makefile.am | 5 +-
common/mllibvirt/Makefile.am | 102 ++
common/mllibvirt/generator.pl | 908 +++++++++++++
common/mllibvirt/libvirt.README | 12 +
common/mllibvirt/libvirt.ml | 1673 ++++++++++++++++++++++++
common/mllibvirt/libvirt.mli | 1647 ++++++++++++++++++++++++
common/mllibvirt/libvirt_c_epilogue.c | 462 +++++++
common/mllibvirt/libvirt_c_oneoffs.c | 1714 +++++++++++++++++++++++++
common/mllibvirt/libvirt_c_prologue.c | 134 ++
configure.ac | 1 +
docs/C_SOURCE_FILES | 1 -
m4/guestfs-ocaml.m4 | 4 -
po/POTFILES | 1 -
v2v/Makefile.am | 26 +-
v2v/copy_to_local.ml | 7 +-
v2v/input_libvirt.ml | 22 +-
v2v/input_libvirt_other.ml | 27 +-
v2v/input_libvirt_other.mli | 5 +-
v2v/input_libvirt_vcenter_https.ml | 13 +-
v2v/input_libvirt_vcenter_https.mli | 2 +-
v2v/input_libvirt_vddk.ml | 15 +-
v2v/input_libvirt_vddk.mli | 4 +-
v2v/input_libvirt_xen_ssh.ml | 13 +-
v2v/input_libvirt_xen_ssh.mli | 2 +-
v2v/libvirt_utils-c.c | 539 --------
v2v/libvirt_utils.ml | 95 +-
v2v/libvirt_utils.mli | 51 +-
v2v/output_libvirt.ml | 56 +-
v2v/parse_libvirt_xml.ml | 23 +-
v2v/parse_libvirt_xml.mli | 11 +-
v2v/test-harness/Makefile.am | 5 +-
v2v/v2v.ml | 12 +-
33 files changed, 6897 insertions(+), 697 deletions(-)
create mode 100644 common/mllibvirt/Makefile.am
create mode 100755 common/mllibvirt/generator.pl
create mode 100644 common/mllibvirt/libvirt.README
create mode 100644 common/mllibvirt/libvirt.ml
create mode 100644 common/mllibvirt/libvirt.mli
create mode 100644 common/mllibvirt/libvirt_c_epilogue.c
create mode 100644 common/mllibvirt/libvirt_c_oneoffs.c
create mode 100644 common/mllibvirt/libvirt_c_prologue.c
delete mode 100644 v2v/libvirt_utils-c.c
--
2.21.0
5 years
[libnbd PATCH 0/4] Various interop fixes
by Eric Blake
Some of these affect attempts to connect to older qemu-nbd versions,
some of them were triggered by manual edits to qemu-nbd source code to
provoke various other compliant (if uncommon) server behaviors.
Eric Blake (4):
starttls: Skip error payload if falling back to unencrypted
states: Reject payload to NBD_REP_ACK
meta-context: Skip error payload if server lacks meta_context
states: Add NBD_OPT_EXPORT_NAME handling
generator/Makefile.am | 1 +
generator/generator | 53 ++++++++++++++
generator/states-newstyle-opt-export-name.c | 73 +++++++++++++++++++
generator/states-newstyle-opt-go.c | 13 +++-
.../states-newstyle-opt-set-meta-context.c | 26 +++++--
generator/states-newstyle-opt-starttls.c | 27 ++++++-
.../states-newstyle-opt-structured-reply.c | 6 ++
generator/states-newstyle.c | 11 ++-
lib/internal.h | 1 +
lib/nbd-protocol.h | 7 ++
10 files changed, 200 insertions(+), 18 deletions(-)
create mode 100644 generator/states-newstyle-opt-export-name.c
--
2.20.1
5 years
[nbdkit PATCH] truncate: Detect large image overflow with round-up
by Eric Blake
It is possible for the round-up parameter to cause our desired size to
exceed 2**63-1. But this error message is cryptic:
$ ./nbdkit -f --filter=truncate pattern $(((1<<63)-1)) round-up=1m &
$ qemu-nbd --list
nbdkit: pattern[1]: error: .get_size function returned invalid value (-9223372036854775808)
qemu-nbd: Failed to read initial magic: Unexpected end-of-file before all bytes were read
Better is the result of this patch:
nbdkit: pattern[1]: error: cannot round size 9223372036854775807 up to next boundary of 1048576
Alas, we are still limited in the fact that we can only detect the
problem on a per-connection basis (at .prepare, rather than at .load)
(because we support plugins that have a different size per
connection); and that nbdkit in general is still rather rude to
clients when .prepare prevents the use of a connection (perhaps nbdkit
should instead be taught to proceed with the socekt, but respond to
NBD_OPT_LIST with 0 items and to NBD_OPT_GO with NBD_REP_ERR_UNKNOWN
or NBD_REP_ERR_SHUTDOWN to give the client a hint that this server has
nothing further to offer).
Signed-off-by: Eric Blake <eblake(a)redhat.com>
---
I pushed this one, but am posting to the list because of the
conversation points it brings up about what to do when .prepare fails
(there's a difference between refusing to start nbdkit because .config
failed, vs. starting but being useless to a client because
.open/.prepare failed).
filters/truncate/truncate.c | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/filters/truncate/truncate.c b/filters/truncate/truncate.c
index 6408c35..bed5a03 100644
--- a/filters/truncate/truncate.c
+++ b/filters/truncate/truncate.c
@@ -38,6 +38,7 @@
#include <string.h>
#include <limits.h>
#include <errno.h>
+#include <inttypes.h>
#include <nbdkit-filter.h>
@@ -172,8 +173,14 @@ truncate_prepare (struct nbdkit_next_ops *next_ops, void *nxdata,
*/
if (truncate_size >= 0)
h->size = truncate_size;
- if (round_up > 0)
+ if (round_up > 0) {
+ if (ROUND_UP (h->size, round_up) > INT64_MAX) {
+ nbdkit_error ("cannot round size %" PRId64 " up to next boundary of %u",
+ h->size, round_up);
+ return -1;
+ }
h->size = ROUND_UP (h->size, round_up);
+ }
if (round_down > 0)
h->size = ROUND_DOWN (h->size, round_down);
--
2.20.1
5 years
[nbdkit PATCH 0/3] Add noparallel filter
by Eric Blake
Being able to programmatically force nbdkit to be less parallel can be
useful during testing. I was less sure about patch 3, but if you like
it, I'm inclined to instead squash it into patch 1. This patch is
written to apply after my NBD_CMD_CACHE work (since I touched the
nocache filter); but can be rearranged if we think this series should
go in first while that one undergoes any adjustments from review.
Eric Blake (3):
server: Allow filters to reduce thread model dynamically
noparallel: Implement new filter
filters: Use only .thread_model, not THREAD_MODEL
docs/nbdkit-filter.pod | 45 ++++++----
filters/fua/nbdkit-fua-filter.pod | 1 +
filters/nocache/nbdkit-nocache-filter.pod | 1 +
filters/noextents/nbdkit-noextents-filter.pod | 1 +
.../noparallel/nbdkit-noparallel-filter.pod | 66 ++++++++++++++
filters/nozero/nbdkit-nozero-filter.pod | 1 +
configure.ac | 2 +
include/nbdkit-filter.h | 7 +-
filters/cow/cow.c | 2 -
filters/delay/delay.c | 2 -
filters/error/error.c | 2 -
filters/fua/fua.c | 2 -
filters/log/log.c | 2 -
filters/nocache/nocache.c | 2 -
filters/noextents/noextents.c | 2 -
filters/noparallel/noparallel.c | 85 +++++++++++++++++++
filters/nozero/nozero.c | 2 -
filters/offset/offset.c | 2 -
filters/partition/partition.c | 2 -
filters/rate/rate.c | 2 -
filters/readahead/readahead.c | 2 -
filters/stats/stats.c | 2 -
filters/truncate/truncate.c | 2 -
filters/xz/xz.c | 6 +-
server/filters.c | 8 +-
server/main.c | 10 ++-
filters/noparallel/Makefile.am | 61 +++++++++++++
tests/test-eflags.sh | 16 ++++
tests/test-parallel-file.sh | 13 ++-
29 files changed, 296 insertions(+), 55 deletions(-)
create mode 100644 filters/noparallel/nbdkit-noparallel-filter.pod
create mode 100644 filters/noparallel/noparallel.c
create mode 100644 filters/noparallel/Makefile.am
--
2.20.1
5 years
[nbdkit PATCH v2 00/24] implement NBD_CMD_CACHE
by Eric Blake
Since v1:
- rework .can_cache to be tri-state, with default of no advertisement
(ripple effect through other patches)
- add a lot more patches in order to round out filter support
And in the meantime, Rich pushed NBD_CMD_CACHE support into libnbd, so
in theory we now have a way to test cache commands through the entire
stack.
Eric Blake (24):
server: Internal hooks for implementing NBD_CMD_CACHE
plugins: Add .cache callback
file, split: Implement .cache with posix_fadvise
plugins: Implement .pread emulation cache
plugins: Implement no-op .cache for in-memory plugins
nbd: Implement NBD_CMD_CACHE passthrough
sh: Implement .cache script callback
ocaml: Implement .cache script callback
plugins: Document lack of .cache in language bindings
filters: Add .cache callback
test-layers: Test .cache usage
test-eflags: Test .can_cache support
blocksize: Implement .cache rounding
cache: Implement .cache
cow: Implement .cache
delay: Implement .cache
error: Implement .cache
log: Implement .cache
offset, partition: Implement .cache
readahead, xz: Implement .can_cache via emulation
stats: Implement .cache
truncate: Implement .cache
filters: Pass through .can_cache for remaining filters
nocache: Implement new filter
docs/nbdkit-filter.pod | 42 +++++--
docs/nbdkit-plugin.pod | 82 +++++++++++++
docs/nbdkit-protocol.pod | 8 ++
filters/cache/nbdkit-cache-filter.pod | 6 +-
filters/cow/nbdkit-cow-filter.pod | 15 ++-
filters/delay/nbdkit-delay-filter.pod | 9 +-
filters/error/nbdkit-error-filter.pod | 8 +-
filters/fua/nbdkit-fua-filter.pod | 5 +-
filters/log/nbdkit-log-filter.pod | 10 +-
filters/nocache/nbdkit-nocache-filter.pod | 69 +++++++++++
filters/noextents/nbdkit-noextents-filter.pod | 2 +
filters/nozero/nbdkit-nozero-filter.pod | 2 +
plugins/lua/nbdkit-lua-plugin.pod | 3 +-
plugins/perl/nbdkit-perl-plugin.pod | 2 +-
plugins/python/nbdkit-python-plugin.pod | 2 +-
plugins/ruby/nbdkit-ruby-plugin.pod | 2 +-
plugins/sh/nbdkit-sh-plugin.pod | 27 ++++-
plugins/tcl/nbdkit-tcl-plugin.pod | 3 +-
configure.ac | 5 +-
common/protocol/protocol.h | 2 +
filters/cache/blk.h | 10 +-
filters/cow/blk.h | 16 ++-
include/nbdkit-common.h | 4 +
include/nbdkit-filter.h | 8 ++
include/nbdkit-plugin.h | 2 +
server/internal.h | 5 +
filters/blocksize/blocksize.c | 29 +++++
filters/cache/blk.c | 50 +++++++-
filters/cache/cache.c | 53 ++++++++
filters/cow/blk.c | 41 ++++++-
filters/cow/cow.c | 90 ++++++++++++++
filters/delay/delay.c | 27 ++++-
filters/error/error.c | 32 ++++-
filters/log/log.c | 29 ++++-
filters/nocache/nocache.c | 113 ++++++++++++++++++
filters/offset/offset.c | 12 +-
filters/partition/partition.c | 12 ++
filters/readahead/readahead.c | 13 ++
filters/stats/stats.c | 23 ++++
filters/truncate/truncate.c | 24 ++++
filters/xz/xz.c | 15 ++-
plugins/data/data.c | 11 ++
plugins/ext2/ext2.c | 10 +-
plugins/file/file.c | 37 ++++++
plugins/floppy/floppy.c | 11 +-
plugins/full/full.c | 13 +-
plugins/iso/iso.c | 10 +-
plugins/linuxdisk/linuxdisk.c | 9 ++
plugins/memory/memory.c | 11 ++
plugins/nbd/nbd.c | 24 ++++
plugins/null/null.c | 13 +-
plugins/ocaml/ocaml.c | 51 ++++++++
plugins/partitioning/partitioning.c | 11 +-
plugins/pattern/pattern.c | 13 +-
plugins/random/random.c | 13 +-
plugins/sh/sh.c | 81 +++++++++++++
plugins/split/split.c | 52 +++++++-
plugins/zero/zero.c | 13 +-
server/filters.c | 59 ++++++++-
server/plugins.c | 44 +++++++
server/protocol-handshake.c | 9 ++
server/protocol.c | 26 ++++
tests/test-layers-filter.c | 22 +++-
tests/test-layers-plugin.c | 17 +++
tests/test-layers.c | 36 ++++++
filters/nocache/Makefile.am | 61 ++++++++++
plugins/ocaml/NBDKit.ml | 16 ++-
plugins/ocaml/NBDKit.mli | 5 +
plugins/sh/example.sh | 7 ++
tests/test-eflags.sh | 36 +++++-
70 files changed, 1570 insertions(+), 63 deletions(-)
create mode 100644 filters/nocache/nbdkit-nocache-filter.pod
create mode 100644 filters/nocache/nocache.c
create mode 100644 filters/nocache/Makefile.am
--
2.20.1
5 years
[PATCH] v2v: -o json: add a simple test for it
by Pino Toscano
Followup of commit f190e08d85556dac293ef15bfeee38e54471570f.
---
v2v/Makefile.am | 2 ++
v2v/test-v2v-o-json.sh | 58 ++++++++++++++++++++++++++++++++++++++++++
2 files changed, 60 insertions(+)
create mode 100755 v2v/test-v2v-o-json.sh
diff --git a/v2v/Makefile.am b/v2v/Makefile.am
index 39511022e..1c37d20b4 100644
--- a/v2v/Makefile.am
+++ b/v2v/Makefile.am
@@ -468,6 +468,7 @@ TESTS += \
test-v2v-networks-and-bridges.sh \
test-v2v-no-copy.sh \
test-v2v-o-glance.sh \
+ test-v2v-o-json.sh \
test-v2v-o-libvirt.sh \
test-v2v-o-null.sh \
test-v2v-o-openstack.sh \
@@ -628,6 +629,7 @@ EXTRA_DIST += \
test-v2v-networks-and-bridges.xml \
test-v2v-no-copy.sh \
test-v2v-o-glance.sh \
+ test-v2v-o-json.sh \
test-v2v-o-libvirt.sh \
test-v2v-o-null.sh \
test-v2v-o-openstack.sh \
diff --git a/v2v/test-v2v-o-json.sh b/v2v/test-v2v-o-json.sh
new file mode 100755
index 000000000..71458a1bb
--- /dev/null
+++ b/v2v/test-v2v-o-json.sh
@@ -0,0 +1,58 @@
+#!/bin/bash -
+# libguestfs virt-v2v test script
+# Copyright (C) 2019 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Test -o json.
+
+set -e
+
+$TEST_FUNCTIONS
+skip_if_skipped
+skip_if_backend uml
+skip_unless_phony_guest windows.img
+skip_unless jq --version
+
+libvirt_uri="test://$abs_top_builddir/test-data/phony-guests/guests.xml"
+
+export VIRT_TOOLS_DATA_DIR="$top_srcdir/test-data/fake-virt-tools"
+
+guestname=windows
+
+d=test-v2v-o-json.d
+rm -rf $d
+mkdir $d
+
+json=$d/$guestname.json
+
+$VG virt-v2v --debug-gc \
+ -i libvirt -ic "$libvirt_uri" windows \
+ -o json -os $d -on $guestname
+
+# Test the disk was created.
+test -f $d/$guestname-sda
+
+# Test the JSON.
+test x$(jq -r '.name' $json) = xwindows
+test x$(jq -r '.inspect.type' $json) = xwindows
+test x$(jq -r '.inspect.distro' $json) = xwindows
+test x$(jq -r '.inspect.osinfo' $json) = xwin7
+test $(jq -r '.disks | length' $json) -eq 1
+test $(jq -r '.nics | length' $json) -eq 1
+test $(jq -r '.removables | length' $json) -eq 0
+
+# Clean up.
+rm -r $d
--
2.21.0
5 years
[nbdkit PATCH 0/2] Avoid oddities with files unaligned to granularity
by Eric Blake
When using a filter that rounds up to alignment boundaries, the tail
bytes of the plugin are difficult to access correctly. Rather than
duplicating lots of code already in the truncate filter, it's easier
to just make the other filters default to rounding down and add doc
links on how to round up instead.
Eric Blake (2):
blocksize: Lift restriction against 0-size file
cache, cow: Round size down
filters/blocksize/nbdkit-blocksize-filter.pod | 15 +++++++++---
filters/cache/nbdkit-cache-filter.pod | 9 +++++++-
filters/cow/nbdkit-cow-filter.pod | 8 +++++++
filters/truncate/nbdkit-truncate-filter.pod | 9 ++++++++
filters/blocksize/blocksize.c | 23 ++++---------------
filters/cache/cache.c | 8 +++++--
filters/cow/cow.c | 6 ++++-
7 files changed, 53 insertions(+), 25 deletions(-)
--
2.20.1
5 years
