[PATCH v2] launch: add support for autodetection of appliance image format
by Pavel Butsykin
This feature allows you to use different image formats for the fixed
appliance. The raw format is used by default.
Signed-off-by: Pavel Butsykin <pbutsykin(a)virtuozzo.com>
---
lib/launch-direct.c | 2 ++
lib/launch-libvirt.c | 19 ++++++++++++-------
m4/guestfs_appliance.m4 | 11 +++++++++++
3 files changed, 25 insertions(+), 7 deletions(-)
diff --git a/lib/launch-direct.c b/lib/launch-direct.c
index 0be662e25..b9b54857a 100644
--- a/lib/launch-direct.c
+++ b/lib/launch-direct.c
@@ -592,7 +592,9 @@ launch_direct (guestfs_h *g, void *datav, const char *arg)
append_list ("id=appliance");
append_list ("cache=unsafe");
append_list ("if=none");
+#ifndef APPLIANCE_FMT_AUTO
append_list ("format=raw");
+#endif
} end_list ();
start_list ("-device") {
append_list ("scsi-hd");
diff --git a/lib/launch-libvirt.c b/lib/launch-libvirt.c
index 4adb2cfb3..030ea6911 100644
--- a/lib/launch-libvirt.c
+++ b/lib/launch-libvirt.c
@@ -212,9 +212,10 @@ get_source_format_or_autodetect (guestfs_h *g, struct drive *drv)
/**
* Create a qcow2 format overlay, with the given C<backing_drive>
- * (file). The C<format> parameter, which must be non-NULL, is the
- * backing file format. This is used to create the appliance overlay,
- * and also for read-only drives.
+ * (file). The C<format> parameter is the backing file format.
+ * The C<format> parameter can be NULL, in this case the backing
+ * format will be determined automatically. This is used to create
+ * the appliance overlay, and also for read-only drives.
*/
static char *
make_qcow2_overlay (guestfs_h *g, const char *backing_drive,
@@ -223,8 +224,6 @@ make_qcow2_overlay (guestfs_h *g, const char *backing_drive,
char *overlay;
struct guestfs_disk_create_argv optargs;
- assert (format != NULL);
-
if (guestfs_int_lazy_make_tmpdir (g) == -1)
return NULL;
@@ -232,8 +231,10 @@ make_qcow2_overlay (guestfs_h *g, const char *backing_drive,
optargs.bitmask = GUESTFS_DISK_CREATE_BACKINGFILE_BITMASK;
optargs.backingfile = backing_drive;
- optargs.bitmask |= GUESTFS_DISK_CREATE_BACKINGFORMAT_BITMASK;
- optargs.backingformat = format;
+ if (format) {
+ optargs.bitmask |= GUESTFS_DISK_CREATE_BACKINGFORMAT_BITMASK;
+ optargs.backingformat = format;
+ }
if (guestfs_disk_create_argv (g, overlay, "qcow2", -1, &optargs) == -1) {
free (overlay);
@@ -461,7 +462,11 @@ launch_libvirt (guestfs_h *g, void *datav, const char *libvirt_uri)
/* Note that appliance can be NULL if using the old-style appliance. */
if (appliance) {
+#ifdef APPLIANCE_FMT_AUTO
+ params.appliance_overlay = make_qcow2_overlay (g, appliance, NULL);
+#else
params.appliance_overlay = make_qcow2_overlay (g, appliance, "raw");
+#endif
if (!params.appliance_overlay)
goto cleanup;
}
diff --git a/m4/guestfs_appliance.m4 b/m4/guestfs_appliance.m4
index 81c43879f..4e1ec8135 100644
--- a/m4/guestfs_appliance.m4
+++ b/m4/guestfs_appliance.m4
@@ -139,3 +139,14 @@ AC_SUBST([GUESTFS_DEFAULT_PATH])
AC_DEFINE_UNQUOTED([GUESTFS_DEFAULT_PATH], ["$GUESTFS_DEFAULT_PATH"],
[Define guestfs default path.])
+
+AC_ARG_ENABLE([appliance-fmt-auto],
+ [AS_HELP_STRING([--enable-appliance-fmt-auto],
+ [enable autodetection of appliance image format @<:@default=no@:>@])],
+ [ENABLE_APPLIANCE_FMT_AUTO="$enableval"],
+ [ENABLE_APPLIANCE_FMT_AUTO=no])
+
+if test "x$ENABLE_APPLIANCE_FMT_AUTO" = "xyes"; then
+ AC_DEFINE([APPLIANCE_FMT_AUTO], [1],
+ [Define to 1 if enabled autodetection of appliance image format.])
+fi
--
2.13.0
4 years, 10 months
1.39 proposal: Let's split up the libguestfs git repo and tarballs
by Richard W.M. Jones
My contention is that the libguestfs git repository is too large and
unwieldy. There are too many separate, unrelated projects and as a
result of that the source has too many dependencies and takes too long
to build and test.
The project divides (sort of) naturally into layers -- the library,
the bindings, the various virt tools -- and could be split along those
lines into separate projects which can then be released and evolve at
their own pace.
My suggested split would be something like this:
* libguestfs: The library, daemon and appliance. That would include
the following directories in a single project:
appliance
bash
contrib
daemon
docs
examples
gnulib
lib
logo
test-tool
tmp
utils
website
* 1 project for each language binding:
csharp
erlang
gobject
golang
haskell
java
lua
ocaml
php
perl
python
ruby
* virt-customize and related tools, we'd probably call this subproject
"virt-builder". It would include virt-builder, virt-customize and
virt-sysprep, since they share a lot of common code.
* 1 project for each of the following items:
small tools written in C
(virt-cat, virt-filesystems, virt-log, virt-ls, virt-tail,
virt-diff, virt-edit, virt-format, guestmount, virt-inspector,
virt-make-fs, virt-rescue)
guestfish
virt-alignment-scan and virt-df
virt-dib
virt-get-kernel
virt-resize
virt-sparsify
virt-v2v and virt-p2v
virt-win-reg
* I'd be inclined to drop the legacy Perl tools virt-tar,
virt-list-filesystems, virt-list-partitions unless someone
especially wished to step forward to maintain them.
* common code and generator: Off to the side we'd somehow need to
package up the common code and the generator for use by all of the
above projects. It wouldn't be a separate project for downstream
packagers, but instead the code would be included (ie. duplicated)
in tarballs and upstream available as a side git repo that you'd
need to include when building (git submodule?). This is somewhat
unspecified.
M4, PO, and tests would be split between the projects as appropriate.
My proposal would be to do this incrementally, rather than all at
once, moving the easier things out first.
Thoughts?
Rich.
--
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
libguestfs lets you edit virtual machines. Supports shell scripting,
bindings from many languages. http://libguestfs.org
5 years
[nbdkit PATCH 0/3] Fix %m usage on BSD
by Eric Blake
Our use of "%m" in various error messages is testament to the
project's initial life on Linux - but other than Cygwin, I know
of no other platforms supporting that glibc extension.
We COULD audit the code and manually turn "%m" into
"%s"/strerror(errno), but that's a lot of churn. Instead, let's
fix the few outliers that can't be easily wrapped, then wrap
the remainder.
While I was able to test this on Linux (both that no wrapper is
used by default, and that faking that %m fails causes the wrapper
to do the right thing), I haven't actually tried it on a BSD box,
hence I'll wait for review before pushing.
Eric Blake (3):
main: Avoid fprintf(%m) for BSD builds
log: Ensure %m sees correct errno
log: Guarantee the operation of %m in nbdkit_error()
docs/nbdkit-filter.pod | 13 +++++++++----
docs/nbdkit-plugin.pod | 13 +++++++++----
configure.ac | 27 +++++++++++++++++++++++++++
src/internal.h | 6 ++++++
src/log-stderr.c | 4 ++--
src/log-syslog.c | 6 +++---
src/log.c | 21 +++++++++++++++++++++
src/main.c | 4 ++--
8 files changed, 79 insertions(+), 15 deletions(-)
--
2.17.2
5 years, 11 months
[PATCH] v2v: -o rhv-upload: decouple name of nbdkit python plugin
by Pino Toscano
Do not assume that the Python plugin of nbdkit has the same name of the
Python interpreter.
Use the default upstream name of nbdkit to identify it; downstream
distributions must adjust this variable, in case they rename the Python
plugin of nbdkit.
---
v2v/output_rhv_upload.ml | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/v2v/output_rhv_upload.ml b/v2v/output_rhv_upload.ml
index f5e0e6b1c..d8f608cff 100644
--- a/v2v/output_rhv_upload.ml
+++ b/v2v/output_rhv_upload.ml
@@ -78,6 +78,7 @@ let parse_output_options options =
{ rhv_cafile; rhv_cluster; rhv_direct; rhv_verifypeer }
+let nbdkit_python_plugin = "python"
let pidfile_timeout = 30
let finalization_timeout = 5*60
@@ -139,14 +140,14 @@ class output_rhv_upload output_alloc output_conn
*)
let error_unless_nbdkit_python_plugin_working () =
let cmd = sprintf "nbdkit %s %s --dump-plugin >/dev/null"
- Python_script.python
+ nbdkit_python_plugin
(quote (Python_script.path plugin_script)) in
debug "%s" cmd;
if Sys.command cmd <> 0 then
error (f_"nbdkit %s plugin is not installed or not working. It is required if you want to use ‘-o rhv-upload’.
See also the virt-v2v-output-rhv(1) manual.")
- Python_script.python
+ nbdkit_python_plugin
in
(* Check that nbdkit was compiled with SELinux support (for the
@@ -209,7 +210,7 @@ See also the virt-v2v-output-rhv(1) manual.")
"--newstyle"; (* use newstyle NBD protocol *)
"--exportname"; "/";
- Python_script.python; (* use the nbdkit Python 3 plugin *)
+ nbdkit_python_plugin; (* use the nbdkit Python plugin *)
Python_script.path plugin_script; (* Python plugin script *)
] in
let args = if verbose () then args @ ["--verbose"] else args in
--
2.17.2
5 years, 11 months
[PATCH nbdkit 0/9] cache: Implement cache-max-size and method of reclaiming space from the cache.
by Richard W.M. Jones
This patch series enhances the cache filter in a few ways, primarily
adding a "cache-on-read" feature (similar to qemu's copyonread); and
adding the ability to limit the cache size and the antecedent of that
which is having a method to reclaim cache blocks.
As the cache is stored as a sparse temporary file, reclaiming cache
blocks simply means punching holes in the temporary file. The tricky
bit is working out where to punch the holes to avoid reclaiming
recently/frequently used blocks.
I believe the status of the patch series is:
[1/9] cache: Rename blk_writeback function.
[2/9] cache: Add cache-on-read mode.
[3/9] common/bitmap: Include <nbdkit-plugin.h>.
[4/9] cache: Split out the block/bitmap code into
- All of the above patches are pretty uncontroversial and ready for
review and upstreaming.
[5/9] cache: Allow this filter to serve requests in
- Probably needs more testing, but should be OK too.
[6/9] common/bitmap: Add bitmap_next function and tests.
[7/9] common/bitmap: Add bitmap_clear function.
[8/9] cache: Implement LRU structure.
[9/9] cache: Implement cache-max-size and method of
- Needs much better testing, and maybe even review of the design.
Rich.
5 years, 11 months
[nbdkit] Rename src to server?
by Richard W.M. Jones
Eric,
What do you think about renaming ‘src’ to ‘server’? I think it would
help when newcomers navigate the source code. (And I hope that my
FOSDEM talk will bring lots of new users and developers, hence why
I've been working on this over the holiday.)
If you agree then I'll push a commit which changes the directory name
and makes the concomitant changes to makefiles, docs etc.
Rich.
--
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
virt-builder quickly builds VMs from scratch
http://libguestfs.org/virt-builder.1.html
5 years, 11 months
[PATCH nbdkit] common: Improve pseudo-random number generation.
by Richard W.M. Jones
Currently we use non-cryptographically secure random numbers in two
places, the error filter (to inject errors at random) and the random
plugin. For this we have used either random_r or a home-brew-ish
Linear Congruential Generator. Use of random_r is problematic on BSDs
because it doesn't exist there. Use of the LCG is simply a bad
choice.
Replace both uses with a better quality and faster PRNG from David
Blackman and Sebastiano Vigna called ‘xoshiro256** 1.0’
(http://xoshiro.di.unimi.it/). This is licensed under a public-domain
license so it compatible with the licensing of nbdkit.
This also fixes a bug in the random plugin where it could never
generate the byte 255 because I used modulo 255 instead of modulo 256
arithmetic. Ooops.
---
plugins/random/nbdkit-random-plugin.pod | 5 +-
common/include/random.h | 79 +++++++++++++++++++++++++
filters/error/error.c | 28 ++++-----
plugins/random/random.c | 46 +++++++-------
common/include/Makefile.am | 1 +
plugins/random/Makefile.am | 1 +
6 files changed, 115 insertions(+), 45 deletions(-)
diff --git a/plugins/random/nbdkit-random-plugin.pod b/plugins/random/nbdkit-random-plugin.pod
index 5dfe801..750daab 100644
--- a/plugins/random/nbdkit-random-plugin.pod
+++ b/plugins/random/nbdkit-random-plugin.pod
@@ -15,9 +15,8 @@ The size of the virtual disk must be specified using the C<size>
parameter. If you specify the C<seed> parameter then you will get the
same random data over multiple runs with the same seed.
-The random data is generated using an I<in>secure method (a Linear
-Congruential Generator). This plugin is mainly good for testing NBD
-clients.
+The random data is generated using an I<insecure> method. This plugin
+is mainly good for testing NBD clients.
=head1 PARAMETERS
diff --git a/common/include/random.h b/common/include/random.h
new file mode 100644
index 0000000..e4cc947
--- /dev/null
+++ b/common/include/random.h
@@ -0,0 +1,79 @@
+/* nbdkit
+ * Copyright (C) 2018 Red Hat Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef NBDKIT_RANDOM_H
+#define NBDKIT_RANDOM_H
+
+#include <stdint.h>
+
+/* Generate pseudo-random numbers, quickly, with explicit state.
+ *
+ * This is based on the xoshiro/xoroshiro generators by David Blackman
+ * and Sebastiano Vigna at http://xoshiro.di.unimi.it/ . Specifically
+ * this is ‘xoshiro256** 1.0’.
+ *
+ * This does _NOT_ generate cryptographically secure random numbers
+ * (CSPRNG) and so should not be used when cryptography or security is
+ * required - use gcrypt if you need those.
+ */
+
+struct random_state {
+ uint64_t s[4];
+};
+
+static inline uint64_t
+rotl (const uint64_t x, int k)
+{
+ return (x << k) | (x >> (64 - k));
+}
+
+/* Returns 64 random bits. Updates the state. */
+uint64_t
+xrandom (struct random_state *state)
+{
+ const uint64_t result_starstar = rotl (state->s[1] * 5, 7) * 9;
+ const uint64_t t = state->s[1] << 17;
+
+ state->s[2] ^= state->s[0];
+ state->s[3] ^= state->s[1];
+ state->s[1] ^= state->s[2];
+ state->s[0] ^= state->s[3];
+
+ state->s[2] ^= t;
+
+ state->s[3] = rotl (state->s[3], 45);
+
+ return result_starstar;
+}
+
+#endif /* NBDKIT_RANDOM_H */
diff --git a/filters/error/error.c b/filters/error/error.c
index 8509a16..6d305b3 100644
--- a/filters/error/error.c
+++ b/filters/error/error.c
@@ -44,6 +44,8 @@
#include <nbdkit-filter.h>
+#include "random.h"
+
#define THREAD_MODEL NBDKIT_THREAD_MODEL_PARALLEL
struct error_settings {
@@ -222,17 +224,13 @@ error_config (nbdkit_next_config *next, void *nxdata,
" Apply settings only to read/write/trim/zero"
struct handle {
-#ifdef __GNU_LIBRARY__
- struct random_data rd;
- char rd_state[32];
-#endif
+ struct random_state random_state;
};
static void *
error_open (nbdkit_next_open *next, void *nxdata, int readonly)
{
struct handle *h;
- time_t t;
if (next (nxdata, readonly) == -1)
return NULL;
@@ -242,11 +240,7 @@ error_open (nbdkit_next_open *next, void *nxdata, int readonly)
nbdkit_error ("malloc: %m");
return NULL;
}
-#ifdef __GNU_LIBRARY__
- memset (&h->rd, 0, sizeof h->rd);
- time (&t);
- initstate_r (t, (char *) &h->rd_state, sizeof h->rd_state, &h->rd);
-#endif
+ memset (&h->random_state, 0, sizeof h->random_state);
return h;
}
@@ -264,7 +258,7 @@ random_error (struct handle *h,
const struct error_settings *error_settings,
const char *fn, int *err)
{
- int32_t rand;
+ uint64_t rand;
if (error_settings->rate <= 0) /* 0% = never inject */
return false;
@@ -278,12 +272,12 @@ random_error (struct handle *h,
if (error_settings->rate >= 1) /* 100% = always inject */
goto inject;
-#ifdef __GNU_LIBRARY__
- random_r (&h->rd, &rand);
-#else
- rand = random ();
-#endif
- if (rand >= error_settings->rate * RAND_MAX)
+ /* To avoid the question if (double)1.0 * UINT64_MAX is
+ * representable in a 64 bit integer, and because we don't need all
+ * this precision anyway, let's work in 32 bits.
+ */
+ rand = xrandom (&h->random_state) & UINT32_MAX;
+ if (rand >= error_settings->rate * UINT32_MAX)
return false;
inject:
diff --git a/plugins/random/random.c b/plugins/random/random.c
index 8adc26e..00a6443 100644
--- a/plugins/random/random.c
+++ b/plugins/random/random.c
@@ -44,32 +44,14 @@
#define NBDKIT_API_VERSION 2
#include <nbdkit-plugin.h>
+#include "random.h"
+
/* The size of disk in bytes (initialized by size=<SIZE> parameter). */
static int64_t size = 0;
/* Seed. */
static uint32_t seed;
-/* We use a Linear Congruential Generator (LCG) to make low quality
- * but easy to compute random numbers. However we're not quite using
- * it in the ordinary way. In order to be able to read any byte of
- * data without needing to run the LCG from the start, the random data
- * is computed from the index and seed through two rounds of LCG:
- *
- * index i LCG(seed) -> +i -> LCG -> LCG -> mod 256 -> b[i]
- * index i+1 LCG(seed) -> +i+1 -> LCG -> LCG -> mod 256 -> b[i+1]
- * etc
- *
- * This LCG is the same one as used in glibc.
- */
-static inline uint32_t
-LCG (uint32_t s)
-{
- s *= 1103515245;
- s += 12345;
- return s;
-}
-
static void
random_load (void)
{
@@ -135,13 +117,27 @@ random_pread (void *handle, void *buf, uint32_t count, uint64_t offset,
{
uint32_t i;
unsigned char *b = buf;
- uint32_t s;
+ uint64_t s;
+ struct random_state state;
for (i = 0; i < count; ++i) {
- s = LCG (seed) + offset + i;
- s = LCG (s);
- s = LCG (s);
- s = s % 255;
+ /* We use nbdkit common/include/random.h to make random numbers.
+ *
+ * However we're not quite using it in the ordinary way. In order
+ * to be able to read any byte of data without needing to run the
+ * PRNG from the start, the random data is computed from the index
+ * and seed through three rounds of PRNG:
+ *
+ * index i PRNG(seed+i) -> PRNG -> PRNG -> mod 256 -> b[i]
+ * index i+1 PRNG(seed+i+1) -> PRNG -> PRNG -> mod 256 -> b[i+1]
+ * etc
+ */
+ memset (&state, 0, sizeof state);
+ state.s[0] = seed + offset + i;
+ xrandom (&state);
+ xrandom (&state);
+ s = xrandom (&state);
+ s &= 255;
b[i] = s;
}
return 0;
diff --git a/common/include/Makefile.am b/common/include/Makefile.am
index c7416fb..6417b8e 100644
--- a/common/include/Makefile.am
+++ b/common/include/Makefile.am
@@ -42,4 +42,5 @@ EXTRA_DIST = \
ispowerof2.h \
iszero.h \
nextnonzero.h \
+ random.h \
rounding.h
diff --git a/plugins/random/Makefile.am b/plugins/random/Makefile.am
index d990158..0a84774 100644
--- a/plugins/random/Makefile.am
+++ b/plugins/random/Makefile.am
@@ -41,6 +41,7 @@ nbdkit_random_plugin_la_SOURCES = \
$(top_srcdir)/include/nbdkit-plugin.h
nbdkit_random_plugin_la_CPPFLAGS = \
+ -I$(top_srcdir)/common/include \
-I$(top_srcdir)/include
nbdkit_random_plugin_la_CFLAGS = \
$(WARNINGS_CFLAGS)
--
2.19.2
5 years, 12 months
