From: "Richard W.M. Jones" <rjones(a)redhat.com&gt; The shell_quote function is moved to a new common/utils directory. Eventually more utility functions can be created here. This change is pure refactoring. --- Makefile.am | 1 + common/utils/Makefile.am | 41 ++++++++++++++++++++++++++++ common/utils/utils.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ common/utils/utils.h | 39 +++++++++++++++++++++++++++ configure.ac | 1 + plugins/iso/Makefile.am | 3 +++ plugins/iso/iso.c | 36 ++----------------------- 7 files changed, 157 insertions(+), 34 deletions(-) create mode 100644 common/utils/Makefile.am create mode 100644 common/utils/utils.c create mode 100644 common/utils/utils.h diff --git a/Makefile.am b/Makefile.am index 653956f..99f0852 100644 --- a/Makefile.am +++ b/Makefile.am @@ -74,6 +74,7 @@ SUBDIRS += \ common/gpt \ common/regions \ common/sparse \ + common/utils \ plugins \ filters endif diff --git a/common/utils/Makefile.am b/common/utils/Makefile.am new file mode 100644 index 0000000..e002586 --- /dev/null +++ b/common/utils/Makefile.am @@ -0,0 +1,41 @@ +# nbdkit +# Copyright (C) 2019 Red Hat Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Red Hat nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +include $(top_srcdir)/common-rules.mk + +noinst_LTLIBRARIES = libutils.la + +libutils_la_SOURCES = \ + utils.c \ + utils.h +libutils_la_CFLAGS = \ + $(WARNINGS_CFLAGS) diff --git a/common/utils/utils.c b/common/utils/utils.c new file mode 100644 index 0000000..458f489 --- /dev/null +++ b/common/utils/utils.c @@ -0,0 +1,70 @@ +/* nbdkit + * Copyright (C) 2018-2019 Red Hat Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Red Hat nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +/* Print str to fp, shell quoting if necessary. This comes from + * libguestfs, but was written by me so I'm relicensing it to a BSD + * license for nbdkit. + */ +void +shell_quote (const char *str, FILE *fp) +{ + const char *safe_chars = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_=,:/"; + size_t i, len; + + /* If the string consists only of safe characters, output it as-is. */ + len = strlen (str); + if (len == strspn (str, safe_chars)) { + fputs (str, fp); + return; + } + + /* Double-quote the string. */ + fputc ('"', fp); + for (i = 0; i < len; ++i) { + switch (str[i]) { + case '$': case '`': case '\\': case '"': + fputc ('\\', fp); + /*FALLTHROUGH*/ + default: + fputc (str[i], fp); + } + } + fputc ('"', fp); +} diff --git a/common/utils/utils.h b/common/utils/utils.h new file mode 100644 index 0000000..a012183 --- /dev/null +++ b/common/utils/utils.h @@ -0,0 +1,39 @@ +/* nbdkit + * Copyright (C) 2018-2019 Red Hat Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Red Hat nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef NBDKIT_UTILS_H +#define NBDKIT_UTILS_H + +extern void shell_quote (const char *str, FILE *fp); + +#endif /* NBDKIT_UTILS_H */ diff --git a/configure.ac b/configure.ac index bca6134..229e38c 100644 --- a/configure.ac +++ b/configure.ac @@ -837,6 +837,7 @@ AC_CONFIG_FILES([Makefile common/include/Makefile common/regions/Makefile common/sparse/Makefile + common/utils/Makefile docs/Makefile include/Makefile plugins/Makefile diff --git a/plugins/iso/Makefile.am b/plugins/iso/Makefile.am index 061c161..673d1ee 100644 --- a/plugins/iso/Makefile.am +++ b/plugins/iso/Makefile.am @@ -43,6 +43,7 @@ nbdkit_iso_plugin_la_SOURCES = \ $(top_srcdir)/include/nbdkit-plugin.h nbdkit_iso_plugin_la_CPPFLAGS = \ + -I$(top_srcdir)/common/utils \ -I$(top_srcdir)/include \ -I. nbdkit_iso_plugin_la_CFLAGS = \ @@ -50,6 +51,8 @@ nbdkit_iso_plugin_la_CFLAGS = \ nbdkit_iso_plugin_la_LDFLAGS = \ -module -avoid-version -shared \ -Wl,--version-script=$(top_srcdir)/plugins/plugins.syms +nbdkit_iso_plugin_la_LIBADD = \ + $(top_builddir)/common/utils/libutils.la if HAVE_POD diff --git a/plugins/iso/iso.c b/plugins/iso/iso.c index 7431b48..53fccf0 100644 --- a/plugins/iso/iso.c +++ b/plugins/iso/iso.c @@ -43,6 +43,8 @@ #include <nbdkit-plugin.h> +#include "utils.h" + /* List of directories parsed from the command line. */ static char **dirs = NULL; static size_t nr_dirs = 0; @@ -59,8 +61,6 @@ static const char *params = NULL; static int fd = -1; /* Construct the temporary ISO. */ -static void shell_quote (const char *str, FILE *fp); - static int make_iso (void) { @@ -136,38 +136,6 @@ make_iso (void) return 0; } -/* Print str to fp, shell quoting if necessary. This comes from - * libguestfs, but was written by me so I'm relicensing it to a BSD - * license for nbdkit. - */ -static void -shell_quote (const char *str, FILE *fp) -{ - const char *safe_chars = - "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_=,:/"; - size_t i, len; - - /* If the string consists only of safe characters, output it as-is. */ - len = strlen (str); - if (len == strspn (str, safe_chars)) { - fputs (str, fp); - return; - } - - /* Double-quote the string. */ - fputc ('"', fp); - for (i = 0; i < len; ++i) { - switch (str[i]) { - case '$': case '`': case '\\': case '"': - fputc ('\\', fp); - /*FALLTHROUGH*/ - default: - fputc (str[i], fp); - } - } - fputc ('"', fp); -} - static void iso_unload (void) { -- 1.8.3.1

From: "Richard W.M. Jones" <rjones(a)redhat.com&gt; This plugin allows you to create a complete ext2 filesystem in a GPT partitioned disk image. This can be attached as a disk to a Linux virtual machine. It is implemented using libext2fs (the same as supermin). Although there is some overlap with nbdkit-iso-plugin and nbdkit-floppy-plugin, the implementations and use cases of all three plugins are sufficiently different that it seems to make sense to add another plugin rather than attempting to extend one of the existing plugins. Largely to avoid user error this plugin is read-only. This is a major difference from the floppy plugin: that plugin allows files to be modified (but not resized or created) and writes those changes through to the backing filesystem. While this plugin could easily be made writable, this would cause almost certain disk corruption when someone connected two clients at the same time. In any case it doesn't make much sense for it to be writable by default since the expectation that writes would somehow modify the original directory on the host filesystem cannot be satisfied by this or any reasonable implementation. Users can add the cow filter on top if they really want writes and know what they are doing: instructions plus disclaimer about this are included in the man page. As mentioned above, this implementation is based on the same idea as the appliance creation code in supermin. Eventually we could replace that supermin code with this plugin, but there are some missing features that would need to be implemented first. --- README | 2 +- TODO | 2 + configure.ac | 4 +- plugins/floppy/nbdkit-floppy-plugin.pod | 1 + plugins/iso/nbdkit-iso-plugin.pod | 6 +- plugins/linuxdisk/Makefile.am | 78 ++ plugins/linuxdisk/filesystem.c | 935 +++++++++++++++++++++ plugins/linuxdisk/linuxdisk.c | 236 ++++++ plugins/linuxdisk/nbdkit-linuxdisk-plugin.pod | 197 +++++ plugins/linuxdisk/partition-gpt.c | 211 +++++ plugins/linuxdisk/virtual-disk.c | 163 ++++ plugins/linuxdisk/virtual-disk.h | 96 +++ .../partitioning/nbdkit-partitioning-plugin.pod | 5 +- tests/Makefile.am | 11 + tests/test-linuxdisk-copy-out.sh | 76 ++ tests/test-linuxdisk.sh | 94 +++ 16 files changed, 2111 insertions(+), 6 deletions(-) create mode 100644 plugins/linuxdisk/Makefile.am create mode 100644 plugins/linuxdisk/filesystem.c create mode 100644 plugins/linuxdisk/linuxdisk.c create mode 100644 plugins/linuxdisk/nbdkit-linuxdisk-plugin.pod create mode 100644 plugins/linuxdisk/partition-gpt.c create mode 100644 plugins/linuxdisk/virtual-disk.c create mode 100644 plugins/linuxdisk/virtual-disk.h create mode 100755 tests/test-linuxdisk-copy-out.sh create mode 100755 tests/test-linuxdisk.sh diff --git a/README b/README index e394a1f..4ba9d2a 100644 --- a/README +++ b/README @@ -96,7 +96,7 @@ For the libguestfs plugin, and to run parts of the test suite: - guestfish (from libguestfs) -For the ext2 plugin: +For the ext2 and linuxdisk plugins: - ext2fs diff --git a/TODO b/TODO index 72e03b6..96066b2 100644 --- a/TODO +++ b/TODO @@ -71,6 +71,8 @@ General ideas for improvements and also look at the implementation of the -swap option in nbd-client. +* Implement extended attributes in the linuxdisk plugin. + Suggestions for plugins ----------------------- diff --git a/configure.ac b/configure.ac index 229e38c..b8400d4 100644 --- a/configure.ac +++ b/configure.ac @@ -735,7 +735,7 @@ dnl Check for guestfish (only needed for some of the tests). AC_CHECK_PROG([GUESTFISH], [guestfish], [guestfish], [no]) AM_CONDITIONAL([HAVE_GUESTFISH], [test "x$GUESTFISH" != "xno"]) -dnl Check for ext2fs and com_err, for the ext2 plugin. +dnl Check for ext2fs and com_err, for the ext2 and linuxdisk plugins. AC_ARG_WITH([ext2], [AS_HELP_STRING([--without-ext2], [disable ext2 plugin @<:@default=check@:>@])], @@ -793,6 +793,7 @@ non_lang_plugins="\ gzip \ iso \ libvirt \ + linuxdisk \ memory \ nbd \ null \ @@ -855,6 +856,7 @@ AC_CONFIG_FILES([Makefile plugins/gzip/Makefile plugins/iso/Makefile plugins/libvirt/Makefile + plugins/linuxdisk/Makefile plugins/lua/Makefile plugins/memory/Makefile plugins/nbd/Makefile diff --git a/plugins/floppy/nbdkit-floppy-plugin.pod b/plugins/floppy/nbdkit-floppy-plugin.pod index ae14946..ff84f68 100644 --- a/plugins/floppy/nbdkit-floppy-plugin.pod +++ b/plugins/floppy/nbdkit-floppy-plugin.pod @@ -74,6 +74,7 @@ important, and it simplifies the implementation greatly. L<nbdkit(1)>, L<nbdkit-plugin(3)>, L<nbdkit-file-plugin(1)>, +L<nbdkit-linuxdisk-plugin(1)>, L<nbdkit-iso-plugin(1)>. =head1 AUTHORS diff --git a/plugins/iso/nbdkit-iso-plugin.pod b/plugins/iso/nbdkit-iso-plugin.pod index 4d9cf41..90e26f0 100644 --- a/plugins/iso/nbdkit-iso-plugin.pod +++ b/plugins/iso/nbdkit-iso-plugin.pod @@ -17,8 +17,9 @@ read-only over the NBD protocol. This plugin uses L<genisoimage(1)> or L<mkisofs(1)> to create the ISO content. -To create a virtual floppy disk instead of a CD, see -L<nbdkit-floppy-plugin(1)>. +To create a FAT-formatted virtual floppy disk instead of a CD, see +L<nbdkit-floppy-plugin(1)>. To create a Linux compatible virtual +disk, see L<nbdkit-linuxdisk-plugin(1)>. =head1 EXAMPLE @@ -96,6 +97,7 @@ L<nbdkit(1)>, L<nbdkit-plugin(3)>, L<nbdkit-file-plugin(1)>, L<nbdkit-floppy-plugin(1)>, +L<nbdkit-linuxdisk-plugin(1)>, L<genisoimage(1)>, L<mkisofs(1)>. diff --git a/plugins/linuxdisk/Makefile.am b/plugins/linuxdisk/Makefile.am new file mode 100644 index 0000000..277efe8 --- /dev/null +++ b/plugins/linuxdisk/Makefile.am @@ -0,0 +1,78 @@ +# nbdkit +# Copyright (C) 2019 Red Hat Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Red Hat nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +include $(top_srcdir)/common-rules.mk + +EXTRA_DIST = nbdkit-linuxdisk-plugin.pod + +if HAVE_EXT2 + +plugin_LTLIBRARIES = nbdkit-linuxdisk-plugin.la + +nbdkit_linuxdisk_plugin_la_SOURCES = \ + filesystem.c \ + linuxdisk.c \ + partition-gpt.c \ + virtual-disk.c \ + virtual-disk.h \ + $(top_srcdir)/include/nbdkit-plugin.h + +nbdkit_linuxdisk_plugin_la_CPPFLAGS = \ + -I$(top_srcdir)/common/gpt \ + -I$(top_srcdir)/common/include \ + -I$(top_srcdir)/common/regions \ + -I$(top_srcdir)/common/utils \ + -I$(top_srcdir)/include +nbdkit_linuxdisk_plugin_la_CFLAGS = \ + $(WARNINGS_CFLAGS) \ + $(EXT2FS_CFLAGS) $(COM_ERR_CFLAGS) +nbdkit_linuxdisk_plugin_la_LIBADD = \ + $(top_builddir)/common/gpt/libgpt.la \ + $(top_builddir)/common/regions/libregions.la \ + $(top_builddir)/common/utils/libutils.la \ + $(EXT2FS_LIBS) $(COM_ERR_LIBS) +nbdkit_linuxdisk_plugin_la_LDFLAGS = \ + -module -avoid-version -shared \ + -Wl,--version-script=$(top_srcdir)/plugins/plugins.syms + +if HAVE_POD + +man_MANS = nbdkit-linuxdisk-plugin.1 +CLEANFILES += $(man_MANS) + +nbdkit-linuxdisk-plugin.1: nbdkit-linuxdisk-plugin.pod + $(PODWRAPPER) --section=1 --man $@ \ + --html $(top_builddir)/html/$@.html \ + $< + +endif HAVE_POD +endif diff --git a/plugins/linuxdisk/filesystem.c b/plugins/linuxdisk/filesystem.c new file mode 100644 index 0000000..f1d3a45 --- /dev/null +++ b/plugins/linuxdisk/filesystem.c @@ -0,0 +1,935 @@ +/* nbdkit + * Copyright (C) 2018-2019 Red Hat Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Red Hat nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <stdint.h> +#include <inttypes.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <dirent.h> +#include <errno.h> +#include <assert.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/sysmacros.h> + +/* Inlining is broken in the ext2fs header file. Disable it by + * defining the following: + */ +#define NO_INLINE_FUNCS +#include <ext2fs.h> + +#include <nbdkit-plugin.h> + +#include "get-current-dir-name.h" +#include "minmax.h" +#include "rounding.h" +#include "utils.h" + +#include "virtual-disk.h" + +int linuxdisk_debug_filesystem; /* -D linuxdisk.filesystem=1 */ + +/* Per-file information collected in the first pass. */ +struct file { + char *pathname; /* Full host path to the file. */ + char *name; /* Base name of the file. */ + ext2_ino_t dir_ino; /* Containing directory inode in target fs. */ + struct stat statbuf; /* File information. */ + + /* These fields are only populated once the file has been created + * in the target filesystem. + */ + ext2_ino_t ino; /* Inode. */ + int ino_flags; /* Inode flags. */ + const char *type; /* File type (used only for debugging). */ +}; + +static int64_t estimate_size (void); +static int mke2fs (const char *filename); +static int visit (const char *dir, ext2_filsys fs, ext2_ino_t dir_ino, + struct file **files, size_t *nr_files); +static int compare_inodes (const void *f1, const void *f2); +static int e2mkdir (ext2_filsys fs, ext2_ino_t dir_ino, const char *name, + const struct stat *statbuf, ext2_ino_t *ino); +static int e2emptyinode (ext2_filsys fs, ext2_ino_t dir_ino, + const char *name, const struct stat *statbuf, + int ino_flags, ext2_ino_t *ino); +static int e2copyfile (ext2_filsys fs, struct file *file); +static int e2hardlink (ext2_filsys fs, struct file *to, struct file *file); +static int e2link (ext2_filsys fs, ext2_ino_t dir_ino, const char *name, + ext2_ino_t ino, int flags); +static int e2copyfiledata (ext2_filsys fs, struct file *file); + +void +load_filesystem (void) +{ + initialize_ext2_error_table (); +} + +int +create_filesystem (struct virtual_disk *disk) +{ + const char *tmpdir; + char *filename; + errcode_t err; + ext2_filsys fs; + size_t i; + struct file *files = NULL; + size_t nr_files = 0; + struct file *last_file; + bool hardlinked; + + /* Estimate the filesystem size and compute the final virtual size + * of the disk. We only need to do this if the user didn't specify + * the exact size on the command line. + */ + if (size == 0 || size_add_estimate) { + int64_t estimate; + + estimate = estimate_size (); + if (estimate == -1) + return -1; + + nbdkit_debug ("filesystem size estimate: %" PRIi64, estimate); + + /* Add 20% to the estimate to account for the overhead of + * filesystem metadata. Also set a minimum size. Note we are + * only wasting virtual space (since this will be stored sparsely + * under $TMDIR) so we can be generous here. + * + * If we decide to create ext3/4 filesystems in future we will + * need to account for the journal here. + */ + estimate = estimate * 6 / 5; + estimate = MAX (estimate, 1024*1024); + + if (size_add_estimate) + size += estimate; + else + size = estimate; + } + + /* The minimum filesystem that libext2fs will let us create is 64M. + * Why?! Anyway it works if I do this, but not otherwise. XXX + */ + size = MAX (size, 64*1024*1024); + + /* Round the final size up to a whole number of sectors. */ + size = ROUND_UP (size, SECTOR_SIZE); + disk->filesystem_size = size; + + nbdkit_debug ("filesystem virtual size: %" PRIi64, size); + + /* Create the filesystem file. */ + tmpdir = getenv ("TMPDIR"); + if (tmpdir == NULL) + tmpdir = LARGE_TMPDIR; + if (asprintf (&filename, "%s/linuxdiskXXXXXX", tmpdir) == -1) { + nbdkit_error ("asprintf: %m"); + return -1; + } + + disk->fd = mkstemp (filename); + if (disk->fd == -1) { + nbdkit_error ("mkstemp: %s: %m", filename); + free (filename); + return -1; + } + if (ftruncate (disk->fd, size) == -1) { + nbdkit_error ("ftruncate: %s: %m", filename); + free (filename); + return -1; + } + + /* Create the filesystem. */ + if (mke2fs (filename) == -1) { + unlink (filename); + free (filename); + return -1; + } + + /* Open the filesystem. */ + err = ext2fs_open (filename, EXT2_FLAG_RW|EXT2_FLAG_64BITS, 0, 0, + unix_io_manager, &fs); + unlink (filename); + free (filename); + if (err) { + nbdkit_error ("ext2fs_open: %s", error_message (err)); + return -1; + } + + err = ext2fs_read_bitmaps (fs); + if (err) { + nbdkit_error ("ext2fs_read_bitmaps: %s", error_message (err)); + ext2fs_close (fs); + return -1; + } + + /* First pass: This creates subdirectories in the filesystem and + * also builds a list of files so we can identify hard links. + */ + for (i = 0; i < nr_dirs; ++i) { + if (visit (dirs[i], fs, EXT2_ROOT_INO, &files, &nr_files) == -1) + return -1; + } + + if (nr_files > 0) { + assert (files != NULL); + + /* Sort the files by device and inode number to identify hard links. */ + qsort (files, nr_files, sizeof (struct file), compare_inodes); + + /* Second pass: Copy/create the files. */ + last_file = NULL; + for (i = 0; i < nr_files; ++i) { + hardlinked = last_file && compare_inodes (last_file, &files[i]) == 0; + + if (!hardlinked) { + /* Normal case: creating a new inode. */ + last_file = &files[i]; + assert (!S_ISDIR (files[i].statbuf.st_mode)); + + if (e2copyfile (fs, &files[i]) == -1) + return -1; + + if (linuxdisk_debug_filesystem) + nbdkit_debug ("%s: <%" PRIu32 ">/%s -> <%" PRIu32 ">", + files[i].type, + files[i].dir_ino, files[i].name, files[i].ino); + } + else { + /* Creating a hard link to an existing inode. */ + if (e2hardlink (fs, last_file, &files[i]) == -1) + return -1; + + if (linuxdisk_debug_filesystem) + nbdkit_debug ("hard link: <%" PRIu32 ">/%s -> <%" PRIu32 ">", + files[i].dir_ino, files[i].name, last_file->ino); + } + } + + for (i = 0; i < nr_files; ++i) { + free (files[i].pathname); + free (files[i].name); + } + free (files); + } + + /* Close the filesystem. Note we don't bother to sync it because + * it's a private temporary file which only we will read. + */ + ext2fs_close2 (fs, EXT2_FLAG_FLUSH_NO_SYNC); + return 0; +} + +/* Use ‘du’ to estimate the size of the filesystem quickly. + * + * Typical output from ‘du -cs dir1 dir2’ is: + * + * 12345 dir1 + * 34567 dir2 + * 46912 total + * + * We ignore everything except the first number on the last line. + */ +static int64_t +estimate_size (void) +{ + char *command = NULL, *line = NULL; + size_t len = 0; + FILE *fp; + size_t i; + int64_t ret; + + /* Create the du command. */ + fp = open_memstream (&command, &len); + if (fp == NULL) { + nbdkit_error ("open_memstream: %m"); + return -1; + } + fprintf (fp, "du -c -k -s"); + for (i = 0; i < nr_dirs; ++i) { + fputc (' ', fp); + shell_quote (dirs[i], fp); + } + if (fclose (fp) == EOF) { + nbdkit_error ("memstream failed: %m"); + return -1; + } + + /* Run the command. */ + nbdkit_debug ("%s", command); + fp = popen (command, "r"); + free (command); + if (fp == NULL) { + nbdkit_error ("du command failed: %m"); + return -1; + } + + /* Ignore everything up to the last line. */ + len = 0; + while (getline (&line, &len, fp) != -1) + /* empty */; + if (ferror (fp)) { + nbdkit_error ("getline failed: %m"); + free (line); + fclose (fp); + return -1; + } + + fclose (fp); + + /* Parse the last line. */ + if (sscanf (line, "%" SCNi64, &ret) != 1 || ret < 0) { + nbdkit_error ("could not parse last line of output: %s", line); + free (line); + return -1; + } + free (line); + + /* Result is in 1K blocks, convert it to bytes. */ + ret *= 1024; + return ret; +} + +static int +mke2fs (const char *filename) +{ + char *command = NULL; + size_t len = 0; + FILE *fp; + int r; + + /* Create the mke2fs command. */ + fp = open_memstream (&command, &len); + if (fp == NULL) { + nbdkit_error ("open_memstream: %m"); + return -1; + } + + fprintf (fp, "mke2fs -t ext2 -F "); + if (label) { + fprintf (fp, "-L "); + shell_quote (label, fp); + fputc (' ', fp); + } + if (!linuxdisk_debug_filesystem) + fprintf (fp, "-q "); /* quiet unless extra debugging enabled */ + shell_quote (filename, fp); + + if (fclose (fp) == EOF) { + nbdkit_error ("memstream failed: %m"); + return -1; + } + + /* Run the command. */ + nbdkit_debug ("%s", command); + r = system (command); + free (command); + + if (WIFEXITED (r) && WEXITSTATUS (r) != 0) { + nbdkit_error ("mke2fs command failed with exit code %d", WEXITSTATUS (r)); + return -1; + } + else if (WIFSIGNALED (r)) { + nbdkit_error ("mke2fs command was killed by signal %d", WTERMSIG (r)); + return -1; + } + else if (WIFSTOPPED (r)) { + nbdkit_error ("mke2fs command was stopped by signal %d", WSTOPSIG (r)); + return -1; + } + + return 0; +} + +static int +visit (const char *dir, ext2_filsys fs, ext2_ino_t dir_ino, + struct file **files, size_t *nr_files) +{ + char *origdir; + DIR *DIR; + struct dirent *d; + struct stat statbuf; + int err; + char *subname = NULL; + + /* Because this is called from config_complete, before nbdkit + * daemonizes or starts any threads, it's safe to use chdir here and + * greatly simplifies the code. However we must chdir back to the + * original directory at the end. + */ + origdir = get_current_dir_name (); + if (origdir == NULL) { + nbdkit_error ("get_current_dir_name: %m"); + goto error0; + } + if (chdir (dir) == -1) { + nbdkit_error ("chdir: %s: %m", dir); + goto error1; + } + + DIR = opendir ("."); + if (DIR == NULL) { + nbdkit_error ("opendir: %s: %m", dir); + goto error1; + } + + while (errno = 0, (d = readdir (DIR)) != NULL) { + if (strcmp (d->d_name, ".") == 0 || + strcmp (d->d_name, "..") == 0) + continue; + + free (subname); + subname = NULL; + if (asprintf (&subname, "%s/%s", dir, d->d_name) == -1) { + nbdkit_error ("asprintf: %m"); + goto error2; + } + + if (lstat (d->d_name, &statbuf) == -1) { + nbdkit_error ("lstat: %s: %m", subname); + goto error2; + } + + if (S_ISDIR (statbuf.st_mode)) { + /* File type: Directory. */ + ext2_ino_t ino; + + /* Create the directory in the target filesystem. */ + if (e2mkdir (fs, dir_ino, d->d_name, &statbuf, &ino) == -1) + goto error2; + + if (linuxdisk_debug_filesystem) + nbdkit_debug ("mkdir: <%" PRIu32 ">/%s -> <%" PRIu32 ">", + dir_ino, d->d_name, ino); + + /* Visit the subdirectory. */ + if (visit (subname, fs, ino, files, nr_files) == -1) + goto error2; + } + else { + /* Any other non-directory is added to files. We will further + * process it in the second pass. + */ + struct file *new_files; + + new_files = realloc (*files, (*nr_files+1) * sizeof (struct file)); + if (new_files == NULL) { + nbdkit_error ("realloc: %m"); + goto error2; + } + *files = new_files; + new_files[*nr_files].name = strdup (d->d_name); + if (new_files[*nr_files].name == NULL) { + nbdkit_error ("strdup: %m"); + goto error2; + } + new_files[*nr_files].pathname = subname; + subname = NULL; /* pass ownership to struct */ + new_files[*nr_files].dir_ino = dir_ino; + new_files[*nr_files].statbuf = statbuf; + (*nr_files)++; + } + } + + /* Did readdir fail? */ + if (errno != 0) { + nbdkit_error ("readdir: %s: %m", dir); + goto error2; + } + + if (closedir (DIR) == -1) { + nbdkit_error ("closedir: %s: %m", dir); + goto error1; + } + + if (chdir (origdir) == -1) { + nbdkit_error ("chdir: %s: %m", origdir); + goto error1; + } + + free (origdir); + free (subname); + + return 0; + + error2: + closedir (DIR); + error1: + err = errno; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-result" + chdir (origdir); +#pragma GCC diagnostic pop + errno = err; + free (origdir); + error0: + free (subname); + return -1; +} + +/* To identify hard links we sort the files array by device and inode + * number (thus hard linked files will be next to each other after the + * sort). Hence this odd sorting function. + */ +static int +compare_inodes (const void *vp1, const void *vp2) +{ + const struct file *f1 = vp1; + const struct file *f2 = vp2; + + if (f1->statbuf.st_dev < f2->statbuf.st_dev) + return -1; + else if (f1->statbuf.st_dev > f2->statbuf.st_dev) + return 1; + else { + if (f1->statbuf.st_ino < f2->statbuf.st_ino) + return -1; + else if (f1->statbuf.st_ino > f2->statbuf.st_ino) + return 1; + else + return 0; + } +} + +static int +e2mkdir (ext2_filsys fs, ext2_ino_t dir_ino, const char *name, + const struct stat *statbuf, ext2_ino_t *ino) +{ + errcode_t err; + mode_t mode = LINUX_S_IFDIR | (statbuf->st_mode & 03777); + ext2_ino_t temp_ino; + struct ext2_inode inode; + + /* It's possible that the directory exists (eg. because the user has + * two identically named subdirectories in two directory + * parameters). If this happens, skip. + */ + err = ext2fs_namei (fs, EXT2_ROOT_INO, dir_ino, name, &temp_ino); + if (err == 0) + return 0; + + /* Create the new inode. */ + err = ext2fs_new_inode (fs, dir_ino, mode, 0, ino); + if (err) { + nbdkit_error ("ext2fs_new_inode: %s", error_message (err)); + return -1; + } + + again: + err = ext2fs_mkdir (fs, dir_ino, *ino, name); + if (err) { + /* http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=217892 */ + if (err == EXT2_ET_DIR_NO_SPACE) { + err = ext2fs_expand_dir (fs, dir_ino); + if (err) { + nbdkit_error ("ext2fs_expand_dir: %s", error_message (err)); + return -1; + } + goto again; + } + nbdkit_error ("ext2fs_mkdir: %s", error_message (err)); + return -1; + } + + /* Copy the permissions etc to the inode. */ + err = ext2fs_read_inode (fs, *ino, &inode); + if (err) { + nbdkit_error ("ext2fs_read_inode: %s", error_message (err)); + return -1; + } + inode.i_mode = mode; + inode.i_uid = statbuf->st_uid; + inode.i_gid = statbuf->st_gid; + inode.i_ctime = statbuf->st_ctime; + inode.i_atime = statbuf->st_atime; + inode.i_mtime = statbuf->st_mtime; + /* XXX nanosecond times? */ + err = ext2fs_write_inode (fs, *ino, &inode); + if (err) { + nbdkit_error ("ext2fs_write_inode: %s", error_message (err)); + return -1; + } + + return 0; +} + +/* A wrapper around ext2fs_new_inode. This creates an inode and a + * directory entry and does some other bookkeeping. This is + * sufficient for specials, but to create a complete regular file or + * symlink you have to do some other stuff. For directories, use + * e2mkdir instead. + */ +static int +e2emptyinode (ext2_filsys fs, ext2_ino_t dir_ino, + const char *name, const struct stat *statbuf, + int ino_flags, ext2_ino_t *ino) +{ + errcode_t err; + struct ext2_inode inode; + int major_, minor_; + + err = ext2fs_new_inode (fs, dir_ino, statbuf->st_mode, 0, ino); + if (err) { + nbdkit_error ("ext2fs_new_inode: %s", error_message (err)); + return -1; + } + + memset (&inode, 0, sizeof inode); + inode.i_mode = statbuf->st_mode; + inode.i_uid = statbuf->st_uid; + inode.i_gid = statbuf->st_gid; + inode.i_blocks = 0; + inode.i_links_count = 1; + /* XXX nanosecond times? */ + inode.i_ctime = statbuf->st_ctime; + inode.i_atime = statbuf->st_atime; + inode.i_mtime = statbuf->st_mtime; + inode.i_size = 0; + if (S_ISBLK (statbuf->st_mode) || S_ISCHR (statbuf->st_mode)) { + major_ = major (statbuf->st_rdev); + minor_ = minor (statbuf->st_rdev); + inode.i_block[0] = + (minor_ & 0xff) | (major_ << 8) | ((minor_ & ~0xff) << 12); + } + else + inode.i_block[0] = 0; + + err = ext2fs_write_new_inode (fs, *ino, &inode); + if (err) { + nbdkit_error ("ext2fs_write_inode: %s", error_message (err)); + return -1; + } + + if (e2link (fs, dir_ino, name, *ino, ino_flags) == -1) + return -1; + + ext2fs_inode_alloc_stats2 (fs, *ino, 1, 0); + + return 0; +} + +/* Copy a file (also symlink, special, etc) from the host to the target. */ +static int +e2copyfile (ext2_filsys fs, struct file *file) +{ + errcode_t err; + ext2_ino_t temp_ino; + + /* This cannot happen because we created directories in the first pass. */ + assert (!S_ISDIR (file->statbuf.st_mode)); + + /* It's possible that the file already exists, if the user has two + * identically named files in two directory parameters. + * + * Supermin overwrites the existing file in this case, but the code + * to do this is complex because it involves deleting the file we + * previously created (and perhaps freeing up its blocks). Supermin + * doesn't handle extended attrs or free them which would be even + * more complex. + * + * Therefore we make this a user error, forbidding the user from + * doing this. + */ + err = ext2fs_namei (fs, EXT2_ROOT_INO, file->dir_ino, file->name, &temp_ino); + if (err == 0) { + nbdkit_error ("linuxdisk: two identically named files were found in different subdirectories, which is forbidden: second filename %s", + file->pathname); + return -1; + } + + /* Regular file. */ + if (S_ISREG (file->statbuf.st_mode)) { + file->ino_flags = EXT2_FT_REG_FILE; + file->type = "regular file"; + if (e2emptyinode (fs, file->dir_ino, file->name, &file->statbuf, + file->ino_flags, &file->ino) == -1) + return -1; + + if (file->statbuf.st_size > 0) { + if (e2copyfiledata (fs, file) == -1) + return -1; + } + } + /* Symbolic link. */ + else if (S_ISLNK (file->statbuf.st_mode)) { + char *buf; + ssize_t r; + + file->ino_flags = EXT2_FT_SYMLINK; + file->type = "symbolic link"; + if (e2emptyinode (fs, file->dir_ino, file->name, &file->statbuf, + file->ino_flags, &file->ino) == -1) + return -1; + + buf = malloc (file->statbuf.st_size + 1); + if (buf == NULL) { + nbdkit_error ("malloc: %m"); + return -1; + } + r = readlink (file->pathname, buf, file->statbuf.st_size); + if (r == -1) { + nbdkit_error ("readlink: %s: %m", file->pathname); + free (buf); + return -1; + } + if (r > file->statbuf.st_size) + r = file->statbuf.st_size; + buf[r] = '\0'; + err = ext2fs_symlink (fs, file->dir_ino, file->ino, NULL, buf); + free (buf); + if (err) { + nbdkit_error ("ext2fs_symlink: %s", error_message (err)); + return -1; + } + } + /* Specials. */ + else if (S_ISBLK (file->statbuf.st_mode)) { + file->ino_flags = EXT2_FT_BLKDEV; + file->type = "block device"; + goto special; + } + else if (S_ISCHR (file->statbuf.st_mode)) { + file->ino_flags = EXT2_FT_CHRDEV; + file->type = "char device"; + goto special; + } + else if (S_ISFIFO (file->statbuf.st_mode)) { + file->ino_flags = EXT2_FT_FIFO; + file->type = "FIFO"; + goto special; + } + else if (S_ISSOCK (file->statbuf.st_mode)) { + file->ino_flags = EXT2_FT_SOCK; + file->type = "socket"; + special: + return e2emptyinode (fs, file->dir_ino, file->name, + &file->statbuf, file->ino_flags, &file->ino); + } + else { + /* Unknown host file type. Skip it but emit a debug message. */ + file->type = "unknown"; + nbdkit_debug ("ignoring unknown file type: %s mode %o", + file->pathname, (unsigned) file->statbuf.st_mode); + } + + return 0; +} + +/* Create a hard link. The new directory entry is ‘file’. The + * existing file/inode is ‘to’. + */ +static int +e2hardlink (ext2_filsys fs, struct file *to, struct file *file) +{ + errcode_t err; + struct ext2_inode inode; + + if (e2link (fs, file->dir_ino, file->name, to->ino, to->ino_flags) == -1) + return -1; + + /* We need to increase the link count in the inode. + * See e2fsprogs/misc/create_inode.c:add_link + */ + err = ext2fs_read_inode (fs, to->ino, &inode); + if (err) { + nbdkit_error ("ext2fs_read_inode: %s", error_message (err)); + return -1; + } + inode.i_links_count++; + err = ext2fs_write_inode (fs, to->ino, &inode); + if (err) { + nbdkit_error ("ext2fs_write_inode: %s", error_message (err)); + return -1; + } + + return 0; +} + +/* This is a helper wrapper around ext2fs_link which expands the + * parent directory if it fills up. + * + * The flags field is required even for hard links because the + * directory stores some data about the file type (returned by + * getdents(2)). + */ +static int +e2link (ext2_filsys fs, ext2_ino_t dir_ino, const char *name, + ext2_ino_t ino, int flags) +{ + errcode_t err; + + again: + err = ext2fs_link (fs, dir_ino, name, ino, flags); + if (err) { + if (err == EXT2_ET_DIR_NO_SPACE) { + err = ext2fs_expand_dir (fs, dir_ino); + if (err) { + nbdkit_error ("ext2fs_expand_dir: %s", error_message (err)); + return -1; + } + goto again; + } + nbdkit_error ("ext2fs_link: %s", error_message (err)); + return -1; + } + + return 0; +} + +/* For regular files, this copies the file data blocks. The inode has + * already been created and linked into the filesystem. + */ +static int +e2copyfiledata (ext2_filsys fs, struct file *file) +{ + int fd = -1; + errcode_t err; + ext2_file_t f; + char buf[BUFSIZ]; + size_t n; + ssize_t r; + unsigned int written; + off_t offset, data_start, data_end; + + fd = open (file->pathname, O_RDONLY); + if (fd == -1) { + /* Supermin skips unreadable files. Here we disallow that. + * However we should probably add options to make this behaviour + * controllable. XXX + */ + nbdkit_error ("open: %s: %m", file->pathname); + goto error; + } + + err = ext2fs_file_open2 (fs, file->ino, NULL, EXT2_FILE_WRITE, &f); + if (err) { + nbdkit_error ("ext2fs_file_open2: %s", error_message (err)); + goto error; + } + + offset = 0; + do { + data_start = lseek (fd, offset, SEEK_DATA); + if (data_start == -1) { + nbdkit_error ("lseek: %s: SEEK_DATA: %m", file->pathname); + goto error; + } + data_end = lseek (fd, data_start, SEEK_HOLE); + if (data_end == -1) { + nbdkit_error ("lseek: %s: SEEK_HOLE: %m", file->pathname); + goto error; + } + + if (linuxdisk_debug_filesystem) + nbdkit_debug ("%s: data region [%" PRIu64 "..%" PRIu64 "]", + file->pathname, + (uint64_t) data_start, (uint64_t) (data_end-1)); + + /* Copy data from [data_start..data_end-1]. */ + n = data_end - data_start; + + if (lseek (fd, data_start, SEEK_SET) == -1) { + nbdkit_error ("lseek: %s: %m", file->pathname); + goto error; + } + err = ext2fs_file_llseek (f, data_start, EXT2_SEEK_SET, NULL); + if (err) { + nbdkit_error ("ext2fs_file_lseek: %s", error_message (err)); + goto error; + } + + while (n > 0) { + r = read (fd, buf, MIN (n, sizeof buf)); + if (r == -1) { + nbdkit_error ("read: %s: %m", file->pathname); + goto error; + } + if (r == 0) { + nbdkit_error ("read: %s: unexpected end of file", file->pathname); + goto error; + } + + err = ext2fs_file_write (f, buf, r, &written); + if (err) { + nbdkit_error ("ext2fs_file_write: %s", error_message (err)); + goto error; + } + if ((ssize_t) written != r) { + nbdkit_error ("ext2fs_file_write: " + "requested write size != bytes written"); + goto error; + } + + n -= written; + } + + offset = data_end; + } while (offset < file->statbuf.st_size); + + if (close (fd) == -1) { + nbdkit_error ("close: %s: %m", file->pathname); + fd = -1; + goto error; + } + fd = -1; + + /* Set the true size of the file in the inode. This should handle + * the case of a sparse file with a hole at the end. + */ + err = ext2fs_file_set_size2 (f, file->statbuf.st_size); + if (err) { + nbdkit_error ("ext2fs_file_set_size2: %s", error_message (err)); + goto error; + } + + err = ext2fs_file_close (f); + if (err) { + nbdkit_error ("ext2fs_file_close: %s", error_message (err)); + goto error; + } + + return 0; + + error: + if (fd >= 0) + close (fd); + return -1; +} diff --git a/plugins/linuxdisk/linuxdisk.c b/plugins/linuxdisk/linuxdisk.c new file mode 100644 index 0000000..36781db --- /dev/null +++ b/plugins/linuxdisk/linuxdisk.c @@ -0,0 +1,236 @@ +/* nbdkit + * Copyright (C) 2019 Red Hat Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Red Hat nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <inttypes.h> +#include <string.h> +#include <unistd.h> +#include <time.h> + +#define NBDKIT_API_VERSION 2 + +#include <nbdkit-plugin.h> + +#include "random.h" +#include "regions.h" + +#include "virtual-disk.h" + +/* Directory, label, size parameters. */ +char **dirs; +size_t nr_dirs; +const char *label; +int64_t size; +bool size_add_estimate; /* if size=+SIZE was used */ + +/* Virtual disk. */ +static struct virtual_disk disk; + +/* Used to create a random GUID for the partition. */ +struct random_state random_state; + +static void +linuxdisk_load (void) +{ + load_filesystem (); + init_virtual_disk (&disk); + xsrandom (time (NULL), &random_state); +} + +static void +linuxdisk_unload (void) +{ + size_t i; + + free_virtual_disk (&disk); + + for (i = 0; i < nr_dirs; ++i) + free (dirs[i]); + free (dirs); +} + +static int +linuxdisk_config (const char *key, const char *value) +{ + if (strcmp (key, "dir") == 0) { + char **new_dirs; + char *dir; + + dir = nbdkit_realpath (value); + if (dir == NULL) + return -1; + + new_dirs = realloc (dirs, (nr_dirs+1) * sizeof (char *)); + if (new_dirs == NULL) { + nbdkit_error ("realloc: %m"); + free (dir); + return -1; + } + dirs = new_dirs; + dirs[nr_dirs] = dir; + nr_dirs++; + } + else if (strcmp (key, "label") == 0) { + label = value; + } + else if (strcmp (key, "size") == 0) { + if (value[0] == '+') { + size_add_estimate = true; + value++; + } + else + size_add_estimate = false; + size = nbdkit_parse_size (value); + if (size == -1) + return -1; + } + else { + nbdkit_error ("unknown parameter '%s'", key); + return -1; + } + + return 0; +} + +static int +linuxdisk_config_complete (void) +{ + if (nr_dirs == 0) { + nbdkit_error ("you must supply the dir=<DIRECTORY> parameter " + "after the plugin name on the command line"); + return -1; + } + + return create_virtual_disk ((const char **) dirs, nr_dirs, label, + size, size_add_estimate, + &disk); +} + +#define linuxdisk_config_help \ + "dir=<DIRECTORY> (required) The directory to serve.\n" \ + "label=<LABEL> The filesystem label.\n" \ + "size=[+]<SIZE> The virtual filesystem size." + +static void * +linuxdisk_open (int readonly) +{ + return NBDKIT_HANDLE_NOT_NEEDED; +} + +#define THREAD_MODEL NBDKIT_THREAD_MODEL_PARALLEL + +/* Get the file size. */ +static int64_t +linuxdisk_get_size (void *handle) +{ + return virtual_size (&disk.regions); +} + +/* Serves the same data over multiple connections. */ +static int +linuxdisk_can_multi_conn (void *handle) +{ + return 1; +} + +/* Read data from the virtual disk. */ +static int +linuxdisk_pread (void *handle, void *buf, uint32_t count, uint64_t offset, + uint32_t flags) +{ + while (count > 0) { + const struct region *region = find_region (&disk.regions, offset); + size_t len; + ssize_t r; + + /* Length to end of region. */ + len = region->end - offset + 1; + if (len > count) + len = count; + + switch (region->type) { + case region_file: + /* We don't use region->u.i since there is only one backing + * file, and we have that open already (in ‘disk.fd’). + */ + r = pread (disk.fd, buf, len, offset - region->start); + if (r == -1) { + nbdkit_error ("pread: %m"); + return -1; + } + if (r == 0) { + nbdkit_error ("pread: unexpected end of file"); + return -1; + } + len = r; + break; + + case region_data: + memcpy (buf, &region->u.data[offset - region->start], len); + break; + + case region_zero: + memset (buf, 0, len); + break; + } + + count -= len; + buf += len; + offset += len; + } + + return 0; +} + +static struct nbdkit_plugin plugin = { + .name = "linuxdisk", + .longname = "nbdkit Linux virtual disk plugin", + .version = PACKAGE_VERSION, + .load = linuxdisk_load, + .unload = linuxdisk_unload, + .config = linuxdisk_config, + .config_complete = linuxdisk_config_complete, + .config_help = linuxdisk_config_help, + .magic_config_key = "dir", + .open = linuxdisk_open, + .get_size = linuxdisk_get_size, + .can_multi_conn = linuxdisk_can_multi_conn, + .pread = linuxdisk_pread, + .errno_is_preserved = 1, +}; + +NBDKIT_REGISTER_PLUGIN(plugin) diff --git a/plugins/linuxdisk/nbdkit-linuxdisk-plugin.pod b/plugins/linuxdisk/nbdkit-linuxdisk-plugin.pod new file mode 100644 index 0000000..5777b82 --- /dev/null +++ b/plugins/linuxdisk/nbdkit-linuxdisk-plugin.pod @@ -0,0 +1,197 @@ +=head1 NAME + +nbdkit-linuxdisk-plugin - create virtual Linux disk from directory + +=head1 SYNOPSIS + + nbdkit linuxdisk [dir=]DIRECTORY [[dir=]DIRECTORY ...] + [label=LABEL] [size=[+]SIZE] + +=head1 DESCRIPTION + +C<nbdkit-linuxdisk-plugin> is a plugin for L<nbdkit(1)> which creates +an ext2-formatted disk image from a directory on the fly. The files +in the specified directory (and subdirectories) appear in the virtual +disk, which is served read-only over the NBD protocol. + +The virtual disk is partitioned with a single GPT partition containing +the filesystem. + +The virtual disk can be used as a Linux root (or other) filesystem. +Most features of Linux filesystems are supported, such as hard links, +symbolic links, block special devices etc. Multiple directories can +be given on the command line and they are merged into the final +filesystem. + +=head1 EXAMPLES + +=over 4 + +=item nbdkit linuxdisk /path/to/directory label=ROOTFS + +Create a virtual disk, giving it a filesystem label. Note that +clients will not be able to modify the filesystem, so it is safe to +share it with multiple clients. + +=item nbdkit --filter=cow linuxdisk /path/to/directory + +Add a writable overlay (see L<nbdkit-cow-filter(1)>, allowing the disk +to be written by the client. B<Multiple clients must not be allowed +to connect at the same time> (even if they all mount it read-only) as +this will cause disk corruption. + +=item nbdkit --filter=cow linuxdisk /path/to/directory size=+1G + +The same but specifying that at least 1G of free space should be +available in the filesystem (not including the space taken by the +initial filesystem). + +=item nbdkit --filter=partition linuxdisk /path/to/directory partition=1 + +Instead of serving a partitioned disk image, serve just the "naked" +filesystem (ie. the first partition, see +L<nbdkit-partition-filter(1)>). + +=item nbdkit -U - linuxdisk /path/to/directory +--run 'qemu-img convert $nbd ext2fs.img' + +This serves nothing. Instead it turns a directory into an ext2 +filesystem image, writing it to F<ext2fs.img> (see +L<nbdkit-captive(1)>). + +The resulting image is a partitioned disk. If you want just the naked +filesystem then add the partition filter as in the previous example. + +=item nbdkit linuxdisk /path/to/dir1 /path/to/dir2 + +Merge two directories into a single disk image. Files in F<dir1> and +F<dir2> will appear in the root directory of the merged disk image. +(About duplicates appearing in both directories, see L</NOTES> below). + +=back + +=head1 PARAMETERS + +=over 4 + +=item [B<dir=>]DIRECTORY + +Specify the directory containing files and subdirectories which will +be added to the virtual disk. Files inside this directory will appear +in the root directory of the virtual disk. + +This parameter is required at least once. If it is given multiple +times then the directories are merged into the final filesystem. + +C<dir=> is a magic config key and may be omitted in most cases. +See L<nbdkit(1)/Magic parameters>. + +=item B<label=>LABEL + +The optional label for the filesystem. + +=item B<size=>SIZE + +=item B<size=+>SIZE + +The total (virtual) size of the filesystem. + +If the C<size> parameter is omitted the plugin will try to size the +filesystem with just enough space to contain the files and directories +that are initially loaded, and there will not be much extra space. + +Using C<size=SIZE> specifies the required virtual size of the whole +filesystem (including initial files and extra space). If this is set +too small for the initial filesystem then the plugin will fail to +start. + +Using C<size=+SIZE> specifies the minimum free space required after +the initial filesystem has been loaded. (The actual free space might +be slightly larger). + +=back + +=head1 NOTES + +=head2 Hard links + +This plugin preserves hard links. That is, if two or more files are +hard linked in the source directory, they will be hard linked in the +resulting filesystem. + +=head2 Symbolic links + +Symbolic links are copied exactly. This means they might not point to +the same file, or to any file, once copied to the filesystem. + +=head2 Sparse files + +This plugin preserves sparseness in files to the extent possible. It +might not always be possible depending on the filesystem block size. +It does not create sparse files if the source file is not sparse. + +=head2 Block/char special devices, FIFOs, sockets, etc. + +These are copied into (ie. recreated in) the filesystem exactly as far +as possible. + +=head2 Users, groups, permissions, file times + +The original file UID, GID, permissions and file times are recreated +as far as possible. Note that UIDs/GIDs will likely map to different +users and groups when read by a virtual machine or other NBD client +machine. + +=head2 Duplicate directories allowed, duplicate files forbidden + +When specifying multiple source directories on the command line: + + nbdkit linuxdisk dir1 dir2 + +it is possible that there will be duplicates in the target filesystem, +eg. F<dir1/dir> and F<dir2/dir> both requiring F</dir> to be created +on the target. For directories, this is permitted. The permissions, +ownership etc of the first directory will be used. + +For files this is not allowed (even if the files are identical). The +plugin will give an error in this case. + +=head1 UNIMPLEMENTED + +=head2 Extended attributes + +Extended attributes are not yet implemented, which means (amongst +other things) that SELinux labels will not be copied to the target +filesystem. This should be fixed in future. + +=head1 ENVIRONMENT VARIABLES + +=over 4 + +=item C<TMPDIR> + +The filesystem image is stored in a temporary file located in +F</var/tmp> by default. You can override this location by setting the +C<TMPDIR> environment variable before starting nbdkit. + +=back + +=head1 SEE ALSO + +L<nbdkit(1)>, +L<nbdkit-plugin(3)>, +L<nbdkit-captive(1)>, +L<nbdkit-cow-filter(1)>, +L<nbdkit-file-plugin(1)>, +L<nbdkit-floppy-plugin(1)>, +L<nbdkit-iso-plugin(1)>, +L<nbdkit-partition-filter(1)>, +L<nbdkit-partitioning-plugin(1)>. + +=head1 AUTHORS + +Richard W.M. Jones + +=head1 COPYRIGHT + +Copyright (C) 2019 Red Hat Inc. diff --git a/plugins/linuxdisk/partition-gpt.c b/plugins/linuxdisk/partition-gpt.c new file mode 100644 index 0000000..3278229 --- /dev/null +++ b/plugins/linuxdisk/partition-gpt.c @@ -0,0 +1,211 @@ +/* nbdkit + * Copyright (C) 2018-2019 Red Hat Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Red Hat nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <stdint.h> +#include <inttypes.h> +#include <string.h> +#include <unistd.h> + +#include <nbdkit-plugin.h> + +#include "efi-crc32.h" +#include "gpt.h" +#include "isaligned.h" +#include "rounding.h" +#include "regions.h" + +#include "virtual-disk.h" + +#define PARTITION_TYPE_GUID "0FC63DAF-8483-4772-8E79-3D69D8477DE4" + +static void create_gpt_protective_mbr (struct virtual_disk *disk, + unsigned char *out); +static void create_gpt_partition_header (struct virtual_disk *disk, + const void *pt, bool is_primary, + unsigned char *out); +static void create_gpt_partition_table (struct virtual_disk *disk, + unsigned char *out); + +/* Initialize the partition table structures. */ +int +create_partition_table (struct virtual_disk *disk) +{ + create_gpt_protective_mbr (disk, disk->protective_mbr); + + create_gpt_partition_table (disk, disk->pt); + + create_gpt_partition_header (disk, disk->pt, true, disk->primary_header); + create_gpt_partition_header (disk, disk->pt, false, disk->secondary_header); + + return 0; +} + +static void +chs_too_large (unsigned char *out) +{ + const int c = 1023, h = 254, s = 63; + + out[0] = h; + out[1] = (c & 0x300) >> 2 | s; + out[2] = c & 0xff; +} + +static void +create_mbr_partition_table_entry (const struct region *region, + bool bootable, int partition_id, + unsigned char *out) +{ + uint64_t start_sector, nr_sectors; + uint32_t u32; + + assert (IS_ALIGNED (region->start, SECTOR_SIZE)); + + start_sector = region->start / SECTOR_SIZE; + nr_sectors = DIV_ROUND_UP (region->len, SECTOR_SIZE); + + assert (start_sector <= UINT32_MAX); + assert (nr_sectors <= UINT32_MAX); + + out[0] = bootable ? 0x80 : 0; + chs_too_large (&out[1]); + out[4] = partition_id; + chs_too_large (&out[5]); + u32 = htole32 (start_sector); + memcpy (&out[8], &u32, 4); + u32 = htole32 (nr_sectors); + memcpy (&out[12], &u32, 4); +} + +static void +create_gpt_protective_mbr (struct virtual_disk *disk, unsigned char *out) +{ + struct region region; + uint64_t end; + + /* Protective MBR creates an MBR partition with partition ID 0xee + * which covers the whole of the disk, or as much of the disk as + * expressible with MBR. + */ + region.start = 512; + end = virtual_size (&disk->regions) - 1; + if (end > UINT32_MAX * SECTOR_SIZE) + end = UINT32_MAX * SECTOR_SIZE; + region.end = end; + region.len = region.end - region.start + 1; + + create_mbr_partition_table_entry (&region, false, 0xee, &out[0x1be]); + + /* Boot sector signature. */ + out[0x1fe] = 0x55; + out[0x1ff] = 0xaa; +} + +static void +create_gpt_partition_header (struct virtual_disk *disk, + const void *pt, bool is_primary, + unsigned char *out) +{ + uint64_t nr_lbas; + struct gpt_header *header = (struct gpt_header *) out; + + nr_lbas = virtual_size (&disk->regions) / SECTOR_SIZE; + + memset (header, 0, sizeof *header); + memcpy (header->signature, GPT_SIGNATURE, sizeof (header->signature)); + memcpy (header->revision, GPT_REVISION, sizeof (header->revision)); + header->header_size = htole32 (sizeof *header); + if (is_primary) { + header->current_lba = htole64 (1); + header->backup_lba = htole64 (nr_lbas - 1); + } + else { + header->current_lba = htole64 (nr_lbas - 1); + header->backup_lba = htole64 (1); + } + header->first_usable_lba = htole64 (34); + header->last_usable_lba = htole64 (nr_lbas - 34); + if (is_primary) + header->partition_entries_lba = htole64 (2); + else + header->partition_entries_lba = htole64 (nr_lbas - 33); + header->nr_partition_entries = htole32 (GPT_MIN_PARTITIONS); + header->size_partition_entry = htole32 (GPT_PT_ENTRY_SIZE); + header->crc_partitions = + htole32 (efi_crc32 (pt, GPT_PT_ENTRY_SIZE * GPT_MIN_PARTITIONS)); + + /* Must be computed last. */ + header->crc = htole32 (efi_crc32 (header, sizeof *header)); +} + +static void +create_gpt_partition_table_entry (const struct region *region, + bool bootable, + char partition_type_guid[16], + char guid[16], + unsigned char *out) +{ + struct gpt_entry *entry = (struct gpt_entry *) out; + + assert (sizeof (struct gpt_entry) == GPT_PT_ENTRY_SIZE); + + memcpy (entry->partition_type_guid, partition_type_guid, 16); + memcpy (entry->unique_guid, guid, 16); + + entry->first_lba = htole64 (region->start / SECTOR_SIZE); + entry->last_lba = htole64 (region->end / SECTOR_SIZE); + entry->attributes = htole64 (bootable ? 4 : 0); +} + +static void +create_gpt_partition_table (struct virtual_disk *disk, unsigned char *out) +{ + size_t j; + + for (j = 0; j < nr_regions (&disk->regions); ++j) { + const struct region *region = get_region (&disk->regions, j); + + /* Find the (only) partition region, which has type region_file. */ + if (region->type == region_file) { + create_gpt_partition_table_entry (region, true, + PARTITION_TYPE_GUID, + disk->guid, + out); + out += GPT_PT_ENTRY_SIZE; + } + } +} diff --git a/plugins/linuxdisk/virtual-disk.c b/plugins/linuxdisk/virtual-disk.c new file mode 100644 index 0000000..915dc72 --- /dev/null +++ b/plugins/linuxdisk/virtual-disk.c @@ -0,0 +1,163 @@ +/* nbdkit + * Copyright (C) 2018-2019 Red Hat Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Red Hat nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <stdint.h> +#include <inttypes.h> +#include <string.h> +#include <unistd.h> + +#include <nbdkit-plugin.h> + +#include "random.h" +#include "regions.h" + +#include "virtual-disk.h" + +static int create_regions (struct virtual_disk *disk); + +void +init_virtual_disk (struct virtual_disk *disk) +{ + memset (disk, 0, sizeof *disk); + disk->fd = -1; + + init_regions (&disk->regions); +} + +int +create_virtual_disk (const char **dirs, size_t nr_dirs, const char *label, + int64_t size, bool size_add_estimate, + struct virtual_disk *disk) +{ + size_t i; + + /* Allocate the partition table structures. We can't fill them in + * until we have created the disk layout. + */ + disk->protective_mbr = calloc (1, SECTOR_SIZE); + disk->primary_header = calloc (1, SECTOR_SIZE); + disk->pt = calloc (1, 32*SECTOR_SIZE); + disk->secondary_header = calloc (1, SECTOR_SIZE); + if (disk->protective_mbr == NULL || + disk->primary_header == NULL || + disk->pt == NULL || + disk->secondary_header == NULL) { + nbdkit_error ("calloc: %m"); + return -1; + } + + /* Create the filesystem. This fills in disk->filesystem_size and + * disk->id. + */ + if (create_filesystem (disk) == -1) + return -1; + + /* Create a random GUID used as "Unique partition GUID". However + * this doesn't follow GUID conventions so in theory could make an + * invalid value. + */ + for (i = 0; i < 16; ++i) + disk->guid[i] = xrandom (&random_state) & 0xff; + + /* Create the virtual disk regions. */ + if (create_regions (disk) == -1) + return -1; + + /* Initialize partition table structures. This depends on + * disk->regions so must be done last. + */ + if (create_partition_table (disk) == -1) + return -1; + + return 0; +} + +void +free_virtual_disk (struct virtual_disk *disk) +{ + free_regions (&disk->regions); + free (disk->protective_mbr); + free (disk->primary_header); + free (disk->pt); + free (disk->secondary_header); + if (disk->fd >= 0) + close (disk->fd); +} + +/* Lay out the final disk. */ +static int +create_regions (struct virtual_disk *disk) +{ + /* Protective MBR. */ + if (append_region_len (&disk->regions, "Protective MBR", + SECTOR_SIZE, 0, 0, + region_data, (void *) disk->protective_mbr) == -1) + return -1; + + /* GPT primary partition table header (LBA 1). */ + if (append_region_len (&disk->regions, "GPT primary header", + SECTOR_SIZE, 0, 0, + region_data, (void *) disk->primary_header) == -1) + return -1; + + /* GPT primary PT (LBA 2..33). */ + if (append_region_len (&disk->regions, "GPT primary PT", + 32*SECTOR_SIZE, 0, 0, + region_data, (void *) disk->pt) == -1) + return -1; + + /* Partition containing the filesystem. Align it to 2048 sectors. */ + if (append_region_len (&disk->regions, "Filesystem", + disk->filesystem_size, 2048*SECTOR_SIZE, 0, + region_file, 0 /* unused */) == -1) + return -1; + + /* GPT secondary PT (LBA -33..-2). */ + if (append_region_len (&disk->regions, "GPT secondary PT", + 32*SECTOR_SIZE, SECTOR_SIZE, 0, + region_data, (void *) disk->pt) == -1) + return -1; + + /* GPT secondary PT header (LBA -1). */ + if (append_region_len (&disk->regions, "GPT secondary header", + SECTOR_SIZE, 0, 0, + region_data, (void *) disk->secondary_header) == -1) + return -1; + + return 0; +} diff --git a/plugins/linuxdisk/virtual-disk.h b/plugins/linuxdisk/virtual-disk.h new file mode 100644 index 0000000..ae0a70e --- /dev/null +++ b/plugins/linuxdisk/virtual-disk.h @@ -0,0 +1,96 @@ +/* nbdkit + * Copyright (C) 2018-2019 Red Hat Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Red Hat nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef NBDKIT_VIRTUAL_DISK_H +#define NBDKIT_VIRTUAL_DISK_H + +#include <stdbool.h> +#include <stdint.h> + +#include "regions.h" + +extern char **dirs; +extern size_t nr_dirs; +extern const char *label; +extern int64_t size; +extern bool size_add_estimate; + +extern struct random_state random_state; + +#define SECTOR_SIZE 512 + +struct virtual_disk { + /* Virtual disk layout. */ + struct regions regions; + + /* Disk protective MBR. */ + uint8_t *protective_mbr; + + /* GPT primary partition table header. */ + uint8_t *primary_header; + + /* GPT primary and secondary (backup) PTs. These are the same. */ + uint8_t *pt; + + /* GPT secondary (backup) PT header. */ + uint8_t *secondary_header; + + /* Size of the filesystem in bytes. */ + uint64_t filesystem_size; + + /* Unique partition GUID. */ + char guid[16]; + + /* File descriptor of the temporary file containing the filesystem. */ + int fd; +}; + +/* virtual-disk.c */ +extern void init_virtual_disk (struct virtual_disk *disk) + __attribute__((__nonnull__ (1))); +extern int create_virtual_disk (const char **dirs, size_t nr_dirs, + const char *label, + int64_t size, bool size_add_estimate, + struct virtual_disk *disk) + __attribute__((__nonnull__ (1, 6))); +extern void free_virtual_disk (struct virtual_disk *disk) + __attribute__((__nonnull__ (1))); + +/* partition-gpt.c */ +extern int create_partition_table (struct virtual_disk *disk); + +/* filesystem.c */ +extern void load_filesystem (void); +extern int create_filesystem (struct virtual_disk *disk); + +#endif /* NBDKIT_VIRTUAL_DISK_H */ diff --git a/plugins/partitioning/nbdkit-partitioning-plugin.pod b/plugins/partitioning/nbdkit-partitioning-plugin.pod index f3d6996..d045e4c 100644 --- a/plugins/partitioning/nbdkit-partitioning-plugin.pod +++ b/plugins/partitioning/nbdkit-partitioning-plugin.pod @@ -19,8 +19,8 @@ If you just want to concatenate files together (without adding a partition table) use L<nbdkit-split-plugin(1)>. If you want to select a single partition from an existing disk, use L<nbdkit-partition-filter(1)>. If you want to create a complete disk -with a filesystem, look at L<nbdkit-floppy-plugin(1)> or -L<nbdkit-iso-plugin(1)>. +with a filesystem, look at L<nbdkit-floppy-plugin(1)>, +L<nbdkit-iso-plugin(1)> or L<nbdkit-linuxdisk-plugin(1)>. The plugin supports read/write access. To limit clients to read-only access use the I<-r> flag. @@ -172,6 +172,7 @@ L<nbdkit(1)>, L<nbdkit-file-plugin(1)>, L<nbdkit-floppy-plugin(1)>, L<nbdkit-iso-plugin(1)>, +L<nbdkit-linuxdisk-plugin(1)>, L<nbdkit-partition-filter(1)>, L<nbdkit-split-plugin(1)>, L<nbdkit-plugin(3)>. diff --git a/tests/Makefile.am b/tests/Makefile.am index c75a9de..6a49e23 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -71,6 +71,8 @@ EXTRA_DIST = \ test-ip.sh \ test-iso.sh \ test-layers.sh \ + test-linuxdisk.sh \ + test-linuxdisk-copy-out.sh \ test-log.sh \ test.lua \ test-memory-largest.sh \ @@ -447,6 +449,15 @@ TESTS += test-iso.sh endif HAVE_GUESTFISH endif HAVE_ISO +# linuxdisk plugin test. +if HAVE_EXT2 +if HAVE_GUESTFISH +TESTS += \ + test-linuxdisk.sh \ + test-linuxdisk-copy-out.sh +endif HAVE_GUESTFISH +endif HAVE_EXT2 + # memory plugin test. LIBGUESTFS_TESTS += test-memory TESTS += test-memory-largest.sh test-memory-largest-for-qemu.sh diff --git a/tests/test-linuxdisk-copy-out.sh b/tests/test-linuxdisk-copy-out.sh new file mode 100755 index 0000000..1b1af56 --- /dev/null +++ b/tests/test-linuxdisk-copy-out.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# nbdkit +# Copyright (C) 2018-2019 Red Hat Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Red Hat nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +# Test the linuxdisk plugin with captive nbdkit, as described +# in the man page. + +source ./functions.sh +set -e +set -x + +requires qemu-img --version + +files="linuxdisk-copy-out.img + linuxdisk-copy-out.test1 linuxdisk-copy-out.test2 + linuxdisk-copy-out.test3 linuxdisk-copy-out.test4" +rm -f $files +cleanup_fn rm -f $files + +nbdkit -f -v -D linuxdisk.filesystem=1 -U - \ + --filter=partition \ + linuxdisk $srcdir/../plugins partition=1 label=ROOT \ + --run 'qemu-img convert $nbd linuxdisk-copy-out.img' + +# Check the disk content. +guestfish --ro -a linuxdisk-copy-out.img -m /dev/sda <<EOF +# Check some known files and directories exist. + ll / + ll /linuxdisk + is-dir /linuxdisk + is-file /linuxdisk/Makefile.am + +# This reads out all the directory entries and all file contents. + tar-out / - | cat >/dev/null + +# Download some files and compare to local copies. + download /linuxdisk/Makefile linuxdisk-copy-out.test1 + download /linuxdisk/Makefile.am linuxdisk-copy-out.test2 + download /linuxdisk/nbdkit-linuxdisk-plugin.pod linuxdisk-copy-out.test3 + download /linuxdisk/filesystem.c linuxdisk-copy-out.test4 +EOF + +# Compare downloaded files to local versions. +cmp linuxdisk-copy-out.test1 $srcdir/../plugins/linuxdisk/Makefile +cmp linuxdisk-copy-out.test2 $srcdir/../plugins/linuxdisk/Makefile.am +cmp linuxdisk-copy-out.test3 $srcdir/../plugins/linuxdisk/nbdkit-linuxdisk-plugin.pod +cmp linuxdisk-copy-out.test4 $srcdir/../plugins/linuxdisk/filesystem.c diff --git a/tests/test-linuxdisk.sh b/tests/test-linuxdisk.sh new file mode 100755 index 0000000..02e0437 --- /dev/null +++ b/tests/test-linuxdisk.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# nbdkit +# Copyright (C) 2018-2019 Red Hat Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Red Hat nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +# Test the linuxdisk plugin. + +source ./functions.sh +set -e +set -x + +requires mkfifo --version + +d=linuxdisk.d +rm -rf $d +cleanup_fn rm -rf $d + +# Create a test directory with some regular files, subdirectories and +# special files. +mkdir $d +mkfifo $d/fifo +mkdir $d/sub +cp $srcdir/Makefile.am $d/sub/Makefile.am +ln $d/sub/Makefile.am $d/sub/hardlink +ln -s $d/sub/Makefile.am $d/sub/symlink + +# It would be nice to use the Unix domain socket to test that the +# socket gets created, but in fact that won't work because this socket +# isn't created until after the plugin creates the virtual disk. +start_nbdkit -P $d/linuxdisk.pid \ + -U $d/linuxdisk.sock \ + -D linuxdisk.filesystem=1 \ + linuxdisk $d + +# Check the disk content. +guestfish --ro --format=raw -a "nbd://?socket=$PWD/$d/linuxdisk.sock" -m /dev/sda1 <<EOF + ll / + ll /sub + +# Check regular files exist. + is-file /sub/Makefile.am + is-file /sub/hardlink +# XXX Test sparse files in future. + +# Check the specials exist. + is-fifo /fifo + is-symlink /sub/symlink +# XXX Test sockets, etc. in future. + +# Check hard linked files. + lstatns /sub/Makefile.am | cat > $d/nlink.1 + lstatns /sub/hardlink | cat > $d/nlink.2 + +# This reads out all the directory entries and all file contents. + tar-out / - | cat >/dev/null + +# Download file and compare to local copy. + download /sub/Makefile.am $d/Makefile.am +EOF + +# Check the two hard linked files have st_nlink == 2. +grep "st_nlink: 2" $d/nlink.1 +grep "st_nlink: 2" $d/nlink.2 + +# Compare downloaded file to local version. +cmp $d/Makefile.am $srcdir/Makefile.am -- 1.8.3.1