[Libguestfs] [PATCH libnbd] PROPOSAL Add nbdcp (NBD copying) tool.

--- .gitignore | 2 + Makefile.am | 1 + TODO | 6 -- configure.ac | 1 + cp/Makefile.am | 52 +++++++++++++++ cp/nbdcp.c | 84 +++++++++++++++++++++++ cp/nbdcp.pod | 177 +++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 317 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 7536021..38923db 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,8 @@ Makefile.in /config.status /config.sub /configure +/cp/nbdcp +/cp/nbdcp.1 /depcomp /docs/*.1 /docs/*.3 diff --git a/Makefile.am b/Makefile.am index 568e735..ed40173 100644 --- a/Makefile.am +++ b/Makefile.am @@ -39,6 +39,7 @@ SUBDIRS = \ tests \ python \ sh \ + cp \ fuse \ ocaml \ ocaml/examples \ diff --git a/TODO b/TODO index 5c97db0..3a6540d 100644 --- a/TODO +++ b/TODO @@ -7,8 +7,6 @@ Bindings in other languages. Example code integrating with ppoll, pollfd, APR pollset (and others?). -Example command line utils to copy in/out (like qemu-img convert). - NBD_OPT_INFO mode (like qemu-nbd -L). NBD resize extension. @@ -22,10 +20,6 @@ Performance: Chart it over various buffer sizes and threads, as that Examine other fuzzers: https://gitlab.com/akihe/radamsa -Should we ship a "nbdcp" copying tool? - - Could upload, download or copy between servers. - - Duplicates functionality already available in qemu-img convert. - nbdfuse: - If you write beyond the end of the virtual file, it returns EIO. - Implement trim/discard. diff --git a/configure.ac b/configure.ac index 77a9103..c3b8caa 100644 --- a/configure.ac +++ b/configure.ac @@ -382,6 +382,7 @@ AC_CONFIG_FILES([sh/nbdsh], AC_CONFIG_FILES([Makefile common/include/Makefile + cp/Makefile docs/Makefile examples/Makefile fuse/Makefile diff --git a/cp/Makefile.am b/cp/Makefile.am new file mode 100644 index 0000000..1f3301c --- /dev/null +++ b/cp/Makefile.am @@ -0,0 +1,52 @@ +# nbd client library in userspace +# Copyright (C) 2013-2020 Red Hat Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +include $(top_srcdir)/subdir-rules.mk + +EXTRA_DIST = \ + nbdcp.pod \ + $(NULL) + +TESTS_ENVIRONMENT = LIBNBD_DEBUG=1 +LOG_COMPILER = $(top_builddir)/run +TESTS = + +bin_PROGRAMS = nbdcp + +nbdcp_SOURCES = nbdcp.c +nbdcp_CPPFLAGS = -I$(top_srcdir)/include +nbdcp_CFLAGS = $(WARNINGS_CFLAGS) +nbdcp_LDADD = $(top_builddir)/lib/libnbd.la + +if HAVE_POD + +man_MANS = \ + nbdcp.1 \ + $(NULL) + +nbdcp.1: nbdcp.pod $(top_builddir)/podwrapper.pl + $(PODWRAPPER) --section=1 --man $@ \ + --html $(top_builddir)/html/$@.html \ + $< + +endif HAVE_POD + +TESTS += \ + $(NULL) + +check-valgrind: + LIBNBD_VALGRIND=1 $(MAKE) check diff --git a/cp/nbdcp.c b/cp/nbdcp.c new file mode 100644 index 0000000..c06380c --- /dev/null +++ b/cp/nbdcp.c @@ -0,0 +1,84 @@ +/* NBD client library in userspace + * Copyright (C) 2013-2020 Red Hat Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <getopt.h> +#include <limits.h> + +#include <libnbd.h> + +#define MAX_REQUEST_SIZE (32 * 1024 * 1024) + +static void __attribute__((noreturn)) +usage (FILE *fp, int exitcode) +{ + fprintf (fp, +"\n" +"Copy between NBD servers and local files:\n" +"\n" +" nbdcp XXX\n" +"\n" +"Please read the nbdcp(1) manual page for full usage.\n" +"\n" +); + exit (exitcode); +} + +static void +display_version (void) +{ + printf ("%s %s\n", PACKAGE_NAME, PACKAGE_VERSION); +} + +int +main (int argc, char *argv[]) +{ + enum { + HELP_OPTION = CHAR_MAX + 1, + }; + const char *short_options = "V"; + const struct option long_options[] = { + { "help", no_argument, NULL, HELP_OPTION }, + { "version", no_argument, NULL, 'V' }, + { NULL } + }; + int c; + + for (;;) { + c = getopt_long (argc, argv, short_options, long_options, NULL); + if (c == -1) + break; + + switch (c) { + case HELP_OPTION: + usage (stdout, EXIT_SUCCESS); + + case 'V': + display_version (); + exit (EXIT_SUCCESS); + + default: + usage (stderr, EXIT_FAILURE); + } + } + + exit (EXIT_FAILURE); +} diff --git a/cp/nbdcp.pod b/cp/nbdcp.pod new file mode 100644 index 0000000..d07e02c --- /dev/null +++ b/cp/nbdcp.pod @@ -0,0 +1,177 @@ +=head1 NAME + +nbdcp - copy between NBD servers and local files + +=head1 SYNOPSIS + + nbdcp [-a|--target-allocation allocated|sparse] + [-m|--multi-conn <n>] [-M|--multi-conn-target <n>] + [-p|--progress-bar] [-S|--sparse-detect <n>] + [-T|--threads <n>] [-z|--target-is-zero] + 'nbd://...'|DISK.IMG 'nbd://...'|DISK.IMG + +=head1 DESCRIPTION + +nbdcp is a utility that can copy quickly between NBD servers and local +raw format files (or block devices). It can copy: + +=over 4 + +=item from NBD server to file (or block device) + +For example, this command copies from the NBD server listening on port +10809 on C<example.com> to a local file called F<disk.img>: + + nbdcp nbd://example.com disk.img + +=item from file (or block device) to NBD server + +For example, this command copies from a local block device F</dev/sda> +to the NBD server listening on Unix domain socket F</tmp/socket>: + + nbdcp /dev/sda 'nbd+unix:///?socket=/tmp/socket' + +=item from NBD server to NBD server + +For example this copies between two different exports on the same NBD +server: + + nbdcp nbd://example.com/export1 nbd://example.com/export2 + +=back + +This program I<cannot>: copy from file to file (use L<cp(1)> or +L<dd(1)>), copy to or from formats other than raw (use L<qemu-img(1)> +convert), or access servers other than NBD servers (also use +L<qemu-img(1)>). + +NBD servers are specified by their URI, following the NBD URI standard +at L<https://github.com/NetworkBlockDevice/nbd/blob/master/doc/uri.md> + +=head2 Controlling sparseness or preallocation in the target + +The options I<-a> (I<--target-allocation>), I<-S> (I<--sparse-detect>) +and I<-z> (I<--target-is-zero>) together control sparseness in the +target file. + +By default nbdcp tries to both preserve sparseness from the source and +will detect runs of allocated zeroes and turn them into sparseness. +To turn off detection of sparseness use S<C<-S 0>>. + +The I<-z> option should be used if and only if you know that the +target block device is zeroed already. This allows an important +optimization where nbdcp can skip zeroing or trimming parts of the +disk that are already zero. + +The I<-a> option is used to control the desired final preallocation +state of the target. The default is S<C<-a sparse>> which makes the +target as sparse as possible. S<C<-a allocated>> makes the target +fully allocated. + +=head1 OPTIONS + +=over 4 + +=item B<--help> + +Display brief command line help and exit. + +=item B<-a allocated> + +=item B<--target-allocation=allocated> + +Make the target fully allocated. + +=item B<-a sparse> + +=item B<--target-allocation=sparse> + +Make the target as sparse as possible. This is the default. See also +L</Controlling sparseness or preallocation in the target>. + +=item B<-m> N + +=item B<--multi-conn=>N + +Enable NBD multi-conn with up to C<N> connections. Only some +NBD servers support this but it can greatly improve performance. + +The default is to enable multi-conn if we detect that the server +supports it, with up to 4 connections. + +=item B<-M> N + +=item B<--multi-conn-target=>N + +If you are copying between NBD servers, use I<-m> to control the +multi-conn setting for the source server, and this option (I<-M>) to +control the multi-conn setting for the target server. + +=item B<-p> + +=item B<--progress-bar> + +Display a progress bar during copying. + +=item B<-p machine:>FD + +=item B<--progress-bar=machine:>FD + +Write a machine-readable progress bar to file descriptor C<FD>. This +progress bar prints lines with the format C<COPIED/TOTAL> (where +C<COPIED> and C<TOTAL> are 64 bit unsigned integers). + +=item B<-S 0> + +=item B<--sparse-detect=0> + +Turn off sparseness detection. + +=item B<-S> N + +=item B<--sparse-detect=>N + +Detect runs of zero bytes of at least size C<N> bytes and turn them +into sparse blocks on the target (if S<C<-a sparse>> is used). +This is the default, with a 512 byte block size. + +=item B<-T> N + +=item B<--threads> N + +Use at most C<N> threads when copying. Usually more threads leads to +better performance, up to the limit of the number of cores on your +machine and the parallelism of the underlying disk or network. The +default is to use the number of online processors. + +=item B<-z> + +=item B<--target-is-zero> + +Declare that the target block device contains only zero bytes (or +sparseness that reads back as zeroes). You must only use this option +if you are sure that this is true, since it means that nbdcp will +enable an optimization where it skips zeroing parts of the disk that +are zero on the source. + +=item B<-V> + +=item B<--version> + +Display the package name and version and exit. + +=back + +=head1 SEE ALSO + +L<qemu-img(1)>, +L<libnbd(3)>, +L<nbdsh(1)>. + +=head1 AUTHORS + +Richard W.M. Jones + +=head1 COPYRIGHT + +Copyright (C) 2020 Red Hat Inc. -- 2.24.1