[Libguestfs] [PATCH v2 17/18] sparsify: Add virt-sparsify --in-place mode.

--- po/POTFILES-ml | 1 + sparsify/Makefile.am | 2 + sparsify/cmdline.ml | 77 +++++++++++++++++------- sparsify/in_place.ml | 143 +++++++++++++++++++++++++++++++++++++++++++++ sparsify/sparsify.ml | 3 + sparsify/virt-sparsify.pod | 63 +++++++++++++++++--- 6 files changed, 259 insertions(+), 30 deletions(-) create mode 100644 sparsify/in_place.ml diff --git a/po/POTFILES-ml b/po/POTFILES-ml index fd2e886..2c37d91 100644 --- a/po/POTFILES-ml +++ b/po/POTFILES-ml @@ -33,6 +33,7 @@ mllib/urandom.ml resize/resize.ml sparsify/cmdline.ml sparsify/copying.ml +sparsify/in_place.ml sparsify/sparsify.ml sysprep/main.ml sysprep/sysprep_operation.ml diff --git a/sparsify/Makefile.am b/sparsify/Makefile.am index 2f32093..811c131 100644 --- a/sparsify/Makefile.am +++ b/sparsify/Makefile.am @@ -28,6 +28,7 @@ CLEANFILES = *~ *.cmi *.cmo *.cmx *.cmxa *.o virt-sparsify SOURCES = \ cmdline.ml \ copying.ml \ + in_place.ml \ sparsify.ml \ statvfs-c.c @@ -46,6 +47,7 @@ deps = \ statvfs-c.o \ cmdline.cmx \ copying.cmx \ + in_place.cmx \ sparsify.cmx if HAVE_OCAMLOPT diff --git a/sparsify/cmdline.ml b/sparsify/cmdline.ml index b714a9e..d25e9e9 100644 --- a/sparsify/cmdline.ml +++ b/sparsify/cmdline.ml @@ -27,7 +27,8 @@ let prog = Filename.basename Sys.executable_name let error fs = error ~prog fs type mode_t = - Mode_copying of string * check_t * bool * string option * string option +| Mode_copying of string * check_t * bool * string option * string option +| Mode_in_place and check_t = [`Ignore|`Continue|`Warn|`Fail] let parse_cmdline () = @@ -54,6 +55,7 @@ let parse_cmdline () = let debug_gc = ref false in let format = ref "" in let ignores = ref [] in + let in_place = ref false in let machine_readable = ref false in let option = ref "" in let quiet = ref false in @@ -69,6 +71,8 @@ let parse_cmdline () = "--debug-gc", Arg.Set debug_gc, " " ^ s_"Debug GC and memory allocations"; "--format", Arg.Set_string format, s_"format" ^ " " ^ s_"Format of input disk"; "--ignore", Arg.String (add ignores), s_"fs" ^ " " ^ s_"Ignore filesystem"; + "--in-place", Arg.Set in_place, " " ^ s_"Modify the disk image in-place"; + "--inplace", Arg.Set in_place, ditto; "--long-options", Arg.Unit display_long_options, " " ^ s_"List long options"; "--machine-readable", Arg.Set machine_readable, " " ^ s_"Make output machine readable"; "-o", Arg.Set_string option, s_"option" ^ " " ^ s_"Add qemu-img options"; @@ -90,6 +94,8 @@ let parse_cmdline () = virt-sparsify [--options] indisk outdisk + virt-sparsify [--options] --in-place disk + A short summary of the options is given below. For detailed help please read the man page virt-sparsify(1). ") @@ -103,6 +109,7 @@ read the man page virt-sparsify(1). let debug_gc = !debug_gc in let format = match !format with "" -> None | str -> Some str in let ignores = List.rev !ignores in + let in_place = !in_place in let machine_readable = !machine_readable in let option = match !option with "" -> None | str -> Some str in let quiet = !quiet in @@ -118,6 +125,7 @@ read the man page virt-sparsify(1). printf "linux-swap\n"; printf "zero\n"; printf "check-tmpdir\n"; + printf "in-place\n"; let g = new G.guestfs () in g#add_drive "/dev/null"; g#launch (); @@ -128,12 +136,14 @@ read the man page virt-sparsify(1). exit 0 ); - (* Verify we got exactly 2 disks. *) + (* Verify we got exactly 1 or 2 disks, depending on the mode. *) let indisk, outdisk = - match List.rev !disks with - | [indisk; outdisk] -> indisk, outdisk + match in_place, List.rev !disks with + | false, [indisk; outdisk] -> indisk, outdisk + | true, [disk] -> disk, "" | _ -> - error "usage is: %s [--options] indisk outdisk" prog in + error "usage is: %s [--options] indisk outdisk OR %s --in-place disk" + prog prog in (* Simple-minded check that the user isn't trying to use the * same disk for input and output. @@ -141,24 +151,49 @@ read the man page virt-sparsify(1). if indisk = outdisk then error (f_"you cannot use the same disk image for input and output"); - (* The input disk must be an absolute path, so we can store the name - * in the overlay disk. - *) let indisk = - if not (Filename.is_relative indisk) then + if not in_place then ( + (* The input disk must be an absolute path, so we can store the name + * in the overlay disk. + *) + let indisk = + if not (Filename.is_relative indisk) then + indisk + else + Sys.getcwd () // indisk in + + (* Check the output is not a block or char special (RHBZ#1056290). *) + if is_block_device outdisk then + error (f_"output '%s' cannot be a block device, it must be a regular file") + outdisk; + + if is_char_device outdisk then + error (f_"output '%s' cannot be a character device, it must be a regular file") + outdisk; + indisk + ) + else ( (* --in-place checks *) + if check_tmpdir <> `Warn then + error (f_"you cannot use --in-place and --check-tmpdir options together"); + + if compress then + error (f_"you cannot use --in-place and --compress options together"); + + if convert <> None then + error (f_"you cannot use --in-place and --convert options together"); + + if option <> None then + error (f_"you cannot use --in-place and -o options together"); + + indisk + ) in + + let mode = + if not in_place then + Mode_copying (outdisk, check_tmpdir, compress, convert, option) else - Sys.getcwd () // indisk in - - (* Check the output is not a block or char special (RHBZ#1056290). *) - if is_block_device outdisk then - error (f_"output '%s' cannot be a block device, it must be a regular file") - outdisk; - - if is_char_device outdisk then - error (f_"output '%s' cannot be a character device, it must be a regular file") - outdisk; + Mode_in_place in indisk, debug_gc, format, ignores, machine_readable, - quiet, verbose, trace, zeroes, - Mode_copying (outdisk, check_tmpdir, compress, convert, option) + quiet, verbose, trace, zeroes, mode diff --git a/sparsify/in_place.ml b/sparsify/in_place.ml new file mode 100644 index 0000000..9f46ad3 --- /dev/null +++ b/sparsify/in_place.ml @@ -0,0 +1,143 @@ +(* virt-sparsify + * Copyright (C) 2011-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +(* This is the virt-sparsify --in-place mode. *) + +open Unix +open Printf + +open Common_gettext.Gettext + +module G = Guestfs + +open Common_utils +open Cmdline + +let run disk format ignores machine_readable quiet verbose trace zeroes = + (* Connect to libguestfs. *) + let g = new G.guestfs () in + if trace then g#set_trace true; + if verbose then g#set_verbose true; + + (* XXX Current limitation of the API. Can remove this hunk in future. *) + let format = + match format with + | Some _ -> format + | None -> Some (g#disk_format disk) in + + g#add_drive ?format ~discard:"enable" disk; + + if not quiet then Progress.set_up_progress_bar ~machine_readable g; + g#launch (); + + (* Discard non-ignored filesystems that we are able to mount, and + * selected swap partitions. + *) + let filesystems = g#list_filesystems () in + let filesystems = List.map fst filesystems in + let filesystems = List.sort compare filesystems in + + let is_ignored fs = + let fs = g#canonical_device_name fs in + List.exists (fun fs' -> fs = g#canonical_device_name fs') ignores + in + + List.iter ( + fun fs -> + if not (is_ignored fs) then ( + if List.mem fs zeroes then ( + if not quiet then + printf (f_"Zeroing %s ...\n%!") fs; + + if not (g#blkdiscardzeroes fs) then + g#zero_device fs; + g#blkdiscard fs + ) else ( + let mounted = + try g#mount_options "discard" fs "/"; true + with _ -> false in + + if mounted then ( + if not quiet then + printf (f_"Trimming %s ...\n%!") fs; + + g#fstrim "/" + ) else ( + let is_linux_x86_swap = + (* Look for the signature for Linux swap on i386. + * Location depends on page size, so it definitely won't + * work on non-x86 architectures (eg. on PPC, page size is + * 64K). Also this avoids hibernated swap space: in those, + * the signature is moved to a different location. + *) + try g#pread_device fs 10 4086L = "SWAPSPACE2" + with _ -> false in + + if is_linux_x86_swap then ( + if not quiet then + printf (f_"Clearing Linux swap on %s ...\n%!") fs; + + (* Don't use mkswap. Just preserve the header containing + * the label, UUID and swap format version (libguestfs + * mkswap may differ from guest's own). + *) + let header = g#pread_device fs 4096 0L in + g#blkdiscard fs; + if g#pwrite_device fs header 0L <> 4096 then + error (f_"pwrite: short write restoring swap partition header") + ) + ) + ); + + g#umount_all () + ) + ) filesystems; + + (* Discard unused space in volume groups. *) + let vgs = g#vgs () in + let vgs = Array.to_list vgs in + let vgs = List.sort compare vgs in + List.iter ( + fun vg -> + if not (List.mem vg ignores) then ( + let lvname = string_random8 () in + let lvdev = "/dev/" ^ vg ^ "/" ^ lvname in + + let created = + try g#lvcreate_free lvname vg 100; true + with _ -> false in + + if created then ( + if not quiet then + printf (f_"Discard space in volgroup %s ...\n%!") vg; + + g#blkdiscard lvdev; + g#sync (); + g#lvremove lvdev + ) + ) + ) vgs; + + g#shutdown (); + g#close (); + + (* Finished. *) + if not quiet then ( + print_newline (); + wrap (s_"Sparsify in-place operation completed with no errors.\n"); + ) diff --git a/sparsify/sparsify.ml b/sparsify/sparsify.ml index faefb23..f148296 100644 --- a/sparsify/sparsify.ml +++ b/sparsify/sparsify.ml @@ -37,6 +37,9 @@ let rec main () = | Mode_copying (outdisk, check_tmpdir, compress, convert, option) -> Copying.run indisk outdisk check_tmpdir compress convert format ignores machine_readable option quiet verbose trace zeroes + | Mode_in_place -> + In_place.run indisk format ignores machine_readable + quiet verbose trace zeroes ); if debug_gc then diff --git a/sparsify/virt-sparsify.pod b/sparsify/virt-sparsify.pod index 84e94c5..c12a15f 100644 --- a/sparsify/virt-sparsify.pod +++ b/sparsify/virt-sparsify.pod @@ -8,6 +8,8 @@ virt-sparsify - Make a virtual machine disk sparse virt-sparsify [--options] indisk outdisk + virt-sparsify [--options] --in-place disk + =head1 DESCRIPTION Virt-sparsify is a tool which can make a virtual machine disk (or any @@ -45,13 +47,6 @@ such as C<du -sh>. It can make a huge difference: =item * -Virt-sparsify does not do in-place modifications. It copies from a -source image to a destination image, leaving the source unchanged. -I<Check that the sparsification was successful before deleting the -source image>. - -=item * - The virtual machine I<must be shut down> before using this tool. =item * @@ -60,6 +55,9 @@ Virt-sparsify may require up to 2x the virtual size of the source disk image (1 temporary copy + 1 destination image). This is in the worst case and usually much less space is required. +If you are using the I<--in-place> option, then large amounts of +temporary space are B<not> required. + =item * Virt-sparsify cannot resize disk images. To do that, use @@ -105,6 +103,11 @@ to ignore (don't zero free space on) certain filesystems by doing: See L<virt-filesystems(1)> to get a list of filesystems within a disk image. +Since virt-sparsify E<ge> 1.26, you can now sparsify a disk image +in place by doing: + + virt-sparsify --in-place disk.img + =head1 OPTIONS =over 4 @@ -147,11 +150,15 @@ B<fail> and exit. =back +You cannot use this option and I<--in-place> together. + =item B<--compress> Compress the output file. This I<only> works if the output format is C<qcow2>. +You cannot use this option and I<--in-place> together. + =item B<--convert> raw =item B<--convert> qcow2 @@ -171,6 +178,8 @@ then virt-sparsify doesn't need to try to guess the input format. For fine-tuning the output format, see: I<--compress>, I<-o>. +You cannot use this option and I<--in-place> together. + =item B<--debug-gc> Debug garbage collection and memory allocation. This is only useful @@ -191,14 +200,23 @@ ensure the format is always specified. =item B<--ignore> volgroup -Ignore the named filesystem. Free space on the filesystem will not be +Ignore the named filesystem. + +When not using I<--in-place>: Free space on the filesystem will not be zeroed, but existing blocks of zeroes will still be sparsified. +When using I<--in-place>, the filesystem is ignored completely. + In the second form, this ignores the named volume group. Use the volume group name without the C</dev/> prefix, eg. I<--ignore vg_foo> You can give this option multiple times. +=item B<--in-place> + +Do in-place sparsification instead of copying sparsification. +See L</IN-PLACE SPARSIFICATION> below. + =item B<--machine-readable> This option is used to make the output more machine friendly @@ -217,6 +235,8 @@ them with commas, eg: virt-sparsify --convert qcow2 \ -o cluster_size=512,preallocation=metadata ... +You cannot use this option and I<--in-place> together. + =item B<-q> =item B<--quiet> @@ -249,6 +269,28 @@ excellent! You can give this option multiple times. =back +=head1 IN-PLACE SPARSIFICATION + +Since virt-sparsify E<ge> 1.26, the tool is able to do in-place +sparsification (instead of copying from an input disk to an output +disk). This is more efficient. However it requires special support +in libguestfs, the kernel and qemu, and it is not able to recover +quite as much space as copying sparsification. So in-place +sparsification is considered to be experimental at this time. + +To use this mode, specify a disk image which will be modified in +place: + + virt-sparsify --in-place disk.img + +Some options are not compatible with this mode: I<--convert>, +I<--compress> and I<-o> because they require wholesale disk format +changes; I<--check-tmpdir> because large amounts of temporary space +are not required. + +In-place sparsification works using discard (a.k.a trim or unmap) +support. + =head1 MACHINE READABLE OUTPUT The I<--machine-readable> option can be used to make the output more @@ -332,6 +374,9 @@ size of the tmpfs mountpoint, eg: mount -o remount,size=10G /tmp +If you are using the I<--in-place> option, then large amounts of +temporary space are B<not> required. + =back For other environment variables, see L<guestfs(3)/ENVIRONMENT VARIABLES>. @@ -355,4 +400,4 @@ Richard W.M. Jones L<http://people.redhat.com/~rjones/> =head1 COPYRIGHT -Copyright (C) 2011-2012 Red Hat Inc. +Copyright (C) 2011-2014 Red Hat Inc. -- 1.8.5.3