On Tue, Feb 14, 2017 at 09:12:02AM +0100, Pino Toscano wrote: FWIW you can do some really weird sh*t with environment variables, such as creating ones whose name contains newlines. That would break the above code: $ env $'A\nB=C' awk 'BEGIN{for (i in ENVIRON) {print i}}' But probably not important enough to worry about, so ACK :-) Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones Read my programming and virtualization blog: http://rwmj.wordpress.com virt-p2v converts physical machines to virtual machines. Boot with a live CD or over the network (PXE) and turn machines into KVM guests. http://libguestfs.org/virt-v2v

It looks like scripts (in particular in the extra-data.d phase) may in-flight add new scripts to the existing ones. As first step to handle this situation, change the path of hooks passed for extra-data.d scripts: instead of the local temporary directory, use the in-guest location that is visible as result of the local fuse-based mount of the guest. --- dib/dib.ml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/dib/dib.ml b/dib/dib.ml index 6a0145d..9884a7a 100644 --- a/dib/dib.ml +++ b/dib/dib.ml @@ -73,7 +73,7 @@ let envvars_string l = let prepare_external ~envvars ~dib_args ~dib_vars ~out_name ~root_label ~rootfs_uuid ~image_cache ~arch ~network ~debug ~fs_type ~checksum - destdir libdir hooksdir fakebindir all_elements element_paths = + destdir libdir fakebindir all_elements element_paths = let network_string = if network then "" else "1" in let checksum_string = if checksum then "1" else "" in @@ -83,6 +83,8 @@ set -e %s mount_dir=$1 shift +hooks_dir=$1 +shift target_dir=$1 shift script=$1 @@ -102,7 +104,7 @@ export DIB_IMAGE_ROOT_FS_UUID=%s export DIB_IMAGE_CACHE=\"%s\" export _LIB=%s export ARCH=%s -export TMP_HOOKS_PATH=%s +export TMP_HOOKS_PATH=\"$hooks_dir\" export DIB_ARGS=\"%s\" export IMAGE_ELEMENT=\"%s\" export ELEMENTS_PATH=\"%s\" @@ -138,7 +140,6 @@ $target_dir/$script image_cache (quote libdir) arch - (quote hooksdir) dib_args (String.concat " " (StringSet.elements all_elements)) (String.concat ":" element_paths) @@ -407,10 +408,14 @@ let run_parts ~debug ~sysroot ~blockdev ~log_file ?(new_wd = "") flush_all (); Buffer.contents outbuf -let run_parts_host ~debug (g : Guestfs.guestfs) hooks_dir hook_name base_mount_dir scripts run_script = - let hook_dir = hooks_dir // hook_name in +let run_parts_host ~debug (g : Guestfs.guestfs) hook_name base_mount_dir scripts run_script = let scripts = List.sort digit_prefix_compare scripts in let mount_dir = base_mount_dir // hook_name in + (* Point to the in-guest hooks, so that changes there can affect + * other phases. + *) + let hooks_dir = mount_dir // "tmp" // "aux" // "hooks" in + let hook_dir = hooks_dir // hook_name in do_mkdir mount_dir; let rec fork_and_run () = @@ -428,7 +433,7 @@ let run_parts_host ~debug (g : Guestfs.guestfs) hooks_dir hook_name base_mount_d let rec loop = function | x :: xs -> message (f_"Running: %s/%s") hook_name x; - let cmd = [ run_script; mount_dir; hook_dir; x ] in + let cmd = [ run_script; mount_dir; hooks_dir; hook_dir; x ] in let retcode = ref 0 in let run () = retcode := run_command cmd in @@ -623,7 +628,7 @@ let main () = ~network:cmdline.network ~debug ~fs_type:cmdline.fs_type ~checksum:cmdline.checksum - tmpdir cmdline.basepath hookstmpdir + tmpdir cmdline.basepath (auxtmpdir // "fake-bin") all_elements cmdline.element_paths; @@ -779,7 +784,7 @@ let main () = if debug >= 1 then ( printf "Running hooks for %s...\n%!" hook; ); - run_parts_host ~debug g hookstmpdir hook tmpdir scripts + run_parts_host ~debug g hook tmpdir scripts (tmpdir // "run-part-extra.sh") with Not_found -> () in -- 2.9.3

Recently, diskimage-builder moved the log cleanup from the 'base' element to disk-image-create proper; the cleanup done is: - truncate any file in /var/log - remove *.log files in /root This was implemented in diskimage-builder upstream as commit 022d93ee822e71245af52c4cf8f8a8e82f599af3. --- dib/dib.ml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dib/dib.ml b/dib/dib.ml index ff2298d..b578750 100644 --- a/dib/dib.ml +++ b/dib/dib.ml @@ -895,6 +895,13 @@ let main () = flush_all (); g#mount blockdev "/"; Array.iter (fun x -> g#rm_rf ("/tmp/" ^ x)) (g#ls "/tmp"); + (* Truncate /var/log files in preparation for first boot. *) + truncate_recursive g "/var/log"; + let non_log fn = + not (String.is_suffix fn ".log") + in + (* Remove root logs. *) + rm_rf_only_files g ~filter:non_log "/root"; flush_all (); -- 2.9.3

On Tue, Feb 14, 2017 at 09:12:07AM +0100, Pino Toscano wrote: This change seems reasonable. Should we: (1) Bind other tar-out parameters? It looks as if numericowner, xattrs, selinux and acls would all be candidates. (2) More importantly for this patch, keep the order of the options compatible with tar-out? This would allow us to pass the optargs_bitmask straight through, if that's an advantage (it may not be). The 'compressed' optarg for tar-out makes this a bit more complicated. Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones Read my programming and virtualization blog: http://rwmj.wordpress.com libguestfs lets you edit virtual machines. Supports shell scripting, bindings from many languages. http://libguestfs.org

Add a new hook for Output_format's to check for prerequisites on the appliance (which can be done only when the appliance is run). --- dib/dib.ml | 2 ++ dib/output_format.ml | 15 +++++++++++++++ dib/output_format.mli | 9 +++++++++ 3 files changed, 26 insertions(+) diff --git a/dib/dib.ml b/dib/dib.ml index b578750..df83ba1 100644 --- a/dib/dib.ml +++ b/dib/dib.ml @@ -703,6 +703,8 @@ let main () = g#launch (); + Output_format.check_formats_appliance_prerequisites cmdline.formats g; + (* Prepare the /aux partition. *) g#mkfs "ext2" "/dev/sdb"; g#mount "/dev/sdb" "/"; diff --git a/dib/output_format.ml b/dib/output_format.ml index f311b8d..851cefc 100644 --- a/dib/output_format.ml +++ b/dib/output_format.ml @@ -27,6 +27,7 @@ type format = { extra_args : extra_arg list; output_to_file : bool; check_prerequisites : (unit -> unit) option; + check_appliance_prerequisites : (Guestfs.guestfs -> unit) option; run_on_filesystem : (Guestfs.guestfs -> string -> string -> unit) option; run_on_file : (string -> (string * string) -> string -> unit) option; } @@ -39,6 +40,7 @@ let defaults = { extra_args = []; output_to_file = true; check_prerequisites = None; + check_appliance_prerequisites = None; run_on_filesystem = None; run_on_file = None; } @@ -129,6 +131,19 @@ let check_formats_prerequisites ~formats = | { check_prerequisites = None } -> () ) formats +let check_formats_appliance_prerequisites ~formats g = + assert !baked; + + (* Run the formats in alphabetical, rather than random order. *) + let formats = List.sort compare_formats (FormatSet.elements formats) in + + List.iter ( + function + | { check_appliance_prerequisites = Some fn } -> + fn g + | { check_appliance_prerequisites = None } -> () + ) formats + let run_formats_on_filesystem ~formats g image_name tmpdir = assert !baked; diff --git a/dib/output_format.mli b/dib/output_format.mli index e935f6f..76683ad 100644 --- a/dib/output_format.mli +++ b/dib/output_format.mli @@ -44,6 +44,11 @@ type format = { to check whether the requirements for this format (available tools, values for command line arguments, etc) are fulfilled. *) + check_appliance_prerequisites : (Guestfs.guestfs -> unit) option; + (** The function which is called after the appliance start to check + whether the requirements in the appliance for this format + (available features, filesystems, etc) are fulfilled. *) + run_on_filesystem : (Guestfs.guestfs -> string -> string -> unit) option; (** The function which is called to perform the export while the guest is mounted. @@ -108,6 +113,10 @@ val set_cardinal : set -> int val check_formats_prerequisites : formats:set -> unit (** Check the prerequisites in all the formats listed in the [formats] set. *) +val check_formats_appliance_prerequisites : formats:set -> Guestfs.guestfs -> unit +(** Check the appliance prerequisites in all the formats listed in the + [formats] set. *) + val run_formats_on_filesystem : formats:set -> Guestfs.guestfs -> string -> string -> unit (** Run the filesystem-based export for all the formats listed in the [formats] set. *) -- 2.9.3

Introduce a new API to create a new squashfs filesystem out of a path in the guest. It can be configured to exclude paths based on patterns, and to select which compression use for the filesystem. The advantage of running mksquashfs directly in the appliance is that ownerships are properly saved, as opposed to tar_out + local untar. --- appliance/packagelist.in | 2 + daemon/Makefile.am | 1 + daemon/squashfs.c | 150 +++++++++++++++++++++++++++++++++++++++++++++++ generator/actions.ml | 26 ++++++++ gobject/Makefile.inc | 2 + lib/MAX_PROC_NR | 2 +- 6 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 daemon/squashfs.c diff --git a/appliance/packagelist.in b/appliance/packagelist.in index 1cf7d9d..8ed3afe 100644 --- a/appliance/packagelist.in +++ b/appliance/packagelist.in @@ -151,6 +151,7 @@ ifelse(SUSE,1, ntfsprogs ntfs-3g reiserfs + squashfs systemd vim xz @@ -253,6 +254,7 @@ rsync scrub sed sleuthkit +squashfs-tools strace syslinux tar diff --git a/daemon/Makefile.am b/daemon/Makefile.am index 3e301ad..5c4ae8e 100644 --- a/daemon/Makefile.am +++ b/daemon/Makefile.am @@ -138,6 +138,7 @@ guestfsd_SOURCES = \ sh.c \ sleep.c \ sleuthkit.c \ + squashfs.c \ stat.c \ statvfs.c \ strings.c \ diff --git a/daemon/squashfs.c b/daemon/squashfs.c new file mode 100644 index 0000000..c307776 --- /dev/null +++ b/daemon/squashfs.c @@ -0,0 +1,150 @@ +/* libguestfs - the guestfsd daemon + * Copyright (C) 2017 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include "daemon.h" +#include "actions.h" +#include "optgroups.h" + +#define MAX_ARGS 64 + +GUESTFSD_EXT_CMD(str_mksquashfs, mksquashfs); + +int +optgroup_squashfs_available (void) +{ + return prog_exists (str_mksquashfs); +} + +/* Takes optional arguments, consult optargs_bitmask. */ +int +do_mksquashfs (const char *path, const char *compress, char *const *excludes) +{ + CLEANUP_FREE char *buf = NULL; + CLEANUP_UNLINK_FREE char *tmpfile = NULL; + CLEANUP_UNLINK_FREE char *exclude_from_file = NULL; + CLEANUP_FREE char *err = NULL; + CLEANUP_FREE char *buffer = NULL; + int r; + const char *argv[MAX_ARGS]; + size_t i = 0; + int fd; + FILE *fp; + + buf = sysroot_path (path); + if (buf == NULL) { + reply_with_perror ("malloc/sysroot_path"); + return -1; + } + + /* /var/tmp is used instead of /tmp, as /tmp is mounted as tmpfs + * and thus a newly created filesystem might not fit in memory. + */ + tmpfile = strdup ("/var/tmp/squashfs.XXXXXX"); + if (tmpfile == NULL) { + reply_with_perror ("strdup"); + return -1; + } + + fd = mkstemp (tmpfile); + if (fd == -1) { + reply_with_perror ("mkstemp"); + return -1; + } + close (fd); + + buffer = malloc (GUESTFS_MAX_CHUNK_SIZE); + if (buffer == NULL) { + reply_with_perror ("malloc/buffer"); + return -1; + } + + ADD_ARG (argv, i, str_mksquashfs); + ADD_ARG (argv, i, buf); + ADD_ARG (argv, i, tmpfile); + ADD_ARG (argv, i, "-noappend"); + ADD_ARG (argv, i, "-root-becomes"); + ADD_ARG (argv, i, buf); + ADD_ARG (argv, i, "-wildcards"); + ADD_ARG (argv, i, "-no-recovery"); + + if (optargs_bitmask & GUESTFS_MKSQUASHFS_COMPRESS_BITMASK) { + ADD_ARG (argv, i, "-comp"); + ADD_ARG (argv, i, compress); + } + + if (optargs_bitmask & GUESTFS_MKSQUASHFS_EXCLUDES_BITMASK) { + exclude_from_file = make_exclude_from_file ("mksquashfs", excludes); + if (!exclude_from_file) + return -1; + + ADD_ARG (argv, i, "-ef"); + ADD_ARG (argv, i, exclude_from_file); + } + + ADD_ARG (argv, i, NULL); + + r = commandv (NULL, &err, argv); + if (r == -1) { + reply_with_error ("%s: %s", path, err); + return -1; + } + + fp = fopen (tmpfile, "r"); + if (fp == NULL) { + reply_with_perror ("%s", tmpfile); + return -1; + } + + /* Now we must send the reply message, before the file contents. After + * this there is no opportunity in the protocol to send any error + * message back. Instead we can only cancel the transfer. + */ + reply (NULL, NULL); + + while ((r = fread (buffer, 1, GUESTFS_MAX_CHUNK_SIZE, fp)) > 0) { + if (send_file_write (buffer, r) < 0) { + fclose (fp); + return -1; + } + } + + if (ferror (fp)) { + fprintf (stderr, "fread: %s: %m\n", tmpfile); + send_file_end (1); /* Cancel. */ + fclose (fp); + return -1; + } + + if (fclose (fp) != 0) { + fprintf (stderr, "fclose: %s: %m\n", tmpfile); + send_file_end (1); /* Cancel. */ + return -1; + } + + if (send_file_end (0)) /* Normal end of file. */ + return -1; + + return 0; +} diff --git a/generator/actions.ml b/generator/actions.ml index fd6cc9f..409f399 100644 --- a/generator/actions.ml +++ b/generator/actions.ml @@ -13292,6 +13292,32 @@ is removed." }; shortdesc = "search the entries associated to the given inode"; longdesc = "Internal function for find_inode." }; + { defaults with + name = "mksquashfs"; added = (1, 35, 25); + style = RErr, [Pathname "path"; FileOut "filename"], [OString "compress"; OStringList "excludes"]; + proc_nr = Some 471; + optional = Some "squashfs"; + cancellable = true; + shortdesc = "create a squashfs filesystem"; + longdesc = "\ +Create a squashfs filesystem for the specified C<path>. + +The optional C<compress> flag controls compression. If not given, +then the output compressed using C<gzip>. Otherwise one +of the following strings may be given to select the compression +type of the squashfs: C<gzip>, C<lzma>, C<lzo>, C<lz4>, C<xz>. + +The other optional arguments are: + +=over 4 + +=item C<excludes> + +A list of wildcards. Files are excluded if they match any of the +wildcards. + +=back" }; + ] (* Non-API meta-commands available only in guestfish. diff --git a/gobject/Makefile.inc b/gobject/Makefile.inc index 76482f7..48cb2ef 100644 --- a/gobject/Makefile.inc +++ b/gobject/Makefile.inc @@ -90,6 +90,7 @@ guestfs_gobject_headers= \ include/guestfs-gobject/optargs-mke2fs.h \ include/guestfs-gobject/optargs-mkfs.h \ include/guestfs-gobject/optargs-mkfs_btrfs.h \ + include/guestfs-gobject/optargs-mksquashfs.h \ include/guestfs-gobject/optargs-mkswap.h \ include/guestfs-gobject/optargs-mktemp.h \ include/guestfs-gobject/optargs-mount_9p.h \ @@ -182,6 +183,7 @@ guestfs_gobject_sources= \ src/optargs-mke2fs.c \ src/optargs-mkfs.c \ src/optargs-mkfs_btrfs.c \ + src/optargs-mksquashfs.c \ src/optargs-mkswap.c \ src/optargs-mktemp.c \ src/optargs-mount_9p.c \ diff --git a/lib/MAX_PROC_NR b/lib/MAX_PROC_NR index 5f476b6..c305aa5 100644 --- a/lib/MAX_PROC_NR +++ b/lib/MAX_PROC_NR @@ -1 +1 @@ -470 +471 -- 2.9.3

Implement the "squash" output format, i.e. a squashfs compressed filesystem. This was implemented in diskimage-builder upstream as commit 9d13084c4183b63587e1f5e4b03395a8df6538f6. --- dib/Makefile.am | 1 + dib/output_format_squashfs.ml | 39 +++++++++++++++++++++++++++++++++++++++ dib/utils.ml | 1 + dib/virt-dib.pod | 6 ++++++ 4 files changed, 47 insertions(+) create mode 100644 dib/output_format_squashfs.ml diff --git a/dib/Makefile.am b/dib/Makefile.am index 470fb52..6780ee2 100644 --- a/dib/Makefile.am +++ b/dib/Makefile.am @@ -31,6 +31,7 @@ formats = \ docker \ qcow2 \ raw \ + squashfs \ tar \ tgz \ vhd diff --git a/dib/output_format_squashfs.ml b/dib/output_format_squashfs.ml new file mode 100644 index 0000000..333e711 --- /dev/null +++ b/dib/output_format_squashfs.ml @@ -0,0 +1,39 @@ +(* virt-dib + * Copyright (C) 2017 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Common_utils +open Common_gettext.Gettext + +open Output_format + +let squashfs_check (g : Guestfs.guestfs) = + g#available [| "squashfs" |] + +let squashfs_run_fs (g : Guestfs.guestfs) filename _ = + message (f_"Compressing the image as squashfs"); + g#mksquashfs ~excludes:[| "sys/*"; "proc/*"; "dev/*" |] ~compress:"xz" + "/" filename + +let fmt = { + defaults with + name = "squashfs"; + check_appliance_prerequisites = Some squashfs_check; + run_on_filesystem = Some squashfs_run_fs; +} + +let () = register_format fmt diff --git a/dib/utils.ml b/dib/utils.ml index 3775a41..da5e738 100644 --- a/dib/utils.ml +++ b/dib/utils.ml @@ -33,6 +33,7 @@ let current_arch () = | arch -> arch let output_filename image_name = function + | "squashfs" -> image_name ^ ".squash" | fmt -> image_name ^ "." ^ fmt let log_filename () = diff --git a/dib/virt-dib.pod b/dib/virt-dib.pod index 2786050..7083e78 100644 --- a/dib/virt-dib.pod +++ b/dib/virt-dib.pod @@ -212,6 +212,12 @@ QEMU's qcow2. Raw disk format. +=item C<squashfs> + +An squashfs filesystem, compressed with XZ. This output format +requires the C<squashfs> feature; see also +L<guestfs(3)/AVAILABILITY>. + =item C<tar> An uncompressed tarball. -- 2.9.3