[Libguestfs] [PATCH 3/3] v2v: add -o json output mode

Add a new output mode to virt-v2v: similar to -o local, the written metadata is a JSON file with the majority of the data that virt-v2v knowns about (or collects) during the conversion. This is meant to be used only when no existing output mode is usable, and a guest needs to be converted to run on KVM anyway. The user of this mode is supposed to use all the data in the JSON, as they contain important details on how even run the guest (e.g. w.r.t. firmware, drivers of disks/NICs, etc). --- v2v/Makefile.am | 4 + v2v/cmdline.ml | 29 +++ v2v/create_json.ml | 348 ++++++++++++++++++++++++++++++++++ v2v/create_json.mli | 29 +++ v2v/output_json.ml | 116 ++++++++++++ v2v/output_json.mli | 31 +++ v2v/virt-v2v-output-local.pod | 50 +++++ v2v/virt-v2v.pod | 15 +- 8 files changed, 620 insertions(+), 2 deletions(-) create mode 100644 v2v/create_json.ml create mode 100644 v2v/create_json.mli create mode 100644 v2v/output_json.ml create mode 100644 v2v/output_json.mli diff --git a/v2v/Makefile.am b/v2v/Makefile.am index f196be81d..53c137fc6 100644 --- a/v2v/Makefile.am +++ b/v2v/Makefile.am @@ -52,6 +52,7 @@ SOURCES_MLI = \ config.mli \ convert_linux.mli \ convert_windows.mli \ + create_json.mli \ create_libvirt_xml.mli \ create_ovf.mli \ DOM.mli \ @@ -75,6 +76,7 @@ SOURCES_MLI = \ networks.mli \ openstack_image_properties.mli \ output_glance.mli \ + output_json.mli \ output_libvirt.mli \ output_local.mli \ output_null.mli \ @@ -117,6 +119,7 @@ SOURCES_ML = \ parse_ovf_from_ova.ml \ parse_ova.ml \ create_ovf.ml \ + create_json.ml \ linux.ml \ windows.ml \ windows_virtio.ml \ @@ -141,6 +144,7 @@ SOURCES_ML = \ convert_windows.ml \ output_null.ml \ output_glance.ml \ + output_json.ml \ output_libvirt.ml \ output_local.ml \ output_qemu.ml \ diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml index 46f6910d0..4d390f249 100644 --- a/v2v/cmdline.ml +++ b/v2v/cmdline.ml @@ -138,6 +138,7 @@ let parse_cmdline () = | "glance" -> output_mode := `Glance | "libvirt" -> output_mode := `Libvirt | "disk" | "local" -> output_mode := `Local + | "json" -> output_mode := `JSON | "null" -> output_mode := `Null | "openstack" | "osp" | "rhosp" -> output_mode := `Openstack | "ovirt" | "rhv" | "rhev" -> output_mode := `RHV @@ -413,6 +414,17 @@ read the man page virt-v2v(1). | `RHV -> no_options (); `RHV | `QEmu -> no_options (); `QEmu + | `JSON -> + if is_query then ( + Output_json.print_output_options (); + exit 0 + ) + else ( + let json_options = + Output_json.parse_output_options output_options in + `JSON json_options + ) + | `Openstack -> if is_query then ( Output_openstack.print_output_options (); @@ -546,6 +558,23 @@ read the man page virt-v2v(1). Output_libvirt.output_libvirt output_conn output_storage, output_format, output_alloc + | `JSON json_options -> + if output_password <> None then + error_option_cannot_be_used_in_output_mode "json" "-op"; + if output_conn <> None then + error_option_cannot_be_used_in_output_mode "json" "-oc"; + let os = + match output_storage with + | None -> + error (f_"-o json: output directory was not specified, use '-os /dir'") + | Some d when not (is_directory d) -> + error (f_"-os %s: output directory does not exist or is not a directory") d + | Some d -> d in + if qemu_boot then + error_option_cannot_be_used_in_output_mode "json" "--qemu-boot"; + Output_json.output_json os json_options, + output_format, output_alloc + | `Local -> if output_password <> None then error_option_cannot_be_used_in_output_mode "local" "-op"; diff --git a/v2v/create_json.ml b/v2v/create_json.ml new file mode 100644 index 000000000..fdf7b12f5 --- /dev/null +++ b/v2v/create_json.ml @@ -0,0 +1,348 @@ +(* virt-v2v + * Copyright (C) 2019 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Std_utils +open C_utils +open Tools_utils + +open Types +open Utils + +module G = Guestfs + +let json_list_of_string_list = + List.map (fun x -> JSON.String x) + +let json_list_of_string_string_list = + List.map (fun (x, y) -> x, JSON.String y) + +let push_optional_string lst name = function + | None -> () + | Some v -> List.push_back lst (name, JSON.String v) + +let push_optional_int lst name = function + | None -> () + | Some v -> List.push_back lst (name, JSON.Int (Int64.of_int v)) + +let json_unknown_string = function + | "unknown" -> JSON.Null + | v -> JSON.String v + +let find_target_disk targets { s_disk_id = id } = + try List.find (fun t -> t.target_overlay.ov_source.s_disk_id = id) targets + with Not_found -> assert false + +let create_json_metadata source targets target_buses + guestcaps inspect target_firmware = + let doc = ref [ + "version", JSON.Int 1L; + "name", JSON.String source.s_name; + "memory", JSON.Int source.s_memory; + "vcpu", JSON.Int (Int64.of_int source.s_vcpu); + ] in + + (match source.s_genid with + | None -> () + | Some genid -> List.push_back doc ("genid", JSON.String genid) + ); + + if source.s_cpu_vendor <> None || source.s_cpu_model <> None || + source.s_cpu_topology <> None then ( + let cpu = ref [] in + + push_optional_string cpu "vendor" source.s_cpu_vendor; + push_optional_string cpu "model" source.s_cpu_model; + (match source.s_cpu_topology with + | None -> () + | Some { s_cpu_sockets; s_cpu_cores; s_cpu_threads } -> + let attrs = [ + "sockets", JSON.Int (Int64.of_int s_cpu_sockets); + "cores", JSON.Int (Int64.of_int s_cpu_cores); + "threads", JSON.Int (Int64.of_int s_cpu_threads); + ] in + List.push_back cpu ("topology", JSON.Dict attrs) + ); + + List.push_back doc ("cpu", JSON.Dict !cpu); + ); + + let firmware = + let firmware_type = + match target_firmware with + | TargetBIOS -> "bios" + | TargetUEFI -> "uefi" in + + let fw = ref [ + "type", JSON.String firmware_type; + ] in + + (match target_firmware with + | TargetBIOS -> () + | TargetUEFI -> + let uefi_firmware = find_uefi_firmware guestcaps.gcaps_arch in + let flags = + List.map ( + function + | Uefi.UEFI_FLAG_SECURE_BOOT_REQUIRED -> "secure_boot_required" + ) uefi_firmware.Uefi.flags in + + let uefi = ref [ + "code", JSON.String uefi_firmware.Uefi.code; + "vars", JSON.String uefi_firmware.Uefi.vars; + "flags", JSON.List (json_list_of_string_list flags); + ] in + + push_optional_string uefi "code-debug" uefi_firmware.Uefi.code_debug; + + List.push_back fw ("uefi", JSON.Dict !uefi) + ); + + !fw in + List.push_back doc ("firmware", JSON.Dict firmware); + + List.push_back doc ("features", + JSON.List (json_list_of_string_list source.s_features)); + + let machine = + match guestcaps.gcaps_machine with + | I440FX -> "pc" + | Q35 -> "q35" + | Virt -> "virt" in + List.push_back doc ("machine", JSON.String machine); + + let disks, removables = + let disks = ref [] + and removables = ref [] in + + let iter_bus bus_name drive_prefix i = function + | BusSlotEmpty -> () + | BusSlotDisk d -> + (* Find the corresponding target disk. *) + let t = find_target_disk targets d in + + let target_file = + match t.target_file with + | TargetFile s -> s + | TargetURI _ -> assert false in + + let disk = [ + "dev", JSON.String (drive_prefix ^ drive_name i); + "bus", JSON.String bus_name; + "format", JSON.String t.target_format; + "file", JSON.String (absolute_path target_file); + ] in + + List.push_back disks (JSON.Dict disk) + + | BusSlotRemovable { s_removable_type = CDROM } -> + let cdrom = [ + "type", JSON.String "cdrom"; + "dev", JSON.String (drive_prefix ^ drive_name i); + "bus", JSON.String bus_name; + ] in + + List.push_back removables (JSON.Dict cdrom) + + | BusSlotRemovable { s_removable_type = Floppy } -> + let floppy = [ + "type", JSON.String "floppy"; + "dev", JSON.String (drive_prefix ^ drive_name i); + ] in + + List.push_back removables (JSON.Dict floppy) + in + + Array.iteri (iter_bus "virtio" "vd") target_buses.target_virtio_blk_bus; + Array.iteri (iter_bus "ide" "hd") target_buses.target_ide_bus; + Array.iteri (iter_bus "scsi" "sd") target_buses.target_scsi_bus; + Array.iteri (iter_bus "floppy" "fd") target_buses.target_floppy_bus; + + !disks, !removables in + List.push_back doc ("disks", JSON.List disks); + List.push_back doc ("removables", JSON.List removables); + + let nics = + List.map ( + fun { s_mac = mac; s_vnet_type = vnet_type; s_nic_model = nic_model; + s_vnet = vnet; } -> + let vnet_type_str = + match vnet_type with + | Bridge -> "bridge" + | Network -> "network" in + + let nic = ref [ + "vnet", JSON.String vnet; + "vnet-type", JSON.String vnet_type_str; + ] in + + let nic_model_str = Option.map string_of_nic_model nic_model in + push_optional_string nic "model" nic_model_str; + + push_optional_string nic "mac" mac; + + JSON.Dict !nic + ) source.s_nics in + List.push_back doc ("nics", JSON.List nics); + + let guestcaps_dict = + let block_bus = + match guestcaps.gcaps_block_bus with + | Virtio_blk -> "virtio-blk" + | Virtio_SCSI -> "virtio-scsi" + | IDE -> "ide" in + let net_bus = + match guestcaps.gcaps_net_bus with + | Virtio_net -> "virtio-net" + | E1000 -> "e1000" + | RTL8139 -> "rtl8139" in + let video = + match guestcaps.gcaps_video with + | QXL -> "qxl" + | Cirrus -> "cirrus" in + let machine = + match guestcaps.gcaps_machine with + | I440FX -> "i440fx" + | Q35 -> "q35" + | Virt -> "virt" in + + [ + "block-bus", JSON.String block_bus; + "net-bus", JSON.String net_bus; + "video", JSON.String video; + "machine", JSON.String machine; + "arch", JSON.String guestcaps.gcaps_arch; + "virtio-rng", JSON.Bool guestcaps.gcaps_virtio_rng; + "virtio-balloon", JSON.Bool guestcaps.gcaps_virtio_balloon; + "isa-pvpanic", JSON.Bool guestcaps.gcaps_isa_pvpanic; + "acpi", JSON.Bool guestcaps.gcaps_acpi; + ] in + List.push_back doc ("guestcaps", JSON.Dict guestcaps_dict); + + (match source.s_sound with + | None -> () + | Some { s_sound_model = model } -> + let sound = [ + "model", JSON.String (string_of_source_sound_model model); + ] in + List.push_back doc ("sound", JSON.Dict sound) + ); + + (match source.s_display with + | None -> () + | Some d -> + let display_type = + match d.s_display_type with + | Window -> "window" + | VNC -> "vnc" + | Spice -> "spice" in + + let display = ref [ + "type", JSON.String display_type; + ] in + + push_optional_string display "keymap" d.s_keymap; + push_optional_string display "password" d.s_password; + + let listen = + match d.s_listen with + | LNoListen -> None + | LAddress address -> + Some [ + "type", JSON.String "address"; + "address", JSON.String address; + ] + | LNetwork network -> + Some [ + "type", JSON.String "network"; + "network", JSON.String network; + ] + | LSocket None -> + Some [ + "type", JSON.String "socket"; + "socket", JSON.Null; + ] + | LSocket (Some socket) -> + Some [ + "type", JSON.String "socket"; + "socket", JSON.String socket; + ] + | LNone -> + Some [ + "type", JSON.String "none"; + ] in + (match listen with + | None -> () + | Some l -> List.push_back display ("listen", JSON.Dict l) + ); + + push_optional_int display "port" d.s_port; + + List.push_back doc ("display", JSON.Dict !display) + ); + + let inspect_dict = + let apps = + List.map ( + fun { G.app2_name = name; app2_display_name = display_name; + app2_epoch = epoch; app2_version = version; + app2_release = release; app2_arch = arch; } -> + JSON.Dict [ + "name", JSON.String name; + "display-name", JSON.String display_name; + "epoch", JSON.Int (Int64.of_int32 epoch); + "version", JSON.String version; + "release", JSON.String release; + "arch", JSON.String arch; + ] + ) inspect.i_apps in + + let firmware_dict = + match inspect.i_firmware with + | I_BIOS -> + [ + "type", JSON.String "bios"; + ] + | I_UEFI devices -> + [ + "type", JSON.String "uefi"; + "devices", JSON.List (json_list_of_string_list devices); + ] in + + [ + "root", JSON.String inspect.i_root; + "type", JSON.String inspect.i_type; + "distro", json_unknown_string inspect.i_distro; + "osinfo", json_unknown_string inspect.i_osinfo; + "arch", JSON.String inspect.i_arch; + "major-version", JSON.Int (Int64.of_int inspect.i_major_version); + "minor-version", JSON.Int (Int64.of_int inspect.i_minor_version); + "package-format", json_unknown_string inspect.i_package_format; + "package-management", json_unknown_string inspect.i_package_management; + "product-name", json_unknown_string inspect.i_product_name; + "product-variant", json_unknown_string inspect.i_product_variant; + "mountpoints", JSON.Dict (json_list_of_string_string_list inspect.i_mountpoints); + "applications", JSON.List apps; + "windows-systemroot", JSON.String inspect.i_windows_systemroot; + "windows-software-hive", JSON.String inspect.i_windows_software_hive; + "windows-system-hive", JSON.String inspect.i_windows_system_hive; + "windows-current-control-set", JSON.String inspect.i_windows_current_control_set; + "firmware", JSON.Dict firmware_dict; + ] in + List.push_back doc ("inspect", JSON.Dict inspect_dict); + + !doc diff --git a/v2v/create_json.mli b/v2v/create_json.mli new file mode 100644 index 000000000..6dbb6e48b --- /dev/null +++ b/v2v/create_json.mli @@ -0,0 +1,29 @@ +(* virt-v2v + * Copyright (C) 2019 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +(** Create JSON metadata for [-o json]. *) + +val create_json_metadata : Types.source -> Types.target list -> + Types.target_buses -> + Types.guestcaps -> + Types.inspect -> + Types.target_firmware -> + JSON.doc +(** [create_json_metadata source targets target_buses guestcaps + inspect target_firmware] creates the JSON with the majority + of the data that virt-v2v used for the conversion. *) diff --git a/v2v/output_json.ml b/v2v/output_json.ml new file mode 100644 index 000000000..ca0bda978 --- /dev/null +++ b/v2v/output_json.ml @@ -0,0 +1,116 @@ +(* virt-v2v + * Copyright (C) 2019 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Printf + +open Std_utils +open Tools_utils +open Common_gettext.Gettext + +open Types +open Utils + +type json_options = { + json_disks_pattern : string; +} + +let print_output_options () = + printf (f_"Output options (-oo) which can be used with -o json: + + -oo json-disks-pattern=PATTERN Pattern for the disks. +") + +let known_pattern_variables = ["DiskNo"; "DiskDeviceName"; "GuestName"] + +let parse_output_options options = + let json_disks_pattern = ref None in + + List.iter ( + function + | "json-disks-pattern", v -> + if !json_disks_pattern <> None then + error (f_"-o json: -oo json-disks-pattern set more than once"); + let vars = + try Var_expander.scan_variables v + with Var_expander.Invalid_variable var -> + error (f_"-o json: -oo json-disks-pattern: invalid variable %%{%s}") + var in + List.iter ( + fun var -> + if not (List.mem var known_pattern_variables) then + error (f_"-o json: -oo json-disks-pattern: unhandled variable %%{%s}") + var + ) vars; + json_disks_pattern := Some v + | k, _ -> + error (f_"-o json: unknown output option ‘-oo %s’") k + ) options; + + let json_disks_pattern = + Option.default "%{GuestName}-%{DiskDeviceName}" !json_disks_pattern in + + { json_disks_pattern } + +class output_json dir json_options = object + inherit output + + method as_options = sprintf "-o json -os %s" dir + + method prepare_targets source overlays _ _ _ _ = + List.mapi ( + fun i (_, ov) -> + let outname = + let vars_fn = function + | "DiskNo" -> Some (string_of_int (i+1)) + | "DiskDeviceName" -> Some ov.ov_sd + | "GuestName" -> Some source.s_name + | _ -> assert false + in + Var_expander.replace_fn json_options.json_disks_pattern vars_fn in + let destname = dir // outname in + mkdir_p (Filename.dirname destname) 0o755; + TargetFile destname + ) overlays + + method supported_firmware = [ TargetBIOS; TargetUEFI ] + + method create_metadata source targets + target_buses guestcaps inspect target_firmware = + let doc = + Create_json.create_json_metadata source targets target_buses + guestcaps inspect target_firmware in + let doc_string = JSON.string_of_doc ~fmt:JSON.Indented doc in + + if verbose () then ( + eprintf "resulting JSON:\n"; + output_string stderr doc_string; + eprintf "\n\n%!"; + ); + + let name = source.s_name in + let file = dir // name ^ ".json" in + + with_open_out file ( + fun chan -> + output_string chan doc_string; + output_char chan '\n' + ) +end + +let output_json = new output_json +let () = Modules_list.register_output_module "json" diff --git a/v2v/output_json.mli b/v2v/output_json.mli new file mode 100644 index 000000000..52f58f2d1 --- /dev/null +++ b/v2v/output_json.mli @@ -0,0 +1,31 @@ +(* virt-v2v + * Copyright (C) 2019 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +(** [-o json] target. *) + +type json_options +(** Miscellaneous extra command line parameters used by json. *) + +val print_output_options : unit -> unit +val parse_output_options : (string * string) list -> json_options +(** Print and parse json -oo options. *) + +val output_json : string -> json_options -> Types.output +(** [output_json directory json_options] creates and returns a new + {!Types.output} object specialized for writing output to local + files with JSON metadata. *) diff --git a/v2v/virt-v2v-output-local.pod b/v2v/virt-v2v-output-local.pod index 7427b1ed7..ec737e1ba 100644 --- a/v2v/virt-v2v-output-local.pod +++ b/v2v/virt-v2v-output-local.pod @@ -11,6 +11,9 @@ or libvirt virt-v2v [-i* options] -o qemu -os DIRECTORY [--qemu-boot] + virt-v2v [-i* options] -o json -os DIRECTORY + [-oo json-disks-pattern=PATTERN] + virt-v2v [-i* options] -o null =head1 DESCRIPTION @@ -54,6 +57,13 @@ above, a shell script is created which contains the raw qemu command you would need to boot the guest. However the shell script is not run, I<unless> you also add the I<--qemu-boot> option. +=item B<-o json -os> C<DIRECTORY> + +This converts the guest to files in C<DIRECTORY>. The metadata +produced is a JSON file containing the majority of the data virt-v2v +gathers during the conversion. +See L</OUTPUT TO JSON> below. + =item B<-o null> The guest is converted, but the final result is thrown away and no @@ -140,6 +150,46 @@ Define the final guest in libvirt: =back +=head1 OUTPUT TO JSON + +The I<-o json> option produces the following files by default: + + NAME.json JSON metadata. + NAME-sda, NAME-sdb, etc. Guest disk(s). + +where C<NAME> is the guest name. + +It is possible to change the pattern of the disks using the +I<-oo json-disks-pattern=...> option: it allows parameters in form of +C<%{...}> variables, for example: + + -oo json-disks-pattern=disk%{DiskNo}.img + +Recognized variables are: + +=over 4 + +=item C<%{DiskNo}> + +The index of the disk, starting from 1. + +=item C<%{DiskDeviceName}> + +The destination device of the disk, e.g. C<sda>, C<sdb>, etc. + +=item C<%{GuestName}> + +The name of the guest. + +=back + +Using a pattern it is possible use subdirectories for the disks, +even with names depending on variables; for example: + + -oo json-disks-pattern=%{GuestName}-%{DiskNo}/disk.img + +The default pattern is C<%{GuestName}-%{DiskDeviceName}>. + =head1 SEE ALSO L<virt-v2v(1)>. diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod index cf9464834..9a555c3be 100644 --- a/v2v/virt-v2v.pod +++ b/v2v/virt-v2v.pod @@ -425,6 +425,17 @@ instead. Set the output method to OpenStack Glance. In this mode the converted guest is uploaded to Glance. See L<virt-v2v-output-openstack(1)>. +=item B<-o> B<json> + +Set the output method to I<json>. + +In this mode, the converted guest is written to a local directory +specified by I<-os /dir> (the directory must exist), with a JSON file +containing the majority of the metadata that virt-v2v gathered during +the conversion. + +See L<virt-v2v-output-local(1)>. + =item B<-o> B<libvirt> Set the output method to I<libvirt>. This is the default. @@ -696,8 +707,8 @@ The location of the storage for the converted guest. For I<-o libvirt>, this is a libvirt directory pool (see S<C<virsh pool-list>>) or pool UUID. -For I<-o local> and I<-o qemu>, this is a directory name. The -directory must exist. +For I<-o json>, I<-o local> and I<-o qemu>, this is a directory name. +The directory must exist. For I<-o rhv-upload>, this is the name of the destination Storage Domain. -- 2.20.1