Simple refactoring, giving the structures uniform names and handling. --- v2v/cmdline.ml | 6 +++--- v2v/input_libvirt.mli | 2 +- v2v/input_libvirt_vddk.ml | 12 ++++++++++++ v2v/input_libvirt_vddk.mli | 15 ++++++++++++++- v2v/output_vdsm.ml | 44 ++++++++++++++++++++++---------------------- v2v/output_vdsm.mli | 6 +++--- v2v/types.ml | 12 ------------ v2v/types.mli | 13 ------------- 8 files changed, 55 insertions(+), 55 deletions(-) diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml index 14c5c0dca..70e9a3c3b 100644 --- a/v2v/cmdline.ml +++ b/v2v/cmdline.ml @@ -329,7 +329,7 @@ read the man page virt-v2v(1). let qemu_boot = !qemu_boot in let root_choice = !root_choice in let vddk_options = - { vddk_config = !vddk_config; + { Input_libvirt_vddk.vddk_config = !vddk_config; vddk_cookie = !vddk_cookie; vddk_libdir = !vddk_libdir; vddk_nfchostport = !vddk_nfchostport; @@ -574,7 +574,7 @@ read the man page virt-v2v(1). | Some s -> s in if vdsm_image_uuids = [] || vdsm_vol_uuids = [] then error (f_"-o vdsm: either --vdsm-vol-uuid or --vdsm-vm-uuid was not specified"); - let vdsm_params = { + let vdsm_options = { Output_vdsm.image_uuids = vdsm_image_uuids; vol_uuids = vdsm_vol_uuids; vm_uuid = vdsm_vm_uuid; @@ -582,7 +582,7 @@ read the man page virt-v2v(1). compat = vdsm_compat; ovf_flavour = vdsm_ovf_flavour; } in - Output_vdsm.output_vdsm os vdsm_params output_alloc, + Output_vdsm.output_vdsm os vdsm_options output_alloc, output_format, output_alloc in { diff --git a/v2v/input_libvirt.mli b/v2v/input_libvirt.mli index 6a521f32e..6f9162482 100644 --- a/v2v/input_libvirt.mli +++ b/v2v/input_libvirt.mli @@ -18,7 +18,7 @@ (** [-i libvirt] source. *) -val input_libvirt : Types.vddk_options -> string option -> string option -> [`VDDK] option -> string -> Types.input +val input_libvirt : Input_libvirt_vddk.vddk_options -> string option -> string option -> [`VDDK] option -> string -> Types.input (** [input_libvirt vddk_options password libvirt_uri input_transport guest] creates and returns a new {!Types.input} object specialized for reading input from libvirt sources. *) diff --git a/v2v/input_libvirt_vddk.ml b/v2v/input_libvirt_vddk.ml index 1e1f5b6bd..a53f3e71d 100644 --- a/v2v/input_libvirt_vddk.ml +++ b/v2v/input_libvirt_vddk.ml @@ -33,6 +33,18 @@ open Xpath_helpers open Printf +type vddk_options = { + vddk_config : string option; + vddk_cookie : string option; + vddk_libdir : string option; + vddk_nfchostport : string option; + vddk_port : string option; + vddk_snapshot : string option; + vddk_thumbprint : string option; + vddk_transports : string option; + vddk_vimapiver : string option; +} + (* Subclass specialized for handling VMware via nbdkit vddk plugin. *) class input_libvirt_vddk vddk_options password libvirt_uri parsed_uri guest = (* The VDDK path. *) diff --git a/v2v/input_libvirt_vddk.mli b/v2v/input_libvirt_vddk.mli index 19a34c202..c8606c72a 100644 --- a/v2v/input_libvirt_vddk.mli +++ b/v2v/input_libvirt_vddk.mli @@ -18,7 +18,20 @@ (** [-i libvirt] when the source is VMware via nbdkit vddk plugin *) -val input_libvirt_vddk : Types.vddk_options -> string option -> string option -> Xml.uri -> string -> Types.input +type vddk_options = { + vddk_config : string option; + vddk_cookie : string option; + vddk_libdir : string option; + vddk_nfchostport : string option; + vddk_port : string option; + vddk_snapshot : string option; + vddk_thumbprint : string option; + vddk_transports : string option; + vddk_vimapiver : string option; +} +(** Various options passed through to the nbdkit vddk plugin unmodified. *) + +val input_libvirt_vddk : vddk_options -> string option -> string option -> Xml.uri -> string -> Types.input (** [input_libvirt_vddk vddk_options password libvirt_uri parsed_uri guest] creates and returns a {!Types.input} object specialized for reading the guest disks using the nbdkit vddk plugin. *) diff --git a/v2v/output_vdsm.ml b/v2v/output_vdsm.ml index 32615fa1f..b76a2e930 100644 --- a/v2v/output_vdsm.ml +++ b/v2v/output_vdsm.ml @@ -26,7 +26,7 @@ open Printf open Types open Utils -type vdsm_params = { +type vdsm_options = { image_uuids : string list; vol_uuids : string list; vm_uuid : string; @@ -35,22 +35,22 @@ type vdsm_params = { ovf_flavour : Create_ovf.ovf_flavour; } -class output_vdsm os vdsm_params output_alloc = +class output_vdsm os vdsm_options output_alloc = object inherit output method as_options = sprintf "-o vdsm -os %s%s%s --vdsm-vm-uuid %s --vdsm-ovf-output %s%s%s" os (String.concat "" - (List.map (sprintf " --vdsm-image-uuid %s") vdsm_params.image_uuids)) + (List.map (sprintf " --vdsm-image-uuid %s") vdsm_options.image_uuids)) (String.concat "" - (List.map (sprintf " --vdsm-vol-uuid %s") vdsm_params.vol_uuids)) - vdsm_params.vm_uuid - vdsm_params.ovf_output - (match vdsm_params.compat with + (List.map (sprintf " --vdsm-vol-uuid %s") vdsm_options.vol_uuids)) + vdsm_options.vm_uuid + vdsm_options.ovf_output + (match vdsm_options.compat with | "0.10" -> "" (* currently this is the default, so don't print it *) | s -> sprintf " --vdsm-compat=%s" s) - (match vdsm_params.ovf_flavour with + (match vdsm_options.ovf_flavour with | Create_ovf.OVirt -> "--vdsm-ovf-flavour=ovf" (* currently this is the default, so don't print it *) | Create_ovf.RHVExportStorageDomain -> "") @@ -82,8 +82,8 @@ object * displaying errors there. *) method prepare_targets _ targets = - if List.length vdsm_params.image_uuids <> List.length targets || - List.length vdsm_params.vol_uuids <> List.length targets then + if List.length vdsm_options.image_uuids <> List.length targets || + List.length vdsm_options.vol_uuids <> List.length targets then error (f_"the number of ‘--vdsm-image-uuid’ and ‘--vdsm-vol-uuid’ parameters passed on the command line has to match the number of guest disk images (for this guest: %d)") (List.length targets); @@ -111,14 +111,14 @@ object if not (is_directory d) then error (f_"image directory (%s) does not exist or is not a directory") d - ) vdsm_params.image_uuids; + ) vdsm_options.image_uuids; (* Note that VDSM has to create this directory too. *) - if not (is_directory vdsm_params.ovf_output) then + if not (is_directory vdsm_options.ovf_output) then error (f_"OVF (metadata) directory (%s) does not exist or is not a directory") - vdsm_params.ovf_output; + vdsm_options.ovf_output; - debug "VDSM: OVF (metadata) directory: %s" vdsm_params.ovf_output; + debug "VDSM: OVF (metadata) directory: %s" vdsm_options.ovf_output; (* The final directory structure should look like this: * /<MP>/<ESD_UUID>/images/ @@ -140,12 +140,12 @@ object debug "VDSM: will export %s to %s" ov_sd target_file; { t with target_file = TargetFile target_file } - ) (List.combine3 targets vdsm_params.image_uuids vdsm_params.vol_uuids) in + ) (List.combine3 targets vdsm_options.image_uuids vdsm_options.vol_uuids) in (* Generate the .meta files associated with each volume. *) let metas = Create_ovf.create_meta_files output_alloc dd_uuid - vdsm_params.image_uuids targets in + vdsm_options.image_uuids targets in List.iter ( fun ({ target_file }, meta) -> let target_file = @@ -166,7 +166,7 @@ object * nodes cannot handle qcow2 v3 (RHBZ#1145582, RHBZ#1400205). *) let compat = - if format <> "qcow2" then compat else Some vdsm_params.compat in + if format <> "qcow2" then compat else Some vdsm_options.compat in g#disk_create ?backingfile ?backingformat ?preallocation ?compat ?clustersize path format size @@ -178,13 +178,13 @@ object (* Create the metadata. *) let ovf = Create_ovf.create_ovf source targets guestcaps inspect output_alloc dd_uuid - vdsm_params.image_uuids - vdsm_params.vol_uuids - vdsm_params.vm_uuid - vdsm_params.ovf_flavour in + vdsm_options.image_uuids + vdsm_options.vol_uuids + vdsm_options.vm_uuid + vdsm_options.ovf_flavour in (* Write it to the metadata file. *) - let file = vdsm_params.ovf_output // vdsm_params.vm_uuid ^ ".ovf" in + let file = vdsm_options.ovf_output // vdsm_options.vm_uuid ^ ".ovf" in with_open_out file (fun chan -> DOM.doc_to_chan chan ovf) end diff --git a/v2v/output_vdsm.mli b/v2v/output_vdsm.mli index f7f6f59ee..6ed684638 100644 --- a/v2v/output_vdsm.mli +++ b/v2v/output_vdsm.mli @@ -18,7 +18,7 @@ (** [-o vdsm] target. *) -type vdsm_params = { +type vdsm_options = { image_uuids : string list; (* --vdsm-image-uuid (multiple) *) vol_uuids : string list; (* --vdsm-vol-uuid (multiple) *) vm_uuid : string; (* --vdsm-vm-uuid *) @@ -28,7 +28,7 @@ type vdsm_params = { } (** Miscellaneous extra command line parameters used by VDSM. *) -val output_vdsm : string -> vdsm_params -> Types.output_allocation -> Types.output -(** [output_vdsm os rhev_params output_alloc] creates and +val output_vdsm : string -> vdsm_options -> Types.output_allocation -> Types.output +(** [output_vdsm os vdsm_options output_alloc] creates and returns a new {!Types.output} object specialized for writing output to Data Domains directly under VDSM control. *) diff --git a/v2v/types.ml b/v2v/types.ml index c1f36fdb1..8c1a069e3 100644 --- a/v2v/types.ml +++ b/v2v/types.ml @@ -473,18 +473,6 @@ type root_choice = AskRoot | SingleRoot | FirstRoot | RootDev of string type output_allocation = Sparse | Preallocated -type vddk_options = { - vddk_config : string option; - vddk_cookie : string option; - vddk_libdir : string option; - vddk_nfchostport : string option; - vddk_port : string option; - vddk_snapshot : string option; - vddk_thumbprint : string option; - vddk_transports : string option; - vddk_vimapiver : string option; -} - class virtual input = object method precheck () = () method virtual as_options : string diff --git a/v2v/types.mli b/v2v/types.mli index a432e167c..c5f99a16b 100644 --- a/v2v/types.mli +++ b/v2v/types.mli @@ -339,19 +339,6 @@ type root_choice = AskRoot | SingleRoot | FirstRoot | RootDev of string type output_allocation = Sparse | Preallocated (** Type of [-oa] (output allocation) option. *) -type vddk_options = { - vddk_config : string option; - vddk_cookie : string option; - vddk_libdir : string option; - vddk_nfchostport : string option; - vddk_port : string option; - vddk_snapshot : string option; - vddk_thumbprint : string option; - vddk_transports : string option; - vddk_vimapiver : string option; -} -(** Various options passed through to the nbdkit vddk plugin unmodified. *) - (** {2 Input object} There is one of these used for the [-i] option. *) -- 2.13.2

On Thursday, 22 March 2018 16:24:22 CET Richard W.M. Jones wrote: Is it really used as global variable? -- Pino Toscano

Currently we have a bunch of ad hoc options like --vddk* and --vdsm* (and proposed to add --rhv*) to handle extra parameters for input and output modes/transports. This complicates the command line parsing and also the clarity of the command line (becauseit's not very obvious which options apply to which side of the conversion). Replace these with a general mechanism for handling input and output options. Thus (for example): --vddk-thumbprint=... becomes -io vddk-thumbprint=... --vdsm-compat=0.10 -oo vdsm-compat=0.10 The responsibility for parsing input and output options moves into the input and output drivers. This improves error checking so it's harder now for wrong flags to be included on the command line when they don't apply to the current mode. The old option names are preserved for compatibility. --- v2v/Makefile.am | 4 + v2v/cmdline.ml | 229 +++++++++++++++++----------------- v2v/input_libvirt.ml | 4 +- v2v/input_libvirt.mli | 4 +- v2v/input_libvirt_vddk.ml | 105 +++++++++------- v2v/input_libvirt_vddk.mli | 16 +-- v2v/output_vdsm.ml | 79 +++++++++++- v2v/output_vdsm.mli | 4 + v2v/test-v2v-docs.sh | 31 ++++- v2v/test-v2v-it-vddk-io-query.sh | 38 ++++++ v2v/test-v2v-o-vdsm-oo-query.sh | 38 ++++++ v2v/test-v2v-o-vdsm-options.sh | 16 +-- v2v/virt-v2v.pod | 258 +++++++++++++++++++++------------------ 13 files changed, 522 insertions(+), 304 deletions(-) diff --git a/v2v/Makefile.am b/v2v/Makefile.am index c2eb31097..6e71cae3c 100644 --- a/v2v/Makefile.am +++ b/v2v/Makefile.am @@ -306,6 +306,8 @@ TESTS = \ test-v2v-i-ova-tar.sh \ test-v2v-i-ova-two-disks.sh \ test-v2v-i-vmx.sh \ + test-v2v-it-vddk-io-query.sh \ + test-v2v-o-vdsm-oo-query.sh \ test-v2v-bad-networks-and-bridges.sh if HAVE_LIBVIRT @@ -471,6 +473,7 @@ EXTRA_DIST += \ test-v2v-i-vmx-4.vmx \ test-v2v-i-vmx-5.vmx \ test-v2v-in-place.sh \ + test-v2v-it-vddk-io-query.sh \ test-v2v-machine-readable.sh \ test-v2v-networks-and-bridges-expected.xml \ test-v2v-networks-and-bridges.sh \ @@ -482,6 +485,7 @@ EXTRA_DIST += \ test-v2v-o-qemu.sh \ test-v2v-o-rhv.ovf.expected \ test-v2v-o-rhv.sh \ + test-v2v-o-vdsm-oo-query.sh \ test-v2v-o-vdsm-options.ovf.expected \ test-v2v-o-vdsm-options.sh \ test-v2v-oa-option.sh \ diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml index 6aecd2aee..dc675fb42 100644 --- a/v2v/cmdline.ml +++ b/v2v/cmdline.ml @@ -69,24 +69,6 @@ let parse_cmdline () = let output_password = ref None in let output_storage = ref None in let password_file = ref None in - let vddk_config = ref None in - let vddk_cookie = ref None in - let vddk_libdir = ref None in - let vddk_nfchostport = ref None in - let vddk_port = ref None in - let vddk_snapshot = ref None in - let vddk_thumbprint = ref None in - let vddk_transports = ref None in - let vddk_vimapiver = ref None in - let vdsm_vm_uuid = ref None in - let vdsm_ovf_output = ref None in (* default "." *) - - let vdsm_compat = ref "0.10" in - let set_vdsm_compat s = vdsm_compat := s in - - let vdsm_ovf_flavour = ref Create_ovf.RHVExportStorageDomain in - let set_vdsm_ovf_flavour arg = - vdsm_ovf_flavour := Create_ovf.ovf_flavour_of_string arg in let set_string_option_once optname optref arg = match !optref with @@ -110,6 +92,15 @@ let parse_cmdline () = error (f_"unknown -i option: %s") s in + let input_options = ref [] in + let set_input_option_compat k v = + input_options := (k, v) :: !input_options + in + let set_input_option option = + let k, v = String.split "=" option in + set_input_option_compat k v + in + let network_map = ref NetworkMap.empty in let add_network, add_bridge = let add flag name t str = @@ -163,6 +154,15 @@ let parse_cmdline () = error (f_"unknown -oa option: %s") s in + let output_options = ref [] in + let set_output_option_compat k v = + output_options := (k, v) :: !output_options + in + let set_output_option option = + let k, v = String.split "=" option in + set_output_option_compat k v + in + let root_choice = ref AskRoot in let set_root_choice = function | "ask" -> root_choice := AskRoot @@ -173,12 +173,6 @@ let parse_cmdline () = error (f_"unknown --root option: %s") s in - let vdsm_image_uuids = ref [] in - let add_vdsm_image_uuid s = List.push_front s vdsm_image_uuids in - - let vdsm_vol_uuids = ref [] in - let add_vdsm_vol_uuid s = List.push_front s vdsm_vol_uuids in - let vmtype_warning _ = warning (f_"the --vmtype option has been removed and now does nothing") in @@ -201,6 +195,8 @@ let parse_cmdline () = s_"Libvirt URI"; [ M"if" ], Getopt.String ("format", set_string_option_once "-if" input_format), s_"Input format (for -i disk)"; + [ M"io" ], Getopt.String ("option[=value]", set_input_option), + s_"Set option for input mode"; [ M"it" ], Getopt.String ("transport", set_string_option_once "-it" input_transport), s_"Input transport"; [ L"in-place" ], Getopt.Set in_place, @@ -223,6 +219,8 @@ let parse_cmdline () = s_"Set output format"; [ M"on" ], Getopt.String ("name", set_string_option_once "-on" output_name), s_"Rename guest when converting"; + [ M"oo" ], Getopt.String ("option[=value]", set_output_option), + s_"Set option for output mode"; [ M"op" ], Getopt.String ("filename", set_string_option_once "-op" output_password), s_"Use password from file to connect to output hypervisor"; [ M"os" ], Getopt.String ("storage", set_string_option_once "-os" output_storage), @@ -236,36 +234,36 @@ let parse_cmdline () = [ L"qemu-boot" ], Getopt.Set qemu_boot, s_"Boot in qemu (-o qemu only)"; [ L"root" ], Getopt.String ("ask|... ", set_root_choice), s_"How to choose root filesystem"; - [ L"vddk-config" ], Getopt.String ("filename", set_string_option_once "--vddk-config" vddk_config), - s_"Set VDDK config file"; - [ L"vddk-cookie" ], Getopt.String ("cookie", set_string_option_once "--vddk-cookie" vddk_cookie), - s_"Set VDDK cookie"; - [ L"vddk-libdir" ], Getopt.String ("libdir", set_string_option_once "--vddk-libdir" vddk_libdir), - s_"Set VDDK library parent directory"; - [ L"vddk-nfchostport" ], Getopt.String ("nfchostport", set_string_option_once "--vddk-nfchostport" vddk_nfchostport), - s_"Set VDDK nfchostport"; - [ L"vddk-port" ], Getopt.String ("port", set_string_option_once "--vddk-port" vddk_port), - s_"Set VDDK port"; - [ L"vddk-snapshot" ], Getopt.String ("snapshot-moref", set_string_option_once "--vddk-snapshot" vddk_snapshot), - s_"Set VDDK snapshot"; - [ L"vddk-thumbprint" ], Getopt.String ("thumbprint", set_string_option_once "--vddk-thumbprint" vddk_thumbprint), - s_"Set VDDK thumbprint"; - [ L"vddk-transports" ], Getopt.String ("transports", set_string_option_once "--vddk-transports" vddk_transports), - s_"Set VDDK transports"; - [ L"vddk-vimapiver" ], Getopt.String ("apiver", set_string_option_once "--vddk-vimapiver" vddk_vimapiver), - s_"Set VDDK vimapiver"; - [ L"vdsm-compat" ], Getopt.Symbol ("0.10|1.1", ["0.10"; "1.1"], set_vdsm_compat), - s_"Write qcow2 with compat=0.10|1.1"; - [ L"vdsm-image-uuid" ], Getopt.String ("uuid", add_vdsm_image_uuid), - s_"Output image UUID(s)"; - [ L"vdsm-vol-uuid" ], Getopt.String ("uuid", add_vdsm_vol_uuid), - s_"Output vol UUID(s)"; - [ L"vdsm-vm-uuid" ], Getopt.String ("uuid", set_string_option_once "--vdsm-vm-uuid" vdsm_vm_uuid), - s_"Output VM UUID"; - [ L"vdsm-ovf-output" ], Getopt.String ("-", set_string_option_once "--vdsm-ovf-output" vdsm_ovf_output), - s_"Output OVF file"; - [ L"vdsm-ovf-flavour" ], Getopt.Symbol (ovf_flavours_str, Create_ovf.ovf_flavours, set_vdsm_ovf_flavour), - s_"Set the type of generated OVF (default rhvexp)"; + [ L"vddk-config" ], Getopt.String ("filename", set_input_option_compat "vddk-config"), + s_"Same as ‘-io vddk-config=filename’"; + [ L"vddk-cookie" ], Getopt.String ("cookie", set_input_option_compat "vddk-cookie"), + s_"Same as ‘-io vddk-cookie=filename’"; + [ L"vddk-libdir" ], Getopt.String ("libdir", set_input_option_compat "vddk-libdir"), + s_"Same as ‘-io vddk-libdir=libdir’"; + [ L"vddk-nfchostport" ], Getopt.String ("nfchostport", set_input_option_compat "vddk-nfchostport"), + s_"Same as ‘-io vddk-nfchostport=nfchostport’"; + [ L"vddk-port" ], Getopt.String ("port", set_input_option_compat "vddk-port"), + s_"Same as ‘-io vddk-port=port’"; + [ L"vddk-snapshot" ], Getopt.String ("snapshot-moref", set_input_option_compat "vddk-snapshot"), + s_"Same as ‘-io vddk-snapshot=snapshot-moref’"; + [ L"vddk-thumbprint" ], Getopt.String ("thumbprint", set_input_option_compat "vddk-thumbprint"), + s_"Same as ‘-io vddk-thumbprint=thumbprint’"; + [ L"vddk-transports" ], Getopt.String ("transports", set_input_option_compat "vddk-transports"), + s_"Same as ‘-io vddk-transports=transports’"; + [ L"vddk-vimapiver" ], Getopt.String ("apiver", set_input_option_compat "vddk-vimapiver"), + s_"Same as ‘-io vddk-vimapiver=apiver’"; + [ L"vdsm-compat" ], Getopt.String ("0.10|1.1", set_output_option_compat "vdsm-compat"), + s_"Same as ‘-oo vdsm-compat=0.10|1.1’"; + [ L"vdsm-image-uuid" ], Getopt.String ("uuid", set_output_option_compat "vdsm-image-uuid"), + s_"Same as ‘-oo vdsm-image-uuid=uuid’"; + [ L"vdsm-vol-uuid" ], Getopt.String ("uuid", set_output_option_compat "vdsm-vol-uuid"), + s_"Same as ‘-oo vdsm-vol-uuid=uuid’"; + [ L"vdsm-vm-uuid" ], Getopt.String ("uuid", set_output_option_compat "vdsm-vm-uuid"), + s_"Same as ‘-oo vdsm-vm-uuid=uuid’"; + [ L"vdsm-ovf-output" ], Getopt.String ("dir", set_output_option_compat "vdsm-ovf-output"), + s_"Same as ‘-oo vdsm-ovf-output=dir’"; + [ L"vdsm-ovf-flavour" ], Getopt.String (ovf_flavours_str, set_output_option_compat "vdsm-ovf-flavour"), + s_"Same as ‘-oo vdsm-ovf-flavour=flavour’"; [ L"vmtype" ], Getopt.String ("-", vmtype_warning), s_"Ignored for backwards compatibility"; ] in @@ -304,6 +302,7 @@ read the man page virt-v2v(1). let input_conn = !input_conn in let input_format = !input_format in let input_mode = !input_mode in + let input_options = List.rev !input_options in let input_transport = match !input_transport with | None -> None @@ -322,6 +321,7 @@ read the man page virt-v2v(1). let output_format = !output_format in let output_mode = !output_mode in let output_name = !output_name in + let output_options = List.rev !output_options in let output_password = !output_password in let output_storage = !output_storage in let password_file = !password_file in @@ -329,22 +329,6 @@ read the man page virt-v2v(1). let print_target = !print_target in let qemu_boot = !qemu_boot in let root_choice = !root_choice in - let vddk_options = - { Input_libvirt_vddk.vddk_config = !vddk_config; - vddk_cookie = !vddk_cookie; - vddk_libdir = !vddk_libdir; - vddk_nfchostport = !vddk_nfchostport; - vddk_port = !vddk_port; - vddk_snapshot = !vddk_snapshot; - vddk_thumbprint = !vddk_thumbprint; - vddk_transports = !vddk_transports; - vddk_vimapiver = !vddk_vimapiver } in - let vdsm_compat = !vdsm_compat in - let vdsm_image_uuids = List.rev !vdsm_image_uuids in - let vdsm_vol_uuids = List.rev !vdsm_vol_uuids in - let vdsm_vm_uuid = !vdsm_vm_uuid in - let vdsm_ovf_output = Option.default "." !vdsm_ovf_output in - let vdsm_ovf_flavour = !vdsm_ovf_flavour in (* No arguments and machine-readable mode? Print out some facts * about what this binary supports. @@ -358,6 +342,7 @@ read the man page virt-v2v(1). printf "colours-option\n"; printf "vdsm-compat-option\n"; printf "in-place\n"; + printf "io/oo\n"; List.iter (printf "input:%s\n") (Modules_list.input_modules ()); List.iter (printf "output:%s\n") (Modules_list.output_modules ()); List.iter (printf "convert:%s\n") (Modules_list.convert_modules ()); @@ -365,6 +350,65 @@ read the man page virt-v2v(1). exit 0 ); + (* Input transport affects whether some input options should or + * should not be used. + *) + let input_transport = + let is_query = input_options = ["?", ""] in + let no_options () = + if is_query then ( + printf (f_"No -io (input options) are supported with this input transport.\n"); + exit 0 + ) + else if input_options <> [] then + error (f_"no -io (input options) are allowed here"); + in + match input_transport with + | None -> no_options (); None + | Some `SSH -> no_options (); Some `SSH + | Some `VDDK -> + if is_query then ( + Input_libvirt_vddk.print_vddk_input_options (); + exit 0 + ) + else ( + let vddk_options = + Input_libvirt_vddk.parse_vddk_input_options input_options in + Some (`VDDK vddk_options) + ) in + + (* Output mode affects whether some output options should or + * should not be used. + *) + let output_mode = + let is_query = output_options = ["?", ""] in + let no_options () = + if is_query then ( + printf (f_"No -oo (output options) are supported in this output mode.\n"); + exit 0 + ) + else if output_options <> [] then + error (f_"no -oo (output options) are allowed here"); + in + match output_mode with + | `Not_set -> no_options (); `Not_set + | `Glance -> no_options (); `Glance + | `Libvirt -> no_options (); `Libvirt + | `Local -> no_options (); `Local + | `Null -> no_options (); `Null + | `RHV -> no_options (); `RHV + | `QEmu -> no_options (); `QEmu + | `VDSM -> + if is_query then ( + Output_vdsm.print_vdsm_output_options (); + exit 0 + ) + else ( + let vdsm_options = + Output_vdsm.parse_vdsm_output_options output_options in + `VDSM vdsm_options + ) in + (* Some options cannot be used with --in-place. *) if in_place then ( if print_target then @@ -379,27 +423,6 @@ read the man page virt-v2v(1). let password = read_first_line_from_file filename in Some password in - (* Input transport affects whether some parameters should or - * should not be used. - *) - (match input_transport with - | None - | Some `SSH -> - if !vddk_config <> None || - !vddk_cookie <> None || - !vddk_libdir <> None || - !vddk_nfchostport <> None || - !vddk_port <> None || - !vddk_snapshot <> None || - !vddk_thumbprint <> None || - !vddk_transports <> None || - !vddk_vimapiver <> None then - error (f_"‘--vddk-*’ options should only be used when conversion via the nbdkit VDDK plugin has been enabled, ie. using ‘-it vddk’.") - | Some `VDDK -> - if !vddk_thumbprint = None then - error (f_"‘--vddk-thumbprint’ is required when using ‘-it vddk’.") - ); - (* Parsing of the argument(s) depends on the input mode. *) let input = match input_mode with @@ -425,11 +448,10 @@ read the man page virt-v2v(1). let input_transport = match input_transport with | None -> None - | Some `VDDK -> Some `VDDK + | (Some (`VDDK _) as vddk) -> vddk | Some `SSH -> error (f_"only ‘-it vddk’ can be used here") in - Input_libvirt.input_libvirt vddk_options password - input_conn input_transport guest + Input_libvirt.input_libvirt password input_conn input_transport guest | `LibvirtXML -> (* -i libvirtxml: Expecting a filename (XML file). *) @@ -460,7 +482,7 @@ read the man page virt-v2v(1). match input_transport with | None -> None | Some `SSH -> Some `SSH - | Some `VDDK -> + | Some (`VDDK _) -> error (f_"only ‘-it ssh’ can be used here") in Input_vmx.input_vmx input_transport arg in @@ -558,7 +580,7 @@ read the man page virt-v2v(1). Output_rhv.output_rhv os output_alloc, output_format, output_alloc - | `VDSM -> + | `VDSM vdsm_options -> if output_password <> None then error_option_cannot_be_used_in_output_mode "vdsm" "-op"; let os = @@ -568,21 +590,6 @@ read the man page virt-v2v(1). | Some d -> d in if qemu_boot then error_option_cannot_be_used_in_output_mode "vdsm" "--qemu-boot"; - let vdsm_vm_uuid = - match vdsm_vm_uuid with - | None -> - error (f_"-o vdsm: --vdsm-image-uuid was not specified") - | Some s -> s in - if vdsm_image_uuids = [] || vdsm_vol_uuids = [] then - error (f_"-o vdsm: either --vdsm-vol-uuid or --vdsm-vm-uuid was not specified"); - let vdsm_options = { - Output_vdsm.image_uuids = vdsm_image_uuids; - vol_uuids = vdsm_vol_uuids; - vm_uuid = vdsm_vm_uuid; - ovf_output = vdsm_ovf_output; - compat = vdsm_compat; - ovf_flavour = vdsm_ovf_flavour; - } in Output_vdsm.output_vdsm os vdsm_options output_alloc, output_format, output_alloc in diff --git a/v2v/input_libvirt.ml b/v2v/input_libvirt.ml index 25c81b924..377257dc2 100644 --- a/v2v/input_libvirt.ml +++ b/v2v/input_libvirt.ml @@ -27,7 +27,7 @@ open Types open Utils (* Choose the right subclass based on the URI. *) -let input_libvirt vddk_options password libvirt_uri input_transport guest = +let input_libvirt password libvirt_uri input_transport guest = match libvirt_uri with | None -> Input_libvirt_other.input_libvirt_other password libvirt_uri guest @@ -53,7 +53,7 @@ let input_libvirt vddk_options password libvirt_uri input_transport guest = password libvirt_uri parsed_uri server guest (* vCenter or ESXi using nbdkit vddk plugin *) - | Some server, Some ("esx"|"gsx"|"vpx"), Some `VDDK -> + | Some server, Some ("esx"|"gsx"|"vpx"), Some (`VDDK vddk_options) -> Input_libvirt_vddk.input_libvirt_vddk vddk_options password libvirt_uri parsed_uri guest diff --git a/v2v/input_libvirt.mli b/v2v/input_libvirt.mli index 6f9162482..08824bb67 100644 --- a/v2v/input_libvirt.mli +++ b/v2v/input_libvirt.mli @@ -18,7 +18,7 @@ (** [-i libvirt] source. *) -val input_libvirt : Input_libvirt_vddk.vddk_options -> string option -> string option -> [`VDDK] option -> string -> Types.input -(** [input_libvirt vddk_options password libvirt_uri input_transport guest] +val input_libvirt : string option -> string option -> [`VDDK of Input_libvirt_vddk.vddk_options] option -> string -> Types.input +(** [input_libvirt password libvirt_uri input_transport guest] creates and returns a new {!Types.input} object specialized for reading input from libvirt sources. *) diff --git a/v2v/input_libvirt_vddk.ml b/v2v/input_libvirt_vddk.ml index a53f3e71d..fa253d242 100644 --- a/v2v/input_libvirt_vddk.ml +++ b/v2v/input_libvirt_vddk.ml @@ -33,22 +33,66 @@ open Xpath_helpers open Printf -type vddk_options = { - vddk_config : string option; - vddk_cookie : string option; - vddk_libdir : string option; - vddk_nfchostport : string option; - vddk_port : string option; - vddk_snapshot : string option; - vddk_thumbprint : string option; - vddk_transports : string option; - vddk_vimapiver : string option; -} +type vddk_options = (string * string) list + +(* List of vddk-* input options. *) +let vddk_option_keys = + [ "vddk-config"; + "vddk-cookie"; + "vddk-libdir"; + "vddk-nfchostport"; + "vddk-port"; + "vddk-snapshot"; + "vddk-thumbprint"; + "vddk-transports"; + "vddk-vimapiver" ] + +let print_vddk_input_options () = + printf (f_"Input options (-io) which can be used with -it vddk: + + -io vddk-thumbprint=xx:xx:xx:... + VDDK server thumbprint (required) + +All other settings are optional: + + -io vddk-config=FILE VDDK configuration file + -io vddk-cookie=COOKIE VDDK cookie + -io vddk-libdir=LIBDIR VDDK library parent directory + -io vddk-nfchostport=PORT VDDK nfchostport + -io vddk-port=PORT VDDK port + -io vddk-snapshot=SNAPSHOT-MOREF + VDDK snapshot moref + -io vddk-transports=MODE:MODE:.. + VDDK transports + -io vddk-vimapiver=APIVER VDDK vimapiver + +Refer to nbdkit-vddk-plugin(1) and the VDDK documentation for further +information on these settings. +") + +let parse_vddk_input_options options = + let keys = List.map fst options in + + (* Check there are no options we don't understand. *) + List.iter ( + fun key -> + if not (String.is_prefix key "vddk-") || + not (List.mem key vddk_option_keys) then + error (f_"-it vddk: ‘-io %s’ is not a valid input option") key + ) keys; + + (* Check no option appears twice. *) + if List.length keys <> List.length (List.sort_uniq keys) then + error (f_"-it vddk: duplicate -io options on the command line"); + + options (* Subclass specialized for handling VMware via nbdkit vddk plugin. *) class input_libvirt_vddk vddk_options password libvirt_uri parsed_uri guest = (* The VDDK path. *) - let libdir = vddk_options.vddk_libdir in + let libdir = + try Some (List.assoc "vddk-libdir" vddk_options) + with Not_found -> None in (* VDDK libraries are located under lib32/ or lib64/ relative to the * libdir. Note this is unrelated to Linux multilib or multiarch. @@ -68,7 +112,7 @@ class input_libvirt_vddk vddk_options password libvirt_uri parsed_uri guest = | None -> () | Some libdir -> if not (is_directory libdir) then - error (f_"‘--vddk-libdir %s’ does not point to a directory. See \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") libdir + error (f_"‘-io vddk-libdir=%s’ does not point to a directory. See \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") libdir ); (match library_path with @@ -122,15 +166,15 @@ See also \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") else error (f_"nbdkit VDDK plugin is not installed or not working. It is required if you want to use VDDK. -It looks like you did not set the right path in the ‘--vddk-libdir’ option, or your copy of the VDDK directory is incomplete. There should be a library called ’<libdir>/%s/libvixDiskLib.so.?’. +It looks like you did not set the right path in the ‘-io vddk-libdir’ option, or your copy of the VDDK directory is incomplete. There should be a library called ’<libdir>/%s/libvixDiskLib.so.?’. See also \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") libNN ) in let error_unless_thumbprint () = - if vddk_options.vddk_thumbprint = None then - error (f_"You must pass the ‘--vddk-thumbprint’ option with the SSL thumbprint of the VMware server. To find the thumbprint, see the nbdkit-vddk-plugin(1) manual. See also \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") + if not (List.mem_assoc "vddk-thumbprint" vddk_options) then + error (f_"You must pass the ‘-io vddk-thumbprint’ option with the SSL thumbprint of the VMware server. To find the thumbprint, see the nbdkit-vddk-plugin(1) manual. See also \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") in (* Check that nbdkit was compiled with SELinux support (for the @@ -147,18 +191,6 @@ See also \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") libNN error (f_"nbdkit was compiled without SELinux support. You will have to recompile nbdkit with libselinux-devel installed, or else set SELinux to Permissive mode while doing the conversion.") in - (* List of passthrough parameters. *) - let vddk_passthrus = - [ "config", (fun { vddk_config } -> vddk_config); - "cookie", (fun { vddk_cookie } -> vddk_cookie); - "libdir", (fun { vddk_libdir } -> vddk_libdir); - "nfchostport", (fun { vddk_nfchostport } -> vddk_nfchostport); - "port", (fun { vddk_port } -> vddk_port); - "snapshot", (fun { vddk_snapshot } -> vddk_snapshot); - "thumbprint", (fun { vddk_thumbprint } -> vddk_thumbprint); - "transports", (fun { vddk_transports } -> vddk_transports); - "vimapiver", (fun { vddk_vimapiver } -> vddk_vimapiver) ] in - object inherit input_libvirt password libvirt_uri guest as super @@ -172,14 +204,9 @@ object method as_options = let pt_options = - String.concat "" ( - List.map ( - fun (name, get_field) -> - match get_field vddk_options with - | None -> "" - | Some field -> sprintf " --vddk-%s %s" name field - ) vddk_passthrus - ) in + String.concat "" + (List.map (fun (k, v) -> + sprintf " -io vddk-%s=%s" k v) vddk_options) in sprintf "%s -it vddk %s" super#as_options (* superclass prints "-i libvirt etc" *) pt_options @@ -284,11 +311,7 @@ object add_arg (sprintf "vm=moref=%s" moref); (* The passthrough parameters. *) - List.iter ( - fun (name, get_field) -> - Option.may (fun field -> add_arg (sprintf "%s=%s" name field)) - (get_field vddk_options) - ) vddk_passthrus; + List.iter (fun (k, v) -> add_arg (sprintf "%s=%s" k v)) vddk_options; get_args () in diff --git a/v2v/input_libvirt_vddk.mli b/v2v/input_libvirt_vddk.mli index c8606c72a..ee2d6651a 100644 --- a/v2v/input_libvirt_vddk.mli +++ b/v2v/input_libvirt_vddk.mli @@ -18,19 +18,13 @@ (** [-i libvirt] when the source is VMware via nbdkit vddk plugin *) -type vddk_options = { - vddk_config : string option; - vddk_cookie : string option; - vddk_libdir : string option; - vddk_nfchostport : string option; - vddk_port : string option; - vddk_snapshot : string option; - vddk_thumbprint : string option; - vddk_transports : string option; - vddk_vimapiver : string option; -} +type vddk_options = (string * string) list (** Various options passed through to the nbdkit vddk plugin unmodified. *) +val print_vddk_input_options : unit -> unit +val parse_vddk_input_options : (string * string) list -> vddk_options +(** Print and parse vddk -io options. *) + val input_libvirt_vddk : vddk_options -> string option -> string option -> Xml.uri -> string -> Types.input (** [input_libvirt_vddk vddk_options password libvirt_uri parsed_uri guest] creates and returns a {!Types.input} object specialized for reading diff --git a/v2v/output_vdsm.ml b/v2v/output_vdsm.ml index b76a2e930..e540f5ec8 100644 --- a/v2v/output_vdsm.ml +++ b/v2v/output_vdsm.ml @@ -35,23 +35,90 @@ type vdsm_options = { ovf_flavour : Create_ovf.ovf_flavour; } +let ovf_flavours_str = String.concat "|" Create_ovf.ovf_flavours + +let print_vdsm_output_options () = + printf (f_"Output options (-oo) which can be used with -o vdsm: + + -oo vdsm-compat=0.10|1.1 Write qcow2 with compat=0.10|1.1 + (default: 0.10) + -oo vdsm-vm-uuid=UUID VM UUID (required) + -oo vdsm-ovf-output=DIR OVF metadata directory (required) + -oo vdsm-ovf-flavour=%s + Set the type of generated OVF (default: rhvexp) + +For each disk you must supply one of each of these options: + + -oo vdsm-image-uuid=UUID Image directory UUID + -oo vdsm-vol-uuid=UUID Disk volume UUID +") ovf_flavours_str + +let parse_vdsm_output_options options = + let vm_uuid = ref None in + let ovf_output = ref None in (* default "." *) + let compat = ref "0.10" in + let ovf_flavour = ref Create_ovf.RHVExportStorageDomain in + let image_uuids = ref [] in + let vol_uuids = ref [] in + + List.iter ( + function + | "vdsm-compat", "0.10" -> compat := "0.10" + | "vdsm-compat", "1.1" -> compat := "1.1" + | "vdsm-compat", v -> + error (f_"-o vdsm: unknown vdsm-compat level ‘%s’") v + | "vdsm-vm-uuid", v -> + if !vm_uuid <> None then + error (f_"-o vdsm: -oo vdsm-vm-uuid set twice"); + vm_uuid := Some v; + | "vdsm-ovf-output", v -> + if !ovf_output <> None then + error (f_"-o vdsm: -oo vdsm-ovf-output set twice"); + ovf_output := Some v; + | "vdsm-ovf-flavour", v -> + ovf_flavour := Create_ovf.ovf_flavour_of_string v + | "vdsm-image-uuid", v -> + List.push_front v image_uuids + | "vdsm-vol-uuid", v -> + List.push_front v vol_uuids + | k, _ -> + error (f_"-o vdsm: unknown output option ‘-oo %s’") k + ) options; + + let compat = !compat in + let image_uuids = List.rev !image_uuids in + let vol_uuids = List.rev !vol_uuids in + if image_uuids = [] || vol_uuids = [] then + error (f_"-o vdsm: either -oo vdsm-vol-uuid or -oo vdsm-vm-uuid was not specified"); + let vm_uuid = + match !vm_uuid with + | None -> + error (f_"-o vdsm: -oo vdsm-image-uuid was not specified") + | Some uuid -> uuid in + let ovf_output = Option.default "." !ovf_output in + let ovf_flavour = !ovf_flavour in + + { image_uuids; vol_uuids; vm_uuid; ovf_output; compat; ovf_flavour } + class output_vdsm os vdsm_options output_alloc = object inherit output method as_options = - sprintf "-o vdsm -os %s%s%s --vdsm-vm-uuid %s --vdsm-ovf-output %s%s%s" os + sprintf "-o vdsm -os %s%s%s -oo vdsm-vm-uuid=%s -oo vdsm-ovf-output=%s%s%s" os (String.concat "" - (List.map (sprintf " --vdsm-image-uuid %s") vdsm_options.image_uuids)) + (List.map (sprintf " -oo vdsm-image-uuid=%s") + vdsm_options.image_uuids)) (String.concat "" - (List.map (sprintf " --vdsm-vol-uuid %s") vdsm_options.vol_uuids)) + (List.map (sprintf " -oo vdsm-vol-uuid=%s") + vdsm_options.vol_uuids)) vdsm_options.vm_uuid vdsm_options.ovf_output (match vdsm_options.compat with | "0.10" -> "" (* currently this is the default, so don't print it *) - | s -> sprintf " --vdsm-compat=%s" s) + | s -> sprintf " -oo vdsm-compat=%s" s) (match vdsm_options.ovf_flavour with - | Create_ovf.OVirt -> "--vdsm-ovf-flavour=ovf" + | Create_ovf.OVirt -> "-oo vdsm-ovf-flavour=ovf" (* currently this is the default, so don't print it *) | Create_ovf.RHVExportStorageDomain -> "") @@ -84,7 +151,7 @@ object method prepare_targets _ targets = if List.length vdsm_options.image_uuids <> List.length targets || List.length vdsm_options.vol_uuids <> List.length targets then - error (f_"the number of ‘--vdsm-image-uuid’ and ‘--vdsm-vol-uuid’ parameters passed on the command line has to match the number of guest disk images (for this guest: %d)") + error (f_"the number of ‘-oo vdsm-image-uuid’ and ‘-oo vdsm-vol-uuid’ parameters passed on the command line has to match the number of guest disk images (for this guest: %d)") (List.length targets); let mp, uuid = diff --git a/v2v/output_vdsm.mli b/v2v/output_vdsm.mli index 6ed684638..90ce8736c 100644 --- a/v2v/output_vdsm.mli +++ b/v2v/output_vdsm.mli @@ -28,6 +28,10 @@ type vdsm_options = { } (** Miscellaneous extra command line parameters used by VDSM. *) +val print_vdsm_output_options : unit -> unit +val parse_vdsm_output_options : (string * string) list -> vdsm_options +(** Print and parse vdsm -oo options. *) + val output_vdsm : string -> vdsm_options -> Types.output_allocation -> Types.output (** [output_vdsm os vdsm_options output_alloc] creates and returns a new {!Types.output} object specialized for writing diff --git a/v2v/test-v2v-docs.sh b/v2v/test-v2v-docs.sh index 0e3bd916a..e1e22b599 100755 --- a/v2v/test-v2v-docs.sh +++ b/v2v/test-v2v-docs.sh @@ -22,4 +22,33 @@ $TEST_FUNCTIONS skip_if_skipped $top_srcdir/podcheck.pl virt-v2v.pod virt-v2v \ - --ignore=--debug-overlay,--ic,--if,--it,--no-trim,--oa,--oc,--of,--on,--op,--os,--vmtype + --ignore=\ +--debug-overlay,\ +--ic,\ +--if,\ +--io,\ +--it,\ +--no-trim,\ +--oa,\ +--oc,\ +--of,\ +--on,\ +--oo,\ +--op,\ +--os,\ +--vddk-config,\ +--vddk-cookie,\ +--vddk-libdir,\ +--vddk-nfchostport,\ +--vddk-port,\ +--vddk-snapshot,\ +--vddk-thumbprint,\ +--vddk-transports,\ +--vddk-vimapiver,\ +--vdsm-compat,\ +--vdsm-image-uuid,\ +--vdsm-ovf-flavour,\ +--vdsm-ovf-output,\ +--vdsm-vm-uuid,\ +--vdsm-vol-uuid,\ +--vmtype diff --git a/v2v/test-v2v-it-vddk-io-query.sh b/v2v/test-v2v-it-vddk-io-query.sh new file mode 100755 index 000000000..014e30207 --- /dev/null +++ b/v2v/test-v2v-it-vddk-io-query.sh @@ -0,0 +1,38 @@ +#!/bin/bash - +# libguestfs virt-v2v test script +# Copyright (C) 2018 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Test -io "?" option. + +set -e + +$TEST_FUNCTIONS +skip_if_skipped + +export VIRT_TOOLS_DATA_DIR="$top_srcdir/test-data/fake-virt-tools" +export VIRTIO_WIN="$top_srcdir/test-data/fake-virtio-win" + +f=test-v2v-it-vddk-io-query.actual +rm -f $f + +$VG virt-v2v --debug-gc \ + -it vddk -io "?" > $f + +grep -- "-io vddk-config" $f +grep -- "-io vddk-thumbprint" $f + +rm $f diff --git a/v2v/test-v2v-o-vdsm-oo-query.sh b/v2v/test-v2v-o-vdsm-oo-query.sh new file mode 100755 index 000000000..5691446ea --- /dev/null +++ b/v2v/test-v2v-o-vdsm-oo-query.sh @@ -0,0 +1,38 @@ +#!/bin/bash - +# libguestfs virt-v2v test script +# Copyright (C) 2018 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Test -oo "?" option. + +set -e + +$TEST_FUNCTIONS +skip_if_skipped + +export VIRT_TOOLS_DATA_DIR="$top_srcdir/test-data/fake-virt-tools" +export VIRTIO_WIN="$top_srcdir/test-data/fake-virtio-win" + +f=test-v2v-o-vdsm-oo-query.actual +rm -f $f + +$VG virt-v2v --debug-gc \ + -o vdsm -oo "?" > $f + +grep -- "-oo vdsm-compat" $f +grep -- "-oo vdsm-image-uuid" $f + +rm $f diff --git a/v2v/test-v2v-o-vdsm-options.sh b/v2v/test-v2v-o-vdsm-options.sh index 106f8694e..ddecfe3a1 100755 --- a/v2v/test-v2v-o-vdsm-options.sh +++ b/v2v/test-v2v-o-vdsm-options.sh @@ -16,7 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# Test -o vdsm options --vdsm-*-uuid +# Test -o vdsm options -oo vdsm-*-uuid set -e set -x @@ -43,19 +43,19 @@ mkdir $d/12345678-1234-1234-1234-123456789abc/master mkdir $d/12345678-1234-1234-1234-123456789abc/master/vms mkdir $d/12345678-1234-1234-1234-123456789abc/master/vms/VM -# The --vdsm-*-uuid options don't actually check that the +# The -oo vdsm-*-uuid options don't actually check that the # parameter is a UUID, which is useful here. $VG virt-v2v --debug-gc \ -i libvirt -ic "$libvirt_uri" windows \ -o vdsm -os $d/12345678-1234-1234-1234-123456789abc \ -of qcow2 \ - --vdsm-image-uuid IMAGE \ - --vdsm-vol-uuid VOL \ - --vdsm-vm-uuid VM \ - --vdsm-ovf-output $d/12345678-1234-1234-1234-123456789abc/master/vms/VM \ - --vdsm-compat=1.1 \ - --vdsm-ovf-flavour=ovirt + -oo vdsm-image-uuid=IMAGE \ + -oo vdsm-vol-uuid=VOL \ + -oo vdsm-vm-uuid=VM \ + -oo vdsm-ovf-output=$d/12345678-1234-1234-1234-123456789abc/master/vms/VM \ + -oo vdsm-compat=1.1 \ + -oo vdsm-ovf-flavour=ovirt # Test the OVF metadata was created. test -f $d/12345678-1234-1234-1234-123456789abc/master/vms/VM/VM.ovf diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod index db7ac5166..4e3daedf0 100644 --- a/v2v/virt-v2v.pod +++ b/v2v/virt-v2v.pod @@ -397,6 +397,47 @@ See L</IN PLACE CONVERSION> below. Conflicts with all I<-o *> options. +=item B<-io> OPTION=VALUE + +Set input option(s) related to the current input mode or transport. +To display short help on what options are available you can use: + + virt-v2v -it vddk -io "?" + +=item B<-io vddk-libdir=>LIBDIR + +Set the VDDK library directory. This directory should I<contain> +subdirectories called F<include>, F<lib64> etc., but do not include +F<lib64> actually in the parameter. + +In most cases this parameter is required when using the I<-it vddk> +(VDDK) transport. See L</INPUT FROM VDDK> below for details. + +=item B<-io vddk-thumbprint=>xx:xx:xx:... + +Set the thumbprint of the remote VMware server. + +This parameter is required when using the I<-it vddk> (VDDK) transport. +See L</INPUT FROM VDDK> below for details. + +=item B<-io vddk-config=>FILENAME + +=item B<-io vddk-cookie=>COOKIE + +=item B<-io vddk-nfchostport=>PORT + +=item B<-io vddk-port=>PORT + +=item B<-io vddk-snapshot=>SNAPSHOT-MOREF + +=item B<-io vddk-transports=>MODE:MODE:... + +=item B<-io vddk-vimapiver=>APIVER + +When using VDDK mode, these options are passed unmodified to the +L<nbdkit(1)> VDDK plugin. Please refer to L<nbdkit-vddk-plugin(1)>. +These are all optional. + =item B<-it> B<ssh> When using I<-i vmx>, this enables the ssh transport. @@ -406,7 +447,7 @@ See L</INPUT FROM VMWARE VMX> below. Use VMware VDDK as a transport to copy the input disks. See L</INPUT FROM VDDK> below. If you use this parameter then you may -need to use other I<--vddk*> options to specify how to connect through +need to use other I<-io vddk*> options to specify how to connect through VDDK. =item B<--keys-from-stdin> @@ -569,6 +610,95 @@ If not specified, then the input format is used. Rename the guest when converting it. If this option is not used then the output name is the same as the input name. +=item B<-oo> OPTION=VALUE + +Set output option(s) related to the current output mode. +To display short help on what options are available you can use: + + virt-v2v -o vdsm -oo "?" + +=item B<-oo vdsm-compat=0.10> + +=item B<-oo vdsm-compat=1.1> + +If I<-o vdsm> and the output format is qcow2, then we add the qcow2 +I<compat=0.10> option to the output file for compatibility with RHEL 6 +(see L<https://bugzilla.redhat.com/1145582>). + +If I<-oo vdsm-compat=1.1> is used then modern qcow2 (I<compat=1.1>) +files are generated instead. + +Currently I<-oo vdsm-compat=0.10> is the default, but this will change +to I<-oo vdsm-compat=1.1> in a future version of virt-v2v (when we can +assume that everyone is using a modern version of qemu). + +B<Note this option only affects I<-o vdsm> output>. All other output +modes (including I<-o rhv>) generate modern qcow2 I<compat=1.1> +files, always. + +If this option is available, then C<vdsm-compat-option> will appear in +the I<--machine-readable> output. + +=item B<-oo vdsm-image-uuid=>UUID + +=item B<-oo vdsm-vol-uuid=>UUID + +=item B<-oo vdsm-vm-uuid=>UUID + +=item B<-oo vdsm-ovf-output=>DIR + +Normally the RHV output mode chooses random UUIDs for the target +guest. However VDSM needs to control the UUIDs and passes these +parameters when virt-v2v runs under VDSM control. The parameters +control: + +=over 4 + +=item * + +the image directory of each guest disk (I<-oo vdsm-image-uuid>) (this +option is passed once for each guest disk) + +=item * + +UUIDs for each guest disk (I<-oo vdsm-vol-uuid>) (this option +is passed once for each guest disk) + +=item * + +the OVF file name (I<-oo vdsm-vm-uuid>). + +=item * + +the OVF output directory (default current directory) (I<-oo vdsm-ovf-output>). + +=back + +The format of UUIDs is: C<12345678-1234-1234-1234-123456789abc> (each +hex digit can be C<0-9> or C<a-f>), conforming to S<OSF DCE 1.1>. + +These options can only be used with I<-o vdsm>. + +=item B<-oo vdsm-ovf-flavour=>flavour + +This option controls the format of the OVF generated at the end of conversion. +Currently there are two possible flavours: + +=over 4 + +=item rhevexp + +The OVF format used in RHV export storage domain. + +=item ovirt + +The OVF format understood by oVirt REST API. + +=back + +For backward compatibility the default is I<rhevexp>, but this may change in +the future. + =item B<-op> file Supply a file containing a password to be used when connecting to the @@ -675,122 +805,6 @@ boot an operating system from the first virtio disk. Specifically, F</boot> must be on the first virtio disk, and it cannot chainload an OS which is not in the first virtio disk. -=item B<--vddk-libdir> LIBDIR - -Set the VDDK library directory. This directory should I<contain> -subdirectories called F<include>, F<lib64> etc., but do not include -F<lib64> actually in the parameter. - -In most cases this parameter is required when using the I<-it vddk> -(VDDK) transport. See L</INPUT FROM VDDK> below for details. - -=item B<--vddk-thumbprint> xx:xx:xx:... - -Set the thumbprint of the remote VMware server. - -This parameter is required when using the I<-it vddk> (VDDK) transport. -See L</INPUT FROM VDDK> below for details. - -=item B<--vddk-config> FILENAME - -=item B<--vddk-cookie> COOKIE - -=item B<--vddk-nfchostport> PORT - -=item B<--vddk-port> PORT - -=item B<--vddk-snapshot> SNAPSHOT-MOREF - -=item B<--vddk-transports> MODE:MODE:... - -=item B<--vddk-vimapiver> APIVER - -When using VDDK mode, these options are passed unmodified to the -L<nbdkit(1)> VDDK plugin. Please refer to L<nbdkit-vddk-plugin(1)>. -These are all optional. - -=item B<--vdsm-compat=0.10> - -=item B<--vdsm-compat=1.1> - -If I<-o vdsm> and the output format is qcow2, then we add the qcow2 -I<compat=0.10> option to the output file for compatibility with RHEL 6 -(see L<https://bugzilla.redhat.com/1145582>). - -If I<--vdsm-compat=1.1> is used then modern qcow2 (I<compat=1.1>) -files are generated instead. - -Currently I<--vdsm-compat=0.10> is the default, but this will change -to I<--vdsm-compat=1.1> in a future version of virt-v2v (when we can -assume that everyone is using a modern version of qemu). - -B<Note this option only affects I<-o vdsm> output>. All other output -modes (including I<-o rhv>) generate modern qcow2 I<compat=1.1> -files, always. - -If this option is available, then C<vdsm-compat-option> will appear in -the I<--machine-readable> output. - -=item B<--vdsm-image-uuid> UUID - -=item B<--vdsm-vol-uuid> UUID - -=item B<--vdsm-vm-uuid> UUID - -=item B<--vdsm-ovf-output> - -Normally the RHV output mode chooses random UUIDs for the target -guest. However VDSM needs to control the UUIDs and passes these -parameters when virt-v2v runs under VDSM control. The parameters -control: - -=over 4 - -=item * - -the image directory of each guest disk (I<--vdsm-image-uuid>) (this -option is passed once for each guest disk) - -=item * - -UUIDs for each guest disk (I<--vdsm-vol-uuid>) (this option -is passed once for each guest disk) - -=item * - -the OVF file name (I<--vdsm-vm-uuid>). - -=item * - -the OVF output directory (default current directory) (I<--vdsm-ovf-output>). - -=back - -The format of UUIDs is: C<12345678-1234-1234-1234-123456789abc> (each -hex digit can be C<0-9> or C<a-f>), conforming to S<OSF DCE 1.1>. - -These options can only be used with I<-o vdsm>. - -=item B<--vdsm-ovf-flavour> flavour - -This option controls the format of the OVF generated at the end of conversion. -Currently there are two possible flavours: - -=over 4 - -=item rhevexp - -The OVF format used in RHV export storage domain. - -=item ovirt - -The OVF format understood by oVirt REST API. - -=back - -For backward compatibility the default is I<rhevexp>, but this may change in -the future. - =item B<-v> =item B<--verbose> @@ -1667,15 +1681,15 @@ SSL thumbprint: $ virt-v2v \ -ic 'vpx://root@vcenter.example.com/Datacenter/esxi?no_verify=1' \ -it vddk \ - --vddk-libdir /path/to/vmware-vix-disklib-distrib \ - --vddk-thumbprint xx:xx:xx:... \ + -io vddk-libdir=/path/to/vmware-vix-disklib-distrib \ + -io vddk-thumbprint=xx:xx:xx:... \ "Windows 2003" \ -o local -os /var/tmp Other options that you might need to add in rare circumstances include -I<--vddk-config>, I<--vddk-cookie>, I<--vddk-nfchostport>, -I<--vddk-port>, I<--vddk-snapshot>, I<--vddk-transports> and -I<--vddk-vimapiver>, which are all explained in the +I<-io vddk-config>, I<-io vddk-cookie>, I<-io vddk-nfchostport>, +I<-io vddk-port>, I<-io vddk-snapshot>, I<-io vddk-transports> and +I<-io vddk-vimapiver>, which are all explained in the L<nbdkit-vddk-plugin(1)> documentation. =head2 VDDK: DEBUGGING VDDK FAILURES -- 2.13.2

PROBLEMS: - -of qcow2 does not work, with multiple problems * needs to set NBD size to something larger than virtual size - Cannot choose the datacenter. - Not tested against imageio which supports zero/trim/flush. This adds a new output mode to virt-v2v. virt-v2v -o rhv-upload streams images directly to an oVirt or RHV >= 4 Data Domain using the oVirt SDK v4. It is more efficient than -o rhv because it does not need to go via the Export Storage Domain, and is possible for humans to use unlike -o vdsm. The implementation uses the Python SDK (‘ovirtsdk4’ module). An nbdkit Python 3 plugin translates NBD calls from qemu into HTTPS requests to oVirt via the SDK. --- .gitignore | 3 + v2v/Makefile.am | 30 ++- v2v/cmdline.ml | 36 +++ v2v/embed.sh | 45 ++++ v2v/output_rhv_upload.ml | 390 ++++++++++++++++++++++++++++ v2v/output_rhv_upload.mli | 33 +++ v2v/output_rhv_upload_createvm_source.mli | 19 ++ v2v/output_rhv_upload_plugin_source.mli | 19 ++ v2v/output_rhv_upload_precheck_source.mli | 19 ++ v2v/rhv-upload-createvm.py | 85 ++++++ v2v/rhv-upload-plugin.py | 414 ++++++++++++++++++++++++++++++ v2v/rhv-upload-precheck.py | 72 ++++++ v2v/test-v2v-o-rhv-upload-oo-query.sh | 38 +++ v2v/test-v2v-python-syntax.sh | 27 ++ v2v/virt-v2v.pod | 132 ++++++++-- 15 files changed, 1344 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index d72447d1d..211376eef 100644 --- a/.gitignore +++ b/.gitignore @@ -654,6 +654,9 @@ Makefile.in /utils/qemu-speed-test/qemu-speed-test /v2v/.depend /v2v/oUnit-* +/v2v/output_rhv_upload_createvm_source.ml +/v2v/output_rhv_upload_plugin_source.ml +/v2v/output_rhv_upload_precheck_source.ml /v2v/real-*.d/ /v2v/real-*.img /v2v/real-*.xml diff --git a/v2v/Makefile.am b/v2v/Makefile.am index 6e71cae3c..f36731750 100644 --- a/v2v/Makefile.am +++ b/v2v/Makefile.am @@ -22,12 +22,19 @@ generator_built = \ uefi.mli BUILT_SOURCES = \ - $(generator_built) + $(generator_built) \ + output_rhv_upload_createvm_source.ml \ + output_rhv_upload_plugin_source.ml \ + output_rhv_upload_precheck_source.ml EXTRA_DIST = \ $(SOURCES_MLI) $(SOURCES_ML) $(SOURCES_C) \ copy_to_local.ml \ copy_to_local.mli \ + embed-code.sh \ + rhv-upload-createvm.py \ + rhv-upload-plugin.py \ + rhv-upload-precheck.py \ v2v_slow_unit_tests.ml \ v2v-slow-unit-tests.sh \ v2v_unit_tests.ml \ @@ -64,6 +71,10 @@ SOURCES_MLI = \ output_null.mli \ output_qemu.mli \ output_rhv.mli \ + output_rhv_upload.mli \ + output_rhv_upload_createvm_source.mli \ + output_rhv_upload_plugin_source.mli \ + output_rhv_upload_precheck_source.mli \ output_vdsm.mli \ parse_ovf_from_ova.mli \ parse_libvirt_xml.mli \ @@ -116,6 +127,10 @@ SOURCES_ML = \ output_local.ml \ output_qemu.ml \ output_rhv.ml \ + output_rhv_upload_createvm_source.ml \ + output_rhv_upload_plugin_source.ml \ + output_rhv_upload_precheck_source.ml \ + output_rhv_upload.ml \ output_vdsm.ml \ inspect_source.ml \ target_bus_assignment.ml \ @@ -126,6 +141,15 @@ SOURCES_C = \ libvirt_utils-c.c \ qemuopts-c.c +# These files are generated and contain rhv-upload-*.py embedded as an +# OCaml string. +output_rhv_upload_createvm_source.ml: rhv-upload-createvm.py + ./embed.sh code $^ $@ +output_rhv_upload_plugin_source.ml: rhv-upload-plugin.py + ./embed.sh code $^ $@ +output_rhv_upload_precheck_source.ml: rhv-upload-precheck.py + ./embed.sh code $^ $@ + if HAVE_OCAML bin_PROGRAMS = virt-v2v virt-v2v-copy-to-local @@ -295,6 +319,7 @@ TESTS_ENVIRONMENT = $(top_builddir)/run --test TESTS = \ test-v2v-docs.sh \ + test-v2v-python-syntax.sh \ test-v2v-i-ova-bad-sha1.sh \ test-v2v-i-ova-bad-sha256.sh \ test-v2v-i-ova-formats.sh \ @@ -307,6 +332,7 @@ TESTS = \ test-v2v-i-ova-two-disks.sh \ test-v2v-i-vmx.sh \ test-v2v-it-vddk-io-query.sh \ + test-v2v-o-rhv-upload-oo-query.sh \ test-v2v-o-vdsm-oo-query.sh \ test-v2v-bad-networks-and-bridges.sh @@ -485,6 +511,7 @@ EXTRA_DIST += \ test-v2v-o-qemu.sh \ test-v2v-o-rhv.ovf.expected \ test-v2v-o-rhv.sh \ + test-v2v-o-rhv-upload-oo-query.sh \ test-v2v-o-vdsm-oo-query.sh \ test-v2v-o-vdsm-options.ovf.expected \ test-v2v-o-vdsm-options.sh \ @@ -494,6 +521,7 @@ EXTRA_DIST += \ test-v2v-print-source.expected \ test-v2v-print-source.sh \ test-v2v-print-source.xml \ + test-v2v-python-syntax.sh \ test-v2v-conversion-of.sh \ test-v2v-sound.sh \ test-v2v-sound.xml \ diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml index dc675fb42..bdd3127f8 100644 --- a/v2v/cmdline.ml +++ b/v2v/cmdline.ml @@ -137,6 +137,8 @@ let parse_cmdline () = | "disk" | "local" -> output_mode := `Local | "null" -> output_mode := `Null | "ovirt" | "rhv" | "rhev" -> output_mode := `RHV + | "ovirt-upload" | "ovirt_upload" | "rhv-upload" | "rhv_upload" -> + output_mode := `RHV_Upload | "qemu" -> output_mode := `QEmu | "vdsm" -> output_mode := `VDSM | s -> @@ -398,6 +400,16 @@ read the man page virt-v2v(1). | `Null -> no_options (); `Null | `RHV -> no_options (); `RHV | `QEmu -> no_options (); `QEmu + | `RHV_Upload -> + if is_query then ( + Output_rhv_upload.print_rhv_output_options (); + exit 0 + ) + else ( + let rhv_options = + Output_rhv_upload.parse_rhv_output_options output_options in + `RHV_Upload rhv_options + ) | `VDSM -> if is_query then ( Output_vdsm.print_vdsm_output_options (); @@ -580,6 +592,30 @@ read the man page virt-v2v(1). Output_rhv.output_rhv os output_alloc, output_format, output_alloc + | `RHV_Upload rhv_options -> + let output_conn = + match output_conn with + | None -> + error (f_"-o rhv-upload: use ‘-oc’ to point to the oVirt or RHV server REST API URL, which is usually https://servername/ovirt-engine/api") + | Some oc -> oc in + (* In theory we could make the password optional in future. *) + let output_password = + match output_password with + | None -> + error (f_"-o rhv-upload: output password file was not specified, use ‘-op’ to point to a file which contains the password used to connect to the oVirt or RHV server") + | Some op -> op in + let os = + match output_storage with + | None -> + error (f_"-o rhv-upload: output storage was not specified, use ‘-os’"); + | Some os -> os in + if qemu_boot then + error_option_cannot_be_used_in_output_mode "rhv-upload" "--qemu-boot"; + Output_rhv_upload.output_rhv_upload output_alloc output_conn + output_password os + rhv_options, + output_format, output_alloc + | `VDSM vdsm_options -> if output_password <> None then error_option_cannot_be_used_in_output_mode "vdsm" "-op"; diff --git a/v2v/embed.sh b/v2v/embed.sh new file mode 100755 index 000000000..0a65cd428 --- /dev/null +++ b/v2v/embed.sh @@ -0,0 +1,45 @@ +#!/bin/bash - +# Embed code or other content into an OCaml file. +# Copyright (C) 2018 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Embed code or other content into an OCaml file. +# +# It is embedded into a string. As OCaml string literals have virtually +# no restrictions on length or content we only have to escape double +# quotes for backslash characters. + +if [ $# -ne 3 ]; then + echo "embed.sh identifier input output" + exit 1 +fi + +ident="$1" +input="$2" +output="$3" + +rm -f "$output" "$output"-t + +exec >"$output"-t + +echo "(* Generated by embed.sh from $input *)" +echo +echo let "$ident" = '"' +sed -e 's/\(["\]\)/\\\1/g' < "$input" +echo '"' + +chmod -w "$output"-t +mv "$output"-t "$output" diff --git a/v2v/output_rhv_upload.ml b/v2v/output_rhv_upload.ml new file mode 100644 index 000000000..0e2dbd78b --- /dev/null +++ b/v2v/output_rhv_upload.ml @@ -0,0 +1,390 @@ +(* virt-v2v + * Copyright (C) 2009-2018 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Printf +open Unix + +open Std_utils +open Tools_utils +open Unix_utils +open Common_gettext.Gettext + +open Types +open Utils + +type rhv_options = { + rhv_cafile : string; + rhv_cluster : string option; + rhv_direct : bool; + rhv_verifypeer : bool; +} + +let print_rhv_output_options () = + printf (f_"Output options (-oo) which can be used with -o rhv-upload: + + -oo rhv-cafile=CA.PEM Set ‘ca.pem’ certificate bundle filename. + -oo rhv-cluster=CLUSTERNAME Set RHV cluster name. + -oo rhv-direct[=true|false] Use direct transfer mode (default: false). + -oo rhv-verifypeer[=true|false] Verify server identity (default: false). +") + +let parse_rhv_output_options options = + let rhv_cafile = ref None in + let rhv_cluster = ref None in + let rhv_direct = ref false in + let rhv_verifypeer = ref false in + + List.iter ( + function + | "rhv-cafile", v -> + if !rhv_cafile <> None then + error (f_"-o rhv-upload: -oo rhv-cafile set twice"); + rhv_cafile := Some v + | "rhv-cluster", v -> + if !rhv_cluster <> None then + error (f_"-o rhv-upload: -oo rhv-cluster set twice"); + rhv_cluster := Some v + | "rhv-direct", "" -> rhv_direct := true + | "rhv-direct", v -> rhv_direct := bool_of_string v + | "rhv-verifypeer", "" -> rhv_verifypeer := true + | "rhv-verifypeer", v -> rhv_verifypeer := bool_of_string v + | k, _ -> + error (f_"-o rhv-upload: unknown output option ‘-oo %s’") k + ) options; + + let rhv_cafile = + match !rhv_cafile with + | Some s -> s + | None -> + error (f_"-o rhv-upload: must use ‘-oo rhv-cafile’ to supply the path to the oVirt or RHV user’s ‘ca.pem’ file") in + let rhv_cluster = !rhv_cluster in + let rhv_direct = !rhv_direct in + let rhv_verifypeer = !rhv_verifypeer in + + { rhv_cafile; rhv_cluster; rhv_direct; rhv_verifypeer } + +let python3 = "python3" (* Defined by PEP 394 *) +let pidfile_timeout = 30 +let finalization_timeout = 5*60 + +class output_rhv_upload output_alloc output_conn + output_password output_storage + rhv_options = + (* Create a temporary directory which will be deleted on exit. *) + let tmpdir = + let base_dir = (open_guestfs ())#get_cachedir () in + let t = Mkdtemp.temp_dir ~base_dir "rhvupload." in + rmdir_on_exit t; + t in + + let diskid_file_of_id id = tmpdir // sprintf "diskid.%d" id in + + (* Write the Python precheck, plugin and create VM to a temporary file. *) + let precheck = + let precheck = tmpdir // "rhv-upload-precheck.py" in + with_open_out + precheck + (fun chan -> output_string chan Output_rhv_upload_precheck_source.code); + precheck in + let plugin = + let plugin = tmpdir // "rhv-upload-plugin.py" in + with_open_out + plugin + (fun chan -> output_string chan Output_rhv_upload_plugin_source.code); + plugin in + let createvm = + let createvm = tmpdir // "rhv-upload-createvm.py" in + with_open_out + createvm + (fun chan -> output_string chan Output_rhv_upload_createvm_source.code); + createvm in + + (* Is SELinux enabled and enforcing on the host? *) + let have_selinux = + 0 = Sys.command "getenforce 2>/dev/null | grep -isq Enforcing" in + + (* Check that nbdkit is available and new enough. *) + let error_unless_nbdkit_working () = + if 0 <> Sys.command "nbdkit --version >/dev/null" then + error (f_"nbdkit is not installed or not working. It is required to use ‘-o rhv-upload’. See \"OUTPUT TO RHV\" in the virt-v2v(1) manual."); + + (* Check it's a new enough version. The latest features we + * require are ‘--exit-with-parent’ and ‘--selinux-label’, both + * added in 1.1.14. (We use 1.1.16 as the minimum here because + * it also adds the selinux=yes|no flag in --dump-config). + *) + let lines = external_command "nbdkit --help" in + let lines = String.concat " " lines in + if String.find lines "exit-with-parent" == -1 || + String.find lines "selinux-label" == -1 then + error (f_"nbdkit is not new enough, you need to upgrade to nbdkit ≥ 1.1.16") + in + + (* Check that the python3 plugin is installed and working + * and can load the plugin script. + *) + let error_unless_nbdkit_python3_working () = + let cmd = sprintf "nbdkit %s %s --dump-plugin >/dev/null" + python3 (quote plugin) in + if Sys.command cmd <> 0 then + error (f_"nbdkit Python 3 plugin is not installed or not working. It is required if you want to use ‘-o rhv-upload’. + +See also \"OUTPUT TO RHV\" in the virt-v2v(1) manual.") + in + + (* Check that nbdkit was compiled with SELinux support (for the + * --selinux-label option). + *) + let error_unless_nbdkit_compiled_with_selinux () = + let lines = external_command "nbdkit --dump-config" in + (* In nbdkit <= 1.1.15 the selinux attribute was not present + * at all in --dump-config output so there was no way to tell. + * Ignore this case because there will be an error later when + * we try to use the --selinux-label parameter. + *) + if List.mem "selinux=no" (List.map String.trim lines) then + error (f_"nbdkit was compiled without SELinux support. You will have to recompile nbdkit with libselinux-devel installed, or else set SELinux to Permissive mode while doing the conversion.") + in + + (* JSON parameters which are invariant between disks. *) + let json_params = [ + "output_conn", JSON.String output_conn; + "output_password", JSON.String output_password; + "output_storage", JSON.String output_storage; + "output_sparse", JSON.Bool (match output_alloc with + | Sparse -> true + | Preallocated -> false); + "rhv_cafile", JSON.String rhv_options.rhv_cafile; + "rhv_cluster", + JSON.String (Option.default "Default" rhv_options.rhv_cluster); + "rhv_direct", JSON.Bool rhv_options.rhv_direct; + + (* The 'Insecure' flag seems to be a number with various possible + * meanings, however we just set it to True/False. + * + * https://github.com/oVirt/ovirt-engine-sdk/blob/19aa7070b80e60a4cfd9104482... + *) + "insecure", JSON.Bool (not rhv_options.rhv_verifypeer); + ] in + + (* nbdkit command line args which are invariant between disks. *) + let nbdkit_args = + let args = [ + "nbdkit"; + + "--foreground"; (* run in foreground *) + "--exit-with-parent"; (* exit when virt-v2v exits *) + "--newstyle"; (* use newstyle NBD protocol *) + "--exportname"; "/"; + + "python3"; (* use the nbdkit Python 3 plugin *) + plugin; (* Python plugin script *) + ] in + let args = if verbose () then args @ ["--verbose"] else args in + let args = + (* label the socket so qemu can open it *) + if have_selinux then + args @ ["--selinux-label"; "system_u:object_r:svirt_t:s0"] + else args in + args in + +object + inherit output + + method precheck () = + error_unless_nbdkit_working (); + error_unless_nbdkit_python3_working (); + if have_selinux then + error_unless_nbdkit_compiled_with_selinux () + + method as_options = + "-o rhv-upload" ^ + (match output_alloc with + | Sparse -> "" (* default, don't need to print it *) + | Preallocated -> " -oa preallocated") ^ + sprintf " -oc %s -op %s -os %s" + output_conn output_password output_storage + + method supported_firmware = [ TargetBIOS ] + + method prepare_targets source targets = + let output_name = source.s_name in + let json_params = + ("output_name", JSON.String output_name) :: json_params in + + (* Python code prechecks. These can't run in #precheck because + * we need to know the name of the virtual machine. + *) + let json_param_file = tmpdir // "params.json" in + with_open_out + json_param_file + (fun chan -> output_string chan (JSON.string_of_doc json_params)); + if run_command [ python3; precheck; json_param_file ] <> 0 then + error (f_"failed server prechecks, see earlier errors"); + + (* Create an nbdkit instance for each disk and set the + * target URI to point to the NBD socket. + *) + List.map ( + fun t -> + let id = t.target_overlay.ov_source.s_disk_id in + let disk_name = sprintf "%s-%03d" output_name id in + let json_params = + ("disk_name", JSON.String disk_name) :: json_params in + + let disk_format = + match t.target_format with + | ("raw" | "qcow2") as fmt -> fmt + | _ -> + error (f_"rhv-upload: -of %s: Only output format ‘raw’ or ‘qcow2’ is supported. If the input is in a different format then force one of these output formats by adding either ‘-of raw’ or ‘-of qcow2’ on the command line.") + t.target_format in + let json_params = + ("disk_format", JSON.String disk_format) :: json_params in + + let disk_size = t.target_overlay.ov_virtual_size in + let json_params = + ("disk_size", JSON.Int64 disk_size) :: json_params in + + (* Ask the plugin to write the disk ID to a special file. *) + let diskid_file = diskid_file_of_id id in + let json_params = + ("diskid_file", JSON.String diskid_file) :: json_params in + + (* Write the JSON parameters to a file. *) + let json_param_file = tmpdir // sprintf "params%d.json" id in + with_open_out + json_param_file + (fun chan -> output_string chan (JSON.string_of_doc json_params)); + + let sock = tmpdir // sprintf "nbdkit%d.sock" id in + let pidfile = tmpdir // sprintf "nbdkit%d.pid" id in + + (* Add common arguments to per-target arguments. *) + let args = + nbdkit_args @ [ "--pidfile"; pidfile; + "--unix"; sock; + sprintf "params=%s" json_param_file ] in + + (* Print the full command we are about to run when debugging. *) + if verbose () then ( + eprintf "running nbdkit:\n"; + List.iter (fun arg -> eprintf " %s" (quote arg)) args; + prerr_newline () + ); + + (* Start an nbdkit instance in the background. By using + * --exit-with-parent we don't have to worry about clean-up. + *) + let args = Array.of_list args in + let pid = fork () in + if pid = 0 then ( + (* Child process (nbdkit). *) + execvp "nbdkit" args + ); + + (* Wait for the pidfile to appear so we know that nbdkit + * is listening for requests. + *) + if not (wait_for_file pidfile pidfile_timeout) then ( + if verbose () then + error (f_"nbdkit did not start up. See previous debugging messages for problems.") + else + error (f_"nbdkit did not start up. There may be errors printed by nbdkit above. + +If the messages above are not sufficient to diagnose the problem then add the ‘virt-v2v -v -x’ options and examine the debugging output carefully.") + ); + + if have_selinux then ( + (* Note that Unix domain sockets have both a file label and + * a socket/process label. Using --selinux-label above + * only set the socket label, but we must also set the file + * label. + *) + ignore ( + run_command ["chcon"; "system_u:object_r:svirt_image_t:s0"; + sock] + ); + ); + (* ... and the regular Unix permissions, in case qemu is + * running as another user. + *) + chmod sock 0o777; + + (* Tell ‘qemu-img convert’ to write to the nbd socket which is + * connected to nbdkit. + *) + let json_params = [ + "file.driver", JSON.String "nbd"; + "file.path", JSON.String sock; + "file.export", JSON.String "/"; + ] in + let target_file = + TargetURI ("json:" ^ JSON.string_of_doc json_params) in + { t with target_file } + ) targets + + method create_metadata source targets _ guestcaps inspect target_firmware = + (* Get the UUIDs of each disk image. These files are written + * out by the nbdkit plugins on successful finalization of the + * transfer. + *) + let nr_disks = List.length targets in + let image_uuids = + List.map ( + fun t -> + let id = t.target_overlay.ov_source.s_disk_id in + let diskid_file = diskid_file_of_id id in + if not (wait_for_file diskid_file finalization_timeout) then + error (f_"transfer of disk %d/%d failed, see earlier error messages") + (id+1) nr_disks; + let diskid = read_whole_file diskid_file in + diskid + ) targets in + + (* We don't have the storage domain UUID, but instead we write + * in a magic value which the Python code (which can get it) + * will substitute. + *) + let sd_uuid = "@SD_UUID@" in + + (* The volume and VM UUIDs are made up. *) + let vol_uuids = List.map (fun _ -> uuidgen ()) targets + and vm_uuid = uuidgen () in + + (* Create the metadata. *) + let ovf = + Create_ovf.create_ovf source targets guestcaps inspect + output_alloc + sd_uuid image_uuids vol_uuids vm_uuid + OVirt in + let ovf = DOM.doc_to_string ovf in + + let json_param_file = tmpdir // "params.json" in + with_open_out + json_param_file + (fun chan -> output_string chan (JSON.string_of_doc json_params)); + + let ovf_file = tmpdir // "vm.ovf" in + with_open_out ovf_file (fun chan -> output_string chan ovf); + if run_command [ python3; createvm; json_param_file; ovf_file ] <> 0 then + error (f_"failed to create virtual machine, see earlier errors") + +end + +let output_rhv_upload = new output_rhv_upload +let () = Modules_list.register_output_module "rhv-upload" diff --git a/v2v/output_rhv_upload.mli b/v2v/output_rhv_upload.mli new file mode 100644 index 000000000..f03e956b3 --- /dev/null +++ b/v2v/output_rhv_upload.mli @@ -0,0 +1,33 @@ +(* virt-v2v + * Copyright (C) 2009-2018 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +(** [-o rhv-upload] target. *) + +type rhv_options +(** Miscellaneous extra command line parameters used by rhv-upload. *) + +val print_rhv_output_options : unit -> unit +val parse_rhv_output_options : (string * string) list -> rhv_options +(** Print and parse rhv-upload -oo options. *) + +val output_rhv_upload : Types.output_allocation -> string -> string -> + string -> rhv_options -> Types.output +(** [output_rhv_upload output_alloc output_conn output_password output_storage + rhv_options] + creates and returns a new {!Types.output} object specialized for writing + output to oVirt or RHV directly via RHV APIs. *) diff --git a/v2v/output_rhv_upload_createvm_source.mli b/v2v/output_rhv_upload_createvm_source.mli new file mode 100644 index 000000000..c1bafa15b --- /dev/null +++ b/v2v/output_rhv_upload_createvm_source.mli @@ -0,0 +1,19 @@ +(* virt-v2v + * Copyright (C) 2018 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +val code : string diff --git a/v2v/output_rhv_upload_plugin_source.mli b/v2v/output_rhv_upload_plugin_source.mli new file mode 100644 index 000000000..c1bafa15b --- /dev/null +++ b/v2v/output_rhv_upload_plugin_source.mli @@ -0,0 +1,19 @@ +(* virt-v2v + * Copyright (C) 2018 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +val code : string diff --git a/v2v/output_rhv_upload_precheck_source.mli b/v2v/output_rhv_upload_precheck_source.mli new file mode 100644 index 000000000..c1bafa15b --- /dev/null +++ b/v2v/output_rhv_upload_precheck_source.mli @@ -0,0 +1,19 @@ +(* virt-v2v + * Copyright (C) 2018 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +val code : string diff --git a/v2v/rhv-upload-createvm.py b/v2v/rhv-upload-createvm.py new file mode 100644 index 000000000..7a80fa5bd --- /dev/null +++ b/v2v/rhv-upload-createvm.py @@ -0,0 +1,85 @@ +# -*- python -*- +# oVirt or RHV upload create VM used by ‘virt-v2v -o rhv-upload’ +# Copyright (C) 2018 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import json +import logging +import ovirtsdk4 as sdk +import ovirtsdk4.types as types +import sys +import time + +from http.client import HTTPSConnection +from urllib.parse import urlparse + +# Parameters are passed in via a JSON doc from the OCaml code. +# Because this Python code ships embedded inside virt-v2v there +# is no formal API here. +params = None +ovf = None # OVF file + +if len(sys.argv) != 3: + raise RuntimeError("incorrect number of parameters") + +# Parameters are passed in via a JSON document. +with open(sys.argv[1], 'r') as fp: + params = json.load(fp) + +# What is passed in is a password file, read the actual password. +with open(params['output_password'], 'r') as fp: + output_password = fp.read() +output_password = output_password.rstrip() + +# Read the OVF document. +with open(sys.argv[2], 'r') as fp: + ovf = fp.read() + +# Parse out the username from the output_conn URL. +parsed = urlparse(params['output_conn']) +username = parsed.username or "admin@internal" + +# Connect to the server. +connection = sdk.Connection( + url = params['output_conn'], + username = username, + password = output_password, + ca_file = params['rhv_cafile'], + log = logging.getLogger(), + insecure = params['insecure'], +) + +system_service = connection.system_service() + +# Get the storage domain UUID and substitute it into the OVF doc. +sds_service = system_service.storage_domains_service() +sd = sds_service.list(search=("name=%s" % params['output_storage']))[0] +sd_uuid = sd.id + +ovf.replace("@SD_UUID@", sd_uuid) + +vms_service = system_service.vms_service() +vm = vms_service.add( + types.Vm( + cluster=types.Cluster(name = params['rhv_cluster']), + initialization=types.Initialization( + configuration = types.Configuration( + type = types.ConfigurationType.OVA, + data = ovf, + ) + ) + ) +) diff --git a/v2v/rhv-upload-plugin.py b/v2v/rhv-upload-plugin.py new file mode 100644 index 000000000..979cbd63b --- /dev/null +++ b/v2v/rhv-upload-plugin.py @@ -0,0 +1,414 @@ +# -*- python -*- +# oVirt or RHV upload nbdkit plugin used by ‘virt-v2v -o rhv-upload’ +# Copyright (C) 2018 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import builtins +import json +import logging +import ovirtsdk4 as sdk +import ovirtsdk4.types as types +import ssl +import sys +import time + +from http.client import HTTPSConnection +from urllib.parse import urlparse + +# Timeout to wait for oVirt disks to change status, or the transfer +# object to finish initializing [seconds]. +timeout = 5*60 + +# Parameters are passed in via a JSON doc from the OCaml code. +# Because this Python code ships embedded inside virt-v2v there +# is no formal API here. +params = None + +def config(key, value): + global params + + if key == "params": + with builtins.open(value, 'r') as fp: + params = json.load(fp) + else: + raise RuntimeError("unknown configuration key '%s'" % key) + +def config_complete(): + if params is None: + raise RuntimeError("missing configuration parameters") + +def open(readonly): + # Parse out the username from the output_conn URL. + parsed = urlparse(params['output_conn']) + username = parsed.username or "admin@internal" + + # Read the password from file. + with builtins.open(params['output_password'], 'r') as fp: + password = fp.read() + password = password.rstrip() + + # Connect to the server. + connection = sdk.Connection( + url = params['output_conn'], + username = username, + password = password, + ca_file = params['rhv_cafile'], + log = logging.getLogger(), + insecure = params['insecure'], + ) + + system_service = connection.system_service() + + # Create the disk. + disks_service = system_service.disks_service() + if params['disk_format'] == "raw": + disk_format = types.DiskFormat.RAW + else: + disk_format = types.DiskFormat.COW + disk = disks_service.add( + disk = types.Disk( + name = params['disk_name'], + description = "Uploaded by virt-v2v", + format = disk_format, + provisioned_size = params['disk_size'], + sparse = params['output_sparse'], + storage_domains = [ + types.StorageDomain( + name = params['output_storage'], + ) + ], + ) + ) + + # Wait till the disk is up, as the transfer can't start if the + # disk is locked: + disk_service = disks_service.disk_service(disk.id) + + endt = time.time() + timeout + while True: + time.sleep(5) + disk = disk_service.get() + if disk.status == types.DiskStatus.OK: + break + if time.time() > endt: + raise RuntimeError("timed out waiting for disk to become unlocked") + + # Get a reference to the transfer service. + transfers_service = system_service.image_transfers_service() + + # Create a new image transfer. + transfer = transfers_service.add( + types.ImageTransfer( + image = types.Image( + id = disk.id + ) + ) + ) + + # Get a reference to the created transfer service. + transfer_service = transfers_service.image_transfer_service(transfer.id) + + # After adding a new transfer for the disk, the transfer's status + # will be INITIALIZING. Wait until the init phase is over. The + # actual transfer can start when its status is "Transferring". + endt = time.time() + timeout + while True: + time.sleep(5) + transfer = transfer_service.get() + if transfer.phase != types.ImageTransferPhase.INITIALIZING: + break + if time.time() > endt: + raise RuntimeError("timed out waiting for transfer status != INITIALIZING") + + # Now we have permission to start the transfer. + if params['rhv_direct']: + if transfer.transfer_url is None: + raise RuntimeError("direct upload to host not supported, requires ovirt-engine >= 4.2 and only works when virt-v2v is run within the oVirt/RHV environment, eg. on an ovirt node.") + destination_url = urlparse(transfer.transfer_url) + else: + destination_url = urlparse(transfer.proxy_url) + + context = ssl.create_default_context() + context.load_verify_locations(cafile = params['rhv_cafile']) + + http = HTTPSConnection( + destination_url.hostname, + destination_url.port, + context = context + ) + + # Save everything we need to make requests in the handle. + return { + 'connection': connection, + 'disk': disk, + 'disk_service': disk_service, + 'failed': False, + 'got_options': False, + 'highestwrite': 0, + 'http': http, + 'needs_auth': not params['rhv_direct'], + 'path': destination_url.path, + 'transfer': transfer, + 'transfer_service': transfer_service, + } + +# Can we issue zero, trim or flush requests? +def get_options(h): + if h['got_options']: + return + h['got_options'] = True + + http = h['http'] + transfer=h['transfer'] + + http.putrequest("OPTIONS", h['path']) + http.putheader("Authorization", transfer.signed_ticket) + http.endheaders() + + r = http.getresponse() + if r.status == 200: + j = json.loads(r.read()) + h['can_zero'] = "zero" in j['features'] + h['can_trim'] = "trim" in j['features'] + h['can_flush'] = "flush" in j['features'] + + # If can_zero is true then it means we're using the new imageio + # which never needs the Authorization header. + if h['can_zero']: + h['needs_auth'] = False + + # Old imageio servers returned 405 Method Not Allowed. If + # we see that we just say that no operations are allowed and + # we will emulate them. + elif r.status == 405: + h['can_zero'] = False + h['can_trim'] = False + h['can_flush'] = False + + else: + raise RuntimeError("could not use OPTIONS request: %d: %s" % + (r.status, r.reason)) + +def can_trim(h): + get_options(h) + return h['can_trim'] + +def can_flush(h): + get_options(h) + return h['can_flush'] + +def get_size(h): + return params['disk_size'] + +# For documentation see: +# https://github.com/oVirt/ovirt-imageio/blob/master/docs/random-io.md +# For examples of working code to read/write from the server, see: +# https://github.com/oVirt/ovirt-imageio/blob/master/daemon/test/server_tes... + +def pread(h, count, offset): + http = h['http'] + transfer=h['transfer'] + transfer_service=h['transfer_service'] + + http.putrequest("GET", h['path']) + if h['needs_auth']: + http.putheader("Authorization", transfer.signed_ticket) + http.putheader("Range", "bytes=%d-%d" % (offset, offset+count-1)) + http.endheaders() + + r = http.getresponse() + # 206 = HTTP Partial Content. + if r.status != 206: + h['transfer_service'].pause() + h['failed'] = True + raise RuntimeError("could not read sector (%d, %d): %d: %s" % + (offset, count, r.status, r.reason)) + return r.read() + +def pwrite(h, buf, offset): + http = h['http'] + transfer=h['transfer'] + transfer_service=h['transfer_service'] + + count = len(buf) + h['highestwrite'] = max(h['highestwrite'], offset+count) + + http.putrequest("PUT", h['path'] + "?flush=n") + if h['needs_auth']: + http.putheader("Authorization", transfer.signed_ticket) + # The oVirt server only uses the first part of the range, and the + # content-length. + http.putheader("Content-Range", "bytes %d-%d/*" % (offset, offset+count-1)) + http.putheader("Content-Length", str(count)) + http.endheaders() + http.send(buf) + + r = http.getresponse() + if r.status != 200: + transfer_service.pause() + h['failed'] = True + raise RuntimeError("could not write sector (%d, %d): %d: %s" % + (offset, count, r.status, r.reason)) + +def zero(h, count, offset, may_trim): + http = h['http'] + transfer=h['transfer'] + transfer_service=h['transfer_service'] + + # Unlike the trim and flush calls, there is no 'can_zero' method + # so nbdkit could call this even if the server doesn't support + # zeroing. If this is the case we must emulate. + if not h['can_zero']: + emulate_zero(h, count, offset) + return + + # Construct the JSON request for zeroing. + buf = json.dumps({'op': "zero", + 'offset': offset, + 'size': count, + 'flush': False}) + + http.putrequest("PATCH", h['path']) + http.putheader("Content-Type", "application/json") + if h['needs_auth']: + http.putheader("Authorization", transfer.signed_ticket) + http.putheader("Content-Length", len(buf)) + http.endheaders() + http.send(buf) + + r = http.getresponse() + if r.status != 200: + transfer_service.pause() + h['failed'] = True + raise RuntimeError("could not zero sector (%d, %d): %d: %s" % + (offset, count, r.status, r.reason)) + +# qemu-img convert starts by trying to zero/trim the whole device. +# Since we've just created a new disk it's safe to ignore these +# requests as long as they are smaller than the highest write seen. +# After that we must emulate them with writes. +def emulate_zero(h, count, offset): + if offset+count < h['highestwrite']: + http.putrequest("PUT", h['path'] + "?flush=n") + http.putheader("Content-Range", + "bytes %d-%d/*" % (offset, offset+count-1)) + http.putheader("Content-Length", str(count)) + http.endheaders() + + buf = bytearray(128*1024) + while count > len(buf): + http.send(buf) + count -= len(buf) + http.send(buffer(buf, 0, count)) + + r = http.getresponse() + if r.status != 200: + transfer_service.pause() + h['failed'] = True + raise RuntimeError("could not write zeroes (%d, %d): %d: %s" % + (offset, count, r.status, r.reason)) + +def trim(h, count, offset): + http = h['http'] + transfer=h['transfer'] + transfer_service=h['transfer_service'] + + # Construct the JSON request for trimming. + buf = json.dumps({'op': "trim", + 'offset': offset, + 'size': count, + 'flush': False}) + + http.putrequest("PATCH", h['path']) + http.putheader("Content-Type", "application/json") + if h['needs_auth']: + http.putheader("Authorization", transfer.signed_ticket) + http.putheader("Content-Length", len(buf)) + http.endheaders() + http.send(buf) + + r = http.getresponse() + if r.status != 200: + transfer_service.pause() + h['failed'] = True + raise RuntimeError("could not trim sector (%d, %d): %d: %s" % + (offset, count, r.status, r.reason)) + +def flush(h): + http = h['http'] + transfer=h['transfer'] + transfer_service=h['transfer_service'] + + # Construct the JSON request for flushing. Note the offset + # and count must be present but are ignored by imageio. + buf = json.dumps({'op': "flush", + 'offset': 0, + 'size': 0, + 'flush': True}) + + http.putrequest("PATCH", h['path']) + http.putheader("Content-Type", "application/json") + if h['needs_auth']: + http.putheader("Authorization", transfer.signed_ticket) + http.putheader("Content-Length", len(buf)) + http.endheaders() + http.send(buf) + + r = http.getresponse() + if r.status != 200: + transfer_service.pause() + h['failed'] = True + raise RuntimeError("could not flush: %d: %s" % (r.status, r.reason)) + +def close(h): + http = h['http'] + connection = h['connection'] + + http.close() + + # If we didn't fail, then finalize the transfer. + if not h['failed']: + disk = h['disk'] + transfer_service=h['transfer_service'] + + transfer_service.finalize() + + # Wait until the transfer disk job is completed since + # only then we can be sure the disk is unlocked. As this + # code is not very clear, what's happening is that we are + # waiting for the transfer object to cease to exist, which + # falls through to the exception case and then we can + # continue. + endt = time.time() + timeout + try: + while True: + time.sleep(1) + tmp = transfer_service.get() + if time.time() > endt: + raise RuntimeError("timed out waiting for transfer to finalize") + except sdk.NotFoundError: + pass + + # Write the disk ID file. Only do this on successful completion. + with builtins.open(params['diskid_file'], 'w') as fp: + fp.write(disk.id) + + # Otherwise if we did fail then we should delete the disk. + else: + disk_service = h['disk_service'] + disk_service.remove() + + connection.close() diff --git a/v2v/rhv-upload-precheck.py b/v2v/rhv-upload-precheck.py new file mode 100644 index 000000000..e3fa65148 --- /dev/null +++ b/v2v/rhv-upload-precheck.py @@ -0,0 +1,72 @@ +# -*- python -*- +# oVirt or RHV pre-upload checks used by ‘virt-v2v -o rhv-upload’ +# Copyright (C) 2018 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import json +import logging +import ovirtsdk4 as sdk +import ovirtsdk4.types as types +import sys +import time + +from http.client import HTTPSConnection +from urllib.parse import urlparse + +# Parameters are passed in via a JSON doc from the OCaml code. +# Because this Python code ships embedded inside virt-v2v there +# is no formal API here. +params = None + +if len(sys.argv) != 2: + raise RuntimeError("incorrect number of parameters") + +# Parameters are passed in via a JSON document. +with open(sys.argv[1], 'r') as fp: + params = json.load(fp) + +# What is passed in is a password file, read the actual password. +with open(params['output_password'], 'r') as fp: + output_password = fp.read() +output_password = output_password.rstrip() + +# Parse out the username from the output_conn URL. +parsed = urlparse(params['output_conn']) +username = parsed.username or "admin@internal" + +# Connect to the server. +connection = sdk.Connection( + url = params['output_conn'], + username = username, + password = output_password, + ca_file = params['rhv_cafile'], + log = logging.getLogger(), + insecure = params['insecure'], +) + +system_service = connection.system_service() + +# Find if a virtual machine already exists with that name. +vms_service = system_service.vms_service() +vms = vms_service.list( + search = ("name=%s" % params['output_name']), +) +if len(vms) > 0: + vm = vms[0] + raise RuntimeError("VM already exists with name ‘%s’, id ‘%s’" % + (params['output_name'], vm.id)) + +# Otherwise everything is OK, exit with no error. diff --git a/v2v/test-v2v-o-rhv-upload-oo-query.sh b/v2v/test-v2v-o-rhv-upload-oo-query.sh new file mode 100755 index 000000000..29d69e1c1 --- /dev/null +++ b/v2v/test-v2v-o-rhv-upload-oo-query.sh @@ -0,0 +1,38 @@ +#!/bin/bash - +# libguestfs virt-v2v test script +# Copyright (C) 2018 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Test -oo "?" option. + +set -e + +$TEST_FUNCTIONS +skip_if_skipped + +export VIRT_TOOLS_DATA_DIR="$top_srcdir/test-data/fake-virt-tools" +export VIRTIO_WIN="$top_srcdir/test-data/fake-virtio-win" + +f=test-v2v-o-rhv-upload-oo-query.actual +rm -f $f + +$VG virt-v2v --debug-gc \ + -o rhv-upload -oo "?" > $f + +grep -- "-oo rhv-cafile" $f +grep -- "-oo rhv-verifypeer" $f + +rm $f diff --git a/v2v/test-v2v-python-syntax.sh b/v2v/test-v2v-python-syntax.sh new file mode 100755 index 000000000..11ecd1f84 --- /dev/null +++ b/v2v/test-v2v-python-syntax.sh @@ -0,0 +1,27 @@ +#!/bin/bash - +# libguestfs +# Copyright (C) 2018 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +set -e + +$TEST_FUNCTIONS +skip_if_skipped + +# Checks the files are syntactically correct, but not very much else. +python3 -m py_compile rhv-upload-createvm.py +python3 -m py_compile rhv-upload-plugin.py +python3 -m py_compile rhv-upload-precheck.py diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod index 4e3daedf0..caf9c983e 100644 --- a/v2v/virt-v2v.pod +++ b/v2v/virt-v2v.pod @@ -6,15 +6,18 @@ virt-v2v - Convert a guest to use KVM virt-v2v -ic vpx://vcenter.example.com/Datacenter/esxi vmware_guest - virt-v2v -ic vpx://vcenter.example.com/Datacenter/esxi vmware_guest \ - -o rhv -os rhv.nfs:/export_domain --network ovirtmgmt - virt-v2v -i libvirtxml guest-domain.xml -o local -os /var/tmp virt-v2v -i disk disk.img -o local -os /var/tmp virt-v2v -i disk disk.img -o glance + virt-v2v -ic vpx://vcenter.example.com/Datacenter/esxi vmware_guest \ + -o rhv-upload -oc https://ovirt-engine.example.com/ovirt-engine/api \ + -os ovirt-data -op /tmp/ovirt-admin-password \ + -oo rhv-cafile=/tmp/ca.pem -oo rhv-direct \ + --network ovirtmgmt + virt-v2v -ic qemu:///system qemu_guest --in-place =head1 DESCRIPTION @@ -52,20 +55,18 @@ For more information see L</INPUT FROM VMWARE VCENTER SERVER> below. =head2 Convert from VMware to RHV/oVirt This is the same as the previous example, except you want to send the -guest to a RHV-M Export Storage Domain which is located remotely -(over NFS) at C<rhv.nfs:/export_domain>. If you are unclear about -the location of the Export Storage Domain you should check the -settings on your RHV-M management console. Guest network +guest to a RHV Data Domain using the RHV REST API. Guest network interface(s) are connected to the target network called C<ovirtmgmt>. virt-v2v -ic vpx://vcenter.example.com/Datacenter/esxi vmware_guest \ - -o rhv -os rhv.nfs:/export_domain --network ovirtmgmt + -o rhv-upload -oc https://ovirt-engine.example.com/ovirt-engine/api \ + -os ovirt-data -op /tmp/ovirt-admin-password \ + -oo rhv-cafile=/tmp/ca.pem -oo rhv-direct \ + --network ovirtmgmt In this case the host running virt-v2v acts as a B<conversion server>. -Note that after conversion, the guest will appear in the RHV-M Export -Storage Domain, from where you will need to import it using the RHV-M -user interface. (See L</OUTPUT TO RHV>). +For more information see L</OUTPUT TO RHV> below. =head2 Convert from ESXi hypervisor over SSH to local libvirt @@ -129,9 +130,9 @@ qemu, do: Xen ───▶│ -i libvirt ──▶ │ │ │ (default) │ ... ───▶│ (default) │ │ │ ──┐ └────────────┘ └────────────┘ │ │ ─┐└──────▶ -o glance - -i libvirtxml ─────────▶ │ │ ┐└─────────▶ -o rhv - -i vmx ────────────────▶ │ │ └──────────▶ -o vdsm - └────────────┘ + -i libvirtxml ─────────▶ │ │ ┐├─────────▶ -o rhv + -i vmx ────────────────▶ │ │ │└─────────▶ -o vdsm + └────────────┘ └──────────▶ -o rhv-upload Virt-v2v has a number of possible input and output modes, selected using the I<-i> and I<-o> options. Only one input and output mode can @@ -164,8 +165,9 @@ libvirt configuration file (mainly for testing). I<-o qemu> writes to a local disk image with a shell script for booting the guest directly in qemu (mainly for testing). -I<-o rhv> is used to write to a RHV / oVirt target. I<-o vdsm> -is only used when virt-v2v runs under VDSM control. +I<-o rhv-upload> is used to write to a RHV / oVirt target. I<-o rhv> +is a legacy method to write to RHV / oVirt E<lt> 4.2. I<-o vdsm> is +only used when virt-v2v runs under VDSM control. I<--in-place> instructs virt-v2v to customize the guest OS in the input virtual machine, instead of creating a new VM in the target hypervisor. @@ -550,6 +552,10 @@ written. This is the same as I<-o rhv>. +=item B<-o> B<ovirt-upload> + +This is the same as I<-o rhv-upload>. + =item B<-o> B<qemu> Set the output method to I<qemu>. @@ -574,6 +580,16 @@ I<-os> parameter must also be used to specify the location of the Export Storage Domain. Note this does not actually import the guest into RHV. You have to do that manually later using the UI. +See L</OUTPUT TO RHV (OLD METHOD)> below. + +=item B<-o> B<rhv-upload> + +Set the output method to I<rhv-upload>. + +The converted guest is written directly to a RHV Data Domain. +This is a faster method than I<-o rhv>, but requires oVirt +or RHV E<ge> 4.2. + See L</OUTPUT TO RHV> below. =item B<-o> B<vdsm> @@ -615,7 +631,33 @@ the output name is the same as the input name. Set output option(s) related to the current output mode. To display short help on what options are available you can use: - virt-v2v -o vdsm -oo "?" + virt-v2v -o rhv-upload -oo "?" + +=item B<-oo rhv-cafile=>F<ca.pem> + +For I<-o rhv-upload> (L<OUTPUT TO RHV>) only, the F<ca.pem> file +(Certificate Authority), copied from F</etc/pki/ovirt-engine/ca.pem> +on the oVirt engine. + +=item B<-oo rhv-cluster=>C<CLUSTERNAME> + +For I<-o rhv-upload> (L<OUTPUT TO RHV>) only, set the RHV Cluster +Name. If not given it uses C<Default>. + +=item B<-oo rhv-direct> + +For I<-o rhv-upload> (L<OUTPUT TO RHV>) only, if this option is given +then virt-v2v will attempt to directly upload the disk to the oVirt +node, otherwise it will proxy the upload through the oVirt engine. +Direct upload requires that you have network access to the oVirt +nodes. Non-direct upload is slightly slower but should work in all +situations. + +=item B<-oo rhv-verifypeer> + +For I<-o rhv-upload> (L<OUTPUT TO RHV>) only, verify the oVirt/RHV +server’s identity by checking the server‘s certificate against the +Certificate Authority. =item B<-oo vdsm-compat=0.10> @@ -1889,6 +1931,62 @@ Define the final guest in libvirt: =head1 OUTPUT TO RHV +This new method to upload guests to oVirt or RHV directly via the REST +API requires oVirt/RHV E<ge> 4.2. + +You need to specify I<-o rhv-upload> as well as the following extra +parameters: + +=over 4 + +=item I<-oc> C<https://ovirt-engine.example.com/ovirt-engine/api> + +The URL of the REST API which is usually the server name with +C</ovirt-engine/api> appended, but might be different if you installed +oVirt Engine on a different path. + +You can optionally add a username and port number to the URL. If the +username is not specified then virt-v2v defaults to using +C<admin@internal> which is the typical superuser account for oVirt +instances. + +=item I<-op> F<password-file> + +A file containing a password to be used when connecting to the oVirt +engine. Note the file should contain the whole password, B<without +any trailing newline>, and for security the file should have mode +C<0600> so that others cannot read it. + +=item I<-os> C<ovirt-data> + +The storage domain. + +=item I<-oo rhv-cafile=>F<ca.pem> + +The F<ca.pem> file (Certificate Authority), copied from +F</etc/pki/ovirt-engine/ca.pem> on the oVirt engine. + +=item I<-oo rhv-cluster=>C<CLUSTERNAME> + +Set the RHV Cluster Name. If not given it uses C<Default>. + +=item I<-oo rhv-direct> + +If this option is given then virt-v2v will attempt to directly upload +the disk to the oVirt node, otherwise it will proxy the upload through +the oVirt engine. Direct upload requires that you have network access +to the oVirt nodes. Non-direct upload is slightly slower but should +work in all situations. + +=item I<-oo rhv-verifypeer> + +Verify the oVirt/RHV server’s identity by checking the server‘s +certificate against the Certificate Authority. + +=back + +=head1 OUTPUT TO RHV (OLD METHOD) + This section only applies to the I<-o rhv> output mode. If you use virt-v2v from the RHV-M user interface, then behind the scenes the import is managed by VDSM using the I<-o vdsm> output mode (which end -- 2.13.2

On Thu, Mar 22, 2018 at 5:25 PM Richard W.M. Jones <rjones(a)redhat.com&gt; wrote: How is this related to the actual file size specified when you create a disk? In block storage, qcow2 image is always on a thin provisioned disk, which in oVirt is a regular logical volume, created at the requested "initial_size": From: https://github.com/oVirt/ovirt-engine-sdk/blob/aba2a83ec94ecac1594adfff62... if image_info["format"] == "qcow2": disk_format = types.DiskFormat.COW else: disk_format = types.DiskFormat.RAW disks_service = connection.system_service().disks_service() disk = disks_service.add( disk=types.Disk( name=os.path.basename(image_path), description='Uploaded disk', format=disk_format, initial_size=image_size, provisioned_size=image_info["virtual-size"], sparse=disk_format == types.DiskFormat.COW, storage_domains=[ types.StorageDomain( name='mydata' ) ] ) ) Internally we do allocated 10% more then the requested initial_size to leave room for qcow2 metadata. See: https://github.com/oVirt/vdsm/blob/3de6f326d7e338b064b11d2a2269500a385709... https://github.com/oVirt/vdsm/blob/3de6f326d7e338b064b11d2a2269500a385709... Do we have other problems? The storage domain belongs to some data center, so I don't think you need to select a data center. [snipped] diff --git a/v2v/rhv-upload-plugin.py b/v2v/rhv-upload-plugin.py It is good practice to separate stdlib imports and 3rd party imports like ovirtsdk, helps to understand the dependencies in modules. [snipped] These will not work in python 2, but you can use six.moves to have code that works on both 2 and 3. [snipped] Can this be None? If ca_file cannot be None, then insecure is not needed, based on Ondra review from earlier version. [snipped] This must be the virtual size. You don't specify initial_size - in this case you get 1G, and most images will fail to upload. The user cannot configure that. This must be based on the image format. The current coded may create images in unsupported combinations, e.g. raw/sparse on block storage, or fail validation in engine. [snipped] +def get_options(h): If we got here, we are working with new daemon or proxy, and both of them do not need auth, so we can set 'needs_auth' to False if OPTIONS returned 200 OK. I would set all the defaults when creating the sate dict. This ensures that we don't forget needs_auth or other keys and crash later with KeyError, and make it easier to understand what is the state managed by the plugin. [snipped] +def pwrite(h, buf, offset): "?flush=n" has effect only if h["can_flush"] is True. Older daemon/proxy do not know support disabling flushing. The docs mention that this query string will be added in 1.3, we are now at 1.2. + # The oVirt server only uses the first part of the range, and the [snipped] > +def zero(h, count, offset, may_trim): > + http = h['http'] > + transfer=h['transfer'] > + transfer_service=h['transfer_service'] > + > + # Unlike the trim and flush calls, there is no 'can_zero' method > + # so nbdkit could call this even if the server doesn't support > + # zeroing. If this is the case we must emulate. > + if not h['can_zero']: > + emulate_zero(h, count, offset) > + return > + > + # Construct the JSON request for zeroing. > + buf = json.dumps({'op': "zero", > + 'offset': offset, > + 'size': count, > + 'flush': False}) > + > + http.putrequest("PATCH", h['path']) > + http.putheader("Content-Type", "application/json") > Only GET and PUT on a server that does not implement OPTIONS need auth. This will work if h['needs_auth'] is set correctly, but I think it is better to include this check only in pread() and pwrite(), and add a comment there that this is need to support legacy versions. I think this comment is not related to this code. Maybe it belongs to write() where we compute the highwrite? This is is used only on old daemon/proxy, the flush has no effect. [snipped] +def trim(h, count, offset): > + http = h['http'] > + transfer=h['transfer'] > + transfer_service=h['transfer_service'] > + > + # Construct the JSON request for trimming. > + buf = json.dumps({'op': "trim", > + 'offset': offset, > + 'size': count, > + 'flush': False}) > + > + http.putrequest("PATCH", h['path']) > + http.putheader("Content-Type", "application/json") > Never needed. [snipped] Should be (discussed in v6) buf = json.dumps({"op": "flush"}) > + > + http.putrequest("PATCH", h['path']) > + http.putheader("Content-Type", "application/json") > Never needed. [snipped] If this raises, we never clean up [snipped] If this raises, we will not remove the disk, or close the connection. [snipped] I did not check all the SDK related code, I'm not very familiar with the SDK. Thanks for creating this, and sorry for the bad documentation on our side :-) Nir