This feature was not documented in the manual page since 2018 (commit c1c4ed4c16e83a83), only in the --help output. It's not needed when it's much more efficient to use either SSH or VDDK instead. --- v2v/copy_to_local.ml | 45 ++++---------------------------------------- 1 file changed, 4 insertions(+), 41 deletions(-) diff --git a/v2v/copy_to_local.ml b/v2v/copy_to_local.ml index f326a2e33..c848914f3 100644 --- a/v2v/copy_to_local.ml +++ b/v2v/copy_to_local.ml @@ -28,7 +28,7 @@ open Getopt.OptionName open Utils open Xpath_helpers -type source_t = Xen_ssh of string | ESXi of string | Test +type source_t = Xen_ssh of string | Test let rec main () = let input_conn = ref None in @@ -61,8 +61,6 @@ Copy the remote guest: virt-v2v-copy-to-local -ic xen+ssh://root@xen.example.com guest - virt-v2v-copy-to-local -ic esx://esxi.example.com guest - Then perform the conversion step: virt-v2v -i libvirtxml guest.xml -o local -os /var/tmp @@ -98,8 +96,6 @@ read the man page virt-v2v-copy-to-local(1). match parsed_uri.Xml.uri_server, parsed_uri.Xml.uri_scheme with | Some server, Some "xen+ssh" -> (* Xen over SSH *) Xen_ssh server - | Some server, Some "esx" -> (* esxi over https *) - ESXi server (* This is just for testing, and is not documented. *) | None, Some "test" -> @@ -107,7 +103,7 @@ read the man page virt-v2v-copy-to-local(1). (* We can probably extend this list in future. *) | _ -> - error (f_"only copies from VMware ESXi or Xen over SSH are supported. See the virt-v2v-copy-to-local(1) manual page.") in + error (f_"only copies from Xen over SSH are supported. See the virt-v2v-copy-to-local(1) manual page.") in (* We expect a single extra argument, which is the guest name. *) let guest_name = @@ -140,25 +136,9 @@ read the man page virt-v2v-copy-to-local(1). debug "libvirt XML after modifying for local disks:\n%s" xml; - (* For VMware ESXi source, we have to massage the disk path. *) let disks = - match source with - | ESXi server -> - let dcpath = - match dcpath with - | Some dcpath -> dcpath - | None -> - error (f_"vcenter: <vmware:datacenterpath> was not found in the XML. You need to upgrade to libvirt ≥ 1.2.20.") in - List.map ( - fun (remote_disk, local_disk) -> - let { VCenter.https_url; sslverify; session_cookie } = - VCenter.map_source dcpath parsed_uri server remote_disk in - debug "esxi: source disk %s (sslverify=%b)" https_url sslverify; - (https_url, local_disk, sslverify, session_cookie) - ) disks - | Test | Xen_ssh _ -> - List.map (fun (remote_disk, local_disk) -> - (remote_disk, local_disk, false, None)) disks in + List.map (fun (remote_disk, local_disk) -> + (remote_disk, local_disk, false, None)) disks in (* Delete the disks on exit, unless we finish everything OK. *) let delete_on_exit = ref true in @@ -200,23 +180,6 @@ read the man page virt-v2v-copy-to-local(1). if shell_command cmd <> 0 then error (f_"ssh copy command failed, see earlier errors"); - | ESXi _ -> - let curl_args = ref [ - "url", Some remote_disk; - "output", Some local_disk; - ] in - if not sslverify then List.push_back curl_args ("insecure", None); - (match cookie with - | None -> () - | Some cookie -> List.push_back curl_args ("cookie", Some cookie) - ); - if quiet () then List.push_back curl_args ("silent", None); - - let curl_h = Curl.create !curl_args in - if verbose () then - Curl.print stderr curl_h; - ignore (Curl.run curl_h) - | Test -> let cmd = [ "cp"; remote_disk; local_disk ] in if run_command cmd <> 0 then -- 2.27.0

For conversions[*] which take longer than 30 minutes it can happen that the HTTPS authorization cookie that we fetched from VMware when we first connect expires. This can especially happen when there are multiple disks, because we may not "touch" (therefore autorenew) the second disk while we are doing the long conversion. This can lead to failures, some of which are silent: again if there are multiple disks, fstrim of the non-system disks can fail silently resulting in the copy step taking a very long time. The solution to this is to use the new nbdkit-curl-plugin cookie-script feature which allows nbdkit to automatically renew the cookie as required. During the conversion or copying steps you may see the cookie being autorenewed: nbdkit: curl[3]: debug: curl: running cookie-script nbdkit: curl[3]: debug: cookie-script returned cookies This removes the ?user and ?password parameters from Nbdkit_sources.- create_curl because they are no longer needed after this change. Note for future: if we need to add them back, we must prevent both user and cookie_script parameters from being used at the same time, because simply having the user parameter will try basic authentication, overriding the cookie, which will either fail (no password) or run very slowly. This change requires nbdkit >= 1.22 which is checked at runtime only if this feature is used. [*] Note here I mean conversions not the total runtime of virt-v2v. When doing the copy the cookie does not expire because it is continuously auto-renewed by VMware as we continuously access the disk (this works differently from systems like Docker where the cookie is only valid from the absolute time when it is first created). This change also implements the cookie-script logic for copying. --- v2v/nbdkit_sources.ml | 34 ++++++++--- v2v/nbdkit_sources.mli | 5 +- v2v/parse_libvirt_xml.ml | 3 +- v2v/vCenter.ml | 129 +++++++++++++++++++-------------------- 4 files changed, 91 insertions(+), 80 deletions(-) diff --git a/v2v/nbdkit_sources.ml b/v2v/nbdkit_sources.ml index 7c177e358..16af5f5c2 100644 --- a/v2v/nbdkit_sources.ml +++ b/v2v/nbdkit_sources.ml @@ -26,7 +26,6 @@ open Types open Utils let nbdkit_min_version = (1, 12, 0) -let nbdkit_min_version_string = "1.12.0" type password = | NoPassword (* no password option at all *) @@ -38,11 +37,16 @@ let error_unless_nbdkit_working () = if not (Nbdkit.is_installed ()) then error (f_"nbdkit is not installed or not working") -let error_unless_nbdkit_min_version config = +let error_unless_nbdkit_version_ge config min_version = let version = Nbdkit.version config in - if version < nbdkit_min_version then - error (f_"nbdkit is too old. nbdkit >= %s is required.") - nbdkit_min_version_string + if version < min_version then ( + let min_major, min_minor, min_release = min_version in + error (f_"nbdkit is too old. nbdkit >= %d.%d.%d is required.") + min_major min_minor min_release + ) + +let error_unless_nbdkit_min_version config = + error_unless_nbdkit_version_ge config nbdkit_min_version let error_unless_nbdkit_plugin_exists plugin = if not (Nbdkit.probe_plugin plugin) then @@ -297,23 +301,35 @@ let create_ssh ?bandwidth ~password ?port ~server ?user path = common_create ?bandwidth password "ssh" (get_args ()) (* Create an nbdkit module specialized for reading from Curl sources. *) -let create_curl ?bandwidth ?cookie ~password ?(sslverify=true) ?user url = +let create_curl ?bandwidth ?cookie_script ?cookie_script_renew + ?(sslverify=true) url = error_unless_nbdkit_plugin_exists "curl"; + (* The cookie* parameters require nbdkit 1.22, so check that early. *) + if cookie_script <> None || cookie_script_renew <> None then ( + let config = Nbdkit.config () in + error_unless_nbdkit_version_ge config (1, 22, 0) + ); + let add_arg, get_args = let args = ref [] in let add_arg (k, v) = List.push_front (k, v) args in let get_args () = List.rev !args in add_arg, get_args in - Option.may (fun s -> add_arg ("user", s)) user; (* https://bugzilla.redhat.com/show_bug.cgi?id=1146007#c10 *) add_arg ("timeout", "2000"); - Option.may (fun s -> add_arg ("cookie", s)) cookie; + Option.may (fun s -> add_arg ("cookie-script", s)) cookie_script; + Option.may (fun i -> add_arg ("cookie-script-renew", string_of_int i)) + cookie_script_renew; if not sslverify then add_arg ("sslverify", "false"); add_arg ("url", url); - common_create ?bandwidth password "curl" (get_args ()) + (* For lots of extra debugging, uncomment one or both lines. *) + (*add_arg ("--debug", "curl.verbose=1");*) + (*add_arg ("--debug", "curl.scripts=1");*) + + common_create ?bandwidth NoPassword "curl" (get_args ()) let run cmd = let sock, _ = Nbdkit.run_unix cmd in diff --git a/v2v/nbdkit_sources.mli b/v2v/nbdkit_sources.mli index 94810ea61..922642df7 100644 --- a/v2v/nbdkit_sources.mli +++ b/v2v/nbdkit_sources.mli @@ -60,10 +60,9 @@ val create_ssh : ?bandwidth:Types.bandwidth -> Note this doesn't run nbdkit yet, it just creates the object. *) val create_curl : ?bandwidth:Types.bandwidth -> - ?cookie:string -> - password:password -> + ?cookie_script:string -> + ?cookie_script_renew:int -> ?sslverify:bool -> - ?user:string -> string -> Nbdkit.cmd (** Create a nbdkit object using the Curl plugin. The required string parameter is the URL. diff --git a/v2v/parse_libvirt_xml.ml b/v2v/parse_libvirt_xml.ml index 0b1368392..fffc5a247 100644 --- a/v2v/parse_libvirt_xml.ml +++ b/v2v/parse_libvirt_xml.ml @@ -319,8 +319,7 @@ let parse_libvirt_xml ?bandwidth ?conn xml = | _, Some port -> invalid_arg "invalid port number in libvirt XML" in sprintf "%s://%s%s%s" driver host port (uri_quote path) in - let nbdkit = Nbdkit_sources.create_curl ?bandwidth ~password:NoPassword - url in + let nbdkit = Nbdkit_sources.create_curl ?bandwidth url in let qemu_uri = Nbdkit_sources.run nbdkit in add_disk qemu_uri format controller P_dont_rewrite | Some protocol, _, _ -> diff --git a/v2v/vCenter.ml b/v2v/vCenter.ml index b41422c1f..6b65f9f94 100644 --- a/v2v/vCenter.ml +++ b/v2v/vCenter.ml @@ -39,11 +39,12 @@ let rec qemu_uri_of_path ?bandwidth ?password_file dcPath uri server path = (* XXX only works if the query string is not URI-quoted *) String.find query "no_verify=1" = -1 in + (* Check the URL exists and authentication info is correct. *) let https_url = let https_url = get_https_url dcPath uri server path in - (* Check the URL exists. *) - let status, _, _ = + let status, dump_response = fetch_headers_from_url password_file uri sslverify https_url in + (* If a disk is actually a snapshot image it will have '-00000n' * appended to its name, e.g.: * [yellow:storage1] RHEL4-X/RHEL4-X-000003.vmdk @@ -51,28 +52,68 @@ let rec qemu_uri_of_path ?bandwidth ?password_file dcPath uri server path = * a 404 and the vmdk name looks like it might be a snapshot, try * again without the snapshot suffix. *) - if status = "404" && PCRE.matches snapshot_re path then ( - let path = PCRE.sub 1 ^ PCRE.sub 2 in - get_https_url dcPath uri server path - ) - else - (* Note that other non-200 status errors will be handled - * in get_session_cookie below, so we don't have to worry - * about them here. - *) - https_url in + let https_url, status, dump_response = + if status = "404" && PCRE.matches snapshot_re path then ( + let path = PCRE.sub 1 ^ PCRE.sub 2 in + let https_url = get_https_url dcPath uri server path in + let status, dump_response = + fetch_headers_from_url password_file uri sslverify https_url in + https_url, status, dump_response + ) + else (https_url, status, dump_response) in - let session_cookie = - get_session_cookie password_file uri sslverify https_url in + if status = "401" then ( + dump_response stderr; + if uri.uri_user <> None then + error (f_"vcenter: incorrect username or password") + else + error (f_"vcenter: incorrect username or password. You might need to specify the username in the URI like this: [vpx|esx|..]://USERNAME@[etc]") + ); - let password = - match password_file with - | None -> Nbdkit_sources.NoPassword - | Some password_file -> Nbdkit_sources.PasswordFile password_file in + if status = "404" then ( + dump_response stderr; + error (f_"vcenter: URL not found: %s") https_url + ); + + if status <> "200" then ( + dump_response stderr; + error (f_"vcenter: invalid response from server: %s") status + ); + + https_url in + + (* Write a cookie script to retrieve the session cookie. + * See nbdkit-curl-plugin(1) "Example: VMware ESXi cookies" + *) + let cookie_script, chan = + Filename.open_temp_file ~perms:0o700 "v2vcs" ".sh" in + unlink_on_exit cookie_script; + let fpf fs = fprintf chan fs in + fpf "#!/bin/sh -\n"; + fpf "\n"; + fpf "curl --head -s"; + if not sslverify then fpf " --insecure"; + (match uri.uri_user, password_file with + | None, None -> () + | Some user, None -> fpf " -u %s" (quote user) + | None, Some password_file -> + fpf " -u \"$LOGNAME\":\"$(cat %s)\"" (quote password_file) + | Some user, Some password_file -> + fpf " -u %s:\"$(cat %s)\"" (quote user) (quote password_file) + ); + fpf " %s" (quote https_url); + fpf " |\n"; + fpf "\tsed -ne %s\n" (quote "{ s/^Set-Cookie: \\([^;]*\\);.*/\\1/ip }"); + close_out chan; + + (* VMware authentication expires after 30 minutes so we must renew + * after < 30 minutes. + *) + let cookie_script_renew = 25*60 in let nbdkit = - Nbdkit_sources.create_curl ?bandwidth ?cookie:session_cookie ~password ~sslverify - ?user:uri.uri_user https_url in + Nbdkit_sources.create_curl ?bandwidth ~cookie_script ~cookie_script_renew + ~sslverify https_url in let qemu_uri = Nbdkit_sources.run nbdkit in (* Return the QEMU URI. *) @@ -98,44 +139,7 @@ and get_https_url dcPath uri server path = (uri_quote path) (uri_quote dcPath) (uri_quote datastore) ) -and get_session_cookie password_file uri sslverify https_url = - let status, headers, dump_response = - fetch_headers_from_url password_file uri sslverify https_url in - - if status = "401" then ( - dump_response stderr; - if uri.uri_user <> None then - error (f_"vcenter: incorrect username or password") - else - error (f_"vcenter: incorrect username or password. You might need to specify the username in the URI like this: [vpx|esx|..]://USERNAME@[etc]") - ); - - if status = "404" then ( - dump_response stderr; - error (f_"vcenter: URL not found: %s") https_url - ); - - if status <> "200" then ( - dump_response stderr; - error (f_"vcenter: invalid response from server: %s") status - ); - - (* Get the cookie. *) - let rec loop = function - | [] -> - dump_response stderr; - warning (f_"vcenter: could not read session cookie from the vCenter Server, conversion may consume all sessions on the server and fail part way through"); - None - | ("set-cookie", cookie) :: _ -> - let cookie, _ = String.split ";" cookie in - Some cookie - - | _ :: headers -> - loop headers - in - loop headers - -(* Fetch the status and reply headers from a URL. *) +(* Fetch the status from a URL. *) and fetch_headers_from_url password_file uri sslverify https_url = let curl_args = ref [ "head", None; @@ -184,11 +188,4 @@ and fetch_headers_from_url password_file uri sslverify https_url = let s = List.hd (List.rev ss) in String.sub s (String.index s ' ' + 1) 3 in - let headers = - List.map ( - fun header -> - let h, c = String.split ": " header in - String.lowercase_ascii h, c - ) headers in - - status, headers, dump_response + status, dump_response -- 2.27.0