Re: [Libguestfs] [PATCH v7 1/1] v2v: ova: don't extract files from OVA if it's not needed

We don't have to always extract all files from the OVA archive. The OVA, as defined in the standard, is plain tar. We can work directly over the tar archive if we use correct 'offset' and 'size' options when defining the backing file for QEMU. This puts much lower requirement on available disk space. Since the virt-v2v behaviour for OVA input now depends on QEMU version available this affects some of the tests. Expected result of the affected also has to depend on the QEMU used thus such tests will have two *.expected files. Signed-off-by: Tomáš Golembiovský <tgolembi(a)redhat.com&gt; ---  mllib/common_utils.ml                   |   9 +++  mllib/common_utils.mli                  |  10 +++  mllib/common_utils_tests.ml             |   7 ++

 test-data/test-utils.sh                 |  19 +++++  v2v/Makefile.am                         |   2 +  v2v/input_ova.ml                        | 109 +++++++++++++++++++++----  v2v/test-v2v-i-ova-formats.sh           |   5 +-  v2v/test-v2v-i-ova-subfolders.expected2 |  18 +++++  v2v/test-v2v-i-ova-subfolders.sh        |  13 ++-  v2v/test-v2v-i-ova-tar.expected         |  18 +++++  v2v/test-v2v-i-ova-tar.expected2        |  18 +++++  v2v/test-v2v-i-ova-tar.ovf              | 138 ++++++++++++++++++++++++++++++++  v2v/test-v2v-i-ova-tar.sh               |  72 +++++++++++++++++  v2v/test-v2v-i-ova-two-disks.expected2  |  19 +++++  v2v/test-v2v-i-ova-two-disks.sh         |  13 ++-  v2v/utils.ml                            |  72 +++++++++++++++++  v2v/utils.mli                           |  12 +++  17 files changed, 530 insertions(+), 24 deletions(-)  create mode 100644 v2v/test-v2v-i-ova-subfolders.expected2  create mode 100644 v2v/test-v2v-i-ova-tar.expected  create mode 100644 v2v/test-v2v-i-ova-tar.expected2  create mode 100644 v2v/test-v2v-i-ova-tar.ovf  create mode 100755 v2v/test-v2v-i-ova-tar.sh  create mode 100644 v2v/test-v2v-i-ova-two-disks.expected2 diff --git a/mllib/common_utils.ml b/mllib/common_utils.ml index e9ae6a4a2..a79abdd7e 100644 --- a/mllib/common_utils.ml +++ b/mllib/common_utils.ml @@ -236,6 +236,15 @@ end  let (//) = Filename.concat  let quote = Filename.quote   +let subdirectory parent path = +  if path = parent then +    "" +  else if String.is_prefix path (parent // "") then +    let len = String.length parent in +    String.sub path (len+1) (String.length path - len-1) +  else +    raise (Invalid_argument (sprintf "%S is not a path prefix of %S" parent path)) +  let ( +^ ) = Int64.add  let ( -^ ) = Int64.sub  let ( *^ ) = Int64.mul diff --git a/mllib/common_utils.mli b/mllib/common_utils.mli index 722e528e5..977ce6576 100644 --- a/mllib/common_utils.mli +++ b/mllib/common_utils.mli @@ -116,6 +116,16 @@ val ( // ) : string -> string -> string  val quote : string -> string  (** Shell-safe quoting of a string (alias for {!Filename.quote}). *)   +val subdirectory : string -> string -> string +(** [subdirectory parent path] returns subdirectory part of [path] relative +    to the [parent]. If [path] and [parent] point to the same directory empty +    string is returned. + +    Note: path normalization on arguments is NOT performed! + +    If [parent] is not a path prefix of [path] the function raises +    [Invalid_argument]. *) +  val ( +^ ) : int64 -> int64 -> int64  val ( -^ ) : int64 -> int64 -> int64  val ( *^ ) : int64 -> int64 -> int64 diff --git a/mllib/common_utils_tests.ml b/mllib/common_utils_tests.ml index 77b0524c1..aacc01e04 100644 --- a/mllib/common_utils_tests.ml +++ b/mllib/common_utils_tests.ml @@ -27,6 +27,12 @@ let assert_equal_int = assert_equal ~printer:(fun x -> string_of_int x)  let assert_equal_int64 = assert_equal ~printer:(fun x -> Int64.to_string x)  let assert_equal_stringlist = assert_equal ~printer:(fun x -> "(" ^ (String.escaped (String.concat "," x)) ^ ")")   +let test_subdirectory ctx = +  assert_equal_string "" (subdirectory "/foo" "/foo"); +  assert_equal_string "" (subdirectory "/foo" "/foo/"); +  assert_equal_string "bar" (subdirectory "/foo" "/foo/bar"); +  assert_equal_string "bar/baz" (subdirectory "/foo" "/foo/bar/baz") +  (* Test Common_utils.int_of_le32 and Common_utils.le32_of_int. *)  let test_le32 ctx =    assert_equal_int64 0x20406080L (int_of_le32 "\x80\x60\x40\x20"); @@ -129,6 +135,7 @@ let test_string_lines_split ctx =  let suite =    "mllib Common_utils" >:::      [ +      "subdirectory" >:: test_subdirectory;        "numeric.le32" >:: test_le32;        "sizes.parse_resize" >:: test_parse_resize;        "sizes.human_size" >:: test_human_size; diff --git a/test-data/test-utils.sh b/test-data/test-utils.sh index 86a5aaf12..1c4abe392 100755 --- a/test-data/test-utils.sh +++ b/test-data/test-utils.sh @@ -54,3 +54,22 @@ do_sha256 ()        ;;    esac  } + +# Returns 0 if QEMU version is greater or equal to the arguments +qemu_is_version() { +    if [ $# -ne 2 ] ; then +        echo "Usage: $0 <major_version> <minor_version>" >&2 +        return 3 +    fi + + +    [[ "$(qemu-img --version)" =~ 'qemu-img version '([0-9]+)\.([0-9]+) ]] || return 2 +    QMAJ=${BASH_REMATCH[1]} +    QMIN=${BASH_REMATCH[2]} + +    if [ \( $QMAJ -gt $1 \) -o \( $QMAJ -eq $1 -a $QMIN -ge $2 \) ] ; then +        return 0 +    fi + +    return 1 +} diff --git a/v2v/Makefile.am b/v2v/Makefile.am index 9189aaf12..d62ac477e 100644 --- a/v2v/Makefile.am +++ b/v2v/Makefile.am @@ -260,6 +260,7 @@ TESTS_ENVIRONMENT = $(top_builddir)/run --test    TESTS = \   test-v2v-docs.sh \ + test-v2v-i-ova-tar.sh \   test-v2v-i-ova-formats.sh \   test-v2v-i-ova-gz.sh \   test-v2v-i-ova-subfolders.sh \ @@ -359,6 +360,7 @@ EXTRA_DIST += \   test-v2v-i-ova-subfolders.expected \   test-v2v-i-ova-subfolders.ovf \   test-v2v-i-ova-subfolders.sh \ + test-v2v-i-ova-tar.sh \   test-v2v-i-ova-two-disks.expected \   test-v2v-i-ova-two-disks.ovf \   test-v2v-i-ova-two-disks.sh \ diff --git a/v2v/input_ova.ml b/v2v/input_ova.ml index 40f723633..d2b48c180 100644 --- a/v2v/input_ova.ml +++ b/v2v/input_ova.ml @@ -39,17 +39,23 @@ object      method source () =   -    let untar ?(format = "") file outdir = -      let cmd = [ "tar"; sprintf "-x%sf" format; file; "-C"; outdir ] in +    (* Untar part or all files from tar archive. If [paths] is specified it is +     * a list of paths in the tar archive. +     *) +    let untar ?(format = "") ?paths file outdir = +      let cmd = +        [ "tar"; sprintf "-x%sf" format; file; "-C"; outdir ] +        @ match paths with None -> [] | Some p -> p in        if run_command cmd <> 0 then -        error (f_"error unpacking %s, see earlier error messages") ova in +        error (f_"error unpacking %s, see earlier error messages") ova +    in        (* Extract ova file. *) -    let exploded = +    let exploded, partial =        (* The spec allows a directory to be specified as an ova.  This         * is also pretty convenient.         *) -      if is_directory ova then ova +      if is_directory ova then ova, false        else (          let uncompress_head zcat file =            let cmd = sprintf "%s %s" zcat (quote file) in @@ -67,11 +73,35 @@ object              tmpfile in   +        (* Untar only ovf and manifest from the archive *) +        let untar_metadata ova outdir = +          let files = +            external_command (sprintf "tar -tf %s" (Filename.quote ova)) in +          let files = +            filter_map (fun f -> +              if Filename.check_suffix f ".ovf" || +                  Filename.check_suffix f ".mf" then +                Some f +              else None +            ) files in +          untar ~paths:files ova outdir +        in +          match detect_file_type ova with          | `Tar ->            (* Normal ovas are tar file (not compressed). *) -          untar ova tmpdir; -          tmpdir +          let qmajor, qminor = qemu_img_version () in +          if qmajor > 2 || (qmajor == 2 && qminor >= 8) then ( +            (* If QEMU is 2.8 or newer we don't have to extract everything. +             * We can access disks inside the tar archive directly. +             *) +            untar_metadata ova tmpdir; +            tmpdir, true +          ) else ( +            untar ova tmpdir; +            tmpdir, false +          ) +          | `Zip ->            (* However, although not permitted by the spec, people ship             * zip files as ova too. @@ -81,7 +111,7 @@ object              [ "-j"; "-d"; tmpdir; ova ] in            if run_command cmd <> 0 then              error (f_"error unpacking %s, see earlier error messages") ova; -          tmpdir +          tmpdir, false          | (`GZip|`XZ) as format ->            let zcat, tar_fmt =              match format with @@ -94,7 +124,7 @@ object            (match tmpfiletype with            | `Tar ->              untar ~format:tar_fmt ova tmpdir; -            tmpdir +            tmpdir, false            | `Zip | `GZip | `XZ | `Unknown ->              error (f_"%s: unsupported file format\n\nFormats which we currently understand for '-i ova' are: tar (uncompressed, compress with gzip or xz), zip") ova            ) @@ -152,6 +182,7 @@ object        fun mf ->          debug "processing manifest %s" mf;          let mf_folder = Filename.dirname mf in +        let mf_subfolder = subdirectory exploded mf_folder in          let chan = open_in mf in          let rec loop () =            let line = input_line chan in @@ -160,7 +191,11 @@ object              let disk = Str.matched_group 2 line in              let expected = Str.matched_group 3 line in              let csum = Checksums.of_string mode expected in -            try Checksums.verify_checksum csum (mf_folder // disk) +            try +              if partial then +                Checksums.verify_checksum csum ~tar:ova (mf_subfolder // disk) +              else +                Checksums.verify_checksum csum (mf_folder // disk)              with Checksums.Mismatched_checksum (_, actual) ->                error (f_"checksum of disk %s does not match manifest %s (actual %s(%s) = %s, expected %s(%s) = %s)")                  disk mf mode disk actual mode disk expected; @@ -283,9 +318,25 @@ object              | Some "gzip" -> true              | Some s -> error (f_"unsupported compression in OVF: %s") s in   -          (* Does the file exist and is it readable? *) -          let filename = ovf_folder // filename in -          Unix.access filename [Unix.R_OK]; +          let partial = +            if compressed && partial then ( +              (* We cannot access compressed disk inside the tar; we have to +               * extract it *) +              untar ~paths:[(subdirectory exploded ovf_folder) // filename] +                ova tmpdir; +              false +            ) +            else +              partial in + +          let filename = +            if partial then +              (subdirectory exploded ovf_folder) // filename +            else ( +              (* Does the file exist and is it readable? *) +              Unix.access (ovf_folder // filename) [Unix.R_OK]; +              ovf_folder // filename +            ) in              (* The spec allows the file to be gzip-compressed, in which case             * we must uncompress it into the tmpdir. @@ -302,9 +353,39 @@ object              )              else filename in   +          let qemu_uri = +            if not partial then ( +              filename +            ) +            else ( +              let offset, size = +                try find_file_in_tar ova filename +                with +                | Not_found -> +                  error (f_"file '%s' not found in the ova") filename +                | Failure msg -> error (f_"%s") msg in +              (* QEMU requires size aligned to 512 bytes. This is safe because +               * tar also works with 512 byte blocks. +               *) +              let size = roundup64 size 512L in +              let doc = [ +                "file", JSON.Dict [ +                  "driver", JSON.String "raw"; +                  "offset", JSON.Int64 offset; +                  "size", JSON.Int64 size; +                  "file", JSON.Dict [ +                    "filename", JSON.String ova] +                  ] +                ] in +              let uri = +                sprintf "json:%s" (JSON.string_of_doc ~fmt:JSON.Compact doc) in +              debug "json: %s" uri; +              uri +            ) in +            let disk = {              s_disk_id = i; -            s_qemu_uri = filename; +            s_qemu_uri = qemu_uri;              s_format = Some "vmdk";              s_controller = controller;            } in diff --git a/v2v/test-v2v-i-ova-formats.sh b/v2v/test-v2v-i-ova-formats.sh index bd3e30ee8..0eec600eb 100755 --- a/v2v/test-v2v-i-ova-formats.sh +++ b/v2v/test-v2v-i-ova-formats.sh @@ -22,7 +22,7 @@ unset CDPATH  export LANG=C  set -e   -formats="tar zip tar-gz tar-xz" +formats="zip tar-gz tar-xz"    if [ -n "$SKIP_TEST_V2V_I_OVA_FORMATS_SH" ]; then      echo "$0: test skipped because environment variable is set" @@ -63,9 +63,6 @@ cp ../test-v2v-i-ova-formats.ovf .    for format in $formats; do      case "$format" in -        tar) -            tar -cf test-$format.ova test-v2v-i-ova-formats.ovf disk1.vmdk disk1.mf -            ;;          zip)              zip -r test test-v2v-i-ova-formats.ovf disk1.vmdk disk1.mf              mv test.zip test-$format.ova diff --git a/v2v/test-v2v-i-ova-subfolders.expected2 b/v2v/test-v2v-i-ova-subfolders.expected2 new file mode 100644 index 000000000..87996d202 --- /dev/null +++ b/v2v/test-v2v-i-ova-subfolders.expected2 @@ -0,0 +1,18 @@ +Source guest information (--print-source option): + +    source name: 2K8R2EESP1_2_Medium +hypervisor type: vmware +         memory: 1073741824 (bytes) +       nr vCPUs: 1 +   CPU features:  +       firmware: uefi +        display:  +          video:  +          sound:  +disks: + json:{ "file": { "driver": "raw", "offset": 2048, "size": 10240, "file": { "filename": "test.ova" } } } (vmdk) [scsi] +removable media: + CD-ROM [ide] in slot 0 +NICs: + Network "Network adapter 1" + diff --git a/v2v/test-v2v-i-ova-subfolders.sh b/v2v/test-v2v-i-ova-subfolders.sh index 4fd1acea3..422959036 100755 --- a/v2v/test-v2v-i-ova-subfolders.sh +++ b/v2v/test-v2v-i-ova-subfolders.sh @@ -56,10 +56,17 @@ popd  # normalize the output.  $VG virt-v2v --debug-gc --quiet \      -i ova $d/test.ova \ -    --print-source | -sed 's,[^ \t]*\(subfolder/disk.*\.vmdk\),\1,' > $d/source +    --print-source > $d/source    # Check the parsed source is what we expect. -diff -u test-v2v-i-ova-subfolders.expected $d/source +if qemu_is_version 2 8 ; then +    # normalize the output +    sed -i -e "s,\"$d/,\"," $d/source +    diff -u test-v2v-i-ova-subfolders.expected2 $d/source +else +    # normalize the output +    sed -i -e 's,[^ \t]*\(subfolder/disk.*\.vmdk\),\1,' $d/source +    diff -u test-v2v-i-ova-subfolders.expected $d/source +fi    rm -rf $d diff --git a/v2v/test-v2v-i-ova-tar.expected b/v2v/test-v2v-i-ova-tar.expected new file mode 100644 index 000000000..7049aeecc --- /dev/null +++ b/v2v/test-v2v-i-ova-tar.expected @@ -0,0 +1,18 @@ +Source guest information (--print-source option): + +    source name: 2K8R2EESP1_2_Medium +hypervisor type: vmware +         memory: 1073741824 (bytes) +       nr vCPUs: 1 +   CPU features:  +       firmware: uefi +        display:  +          video:  +          sound:  +disks: + disk1.vmdk (vmdk) [scsi] +removable media: + CD-ROM [ide] in slot 0 +NICs: + Network "Network adapter 1" + diff --git a/v2v/test-v2v-i-ova-tar.expected2 b/v2v/test-v2v-i-ova-tar.expected2 new file mode 100644 index 000000000..1aa54dc37 --- /dev/null +++ b/v2v/test-v2v-i-ova-tar.expected2 @@ -0,0 +1,18 @@ +Source guest information (--print-source option): + +    source name: 2K8R2EESP1_2_Medium +hypervisor type: vmware +         memory: 1073741824 (bytes) +       nr vCPUs: 1 +   CPU features:  +       firmware: uefi +        display:  +          video:  +          sound:  +disks: + json:{ "file": { "driver": "raw", "offset": 9216, "size": 10240, "file": { "filename": "test-tar.ova" } } } (vmdk) [scsi] +removable media: + CD-ROM [ide] in slot 0 +NICs: + Network "Network adapter 1" + diff --git a/v2v/test-v2v-i-ova-tar.ovf b/v2v/test-v2v-i-ova-tar.ovf new file mode 100644 index 000000000..4827c7e9b --- /dev/null +++ b/v2v/test-v2v-i-ova-tar.ovf @@ -0,0 +1,138 @@ +<?xml version="1.0" encoding="UTF-8"?> +<Envelope vmw:buildId="build-1750787" xmlns="http://schemas.dmtf.org/ovf/envelope/1" xmlns:cim="http://schemas.dmtf.org/wbem/wscim/1/common" xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1" xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_Re... xmlns:vmw="http://www.vmware.com/schema/ovf" xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim- schema/2/CIM_VirtualSystemSettingData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> +  <References> +    <File ovf:href="disk1.vmdk" ovf:id="file1" ovf:size="7804077568"/> +  </References> +  <DiskSection> +    <Info>Virtual disk information</Info> +    <Disk ovf:capacity="50" ovf:capacityAllocationUnits="byte * 2^30" ovf:diskId="vmdisk1" ovf:fileRef="file1" ovf:format="http://www.vmware.com/interfaces/specifications/vmdk.htm... ovf:populatedSize="18975752192"/> +  </DiskSection> +  <NetworkSection> +    <Info>The list of logical networks</Info> +    <Network ovf:name="PG-VLAN60"> +      <Description>The PG-VLAN60 network</Description> +    </Network> +  </NetworkSection> +  <VirtualSystem ovf:id="2K8R2EESP1_2_Medium"> +    <Info>A virtual machine</Info> +    <Name>2K8R2EESP1_2_Medium</Name> +    <OperatingSystemSection ovf:id="103" vmw:osType="windows7Server64Guest"> +      <Info>The kind of installed guest operating system</Info> +      <Description>Microsoft Windows Server 2008 R2 (64-bit)</Description> +    </OperatingSystemSection> +    <VirtualHardwareSection> +      <Info>Virtual hardware requirements</Info> +      <System> +        <vssd:ElementName>Virtual Hardware Family</vssd:ElementName> +        <vssd:InstanceID>0</vssd:InstanceID> +        <vssd:VirtualSystemIdentifier>2K8R2EESP1_2_Medium</vssd:VirtualSystemIdentifier> +        <vssd:VirtualSystemType>vmx-10</vssd:VirtualSystemType> +      </System> +      <Item> +        <rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits> +        <rasd:Description>Number of Virtual CPUs</rasd:Description> +        <rasd:ElementName>1 virtual CPU(s)</rasd:ElementName> +        <rasd:InstanceID>1</rasd:InstanceID> +        <rasd:ResourceType>3</rasd:ResourceType> +        <rasd:VirtualQuantity>1</rasd:VirtualQuantity> +      </Item> +      <Item> +        <rasd:AllocationUnits>byte * 2^20</rasd:AllocationUnits> +        <rasd:Description>Memory Size</rasd:Description> +        <rasd:ElementName>1024MB of memory</rasd:ElementName> +        <rasd:InstanceID>2</rasd:InstanceID> +        <rasd:ResourceType>4</rasd:ResourceType> +        <rasd:VirtualQuantity>1024</rasd:VirtualQuantity> +      </Item> +      <Item> +        <rasd:Address>0</rasd:Address> +        <rasd:Description>SCSI Controller</rasd:Description> +        <rasd:ElementName>SCSI controller 0</rasd:ElementName> +        <rasd:InstanceID>3</rasd:InstanceID> +        <rasd:ResourceSubType>lsilogicsas</rasd:ResourceSubType> +        <rasd:ResourceType>6</rasd:ResourceType> +        <vmw:Config ovf:required="false" vmw:key="slotInfo.pciSlotNumber" vmw:value="160"/> +      </Item> +      <Item> +        <rasd:Address>1</rasd:Address> +        <rasd:Description>IDE Controller</rasd:Description> +        <rasd:ElementName>IDE 1</rasd:ElementName> +        <rasd:InstanceID>4</rasd:InstanceID> +        <rasd:ResourceType>5</rasd:ResourceType> +      </Item> +      <Item> +        <rasd:Address>0</rasd:Address> +        <rasd:Description>IDE Controller</rasd:Description> +        <rasd:ElementName>IDE 0</rasd:ElementName> +        <rasd:InstanceID>5</rasd:InstanceID> +        <rasd:ResourceType>5</rasd:ResourceType> +      </Item> +      <Item ovf:required="false"> +        <rasd:AutomaticAllocation>false</rasd:AutomaticAllocation> +        <rasd:ElementName>Video card</rasd:ElementName> +        <rasd:InstanceID>6</rasd:InstanceID> +        <rasd:ResourceType>24</rasd:ResourceType> +        <vmw:Config ovf:required="false" vmw:key="enable3DSupport" vmw:value="false"/> +        <vmw:Config ovf:required="false" vmw:key="use3dRenderer" vmw:value="automatic"/> +        <vmw:Config ovf:required="false" vmw:key="useAutoDetect" vmw:value="true"/> +        <vmw:Config ovf:required="false" vmw:key="videoRamSizeInKB" vmw:value="4096"/> +      </Item> +      <Item ovf:required="false"> +        <rasd:AutomaticAllocation>false</rasd:AutomaticAllocation> +        <rasd:ElementName>VMCI device</rasd:ElementName> +        <rasd:InstanceID>7</rasd:InstanceID> +        <rasd:ResourceSubType>vmware.vmci</rasd:ResourceSubType> +        <rasd:ResourceType>1</rasd:ResourceType> +        <vmw:Config ovf:required="false" vmw:key="allowUnrestrictedCommunication" vmw:value="false"/> +        <vmw:Config ovf:required="false" vmw:key="slotInfo.pciSlotNumber" vmw:value="32"/> +      </Item> +      <Item ovf:required="false"> +        <rasd:AddressOnParent>0</rasd:AddressOnParent> +        <rasd:AutomaticAllocation>false</rasd:AutomaticAllocation> +        <rasd:ElementName>CD/DVD drive 1</rasd:ElementName> +        <rasd:InstanceID>8</rasd:InstanceID> +        <rasd:Parent>4</rasd:Parent> +        <rasd:ResourceSubType>vmware.cdrom.atapi</rasd:ResourceSubType> +        <rasd:ResourceType>15</rasd:ResourceType> +      </Item> +      <Item> +        <rasd:AddressOnParent>0</rasd:AddressOnParent> +        <rasd:ElementName>Hard disk 1</rasd:ElementName> +        <rasd:HostResource>ovf:/disk/vmdisk1</rasd:HostResource> +        <rasd:InstanceID>9</rasd:InstanceID> +        <rasd:Parent>3</rasd:Parent> +        <rasd:ResourceType>17</rasd:ResourceType> +        <vmw:Config ovf:required="false" vmw:key="backing.writeThrough" vmw:value="false"/> +      </Item> +      <Item> +        <rasd:AddressOnParent>7</rasd:AddressOnParent> +        <rasd:AutomaticAllocation>true</rasd:AutomaticAllocation> +        <rasd:Connection>PG-VLAN60</rasd:Connection> +        <rasd:Description>E1000 ethernet adapter on &quot;PG-VLAN60&quot;</rasd:Description> +        <rasd:ElementName>Network adapter 1</rasd:ElementName> +        <rasd:InstanceID>11</rasd:InstanceID> +        <rasd:ResourceSubType>E1000</rasd:ResourceSubType> +        <rasd:ResourceType>10</rasd:ResourceType> +        <vmw:Config ovf:required="false" vmw:key="slotInfo.pciSlotNumber" vmw:value="33"/> +        <vmw:Config ovf:required="false" vmw:key="wakeOnLanEnabled" vmw:value="true"/> +      </Item> +      <vmw:Config ovf:required="false" vmw:key="cpuHotAddEnabled" vmw:value="false"/> +      <vmw:Config ovf:required="false" vmw:key="cpuHotRemoveEnabled" vmw:value="false"/> +      <vmw:Config ovf:required="false" vmw:key="firmware" vmw:value="efi"/> +      <vmw:Config ovf:required="false" vmw:key="virtualICH7MPresent" vmw:value="false"/> +      <vmw:Config ovf:required="false" vmw:key="virtualSMCPresent" vmw:value="false"/> +      <vmw:Config ovf:required="false" vmw:key="memoryHotAddEnabled" vmw:value="false"/> +      <vmw:Config ovf:required="false" vmw:key="nestedHVEnabled" vmw:value="false"/> +      <vmw:Config ovf:required="false" vmw:key="powerOpInfo.powerOffType" vmw:value="soft"/> +      <vmw:Config ovf:required="false" vmw:key="powerOpInfo.resetType" vmw:value="soft"/> +      <vmw:Config ovf:required="false" vmw:key="powerOpInfo.standbyAction" vmw:value="checkpoint"/> +      <vmw:Config ovf:required="false" vmw:key="powerOpInfo.suspendType" vmw:value="hard"/> +      <vmw:Config ovf:required="false" vmw:key="tools.afterPowerOn" vmw:value="true"/> +      <vmw:Config ovf:required="false" vmw:key="tools.afterResume" vmw:value="true"/> +      <vmw:Config ovf:required="false" vmw:key="tools.beforeGuestShutdown" vmw:value="true"/> +      <vmw:Config ovf:required="false" vmw:key="tools.beforeGuestStandby" vmw:value="true"/> +      <vmw:Config ovf:required="false" vmw:key="tools.syncTimeWithHost" vmw:value="false"/> +      <vmw:Config ovf:required="false" vmw:key="tools.toolsUpgradePolicy" vmw:value="upgradeAtPowerCycle"/> +    </VirtualHardwareSection> +  </VirtualSystem> +</Envelope>                                  diff --git a/v2v/test-v2v-i-ova-tar.sh b/v2v/test-v2v-i-ova-tar.sh new file mode 100755 index 000000000..c3b0588f4 --- /dev/null +++ b/v2v/test-v2v-i-ova-tar.sh @@ -0,0 +1,72 @@ +#!/bin/bash - +# libguestfs virt-v2v test script +# Copyright (C) 2014-2016 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Test -i ova option with ova file compressed in different ways + +unset CDPATH +export LANG=C +set -e + +if [ -n "$SKIP_TEST_V2V_I_OVA_FORMATS_SH" ]; then +    echo "$0: test skipped because environment variable is set" +    exit 77 +fi + +if [ "$(guestfish get-backend)" = "uml" ]; then +    echo "$0: test skipped because UML backend does not support network" +    exit 77 +fi + +export VIRT_TOOLS_DATA_DIR="$srcdir/../test-data/fake-virt-tools" + +. $srcdir/../test-data/test-utils.sh + +d=test-v2v-i-ova-tar.d +rm -rf $d +mkdir $d + +pushd $d + +# Create a phony OVA.  This is only a test of source parsing, not +# conversion, so the contents of the disks doesn't matter. +truncate -s 10k disk1.vmdk +sha=`do_sha1 disk1.vmdk` +echo -e "SHA1(disk1.vmdk)= $sha\r" > disk1.mf +cp ../test-v2v-i-ova-tar.ovf . +tar -cf test-tar.ova test-v2v-i-ova-tar.ovf disk1.vmdk disk1.mf + +popd + +# Run virt-v2v but only as far as the --print-source stage +$VG virt-v2v --debug-gc --quiet \ +    -i ova $d/test-tar.ova \ +    --print-source > $d/source + +# Check the parsed source is what we expect. +if qemu_is_version 2 8 ; then +    # normalize the output +    sed -i -e "s,\"$d/,\"," $d/source +    diff -u test-v2v-i-ova-tar.expected2 $d/source +else +    # normalize the output +    sed -i -e 's,[^ \t]*\(disk.*.vmdk\),\1,' $d/source +    diff -u test-v2v-i-ova-tar.expected $d/source +fi + + +rm -rf $d diff --git a/v2v/test-v2v-i-ova-two-disks.expected2 b/v2v/test-v2v-i-ova-two-disks.expected2 new file mode 100644 index 000000000..b12ca1bd6 --- /dev/null +++ b/v2v/test-v2v-i-ova-two-disks.expected2 @@ -0,0 +1,19 @@ +Source guest information (--print-source option): + +    source name: 2K8R2EESP1_2_Medium +hypervisor type: vmware +         memory: 1073741824 (bytes) +       nr vCPUs: 1 +   CPU features:  +       firmware: bios +        display:  +          video:  +          sound:  +disks: + json:{ "file": { "driver": "raw", "offset": 9728, "size": 10240, "file": { "filename": "test.ova" } } } (vmdk) [scsi] + json:{ "file": { "driver": "raw", "offset": 21504, "size": 102400, "file": { "filename": "test.ova" } } } (vmdk) [scsi] +removable media: + CD-ROM [ide] in slot 0 +NICs: + Network "Network adapter 1" + diff --git a/v2v/test-v2v-i-ova-two-disks.sh b/v2v/test-v2v-i-ova-two-disks.sh index 26dd19860..310aff1b9 100755 --- a/v2v/test-v2v-i-ova-two-disks.sh +++ b/v2v/test-v2v-i-ova-two-disks.sh @@ -60,10 +60,17 @@ popd  # normalize the output.  $VG virt-v2v --debug-gc --quiet \      -i ova $d/test.ova \ -    --print-source | -sed 's,[^ \t]*\(disk.*.vmdk\),\1,' > $d/source +    --print-source  > $d/source    # Check the parsed source is what we expect. -diff -u test-v2v-i-ova-two-disks.expected $d/source +if qemu_is_version 2 8 ; then +    # normalize the output +    sed -i -e "s,\"$d/,\"," $d/source +    diff -u test-v2v-i-ova-two-disks.expected2 $d/source +else +    # normalize the output +    sed -i -e 's,[^ \t]*\(disk.*.vmdk\),\1,' $d/source +    diff -u test-v2v-i-ova-two-disks.expected $d/source +fi    rm -rf $d diff --git a/v2v/utils.ml b/v2v/utils.ml index 17ad8a29c..111dc0ea9 100644 --- a/v2v/utils.ml +++ b/v2v/utils.ml @@ -90,3 +90,75 @@ let du filename =    match lines with    | line::_ -> Int64.of_string line    | [] -> invalid_arg filename + +let qemu_img_version () = +  let lines = external_command "qemu-img --version" in +  match lines with +  | [] -> error ("'qemu-img --version' returned no output") +  | line :: _ -> +      let rex = Str.regexp +        "qemu-img version \\([0-9]+\\)\\.\\([0-9]+\\)" in +      if Str.string_match rex line 0 then ( +        try +          int_of_string (Str.matched_group 1 line), +          int_of_string (Str.matched_group 2 line) +        with Failure _ -> +          warning (f_"failed to parse qemu-img version(%S), assuming 0.9") +            line; +          0, 9 +      ) else ( +        warning (f_"failed to read qemu-img version(%S), assuming 0.9") +          line; +        0, 9 +      ) + +let find_file_in_tar tar filename = +  let lines = external_command (sprintf "tar tRvf %s" (Filename.quote tar)) in +  let rec loop lines = +    match lines with +    | [] -> raise Not_found +    | line :: lines -> ( +      (* Lines have the form: +        * block <offset>: <perms> <owner>/<group> <size> <mdate> <mtime> <file> +        *) +      let elems = Str.bounded_split (Str.regexp " +") line 8 in +      if List.length elems = 8 && List.hd elems = "block" then ( +        let elems = Array.of_list elems in +        let offset = elems.(1) in +        let size = elems.(4) in +        let fname = elems.(7) in + +        if fname <> filename then +          loop lines +        else ( +          let offset = +            try +              (* There should be a colon at the end *) +              let i = String.rindex offset ':' in +              if i == (String.length offset)-1 then +                Int64.of_string (String.sub offset 0 i) +              else +                raise (Failure "colon at wrong position") +            with Failure _ | Not_found -> +              raise (Failure (sprintf "invalid offset returned by tar: %S" +                offset)) in + +          let size = +            try Int64.of_string size +            with Failure _ -> +              raise (Failure (sprintf +                "invalid size returned by tar: %S" size)) in + +          (* Note: Offset is actualy block number and there is a single +            * block with tar header at the beginning of the file. So skip +            * the header and convert the block number to bytes before +            * returning. +            *) +          (offset +^ 1L) *^ 512L, size +        ) +      ) else +        raise (Failure (sprintf +          "failed to parse line returned by tar: %S" line)) +    ) +  in +  loop lines diff --git a/v2v/utils.mli b/v2v/utils.mli index 5eacb4aec..a9272568e 100644 --- a/v2v/utils.mli +++ b/v2v/utils.mli @@ -50,3 +50,15 @@ val du : string -> int64        This can raise either [Failure] or [Invalid_argument] in case      of errors. *) + +val qemu_img_version : unit -> int * int +(** Returns version of qemu-img as a tuple (major, minor). + +    In case of error (0,9) is returned. *) + +val find_file_in_tar : string -> string -> int64 * int64 +(** [find_file_in_tar tar filename] looks up file in [tar] archive and returns +    a tuple containing which byte it starts and how long the file is. + +    Function raises [Not_found] if there is no such file inside [tar] and +    [Failure] if there is any error parsing the tar output. *)