This way it is possible to change where the matching start, instead of always assuming it is the beginning. --- common/mlpcre/PCRE.ml | 2 +- common/mlpcre/PCRE.mli | 5 ++++- common/mlpcre/pcre-c.c | 16 +++++++++++++--- common/mlpcre/pcre_tests.ml | 15 ++++++++++++--- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/common/mlpcre/PCRE.ml b/common/mlpcre/PCRE.ml index b054928f9..33074af1c 100644 --- a/common/mlpcre/PCRE.ml +++ b/common/mlpcre/PCRE.ml @@ -23,7 +23,7 @@ exception Error of string * int type regexp external compile : ?anchored:bool -> ?caseless:bool -> ?dotall:bool -> ?extended:bool -> ?multiline:bool -> string -> regexp = "guestfs_int_pcre_compile_byte" "guestfs_int_pcre_compile" -external matches : regexp -> string -> bool = "guestfs_int_pcre_matches" +external matches : ?offset:int -> regexp -> string -> bool = "guestfs_int_pcre_matches" external sub : int -> string = "guestfs_int_pcre_sub" external subi : int -> int * int = "guestfs_int_pcre_subi" diff --git a/common/mlpcre/PCRE.mli b/common/mlpcre/PCRE.mli index eacb6fd90..e10d512fc 100644 --- a/common/mlpcre/PCRE.mli +++ b/common/mlpcre/PCRE.mli @@ -62,7 +62,7 @@ val compile : ?anchored:bool -> ?caseless:bool -> ?dotall:bool -> ?extended:bool See pcreapi(3) for details of what they do. All flags default to false. *) -val matches : regexp -> string -> bool +val matches : ?offset:int -> regexp -> string -> bool (** Test whether the regular expression matches the string. This returns true if the regexp matches or false otherwise. @@ -71,6 +71,9 @@ val matches : regexp -> string -> bool or the thread/program exits. You can call {!sub} to return these substrings. + The [?offset] flag is used to change the start of the search, + which by default is at the beginning of the string (position 0). + This can raise {!Error} if PCRE returns an error. *) val sub : int -> string diff --git a/common/mlpcre/pcre-c.c b/common/mlpcre/pcre-c.c index 0762a8341..be054a004 100644 --- a/common/mlpcre/pcre-c.c +++ b/common/mlpcre/pcre-c.c @@ -121,6 +121,15 @@ is_Some_true (value v) Bool_val (Field (v, 0)) /* Some true */; } +static int +Optint_val (value intv, int defval) +{ + if (intv == Val_int (0)) /* None */ + return defval; + else /* Some int */ + return Int_val (Field (intv, 0)); +} + value guestfs_int_pcre_compile (value anchoredv, value caselessv, value dotallv, value extendedv, value multilinev, @@ -165,9 +174,9 @@ guestfs_int_pcre_compile_byte (value *argv, int argn) } value -guestfs_int_pcre_matches (value rev, value strv) +guestfs_int_pcre_matches (value offsetv, value rev, value strv) { - CAMLparam2 (rev, strv); + CAMLparam3 (offsetv, rev, strv); pcre *re = Regexp_val (rev); struct last_match *m, *oldm; size_t len = caml_string_length (strv); @@ -205,7 +214,8 @@ guestfs_int_pcre_matches (value rev, value strv) caml_raise_out_of_memory (); } - m->r = pcre_exec (re, NULL, m->subject, len, 0, 0, m->vec, veclen); + m->r = pcre_exec (re, NULL, m->subject, len, Optint_val (offsetv, 0), 0, + m->vec, veclen); if (m->r < 0 && m->r != PCRE_ERROR_NOMATCH) { int ret = m->r; free_last_match (m); diff --git a/common/mlpcre/pcre_tests.ml b/common/mlpcre/pcre_tests.ml index 346019c40..f199ad63a 100644 --- a/common/mlpcre/pcre_tests.ml +++ b/common/mlpcre/pcre_tests.ml @@ -18,6 +18,10 @@ open Printf +let optget def = function + | None -> def + | Some v -> v + let compile ?(anchored = false) ?(caseless = false) ?(dotall = false) ?(extended = false) ?(multiline = false) patt = @@ -30,9 +34,9 @@ let compile ?(anchored = false) ?(caseless = false) patt; PCRE.compile ~anchored ~caseless ~dotall ~extended ~multiline patt -let matches re str = - eprintf "PCRE.matches %s ->%!" str; - let r = PCRE.matches re str in +let matches ?offset re str = + eprintf "PCRE.matches %s, %d ->%!" str (optget 0 offset); + let r = PCRE.matches ?offset re str in eprintf " %b\n%!" r; r @@ -103,6 +107,11 @@ let () = assert (subi 1 = (2, 3)); assert (subi 2 = (3, 3)); + assert (matches ~offset:5 re0 "aaabcabc" = true); + assert (sub 0 = "ab"); + + assert (matches ~offset:5 re0 "aaabcbaac" = false); + assert (replace re0 "dd" "abcabcaabccca" = "ddcabcaabccca"); assert (replace ~global:true re0 "dd" "abcabcaabccca" = "ddcddcddccca"); -- 2.20.1

This helper module provides a facility to replace %{FOO}-like variables in text strings with user-provided content. --- .gitignore | 1 + v2v/Makefile.am | 32 +++++++++++- v2v/dummy.c | 2 + v2v/var_expander.ml | 69 +++++++++++++++++++++++++ v2v/var_expander.mli | 82 ++++++++++++++++++++++++++++++ v2v/var_expander_tests.ml | 103 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 287 insertions(+), 2 deletions(-) create mode 100644 v2v/dummy.c create mode 100644 v2v/var_expander.ml create mode 100644 v2v/var_expander.mli create mode 100644 v2v/var_expander_tests.ml diff --git a/.gitignore b/.gitignore index 29d3e3aae..9a448fc4e 100644 --- a/.gitignore +++ b/.gitignore @@ -694,6 +694,7 @@ Makefile.in /v2v/uefi.ml /v2v/uefi.mli /v2v/v2v_unit_tests +/v2v/var_expander_tests /v2v/virt-v2v /v2v/virt-v2v.1 /v2v/virt-v2v-copy-to-local diff --git a/v2v/Makefile.am b/v2v/Makefile.am index 2312812fb..f196be81d 100644 --- a/v2v/Makefile.am +++ b/v2v/Makefile.am @@ -98,6 +98,7 @@ SOURCES_MLI = \ utils.mli \ v2v.mli \ vCenter.mli \ + var_expander.mli \ windows.mli \ windows_virtio.mli @@ -106,6 +107,7 @@ SOURCES_ML = \ types.ml \ uefi.ml \ utils.ml \ + var_expander.ml \ python_script.ml \ name_from_disk.ml \ vCenter.ml \ @@ -442,7 +444,7 @@ TESTS += \ endif if HAVE_OCAML_PKG_OUNIT -TESTS += v2v_unit_tests +TESTS += v2v_unit_tests var_expander_tests endif if ENABLE_APPLIANCE @@ -651,7 +653,7 @@ EXTRA_DIST += \ # Unit tests. check_PROGRAMS = if HAVE_OCAML_PKG_OUNIT -check_PROGRAMS += v2v_unit_tests +check_PROGRAMS += v2v_unit_tests var_expander_tests endif v2v_unit_tests_BOBJECTS = \ @@ -671,13 +673,28 @@ v2v_unit_tests_SOURCES = $(virt_v2v_SOURCES) v2v_unit_tests_CPPFLAGS = $(virt_v2v_CPPFLAGS) v2v_unit_tests_CFLAGS = $(virt_v2v_CFLAGS) +var_expander_tests_BOBJECTS = \ + var_expander.cmo \ + var_expander_tests.cmo +var_expander_tests_XOBJECTS = $(var_expander_tests_BOBJECTS:.cmo=.cmx) + +var_expander_tests_SOURCES = dummy.c +var_expander_tests_CPPFLAGS = $(virt_v2v_CPPFLAGS) +var_expander_tests_CFLAGS = $(virt_v2v_CFLAGS) + if !HAVE_OCAMLOPT # Can't call this v2v_unit_tests_OBJECTS because automake gets confused. v2v_unit_tests_THEOBJECTS = $(v2v_unit_tests_BOBJECTS) v2v_unit_tests.cmo: OCAMLPACKAGES += -package oUnit + +var_expander_tests_THEOBJECTS = $(var_expander_tests_BOBJECTS) +var_expander_tests.cmo: OCAMLPACKAGES += -package oUnit else v2v_unit_tests_THEOBJECTS = $(v2v_unit_tests_XOBJECTS) v2v_unit_tests.cmx: OCAMLPACKAGES += -package oUnit + +var_expander_tests_THEOBJECTS = $(var_expander_tests_XOBJECTS) +var_expander_tests.cmx: OCAMLPACKAGES += -package oUnit endif v2v_unit_tests_DEPENDENCIES = \ @@ -696,6 +713,17 @@ v2v_unit_tests_LINK = \ $(OCAMLLINKFLAGS) \ $(v2v_unit_tests_THEOBJECTS) -o $@ +var_expander_tests_DEPENDENCIES = \ + $(var_expander_tests_THEOBJECTS) \ + ../common/mlpcre/mlpcre.$(MLARCHIVE) \ + $(top_srcdir)/ocaml-link.sh +var_expander_tests_LINK = \ + $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \ + $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) \ + $(OCAMLPACKAGES) -package oUnit \ + $(OCAMLLINKFLAGS) \ + $(var_expander_tests_THEOBJECTS) -o $@ + # Dependencies. .depend: \ $(srcdir)/*.mli \ diff --git a/v2v/dummy.c b/v2v/dummy.c new file mode 100644 index 000000000..ebab6198c --- /dev/null +++ b/v2v/dummy.c @@ -0,0 +1,2 @@ +/* Dummy source, to be used for OCaml-based tools with no C sources. */ +enum { foo = 1 }; diff --git a/v2v/var_expander.ml b/v2v/var_expander.ml new file mode 100644 index 000000000..c47b8822b --- /dev/null +++ b/v2v/var_expander.ml @@ -0,0 +1,69 @@ +(* virt-v2v + * Copyright (C) 2019 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Std_utils + +exception Invalid_variable of string + +let var_re = PCRE.compile "%{([^}]+)}" + +let check_variable var = + String.iter ( + function + | '0'..'9' + | 'a'..'z' + | 'A'..'Z' + | '_' + | '-' -> () + | _ -> raise (Invalid_variable var) + ) var + +let scan_variables str = + let res = ref [] in + let offset = ref 0 in + while PCRE.matches ~offset:!offset var_re str; do + let var = PCRE.sub 1 in + check_variable var; + let _, end_ = PCRE.subi 0 in + List.push_back res var; + offset := end_ + done; + List.remove_duplicates !res + +let replace_fn str fn = + let res = ref str in + let offset = ref 0 in + while PCRE.matches ~offset:!offset var_re !res; do + let var = PCRE.sub 1 in + check_variable var; + let start_, end_ = PCRE.subi 0 in + match fn var with + | None -> + offset := end_ + | Some text -> + res := (String.sub !res 0 start_) ^ text ^ (String.sub !res end_ (String.length !res - end_)); + offset := start_ + String.length text + done; + !res + +let replace_list str lst = + let fn var = + try Some (List.assoc var lst) + with Not_found -> None + in + replace_fn str fn diff --git a/v2v/var_expander.mli b/v2v/var_expander.mli new file mode 100644 index 000000000..80aa33c2c --- /dev/null +++ b/v2v/var_expander.mli @@ -0,0 +1,82 @@ +(* virt-v2v + * Copyright (C) 2019 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +(** Simple variable expander. + + This module provides the support to expand variables in strings, + specified in the form of [%{name}]. + + For example: + +{v +let str = "variable-%{INDEX} in %{INDEX} replaced %{INDEX} times" +let index = ref 0 +let fn = function + | "INDEX" -> + incr index; + Some (string_of_int !index) + | _ -> None +in +let str = Var_expander.replace_fn str fn +(* now str is "variable-1 in 2 replaced 3 times" *) +v} + + The names of variables can contain only ASCII letters (uppercase, + and lowercase), digits, underscores, and dashes. + + The replacement is done in a single pass: this means that if a + variable is replaced with the text of a variable, that new text + is kept as is in the final output. In practice: + +{v +let str = "%{VAR}" +let str = Var_expander.replace_list str [("VAR", "%{VAR}")] +(* now str is "%{VAR}" *) +v} +*) + +exception Invalid_variable of string +(** Invalid variable name error. + + In case a variable contains characters not allowed, then this + exception with the actual unacceptable variable. *) + +val scan_variables : string -> string list +(** Scan the pattern string for all the variables available. + + This can raise {!Invalid_variable} in case there are invalid + variable names. *) + +val replace_fn : string -> (string -> string option) -> string +(** Replaces a string expanding all the variables. + + The replacement function specify how a variable is replaced; + if [None] is returned, then that variable is not replaced. + + This can raise {!Invalid_variable} in case there are invalid + variable names. *) + +val replace_list : string -> (string * string) list -> string +(** Replaces a string expanding all the variables. + + The replacement list specify how a variable is replaced; + if it is not specified in the list, then that variable is not + replaced. + + This can raise {!Invalid_variable} in case there are invalid + variable names. *) diff --git a/v2v/var_expander_tests.ml b/v2v/var_expander_tests.ml new file mode 100644 index 000000000..d9dec9e7c --- /dev/null +++ b/v2v/var_expander_tests.ml @@ -0,0 +1,103 @@ +(* virt-v2v + * Copyright (C) 2019 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open OUnit + +let assert_equal_string = assert_equal ~printer:(fun x -> x) +let assert_equal_stringlist = assert_equal ~printer:(fun x -> "(" ^ (String.escaped (String.concat "," x)) ^ ")") + +let replace_none_fn _ = None +let replace_empty_fn _ = Some "" + +let test_no_replacement () = + assert_equal_string "" (Var_expander.replace_fn "" replace_none_fn); + assert_equal_string "x" (Var_expander.replace_fn "x" replace_none_fn); + assert_equal_string "%{}" (Var_expander.replace_fn "%{}" replace_none_fn); + assert_equal_string "%{EMPTY}" (Var_expander.replace_fn "%{EMPTY}" replace_none_fn); + assert_equal_string "%{EMPTY} %{no}" (Var_expander.replace_fn "%{EMPTY} %{no}" replace_none_fn); + assert_equal_string "a %{EMPTY} b" (Var_expander.replace_fn "a %{EMPTY} b" replace_none_fn); + () + +let test_replacements () = + assert_equal_string "" (Var_expander.replace_fn "%{EMPTY}" replace_empty_fn); + assert_equal_string "x " (Var_expander.replace_fn "x %{EMPTY}" replace_empty_fn); + assert_equal_string "xy" (Var_expander.replace_fn "x%{EMPTY}y" replace_empty_fn); + assert_equal_string "x<->y" (Var_expander.replace_fn "x%{FOO}y" (function | "FOO" -> Some "<->" | _ -> None)); + assert_equal_string "a x b" (Var_expander.replace_fn "a %{FOO} b" (function | "FOO" -> Some "x" | _ -> None)); + assert_equal_string "%{FOO} x" (Var_expander.replace_fn "%{FOO} %{BAR}" (function | "BAR" -> Some "x" | _ -> None)); + assert_equal_string "%{FOO}" (Var_expander.replace_fn "%{BAR}" (function | "BAR" -> Some "%{FOO}" | _ -> None)); + assert_equal_string "%{FOO} x" (Var_expander.replace_fn "%{BAR} %{FOO}" (function | "BAR" -> Some "%{FOO}" | "FOO" -> Some "x" | _ -> None)); + begin + let str = "%{INDEX}, %{INDEX}, %{INDEX}" in + let index = ref 0 in + let fn = function + | "INDEX" -> + incr index; + Some (string_of_int !index) + | _ -> None + in + assert_equal_string "1, 2, 3" (Var_expander.replace_fn str fn) + end; + () + +let test_list () = + assert_equal_string "x %{NONE}" (Var_expander.replace_list "%{FOO} %{NONE}" [("FOO", "x")]); + () + +let test_scan_variables () = + let assert_invalid_variable var = + let str = "%{" ^ var ^ "}" in + assert_raises (Var_expander.Invalid_variable var) + (fun () -> Var_expander.scan_variables str) + in + assert_equal_stringlist [] (Var_expander.scan_variables ""); + assert_equal_stringlist [] (Var_expander.scan_variables "foo"); + assert_equal_stringlist ["FOO"] (Var_expander.scan_variables "%{FOO}"); + assert_equal_stringlist ["FOO"; "BAR"] (Var_expander.scan_variables "%{FOO} %{BAR}"); + assert_equal_stringlist ["FOO"; "BAR"] (Var_expander.scan_variables "%{FOO} %{BAR} %{FOO}"); + assert_invalid_variable "FOO/BAR"; + () + +let test_errors () = + let assert_invalid_variable var = + let str = "%{" ^ var ^ "}" in + assert_raises (Var_expander.Invalid_variable var) + (fun () -> Var_expander.replace_fn str replace_none_fn) + in + assert_invalid_variable "FOO/BAR"; + assert_invalid_variable "FOO:BAR"; + assert_invalid_variable "FOO(BAR"; + assert_invalid_variable "FOO)BAR"; + assert_invalid_variable "FOO@BAR"; + () + +(* Suites declaration. *) +let suite = + TestList ([ + "basic" >::: [ + "no_replacement" >:: test_no_replacement; + "replacements" >:: test_replacements; + "list" >:: test_list; + "scan_variables" >:: test_scan_variables; + "errors" >:: test_errors; + ]; + ]) + +let () = + ignore (run_test_tt_main suite); + Printf.fprintf stderr "\n" -- 2.20.1

Add a new output mode to virt-v2v: similar to -o local, the written metadata is a JSON file with the majority of the data that virt-v2v knowns about (or collects) during the conversion. This is meant to be used only when no existing output mode is usable, and a guest needs to be converted to run on KVM anyway. The user of this mode is supposed to use all the data in the JSON, as they contain important details on how even run the guest (e.g. w.r.t. firmware, drivers of disks/NICs, etc). --- v2v/Makefile.am | 4 + v2v/cmdline.ml | 29 +++ v2v/create_json.ml | 348 ++++++++++++++++++++++++++++++++++ v2v/create_json.mli | 29 +++ v2v/output_json.ml | 116 ++++++++++++ v2v/output_json.mli | 31 +++ v2v/virt-v2v-output-local.pod | 50 +++++ v2v/virt-v2v.pod | 15 +- 8 files changed, 620 insertions(+), 2 deletions(-) create mode 100644 v2v/create_json.ml create mode 100644 v2v/create_json.mli create mode 100644 v2v/output_json.ml create mode 100644 v2v/output_json.mli diff --git a/v2v/Makefile.am b/v2v/Makefile.am index f196be81d..53c137fc6 100644 --- a/v2v/Makefile.am +++ b/v2v/Makefile.am @@ -52,6 +52,7 @@ SOURCES_MLI = \ config.mli \ convert_linux.mli \ convert_windows.mli \ + create_json.mli \ create_libvirt_xml.mli \ create_ovf.mli \ DOM.mli \ @@ -75,6 +76,7 @@ SOURCES_MLI = \ networks.mli \ openstack_image_properties.mli \ output_glance.mli \ + output_json.mli \ output_libvirt.mli \ output_local.mli \ output_null.mli \ @@ -117,6 +119,7 @@ SOURCES_ML = \ parse_ovf_from_ova.ml \ parse_ova.ml \ create_ovf.ml \ + create_json.ml \ linux.ml \ windows.ml \ windows_virtio.ml \ @@ -141,6 +144,7 @@ SOURCES_ML = \ convert_windows.ml \ output_null.ml \ output_glance.ml \ + output_json.ml \ output_libvirt.ml \ output_local.ml \ output_qemu.ml \ diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml index 46f6910d0..4d390f249 100644 --- a/v2v/cmdline.ml +++ b/v2v/cmdline.ml @@ -138,6 +138,7 @@ let parse_cmdline () = | "glance" -> output_mode := `Glance | "libvirt" -> output_mode := `Libvirt | "disk" | "local" -> output_mode := `Local + | "json" -> output_mode := `JSON | "null" -> output_mode := `Null | "openstack" | "osp" | "rhosp" -> output_mode := `Openstack | "ovirt" | "rhv" | "rhev" -> output_mode := `RHV @@ -413,6 +414,17 @@ read the man page virt-v2v(1). | `RHV -> no_options (); `RHV | `QEmu -> no_options (); `QEmu + | `JSON -> + if is_query then ( + Output_json.print_output_options (); + exit 0 + ) + else ( + let json_options = + Output_json.parse_output_options output_options in + `JSON json_options + ) + | `Openstack -> if is_query then ( Output_openstack.print_output_options (); @@ -546,6 +558,23 @@ read the man page virt-v2v(1). Output_libvirt.output_libvirt output_conn output_storage, output_format, output_alloc + | `JSON json_options -> + if output_password <> None then + error_option_cannot_be_used_in_output_mode "json" "-op"; + if output_conn <> None then + error_option_cannot_be_used_in_output_mode "json" "-oc"; + let os = + match output_storage with + | None -> + error (f_"-o json: output directory was not specified, use '-os /dir'") + | Some d when not (is_directory d) -> + error (f_"-os %s: output directory does not exist or is not a directory") d + | Some d -> d in + if qemu_boot then + error_option_cannot_be_used_in_output_mode "json" "--qemu-boot"; + Output_json.output_json os json_options, + output_format, output_alloc + | `Local -> if output_password <> None then error_option_cannot_be_used_in_output_mode "local" "-op"; diff --git a/v2v/create_json.ml b/v2v/create_json.ml new file mode 100644 index 000000000..fdf7b12f5 --- /dev/null +++ b/v2v/create_json.ml @@ -0,0 +1,348 @@ +(* virt-v2v + * Copyright (C) 2019 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Std_utils +open C_utils +open Tools_utils + +open Types +open Utils + +module G = Guestfs + +let json_list_of_string_list = + List.map (fun x -> JSON.String x) + +let json_list_of_string_string_list = + List.map (fun (x, y) -> x, JSON.String y) + +let push_optional_string lst name = function + | None -> () + | Some v -> List.push_back lst (name, JSON.String v) + +let push_optional_int lst name = function + | None -> () + | Some v -> List.push_back lst (name, JSON.Int (Int64.of_int v)) + +let json_unknown_string = function + | "unknown" -> JSON.Null + | v -> JSON.String v + +let find_target_disk targets { s_disk_id = id } = + try List.find (fun t -> t.target_overlay.ov_source.s_disk_id = id) targets + with Not_found -> assert false + +let create_json_metadata source targets target_buses + guestcaps inspect target_firmware = + let doc = ref [ + "version", JSON.Int 1L; + "name", JSON.String source.s_name; + "memory", JSON.Int source.s_memory; + "vcpu", JSON.Int (Int64.of_int source.s_vcpu); + ] in + + (match source.s_genid with + | None -> () + | Some genid -> List.push_back doc ("genid", JSON.String genid) + ); + + if source.s_cpu_vendor <> None || source.s_cpu_model <> None || + source.s_cpu_topology <> None then ( + let cpu = ref [] in + + push_optional_string cpu "vendor" source.s_cpu_vendor; + push_optional_string cpu "model" source.s_cpu_model; + (match source.s_cpu_topology with + | None -> () + | Some { s_cpu_sockets; s_cpu_cores; s_cpu_threads } -> + let attrs = [ + "sockets", JSON.Int (Int64.of_int s_cpu_sockets); + "cores", JSON.Int (Int64.of_int s_cpu_cores); + "threads", JSON.Int (Int64.of_int s_cpu_threads); + ] in + List.push_back cpu ("topology", JSON.Dict attrs) + ); + + List.push_back doc ("cpu", JSON.Dict !cpu); + ); + + let firmware = + let firmware_type = + match target_firmware with + | TargetBIOS -> "bios" + | TargetUEFI -> "uefi" in + + let fw = ref [ + "type", JSON.String firmware_type; + ] in + + (match target_firmware with + | TargetBIOS -> () + | TargetUEFI -> + let uefi_firmware = find_uefi_firmware guestcaps.gcaps_arch in + let flags = + List.map ( + function + | Uefi.UEFI_FLAG_SECURE_BOOT_REQUIRED -> "secure_boot_required" + ) uefi_firmware.Uefi.flags in + + let uefi = ref [ + "code", JSON.String uefi_firmware.Uefi.code; + "vars", JSON.String uefi_firmware.Uefi.vars; + "flags", JSON.List (json_list_of_string_list flags); + ] in + + push_optional_string uefi "code-debug" uefi_firmware.Uefi.code_debug; + + List.push_back fw ("uefi", JSON.Dict !uefi) + ); + + !fw in + List.push_back doc ("firmware", JSON.Dict firmware); + + List.push_back doc ("features", + JSON.List (json_list_of_string_list source.s_features)); + + let machine = + match guestcaps.gcaps_machine with + | I440FX -> "pc" + | Q35 -> "q35" + | Virt -> "virt" in + List.push_back doc ("machine", JSON.String machine); + + let disks, removables = + let disks = ref [] + and removables = ref [] in + + let iter_bus bus_name drive_prefix i = function + | BusSlotEmpty -> () + | BusSlotDisk d -> + (* Find the corresponding target disk. *) + let t = find_target_disk targets d in + + let target_file = + match t.target_file with + | TargetFile s -> s + | TargetURI _ -> assert false in + + let disk = [ + "dev", JSON.String (drive_prefix ^ drive_name i); + "bus", JSON.String bus_name; + "format", JSON.String t.target_format; + "file", JSON.String (absolute_path target_file); + ] in + + List.push_back disks (JSON.Dict disk) + + | BusSlotRemovable { s_removable_type = CDROM } -> + let cdrom = [ + "type", JSON.String "cdrom"; + "dev", JSON.String (drive_prefix ^ drive_name i); + "bus", JSON.String bus_name; + ] in + + List.push_back removables (JSON.Dict cdrom) + + | BusSlotRemovable { s_removable_type = Floppy } -> + let floppy = [ + "type", JSON.String "floppy"; + "dev", JSON.String (drive_prefix ^ drive_name i); + ] in + + List.push_back removables (JSON.Dict floppy) + in + + Array.iteri (iter_bus "virtio" "vd") target_buses.target_virtio_blk_bus; + Array.iteri (iter_bus "ide" "hd") target_buses.target_ide_bus; + Array.iteri (iter_bus "scsi" "sd") target_buses.target_scsi_bus; + Array.iteri (iter_bus "floppy" "fd") target_buses.target_floppy_bus; + + !disks, !removables in + List.push_back doc ("disks", JSON.List disks); + List.push_back doc ("removables", JSON.List removables); + + let nics = + List.map ( + fun { s_mac = mac; s_vnet_type = vnet_type; s_nic_model = nic_model; + s_vnet = vnet; } -> + let vnet_type_str = + match vnet_type with + | Bridge -> "bridge" + | Network -> "network" in + + let nic = ref [ + "vnet", JSON.String vnet; + "vnet-type", JSON.String vnet_type_str; + ] in + + let nic_model_str = Option.map string_of_nic_model nic_model in + push_optional_string nic "model" nic_model_str; + + push_optional_string nic "mac" mac; + + JSON.Dict !nic + ) source.s_nics in + List.push_back doc ("nics", JSON.List nics); + + let guestcaps_dict = + let block_bus = + match guestcaps.gcaps_block_bus with + | Virtio_blk -> "virtio-blk" + | Virtio_SCSI -> "virtio-scsi" + | IDE -> "ide" in + let net_bus = + match guestcaps.gcaps_net_bus with + | Virtio_net -> "virtio-net" + | E1000 -> "e1000" + | RTL8139 -> "rtl8139" in + let video = + match guestcaps.gcaps_video with + | QXL -> "qxl" + | Cirrus -> "cirrus" in + let machine = + match guestcaps.gcaps_machine with + | I440FX -> "i440fx" + | Q35 -> "q35" + | Virt -> "virt" in + + [ + "block-bus", JSON.String block_bus; + "net-bus", JSON.String net_bus; + "video", JSON.String video; + "machine", JSON.String machine; + "arch", JSON.String guestcaps.gcaps_arch; + "virtio-rng", JSON.Bool guestcaps.gcaps_virtio_rng; + "virtio-balloon", JSON.Bool guestcaps.gcaps_virtio_balloon; + "isa-pvpanic", JSON.Bool guestcaps.gcaps_isa_pvpanic; + "acpi", JSON.Bool guestcaps.gcaps_acpi; + ] in + List.push_back doc ("guestcaps", JSON.Dict guestcaps_dict); + + (match source.s_sound with + | None -> () + | Some { s_sound_model = model } -> + let sound = [ + "model", JSON.String (string_of_source_sound_model model); + ] in + List.push_back doc ("sound", JSON.Dict sound) + ); + + (match source.s_display with + | None -> () + | Some d -> + let display_type = + match d.s_display_type with + | Window -> "window" + | VNC -> "vnc" + | Spice -> "spice" in + + let display = ref [ + "type", JSON.String display_type; + ] in + + push_optional_string display "keymap" d.s_keymap; + push_optional_string display "password" d.s_password; + + let listen = + match d.s_listen with + | LNoListen -> None + | LAddress address -> + Some [ + "type", JSON.String "address"; + "address", JSON.String address; + ] + | LNetwork network -> + Some [ + "type", JSON.String "network"; + "network", JSON.String network; + ] + | LSocket None -> + Some [ + "type", JSON.String "socket"; + "socket", JSON.Null; + ] + | LSocket (Some socket) -> + Some [ + "type", JSON.String "socket"; + "socket", JSON.String socket; + ] + | LNone -> + Some [ + "type", JSON.String "none"; + ] in + (match listen with + | None -> () + | Some l -> List.push_back display ("listen", JSON.Dict l) + ); + + push_optional_int display "port" d.s_port; + + List.push_back doc ("display", JSON.Dict !display) + ); + + let inspect_dict = + let apps = + List.map ( + fun { G.app2_name = name; app2_display_name = display_name; + app2_epoch = epoch; app2_version = version; + app2_release = release; app2_arch = arch; } -> + JSON.Dict [ + "name", JSON.String name; + "display-name", JSON.String display_name; + "epoch", JSON.Int (Int64.of_int32 epoch); + "version", JSON.String version; + "release", JSON.String release; + "arch", JSON.String arch; + ] + ) inspect.i_apps in + + let firmware_dict = + match inspect.i_firmware with + | I_BIOS -> + [ + "type", JSON.String "bios"; + ] + | I_UEFI devices -> + [ + "type", JSON.String "uefi"; + "devices", JSON.List (json_list_of_string_list devices); + ] in + + [ + "root", JSON.String inspect.i_root; + "type", JSON.String inspect.i_type; + "distro", json_unknown_string inspect.i_distro; + "osinfo", json_unknown_string inspect.i_osinfo; + "arch", JSON.String inspect.i_arch; + "major-version", JSON.Int (Int64.of_int inspect.i_major_version); + "minor-version", JSON.Int (Int64.of_int inspect.i_minor_version); + "package-format", json_unknown_string inspect.i_package_format; + "package-management", json_unknown_string inspect.i_package_management; + "product-name", json_unknown_string inspect.i_product_name; + "product-variant", json_unknown_string inspect.i_product_variant; + "mountpoints", JSON.Dict (json_list_of_string_string_list inspect.i_mountpoints); + "applications", JSON.List apps; + "windows-systemroot", JSON.String inspect.i_windows_systemroot; + "windows-software-hive", JSON.String inspect.i_windows_software_hive; + "windows-system-hive", JSON.String inspect.i_windows_system_hive; + "windows-current-control-set", JSON.String inspect.i_windows_current_control_set; + "firmware", JSON.Dict firmware_dict; + ] in + List.push_back doc ("inspect", JSON.Dict inspect_dict); + + !doc diff --git a/v2v/create_json.mli b/v2v/create_json.mli new file mode 100644 index 000000000..6dbb6e48b --- /dev/null +++ b/v2v/create_json.mli @@ -0,0 +1,29 @@ +(* virt-v2v + * Copyright (C) 2019 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +(** Create JSON metadata for [-o json]. *) + +val create_json_metadata : Types.source -> Types.target list -> + Types.target_buses -> + Types.guestcaps -> + Types.inspect -> + Types.target_firmware -> + JSON.doc +(** [create_json_metadata source targets target_buses guestcaps + inspect target_firmware] creates the JSON with the majority + of the data that virt-v2v used for the conversion. *) diff --git a/v2v/output_json.ml b/v2v/output_json.ml new file mode 100644 index 000000000..ca0bda978 --- /dev/null +++ b/v2v/output_json.ml @@ -0,0 +1,116 @@ +(* virt-v2v + * Copyright (C) 2019 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Printf + +open Std_utils +open Tools_utils +open Common_gettext.Gettext + +open Types +open Utils + +type json_options = { + json_disks_pattern : string; +} + +let print_output_options () = + printf (f_"Output options (-oo) which can be used with -o json: + + -oo json-disks-pattern=PATTERN Pattern for the disks. +") + +let known_pattern_variables = ["DiskNo"; "DiskDeviceName"; "GuestName"] + +let parse_output_options options = + let json_disks_pattern = ref None in + + List.iter ( + function + | "json-disks-pattern", v -> + if !json_disks_pattern <> None then + error (f_"-o json: -oo json-disks-pattern set more than once"); + let vars = + try Var_expander.scan_variables v + with Var_expander.Invalid_variable var -> + error (f_"-o json: -oo json-disks-pattern: invalid variable %%{%s}") + var in + List.iter ( + fun var -> + if not (List.mem var known_pattern_variables) then + error (f_"-o json: -oo json-disks-pattern: unhandled variable %%{%s}") + var + ) vars; + json_disks_pattern := Some v + | k, _ -> + error (f_"-o json: unknown output option ‘-oo %s’") k + ) options; + + let json_disks_pattern = + Option.default "%{GuestName}-%{DiskDeviceName}" !json_disks_pattern in + + { json_disks_pattern } + +class output_json dir json_options = object + inherit output + + method as_options = sprintf "-o json -os %s" dir + + method prepare_targets source overlays _ _ _ _ = + List.mapi ( + fun i (_, ov) -> + let outname = + let vars_fn = function + | "DiskNo" -> Some (string_of_int (i+1)) + | "DiskDeviceName" -> Some ov.ov_sd + | "GuestName" -> Some source.s_name + | _ -> assert false + in + Var_expander.replace_fn json_options.json_disks_pattern vars_fn in + let destname = dir // outname in + mkdir_p (Filename.dirname destname) 0o755; + TargetFile destname + ) overlays + + method supported_firmware = [ TargetBIOS; TargetUEFI ] + + method create_metadata source targets + target_buses guestcaps inspect target_firmware = + let doc = + Create_json.create_json_metadata source targets target_buses + guestcaps inspect target_firmware in + let doc_string = JSON.string_of_doc ~fmt:JSON.Indented doc in + + if verbose () then ( + eprintf "resulting JSON:\n"; + output_string stderr doc_string; + eprintf "\n\n%!"; + ); + + let name = source.s_name in + let file = dir // name ^ ".json" in + + with_open_out file ( + fun chan -> + output_string chan doc_string; + output_char chan '\n' + ) +end + +let output_json = new output_json +let () = Modules_list.register_output_module "json" diff --git a/v2v/output_json.mli b/v2v/output_json.mli new file mode 100644 index 000000000..52f58f2d1 --- /dev/null +++ b/v2v/output_json.mli @@ -0,0 +1,31 @@ +(* virt-v2v + * Copyright (C) 2019 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +(** [-o json] target. *) + +type json_options +(** Miscellaneous extra command line parameters used by json. *) + +val print_output_options : unit -> unit +val parse_output_options : (string * string) list -> json_options +(** Print and parse json -oo options. *) + +val output_json : string -> json_options -> Types.output +(** [output_json directory json_options] creates and returns a new + {!Types.output} object specialized for writing output to local + files with JSON metadata. *) diff --git a/v2v/virt-v2v-output-local.pod b/v2v/virt-v2v-output-local.pod index 7427b1ed7..ec737e1ba 100644 --- a/v2v/virt-v2v-output-local.pod +++ b/v2v/virt-v2v-output-local.pod @@ -11,6 +11,9 @@ or libvirt virt-v2v [-i* options] -o qemu -os DIRECTORY [--qemu-boot] + virt-v2v [-i* options] -o json -os DIRECTORY + [-oo json-disks-pattern=PATTERN] + virt-v2v [-i* options] -o null =head1 DESCRIPTION @@ -54,6 +57,13 @@ above, a shell script is created which contains the raw qemu command you would need to boot the guest. However the shell script is not run, I<unless> you also add the I<--qemu-boot> option. +=item B<-o json -os> C<DIRECTORY> + +This converts the guest to files in C<DIRECTORY>. The metadata +produced is a JSON file containing the majority of the data virt-v2v +gathers during the conversion. +See L</OUTPUT TO JSON> below. + =item B<-o null> The guest is converted, but the final result is thrown away and no @@ -140,6 +150,46 @@ Define the final guest in libvirt: =back +=head1 OUTPUT TO JSON + +The I<-o json> option produces the following files by default: + + NAME.json JSON metadata. + NAME-sda, NAME-sdb, etc. Guest disk(s). + +where C<NAME> is the guest name. + +It is possible to change the pattern of the disks using the +I<-oo json-disks-pattern=...> option: it allows parameters in form of +C<%{...}> variables, for example: + + -oo json-disks-pattern=disk%{DiskNo}.img + +Recognized variables are: + +=over 4 + +=item C<%{DiskNo}> + +The index of the disk, starting from 1. + +=item C<%{DiskDeviceName}> + +The destination device of the disk, e.g. C<sda>, C<sdb>, etc. + +=item C<%{GuestName}> + +The name of the guest. + +=back + +Using a pattern it is possible use subdirectories for the disks, +even with names depending on variables; for example: + + -oo json-disks-pattern=%{GuestName}-%{DiskNo}/disk.img + +The default pattern is C<%{GuestName}-%{DiskDeviceName}>. + =head1 SEE ALSO L<virt-v2v(1)>. diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod index cf9464834..9a555c3be 100644 --- a/v2v/virt-v2v.pod +++ b/v2v/virt-v2v.pod @@ -425,6 +425,17 @@ instead. Set the output method to OpenStack Glance. In this mode the converted guest is uploaded to Glance. See L<virt-v2v-output-openstack(1)>. +=item B<-o> B<json> + +Set the output method to I<json>. + +In this mode, the converted guest is written to a local directory +specified by I<-os /dir> (the directory must exist), with a JSON file +containing the majority of the metadata that virt-v2v gathered during +the conversion. + +See L<virt-v2v-output-local(1)>. + =item B<-o> B<libvirt> Set the output method to I<libvirt>. This is the default. @@ -696,8 +707,8 @@ The location of the storage for the converted guest. For I<-o libvirt>, this is a libvirt directory pool (see S<C<virsh pool-list>>) or pool UUID. -For I<-o local> and I<-o qemu>, this is a directory name. The -directory must exist. +For I<-o json>, I<-o local> and I<-o qemu>, this is a directory name. +The directory must exist. For I<-o rhv-upload>, this is the name of the destination Storage Domain. -- 2.20.1