Change Index.arch to the type (Arch of string | GuessedArch of string). In a future commit, the index parser will allow arch not to be set for some cases. Thus arch value will be guessed by inspecting the image. However we need to distinguish between a set value and a guessed one. Using this new type will help it: match arch with | Arch s -> (* This is a set value *) | GuessedArch s -> (* This is a guessed value *) --- builder/builder.ml | 6 +++--- builder/cache.ml | 6 ++++-- builder/cache.mli | 6 +++--- builder/downloader.mli | 2 +- builder/index.ml | 9 +++++++-- builder/index.mli | 8 +++++++- builder/index_parser.ml | 4 ++-- builder/list_entries.ml | 6 +++--- builder/simplestreams_parser.ml | 2 +- 9 files changed, 31 insertions(+), 18 deletions(-) diff --git a/builder/builder.ml b/builder/builder.ml index 3f7c79bc9..41c0a4ccc 100644 --- a/builder/builder.ml +++ b/builder/builder.ml @@ -94,7 +94,7 @@ let selected_cli_item cmdline index = let item = try List.find ( fun (name, { Index.arch = a }) -> - name = arg && cmdline.arch = normalize_arch a + name = arg && cmdline.arch = normalize_arch (Index.string_of_arch a) ) index with Not_found -> error (f_"cannot find os-version ‘%s’ with architecture ‘%s’.\nUse --list to list available guest types.") @@ -252,7 +252,7 @@ let main () = List.iter ( fun (name, { Index.revision; file_uri; proxy }) -> - let template = name, cmdline.arch, revision in + let template = name, Index.Arch cmdline.arch, revision in message (f_"Downloading: %s") file_uri; let progress_bar = not (quiet ()) in ignore (Downloader.download downloader ~template ~progress_bar @@ -300,7 +300,7 @@ let main () = let template = let template, delete_on_exit = let { Index.revision; file_uri; proxy } = entry in - let template = arg, cmdline.arch, revision in + let template = arg, Index.Arch cmdline.arch, revision in message (f_"Downloading: %s") file_uri; let progress_bar = not (quiet ()) in Downloader.download downloader ~template ~progress_bar ~proxy diff --git a/builder/cache.ml b/builder/cache.ml index dbd222fda..d2693b943 100644 --- a/builder/cache.ml +++ b/builder/cache.ml @@ -41,7 +41,9 @@ let create ~directory = } let cache_of_name t name arch revision = - t.directory // sprintf "%s.%s.%s" name arch (string_of_revision revision) + t.directory // sprintf "%s.%s.%s" name + (Index.string_of_arch arch) + (string_of_revision revision) let is_cached t name arch revision = let filename = cache_of_name t name arch revision in @@ -54,6 +56,6 @@ let print_item_status t ~header l = List.iter ( fun (name, arch, revision) -> let cached = is_cached t name arch revision in - printf "%-24s %-10s %s\n" name arch + printf "%-24s %-10s %s\n" name (Index.string_of_arch arch) (if cached then s_"cached" else (*s_*)"no") ) l diff --git a/builder/cache.mli b/builder/cache.mli index f27fc235b..f88cbdf2f 100644 --- a/builder/cache.mli +++ b/builder/cache.mli @@ -27,16 +27,16 @@ type t val create : directory:string -> t (** Create the abstract type. *) -val cache_of_name : t -> string -> string -> Utils.revision -> string +val cache_of_name : t -> string -> Index.arch -> Utils.revision -> string (** [cache_of_name t name arch revision] return the filename of the cached file. (Note: It doesn't check if the filename exists, this is just a simple string transformation). *) -val is_cached : t -> string -> string -> Utils.revision -> bool +val is_cached : t -> string -> Index.arch -> Utils.revision -> bool (** [is_cached t name arch revision] return whether the file with specified name, architecture and revision is cached. *) -val print_item_status : t -> header:bool -> (string * string * Utils.revision) list -> unit +val print_item_status : t -> header:bool -> (string * Index.arch * Utils.revision) list -> unit (** [print_item_status t header items] print the status in the cache of the specified items (which are tuples of name, architecture, and revision). diff --git a/builder/downloader.mli b/builder/downloader.mli index 7f39f7e36..a199440cb 100644 --- a/builder/downloader.mli +++ b/builder/downloader.mli @@ -27,7 +27,7 @@ type t val create : curl:string -> tmpdir:string -> cache:Cache.t option -> t (** Create the abstract type. *) -val download : t -> ?template:(string*string*Utils.revision) -> ?progress_bar:bool -> ?proxy:Curl.proxy -> uri -> (filename * bool) +val download : t -> ?template:string * Index.arch * Utils.revision -> ?progress_bar:bool -> ?proxy:Curl.proxy -> uri -> filename * bool (** Download the URI, returning the downloaded filename and a temporary file flag. The temporary file flag is [true] iff the downloaded file is temporary and should be deleted by the diff --git a/builder/index.ml b/builder/index.ml index 84f66c265..65d97ea7e 100644 --- a/builder/index.ml +++ b/builder/index.ml @@ -30,7 +30,7 @@ and entry = { printable_name : string option; (* the name= field *) osinfo : string option; file_uri : string; - arch : string; + arch : arch; signature_uri : string option; (* deprecated, will be removed in 1.26 *) checksums : Checksums.csum_t list option; revision : Utils.revision; @@ -46,6 +46,11 @@ and entry = { sigchecker : Sigchecker.t; proxy : Curl.proxy; } +and arch = + | Arch of string + | GuessedArch of string + +let string_of_arch = function Arch a | GuessedArch a -> a let print_entry chan (name, { printable_name; file_uri; arch; osinfo; signature_uri; checksums; revision; format; @@ -56,7 +61,7 @@ let print_entry chan (name, { printable_name; file_uri; arch; osinfo; Option.may (fp "name=%s\n") printable_name; Option.may (fp "osinfo=%s\n") osinfo; fp "file=%s\n" file_uri; - fp "arch=%s\n" arch; + fp "arch=%s\n" (string_of_arch arch); Option.may (fp "sig=%s\n") signature_uri; Option.may ( List.iter ( diff --git a/builder/index.mli b/builder/index.mli index 6202d636e..df921981f 100644 --- a/builder/index.mli +++ b/builder/index.mli @@ -21,7 +21,7 @@ and entry = { printable_name : string option; (* the name= field *) osinfo : string option; file_uri : string; - arch : string; + arch : arch; signature_uri : string option; (* deprecated, will be removed in 1.26 *) checksums : Checksums.csum_t list option; revision : Utils.revision; @@ -37,6 +37,12 @@ and entry = { sigchecker : Sigchecker.t; proxy : Curl.proxy; } +and arch = + | Arch of string + | GuessedArch of string + +val string_of_arch : arch -> string +(** [string_of_arch a]Get the string value of [a]. *) val print_entry : out_channel -> (string * entry) -> unit (** Debugging helper function dumping an index entry to a stream. diff --git a/builder/index_parser.ml b/builder/index_parser.ml index f76aed65d..d79c807e4 100644 --- a/builder/index_parser.ml +++ b/builder/index_parser.ml @@ -97,7 +97,7 @@ let get_index ~downloader ~sigchecker { Sources.uri; proxy } = eprintf (f_"%s: no ‘file’ (URI) entry for ‘%s’\n") prog n; corrupt_file () in let arch = - try List.assoc ("arch", None) fields + try Index.Arch (List.assoc ("arch", None) fields) with Not_found -> eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n; corrupt_file () in @@ -236,7 +236,7 @@ let write_entry chan (name, { Index.printable_name; file_uri; arch; osinfo; Option.may (fp "name=%s\n") printable_name; Option.may (fp "osinfo=%s\n") osinfo; fp "file=%s\n" file_uri; - fp "arch=%s\n" arch; + fp "arch=%s\n" (Index.string_of_arch arch); Option.may (fp "sig=%s\n") signature_uri; (match checksums with | None -> () diff --git a/builder/list_entries.ml b/builder/list_entries.ml index af1d2419b..c64d554a2 100644 --- a/builder/list_entries.ml +++ b/builder/list_entries.ml @@ -46,7 +46,7 @@ and list_entries_short index = fun (name, { Index.printable_name; arch; hidden }) -> if not hidden then ( printf "%-24s" name; - printf " %-10s" arch; + printf " %-10s" (Index.string_of_arch arch); Option.may (printf " %s") printable_name; printf "\n" ) @@ -74,7 +74,7 @@ and list_entries_long ~sources index = if not hidden then ( printf "%-24s %s\n" "os-version:" name; Option.may (printf "%-24s %s\n" (s_"Full name:")) printable_name; - printf "%-24s %s\n" (s_"Architecture:") arch; + printf "%-24s %s\n" (s_"Architecture:") (Index.string_of_arch arch); printf "%-24s %s\n" (s_"Minimum/default size:") (human_size size); Option.may (fun size -> printf "%-24s %s\n" (s_"Download size:") (human_size size) @@ -116,7 +116,7 @@ and list_entries_json ~sources index = match printable_name with | None -> item | Some str -> ("full-name", JSON.String str) :: item in - let item = ("arch", JSON.String arch) :: item in + let item = ("arch", JSON.String (Index.string_of_arch arch)) :: item in let item = ("size", JSON.Int64 size) :: item in let item = match compressed_size with diff --git a/builder/simplestreams_parser.ml b/builder/simplestreams_parser.ml index 75592e377..996c334f5 100644 --- a/builder/simplestreams_parser.ml +++ b/builder/simplestreams_parser.ml @@ -83,7 +83,7 @@ let get_index ~downloader ~sigchecker { Sources.uri; proxy } = let products = Array.to_list products_node in filter_map ( fun (prod, prod_desc) -> - let arch = object_get_string "arch" prod_desc in + let arch = Index.Arch (object_get_string "arch" prod_desc) in let prods = Array.to_list (object_get_object "versions" prod_desc) in let prods = filter_map ( fun (rel, rel_desc) -> -- 2.15.0

virt-builder-repository allows users to easily create or update a virt-builder source repository out of disk images. The tool can be run in either interactive or automated mode. --- .gitignore | 4 + builder/Makefile.am | 87 ++++- builder/repository_main.ml | 595 ++++++++++++++++++++++++++++++++ builder/test-docs.sh | 2 + builder/test-virt-builder-repository.sh | 100 ++++++ builder/utils.ml | 4 + builder/utils.mli | 3 + builder/virt-builder-repository.pod | 213 ++++++++++++ builder/virt-builder.pod | 4 + fish/guestfish.pod | 1 + installcheck.sh.in | 1 + lib/guestfs.pod | 1 + 12 files changed, 1013 insertions(+), 2 deletions(-) create mode 100644 builder/repository_main.ml create mode 100755 builder/test-virt-builder-repository.sh create mode 100644 builder/virt-builder-repository.pod diff --git a/.gitignore b/.gitignore index f3569aa73..e54ea2d45 100644 --- a/.gitignore +++ b/.gitignore @@ -98,14 +98,18 @@ Makefile.in /builder/oUnit-* /builder/*.out /builder/*.qcow2 +/builder/repository-testdata /builder/stamp-virt-builder.pod +/builder/stamp-virt-builder-repository.pod /builder/stamp-virt-index-validate.pod /builder/test-config/virt-builder/repos.d/test-index.conf /builder/test-console-*.sh /builder/test-simplestreams/virt-builder/repos.d/cirros.conf /builder/test-website/virt-builder/repos.d/libguestfs.conf /builder/virt-builder +/builder/virt-builder-repository /builder/virt-builder.1 +/builder/virt-builder-repository.1 /builder/virt-index-validate /builder/virt-index-validate.1 /builder/*.xz diff --git a/builder/Makefile.am b/builder/Makefile.am index be1afdff5..b4a7c731f 100644 --- a/builder/Makefile.am +++ b/builder/Makefile.am @@ -21,6 +21,8 @@ AM_YFLAGS = -d EXTRA_DIST = \ $(SOURCES_MLI) $(SOURCES_ML) $(SOURCES_C) \ + $(REPOSITORY_SOURCES_ML) \ + $(REPOSITORY_SOURCES_MLI) \ index_parser_tests.ml \ libguestfs.gpg \ opensuse.gpg \ @@ -33,6 +35,7 @@ EXTRA_DIST = \ test-virt-builder-list.sh \ test-virt-builder-list-simplestreams.sh \ test-virt-builder-planner.sh \ + test-virt-builder-repository.sh \ test-virt-index-validate.sh \ test-virt-index-validate-bad-1 \ test-virt-index-validate-good-1 \ @@ -40,6 +43,7 @@ EXTRA_DIST = \ test-virt-index-validate-good-3 \ test-virt-index-validate-good-4 \ virt-builder.pod \ + virt-builder-repository.pod \ virt-index-validate.pod \ yajl_tests.ml @@ -94,13 +98,45 @@ SOURCES_C = \ setlocale-c.c \ yajl-c.c +REPOSITORY_SOURCES_ML = \ + yajl.ml \ + utils.ml \ + index.ml \ + cache.ml \ + downloader.ml \ + sigchecker.ml \ + ini_reader.ml \ + index_parser.ml \ + paths.ml \ + sources.ml \ + osinfo_config.ml \ + osinfo.ml \ + repository_main.ml + +REPOSITORY_SOURCES_MLI = \ + cache.mli \ + downloader.mli \ + index.mli \ + index_parser.mli \ + ini_reader.mli \ + sigchecker.mli \ + sources.mli \ + yajl.mli + +REPOSITORY_SOURCES_C = \ + index-scan.c \ + index-struct.c \ + index-parse.c \ + index-parser-c.c \ + yajl-c.c + man_MANS = noinst_DATA = bin_PROGRAMS = if HAVE_OCAML -bin_PROGRAMS += virt-builder +bin_PROGRAMS += virt-builder virt-builder-repository virt_builder_SOURCES = $(SOURCES_C) virt_builder_CPPFLAGS = \ @@ -123,12 +159,31 @@ virt_builder_CFLAGS = \ BOBJECTS = $(SOURCES_ML:.ml=.cmo) XOBJECTS = $(BOBJECTS:.cmo=.cmx) +virt_builder_repository_SOURCES = $(REPOSITORY_SOURCES_C) +virt_builder_repository_CPPFLAGS = \ + -I. \ + -I$(top_builddir) \ + -I$(top_srcdir)/gnulib/lib -I$(top_builddir)/gnulib/lib \ + -I$(shell $(OCAMLC) -where) \ + -I$(top_srcdir)/gnulib/lib \ + -I$(top_srcdir)/lib +virt_builder_repository_CFLAGS = \ + -pthread \ + $(WARN_CFLAGS) $(WERROR_CFLAGS) \ + -Wno-unused-macros \ + $(LIBTINFO_CFLAGS) \ + $(LIBXML2_CFLAGS) \ + $(YAJL_CFLAGS) +REPOSITORY_BOBJECTS = $(REPOSITORY_SOURCES_ML:.ml=.cmo) +REPOSITORY_XOBJECTS = $(REPOSITORY_BOBJECTS:.cmo=.cmx) + # -I $(top_builddir)/lib/.libs is a hack which forces corresponding -L # option to be passed to gcc, so we don't try linking against an # installed copy of libguestfs. OCAMLPACKAGES = \ -package str,unix \ -I $(top_builddir)/common/utils/.libs \ + -I $(top_builddir)/common/mlxml \ -I $(top_builddir)/lib/.libs \ -I $(top_builddir)/gnulib/lib/.libs \ -I $(top_builddir)/ocaml \ @@ -161,13 +216,16 @@ OCAMLFLAGS = $(OCAML_FLAGS) $(OCAML_WARN_ERROR) if !HAVE_OCAMLOPT OBJECTS = $(BOBJECTS) +REPOSITORY_OBJECTS = $(REPOSITORY_BOBJECTS) else OBJECTS = $(XOBJECTS) +REPOSITORY_OBJECTS = $(REPOSITORY_XOBJECTS) endif OCAMLLINKFLAGS = \ mlgettext.$(MLARCHIVE) \ mlpcre.$(MLARCHIVE) \ + mlxml.$(MLARCHIVE) \ mlstdutils.$(MLARCHIVE) \ mlguestfs.$(MLARCHIVE) \ mlcutils.$(MLARCHIVE) \ @@ -189,6 +247,16 @@ virt_builder_LINK = \ $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) $(OCAMLLINKFLAGS) \ $(OBJECTS) -o $@ +virt_builder_repository_DEPENDENCIES = \ + $(REPOSITORY_OBJECTS) \ + ../common/mltools/mltools.$(MLARCHIVE) \ + ../common/mlxml/mlxml.$(MLARCHIVE) \ + $(top_srcdir)/ocaml-link.sh +virt_builder_repository_LINK = \ + $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \ + $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) $(OCAMLLINKFLAGS) \ + $(REPOSITORY_OBJECTS) -o $@ + # Manual pages and HTML files for the website. man_MANS += virt-builder.1 @@ -207,6 +275,20 @@ stamp-virt-builder.pod: virt-builder.pod $(top_srcdir)/customize/customize-synop $< touch $@ +man_MANS += virt-builder-repository.1 +noinst_DATA += $(top_builddir)/website/virt-builder-repository.1.html + +virt-builder-repository.1 $(top_builddir)/website/virt-builder-repository.1.html: stamp-virt-builder-repository.pod + +stamp-virt-builder-repository.pod: virt-builder-repository.pod + $(PODWRAPPER) \ + --man virt-builder-repository.1 \ + --html $(top_builddir)/website/virt-builder-repository.1.html \ + --license GPLv2+ \ + --warning safe \ + $< + touch $@ + # Tests. TESTS_ENVIRONMENT = $(top_builddir)/run --test @@ -325,7 +407,8 @@ check-valgrind: SLOW_TESTS = \ $(console_test_scripts) \ - test-virt-builder-planner.sh + test-virt-builder-planner.sh \ + test-virt-builder-repository.sh check-slow: $(MAKE) check TESTS="$(SLOW_TESTS)" SLOW=1 diff --git a/builder/repository_main.ml b/builder/repository_main.ml new file mode 100644 index 000000000..a4bd16586 --- /dev/null +++ b/builder/repository_main.ml @@ -0,0 +1,595 @@ +(* virt-builder + * Copyright (C) 2016-2017 SUSE Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Std_utils +open Common_gettext.Gettext +open Tools_utils +open Unix_utils +open Getopt.OptionName +open Utils +open Yajl +open Xpath_helpers + +open Printf + +type cmdline = { + gpg : string; + gpgkey : string option; + interactive : bool; + compression : bool; + repo : string; +} + +type disk_image_info = { + format : string; + size : int64; +} + +let parse_cmdline () = + let gpg = ref "gpg" in + let gpgkey = ref None in + let set_gpgkey arg = gpgkey := Some arg in + + let interactive = ref false in + let compression = ref true in + let machine_readable = ref false in + + let argspec = [ + [ L"gpg" ], Getopt.Set_string ("gpg", gpg), s_"Set GPG binary/command"; + [ S 'K'; L"gpg-key" ], Getopt.String ("gpgkey", set_gpgkey), + s_"ID of the GPG key to sign the repo with"; + [ S 'i'; L"interactive" ], Getopt.Set interactive, s_"Ask the user about missing data"; + [ L"no-compression" ], Getopt.Clear compression, s_"Don’t compress the new images in the index"; + [ L"machine-readable" ], Getopt.Set machine_readable, s_"Make output machine readable"; + ] in + + let args = ref [] in + let anon_fun s = push_front s args in + let usage_msg = + sprintf (f_"\ +%s: create a repository for virt-builder + + virt-builder-repository REPOSITORY_PATH + +A short summary of the options is given below. For detailed help please +read the man page virt-builder-repository(1). +") + prog in + let opthandle = create_standard_options argspec ~anon_fun usage_msg in + Getopt.parse opthandle; + + (* Machine-readable mode? Print out some facts about what + * this binary supports. + *) + if !machine_readable then ( + printf "virt-builder-repository\n"; + exit 0 + ); + + (* Dereference options. *) + let args = List.rev !args in + let gpg = !gpg in + let gpgkey = !gpgkey in + let interactive = !interactive in + let compression = !compression in + + (* Check options *) + let repo = + match args with + | [repo] -> repo + | [] -> + error (f_"virt-builder-repository /path/to/repo + +Use ‘/path/to/repo’ to point to the repository folder.") + | _ -> + error (f_"too many parameters, only one path to repository is allowed") in + + { + gpg = gpg; + gpgkey = gpgkey; + interactive = interactive; + compression = compression; + repo = repo; + } + +let do_mv src dest = + let cmd = [ "mv"; src; dest ] in + let r = run_command cmd in + if r <> 0 then + error (f_"moving file ‘%s’ to ‘%s’ failed") src dest + +let checksums_get_sha512 = function + | None -> None + | Some csums -> + let rec loop = function + | [] -> None + | Checksums.SHA512 csum :: _ -> Some (Checksums.SHA512 csum) + | _ :: rest -> loop rest + in + loop csums + +let osinfo_ids = ref None + +let rec osinfo_get_short_ids () = + match !osinfo_ids with + | Some ids -> ids + | None -> + osinfo_ids := + Some ( + Osinfo.fold ( + fun set filepath -> + let doc = Xml.parse_file filepath in + let xpathctx = Xml.xpath_new_context doc in + let nodes = xpath_get_nodes xpathctx "/libosinfo/os/short-id" in + List.fold_left ( + fun set node -> + let id = Xml.node_as_string node in + StringSet.add id set + ) set nodes + ) StringSet.empty + ); + osinfo_get_short_ids () + +let compress_to file outdir = + let outimg = outdir // Filename.basename file ^ ".xz" in + + info "Compressing ..."; + let cmd = [ "xz"; "-f"; "--best"; "--block-size=16777216"; "-c"; file ] in + let file_flags = [ Unix.O_WRONLY; Unix.O_CREAT; Unix.O_TRUNC; ] in + let outfd = Unix.openfile outimg file_flags 0o666 in + let res = run_command cmd ~stdout_chan:outfd in + if res <> 0 then + error (f_"‘xz’ command failed"); + outimg + +let get_mime_type filepath = + let file_cmd = "file --mime-type --brief " ^ (quote filepath) in + match external_command file_cmd with + | [] -> None + | line :: _ -> Some line + +let get_disk_image_info filepath = + let infos = get_image_infos filepath in + { + format = object_get_string "format" infos; + size = object_get_number "virtual-size" infos + } + +let compute_short_id distro major minor = + match distro with + | "centos" when major >= 7 -> + sprintf "%s%d.0" distro major + | "debian" when major >= 4 -> + sprintf "%s%d" distro major + | ("fedora"|"mageia") -> + sprintf "%s%d" distro major + | "sles" when major = 0 -> + sprintf "%s%d" distro major + | "sles" -> + sprintf "%s%dsp%d" distro major minor + | "ubuntu" -> + sprintf "%s%d.%02d" distro major minor + | _ (* Any other combination. *) -> + sprintf "%s%d.%d" distro major minor + +let cmp a b = + Index.string_of_arch a = Index.string_of_arch b + +let has_entry id arch index = + List.exists ( + fun (item_id, { Index.arch = item_arch }) -> + item_id = id && cmp item_arch arch + ) index + +let process_image acc_entries filename repo tmprepo index interactive + compression sigchecker = + message (f_"Preparing %s") filename; + + let filepath = repo // filename in + let { format; size } = get_disk_image_info filepath in + let out_path = + if not compression then filepath + else compress_to filepath tmprepo in + let out_filename = Filename.basename out_path in + let checksum = Checksums.compute_checksum "sha512" out_path in + let compressed_size = (Unix.LargeFile.stat out_path).Unix.LargeFile.st_size in + + let ask ~default ?values message = + printf "%s [%s] " message default; + (match values with + | None -> () + | Some x -> + printf (f_"Choose one from the list below:\n %s\n") + (String.concat "\n " x)); + let value = read_line () in + + if value = "" then + default + else + value + in + + let re_valid_id = PCRE.compile ~anchored:true "[-a-zA-Z0-9_.]+" in + let rec ask_id default = + let id = ask (s_"Identifier: ") ~default in + if not (PCRE.matches re_valid_id id) then ( + warning (f_"Allowed characters are letters, digits, - _ and ."); + ask_id default + ) else + id in + + let ask_arch guess = + let arches = [ "x86_64"; "aarch64"; "armv7l"; "i686"; "ppc64"; "ppc64le"; "s390x" ] in + Index.Arch (ask (s_"Architecture: ") ~default:guess ~values:arches) + in + + let ask_osinfo default = + let osinfo = ask (s_ "osinfo short ID: ") ~default in + let osinfo_ids = osinfo_get_short_ids () in + if not (StringSet.mem osinfo osinfo_ids) then + warning (f_"‘%s’ is not a recognized osinfo OS id; using it anyway") osinfo; + osinfo in + + let extract_entry_data ?entry () = + message (f_"Extracting data from the image..."); + let g = Tools_utils.open_guestfs () in + g#add_drive_ro filepath; + g#launch (); + + let roots = g#inspect_os () in + let nroots = Array.length roots in + if nroots <> 1 then + error (f_"virt-builder template images must have one and only one root file system, found %d") + nroots; + + let root = Array.get roots 0 in + let inspected_arch = g#inspect_get_arch root in + let product = g#inspect_get_product_name root in + let distro = g#inspect_get_distro root in + let version_major = g#inspect_get_major_version root in + let version_minor = g#inspect_get_minor_version root in + let lvs = g#lvs () in + let filesystems = g#inspect_get_filesystems root in + + let shortid = compute_short_id distro version_major version_minor in + + g#close (); + + let id = + match entry with + | Some (id, _) -> id + | None -> ( + if interactive then ask_id shortid + else error (f_"missing image identifier") + ) in + + let arch = + match entry with + | Some (_, { Index.arch }) -> ( + match arch with + | Index.Arch arch -> Index.Arch arch + | Index.GuessedArch arch -> + if interactive then ask_arch arch + else Index.Arch arch ) + | None -> + if interactive then ask_arch inspected_arch + else Index.Arch inspected_arch in + + if has_entry id arch acc_entries then ( + let arch = + match arch with + | Index.Arch arch + | Index.GuessedArch arch -> arch in + error (f_"Already existing image with id %s and architecture %s") id arch + ); + + let printable_name = + match entry with + | Some (_, { Index.printable_name }) -> + if printable_name = None then + if interactive then Some (ask (s_"Display name: ") ~default:product) + else Some product + else + printable_name + | None -> Some product in + + let osinfo = + match entry with + | Some (_, { Index.osinfo }) -> + if osinfo = None then + Some (if interactive then ask_osinfo shortid else shortid) + else + osinfo + | None -> + Some (if interactive then ask_osinfo shortid else shortid) in + + let expand = + match entry with + | Some (_, { Index.expand }) -> + if expand = None then + if interactive then + Some (ask (s_"Expandable partition: ") ~default:root + ~values:(Array.to_list filesystems)) + else Some root + else + expand + | None -> + if interactive then + Some (ask (s_"Expandable partition: ") ~default:root + ~values:(Array.to_list filesystems)) + else Some root in + + let lvexpand = + if lvs = [||] then + None + else + match entry with + | Some (_, { Index.lvexpand }) -> + if lvexpand = None then + if interactive then + Some (ask (s_"Expandable volume: ") ~values:(Array.to_list lvs) + ~default:(Array.get lvs 0)) + else Some (Array.get lvs 0) + else + lvexpand + | None -> + if interactive then + Some (ask (s_"Expandable volume: ") ~values:(Array.to_list lvs) + ~default:(Array.get lvs 0)) + else Some (Array.get lvs 0) in + + let revision = + match entry with + | Some (_, { Index.revision }) -> + Utils.increment_revision revision + | None -> Rev_int 1 in + + let notes = + match entry with + | Some (_, { Index.notes }) -> notes + | None -> [] in + + let hidden = + match entry with + | Some (_, { Index.hidden }) -> hidden + | None -> false in + + let aliases = + match entry with + | Some (_, { Index.aliases }) -> aliases + | None -> None in + + (id, { Index.printable_name; + osinfo; + file_uri = Filename.basename out_path; + arch; + signature_uri = None; + checksums = Some [checksum]; + revision; + format = Some format; + size; + compressed_size = Some compressed_size; + expand; + lvexpand; + notes; + hidden; + aliases; + sigchecker; + proxy = Curl.SystemProxy }) + in + + (* Do we have an entry for that file already? *) + let file_entry = + try + List.hd ( + List.filter ( + fun (_, { Index.file_uri }) -> + let basename = Filename.basename file_uri in + basename = out_filename || basename = filename + ) index + ) + with + | Failure _ -> extract_entry_data () in + + let _, { Index.checksums } = file_entry in + let old_checksum = checksums_get_sha512 checksums in + + match old_checksum with + | Some old_sum -> + if old_sum = checksum then + let id, entry = file_entry in + (id, { entry with Index.file_uri = out_filename }) + else + extract_entry_data ~entry:file_entry () + | None -> + extract_entry_data ~entry:file_entry () + +let main () = + let cmdline = parse_cmdline () in + + (* If debugging, echo the command line arguments. *) + debug "command line: %s" (String.concat " " (Array.to_list Sys.argv)); + + (* Check that the paths are existing *) + if not (Sys.file_exists cmdline.repo) then + error (f_"repository folder ‘%s’ doesn’t exist") cmdline.repo; + + (* Create a temporary folder to work in *) + let tmpdir = Mkdtemp.temp_dir ~base_dir:cmdline.repo + "virt-builder-repository." in + rmdir_on_exit tmpdir; + + let tmprepo = tmpdir // "repo" in + mkdir_p tmprepo 0o700; + + let sigchecker = Sigchecker.create ~gpg:cmdline.gpg + ~check_signature:false + ~gpgkey:No_Key + ~tmpdir in + + let index = + try + let index_filename = + List.find ( + fun filename -> Sys.file_exists (cmdline.repo // filename) + ) [ "index.asc"; "index" ] in + + let downloader = Downloader.create ~curl:"do-not-use-curl" + ~cache:None ~tmpdir in + + let source = { Sources.name = index_filename; + uri = cmdline.repo // index_filename; + gpgkey = No_Key; + proxy = Curl.SystemProxy; + format = Sources.FormatNative } in + + Index_parser.get_index ~downloader ~sigchecker ~template:true source + with Not_found -> [] in + + (* Check for index/interactive consistency *) + if not cmdline.interactive && index = [] then + error (f_"the repository must contain an index file when running in automated mode"); + + debug "Searching for images ..."; + + let images = + let is_supported_format file = + let extension = last_part_of file '.' in + match extension with + | Some ext -> List.mem ext [ "qcow2"; "raw"; "img" ] + | None -> + match get_mime_type file with + | None -> false + | Some mime -> mime = "application/octet-stream" in + let is_new file = + try + let _, { Index.checksums } = + List.find ( + fun (_, { Index.file_uri }) -> + Filename.basename file_uri = file + ) index in + let checksum = checksums_get_sha512 checksums in + let path = cmdline.repo // file in + let file_checksum = Checksums.compute_checksum "sha512" path in + match checksum with + | None -> true + | Some sum -> sum <> file_checksum + with Not_found -> true in + let files = Array.to_list (Sys.readdir cmdline.repo) in + let files = List.filter ( + fun file -> is_regular_file (cmdline.repo // file) + ) files in + List.filter ( + fun file -> is_supported_format (cmdline.repo // file) && is_new file + ) files in + + if images = [] then ( + info (f_ "No new image found"); + exit 0 + ); + + info (f_ "Found new images: %s") (String.concat " " images); + + with_open_out (tmprepo // "index") ( + fun index_channel -> + (* Generate entries for uncompressed images *) + let images_entries = List.fold_right ( + fun filename acc -> + let image_entry = process_image acc + filename + cmdline.repo + tmprepo + index + cmdline.interactive + cmdline.compression + sigchecker in + image_entry :: acc + ) images [] in + + (* Filter out entries for newly found images and entries + without a corresponding image file or with empty arch *) + let index = List.filter ( + fun (id, { Index.arch; file_uri }) -> + not (has_entry id arch images_entries) && Sys.file_exists file_uri + ) index in + + (* Convert all URIs back to relative ones *) + let index = List.map ( + fun (id, entry) -> + let { Index.file_uri } = entry in + let rel_path = + try + subdirectory cmdline.repo file_uri + with + | Invalid_argument _ -> + file_uri in + let rel_entry = { entry with Index.file_uri = rel_path } in + (id, rel_entry) + ) index in + + (* Write all the entries *) + List.iter ( + fun entry -> + Index_parser.write_entry index_channel entry; + ) (index @ images_entries); + ); + + (* GPG sign the generated index *) + (match cmdline.gpgkey with + | None -> + debug "Skip index signing" + | Some gpgkey -> + message (f_"Signing index with the GPG key %s") gpgkey; + let cmd = sprintf "%s --armor --output %s --export %s" + (quote (cmdline.gpg // "index.gpg")) + (quote tmprepo) (quote gpgkey) in + if shell_command cmd <> 0 then + error (f_"failed to export the GPG key %s") gpgkey; + + let cmd = sprintf "%s --armor --default-key %s --clearsign %s" + (quote cmdline.gpg) (quote gpgkey) + (quote (tmprepo // "index" )) in + if shell_command cmd <> 0 then + error (f_"failed to sign index"); + ); + + message (f_"Creating index backup copy"); + + List.iter ( + fun filename -> + let filepath = cmdline.repo // filename in + if Sys.file_exists filepath then + do_mv filepath (filepath ^ ".bak") + ) ["index"; "index.asc"]; + + message (f_"Moving files to final destination"); + + Array.iter ( + fun filename -> + do_mv (tmprepo // filename) cmdline.repo + ) (Sys.readdir tmprepo); + + debug "Cleanup"; + + (* Remove the processed image files *) + if cmdline.compression then + List.iter ( + fun filename -> Sys.remove (cmdline.repo // filename) + ) images + +let () = run_main_and_handle_errors main diff --git a/builder/test-docs.sh b/builder/test-docs.sh index 884135de6..6f39b906d 100755 --- a/builder/test-docs.sh +++ b/builder/test-docs.sh @@ -25,3 +25,5 @@ $top_srcdir/podcheck.pl virt-builder.pod virt-builder \ --insert $top_srcdir/customize/customize-synopsis.pod:__CUSTOMIZE_SYNOPSIS__ \ --insert $top_srcdir/customize/customize-options.pod:__CUSTOMIZE_OPTIONS__ \ --ignore=--check-signatures,--no-check-signatures + +$srcdir/../podcheck.pl virt-builder-repository.pod virt-builder-repository diff --git a/builder/test-virt-builder-repository.sh b/builder/test-virt-builder-repository.sh new file mode 100755 index 000000000..0845c9d98 --- /dev/null +++ b/builder/test-virt-builder-repository.sh @@ -0,0 +1,100 @@ +#!/bin/bash - +# libguestfs +# Copyright (C) 2017 SUSE Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +set -e + +$TEST_FUNCTIONS +slow_test +skip_if_skipped "$script" + +test_data=repository-testdata +rm -rf $test_data +mkdir $test_data + +# Make a copy of the Fedora image +cp ../test-data/phony-guests/fedora.img $test_data + +# Create minimal index file +cat > $test_data/index << EOF +[fedora] +file=fedora.img +EOF + +# Run virt-builder-repository (no compression, interactive) +echo 'x86_64 +Fedora Test Image +fedora14 +/dev/sda1 +/dev/VG/Root +' | virt-builder-repository -v -x --no-compression -i $test_data + +assert_config () { + item=$1 + regex=$2 + + sed -n -e "/\[$item]/,/^$/p" $test_data/index | grep "$regex" +} + +# Check the generated index file +assert_config 'fedora' 'revision=1' +assert_config 'fedora' 'arch=x86_64' +assert_config 'fedora' 'name=Fedora Test Image' +assert_config 'fedora' 'osinfo=fedora14' +assert_config 'fedora' 'checksum' +assert_config 'fedora' 'format=raw' +assert_config 'fedora' '^size=' +assert_config 'fedora' 'compressed_size=' +assert_config 'fedora' 'expand=/dev/' + + +# Copy the debian image and add the minimal piece to index +cp ../test-data/phony-guests/debian.img $test_data + +cat >> $test_data/index << EOF + +[debian] +file=debian.img +EOF + +# Run virt-builder-repository again +echo 'x86_64 +Debian Test Image +debian9 + +' | virt-builder-repository --no-compression -i $test_data + +# Check that the new image is complete and the first one hasn't changed +assert_config 'fedora' 'revision=1' + +assert_config 'debian' 'revision=1' +assert_config 'debian' 'checksum' + +# Modify the fedora image +export EDITOR='echo newline >>' +virt-edit -a $test_data/fedora.img /etc/test3 + +# Rerun the tool (with compression) +virt-builder-repository -i $test_data + +# Check that the revision, file and size have been updated +assert_config 'fedora' 'revision=2' +assert_config 'fedora' 'file=fedora.img.xz' +test -e $test_data/fedora.img.xz +! test -e $test_data/fedora.img + +rm -rf $test_data diff --git a/builder/utils.ml b/builder/utils.ml index d0e51a049..4cca4c21d 100644 --- a/builder/utils.ml +++ b/builder/utils.ml @@ -35,6 +35,10 @@ let string_of_revision = function | Rev_int n -> string_of_int n | Rev_string s -> s +let increment_revision = function + | Rev_int n -> Rev_int (n + 1) + | Rev_string s -> Rev_int ((int_of_string s) + 1) + let get_image_infos filepath = let qemuimg_cmd = "qemu-img info --output json " ^ quote filepath in let lines = external_command qemuimg_cmd in diff --git a/builder/utils.mli b/builder/utils.mli index 5c7bfa1e4..f9c828dd0 100644 --- a/builder/utils.mli +++ b/builder/utils.mli @@ -32,3 +32,6 @@ val string_of_revision : revision -> string val get_image_infos : string -> Yajl.yajl_val (** [get_image_infos path] Run qemu-img info on the image pointed at path as YAJL tree. *) + +val increment_revision : revision -> revision +(** Add one to the revision number *) diff --git a/builder/virt-builder-repository.pod b/builder/virt-builder-repository.pod new file mode 100644 index 000000000..11fec8f07 --- /dev/null +++ b/builder/virt-builder-repository.pod @@ -0,0 +1,213 @@ +=begin html + +<img src="virt-builder.svg" width="250" + style="float: right; clear: right;" /> + +=end html + +=head1 NAME + +virt-builder-repository - Build virt-builder source repository easily + +=head1 SYNOPSIS + + virt-builder-repository /path/to/repository + [-i|--interactive] [--gpg-key KEYID] + +=head1 DESCRIPTION + +Virt-builder is a tool for quickly building new virtual machines. It can +be configured to use template repositories. However creating and +maintaining a repository involves many tasks which can be automated. +virt-builder-repository is a tool helping to manage these repositories. + +Virt-builder-repository loops over the files in the directory specified +as argument, compresses the files with a name ending by C<qcow2>, C<raw>, +C<img> or without extension, extracts data from them and creates or +updates the C<index> file. + +Some of the image-related data needed for the index file can’t be +computed from the image file. virt-builder-repository first tries to +find them in the existing index file. If data are still missing after +this, they are prompted in interactive mode, otherwise an error will +be triggered. + +If a C<KEYID> is provided, the generated index file will be signed +with this GPG key. + +=head1 EXAMPLES + +=head2 Create the initial repository + +Create a folder and copy the disk image template files in it. Then +run a command like the following one: + + virt-builder-repository --gpg-key &quot;joe(a)hacker.org&quot; -i /path/to/folder + +Note that this example command runs in interactive mode. To run in +automated mode, a minimal index file needs to be created before running +the command containing sections like this one: + + [template_id] + file=template_filename.qcow.xz + +The file value needs to match the image name extended with the C<.xz> +suffix if the I<--no-compression> parameter is not provided or the +image name if no compression is involved. Other optional data can be +prefilled. Default values are computed by inspecting the disk image. +For more informations, see +L<virt-builder(1)/Creating and signing the index file>. + +=head2 Update images in an existing repository + +In this use case, an new image or a new revision of an existing image +needs to be added to the repository. Place the corresponding image +template files in the repository folder. + +To update the revision of an image, the file needs to have the same +name than the existing one (without the C<xz> extension). + +As in the repository creation use case, a minimal fragment can be +added to the index file for the automated mode. This can be done +on the signed index even if it may sound a strange idea: the index +will be signed again by the tool. + +To remove an image from the repository, just remove the corresponding +image file before running virt-builder-repository. + +Then running the following command will complete and update the index +file: + + virt-builder-repository --gpg-key &quot;joe(a)hacker.org&quot; -i /path/to/folder + +virt-builder-repository works in a temporary folder inside the repository +one. If anything wrong happens when running the tool, the repository is +left untouched. + +=head1 OPTIONS + +=over 4 + +=item B<--help> + +Display help. + +=item B<--gpg> GPG + +Specify an alternate L<gpg(1)> (GNU Privacy Guard) binary. You can +also use this to add gpg parameters, for example to specify an +alternate home directory: + + virt-builder-repository --gpg "gpg --homedir /tmp" [...] + +This can also be used to avoid gpg asking for the key passphrase: + + virt-builder-repository --gpg "gpg --passphrase-file /tmp/pass --batch" [...] + +=item B<-K> KEYID + +=item B<--gpg-key> KEYID + +Specify the GPG key to be used to sign the repository index file. +If not provided, the index will left unsigned. C<KEYID> is used to +identify the GPG key to use. This value is passed to gpg’s +I<--default-key> option and can thus be an email address or a +fingerprint. + +B<NOTE>: by default, virt-builder-repository searches for the key +in the user’s GPG keyring. + +=item B<-i> + +=item B<--interactive> + +Prompt for missing data. Default values are computed from the disk +image. + +When prompted for data, inputting C<-> corresponds to leaving the +value empty. This can be used to avoid setting the default computed value. + +=item B<--keep-index> + +When using a GPG key, don’t remove the unsigned index. + +=item B<--no-compression> + +Don’t compress the template images. + +=item B<--machine-readable> + +This option is used to make the output more machine friendly +when being parsed by other programs. See +L</MACHINE READABLE OUTPUT> below. + + +=item B<--colors> + +=item B<--colours> + +Use ANSI colour sequences to colourize messages. This is the default +when the output is a tty. If the output of the program is redirected +to a file, ANSI colour sequences are disabled unless you use this +option. + +=item B<-q> + +=item B<--quiet> + +Don’t print ordinary progress messages. + +=item B<-v> + +=item B<--verbose> + +Enable debug messages and/or produce verbose output. + +When reporting bugs, use this option and attach the complete output to +your bug report. + +=item B<-V> + +=item B<--version> + +Display version number and exit. + +=item B<-x> + +Enable tracing of libguestfs API calls. + + +=back + +=head1 MACHINE READABLE OUTPUT + +The I<--machine-readable> option can be used to make the output more +machine friendly, which is useful when calling virt-builder-repository from +other programs, GUIs etc. + +Use the option on its own to query the capabilities of the +virt-builder-repository binary. Typical output looks like this: + + $ virt-builder-repository --machine-readable + virt-builder-repository + +A list of features is printed, one per line, and the program exits +with status 0. + +=head1 EXIT STATUS + +This program returns 0 if successful, or non-zero if there was an +error. + +=head1 SEE ALSO + +L<virt-builder(1)> +L<http://libguestfs.org/>. + +=head1 AUTHOR + +Cédric Bosdonnat L<mailto:cbosdonnat@suse.com> + +=head1 COPYRIGHT + +Copyright (C) 2016-2017 SUSE Inc. diff --git a/builder/virt-builder.pod b/builder/virt-builder.pod index 74bf7bb11..1ed18a7c7 100644 --- a/builder/virt-builder.pod +++ b/builder/virt-builder.pod @@ -1319,6 +1319,9 @@ digital signature): The part in square brackets is the C<os-version>, which is the same string that is used on the virt-builder command line to build that OS. +The index file creation and signature can be eased with the +L<virt-builder-repository(1)> tool. + After preparing the C<index> file in the correct format, clearsign it using the following command: @@ -1875,6 +1878,7 @@ error. L<guestfs(3)>, L<guestfish(1)>, L<guestmount(1)>, +L<virt-builder-repository(1)>, L<virt-copy-out(1)>, L<virt-customize(1)>, L<virt-get-kernel(1)>, diff --git a/fish/guestfish.pod b/fish/guestfish.pod index 11f2bbeb5..c37189bd8 100644 --- a/fish/guestfish.pod +++ b/fish/guestfish.pod @@ -1617,6 +1617,7 @@ L<guestfs(3)>, L<http://libguestfs.org/>;, L<virt-alignment-scan(1)>, L<virt-builder(1)>, +L<virt-builder-repository(1)>, L<virt-cat(1)>, L<virt-copy-in(1)>, L<virt-copy-out(1)>, diff --git a/installcheck.sh.in b/installcheck.sh.in index 6b05ab812..a4829cda6 100644 --- a/installcheck.sh.in +++ b/installcheck.sh.in @@ -46,6 +46,7 @@ cp @bindir@/guestfish fish/ cp @bindir@/guestmount fuse/ cp @bindir@/virt-alignment-scan align/ cp @bindir@/virt-builder builder/ +cp @bindir@/virt-builder-repository builder/ cp @bindir@/virt-cat cat/ cp @bindir@/virt-copy-in fish/ cp @bindir@/virt-copy-out fish/ diff --git a/lib/guestfs.pod b/lib/guestfs.pod index 8d31f3200..55467e92e 100644 --- a/lib/guestfs.pod +++ b/lib/guestfs.pod @@ -3417,6 +3417,7 @@ L<guestfish(1)>, L<guestmount(1)>, L<virt-alignment-scan(1)>, L<virt-builder(1)>, +L<virt-builder-repository(1)>, L<virt-cat(1)>, L<virt-copy-in(1)>, L<virt-copy-out(1)>, -- 2.15.0