[Libguestfs] [PATCH 10/10] builder: support Simple Streams v1.0 as index metadata

Add a simple YAJL<->OCaml bridge to expose the JSON parsing function, and use it to parse the JSON indexes of the Simple Streams format. Read only datatype=image-downloads contents, and only the latest versions of each content available as disk image (disk.img or disk1.img). --- builder/Makefile.am | 13 ++- builder/builder.ml | 2 + builder/simplestreams_parser.ml | 207 +++++++++++++++++++++++++++++++++++++++ builder/simplestreams_parser.mli | 19 ++++ builder/sources.ml | 9 ++ builder/sources.mli | 1 + builder/virt-builder.pod | 7 ++ builder/yajl-c.c | 141 ++++++++++++++++++++++++++ builder/yajl.ml | 30 ++++++ builder/yajl.mli | 33 +++++++ po/POTFILES | 1 + po/POTFILES-ml | 2 + 12 files changed, 462 insertions(+), 3 deletions(-) create mode 100644 builder/simplestreams_parser.ml create mode 100644 builder/simplestreams_parser.mli create mode 100644 builder/yajl-c.c create mode 100644 builder/yajl.ml create mode 100644 builder/yajl.mli diff --git a/builder/Makefile.am b/builder/Makefile.am index 597b943..98a444f 100644 --- a/builder/Makefile.am +++ b/builder/Makefile.am @@ -48,7 +48,9 @@ SOURCES_MLI = \ pxzcat.mli \ setlocale.mli \ sigchecker.mli \ - sources.mli + simplestreams_parser.mli \ + sources.mli \ + yajl.mli SOURCES_ML = \ utils.ml \ @@ -57,6 +59,7 @@ SOURCES_ML = \ checksums.ml \ index.ml \ ini_reader.ml \ + yajl.ml \ paths.ml \ languages.ml \ cache.ml \ @@ -64,6 +67,7 @@ SOURCES_ML = \ downloader.ml \ sigchecker.ml \ index_parser.ml \ + simplestreams_parser.ml \ list_entries.ml \ cmdline.ml \ builder.ml @@ -81,7 +85,8 @@ SOURCES_C = \ index-parse.c \ index-parser-c.c \ pxzcat-c.c \ - setlocale-c.c + setlocale-c.c \ + yajl-c.c man_MANS = noinst_DATA = @@ -106,7 +111,8 @@ virt_builder_CFLAGS = \ -Wno-unused-macros \ $(LIBLZMA_CFLAGS) \ $(LIBTINFO_CFLAGS) \ - $(LIBXML2_CFLAGS) + $(LIBXML2_CFLAGS) \ + $(YAJL_CFLAGS) BOBJECTS = \ $(top_builddir)/mllib/libdir.cmo \ @@ -156,6 +162,7 @@ OCAMLCLIBS = \ $(LIBCRYPT_LIBS) \ $(LIBLZMA_LIBS) \ $(LIBXML2_LIBS) \ + $(YAJL_LIBS) \ $(LIBINTL) \ -lgnu diff --git a/builder/builder.ml b/builder/builder.ml index a30dbd1..1e6a426 100644 --- a/builder/builder.ml +++ b/builder/builder.ml @@ -175,6 +175,8 @@ let main () = match source.Sources.format with | Sources.FormatNative -> Index_parser.get_index ~downloader ~sigchecker source + | Sources.FormatSimpleStreams -> + Simplestreams_parser.get_index ~downloader ~sigchecker source ) sources ) in let index = remove_duplicates index in diff --git a/builder/simplestreams_parser.ml b/builder/simplestreams_parser.ml new file mode 100644 index 0000000..5f8e725 --- /dev/null +++ b/builder/simplestreams_parser.ml @@ -0,0 +1,207 @@ +(* virt-builder + * Copyright (C) 2015 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Common_gettext.Gettext +open Common_utils + +open Yajl + +open Printf + +let ensure_trailing_slash str = + if String.length str > 0 && str.[String.length str - 1] <> '/' then str ^ "/" + else str + +let object_find_optional key = function + | Yajl_object o -> + (match List.filter (fun (k, _) -> k = key) (Array.to_list o) with + | [(k, v)] -> Some v + | [] -> None + | _ -> error "more than one '%s' node" key) + | _ -> error "value for key '%s' is not an object" key + +let object_find key yv = + (match object_find_optional key yv with + | None -> error "missing '%s' node" key + | Some v -> v + ) + +let object_get_string key yv = + (match object_find key yv with + | Yajl_string s -> s + | _ -> error "key '%s' is not a string" key + ) + +let object_get_string_optional key yv = + (match object_find_optional key yv with + | None -> None + | Some (Yajl_string s) -> Some s + | Some _ -> error "key '%s' is not a string" key + ) + +let object_find_object key yv = + (match object_find key yv with + | Yajl_object _ as o -> o + | _ -> error "key '%s' is not an object" key + ) + +let object_find_object_optional key yv = + (match object_find_optional key yv with + | None -> None + | Some (Yajl_object _ as o) -> Some o + | Some _ -> error "key '%s' is not an object" key + ) + +let object_find_objects fn = function + | Yajl_object o -> filter_map fn (Array.to_list o) + | _ -> error "value is not an object" + +let object_get_object key yv = + (match object_find_object key yv with + | Yajl_object o -> o + | _ -> assert false (* object_find_object already errors out. *) + ) + +let object_get_number key yv = + (match object_find key yv with + | Yajl_number n -> n + | Yajl_double d -> Int64.of_float d + | _ -> error "key '%s' is not an integer" key + ) + +let get_index ~downloader ~sigchecker + { Sources.uri = uri; proxy = proxy } = + + let uri = ensure_trailing_slash uri in + + let download_and_parse uri = + let tmpfile, delete_tmpfile = Downloader.download downloader ~proxy uri in + if delete_tmpfile then + unlink_on_exit tmpfile; + let file = + if Sigchecker.verifying_signatures sigchecker then ( + let tmpunsigned = Sigchecker.verify_and_remove_signature sigchecker tmpfile in + match tmpunsigned with + | None -> assert false (* only when not verifying signatures *) + | Some f -> f + ) else + tmpfile in + yajl_tree_parse (read_whole_file file) in + + let downloads = + let uri_index = + if Sigchecker.verifying_signatures sigchecker then + uri ^ "streams/v1/index.sjson" + else + uri ^ "streams/v1/index.json" in + let tree = download_and_parse uri_index in + + let format = object_get_string "format" tree in + if format <> "index:1.0" then + error (f_"%s does not point to a Simple Streams (index) v1.0 JSON file") uri; + + let index = Array.to_list (object_get_object "index" tree) in + filter_map ( + fun (_, desc) -> + let datatype = object_get_string "datatype" desc in + match datatype with + | "image-downloads" -> Some (object_get_string "path" desc) + | _ -> None + ) index in + + let scan_product_list path = + let tree = download_and_parse (uri ^ path) in + + let format = object_get_string "format" tree in + if format <> "products:1.0" then + error (f_"%s does not point to a Simple Streams (products) v1.0 JSON file") uri; + + let products_node = object_get_object "products" tree in + + let products = Array.to_list products_node in + filter_map ( + fun (prod, prod_desc) -> + let arch = + match object_get_string "arch" prod_desc with + | "amd64" -> "x86_64" + | a -> a in + let prods = Array.to_list (object_get_object "versions" prod_desc) in + let prods = filter_map ( + fun (rel, rel_desc) -> + let pubname = + match object_get_string_optional "pubname" rel_desc with + | Some p -> p + | None -> object_get_string "pubname" prod_desc in + let items = object_find_object "items" rel_desc in + let disk_items = object_find_objects ( + function + | (("disk.img"|"disk1.img"), v) -> Some v + | _ -> None + ) items in + (match disk_items with + | [] -> None + | disk_item :: _ -> + let disk_path = object_get_string "path" disk_item in + let disk_size = object_get_number "size" disk_item in + let checksums = object_find_objects ( + function + | ("sha256", Yajl_string c) -> Some (Checksums.SHA256 c) + | ("sha512", Yajl_string c) -> Some (Checksums.SHA512 c) + | _ -> None + ) disk_item in + let checksums = + match checksums with + | [] -> None + | x -> Some x in + let entry = { + Index.printable_name = Some pubname; + osinfo = None; + file_uri = uri ^ disk_path; + arch = arch; + signature_uri = None; + checksums = checksums; + revision = 0; (* XXX handle revisions as strings *) + format = None; + size = disk_size; + compressed_size = None; + expand = None; + lvexpand = None; + notes = []; + hidden = false; + aliases = Some [pubname;]; + sigchecker = sigchecker; + proxy = proxy; + } in + Some (rel, (prod, entry)) + ) + ) prods in + (* Select the disk image with the bigger version (i.e. usually + * the most recent one. *) + let reverse_revision_compare (rev1, _) (rev2, _) = compare rev2 rev1 in + let prods = List.sort reverse_revision_compare prods in + match prods with + | [] -> None + | (_, entry) :: _ -> Some entry + ) products in + + let entries = List.flatten (List.map scan_product_list downloads) in + if verbose () then ( + printf "simplestreams tree (%s) after parsing:\n" uri; + List.iter (Index.print_entry Pervasives.stdout) entries + ); + entries diff --git a/builder/simplestreams_parser.mli b/builder/simplestreams_parser.mli new file mode 100644 index 0000000..a4b91ba --- /dev/null +++ b/builder/simplestreams_parser.mli @@ -0,0 +1,19 @@ +(* virt-builder + * Copyright (C) 2015 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> Sources.source -> Index.index diff --git a/builder/sources.ml b/builder/sources.ml index b21e8fc..149db6f 100644 --- a/builder/sources.ml +++ b/builder/sources.ml @@ -31,6 +31,7 @@ type source = { } and source_format = | FormatNative +| FormatSimpleStreams module StringSet = Set.Make (String) @@ -82,6 +83,14 @@ let parse_conf file = try (match (List.assoc ("format", None) fields) with | "native" | "" -> FormatNative + | "simplestreams" as fmt -> + if not (Yajl.yajl_is_available ()) then ( + if verbose () then ( + eprintf (f_"%s: repository type '%s' not supported (missing YAJL support), skipping it\n") prog fmt; + ); + invalid_arg fmt + ) else + FormatSimpleStreams | fmt -> if verbose () then ( eprintf (f_"%s: unknown repository type '%s' in %s, skipping it\n") prog fmt file; diff --git a/builder/sources.mli b/builder/sources.mli index e861310..e621a9f 100644 --- a/builder/sources.mli +++ b/builder/sources.mli @@ -25,5 +25,6 @@ type source = { } and source_format = | FormatNative +| FormatSimpleStreams val read_sources : unit -> source list diff --git a/builder/virt-builder.pod b/builder/virt-builder.pod index fc49d4d..0de643a 100644 --- a/builder/virt-builder.pod +++ b/builder/virt-builder.pod @@ -1181,6 +1181,13 @@ The possible values are: The native format of the C<virt-builder> repository. See also L</Creating and signing the index file> below. +=item B<simplestreams> + +The URI represents the root of a Simple Streams v1.0 tree of metadata. + +For more information about Simple Streams, see also +L<https://launchpad.net/simplestreams>. + =back If not present, the assumed value is C<native>. diff --git a/builder/yajl-c.c b/builder/yajl-c.c new file mode 100644 index 0000000..cb47efa --- /dev/null +++ b/builder/yajl-c.c @@ -0,0 +1,141 @@ +/* virt-builder + * Copyright (C) 2015 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <config.h> + +#include <caml/alloc.h> +#include <caml/fail.h> +#include <caml/memory.h> +#include <caml/mlvalues.h> + +#if HAVE_YAJL +#include <yajl/yajl_tree.h> +#endif + +#include <stdio.h> +#include <string.h> + +#define Val_none (Val_int (0)) + +extern value virt_builder_yajl_tree_parse (value stringv); +extern value virt_builder_yajl_is_available (value unit); + +#if HAVE_YAJL +static value +convert_yajl_value (yajl_val val, int level) +{ + CAMLparam0 (); + CAMLlocal4 (rv, lv, v, sv); + + if (level > 20) + caml_invalid_argument ("too many levels of object/array nesting"); + + if (YAJL_IS_OBJECT (val)) { + size_t len = YAJL_GET_OBJECT(val)->len; + size_t i; + rv = caml_alloc (1, 3); + lv = caml_alloc_tuple (len); + for (i = 0; i < len; ++i) { + v = caml_alloc_tuple (2); + sv = caml_copy_string (YAJL_GET_OBJECT(val)->keys[i]); + Store_field (v, 0, sv); + sv = convert_yajl_value (YAJL_GET_OBJECT(val)->values[i], level + 1); + Store_field (v, 1, sv); + Store_field (lv, i, v); + } + Store_field (rv, 0, lv); + } else if (YAJL_IS_ARRAY (val)) { + size_t len = YAJL_GET_ARRAY(val)->len; + size_t i; + rv = caml_alloc (1, 4); + lv = caml_alloc_tuple (len); + for (i = 0; i < len; ++i) { + v = convert_yajl_value (YAJL_GET_ARRAY(val)->values[i], level + 1); + Store_field (lv, i, v); + } + Store_field (rv, 0, lv); + } else if (YAJL_IS_STRING (val)) { + rv = caml_alloc (1, 0); + v = caml_copy_string (YAJL_GET_STRING(val)); + Store_field (rv, 0, v); + } else if (YAJL_IS_DOUBLE (val)) { + rv = caml_alloc (1, 2); + lv = caml_alloc_tuple (1); + Store_double_field (lv, 0, YAJL_GET_DOUBLE(val)); + Store_field (rv, 0, lv); + } else if (YAJL_IS_INTEGER (val)) { + rv = caml_alloc (1, 1); + v = caml_copy_int64 (YAJL_GET_INTEGER(val)); + Store_field (rv, 0, v); + } else if (YAJL_IS_TRUE (val)) { + rv = caml_alloc (1, 5); + Store_field (rv, 0, Val_true); + } else if (YAJL_IS_FALSE (val)) { + rv = caml_alloc (1, 5); + Store_field (rv, 0, Val_false); + } else + rv = Val_none; + + CAMLreturn (rv); +} + +value +virt_builder_yajl_is_available (value unit) +{ + return Val_true; +} + +value +virt_builder_yajl_tree_parse (value stringv) +{ + CAMLparam1 (stringv); + CAMLlocal1 (rv); + yajl_val tree; + char error_buf[256]; + + tree = yajl_tree_parse (String_val (stringv), error_buf, sizeof error_buf); + if (tree == NULL) { + char buf[256 + sizeof error_buf]; + if (strlen (error_buf) > 0) + snprintf (buf, sizeof buf, "JSON parse error: %s", error_buf); + else + snprintf (buf, sizeof buf, "unknown JSON parse error"); + caml_invalid_argument (buf); + } + + rv = convert_yajl_value (tree, 1); + yajl_tree_free (tree); + + CAMLreturn (rv); +} + +#else + +value +virt_builder_yajl_is_available (value unit) +{ + return Val_false; +} + +value +virt_builder_yajl_tree_parse (value stringv) +{ + caml_invalid_argument ("virt-builder was compiled without yajl support"); +} + +#endif diff --git a/builder/yajl.ml b/builder/yajl.ml new file mode 100644 index 0000000..f2d5c2b --- /dev/null +++ b/builder/yajl.ml @@ -0,0 +1,30 @@ +(* virt-builder + * Copyright (C) 2015 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +type yajl_val = +| Yajl_null +| Yajl_string of string +| Yajl_number of int64 +| Yajl_double of float +| Yajl_object of (string * yajl_val) array +| Yajl_array of yajl_val array +| Yajl_bool of bool + +external yajl_is_available : unit -> bool = "virt_builder_yajl_is_available" "noalloc" + +external yajl_tree_parse : string -> yajl_val = "virt_builder_yajl_tree_parse" diff --git a/builder/yajl.mli b/builder/yajl.mli new file mode 100644 index 0000000..aaa9389 --- /dev/null +++ b/builder/yajl.mli @@ -0,0 +1,33 @@ +(* virt-builder + * Copyright (C) 2015 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +type yajl_val = +| Yajl_null +| Yajl_string of string +| Yajl_number of int64 +| Yajl_double of float +| Yajl_object of (string * yajl_val) array +| Yajl_array of yajl_val array +| Yajl_bool of bool + +val yajl_is_available : unit -> bool +(** Is YAJL built in? If not, calling any of the other yajl_* + functions will result in an error. *) + +val yajl_tree_parse : string -> yajl_val +(** Parse the JSON string. *) diff --git a/po/POTFILES b/po/POTFILES index 7f1580c..8fb68a4 100644 --- a/po/POTFILES +++ b/po/POTFILES @@ -6,6 +6,7 @@ builder/index-struct.c builder/index-validate.c builder/pxzcat-c.c builder/setlocale-c.c +builder/yajl-c.c cat/cat.c cat/filesystems.c cat/log.c diff --git a/po/POTFILES-ml b/po/POTFILES-ml index 7933c8e..87f10ee 100644 --- a/po/POTFILES-ml +++ b/po/POTFILES-ml @@ -12,8 +12,10 @@ builder/paths.ml builder/pxzcat.ml builder/setlocale.ml builder/sigchecker.ml +builder/simplestreams_parser.ml builder/sources.ml builder/utils.ml +builder/yajl.ml customize/crypt.ml customize/customize_cmdline.ml customize/customize_main.ml -- 2.1.0