[PATCH 0/4 v3] builder: support for Simple Streams metadata
Hi, this series adds a basic support for Simple Streams v1.0 metadata files. This makes it possible to create a repository .conf files with [cirros] uri=http://download.cirros-cloud.net format=simplestreams to read the latest version of each CirrOS image. Thanks, Pino Toscano (4): builder: add non-int revisions builder: add simple libyajl binding build: expose HAVE_YAJL to automake builder: support Simple Streams v1.0 as index metadata .gitignore | 3 + builder/Makefile.am | 51 ++- builder/builder.ml | 11 +- builder/cache.ml | 2 +- builder/cache.mli | 6 +- builder/downloader.mli | 2 +- builder/index.ml | 4 +- builder/index.mli | 2 +- builder/index_parser.ml | 4 +- builder/simplestreams_parser.ml | 204 ++++++++++ builder/simplestreams_parser.mli | 19 + builder/sources.ml | 9 + builder/sources.mli | 1 + builder/test-simplestreams/streams/v1/index.json | 18 + .../v1/net.cirros-cloud:released:download.json | 429 +++++++++++++++++++++ .../virt-builder/repos.d/cirros.conf.in | 3 + builder/test-virt-builder-list-simplestreams.sh | 108 ++++++ builder/utils.ml | 7 + builder/virt-builder.pod | 7 + builder/yajl-c.c | 143 +++++++ builder/yajl.ml | 30 ++ builder/yajl.mli | 33 ++ builder/yajl_tests.ml | 139 +++++++ configure.ac | 2 + po/POTFILES | 1 + po/POTFILES-ml | 3 + 26 files changed, 1226 insertions(+), 15 deletions(-) create mode 100644 builder/simplestreams_parser.ml create mode 100644 builder/simplestreams_parser.mli create mode 100644 builder/test-simplestreams/streams/v1/index.json create mode 100644 builder/test-simplestreams/streams/v1/net.cirros-cloud:released:download.json create mode 100644 builder/test-simplestreams/virt-builder/repos.d/cirros.conf.in create mode 100755 builder/test-virt-builder-list-simplestreams.sh create mode 100644 builder/yajl-c.c create mode 100644 builder/yajl.ml create mode 100644 builder/yajl.mli create mode 100644 builder/yajl_tests.ml -- 2.1.0
Add support for non-integer revisions of entries, comparing them as integer when possible. --- builder/builder.ml | 9 ++++++++- builder/cache.ml | 2 +- builder/cache.mli | 6 +++--- builder/downloader.mli | 2 +- builder/index.ml | 4 ++-- builder/index.mli | 2 +- builder/index_parser.ml | 4 ++-- builder/utils.ml | 7 +++++++ 8 files changed, 25 insertions(+), 11 deletions(-) diff --git a/builder/builder.ml b/builder/builder.ml index d59380b..dcfd437 100644 --- a/builder/builder.ml +++ b/builder/builder.ml @@ -34,6 +34,13 @@ open Printf let () = Random.self_init () let remove_duplicates index = + let compare_revisions rev1 rev2 = + match rev1, rev2 with + | Rev_int n1, Rev_int n2 -> compare n1 n2 + | Rev_string s1, Rev_int n2 -> compare s1 (string_of_int n2) + | Rev_int n1, Rev_string s2 -> compare (string_of_int n1) s2 + | Rev_string s1, Rev_string s2 -> compare s1 s2 + in (* Fill an hash with the higher revision of the available * (name, arch) tuples, so it possible to ignore duplicates, * and versions with a lower revision. @@ -44,7 +51,7 @@ let remove_duplicates index = let id = name, arch in try let rev = Hashtbl.find nseen id in - if revision > rev then + if compare_revisions rev revision > 0 then Hashtbl.replace nseen id revision with Not_found -> Hashtbl.add nseen id revision diff --git a/builder/cache.ml b/builder/cache.ml index e73bcfd..8b63a86 100644 --- a/builder/cache.ml +++ b/builder/cache.ml @@ -40,7 +40,7 @@ let create ~directory = } let cache_of_name t name arch revision = - t.directory // sprintf "%s.%s.%d" name arch revision + t.directory // sprintf "%s.%s.%s" name arch (string_of_revision revision) let is_cached t name arch revision = let filename = cache_of_name t name arch revision in diff --git a/builder/cache.mli b/builder/cache.mli index 7edc670..1ab8ede 100644 --- a/builder/cache.mli +++ b/builder/cache.mli @@ -27,16 +27,16 @@ type t val create : directory:string -> t (** Create the abstract type. *) -val cache_of_name : t -> string -> string -> int -> string +val cache_of_name : t -> string -> string -> Utils.revision -> string (** [cache_of_name t name arch revision] return the filename of the cached file. (Note: It doesn't check if the filename exists, this is just a simple string transformation). *) -val is_cached : t -> string -> string -> int -> bool +val is_cached : t -> string -> string -> Utils.revision -> bool (** [is_cached t name arch revision] return whether the file with specified name, architecture and revision is cached. *) -val print_item_status : t -> header:bool -> (string * string * int) list -> unit +val print_item_status : t -> header:bool -> (string * string * Utils.revision) list -> unit (** [print_item_status t header items] print the status in the cache of the specified items (which are tuples of name, architecture, and revision). diff --git a/builder/downloader.mli b/builder/downloader.mli index 5e3cdaa..11ec498 100644 --- a/builder/downloader.mli +++ b/builder/downloader.mli @@ -35,7 +35,7 @@ type proxy_mode = val create : curl:string -> cache:Cache.t option -> t (** Create the abstract type. *) -val download : t -> ?template:(string*string*int) -> ?progress_bar:bool -> ?proxy:proxy_mode -> uri -> (filename * bool) +val download : t -> ?template:(string*string*Utils.revision) -> ?progress_bar:bool -> ?proxy:proxy_mode -> uri -> (filename * bool) (** Download the URI, returning the downloaded filename and a temporary file flag. The temporary file flag is [true] iff the downloaded file is temporary and should be deleted by the diff --git a/builder/index.ml b/builder/index.ml index 3e8cb85..c59d6dd 100644 --- a/builder/index.ml +++ b/builder/index.ml @@ -32,7 +32,7 @@ and entry = { arch : string; signature_uri : string option; (* deprecated, will be removed in 1.26 *) checksums : Checksums.csum_t list option; - revision : int; + revision : Utils.revision; format : string option; size : int64; compressed_size : int64 option; @@ -86,7 +86,7 @@ let print_entry chan (name, { printable_name = printable_name; (Checksums.string_of_csum_t c) (Checksums.string_of_csum c) ) checksums ); - fp "revision=%d\n" revision; + fp "revision=%s\n" (string_of_revision revision); (match format with | None -> () | Some format -> fp "format=%s\n" format diff --git a/builder/index.mli b/builder/index.mli index 10ed15a..fadcad9 100644 --- a/builder/index.mli +++ b/builder/index.mli @@ -24,7 +24,7 @@ and entry = { arch : string; signature_uri : string option; (* deprecated, will be removed in 1.26 *) checksums : Checksums.csum_t list option; - revision : int; + revision : Utils.revision; format : string option; size : int64; compressed_size : int64 option; diff --git a/builder/index_parser.ml b/builder/index_parser.ml index 845d0e9..2c78fd9 100644 --- a/builder/index_parser.ml +++ b/builder/index_parser.ml @@ -112,9 +112,9 @@ let get_index ~downloader ~sigchecker try Some (List.assoc ("checksum", None) fields) with Not_found -> None in let revision = - try int_of_string (List.assoc ("revision", None) fields) + try Rev_int (int_of_string (List.assoc ("revision", None) fields)) with - | Not_found -> 1 + | Not_found -> Rev_int 1 | Failure "int_of_string" -> eprintf (f_"%s: cannot parse 'revision' field for '%s'\n") prog n; corrupt_file () in diff --git a/builder/utils.ml b/builder/utils.ml index a6628eb..986bf68 100644 --- a/builder/utils.ml +++ b/builder/utils.ml @@ -26,5 +26,12 @@ type gpgkey_type = | No_Key | Fingerprint of string | KeyFile of string +and revision = + | Rev_int of int + | Rev_string of string let quote = Filename.quote + +let string_of_revision = function + | Rev_int n -> string_of_int n + | Rev_string s -> s -- 2.1.0
Only yajl_val and yajl_tree_parse are exposed for now. --- .gitignore | 2 + builder/Makefile.am | 45 ++++++++++++++-- builder/yajl-c.c | 143 ++++++++++++++++++++++++++++++++++++++++++++++++++ builder/yajl.ml | 30 +++++++++++ builder/yajl.mli | 33 ++++++++++++ builder/yajl_tests.ml | 139 ++++++++++++++++++++++++++++++++++++++++++++++++ po/POTFILES | 1 + po/POTFILES-ml | 2 + 8 files changed, 391 insertions(+), 4 deletions(-) create mode 100644 builder/yajl-c.c create mode 100644 builder/yajl.ml create mode 100644 builder/yajl.mli create mode 100644 builder/yajl_tests.ml diff --git a/.gitignore b/.gitignore index e502018..db8d0a2 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,7 @@ Makefile.in /builder/index-parse.h /builder/index-scan.c /builder/libguestfs.conf +/builder/oUnit-* /builder/*.qcow2 /builder/stamp-virt-builder.pod /builder/stamp-virt-index-validate.pod @@ -70,6 +71,7 @@ Makefile.in /builder/virt-index-validate /builder/virt-index-validate.1 /builder/*.xz +/builder/yajl_tests /cat/stamp-virt-*.pod /cat/virt-cat /cat/virt-cat.1 diff --git a/builder/Makefile.am b/builder/Makefile.am index f48efb0..366b8db 100644 --- a/builder/Makefile.am +++ b/builder/Makefile.am @@ -32,7 +32,8 @@ EXTRA_DIST = \ test-virt-index-validate-good-2 \ test-virt-index-validate-good-3 \ virt-builder.pod \ - virt-index-validate.pod + virt-index-validate.pod \ + yajl_tests.ml CLEANFILES = *~ *.annot *.cmi *.cmo *.cmx *.cmxa *.o virt-builder @@ -48,7 +49,8 @@ SOURCES_MLI = \ pxzcat.mli \ setlocale.mli \ sigchecker.mli \ - sources.mli + sources.mli \ + yajl.mli SOURCES_ML = \ utils.ml \ @@ -57,6 +59,7 @@ SOURCES_ML = \ checksums.ml \ index.ml \ ini_reader.ml \ + yajl.ml \ paths.ml \ languages.ml \ cache.ml \ @@ -81,7 +84,8 @@ SOURCES_C = \ index-parse.c \ index-parser-c.c \ pxzcat-c.c \ - setlocale-c.c + setlocale-c.c \ + yajl-c.c man_MANS = noinst_DATA = @@ -106,7 +110,8 @@ virt_builder_CFLAGS = \ -Wno-unused-macros \ $(LIBLZMA_CFLAGS) \ $(LIBTINFO_CFLAGS) \ - $(LIBXML2_CFLAGS) + $(LIBXML2_CFLAGS) \ + $(YAJL_CFLAGS) BOBJECTS = \ $(top_builddir)/mllib/libdir.cmo \ @@ -145,9 +150,13 @@ OCAMLPACKAGES = \ -I $(top_builddir)/ocaml \ -I $(top_builddir)/mllib \ -I $(top_builddir)/customize +OCAMLPACKAGES_TESTS = if HAVE_OCAML_PKG_GETTEXT OCAMLPACKAGES += -package gettext-stub endif +if HAVE_OCAML_PKG_OUNIT +OCAMLPACKAGES_TESTS += -package oUnit +endif OCAMLCLIBS = \ -pthread -lpthread \ @@ -156,6 +165,7 @@ OCAMLCLIBS = \ $(LIBCRYPT_LIBS) \ $(LIBLZMA_LIBS) \ $(LIBXML2_LIBS) \ + $(YAJL_LIBS) \ $(LIBINTL) \ -lgnu @@ -232,13 +242,40 @@ fedora.qcow2.xz: fedora.qcow2 xz --best -c $< > $@-t mv $@-t $@ +yajl_tests_SOURCES = yajl-c.c +yajl_tests_CPPFLAGS = $(virt_builder_CPPFLAGS) +yajl_tests_BOBJECTS = \ + yajl.cmo \ + yajl_tests.cmo +yajl_tests_XOBJECTS = $(yajl_tests_BOBJECTS:.cmo=.cmx) + +# Can't call the following as <test>_OBJECTS because automake gets confused. +if HAVE_OCAMLOPT +yajl_tests_THEOBJECTS = $(yajl_tests_XOBJECTS) +yajl_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS) +else +yajl_tests_THEOBJECTS = $(yajl_tests_BOBJECTS) +yajl_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS) +endif + +yajl_tests_DEPENDENCIES = $(yajl_tests_THEOBJECTS) $(top_srcdir)/ocaml-link.sh +yajl_tests_LINK = \ + $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \ + $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) $(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \ + $(yajl_tests_THEOBJECTS) -o $@ + TESTS = \ test-virt-builder-list.sh \ test-virt-index-validate.sh +check_PROGRAMS = if ENABLE_APPLIANCE TESTS += test-virt-builder.sh endif ENABLE_APPLIANCE +if HAVE_OCAML_PKG_OUNIT +check_PROGRAMS += yajl_tests +TESTS += yajl_tests +endif check-valgrind: $(MAKE) VG="$(top_builddir)/run @VG@" check diff --git a/builder/yajl-c.c b/builder/yajl-c.c new file mode 100644 index 0000000..6a96d59 --- /dev/null +++ b/builder/yajl-c.c @@ -0,0 +1,143 @@ +/* virt-builder + * Copyright (C) 2015 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <config.h> + +#include <caml/alloc.h> +#include <caml/fail.h> +#include <caml/memory.h> +#include <caml/mlvalues.h> + +#if HAVE_YAJL +#include <yajl/yajl_tree.h> +#endif + +#include <stdio.h> +#include <string.h> + +#define Val_none (Val_int (0)) + +extern value virt_builder_yajl_is_available (value unit); +extern value virt_builder_yajl_tree_parse (value stringv); + +#if HAVE_YAJL +static value +convert_yajl_value (yajl_val val, int level) +{ + CAMLparam0 (); + CAMLlocal4 (rv, lv, v, sv); + + if (level > 20) + caml_invalid_argument ("too many levels of object/array nesting"); + + if (YAJL_IS_OBJECT (val)) { + size_t len = YAJL_GET_OBJECT(val)->len; + size_t i; + rv = caml_alloc (1, 3); + lv = caml_alloc_tuple (len); + for (i = 0; i < len; ++i) { + v = caml_alloc_tuple (2); + sv = caml_copy_string (YAJL_GET_OBJECT(val)->keys[i]); + Store_field (v, 0, sv); + sv = convert_yajl_value (YAJL_GET_OBJECT(val)->values[i], level + 1); + Store_field (v, 1, sv); + Store_field (lv, i, v); + } + Store_field (rv, 0, lv); + } else if (YAJL_IS_ARRAY (val)) { + size_t len = YAJL_GET_ARRAY(val)->len; + size_t i; + rv = caml_alloc (1, 4); + lv = caml_alloc_tuple (len); + for (i = 0; i < len; ++i) { + v = convert_yajl_value (YAJL_GET_ARRAY(val)->values[i], level + 1); + Store_field (lv, i, v); + } + Store_field (rv, 0, lv); + } else if (YAJL_IS_STRING (val)) { + rv = caml_alloc (1, 0); + v = caml_copy_string (YAJL_GET_STRING(val)); + Store_field (rv, 0, v); + } else if (YAJL_IS_DOUBLE (val)) { + rv = caml_alloc (1, 2); + lv = caml_alloc_tuple (1); + Store_double_field (lv, 0, YAJL_GET_DOUBLE(val)); + Store_field (rv, 0, lv); + } else if (YAJL_IS_INTEGER (val)) { + rv = caml_alloc (1, 1); + v = caml_copy_int64 (YAJL_GET_INTEGER(val)); + Store_field (rv, 0, v); + } else if (YAJL_IS_TRUE (val)) { + rv = caml_alloc (1, 5); + Store_field (rv, 0, Val_true); + } else if (YAJL_IS_FALSE (val)) { + rv = caml_alloc (1, 5); + Store_field (rv, 0, Val_false); + } else + rv = Val_none; + + CAMLreturn (rv); +} + +value +virt_builder_yajl_is_available (value unit) +{ + /* NB: noalloc */ + return Val_true; +} + +value +virt_builder_yajl_tree_parse (value stringv) +{ + CAMLparam1 (stringv); + CAMLlocal1 (rv); + yajl_val tree; + char error_buf[256]; + + tree = yajl_tree_parse (String_val (stringv), error_buf, sizeof error_buf); + if (tree == NULL) { + char buf[256 + sizeof error_buf]; + if (strlen (error_buf) > 0) + snprintf (buf, sizeof buf, "JSON parse error: %s", error_buf); + else + snprintf (buf, sizeof buf, "unknown JSON parse error"); + caml_invalid_argument (buf); + } + + rv = convert_yajl_value (tree, 1); + yajl_tree_free (tree); + + CAMLreturn (rv); +} + +#else + +value +virt_builder_yajl_is_available (value unit) +{ + /* NB: noalloc */ + return Val_false; +} + +value +virt_builder_yajl_tree_parse (value stringv) +{ + caml_invalid_argument ("virt-builder was compiled without yajl support"); +} + +#endif diff --git a/builder/yajl.ml b/builder/yajl.ml new file mode 100644 index 0000000..f2d5c2b --- /dev/null +++ b/builder/yajl.ml @@ -0,0 +1,30 @@ +(* virt-builder + * Copyright (C) 2015 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +type yajl_val = +| Yajl_null +| Yajl_string of string +| Yajl_number of int64 +| Yajl_double of float +| Yajl_object of (string * yajl_val) array +| Yajl_array of yajl_val array +| Yajl_bool of bool + +external yajl_is_available : unit -> bool = "virt_builder_yajl_is_available" "noalloc" + +external yajl_tree_parse : string -> yajl_val = "virt_builder_yajl_tree_parse" diff --git a/builder/yajl.mli b/builder/yajl.mli new file mode 100644 index 0000000..aaa9389 --- /dev/null +++ b/builder/yajl.mli @@ -0,0 +1,33 @@ +(* virt-builder + * Copyright (C) 2015 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +type yajl_val = +| Yajl_null +| Yajl_string of string +| Yajl_number of int64 +| Yajl_double of float +| Yajl_object of (string * yajl_val) array +| Yajl_array of yajl_val array +| Yajl_bool of bool + +val yajl_is_available : unit -> bool +(** Is YAJL built in? If not, calling any of the other yajl_* + functions will result in an error. *) + +val yajl_tree_parse : string -> yajl_val +(** Parse the JSON string. *) diff --git a/builder/yajl_tests.ml b/builder/yajl_tests.ml new file mode 100644 index 0000000..344a8db --- /dev/null +++ b/builder/yajl_tests.ml @@ -0,0 +1,139 @@ +(* virt-builder + * Copyright (C) 2015 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +(* This file tests the Yajl module. *) + +open OUnit2 +open Yajl + +(* Utils. *) +let assert_equal_string = assert_equal ~printer:(fun x -> x) +let assert_equal_int = assert_equal ~printer:(fun x -> string_of_int x) +let assert_equal_int64 = assert_equal ~printer:(fun x -> Int64.to_string x) +let assert_equal_bool = assert_equal ~printer:(fun x -> string_of_bool x) + +let string_of_yajl_val_type = function + | Yajl_null -> "null" + | Yajl_string _ -> "string" + | Yajl_number _ -> "number" + | Yajl_double _ -> "float" + | Yajl_object _ -> "object" + | Yajl_array _ -> "array" + | Yajl_bool _ -> "bool" +let type_mismatch_string exp value = + Printf.sprintf "value is not %s but %s" exp (string_of_yajl_val_type value) + +let assert_raises_invalid_argument str = + (* Replace the Invalid_argument string with a fixed one, just to check + * whether the exception has been raised. + *) + let mock = "parse_error" in + let wrapped_tree_parse str = + try yajl_tree_parse str + with Invalid_argument _ -> raise (Invalid_argument mock) in + assert_raises (Invalid_argument mock) (fun () -> wrapped_tree_parse str) +let assert_raises_nested str = + let err = "too many levels of object/array nesting" in + assert_raises (Invalid_argument err) (fun () -> yajl_tree_parse str) + +let assert_is_object value = + assert_bool + (type_mismatch_string "object" value) + (match value with | Yajl_object _ -> true | _ -> false) +let assert_is_string exp = function + | Yajl_string s -> assert_equal_string exp s + | _ as v -> assert_failure (type_mismatch_string "string" v) +let assert_is_number exp = function + | Yajl_number n -> assert_equal_int64 exp n + | Yajl_double d -> assert_equal_int64 exp (Int64.of_float d) + | _ as v -> assert_failure (type_mismatch_string "number/double" v) +let assert_is_array value = + assert_bool + (type_mismatch_string "array" value) + (match value with | Yajl_array _ -> true | _ -> false) +let assert_is_bool exp = function + | Yajl_bool b -> assert_equal_bool exp b + | _ as v -> assert_failure (type_mismatch_string "bool" v) + +let get_object_list = function + | Yajl_object x -> x + | _ as v -> assert_failure (type_mismatch_string "object" v) +let get_array = function + | Yajl_array x -> x + | _ as v -> assert_failure (type_mismatch_string "array" v) + + +let test_tree_parse_invalid ctx = + assert_raises_invalid_argument ""; + assert_raises_invalid_argument "invalid"; + assert_raises_invalid_argument ":5"; + + (* Nested objects/arrays. *) + let str = "[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]" in + assert_raises_nested str; + let str = "{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":5}}}}}}}}}}}}}}}}}}}}}" in + assert_raises_nested str + +let test_tree_parse_basic ctx = + let value = yajl_tree_parse "{}" in + assert_is_object value; + + let value = yajl_tree_parse "\"foo\"" in + assert_is_string "foo" value; + + let value = yajl_tree_parse "[]" in + assert_is_array value + +let test_tree_parse_inspect ctx = + let value = yajl_tree_parse "{\"foo\":5}" in + let l = get_object_list value in + assert_equal_int 1 (Array.length l); + assert_equal_string "foo" (fst (l.(0))); + assert_is_number 5_L (snd (l.(0))); + + let value = yajl_tree_parse "[\"foo\", true]" in + let a = get_array value in + assert_equal_int 2 (Array.length a); + assert_is_string "foo" (a.(0)); + assert_is_bool true (a.(1)); + + let value = yajl_tree_parse "{\"foo\":[false, {}, 10], \"second\":2}" in + let l = get_object_list value in + assert_equal_int 2 (Array.length l); + assert_equal_string "foo" (fst (l.(0))); + let a = get_array (snd (l.(0))) in + assert_equal_int 3 (Array.length a); + assert_is_bool false (a.(0)); + assert_is_object (a.(1)); + assert_is_number 10_L (a.(2)); + assert_equal_string "second" (fst (l.(1))); + assert_is_number 2_L (snd (l.(1))) + +(* Suites declaration. *) +let suite = + "builder Yajl" >::: + [ + "tree_parse.invalid" >:: test_tree_parse_invalid; + "tree_parse.basic" >:: test_tree_parse_basic; + "tree_parse.inspect" >:: test_tree_parse_inspect; + ] + +let () = + if not (yajl_is_available ()) then + exit 77; + run_test_tt_main suite diff --git a/po/POTFILES b/po/POTFILES index 6a0a3fc..bb68183 100644 --- a/po/POTFILES +++ b/po/POTFILES @@ -6,6 +6,7 @@ builder/index-struct.c builder/index-validate.c builder/pxzcat-c.c builder/setlocale-c.c +builder/yajl-c.c cat/cat.c cat/filesystems.c cat/log.c diff --git a/po/POTFILES-ml b/po/POTFILES-ml index 7933c8e..ff08a53 100644 --- a/po/POTFILES-ml +++ b/po/POTFILES-ml @@ -14,6 +14,8 @@ builder/setlocale.ml builder/sigchecker.ml builder/sources.ml builder/utils.ml +builder/yajl.ml +builder/yajl_tests.ml customize/crypt.ml customize/customize_cmdline.ml customize/customize_main.ml -- 2.1.0
This way it can be used in automake conditionals. --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index 739cf06..2ce2456 100644 --- a/configure.ac +++ b/configure.ac @@ -1072,6 +1072,7 @@ PKG_CHECK_MODULES([YAJL], [yajl >= 2.0.4], [ AC_SUBST([YAJL_LIBS]) AC_DEFINE([HAVE_YAJL],[1],[Define to 1 if you have yajl.]) ],[AC_MSG_WARN([yajl not found, some features will be disabled])]) +AM_CONDITIONAL([HAVE_YAJL],[test "x$YAJL_LIBS" != "x"]) dnl Check for C++ (optional, we just use this to test the header works). AC_PROG_CXX -- 2.1.0
Add a new "simplestreams" repository type, and a simple parser for fetching and reading the JSON indexes of the Simple Streams v1.0 format. Read only datatype=image-downloads contents, and only the latest versions of each content available as disk image (disk.img or disk1.img). Add a simple test, using the "released" images from the CirrOS project. --- .gitignore | 1 + builder/Makefile.am | 6 + builder/builder.ml | 2 + builder/simplestreams_parser.ml | 204 ++++++++++ builder/simplestreams_parser.mli | 19 + builder/sources.ml | 9 + builder/sources.mli | 1 + builder/test-simplestreams/streams/v1/index.json | 18 + .../v1/net.cirros-cloud:released:download.json | 429 +++++++++++++++++++++ .../virt-builder/repos.d/cirros.conf.in | 3 + builder/test-virt-builder-list-simplestreams.sh | 108 ++++++ builder/virt-builder.pod | 7 + configure.ac | 1 + po/POTFILES-ml | 1 + 14 files changed, 809 insertions(+) create mode 100644 builder/simplestreams_parser.ml create mode 100644 builder/simplestreams_parser.mli create mode 100644 builder/test-simplestreams/streams/v1/index.json create mode 100644 builder/test-simplestreams/streams/v1/net.cirros-cloud:released:download.json create mode 100644 builder/test-simplestreams/virt-builder/repos.d/cirros.conf.in create mode 100755 builder/test-virt-builder-list-simplestreams.sh diff --git a/.gitignore b/.gitignore index db8d0a2..a430f6e 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,7 @@ Makefile.in /builder/stamp-virt-builder.pod /builder/stamp-virt-index-validate.pod /builder/test-config/virt-builder/repos.d/test-index.conf +/builder/test-simplestreams/virt-builder/repos.d/cirros.conf /builder/test-website/virt-builder/repos.d/libguestfs.conf /builder/virt-builder /builder/virt-builder.1 diff --git a/builder/Makefile.am b/builder/Makefile.am index 366b8db..e7cc444 100644 --- a/builder/Makefile.am +++ b/builder/Makefile.am @@ -49,6 +49,7 @@ SOURCES_MLI = \ pxzcat.mli \ setlocale.mli \ sigchecker.mli \ + simplestreams_parser.mli \ sources.mli \ yajl.mli @@ -67,6 +68,7 @@ SOURCES_ML = \ downloader.ml \ sigchecker.ml \ index_parser.ml \ + simplestreams_parser.ml \ list_entries.ml \ cmdline.ml \ builder.ml @@ -269,6 +271,10 @@ TESTS = \ test-virt-index-validate.sh check_PROGRAMS = +if HAVE_YAJL +TESTS += test-virt-builder-list-simplestreams.sh +endif + if ENABLE_APPLIANCE TESTS += test-virt-builder.sh endif ENABLE_APPLIANCE diff --git a/builder/builder.ml b/builder/builder.ml index dcfd437..7d347b3 100644 --- a/builder/builder.ml +++ b/builder/builder.ml @@ -182,6 +182,8 @@ let main () = match source.Sources.format with | Sources.FormatNative -> Index_parser.get_index ~downloader ~sigchecker source + | Sources.FormatSimpleStreams -> + Simplestreams_parser.get_index ~downloader ~sigchecker source ) sources ) in let index = remove_duplicates index in diff --git a/builder/simplestreams_parser.ml b/builder/simplestreams_parser.ml new file mode 100644 index 0000000..13e0b5d --- /dev/null +++ b/builder/simplestreams_parser.ml @@ -0,0 +1,204 @@ +(* virt-builder + * Copyright (C) 2015 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Common_gettext.Gettext +open Common_utils + +open Yajl +open Utils + +open Printf + +let ensure_trailing_slash str = + if String.length str > 0 && str.[String.length str - 1] <> '/' then str ^ "/" + else str + +let object_find_optional key = function + | Yajl_object o -> + (match List.filter (fun (k, _) -> k = key) (Array.to_list o) with + | [(k, v)] -> Some v + | [] -> None + | _ -> error (f_"more than value for the key '%s'") key) + | _ -> error (f_"the value of the key '%s' is not an object") key + +let object_find key yv = + match object_find_optional key yv with + | None -> error (f_"missing value for the key '%s'") key + | Some v -> v + +let object_get_string key yv = + match object_find key yv with + | Yajl_string s -> s + | _ -> error (f_"the value for the key '%s' is not a string") key + +let object_find_object key yv = + match object_find key yv with + | Yajl_object _ as o -> o + | _ -> error (f_"the value for the key '%s' is not an object") key + +let object_find_objects fn = function + | Yajl_object o -> filter_map fn (Array.to_list o) + | _ -> error (f_"the value is not an object") + +let object_get_object key yv = + match object_find_object key yv with + | Yajl_object o -> o + | _ -> assert false (* object_find_object already errors out. *) + +let object_get_number key yv = + match object_find key yv with + | Yajl_number n -> n + | Yajl_double d -> Int64.of_float d + | _ -> error (f_"the value for the key '%s' is not an integer") key + +let objects_get_string key yvs = + let rec loop = function + | [] -> None + | x :: xs -> + (match object_find_optional key x with + | Some (Yajl_string s) -> Some s + | Some _ -> error (f_"the value for key '%s' is not a string as expected") key + | None -> loop xs + ) + in + match loop yvs with + | Some s -> s + | None -> error (f_"the key '%s' was not found in a list of objects") key + +let get_index ~downloader ~sigchecker + { Sources.uri = uri; proxy = proxy } = + + let uri = ensure_trailing_slash uri in + + let download_and_parse uri = + let tmpfile, delete_tmpfile = Downloader.download downloader ~proxy uri in + if delete_tmpfile then + unlink_on_exit tmpfile; + let file = + if Sigchecker.verifying_signatures sigchecker then ( + let tmpunsigned = + Sigchecker.verify_and_remove_signature sigchecker tmpfile in + match tmpunsigned with + | None -> assert false (* only when not verifying signatures *) + | Some f -> f + ) else + tmpfile in + yajl_tree_parse (read_whole_file file) in + + let downloads = + let uri_index = + if Sigchecker.verifying_signatures sigchecker then + uri ^ "streams/v1/index.sjson" + else + uri ^ "streams/v1/index.json" in + let tree = download_and_parse uri_index in + + let format = object_get_string "format" tree in + if format <> "index:1.0" then + error (f_"%s is not a Simple Streams (index) v1.0 JSON file (format: %s)") + uri format; + + let index = Array.to_list (object_get_object "index" tree) in + filter_map ( + fun (_, desc) -> + let format = object_get_string "format" desc in + let datatype = object_get_string "datatype" desc in + match format, datatype with + | "products:1.0", "image-downloads" -> + Some (object_get_string "path" desc) + | _ -> None + ) index in + + let scan_product_list path = + let tree = download_and_parse (uri ^ path) in + + let format = object_get_string "format" tree in + if format <> "products:1.0" then + error (f_"%s is not a Simple Streams (products) v1.0 JSON file (format: %s)") + uri format; + + let products_node = object_get_object "products" tree in + + let products = Array.to_list products_node in + filter_map ( + fun (prod, prod_desc) -> + let arch = object_get_string "arch" prod_desc in + let prods = Array.to_list (object_get_object "versions" prod_desc) in + let prods = filter_map ( + fun (rel, rel_desc) -> + let pubname = objects_get_string "pubname" [rel_desc; prod_desc] in + let items = object_find_object "items" rel_desc in + let disk_items = object_find_objects ( + function + | (("disk.img"|"disk1.img"), v) -> Some v + | _ -> None + ) items in + (match disk_items with + | [] -> None + | disk_item :: _ -> + let printable_name = Some pubname in + let file_uri = uri ^ (object_get_string "path" disk_item) in + let checksums = + let checksums = object_find_objects ( + function + | ("sha256", Yajl_string c) -> Some (Checksums.SHA256 c) + | ("sha512", Yajl_string c) -> Some (Checksums.SHA512 c) + | _ -> None + ) disk_item in + match checksums with + | [] -> None + | x -> Some x in + let revision = Rev_string rel in + let size = object_get_number "size" disk_item in + let aliases = Some [pubname;] in + + let entry = { Index.printable_name = printable_name; + osinfo = None; + file_uri = file_uri; + arch = arch; + signature_uri = None; + checksums = checksums; + revision = revision; + format = None; + size = size; + compressed_size = None; + expand = None; + lvexpand = None; + notes = []; + hidden = false; + aliases = aliases; + sigchecker = sigchecker; + proxy = proxy; } in + Some (rel, (prod, entry)) + ) + ) prods in + (* Select the disk image with the bigger version (i.e. usually + * the most recent one. *) + let reverse_revision_compare (rev1, _) (rev2, _) = compare rev2 rev1 in + let prods = List.sort reverse_revision_compare prods in + match prods with + | [] -> None + | (_, entry) :: _ -> Some entry + ) products in + + let entries = List.flatten (List.map scan_product_list downloads) in + if verbose () then ( + printf "simplestreams tree (%s) after parsing:\n" uri; + List.iter (Index.print_entry Pervasives.stdout) entries + ); + entries diff --git a/builder/simplestreams_parser.mli b/builder/simplestreams_parser.mli new file mode 100644 index 0000000..a4b91ba --- /dev/null +++ b/builder/simplestreams_parser.mli @@ -0,0 +1,19 @@ +(* virt-builder + * Copyright (C) 2015 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> Sources.source -> Index.index diff --git a/builder/sources.ml b/builder/sources.ml index b21e8fc..149db6f 100644 --- a/builder/sources.ml +++ b/builder/sources.ml @@ -31,6 +31,7 @@ type source = { } and source_format = | FormatNative +| FormatSimpleStreams module StringSet = Set.Make (String) @@ -82,6 +83,14 @@ let parse_conf file = try (match (List.assoc ("format", None) fields) with | "native" | "" -> FormatNative + | "simplestreams" as fmt -> + if not (Yajl.yajl_is_available ()) then ( + if verbose () then ( + eprintf (f_"%s: repository type '%s' not supported (missing YAJL support), skipping it\n") prog fmt; + ); + invalid_arg fmt + ) else + FormatSimpleStreams | fmt -> if verbose () then ( eprintf (f_"%s: unknown repository type '%s' in %s, skipping it\n") prog fmt file; diff --git a/builder/sources.mli b/builder/sources.mli index e861310..e621a9f 100644 --- a/builder/sources.mli +++ b/builder/sources.mli @@ -25,5 +25,6 @@ type source = { } and source_format = | FormatNative +| FormatSimpleStreams val read_sources : unit -> source list diff --git a/builder/test-simplestreams/streams/v1/index.json b/builder/test-simplestreams/streams/v1/index.json new file mode 100644 index 0000000..3180af7 --- /dev/null +++ b/builder/test-simplestreams/streams/v1/index.json @@ -0,0 +1,18 @@ +{ + "format": "index:1.0", + "updated": "Fri, 08 May 2015 13:20:51 +0000", + "index": { + "net.cirros-cloud:released:download": { + "products": [ + "net.cirros-cloud:standard:0.3:i386", + "net.cirros-cloud:standard:0.3:x86_64", + "net.cirros-cloud:standard:0.3:powerpc", + "net.cirros-cloud:standard:0.3:arm" + ], + "datatype": "image-downloads", + "format": "products:1.0", + "updated": "Fri, 08 May 2015 13:20:51 +0000", + "path": "streams/v1/net.cirros-cloud:released:download.json" + } + } +} diff --git a/builder/test-simplestreams/streams/v1/net.cirros-cloud:released:download.json b/builder/test-simplestreams/streams/v1/net.cirros-cloud:released:download.json new file mode 100644 index 0000000..0fb57b4 --- /dev/null +++ b/builder/test-simplestreams/streams/v1/net.cirros-cloud:released:download.json @@ -0,0 +1,429 @@ +{ + "products": { + "net.cirros-cloud:standard:0.3:i386": { + "arch": "i386", + "versions": { + "20140908": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "785c524f55bf2493dcddeedba25172f1", + "sha256": "bc20b950cc20e66e0b915748ad6b1ea2360ec03f08ffe6c68dc1b4a82331ab66", + "size": 3204602, + "path": "0.3.3/cirros-0.3.3-i386-lxc.tar.gz" + }, + "disk.img": { + "ftype": "disk.img", + "md5": "283c77db6fc79b2d47c585ec241e7edc", + "sha256": "837cc3c79de14de62b8691c08a6913b7c0c62427870c7c1921ac43b9580748a9", + "size": 12268032, + "path": "0.3.3/cirros-0.3.3-i386-disk.img" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "861ef94d90ea220ae72b84d258ff5620", + "sha256": "5c7e685d42f0f1c5c70398e8b7388fc21b84dcbb4dc4a37c1c1645fe5c5ed3a4", + "size": 8231317, + "path": "0.3.3/cirros-0.3.3-i386-uec.tar.gz" + } + }, + "version": "0.3.3", + "pubname": "cirros-0.3.3-i386" + }, + "20150422": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "ffe02ef2b49cb34881f96e2b4c69383c", + "sha256": "5760cae9b9701dc09f495c3787f1da271800dcbc040eb793ddf4eb636689148a", + "size": 3210378, + "path": "0.3.4/cirros-0.3.4-i386-lxc.tar.gz" + }, + "disk.img": { + "ftype": "disk.img", + "md5": "79b4436412283bb63c2cba4ac796bcd9", + "sha256": "48c279ff502ea4c5e3e09d58c9e3a2ab615452dae2ed8fe347f68c136c6a5a0b", + "size": 12506112, + "path": "0.3.4/cirros-0.3.4-i386-disk.img" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "c1630b33986cc09df2e0cbca7f5c5346", + "sha256": "f1cad519da6632b1d0ebad2f892534b732b5f9dd54d27f532d2963a0748f246e", + "size": 8241360, + "path": "0.3.4/cirros-0.3.4-i386-uec.tar.gz" + } + }, + "version": "0.3.4", + "pubname": "cirros-0.3.4-i386" + }, + "20140317": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "55023b89a6121b30603859bb80f5c8d3", + "sha256": "136f8dfff85b8d509a0d0a0264aea79ae94d9b7c3ea5745fbd478ae600adeb20", + "size": 3201206, + "path": "0.3.2/cirros-0.3.2-i386-lxc.tar.gz" + }, + "disk.img": { + "ftype": "disk.img", + "md5": "b0a256d40e078d66576f7ae91c296e2b", + "sha256": "f0803c2d179c8a02d029239d35fc3e752cc81ad3436ea52b757e11685ca7c074", + "size": 12336128, + "path": "0.3.2/cirros-0.3.2-i386-disk.img" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "dcbcc559fc3bc468d9149bd7191aaa0c", + "sha256": "74ba8b67e0dce67f16a6bc5942bee90cbe31381f932aac1ba51bbc774aad5046", + "size": 8221995, + "path": "0.3.2/cirros-0.3.2-i386-uec.tar.gz" + } + }, + "version": "0.3.2", + "pubname": "cirros-0.3.2-i386" + }, + "20111020": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "e760bf470841f57c2c1bb426d407d169", + "sha256": "cac7887628527604b65a89e8caa34096d51c2dc1acfe405c15db2fc58495142a", + "size": 1845928, + "path": "0.3.0/cirros-0.3.0-i386-lxc.tar.gz" + }, + "disk.img": { + "ftype": "disk.img", + "md5": "90169ba6f09b5906a7f0755bd00bf2c3", + "sha256": "3309675d0d409128b1c2651d576bc8092ca9ab93e15f3d3aa458f40947569b61", + "size": 9159168, + "path": "0.3.0/cirros-0.3.0-i386-disk.img" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "115ca6afa47089dc083c0dc9f9b7ff03", + "sha256": "b57e0acb32852f89734ff11a511ae0897e8cecb41882d03551649289b6854a1b", + "size": 6596586, + "path": "0.3.0/cirros-0.3.0-i386-uec.tar.gz" + } + }, + "version": "0.3.0", + "pubname": "cirros-0.3.0-i386" + }, + "20130207": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "fc404d9fc9dc5f0eb33ebaa03920a046", + "sha256": "70a8c9c175589f7ac7054c6151cf2bb7eb9e210cefbe310446df2fb1a436b504", + "size": 3191593, + "path": "0.3.1/cirros-0.3.1-i386-lxc.tar.gz" + }, + "disk.img": { + "ftype": "disk.img", + "md5": "6ba617eafc992e33e7c141c679225e53", + "sha256": "b8aa1ce5d11939eaa01205fc31348532a31b82790921d45ceb397fbe76492787", + "size": 12251136, + "path": "0.3.1/cirros-0.3.1-i386-disk.img" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "52845de5142e58faf211e135d2b45721", + "sha256": "88dda2e505b862a95d0e0044013addcaa3200e602150c9d73e32c2e29345d6f3", + "size": 8197543, + "path": "0.3.1/cirros-0.3.1-i386-uec.tar.gz" + } + }, + "version": "0.3.1", + "pubname": "cirros-0.3.1-i386" + } + }, + "stream": "released" + }, + "net.cirros-cloud:standard:0.3:x86_64": { + "arch": "x86_64", + "versions": { + "20140908": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "600363faf89daa6adea87e2dd77263b4", + "sha256": "66255c2f9404484996e646bb825fd2f1fd9d66583fecd3e9e0789376cf055fd4", + "size": 3544884, + "path": "0.3.3/cirros-0.3.3-x86_64-lxc.tar.gz" + }, + "disk.img": { + "ftype": "disk.img", + "md5": "133eae9fb1c98f45894a4e60d8736619", + "sha256": "f11286e2bd317ee1a1d0469a6b182b33bda4af6f35ba224ca49d75752c81e20a", + "size": 13200896, + "path": "0.3.3/cirros-0.3.3-x86_64-disk.img" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "4f3371e52d60e5d47a6fbdd7d6a33973", + "sha256": "6bd27854862c30e25f0312defb9e86f9176b503751f040e323ab83322aa07dae", + "size": 8668951, + "path": "0.3.3/cirros-0.3.3-x86_64-uec.tar.gz" + } + }, + "version": "0.3.3", + "pubname": "cirros-0.3.3-x86_64" + }, + "20150422": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "6554ba2d15401c8caa90212ace985593", + "sha256": "e8172c603fad47f4c95e67bd2751c2977e07164c9ebdfcf3f9c1d7ff598ed217", + "size": 3555536, + "path": "0.3.4/cirros-0.3.4-x86_64-lxc.tar.gz" + }, + "disk.img": { + "ftype": "disk.img", + "md5": "ee1eca47dc88f4879d8a229cc70a07c6", + "sha256": "34987d0d5702f8813f3ff9efe90e9e39e6926ec78658763580a79face67f3394", + "size": 13287936, + "path": "0.3.4/cirros-0.3.4-x86_64-disk.img" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "067d511efcc4acd034f7d64b716273bf", + "sha256": "95e77c7deaf0f515f959ffe329918d5dd23e417503d1d45e926a888853c90710", + "size": 8683894, + "path": "0.3.4/cirros-0.3.4-x86_64-uec.tar.gz" + } + }, + "version": "0.3.4", + "pubname": "cirros-0.3.4-x86_64" + }, + "20140317": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "174a08e827f22b20ebd774f90f1c9181", + "sha256": "03eff2a3603164fa02c00ca76fdcbd611b2ae8a6a80406753ec35eb275648d2e", + "size": 3538286, + "path": "0.3.2/cirros-0.3.2-x86_64-lxc.tar.gz" + }, + "disk.img": { + "ftype": "disk.img", + "md5": "64d7c1cd2b6f60c92c14662941cb7913", + "sha256": "a2ca56aeded5a5bcaa6104fb14ec07b1ceb65222e2890bef8a89b8d2da729ad5", + "size": 13167616, + "path": "0.3.2/cirros-0.3.2-x86_64-disk.img" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "b369559e3ffa9208b4eb2224c949f579", + "sha256": "f462729fc2f071081dbc55932e07437e265263ef2e688306c353a1f152162b03", + "size": 8655821, + "path": "0.3.2/cirros-0.3.2-x86_64-uec.tar.gz" + } + }, + "version": "0.3.2", + "pubname": "cirros-0.3.2-x86_64" + }, + "20111020": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "1a480e5150db4d93be2aa3c9ced94fa1", + "sha256": "0e78796a30641dd7184f752c302a87d66f1eba9985c876911e4f26b4d8ba4a88", + "size": 2115217, + "path": "0.3.0/cirros-0.3.0-x86_64-lxc.tar.gz" + }, + "disk.img": { + "ftype": "disk.img", + "md5": "50bdc35edb03a38d91b1b071afb20a3c", + "sha256": "648782e9287288630250d07531fed9944ecc3986764a6664f0bf6c050ec06afd", + "size": 9761280, + "path": "0.3.0/cirros-0.3.0-x86_64-disk.img" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "f56d3cffa47b7d209d2b6905628f07b9", + "sha256": "043a3e090a5d76d23758a3919fcaff93f77ce7b97594d9d10fc8d00e85f83191", + "size": 6957349, + "path": "0.3.0/cirros-0.3.0-x86_64-uec.tar.gz" + } + }, + "version": "0.3.0", + "pubname": "cirros-0.3.0-x86_64" + }, + "20130207": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "38252f1a49fec0ebedebc820854497e0", + "sha256": "a086fcf2758468972d45957dc78ec6317c06f356930dbbc6cad6a8d1855f135e", + "size": 3534564, + "path": "0.3.1/cirros-0.3.1-x86_64-lxc.tar.gz" + }, + "disk.img": { + "ftype": "disk.img", + "md5": "d972013792949d0d3ba628fbe8685bce", + "sha256": "e01302fb2d2b13ae65226a0300335172e4487bbe60bb1e5c8b0843a25f126d34", + "size": 13147648, + "path": "0.3.1/cirros-0.3.1-x86_64-disk.img" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "e1849016cb71a00808093b7bf986f36a", + "sha256": "51eb03b83123a68d4f866c7c15b195204e62db9e33475509a38b79b3122cde38", + "size": 8633554, + "path": "0.3.1/cirros-0.3.1-x86_64-uec.tar.gz" + } + }, + "version": "0.3.1", + "pubname": "cirros-0.3.1-x86_64" + } + }, + "stream": "released" + }, + "net.cirros-cloud:standard:0.3:powerpc": { + "arch": "powerpc", + "versions": { + "20150422": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "7e3a147b5665a1e14fe6d0ffef9ca410", + "sha256": "b1ab58ebb2a3f253d15cfa13f0cd0de883ce6e80f42ce2e3889e880423e95462", + "size": 3603000, + "path": "0.3.4/cirros-0.3.4-powerpc-lxc.tar.gz" + }, + "disk.img": { + "ftype": "disk.img", + "md5": "453b21916c47c6ff8c615f8a5f7b76d2", + "sha256": "5445d77104d19ccdc2dbc943a3d0735c0dadd1227466a3f3e109c19671065f18", + "size": 17145856, + "path": "0.3.4/cirros-0.3.4-powerpc-disk.img" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "2f8a48928a6b0b2c9afbcb7255ff2aec", + "sha256": "80444c3ac8277d87eed0963567206ea5b009ecf4fdbc71fe8c4df020b42e3215", + "size": 12032417, + "path": "0.3.4/cirros-0.3.4-powerpc-uec.tar.gz" + } + } + } + }, + "version": "0.3.4", + "stream": "released", + "pubname": "cirros-0.3.4-powerpc" + }, + "net.cirros-cloud:standard:0.3:arm": { + "arch": "arm", + "versions": { + "20140908": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "7f2fe450d9a0b1688f2a4df6748690f5", + "sha256": "c839db36d5c96651d1dbcfc8a553678b010c7bb49416ac3099282858003fa021", + "size": 3482025, + "path": "0.3.3/cirros-0.3.3-arm-lxc.tar.gz" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "0437419d2371771b7406fade2d6db7d1", + "sha256": "4a0987818f3a834d9907b93729beb79426e58be9d46b3c207241bc4b1ceec52f", + "size": 7351177, + "path": "0.3.3/cirros-0.3.3-arm-uec.tar.gz" + } + }, + "version": "0.3.3", + "pubname": "cirros-0.3.3-arm" + }, + "20150422": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "5fd856d158dc6787264a1b4236126aab", + "sha256": "06c780e8c7166e16154bd0d3a6d1b0031e219269e990c5b8864a82925285486c", + "size": 3489736, + "path": "0.3.4/cirros-0.3.4-arm-lxc.tar.gz" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "1a2aea370f9f2d95d837f6b84bef658d", + "sha256": "6dd3bad904671aee13317c187d933b3efb1ac39a0d81fd5ac9a1fc694e3cb7ff", + "size": 7357906, + "path": "0.3.4/cirros-0.3.4-arm-uec.tar.gz" + } + }, + "version": "0.3.4", + "pubname": "cirros-0.3.4-arm" + }, + "20140317": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "bfb2ecb203cf27f4786b43a558b1ffe8", + "sha256": "a916b3d268793977001e2f09638e4a3e3953cc51d0d53d0bfb8c14cfa7105e86", + "size": 3474403, + "path": "0.3.2/cirros-0.3.2-arm-lxc.tar.gz" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "171719bfd4b3ec613f5fac321e4216de", + "sha256": "7f74826638e153ec58230bae9eb21805f875ed12eab299140f978e45597cb5cd", + "size": 7340357, + "path": "0.3.2/cirros-0.3.2-arm-uec.tar.gz" + } + }, + "version": "0.3.2", + "pubname": "cirros-0.3.2-arm" + }, + "20111020": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "91add49e56cbe6b5004015a4d2f51dbc", + "sha256": "fcd3723c956a1c232730dc28513b466657cbe984232ba2fcc30a4e1f55aa91e9", + "size": 2043822, + "path": "0.3.0/cirros-0.3.0-arm-lxc.tar.gz" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "c31e05f7829ad45f9d9995c35d232769", + "sha256": "b871823406f818430f57744333b1bb17ce0047e551a316f316641f1bd70d9152", + "size": 5761642, + "path": "0.3.0/cirros-0.3.0-arm-uec.tar.gz" + } + }, + "version": "0.3.0", + "pubname": "cirros-0.3.0-arm" + }, + "20130207": { + "items": { + "lxc.tar.gz": { + "ftype": "lxc.tar.gz", + "md5": "7ddea367ecb7ecb91554e18bed7c71bd", + "sha256": "2060e59e642b3b2bdf6e34aba3ed15f468bc6f9a8417fc196d01d29b2075493e", + "size": 3466149, + "path": "0.3.1/cirros-0.3.1-arm-lxc.tar.gz" + }, + "uec.tar.gz": { + "ftype": "uec.tar.gz", + "md5": "d04e6f26aed123bba2c096581b269e7f", + "sha256": "09dcd3ea6f1d48b3519232973e4dc00fc5e73cbea974cda6b5f7cfa380c6b428", + "size": 7314471, + "path": "0.3.1/cirros-0.3.1-arm-uec.tar.gz" + } + }, + "version": "0.3.1", + "pubname": "cirros-0.3.1-arm" + } + }, + "stream": "released" + } + }, + "datatype": "image-downloads", + "format": "products:1.0", + "updated": "Fri, 08 May 2015 13:20:51 +0000", + "content_id": "net.cirros-cloud:released:download" +} diff --git a/builder/test-simplestreams/virt-builder/repos.d/cirros.conf.in b/builder/test-simplestreams/virt-builder/repos.d/cirros.conf.in new file mode 100644 index 0000000..a6f2765 --- /dev/null +++ b/builder/test-simplestreams/virt-builder/repos.d/cirros.conf.in @@ -0,0 +1,3 @@ +[test-cirros] +uri=file://@abs_top_builddir@/builder/test-simplestreams +format=simplestreams diff --git a/builder/test-virt-builder-list-simplestreams.sh b/builder/test-virt-builder-list-simplestreams.sh new file mode 100755 index 0000000..24cf44c --- /dev/null +++ b/builder/test-virt-builder-list-simplestreams.sh @@ -0,0 +1,108 @@ +#!/bin/bash - +# libguestfs virt-builder test script +# Copyright (C) 2015 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +export LANG=C +set -e + +abs_builddir=$(pwd) + +export XDG_CONFIG_HOME= +export XDG_CONFIG_DIRS="$abs_builddir/test-simplestreams" + +short_list=$($VG virt-builder --no-check-signature --no-cache --list) + +if [ "$short_list" != "net.cirros-cloud:standard:0.3:i386 i386 cirros-0.3.4-i386 +net.cirros-cloud:standard:0.3:x86_64 x86_64 cirros-0.3.4-x86_64 +net.cirros-cloud:standard:0.3:powerpc powerpc cirros-0.3.4-powerpc" ]; then + echo "$0: unexpected --list output:" + echo "$short_list" + exit 1 +fi + +long_list=$(virt-builder --no-check-signature --no-cache --list --long) + +if [ "$long_list" != "Source URI: file://$abs_builddir/test-simplestreams + +os-version: net.cirros-cloud:standard:0.3:i386 +Full name: cirros-0.3.4-i386 +Architecture: i386 +Minimum/default size: 11.9M +Aliases: cirros-0.3.4-i386 + +os-version: net.cirros-cloud:standard:0.3:x86_64 +Full name: cirros-0.3.4-x86_64 +Architecture: x86_64 +Minimum/default size: 12.7M +Aliases: cirros-0.3.4-x86_64 + +os-version: net.cirros-cloud:standard:0.3:powerpc +Full name: cirros-0.3.4-powerpc +Architecture: powerpc +Minimum/default size: 16.4M +Aliases: cirros-0.3.4-powerpc" ]; then + echo "$0: unexpected --list --long output:" + echo "$long_list" + exit 1 +fi + +json_list=$(virt-builder --no-check-signature --no-cache --list --list-format json) + +if [ "$json_list" != "{ + \"version\": 1, + \"sources\": [ + { + \"uri\": \"file://$abs_builddir/test-simplestreams\" + } + ], + \"templates\": [ + { + \"os-version\": \"net.cirros-cloud:standard:0.3:i386\", + \"full-name\": \"cirros-0.3.4-i386\", + \"arch\": \"i386\", + \"size\": 12506112, + \"aliases\": [ + \"cirros-0.3.4-i386\" + ], + \"hidden\": false + }, + { + \"os-version\": \"net.cirros-cloud:standard:0.3:x86_64\", + \"full-name\": \"cirros-0.3.4-x86_64\", + \"arch\": \"x86_64\", + \"size\": 13287936, + \"aliases\": [ + \"cirros-0.3.4-x86_64\" + ], + \"hidden\": false + }, + { + \"os-version\": \"net.cirros-cloud:standard:0.3:powerpc\", + \"full-name\": \"cirros-0.3.4-powerpc\", + \"arch\": \"powerpc\", + \"size\": 17145856, + \"aliases\": [ + \"cirros-0.3.4-powerpc\" + ], + \"hidden\": false + } + ] +}" ]; then + echo "$0: unexpected --list --format json output:" + echo "$json_list" + exit 1 +fi diff --git a/builder/virt-builder.pod b/builder/virt-builder.pod index 710f006..67e017f 100644 --- a/builder/virt-builder.pod +++ b/builder/virt-builder.pod @@ -1183,6 +1183,13 @@ The possible values are: The native format of the C<virt-builder> repository. See also L</Creating and signing the index file> below. +=item B<simplestreams> + +The URI represents the root of a Simple Streams v1.0 tree of metadata. + +For more information about Simple Streams, see also +L<https://launchpad.net/simplestreams>. + =back If not present, the assumed value is C<native>. diff --git a/configure.ac b/configure.ac index 2ce2456..e057660 100644 --- a/configure.ac +++ b/configure.ac @@ -1724,6 +1724,7 @@ AC_CONFIG_FILES([Makefile builder/Makefile builder/libguestfs.conf builder/test-config/virt-builder/repos.d/test-index.conf + builder/test-simplestreams/virt-builder/repos.d/cirros.conf builder/test-website/virt-builder/repos.d/libguestfs.conf builder/website/Makefile cat/Makefile diff --git a/po/POTFILES-ml b/po/POTFILES-ml index ff08a53..3f0038e 100644 --- a/po/POTFILES-ml +++ b/po/POTFILES-ml @@ -12,6 +12,7 @@ builder/paths.ml builder/pxzcat.ml builder/setlocale.ml builder/sigchecker.ml +builder/simplestreams_parser.ml builder/sources.ml builder/utils.ml builder/yajl.ml -- 2.1.0
Looks good, ACK series. Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones Read my programming and virtualization blog: http://rwmj.wordpress.com virt-df lists disk usage of guests without needing to install any software inside the virtual machine. Supports Linux and Windows. http://people.redhat.com/~rjones/virt-df/
participants (2)
-
Pino Toscano -
Richard W.M. Jones