Split lib/osinfo.c to provide an API for other pieces of code (namely mllib) to reuse it. The ISO-related processing is thus moved into a lib/osinfo-iso.c file. --- lib/Makefile.am | 2 + lib/osinfo-iso.c | 462 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/osinfo.c | 420 +------------------------------------------------- lib/osinfo.h | 27 ++++ 4 files changed, 493 insertions(+), 418 deletions(-) create mode 100644 lib/osinfo-iso.c create mode 100644 lib/osinfo.h diff --git a/lib/Makefile.am b/lib/Makefile.am index 063706f8f..dd5f9fb92 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -112,7 +112,9 @@ libguestfs_la_SOURCES = \ lpj.c \ match.c \ mountable.c \ + osinfo.h \ osinfo.c \ + osinfo-iso.c \ private-data.c \ proto.c \ qemu.c \ diff --git a/lib/osinfo-iso.c b/lib/osinfo-iso.c new file mode 100644 index 000000000..059d72def --- /dev/null +++ b/lib/osinfo-iso.c @@ -0,0 +1,462 @@ +/* libguestfs + * Copyright (C) 2012 Red Hat Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* Read libosinfo XML files to parse out just the + * os/media/iso/system-id and os/media/iso/volume-id fields, which we + * can then use to map install media to operating systems. + * + * Note some assumptions here: + * + * (1) We have to do some translation of the distro names and versions + * stored in the libosinfo files and the standard names returned by + * libguestfs. + * + * (2) Media detection is only part of the story. We may still need + * to inspect inside the image. + * + * (3) We only read the XML database files (at most) once per process, + * and keep them cached. They are only read at all if someone tries + * to inspect a CD/DVD/ISO. + * + * XXX Currently the database is not freed when the program exits / + * library is unloaded, although we should probably do that. + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <dirent.h> +#include <assert.h> +#include <sys/types.h> +#include <libintl.h> +#include <sys/stat.h> + +#include <libxml/parser.h> +#include <libxml/xpath.h> + +#include "ignore-value.h" +#include "glthread/lock.h" +#include "c-ctype.h" + +#include "guestfs.h" +#include "guestfs-internal.h" + +#include "osinfo.h" + +gl_lock_define_initialized (static, osinfo_db_lock); +static ssize_t osinfo_db_size = 0; /* 0 = unread, -1 = error, >= 1 = #records */ +static struct osinfo *osinfo_db = NULL; + +static void free_osinfo_db_entry (struct osinfo *); + +#define XMLSTREQ(a,b) (xmlStrEqual((a),(b)) == 1) + +static int read_osinfo_db_xml (guestfs_h *g, const char *pathname, void *data); + +/* Given one or more fields from the header of a CD/DVD/ISO, look up + * the media in the libosinfo database and return our best guess for + * the operating system. + * + * This returns: + * -1 => a fatal error ('error' has been called, caller must not ignore it) + * 0 => could not locate the OS + * 1 => matching OS found, the osinfo_ret pointer has been filled in + */ +int +guestfs_int_osinfo_map (guestfs_h *g, const struct guestfs_isoinfo *isoinfo, + const struct osinfo **osinfo_ret) +{ + size_t i; + + /* We only need to lock the database when reading it for the first time. */ + gl_lock_lock (osinfo_db_lock); + if (osinfo_db_size == 0) { + if (read_osinfo_db (g, read_osinfo_db_xml, NULL) == -1) { + /* Fatal error: free any database entries which have been read, and + * mark the database as having a permanent error. + */ + if (osinfo_db_size > 0) { + for (i = 0; i < (size_t) osinfo_db_size; ++i) + free_osinfo_db_entry (&osinfo_db[i]); + } + free (osinfo_db); + osinfo_db = NULL; + osinfo_db_size = -1; + gl_lock_unlock (osinfo_db_lock); + return -1; + } + } + gl_lock_unlock (osinfo_db_lock); + + if (osinfo_db_size <= 0) + return 0; + + /* Look in the database to see if we can find a match. */ + for (i = 0; i < (size_t) osinfo_db_size; ++i) { + if (osinfo_db[i].re_system_id) { + if (!isoinfo->iso_system_id || + !match (g, isoinfo->iso_system_id, osinfo_db[i].re_system_id)) + continue; + } + + if (osinfo_db[i].re_volume_id) { + if (!isoinfo->iso_volume_id || + !match (g, isoinfo->iso_volume_id, osinfo_db[i].re_volume_id)) + continue; + } + + if (osinfo_db[i].re_publisher_id) { + if (!isoinfo->iso_publisher_id || + !match (g, isoinfo->iso_publisher_id, osinfo_db[i].re_publisher_id)) + continue; + } + + if (osinfo_db[i].re_application_id) { + if (!isoinfo->iso_application_id || + !match (g, isoinfo->iso_application_id, osinfo_db[i].re_application_id)) + continue; + } + + debug (g, "osinfo: mapped disk to database entry %zu", i); + + if (osinfo_ret) + *osinfo_ret = &osinfo_db[i]; + return 1; + } + + debug (g, "osinfo: no mapping found"); + + return 0; +} + +static int read_iso_node (guestfs_h *g, xmlNodePtr iso_node, struct osinfo *osinfo); +static int read_media_node (guestfs_h *g, xmlXPathContextPtr xpathCtx, xmlNodePtr media_node, struct osinfo *osinfo); +static int read_os_node (guestfs_h *g, xmlXPathContextPtr xpathCtx, xmlNodePtr os_node, struct osinfo *osinfo); + +/* Read a single XML file from pathname (which is a full path). + * Only memory allocation failures are fatal errors here. + */ +static int +read_osinfo_db_xml (guestfs_h *g, const char *pathname, void *opaque) +{ + CLEANUP_XMLFREEDOC xmlDocPtr doc = NULL; + CLEANUP_XMLXPATHFREECONTEXT xmlXPathContextPtr xpathCtx = NULL; + CLEANUP_XMLXPATHFREEOBJECT xmlXPathObjectPtr xpathObj = NULL; + xmlNodeSetPtr nodes; + xmlNodePtr iso_node, media_node, os_node; + struct osinfo *osinfo; + size_t i; + + doc = xmlReadFile (pathname, NULL, XML_PARSE_NONET); + if (doc == NULL) { + debug (g, "osinfo: unable to parse XML file %s", pathname); + return 0; + } + + xpathCtx = xmlXPathNewContext (doc); + if (xpathCtx == NULL) { + error (g, _("osinfo: unable to create new XPath context")); + return -1; + } + + /* Get all <iso> nodes at any depth, then use the parent pointers in + * order to work back up the tree. + */ + xpathObj = xmlXPathEvalExpression (BAD_CAST "/libosinfo/os/media/iso", + xpathCtx); + if (xpathObj == NULL) { + error (g, _("osinfo: %s: unable to evaluate XPath expression"), + pathname); + return -1; + } + + nodes = xpathObj->nodesetval; + + if (nodes != NULL) { + for (i = 0; i < (size_t) nodes->nodeNr; ++i) { + iso_node = nodes->nodeTab[i]; + assert (iso_node != NULL); + assert (STREQ ((const char *) iso_node->name, "iso")); + assert (iso_node->type == XML_ELEMENT_NODE); + + media_node = iso_node->parent; + assert (media_node != NULL); + assert (STREQ ((const char *) media_node->name, "media")); + assert (media_node->type == XML_ELEMENT_NODE); + + os_node = media_node->parent; + assert (os_node != NULL); + assert (STREQ ((const char *) os_node->name, "os")); + assert (os_node->type == XML_ELEMENT_NODE); + + /* Allocate an osinfo record. */ + osinfo_db_size++; + osinfo_db = safe_realloc (g, osinfo_db, + sizeof (struct osinfo) * osinfo_db_size); + osinfo = &osinfo_db[osinfo_db_size-1]; + memset (osinfo, 0, sizeof *osinfo); + + /* Read XML fields into the new osinfo record. */ + if (read_iso_node (g, iso_node, osinfo) == -1 || + read_media_node (g, xpathCtx, media_node, osinfo) == -1 || + read_os_node (g, xpathCtx, os_node, osinfo) == -1) { + free_osinfo_db_entry (osinfo); + osinfo_db_size--; + return -1; + } + +#if 0 + debug (g, "osinfo: %s: %s%s%s%s=> arch %s live %s installer %s product %s type %d distro %d version %d.%d", + pathname, + osinfo->re_system_id ? "<system-id/> " : "", + osinfo->re_volume_id ? "<volume-id/> " : "", + osinfo->re_publisher_id ? "<publisher-id/> " : "", + osinfo->re_application_id ? "<application-id/> " : "", + osinfo->arch ? osinfo->arch : "(none)", + osinfo->is_live_disk ? "true" : "false", + osinfo->is_installer ? "true" : "false", + osinfo->product_name ? osinfo->product_name : "(none)", + (int) osinfo->type, (int) osinfo->distro, + osinfo->major_version, osinfo->minor_version); +#endif + } + } + + return 0; +} + +static int compile_re (guestfs_h *g, xmlNodePtr child, pcre **re); + +/* Read the regular expressions under the <iso> node. libosinfo + * itself uses the glib function 'g_regex_match_simple'. That appears + * to implement PCRE, however I have not checked in detail. + */ +static int +read_iso_node (guestfs_h *g, xmlNodePtr iso_node, struct osinfo *osinfo) +{ + xmlNodePtr child; + + for (child = iso_node->children; child; child = child->next) { + if (STREQ ((const char *) child->name, "system-id")) { + if (compile_re (g, child, &osinfo->re_system_id) == -1) + return -1; + } + else if (STREQ ((const char *) child->name, "volume-id")) { + if (compile_re (g, child, &osinfo->re_volume_id) == -1) + return -1; + } + else if (STREQ ((const char *) child->name, "publisher-id")) { + if (compile_re (g, child, &osinfo->re_publisher_id) == -1) + return -1; + } + else if (STREQ ((const char *) child->name, "application-id")) { + if (compile_re (g, child, &osinfo->re_application_id) == -1) + return -1; + } + } + + return 0; +} + +static int +compile_re (guestfs_h *g, xmlNodePtr node, pcre **re) +{ + const char *err; + int offset; + CLEANUP_FREE char *content = (char *) xmlNodeGetContent (node); + + if (content) { + *re = pcre_compile (content, 0, &err, &offset, NULL); + if (*re == NULL) + debug (g, "osinfo: could not parse regular expression '%s': %s (ignored)", + content, err); + } + + return 0; +} + +/* Read the attributes of the <media/> node. */ +static int +read_media_node (guestfs_h *g, xmlXPathContextPtr xpathCtx, + xmlNodePtr media_node, struct osinfo *osinfo) +{ + osinfo->arch = (char *) xmlGetProp (media_node, BAD_CAST "arch"); + + osinfo->is_live_disk = 0; /* If no 'live' attr, defaults to false. */ + { + CLEANUP_XMLFREE xmlChar *content = NULL; + content = xmlGetProp (media_node, BAD_CAST "live"); + if (content) + osinfo->is_live_disk = XMLSTREQ (content, BAD_CAST "true"); + } + + osinfo->is_installer = true; /* If no 'installer' attr, defaults to true. */ + { + CLEANUP_XMLFREE xmlChar *content = NULL; + content = xmlGetProp (media_node, BAD_CAST "installer"); + if (content) + osinfo->is_installer = XMLSTREQ (content, BAD_CAST "true"); + } + + return 0; +} + +static int parse_version (guestfs_h *g, xmlNodePtr node, struct osinfo *osinfo); +static int parse_family (guestfs_h *g, xmlNodePtr node, struct osinfo *osinfo); +static int parse_distro (guestfs_h *g, xmlNodePtr node, struct osinfo *osinfo); + +/* Read some fields under the <os/> node. */ +static int +read_os_node (guestfs_h *g, xmlXPathContextPtr xpathCtx, + xmlNodePtr os_node, struct osinfo *osinfo) +{ + xmlNodePtr child; + + for (child = os_node->children; child; child = child->next) { + if (STREQ ((const char *) child->name, "name")) + osinfo->product_name = (char *) xmlNodeGetContent (child); + else if (STREQ ((const char *) child->name, "version")) { + if (parse_version (g, child, osinfo) == -1) + return -1; + } + else if (STREQ ((const char *) child->name, "family")) { + if (parse_family (g, child, osinfo) == -1) + return -1; + } + else if (STREQ ((const char *) child->name, "distro")) { + if (parse_distro (g, child, osinfo) == -1) + return -1; + } + } + + return 0; +} + +static int +parse_version (guestfs_h *g, xmlNodePtr node, struct osinfo *osinfo) +{ + CLEANUP_FREE char *content = NULL; + + content = (char *) xmlNodeGetContent (node); + /* We parse either "X.Y" or "X" as version strings, so try to parse + * only if the first character is a digit. + */ + if (content && c_isdigit (content[0])) { + struct version version; + const int res = guestfs_int_version_from_x_y_or_x (g, &version, content); + if (res < 0) + return -1; + else if (res > 0) { + osinfo->major_version = version.v_major; + osinfo->minor_version = version.v_minor; + } + } + + return 0; +} + +static int +parse_family (guestfs_h *g, xmlNodePtr node, struct osinfo *osinfo) +{ + CLEANUP_FREE char *content = NULL; + + osinfo->type = OS_TYPE_UNKNOWN; + + content = (char *) xmlNodeGetContent (node); + if (content) { + if (STREQ (content, "linux")) + osinfo->type = OS_TYPE_LINUX; + else if (STRPREFIX (content, "win")) + osinfo->type = OS_TYPE_WINDOWS; + else if (STREQ (content, "freebsd")) + osinfo->type = OS_TYPE_FREEBSD; + else if (STREQ (content, "netbsd")) + osinfo->type = OS_TYPE_NETBSD; + else if (STREQ (content, "msdos")) + osinfo->type = OS_TYPE_DOS; + else if (STREQ (content, "openbsd")) + osinfo->type = OS_TYPE_OPENBSD; + else + debug (g, "osinfo: warning: unknown <family> '%s'", content); + } + + return 0; +} + +static int +parse_distro (guestfs_h *g, xmlNodePtr node, struct osinfo *osinfo) +{ + CLEANUP_FREE char *content = NULL; + + osinfo->distro = OS_DISTRO_UNKNOWN; + + content = (char *) xmlNodeGetContent (node); + if (content) { + if (STREQ (content, "altlinux")) + osinfo->distro = OS_DISTRO_ALTLINUX; + else if (STREQ (content, "centos")) + osinfo->distro = OS_DISTRO_CENTOS; + else if (STREQ (content, "debian")) + osinfo->distro = OS_DISTRO_DEBIAN; + else if (STREQ (content, "fedora")) + osinfo->distro = OS_DISTRO_FEDORA; + else if (STREQ (content, "freebsd")) + osinfo->distro = OS_DISTRO_FREEBSD; + else if (STREQ (content, "mageia")) + osinfo->distro = OS_DISTRO_MAGEIA; + else if (STREQ (content, "mandriva")) + osinfo->distro = OS_DISTRO_MANDRIVA; + else if (STREQ (content, "netbsd")) + osinfo->distro = OS_DISTRO_NETBSD; + else if (STREQ (content, "openbsd")) + osinfo->distro = OS_DISTRO_OPENBSD; + else if (STREQ (content, "opensuse")) + osinfo->distro = OS_DISTRO_OPENSUSE; + else if (STREQ (content, "rhel")) + osinfo->distro = OS_DISTRO_RHEL; + else if (STREQ (content, "sled") || STREQ (content, "sles")) + osinfo->distro = OS_DISTRO_SLES; + else if (STREQ (content, "ubuntu")) + osinfo->distro = OS_DISTRO_UBUNTU; + else if (STRPREFIX (content, "win")) + osinfo->distro = OS_DISTRO_WINDOWS; + else + debug (g, "osinfo: warning: unknown <distro> '%s'", content); + } + + return 0; +} + +static void +free_osinfo_db_entry (struct osinfo *osinfo) +{ + free (osinfo->product_name); + free (osinfo->arch); + + if (osinfo->re_system_id) + pcre_free (osinfo->re_system_id); + if (osinfo->re_volume_id) + pcre_free (osinfo->re_volume_id); + if (osinfo->re_publisher_id) + pcre_free (osinfo->re_publisher_id); + if (osinfo->re_application_id) + pcre_free (osinfo->re_application_id); +} diff --git a/lib/osinfo.c b/lib/osinfo.c index 3514585c7..5ccb554be 100644 --- a/lib/osinfo.c +++ b/lib/osinfo.c @@ -29,20 +29,6 @@ * safe(-ish) since the media identifiers always change for every * release of an OS. We can easily add support for this if it becomes * necessary. - * - * (3) We have to do some translation of the distro names and versions - * stored in the libosinfo files and the standard names returned by - * libguestfs. - * - * (4) Media detection is only part of the story. We may still need - * to inspect inside the image. - * - * (5) We only read the XML database files (at most) once per process, - * and keep them cached. They are only read at all if someone tries - * to inspect a CD/DVD/ISO. - * - * XXX Currently the database is not freed when the program exits / - * library is unloaded, although we should probably do that. */ #include <config.h> @@ -58,101 +44,14 @@ #include <libintl.h> #include <sys/stat.h> -#include <libxml/parser.h> -#include <libxml/xpath.h> - #include "ignore-value.h" -#include "glthread/lock.h" #include "c-ctype.h" #include "guestfs.h" #include "guestfs-internal.h" -gl_lock_define_initialized (static, osinfo_db_lock); -static ssize_t osinfo_db_size = 0; /* 0 = unread, -1 = error, >= 1 = #records */ -static struct osinfo *osinfo_db = NULL; - -#define XMLSTREQ(a,b) (xmlStrEqual((a),(b)) == 1) -typedef int (*read_osinfo_db_callback) (guestfs_h *g, const char *path, void *opaque); - -static int read_osinfo_db (guestfs_h *g, read_osinfo_db_callback callback, void *opaque); -static void free_osinfo_db_entry (struct osinfo *); - -/* Given one or more fields from the header of a CD/DVD/ISO, look up - * the media in the libosinfo database and return our best guess for - * the operating system. - * - * This returns: - * -1 => a fatal error ('error' has been called, caller must not ignore it) - * 0 => could not locate the OS - * 1 => matching OS found, the osinfo_ret pointer has been filled in - */ -int -guestfs_int_osinfo_map (guestfs_h *g, const struct guestfs_isoinfo *isoinfo, - const struct osinfo **osinfo_ret) -{ - size_t i; - - /* We only need to lock the database when reading it for the first time. */ - gl_lock_lock (osinfo_db_lock); - if (osinfo_db_size == 0) { - if (read_osinfo_db (g, read_osinfo_db_xml, NULL) == -1) { - /* Fatal error: free any database entries which have been read, and - * mark the database as having a permanent error. - */ - if (osinfo_db_size > 0) { - for (i = 0; i < (size_t) osinfo_db_size; ++i) - free_osinfo_db_entry (&osinfo_db[i]); - } - free (osinfo_db); - osinfo_db = NULL; - osinfo_db_size = -1; - gl_lock_unlock (osinfo_db_lock); - return -1; - } - } - gl_lock_unlock (osinfo_db_lock); +#include "osinfo.h" - if (osinfo_db_size <= 0) - return 0; - - /* Look in the database to see if we can find a match. */ - for (i = 0; i < (size_t) osinfo_db_size; ++i) { - if (osinfo_db[i].re_system_id) { - if (!isoinfo->iso_system_id || - !match (g, isoinfo->iso_system_id, osinfo_db[i].re_system_id)) - continue; - } - - if (osinfo_db[i].re_volume_id) { - if (!isoinfo->iso_volume_id || - !match (g, isoinfo->iso_volume_id, osinfo_db[i].re_volume_id)) - continue; - } - - if (osinfo_db[i].re_publisher_id) { - if (!isoinfo->iso_publisher_id || - !match (g, isoinfo->iso_publisher_id, osinfo_db[i].re_publisher_id)) - continue; - } - - if (osinfo_db[i].re_application_id) { - if (!isoinfo->iso_application_id || - !match (g, isoinfo->iso_application_id, osinfo_db[i].re_application_id)) - continue; - } - - debug (g, "osinfo: mapped disk to database entry %zu", i); - - if (osinfo_ret) - *osinfo_ret = &osinfo_db[i]; - return 1; - } - - debug (g, "osinfo: no mapping found"); - - return 0; -} /* Read the libosinfo XML database files. The lock is held while * this is called. @@ -168,12 +67,11 @@ guestfs_int_osinfo_map (guestfs_h *g, const struct guestfs_isoinfo *isoinfo, * Try to use the shared osinfo database layout (and location) first: * https://gitlab.com/libosinfo/libosinfo/blob/master/docs/database-layout.txt */ -static int read_osinfo_db_xml (guestfs_h *g, const char *pathname, void *data); static int read_osinfo_db_flat (guestfs_h *g, const char *directory, read_osinfo_db_callback callback, void *opaque); static int read_osinfo_db_three_levels (guestfs_h *g, const char *directory, read_osinfo_db_callback callback, void *opaque); static int read_osinfo_db_directory (guestfs_h *g, const char *directory, read_osinfo_db_callback callback, void *opaque); -static int +int read_osinfo_db (guestfs_h *g, read_osinfo_db_callback callback, void *opaque) { @@ -337,317 +235,3 @@ read_osinfo_db_directory (guestfs_h *g, const char *directory, return -1; } - -static int read_iso_node (guestfs_h *g, xmlNodePtr iso_node, struct osinfo *osinfo); -static int read_media_node (guestfs_h *g, xmlXPathContextPtr xpathCtx, xmlNodePtr media_node, struct osinfo *osinfo); -static int read_os_node (guestfs_h *g, xmlXPathContextPtr xpathCtx, xmlNodePtr os_node, struct osinfo *osinfo); - -/* Read a single XML file from pathname (which is a full path). - * Only memory allocation failures are fatal errors here. - */ -static int -read_osinfo_db_xml (guestfs_h *g, const char *pathname, void *opaque) -{ - CLEANUP_XMLFREEDOC xmlDocPtr doc = NULL; - CLEANUP_XMLXPATHFREECONTEXT xmlXPathContextPtr xpathCtx = NULL; - CLEANUP_XMLXPATHFREEOBJECT xmlXPathObjectPtr xpathObj = NULL; - xmlNodeSetPtr nodes; - xmlNodePtr iso_node, media_node, os_node; - struct osinfo *osinfo; - size_t i; - - doc = xmlReadFile (pathname, NULL, XML_PARSE_NONET); - if (doc == NULL) { - debug (g, "osinfo: unable to parse XML file %s", pathname); - return 0; - } - - xpathCtx = xmlXPathNewContext (doc); - if (xpathCtx == NULL) { - error (g, _("osinfo: unable to create new XPath context")); - return -1; - } - - /* Get all <iso> nodes at any depth, then use the parent pointers in - * order to work back up the tree. - */ - xpathObj = xmlXPathEvalExpression (BAD_CAST "/libosinfo/os/media/iso", - xpathCtx); - if (xpathObj == NULL) { - error (g, _("osinfo: %s: unable to evaluate XPath expression"), - pathname); - return -1; - } - - nodes = xpathObj->nodesetval; - - if (nodes != NULL) { - for (i = 0; i < (size_t) nodes->nodeNr; ++i) { - iso_node = nodes->nodeTab[i]; - assert (iso_node != NULL); - assert (STREQ ((const char *) iso_node->name, "iso")); - assert (iso_node->type == XML_ELEMENT_NODE); - - media_node = iso_node->parent; - assert (media_node != NULL); - assert (STREQ ((const char *) media_node->name, "media")); - assert (media_node->type == XML_ELEMENT_NODE); - - os_node = media_node->parent; - assert (os_node != NULL); - assert (STREQ ((const char *) os_node->name, "os")); - assert (os_node->type == XML_ELEMENT_NODE); - - /* Allocate an osinfo record. */ - osinfo_db_size++; - osinfo_db = safe_realloc (g, osinfo_db, - sizeof (struct osinfo) * osinfo_db_size); - osinfo = &osinfo_db[osinfo_db_size-1]; - memset (osinfo, 0, sizeof *osinfo); - - /* Read XML fields into the new osinfo record. */ - if (read_iso_node (g, iso_node, osinfo) == -1 || - read_media_node (g, xpathCtx, media_node, osinfo) == -1 || - read_os_node (g, xpathCtx, os_node, osinfo) == -1) { - free_osinfo_db_entry (osinfo); - osinfo_db_size--; - return -1; - } - -#if 0 - debug (g, "osinfo: %s: %s%s%s%s=> arch %s live %s installer %s product %s type %d distro %d version %d.%d", - pathname, - osinfo->re_system_id ? "<system-id/> " : "", - osinfo->re_volume_id ? "<volume-id/> " : "", - osinfo->re_publisher_id ? "<publisher-id/> " : "", - osinfo->re_application_id ? "<application-id/> " : "", - osinfo->arch ? osinfo->arch : "(none)", - osinfo->is_live_disk ? "true" : "false", - osinfo->is_installer ? "true" : "false", - osinfo->product_name ? osinfo->product_name : "(none)", - (int) osinfo->type, (int) osinfo->distro, - osinfo->major_version, osinfo->minor_version); -#endif - } - } - - return 0; -} - -static int compile_re (guestfs_h *g, xmlNodePtr child, pcre **re); - -/* Read the regular expressions under the <iso> node. libosinfo - * itself uses the glib function 'g_regex_match_simple'. That appears - * to implement PCRE, however I have not checked in detail. - */ -static int -read_iso_node (guestfs_h *g, xmlNodePtr iso_node, struct osinfo *osinfo) -{ - xmlNodePtr child; - - for (child = iso_node->children; child; child = child->next) { - if (STREQ ((const char *) child->name, "system-id")) { - if (compile_re (g, child, &osinfo->re_system_id) == -1) - return -1; - } - else if (STREQ ((const char *) child->name, "volume-id")) { - if (compile_re (g, child, &osinfo->re_volume_id) == -1) - return -1; - } - else if (STREQ ((const char *) child->name, "publisher-id")) { - if (compile_re (g, child, &osinfo->re_publisher_id) == -1) - return -1; - } - else if (STREQ ((const char *) child->name, "application-id")) { - if (compile_re (g, child, &osinfo->re_application_id) == -1) - return -1; - } - } - - return 0; -} - -static int -compile_re (guestfs_h *g, xmlNodePtr node, pcre **re) -{ - const char *err; - int offset; - CLEANUP_FREE char *content = (char *) xmlNodeGetContent (node); - - if (content) { - *re = pcre_compile (content, 0, &err, &offset, NULL); - if (*re == NULL) - debug (g, "osinfo: could not parse regular expression '%s': %s (ignored)", - content, err); - } - - return 0; -} - -/* Read the attributes of the <media/> node. */ -static int -read_media_node (guestfs_h *g, xmlXPathContextPtr xpathCtx, - xmlNodePtr media_node, struct osinfo *osinfo) -{ - osinfo->arch = (char *) xmlGetProp (media_node, BAD_CAST "arch"); - - osinfo->is_live_disk = 0; /* If no 'live' attr, defaults to false. */ - { - CLEANUP_XMLFREE xmlChar *content = NULL; - content = xmlGetProp (media_node, BAD_CAST "live"); - if (content) - osinfo->is_live_disk = XMLSTREQ (content, BAD_CAST "true"); - } - - osinfo->is_installer = true; /* If no 'installer' attr, defaults to true. */ - { - CLEANUP_XMLFREE xmlChar *content = NULL; - content = xmlGetProp (media_node, BAD_CAST "installer"); - if (content) - osinfo->is_installer = XMLSTREQ (content, BAD_CAST "true"); - } - - return 0; -} - -static int parse_version (guestfs_h *g, xmlNodePtr node, struct osinfo *osinfo); -static int parse_family (guestfs_h *g, xmlNodePtr node, struct osinfo *osinfo); -static int parse_distro (guestfs_h *g, xmlNodePtr node, struct osinfo *osinfo); - -/* Read some fields under the <os/> node. */ -static int -read_os_node (guestfs_h *g, xmlXPathContextPtr xpathCtx, - xmlNodePtr os_node, struct osinfo *osinfo) -{ - xmlNodePtr child; - - for (child = os_node->children; child; child = child->next) { - if (STREQ ((const char *) child->name, "name")) - osinfo->product_name = (char *) xmlNodeGetContent (child); - else if (STREQ ((const char *) child->name, "version")) { - if (parse_version (g, child, osinfo) == -1) - return -1; - } - else if (STREQ ((const char *) child->name, "family")) { - if (parse_family (g, child, osinfo) == -1) - return -1; - } - else if (STREQ ((const char *) child->name, "distro")) { - if (parse_distro (g, child, osinfo) == -1) - return -1; - } - } - - return 0; -} - -static int -parse_version (guestfs_h *g, xmlNodePtr node, struct osinfo *osinfo) -{ - CLEANUP_FREE char *content = NULL; - - content = (char *) xmlNodeGetContent (node); - /* We parse either "X.Y" or "X" as version strings, so try to parse - * only if the first character is a digit. - */ - if (content && c_isdigit (content[0])) { - struct version version; - const int res = guestfs_int_version_from_x_y_or_x (g, &version, content); - if (res < 0) - return -1; - else if (res > 0) { - osinfo->major_version = version.v_major; - osinfo->minor_version = version.v_minor; - } - } - - return 0; -} - -static int -parse_family (guestfs_h *g, xmlNodePtr node, struct osinfo *osinfo) -{ - CLEANUP_FREE char *content = NULL; - - osinfo->type = OS_TYPE_UNKNOWN; - - content = (char *) xmlNodeGetContent (node); - if (content) { - if (STREQ (content, "linux")) - osinfo->type = OS_TYPE_LINUX; - else if (STRPREFIX (content, "win")) - osinfo->type = OS_TYPE_WINDOWS; - else if (STREQ (content, "freebsd")) - osinfo->type = OS_TYPE_FREEBSD; - else if (STREQ (content, "netbsd")) - osinfo->type = OS_TYPE_NETBSD; - else if (STREQ (content, "msdos")) - osinfo->type = OS_TYPE_DOS; - else if (STREQ (content, "openbsd")) - osinfo->type = OS_TYPE_OPENBSD; - else - debug (g, "osinfo: warning: unknown <family> '%s'", content); - } - - return 0; -} - -static int -parse_distro (guestfs_h *g, xmlNodePtr node, struct osinfo *osinfo) -{ - CLEANUP_FREE char *content = NULL; - - osinfo->distro = OS_DISTRO_UNKNOWN; - - content = (char *) xmlNodeGetContent (node); - if (content) { - if (STREQ (content, "altlinux")) - osinfo->distro = OS_DISTRO_ALTLINUX; - else if (STREQ (content, "centos")) - osinfo->distro = OS_DISTRO_CENTOS; - else if (STREQ (content, "debian")) - osinfo->distro = OS_DISTRO_DEBIAN; - else if (STREQ (content, "fedora")) - osinfo->distro = OS_DISTRO_FEDORA; - else if (STREQ (content, "freebsd")) - osinfo->distro = OS_DISTRO_FREEBSD; - else if (STREQ (content, "mageia")) - osinfo->distro = OS_DISTRO_MAGEIA; - else if (STREQ (content, "mandriva")) - osinfo->distro = OS_DISTRO_MANDRIVA; - else if (STREQ (content, "netbsd")) - osinfo->distro = OS_DISTRO_NETBSD; - else if (STREQ (content, "openbsd")) - osinfo->distro = OS_DISTRO_OPENBSD; - else if (STREQ (content, "opensuse")) - osinfo->distro = OS_DISTRO_OPENSUSE; - else if (STREQ (content, "rhel")) - osinfo->distro = OS_DISTRO_RHEL; - else if (STREQ (content, "sled") || STREQ (content, "sles")) - osinfo->distro = OS_DISTRO_SLES; - else if (STREQ (content, "ubuntu")) - osinfo->distro = OS_DISTRO_UBUNTU; - else if (STRPREFIX (content, "win")) - osinfo->distro = OS_DISTRO_WINDOWS; - else - debug (g, "osinfo: warning: unknown <distro> '%s'", content); - } - - return 0; -} - -static void -free_osinfo_db_entry (struct osinfo *osinfo) -{ - free (osinfo->product_name); - free (osinfo->arch); - - if (osinfo->re_system_id) - pcre_free (osinfo->re_system_id); - if (osinfo->re_volume_id) - pcre_free (osinfo->re_volume_id); - if (osinfo->re_publisher_id) - pcre_free (osinfo->re_publisher_id); - if (osinfo->re_application_id) - pcre_free (osinfo->re_application_id); -} diff --git a/lib/osinfo.h b/lib/osinfo.h new file mode 100644 index 000000000..68846747e --- /dev/null +++ b/lib/osinfo.h @@ -0,0 +1,27 @@ +/* libguestfs + * Copyright (C) 2017 Red Hat Inc. + * Copyright (C) 2017 SUSE Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef OSINFO_H +#define OSINFO_H + +typedef int (*read_osinfo_db_callback) (guestfs_h *g, const char *path, void *opaque); + +extern int read_osinfo_db (guestfs_h *g, read_osinfo_db_callback callback, void *opaque); + +#endif /* OSINFO_H */ -- 2.12.0

Rename test-virt-builder-docs.sh into test-docs.sh to include test for another tool's documentation. --- builder/Makefile.am | 4 ++-- builder/{test-virt-builder-docs.sh => test-docs.sh} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename builder/{test-virt-builder-docs.sh => test-docs.sh} (100%) diff --git a/builder/Makefile.am b/builder/Makefile.am index d56b394b7..218f64b4c 100644 --- a/builder/Makefile.am +++ b/builder/Makefile.am @@ -28,7 +28,7 @@ EXTRA_DIST = \ test-simplestreams/streams/v1/index.json \ test-simplestreams/streams/v1/net.cirros-cloud_released_download.json \ test-virt-builder.sh \ - test-virt-builder-docs.sh \ + test-docs.sh \ test-virt-builder-list.sh \ test-virt-builder-list-simplestreams.sh \ test-virt-builder-planner.sh \ @@ -237,7 +237,7 @@ yajl_tests_LINK = \ $(yajl_tests_THEOBJECTS) -o $@ TESTS = \ - test-virt-builder-docs.sh \ + test-docs.sh \ test-virt-builder-list.sh \ test-virt-index-validate.sh \ $(SLOW_TESTS) diff --git a/builder/test-virt-builder-docs.sh b/builder/test-docs.sh similarity index 100% rename from builder/test-virt-builder-docs.sh rename to builder/test-docs.sh -- 2.12.0

Add a function to properly write virt-builder source index entries. Note that this function is very similar to Index.print_entry that is meant for debugging purposes. --- .gitignore | 1 + builder/Makefile.am | 36 +++++++++++- builder/index.mli | 3 + builder/index_parser.ml | 54 ++++++++++++++++++ builder/index_parser.mli | 4 ++ builder/index_parser_tests.ml | 129 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 builder/index_parser_tests.ml diff --git a/.gitignore b/.gitignore index c82745ee8..12322e56b 100644 --- a/.gitignore +++ b/.gitignore @@ -104,6 +104,7 @@ Makefile.in /builder/virt-index-validate /builder/virt-index-validate.1 /builder/*.xz +/builder/index_parser_tests /builder/yajl_tests /cat/stamp-virt-*.pod /cat/virt-cat diff --git a/builder/Makefile.am b/builder/Makefile.am index 218f64b4c..bf4ccb7d7 100644 --- a/builder/Makefile.am +++ b/builder/Makefile.am @@ -217,13 +217,36 @@ yajl_tests_BOBJECTS = \ yajl_tests.cmo yajl_tests_XOBJECTS = $(yajl_tests_BOBJECTS:.cmo=.cmx) +index_parser_tests_SOURCES = \ + index-scan.c \ + index-struct.c \ + index-parser-c.c \ + index-parse.c +index_parser_tests_CPPFLAGS = $(virt_builder_CPPFLAGS) +index_parser_tests_BOBJECTS = \ + utils.cmo \ + cache.cmo \ + downloader.cmo \ + sigchecker.cmo \ + index.cmo \ + ini_reader.cmo \ + index_parser.cmo \ + index_parser_tests.cmo +index_parser_tests_XOBJECTS = $(index_parser_tests_BOBJECTS:.cmo=.cmx) + # Can't call the following as <test>_OBJECTS because automake gets confused. if HAVE_OCAMLOPT yajl_tests_THEOBJECTS = $(yajl_tests_XOBJECTS) yajl_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS) + +index_parser_tests_THEOBJECTS = $(index_parser_tests_XOBJECTS) +index_parser_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS) else yajl_tests_THEOBJECTS = $(yajl_tests_BOBJECTS) yajl_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS) + +index_parser_tests_THEOBJECTS = $(index_parser_tests_BOBJECTS) +index_parser_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS) endif yajl_tests_DEPENDENCIES = \ @@ -236,6 +259,15 @@ yajl_tests_LINK = \ $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) $(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \ $(yajl_tests_THEOBJECTS) -o $@ +index_parser_tests_DEPENDENCIES = \ + $(index_parser_tests_THEOBJECTS) \ + ../mllib/mllib.$(MLARCHIVE) \ + $(top_srcdir)/ocaml-link.sh +index_parser_tests_LINK = \ + $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \ + $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) $(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \ + $(index_parser_tests_THEOBJECTS) -o $@ + TESTS = \ test-docs.sh \ test-virt-builder-list.sh \ @@ -249,8 +281,8 @@ if ENABLE_APPLIANCE TESTS += test-virt-builder.sh endif ENABLE_APPLIANCE if HAVE_OCAML_PKG_OUNIT -check_PROGRAMS += yajl_tests -TESTS += yajl_tests +check_PROGRAMS += yajl_tests index_parser_tests +TESTS += yajl_tests index_parser_tests endif check-valgrind: diff --git a/builder/index.mli b/builder/index.mli index ff5ec4a35..6202d636e 100644 --- a/builder/index.mli +++ b/builder/index.mli @@ -39,3 +39,6 @@ and entry = { } val print_entry : out_channel -> (string * entry) -> unit +(** Debugging helper function dumping an index entry to a stream. + To write entries for non-debugging purpose, use the + [Index_parser.write_entry] function. *) diff --git a/builder/index_parser.ml b/builder/index_parser.ml index 26dd1f653..5bfd50b28 100644 --- a/builder/index_parser.ml +++ b/builder/index_parser.ml @@ -234,3 +234,57 @@ let get_index ~downloader ~sigchecker ~template in get_index () + +let write_entry chan (name, { Index.printable_name = printable_name; + file_uri = file_uri; + arch = arch; + osinfo = osinfo; + signature_uri = signature_uri; + checksums = checksums; + revision = revision; + format = format; + size = size; + compressed_size = compressed_size; + expand = expand; + lvexpand = lvexpand; + notes = notes; + aliases = aliases; + hidden = hidden }) = + let fp fs = fprintf chan fs in + fp "[%s]\n" name; + may (fp "name=%s\n") printable_name; + may (fp "osinfo=%s\n") osinfo; + fp "file=%s\n" file_uri; + fp "arch=%s\n" arch; + may (fp "sig=%s\n") signature_uri; + (match checksums with + | None -> () + | Some checksums -> + List.iter ( + fun c -> + fp "checksum[%s]=%s\n" + (Checksums.string_of_csum_t c) (Checksums.string_of_csum c) + ) checksums + ); + fp "revision=%s\n" (string_of_revision revision); + may (fp "format=%s\n") format; + fp "size=%Ld\n" size; + may (fp "compressed_size=%Ld\n") compressed_size; + may (fp "expand=%s\n") expand; + may (fp "lvexpand=%s\n") lvexpand; + + let format_notes notes = + String.concat "\n " (String.nsplit "\n" notes) in + + List.iter ( + fun (lang, notes) -> + match lang with + | "" -> fp "notes=%s\n" (format_notes notes) + | lang -> fp "notes[%s]=%s\n" lang (format_notes notes) + ) notes; + (match aliases with + | None -> () + | Some l -> fp "aliases=%s\n" (String.concat " " l) + ); + if hidden then fp "hidden=true\n"; + fp "\n" diff --git a/builder/index_parser.mli b/builder/index_parser.mli index aa5f84730..ae757ad6f 100644 --- a/builder/index_parser.mli +++ b/builder/index_parser.mli @@ -19,3 +19,7 @@ val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> template:bool -> Sources.source -> Index.index (** [get_index download sigchecker source] will parse the source index file into an index entry list. *) + +val write_entry : out_channel -> (string * Index.entry) -> unit +(** [write_entry chan entry] writes the index entry to the chan output + stream.*) diff --git a/builder/index_parser_tests.ml b/builder/index_parser_tests.ml new file mode 100644 index 000000000..c4352d752 --- /dev/null +++ b/builder/index_parser_tests.ml @@ -0,0 +1,129 @@ +(* builder + * Copyright (C) 2017 SUSE Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +(* This file tests the Index_parser module. *) + +open OUnit2 +open Printf +open Unix_utils +open Common_utils + +let tmpdir = Mkdtemp.temp_dir "guestfs-tests." "";; +rmdir_on_exit tmpdir + +let dummy_sigchecker = Sigchecker.create ~gpg:"gpg" + ~check_signature:false + ~gpgkey:Utils.No_Key + ~tmpdir + +let dummy_downloader = Downloader.create ~curl:"do-not-use-curl" + ~cache:None ~tmpdir + +(* Utils. *) +let write_entries file entries = + let chan = open_out (tmpdir // file) in + List.iter ( + fun (entry) -> + Index_parser.write_entry chan entry; + ) entries; + close_out chan + +let read_file file = + read_whole_file (tmpdir // "out") + +let parse_file file = + let source = { Sources.name = "input"; + uri = tmpdir // file; + gpgkey = Utils.No_Key; + proxy = Curl.SystemProxy; + format = Sources.FormatNative } in + let entries = Index_parser.get_index ~downloader:dummy_downloader + ~sigchecker:dummy_sigchecker + ~template:false + source in + List.map ( + fun (id, e) -> (id, { e with Index.file_uri = Filename.basename e.Index.file_uri }) + ) entries + +let format_entries entries = + let format_entry entry = + write_entries "out" [entry]; + read_file "out" in + List.map format_entry entries + +let assert_equal_string = assert_equal ~printer:(fun x -> sprintf "\"%s\"" x) +let assert_equal_list formatter = + let printer = ( + fun x -> "(" ^ (String.escaped (String.concat "," (formatter x))) ^ ")" + ) in + assert_equal ~printer + +let test_write_complete ctx = + let entry = + ("test-id", { Index.printable_name = Some "test_name"; + osinfo = Some "osinfo_data"; + file_uri = "image_path"; + arch = "test_arch"; + signature_uri = None; + checksums = Some [Checksums.SHA512 "512checksum"]; + revision = Utils.Rev_int 42; + format = Some "qcow2"; + size = Int64.of_int 123456; + compressed_size = Some (Int64.of_int 12345); + expand = Some "/dev/sda1"; + lvexpand = Some "/some/lv"; + notes = [ ("", "Notes split\non several lines\n\n with starting space ") ]; + hidden = false; + aliases = Some ["alias1"; "alias2"]; + sigchecker = dummy_sigchecker; + proxy = Curl.SystemProxy }) in + + write_entries "out" [entry]; + let actual = read_file "out" in + let expected = "[test-id] +name=test_name +osinfo=osinfo_data +file=image_path +arch=test_arch +checksum[sha512]=512checksum +revision=42 +format=qcow2 +size=123456 +compressed_size=12345 +expand=/dev/sda1 +lvexpand=/some/lv +notes=Notes split + on several lines + + with starting space +aliases=alias1 alias2 + +" in + assert_equal_string expected actual; + + let parsed_entries = parse_file "out" in + assert_equal_list format_entries [entry] parsed_entries + +let suite = + "builder Index_parser" >::: + [ + "write.complete" >:: test_write_complete; + ] + +let () = + run_test_tt_main suite -- 2.12.0

--- mllib/common_utils.ml | 6 ++++++ mllib/common_utils.mli | 3 +++ 2 files changed, 9 insertions(+) diff --git a/mllib/common_utils.ml b/mllib/common_utils.ml index 945728b5e..7932707c9 100644 --- a/mllib/common_utils.ml +++ b/mllib/common_utils.ml @@ -1172,3 +1172,9 @@ let do_cp src destdir = let cmd = [ "cp"; "-t"; destdir; "-a"; src ] in if run_command cmd <> 0 then error (f_"copy of %s to %s failed") src destdir + +let do_mv src dest = + let cmd = [ "mv"; src; dest ] in + let r = run_command cmd in + if r <> 0 then + error (f_"moving file '%s' to '%s' failed") src dest diff --git a/mllib/common_utils.mli b/mllib/common_utils.mli index 5c376fcb3..a98daad03 100644 --- a/mllib/common_utils.mli +++ b/mllib/common_utils.mli @@ -495,3 +495,6 @@ val inspect_decrypt : Guestfs.guestfs -> unit val do_cp : string -> string -> unit (** Run the cp command, and exit with an error if it failed *) + +val do_mv : string -> string -> unit +(** Run the mv command, and exit with an error if it failed *) -- 2.12.0

virt-builder-repository allows users to easily create or update a virt-builder source repository out of disk images. The tool can be run in either interactive or automated mode. --- .gitignore | 3 + builder/Makefile.am | 81 +++++- builder/repository_main.ml | 488 ++++++++++++++++++++++++++++++++++++ builder/test-docs.sh | 2 + builder/virt-builder-repository.pod | 183 ++++++++++++++ 5 files changed, 756 insertions(+), 1 deletion(-) create mode 100644 builder/repository_main.ml create mode 100644 builder/virt-builder-repository.pod diff --git a/.gitignore b/.gitignore index 12322e56b..382eaec37 100644 --- a/.gitignore +++ b/.gitignore @@ -94,13 +94,16 @@ Makefile.in /builder/oUnit-* /builder/*.qcow2 /builder/stamp-virt-builder.pod +/builder/stamp-virt-builder-repository.pod /builder/stamp-virt-index-validate.pod /builder/test-config/virt-builder/repos.d/test-index.conf /builder/test-console-*.sh /builder/test-simplestreams/virt-builder/repos.d/cirros.conf /builder/test-website/virt-builder/repos.d/libguestfs.conf /builder/virt-builder +/builder/virt-builder-repository /builder/virt-builder.1 +/builder/virt-builder-repository.1 /builder/virt-index-validate /builder/virt-index-validate.1 /builder/*.xz diff --git a/builder/Makefile.am b/builder/Makefile.am index bf4ccb7d7..9ae8fe272 100644 --- a/builder/Makefile.am +++ b/builder/Makefile.am @@ -21,6 +21,8 @@ AM_YFLAGS = -d EXTRA_DIST = \ $(SOURCES_MLI) $(SOURCES_ML) $(SOURCES_C) \ + $(REPOSITORY_SOURCES_ML) \ + $(REPOSITORY_SOURCES_MLI) \ libguestfs.gpg \ opensuse.gpg \ test-console.sh \ @@ -38,6 +40,7 @@ EXTRA_DIST = \ test-virt-index-validate-good-2 \ test-virt-index-validate-good-3 \ virt-builder.pod \ + virt-builder-repository.pod \ virt-index-validate.pod \ yajl_tests.ml @@ -85,13 +88,44 @@ SOURCES_C = \ setlocale-c.c \ yajl-c.c +REPOSITORY_SOURCES_ML = \ + utils.ml \ + index.ml \ + cache.ml \ + downloader.ml \ + sigchecker.ml \ + ini_reader.ml \ + index_parser.ml \ + yajl.ml \ + paths.ml \ + sources.ml \ + repository_main.ml + +REPOSITORY_SOURCES_MLI = \ + cache.mli \ + downloader.mli \ + index.mli \ + index_parser.mli \ + ini_reader.mli \ + sigchecker.mli \ + sources.mli \ + yajl.mli + +REPOSITORY_SOURCES_C = \ + index-scan.c \ + index-struct.c \ + index-parse.c \ + index-parser-c.c \ + yajl-c.c + + man_MANS = noinst_DATA = bin_PROGRAMS = if HAVE_OCAML -bin_PROGRAMS += virt-builder +bin_PROGRAMS += virt-builder virt-builder-repository virt_builder_SOURCES = $(SOURCES_C) virt_builder_CPPFLAGS = \ @@ -115,6 +149,26 @@ virt_builder_CFLAGS = \ BOBJECTS = $(SOURCES_ML:.ml=.cmo) XOBJECTS = $(BOBJECTS:.cmo=.cmx) +virt_builder_repository_SOURCES = $(REPOSITORY_SOURCES_C) +virt_builder_repository_CPPFLAGS = \ + -I. \ + -I$(top_builddir) \ + -I$(top_srcdir)/gnulib/lib -I$(top_builddir)/gnulib/lib \ + -I$(shell $(OCAMLC) -where) \ + -I$(top_srcdir)/gnulib/lib \ + -I$(top_srcdir)/src \ + -I$(top_srcdir)/fish +virt_builder_repository_CFLAGS = \ + -pthread \ + $(WARN_CFLAGS) $(WERROR_CFLAGS) \ + -Wno-unused-macros \ + $(LIBLZMA_CFLAGS) \ + $(LIBTINFO_CFLAGS) \ + $(LIBXML2_CFLAGS) \ + $(YAJL_CFLAGS) +REPOSITORY_BOBJECTS = $(REPOSITORY_SOURCES_ML:.ml=.cmo) +REPOSITORY_XOBJECTS = $(REPOSITORY_BOBJECTS:.cmo=.cmx) + # -I $(top_builddir)/lib/.libs is a hack which forces corresponding -L # option to be passed to gcc, so we don't try linking against an # installed copy of libguestfs. @@ -149,8 +203,10 @@ OCAMLFLAGS = $(OCAML_FLAGS) $(OCAML_WARN_ERROR) if !HAVE_OCAMLOPT OBJECTS = $(BOBJECTS) +REPOSITORY_OBJECTS = $(REPOSITORY_BOBJECTS) else OBJECTS = $(XOBJECTS) +REPOSITORY_OBJECTS = $(REPOSITORY_XOBJECTS) endif OCAMLLINKFLAGS = mlguestfs.$(MLARCHIVE) mllib.$(MLARCHIVE) customize.$(MLARCHIVE) $(LINK_CUSTOM_OCAMLC_ONLY) @@ -165,6 +221,15 @@ virt_builder_LINK = \ $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) $(OCAMLLINKFLAGS) \ $(OBJECTS) -o $@ +virt_builder_repository_DEPENDENCIES = \ + $(REPOSITORY_OBJECTS) \ + ../mllib/mllib.$(MLARCHIVE) \ + $(top_srcdir)/ocaml-link.sh +virt_builder_repository_LINK = \ + $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \ + $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) $(OCAMLLINKFLAGS) \ + $(REPOSITORY_OBJECTS) -o $@ + # Manual pages and HTML files for the website. man_MANS += virt-builder.1 @@ -183,6 +248,20 @@ stamp-virt-builder.pod: virt-builder.pod $(top_srcdir)/customize/customize-synop $< touch $@ +man_MANS += virt-builder-repository.1 +noinst_DATA += $(top_builddir)/website/virt-builder-repository.1.html + +virt-builder-repository.1 $(top_builddir)/website/virt-builder-repository.1.html: stamp-virt-builder-repository.pod + +stamp-virt-builder-repository.pod: virt-builder-repository.pod + $(PODWRAPPER) \ + --man virt-builder-repository.1 \ + --html $(top_builddir)/website/virt-builder-repository.1.html \ + --license GPLv2+ \ + --warning safe \ + $< + touch $@ + # Tests. TESTS_ENVIRONMENT = $(top_builddir)/run --test diff --git a/builder/repository_main.ml b/builder/repository_main.ml new file mode 100644 index 000000000..87bcc5e15 --- /dev/null +++ b/builder/repository_main.ml @@ -0,0 +1,488 @@ +(* virt-builder + * Copyright (C) 2016-2017 SUSE Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Common_gettext.Gettext +open Common_utils +open Unix_utils +open Getopt.OptionName +open Utils +open Yajl +open Xpath_helpers + +open Printf + +module StringSet = Set.Make(String) + +type cmdline = { + gpg : string; + gpgkey : string option; + interactive : bool; + keep_unsigned : bool; + repo : string; +} + +type disk_image_info = { + format : string; + size : int64; +} + +let parse_cmdline () = + let gpg = ref "gpg" in + let gpgkey = ref "" in + let interactive = ref false in + let keep_unsigned = ref false in + + let argspec = [ + [ L"gpg" ], Getopt.Set_string ("gpg", gpg), s_"Set GPG binary/command"; + [ S 'K'; L"gpg-key" ], Getopt.Set_string ("gpgkey", gpgkey), + s_"ID of the GPG key to sign the repo with"; + [ S 'i'; L"interactive" ], Getopt.Set interactive, s_"Ask the user about missing data"; + [ L"keep-index" ], Getopt.Set keep_unsigned, s_"Keep unsigned index"; + ] in + + let args = ref [] in + let anon_fun s = push_front s args in + let usage_msg = + sprintf (f_"\ +%s: create a repository for virt-builder + + virt-builder-repository REPOSITORY_PATH + +A short summary of the options is given below. For detailed help please +read the man page virt-builder-repository(1). +") + prog in + let opthandle = create_standard_options argspec ~anon_fun usage_msg in + Getopt.parse opthandle; + + (* Dereference options. *) + let args = List.rev !args in + let gpg = !gpg in + let gpgkey = match !gpgkey with "" -> None | s -> Some s in + let interactive = !interactive in + let keep_unsigned = !keep_unsigned in + + (* Check options *) + let repo = + (match args with + | [repo] -> repo + | [] -> + error (f_"virt-builder-repository /path/to/repo\nUse '/path/to/repo' to point to the repository folder.") + | _ -> + error (f_"too many parameters, only one path to repository is allowed") + ) in + + { + gpg = gpg; + gpgkey = gpgkey; + interactive = interactive; + keep_unsigned = keep_unsigned; + repo = repo; + } + +let increment_revision revision = + match revision with + | Utils.Rev_int n -> Utils.Rev_int (n + 1) + | Utils.Rev_string s -> + if Str.string_match (Str.regexp "^\\(.*[-._]\\)\\([0-9]+\\)$") s 0 then + let prefix = Str.matched_group 1 s in + let suffix = int_of_string (Str.matched_group 2 s) in + Utils.Rev_string (prefix ^ (string_of_int (suffix + 1))) + else + Utils.Rev_string (s ^ ".1") + +let osinfo_ids = ref None + +let osinfo_get_short_ids () = + match !osinfo_ids with + | Some ids -> ids + | None -> ( + let set = ref StringSet.empty in + let g = open_guestfs () in + + Osinfo.read_osinfo_db g#ocaml_handle ( + fun filepath -> + let doc = Xml.parse_file filepath in + let xpathctx = Xml.xpath_new_context doc in + let nodes = xpath_get_nodes xpathctx "/libosinfo/os/short-id" in + List.iter ( + fun node -> + let id = Xml.node_as_string node in + set := StringSet.add id !set + ) nodes + ); + g#close (); + osinfo_ids := Some (!set); + !set + ) + +let compress_to file outdir = + info "Copying image to temporary folder ...%!"; + let outimg = outdir // (Filename.basename file) in + do_cp file outdir; + + info "Compressing ...%!"; + let cmd = [ "xz"; "-f"; "--best"; + "--block-size=16777216"; outimg ] in + if run_command cmd <> 0 then + exit 1; + outimg ^ ".xz" + +let get_disk_image_info filepath = + let qemuimg_cmd = "qemu-img info --output json " ^ (quote filepath) in + let lines = external_command qemuimg_cmd in + let line = String.concat "\n" lines in + let infos = yajl_tree_parse line in + { + format = object_get_string "format" infos; + size = object_get_number "virtual-size" infos + } + +let process_image filename repo tmprepo index interactive sigchecker = + message (f_"Preparing %s") filename; + + let filepath = repo // filename in + let { format = format; size = size } = get_disk_image_info filepath in + let xz_path = compress_to filepath tmprepo in + let checksum = Checksums.compute_checksum "sha512" xz_path in + let compressed_size = (Unix.LargeFile.stat xz_path).Unix.LargeFile.st_size in + + let ask message = + printf message; + let value = read_line () in + match value with + | "" -> None + | s -> Some s + in + + let rec ask_id () = + printf (f_"Identifier: "); + let id = read_line () in + if not (Str.string_match (Str.regexp "[a-zA-Z0-9-_.]+") id 0) then ( + warning (f_"Allowed characters are letters, digits, - _ and ."); + ask_id (); + ) else + id; + in + + let ask_arch () = + printf (f_"Architecture. Choose one from the list below:\n"); + let arches = ["x86_64"; "aarch64"; "armv7l"; "i686"; "ppc64"; "ppc64le"; "s390x" ] in + iteri ( + fun i arch -> printf " [%d] %s\n" (i + 1) arch + ) arches; + + let input = read_line () in + if input = "exit" || input = "q" || input = "quit" then + exit 0 + else + try + let i = int_of_string input in + List.nth arches (i - 1) + with Failure _ -> input + in + + let ask_osinfo () = + printf (f_ "osinfo short ID (can be found with osinfo-query os command): "); + let value = read_line () in + match value with + | "" -> None + | osinfo -> + let osinfo_ids = osinfo_get_short_ids () in + if not (StringSet.mem osinfo osinfo_ids) then + warning (f_"'%s' is not a recognized osinfo OS id; using it anyway") osinfo; + Some osinfo in + + (* Do we have an entry for that file already? *) + let file_entry = + try + let xzfn filename = Filename.basename filename ^ ".xz" in + List.hd ( + List.filter ( + fun (id, { Index.file_uri=file_uri }) -> + (Filename.basename file_uri) = (xzfn (Filename.basename filename)) + ) index + ) + with + | Failure _ -> + let entry = { Index.printable_name = None; + osinfo = None; + file_uri = ""; + arch = ""; + signature_uri = None; + checksums = None; + revision = Utils.Rev_int 0; + format = Some format; + size = size; + compressed_size = Some compressed_size; + expand = None; + lvexpand = None; + notes = []; + hidden = false; + aliases = None; + sigchecker = sigchecker; + proxy = Curl.UnsetProxy } in + ("", entry) in + + let id, { Index.printable_name = printable_name; + osinfo = osinfo; + arch = arch; + checksums = checksums; + revision = revision; + expand = expand; + lvexpand = lvexpand; + notes = notes; + hidden = hidden; + aliases = aliases } = file_entry in + + let old_checksum = + match checksums with + | Some csums -> ( + try + let csum = List.find ( + fun c -> + match c with + | Checksums.SHA512 _ -> true + | _ -> false + ) csums in + csum + with + | _ -> Checksums.SHA512 "" + ) + | None -> Checksums.SHA512 "" in + + let id = + if id = "" then ( + if interactive then ask_id () + else error (f_"missing image identifier"); + ) else id in + + let printable_name = + if printable_name = None && interactive then + ask (f_"Display name: ") + else + printable_name in + + let arch = + if arch = "" then ( + if interactive then ask_arch () + else error (f_"missing architecture for %s") id; + ) else arch in + + let osinfo = + if osinfo = None && interactive then + ask_osinfo () + else + osinfo in + + let expand = + if expand = None && interactive then + ask (f_"Expandable partition: ") + else + expand in + + let lvexpand = + if lvexpand = None && interactive then + ask (f_"Expandable volume: ") + else + lvexpand in + + let revision = + if old_checksum <> checksum then + increment_revision revision + else + revision in + + (id, { Index.printable_name = printable_name; + osinfo = osinfo; + file_uri = Filename.basename xz_path; + arch = arch; + signature_uri = None; + checksums = Some [checksum]; + revision = revision; + format = Some format; + size = size; + compressed_size = Some compressed_size; + expand = expand; + lvexpand = lvexpand; + notes = notes; + hidden = hidden; + aliases = aliases; + sigchecker = sigchecker; + proxy = Curl.UnsetProxy }) + +let has_entry id arch index = + List.exists ( + fun (item_id, entry) -> + let { Index.arch = item_arch } = entry in + item_id = id && item_arch = arch + ) index + + +let main () = + let cmdline = parse_cmdline () in + + (* If debugging, echo the command line arguments. *) + debug "command line: %s\n" (String.concat " " (Array.to_list Sys.argv)); + + (* Check that the paths are existing *) + if not (Sys.file_exists cmdline.repo) then + error (f_"repository folder '%s' doesn't exist") cmdline.repo; + + (* Create a temporary folder to work in *) + let tmpdir = Mkdtemp.temp_dir ~base_dir:cmdline.repo + "virt-builder-repository." "" in + rmdir_on_exit tmpdir; + + let tmprepo = tmpdir // "repo" in + mkdir_p tmprepo 0o700; + + let sigchecker = Sigchecker.create ~gpg:cmdline.gpg + ~check_signature:false + ~gpgkey:No_Key + ~tmpdir in + + let index = + try + let index_filename = + List.find ( + fun filename -> Sys.file_exists (cmdline.repo // filename) + ) [ "index.asc"; "index" ] in + + let downloader = Downloader.create ~curl:"do-not-use-curl" + ~cache:None ~tmpdir in + + let source = { Sources.name = "input"; + uri = cmdline.repo // index_filename; + gpgkey = No_Key; + proxy = Curl.SystemProxy; + format = Sources.FormatNative } in + + Index_parser.get_index ~downloader ~sigchecker ~template:true source + with Not_found -> [] in + + (* Check for index/interactive consistency *) + if not cmdline.interactive && index == [] then + error (f_"the repository needs to contain an index file when running in automated mode"); + + debug "Searching for images ...\n"; + + let images = + let is_supported_format file = + let extension = last_part_of file '.' in + match extension with + | Some ext -> List.mem ext [ "qcow2"; "raw"; "img" ] + | None -> file <> "index" in + let files = Array.to_list (Sys.readdir cmdline.repo) in + List.filter is_supported_format files in + + if (List.length images) = 0 then ( + info (f_ "Found no new image"); + exit 0 + ); + + info (f_ "Found new images: %s") (String.concat " " images); + + let outindex_path = tmprepo // "index" in + let index_channel = open_out outindex_path in + + (* Generate entries for uncompressed images *) + let images_entries = List.map ( + fun filename -> + process_image filename cmdline.repo tmprepo + index cmdline.interactive sigchecker + ) images in + + (* Filter out entries for newly found images and entries + without a corresponding image file *) + let index = List.filter ( + fun (id, entry) -> + let { Index.arch = arch } = entry in + let { Index.file_uri = file_uri } = entry in + not (has_entry id arch images_entries) && Sys.file_exists file_uri + ) index in + + (* Convert all URIs back to relative ones *) + let index = List.map ( + fun (id, entry) -> + let { Index.file_uri = file_uri } = entry in + let rel_path = + try + subdirectory cmdline.repo file_uri + with + | Invalid_argument _ -> + file_uri in + let rel_entry = { entry with Index.file_uri = rel_path } in + (id, rel_entry) + ) index in + + (* Write all the entries *) + List.iter ( + fun entry -> + Index_parser.write_entry index_channel entry; + ) (index @ images_entries); + + close_out index_channel; + + (* GPG sign the generated index *) + (match cmdline.gpgkey with + | None -> + debug "Skip index signing" + | Some gpgkey -> + message (f_"Signing index with GPG key %s") gpgkey; + let cmd = [ cmdline.gpg; "--armor"; + "--output"; (tmprepo // "index.gpg"); + "--export"; gpgkey ] in + if run_command cmd <> 0 then + error (f_"failed to export GPG key %s") gpgkey; + + let cmd = [ cmdline.gpg; "--armor"; + "--default-key"; gpgkey; + "--clearsign"; (tmprepo // "index") ] in + if run_command cmd <> 0 then + error (f_"failed to sign index"); + + (* Remove the index file since we have the signed version of it *) + if not cmdline.keep_unsigned then + Sys.remove (tmprepo // "index") + ); + + message (f_"Creating index backup copy"); + + List.iter ( + fun filename -> + let filepath = cmdline.repo // filename in + if Sys.file_exists filepath then + do_mv filepath (filepath ^ ".bak") + ) ["index"; "index.asc"]; + + message (f_"Moving files to final destination"); + + Array.iter ( + fun filename -> + do_mv (tmprepo // filename) cmdline.repo + ) (Sys.readdir tmprepo); + + debug "Cleanup"; + + (* Remove the processed image files *) + List.iter ( + fun filename -> Sys.remove (cmdline.repo // filename) + ) images + +let () = run_main_and_handle_errors main diff --git a/builder/test-docs.sh b/builder/test-docs.sh index 884135de6..6f39b906d 100755 --- a/builder/test-docs.sh +++ b/builder/test-docs.sh @@ -25,3 +25,5 @@ $top_srcdir/podcheck.pl virt-builder.pod virt-builder \ --insert $top_srcdir/customize/customize-synopsis.pod:__CUSTOMIZE_SYNOPSIS__ \ --insert $top_srcdir/customize/customize-options.pod:__CUSTOMIZE_OPTIONS__ \ --ignore=--check-signatures,--no-check-signatures + +$srcdir/../podcheck.pl virt-builder-repository.pod virt-builder-repository diff --git a/builder/virt-builder-repository.pod b/builder/virt-builder-repository.pod new file mode 100644 index 000000000..b5e790403 --- /dev/null +++ b/builder/virt-builder-repository.pod @@ -0,0 +1,183 @@ +=begin html + +<img src="virt-builder.svg" width="250" + style="float: right; clear: right;" /> + +=end html + +=head1 NAME + +virt-builder-repository - Build virt-builder source repository easily + +=head1 SYNOPSIS + + virt-builder-repository /path/to/repository + [-i|--interactive] [--gpg-key KEYID] + +=head1 DESCRIPTION + +Virt-builder is a tool for quickly building new virtual machines. It can +be configured to use template repositories. However creating and +maintaining a repository involves many tasks which can be automated. +virt-builder-repository is a tool helping to manage these repositories. + +Virt-builder-repository loops over the files in the C</path/to/repository> +folder, compresses the files with a name ending by C<qcow2>, C<raw> or +C<img>, extracts data from them and creates or updates the C<index> file. + +Some of the image-related data needed for the index file can't be +computed from the image file. virt-builder-repository first tries to +find them in the existing index file. If data are still missing after +this, they are prompted in interactive mode, otherwise an error will +be triggered. + +If a C<KEYID> is provided, the generated index file will be signed +with this GPG key. + +=head1 EXAMPLES + +=head2 Create the initial repository + +Create a folder and copy the disk image template files in it. Then +run a command like the following one: + + virt-builder-repository --gpg-key &quot;joe(a)hacker.org&quot; -i /path/to/folder + +Note that this example command runs in interactive mode. To run in +automated mode, a minimal index file needs to be created before running +the command containing sections like this one: + + [template_id] + name=template display name + file=template_filename.qcow.xz + arch=x86_64 + revision=0 + expand=/dev/sda3 + +The file value needs to match the image name extended with the ".xz" +suffix. Other optional data can be prefilled, for more informations, +see L<virt-builder(1)/Creating and signing the index file>. + +=head2 Update images in an existing repository + +In this use case, an new image or a new revision of an existing image +needs to be added to the repository. Place the corresponding image +template files in the repository folder. + +To update the revision of an image, the file needs to have the same +name than the existing one (without the C<xz> extension). + +As in the repository creation use case, a minimal fragment can be +added to the index file for the automated mode. This can be done +on the signed index even if it may sound a strange idea: the index +will be resigned by the tool. + +To remove an image from the repository, just remove the corresponding +compressed image file before running virt-builder-repository. + +Then running the following command will complete and update the index +file: + + virt-builder-repository --gpg-key &quot;joe(a)hacker.org&quot; -i /path/to/folder + +virt-builder-repository works in a temporary folder inside the repository +one. If anything wrong happens when running the tool, the repository is +left untouched. + +=head1 OPTIONS + +=over 4 + +=item B<--help> + +Display help. + +=item B<--gpg> GPG + +Specify an alternate L<gpg(1)> (GNU Privacy Guard) binary. You can +also use this to add gpg parameters, for example to specify an +alternate home directory: + + virt-builder-repository --gpg "gpg --homedir /tmp" [...] + +This can also be used to avoid gpg asking for the key passphrase: + + virt-builder-repository --gpg "gpg --passphrase-file /tmp/pass --batch" [...] + +=item B<-K> KEYID + +=item B<--gpg-key> KEYID + +Specify the GPG key to be used to sign the repository index file. +If not provided, the index will left unsigned. C<KEYID> is used to +identify the GPG key to use. This value is passed to gpg's +C<--default-key> option and can can thus be an email address or a +fingerprint. + +B<NOTE>: by default, virt-builder-repository searches for the key +in the user's GPG key ring. + +=item B<--keep-index> + +When using a GPG key, don't remove the unsigned index. + +=item B<-i> + +=item B<--interactive> + +Prompt for missing data. + +=item B<--colors> + +=item B<--colours> + +Use ANSI colour sequences to colourize messages. This is the default +when the output is a tty. If the output of the program is redirected +to a file, ANSI colour sequences are disabled unless you use this +option. + +=item B<-q> + +=item B<--quiet> + +Don't print ordinary progress messages. + +=item B<-v> + +=item B<--verbose> + +Enable debug messages and/or produce verbose output. + +When reporting bugs, use this option and attach the complete output to +your bug report. + +=item B<-V> + +=item B<--version> + +Display version number and exit. + +=item B<-x> + +Enable tracing of libguestfs API calls. + + +=back + +=head1 EXIT STATUS + +This program returns 0 if successful, or non-zero if there was an +error. + +=head1 SEE ALSO + +L<virt-builder(1)> +L<http://libguestfs.org/>. + +=head1 AUTHOR + +Cédric Bosdonnat L<mailto:cbosdonnat@suse.com> + +=head1 COPYRIGHT + +Copyright (C) 2016-2017 SUSE Inc. -- 2.12.0

On Tue, 2017-04-04 at 16:23 +0200, Pino Toscano wrote: OK, comments fixed, stacked locally waiting for next round. -- Cedric

On Thursday, 23 March 2017 10:02:48 CEST Cédric Bosdonnat wrote: It would be nice to specify to not compress images at all -- for example I have a couple of local repositories with plain qcow2 images, and it's easier to handle without the need to compress them every time a new version of an image is created. In this case may require further work, as you should skip non-compressed images already handled, avoid copying them from the temporary directory, etc. Also, it could make use of libguestfs to inspect the new images, and thus suggest something for few data: - OS name, for the ID - OS name & version, for suggesting an initial osinfo (after all, most of them follow few patterns) - product name, for the display name - architecture - list of partitions with filesytems, for the expandable filesystem - list of LVs, for the expandable volume especially that it could skip asking few based on that (e.g. pick the architecture, or skip the expandable volume if there are no LVs). s/src/lib/, following the big renaming of the src directory that happened shortly before 1.36. I recommend to use configure libguestfs with --enable-werror when building from git, so issues like this can be spotted easily: cc1: error: ../src: No such file or directory [-Werror=missing-include-dirs] What is it for? No need for round brackets here, the let ... in already encloses it. This is slightly inefficent, as it copies the image fully -- just xz the original image piping the process output to the destination file. I think at this point it should warn (or even fail) when the current image is the same as an existing image. Note the identification for this is the (identifier,architecture) pair. Spacing around '='. Also 'id' is not used, so can be replaced with '_'. In a previous review, I meant that you can cache the whole result of xzfn (Filename.basename filename) as it is the same while iterating through 'index'. Extra variable 'csum'. 'entry' can be unpacked directly as argument (like done already above). Minor nitpick: it could use 'index_filename' instead of "input". Most probably there should be "must" instead of "needs" here, since it's a requirement. Here it can be 'if images == [] then'. "No new images found" sounds more natural IMHO. As above, 'entry' can be unpacked together in the function declaration. Unfortunately you cannot use run_command with cmdline.gpg, as it can be also a string with spaces (e.g. 'gpg --homedir /foo'), and thus it cannot be considered a single parameter. Use shell_command here, see also builder/sigchecker.ml. Enough comments for now, more will come the upcoming days. Thanks, -- Pino Toscano