[PATCH] febootstrap: Use contents of installed Debian packages instead of downloading and unpacking them.
--- src/febootstrap_debian.ml | 65 +++++++++++++++++++++++++++++++++++++------- 1 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/febootstrap_debian.ml b/src/febootstrap_debian.ml index 23f3593..bb10f15 100644 --- a/src/febootstrap_debian.ml +++ b/src/febootstrap_debian.ml @@ -28,6 +28,9 @@ open Febootstrap_cmdline (* Create a temporary directory for use by all the functions in this file. *) let tmpdir = tmpdir () +let installed_pkgs = + run_command_get_lines "dpkg-query --show --showformat='${Package}\\n'" + let debian_detect () = file_exists "/etc/debian_version" && Config.aptitude <> "no" && Config.apt_cache <> "no" && Config.dpkg <> "no" @@ -51,18 +54,29 @@ let rec debian_resolve_dependencies_and_download names = not (List.exists (fun re -> Str.string_match re name 0) excludes) ) pkgs in + let present_pkgs, download_pkgs = List.partition ( + fun pkg -> List.exists ((=) pkg) installed_pkgs + ) pkgs in + + debug "wanted packages (present / download): %s / %s\n" + (String.concat " " present_pkgs) + (String.concat " " download_pkgs); + (* Download the packages. *) - let cmd = - sprintf "umask 0000; cd %s && %s download %s" - (Filename.quote tmpdir) - Config.aptitude - (String.concat " " (List.map Filename.quote pkgs)) in - run_command cmd; + if (List.length download_pkgs > 0) + then ( + let cmd = + sprintf "umask 0000; cd %s && %s download %s" + (Filename.quote tmpdir) + Config.aptitude + (String.concat " " (List.map Filename.quote download_pkgs)) in + run_command cmd + ); (* Find out what aptitude downloaded. *) let files = Sys.readdir tmpdir in - let pkgs = List.map ( + let download_pkgs = List.map ( fun pkg -> (* Look for 'pkg_*.deb' in the list of files. *) let pre = pkg ^ "_" in @@ -79,9 +93,9 @@ let rec debian_resolve_dependencies_and_download names = exit 1 with Exit -> !r - ) pkgs in + ) download_pkgs in - List.sort compare pkgs + List.sort compare (List.append present_pkgs download_pkgs) (* On Ubuntu 10.04 LTS, apt-cache depends --recurse is broken. It * doesn't return the full list of dependencies. Therefore recurse @@ -106,7 +120,7 @@ and workaround_broken_apt_cache_depends_recurse names = else names -let debian_list_files pkg = +let debian_list_files_downloaded pkg = debug "unpacking %s ..." pkg; (* We actually need to extract the file in order to get the @@ -147,9 +161,38 @@ let debian_list_files pkg = files +let debian_list_files_installed pkg = + debug "using installed package %s ..." pkg; + let cmd = sprintf "dpkg-query --listfiles %s" pkg in + let lines = run_command_get_lines cmd in + (* filter out lines not directly describing fs objects such as + "package diverts others to: /path/to/..." *) + let lines = List.filter ( + fun l -> l.[0] = '/' && l.[1] != '.' + ) lines in + let files = List.map ( + fun path -> + let statbuf = lstat path in + let is_dir = statbuf.st_kind = S_DIR in + let config = statbuf.st_kind = S_REG && string_prefix "/etc/" path in + let mode = statbuf.st_perm in + (path, { ft_dir = is_dir; ft_config = config; ft_mode = mode; + ft_ghost = false; ft_size = statbuf.st_size }) + ) lines in + files + +let debian_list_files pkg = + if List.exists ((=) pkg) installed_pkgs then + debian_list_files_installed pkg + else + debian_list_files_downloaded pkg + (* Easy because we already unpacked the archive above. *) let debian_get_file_from_package pkg file = - tmpdir // pkg ^ ".d" // file + if List.exists (fun p -> p = pkg) installed_pkgs then + file + else + tmpdir // pkg ^ ".d" // file let () = let ph = { -- 1.7.5.4
On Mon, Sep 12, 2011 at 11:58:09PM +0200, Hilko Bengen wrote:
--- src/febootstrap_debian.ml | 65 +++++++++++++++++++++++++++++++++++++------- 1 files changed, 54 insertions(+), 11 deletions(-)
Update on this one since I've been busy investigating a serious bug in RHEL. We decided that this functionality was great, but that it should be available optionally, and across at least the Debian and Yum backends. It would be useful for Fedora builds too. ---- On a separate note, febootstrap-supermin-helper could use a 9pfs backend. This would make boot times even faster, while reducing the complexity of the boot process -- in other words, it's all upside. It seems that the hard bit of this is modifying the 9p server inside qemu to make it more flexible. It would be good if the 9p server could be driven a text configuration file: /etc directory /etc/passwd replace /tmp/base/etc/passwd mode 0644 owner root group root /usr/bin directory /usr/bin/ls host /usr/bin/vi host This describes to qemu how to synthesize a filesystem. Some files comes from host files directly (hostfiles), and some files are replacement files (from base.img). Then all the febootstrap-supermin-helper backend would need to do would be to unpack base.img and construct this simple text config file. Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones virt-top is 'top' for virtual machines. Tiny program with many powerful monitoring features, net stats, disk stats, logging, etc. http://et.redhat.com/~rjones/virt-top
--- src/febootstrap.ml | 4 ++-- src/febootstrap_cmdline.ml | 4 ++++ src/febootstrap_cmdline.mli | 3 +++ src/febootstrap_debian.ml | 8 ++++---- src/febootstrap_package_handlers.ml | 4 ++-- src/febootstrap_package_handlers.mli | 4 ++-- src/febootstrap_pacman.ml | 6 ++++-- src/febootstrap_yum_rpm.ml | 5 +++-- 8 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/febootstrap.ml b/src/febootstrap.ml index 7e48206..9b7dc88 100644 --- a/src/febootstrap.ml +++ b/src/febootstrap.ml @@ -67,7 +67,7 @@ let () = List.flatten ( List.map ( fun pkg -> - let files = ph.ph_list_files pkg in + let files = (ph.ph_list_files ~use_installed:use_installed pkg) in List.map (fun (filename, ft) -> filename, ft, pkg) files ) packages ) in @@ -320,7 +320,7 @@ let () = * original file from the package. *) else if config then ( - let outfile = ph.ph_get_file_from_package pkg path in + let outfile = ph.ph_get_file_from_package ~use_installed:use_installed pkg path in (* Note that the output config file might not be a regular file. *) let statbuf = lstat outfile in diff --git a/src/febootstrap_cmdline.ml b/src/febootstrap_cmdline.ml index 667e297..939afb7 100644 --- a/src/febootstrap_cmdline.ml +++ b/src/febootstrap_cmdline.ml @@ -23,6 +23,7 @@ let names_mode = ref false let outputdir = ref "." let packages = ref [] let save_temps = ref false +let use_installed = ref false let verbose = ref false let warnings = ref true let yum_config = ref None @@ -50,6 +51,8 @@ let argspec = Arg.align [ " Don't delete temporary files and directories on exit."; "--save-temps", Arg.Set save_temps, " Don't delete temporary files and directories on exit."; + "--use-installed", Arg.Set use_installed, + " Inspect installed packages for determining contents."; "-v", Arg.Set verbose, " Enable verbose output"; "--verbose", Arg.Set verbose, @@ -89,6 +92,7 @@ let names_mode = !names_mode let outputdir = !outputdir let packages = List.rev !packages let save_temps = !save_temps +let use_installed = !use_installed let verbose = !verbose let warnings = !warnings let yum_config = !yum_config diff --git a/src/febootstrap_cmdline.mli b/src/febootstrap_cmdline.mli index d948d80..a545012 100644 --- a/src/febootstrap_cmdline.mli +++ b/src/febootstrap_cmdline.mli @@ -38,6 +38,9 @@ val packages : string list val save_temps : bool (** True if [--save-temps] was given on the command line. *) +val use_installed : bool + (** True if [--use-installed] was given on the command line *) + val verbose : bool (** True if [--verbose] was given on the command line. See also {!debug}. *) diff --git a/src/febootstrap_debian.ml b/src/febootstrap_debian.ml index bb10f15..7482ed4 100644 --- a/src/febootstrap_debian.ml +++ b/src/febootstrap_debian.ml @@ -181,15 +181,15 @@ let debian_list_files_installed pkg = ) lines in files -let debian_list_files pkg = - if List.exists ((=) pkg) installed_pkgs then +let debian_list_files ?(use_installed=false) pkg = + if use_installed && List.exists ((=) pkg) installed_pkgs then debian_list_files_installed pkg else debian_list_files_downloaded pkg (* Easy because we already unpacked the archive above. *) -let debian_get_file_from_package pkg file = - if List.exists (fun p -> p = pkg) installed_pkgs then +let debian_get_file_from_package ?(use_installed=false) pkg file = + if use_installed && List.exists (fun p -> p = pkg) installed_pkgs then file else tmpdir // pkg ^ ".d" // file diff --git a/src/febootstrap_package_handlers.ml b/src/febootstrap_package_handlers.ml index ad3a233..f627d2f 100644 --- a/src/febootstrap_package_handlers.ml +++ b/src/febootstrap_package_handlers.ml @@ -25,8 +25,8 @@ open Febootstrap_cmdline type package_handler = { ph_detect : unit -> bool; ph_resolve_dependencies_and_download : string list -> string list; - ph_list_files : string -> (string * file_type) list; - ph_get_file_from_package : string -> string -> string + ph_list_files : ?use_installed:bool -> string -> (string * file_type) list; + ph_get_file_from_package : ?use_installed:bool -> string -> string -> string } and file_type = { ft_dir : bool; diff --git a/src/febootstrap_package_handlers.mli b/src/febootstrap_package_handlers.mli index c28d81f..ebf0386 100644 --- a/src/febootstrap_package_handlers.mli +++ b/src/febootstrap_package_handlers.mli @@ -31,11 +31,11 @@ type package_handler = { Note this should also process the [excludes] list. *) - ph_list_files : string -> (string * file_type) list; + ph_list_files : ?use_installed:bool -> string -> (string * file_type) list; (** [ph_list_files pkg] lists the files and file metadata in the package called [pkg] (a package file). *) - ph_get_file_from_package : string -> string -> string; + ph_get_file_from_package : ?use_installed:bool -> string -> string -> string; (** [ph_get_file_from_package pkg file] extracts the single named file [file] from [pkg]. The path of the extracted file is returned. *) diff --git a/src/febootstrap_pacman.ml b/src/febootstrap_pacman.ml index 6691ebe..002ea6d 100644 --- a/src/febootstrap_pacman.ml +++ b/src/febootstrap_pacman.ml @@ -71,7 +71,8 @@ let pacman_resolve_dependencies_and_download names = List.sort compare pkgs -let pacman_list_files pkg = +(* fixme: use_installed *) +let pacman_list_files ?(use_installed=false) pkg = debug "unpacking %s ..." pkg; (* We actually need to extract the file in order to get the @@ -117,7 +118,8 @@ let pacman_list_files pkg = files (* Easy because we already unpacked the archive above. *) -let pacman_get_file_from_package pkg file = +(* fixme: use_installed *) +let pacman_get_file_from_package ?(use_installed=false) pkg file = tmpdir // pkg ^ ".d" // file let () = diff --git a/src/febootstrap_yum_rpm.ml b/src/febootstrap_yum_rpm.ml index 028492a..d208e8e 100644 --- a/src/febootstrap_yum_rpm.ml +++ b/src/febootstrap_yum_rpm.ml @@ -172,7 +172,7 @@ if verbose: sprintf "%s/%s-%s-%s.%s.rpm" tmpdir name version release arch ) pkgs -let rec yum_rpm_list_files pkg = +let rec yum_rpm_list_files ?(use_installed=false) pkg = (* Run rpm -qlp with some extra magic. *) let cmd = sprintf "rpm -q --qf '[%%{FILENAMES} %%{FILEFLAGS:fflags} %%{FILEMODES} %%{FILESIZES}\\n]' -p %s" @@ -228,7 +228,8 @@ let rec yum_rpm_list_files pkg = files -let yum_rpm_get_file_from_package pkg file = +(* fixme use_installed *) +let yum_rpm_get_file_from_package ?(use_installed=false) pkg file = debug "extracting %s from %s ..." file (Filename.basename pkg); let outfile = tmpdir // file in -- 1.7.6.3
On Thu, Sep 22, 2011 at 08:55:30PM +0200, Hilko Bengen wrote:
+ let files = (ph.ph_list_files ~use_installed:use_installed pkg) in
In OCaml, f ~a:a can be (and normally is) abbreviated to: f ~a so you can just write 'ph.ph_list_files ~use_installed pkg' here.
+(* fixme: use_installed *) +let pacman_list_files ?(use_installed=false) pkg =
Instead of the comment, I'd prefer: let pacman_list_files ?(use_installed=false) pkg = if use_installed then failwith "pacman driver doesn't support --use-installed"; ...rest... Also this needs an addition to febootstrap.pod (the man page). Apart from those things, the patch looks good. Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones virt-df lists disk usage of guests without needing to install any software inside the virtual machine. Supports Linux and Windows. http://et.redhat.com/~rjones/virt-df/
--- src/febootstrap.ml | 4 ++-- src/febootstrap.pod | 4 ++++ src/febootstrap_cmdline.ml | 2 +- src/febootstrap_pacman.ml | 8 ++++++-- src/febootstrap_yum_rpm.ml | 7 ++++++- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/febootstrap.ml b/src/febootstrap.ml index 9b7dc88..82b80e2 100644 --- a/src/febootstrap.ml +++ b/src/febootstrap.ml @@ -67,7 +67,7 @@ let () = List.flatten ( List.map ( fun pkg -> - let files = (ph.ph_list_files ~use_installed:use_installed pkg) in + let files = (ph.ph_list_files ~use_installed pkg) in List.map (fun (filename, ft) -> filename, ft, pkg) files ) packages ) in @@ -320,7 +320,7 @@ let () = * original file from the package. *) else if config then ( - let outfile = ph.ph_get_file_from_package ~use_installed:use_installed pkg path in + let outfile = ph.ph_get_file_from_package ~use_installed pkg path in (* Note that the output config file might not be a regular file. *) let statbuf = lstat outfile in diff --git a/src/febootstrap.pod b/src/febootstrap.pod index ac97f48..ae6c56c 100644 --- a/src/febootstrap.pod +++ b/src/febootstrap.pod @@ -96,6 +96,10 @@ output directory then they will be overwritten. Don't remove temporary files and directories on exit. This is useful for debugging. +=item B<--use-installed> + +Inspect already installed packages for determining contents. + =item B<-v> =item B<--verbose> diff --git a/src/febootstrap_cmdline.ml b/src/febootstrap_cmdline.ml index 939afb7..fc18bbd 100644 --- a/src/febootstrap_cmdline.ml +++ b/src/febootstrap_cmdline.ml @@ -52,7 +52,7 @@ let argspec = Arg.align [ "--save-temps", Arg.Set save_temps, " Don't delete temporary files and directories on exit."; "--use-installed", Arg.Set use_installed, - " Inspect installed packages for determining contents."; + " Inspect already installed packages for determining contents."; "-v", Arg.Set verbose, " Enable verbose output"; "--verbose", Arg.Set verbose, diff --git a/src/febootstrap_pacman.ml b/src/febootstrap_pacman.ml index 002ea6d..657f4d7 100644 --- a/src/febootstrap_pacman.ml +++ b/src/febootstrap_pacman.ml @@ -71,8 +71,10 @@ let pacman_resolve_dependencies_and_download names = List.sort compare pkgs -(* fixme: use_installed *) let pacman_list_files ?(use_installed=false) pkg = + if use_installed then + failwith "pacman driver doesn't support --use-installed"; + debug "unpacking %s ..." pkg; (* We actually need to extract the file in order to get the @@ -118,8 +120,10 @@ let pacman_list_files ?(use_installed=false) pkg = files (* Easy because we already unpacked the archive above. *) -(* fixme: use_installed *) let pacman_get_file_from_package ?(use_installed=false) pkg file = + if use_installed then + failwith "pacman driver doesn't support --use-installed"; + tmpdir // pkg ^ ".d" // file let () = diff --git a/src/febootstrap_yum_rpm.ml b/src/febootstrap_yum_rpm.ml index d208e8e..815c5ba 100644 --- a/src/febootstrap_yum_rpm.ml +++ b/src/febootstrap_yum_rpm.ml @@ -173,6 +173,9 @@ if verbose: ) pkgs let rec yum_rpm_list_files ?(use_installed=false) pkg = + if use_installed then + failwith "yum_rpm driver doesn't support --use-installed"; + (* Run rpm -qlp with some extra magic. *) let cmd = sprintf "rpm -q --qf '[%%{FILENAMES} %%{FILEFLAGS:fflags} %%{FILEMODES} %%{FILESIZES}\\n]' -p %s" @@ -228,8 +231,10 @@ let rec yum_rpm_list_files ?(use_installed=false) pkg = files -(* fixme use_installed *) let yum_rpm_get_file_from_package ?(use_installed=false) pkg file = + if use_installed then + failwith "yum_rpm driver doesn't support --use-installed"; + debug "extracting %s from %s ..." file (Filename.basename pkg); let outfile = tmpdir // file in -- 1.7.6.3
Thanks, ACKed and pushed. I made a couple of minor changes, and squashed the three patches together into a single commit. Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones libguestfs lets you edit virtual machines. Supports shell scripting, bindings from many languages. http://libguestfs.org
participants (2)
-
Hilko Bengen -
Richard W.M. Jones