[PATCH] febootstrap: Use contents of installed Debian packages instead of downloading and unpacking them.
4:58 p.m.
---
src/febootstrap_debian.ml | 65 +++++++++++++++++++++++++++++++++++++-------
1 files changed, 54 insertions(+), 11 deletions(-)
diff --git a/src/febootstrap_debian.ml b/src/febootstrap_debian.ml
index 23f3593..bb10f15 100644
--- a/src/febootstrap_debian.ml
+++ b/src/febootstrap_debian.ml
@@ -28,6 +28,9 @@ open Febootstrap_cmdline
(* Create a temporary directory for use by all the functions in this file. *)
let tmpdir = tmpdir ()
+let installed_pkgs =
+ run_command_get_lines "dpkg-query --show
--showformat='${Package}\\n'"
+
let debian_detect () =
file_exists "/etc/debian_version" &&
Config.aptitude <> "no" && Config.apt_cache <>
"no" && Config.dpkg <> "no"
@@ -51,18 +54,29 @@ let rec debian_resolve_dependencies_and_download names =
not (List.exists (fun re -> Str.string_match re name 0) excludes)
) pkgs in
+ let present_pkgs, download_pkgs = List.partition (
+ fun pkg -> List.exists ((=) pkg) installed_pkgs
+ ) pkgs in
+
+ debug "wanted packages (present / download): %s / %s\n"
+ (String.concat " " present_pkgs)
+ (String.concat " " download_pkgs);
+
(* Download the packages. *)
- let cmd =
- sprintf "umask 0000; cd %s && %s download %s"
- (Filename.quote tmpdir)
- Config.aptitude
- (String.concat " " (List.map Filename.quote pkgs)) in
- run_command cmd;
+ if (List.length download_pkgs > 0)
+ then (
+ let cmd =
+ sprintf "umask 0000; cd %s && %s download %s"
+ (Filename.quote tmpdir)
+ Config.aptitude
+ (String.concat " " (List.map Filename.quote download_pkgs)) in
+ run_command cmd
+ );
(* Find out what aptitude downloaded. *)
let files = Sys.readdir tmpdir in
- let pkgs = List.map (
+ let download_pkgs = List.map (
fun pkg ->
(* Look for 'pkg_*.deb' in the list of files. *)
let pre = pkg ^ "_" in
@@ -79,9 +93,9 @@ let rec debian_resolve_dependencies_and_download names =
exit 1
with
Exit -> !r
- ) pkgs in
+ ) download_pkgs in
- List.sort compare pkgs
+ List.sort compare (List.append present_pkgs download_pkgs)
(* On Ubuntu 10.04 LTS, apt-cache depends --recurse is broken. It
* doesn't return the full list of dependencies. Therefore recurse
@@ -106,7 +120,7 @@ and workaround_broken_apt_cache_depends_recurse names =
else
names
-let debian_list_files pkg =
+let debian_list_files_downloaded pkg =
debug "unpacking %s ..." pkg;
(* We actually need to extract the file in order to get the
@@ -147,9 +161,38 @@ let debian_list_files pkg =
files
+let debian_list_files_installed pkg =
+ debug "using installed package %s ..." pkg;
+ let cmd = sprintf "dpkg-query --listfiles %s" pkg in
+ let lines = run_command_get_lines cmd in
+ (* filter out lines not directly describing fs objects such as
+ "package diverts others to: /path/to/..." *)
+ let lines = List.filter (
+ fun l -> l.[0] = '/' && l.[1] != '.'
+ ) lines in
+ let files = List.map (
+ fun path ->
+ let statbuf = lstat path in
+ let is_dir = statbuf.st_kind = S_DIR in
+ let config = statbuf.st_kind = S_REG && string_prefix "/etc/"
path in
+ let mode = statbuf.st_perm in
+ (path, { ft_dir = is_dir; ft_config = config; ft_mode = mode;
+ ft_ghost = false; ft_size = statbuf.st_size })
+ ) lines in
+ files
+
+let debian_list_files pkg =
+ if List.exists ((=) pkg) installed_pkgs then
+ debian_list_files_installed pkg
+ else
+ debian_list_files_downloaded pkg
+
(* Easy because we already unpacked the archive above. *)
let debian_get_file_from_package pkg file =
- tmpdir // pkg ^ ".d" // file
+ if List.exists (fun p -> p = pkg) installed_pkgs then
+ file
+ else
+ tmpdir // pkg ^ ".d" // file
let () =
let ph = {
--
1.7.5.4
6:32 a.m.
On Mon, Sep 12, 2011 at 11:58:09PM +0200, Hilko Bengen wrote:
---
src/febootstrap_debian.ml | 65 +++++++++++++++++++++++++++++++++++++-------
1 files changed, 54 insertions(+), 11 deletions(-)
Update on this one since I've been busy investigating a serious
bug in RHEL.
We decided that this functionality was great, but that it should be
available optionally, and across at least the Debian and Yum backends.
It would be useful for Fedora builds too.
----
On a separate note, febootstrap-supermin-helper could use a 9pfs
backend. This would make boot times even faster, while reducing the
complexity of the boot process -- in other words, it's all upside.
It seems that the hard bit of this is modifying the 9p server inside
qemu to make it more flexible. It would be good if the 9p server
could be driven a text configuration file:
/etc directory
/etc/passwd replace /tmp/base/etc/passwd mode 0644 owner root group root
/usr/bin directory
/usr/bin/ls host
/usr/bin/vi host
This describes to qemu how to synthesize a filesystem. Some files
comes from host files directly (hostfiles), and some files are
replacement files (from base.img).
Then all the febootstrap-supermin-helper backend would need to do
would be to unpack base.img and construct this simple text config
file.
Rich.
--
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
virt-top is 'top' for virtual machines. Tiny program with many
powerful monitoring features, net stats, disk stats, logging, etc.
http://et.redhat.com/~rjones/virt-top
1:55 p.m.
New subject: [PATCH] Added --use-installed switch (which for now only works for Debian).
---
src/febootstrap.ml | 4 ++--
src/febootstrap_cmdline.ml | 4 ++++
src/febootstrap_cmdline.mli | 3 +++
src/febootstrap_debian.ml | 8 ++++----
src/febootstrap_package_handlers.ml | 4 ++--
src/febootstrap_package_handlers.mli | 4 ++--
src/febootstrap_pacman.ml | 6 ++++--
src/febootstrap_yum_rpm.ml | 5 +++--
8 files changed, 24 insertions(+), 14 deletions(-)
diff --git a/src/febootstrap.ml b/src/febootstrap.ml
index 7e48206..9b7dc88 100644
--- a/src/febootstrap.ml
+++ b/src/febootstrap.ml
@@ -67,7 +67,7 @@ let () =
List.flatten (
List.map (
fun pkg ->
- let files = ph.ph_list_files pkg in
+ let files = (ph.ph_list_files ~use_installed:use_installed pkg) in
List.map (fun (filename, ft) -> filename, ft, pkg) files
) packages
) in
@@ -320,7 +320,7 @@ let () =
* original file from the package.
*)
else if config then (
- let outfile = ph.ph_get_file_from_package pkg path in
+ let outfile = ph.ph_get_file_from_package ~use_installed:use_installed pkg path
in
(* Note that the output config file might not be a regular file. *)
let statbuf = lstat outfile in
diff --git a/src/febootstrap_cmdline.ml b/src/febootstrap_cmdline.ml
index 667e297..939afb7 100644
--- a/src/febootstrap_cmdline.ml
+++ b/src/febootstrap_cmdline.ml
@@ -23,6 +23,7 @@ let names_mode = ref false
let outputdir = ref "."
let packages = ref []
let save_temps = ref false
+let use_installed = ref false
let verbose = ref false
let warnings = ref true
let yum_config = ref None
@@ -50,6 +51,8 @@ let argspec = Arg.align [
" Don't delete temporary files and directories on exit.";
"--save-temps", Arg.Set save_temps,
" Don't delete temporary files and directories on exit.";
+ "--use-installed", Arg.Set use_installed,
+ " Inspect installed packages for determining contents.";
"-v", Arg.Set verbose,
" Enable verbose output";
"--verbose", Arg.Set verbose,
@@ -89,6 +92,7 @@ let names_mode = !names_mode
let outputdir = !outputdir
let packages = List.rev !packages
let save_temps = !save_temps
+let use_installed = !use_installed
let verbose = !verbose
let warnings = !warnings
let yum_config = !yum_config
diff --git a/src/febootstrap_cmdline.mli b/src/febootstrap_cmdline.mli
index d948d80..a545012 100644
--- a/src/febootstrap_cmdline.mli
+++ b/src/febootstrap_cmdline.mli
@@ -38,6 +38,9 @@ val packages : string list
val save_temps : bool
(** True if [--save-temps] was given on the command line. *)
+val use_installed : bool
+ (** True if [--use-installed] was given on the command line *)
+
val verbose : bool
(** True if [--verbose] was given on the command line.
See also {!debug}. *)
diff --git a/src/febootstrap_debian.ml b/src/febootstrap_debian.ml
index bb10f15..7482ed4 100644
--- a/src/febootstrap_debian.ml
+++ b/src/febootstrap_debian.ml
@@ -181,15 +181,15 @@ let debian_list_files_installed pkg =
) lines in
files
-let debian_list_files pkg =
- if List.exists ((=) pkg) installed_pkgs then
+let debian_list_files ?(use_installed=false) pkg =
+ if use_installed && List.exists ((=) pkg) installed_pkgs then
debian_list_files_installed pkg
else
debian_list_files_downloaded pkg
(* Easy because we already unpacked the archive above. *)
-let debian_get_file_from_package pkg file =
- if List.exists (fun p -> p = pkg) installed_pkgs then
+let debian_get_file_from_package ?(use_installed=false) pkg file =
+ if use_installed && List.exists (fun p -> p = pkg) installed_pkgs then
file
else
tmpdir // pkg ^ ".d" // file
diff --git a/src/febootstrap_package_handlers.ml b/src/febootstrap_package_handlers.ml
index ad3a233..f627d2f 100644
--- a/src/febootstrap_package_handlers.ml
+++ b/src/febootstrap_package_handlers.ml
@@ -25,8 +25,8 @@ open Febootstrap_cmdline
type package_handler = {
ph_detect : unit -> bool;
ph_resolve_dependencies_and_download : string list -> string list;
- ph_list_files : string -> (string * file_type) list;
- ph_get_file_from_package : string -> string -> string
+ ph_list_files : ?use_installed:bool -> string -> (string * file_type) list;
+ ph_get_file_from_package : ?use_installed:bool -> string -> string -> string
}
and file_type = {
ft_dir : bool;
diff --git a/src/febootstrap_package_handlers.mli b/src/febootstrap_package_handlers.mli
index c28d81f..ebf0386 100644
--- a/src/febootstrap_package_handlers.mli
+++ b/src/febootstrap_package_handlers.mli
@@ -31,11 +31,11 @@ type package_handler = {
Note this should also process the [excludes] list. *)
- ph_list_files : string -> (string * file_type) list;
+ ph_list_files : ?use_installed:bool -> string -> (string * file_type) list;
(** [ph_list_files pkg] lists the files and file metadata in the
package called [pkg] (a package file). *)
- ph_get_file_from_package : string -> string -> string;
+ ph_get_file_from_package : ?use_installed:bool -> string -> string -> string;
(** [ph_get_file_from_package pkg file] extracts the
single named file [file] from [pkg]. The path of the
extracted file is returned. *)
diff --git a/src/febootstrap_pacman.ml b/src/febootstrap_pacman.ml
index 6691ebe..002ea6d 100644
--- a/src/febootstrap_pacman.ml
+++ b/src/febootstrap_pacman.ml
@@ -71,7 +71,8 @@ let pacman_resolve_dependencies_and_download names =
List.sort compare pkgs
-let pacman_list_files pkg =
+(* fixme: use_installed *)
+let pacman_list_files ?(use_installed=false) pkg =
debug "unpacking %s ..." pkg;
(* We actually need to extract the file in order to get the
@@ -117,7 +118,8 @@ let pacman_list_files pkg =
files
(* Easy because we already unpacked the archive above. *)
-let pacman_get_file_from_package pkg file =
+(* fixme: use_installed *)
+let pacman_get_file_from_package ?(use_installed=false) pkg file =
tmpdir // pkg ^ ".d" // file
let () =
diff --git a/src/febootstrap_yum_rpm.ml b/src/febootstrap_yum_rpm.ml
index 028492a..d208e8e 100644
--- a/src/febootstrap_yum_rpm.ml
+++ b/src/febootstrap_yum_rpm.ml
@@ -172,7 +172,7 @@ if verbose:
sprintf "%s/%s-%s-%s.%s.rpm" tmpdir name version release arch
) pkgs
-let rec yum_rpm_list_files pkg =
+let rec yum_rpm_list_files ?(use_installed=false) pkg =
(* Run rpm -qlp with some extra magic. *)
let cmd =
sprintf "rpm -q --qf '[%%{FILENAMES} %%{FILEFLAGS:fflags} %%{FILEMODES}
%%{FILESIZES}\\n]' -p %s"
@@ -228,7 +228,8 @@ let rec yum_rpm_list_files pkg =
files
-let yum_rpm_get_file_from_package pkg file =
+(* fixme use_installed *)
+let yum_rpm_get_file_from_package ?(use_installed=false) pkg file =
debug "extracting %s from %s ..." file (Filename.basename pkg);
let outfile = tmpdir // file in
--
1.7.6.3
4:43 p.m.
New subject: [PATCH] Added --use-installed switch (which for now only works for Debian).
On Thu, Sep 22, 2011 at 08:55:30PM +0200, Hilko Bengen wrote:
+ let files = (ph.ph_list_files ~use_installed:use_installed
pkg) in
In OCaml,
f ~a:a
can be (and normally is) abbreviated to:
f ~a
so you can just write 'ph.ph_list_files ~use_installed pkg' here.
+(* fixme: use_installed *)
+let pacman_list_files ?(use_installed=false) pkg =
Instead of the comment, I'd prefer:
let pacman_list_files ?(use_installed=false) pkg =
if use_installed then
failwith "pacman driver doesn't support --use-installed";
...rest...
Also this needs an addition to febootstrap.pod (the man page).
Apart from those things, the patch looks good.
Rich.
--
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
virt-df lists disk usage of guests without needing to install any
software inside the virtual machine. Supports Linux and Windows.
http://et.redhat.com/~rjones/virt-df/
12:15 p.m.
New subject: [PATCH] Incorporated suggestions for --use-installed
---
src/febootstrap.ml | 4 ++--
src/febootstrap.pod | 4 ++++
src/febootstrap_cmdline.ml | 2 +-
src/febootstrap_pacman.ml | 8 ++++++--
src/febootstrap_yum_rpm.ml | 7 ++++++-
5 files changed, 19 insertions(+), 6 deletions(-)
diff --git a/src/febootstrap.ml b/src/febootstrap.ml
index 9b7dc88..82b80e2 100644
--- a/src/febootstrap.ml
+++ b/src/febootstrap.ml
@@ -67,7 +67,7 @@ let () =
List.flatten (
List.map (
fun pkg ->
- let files = (ph.ph_list_files ~use_installed:use_installed pkg) in
+ let files = (ph.ph_list_files ~use_installed pkg) in
List.map (fun (filename, ft) -> filename, ft, pkg) files
) packages
) in
@@ -320,7 +320,7 @@ let () =
* original file from the package.
*)
else if config then (
- let outfile = ph.ph_get_file_from_package ~use_installed:use_installed pkg path
in
+ let outfile = ph.ph_get_file_from_package ~use_installed pkg path in
(* Note that the output config file might not be a regular file. *)
let statbuf = lstat outfile in
diff --git a/src/febootstrap.pod b/src/febootstrap.pod
index ac97f48..ae6c56c 100644
--- a/src/febootstrap.pod
+++ b/src/febootstrap.pod
@@ -96,6 +96,10 @@ output directory then they will be overwritten.
Don't remove temporary files and directories on exit. This is useful
for debugging.
+=item B<--use-installed>
+
+Inspect already installed packages for determining contents.
+
=item B<-v>
=item B<--verbose>
diff --git a/src/febootstrap_cmdline.ml b/src/febootstrap_cmdline.ml
index 939afb7..fc18bbd 100644
--- a/src/febootstrap_cmdline.ml
+++ b/src/febootstrap_cmdline.ml
@@ -52,7 +52,7 @@ let argspec = Arg.align [
"--save-temps", Arg.Set save_temps,
" Don't delete temporary files and directories on exit.";
"--use-installed", Arg.Set use_installed,
- " Inspect installed packages for determining contents.";
+ " Inspect already installed packages for determining contents.";
"-v", Arg.Set verbose,
" Enable verbose output";
"--verbose", Arg.Set verbose,
diff --git a/src/febootstrap_pacman.ml b/src/febootstrap_pacman.ml
index 002ea6d..657f4d7 100644
--- a/src/febootstrap_pacman.ml
+++ b/src/febootstrap_pacman.ml
@@ -71,8 +71,10 @@ let pacman_resolve_dependencies_and_download names =
List.sort compare pkgs
-(* fixme: use_installed *)
let pacman_list_files ?(use_installed=false) pkg =
+ if use_installed then
+ failwith "pacman driver doesn't support --use-installed";
+
debug "unpacking %s ..." pkg;
(* We actually need to extract the file in order to get the
@@ -118,8 +120,10 @@ let pacman_list_files ?(use_installed=false) pkg =
files
(* Easy because we already unpacked the archive above. *)
-(* fixme: use_installed *)
let pacman_get_file_from_package ?(use_installed=false) pkg file =
+ if use_installed then
+ failwith "pacman driver doesn't support --use-installed";
+
tmpdir // pkg ^ ".d" // file
let () =
diff --git a/src/febootstrap_yum_rpm.ml b/src/febootstrap_yum_rpm.ml
index d208e8e..815c5ba 100644
--- a/src/febootstrap_yum_rpm.ml
+++ b/src/febootstrap_yum_rpm.ml
@@ -173,6 +173,9 @@ if verbose:
) pkgs
let rec yum_rpm_list_files ?(use_installed=false) pkg =
+ if use_installed then
+ failwith "yum_rpm driver doesn't support --use-installed";
+
(* Run rpm -qlp with some extra magic. *)
let cmd =
sprintf "rpm -q --qf '[%%{FILENAMES} %%{FILEFLAGS:fflags} %%{FILEMODES}
%%{FILESIZES}\\n]' -p %s"
@@ -228,8 +231,10 @@ let rec yum_rpm_list_files ?(use_installed=false) pkg =
files
-(* fixme use_installed *)
let yum_rpm_get_file_from_package ?(use_installed=false) pkg file =
+ if use_installed then
+ failwith "yum_rpm driver doesn't support --use-installed";
+
debug "extracting %s from %s ..." file (Filename.basename pkg);
let outfile = tmpdir // file in
--
1.7.6.3
1:30 p.m.
New subject: [PATCH] Incorporated suggestions for --use-installed
Thanks, ACKed and pushed.
I made a couple of minor changes, and squashed the three patches
together into a single commit.
Rich.
--
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
libguestfs lets you edit virtual machines. Supports shell scripting,
bindings from many languages. http://libguestfs.org
4842
days inactive
4853
days old
5 comments
2 participants
participants (2)
-
Hilko Bengen -
Richard W.M. Jones