February 2014 - Libguestfs - Libguestfs List Archives

by Richard W.M. Jones

This requires the new version of supermin (5.1.0). --- .gitignore | 6 +- README | 5 +- appliance/Makefile.am | 102 ++-- appliance/excludefiles.in | 29 ++ appliance/excludelist.in | 57 --- appliance/hostfiles.in | 13 + appliance/libguestfs-make-fixed-appliance.in | 2 +- appliance/libguestfs-make-fixed-appliance.pod | 3 +- appliance/make.sh.in | 36 +- appliance/packagelist.in | 88 ++-- configure.ac | 79 +-- examples/guestfs-performance.pod | 11 +- fish/guestfish.pod | 10 +- src/appliance.c | 675 ++++---------------------- src/guestfs.pod | 21 +- test-tool/libguestfs-test-tool.pod | 3 +- 16 files changed, 274 insertions(+), 866 deletions(-) create mode 100644 appliance/excludefiles.in delete mode 100644 appliance/excludelist.in create mode 100644 appliance/hostfiles.in diff --git a/.gitignore b/.gitignore index 1ee7775..2813872 100644 --- a/.gitignore +++ b/.gitignore @@ -37,7 +37,10 @@ Makefile.in /align/stamp-virt-alignment-scan.pod /align/virt-alignment-scan /align/virt-alignment-scan.1 -/appliance/excludelist +/appliance/daemon.tar.gz +/appliance/excludefiles +/appliance/hostfiles +/appliance/init.tar.gz /appliance/libguestfs-make-fixed-appliance /appliance/libguestfs-make-fixed-appliance.1 /appliance/make.sh @@ -45,6 +48,7 @@ Makefile.in /appliance/stamp-libguestfs-make-fixed-appliance.pod /appliance/stamp-supermin /appliance/supermin.d +/appliance/udev-rules.tar.gz /autom4te.cache /bash/virt-builder /bash/virt-cat diff --git a/README b/README index 3f477f0..8fdc041 100644 --- a/README +++ b/README @@ -61,12 +61,11 @@ The full requirements are described below. | | | | - virtio-block | | | | | - virtio-net | +--------------+-------------+---+-----------------------------------------+ -| supermin | 4.1.0 | R | This is required on all distros. | -| febootstrap | 3.20 | | 'supermin' is the new name for | +| supermin | 5.1.0 | R | This is required on all distros. | +| | | | 'supermin' is the new name for | | | | | 'febootstrap'. | | | | | For alternatives, see: | | | | | libguestfs.org/download/binaries/appliance/ -| | | | febootstrap 2.x WILL NOT WORK | +--------------+-------------+---+-----------------------------------------+ | glibc | | R | We use various glibc-isms. | | | | | Also glibc provides XDR, rpcgen. | diff --git a/appliance/Makefile.am b/appliance/Makefile.am index a9cc9fe..b98ddf5 100644 --- a/appliance/Makefile.am +++ b/appliance/Makefile.am @@ -1,5 +1,5 @@ # libguestfs -# Copyright (C) 2009 Red Hat Inc. +# Copyright (C) 2009-2014 Red Hat Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,8 +19,9 @@ include $(top_srcdir)/subdir-rules.mk EXTRA_DIST = \ 99-guestfs-serial.rules \ - excludelist.in \ + excludefiles.in \ guestfsd.suppressions \ + hostfiles.in \ init \ libguestfs-make-fixed-appliance.in \ libguestfs-make-fixed-appliance.pod \ @@ -32,24 +33,42 @@ superminfsdir = $(libdir)/guestfs/supermin.d fs_DATA = superminfs_DATA = \ - supermin.d/init.img \ - supermin.d/udev-rules.img - -if SUPERMIN_HELPER_COMPRESSED_CPIO -GZ = .gz -endif + supermin.d/init.tar.gz \ + supermin.d/udev-rules.tar.gz if ENABLE_DAEMON superminfs_DATA += \ - supermin.d/daemon.img$(GZ) + supermin.d/daemon.tar.gz endif if ENABLE_APPLIANCE superminfs_DATA += \ - supermin.d/base.img$(GZ) \ + supermin.d/base.tar.gz \ + supermin.d/packages \ + supermin.d/excludefiles \ supermin.d/hostfiles endif +supermin.d/base.tar.gz \ +supermin.d/daemon.tar.gz \ +supermin.d/excludefiles \ +supermin.d/hostfiles \ +supermin.d/init.tar.gz \ +supermin.d/packages \ +supermin.d/udev-rules.tar.gz: stamp-supermin +stamp-supermin: make.sh \ + packagelist \ + hostfiles \ + excludefiles \ + daemon.tar.gz \ + init.tar.gz \ + udev-rules.tar.gz + rm -f $@ supermin.d/base.tar.gz supermin.d/packages + ./make.sh + cp -t supermin.d \ + daemon.tar.gz excludefiles hostfiles init.tar.gz udev-rules.tar.gz + touch $@ + # This used to be a configure-generated file. However config.status # always touches the destination file, which means the appliance got # rebuilt too often. @@ -65,57 +84,47 @@ if VALGRIND_DAEMON PACKAGELIST_CPP_FLAGS += -DVALGRIND_DAEMON=1 endif +excludefiles: excludefiles.in Makefile + m4 $(PACKAGELIST_CPP_FLAGS) $< | \ + grep -v '^[[:space:]]*$$' | grep -v '^#' > $@-t + cmp -s $@ $@-t || mv $@-t $@ + rm -f $@-t + +hostfiles: hostfiles.in Makefile + m4 $(PACKAGELIST_CPP_FLAGS) $< | \ + grep -v '^[[:space:]]*$$' | grep -v '^#' > $@-t + cmp -s $@ $@-t || mv $@-t $@ + rm -f $@-t + packagelist: packagelist.in Makefile - cpp -undef $(PACKAGELIST_CPP_FLAGS) < $< | \ + m4 $(PACKAGELIST_CPP_FLAGS) $< | \ grep -v '^[[:space:]]*$$' | grep -v '^#' > $@-t cmp -s $@ $@-t || mv $@-t $@ rm -f $@-t -excludelist: excludelist.in Makefile - cpp -undef $(PACKAGELIST_CPP_FLAGS) < $< | \ - grep -v '^[[:space:]]*$$' | grep -v '^#' > $@-t - cmp -s $@ $@-t || mv $@-t $@ - rm -f $@-t - -supermin.d/base.img$(GZ) supermin.d/hostfiles: stamp-supermin -stamp-supermin: make.sh packagelist excludelist - rm -f $@ supermin.d/base.img$(GZ) supermin.d/hostfiles - ./make.sh -if SUPERMIN_HELPER_COMPRESSED_CPIO - gzip -9 supermin.d/base.img -endif - touch $@ - -supermin.d/daemon.img$(GZ): ../daemon/guestfsd guestfsd.suppressions - rm -f $@ $@-t $@-tt +daemon.tar.gz: ../daemon/guestfsd guestfsd.suppressions + rm -f $@ $@-t rm -rf tmp-d mkdir -p tmp-d$(DAEMON_SUPERMIN_DIR) tmp-d/etc ln ../daemon/guestfsd tmp-d$(DAEMON_SUPERMIN_DIR)/guestfsd ln $(srcdir)/guestfsd.suppressions tmp-d/etc/guestfsd.suppressions - ( cd tmp-d && find | cpio --quiet -o -H newc ) > $@-t + ( cd tmp-d && tar zcf - * ) > $@-t rm -r tmp-d -if SUPERMIN_HELPER_COMPRESSED_CPIO - gzip -9 -c $@-t > $@-tt - mv $@-tt $@-t -endif mv $@-t $@ -supermin.d/init.img: init - rm -rf init.tmp $@ $@-t - mkdir init.tmp - cp $< init.tmp - (cd init.tmp; echo "init" | cpio --quiet -o -H newc) > $@-t - rm -r init.tmp +init.tar.gz: init + rm -f $@ $@-t + tar zcf $@-t init mv $@-t $@ # We should put this file in /lib/udev/rules.d, but put it in /etc so # we don't have to deal with all the UsrMove crap in Fedora. -supermin.d/udev-rules.img: 99-guestfs-serial.rules +udev-rules.tar.gz: 99-guestfs-serial.rules rm -f $@ $@-t rm -rf tmp-u mkdir -p tmp-u/etc/udev/rules.d for f in $^; do ln $$f tmp-u/etc/udev/rules.d/$$(basename $$f); done - ( cd tmp-u && find | cpio --quiet -o -H newc ) > $@-t + ( cd tmp-u && tar zcf - etc ) > $@-t rm -r tmp-u mv $@-t $@ @@ -144,7 +153,14 @@ stamp-libguestfs-make-fixed-appliance.pod: libguestfs-make-fixed-appliance.pod # Make clean. -CLEANFILES = packagelist excludelist \ +CLEANFILES = \ + *~ \ + daemon.tar.gz \ + excludefiles \ + hostfiles \ + init.tar.gz \ libguestfs-make-fixed-appliance.1 \ + packagelist \ stamp-libguestfs-make-fixed-appliance.pod \ - supermin.d/* + supermin.d/* \ + udev-rules.tar.gz diff --git a/appliance/excludefiles.in b/appliance/excludefiles.in new file mode 100644 index 0000000..9fce4a3 --- /dev/null +++ b/appliance/excludefiles.in @@ -0,0 +1,29 @@ +dnl This is the list of files excluded from the appliance, even if +dnl they appear in packagelist.in (or more likely, as dependencies of +dnl packages in packagelist.in). +dnl +dnl List is a list of wildcards, one per line, prefixed by a '-' character. +dnl +dnl This file is processed by m4 with one of the +dnl following symbols defined (depending on the distro): +dnl +dnl REDHAT=1 For Fedora, RHEL, EPEL and workalikes. +dnl DEBIAN=1 For Debian. +dnl UBUNTU=1 For Ubuntu. +dnl ARCHLINUX=1 For Archlinux. +dnl +dnl Note that any matching file will be dropped from the appliance. +dnl Of course, this may break the appliance, so be careful. + +dnl The right kernel modules are added back by supermin. +-/boot/* +-/lib/modules/* + +-/usr/lib/locale/* +-/usr/share/locale/* +-/usr/share/man/* +-/usr/share/doc/* +-/usr/share/info/* +-/usr/share/gnome/help/* +-/usr/share/cracklib/* +-/usr/share/i18n/* diff --git a/appliance/excludelist.in b/appliance/excludelist.in deleted file mode 100644 index aa4c406..0000000 --- a/appliance/excludelist.in +++ /dev/null @@ -1,57 +0,0 @@ -/* This is the list of distro packages which are - * excluded from the appliance, even if they appear in - * packagelist.in (or more likely, as dependencies of - * packages in packagelist.in). - * - * List is a list of basic regular expressions, one per line. - * - * This file is processed by cpp with one of the - * following symbols defined (depending on the distro): - * - * REDHAT=1 For Fedora, RHEL, EPEL and workalikes. - * DEBIAN=1 For Debian. - * UBUNTU=1 For Ubuntu. - * ARCHLINUX=1 For Archlinux. - * - * Note that any file provided by one of these packages will - * be dropped from the appliance. Of course, this may break - * the appliance, so be careful. Other files are also dropped - * from the appliance such as docs and man pages: see 'make.sh.in' - * for the full details. - */ - -/* Basically the same with a few minor tweaks. */ -#ifdef UBUNTU -#define DEBIAN 1 -#endif - -/* Don't need any Perl or Python appearing in the appliance. */ -^perl -^python - -/* Plymouth is a graphical boot thing - not needed. */ -^plymouth - -/* Linux firmware. */ -^linux-firmware - -/* Keyboard maps - appliance is not interactive. */ -^kbd-misc - -#ifdef REDHAT - -/* Linux kernel. febootstrap <= 3.18 used to exclude the kernel - * package (only) by default, but since 3.19 it doesn't do this any - * longer. - */ -^kernel - -^fedora-logos -^redhat-logos -^dracut - -#endif - -#ifdef DEBIAN -^file-rc -#endif diff --git a/appliance/hostfiles.in b/appliance/hostfiles.in new file mode 100644 index 0000000..5ad46ea --- /dev/null +++ b/appliance/hostfiles.in @@ -0,0 +1,13 @@ +dnl This is the list of extra files added to appliance. +dnl +dnl List is a list of wildcards, one per line. +dnl +dnl This file is processed by m4 with one of the +dnl following symbols defined (depending on the distro): +dnl +dnl REDHAT=1 For Fedora, RHEL, EPEL and workalikes. +dnl DEBIAN=1 For Debian. +dnl UBUNTU=1 For Ubuntu. +dnl ARCHLINUX=1 For Archlinux. + +/usr/share/augeas/lenses/*.aug diff --git a/appliance/libguestfs-make-fixed-appliance.in b/appliance/libguestfs-make-fixed-appliance.in index 121edc5..500f406 100644 --- a/appliance/libguestfs-make-fixed-appliance.in +++ b/appliance/libguestfs-make-fixed-appliance.in @@ -105,7 +105,7 @@ guestfish -a /dev/null run # Find the location of the appliance. cachedir="$(guestfish get-cachedir)" euid="$(id -u)" -appliancedir="$cachedir/.guestfs-$euid" +appliancedir="$cachedir/.guestfs-$euid/appliance.d" cp "$appliancedir/kernel" "$outputdir/kernel" cp "$appliancedir/initrd" "$outputdir/initrd" diff --git a/appliance/libguestfs-make-fixed-appliance.pod b/appliance/libguestfs-make-fixed-appliance.pod index e5a216d..ab0b0c9 100644 --- a/appliance/libguestfs-make-fixed-appliance.pod +++ b/appliance/libguestfs-make-fixed-appliance.pod @@ -130,7 +130,7 @@ be set using the C<LIBGUESTFS_PATH> environment variable. Normally a supermin appliance is located on this path (see L<supermin(1)/SUPERMIN APPLIANCE>). libguestfs reconstructs this -into a full appliance by running L<supermin-helper(1)>. +into a full appliance by running C<supermin --build>. However, a simpler "fixed appliance" can also be used. libguestfs detects this by looking for a directory on the path containing four @@ -167,7 +167,6 @@ libguestfs, please see the L<guestfs(3)> manual page. L<guestfs(3)>, L<supermin(1)>, -L<supermin-helper(1)>, L<xz(1)>, L<http://libguestfs.org/>, L<http://qemu.org/>. diff --git a/appliance/make.sh.in b/appliance/make.sh.in index 7fa494e..e23d368 100755 --- a/appliance/make.sh.in +++ b/appliance/make.sh.in @@ -20,42 +20,14 @@ unset CDPATH set -e -# Turn excludelist file into command line arguments. -exec 5<excludelist -while read regexp <&5; do - excludes="$excludes --exclude $regexp" -done -exec 5<&- +# Run supermin. -# Run supermin on the package list. -# NB: Keep using --yum-config (deprecated alias) here since both old -# and new supermin still support it. if [ "x@SUPERMIN_PACKAGER_CONFIG@" != "xno" ]; then - extra="--yum-config @SUPERMIN_PACKAGER_CONFIG@" + extra="--packager-config @SUPERMIN_PACKAGER_CONFIG@" fi if [ "x@SUPERMIN_EXTRA_OPTIONS@" != "xno" ]; then extra="$extra @SUPERMIN_EXTRA_OPTIONS@" fi -echo @SUPERMIN@ -v -o supermin.d --names $(< packagelist ) $excludes $extra -@SUPERMIN@ -v -o supermin.d --names $(< packagelist ) $excludes $extra - -# Remove some things that we don't want in the appliance. This is -# copied from the old febootstrap-minimize. However minimization is -# not so important now that we are caching the appliance. -< supermin.d/hostfiles \ -grep -v '^/usr/lib/locale' | -grep -v '^/usr/share/locale' | -grep -v '^/usr/share/man/' | -grep -v '^/usr/share/doc/' | -grep -v '^/usr/share/info/' | -grep -v '^/usr/share/gnome/help/' | -grep -v '^/usr/share/cracklib/' | -grep -v '^/usr/share/i18n/' > supermin.d/hostfiles-t - -# Include any Augeas lenses from the host. -if grep -q /usr/share/augeas/lenses supermin.d/hostfiles-t; then - echo "/usr/share/augeas/lenses/*.aug" >> supermin.d/hostfiles-t -fi - -mv supermin.d/hostfiles-t supermin.d/hostfiles +echo @SUPERMIN@ --prepare -v -o supermin.d $(< packagelist ) $extra +@SUPERMIN@ --prepare -v -o supermin.d $(< packagelist ) $extra diff --git a/appliance/packagelist.in b/appliance/packagelist.in index 2963033..d26bffb 100644 --- a/appliance/packagelist.in +++ b/appliance/packagelist.in @@ -1,34 +1,31 @@ -/* This is the list of distro packages which are - * installed on the appliance. - * - * This file is processed by cpp with one of the - * following symbols defined (depending on the distro): - * - * REDHAT=1 For Fedora, RHEL, EPEL and workalikes. - * DEBIAN=1 For Debian. - * UBUNTU=1 For Ubuntu. - * ARCHLINUX=1 For Archlinux. - * - * There is also a list of packages which are excluded if they appear - * as dependencies of the packages below. See: excludelist.in - * - * To add arbitrary extra packages, use: - * - * ./configure --with-extra-packages="gdb valgrind [etc]" - */ +dnl This is the list of distro packages which are +dnl installed on the appliance. +dnl +dnl This file is processed by m4 with one of the +dnl following symbols defined (depending on the distro): +dnl +dnl REDHAT=1 For Fedora, RHEL, EPEL and workalikes. +dnl DEBIAN=1 For Debian. +dnl UBUNTU=1 For Ubuntu. +dnl ARCHLINUX=1 For Archlinux. +dnl +dnl There is also a list of packages which are excluded if they appear +dnl as dependencies of the packages below. See: excludelist.in +dnl +dnl To add arbitrary extra packages, use: +dnl +dnl ./configure --with-extra-packages="gdb valgrind [etc]" -/* Basically the same with a few minor tweaks. */ -#ifdef UBUNTU -#define DEBIAN 1 -#endif +dnl Basically the same with a few minor tweaks. +ifelse(UBUNTU,1,define(DEBIAN,1)) -#ifdef REDHAT +ifelse(REDHAT,1, augeas-libs btrfs-progs cryptsetup - cryptsetup-luks /* old name used before Fedora 17 */ + cryptsetup-luks dnl old name used before Fedora 17 e2fsprogs - /* e4fsprogs only exists on RHEL 5, will be ignored everywhere else. */ + dnl e4fsprogs only exists on RHEL 5, will be ignored everywhere else. e4fsprogs genisoimage gfs-utils @@ -40,7 +37,7 @@ iputils kernel libcap - libldm /* only Fedora has this for now, but we should add it to others later*/ + libldm dnl only Fedora for now, others later nilfs-utils ntfsprogs ntfs-3g @@ -49,14 +46,14 @@ reiserfs-utils libselinux syslinux-extlinux - systemd /* for /sbin/reboot and udevd */ + systemd dnl for /sbin/reboot and udevd vim-minimal xz yajl zfs-fuse -#endif /* REDHAT */ +) -#ifdef DEBIAN +ifelse(DEBIAN,1, bsdmainutils btrfs-tools cryptsetup @@ -74,21 +71,21 @@ libpcre3 libyajl2 linux-image - /* syslinux 'suggests' mtools, but in reality it's a hard dependency: */ + dnl syslinux 'suggests' mtools, but in reality it's a hard dependency: mtools nilfs-tools ntfs-3g ntfsprogs openssh-client reiserfsprogs - sysvinit /* for /sbin/reboot */ + sysvinit dnl for /sbin/reboot ufsutils vim-tiny xz-utils zfs-fuse -#endif /* DEBIAN */ +) -#ifdef ARCHLINUX +ifelse(ARCHLINUX,1, augeas btrfs-progs cdrkit @@ -111,9 +108,9 @@ xz yajl zfs-fuse -#endif /* ARCHLINUX */ +) -#ifdef FRUGALWARE +ifelse(FRUGALWARE,1, augeas btrfs-progs cryptsetup-luks @@ -169,7 +166,7 @@ tar util-linux xfsprogs -#endif /* FRUGALWARE */ +) acl attr @@ -196,11 +193,9 @@ lvm2 lzop mdadm module-init-tools -/* -Enabling this pulls out 140 extra packages -into the appliance: -ocfs2-tools -*/ +dnl Enabling this pulls out 140 extra packages +dnl into the appliance: +dnl ocfs2-tools parted procps procps-ng @@ -214,17 +209,10 @@ tar udev util-linux util-linux-ng -#ifndef UBUNTU -/* on Ubuntu contains a file in /lib64 which conflicts with libc6 that has - * /lib64 as a symbolic link - */ xfsprogs -#endif zerofree -#ifdef VALGRIND_DAEMON -valgrind -#endif +ifelse(VALGRIND_DAEMON,1,valgrind) -/* Define this by doing: ./configure --with-extra-packages="..." */ +dnl Define this by doing: ./configure --with-extra-packages="..." EXTRA_PACKAGES diff --git a/configure.ac b/configure.ac index 96ae786..3d83d0a 100644 --- a/configure.ac +++ b/configure.ac @@ -447,45 +447,10 @@ AM_CONDITIONAL([ENABLE_APPLIANCE],[test "x$ENABLE_APPLIANCE" = "xyes"]) AC_MSG_RESULT([$ENABLE_APPLIANCE]) AC_SUBST([ENABLE_APPLIANCE]) -dnl Check for supermin >= 4.1.0 or febootstrap >= 3.20. -AC_CHECK_PROGS([SUPERMIN], - [supermin febootstrap],[no]) -AC_PATH_PROGS([SUPERMIN_HELPER], - [supermin-helper febootstrap-supermin-helper],[no]) - -dnl supermin >= 4.1.4 supports compressed cpio images. -AC_MSG_CHECKING([for supermin-helper version]) -supermin_helper_version=`$SUPERMIN_HELPER --version | awk '{print $2}'` -AC_MSG_RESULT([$supermin_helper_version]) -AC_MSG_CHECKING([if supermin-helper supports compressed cpio images]) -supermin_helper_version_int=`echo "$supermin_helper_version" | awk -F. '{print $1 * 1000000 + $2 * 1000 + $3}'` -echo supermin_helper_version_int=$supermin_helper_version_int >&AS_MESSAGE_LOG_FD -if test $supermin_helper_version_int -ge 4001004; then - supermin_helper_compressed_cpio=yes -else - supermin_helper_compressed_cpio=no -fi -AC_MSG_RESULT([$supermin_helper_compressed_cpio]) -AM_CONDITIONAL([SUPERMIN_HELPER_COMPRESSED_CPIO], - [test "x$supermin_helper_compressed_cpio" = "xyes"]) - -dnl supermin >= 4.1.5 supports device trees and uses a new style command -dnl syntax. -AC_MSG_CHECKING([if supermin-helper supports device trees and new style command syntax]) -if test $supermin_helper_version_int -ge 4001005; then - supermin_helper_new_style_syntax=yes - AC_DEFINE([SUPERMIN_HELPER_NEW_STYLE_SYNTAX],[1], - [Define to 1 if you have supermin-helper >= 4.1.5.]) -else - supermin_helper_new_style_syntax=no -fi -AC_MSG_RESULT([$supermin_helper_new_style_syntax]) +dnl Check for supermin >= 5.1.0. +AC_CHECK_PROGS([SUPERMIN],[supermin],[no]) dnl Pass supermin --packager-config option. -dnl -dnl Note that in febootstrap >= 3.21 / supermin >= 4.1.0, this option -dnl is now called --packager-config, although --yum-config can still -dnl be used as a deprecated alias. SUPERMIN_PACKAGER_CONFIG=no AC_MSG_CHECKING([for --with-supermin-packager-config option]) @@ -496,22 +461,6 @@ AC_ARG_WITH([supermin-packager-config], AC_MSG_RESULT([$SUPERMIN_PACKAGER_CONFIG"])], [AC_MSG_RESULT([not set])]) -AC_MSG_CHECKING([for --with-febootstrap-yum-config option (deprecated)]) -AC_ARG_WITH([febootstrap-yum-config], - [AS_HELP_STRING([--with-febootstrap-yum-config=FILE], - [pass supermin --packager-config option @<:@default=no@:>@])], - [SUPERMIN_PACKAGER_CONFIG="$withval" - AC_MSG_RESULT([$SUPERMIN_PACKAGER_CONFIG"])], - [AC_MSG_RESULT([not set])]) - -AC_MSG_CHECKING([for --with-febootstrap-packager-config option (deprecated)]) -AC_ARG_WITH([febootstrap-packager-config], - [AS_HELP_STRING([--with-febootstrap-packager-config=FILE], - [pass supermin --packager-config option @<:@default=no@:>@])], - [SUPERMIN_PACKAGER_CONFIG="$withval" - AC_MSG_RESULT([$SUPERMIN_PACKAGER_CONFIG"])], - [AC_MSG_RESULT([not set])]) - AC_SUBST([SUPERMIN_PACKAGER_CONFIG]) dnl Pass additional supermin options. @@ -528,21 +477,35 @@ AC_ARG_WITH([supermin-extra-options], AC_SUBST([SUPERMIN_EXTRA_OPTIONS]) if test "x$ENABLE_APPLIANCE" = "xyes"; then + supermin_major_min=5 + supermin_minor_min=1 + supermin_min=$supermin_major_min.$supermin_minor_min + test "x$SUPERMIN" = "xno" && - AC_MSG_ERROR([supermin (formerly called febootstrap) must be installed]) - dnl febootstrap 2.x did not support the --version parameter + AC_MSG_ERROR([supermin >= $supermin_min must be installed]) + + AC_MSG_CHECKING([supermin is new enough]) $SUPERMIN --version >&AS_MESSAGE_LOG_FD 2>&1 || - AC_MSG_ERROR([supermin (formerly called febootstrap) >= 3.20 must be installed, your version is too old]) + AC_MSG_ERROR([supermin >= $supermin_min must be installed, your version is too old]) + supermin_major="`$SUPERMIN --version | awk '{print $2}' | awk -F. '{print $1}'`" + supermin_minor="`$SUPERMIN --version | awk '{print $2}' | awk -F. '{print $2}'`" + AC_MSG_RESULT([$supermin_major.$supermin_minor]) + + if test "$supermin_major" -lt "$supermin_major_min" || \ + ( test "$supermin_major" -eq "$supermin_major_min" && test "$supermin_minor" -lt "$supermin_minor_min" ); then + AC_MSG_ERROR([supermin >= $supermin_min must be installed, your version is too old]) + fi fi -AC_DEFINE_UNQUOTED([SUPERMIN_HELPER],["$SUPERMIN_HELPER"],[Name of supermin-helper program.]) +AC_DEFINE_UNQUOTED([SUPERMIN],["$SUPERMIN"],[Name of supermin program]) dnl Which distro? dnl dnl This used to be Very Important but is now just used to select dnl which packages to install in the appliance, since the package dnl names vary slightly across distros. (See -dnl appliance/packagelist.in and appliance/excludelist.in) +dnl appliance/packagelist.in, appliance/excludefiles.in, +dnl appliance/hostfiles.in) AC_MSG_CHECKING([which Linux distro for package names]) DISTRO=REDHAT if test -f /etc/debian_version; then diff --git a/examples/guestfs-performance.pod b/examples/guestfs-performance.pod index b2a5058..9f47e3c 100644 --- a/examples/guestfs-performance.pod +++ b/examples/guestfs-performance.pod @@ -153,7 +153,7 @@ L<libguestfs-make-fixed-appliance(1)>). In our testing we did not find that using a fixed appliance gave any measurable performance benefit, even when the appliance was located in -memory (ie. on C</dev/shm>). However there are three points to +memory (ie. on C</dev/shm>). However there are two points to consider: =over 4 @@ -166,14 +166,6 @@ times. =item 2. -By default libguestfs (or rather, L<supermin-helper(1)>) -searches over the root filesystem to find out if any host files have -changed and if it needs to rebuild the appliance. If these files are -not cached and the root filesystem is on an HDD, then this generates -lots of seeks. Using a fixed appliance avoids this. - -=item 3. - The appliance is loaded on demand. A simple test such as: time guestfish -a /dev/null run @@ -567,7 +559,6 @@ bit. =head1 SEE ALSO L<supermin(1)>, -L<supermin-helper(1)>, L<guestfish(1)>, L<guestfs(3)>, L<guestfs-examples(3)>, diff --git a/fish/guestfish.pod b/fish/guestfish.pod index 31add50..a0a884d 100644 --- a/fish/guestfish.pod +++ b/fish/guestfish.pod @@ -1525,10 +1525,10 @@ the path of qemu/KVM. =item SUPERMIN_MODULES These two environment variables allow the kernel that libguestfs uses -in the appliance to be selected. If C<$SUPERMIN_KERNEL> is not -set, then the most recent host kernel is chosen. For more information -about kernel selection, see L<supermin-helper(1)>. This -feature is only available in supermin / febootstrap E<ge> 3.8. +in the appliance to be selected. If C<$SUPERMIN_KERNEL> is not set, +then the most recent host kernel is chosen. For more information +about kernel selection, see L<supermin(1)>. This feature is only +available in supermin / febootstrap E<ge> 3.8. =item TMPDIR @@ -1629,7 +1629,7 @@ L<virt-win-reg(1)>, L<libguestfs-tools.conf(5)>, L<display(1)>, L<hexedit(1)>, -L<supermin-helper(1)>. +L<supermin(1)>. =head1 AUTHORS diff --git a/src/appliance.c b/src/appliance.c index 5115ebe..20815ec 100644 --- a/src/appliance.c +++ b/src/appliance.c @@ -52,18 +52,8 @@ static int dir_contains_files (const char *dir, ...); static int contains_old_style_appliance (guestfs_h *g, const char *path, void *data); static int contains_fixed_appliance (guestfs_h *g, const char *path, void *data); static int contains_supermin_appliance (guestfs_h *g, const char *path, void *data); -static char *calculate_supermin_checksum (guestfs_h *g, const char *supermin_path); -static int check_for_cached_appliance (guestfs_h *g, const char *supermin_path, const char *checksum, uid_t uid, char **kernel, char **dtb, char **initrd, char **appliance); -static int build_supermin_appliance (guestfs_h *g, const char *supermin_path, const char *checksum, uid_t uid, char **kernel, char **dtb, char **initrd, char **appliance); -static int hard_link_to_cached_appliance (guestfs_h *g, const char *cachedir, char **kernel, char **dtb, char **initrd, char **appliance); -static int run_supermin_helper (guestfs_h *g, const char *supermin_path, const char *cachedir); - -/* RHBZ#790721: It makes no sense to have multiple threads racing to - * build the appliance from within a single process, and the code - * isn't safe for that anyway. Therefore put a thread lock around - * appliance building. - */ -gl_lock_define_initialized (static, building_lock); +static int build_supermin_appliance (guestfs_h *g, const char *supermin_path, uid_t uid, char **kernel, char **dtb, char **initrd, char **appliance); +static int run_supermin_build (guestfs_h *g, const char *lockfile, const char *appliancedir, const char *supermin_path); /* Locate or build the appliance. * @@ -83,60 +73,31 @@ gl_lock_define_initialized (static, building_lock); * * (1) Look for the first element of g->path which contains a * supermin appliance skeleton. If no element has this, skip - * straight to step (5). + * straight to step (3). * - * (2) Calculate the checksum of this supermin appliance. + * (2) Call 'supermin --build' to build the full appliance (if it + * needs to be rebuilt). If this is successful, return the full + * appliance. * - * (3) Check whether a cached appliance with the checksum calculated - * in (2) exists and passes basic security checks. If so, return - * this appliance. - * - * (4) Try to build the supermin appliance. If this is successful, - * return it. - * - * (5) Check each element of g->path, looking for a fixed appliance. + * (3) Check each element of g->path, looking for a fixed appliance. * If one is found, return it. * - * (6) Check each element of g->path, looking for an old-style appliance. + * (4) Check each element of g->path, looking for an old-style appliance. * If one is found, return it. * * The supermin appliance cache directory lives in * $TMPDIR/.guestfs-$UID/ and consists of up to five files: * - * $TMPDIR/.guestfs-$UID/checksum - the checksum - * $TMPDIR/.guestfs-$UID/kernel - the kernel - * $TMPDIR/.guestfs-$UID/dtb - the device tree (on ARM) - * $TMPDIR/.guestfs-$UID/initrd - the supermin initrd - * $TMPDIR/.guestfs-$UID/root - the appliance - * - * Since multiple instances of libguestfs with the same UID may be - * racing to create an appliance, we need to be careful when building - * and using the appliance. - * - * If a cached appliance with checksum exists (step (2) above) then we - * make a hard link to it with our current PID, so that we have a copy - * even if the appliance is replaced by another process building an - * appliance afterwards: - * - * $TMPDIR/.guestfs-$UID/kernel.$PID - * $TMPDIR/.guestfs-$UID/dtb.$PID - * $TMPDIR/.guestfs-$UID/initrd.$PID - * $TMPDIR/.guestfs-$UID/root.$PID - * - * A lock is taken on "checksum" while we perform the link. - * - * Linked files are deleted by a garbage collection sweep which can be - * initiated by any libguestfs process with the same UID when the - * corresponding PID no longer exists. (This is safe: the parent is - * always around in guestfs_launch() while qemu is starting up, and - * after that qemu will either have finished with the files or be - * holding them open, so we can unlink them). - * - * When building a new appliance (step (3)), it is built into randomly - * named temporary files in the $TMPDIR. Then a lock is acquired on - * $TMPDIR/.guestfs-$UID/checksum (this file being created if - * necessary), the files are renamed into their final location, and - * the lock is released. + * $TMPDIR/.guestfs-$UID/lock - the supermin lock file + * $TMPDIR/.guestfs-$UID/appliance.d/kernel - the kernel + * $TMPDIR/.guestfs-$UID/appliance.d/dtb - the device tree (on ARM) + * $TMPDIR/.guestfs-$UID/appliance.d/initrd - the supermin initrd + * $TMPDIR/.guestfs-$UID/appliance.d/root - the appliance + * + * Multiple instances of libguestfs with the same UID may be racing to + * create an appliance. However (since supermin >= 5) supermin + * provides a --lock flag and atomic update of the appliance.d + * subdirectory. */ int guestfs___build_appliance (guestfs_h *g, @@ -145,14 +106,9 @@ guestfs___build_appliance (guestfs_h *g, char **initrd_rtn, char **appliance_rtn) { - int r; char *kernel, *dtb, *initrd, *appliance; - gl_lock_lock (building_lock); - r = build_appliance (g, &kernel, &dtb, &initrd, &appliance); - gl_lock_unlock (building_lock); - - if (r == -1) + if (build_appliance (g, &kernel, &dtb, &initrd, &appliance) == -1) return -1; /* Don't assign these until we know we're going to succeed, to avoid @@ -182,24 +138,12 @@ build_appliance (guestfs_h *g, if (r == -1) return -1; - if (r == 1) { - /* Step (2): calculate checksum. */ - CLEANUP_FREE char *checksum = - calculate_supermin_checksum (g, supermin_path); - if (checksum) { - /* Step (3): cached appliance exists? */ - r = check_for_cached_appliance (g, supermin_path, checksum, uid, - kernel, dtb, initrd, appliance); - if (r != 0) - return r == 1 ? 0 : -1; + if (r == 1) + /* Step (2): build supermin appliance. */ + return build_supermin_appliance (g, supermin_path, uid, + kernel, dtb, initrd, appliance); - /* Step (4): build supermin appliance. */ - return build_supermin_appliance (g, supermin_path, checksum, uid, - kernel, dtb, initrd, appliance); - } - } - - /* Step (5). */ + /* Step (3). */ r = find_path (g, contains_fixed_appliance, NULL, &path); if (r == -1) return -1; @@ -223,7 +167,7 @@ build_appliance (guestfs_h *g, return 0; } - /* Step (6). */ + /* Step (4). */ r = find_path (g, contains_old_style_appliance, NULL, &path); if (r == -1) return -1; @@ -264,82 +208,33 @@ contains_supermin_appliance (guestfs_h *g, const char *path, void *data) return dir_contains_files (path, "supermin.d", NULL); } -#define MAX_CHECKSUM_LEN 256 - -static void -read_checksum (guestfs_h *g, void *checksumv, const char *line, size_t len) -{ - char *checksum = checksumv; - - if (len > MAX_CHECKSUM_LEN) - return; - strcpy (checksum, line); -} - -static int -process_exists (int pid) -{ - if (kill (pid, 0) == 0) - return 1; - - if (errno == ESRCH) - return 0; - - return -1; -} - -/* Garbage collect appliance hard links. Files that match - * (kernel|dtb|initrd|root).$PID where the corresponding PID doesn't - * exist are deleted. Note that errors in this function don't matter. - * There may also be other libguestfs processes racing to do the same - * thing here. +/* Build supermin appliance from supermin_path to $TMPDIR/.guestfs-$UID. + * + * Returns: + * 0 = built + * -1 = error (aborts launch) */ -static void -garbage_collect_appliances (const char *cachedir) -{ - DIR *dir; - struct dirent *d; - int pid; - - dir = opendir (cachedir); - if (dir == NULL) - return; - - while ((d = readdir (dir)) != NULL) { - if (sscanf (d->d_name, "kernel.%d", &pid) == 1 && - process_exists (pid) == 0) - unlinkat (dirfd (dir), d->d_name, 0); - else if (sscanf (d->d_name, "dtb.%d", &pid) == 1 && - process_exists (pid) == 0) - unlinkat (dirfd (dir), d->d_name, 0); - else if (sscanf (d->d_name, "initrd.%d", &pid) == 1 && - process_exists (pid) == 0) - unlinkat (dirfd (dir), d->d_name, 0); - else if (sscanf (d->d_name, "root.%d", &pid) == 1 && - process_exists (pid) == 0) - unlinkat (dirfd (dir), d->d_name, 0); - } - - closedir (dir); -} - static int -check_for_cached_appliance (guestfs_h *g, - const char *supermin_path, const char *checksum, - uid_t uid, - char **kernel, char **dtb, - char **initrd, char **appliance) +build_supermin_appliance (guestfs_h *g, + const char *supermin_path, + uid_t uid, + char **kernel, char **dtb, + char **initrd, char **appliance) { CLEANUP_FREE char *tmpdir = guestfs_get_cachedir (g); + struct stat statbuf; + size_t len; /* len must be longer than the length of any pathname we can * generate in this function. */ - size_t len = strlen (tmpdir) + 128; + len = strlen (tmpdir) + 128; char cachedir[len]; snprintf (cachedir, len, "%s/.guestfs-%d", tmpdir, uid); - char filename[len]; - snprintf (filename, len, "%s/checksum", cachedir); + char lockfile[len]; + snprintf (lockfile, len, "%s/lock", cachedir); + char appliancedir[len]; + snprintf (appliancedir, len, "%s/appliance.d", cachedir); ignore_value (mkdir (cachedir, 0755)); ignore_value (chmod (cachedir, 0755)); /* RHBZ#921292 */ @@ -347,17 +242,16 @@ check_for_cached_appliance (guestfs_h *g, /* See if the cache directory exists and passes some simple checks * to make sure it has not been tampered with. */ - struct stat statbuf; if (lstat (cachedir, &statbuf) == -1) return 0; if (statbuf.st_uid != uid) { error (g, _("security: cached appliance %s is not owned by UID %d"), - filename, uid); + cachedir, uid); return -1; } if (!S_ISDIR (statbuf.st_mode)) { error (g, _("security: cached appliance %s is not a directory (mode %o)"), - filename, statbuf.st_mode); + cachedir, statbuf.st_mode); return -1; } if ((statbuf.st_mode & 0022) != 0) { @@ -366,302 +260,43 @@ check_for_cached_appliance (guestfs_h *g, return -1; } - (void) utime (cachedir, NULL); - - garbage_collect_appliances (cachedir); - - /* Try to open and acquire a lock on the checksum file. */ - int fd = open (filename, O_RDONLY|O_CLOEXEC); - if (fd == -1) - return 0; -#ifdef HAVE_FUTIMENS - (void) futimens (fd, NULL); -#else - (void) futimes (fd, NULL); -#endif - struct flock fl; - fl.l_type = F_RDLCK; - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 1; - again: - if (fcntl (fd, F_SETLKW, &fl) == -1) { - if (errno == EINTR) - goto again; - perrorf (g, "fcntl: F_SETLKW: %s", filename); - close (fd); - return -1; - } - - /* Read the checksum file. */ - size_t clen = strlen (checksum); - char checksum_on_disk[clen]; - ssize_t rr = read (fd, checksum_on_disk, clen); - if (rr == -1) { - perrorf (g, "read: %s", filename); - close (fd); - return -1; - } - if ((size_t) rr != clen) { - close (fd); - return 0; - } - - if (memcmp (checksum, checksum_on_disk, clen) != 0) { - close (fd); - return 0; - } - - /* At this point, cachedir exists, and checksum matches, and we have - * a read lock on the checksum file. Make hard links to the files. - */ - if (hard_link_to_cached_appliance (g, cachedir, - kernel, dtb, initrd, appliance) == -1) { - close (fd); - return -1; - } - - /* Releases the lock on checksum. */ - if (close (fd) == -1) { - perrorf (g, "close"); - /* Allocated in hard_link_to_cached_appliance above, must be - * freed along this error path. - */ - free (*kernel); - free (*dtb); - free (*initrd); - free (*appliance); - return -1; - } - - /* Exists! */ - return 1; -} - -/* Build supermin appliance from supermin_path to $TMPDIR/.guestfs-$UID. - * - * Returns: - * 0 = built - * -1 = error (aborts launch) - */ -static int -build_supermin_appliance (guestfs_h *g, - const char *supermin_path, const char *checksum, - uid_t uid, - char **kernel, char **dtb, - char **initrd, char **appliance) -{ - CLEANUP_FREE char *tmpdir = guestfs_get_cachedir (g); - size_t len; - + (void) utimes (cachedir, NULL); if (g->verbose) guestfs___print_timestamped_message (g, "begin building supermin appliance"); - /* len must be longer than the length of any pathname we can - * generate in this function. - */ - len = strlen (tmpdir) + 128; - - /* Build the appliance into a temporary directory. */ - char tmpcd[len]; - snprintf (tmpcd, len, "%s/guestfs.XXXXXX", tmpdir); - - if (mkdtemp (tmpcd) == NULL) { - perrorf (g, "mkdtemp"); - return -1; - } - + /* Build the appliance if it needs to be built. */ if (g->verbose) - guestfs___print_timestamped_message (g, "run supermin-helper"); + guestfs___print_timestamped_message (g, "run supermin"); - int r = run_supermin_helper (g, supermin_path, tmpcd); - if (r == -1) { - guestfs___recursive_remove_dir (g, tmpcd); + if (run_supermin_build (g, lockfile, appliancedir, supermin_path) == -1) return -1; - } if (g->verbose) guestfs___print_timestamped_message (g, "finished building supermin appliance"); - char cachedir[len]; - snprintf (cachedir, len, "%s/.guestfs-%d", tmpdir, uid); - char filename[len]; - char filename2[len]; - snprintf (filename, len, "%s/checksum", cachedir); - - /* Open and acquire write lock on checksum file. The file might - * not exist, in which case we want to create it. - */ - int fd = open (filename, O_WRONLY|O_CREAT|O_NOCTTY|O_CLOEXEC, 0644); - if (fd == -1) { - perrorf (g, "open: %s", filename); - guestfs___recursive_remove_dir (g, tmpcd); - return -1; - } - struct flock fl; - fl.l_type = F_WRLCK; - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 1; - again: - if (fcntl (fd, F_SETLKW, &fl) == -1) { - if (errno == EINTR) - goto again; - perrorf (g, "fcntl: F_SETLKW: %s", filename); - close (fd); - guestfs___recursive_remove_dir (g, tmpcd); - return -1; - } - - /* At this point we have acquired a write lock on the checksum - * file so we go ahead and replace it with the new checksum, and - * rename in appliance files into this directory. - */ - size_t clen = strlen (checksum); - if (ftruncate (fd, clen) == -1) { - perrorf (g, "ftruncate: %s", filename); - close (fd); - guestfs___recursive_remove_dir (g, tmpcd); - return -1; - } - - ssize_t rr = write (fd, checksum, clen); - if (rr == -1) { - perrorf (g, "write: %s", filename); - close (fd); - guestfs___recursive_remove_dir (g, tmpcd); - return -1; - } - if ((size_t) rr != clen) { - error (g, "partial write: %s", filename); - close (fd); - guestfs___recursive_remove_dir (g, tmpcd); - return -1; - } - - snprintf (filename, len, "%s/kernel", tmpcd); - snprintf (filename2, len, "%s/kernel", cachedir); - unlink (filename2); - if (rename (filename, filename2) == -1) { - perrorf (g, "rename: %s %s", filename, filename2); - close (fd); - guestfs___recursive_remove_dir (g, tmpcd); - return -1; - } - -#ifdef DTB_WILDCARD - snprintf (filename, len, "%s/dtb", tmpcd); - snprintf (filename2, len, "%s/dtb", cachedir); - unlink (filename2); - if (rename (filename, filename2) == -1) { - perrorf (g, "rename: %s %s", filename, filename2); - close (fd); - guestfs___recursive_remove_dir (g, tmpcd); - return -1; - } -#endif - - snprintf (filename, len, "%s/initrd", tmpcd); - snprintf (filename2, len, "%s/initrd", cachedir); - unlink (filename2); - if (rename (filename, filename2) == -1) { - perrorf (g, "rename: %s %s", filename, filename2); - close (fd); - guestfs___recursive_remove_dir (g, tmpcd); - return -1; - } - - snprintf (filename, len, "%s/root", tmpcd); - snprintf (filename2, len, "%s/root", cachedir); - unlink (filename2); - if (rename (filename, filename2) == -1) { - perrorf (g, "rename: %s %s", filename, filename2); - close (fd); - guestfs___recursive_remove_dir (g, tmpcd); - return -1; - } - - guestfs___recursive_remove_dir (g, tmpcd); - - /* Now finish off by linking to the cached appliance and returning it. */ - if (hard_link_to_cached_appliance (g, cachedir, - kernel, dtb, initrd, appliance) == -1) { - close (fd); - return -1; - } - - /* Releases the lock on checksum. */ - if (close (fd) == -1) { - perrorf (g, "close"); - /* Allocated in hard_link_to_cached_appliance above, must be - * freed along this error path. - */ - free (*kernel); - free (*dtb); - free (*initrd); - free (*appliance); - return -1; - } - - return 0; -} - -/* NB: lock on checksum file must be held when this is called. */ -static int -hard_link_to_cached_appliance (guestfs_h *g, - const char *cachedir, - char **kernel, char **dtb, - char **initrd, char **appliance) -{ - pid_t pid = getpid (); - size_t len = strlen (cachedir) + 32; - + /* Return the appliance filenames. */ *kernel = safe_malloc (g, len); +#ifdef DTB_WILDCARD *dtb = safe_malloc (g, len); +#else + *dtb = NULL; +#endif *initrd = safe_malloc (g, len); *appliance = safe_malloc (g, len); - snprintf (*kernel, len, "%s/kernel.%d", cachedir, pid); - snprintf (*dtb, len, "%s/dtb.%d", cachedir, pid); - snprintf (*initrd, len, "%s/initrd.%d", cachedir, pid); - snprintf (*appliance, len, "%s/root.%d", cachedir, pid); + snprintf (*kernel, len, "%s/kernel", appliancedir); +#ifdef DTB_WILDCARD + snprintf (*dtb, len, "%s/dtb", appliancedir); +#endif + snprintf (*initrd, len, "%s/initrd", appliancedir); + snprintf (*appliance, len, "%s/root", appliancedir); + + /* Touch the files so they don't get deleted (as they are in /var/tmp). */ + (void) utimes (*kernel, NULL); +#ifdef DTB_WILDCARD + (void) utimes (*dtb, NULL); +#endif + (void) utimes (*initrd, NULL); - char filename[len]; - snprintf (filename, len, "%s/kernel", cachedir); - (void) unlink (*kernel); - if (link (filename, *kernel) == -1) { - perrorf (g, "link: %s %s", filename, *kernel); - goto error; - } - (void) utimes (filename, NULL); - - snprintf (filename, len, "%s/dtb", cachedir); - (void) unlink (*dtb); - if (link (filename, *dtb) == -1) { - if (errno == ENOENT) { - /* dtb doesn't exist -- this is OK */ - free (*dtb); - *dtb = NULL; - } else { - perrorf (g, "link: %s %s", filename, *kernel); - goto error; - } - } - (void) utimes (filename, NULL); - - snprintf (filename, len, "%s/initrd", cachedir); - (void) unlink (*initrd); - if (link (filename, *initrd) == -1) { - perrorf (g, "link: %s %s", filename, *initrd); - goto error; - } - (void) utime (filename, NULL); - - snprintf (filename, len, "%s/root", cachedir); - (void) unlink (*appliance); - if (link (filename, *appliance) == -1) { - perrorf (g, "link: %s %s", filename, *appliance); - goto error; - } /* Checking backend != "uml" is a big hack. UML encodes the mtime * of the original backing file (in this case, the appliance) in the * COW file, and checks it when adding it to the VM. If there are @@ -675,211 +310,69 @@ hard_link_to_cached_appliance (guestfs_h *g, * XXX */ if (STRNEQ (g->backend, "uml")) - (void) utime (filename, NULL); + (void) utimes (*appliance, NULL); return 0; - - error: - free (*kernel); - free (*dtb); - free (*initrd); - free (*appliance); - return -1; } -/* Run supermin-helper and tell it to generate the +/* Run supermin --build and tell it to generate the * appliance. */ - -#ifdef SUPERMIN_HELPER_NEW_STYLE_SYNTAX - -/* supermin_path is a path which is known to contain a supermin - * appliance. Using supermin-helper -f checksum calculate - * the checksum so we can see if it is cached. - */ -static char * -calculate_supermin_checksum (guestfs_h *g, const char *supermin_path) -{ - size_t len; - CLEANUP_CMD_CLOSE struct command *cmd = guestfs___new_command (g); - int pass_u_g_args = getuid () != geteuid () || getgid () != getegid (); - char checksum[MAX_CHECKSUM_LEN + 1] = { 0 }; - - guestfs___cmd_add_arg (cmd, SUPERMIN_HELPER); - if (g->verbose) - guestfs___cmd_add_arg (cmd, "--verbose"); - if (pass_u_g_args) { - guestfs___cmd_add_arg (cmd, "-u"); - guestfs___cmd_add_arg_format (cmd, "%d", geteuid ()); - guestfs___cmd_add_arg (cmd, "-g"); - guestfs___cmd_add_arg_format (cmd, "%d", getegid ()); - } - guestfs___cmd_add_arg (cmd, "-f"); - guestfs___cmd_add_arg (cmd, "checksum"); - guestfs___cmd_add_arg (cmd, "--host-cpu"); - guestfs___cmd_add_arg (cmd, host_cpu); - guestfs___cmd_add_arg_format (cmd, "%s/supermin.d", supermin_path); - guestfs___cmd_set_stdout_callback (cmd, read_checksum, checksum, 0); - - /* Errors here are non-fatal, so we don't need to call error(). */ - if (guestfs___cmd_run (cmd) == -1) - return NULL; - - debug (g, "checksum of existing appliance: %s", checksum); - - len = strlen (checksum); - if (len < 16) { /* sanity check */ - warning (g, "supermin-helper -f checksum returned a short string"); - return NULL; - } - - return safe_strndup (g, checksum, len); -} - static int -run_supermin_helper (guestfs_h *g, const char *supermin_path, - const char *cachedir) +run_supermin_build (guestfs_h *g, + const char *lockfile, + const char *appliancedir, + const char *supermin_path) { CLEANUP_CMD_CLOSE struct command *cmd = guestfs___new_command (g); int r; +#if 0 /* not supported in supermin 5 yet XXX */ uid_t uid = getuid (); uid_t euid = geteuid (); gid_t gid = getgid (); gid_t egid = getegid (); int pass_u_g_args = uid != euid || gid != egid; +#endif - guestfs___cmd_add_arg (cmd, SUPERMIN_HELPER); + guestfs___cmd_add_arg (cmd, SUPERMIN); + guestfs___cmd_add_arg (cmd, "--build"); if (g->verbose) guestfs___cmd_add_arg (cmd, "--verbose"); +#if 0 if (pass_u_g_args) { guestfs___cmd_add_arg (cmd, "-u"); guestfs___cmd_add_arg_format (cmd, "%d", euid); guestfs___cmd_add_arg (cmd, "-g"); guestfs___cmd_add_arg_format (cmd, "%d", egid); } +#endif guestfs___cmd_add_arg (cmd, "--copy-kernel"); guestfs___cmd_add_arg (cmd, "-f"); guestfs___cmd_add_arg (cmd, "ext2"); guestfs___cmd_add_arg (cmd, "--host-cpu"); guestfs___cmd_add_arg (cmd, host_cpu); + guestfs___cmd_add_arg (cmd, "--if-newer"); + guestfs___cmd_add_arg (cmd, "--lock"); + guestfs___cmd_add_arg (cmd, lockfile); #ifdef DTB_WILDCARD guestfs___cmd_add_arg (cmd, "--dtb"); guestfs___cmd_add_arg (cmd, DTB_WILDCARD); #endif guestfs___cmd_add_arg_format (cmd, "%s/supermin.d", supermin_path); - guestfs___cmd_add_arg (cmd, "--output-kernel"); - guestfs___cmd_add_arg_format (cmd, "%s/kernel", cachedir); -#ifdef DTB_WILDCARD - guestfs___cmd_add_arg (cmd, "--output-dtb"); - guestfs___cmd_add_arg_format (cmd, "%s/dtb", cachedir); -#endif - guestfs___cmd_add_arg (cmd, "--output-initrd"); - guestfs___cmd_add_arg_format (cmd, "%s/initrd", cachedir); - guestfs___cmd_add_arg (cmd, "--output-appliance"); - guestfs___cmd_add_arg_format (cmd, "%s/root", cachedir); + guestfs___cmd_add_arg (cmd, "-o"); + guestfs___cmd_add_arg (cmd, appliancedir); r = guestfs___cmd_run (cmd); if (r == -1) return -1; if (!WIFEXITED (r) || WEXITSTATUS (r) != 0) { - guestfs___external_command_failed (g, r, SUPERMIN_HELPER, NULL); + guestfs___external_command_failed (g, r, SUPERMIN, NULL); return -1; } return 0; } -#else /* ! SUPERMIN_HELPER_NEW_STYLE_SYNTAX */ - -#ifdef DTB_WILDCARD -#error "This architecture has device trees, so requires supermin-helper >= 4.1.5" -#endif - -/* supermin_path is a path which is known to contain a supermin - * appliance. Using supermin-helper -f checksum calculate - * the checksum so we can see if it is cached. - */ -static char * -calculate_supermin_checksum (guestfs_h *g, const char *supermin_path) -{ - size_t len; - CLEANUP_CMD_CLOSE struct command *cmd = guestfs___new_command (g); - int pass_u_g_args = getuid () != geteuid () || getgid () != getegid (); - char checksum[MAX_CHECKSUM_LEN + 1] = { 0 }; - - guestfs___cmd_add_arg (cmd, SUPERMIN_HELPER); - if (g->verbose) - guestfs___cmd_add_arg (cmd, "--verbose"); - if (pass_u_g_args) { - guestfs___cmd_add_arg (cmd, "-u"); - guestfs___cmd_add_arg_format (cmd, "%d", geteuid ()); - guestfs___cmd_add_arg (cmd, "-g"); - guestfs___cmd_add_arg_format (cmd, "%d", getegid ()); - } - guestfs___cmd_add_arg (cmd, "-f"); - guestfs___cmd_add_arg (cmd, "checksum"); - guestfs___cmd_add_arg_format (cmd, "%s/supermin.d", supermin_path); - guestfs___cmd_add_arg (cmd, host_cpu); - guestfs___cmd_set_stdout_callback (cmd, read_checksum, checksum, 0); - - /* Errors here are non-fatal, so we don't need to call error(). */ - if (guestfs___cmd_run (cmd) == -1) - return NULL; - - debug (g, "checksum of existing appliance: %s", checksum); - - len = strlen (checksum); - if (len < 16) { /* sanity check */ - warning (g, "supermin-helper -f checksum returned a short string"); - return NULL; - } - - return safe_strndup (g, checksum, len); -} - -static int -run_supermin_helper (guestfs_h *g, const char *supermin_path, - const char *cachedir) -{ - CLEANUP_CMD_CLOSE struct command *cmd = guestfs___new_command (g); - int r; - uid_t uid = getuid (); - uid_t euid = geteuid (); - gid_t gid = getgid (); - gid_t egid = getegid (); - int pass_u_g_args = uid != euid || gid != egid; - - guestfs___cmd_add_arg (cmd, SUPERMIN_HELPER); - if (g->verbose) - guestfs___cmd_add_arg (cmd, "--verbose"); - if (pass_u_g_args) { - guestfs___cmd_add_arg (cmd, "-u"); - guestfs___cmd_add_arg_format (cmd, "%d", euid); - guestfs___cmd_add_arg (cmd, "-g"); - guestfs___cmd_add_arg_format (cmd, "%d", egid); - } - guestfs___cmd_add_arg (cmd, "--copy-kernel"); - guestfs___cmd_add_arg (cmd, "-f"); - guestfs___cmd_add_arg (cmd, "ext2"); - guestfs___cmd_add_arg_format (cmd, "%s/supermin.d", supermin_path); - guestfs___cmd_add_arg (cmd, host_cpu); - guestfs___cmd_add_arg_format (cmd, "%s/kernel", cachedir); - guestfs___cmd_add_arg_format (cmd, "%s/initrd", cachedir); - guestfs___cmd_add_arg_format (cmd, "%s/root", cachedir); - - r = guestfs___cmd_run (cmd); - if (r == -1) - return -1; - if (!WIFEXITED (r) || WEXITSTATUS (r) != 0) { - guestfs___external_command_failed (g, r, SUPERMIN_HELPER, NULL); - return -1; - } - - return 0; -} - -#endif /* ! SUPERMIN_HELPER_NEW_STYLE_SYNTAX */ - /* Search elements of g->path, returning the first path element which * matches the predicate function 'pred'. * diff --git a/src/guestfs.pod b/src/guestfs.pod index 4d95a2f..1e6ec3f 100644 --- a/src/guestfs.pod +++ b/src/guestfs.pod @@ -3477,15 +3477,14 @@ debugging (set the environment variable C<LIBGUESTFS_DEBUG=1>). =item Create the appliance -C<supermin-helper> is invoked to create the kernel, a -small initrd and the appliance. +C<supermin --build> is invoked to create the kernel, a small initrd +and the appliance. The appliance is cached in C</var/tmp/.guestfs-E<lt>UIDE<gt>> (or in another directory if C<LIBGUESTFS_CACHEDIR> or C<TMPDIR> are set). For a complete description of how the appliance is created and cached, -read the L<supermin(1)> and L<supermin-helper(1)> man -pages. +read the L<supermin(1)> man page. =item Start qemu and boot the kernel @@ -3493,12 +3492,12 @@ qemu is invoked to boot the kernel. =item Run the initrd -C<supermin-helper> builds a small initrd. The initrd is -not the appliance. The purpose of the initrd is to load enough kernel -modules in order that the appliance itself can be mounted and started. +C<supermin --build> builds a small initrd. The initrd is not the +appliance. The purpose of the initrd is to load enough kernel modules +in order that the appliance itself can be mounted and started. The initrd is a cpio archive called -C</var/tmp/.guestfs-E<lt>UIDE<gt>/initrd>. +C</var/tmp/.guestfs-E<lt>UIDE<gt>/appliance.d/initrd>. When the initrd has started you will see messages showing that kernel modules are being loaded, similar to this: @@ -3512,7 +3511,8 @@ modules are being loaded, similar to this: The appliance is a sparse file containing an ext2 filesystem which contains a familiar (although reduced in size) Linux operating system. -It would normally be called C</var/tmp/.guestfs-E<lt>UIDE<gt>/root>. +It would normally be called +C</var/tmp/.guestfs-E<lt>UIDE<gt>/appliance.d/root>. The regular disks being inspected by libguestfs are the first devices exposed by qemu (eg. as C</dev/vda>). @@ -4706,7 +4706,7 @@ environment which tends to break everything. These two environment variables allow the kernel that libguestfs uses in the appliance to be selected. If C<$SUPERMIN_KERNEL> is not set, then the most recent host kernel is chosen. For more information -about kernel selection, see L<supermin-helper(1)>. This +about kernel selection, see L<supermin(1)>. This feature is only available in supermin / febootstrap E<ge> 3.8. =item TMPDIR @@ -4758,7 +4758,6 @@ L<guestfs-testing(1)>, L<libguestfs-test-tool(1)>, L<libguestfs-make-fixed-appliance(1)>, L<supermin(1)>, -L<supermin-helper(1)>, L<qemu(1)>, L<hivex(3)>, L<stap(1)>, diff --git a/test-tool/libguestfs-test-tool.pod b/test-tool/libguestfs-test-tool.pod index c428468..2ac299b 100644 --- a/test-tool/libguestfs-test-tool.pod +++ b/test-tool/libguestfs-test-tool.pod @@ -87,8 +87,7 @@ variables C<SUPERMIN_KERNEL> and/or C<SUPERMIN_MODULES> (C<FEBOOTSTRAP_KERNEL> and C<FEBOOTSTRAP_MODULES> if still using the old febootstrap 3.21 program). -Refer to L<supermin-helper(1)/ENVIRONMENT VARIABLES> -for further information. +Refer to L<supermin(1)/ENVIRONMENT VARIABLES> for further information. =head1 TRYING OUT A DIFFERENT VERSION OF LIBVIRT -- 1.8.5.3

10 years, 8 months

1
0
0 / 0

Problem with febootstrap version while compiling guestfsd in centos6.5

by Safa Rekik

Hello, I am try to compile guestfsd using this link<http://thr3ads.net/libguestfs/2012/09/2100168-Re-compile-guestfsd>. I am using libguestfs-1.24.4.tar.gz After running ./configure with disable options i get a message claiming a febootstrap >=3.2. I already installed febootstrap-3.21-4.el6x86_64.rpm for centos6.5. In the rpm description i found a note telling that this rpm will provide febootstrap 2.11 and 3.21, and indeed i have too commands febbotstrap(2.11) and febootstrap3(3.21). So could it be a version conflict problem? is there a way to make the ./configure script point to 3.21 version? Any help would be much appreciated. Thanks in advance ! *Note : the kernel of centos6.5 is 2.6.32, which is lower than the one required in README file . But i don't know whether it may be the source of the problem* -- Safa REKIK

10 years, 8 months

2
2
0 / 0

[PATCH supermin v3] Supermin 5 rewrite.

by Richard W.M. Jones

This is the first really working version of supermin 5. This version can prepare a supermin appliance, and build a full appliance from a supermin appliance. See examples/build-basic-vm.sh for example usage. Read the updated supermin(1) man page for the details. Among the improvements: - Package names are used instead of hostfiles. This should improve robustness, since adding a new file to a package or moving files around won't break the appliance. From the libguestfs point of view this will also allow us to split libguestfs dependencies into subpackges (libguestfs-xfs, libguestfs-ntfs, etc). - It uses the current package database to do dependency resolution, (instead of yum/apt). This is considerably faster. - It only needs to download packages that contain configuration files, which results in much less network bandwidth used. (--use-installed is supported as before to completely eliminate network usage) - --if-newer and --lock options move the locking into supermin (previously it was done in libguestfs). This makes supermin much easier to consume from other programs. - The --if-newer option checks the RPM database instead of iterating over the filesystem, making it much faster (a single stat versus 1000s of stats across the filesystem). - You can create a chroot filesystem from a supermin appliance. Currently missing: - support for Debian / dpkg - support for other distros (that were supported by v4) I intend to add at least Debian support before pushing anything. Some dropped features: - -f cpio is no longer supported. We didn't use this in libguestfs since 2010. It would not be hard to add this back. - --user/--group options. We added these for the benefit of virt-v2v but it's not clear if they are needed. Easy to add back if required. - You can no longer create a supermin from package files. This was never used by callers, as far as I know. - You can no longer exclude packages after depsolving. However you can still exclude files, which is nearly the same thing in practice. Rich.

10 years, 8 months

1
1
0 / 0

[PATCH supermin v2 NOT TO BE APPLIED] Supermin 5 rewrite.

by Richard W.M. Jones

--- .gitignore | 2 + .gitmodules | 3 - HACKING | 41 +++ Makefile.am | 6 +- README | 7 +- TODO | 64 +--- autobuild.sh | 65 ---- autogen.sh | 12 - configure.ac | 27 +- gnulib | 1 - helper/Makefile.am | 88 ----- helper/README | 10 - helper/appliance.c | 330 ------------------- helper/bin2s.pl | 45 --- helper/checksum.c | 119 ------- helper/cpio.c | 279 ---------------- helper/ext2.c | 538 ------------------------------- helper/ext2cpio.c | 418 ------------------------ helper/ext2initrd.c | 281 ---------------- helper/ext2internal.h | 45 --- helper/helper.h | 85 ----- helper/init.c | 537 ------------------------------- helper/kernel.c | 454 -------------------------- helper/main.c | 475 --------------------------- helper/supermin-helper.pod | 351 -------------------- helper/utils.c | 314 ------------------ lib/.gitignore | 175 ---------- m4/.gitignore | 114 ------- m4/gnulib-cache.m4 | 55 ---- src/Makefile.am | 139 +++++--- src/bin2s.pl | 45 +++ src/build.ml | 390 +++++++++++++++++++++++ src/chroot.ml | 77 +++++ src/config.ml.in | 7 +- src/ext2.ml | 80 +++++ src/ext2_initrd.ml | 156 +++++++++ src/ext2fs-c.c | 655 ++++++++++++++++++++++++++++++++++++++ src/ext2fs.ml | 26 ++ src/ext2fs.mli | 33 ++ src/ext2init-c.c | 44 +++ src/ext2init.ml | 19 ++ src/ext2init.mli | 19 ++ src/fnmatch-c.c | 64 ++++ src/fnmatch.ml | 28 ++ src/fnmatch.mli | 27 ++ src/glob-c.c | 84 +++++ src/glob.ml | 28 ++ src/glob.mli | 27 ++ src/init.c | 537 +++++++++++++++++++++++++++++++ src/kernel.ml | 238 ++++++++++++++ src/package_handler.ml | 129 ++++++++ src/package_handler.mli | 157 +++++++++ src/prepare.ml | 160 ++++++++++ src/realpath-c.c | 44 +++ src/realpath.ml | 19 ++ src/realpath.mli | 19 ++ src/rpm.ml | 192 +++++++++++ src/supermin.ml | 586 ++++++++++------------------------ src/supermin.pod | 463 +++++++++++++++++---------- src/supermin_cmdline.ml | 120 ------- src/supermin_cmdline.mli | 60 ---- src/supermin_debian.ml | 231 -------------- src/supermin_package_handlers.ml | 81 ----- src/supermin_package_handlers.mli | 73 ----- src/supermin_pacman.ml | 145 --------- src/supermin_pacman_g2.ml | 149 --------- src/supermin_urpmi_rpm.ml | 132 -------- src/supermin_utils.ml | 158 --------- src/supermin_utils.mli | 78 ----- src/supermin_yum_rpm.ml | 262 --------------- src/supermin_zypp_rpm.ml | 269 ---------------- src/types.ml | 19 ++ src/utils.ml | 194 +++++++++++ src/utils.mli | 90 ++++++ 74 files changed, 4226 insertions(+), 7268 deletions(-) delete mode 100644 .gitmodules create mode 100644 HACKING delete mode 100755 autobuild.sh delete mode 100755 autogen.sh delete mode 160000 gnulib delete mode 100644 helper/Makefile.am delete mode 100644 helper/README delete mode 100644 helper/appliance.c delete mode 100755 helper/bin2s.pl delete mode 100644 helper/checksum.c delete mode 100644 helper/cpio.c delete mode 100644 helper/ext2.c delete mode 100644 helper/ext2cpio.c delete mode 100644 helper/ext2initrd.c delete mode 100644 helper/ext2internal.h delete mode 100644 helper/helper.h delete mode 100644 helper/init.c delete mode 100644 helper/kernel.c delete mode 100644 helper/main.c delete mode 100644 helper/supermin-helper.pod delete mode 100644 helper/utils.c delete mode 100644 lib/.gitignore delete mode 100644 m4/.gitignore delete mode 100644 m4/gnulib-cache.m4 create mode 100755 src/bin2s.pl create mode 100644 src/build.ml create mode 100644 src/chroot.ml create mode 100644 src/ext2.ml create mode 100644 src/ext2_initrd.ml create mode 100644 src/ext2fs-c.c create mode 100644 src/ext2fs.ml create mode 100644 src/ext2fs.mli create mode 100644 src/ext2init-c.c create mode 100644 src/ext2init.ml create mode 100644 src/ext2init.mli create mode 100644 src/fnmatch-c.c create mode 100644 src/fnmatch.ml create mode 100644 src/fnmatch.mli create mode 100644 src/glob-c.c create mode 100644 src/glob.ml create mode 100644 src/glob.mli create mode 100644 src/init.c create mode 100644 src/kernel.ml create mode 100644 src/package_handler.ml create mode 100644 src/package_handler.mli create mode 100644 src/prepare.ml create mode 100644 src/realpath-c.c create mode 100644 src/realpath.ml create mode 100644 src/realpath.mli create mode 100644 src/rpm.ml delete mode 100644 src/supermin_cmdline.ml delete mode 100644 src/supermin_cmdline.mli delete mode 100644 src/supermin_debian.ml delete mode 100644 src/supermin_package_handlers.ml delete mode 100644 src/supermin_package_handlers.mli delete mode 100644 src/supermin_pacman.ml delete mode 100644 src/supermin_pacman_g2.ml delete mode 100644 src/supermin_urpmi_rpm.ml delete mode 100644 src/supermin_utils.ml delete mode 100644 src/supermin_utils.mli delete mode 100644 src/supermin_yum_rpm.ml delete mode 100644 src/supermin_zypp_rpm.ml create mode 100644 src/types.ml create mode 100644 src/utils.ml create mode 100644 src/utils.mli diff --git a/.gitignore b/.gitignore index 5d7aef0..6f26c61 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,8 @@ pod2htm?.tmp /snippet/ /src/.depend /src/config.ml +/src/ext2init-bin.S +/src/init /src/supermin /src/supermin.1 /stamp-h1 diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index acb2669..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "gnulib"] - path = gnulib - url = git://git.sv.gnu.org/gnulib.git diff --git a/HACKING b/HACKING new file mode 100644 index 0000000..81831ae --- /dev/null +++ b/HACKING @@ -0,0 +1,41 @@ +Send patches to the libguestfs mailing list. + +Overview of the source files: + + supermin.ml main program, argument parsing and coordination + + | + +- prepare.ml Prepare mode (--prepare option) + | + +- build.ml Build mode (--build option) + | + +- chroot.ml Build a chroot (--build -f chroot) + | + +- ext2.ml Build an ext2 fs (--build -f ext2) + | + +- kernel.ml Find the right kernel to use + | + +- ext2_initrd.ml Build a minimal initrd + +Libraries used by both modes: + + | + +- package_hander.ml Package manager interface, for resolving + | | package dependencies, list of files, etc. + | | + | +- rpm.ml Package manager implementation for RPM distros + | | + | +- dpkg.ml Package manager implementation for dpkg distros + | | + | +- etc. + | + +- config.ml Configuration (from autoconf) + | + +- types.ml Some global type declarations + | + +- utils.ml Some utility functions + | + +- fnmatch.ml Interface to fnmatch(3) + | | + | +- fnmatch-c.c Binding to fnmatch(3) + +- etc. And other C bindings ... diff --git a/Makefile.am b/Makefile.am index e1aee63..95455f4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -19,20 +19,18 @@ ACLOCAL_AMFLAGS = -I m4 -SUBDIRS = lib src helper examples tests +SUBDIRS = src examples tests EXTRA_DIST = \ .gitignore \ .gitmodules \ autogen.sh \ html/pod.css \ - m4/gnulib-cache.m4 \ $(SOURCES) # Maintainer website update. HTMLFILES = \ - html/supermin.1.html \ - html/supermin-helper.1.html + html/supermin.1.html WEBSITEDIR = $(HOME)/d/redhat/websites/libguestfs diff --git a/README b/README index 1b63235..4700e52 100644 --- a/README +++ b/README @@ -10,6 +10,10 @@ second when you need to boot one of them. A complete description is in the supermin(1) man page. +IMPORTANT NOTE FOR USERS OF SUPERMIN 4.x: supermin 5.x is a rewrite of + supermin 4. It is compatible at a high level with supermin 4 / + febootstrap 3, but requires some command line adjustments. + IMPORTANT NOTE FOR USERS OF FEBOOTSTRAP 3.x: supermin 4.x is just an evolution of febootstrap 3.x (really we just renamed it). The previous febootstrap program is now called @@ -49,8 +53,7 @@ Requirements For Fedora/RHEL: rpm - yum - yumdownloader + yumdownloader (from yum-utils) For Debian/Ubuntu: diff --git a/TODO b/TODO index d79440f..03aefaa 100644 --- a/TODO +++ b/TODO @@ -1,34 +1,10 @@ -Ideas for a future version of supermin. +For supermin 5 rewrite: - -hostfiles ---------- - -'hostfiles' causes lots of trouble, because it bakes path dependencies -into the supermin appliance. - -We propose to replace this with a list of root packages. Lines in -hostfiles beginning with a "+" sign are a root package, eg: - - +bash - +coreutils - -We then query RPM or dpkg to get a list of hostfiles at appliance boot -time. eg. For rpm we'd do the rpmlib equivalents of: - - rpm -ql bash # list of files in bash - rpm -qR bash # what bash requires - rpm -q --whatprovides <depN> # recursively look up each requires - -(If this is too slow, aggressively cache the results) - -Some files still need to be stored at build time, basically %config -files. - -Unclear what to do about kernels if we make this change. - -'hostfiles' may still be needed for a handful of files that we really -want to copy in. Notable ones: /etc/localtime and /etc/resolv.conf. + - FIX SUPERMIN_KERNEL ETC ENVIRONMENT VARIABLES + - support for debian + - support for other distros + - locking + - check RPM package database and only rebuild when necessary Store %post scripts @@ -38,21 +14,13 @@ Can we get the %post scripts and store them in a directory in the appliance? -Directories ------------ - -Reconstruction is tied to having directories being created before they -are used. We should try to remove this limitation as it's really -quite unnecessary. - - -Use libsolv or hawkey for dependency resolution ------------------------------------------------ - -Using Python + yum to resolve RPM dependencies is sucky and slow. We -should look at the combination of DNF, libsolv and hawkey to replace -them: - -https://fedoraproject.org/wiki/Features/DNF -https://github.com/openSUSE/libsolv -https://github.com/akozumpl/hawkey +Some features of supermin 4 which were cut in supermin 5 +-------------------------------------------------------- + + - write an initrd file (ie. '-f cpio') + - install from a set of package files + - exclude packages after dep solving + - save-temps + - --output-{appliance,dtb,initrd,kernel} options + - --user/--group options + - --kmods option diff --git a/autobuild.sh b/autobuild.sh deleted file mode 100755 index 7df080e..0000000 --- a/autobuild.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - - -PROJECT=supermin -MAILTO=libguestfs(a)redhat.com -HOSTNAME="$(hostname -s)" - -#---------------------------------------------------------------------- -# Helper functions. - -failed () -{ - mail -s "$HOSTNAME $PROJECT FAILED $1 $gitsha" $MAILTO < local-log -} - -ok () -{ - mail -s "$HOSTNAME $PROJECT success $gitsha" $MAILTO < local-log -} - -#---------------------------------------------------------------------- - -set -e -set -x - -rm -f local-log -cat > local-log <<EOF - -This is an automatic message generated by the builder on -$HOSTNAME for $PROJECT. Log files from the build -follow below. - -$(uname -a) -$(date) - ------ - -EOF -exec >> local-log 2>&1 - -# Pull from the public repo so that we don't need ssh-agent. -git pull --rebase git://github.com/libguestfs/supermin.git -git clean -d -f - -# The git version we are building. -gitsha=$(git log|head -1|awk '{print $2}') - -# Do the configure step. -./autogen.sh || { - failed "configure step" - exit 1 -} - -# Do the build step. -make || { - failed "build step" - exit 1 -} - -# Run the tests. -make check || { - failed "tests" - exit 1 -} - -ok diff --git a/autogen.sh b/autogen.sh deleted file mode 100755 index fce31bb..0000000 --- a/autogen.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - - -if [ -z "$(ls gnulib 2>/dev/null)" ] -then - git clone git://git.savannah.gnu.org/gnulib.git -fi - -./gnulib/gnulib-tool --update - -export AUTOMAKE='automake --foreign --add-missing' -autoreconf -./configure "$@" diff --git a/configure.ac b/configure.ac index 56822fa..0ef3c05 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ dnl supermin configure.ac -dnl (C) Copyright 2009-2013 Red Hat Inc. +dnl (C) Copyright 2009-2014 Red Hat Inc. dnl dnl This program is free software; you can redistribute it and/or modify dnl it under the terms of the GNU General Public License as published by @@ -17,12 +17,11 @@ dnl Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. dnl dnl Written by Richard W.M. Jones <rjones(a)redhat.com> -AC_INIT([supermin],[4.1.6]) +AC_INIT([supermin],[5.1.0]) AM_INIT_AUTOMAKE(foreign) dnl Check for basic C environment. AC_PROG_CC_STDC -gl_EARLY AC_PROG_INSTALL AC_PROG_CPP @@ -35,10 +34,11 @@ AM_PROG_CC_C_O dnl Check support for 64 bit file offsets. AC_SYS_LARGEFILE -gl_INIT +dnl Enable GNU stuff. +AC_USE_SYSTEM_EXTENSIONS -dnl Define a C symbol for the host CPU architecture. -AC_DEFINE_UNQUOTED([host_cpu],["$host_cpu"],[Host architecture.]) +dnl Define the host CPU architecture (defines host_cpu). +AC_CANONICAL_HOST # Define $(SED). m4_ifdef([AC_PROG_SED],[ @@ -59,10 +59,6 @@ if test "$OCAMLFIND" = "no"; then AC_MSG_ERROR([You must install OCaml findlib (the ocamlfind command)]) fi -dnl Optional OCaml packages. -AC_CHECK_OCAML_PKG([inifiles]) -AM_CONDITIONAL([HAVE_OCAML_INIFILES], [test "x$OCAML_PKG_inifiles" != "xno"]) - dnl Optional programs. AC_CHECK_PROG(PERLDOC,[perldoc],[perldoc],[no]) if test "x$PERLDOC" = "xno" ; then @@ -71,7 +67,6 @@ fi AM_CONDITIONAL(HAVE_PERLDOC,[test "$perldoc" != "no"]) dnl For yum-rpm handler. -AC_CHECK_PROG(YUM,[yum],[yum],[no]) AC_CHECK_PROG(RPM,[rpm],[rpm],[no]) AC_CHECK_PROG(YUMDOWNLOADER,[yumdownloader],[yumdownloader],[no]) @@ -187,8 +182,7 @@ AC_PATH_PROG([MKE2FS],[mke2fs],[no], if test "x$MKE2FS" = "xno" ; then AC_MSG_FAILURE([mke2fs program not found]) fi -AC_DEFINE_UNQUOTED([MKE2FS],["$MKE2FS"], - [Full path to the mke2fs program.]) +AC_SUBST([MKE2FS]) dnl RHEL 5 mke2fs needed -T <fs> instead of -t <fs>. Unhelpfully dnl the --help output doesn't mention this, so we have to test it. @@ -204,14 +198,13 @@ else fi rm conftest.img AC_MSG_RESULT([$MKE2FS_T_OPTION]) -AC_DEFINE_UNQUOTED([MKE2FS_T_OPTION],["$MKE2FS_T_OPTION"], - [mke2fs option used to specify filesystem type (usually "-t")]) +AC_SUBST([MKE2FS_T_OPTION]) dnl ext2fs, com_err. PKG_CHECK_MODULES([EXT2FS], [ext2fs]) PKG_CHECK_MODULES([COM_ERR], [com_err]) -dnl Optional ext2fs_close2 function. +dnl Requires ext2fs_close2 function, added in 2011. old_LIBS="$LIBS" LIBS="$EXT2FS_LIBS $COM_ERR_LIBS" AC_CHECK_FUNCS([ext2fs_close2]) @@ -235,8 +228,6 @@ AM_CONDITIONAL([NETWORK_TESTS], AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([Makefile examples/Makefile - helper/Makefile - lib/Makefile src/config.ml src/Makefile tests/Makefile]) diff --git a/gnulib b/gnulib deleted file mode 160000 index 0ac90c5..0000000 --- a/gnulib +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0ac90c5a98030c998f3e1db3a0d7f19d4630b6b6 diff --git a/helper/Makefile.am b/helper/Makefile.am deleted file mode 100644 index 9e2acde..0000000 --- a/helper/Makefile.am +++ /dev/null @@ -1,88 +0,0 @@ -# supermin Makefile.am -# (C) Copyright 2010-2013 Red Hat Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -# -# Written by Richard W.M. Jones <rjones(a)redhat.com> - -bin_PROGRAMS = \ - supermin-helper - -supermin_helper_SOURCES = \ - helper.h \ - appliance.c \ - checksum.c \ - cpio.c \ - ext2.c \ - ext2cpio.c \ - ext2initrd.c \ - ext2internal.h \ - kernel.c \ - main.c \ - utils.c -supermin_helper_CFLAGS = \ - -Wall $(EXT2FS_CFLAGS) $(COM_ERR_CFLAGS) -I../lib -supermin_helper_LDADD = \ - ext2init.o $(EXT2FS_LIBS) $(COM_ERR_LIBS) $(ZLIB_LIBS) \ - $(LTLIBINTL) -L../lib -lgnu - -# init "script" used by ext2 initrd. -noinst_PROGRAMS = init -init_SOURCES = init.c -init_CFLAGS = -static -init_LDFLAGS = -static -init_LDADD = $(ZLIB_STATIC_LIBS) $(LZMA_STATIC_LIBS) - -CLEANFILES = ext2init.S - -ext2init.o: ext2init.S - $(CC) -o $@ -c $< - -ext2init.S: init - strip --strip-all $< - @file $< | grep -isq static || \ - (echo "*** error: init is not staticly linked"; exit 1) - ./bin2s.pl $< $@ - -man_MANS = \ - supermin-helper.1 - -if HAVE_PERLDOC - -supermin-helper.1: supermin-helper.pod - pod2man \ - -u \ - --section 1 \ - -c "Virtualization Support" \ - --release "$(PACKAGE_NAME)-$(PACKAGE_VERSION)" \ - $< > $@ - -noinst_DATA = \ - $(top_builddir)/html/supermin-helper.1.html - -$(top_builddir)/html/supermin-helper.1.html: supermin-helper.pod - mkdir -p $(top_builddir)/html - cd $(top_builddir) && pod2html \ - --css 'pod.css' \ - --htmldir html \ - --outfile html/supermin-helper.1.html \ - helper/supermin-helper.pod - -endif - -EXTRA_DIST = \ - supermin-helper.1 \ - supermin-helper.pod \ - bin2s.pl diff --git a/helper/README b/helper/README deleted file mode 100644 index a8a7562..0000000 --- a/helper/README +++ /dev/null @@ -1,10 +0,0 @@ -This directory contains the supermin-helper reimplementation in C. - -This program builds the supermin appliance on the fly each time the -appliance runs (or in recent versions of libguestfs, the appliance is -cached as long as the host files don't change). - -*NOTE*: This program is designed to be very short-lived, and so we -don't normally bother to free up any memory that we allocate. That's -not completely true - we free up stuff if it's obvious and easy to -free up, and ignore the rest. diff --git a/helper/appliance.c b/helper/appliance.c deleted file mode 100644 index 200de96..0000000 --- a/helper/appliance.c +++ /dev/null @@ -1,330 +0,0 @@ -/* supermin-helper reimplementation in C. - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include <config.h> - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> -#include <fcntl.h> -#include <errno.h> -#include <dirent.h> -#include <fnmatch.h> -#include <sys/stat.h> -#include <assert.h> - -#include "error.h" -#include "fts_.h" -#include "xalloc.h" -#include "xvasprintf.h" - -#include "helper.h" - -static void iterate_inputs (char **inputs, int nr_inputs, struct writer *); -static void iterate_input_directory (const char *dirname, int dirfd, struct writer *); -static void add_kernel_modules (const char *whitelist, const char *modpath, struct writer *); -static void add_hostfiles (const char *hostfiles_file, struct writer *); - -/* Create the appliance. - * - * The initrd consists of these components concatenated together: - * - * (1) The base skeleton appliance that we constructed at build time. - * format = plain cpio - * (2) The host files which match wildcards in *.supermin.hostfiles. - * input format = plain text, output format = plain cpio - * (3) The modules from modpath which are on the module whitelist. - * output format = plain cpio - * - * The original shell script used the external cpio program to create - * parts (2) and (3), but we have decided it's going to be faster if - * we just write out the data outselves. The reasons are that - * external cpio is slow (particularly when used with SELinux because - * it does 512 byte reads), and the format that we're writing is - * narrow and well understood, because we only care that the Linux - * kernel can read it. - * - * This version contains some improvements over the C version written - * for libguestfs, in that we can have multiple base images (or - * hostfiles) or use a directory to store these files. - */ -void -create_appliance (const char *hostcpu, - char **inputs, int nr_inputs, - const char *whitelist, - const char *modpath, - const char *initrd, - const char *appliance, - struct writer *writer) -{ - writer->wr_start (hostcpu, appliance, modpath, initrd); - - iterate_inputs (inputs, nr_inputs, writer); - - writer->wr_file ("/lib/modules"); - /* Kernel modules (3). */ - add_kernel_modules (whitelist, modpath, writer); - - writer->wr_end (); -} - -/* Iterate over the inputs to find out what they are, visiting - * directories if specified. - */ -static void -iterate_inputs (char **inputs, int nr_inputs, struct writer *writer) -{ - int i; - for (i = 0; i < nr_inputs; ++i) { - if (verbose) - print_timestamped_message ("visiting %s", inputs[i]); - - int fd = open (inputs[i], O_RDONLY); - if (fd == -1) - error (EXIT_FAILURE, errno, "open: %s", inputs[i]); - - struct stat statbuf; - if (fstat (fd, &statbuf) == -1) - error (EXIT_FAILURE, errno, "fstat: %s", inputs[i]); - - /* Directory? */ - if (S_ISDIR (statbuf.st_mode)) - iterate_input_directory (inputs[i], fd, writer); - else if (S_ISREG (statbuf.st_mode)) { - /* Is it a cpio file? */ - char buf[6]; - ssize_t r = read (fd, buf, 6); - if (r >= 6 && memcmp (buf, "070701", 6) == 0) - /* Yes, a cpio file. This is a skeleton appliance, case (1). */ - cpio: - writer->wr_cpio_file (inputs[i]); - else if (r >= 2 && memcmp (buf, "\x1f\x8b", 2) == 0) - /* A gzip-compressed file. This must be cpio; case (1). */ - goto cpio; - else - /* No, must be hostfiles, case (2). */ - add_hostfiles (inputs[i], writer); - } - else - error (EXIT_FAILURE, 0, "%s: input is not a regular file or directory", - inputs[i]); - - close (fd); - } -} - -static int -string_compare (const void *p1, const void *p2) -{ - return strcmp (* (char * const *) p1, * (char * const *) p2); -} - -static void -iterate_input_directory (const char *dirname, int dirfd, struct writer *writer) -{ - DIR *dir = fdopendir (dirfd); - if (dir == NULL) - error (EXIT_FAILURE, errno, "fdopendir: %s", dirname); - - char **entries = NULL; - size_t nr_entries = 0, nr_alloc = 0; - - struct dirent *d; - while ((errno = 0, d = readdir (dir)) != NULL) { - if (d->d_name[0] == '.') /* ignore ., .. and any hidden files. */ - continue; - - /* Ignore *~ files created by editors. */ - size_t len = strlen (d->d_name); - if (len > 0 && d->d_name[len-1] == '~') - continue; - - add_string (&entries, &nr_entries, &nr_alloc, d->d_name); - } - - if (errno != 0) - error (EXIT_FAILURE, errno, "readdir: %s", dirname); - - if (closedir (dir) == -1) - error (EXIT_FAILURE, errno, "closedir: %s", dirname); - - add_string (&entries, &nr_entries, &nr_alloc, NULL); - - /* Visit directory entries in order. In febootstrap <= 2.8 we - * didn't impose any order, but that led to some difficult - * heisenbugs. - */ - sort (entries, string_compare); - - char path[PATH_MAX]; - char *inputs[] = { path }; - size_t len = strlen (dirname); - - if (len + 1 >= PATH_MAX) - error (EXIT_FAILURE, 0, "%s: directory name too long", __func__); - - strcpy (path, dirname); - path[len++] = '/'; - - size_t i; - for (i = 0; entries[i] != NULL; ++i) { - size_t len2 = strlen (entries[i]); - - if (len + 1 + len2 >= PATH_MAX) - error (EXIT_FAILURE, 0, "%s: path name too long", __func__); - - strcpy (&path[len], entries[i]); - - iterate_inputs (inputs, 1, writer); - } -} - -/* Copy kernel modules. - * - * Find every file under modpath. - * - * Exclude all *.ko files, *except* ones which match names in - * the whitelist (which may contain wildcards). Include all - * other files. - * - * Add chosen files to the output. - * - * whitelist_file may be NULL, to include ALL kernel modules. - */ -static void -add_kernel_modules (const char *whitelist_file, const char *modpath, - struct writer *writer) -{ - if (verbose) - print_timestamped_message ("adding kernel modules"); - - char **whitelist = NULL; - if (whitelist_file != NULL) - whitelist = load_file (whitelist_file); - - char *paths[2] = { (char *) modpath, NULL }; - FTS *fts = fts_open (paths, FTS_COMFOLLOW|FTS_PHYSICAL, NULL); - if (fts == NULL) - error (EXIT_FAILURE, errno, "add_kernel_modules: fts_open: %s", modpath); - - for (;;) { - errno = 0; - FTSENT *entry = fts_read (fts); - if (entry == NULL && errno != 0) - error (EXIT_FAILURE, errno, "add_kernel_modules: fts_read: %s", modpath); - if (entry == NULL) - break; - - /* Ignore directories being visited in post-order. */ - if (entry->fts_info & FTS_DP) - continue; - - /* Is it a *.ko file? */ - if (entry->fts_namelen >= 3 && - entry->fts_name[entry->fts_namelen-3] == '.' && - entry->fts_name[entry->fts_namelen-2] == 'k' && - entry->fts_name[entry->fts_namelen-1] == 'o') { - if (whitelist) { - /* Is it a *.ko file which is on the whitelist? */ - size_t j; - for (j = 0; whitelist[j] != NULL; ++j) { - int r; - r = fnmatch (whitelist[j], entry->fts_name, 0); - if (r == 0) { - /* It's on the whitelist, so include it. */ - if (verbose >= 2) - fprintf (stderr, "including kernel module %s (matches whitelist entry %s)\n", - entry->fts_name, whitelist[j]); - writer->wr_fts_entry (entry); - break; - } else if (r != FNM_NOMATCH) - error (EXIT_FAILURE, 0, "internal error: fnmatch ('%s', '%s', %d) returned unexpected non-zero value %d\n", - whitelist[j], entry->fts_name, 0, r); - } /* for (j) */ - } else { /* whitelist == NULL, always include */ - if (verbose >= 2) - fprintf (stderr, "including kernel module %s\n", entry->fts_name); - writer->wr_fts_entry (entry); - } - } else - /* It's some other sort of file, or a directory, always include. */ - writer->wr_fts_entry (entry); - } - - if (fts_close (fts) == -1) - error (EXIT_FAILURE, errno, "add_kernel_modules: fts_close: %s", modpath); -} - -/* Copy the host files. - * - * Read the list of entries in hostfiles (which may contain - * wildcards). Look them up in the filesystem, and add those files - * that exist. Ignore any files that don't exist or are not readable. - */ -static void -add_hostfiles (const char *hostfiles_file, struct writer *writer) -{ - char **hostfiles = load_file (hostfiles_file); - - /* Hostfiles list can contain "." before each path - ignore it. - * It also contains each directory name before we enter it. But - * we don't read that until we see a wildcard for that directory. - */ - size_t i, j; - for (i = 0; hostfiles[i] != NULL; ++i) { - char *hostfile = hostfiles[i]; - if (hostfile[0] == '.') - hostfile++; - - struct stat statbuf; - - /* Is it a wildcard? */ - if (strchr (hostfile, '*') || strchr (hostfile, '?')) { - char *dirname = xstrdup (hostfile); - char *patt = strrchr (dirname, '/'); - if (!patt) - error (EXIT_FAILURE, 0, "%s: line %zu: invalid pattern\n(is this file a supermin appliance hostfiles file?)", - hostfiles_file, i+1); - *patt++ = '\0'; - - char **files = read_dir (dirname); - files = filter_fnmatch (files, patt, FNM_NOESCAPE); - - /* Add matching files. */ - for (j = 0; files[j] != NULL; ++j) { - char *tmp = xasprintf ("%s/%s", dirname, files[j]); - - if (verbose >= 2) - fprintf (stderr, "including host file %s (matches %s)\n", tmp, patt); - - writer->wr_file (tmp); - - free (tmp); - } - } - /* Else does this file/directory/whatever exist? */ - else if (lstat (hostfile, &statbuf) == 0) { - if (verbose >= 2) - fprintf (stderr, "including host file %s (directly referenced)\n", - hostfile); - - writer->wr_file_stat (hostfile, &statbuf); - } /* Ignore files that don't exist. */ - } -} diff --git a/helper/bin2s.pl b/helper/bin2s.pl deleted file mode 100755 index 2c78b5e..0000000 --- a/helper/bin2s.pl +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/perl - -# This script creates a source file for the GNU assembler which shuold -# result in an object file equivalent to that of -# -# objcopy -I binary -B $(DEFAULT_ARCH) -O $(ELF_DEFAULT_ARCH) <in> <out> - -use strict; -use warnings; - -die "usage: $0 <in> <out>\n" if @ARGV != 2; - -my ($infile, $outfile) = @ARGV; -my ($buf, $i, $sz); -open my $ifh, '<', $infile or die "open $infile: $!"; -open my $ofh, '>', $outfile or die "open $outfile: $!"; - -print $ofh <<"EOF"; -/* This file has been automatically generated from $infile by $0 */ - -\t.globl\t_binary_${infile}_start -\t.globl\t_binary_${infile}_end -\t.globl\t_binary_${infile}_size - -\t.section\t.data -_binary_${infile}_start: -EOF - -$sz = 0; -while ( $i = read $ifh, $buf, 12 ) { - print $ofh "\t.byte\t" - . join( ',', map { sprintf '0x%02x', ord $_ } split //, $buf ) . "\n"; - $sz += $i; -} -die "read $infile (at offset $sz): $!\n" if not defined $i; -close $ifh; - -print $ofh <<"EOF"; - -_binary_${infile}_end: - -\t.equ _binary_${infile}_size, $sz -EOF - -close $ofh; diff --git a/helper/checksum.c b/helper/checksum.c deleted file mode 100644 index 11f4494..0000000 --- a/helper/checksum.c +++ /dev/null @@ -1,119 +0,0 @@ -/* supermin-helper reimplementation in C. - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include <config.h> - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <inttypes.h> -#include <unistd.h> -#include <errno.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <assert.h> - -#include "error.h" - -#include "helper.h" - -static FILE *pp = NULL; - -/* This is the command we run to calculate the SHA. Note that we sort - * the rows first so that the checksum is roughly stable, since the - * order that we output files might not be (eg. because we rely on the - * ordering of readdir). Uncomment the second line to see the output - * before hashing. - */ -static const char *shacmd = "sort | sha256sum | awk '{print $1}'"; -//static const char *shacmd = "sort | cat"; - -static void -checksum_start (const char *hostcpu, const char *appliance, - const char *modpath, const char *initrd) -{ - pp = popen (shacmd, "w"); - if (pp == NULL) - error (EXIT_FAILURE, errno, "popen: command failed: %s", shacmd); - - fprintf (pp, "%s %s %s %d\n", - PACKAGE_STRING, hostcpu, modpath, geteuid ()); -} - -static void -checksum_end (void) -{ - if (pclose (pp) == -1) - error (EXIT_FAILURE, errno, "pclose: command failed: %s", shacmd); - pp = NULL; -} - -static void -checksum_file_stat (const char *filename, const struct stat *statbuf) -{ - /* Publically writable directories (ie. /tmp) and special files - * don't have stable times. Since we only care about some - * attributes of directories and special files, we vary the output - * accordingly. - */ - if (S_ISREG (statbuf->st_mode)) - fprintf (pp, "%s %ld %ld %d %d %" PRIu64 " %o\n", - filename, - (long) statbuf->st_ctime, (long) statbuf->st_mtime, - statbuf->st_uid, statbuf->st_gid, (uint64_t) statbuf->st_size, - statbuf->st_mode); - else - fprintf (pp, "%s %d %d %o\n", - filename, - statbuf->st_uid, statbuf->st_gid, - statbuf->st_mode); -} - -static void -checksum_file (const char *filename) -{ - struct stat statbuf; - - if (lstat (filename, &statbuf) == -1) - error (EXIT_FAILURE, errno, "lstat: %s", filename); - checksum_file_stat (filename, &statbuf); -} - -static void -checksum_fts_entry (FTSENT *entry) -{ - if (entry->fts_info & FTS_NS || entry->fts_info & FTS_NSOK) - checksum_file (entry->fts_path); - else - checksum_file_stat (entry->fts_path, entry->fts_statp); -} - -static void -checksum_cpio_file (const char *cpio_file) -{ - checksum_file (cpio_file); -} - -struct writer checksum_writer = { - .wr_start = checksum_start, - .wr_end = checksum_end, - .wr_file = checksum_file, - .wr_file_stat = checksum_file_stat, - .wr_fts_entry = checksum_fts_entry, - .wr_cpio_file = checksum_cpio_file, -}; diff --git a/helper/cpio.c b/helper/cpio.c deleted file mode 100644 index 55db96a..0000000 --- a/helper/cpio.c +++ /dev/null @@ -1,279 +0,0 @@ -/* supermin-helper reimplementation in C. - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include <config.h> - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> -#include <limits.h> -#include <fcntl.h> -#include <errno.h> -#include <sys/stat.h> -#include <assert.h> - -#include "error.h" -#include "fts_.h" -#include "full-write.h" -#include "xalloc.h" -#include "xvasprintf.h" - -#include "helper.h" - -/* Buffer size used in copy operations throughout. Large for - * greatest efficiency. - */ -#define BUFFER_SIZE 65536 - -static int out_fd = -1; -static off_t out_offset = 0; - -static void write_file_to_fd (const char *filename); -static void write_file_len_to_fd (const char *filename, size_t len); -static void write_padding (size_t len); -static void cpio_append_fts_entry (FTSENT *entry); -static void cpio_append_stat (const char *filename, const struct stat *); -static void cpio_append (const char *filename); -static void cpio_append_trailer (void); - -/* Copy contents of buffer to out_fd and keep out_offset correct. */ -static void -write_to_fd (const void *buffer, size_t len) -{ - if (full_write (out_fd, buffer, len) != len) - error (EXIT_FAILURE, errno, "write"); - out_offset += len; -} - -/* Copy contents of file to out_fd. */ -static void -write_file_to_fd (const char *filename) -{ - char buffer[BUFFER_SIZE]; - int fd2; - ssize_t r; - - if (verbose >= 2) - fprintf (stderr, "write_file_to_fd %s -> %d\n", filename, out_fd); - - fd2 = open (filename, O_RDONLY); - if (fd2 == -1) - error (EXIT_FAILURE, errno, "open: %s", filename); - for (;;) { - r = read (fd2, buffer, sizeof buffer); - if (r == 0) - break; - if (r == -1) { - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) - continue; - error (EXIT_FAILURE, errno, "read: %s", filename); - } - write_to_fd (buffer, r); - } - - if (close (fd2) == -1) - error (EXIT_FAILURE, errno, "close: %s", filename); -} - -/* Copy file of given length to output, and fail if the file has - * changed size. - */ -static void -write_file_len_to_fd (const char *filename, size_t len) -{ - char buffer[BUFFER_SIZE]; - size_t count = 0; - - if (verbose >= 2) - fprintf (stderr, "write_file_to_fd %s -> %d\n", filename, out_fd); - - int fd2 = open (filename, O_RDONLY); - if (fd2 == -1) - error (EXIT_FAILURE, errno, "open: %s", filename); - for (;;) { - ssize_t r = read (fd2, buffer, sizeof buffer); - if (r == 0) - break; - if (r == -1) { - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) - continue; - error (EXIT_FAILURE, errno, "read: %s", filename); - } - write_to_fd (buffer, r); - count += r; - if (count > len) - error (EXIT_FAILURE, 0, "write_file_len_to_fd: %s: file has increased in size\n", filename); - } - - if (close (fd2) == -1) - error (EXIT_FAILURE, errno, "close: %s", filename); - - if (count != len) - error (EXIT_FAILURE, 0, "supermin-helper: write_file_len_to_fd: %s: file has changed size\n", filename); -} - -/* Append the file pointed to by FTSENT to the cpio output. */ -static void -cpio_append_fts_entry (FTSENT *entry) -{ - if (entry->fts_info & FTS_NS || entry->fts_info & FTS_NSOK) - cpio_append (entry->fts_path); - else - cpio_append_stat (entry->fts_path, entry->fts_statp); -} - -/* Append the file named 'filename' to the cpio output. */ -static void -cpio_append (const char *filename) -{ - struct stat statbuf; - - if (lstat (filename, &statbuf) == -1) - error (EXIT_FAILURE, errno, "lstat: %s", filename); - cpio_append_stat (filename, &statbuf); -} - -/* Append the file to the cpio output. */ -#define PADDING(len) ((((len) + 3) & ~3) - (len)) - -#define CPIO_HEADER_LEN (6 + 13*8) - -static void -cpio_append_stat (const char *filename, const struct stat *statbuf) -{ - const char *orig_filename = filename; - - if (*filename == '/') - filename++; - if (*filename == '\0') - filename = "."; - - if (verbose >= 2) - fprintf (stderr, "cpio_append_stat %s 0%o -> %d\n", - orig_filename, statbuf->st_mode, out_fd); - - /* Regular files and symlinks are the only ones that have a "body" - * in this cpio entry. - */ - int has_body = S_ISREG (statbuf->st_mode) || S_ISLNK (statbuf->st_mode); - - size_t len = strlen (filename) + 1; - - char header[CPIO_HEADER_LEN + 1]; - snprintf (header, sizeof header, - "070701" /* magic */ - "%08X" /* inode */ - "%08X" /* mode */ - "%08X" "%08X" /* uid, gid */ - "%08X" /* nlink */ - "%08X" /* mtime */ - "%08X" /* file length */ - "%08X" "%08X" /* device holding file major, minor */ - "%08X" "%08X" /* for specials, device major, minor */ - "%08X" /* name length (including \0 byte) */ - "%08X", /* checksum (not used by the kernel) */ - (unsigned) statbuf->st_ino, statbuf->st_mode, - statbuf->st_uid, statbuf->st_gid, - (unsigned) statbuf->st_nlink, (unsigned) statbuf->st_mtime, - has_body ? (unsigned) statbuf->st_size : 0, - major (statbuf->st_dev), minor (statbuf->st_dev), - major (statbuf->st_rdev), minor (statbuf->st_rdev), - (unsigned) len, 0); - - /* Write the header. */ - write_to_fd (header, CPIO_HEADER_LEN); - - /* Follow with the filename, and pad it. */ - write_to_fd (filename, len); - size_t padding_len = PADDING (CPIO_HEADER_LEN + len); - write_padding (padding_len); - - /* Follow with the file or symlink content, and pad it. */ - if (has_body) { - if (S_ISREG (statbuf->st_mode)) - write_file_len_to_fd (orig_filename, statbuf->st_size); - else if (S_ISLNK (statbuf->st_mode)) { - char tmp[PATH_MAX]; - if (readlink (orig_filename, tmp, sizeof tmp) == -1) - error (EXIT_FAILURE, errno, "readlink: %s", orig_filename); - write_to_fd (tmp, statbuf->st_size); - } - - padding_len = PADDING (statbuf->st_size); - write_padding (padding_len); - } -} - -/* CPIO voodoo. */ -static void -cpio_append_trailer (void) -{ - struct stat statbuf; - memset (&statbuf, 0, sizeof statbuf); - statbuf.st_nlink = 1; - cpio_append_stat ("TRAILER!!!", &statbuf); - - /* CPIO seems to pad up to the next block boundary, ie. up to - * the next 512 bytes. - */ - write_padding (((out_offset + 511) & ~511) - out_offset); - assert ((out_offset & 511) == 0); -} - -/* Write 'len' bytes of zeroes out. */ -static void -write_padding (size_t len) -{ - static const char buffer[512] = { 0 }; - - while (len > 0) { - size_t n = len < sizeof buffer ? len : sizeof buffer; - write_to_fd (buffer, n); - len -= n; - } -} - -static void -cpio_start (const char *hostcpu, const char *appliance, - const char *modpath, const char *initrd) -{ - out_fd = open (initrd, O_WRONLY | O_CREAT | O_TRUNC | O_NOCTTY, 0644); - if (out_fd == -1) - error (EXIT_FAILURE, errno, "open: %s", initrd); - out_offset = 0; -} - -static void -cpio_end (void) -{ - cpio_append_trailer (); - - /* Finish off and close output file. */ - if (close (out_fd) == -1) - error (EXIT_FAILURE, errno, "close"); -} - -struct writer cpio_writer = { - .wr_start = cpio_start, - .wr_end = cpio_end, - .wr_file = cpio_append, - .wr_file_stat = cpio_append_stat, - .wr_fts_entry = cpio_append_fts_entry, - .wr_cpio_file = write_file_to_fd, -}; diff --git a/helper/ext2.c b/helper/ext2.c deleted file mode 100644 index 2d75fb2..0000000 --- a/helper/ext2.c +++ /dev/null @@ -1,538 +0,0 @@ -/* supermin-helper reimplementation in C. - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include <config.h> - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <fcntl.h> -#include <unistd.h> -#include <errno.h> -#include <limits.h> -#include <sys/stat.h> -#include <assert.h> - -#include "error.h" -#include "fts_.h" -#include "xvasprintf.h" - -#include "helper.h" -#include "ext2internal.h" - -ext2_filsys fs; - -/* The ext2 image that we build always has a fixed size, and we 'hope' - * that the files fit in (otherwise we'll get an error). Note that - * the file is sparsely allocated. - * - * The downside of allocating a very large initial disk is that the - * fixed overhead of ext2 is larger (since ext2 calculates it based on - * the size of the disk). For a 4GB disk the overhead is - * approximately 66MB. - * - * In future, make this configurable, or determine it from the input - * files (XXX). - */ -#define APPLIANCE_SIZE ((off_t)4*1024*1024*1024) - -static void -ext2_start (const char *hostcpu, const char *appliance, - const char *modpath, const char *initrd) -{ - initialize_ext2_error_table (); - - /* Make the initrd. */ - ext2_make_initrd (modpath, initrd); - - /* Make the appliance sparse image. */ - int fd = open (appliance, O_WRONLY | O_CREAT | O_TRUNC | O_NOCTTY, 0644); - if (fd == -1) - error (EXIT_FAILURE, errno, "open: %s", appliance); - - if (lseek (fd, APPLIANCE_SIZE - 1, SEEK_SET) == (off_t) -1) - error (EXIT_FAILURE, errno, "lseek"); - - char c = 0; - if (write (fd, &c, 1) != 1) - error (EXIT_FAILURE, errno, "write"); - - if (close (fd) == -1) - error (EXIT_FAILURE, errno, "close"); - - /* Run mke2fs on the file. - * XXX Quoting, but this string doesn't come from an untrusted source. - */ - char *cmd = xasprintf ("%s %s ext2 -F%s '%s'", - MKE2FS, MKE2FS_T_OPTION, - verbose >= 2 ? "" : "q", - appliance); - int r = system (cmd); - if (r == -1 || WEXITSTATUS (r) != 0) - error (EXIT_FAILURE, 0, "%s: failed", cmd); - free (cmd); - - if (verbose) - print_timestamped_message ("finished mke2fs"); - - /* Open the filesystem. */ - int fs_flags = EXT2_FLAG_RW; -#ifdef EXT2_FLAG_64BITS - fs_flags |= EXT2_FLAG_64BITS; -#endif - errcode_t err = - ext2fs_open (appliance, fs_flags, 0, 0, unix_io_manager, &fs); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_open: %s", error_message (err)); - - /* Bitmaps are not loaded by default, so load them. ext2fs_close will - * write out any changes. - */ - err = ext2fs_read_bitmaps (fs); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_read_bitmaps: %s", error_message (err)); -} - -static void -ext2_end (void) -{ - if (verbose) - print_timestamped_message ("closing ext2 filesystem"); - - /* Write out changes and close. */ - errcode_t err; -#ifdef HAVE_EXT2FS_CLOSE2 - err = ext2fs_close2 (fs, EXT2_FLAG_FLUSH_NO_SYNC); -#else - err = ext2fs_close (fs); -#endif - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_close: %s", error_message (err)); -} - -void -ext2_mkdir (ext2_ino_t dir_ino, const char *dirname, const char *basename, - mode_t mode, uid_t uid, gid_t gid, - time_t ctime, time_t atime, time_t mtime) -{ - errcode_t err; - - mode = LINUX_S_IFDIR | (mode & 03777); - - /* Does the directory exist? This is legitimate: we just skip - * this case. - */ - ext2_ino_t ino; - err = ext2fs_namei (fs, EXT2_ROOT_INO, dir_ino, basename, &ino); - if (err == 0) - return; /* skip */ - - /* Otherwise, create it. */ - err = ext2fs_new_inode (fs, dir_ino, mode, 0, &ino); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_new_inode: %s", error_message (err)); - - try_again: - err = ext2fs_mkdir (fs, dir_ino, ino, basename); - if (err != 0) { - /* See: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=217892 */ - if (err == EXT2_ET_DIR_NO_SPACE) { - err = ext2fs_expand_dir (fs, dir_ino); - if (err) - error (EXIT_FAILURE, 0, "ext2fs_expand_dir: %s/%s: %s", - dirname, basename, error_message (err)); - goto try_again; - } else - error (EXIT_FAILURE, 0, "ext2fs_mkdir: %s/%s: %s", - dirname, basename, error_message (err)); - } - - /* Copy the final permissions, UID etc. to the inode. */ - struct ext2_inode inode; - err = ext2fs_read_inode (fs, ino, &inode); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_read_inode: %s", error_message (err)); - inode.i_mode = mode; - inode.i_uid = uid; - inode.i_gid = gid; - inode.i_ctime = ctime; - inode.i_atime = atime; - inode.i_mtime = mtime; - err = ext2fs_write_inode (fs, ino, &inode); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_write_inode: %s", error_message (err)); -} - -void -ext2_empty_inode (ext2_ino_t dir_ino, const char *dirname, const char *basename, - mode_t mode, uid_t uid, gid_t gid, - time_t ctime, time_t atime, time_t mtime, - int major, int minor, int dir_ft, ext2_ino_t *ino_ret) -{ - errcode_t err; - struct ext2_inode inode; - ext2_ino_t ino; - - err = ext2fs_new_inode (fs, dir_ino, mode, 0, &ino); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_new_inode: %s", error_message (err)); - - memset (&inode, 0, sizeof inode); - inode.i_mode = mode; - inode.i_uid = uid; - inode.i_gid = gid; - inode.i_blocks = 0; - inode.i_links_count = 1; - inode.i_ctime = ctime; - inode.i_atime = atime; - inode.i_mtime = mtime; - inode.i_size = 0; - inode.i_block[0] = (minor & 0xff) | (major << 8) | ((minor & ~0xff) << 12); - - err = ext2fs_write_new_inode (fs, ino, &inode); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_write_inode: %s", error_message (err)); - - ext2_link (dir_ino, basename, ino, dir_ft); - - ext2fs_inode_alloc_stats2 (fs, ino, 1, 0); - - if (ino_ret) - *ino_ret = ino; -} - -/* You must create the file first with ext2_empty_inode. */ -void -ext2_write_file (ext2_ino_t ino, const char *buf, size_t size, - const char *orig_filename) -{ - errcode_t err; - ext2_file_t file; - err = ext2fs_file_open2 (fs, ino, NULL, EXT2_FILE_WRITE, &file); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_file_open2: %s: %s", - orig_filename, error_message (err)); - - /* ext2fs_file_write cannot deal with partial writes. You have - * to write the entire file in a single call. - */ - unsigned int written; - err = ext2fs_file_write (file, buf, size, &written); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_file_write: %s: %s\n" - "Block allocation failures can happen here for several reasons:\n" - " - /lib/modules contains modules with debug that makes the modules very large\n" - " - a file listed in 'hostfiles' is unexpectedly very large\n" - " - too many packages added to the supermin appliance", - orig_filename, error_message (err)); - if ((size_t) written != size) - error (EXIT_FAILURE, 0, - "ext2fs_file_write: %s: size = %zu != written = %u\n", - orig_filename, size, written); - - err = ext2fs_file_flush (file); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_file_flush: %s: %s", - orig_filename, error_message (err)); - err = ext2fs_file_close (file); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_file_close: %s: %s", - orig_filename, error_message (err)); - - /* Update the true size in the inode. */ - struct ext2_inode inode; - err = ext2fs_read_inode (fs, ino, &inode); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_read_inode: %s: %s", - orig_filename, error_message (err)); - inode.i_size = size; - err = ext2fs_write_inode (fs, ino, &inode); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_write_inode: %s: %s", - orig_filename, error_message (err)); -} - -/* This is just a wrapper around ext2fs_link which calls - * ext2fs_expand_dir as necessary if the directory fills up. See - * definition of expand_dir in the sources of debugfs. - */ -void -ext2_link (ext2_ino_t dir_ino, const char *basename, ext2_ino_t ino, int dir_ft) -{ - errcode_t err; - - again: - err = ext2fs_link (fs, dir_ino, basename, ino, dir_ft); - - if (err == EXT2_ET_DIR_NO_SPACE) { - err = ext2fs_expand_dir (fs, dir_ino); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2_link: ext2fs_expand_dir: %s: %s", - basename, error_message (err)); - goto again; - } - - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_link: %s: %s", - basename, error_message (err)); -} - -static int -release_block (ext2_filsys fs, blk_t *blocknr, - int blockcnt, void *private) -{ - blk_t block; - - block = *blocknr; - ext2fs_block_alloc_stats (fs, block, -1); - return 0; -} - -/* unlink or rmdir path, if it exists. */ -void -ext2_clean_path (ext2_ino_t dir_ino, - const char *dirname, const char *basename, - int isdir) -{ - errcode_t err; - - ext2_ino_t ino; - err = ext2fs_lookup (fs, dir_ino, basename, strlen (basename), - NULL, &ino); - if (err == EXT2_ET_FILE_NOT_FOUND) - return; - - if (!isdir) { - struct ext2_inode inode; - err = ext2fs_read_inode (fs, ino, &inode); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_read_inode: %s", error_message (err)); - inode.i_links_count--; - err = ext2fs_write_inode (fs, ino, &inode); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_write_inode: %s", error_message (err)); - - err = ext2fs_unlink (fs, dir_ino, basename, 0, 0); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_unlink_inode: %s", error_message (err)); - - if (inode.i_links_count == 0) { - inode.i_dtime = time (NULL); - err = ext2fs_write_inode (fs, ino, &inode); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2fs_write_inode: %s", error_message (err)); - - if (ext2fs_inode_has_valid_blocks (&inode)) { - int flags = 0; - /* From the docs: "BLOCK_FLAG_READ_ONLY is a promise by the - * caller that it will not modify returned block number." - * RHEL 5 does not have this flag, so just omit it if it is - * not defined. - */ -#ifdef BLOCK_FLAG_READ_ONLY - flags |= BLOCK_FLAG_READ_ONLY; -#endif - ext2fs_block_iterate (fs, ino, flags, NULL, - release_block, NULL); - } - - ext2fs_inode_alloc_stats2 (fs, ino, -1, isdir); - } - } - /* else it's a directory, what to do? XXX */ -} - -/* Read in the whole file into memory. Check the size is still 'size'. */ -static char * -read_whole_file (const char *filename, size_t size) -{ - char *buf = malloc (size); - if (buf == NULL) - error (EXIT_FAILURE, errno, "malloc"); - - int fd = open (filename, O_RDONLY); - if (fd == -1) - error (EXIT_FAILURE, errno, "open: %s", filename); - - size_t n = 0; - char *p = buf; - - while (n < size) { - ssize_t r = read (fd, p, size - n); - if (r == -1) - error (EXIT_FAILURE, errno, "read: %s", filename); - if (r == 0) - error (EXIT_FAILURE, 0, - "error: file has changed size unexpectedly: %s", filename); - n += r; - p += r; - } - - if (close (fd) == -1) - error (EXIT_FAILURE, errno, "close: %s", filename); - - return buf; -} - -/* Add a file (or directory etc) from the host. */ -static void -ext2_file_stat (const char *orig_filename, const struct stat *statbuf) -{ - errcode_t err; - - if (verbose >= 2) - fprintf (stderr, "ext2_file_stat %s 0%o\n", - orig_filename, statbuf->st_mode); - - /* Sanity check the path. These rules are always true for the paths - * passed to us here from the appliance layer. The assertions just - * verify that the rules haven't changed. - */ - size_t n = strlen (orig_filename); - assert (n <= PATH_MAX); - assert (n > 0); - assert (orig_filename[0] == '/'); /* always absolute path */ - assert (n == 1 || orig_filename[n-1] != '/'); /* no trailing slash */ - - /* Don't make the root directory, it always exists. This simplifies - * the code that follows. - */ - if (n == 1) return; - - const char *dirname, *basename; - const char *p = strrchr (orig_filename, '/'); - ext2_ino_t dir_ino; - if (orig_filename == p) { /* "/foo" */ - dirname = "/"; - basename = orig_filename+1; - dir_ino = EXT2_ROOT_INO; - } else { /* "/foo/bar" */ - dirname = strndup (orig_filename, p-orig_filename); - basename = p+1; - - /* If the parent directory is a symlink to another directory, then - * we want to look up the target directory. (RHBZ#698089). - */ - struct stat stat1, stat2; - if (lstat (dirname, &stat1) == 0 && S_ISLNK (stat1.st_mode) && - stat (dirname, &stat2) == 0 && S_ISDIR (stat2.st_mode)) { - char *new_dirname = malloc (PATH_MAX+1); - ssize_t r = readlink (dirname, new_dirname, PATH_MAX+1); - if (r == -1) - error (EXIT_FAILURE, errno, "readlink: %s", orig_filename); - new_dirname[r] = '\0'; - dirname = new_dirname; - } - - /* Look up the parent directory. */ - err = ext2fs_namei (fs, EXT2_ROOT_INO, EXT2_ROOT_INO, dirname, &dir_ino); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2: parent directory not found: %s: %s", - dirname, error_message (err)); - } - - ext2_clean_path (dir_ino, dirname, basename, S_ISDIR (statbuf->st_mode)); - - int dir_ft; - - /* Create regular file. */ - if (S_ISREG (statbuf->st_mode)) { - /* XXX Hard links get duplicated here. */ - ext2_ino_t ino; - ext2_empty_inode (dir_ino, dirname, basename, - statbuf->st_mode, statbuf->st_uid, statbuf->st_gid, - statbuf->st_ctime, statbuf->st_atime, statbuf->st_mtime, - 0, 0, EXT2_FT_REG_FILE, &ino); - - if (statbuf->st_size > 0) { - char *buf = read_whole_file (orig_filename, statbuf->st_size); - ext2_write_file (ino, buf, statbuf->st_size, orig_filename); - free (buf); - } - } - /* Create a symlink. */ - else if (S_ISLNK (statbuf->st_mode)) { - ext2_ino_t ino; - ext2_empty_inode (dir_ino, dirname, basename, - statbuf->st_mode, statbuf->st_uid, statbuf->st_gid, - statbuf->st_ctime, statbuf->st_atime, statbuf->st_mtime, - 0, 0, EXT2_FT_SYMLINK, &ino); - - char buf[PATH_MAX+1]; - ssize_t r = readlink (orig_filename, buf, sizeof buf); - if (r == -1) - error (EXIT_FAILURE, errno, "readlink: %s", orig_filename); - ext2_write_file (ino, buf, r, orig_filename); - } - /* Create directory. */ - else if (S_ISDIR (statbuf->st_mode)) - ext2_mkdir (dir_ino, dirname, basename, - statbuf->st_mode, statbuf->st_uid, statbuf->st_gid, - statbuf->st_ctime, statbuf->st_atime, statbuf->st_mtime); - /* Create a special file. */ - else if (S_ISBLK (statbuf->st_mode)) { - dir_ft = EXT2_FT_BLKDEV; - goto make_special; - } - else if (S_ISCHR (statbuf->st_mode)) { - dir_ft = EXT2_FT_CHRDEV; - goto make_special; - } else if (S_ISFIFO (statbuf->st_mode)) { - dir_ft = EXT2_FT_FIFO; - goto make_special; - } else if (S_ISSOCK (statbuf->st_mode)) { - dir_ft = EXT2_FT_SOCK; - make_special: - ext2_empty_inode (dir_ino, dirname, basename, - statbuf->st_mode, statbuf->st_uid, statbuf->st_gid, - statbuf->st_ctime, statbuf->st_atime, statbuf->st_mtime, - major (statbuf->st_rdev), minor (statbuf->st_rdev), - dir_ft, NULL); - } -} - -static void -ext2_file (const char *filename) -{ - struct stat statbuf; - - if (lstat (filename, &statbuf) == -1) - error (EXIT_FAILURE, errno, "lstat: %s", filename); - ext2_file_stat (filename, &statbuf); -} - -/* In theory this could be optimized to avoid a namei lookup, but - * it probably wouldn't make much difference. - */ -static void -ext2_fts_entry (FTSENT *entry) -{ - if (entry->fts_info & FTS_NS || entry->fts_info & FTS_NSOK) - ext2_file (entry->fts_path); - else - ext2_file_stat (entry->fts_path, entry->fts_statp); -} - -struct writer ext2_writer = { - .wr_start = ext2_start, - .wr_end = ext2_end, - .wr_file = ext2_file, - .wr_file_stat = ext2_file_stat, - .wr_fts_entry = ext2_fts_entry, - .wr_cpio_file = ext2_cpio_file, -}; diff --git a/helper/ext2cpio.c b/helper/ext2cpio.c deleted file mode 100644 index 9f36aea..0000000 --- a/helper/ext2cpio.c +++ /dev/null @@ -1,418 +0,0 @@ -/* supermin-helper reimplementation in C. - * Copyright (C) 2009-2013, 2012 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include <config.h> - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <fcntl.h> -#include <unistd.h> -#include <errno.h> -#include <limits.h> -#include <sys/stat.h> -#include <assert.h> - -#ifdef HAVE_ZLIB -#include <zlib.h> -#endif - -#include "error.h" - -#include "helper.h" -#include "ext2internal.h" - -/* This function must unpack the cpio file and add the files it - * contains to the ext2 filesystem. Essentially this is doing the - * same thing as the kernel init/initramfs.c code. Note that we - * assume that the cpio is 'newc' format and can't/won't deal with - * anything else. All this cpio parsing code is copied to some extent - * from init/initramfs.c in the kernel. - */ -#define N_ALIGN(len) ((((len) + 1) & ~3) + 2) - -static unsigned long cpio_ino, nlink; -static mode_t mode; -static unsigned long body_len, name_len; -static uid_t uid; -static gid_t gid; -static time_t mtime; -static int dev_major, dev_minor, rdev_major, rdev_minor; -static loff_t curr, next_header; -#ifdef HAVE_ZLIB -static gzFile gzfp; -#else -static FILE *fp; -#endif -static const char *input_file; - -static int xread (void *buffer, size_t size); -static void parse_header (char *s); -static int parse_next_entry (void); -static void skip_to_next_header (void); -static void read_file (void); -static char *read_whole_body (void); -static ext2_ino_t maybe_link (void); -static void add_link (ext2_ino_t real_ino); -static void clear_links (void); - -void -ext2_cpio_file (const char *cpio_file) -{ - /* Save this for error messages in xread. */ - input_file = cpio_file; - -#ifdef HAVE_ZLIB - gzfp = gzopen (cpio_file, "rb"); - if (gzfp == NULL) - error (EXIT_FAILURE, errno, "open: %s", cpio_file); -#else - fp = fopen (cpio_file, "r"); - if (fp == NULL) - error (EXIT_FAILURE, errno, "open: %s", cpio_file); -#endif - - curr = 0; - while (parse_next_entry ()) - ; - -#ifdef HAVE_ZLIB - gzclose (gzfp); -#else - fclose (fp); -#endif -} - -/* Read 'size' bytes from the handle and write it into 'buffer'. If - * the read fails or there is a partial read, exit with an error. If - * end of file, return 0. If the full data was read, return > 0. - */ -static int -xread (void *buffer, size_t size) -{ -#ifdef HAVE_ZLIB - int r; - - r = gzread (gzfp, buffer, size); - if (r == -1) { - int errnum; - const char *errstr; - - errstr = gzerror (gzfp, &errnum); - error (EXIT_FAILURE, 0, "gzread: %s: %s (%d)", input_file, errstr, errnum); - } - if (r == 0) - return 0; - if ((size_t) r < size) - error (EXIT_FAILURE, 0, "gzread: %s: unexpected end of file", input_file); -#else - clearerr (fp); - if (fread (buffer, size, 1, fp) != 1) { - if (feof (fp)) - return 0; - error (EXIT_FAILURE, errno, "fread: %s: read failure", input_file); - } -#endif - return 1; -} - -static int -parse_next_entry (void) -{ - char header[110]; - - /* Skip padding and synchronize with the next header. */ - again: - if (xread (&header[0], 4) == 0) - return 0; - - curr += 4; - if (memcmp (header, "\0\0\0\0", 4) == 0) - goto again; - - /* Read the rest of the header field. */ - if (xread (&header[4], sizeof header - 4) == 0) - error (EXIT_FAILURE, errno, "%s: unexpected end of file reading cpio file", - input_file); - curr += sizeof header - 4; - - if (verbose >= 2) { - char header2[sizeof header + 1]; - memcpy (header2, header, sizeof header); - header2[sizeof header] = '\0'; - fprintf (stderr, "cpio header %s\n", header2); - } - - if (memcmp (header, "070707", 6) == 0) - error (EXIT_FAILURE, 0, "incorrect cpio method: use -H newc option"); - if (memcmp (header, "070701", 6) != 0) - error (EXIT_FAILURE, 0, "input is not a cpio file"); - - parse_header (header); - - next_header = curr + N_ALIGN(name_len) + body_len; - next_header = (next_header + 3) & ~3; - if (name_len <= 0 || name_len > PATH_MAX) - skip_to_next_header (); - else if (S_ISLNK (mode)) { - if (body_len <= 0 || body_len > PATH_MAX) - skip_to_next_header (); - else - read_file (); - } - else if (!S_ISREG (mode) && body_len > 0) - skip_to_next_header (); /* only regular files have bodies */ - else - read_file (); /* could be file, directory, block special, ... */ - - return 1; -} - -static void -parse_header (char *s) -{ - unsigned long parsed[12]; - char buf[9]; - int i; - - buf[8] = '\0'; - for (i = 0, s += 6; i < 12; i++, s += 8) { - memcpy (buf, s, 8); - parsed[i] = strtoul (buf, NULL, 16); - } - cpio_ino = parsed[0]; /* fake inode number from cpio file */ - mode = parsed[1]; - uid = parsed[2]; - gid = parsed[3]; - nlink = parsed[4]; - mtime = parsed[5]; - body_len = parsed[6]; - dev_major = parsed[7]; - dev_minor = parsed[8]; - rdev_major = parsed[9]; - rdev_minor = parsed[10]; - name_len = parsed[11]; -} - -static void -skip_to_next_header (void) -{ - char buf[65536]; - - while (curr < next_header) { - size_t bytes = (size_t) (next_header - curr); - if (bytes > sizeof buf) - bytes = sizeof buf; - if (xread (buf, bytes) == 0) - error (EXIT_FAILURE, errno, "%s: unexpected end of cpio file", - input_file); - curr += bytes; - } -} - -/* Read any sort of file. The body will only be present for - * regular files and symlinks. - */ -static void -read_file (void) -{ - errcode_t err; - int dir_ft; - char name[N_ALIGN(name_len)+1]; /* asserted above this is <= PATH_MAX */ - - if (xread (name, N_ALIGN(name_len)) == 0) - error (EXIT_FAILURE, errno, "%s: unexpected end of file reading name field in cpio file", - input_file); - curr += N_ALIGN(name_len); - - name[name_len] = '\0'; - - if (verbose >= 2) - fprintf (stderr, "ext2 read_file %s %o\n", name, mode); - - if (strcmp (name, "TRAILER!!!") == 0) { - clear_links (); - goto skip; - } - - /* The name will be something like "bin/ls" or "./bin/ls". It won't - * (ever?) be an absolute path. Skip leading parts, and if it refers - * to the root directory just skip it entirely. - */ - char *dirname = name, *basename; - if (*dirname == '.') - dirname++; - if (*dirname == '/') - dirname++; - if (*dirname == '\0') - goto skip; - - ext2_ino_t dir_ino; - basename = strrchr (dirname, '/'); - if (basename == NULL) { - basename = dirname; - dir_ino = EXT2_ROOT_INO; - } else { - *basename++ = '\0'; - - /* Look up the parent directory. */ - err = ext2fs_namei (fs, EXT2_ROOT_INO, EXT2_ROOT_INO, dirname, &dir_ino); - if (err != 0) - error (EXIT_FAILURE, 0, "ext2: parent directory not found: %s: %s", - dirname, error_message (err)); - } - - if (verbose >= 2) - fprintf (stderr, "ext2 read_file dirname %s basename %s\n", - dirname, basename); - - ext2_clean_path (dir_ino, dirname, basename, S_ISDIR (mode)); - - /* Create a regular file. */ - if (S_ISREG (mode)) { - ext2_ino_t ml = maybe_link (); - ext2_ino_t ino; - if (ml <= 1) { - ext2_empty_inode (dir_ino, dirname, basename, - mode, uid, gid, mtime, mtime, mtime, - 0, 0, EXT2_FT_REG_FILE, &ino); - if (ml == 1) - add_link (ino); - } - else /* ml >= 2 */ { - /* It's a hard link back to a previous file. */ - ino = ml; - ext2_link (dir_ino, basename, ino, EXT2_FT_REG_FILE); - } - - if (body_len) { - char *buf = read_whole_body (); - ext2_write_file (ino, buf, body_len, name); - free (buf); - } - } - /* Create a symlink. */ - else if (S_ISLNK (mode)) { - ext2_ino_t ino; - ext2_empty_inode (dir_ino, dirname, basename, - mode, uid, gid, mtime, mtime, mtime, - 0, 0, EXT2_FT_SYMLINK, &ino); - - char *buf = read_whole_body (); - ext2_write_file (ino, buf, body_len, name); - free (buf); - } - /* Create a directory. */ - else if (S_ISDIR (mode)) { - ext2_mkdir (dir_ino, dirname, basename, - mode, uid, gid, mtime, mtime, mtime); - } - /* Create a special file. */ - else if (S_ISBLK (mode)) { - dir_ft = EXT2_FT_BLKDEV; - goto make_special; - } - else if (S_ISCHR (mode)) { - dir_ft = EXT2_FT_CHRDEV; - goto make_special; - } else if (S_ISFIFO (mode)) { - dir_ft = EXT2_FT_FIFO; - goto make_special; - } else if (S_ISSOCK (mode)) { - dir_ft = EXT2_FT_SOCK; - make_special: - /* Just like the kernel, we ignore special files with nlink > 1. */ - if (maybe_link () == 0) - ext2_empty_inode (dir_ino, dirname, basename, - mode, uid, gid, mtime, mtime, mtime, - rdev_major, rdev_minor, dir_ft, NULL); - } - - skip: - skip_to_next_header (); -} - -static char * -read_whole_body (void) -{ - char *buf = malloc (body_len); - if (buf == NULL) - error (EXIT_FAILURE, errno, "malloc"); - - if (xread (buf, body_len) == 0) - error (EXIT_FAILURE, errno, "%s: unexpected end of file reading body in cpio file", - input_file); - curr += body_len; - - return buf; -} - -struct links { - struct links *next; - unsigned long cpio_ino; /* fake ino from cpio file */ - int minor; - int major; - ext2_ino_t real_ino; /* real inode number on ext2 filesystem */ -}; -static struct links *links_head = NULL; - -/* If it's a hard link, return the linked inode number in the real - * ext2 filesystem. - * - * Returns: 0 = not a hard link - * 1 = possible unresolved hard link - * inode number = resolved hard link to this inode - */ -static ext2_ino_t -maybe_link (void) -{ - if (nlink >= 2) { - struct links *p; - for (p = links_head; p; p = p->next) { - if (p->cpio_ino != cpio_ino) - continue; - if (p->minor != dev_minor) - continue; - if (p->major != dev_major) - continue; - return p->real_ino; - } - return 1; - } - - return 0; -} - -static void -add_link (ext2_ino_t real_ino) -{ - struct links *p = malloc (sizeof (*p)); - p->cpio_ino = cpio_ino; - p->minor = dev_minor; - p->major = dev_major; - p->real_ino = real_ino; - p->next = links_head; - links_head = p; -} - -static void -clear_links (void) -{ - /* Don't bother to free the linked list in this short-lived program. */ - links_head = NULL; -} diff --git a/helper/ext2initrd.c b/helper/ext2initrd.c deleted file mode 100644 index 8dab5e9..0000000 --- a/helper/ext2initrd.c +++ /dev/null @@ -1,281 +0,0 @@ -/* supermin-helper reimplementation in C. - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -/* ext2 requires a small initrd in order to boot. This builds it. */ - -#include <config.h> - -#include <stdio.h> -#include <stdlib.h> -#include <fcntl.h> -#include <unistd.h> -#include <dirent.h> -#include <errno.h> -#include <assert.h> -#include <fnmatch.h> - -#include "error.h" -#include "full-write.h" -#include "xalloc.h" -#include "xvasprintf.h" - -#include "helper.h" -#include "ext2internal.h" - -static void read_module_deps (const char *modpath); -static void free_module_deps (void); -static void add_module_dep (const char *name, const char *dep); -static struct module * add_module (const char *name); -static struct module * find_module (const char *name); -static void print_module_load_order (FILE *f, FILE *pp, struct module *m); - -/* The init binary. */ -extern char _binary_init_start, _binary_init_end, _binary_init_size; - -/* The list of modules (wildcards) we consider for inclusion in the - * mini initrd. Only what is needed in order to find a device with an - * ext2 filesystem on it. - */ -static const char *kmods[] = { - "ext2.ko*", - "ext4.ko*", /* CONFIG_EXT4_USE_FOR_EXT23=y option might be set */ - "virtio*.ko*", - "ide*.ko*", - "libata*.ko*", - "piix*.ko*", - "scsi_transport_spi.ko*", - "scsi_mod.ko*", - "sd_mod.ko*", - "sym53c8xx.ko*", - "ata_piix.ko*", - "sr_mod.ko*", - "mbcache.ko*", - "crc*.ko*", - "libcrc*.ko*", - "ibmvscsic.ko*", - "megaraid*.ko*", - NULL -}; - -/* Module dependencies. */ -struct module { - struct module *next; - struct moddep *deps; - char *name; - int visited; -}; -struct module *modules = NULL; - -struct moddep { - struct moddep *next; - struct module *dep; -}; - -void -ext2_make_initrd (const char *modpath, const char *initrd) -{ - char dir[] = "/tmp/ext2initrdXXXXXX"; - if (mkdtemp (dir) == NULL) - error (EXIT_FAILURE, errno, "mkdtemp"); - - read_module_deps (modpath); - add_module (""); - int i; - struct module *m; - for (i = 0; kmods[i] != NULL; ++i) { - for (m = modules; m; m = m->next) { - char *n = strrchr (m->name, '/'); - if (n) - n += 1; - else - n = m->name; - if (fnmatch (kmods[i], n, FNM_PATHNAME) == 0) { - if (verbose >= 2) - fprintf (stderr, "Adding top-level dependency %s (%s)\n", m->name, kmods[i]); - add_module_dep ("", m->name); - } - } - } - - char *cmd = xasprintf ("cd %s; xargs cp -t %s", modpath, dir); - char *outfile = xasprintf ("%s/modules", dir); - if (verbose >= 2) fprintf (stderr, "writing to %s\n", cmd); - - FILE *f = fopen (outfile, "w"); - if (f == NULL) - error (EXIT_FAILURE, errno, "failed to create modules list (%s)", outfile); - free (outfile); - FILE *pp = popen (cmd, "w"); - if (pp == NULL) - error (EXIT_FAILURE, errno, "failed to create pipe (%s)", cmd); - - /* The "pseudo" module depends on all modules matched by the contents of kmods */ - struct module *pseudo = find_module (""); - print_module_load_order (pp, f, pseudo); - fclose (pp); - pclose (f); - - free (cmd); - free_module_deps (); - - /* Copy in the init program, linked into this program as a data blob. */ - char *init = xasprintf ("%s/init", dir); - int fd = open (init, O_WRONLY|O_TRUNC|O_CREAT|O_NOCTTY, 0755); - if (fd == -1) - error (EXIT_FAILURE, errno, "open: %s", init); - - size_t n = (size_t) &_binary_init_size; - if (full_write (fd, &_binary_init_start, n) != n) - error (EXIT_FAILURE, errno, "write: %s", init); - - if (close (fd) == -1) - error (EXIT_FAILURE, errno, "close: %s", init); - - free (init); - - /* Build the cpio file. */ - cmd = xasprintf ("(cd %s && (echo . ; ls -1)" - " | cpio --quiet -o -H newc) > '%s'", - dir, initrd); - if (verbose >= 2) fprintf (stderr, "%s\n", cmd); - int r = system (cmd); - if (r == -1 || WEXITSTATUS (r) != 0) - error (EXIT_FAILURE, 0, "ext2_make_initrd: cpio failed"); - free (cmd); - - /* Construction of 'dir' above ensures this is safe. */ - cmd = xasprintf ("rm -rf %s", dir); - if (verbose >= 2) fprintf (stderr, "%s\n", cmd); - system (cmd); - free (cmd); -} - -static void -free_module_deps (void) -{ - /* Short-lived program, don't bother to free it. */ - modules = NULL; -} - -/* Read modules.dep into internal structure. */ -static void -read_module_deps (const char *modpath) -{ - free_module_deps (); - - char *filename = xasprintf ("%s/modules.dep", modpath); - FILE *fp = fopen (filename, "r"); - free (filename); - if (fp == NULL) - error (EXIT_FAILURE, errno, "open: %s/modules.dep", modpath); - - char *line = NULL; - size_t llen = 0; - ssize_t len; - while ((len = getline (&line, &llen, fp)) != -1) { - if (len > 0 && line[len-1] == '\n') - line[--len] = '\0'; - - char *name = strtok (line, ": "); - if (!name) continue; - - add_module (name); - char *dep; - while ((dep = strtok (NULL, " ")) != NULL) { - add_module_dep (name, dep); - } - } - - free (line); - fclose (fp); -} - -static struct module * -add_module (const char *name) -{ - struct module *m = find_module (name); - if (m) - return m; - m = xmalloc (sizeof *m); - m->name = xstrdup (name); - m->deps = NULL; - m->next = modules; - m->visited = 0; - modules = m; - return m; -} - -static struct module * -find_module (const char *name) -{ - struct module *m; - for (m = modules; m; m = m->next) { - if (strcmp (name, m->name) == 0) - break; - } - return m; -} - -/* Module 'name' requires 'dep' to be loaded first. */ -static void -add_module_dep (const char *name, const char *dep) -{ - if (verbose >= 2) fprintf (stderr, "add_module_dep %s: %s\n", name, dep); - struct module *m1 = add_module (name); - struct module *m2 = add_module (dep); - struct moddep *d; - for (d = m1->deps; d; d = d->next) { - if (d->dep == m2) - return; - } - d = xmalloc (sizeof *d); - d->next = m1->deps; - d->dep = m2; - m1->deps = d; - return; -} - -/* DFS on the dependency graph */ -static void -print_module_load_order (FILE *pipe, FILE *list, struct module *m) -{ - if (m->visited) - return; - - struct moddep *d; - for (d = m->deps; d; d = d->next) - print_module_load_order (pipe, list, d->dep); - - if (m->name[0] == 0) - return; - - char *basename = strrchr (m->name, '/'); - if (basename) - ++basename; - else - basename = m->name; - - fputs (m->name, pipe); - fputc ('\n', pipe); - fputs (basename, list); - fputc ('\n', list); - m->visited = 1; - - if (verbose >= 2) - fprintf (stderr, "print_module_load_order: %s %s\n", m->name, basename); -} diff --git a/helper/ext2internal.h b/helper/ext2internal.h deleted file mode 100644 index 320a091..0000000 --- a/helper/ext2internal.h +++ /dev/null @@ -1,45 +0,0 @@ -/* supermin-helper reimplementation in C. - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -/* This is a private interface used between the parts of the ext2 plugin. */ - -#ifndef SUPERMIN_EXT2INTERNAL_H -#define SUPERMIN_EXT2INTERNAL_H - -/* Inlining is broken in the ext2fs header file. Disable it by - * defining the following: - */ -#define NO_INLINE_FUNCS -#include <ext2fs/ext2fs.h> - -/* ext2.c */ -extern ext2_filsys fs; - -extern void ext2_mkdir (ext2_ino_t dir_ino, const char *dirname, const char *basename, mode_t mode, uid_t uid, gid_t gid, time_t ctime, time_t atime, time_t mtime); -extern void ext2_empty_inode (ext2_ino_t dir_ino, const char *dirname, const char *basename, mode_t mode, uid_t uid, gid_t gid, time_t ctime, time_t atime, time_t mtime, int major, int minor, int dir_ft, ext2_ino_t *ino_ret); -extern void ext2_write_file (ext2_ino_t ino, const char *buf, size_t size, const char *orig_filename); -extern void ext2_link (ext2_ino_t dir_ino, const char *basename, ext2_ino_t ino, int dir_ft); -extern void ext2_clean_path (ext2_ino_t dir_ino, const char *dirname, const char *basename, int isdir); - -/* ext2cpio.c */ -extern void ext2_cpio_file (const char *cpio_file); - -/* ext2initrd.c */ -extern void ext2_make_initrd (const char *modpath, const char *initrd); - -#endif /* SUPERMIN_EXT2INTERNAL_H */ diff --git a/helper/helper.h b/helper/helper.h deleted file mode 100644 index 94ad9e1..0000000 --- a/helper/helper.h +++ /dev/null @@ -1,85 +0,0 @@ -/* supermin-helper reimplementation in C. - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#ifndef SUPERMIN_HELPER_H -#define SUPERMIN_HELPER_H - -#include <sys/stat.h> -#include "fts_.h" - -struct writer { - /* Start building a new appliance. - * 'appliance' is the output appliance. - * 'initrd' is the mini-initrd to create (only used for ext2 output). - * 'modpath' is the kernel module path. - */ - void (*wr_start) (const char *hostcpu, const char *appliance, - const char *modpath, const char *initrd); - - /* Finish off the appliance. */ - void (*wr_end) (void); - - /* Append the named host file to the appliance being built. The - * wr_file_stat form is used where we have already stat'd this file, - * to avoid having to stat it a second time. The wr_fts_entry form - * is used where the caller has an FTSENT. - */ - void (*wr_file) (const char *filename); - void (*wr_file_stat) (const char *filename, const struct stat *); - void (*wr_fts_entry) (FTSENT *entry); - - /* Append the contents of cpio file to the appliance being built. */ - void (*wr_cpio_file) (const char *cpio_file); -}; - -/* main.c */ -extern struct timeval start_t; -extern int verbose; -extern int copy_kernel; - -/* appliance.c */ -extern void create_appliance (const char *hostcpu, char **inputs, int nr_inputs, const char *whitelist, const char *modpath, const char *initrd, const char *appliance, struct writer *writer); - -/* checksum.c */ -extern struct writer checksum_writer; - -/* cpio.c */ -extern struct writer cpio_writer; - -/* ext2.c */ -extern struct writer ext2_writer; - -/* kernel.c */ -extern const char *create_kernel (const char *hostcpu, const char *kernel, const char *dtb_wildcard, const char *dtb); - -/* utils.c */ -extern void print_timestamped_message (const char *fs, ...); -extern int64_t timeval_diff (const struct timeval *x, const struct timeval *y); -extern int reverse_filevercmp (const void *p1, const void *p2); -extern void add_string (char ***argv, size_t *n_used, size_t *n_alloc, const char *str); -extern size_t count_strings (char *const *argv); -extern char **read_dir (const char *name); -extern char **filter (char **strings, int (*)(const char *)); -extern char **filter_fnmatch (char **strings, const char *patt, int flags); -extern char **filter_notmatching_substring (char **strings, const char *sub); -extern void sort (char **strings, int (*compare) (const void *, const void *)); -extern int isdir (const char *path); -extern int isfile (const char *path); -extern char **load_file (const char *filename); - -#endif /* SUPERMIN_HELPER_H */ diff --git a/helper/init.c b/helper/init.c deleted file mode 100644 index f979f72..0000000 --- a/helper/init.c +++ /dev/null @@ -1,537 +0,0 @@ -/* supermin-helper reimplementation in C. - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -/* This very minimal init "script" goes in the mini-initrd used to - * boot the ext2-based appliance. Note we have no shell, so we cannot - * use system(3) to run external commands. In fact, we don't have - * very much at all, except this program, and some kernel modules. - */ - -#include <config.h> - -#include <stdio.h> -#include <stdlib.h> -#include <stdint.h> -#include <string.h> -#include <inttypes.h> -#include <unistd.h> -#include <errno.h> -#include <fcntl.h> -#include <dirent.h> -#include <time.h> -#include <sys/types.h> -#include <sys/mount.h> -#include <sys/stat.h> -#include <sys/wait.h> - -#include <asm/unistd.h> - -#ifdef HAVE_ZLIB_STATIC -#include <zlib.h> -#endif - -#ifdef HAVE_LZMA_STATIC -#include <lzma.h> -#endif - -/* Maximum time to wait for the root device to appear (seconds). - * - * On slow machines with lots of disks (Koji running the 255 disk test - * in libguestfs) this really can take several minutes. - * - * Note that the actual wait time is approximately double the number - * given here because there is a delay which doubles until it reaches - * this value. - */ -#define MAX_ROOT_WAIT 300 - -extern long init_module (void *, unsigned long, const char *); - -/* translation taken from module-init-tools/insmod.c */ -static const char *moderror(int err) -{ - switch (err) { - case ENOEXEC: - return "Invalid module format"; - case ENOENT: - return "Unknown symbol in module"; - case ESRCH: - return "Module has wrong symbol version"; - case EINVAL: - return "Invalid parameters"; - default: - return strerror(err); - } -} - -/* Leave this enabled for now. When we get more confident in the boot - * process we can turn this off or make it configurable. - */ -#define verbose 1 - -static void mount_proc (void); -static void print_uptime (void); -static void read_cmdline (void); -static void insmod (const char *filename); -static void show_directory (const char *dir); - -static char cmdline[1024]; -static char line[1024]; - -int -main () -{ - mount_proc (); - - print_uptime (); - fprintf (stderr, "supermin: ext2 mini initrd starting up: " - PACKAGE_VERSION -#ifdef HAVE_ZLIB_STATIC - " zlib" -#endif -#ifdef HAVE_LZMA_STATIC - " xz" -#endif - "\n"); - - read_cmdline (); - - /* Create some fixed directories. */ - mkdir ("/dev", 0755); - mkdir ("/root", 0755); - mkdir ("/sys", 0755); - - /* Mount /sys. */ - if (verbose) - fprintf (stderr, "supermin: mounting /sys\n"); - if (mount ("sysfs", "/sys", "sysfs", 0, "") == -1) { - perror ("mount: /sys"); - exit (EXIT_FAILURE); - } - - FILE *fp = fopen ("/modules", "r"); - if (fp == NULL) { - perror ("fopen: /modules"); - exit (EXIT_FAILURE); - } - while (fgets (line, sizeof line, fp)) { - size_t n = strlen (line); - if (n > 0 && line[n-1] == '\n') - line[--n] = '\0'; - - /* XXX Because of the way we construct the module list, the - * "modules" file can contain non-existent modules. Ignore those - * for now. Really we should add them as missing dependencies. - * See ext2initrd.c:ext2_make_initrd(). - */ - if (access (line, R_OK) == 0) - insmod (line); - else - fprintf (stderr, "skipped %s, module is missing\n", line); - } - fclose (fp); - - /* Look for the ext2 filesystem device. It's always the last one - * that was added. Modern versions of libguestfs supply the - * expected name of the root device on the command line - * ("root=/dev/..."). For virtio-scsi this is required, because we - * must wait for the device to appear after the module is loaded. - */ - char *root, *path; - size_t len; - root = strstr (cmdline, "root="); - if (root) { - root += 5; - if (strncmp (root, "/dev/", 5) == 0) - root += 5; - len = strcspn (root, " "); - root[len] = '\0'; - - asprintf (&path, "/sys/block/%s/dev", root); - - uint64_t delay_ns = 250000; - while (delay_ns <= MAX_ROOT_WAIT * UINT64_C(1000000000)) { - fp = fopen (path, "r"); - if (fp != NULL) - goto found; - - if (delay_ns > 1000000000) - fprintf (stderr, - "supermin: waiting another %" PRIu64 " ns for %s to appear\n", - delay_ns, path); - - struct timespec t; - t.tv_sec = delay_ns / 1000000000; - t.tv_nsec = delay_ns % 1000000000; - nanosleep (&t, NULL); - delay_ns *= 2; - } - } - else { - path = strdup ("/sys/block/xdx/dev"); - - char class[3] = { 'v', 's', 'h' }; - size_t i, j; - fp = NULL; - for (i = 0; i < sizeof class; ++i) { - for (j = 'z'; j >= 'a'; --j) { - path[11] = class[i]; - path[13] = j; - fp = fopen (path, "r"); - if (fp != NULL) - goto found; - } - } - } - - fprintf (stderr, - "supermin: no ext2 root device found\n" - "Please include FULL verbose output in your bug report.\n"); - exit (EXIT_FAILURE); - - found: - if (verbose) - fprintf (stderr, "supermin: picked %s as root device\n", path); - - fgets (line, sizeof line, fp); - int major = atoi (line); - char *p = line + strcspn (line, ":") + 1; - int minor = atoi (p); - - fclose (fp); - if (umount ("/sys") == -1) { - perror ("umount: /sys"); - exit (EXIT_FAILURE); - } - - if (verbose) - fprintf (stderr, "supermin: creating /dev/root as block special %d:%d\n", - major, minor); - - if (mknod ("/dev/root", S_IFBLK|0700, makedev (major, minor)) == -1) { - perror ("mknod: /dev/root"); - exit (EXIT_FAILURE); - } - - /* Mount new root and chroot to it. */ - if (verbose) - fprintf (stderr, "supermin: mounting new root on /root\n"); - if (mount ("/dev/root", "/root", "ext2", MS_NOATIME, "") == -1) { - perror ("mount: /root"); - exit (EXIT_FAILURE); - } - - /* Note that pivot_root won't work. See the note in - * Documentation/filesystems/ramfs-rootfs-initramfs.txt - * We could remove the old initramfs files, but let's not bother. - */ - if (verbose) - fprintf (stderr, "supermin: chroot\n"); - - if (chroot ("/root") == -1) { - perror ("chroot: /root"); - exit (EXIT_FAILURE); - } - - chdir ("/"); - - /* Run /init from ext2 filesystem. */ - execl ("/init", "init", NULL); - perror ("execl: /init"); - - /* /init failed to execute, but why? Before we ditch, print some - * debug. Although we have a full appliance, the fact that /init - * failed to run means we may not be able to run any commands. - */ - show_directory ("/"); - show_directory ("/bin"); - show_directory ("/lib"); - show_directory ("/lib64"); - fflush (stderr); - - exit (EXIT_FAILURE); -} - -#if HAVE_LZMA_STATIC -static int -ends_with (const char *str, const char *suffix) -{ - if (!str || !suffix) - return 0; - size_t lenstr = strlen (str); - size_t lensuffix = strlen (suffix); - if (lensuffix > lenstr) - return 0; - return strncmp (str + lenstr - lensuffix, suffix, lensuffix) == 0; -} -#endif - -static void -insmod (const char *filename) -{ - size_t size; - - if (verbose) - fprintf (stderr, "supermin: internal insmod %s\n", filename); - -#ifdef HAVE_ZLIB_STATIC - int capacity = 64*1024; - char *buf = (char *) malloc (capacity); - int tmpsize = 8 * 1024; - char tmp[tmpsize]; - int num; - - errno = 0; - size = 0; - -#ifdef HAVE_LZMA_STATIC - if (ends_with(filename, ".xz")) { - lzma_stream strm = LZMA_STREAM_INIT; - lzma_ret ret = lzma_stream_decoder(&strm, UINT64_MAX, - LZMA_CONCATENATED); - if (verbose) - fprintf (stderr, "supermin: running xz\n"); - FILE *fd = fopen (filename, "r"); - if (!fd) { - perror("popen failed"); - exit (EXIT_FAILURE); - } - char tmp_out[tmpsize]; - strm.avail_in = 0; - strm.next_out = tmp_out; - strm.avail_out = tmpsize; - - lzma_action action = LZMA_RUN; - - while (1) { - if (strm.avail_in == 0) { - strm.next_in = tmp; - strm.avail_in = fread(tmp, 1, tmpsize, fd); - - if (ferror(fd)) { - // POSIX says that fread() sets errno if - // an error occurred. ferror() doesn't - // touch errno. - perror("Error reading input file"); - exit (EXIT_FAILURE); - } - if (feof(fd)) action = LZMA_FINISH; - } - - ret = lzma_code(&strm, action); - - // Write and check write error before checking decoder error. - // This way as much data as possible gets written to output - // even if decoder detected an error. - if (strm.avail_out == 0 || ret != LZMA_OK) { - const size_t num = tmpsize - strm.avail_out; - if (num > capacity) { - buf = (char*) realloc (buf, size*2); - capacity = size; - } - memcpy (buf+size, tmp_out, num); - capacity -= num; - size += num; - strm.next_out = tmp_out; - strm.avail_out = tmpsize; - } - if (ret != LZMA_OK) { - if (ret == LZMA_STREAM_END) { - break; - } else { - perror("internal error"); - exit(EXIT_FAILURE); - } - } - } - fclose (fd); - if (verbose) - fprintf (stderr, "done with xz %d read\n", size); - } else { -#endif - gzFile gzfp = gzopen (filename, "rb"); - if (gzfp == NULL) { - fprintf (stderr, "insmod: gzopen failed: %s", filename); - exit (EXIT_FAILURE); - } - while ((num = gzread (gzfp, tmp, tmpsize)) > 0) { - if (num > capacity) { - buf = (char*) realloc (buf, size*2); - capacity = size; - } - memcpy (buf+size, tmp, num); - capacity -= num; - size += num; - } - if (num == -1) { - perror ("insmod: gzread"); - exit (EXIT_FAILURE); - } - gzclose (gzfp); -#ifdef HAVE_LZMA_STATIC -} -#endif - -#else - int fd = open (filename, O_RDONLY); - if (fd == -1) { - fprintf (stderr, "insmod: open: %s: %m\n", filename); - exit (EXIT_FAILURE); - } - struct stat st; - if (fstat (fd, &st) == -1) { - perror ("insmod: fstat"); - exit (EXIT_FAILURE); - } - size = st.st_size; - char buf[size]; - size_t offset = 0; - do { - ssize_t rc = read (fd, buf + offset, size - offset); - if (rc == -1) { - perror ("insmod: read"); - exit (EXIT_FAILURE); - } - offset += rc; - } while (offset < size); - close (fd); -#endif - - if (init_module (buf, size, "") != 0) { - fprintf (stderr, "insmod: init_module: %s: %s\n", filename, moderror (errno)); - /* However ignore the error because this can just happen because - * of a missing device. - */ - } - -#ifdef HAVE_ZLIB_STATIC - free (buf); -#endif -} - -/* Mount /proc unless it's mounted already. */ -static void -mount_proc (void) -{ - if (access ("/proc/uptime", R_OK) == -1) { - mkdir ("/proc", 0755); - - if (verbose) - fprintf (stderr, "supermin: mounting /proc\n"); - - if (mount ("proc", "/proc", "proc", 0, "") == -1) { - perror ("mount: /proc"); - /* Non-fatal. */ - } - } -} - -/* Print contents of /proc/uptime. */ -static void -print_uptime (void) -{ - FILE *fp = fopen ("/proc/uptime", "r"); - if (fp == NULL) { - perror ("/proc/uptime"); - return; - } - - fgets (line, sizeof line, fp); - fclose (fp); - - fprintf (stderr, "supermin: uptime: %s", line); -} - -/* Read /proc/cmdline into cmdline global (or at least the first 1024 - * bytes of it). - */ -static void -read_cmdline (void) -{ - FILE *fp = fopen ("/proc/cmdline", "r"); - if (fp == NULL) { - perror ("/proc/cmdline"); - return; - } - - fgets (cmdline, sizeof cmdline, fp); - fclose (fp); - - fprintf (stderr, "supermin: cmdline: %s", cmdline); -} - -/* Display a directory on stderr. This is used for debugging only. */ -static char -dirtype (int dt) -{ - switch (dt) { - case DT_BLK: return 'b'; - case DT_CHR: return 'c'; - case DT_DIR: return 'd'; - case DT_FIFO: return 'p'; - case DT_LNK: return 'l'; - case DT_REG: return '-'; - case DT_SOCK: return 's'; - case DT_UNKNOWN: return 'u'; - default: return '?'; - } -} - -static void -show_directory (const char *dirname) -{ - DIR *dir; - struct dirent *d; - struct stat statbuf; - char link[PATH_MAX+1]; - ssize_t n; - - fprintf (stderr, "supermin: debug: listing directory %s\n", dirname); - - if (chdir (dirname) == -1) { - perror (dirname); - return; - } - - dir = opendir ("."); - if (!dir) { - perror (dirname); - chdir ("/"); - return; - } - - while ((d = readdir (dir)) != NULL) { - fprintf (stderr, "%5lu %c %-16s", d->d_ino, dirtype (d->d_type), d->d_name); - if (lstat (d->d_name, &statbuf) >= 0) { - fprintf (stderr, " %06o %ld %d:%d", - statbuf.st_mode, statbuf.st_size, - statbuf.st_uid, statbuf.st_gid); - if (S_ISLNK (statbuf.st_mode)) { - n = readlink (d->d_name, link, PATH_MAX); - if (n >= 0) { - link[n] = '\0'; - fprintf (stderr, " -> %s", link); - } - } - } - fprintf (stderr, "\n"); - } - - closedir (dir); - chdir ("/"); -} diff --git a/helper/kernel.c b/helper/kernel.c deleted file mode 100644 index 5c8175d..0000000 --- a/helper/kernel.c +++ /dev/null @@ -1,454 +0,0 @@ -/* supermin-helper reimplementation in C. - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include <config.h> - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <fcntl.h> -#include <fnmatch.h> -#include <unistd.h> -#include <errno.h> -#include <sys/utsname.h> -#include <assert.h> - -#include "error.h" -#include "xvasprintf.h" -#include "full-write.h" - -#include "helper.h" - -#ifndef O_CLOEXEC -#define O_CLOEXEC 0 -#endif - -/* Directory containing candidate kernels. We could make this - * configurable at some point. - */ -#define KERNELDIR "/boot" -#define MODULESDIR "/lib/modules" - -static char* get_kernel_version (char* filename); -static const char *create_kernel_from_env (const char *hostcpu, const char *kernel, const char *kernel_env, const char *modpath_env, const char *dtb_wildcard, const char *dtb); -static void copy_or_symlink_file (const char *what, const char *from, const char *to); - -static char * -get_modpath (const char *kernel_name) -{ - /* Ignore "vmlinuz-" at the beginning of the kernel name. */ - const char *version = &kernel_name[8]; - - /* /lib/modules/<version> */ - char *modpath = xasprintf (MODULESDIR "/%s", version); - if (!modpath) { - perror ("xasprintf"); - exit (EXIT_FAILURE); - } - - if (! isdir (modpath)) { - char* path; - char* version; - path = xasprintf (KERNELDIR "/%s", kernel_name); - if (!path) { - perror ("xasprintf"); - exit (EXIT_FAILURE); - } - version = get_kernel_version (path); - free (path); - if (version != NULL) { - free (modpath); - modpath = xasprintf (MODULESDIR "/%s", version); - free (version); - if (!path) { - perror ("xasprintf"); - exit (EXIT_FAILURE); - } - } - } - - return modpath; -} - -/* kernel_name is "vmlinuz-*". Check if there is a corresponding - * module path in /lib/modules. - */ -static int -has_modpath (const char *kernel_name) -{ - char *modpath = get_modpath (kernel_name); - - if (verbose) - fprintf (stderr, "checking modpath %s is a directory\n", modpath); - - int r = isdir (modpath); - - if (r) { - free (modpath); - return 1; - } - else { - if (verbose) - fprintf (stderr, "ignoring %s (no modpath %s)\n", kernel_name, modpath); - free (modpath); - return 0; - } -} - -/* Try to find a device tree file in the kernel's dtb directory which - * matches the 'dtb_wildcard' pattern. Copy the file (or symlink it) to - * 'dtb'. - * - * The dtb directory is formed by replacing "vmlinuz-" at the - * beginning of the 'kernel' filename with "dtb-". - * - * We can override the whole lot by setting $SUPERMIN_DTB. - */ -static void -get_dtb (const char *kernel, const char *dtb_wildcard, const char *dtb) -{ - if (!dtb_wildcard) - return; - - char *dtb_env = getenv ("SUPERMIN_DTB"); - if (dtb_env) { - copy_or_symlink_file ("dtb", dtb_env, dtb); - return; - } - - assert (dtb); /* command line arg parsing should ensure this */ - assert (kernel != NULL); - - /* Replace /vmlinuz- with /dtb- */ - if (strncmp (kernel, "vmlinuz-", 8) != 0) { - no_dtb_dir: - fprintf (stderr, - "supermin-helper: failed to find a dtb (device tree) directory.\n" - "I expected to take '%s'\n" - "and replace vmlinuz- with dtb- to form a directory.\n" - "You can set SUPERMIN_DTB to point to the dtb *file* that should\n" - "be used.\n", - kernel); - exit (EXIT_FAILURE); - } - - char *dtb_dir; - if (asprintf (&dtb_dir, KERNELDIR "/dtb-%s", &kernel[8]) == -1) { - perror ("asprintf"); - exit (EXIT_FAILURE); - } - - /* It should be a directory. */ - if (!isdir (dtb_dir)) - goto no_dtb_dir; - - if (verbose) - fprintf (stderr, "looking for dtb matching %s in %s\n", - dtb_wildcard, dtb_dir); - - /* Look for the wildcard match. - * XXX Should probably do some sorting here and/or worry if there - * are multiple matches. - */ - char **all_files = read_dir (dtb_dir); - char **candidates = filter_fnmatch (all_files, dtb_wildcard, FNM_NOESCAPE); - - if (candidates[0] == NULL) { - fprintf (stderr, - "supermin-helper: failed to find a matching device tree.\n" - "I looked for a file matching '%s' in directory '%s'\n" - "You can set SUPERMIN_DTB to point to the dtb file that should\n" - "be used.\n", - dtb_wildcard, dtb_dir); - exit (EXIT_FAILURE); - } - - if (verbose) - fprintf (stderr, "picked dtb %s\n", candidates[0]); - - char *tmp = xasprintf ("%s/%s", dtb_dir, candidates[0]); - copy_or_symlink_file ("dtb", tmp, dtb); - free (tmp); -} - -/* Create the kernel and device tree files. This chooses an - * appropriate kernel, and optionally a device tree, and makes a - * symlink to them (or copies them if --copy-kernel was passed). - * - * Look for the most recent kernel named vmlinuz-*.<arch>* which has a - * corresponding directory in /lib/modules/. If the architecture is - * x86, look for any x86 kernel. - * - * RHEL 5 didn't append the arch to the kernel name, so look for - * kernels without arch second. - * - * If no suitable kernel can be found, exit with an error. - * - * This function returns the module path (ie. /lib/modules/<version>). - */ -const char * -create_kernel (const char *hostcpu, const char *kernel, - const char *dtb_wildcard, const char *dtb) -{ - int is_x86; /* x86 but not x86-64 */ - int is_arm; /* arm */ - - is_x86 = - hostcpu[0] == 'i' && hostcpu[2] == '8' && hostcpu[3] == '6' && - hostcpu[4] == '\0'; - is_arm = hostcpu[0] == 'a' && hostcpu[1] == 'r' && hostcpu[2] == 'm'; - - /* Override kernel selection using environment variables? */ - char *kernel_env = getenv ("SUPERMIN_KERNEL"); - if (kernel_env) { - char *modpath_env = getenv ("SUPERMIN_MODULES"); - return create_kernel_from_env (hostcpu, kernel, kernel_env, modpath_env, - dtb_wildcard, dtb); - } - - /* In original: ls -1dvr /boot/vmlinuz-*.$arch* 2>/dev/null | grep -v xen */ - const char *patt; - if (is_x86) - patt = "vmlinuz-*.i?86*"; - else - patt = xasprintf ("vmlinuz-*.%s*", hostcpu); - - char **all_files = read_dir (KERNELDIR); - char **candidates; - candidates = filter_fnmatch (all_files, patt, FNM_NOESCAPE); - candidates = filter_notmatching_substring (candidates, "xen"); - if (is_arm) { - candidates = filter_notmatching_substring (candidates, "lpae"); - candidates = filter_notmatching_substring (candidates, "tegra"); - } - candidates = filter (candidates, has_modpath); - - if (candidates[0] == NULL) { - /* In original: ls -1dvr /boot/vmlinuz-* 2>/dev/null | grep -v xen */ - patt = "vmlinuz-*"; - candidates = filter_fnmatch (all_files, patt, FNM_NOESCAPE); - candidates = filter_notmatching_substring (candidates, "xen"); - if (is_arm) { - candidates = filter_notmatching_substring (candidates, "lpae"); - candidates = filter_notmatching_substring (candidates, "tegra"); - } - candidates = filter (candidates, has_modpath); - - if (candidates[0] == NULL) - goto no_kernels; - } - - sort (candidates, reverse_filevercmp); - - if (verbose) - fprintf (stderr, "picked kernel %s\n", candidates[0]); - - if (kernel) { - /* Choose the first candidate. */ - char *tmp = xasprintf (KERNELDIR "/%s", candidates[0]); - copy_or_symlink_file ("kernel", tmp, kernel); - free (tmp); - } - - /* Device tree. */ - get_dtb (candidates[0], dtb_wildcard, dtb); - - return get_modpath (candidates[0]); - - /* Print more diagnostics here than the old script did. */ - no_kernels: - fprintf (stderr, - "supermin-helper: failed to find a suitable kernel.\n" - "I looked for kernels in " KERNELDIR " and modules in " MODULESDIR - ".\n" - "If this is a Xen guest, and you only have Xen domU kernels\n" - "installed, try installing a fullvirt kernel (only for\n" - "supermin use, you shouldn't boot the Xen guest with it).\n"); - exit (EXIT_FAILURE); -} - -/* Select the kernel from environment variables set by the user. - * modpath_env may be NULL, in which case we attempt to work it out - * from kernel_env. - */ -static const char * -create_kernel_from_env (const char *hostcpu, const char *kernel, - const char *kernel_env, const char *modpath_env, - const char *dtb_wildcard, const char *dtb) -{ - if (verbose) { - fprintf (stderr, - "supermin-helper: using environment variable(s) SUPERMIN_* to\n" - "select kernel %s", kernel_env); - if (modpath_env) - fprintf (stderr, " and module path %s", modpath_env); - fprintf (stderr, "\n"); - } - - if (!isfile (kernel_env)) { - fprintf (stderr, - "supermin-helper: %s: not a regular file\n" - "(what is $SUPERMIN_KERNEL set to?)\n", kernel_env); - exit (EXIT_FAILURE); - } - - if (!modpath_env) { - /* Try to guess modpath from kernel path. */ - const char *p = strrchr (kernel_env, '/'); - if (p) p++; else p = kernel_env; - - /* NB: We need the extra test to ensure calling get_modpath is safe. */ - if (strncmp (p, "vmlinuz-", 8) != 0) { - fprintf (stderr, - "supermin-helper: cannot guess module path.\n" - "Set $SUPERMIN_MODULES to the modules directory corresponding to\n" - "kernel %s, or unset $SUPERMIN_KERNEL to autoselect a kernel.\n", - kernel_env); - exit (EXIT_FAILURE); - } - - modpath_env = get_modpath (p); - } - - if (!isdir (modpath_env)) { - fprintf (stderr, - "supermin-helper: %s: not a directory\n" - "(what is $SUPERMIN_MODULES set to?)\n", modpath_env); - exit (EXIT_FAILURE); - } - - /* Create the symlink. */ - if (kernel) - copy_or_symlink_file ("kernel", kernel_env, kernel); - - /* Device tree. */ - get_dtb (kernel, dtb_wildcard, dtb); - - return modpath_env; -} - -static void -copy_or_symlink_file (const char *what, const char *from, const char *to) -{ - int fd1, fd2; - char buf[BUFSIZ]; - ssize_t r; - - if (verbose >= 2) - fprintf (stderr, "%s %s %s -> %s\n", - !copy_kernel ? "symlink" : "copy", what, from, to); - - if (!copy_kernel) { - if (symlink (from, to) == -1) - error (EXIT_FAILURE, errno, "creating %s symlink %s %s", what, from, to); - } - else { - fd1 = open (from, O_RDONLY | O_CLOEXEC); - if (fd1 == -1) - error (EXIT_FAILURE, errno, "open: %s", from); - - fd2 = open (to, O_WRONLY|O_CREAT|O_NOCTTY|O_CLOEXEC, 0644); - if (fd2 == -1) - error (EXIT_FAILURE, errno, "open: %s", to); - - while ((r = read (fd1, buf, sizeof buf)) > 0) { - if (full_write (fd2, buf, r) != r) - error (EXIT_FAILURE, errno, "write: %s", to); - } - - if (r == -1) - error (EXIT_FAILURE, errno, "read: %s", from); - - if (close (fd1) == -1) - error (EXIT_FAILURE, errno, "close: %s", from); - - if (close (fd2) == -1) - error (EXIT_FAILURE, errno, "close: %s", to); - } -} - -/* Read an unsigned little endian short at a specified offset in a file. - * Returns a non-negative int on success or -1 on failure. - */ -static int -read_leshort (FILE* fp, int offset) -{ - char buf[2]; - if (fseek (fp, offset, SEEK_SET) != 0 || - fread (buf, sizeof(char), 2, fp) != 2) - { - return -1; - } - return ((buf[1] & 0xFF) << 8) | (buf[0] & 0xFF); -} - -/* Extract the kernel version from a Linux kernel file. - * Returns a malloc'd string containing the version or NULL if the - * file can't be read, is not a Linux kernel, or the version can't - * be found. - * - * See ftp://ftp.astron.com/pub/file/file-<ver>.tar.gz - * (file-<ver>/magic/Magdir/linux) for the rules used to find the - * version number: - * 514 string HdrS Linux kernel - * >518 leshort >0x1ff - * >>(526.s+0x200) string >\0 version %s, - * - * Bugs: probably limited to x86 kernels. - */ -static char* -get_kernel_version (char* filename) -{ - FILE* fp; - int size = 132; - char buf[size]; - int offset; - - fp = fopen (filename, "rb"); - - if (fseek (fp, 514, SEEK_SET) != 0 || - fgets (buf, size, fp) == NULL || - strncmp (buf, "HdrS", 4) != 0 || - read_leshort (fp, 518) < 0x1FF) - { - /* not a Linux kernel */ - fclose (fp); - return NULL; - } - - offset = read_leshort (fp, 526); - if (offset == -1) - { - /* can't read version offset */ - fclose (fp); - return NULL; - } - - if (fseek (fp, offset + 0x200, SEEK_SET) != 0 || - fgets (buf, size, fp) == NULL) - { - /* can't read version string */ - fclose (fp); - return NULL; - } - - fclose (fp); - - buf[strcspn (buf, " \t\n")] = '\0'; - return strdup (buf); -} diff --git a/helper/main.c b/helper/main.c deleted file mode 100644 index 8107dee..0000000 --- a/helper/main.c +++ /dev/null @@ -1,475 +0,0 @@ -/* supermin-helper reimplementation in C. - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include <config.h> - -#include <stdio.h> -#include <stdlib.h> -#include <stdbool.h> -#include <string.h> -#include <errno.h> -#include <unistd.h> -#include <getopt.h> -#include <limits.h> -#include <sys/types.h> -#include <sys/time.h> -#include <assert.h> -#include <grp.h> -#include <pwd.h> - -#include "error.h" -#include "xstrtol.h" - -#include "helper.h" - -struct timeval start_t; -int verbose = 0; -int copy_kernel = 0; - -enum { HELP_OPTION = CHAR_MAX + 1 }; - -static const char *options = "f:g:k:o:u:vV"; -static const struct option long_options[] = { - { "help", 0, 0, HELP_OPTION }, - { "copy-kernel", 0, 0, 0 }, - { "dtb", required_argument, 0, 0 }, - { "format", required_argument, 0, 'f' }, - { "group", required_argument, 0, 'g' }, - { "host-cpu", required_argument, 0, 0 }, - { "kmods", required_argument, 0, 'k' }, - { "output-appliance", required_argument, 0, 0 }, - { "output-dtb", required_argument, 0, 0 }, - { "output-initrd", required_argument, 0, 0 }, - { "output-kernel", required_argument, 0, 0 }, - { "user", required_argument, 0, 'u' }, - { "verbose", 0, 0, 'v' }, - { "version", 0, 0, 'V' }, - { 0, 0, 0, 0 } -}; - -static void -usage (FILE *f, const char *progname) -{ - fprintf (f, - "%s: build the supermin appliance on the fly\n" - "\n" - "Usage:\n" - " %s [-f cpio|ext2] -o outputdir input [input...]\n" - "or:\n" - " %s [-f cpio|ext2] --output-kernel kernel \\\n" - " [--output-dtb dtb] --output-initrd initrd \\\n" - " [--output-appliance appliance] input [input...]\n" - "or:\n" - " %s -f checksum input [input ...]\n" - "or:\n" - " %s --help\n" - " %s --version\n" - "\n" - "This program is used to build the full appliance from the supermin appliance.\n" - "\n" - "Options:\n" - " --help\n" - " Display this help text and exit.\n" - " --copy-kernel\n" - " Copy the kernel & device tree instead of symlinking to it.\n" - " --dtb wildcard\n" - " Search for a device tree matching wildcard.\n" - " -f cpio|ext2|checksum | --format cpio|ext2|checksum\n" - " Specify output format (default: cpio).\n" - " --host-cpu cpu\n" - " Host CPU type (default: " host_cpu ").\n" - " -k file | --kmods file\n" - " Specify kernel module whitelist.\n" - " -o outputdir\n" - " Write output to outputdir/kernel etc.\n" - " --output-appliance path\n" - " Write appliance to path (overrides -o).\n" - " --output-dtb path\n" - " Write device tree to path (overrides -o).\n" - " --output-initrd path\n" - " Write initrd to path (overrides -o).\n" - " --output-kernel path\n" - " Write kernel to path (overrides -o).\n" - " -u user | --user user\n" - " -g group | --group group\n" - " The user name or uid, and group name or gid the appliance will\n" - " run as. Use of these options requires root privileges.\n" - " --verbose | -v\n" - " Enable verbose messages (give multiple times for more verbosity).\n" - " --version | -V\n" - " Display version number and exit.\n", - progname, progname, progname, progname, progname, progname); -} - -static uid_t -parseuser (const char *id, const char *progname) -{ - struct passwd *pwd; - int saved_errno; - - errno = 0; - pwd = getpwnam (id); - - if (NULL == pwd) { - saved_errno = errno; - - long val; - int err = xstrtol (id, NULL, 10, &val, ""); - if (err == LONGINT_OK) - return (uid_t) val; - - fprintf (stderr, "%s: -u option: %s is not a valid user name or uid", - progname, id); - if (saved_errno != 0) - fprintf (stderr, " (getpwnam error: %s)", strerror (saved_errno)); - fprintf (stderr, "\n"); - exit (EXIT_FAILURE); - } - - return pwd->pw_uid; -} - -static gid_t -parsegroup (const char *id, const char *progname) -{ - struct group *grp; - int saved_errno; - - errno = 0; - grp = getgrnam (id); - - if (NULL == grp) { - saved_errno = errno; - - long val; - int err = xstrtol (id, NULL, 10, &val, ""); - if (err == LONGINT_OK) - return (gid_t) val; - - fprintf (stderr, "%s: -g option: %s is not a valid group name or gid", - progname, id); - if (saved_errno != 0) - fprintf (stderr, " (getgrnam error: %s)", strerror (saved_errno)); - fprintf (stderr, "\n"); - exit (EXIT_FAILURE); - } - - return grp->gr_gid; -} - -int -main (int argc, char *argv[]) -{ - /* First thing: start the clock. */ - gettimeofday (&start_t, NULL); - - const char *format = "cpio"; - const char *whitelist = NULL; - - /* For the reason this was originally added, see - * https://bugzilla.redhat.com/show_bug.cgi?id=558593 - */ - const char *hostcpu = host_cpu; - - /* Output files. */ - char *kernel = NULL, *dtb = NULL, *initrd = NULL, *appliance = NULL; - const char *output_dir = NULL; - - /* Device tree wildcard (--dtb argument). */ - const char *dtb_wildcard = NULL; - - uid_t euid = geteuid (); - gid_t egid = getegid (); - - bool old_style = true; - - /* Command line arguments. */ - for (;;) { - int option_index; - int c = getopt_long (argc, argv, options, long_options, &option_index); - if (c == -1) break; - - switch (c) { - case HELP_OPTION: - usage (stdout, argv[0]); - exit (EXIT_SUCCESS); - - case 0: /* options which are long only */ - if (strcmp (long_options[option_index].name, "copy-kernel") == 0) { - copy_kernel = 1; - } - else if (strcmp (long_options[option_index].name, "dtb") == 0) { - dtb_wildcard = optarg; - old_style = false; /* --dtb + old-style wouldn't work anyway */ - } - else if (strcmp (long_options[option_index].name, "host-cpu") == 0) { - hostcpu = optarg; - old_style = false; - } - else if (strcmp (long_options[option_index].name, "output-kernel") == 0) { - kernel = optarg; - old_style = false; - } - else if (strcmp (long_options[option_index].name, "output-dtb") == 0) { - dtb = optarg; - old_style = false; - } - else if (strcmp (long_options[option_index].name, "output-initrd") == 0) { - initrd = optarg; - old_style = false; - } - else if (strcmp (long_options[option_index].name, "output-appliance") == 0) { - appliance = optarg; - old_style = false; - } - else { - fprintf (stderr, "%s: unknown long option: %s (%d)\n", - argv[0], long_options[option_index].name, option_index); - exit (EXIT_FAILURE); - } - break; - - case 'f': - format = optarg; - break; - - case 'u': - euid = parseuser (optarg, argv[0]); - break; - - case 'g': - egid = parsegroup (optarg, argv[0]); - break; - - case 'k': - whitelist = optarg; - break; - - case 'o': - output_dir = optarg; - old_style = false; - break; - - case 'v': - verbose++; - break; - - case 'V': - printf (PACKAGE_NAME " " PACKAGE_VERSION "\n"); - exit (EXIT_SUCCESS); - - default: - usage (stderr, argv[0]); - exit (EXIT_FAILURE); - } - } - - /* Select the correct writer module. */ - struct writer *writer; - bool needs_kernel; - bool needs_initrd; - bool needs_appliance; - - bool needs_dtb = dtb_wildcard != NULL; - - if (strcmp (format, "cpio") == 0) { - writer = &cpio_writer; - needs_kernel = true; - needs_initrd = true; - needs_appliance = false; - } - else if (strcmp (format, "ext2") == 0) { - writer = &ext2_writer; - needs_kernel = true; - needs_initrd = true; - needs_appliance = true; - } - else if (strcmp (format, "checksum") == 0) { - writer = &checksum_writer; - needs_kernel = false; - needs_initrd = false; - needs_appliance = false; - } - else { - fprintf (stderr, - "%s: incorrect output format (-f): must be cpio|ext2|checksum\n", - argv[0]); - exit (EXIT_FAILURE); - } - - char **inputs = &argv[optind]; - int nr_inputs; - - /* Old-style arguments? */ - if (old_style) { - int nr_outputs; - - if (strcmp (format, "cpio") == 0) - nr_outputs = 2; /* kernel and appliance (== initrd) */ - else if (strcmp (format, "ext2") == 0) - nr_outputs = 3; /* kernel, initrd, appliance */ - else if (strcmp (format, "checksum") == 0) - nr_outputs = 0; /* (none) */ - else - abort (); - - /* [optind .. optind+nr_inputs-1] hostcpu [argc-nr_outputs-1 .. argc-1] - * <---- nr_inputs ----> 1 <---- nr_outputs ----> - */ - nr_inputs = argc - nr_outputs - 1 - optind; - char **outputs = &argv[optind+nr_inputs+1]; - /*assert (outputs [nr_outputs] == NULL); - assert (inputs [nr_inputs + 1 + nr_outputs] == NULL);*/ - - if (nr_outputs > 0) - kernel = outputs[0]; - if (nr_outputs > 1) - initrd = outputs[1]; - if (nr_outputs > 2) - appliance = outputs[2]; - } - /* New-style? Check all outputs were defined. */ - else { - if (needs_kernel && !kernel) { - if (!output_dir) { - no_output_dir: - fprintf (stderr, "%s: use -o to specify output directory or --output-[kernel|dtb|initrd|appliance]\n", argv[0]); - exit (EXIT_FAILURE); - } - if (asprintf (&kernel, "%s/kernel", output_dir) == -1) { - perror ("asprintf"); - exit (EXIT_FAILURE); - } - } - - if (needs_dtb && !dtb) { - if (!output_dir) - goto no_output_dir; - if (asprintf (&dtb, "%s/dtb", output_dir) == -1) { - perror ("asprintf"); - exit (EXIT_FAILURE); - } - } - - if (needs_initrd && !initrd) { - if (!output_dir) - goto no_output_dir; - if (asprintf (&initrd, "%s/initrd", output_dir) == -1) { - perror ("asprintf"); - exit (EXIT_FAILURE); - } - } - - if (needs_appliance && !appliance) { - if (!output_dir) - goto no_output_dir; - if (asprintf (&appliance, "%s/appliance", output_dir) == -1) { - perror ("asprintf"); - exit (EXIT_FAILURE); - } - } - - nr_inputs = argc - optind; - } - - if (nr_inputs < 1) { - fprintf (stderr, "%s: not enough files specified on the command line\n", - argv[0]); - exit (EXIT_FAILURE); - } - - if (verbose) { - print_timestamped_message ("whitelist = %s", - whitelist ? : "(not specified)"); - print_timestamped_message ("host_cpu = %s", hostcpu); - print_timestamped_message ("dtb_wildcard = %s", - dtb_wildcard ? : "(not specified)"); - print_timestamped_message ("inputs:"); - int i; - for (i = 0; i < nr_inputs; ++i) - print_timestamped_message ("inputs[%d] = %s", i, inputs[i]); - print_timestamped_message ("outputs:"); - print_timestamped_message ("kernel = %s", kernel ? : "(none)"); - print_timestamped_message ("dtb = %s", dtb ? : "(none)"); - print_timestamped_message ("initrd = %s", initrd ? : "(none)"); - print_timestamped_message ("appliance = %s", appliance ? : "(none)"); - } - - /* We need to set the real, not effective, uid here to work round a - * misfeature in bash. bash will automatically reset euid to uid when - * invoked. As shell is used in places by supermin-helper, this - * results in code running with varying privilege. */ - uid_t uid = getuid (); - gid_t gid = getgid (); - - if (uid != euid || gid != egid) { - if (uid != 0) { - fprintf (stderr, "The -u and -g options require root privileges.\n"); - usage (stderr, argv[0]); - exit (EXIT_FAILURE); - } - - /* Need to become root first because setgid and setuid require it */ - if (seteuid (0) == -1) { - perror ("seteuid"); - exit (EXIT_FAILURE); - } - - /* Set gid and uid to command-line parameters */ - if (setgid (egid) == -1) { - perror ("setgid"); - exit (EXIT_FAILURE); - } - - /* Kill supplemental groups from parent process (RHBZ#902476). */ - if (setgroups (1, &egid) == -1) { - perror ("setgroups"); - exit (EXIT_FAILURE); - } - - if (setuid (euid) == -1) { - perror ("setuid"); - exit (EXIT_FAILURE); - } - } - - /* Remove the output files if they exist. */ - if (kernel) - unlink (kernel); - if (dtb) - unlink (dtb); - if (initrd) - unlink (initrd); - if (appliance) - unlink (appliance); - - /* Create kernel output file. */ - const char *modpath = create_kernel (hostcpu, kernel, dtb_wildcard, dtb); - - if (verbose) - print_timestamped_message ("finished creating kernel"); - - /* Create the appliance. */ - create_appliance (hostcpu, inputs, nr_inputs, whitelist, modpath, - initrd, appliance, writer); - - if (verbose) - print_timestamped_message ("finished creating appliance"); - - exit (EXIT_SUCCESS); -} diff --git a/helper/supermin-helper.pod b/helper/supermin-helper.pod deleted file mode 100644 index f670d4b..0000000 --- a/helper/supermin-helper.pod +++ /dev/null @@ -1,351 +0,0 @@ -=head1 NAME - -supermin-helper - Reconstruct initramfs from supermin appliance. - -=head1 SYNOPSIS - -New style (since supermin 4.1.5): - - supermin-helper [-f cpio|ext2] -o outputdir input [input...] - -or: - - supermin-helper [-f cpio|ext2] --output-kernel kernel \ - [--output-dtb dtb] --output-initrd initrd \ - [--output-appliance appliance] input [input...] - -or: - - supermin-helper -f checksum input [input ...] - -Old style (still supported in this version but deprecated): - - supermin-helper [-f cpio] supermin.img hostfiles.txt host_cpu kernel initrd - supermin-helper [-f cpio] input [...] host_cpu kernel initrd - - supermin-helper -f ext2 input [...] host_cpu kernel initrd appliance - - supermin-helper -f checksum input [...] host_cpu - -=head1 DESCRIPTION - -I<supermin-helper> reconstructs a bootable kernel and initramfs from a -supermin appliance. First you should be familiar with L<supermin(1)>. - -=head1 PARAMETERS - -Specify the I<input> file(s), and I<-o> or I<--output-*> flags -indicating where you want the appliance to be written. - -Use the I<-f> option to select what type of appliance you want. - -C<supermin.img> and C<hostfiles.txt> are the input files which -describe the supermin appliance. (You can also use a directory name -here which is searched for files). - -To write the appliance to a directory, use I<-o outputdir>. The -directory should already exist. Files called C<outputdir/kernel>, -C<outputdir/dtb>, C<outputdir/initrd> and/or C<outputdir/appliance> -will be written. (Not all files are written, it depends on what kind -of appliance you asked for and what architecture you are running on) - -To write files with specific names instead, use the -I<--output-kernel>, I<--output-dtb>, I<--output-initrd> and/or -I<--output-appliance> options. - -=head1 OPTIONS - -=over 4 - -=item B<--help> - -Display brief command line usage, and exit. - -=item B<--copy-kernel> - -Copy the kernel (and device tree, if created) instead of symlinking to -the kernel in C</boot>. - -This is fractionally slower, but is necessary if you want to change -the permissions or SELinux label on the kernel or device tree. - -=item B<--dtb wildcard> - -If specified, search for a device tree which is compatible with the -selected kernel and the name of which matches the given wildcard. You -can use a wildcard such as C<vexpress-*a9*.dtb> which would match -C<vexpress-v2p-ca9.dtb>. - -Notes: - -=over 4 - -=item * - -You may need to quote the wildcard to prevent it from being expanded -by your shell. - -=item * - -If no I<--dtb> option is given, no device tree will be looked for. - -=item * - -You only need a device tree on architectures such as ARM and PowerPC -which use them. On other architectures, don't use this option. - -=item * - -If you use this option and no compatible device tree can be found, -supermin-helper will exit with an error. - -=back - -=item B<-f fmt> - -=item B<--format fmt> - -Select the output format for the appliance. Possible formats are: - -=over 4 - -=item cpio - -A Linux initramfs. This is the default. - -In this case you have to supply output names for the C<kernel> and -C<initrd>. The C<initrd> is the appliance. - -Note that L<cpio(1)> might not be able to extract this file fully. -The format used by the Linux kernel is not quite a true cpio file. - -=item ext2 - -An ext2 filesystem. - -In this case you have to supply output names for the C<kernel>, a -small C<initrd> which is used just to locate the appliance, and the -C<appliance> (the ext2 filesystem). - -=item checksum - -Output a checksum. - -This prints a checksum which only changes when one of the input files -changes. - -You can use this in order to cache the output of a previous run of -this program: computing the checksum is much quicker than building an -appliance, and you only need to invalidate the cache (and consequently -rebuild the appliance) when the checksum changes. Note that the -host_cpu and the UID of the current user are included in the checksum. - -=back - -=item B<--host-cpu cpu> - -Specify the host CPU (eg. C<i686>, C<x86_64>). This is used as a -substring match when searching for compatible kernels. If not -specified, it defaults to the host CPU that supermin-helper was -compiled on. - -=item B<-k file> - -=item B<--kmods file> - -If this option is specified, then C<file> should be a list of -wildcards matching kernel module names, eg: - - virtio*.ko - scsi*.ko - piix.ko - -In this case, only kernel modules matching those wildcards will be -included in the output appliance. Note: You must resolve any -dependencies yourself as this does not pull in dependent modules -automatically. - -If this option is not specified, then every kernel module from the -host will be included. This is safer, but can produce rather large -appliances which need a lot more memory to boot. - -=item B<-o outputdir> - -Write the appliance to the named directory. Two or more of the -following files will be created (the exact files created depends on -the type of appliance you asked for and the architecture): - -=over 4 - -=item C<outputdir/kernel> - -(ie. A file literally called C<kernel> in the directory I<outputdir> -that you specified). This is usually a symlink to the kernel, unless -you gave the I<--copy-kernel> option. - -=item C<outputdir/dtb> - -The device tree. See also the I<--dtb> option. - -This is only created on architectures that use device trees, eg. ARM. - -This is usually a symlink to the device tree binary file, unless you -gave the I<--copy-kernel> option. - -=item C<outputdir/initrd> - -The initrd. For I<-f cpio> this also contains the full appliance. -For I<-f ext2> this is just a small initrd which is sufficient to find -and mount the appliance disk. - -=item C<outputdir/appliance> - -The appliance disk (only for I<-f ext2>). - -=back - -=item B<--output-kernel kernel> - -Instead of using the literal hard-coded name C<kernel>, write the -kernel to the named path. -This overrides the I<-o outputdir> option (if present). - -=item B<--output-dtb dtb> - -Instead of using the literal hard-coded name C<dtb>, write the -device tree to the named path. -This overrides the I<-o outputdir> option (if present). - -=item B<--output-initrd initrd> - -Instead of using the literal hard-coded name C<initrd>, write the -initrd to the named path. -This overrides the I<-o outputdir> option (if present). - -=item B<--output-initrd appliance> - -Instead of using the literal hard-coded name C<appliance>, write the -initrd to the named path. -This overrides the I<-o outputdir> option (if present). - -=item B<-u user> - -=item B<--user user> - -=item B<-g group> - -=item B<--group group> - -Run supermin-helper as an alternate user and/or group. C<user> and -C<group> can be specified as either a name, which will be resolved -using the system name service, or a uid/gid. Use of these options -requires root privileges. - -Use of these options is required if running supermin-helper as root -with the effective uid/gid set to non-root. Bash will reset the -effective uid/gid to the real uid/gid when invoked. As -supermin-helper uses bash in parts, this will result in the creation -of an appliance with a mixture of ownerships. - -=item B<-v> - -=item B<--verbose> - -Enable verbose messages (give multiple times for more verbosity). - -=item B<-V> - -=item B<--version> - -Display version number and exit. - -=back - -=head1 COMPRESSED INPUT FILES - -supermin-helper E<ge> 4.1.4 supports gzip-compressed input cpio image -files. C<hostfiles> cannot be compressed. - -Compressing input files saves space, but can make supermin-helper run -fractionally slower. - -=head1 SPEED - -In libguestfs, on a mid-range Intel-based PC, we reconstruct the -initramfs using this script in around 1/5th of a second (assuming a -"hot cache" - it's rather slower when run the first time on a cold -cache). - -Some tips to improve performance: - -=over 4 - -=item * - -Use a kernel module whitelist (the C<--kmods> option), and only -list the kernel modules you really need. - -=item * - -Minimize the appliance, removing as much extraneous junk as possible. - -=back - -=head1 ENVIRONMENT VARIABLES - -=over 4 - -=item SUPERMIN_KERNEL - -If this environment variable is set, then automatic selection of the -kernel is bypassed and this kernel is used. - -The environment variable should point to a kernel file, -eg. C</boot/vmlinuz-3.0.x86_64> - -The corresponding module path is guessed from the kernel name, but you -can override that by setting C<SUPERMIN_MODULES>. - -=item SUPERMIN_MODULES - -If C<SUPERMIN_KERNEL> and C<SUPERMIN_MODULES> are both set, then -automatic selection of the kernel is bypassed and the kernel and -module path are set to these values. - -The environment variable should point to a module directory, -eg. C</lib/modules/3.0.x86_64/> - -This has no effect if C<SUPERMIN_KERNEL> is not set. - -=item SUPERMIN_DTB - -Force the given device tree file to be used. - -=back - -=head1 SEE ALSO - -L<supermin(1)>. - -=head1 AUTHORS - -Richard W.M. Jones <rjones @ redhat . com> - -=head1 COPYRIGHT - -(C) Copyright 2009-2013 Red Hat Inc., -L<http://people.redhat.com/~rjones/supermin>. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. diff --git a/helper/utils.c b/helper/utils.c deleted file mode 100644 index f2450a6..0000000 --- a/helper/utils.c +++ /dev/null @@ -1,314 +0,0 @@ -/* supermin-helper reimplementation in C. - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include <config.h> - -#include <stdio.h> -#include <stdlib.h> -#include <stdarg.h> -#include <stdint.h> -#include <string.h> -#include <dirent.h> -#include <errno.h> -#include <fnmatch.h> -#include <inttypes.h> -#include <sys/time.h> -#include <sys/stat.h> -#include <assert.h> - -#include "error.h" -#include "filevercmp.h" -#include "hash.h" -#include "hash-pjw.h" -#include "xalloc.h" - -#include "helper.h" - -/* Compute Y - X and return the result in milliseconds. - * Approximately the same as this code: - * http://www.mpp.mpg.de/~huber/util/timevaldiff.c - */ -int64_t -timeval_diff (const struct timeval *x, const struct timeval *y) -{ - int64_t msec; - - msec = (y->tv_sec - x->tv_sec) * 1000; - msec += (y->tv_usec - x->tv_usec) / 1000; - return msec; -} - -void -print_timestamped_message (const char *fs, ...) -{ - struct timeval tv; - gettimeofday (&tv, NULL); - - va_list args; - char *msg; - int err; - - va_start (args, fs); - err = vasprintf (&msg, fs, args); - va_end (args); - - if (err < 0) return; - - fprintf (stderr, "supermin helper [%05" PRIi64 "ms] %s\n", - timeval_diff (&start_t, &tv), msg); - - free (msg); -} - -int -reverse_filevercmp (const void *p1, const void *p2) -{ - const char *s1 = * (char * const *) p1; - const char *s2 = * (char * const *) p2; - - /* Note, arguments are reversed to achieve a reverse sort. */ - return filevercmp (s2, s1); -} - -void -add_string (char ***argv, size_t *n_used, size_t *n_alloc, const char *str) -{ - char *new_str; - - if (*n_used >= *n_alloc) - *argv = x2nrealloc (*argv, n_alloc, sizeof (char *)); - - if (str) - new_str = xstrdup (str); - else - new_str = NULL; - - (*argv)[*n_used] = new_str; - - (*n_used)++; -} - -size_t -count_strings (char *const *argv) -{ - size_t argc; - - for (argc = 0; argv[argc] != NULL; ++argc) - ; - return argc; -} - -struct dir_cache { - char *path; - char **files; -}; - -static size_t -dir_cache_hash (void const *x, size_t table_size) -{ - struct dir_cache const *p = x; - return hash_pjw (p->path, table_size); -} - -static bool -dir_cache_compare (void const *x, void const *y) -{ - struct dir_cache const *p = x; - struct dir_cache const *q = y; - return strcmp (p->path, q->path) == 0; -} - -/* Read a directory into a list of strings. - * - * Previously looked up directories are cached and returned quickly, - * saving some considerable amount of time compared to reading the - * directory over again. However this means you really must not - * alter the array of strings that are returned. - * - * Returns an empty list if the directory cannot be opened. - */ -char ** -read_dir (const char *name) -{ - static Hash_table *ht = NULL; - - if (!ht) - ht = hash_initialize (1024, NULL, dir_cache_hash, dir_cache_compare, NULL); - - struct dir_cache key = { .path = (char *) name }; - struct dir_cache *p = hash_lookup (ht, &key); - if (p) - return p->files; - - char **files = NULL; - size_t n_used = 0, n_alloc = 0; - - DIR *dir = opendir (name); - if (!dir) { - /* If it fails to open, that's OK, skip to the end. */ - /*perror (name);*/ - goto done; - } - - for (;;) { - errno = 0; - struct dirent *d = readdir (dir); - if (d == NULL) { - if (errno != 0) - /* But if it fails here, after opening and potentially reading - * part of the directory, that's a proper failure - inform the - * user and exit. - */ - error (EXIT_FAILURE, errno, "%s", name); - break; - } - - add_string (&files, &n_used, &n_alloc, d->d_name); - } - - if (closedir (dir) == -1) - error (EXIT_FAILURE, errno, "closedir: %s", name); - - done: - /* NULL-terminate the array. */ - add_string (&files, &n_used, &n_alloc, NULL); - - /* Add it to the hash for next time. */ - p = xmalloc (sizeof *p); - p->path = (char *) name; - p->files = files; - p = hash_insert (ht, p); - assert (p != NULL); - - return files; -} - -/* Filter a list of strings, returning only those where f(s) != 0. */ -char ** -filter (char **strings, int (*f) (const char *)) -{ - char **out = NULL; - size_t n_used = 0, n_alloc = 0; - - int i; - for (i = 0; strings[i] != NULL; ++i) { - if (f (strings[i]) != 0) - add_string (&out, &n_used, &n_alloc, strings[i]); - } - - add_string (&out, &n_used, &n_alloc, NULL); - return out; -} - -/* Filter a list of strings and return only those matching the wildcard. */ -char ** -filter_fnmatch (char **strings, const char *patt, int flags) -{ - char **out = NULL; - size_t n_used = 0, n_alloc = 0; - - int i, r; - for (i = 0; strings[i] != NULL; ++i) { - r = fnmatch (patt, strings[i], flags); - if (r == 0) - add_string (&out, &n_used, &n_alloc, strings[i]); - else if (r != FNM_NOMATCH) - error (EXIT_FAILURE, 0, "internal error: fnmatch ('%s', '%s', %d) returned unexpected non-zero value %d\n", - patt, strings[i], flags, r); - } - - add_string (&out, &n_used, &n_alloc, NULL); - return out; -} - -/* Filter a list of strings and return only those which DON'T contain sub. */ -char ** -filter_notmatching_substring (char **strings, const char *sub) -{ - char **out = NULL; - size_t n_used = 0, n_alloc = 0; - - int i; - for (i = 0; strings[i] != NULL; ++i) { - if (strstr (strings[i], sub) == NULL) - add_string (&out, &n_used, &n_alloc, strings[i]); - } - - add_string (&out, &n_used, &n_alloc, NULL); - return out; -} - -/* Sort a list of strings, in place, with the comparison function supplied. */ -void -sort (char **strings, int (*compare) (const void *, const void *)) -{ - qsort (strings, count_strings (strings), sizeof (char *), compare); -} - -/* Return true iff path exists and is a directory. This version - * follows symlinks. - */ -int -isdir (const char *path) -{ - struct stat statbuf; - - if (stat (path, &statbuf) == -1) - return 0; - - return S_ISDIR (statbuf.st_mode); -} - -/* Return true iff path exists and is a regular file. This version - * follows symlinks. - */ -int -isfile (const char *path) -{ - struct stat statbuf; - - if (stat (path, &statbuf) == -1) - return 0; - - return S_ISREG (statbuf.st_mode); -} - -/* Load in a file, returning a list of lines. */ -char ** -load_file (const char *filename) -{ - char **lines = 0; - size_t n_used = 0, n_alloc = 0; - - FILE *fp; - fp = fopen (filename, "r"); - if (fp == NULL) - error (EXIT_FAILURE, errno, "fopen: %s", filename); - - char line[4096]; - while (fgets (line, sizeof line, fp)) { - size_t len = strlen (line); - if (len > 0 && line[len-1] == '\n') - line[len-1] = '\0'; - add_string (&lines, &n_used, &n_alloc, line); - } - fclose (fp); - - add_string (&lines, &n_used, &n_alloc, NULL); - return lines; -} diff --git a/lib/.gitignore b/lib/.gitignore deleted file mode 100644 index efdd83a..0000000 --- a/lib/.gitignore +++ /dev/null @@ -1,175 +0,0 @@ -/alloca.in.h -/asnprintf.c -/asprintf.c -/at-func.c -/basename-lgpl.c -/bitrotate.h -/c-ctype.c -/c-ctype.h -/chdir-long.c -/chdir-long.h -/chown.c -/cloexec.c -/cloexec.h -/close.c -/closedir.c -/close-hook.c -/close-hook.h -/creat-safer.c -/cycle-check.c -/cycle-check.h -/dev-ino.h -/dirent--.h -/dirent.in.h -/dirent-private.h -/dirent-safer.h -/dirfd.c -/dirname.h -/dirname-lgpl.c -/dosname.h -/dummy.c -/dup2.c -/dup.c -/dup-safer.c -/errno.in.h -/error.c -/error.h -/exitfail.c -/exitfail.h -/fchdir.c -/fchmodat.c -/fchownat.c -/fchown-stub.c -/fclose.c -/fcntl.c -/fcntl--.h -/fcntl.in.h -/fcntl-safer.h -/fd-hook.c -/fd-hook.h -/fdopendir.c -/fd-safer.c -/filenamecat.h -/filenamecat-lgpl.c -/filename.h -/filevercmp.c -/filevercmp.h -/float.c -/float+.h -/float.in.h -/fstatat.c -/fstat.c -/fts.c -/fts-cycle.c -/fts_.h -/full-write.c -/full-write.h -/getcwd.c -/getcwd-lgpl.c -/getdtablesize.c -/getopt1.c -/getopt.c -/getopt.h -/getopt.in.h -/getopt_int.h -/gettext.h -/hash.c -/hash.h -/hash-pjw.c -/hash-pjw.h -/intprops.h -/inttypes.h -/inttypes.in.h -/i-ring.c -/i-ring.h -/itold.c -/lchown.c -/lstat.c -/Makefile.am -/malloc.c -/memchr.c -/memchr.valgrind -/mempcpy.c -/memrchr.c -/mkdirat.c -/mkdir.c -/msvc-inval.c -/msvc-inval.h -/msvc-nothrow.c -/msvc-nothrow.h -/openat.c -/openat-die.c -/openat.h -/openat-priv.h -/openat-proc.c -/openat-safer.c -/open.c -/opendir.c -/opendir-safer.c -/open-safer.c -/pathmax.h -/pipe-safer.c -/printf-args.c -/printf-args.h -/printf-parse.c -/printf-parse.h -/raise.c -/readdir.c -/realloc.c -/rmdir.c -/safe-read.c -/safe-read.h -/safe-write.c -/safe-write.h -/same-inode.h -/save-cwd.c -/save-cwd.h -/signal.h -/signal.in.h -/size_max.h -/stat.c -/stdarg.in.h -/stdbool.in.h -/stddef.in.h -/stdint.in.h -/stdio.in.h -/stdio-write.c -/stdlib.in.h -/strdup.c -/strerror.c -/strerror-override.c -/strerror-override.h -/string.in.h -/stripslash.c -/sys_stat.in.h -/sys_types.in.h -/sys_wait.in.h -/time.in.h -/unistd--.h -/unistd.in.h -/unistd-safer.h -/unlinkat.c -/unlink.c -/vasnprintf.c -/vasnprintf.h -/vasprintf.c -/verify.h -/wchar.in.h -/write.c -/xalloc-die.c -/xalloc.h -/xalloc-oversized.h -/xasprintf.c -/xgetcwd.c -/xgetcwd.h -/xmalloc.c -/xsize.h -/xstrtol.c -/xstrtol-error.c -/xstrtol.h -/xstrtoul.c -/xvasprintf.c -/xvasprintf.h -/bitrotate.c -/unistd.c -/xsize.c diff --git a/m4/.gitignore b/m4/.gitignore deleted file mode 100644 index fc143b6..0000000 --- a/m4/.gitignore +++ /dev/null @@ -1,114 +0,0 @@ -/00gnulib.m4 -/alloca.m4 -/asm-underscore.m4 -/chdir-long.m4 -/chown.m4 -/cloexec.m4 -/closedir.m4 -/close.m4 -/cycle-check.m4 -/d-ino.m4 -/dirent_h.m4 -/dirent-safer.m4 -/dirfd.m4 -/dirname.m4 -/dos.m4 -/double-slash-root.m4 -/d-type.m4 -/dup2.m4 -/dup.m4 -/errno_h.m4 -/error.m4 -/exponentd.m4 -/extensions.m4 -/fchdir.m4 -/fclose.m4 -/fcntl_h.m4 -/fcntl.m4 -/fcntl-o.m4 -/fcntl-safer.m4 -/fdopendir.m4 -/filenamecat.m4 -/float_h.m4 -/fstat.m4 -/fts.m4 -/getcwd-abort-bug.m4 -/getcwd.m4 -/getcwd-path-max.m4 -/getdtablesize.m4 -/getopt.m4 -/gnulib-common.m4 -/gnulib-comp.m4 -/gnulib-tool.m4 -/hash.m4 -/include_next.m4 -/intmax_t.m4 -/inttypes_h.m4 -/inttypes.m4 -/inttypes-pri.m4 -/i-ring.m4 -/largefile.m4 -/lchown.m4 -/longlong.m4 -/lstat.m4 -/malloc.m4 -/math_h.m4 -/memchr.m4 -/mempcpy.m4 -/memrchr.m4 -/mkdir.m4 -/mmap-anon.m4 -/mode_t.m4 -/msvc-inval.m4 -/msvc-nothrow.m4 -/multiarch.m4 -/nocrash.m4 -/off_t.m4 -/onceonly.m4 -/openat.m4 -/opendir.m4 -/open.m4 -/pathmax.m4 -/printf.m4 -/raise.m4 -/readdir.m4 -/realloc.m4 -/rmdir.m4 -/safe-read.m4 -/safe-write.m4 -/save-cwd.m4 -/signal_h.m4 -/size_max.m4 -/ssize_t.m4 -/stat.m4 -/stdarg.m4 -/stdbool.m4 -/stddef_h.m4 -/stdint_h.m4 -/stdint.m4 -/stdio_h.m4 -/stdlib_h.m4 -/strdup.m4 -/strerror.m4 -/string_h.m4 -/sys_socket_h.m4 -/sys_stat_h.m4 -/sys_types_h.m4 -/sys_wait_h.m4 -/time_h.m4 -/unistd_h.m4 -/unistd-safer.m4 -/unlink.m4 -/vasnprintf.m4 -/vasprintf.m4 -/warn-on-use.m4 -/wchar_h.m4 -/wchar_t.m4 -/wint_t.m4 -/write.m4 -/xalloc.m4 -/xgetcwd.m4 -/xsize.m4 -/xstrtol.m4 -/xvasprintf.m4 -/extern-inline.m4 diff --git a/m4/gnulib-cache.m4 b/m4/gnulib-cache.m4 deleted file mode 100644 index ff80694..0000000 --- a/m4/gnulib-cache.m4 +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (C) 2002-2013 Free Software Foundation, Inc. -# -# This file is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This file is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this file. If not, see <http://www.gnu.org/licenses/>. -# -# As a special exception to the GNU General Public License, -# this file may be distributed as part of a program that -# contains a configuration script generated by Autoconf, under -# the same distribution terms as the rest of that program. -# -# Generated by gnulib-tool. -# -# This file represents the specification of how gnulib-tool is used. -# It acts as a cache: It is written and read by gnulib-tool. -# In projects that use version control, this file is meant to be put under -# version control, like the configure.ac and various Makefile.am files. - - -# Specification in the form of a command-line invocation: -# gnulib-tool --import --dir=. --lib=libgnu --source-base=lib --m4-base=m4 --doc-base=doc --tests-base=tests --aux-dir=. --no-conditional-dependencies --no-libtool --macro-prefix=gl error filevercmp fts full-write hash hash-pjw xalloc xstrtol xvasprintf - -# Specification in the form of a few gnulib-tool.m4 macro invocations: -gl_LOCAL_DIR([]) -gl_MODULES([ - error - filevercmp - fts - full-write - hash - hash-pjw - xalloc - xstrtol - xvasprintf -]) -gl_AVOID([]) -gl_SOURCE_BASE([lib]) -gl_M4_BASE([m4]) -gl_PO_BASE([]) -gl_DOC_BASE([doc]) -gl_TESTS_BASE([tests]) -gl_LIB([libgnu]) -gl_MAKEFILE_NAME([]) -gl_MACRO_PREFIX([gl]) -gl_PO_DOMAIN([]) -gl_WITNESS_C_MACRO([]) diff --git a/src/Makefile.am b/src/Makefile.am index ca8041f..8abb1c5 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,5 +1,5 @@ # supermin Makefile.am -# (C) Copyright 2009-2013 Red Hat Inc. +# (C) Copyright 2009-2014 Red Hat Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,67 +19,81 @@ EXTRA_DIST = \ .depend \ - config.ml \ + config.ml.in \ supermin.1 \ supermin.pod \ - supermin.ml \ - supermin_cmdline.mli \ - supermin_cmdline.ml \ - supermin_debian.ml \ - supermin_package_handlers.mli \ - supermin_package_handlers.ml \ - supermin_pacman.ml \ - supermin_pacman_g2.ml \ - supermin_urpmi_rpm.ml \ - supermin_utils.mli \ - supermin_utils.ml \ - supermin_yum_rpm.ml \ - supermin_zypp_rpm.ml + $(SOURCES) # Note these must be in build dependency order. SOURCES = \ + ext2fs-c.c \ + ext2fs.ml \ + ext2fs.mli \ + ext2init-c.c \ + ext2init.ml \ + ext2init.mli \ + fnmatch-c.c \ + fnmatch.ml \ + fnmatch.mli \ + glob-c.c \ + glob.ml \ + glob.mli \ + realpath-c.c \ + realpath.ml \ + realpath.mli \ config.ml \ - supermin_cmdline.mli \ - supermin_cmdline.ml \ - supermin_utils.mli \ - supermin_utils.ml \ - supermin_package_handlers.mli \ - supermin_package_handlers.ml + utils.ml \ + utils.mli \ + types.ml \ + package_handler.ml \ + package_handler.mli \ + rpm.ml \ + prepare.ml \ + chroot.ml \ + kernel.ml \ + ext2_initrd.ml \ + ext2.ml \ + build.ml \ + supermin.ml + # Can't use filter for this because of automake brokenness. SOURCES_ML = \ + ext2fs.ml \ + ext2init.ml \ + fnmatch.ml \ + glob.ml \ + realpath.ml \ config.ml \ - supermin_cmdline.ml \ - supermin_utils.ml \ - supermin_package_handlers.ml - -if HAVE_OCAML_INIFILES -SOURCES += \ - supermin_zypp_rpm.ml -SOURCES_ML += \ - supermin_zypp_rpm.ml -endif - -SOURCES += \ - supermin_yum_rpm.ml \ - supermin_urpmi_rpm.ml \ - supermin_debian.ml \ - supermin_pacman.ml \ - supermin_pacman_g2.ml \ - supermin.ml -SOURCES_ML += \ - supermin_yum_rpm.ml \ - supermin_urpmi_rpm.ml \ - supermin_debian.ml \ - supermin_pacman.ml \ - supermin_pacman_g2.ml \ + utils.ml \ + types.ml \ + package_handler.ml \ + rpm.ml \ + prepare.ml \ + chroot.ml \ + kernel.ml \ + ext2_initrd.ml \ + ext2.ml \ + build.ml \ supermin.ml +SOURCES_C = \ + ext2fs-c.c \ + ext2init-c.c \ + fnmatch-c.c \ + glob-c.c \ + realpath-c.c + CLEANFILES = *~ *.cmi *.cmo *.cmx *.o supermin man_MANS = \ supermin.1 -bin_SCRIPTS = supermin +bin_PROGRAMS = supermin + +supermin_SOURCES = $(SOURCES_C) +supermin_CFLAGS = \ + -I$(shell $(OCAMLC) -where) \ + $(EXT2FS_CFLAGS) $(COM_ERR_CFLAGS) BOBJECTS = $(SOURCES_ML:.ml=.cmo) XOBJECTS = $(SOURCES_ML:.ml=.cmx) @@ -93,14 +107,17 @@ BEST = opt endif OCAMLPACKAGES = -package unix,str -if HAVE_OCAML_INIFILES -OCAMLPACKAGES += -package inifiles -endif OCAMLFLAGS = -warn-error CDEFLMPSUVXYZ -supermin: $(OBJECTS) - $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) -linkpkg \ - $^ -o $@ +supermin_DEPENDENCIES = $(OBJECTS) ext2init-bin.o + +supermin_LDADD = ext2init-bin.o + +supermin_LINK = \ + $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) \ + $(OCAMLPACKAGES) -linkpkg \ + -cclib '$(EXT2FS_LIBS) $(COM_ERR_LIBS)' \ + -o $@ $(OBJECTS) .mli.cmi: $(OCAMLFIND) ocamlc $(OCAMLFLAGS) $(OCAMLPACKAGES) -c $< -o $@ @@ -109,6 +126,24 @@ supermin: $(OBJECTS) .ml.cmx: $(OCAMLFIND) ocamlopt $(OCAMLFLAGS) $(OCAMLPACKAGES) -c $< -o $@ +# init "script" used by ext2 initrd. +noinst_PROGRAMS = init +init_SOURCES = init.c +init_CFLAGS = -static +init_LDFLAGS = -static +init_LDADD = $(ZLIB_STATIC_LIBS) $(LZMA_STATIC_LIBS) + +CLEANFILES += ext2init-bin.S + +ext2init-bin.o: ext2init-bin.S + $(CC) -o $@ -c $< + +ext2init-bin.S: init + strip --strip-all $< + @file $< | grep -isq static || \ + (echo "*** error: init is not staticly linked"; exit 1) + ./bin2s.pl $< $@ + depend: .depend .depend: $(SOURCES) diff --git a/src/bin2s.pl b/src/bin2s.pl new file mode 100755 index 0000000..2c78b5e --- /dev/null +++ b/src/bin2s.pl @@ -0,0 +1,45 @@ +#!/usr/bin/perl + +# This script creates a source file for the GNU assembler which shuold +# result in an object file equivalent to that of +# +# objcopy -I binary -B $(DEFAULT_ARCH) -O $(ELF_DEFAULT_ARCH) <in> <out> + +use strict; +use warnings; + +die "usage: $0 <in> <out>\n" if @ARGV != 2; + +my ($infile, $outfile) = @ARGV; +my ($buf, $i, $sz); +open my $ifh, '<', $infile or die "open $infile: $!"; +open my $ofh, '>', $outfile or die "open $outfile: $!"; + +print $ofh <<"EOF"; +/* This file has been automatically generated from $infile by $0 */ + +\t.globl\t_binary_${infile}_start +\t.globl\t_binary_${infile}_end +\t.globl\t_binary_${infile}_size + +\t.section\t.data +_binary_${infile}_start: +EOF + +$sz = 0; +while ( $i = read $ifh, $buf, 12 ) { + print $ofh "\t.byte\t" + . join( ',', map { sprintf '0x%02x', ord $_ } split //, $buf ) . "\n"; + $sz += $i; +} +die "read $infile (at offset $sz): $!\n" if not defined $i; +close $ifh; + +print $ofh <<"EOF"; + +_binary_${infile}_end: + +\t.equ _binary_${infile}_size, $sz +EOF + +close $ofh; diff --git a/src/build.ml b/src/build.ml new file mode 100644 index 0000000..a460083 --- /dev/null +++ b/src/build.ml @@ -0,0 +1,390 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +open Unix +open Printf + +open Utils +open Types +open Package_handler +open Fnmatch +open Glob +open Realpath + +type appliance = { + excludefiles : string list; (* list of wildcards *) + hostfiles : string list; (* list of wildcards *) + packages : string list; (* list of package names *) + (* Note that base images don't appear here because they are unpacked + * into a build directory as we discover them. + *) +} + +let empty_appliance = { excludefiles = []; hostfiles = []; packages = [] } + +type file_type = +| GZip of file_content +| XZ of file_content +| Uncompressed of file_content +and file_content = +| Base_image (* a tarball *) +| Packages +| Hostfiles +| Excludefiles + +let rec string_of_file_type = function + | GZip c -> sprintf "gzip %s" (string_of_file_content c) + | XZ c -> sprintf "xz %s" (string_of_file_content c) + | Uncompressed c -> sprintf "uncompressed %s" (string_of_file_content c) +and string_of_file_content = function + | Base_image -> "base image (tar)" + | Packages -> "packages" + | Hostfiles -> "hostfiles" + | Excludefiles -> "excludefiles" + +let rec build (copy_kernel, debug, dtb_wildcard, format, host_cpu, outputdir, + packager_config, tmpdir, use_installed) inputs = + if debug >= 1 then + printf "supermin: build: %s\n%!" (String.concat " " inputs); + + if inputs = [] then ( + eprintf "supermin: build: no input supermin appliance specified\n"; + exit 1; + ); + + (* When base images are seen, they are unpacked into this temporary + * directory. But to speed things up, when we are building a chroot, + * set the basedir to be the output directory, so we avoid copying + * from a temporary directory to the output directory. + *) + let basedir = + match format with + | Chroot -> + outputdir + | Ext2 -> + let basedir = tmpdir // "base.d" in + mkdir basedir 0o755; + basedir in + + (* Read the supermin appliance, ie. the input files and/or + * directories that make up the appliance. + *) + let appliance = read_appliance debug basedir empty_appliance inputs in + + (* Resolve dependencies in the list of packages. *) + let ph = get_package_handler () in + let packages = filter_map ph.ph_package_of_string appliance.packages in + let packages = + let packages = package_set_of_list packages in + let dependencies = ph.ph_get_all_requires packages in + PackageSet.union packages dependencies in + + if debug >= 1 then + printf "supermin: build: %d packages, including dependencies\n%!" + (PackageSet.cardinal packages); + + (* List the files in each package. *) + let files = + PackageSet.fold ( + fun pkg files -> + let fs = ph.ph_get_files pkg in + let fs = List.map (fun { ft_path = path } -> path) fs in + fs :: files + ) packages [] in + let files = List.flatten files in + + if debug >= 1 then + printf "supermin: build: %d files\n%!" (List.length files); + + (* Remove files from the list which don't exist on the host or are + * unreadable to us. + *) + let files = + List.filter ( + fun path -> + try ignore (lstat path); true + with Unix_error (err, fn, _) -> false + ) files in + + if debug >= 1 then + printf "supermin: build: %d files, after removing unreadable files\n%!" + (List.length files); + + (* Remove excludefiles from the list. Notes: (1) The current + * implementation does not apply excludefiles to the base image. (2) + * The current implementation does not apply excludefiles to the + * hostfiles (see below). + *) + let files = + if appliance.excludefiles = [] then files + else ( + let fn_flags = [FNM_NOESCAPE] in + List.filter ( + fun path -> + List.for_all ( + fun pattern -> not (fnmatch pattern path fn_flags) + ) appliance.excludefiles + ) files + ) in + + if debug >= 1 then + printf "supermin: build: %d files, after matching excludefiles\n%!" + (List.length files); + + (* Add hostfiles. This may contain wildcards too. *) + let files = + if appliance.hostfiles = [] then files + else ( + let hostfiles = List.map ( + fun pattern -> glob pattern [GLOB_NOESCAPE] + ) appliance.hostfiles in + let hostfiles = List.map Array.to_list hostfiles in + let hostfiles = List.flatten hostfiles in + files @ hostfiles + ) in + + if debug >= 1 then + printf "supermin: build: %d files, after adding hostfiles\n%!" + (List.length files); + + (* Difficult to explain what this does. See comment below. *) + let files = munge files in + + if debug >= 1 then + printf "supermin: build: %d files, after munging\n%!" + (List.length files); + + (* Depending on the format, we build the appliance in different ways. *) + match format with + | Chroot -> + (* chroot doesn't need an external kernel or initrd *) + Chroot.build_chroot debug files outputdir + + | Ext2 -> + let kernel = outputdir // "kernel" + and dtb = outputdir // "dtb" + and appliance = outputdir // "root" + and initrd = outputdir // "initrd" in + let kernel_version, modpath = + Kernel.build_kernel debug host_cpu dtb_wildcard copy_kernel kernel dtb in + Ext2.build_ext2 debug basedir files modpath kernel_version appliance; + Ext2_initrd.build_initrd debug tmpdir modpath initrd + +and read_appliance debug basedir appliance = function + | [] -> appliance + + | dir :: rest when Sys.is_directory dir -> + let inputs = Array.to_list (Sys.readdir dir) in + let inputs = List.sort compare inputs in + let inputs = List.map ((//) dir) inputs in + read_appliance debug basedir appliance (inputs @ rest) + + | file :: rest -> + let file_type = get_file_type file in + + if debug >= 1 then + printf "supermin: build: visiting %s type %s\n%!" + file (string_of_file_type file_type); + + (* Depending on the file type, read or unpack the file. *) + let appliance = + match file_type with + | Uncompressed ((Packages|Hostfiles|Excludefiles) as t) -> + let chan = open_in file in + let lines = input_all_lines chan in + close_in chan; + update_appliance appliance lines t + | GZip ((Packages|Hostfiles|Excludefiles) as t) -> + let cmd = sprintf "zcat %s" (quote file) in + let lines = run_command_get_lines cmd in + update_appliance appliance lines t + | XZ ((Packages|Hostfiles|Excludefiles) as t) -> + let cmd = sprintf "xzcat %s" (quote file) in + let lines = run_command_get_lines cmd in + update_appliance appliance lines t + | Uncompressed Base_image -> + let cmd = sprintf "tar -C %s -xf %s" (quote basedir) (quote file) in + run_command cmd; + appliance + | GZip Base_image -> + let cmd = + sprintf "zcat %s | tar -C %s -xf -" (quote file) (quote basedir) in + run_command cmd; + appliance + | XZ Base_image -> + let cmd = + sprintf "xzcat %s | tar -C %s -xf -" (quote file) (quote basedir) in + run_command cmd; + appliance in + + read_appliance debug basedir appliance rest + +and update_appliance appliance lines = function + | Packages -> + { appliance with packages = appliance.packages @ lines } + | Hostfiles -> + { appliance with hostfiles = appliance.hostfiles @ lines } + | Excludefiles -> + let lines = List.map ( + fun path -> + let n = String.length path in + if n < 1 || path.[0] <> '-' then ( + eprintf "supermin: excludefiles line does not start with '-'\n"; + exit 1 + ); + String.sub path 1 (n-1) + ) lines in + { appliance with excludefiles = appliance.excludefiles @ lines } + | Base_image -> assert false + +(* Determine the [file_type] of [file], or exit with an error. *) +and get_file_type file = + let chan = open_in file in + let buf = String.create 512 in + let len = input chan buf 0 (String.length buf) in + close_in chan; + + if len >= 3 && buf.[0] = '\x1f' && buf.[1] = '\x8b' && buf.[2] = '\x08' + then (* gzip-compressed file *) + GZip (get_compressed_file_content "zcat" file) + else if len >= 6 && buf.[0] = '\xfd' && buf.[1] = '7' && buf.[2] = 'z' && + buf.[3] = 'X' && buf.[4] = 'Z' && buf.[5] = '\000' + then (* xz-compressed file *) + XZ (get_compressed_file_content "xzcat" file) + else + Uncompressed (get_file_content file buf len) + +and get_file_content file buf len = + if len >= 2 && buf.[0] = '/' then Hostfiles + else if len >= 2 && buf.[0] = '-' then Excludefiles + else if len >= 1 && isalnum buf.[0] then Packages + else if len >= 261 && buf.[257] = 'u' && buf.[258] = 's' && + buf.[259] = 't' && buf.[260] = 'a' && buf.[261] = 'r' + then (* tar file *) + Base_image + else ( + eprintf "supermin: %s: unknown file type in supermin directory\n" file; + exit 1 + ) + +and get_compressed_file_content zcat file = + let cmd = sprintf "%s %s" zcat (quote file) in + let chan = open_process_in cmd in + let buf = String.create 512 in + let len = input chan buf 0 (String.length buf) in + (* We're expecting the subprocess to fail because we close the pipe + * early, so: + *) + ignore (close_process_in chan); + + get_file_content file buf len + +and isalnum = function + | '0'..'9' | 'a'..'z' | 'A'..'Z' -> true + | _ -> false + +(* The files may not be listed in an order that allows us to run + * through the list (even if we sorted it). The particular problem is + * where you have: + * + * - /lib is a symlink to /usr/lib + * - /lib/modules exists + * - /usr/lib + * + * The problem is that /lib is created as a symlink to a directory + * that doesn't yet exist (/usr/lib), and so it fails when you + * try to create /lib/modules (ie. really /usr/lib/modules). + * + * A second problem is that intermediate directories are not + * necessarily listed. eg. "/foo/bar/baz" might appear, without the + * parent directories appearing in the list. This can happen + * because of excludefiles, or simply packaging mistakes in + * the distro. + * + * We create intermediate directories simply by examining the + * file list. Symlinks to not-yet-existing directories are + * handled by adding the target directory into the list before the + * symlink. + *) +and munge files = + let files = List.sort compare files in + + let rec stat_is_dir dir = + try (stat dir).st_kind = S_DIR with Unix_error _ -> false + and is_lnk_to_dir dir = + try stat_is_dir dir && (lstat dir).st_kind = S_LNK + with Unix_error _ -> false + in + + let insert_dir, dir_seen = + let h = Hashtbl.create (List.length files) in + let insert_dir dir = Hashtbl.replace h dir true in + let dir_seen dir = Hashtbl.mem h dir in + insert_dir, dir_seen + in + + let rec loop = function + | [] -> [] + + | "/" :: rest -> + (* This is just to avoid a corner-case in subsequent rules. *) + loop rest + + | dir :: rest when stat_is_dir dir && dir_seen dir -> + dir :: loop rest + + | dir :: rest when is_lnk_to_dir dir -> + insert_dir dir; + + (* Symlink to a directory. Insert the target directory before + * if we've not seen it yet. + *) + let target = readlink dir in + let parent = Filename.dirname dir in + (* Make the target an absolute path. *) + let target = + if String.length target < 1 || target.[0] <> '/' then + realpath (parent // target) + else + target in + if not (dir_seen target) then + loop (target :: dir :: rest) + else + dir :: loop rest + + | dir :: rest when stat_is_dir dir -> + insert_dir dir; + + (* Have we seen the parent? *) + let parent = Filename.dirname dir in + if not (dir_seen parent) then + loop (parent :: dir :: rest) + else + dir :: loop rest + + | file :: rest -> + (* Have we seen this parent directory before? *) + let dir = Filename.dirname file in + if not (dir_seen dir) then + loop (dir :: file :: rest) + else + file :: loop rest + in + let files = loop files in + + files diff --git a/src/chroot.ml b/src/chroot.ml new file mode 100644 index 0000000..1e1ddb2 --- /dev/null +++ b/src/chroot.ml @@ -0,0 +1,77 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +open Unix +open Printf + +open Utils + +let build_chroot debug files outputdir = + List.iter ( + fun path -> + try + let st = lstat path in + let opath = outputdir // path in + match st.st_kind with + | S_DIR -> + (* Note we fix up the permissions of directories in a second + * pass, otherwise we risk creating a directory that we are + * unable to write inside. GNU tar does the same thing! + *) + if debug >= 2 then printf "supermin: chroot: mkdir %s\n%!" opath; + mkdir opath 0o700 + + | S_LNK -> + let link = readlink path in + (* Need to turn absolute links into relative links, so they + * always work, whether or not you are in a chroot. + *) + let link = + if String.length link < 1 || link.[0] <> '/' then + link + else ( + let link = ref link in + for i = 1 to String.length path - 1 do + if path.[i] = '/' then link := "../" ^ !link + done; + !link + ) in + + if debug >= 2 then + printf "supermin: chroot: link %s -> %s\n%!" opath link; + symlink link opath + + | S_REG | S_CHR | S_BLK | S_FIFO | S_SOCK -> + if debug >= 2 then printf "supermin: chroot: copy %s\n%!" opath; + let cmd = sprintf "cp -p %s %s" path opath in + ignore (Sys.command cmd) + with Unix_error _ -> () + ) files; + + (* Second pass: fix up directory permissions in reverse. *) + let dirs = filter_map ( + fun path -> + let st = lstat path in + if st.st_kind = S_DIR then Some (path, st) else None + ) files in + List.iter ( + fun (path, st) -> + let opath = outputdir // path in + (try chown opath st.st_uid st.st_gid with Unix_error _ -> ()); + (try chmod opath st.st_perm with Unix_error _ -> ()) + ) (List.rev dirs) diff --git a/src/config.ml.in b/src/config.ml.in index fe7cda3..0995b99 100644 --- a/src/config.ml.in +++ b/src/config.ml.in @@ -1,5 +1,5 @@ -(* supermin 4 - * Copyright (C) 2009-2013 Red Hat Inc. +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. * @configure_input@ * * This program is free software; you can redistribute it and/or modify @@ -20,7 +20,6 @@ let package_name = "@PACKAGE_NAME@" let package_version = "@PACKAGE_VERSION@" let zypper = "@ZYPPER@" -let yum = "@YUM@" let urpmi = "@URPMI@" let rpm = "@RPM@" let yumdownloader = "@YUMDOWNLOADER@" @@ -31,3 +30,5 @@ let apt_cache_depends_recurse_broken = @APT_CACHE_DEPENDS_RECURSE_BROKEN@ let pacman = "@PACMAN@" let pacman_g2 = "@PACMAN_G2@" let host_cpu = "@host_cpu@" +let mke2fs = "@MKE2FS@" +let mke2fs_t_option = "@MKE2FS_T_OPTION@" diff --git a/src/ext2.ml b/src/ext2.ml new file mode 100644 index 0000000..701f52e --- /dev/null +++ b/src/ext2.ml @@ -0,0 +1,80 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +open Unix +open Printf + +open Utils +open Ext2fs + +(* The ext2 image that we build always has a fixed size, and we 'hope' + * that the files fit in (otherwise we'll get an error). Note that + * the file is sparsely allocated. + * + * The downside of allocating a very large initial disk is that the + * fixed overhead of ext2 is larger (since ext2 calculates it based on + * the size of the disk). For a 4GB disk the overhead is + * approximately 66MB. + * + * In future, make this configurable, or determine it from the input + * files (XXX). + *) +let appliance_size = 4L *^ 1024L *^ 1024L *^ 1024L + +let build_ext2 debug basedir files modpath kernel_version appliance = + if debug >= 1 then + printf "supermin: ext2: creating empty ext2 filesystem '%s'\n%!" appliance; + + let fd = openfile appliance [O_WRONLY;O_CREAT;O_TRUNC;O_NOCTTY] 0o644 in + LargeFile.ftruncate fd appliance_size; + close fd; + + let cmd = + sprintf "%s %s ext2 -F%s %s" + Config.mke2fs Config.mke2fs_t_option + (if debug >= 2 then "" else "q") + appliance in + run_command cmd; + + let fs = ext2fs_open appliance in + ext2fs_read_bitmaps fs; + + if debug >= 1 then + printf "supermin: ext2: populating from base image\n%!"; + + (* Read files from the base image, which has been unpacked into a + * directory for us. + *) + ext2fs_copy_dir_recursively_from_host fs basedir "/"; + + if debug >= 1 then + printf "supermin: ext2: copying files from host filesystem\n%!"; + + (* Copy files from host filesystem. *) + List.iter (fun path -> ext2fs_copy_file_from_host fs path path) files; + + if debug >= 1 then + printf "supermin: ext2: copying kernel modules\n%!"; + + (* Import the kernel modules. *) + ext2fs_copy_file_from_host fs "/lib" "/lib"; + ext2fs_copy_file_from_host fs "/lib/modules" "/lib/modules"; + ext2fs_copy_dir_recursively_from_host fs + modpath ("/lib/modules/" ^ kernel_version); + + ext2fs_close fs diff --git a/src/ext2_initrd.ml b/src/ext2_initrd.ml new file mode 100644 index 0000000..2fb938e --- /dev/null +++ b/src/ext2_initrd.ml @@ -0,0 +1,156 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +open Unix +open Printf + +open Utils +open Ext2fs +open Ext2init +open Fnmatch + +module StringSet = Set.Make (String) +module StringMap = Map.Make (String) + +let string_set_of_list strs = List.fold_right StringSet.add strs StringSet.empty +let keys map = StringMap.fold (fun k _ ks -> k :: ks) map [] + +(* The list of modules (wildcards) we consider for inclusion in the + * mini initrd. Only what is needed in order to find a device with an + * ext2 filesystem on it. + *) +let kmods = [ + "ext2.ko*"; + "ext4.ko*"; (* CONFIG_EXT4_USE_FOR_EXT23=y option might be set *) + "virtio*.ko*"; + "ide*.ko*"; + "libata*.ko*"; + "piix*.ko*"; + "scsi_transport_spi.ko*"; + "scsi_mod.ko*"; + "sd_mod.ko*"; + "sym53c8xx.ko*"; + "ata_piix.ko*"; + "sr_mod.ko*"; + "mbcache.ko*"; + "crc*.ko*"; + "libcrc*.ko*"; + "ibmvscsic.ko*"; + "megaraid*.ko*"; +] + +let rec build_initrd debug tmpdir modpath initrd = + if debug >= 1 then + printf "supermin: ext2: creating minimal initrd '%s'\n%!" initrd; + + let initdir = tmpdir // "init.d" in + mkdir initdir 0o755; + + (* Read modules.dep file. *) + let moddeps = read_module_deps modpath in + + (* Create a set of top-level modules, that is any module which + * matches a pattern in kmods. + *) + let topset = + let mods = keys moddeps in + List.fold_left ( + fun topset modl -> + let m = Filename.basename modl in + let matches wildcard = fnmatch wildcard m [FNM_PATHNAME] in + if List.exists matches kmods then + StringSet.add modl topset + else + topset + ) StringSet.empty mods in + + (* Do depth-first search to locate the modules we need to load. Keep + * track of which modules we've added so we don't add them twice. + *) + let visited = Hashtbl.create 13 in + let loaded = ref 0 in + let chan = open_out (initdir // "modules") in + let rec visit set = + StringSet.iter ( + fun modl -> + if not (Hashtbl.mem visited modl) then ( + Hashtbl.add visited modl true; + + if debug >= 2 then + printf "supermin: ext2: initrd: visiting module %s\n%!" modl; + + (* Visit dependencies first. *) + let deps = + try StringMap.find modl moddeps + with Not_found -> StringSet.empty in + visit deps; + + (* Copy module to the init directory. *) + let cmd = + sprintf "cp -t %s %s" (quote initdir) (quote (modpath // modl)) in + run_command cmd; + + (* Write module name to 'modules' file. *) + fprintf chan "%s\n" (Filename.basename modl); + incr loaded + ) + ) set + in + visit topset; + close_out chan; + + if debug >= 1 then + printf "supermin: ext2: wrote %d modules to minimal initrd\n%!" !loaded; + + (* This is the binary blob containing the init "script". *) + let init = binary_init () in + let initname = initdir // "init" in + let chan = open_out initname in + output_string chan init; + close_out chan; + chmod initname 0o755; + + (* Build the cpio file. *) + let cmd = + sprintf "(cd %s && (echo .; ls -1) | cpio --quiet -o -H newc) > %s" + (quote initdir) (quote initrd) in + run_command cmd + +(* Read modules.dep into internal structure. *) +and read_module_deps modpath = + let modules_dep = modpath // "modules.dep" in + let chan = open_in modules_dep in + let lines = input_all_lines chan in + close_in chan; + List.fold_left ( + fun map line -> + let i = String.index line ':' in + let modl = String.sub line 0 i in + let deps = String.sub line (i+1) (String.length line - (i+1)) in + let deps = + if deps <> "" && deps <> " " then ( + let deps = + let len = String.length deps in + if len >= 1 && deps.[0] = ' ' then String.sub deps 1 (len-1) + else deps in + let deps = string_split " " deps in + string_set_of_list deps + ) + else StringSet.empty in + StringMap.add modl deps map + ) StringMap.empty lines diff --git a/src/ext2fs-c.c b/src/ext2fs-c.c new file mode 100644 index 0000000..828804e --- /dev/null +++ b/src/ext2fs-c.c @@ -0,0 +1,655 @@ +/* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <time.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fts.h> +#include <limits.h> +#include <errno.h> +#include <assert.h> + +/* Inlining is broken in the ext2fs header file. Disable it by + * defining the following: + */ +#define NO_INLINE_FUNCS +#include <ext2fs.h> + +/* Defining CAML_NAME_SPACE prevents <caml/compatibility.h> from + * defining some macros that conflict with ext2fs macros. + */ +#define CAML_NAME_SPACE +#include <caml/alloc.h> +#include <caml/custom.h> +#include <caml/fail.h> +#include <caml/memory.h> +#include <caml/mlvalues.h> +#include <caml/unixsupport.h> + +static void initialize (void) __attribute__((constructor)); + +static void +initialize (void) +{ + initialize_ext2_error_table (); +} + +static void ext2_error_to_exception (const char *fn, errcode_t err) __attribute__((noreturn)); + +static void +ext2_error_to_exception (const char *fn, errcode_t err) +{ + fprintf (stderr, "supermin: %s: %s\n", fn, error_message (err)); + caml_failwith (fn); +} + +static void ext2_handle_closed (void) __attribute__((noreturn)); + +static void +ext2_handle_closed (void) +{ + caml_failwith ("ext2fs: function called on a closed handle"); +} + +#define Ext2fs_val(v) (*((ext2_filsys *)Data_custom_val(v))) + +static void +ext2_finalize (value fsv) +{ + ext2_filsys fs = Ext2fs_val (fsv); + + if (fs) { +#ifdef HAVE_EXT2FS_CLOSE2 + ext2fs_close2 (fs, EXT2_FLAG_FLUSH_NO_SYNC); +#else + ext2fs_close (fs); +#endif + } +} + +static struct custom_operations ext2_custom_operations = { + (char *) "ext2fs_custom_operations", + ext2_finalize, + custom_compare_default, + custom_hash_default, + custom_serialize_default, + custom_deserialize_default +}; + +static value +Val_ext2fs (ext2_filsys fs) +{ + CAMLparam0 (); + CAMLlocal1 (fsv); + + fsv = caml_alloc_custom (&ext2_custom_operations, + sizeof (ext2_filsys), 0, 1); + Ext2fs_val (fsv) = fs; + CAMLreturn (fsv); +} + +value +supermin_ext2fs_open (value filev) +{ + CAMLparam1 (filev); + CAMLlocal1 (fsv); + int fs_flags = EXT2_FLAG_RW; + errcode_t err; + ext2_filsys fs; + +#ifdef EXT2_FLAG_64BITS + fs_flags |= EXT2_FLAG_64BITS; +#endif + + err = ext2fs_open (String_val (filev), fs_flags, 0, 0, + unix_io_manager, &fs); + if (err != 0) + ext2_error_to_exception ("ext2fs_open", err); + + fsv = Val_ext2fs (fs); + CAMLreturn (fsv); +} + +value +supermin_ext2fs_close (value fsv) +{ + CAMLparam1 (fsv); + + ext2_finalize (fsv); + + /* So we don't double-free in the finalizer. */ + Ext2fs_val (fsv) = NULL; + + CAMLreturn (Val_unit); +} + +value +supermin_ext2fs_read_bitmaps (value fsv) +{ + CAMLparam1 (fsv); + ext2_filsys fs; + errcode_t err; + + fs = Ext2fs_val (fsv); + if (fs == NULL) + ext2_handle_closed (); + + err = ext2fs_read_bitmaps (fs); + if (err != 0) + ext2_error_to_exception ("ext2fs_read_bitmaps", err); + + CAMLreturn (Val_unit); +} + +static void ext2_mkdir (ext2_filsys fs, ext2_ino_t dir_ino, const char *dirname, const char *basename, mode_t mode, uid_t uid, gid_t gid, time_t ctime, time_t atime, time_t mtime); +static void ext2_empty_inode (ext2_filsys fs, ext2_ino_t dir_ino, const char *dirname, const char *basename, mode_t mode, uid_t uid, gid_t gid, time_t ctime, time_t atime, time_t mtime, int major, int minor, int dir_ft, ext2_ino_t *ino_ret); +static void ext2_write_file (ext2_filsys fs, ext2_ino_t ino, const char *buf, size_t size, const char *filename); +static void ext2_link (ext2_filsys fs, ext2_ino_t dir_ino, const char *basename, ext2_ino_t ino, int dir_ft); +static void ext2_clean_path (ext2_filsys fs, ext2_ino_t dir_ino, const char *dirname, const char *basename, int isdir); +static void ext2_copy_file (ext2_filsys fs, const char *src, const char *dest); + +/* Copy the host filesystem file/directory 'src' to the destination + * 'dest'. Directories are NOT copied recursively - the directory is + * simply created. See function below for recursive copy. + */ +value +supermin_ext2fs_copy_file_from_host (value fsv, value srcv, value destv) +{ + CAMLparam3 (fsv, srcv, destv); + const char *src = String_val (srcv); + const char *dest = String_val (destv); + ext2_filsys fs; + + fs = Ext2fs_val (fsv); + if (fs == NULL) + ext2_handle_closed (); + + ext2_copy_file (fs, src, dest); + + CAMLreturn (Val_unit); +} + +/* Copy the host directory 'srcdir' to the destination directory + * 'destdir'. The copy is done recursively. + */ +value +supermin_ext2fs_copy_dir_recursively_from_host (value fsv, + value srcdirv, value destdirv) +{ + CAMLparam3 (fsv, srcdirv, destdirv); + const char *srcdir = String_val (srcdirv); + const char *destdir = String_val (destdirv); + size_t srclen = strlen (srcdir); + ext2_filsys fs; + char *paths[2]; + FTS *fts; + FTSENT *entry; + const char *srcpath; + char *destpath; + size_t n; + int r; + + fs = Ext2fs_val (fsv); + if (fs == NULL) + ext2_handle_closed (); + + paths[0] = (char *) srcdir; + paths[1] = NULL; + fts = fts_open (paths, FTS_COMFOLLOW|FTS_PHYSICAL, NULL); + if (fts == NULL) + unix_error (errno, (char *) "fts_open", srcdirv); + + for (;;) { + errno = 0; + entry = fts_read (fts); + if (entry == NULL && errno != 0) + unix_error (errno, (char *) "fts_read", srcdirv); + if (entry == NULL) + break; + + /* Ignore directories being visited in post-order. */ + if (entry->fts_info & FTS_DP) + continue; + + /* Copy the file. */ + srcpath = entry->fts_path + srclen; + if (strcmp (destdir, "/") == 0) { + if (srcpath == '\0') + r = asprintf (&destpath, "/"); + else + r = asprintf (&destpath, "/%s", srcpath + 1); + } else { + if (srcpath == '\0') + r = asprintf (&destpath, "%s", destdir); + else + r = asprintf (&destpath, "%s/%s", destdir, srcpath + 1); + } + if (r == -1) + caml_raise_out_of_memory (); + + /* Destpath must not have a trailing '/' */ + n = strlen (destpath); + if (n >= 1 && destpath[n-1] == '/') destpath[n-1] = '\0'; + + ext2_copy_file (fs, entry->fts_path, destpath); + free (destpath); + } + + if (fts_close (fts) == -1) + unix_error (errno, (char *) "fts_close", srcdirv); + + CAMLreturn (Val_unit); +} + +static void +ext2_mkdir (ext2_filsys fs, + ext2_ino_t dir_ino, const char *dirname, const char *basename, + mode_t mode, uid_t uid, gid_t gid, + time_t ctime, time_t atime, time_t mtime) +{ + errcode_t err; + + mode = LINUX_S_IFDIR | (mode & 03777); + + /* Does the directory exist? This is legitimate: we just skip + * this case. + */ + ext2_ino_t ino; + err = ext2fs_namei (fs, EXT2_ROOT_INO, dir_ino, basename, &ino); + if (err == 0) + return; /* skip */ + + /* Otherwise, create it. */ + err = ext2fs_new_inode (fs, dir_ino, mode, 0, &ino); + if (err != 0) + ext2_error_to_exception ("ext2fs_new_inode", err); + + try_again: + err = ext2fs_mkdir (fs, dir_ino, ino, basename); + if (err != 0) { + /* See: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=217892 */ + if (err == EXT2_ET_DIR_NO_SPACE) { + err = ext2fs_expand_dir (fs, dir_ino); + if (err) + ext2_error_to_exception ("ext2fs_expand_dir", err); + goto try_again; + } else + ext2_error_to_exception ("ext2fs_mkdir", err); + } + + /* Copy the final permissions, UID etc. to the inode. */ + struct ext2_inode inode; + err = ext2fs_read_inode (fs, ino, &inode); + if (err != 0) + ext2_error_to_exception ("ext2fs_read_inode", err); + inode.i_mode = mode; + inode.i_uid = uid; + inode.i_gid = gid; + inode.i_ctime = ctime; + inode.i_atime = atime; + inode.i_mtime = mtime; + err = ext2fs_write_inode (fs, ino, &inode); + if (err != 0) + ext2_error_to_exception ("ext2fs_write_inode", err); +} + +static void +ext2_empty_inode (ext2_filsys fs, + ext2_ino_t dir_ino, const char *dirname, const char *basename, + mode_t mode, uid_t uid, gid_t gid, + time_t ctime, time_t atime, time_t mtime, + int major, int minor, int dir_ft, ext2_ino_t *ino_ret) +{ + errcode_t err; + struct ext2_inode inode; + ext2_ino_t ino; + + err = ext2fs_new_inode (fs, dir_ino, mode, 0, &ino); + if (err != 0) + ext2_error_to_exception ("ext2fs_new_inode", err); + + memset (&inode, 0, sizeof inode); + inode.i_mode = mode; + inode.i_uid = uid; + inode.i_gid = gid; + inode.i_blocks = 0; + inode.i_links_count = 1; + inode.i_ctime = ctime; + inode.i_atime = atime; + inode.i_mtime = mtime; + inode.i_size = 0; + inode.i_block[0] = (minor & 0xff) | (major << 8) | ((minor & ~0xff) << 12); + + err = ext2fs_write_new_inode (fs, ino, &inode); + if (err != 0) + ext2_error_to_exception ("ext2fs_write_inode", err); + + ext2_link (fs, dir_ino, basename, ino, dir_ft); + + ext2fs_inode_alloc_stats2 (fs, ino, 1, 0); + + if (ino_ret) + *ino_ret = ino; +} + +/* You must create the file first with ext2_empty_inode. */ +static void +ext2_write_file (ext2_filsys fs, + ext2_ino_t ino, const char *buf, size_t size, + const char *filename) +{ + errcode_t err; + ext2_file_t file; + err = ext2fs_file_open2 (fs, ino, NULL, EXT2_FILE_WRITE, &file); + if (err != 0) + ext2_error_to_exception ("ext2fs_file_open2", err); + + /* ext2fs_file_write cannot deal with partial writes. You have + * to write the entire file in a single call. + */ + unsigned int written; + err = ext2fs_file_write (file, buf, size, &written); + if (err != 0) + ext2_error_to_exception ("ext2fs_file_write", err); + if ((size_t) written != size) + caml_failwith ("ext2fs_file_write: file size != bytes written"); + + err = ext2fs_file_flush (file); + if (err != 0) + ext2_error_to_exception ("ext2fs_file_flush", err); + err = ext2fs_file_close (file); + if (err != 0) + ext2_error_to_exception ("ext2fs_file_close", err); + + /* Update the true size in the inode. */ + struct ext2_inode inode; + err = ext2fs_read_inode (fs, ino, &inode); + if (err != 0) + ext2_error_to_exception ("ext2fs_read_inode", err); + inode.i_size = size; + err = ext2fs_write_inode (fs, ino, &inode); + if (err != 0) + ext2_error_to_exception ("ext2fs_write_inode", err); +} + +/* This is just a wrapper around ext2fs_link which calls + * ext2fs_expand_dir as necessary if the directory fills up. See + * definition of expand_dir in the sources of debugfs. + */ +static void +ext2_link (ext2_filsys fs, + ext2_ino_t dir_ino, const char *basename, ext2_ino_t ino, int dir_ft) +{ + errcode_t err; + + again: + err = ext2fs_link (fs, dir_ino, basename, ino, dir_ft); + + if (err == EXT2_ET_DIR_NO_SPACE) { + err = ext2fs_expand_dir (fs, dir_ino); + if (err != 0) + ext2_error_to_exception ("ext2fs_expand_dir", err); + goto again; + } + + if (err != 0) + ext2_error_to_exception ("ext2fs_link", err); +} + +static int +release_block (ext2_filsys fs, blk_t *blocknr, + int blockcnt, void *private) +{ + blk_t block; + + block = *blocknr; + ext2fs_block_alloc_stats (fs, block, -1); + return 0; +} + +/* unlink or rmdir path, if it exists. */ +static void +ext2_clean_path (ext2_filsys fs, ext2_ino_t dir_ino, + const char *dirname, const char *basename, + int isdir) +{ + errcode_t err; + + ext2_ino_t ino; + err = ext2fs_lookup (fs, dir_ino, basename, strlen (basename), + NULL, &ino); + if (err == EXT2_ET_FILE_NOT_FOUND) + return; + + if (!isdir) { + struct ext2_inode inode; + err = ext2fs_read_inode (fs, ino, &inode); + if (err != 0) + ext2_error_to_exception ("ext2fs_read_inode", err); + inode.i_links_count--; + err = ext2fs_write_inode (fs, ino, &inode); + if (err != 0) + ext2_error_to_exception ("ext2fs_write_inode", err); + + err = ext2fs_unlink (fs, dir_ino, basename, 0, 0); + if (err != 0) + ext2_error_to_exception ("ext2fs_unlink_inode", err); + + if (inode.i_links_count == 0) { + inode.i_dtime = time (NULL); + err = ext2fs_write_inode (fs, ino, &inode); + if (err != 0) + ext2_error_to_exception ("ext2fs_write_inode", err); + + if (ext2fs_inode_has_valid_blocks (&inode)) { + int flags = 0; + /* From the docs: "BLOCK_FLAG_READ_ONLY is a promise by the + * caller that it will not modify returned block number." + * RHEL 5 does not have this flag, so just omit it if it is + * not defined. + */ +#ifdef BLOCK_FLAG_READ_ONLY + flags |= BLOCK_FLAG_READ_ONLY; +#endif + ext2fs_block_iterate (fs, ino, flags, NULL, + release_block, NULL); + } + + ext2fs_inode_alloc_stats2 (fs, ino, -1, isdir); + } + } + /* else it's a directory, what to do? XXX */ +} + +/* Read in the whole file into memory. Check the size is still 'size'. */ +static char * +read_whole_file (const char *filename, size_t size) +{ + char *buf = malloc (size); + if (buf == NULL) + caml_raise_out_of_memory (); + + int fd = open (filename, O_RDONLY); + if (fd == -1) { + /* Skip unreadable files. */ + fprintf (stderr, "supermin: open: %s: %m\n", filename); + return NULL; + } + + size_t n = 0; + char *p = buf; + + while (n < size) { + ssize_t r = read (fd, p, size - n); + if (r == -1) + unix_error (errno, (char *) "read", caml_copy_string (filename)); + if (r == 0) { + fprintf (stderr, "supermin: end of file reading '%s'\n", filename); + caml_invalid_argument ("ext2fs: file has changed size unexpectedly"); + } + n += r; + p += r; + } + + if (close (fd) == -1) + unix_error (errno, (char *) "close", caml_copy_string (filename)); + + return buf; +} + +/* Copy a file (or directory etc) from the host. */ +static void +ext2_copy_file (ext2_filsys fs, const char *src, const char *dest) +{ + errcode_t err; + struct stat statbuf; + + if (lstat (src, &statbuf) == -1) + unix_error (errno, (char *) "lstat", caml_copy_string (src)); + +#if 0 + /* if debug >= 3 */ + fprintf (stderr, "ext2_copy_file %s %s 0%o\n", src, dest, statbuf.st_mode); +#endif + + /* Sanity check the path. These rules are always true for the paths + * passed to us here from the appliance layer. The assertions just + * verify that the rules haven't changed. + */ + size_t n = strlen (dest); + assert (n <= PATH_MAX); + assert (n > 0); + assert (dest[0] == '/'); /* always absolute path */ + assert (n == 1 || dest[n-1] != '/'); /* no trailing slash */ + + /* Don't make the root directory, it always exists. This simplifies + * the code that follows. + */ + if (n == 1) return; + + /* XXX There is some confusion between 'src' and 'dest' in the + * following code, meaning we are likely following the structure of + * the host filesystem when adding files from the base image. This + * is because the code was copied and pasted from supermin 4 where + * there was no such distinction. + */ + const char *dirname, *basename; + const char *p = strrchr (dest, '/'); + ext2_ino_t dir_ino; + if (dest == p) { /* "/foo" */ + dirname = "/"; + basename = dest+1; + dir_ino = EXT2_ROOT_INO; + } else { /* "/foo/bar" */ + dirname = strndup (dest, p-dest); + basename = p+1; + + /* If the parent directory is a symlink to another directory, then + * we want to look up the target directory. (RHBZ#698089). + */ + struct stat stat1, stat2; + if (lstat (dirname, &stat1) == 0 && S_ISLNK (stat1.st_mode) && + stat (dirname, &stat2) == 0 && S_ISDIR (stat2.st_mode)) { + char *new_dirname = malloc (PATH_MAX+1); + ssize_t r = readlink (dirname, new_dirname, PATH_MAX+1); + if (r == -1) + unix_error (errno, (char *) "readlink", caml_copy_string (dest)); + new_dirname[r] = '\0'; + dirname = new_dirname; + } + + /* Look up the parent directory. */ + err = ext2fs_namei (fs, EXT2_ROOT_INO, EXT2_ROOT_INO, dirname, &dir_ino); + if (err != 0) + ext2_error_to_exception ("ext2fs_namei: parent directory not found", err); + } + + ext2_clean_path (fs, dir_ino, dirname, basename, S_ISDIR (statbuf.st_mode)); + + int dir_ft; + + /* Create regular file. */ + if (S_ISREG (statbuf.st_mode)) { + /* XXX Hard links get duplicated here. */ + ext2_ino_t ino; + char *buf = NULL; + + if (statbuf.st_size > 0) { + buf = read_whole_file (src, statbuf.st_size); + if (buf == NULL) + goto skip_unreadable_file; + } + + ext2_empty_inode (fs, dir_ino, dirname, basename, + statbuf.st_mode, statbuf.st_uid, statbuf.st_gid, + statbuf.st_ctime, statbuf.st_atime, statbuf.st_mtime, + 0, 0, EXT2_FT_REG_FILE, &ino); + + if (statbuf.st_size > 0) { + ext2_write_file (fs, ino, buf, statbuf.st_size, dest); + free (buf); + } + skip_unreadable_file: ; + } + /* Create a symlink. */ + else if (S_ISLNK (statbuf.st_mode)) { + ext2_ino_t ino; + ext2_empty_inode (fs, dir_ino, dirname, basename, + statbuf.st_mode, statbuf.st_uid, statbuf.st_gid, + statbuf.st_ctime, statbuf.st_atime, statbuf.st_mtime, + 0, 0, EXT2_FT_SYMLINK, &ino); + + char buf[PATH_MAX+1]; + ssize_t r = readlink (src, buf, sizeof buf); + if (r == -1) + unix_error (errno, (char *) "readlink", caml_copy_string (src)); + ext2_write_file (fs, ino, buf, r, dest); + } + /* Create directory. */ + else if (S_ISDIR (statbuf.st_mode)) + ext2_mkdir (fs, dir_ino, dirname, basename, + statbuf.st_mode, statbuf.st_uid, statbuf.st_gid, + statbuf.st_ctime, statbuf.st_atime, statbuf.st_mtime); + /* Create a special file. */ + else if (S_ISBLK (statbuf.st_mode)) { + dir_ft = EXT2_FT_BLKDEV; + goto make_special; + } + else if (S_ISCHR (statbuf.st_mode)) { + dir_ft = EXT2_FT_CHRDEV; + goto make_special; + } else if (S_ISFIFO (statbuf.st_mode)) { + dir_ft = EXT2_FT_FIFO; + goto make_special; + } else if (S_ISSOCK (statbuf.st_mode)) { + dir_ft = EXT2_FT_SOCK; + make_special: + ext2_empty_inode (fs, dir_ino, dirname, basename, + statbuf.st_mode, statbuf.st_uid, statbuf.st_gid, + statbuf.st_ctime, statbuf.st_atime, statbuf.st_mtime, + major (statbuf.st_rdev), minor (statbuf.st_rdev), + dir_ft, NULL); + } +} diff --git a/src/ext2fs.ml b/src/ext2fs.ml new file mode 100644 index 0000000..cf7c546 --- /dev/null +++ b/src/ext2fs.ml @@ -0,0 +1,26 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +type t + +external ext2fs_open : string -> t = "supermin_ext2fs_open" +external ext2fs_close : t -> unit = "supermin_ext2fs_close" + +external ext2fs_read_bitmaps : t -> unit = "supermin_ext2fs_read_bitmaps" +external ext2fs_copy_file_from_host : t -> string -> string -> unit = "supermin_ext2fs_copy_file_from_host" +external ext2fs_copy_dir_recursively_from_host : t -> string -> string -> unit = "supermin_ext2fs_copy_dir_recursively_from_host" diff --git a/src/ext2fs.mli b/src/ext2fs.mli new file mode 100644 index 0000000..ebcaf1b --- /dev/null +++ b/src/ext2fs.mli @@ -0,0 +1,33 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +(** {2 The [Ext2fs] module} + + The [Ext2fs] module provides a slightly simplified interface to + the ext2fs library. Where we don't use flags/parameters/etc they + are not exposed to OCaml. +*) + +type t + +val ext2fs_open : string -> t +val ext2fs_close : t -> unit + +val ext2fs_read_bitmaps : t -> unit +val ext2fs_copy_file_from_host : t -> string -> string -> unit +val ext2fs_copy_dir_recursively_from_host : t -> string -> string -> unit diff --git a/src/ext2init-c.c b/src/ext2init-c.c new file mode 100644 index 0000000..c310ed2 --- /dev/null +++ b/src/ext2init-c.c @@ -0,0 +1,44 @@ +/* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <caml/alloc.h> +#include <caml/memory.h> + +/* The init binary. + * See: bin2s.pl, init.c. + */ +extern char _binary_init_start, _binary_init_end, _binary_init_size; + +value +supermin_binary_init (value unitv) +{ + CAMLparam1 (unitv); + CAMLlocal1 (sv); + size_t n = (size_t) &_binary_init_size; + + sv = caml_alloc_string (n); + memcpy (String_val (sv), &_binary_init_start, n); + + CAMLreturn (sv); +} diff --git a/src/ext2init.ml b/src/ext2init.ml new file mode 100644 index 0000000..62985b0 --- /dev/null +++ b/src/ext2init.ml @@ -0,0 +1,19 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +external binary_init : unit -> string = "supermin_binary_init" diff --git a/src/ext2init.mli b/src/ext2init.mli new file mode 100644 index 0000000..a75c930 --- /dev/null +++ b/src/ext2init.mli @@ -0,0 +1,19 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +val binary_init : unit -> string diff --git a/src/fnmatch-c.c b/src/fnmatch-c.c new file mode 100644 index 0000000..c321343 --- /dev/null +++ b/src/fnmatch-c.c @@ -0,0 +1,64 @@ +/* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <config.h> + +#include <stdlib.h> +#include <fnmatch.h> +#include <errno.h> + +#include <caml/alloc.h> +#include <caml/memory.h> +#include <caml/mlvalues.h> +#include <caml/unixsupport.h> + +/* NB: These flags must appear in the same order as fnmatch.ml */ +static int flags[] = { + FNM_NOESCAPE, + FNM_PATHNAME, + FNM_PERIOD, + FNM_FILE_NAME, + FNM_LEADING_DIR, + FNM_CASEFOLD, +}; + +value +supermin_fnmatch (value patternv, value strv, value flagsv) +{ + CAMLparam3 (patternv, strv, flagsv); + int f = 0, r; + + /* Convert flags to bitmask. */ + while (flagsv != Val_int (0)) { + f |= flags[Int_val (Field (flagsv, 0))]; + flagsv = Field (flagsv, 1); + } + + r = fnmatch (String_val (patternv), String_val (strv), f); + + if (r == 0) + CAMLreturn (Val_true); + else if (r == FNM_NOMATCH) + CAMLreturn (Val_false); + else { + /* XXX The fnmatch specification doesn't mention what errors can + * be returned by fnmatch. Assume they are errnos for now. + */ + unix_error (errno, (char *) "fnmatch", patternv); + } +} diff --git a/src/fnmatch.ml b/src/fnmatch.ml new file mode 100644 index 0000000..2f10a97 --- /dev/null +++ b/src/fnmatch.ml @@ -0,0 +1,28 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +(* NB: These flags must appear in the same order as fnmatch-c.c *) +type flag = +| FNM_NOESCAPE +| FNM_PATHNAME +| FNM_PERIOD +| FNM_FILE_NAME +| FNM_LEADING_DIR +| FNM_CASEFOLD + +external fnmatch : string -> string -> flag list -> bool = "supermin_fnmatch" diff --git a/src/fnmatch.mli b/src/fnmatch.mli new file mode 100644 index 0000000..b3dce8d --- /dev/null +++ b/src/fnmatch.mli @@ -0,0 +1,27 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +type flag = +| FNM_NOESCAPE +| FNM_PATHNAME +| FNM_PERIOD +| FNM_FILE_NAME +| FNM_LEADING_DIR +| FNM_CASEFOLD + +val fnmatch : string -> string -> flag list -> bool diff --git a/src/glob-c.c b/src/glob-c.c new file mode 100644 index 0000000..7ee313f --- /dev/null +++ b/src/glob-c.c @@ -0,0 +1,84 @@ +/* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <config.h> + +#include <stdlib.h> +#include <string.h> +#include <glob.h> +#include <assert.h> + +#include <caml/alloc.h> +#include <caml/fail.h> +#include <caml/memory.h> +#include <caml/mlvalues.h> + +/* NB: These flags must appear in the same order as glob.ml */ +static int flags[] = { + GLOB_ERR, + GLOB_MARK, + GLOB_NOSORT, + GLOB_NOCHECK, + GLOB_NOESCAPE, + GLOB_PERIOD, +}; + +value +supermin_glob (value patternv, value flagsv) +{ + CAMLparam2 (patternv, flagsv); + CAMLlocal2 (rv, sv); + int f = 0, r; + size_t i; + glob_t g; + + memset (&g, 0, sizeof g); + + /* Convert flags to bitmask. */ + while (flagsv != Val_int (0)) { + f |= flags[Int_val (Field (flagsv, 0))]; + flagsv = Field (flagsv, 1); + } + + r = glob (String_val (patternv), f, NULL, &g); + + if (r == 0 || r == GLOB_NOMATCH) { + if (r == GLOB_NOMATCH) + assert (g.gl_pathc == 0); + + rv = caml_alloc (g.gl_pathc, 0); + for (i = 0; i < g.gl_pathc; ++i) { + sv = caml_copy_string (g.gl_pathv[i]); + Store_field (rv, i, sv); + } + + globfree (&g); + + CAMLreturn (rv); + } + + /* An error occurred. */ + globfree (&g); + + if (r == GLOB_NOSPACE) + caml_raise_out_of_memory (); + else if (r == GLOB_ABORTED) + caml_failwith ("glob: read error"); + else + caml_failwith ("glob: unknown error"); +} diff --git a/src/glob.ml b/src/glob.ml new file mode 100644 index 0000000..e8ba616 --- /dev/null +++ b/src/glob.ml @@ -0,0 +1,28 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +(* NB: These flags must appear in the same order as fnmatch-c.c *) +type flag = +| GLOB_ERR +| GLOB_MARK +| GLOB_NOSORT +| GLOB_NOCHECK +| GLOB_NOESCAPE +| GLOB_PERIOD + +external glob : string -> flag list -> string array = "supermin_glob" diff --git a/src/glob.mli b/src/glob.mli new file mode 100644 index 0000000..3164762 --- /dev/null +++ b/src/glob.mli @@ -0,0 +1,27 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +type flag = +| GLOB_ERR +| GLOB_MARK +| GLOB_NOSORT +| GLOB_NOCHECK +| GLOB_NOESCAPE +| GLOB_PERIOD + +val glob : string -> flag list -> string array diff --git a/src/init.c b/src/init.c new file mode 100644 index 0000000..1f9f2c1 --- /dev/null +++ b/src/init.c @@ -0,0 +1,537 @@ +/* supermin-helper reimplementation in C. + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +/* This very minimal init "script" goes in the mini-initrd used to + * boot the ext2-based appliance. Note we have no shell, so we cannot + * use system(3) to run external commands. In fact, we don't have + * very much at all, except this program, and some kernel modules. + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <string.h> +#include <inttypes.h> +#include <unistd.h> +#include <errno.h> +#include <fcntl.h> +#include <dirent.h> +#include <time.h> +#include <sys/types.h> +#include <sys/mount.h> +#include <sys/stat.h> +#include <sys/wait.h> + +#include <asm/unistd.h> + +#ifdef HAVE_ZLIB_STATIC +#include <zlib.h> +#endif + +#ifdef HAVE_LZMA_STATIC +#include <lzma.h> +#endif + +/* Maximum time to wait for the root device to appear (seconds). + * + * On slow machines with lots of disks (Koji running the 255 disk test + * in libguestfs) this really can take several minutes. + * + * Note that the actual wait time is approximately double the number + * given here because there is a delay which doubles until it reaches + * this value. + */ +#define MAX_ROOT_WAIT 300 + +extern long init_module (void *, unsigned long, const char *); + +/* translation taken from module-init-tools/insmod.c */ +static const char *moderror(int err) +{ + switch (err) { + case ENOEXEC: + return "Invalid module format"; + case ENOENT: + return "Unknown symbol in module"; + case ESRCH: + return "Module has wrong symbol version"; + case EINVAL: + return "Invalid parameters"; + default: + return strerror(err); + } +} + +/* Leave this enabled for now. When we get more confident in the boot + * process we can turn this off or make it configurable. + */ +#define verbose 1 + +static void mount_proc (void); +static void print_uptime (void); +static void read_cmdline (void); +static void insmod (const char *filename); +static void show_directory (const char *dir); + +static char cmdline[1024]; +static char line[1024]; + +int +main () +{ + mount_proc (); + + print_uptime (); + fprintf (stderr, "supermin: ext2 mini initrd starting up: " + PACKAGE_VERSION +#ifdef HAVE_ZLIB_STATIC + " zlib" +#endif +#ifdef HAVE_LZMA_STATIC + " xz" +#endif + "\n"); + + read_cmdline (); + + /* Create some fixed directories. */ + mkdir ("/dev", 0755); + mkdir ("/root", 0755); + mkdir ("/sys", 0755); + + /* Mount /sys. */ + if (verbose) + fprintf (stderr, "supermin: mounting /sys\n"); + if (mount ("sysfs", "/sys", "sysfs", 0, "") == -1) { + perror ("mount: /sys"); + exit (EXIT_FAILURE); + } + + FILE *fp = fopen ("/modules", "r"); + if (fp == NULL) { + perror ("fopen: /modules"); + exit (EXIT_FAILURE); + } + while (fgets (line, sizeof line, fp)) { + size_t n = strlen (line); + if (n > 0 && line[n-1] == '\n') + line[--n] = '\0'; + + /* XXX Because of the way we construct the module list, the + * "modules" file can contain non-existent modules. Ignore those + * for now. Really we should add them as missing dependencies. + * See ext2initrd.c:ext2_make_initrd(). + */ + if (access (line, R_OK) == 0) + insmod (line); + else + fprintf (stderr, "skipped %s, module is missing\n", line); + } + fclose (fp); + + /* Look for the ext2 filesystem device. It's always the last one + * that was added. Modern versions of libguestfs supply the + * expected name of the root device on the command line + * ("root=/dev/..."). For virtio-scsi this is required, because we + * must wait for the device to appear after the module is loaded. + */ + char *root, *path; + size_t len; + root = strstr (cmdline, "root="); + if (root) { + root += 5; + if (strncmp (root, "/dev/", 5) == 0) + root += 5; + len = strcspn (root, " "); + root[len] = '\0'; + + asprintf (&path, "/sys/block/%s/dev", root); + + uint64_t delay_ns = 250000; + while (delay_ns <= MAX_ROOT_WAIT * UINT64_C(1000000000)) { + fp = fopen (path, "r"); + if (fp != NULL) + goto found; + + if (delay_ns > 1000000000) + fprintf (stderr, + "supermin: waiting another %" PRIu64 " ns for %s to appear\n", + delay_ns, path); + + struct timespec t; + t.tv_sec = delay_ns / 1000000000; + t.tv_nsec = delay_ns % 1000000000; + nanosleep (&t, NULL); + delay_ns *= 2; + } + } + else { + path = strdup ("/sys/block/xdx/dev"); + + char class[3] = { 'v', 's', 'h' }; + size_t i, j; + fp = NULL; + for (i = 0; i < sizeof class; ++i) { + for (j = 'z'; j >= 'a'; --j) { + path[11] = class[i]; + path[13] = j; + fp = fopen (path, "r"); + if (fp != NULL) + goto found; + } + } + } + + fprintf (stderr, + "supermin: no ext2 root device found\n" + "Please include FULL verbose output in your bug report.\n"); + exit (EXIT_FAILURE); + + found: + if (verbose) + fprintf (stderr, "supermin: picked %s as root device\n", path); + + fgets (line, sizeof line, fp); + int major = atoi (line); + char *p = line + strcspn (line, ":") + 1; + int minor = atoi (p); + + fclose (fp); + if (umount ("/sys") == -1) { + perror ("umount: /sys"); + exit (EXIT_FAILURE); + } + + if (verbose) + fprintf (stderr, "supermin: creating /dev/root as block special %d:%d\n", + major, minor); + + if (mknod ("/dev/root", S_IFBLK|0700, makedev (major, minor)) == -1) { + perror ("mknod: /dev/root"); + exit (EXIT_FAILURE); + } + + /* Mount new root and chroot to it. */ + if (verbose) + fprintf (stderr, "supermin: mounting new root on /root\n"); + if (mount ("/dev/root", "/root", "ext2", MS_NOATIME, "") == -1) { + perror ("mount: /root"); + exit (EXIT_FAILURE); + } + + /* Note that pivot_root won't work. See the note in + * Documentation/filesystems/ramfs-rootfs-initramfs.txt + * We could remove the old initramfs files, but let's not bother. + */ + if (verbose) + fprintf (stderr, "supermin: chroot\n"); + + if (chroot ("/root") == -1) { + perror ("chroot: /root"); + exit (EXIT_FAILURE); + } + + chdir ("/"); + + /* Run /init from ext2 filesystem. */ + execl ("/init", "init", NULL); + perror ("execl: /init"); + + /* /init failed to execute, but why? Before we ditch, print some + * debug. Although we have a full appliance, the fact that /init + * failed to run means we may not be able to run any commands. + */ + show_directory ("/"); + show_directory ("/bin"); + show_directory ("/lib"); + show_directory ("/lib64"); + fflush (stderr); + + exit (EXIT_FAILURE); +} + +#if HAVE_LZMA_STATIC +static int +ends_with (const char *str, const char *suffix) +{ + if (!str || !suffix) + return 0; + size_t lenstr = strlen (str); + size_t lensuffix = strlen (suffix); + if (lensuffix > lenstr) + return 0; + return strncmp (str + lenstr - lensuffix, suffix, lensuffix) == 0; +} +#endif + +static void +insmod (const char *filename) +{ + size_t size; + + if (verbose) + fprintf (stderr, "supermin: internal insmod %s\n", filename); + +#ifdef HAVE_ZLIB_STATIC + int capacity = 64*1024; + char *buf = (char *) malloc (capacity); + int tmpsize = 8 * 1024; + char tmp[tmpsize]; + int num; + + errno = 0; + size = 0; + +#ifdef HAVE_LZMA_STATIC + if (ends_with(filename, ".xz")) { + lzma_stream strm = LZMA_STREAM_INIT; + lzma_ret ret = lzma_stream_decoder(&strm, UINT64_MAX, + LZMA_CONCATENATED); + if (verbose) + fprintf (stderr, "supermin: running xz\n"); + FILE *fd = fopen (filename, "r"); + if (!fd) { + perror("popen failed"); + exit (EXIT_FAILURE); + } + char tmp_out[tmpsize]; + strm.avail_in = 0; + strm.next_out = tmp_out; + strm.avail_out = tmpsize; + + lzma_action action = LZMA_RUN; + + while (1) { + if (strm.avail_in == 0) { + strm.next_in = tmp; + strm.avail_in = fread(tmp, 1, tmpsize, fd); + + if (ferror(fd)) { + // POSIX says that fread() sets errno if + // an error occurred. ferror() doesn't + // touch errno. + perror("Error reading input file"); + exit (EXIT_FAILURE); + } + if (feof(fd)) action = LZMA_FINISH; + } + + ret = lzma_code(&strm, action); + + // Write and check write error before checking decoder error. + // This way as much data as possible gets written to output + // even if decoder detected an error. + if (strm.avail_out == 0 || ret != LZMA_OK) { + const size_t num = tmpsize - strm.avail_out; + if (num > capacity) { + buf = (char*) realloc (buf, size*2); + capacity = size; + } + memcpy (buf+size, tmp_out, num); + capacity -= num; + size += num; + strm.next_out = tmp_out; + strm.avail_out = tmpsize; + } + if (ret != LZMA_OK) { + if (ret == LZMA_STREAM_END) { + break; + } else { + perror("internal error"); + exit(EXIT_FAILURE); + } + } + } + fclose (fd); + if (verbose) + fprintf (stderr, "done with xz %d read\n", size); + } else { +#endif + gzFile gzfp = gzopen (filename, "rb"); + if (gzfp == NULL) { + fprintf (stderr, "insmod: gzopen failed: %s", filename); + exit (EXIT_FAILURE); + } + while ((num = gzread (gzfp, tmp, tmpsize)) > 0) { + if (num > capacity) { + buf = (char*) realloc (buf, size*2); + capacity = size; + } + memcpy (buf+size, tmp, num); + capacity -= num; + size += num; + } + if (num == -1) { + perror ("insmod: gzread"); + exit (EXIT_FAILURE); + } + gzclose (gzfp); +#ifdef HAVE_LZMA_STATIC +} +#endif + +#else + int fd = open (filename, O_RDONLY); + if (fd == -1) { + fprintf (stderr, "insmod: open: %s: %m\n", filename); + exit (EXIT_FAILURE); + } + struct stat st; + if (fstat (fd, &st) == -1) { + perror ("insmod: fstat"); + exit (EXIT_FAILURE); + } + size = st.st_size; + char buf[size]; + size_t offset = 0; + do { + ssize_t rc = read (fd, buf + offset, size - offset); + if (rc == -1) { + perror ("insmod: read"); + exit (EXIT_FAILURE); + } + offset += rc; + } while (offset < size); + close (fd); +#endif + + if (init_module (buf, size, "") != 0) { + fprintf (stderr, "insmod: init_module: %s: %s\n", filename, moderror (errno)); + /* However ignore the error because this can just happen because + * of a missing device. + */ + } + +#ifdef HAVE_ZLIB_STATIC + free (buf); +#endif +} + +/* Mount /proc unless it's mounted already. */ +static void +mount_proc (void) +{ + if (access ("/proc/uptime", R_OK) == -1) { + mkdir ("/proc", 0755); + + if (verbose) + fprintf (stderr, "supermin: mounting /proc\n"); + + if (mount ("proc", "/proc", "proc", 0, "") == -1) { + perror ("mount: /proc"); + /* Non-fatal. */ + } + } +} + +/* Print contents of /proc/uptime. */ +static void +print_uptime (void) +{ + FILE *fp = fopen ("/proc/uptime", "r"); + if (fp == NULL) { + perror ("/proc/uptime"); + return; + } + + fgets (line, sizeof line, fp); + fclose (fp); + + fprintf (stderr, "supermin: uptime: %s", line); +} + +/* Read /proc/cmdline into cmdline global (or at least the first 1024 + * bytes of it). + */ +static void +read_cmdline (void) +{ + FILE *fp = fopen ("/proc/cmdline", "r"); + if (fp == NULL) { + perror ("/proc/cmdline"); + return; + } + + fgets (cmdline, sizeof cmdline, fp); + fclose (fp); + + fprintf (stderr, "supermin: cmdline: %s", cmdline); +} + +/* Display a directory on stderr. This is used for debugging only. */ +static char +dirtype (int dt) +{ + switch (dt) { + case DT_BLK: return 'b'; + case DT_CHR: return 'c'; + case DT_DIR: return 'd'; + case DT_FIFO: return 'p'; + case DT_LNK: return 'l'; + case DT_REG: return '-'; + case DT_SOCK: return 's'; + case DT_UNKNOWN: return 'u'; + default: return '?'; + } +} + +static void +show_directory (const char *dirname) +{ + DIR *dir; + struct dirent *d; + struct stat statbuf; + char link[PATH_MAX+1]; + ssize_t n; + + fprintf (stderr, "supermin: debug: listing directory %s\n", dirname); + + if (chdir (dirname) == -1) { + perror (dirname); + return; + } + + dir = opendir ("."); + if (!dir) { + perror (dirname); + chdir ("/"); + return; + } + + while ((d = readdir (dir)) != NULL) { + fprintf (stderr, "%5lu %c %-16s", d->d_ino, dirtype (d->d_type), d->d_name); + if (lstat (d->d_name, &statbuf) >= 0) { + fprintf (stderr, " %06o %ld %d:%d", + statbuf.st_mode, statbuf.st_size, + statbuf.st_uid, statbuf.st_gid); + if (S_ISLNK (statbuf.st_mode)) { + n = readlink (d->d_name, link, PATH_MAX); + if (n >= 0) { + link[n] = '\0'; + fprintf (stderr, " -> %s", link); + } + } + } + fprintf (stderr, "\n"); + } + + closedir (dir); + chdir ("/"); +} diff --git a/src/kernel.ml b/src/kernel.ml new file mode 100644 index 0000000..412ca3c --- /dev/null +++ b/src/kernel.ml @@ -0,0 +1,238 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +open Unix +open Printf + +open Utils +open Ext2fs +open Fnmatch + +let rec build_kernel debug host_cpu dtb_wildcard copy_kernel kernel dtb = + try + let kernel_env = getenv "SUPERMIN_KERNEL" in + build_kernel_from_env debug host_cpu dtb_wildcard copy_kernel + kernel_env kernel dtb + with Not_found -> + let is_x86 = + String.length host_cpu = 4 && + host_cpu.[0] = 'i' && host_cpu.[2] = '8' && host_cpu.[3] = '6' in + let is_arm = + String.length host_cpu >= 3 && + host_cpu.[0] = 'a' && host_cpu.[1] = 'r' && host_cpu.[2] = 'm' in + + let all_files = Sys.readdir "/boot" in + let all_files = Array.to_list all_files in + + let filter_pattern patt = + let files = + List.filter + (fun filename -> fnmatch patt filename [FNM_NOESCAPE]) all_files in + let files = + List.filter (fun filename -> find filename "xen" = -1) files in + let files = + if not is_arm then files + else ( + List.filter (fun filename -> + find filename "lpae" = -1 && find filename "tegra" = -1 + ) files + ) in + List.filter (fun filename -> has_modpath filename) files + in + + (* In original: ls -1dvr /boot/vmlinuz-*.$arch* 2>/dev/null | grep -v xen *) + let patt = + if is_x86 then "vmlinuz-*.i?86*" else "vmlinuz-*." ^ host_cpu ^ "*" in + let files = filter_pattern patt in + + let files = + if files <> [] then files + else ( + (* In original: ls -1dvr /boot/vmlinuz-* 2>/dev/null | grep -v xen *) + let patt = "vmlinuz-*" in + filter_pattern patt + ) in + + if files = [] then no_kernels (); + + let files = List.sort (fun a b -> compare_version b a) files in + let kernel_name = List.hd files in + + if debug >= 1 then + printf "supermin: kernel: picked kernel %s\n%!" kernel_name; + + copy_or_symlink_file copy_kernel ("/boot" // kernel_name) kernel; + + (match dtb_wildcard with + | None -> () + | Some wildcard -> + (* Replace vmlinuz- with dtb- *) + if not (string_prefix "vmlinuz-" kernel_name) then + no_dtb_dir kernel_name; + let dtb_dir = + "/boot/dtb-" ^ + String.sub kernel_name 8 (String.length kernel_name - 8) in + if not (dir_exists dtb_dir) then + no_dtb_dir kernel_name; + + let all_files = Sys.readdir dtb_dir in + let all_files = Array.to_list all_files in + + let files = + List.filter + (fun filename -> fnmatch wildcard filename [FNM_NOESCAPE]) + all_files in + if files = [] then + no_dtb dtb_dir wildcard; + + let dtb_name = List.hd files in + + if debug >= 1 then + printf "supermin: kernel: picked dtb %s\n%!" dtb_name; + + copy_or_symlink_file copy_kernel (dtb_dir // dtb_name) dtb; + ); + + let kernel_version = get_kernel_version kernel_name in + let modpath = "/lib/modules/" ^ kernel_version in + if debug >= 1 then ( + printf "supermin: kernel: kernel_version %s\n" kernel_version; + printf "supermin: kernel: modules %s\n%!" modpath; + ); + (kernel_version, modpath) + +and no_kernels () = + eprintf "\ +supermin: failed to find a suitable kernel. + +I looked for kernels in /bool and modules in /lib/modules. + +If this is a Xen guest, and you only have Xen domU kernels +installed, try installing a fullvirt kernel (only for +supermin use, you shouldn't boot the Xen guest with it).\n"; + exit 1 + +and no_dtb_dir kernel_name = + eprintf "\ +supermin: failed to find a dtb (device tree) directory. + +I expected to take '%s' and to +replace vmlinuz- with dtb- to form a directory. + +You can set SUPERMIN_KERNEL, SUPERMIN_MODULES and SUPERMIN_DTB +to override automatic selection. See supermin(1).\n" + kernel_name; + exit 1 + +and no_dtb dtb_dir wildcard = + eprintf "\ +supermin: failed to find a matching device tree. + +I looked for a file matching '%s' in directory '%s'. + +You can set SUPERMIN_KERNEL, SUPERMIN_MODULES and SUPERMIN_DTB +to override automatic selection. See supermin(1).\n" + wildcard dtb_dir; + exit 1 + +and build_kernel_from_env debug host_cpu dtb_wildcard copy_kernel kernel_env + kernel dtb = + assert false + +and has_modpath kernel_name = + try + let kv = get_kernel_version kernel_name in + dir_exists ("/lib/modules/" ^ kv) + with + | Not_found -> false + +and get_kernel_version kernel_name = + if string_prefix "vmlinuz-" kernel_name then ( + let kv = String.sub kernel_name 8 (String.length kernel_name - 8) in + if dir_exists ("/lib/modules/" ^ kv) then kv + else get_kernel_version_from_kernel kernel_name + ) else get_kernel_version_from_kernel kernel_name + +(* Extract the kernel version from a Linux kernel file. + * + * Returns a string containing the version or [Not_found] if the + * file can't be read, is not a Linux kernel, or the version can't + * be found. + * + * See ftp://ftp.astron.com/pub/file/file-<ver>.tar.gz + * (file-<ver>/magic/Magdir/linux) for the rules used to find the + * version number: + * 514 string HdrS Linux kernel + * >518 leshort >0x1ff + * >>(526.s+0x200) string >\0 version %s, + * + * Bugs: probably limited to x86 kernels. + *) +and get_kernel_version_from_kernel kernel_name = + try + let chan = open_in ("/boot" // kernel_name) in + let buf = read_string chan 514 4 in + if buf <> "HdrS" then ( + close_in chan; + raise Not_found + ); + let s = read_leshort chan 518 in + if s < 0x1ff then ( + close_in chan; + raise Not_found + ); + let offset = read_leshort chan 526 in + if offset < 0 then ( + close_in chan; + raise Not_found + ); + let buf = read_string chan (offset + 0x200) 132 in + close_in chan; + let rec loop i = + if i < 132 then ( + if buf.[i] = '\000' || buf.[i] = ' ' && + buf.[i] = '\t' && buf.[i] = '\n' then + String.sub buf 0 i + else + loop (i+1) + ) + else raise Not_found + in + loop 0 + with + | Sys_error _ -> raise Not_found + | Invalid_argument _ -> raise Not_found + +(* Read an unsigned little endian short at a specified offset in a file. *) +and read_leshort chan offset = + let buf = read_string chan offset 2 in + (Char.code buf.[1] lsl 8) lor Char.code buf.[0] + +and read_string chan offset len = + seek_in chan offset; + let buf = String.create len in + really_input chan buf 0 len; + buf + +and copy_or_symlink_file copy_kernel src dest = + if not copy_kernel then + symlink src dest + else ( + let cmd = sprintf "cp -p %s %s" (quote src) (quote dest) in + run_command cmd + ) diff --git a/src/package_handler.ml b/src/package_handler.ml new file mode 100644 index 0000000..8c83f46 --- /dev/null +++ b/src/package_handler.ml @@ -0,0 +1,129 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +open Printf + +open Utils + +type package = int + +module PackageSet = Set.Make ( + struct + type t = package + let compare = compare + end +) + +let package_set_of_list pkgs = + List.fold_right PackageSet.add pkgs PackageSet.empty + +type settings = { + debug : int; + tmpdir : string; + packager_config : string option; +} + +let no_settings = + { debug = 0; tmpdir = "/nowhere"; packager_config = None; } + +type file = { + ft_path : string; + ft_config : bool; +} + +type package_handler = { + ph_detect : unit -> bool; + ph_init : settings -> unit; + ph_package_of_string : string -> package option; + ph_package_to_string : package -> string; + ph_package_name : package -> string; + ph_get_requires : package -> PackageSet.t; + ph_get_all_requires : PackageSet.t -> PackageSet.t; + ph_get_files : package -> file list; + ph_get_all_files : PackageSet.t -> file list; + ph_download_package : package -> string -> unit; + ph_download_all_packages : PackageSet.t -> string -> unit; +} + +(* Suggested memoization functions. *) +let get_memo_functions () = + let id = ref 0 in + let h1 = Hashtbl.create 13 and h2 = Hashtbl.create 13 in + let internal_of_pkg pkg = + try Hashtbl.find h1 pkg with Not_found -> assert false + in + let pkg_of_internal internal = + try Hashtbl.find h2 internal + with Not_found -> + let id = incr id; !id in + Hashtbl.add h2 internal id; + Hashtbl.add h1 id internal; + id + in + internal_of_pkg, pkg_of_internal + +let handlers = ref [] +let register_package_handler name ph = handlers := (name, ph) :: !handlers + +let handler = ref None + +let check_system settings = + try + let (_, ph) as h = List.find (fun (_, ph) -> ph.ph_detect ()) !handlers in + handler := Some h; + ph.ph_init settings + with Not_found -> + eprintf "\ +supermin: could not detect package manager used by this system or distro. + +If this is a new Linux distro, or not Linux, or a Linux distro that uses +an unusual packaging format then you may need to port supermin. If +you are expecting that supermin should work on this system or distro +then it may be that the package detection code is not working. +"; + exit 1 + +let rec get_package_handler () = + match !handler with + | Some (_, ph) -> ph + | None -> assert false + +let rec get_package_handler_name () = + match !handler with + | Some (name, _) -> name + | None -> assert false + +let default_get_all_requires pkgs = + let ph = get_package_handler () in + PackageSet.fold + (fun pkg -> PackageSet.union (ph.ph_get_requires pkg)) + pkgs PackageSet.empty + +let default_get_all_files pkgs = + let ph = get_package_handler () in + PackageSet.fold ( + fun pkg xs -> + let files = ph.ph_get_files pkg in + files @ xs + ) pkgs [] + +let default_download_all_packages pkgs dir = + let ph = get_package_handler () in + PackageSet.iter ( + fun pkg -> ph.ph_download_package pkg dir + ) pkgs diff --git a/src/package_handler.mli b/src/package_handler.mli new file mode 100644 index 0000000..7ed6272 --- /dev/null +++ b/src/package_handler.mli @@ -0,0 +1,157 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +(** {2 Package handlers.} *) + +type package = int + +module PackageSet : sig + type elt = package + type t + val empty : t + val is_empty : t -> bool + val mem : elt -> t -> bool + val add : elt -> t -> t + val singleton : elt -> t + val remove : elt -> t -> t + val union : t -> t -> t + val inter : t -> t -> t + val diff : t -> t -> t + val compare : t -> t -> int + val equal : t -> t -> bool + val subset : t -> t -> bool + val iter : (elt -> unit) -> t -> unit + val fold : (elt -> 'a -> 'a) -> t -> 'a -> 'a + val for_all : (elt -> bool) -> t -> bool + val exists : (elt -> bool) -> t -> bool + val filter : (elt -> bool) -> t -> t + val partition : (elt -> bool) -> t -> t * t + val cardinal : t -> int + val elements : t -> elt list + val min_elt : t -> elt + val max_elt : t -> elt + val choose : t -> elt + val split : elt -> t -> t * bool * t +end + +val package_set_of_list : package list -> PackageSet.t + +(** Package handler settings, passed to [ph_init] function. *) +type settings = { + debug : int; (** Debugging level (-v option). *) + tmpdir : string; + (** A scratch directory, where the package handler may write any + files or directories it needs. The directory exists already, so + does not need to be created. It is deleted automatically when + the program exits. *) + packager_config : string option; + (** The --packager-config command line option, if present. *) +} + +val no_settings : settings +(** An empty settings struct. *) + +(** Files (also directories and other filesystem objects) that are + part of a particular package. Note that the package is always + installed when we query it, so to find out things like the file + type, size and mode you just need to [lstat file.ft_path]. *) +type file = { + ft_path : string; + (** File path. *) + + ft_config : bool; + (** Flag to indicate this is a configuration file. In some package + managers (RPM) this is stored in package metadata. In others + (dpkg) we guess it based on the filename. *) +} + +(** Package handlers are modules that implement this structure and + call {!register_package_handler}. *) +type package_handler = { + ph_detect : unit -> bool; + (** The package handler should return true if the system uses this + package manager. *) + + ph_init : settings -> unit; + (** This is called when this package handler is chosen and + initializes. The [settings] parameter is a struct of general + settings and configuration. *) + + ph_package_of_string : string -> package option; + (** Convert a string (from user input) into a package object. If + the package is not installed or the string is otherwise + incorrect this returns [None]. *) + + ph_package_to_string : package -> string; + (** Convert package back to a printable string. {b Only} use this + for debugging and printing errors. Use {!ph_package_name} for a + reproducible name that can be written to packagelist. *) + + ph_package_name : package -> string; + (** Return the name of the package, for writing to packagelist. *) + + ph_get_requires : package -> PackageSet.t; + (** Given a single installed package, return the names of the + installed packages that are dependencies of this package. *) + + ph_get_all_requires : PackageSet.t -> PackageSet.t; + (** Given a list of installed packages, return the combined list of + names of installed packages which are dependencies. Package + handlers may override the default implementation (below) if they + have a more efficient way to implement it. *) + + ph_get_files : package -> file list; + (** Given a single package, list out the files in that package + (including package management metadata). *) + + ph_get_all_files : PackageSet.t -> file list; + (** Same as {!ph_get_files}, but return a combined list from all + packages in the set. Package handlers may override the default + implementation (below) if they have a more efficient way to + implement it. *) + + ph_download_package : package -> string -> unit; + (** [ph_download_package package dir] downloads the named package + from the repository, and unpack it in the given [dir]. + + When [--use-installed] option is used, this will not be called. *) + + ph_download_all_packages : PackageSet.t -> string -> unit; + (** [ph_download_all_packages packages dir] downloads all the + packages and unpacks them into a single directory tree. *) +} + +(** Defaults for some ph_* functions if the package handler does not + want to supply them. *) +val default_get_all_requires : PackageSet.t -> PackageSet.t +val default_get_all_files : PackageSet.t -> file list +val default_download_all_packages : PackageSet.t -> string -> unit + +(** Package handlers could use these memoization functions to convert + from the {!package} type to an internal struct and back again, or + they can implement their own. *) +val get_memo_functions : unit -> (package -> 'a) * ('a -> package) + +(** At program start-up, all package handlers register themselves here. *) +val register_package_handler : string -> package_handler -> unit + +val check_system : settings -> unit + +val get_package_handler : unit -> package_handler + +val get_package_handler_name : unit -> string diff --git a/src/prepare.ml b/src/prepare.ml new file mode 100644 index 0000000..9d03fc9 --- /dev/null +++ b/src/prepare.ml @@ -0,0 +1,160 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +open Printf + +open Package_handler +open Utils + +let prepare (copy_kernel, debug, dtb_wildcard, format, host_cpu, outputdir, + packager_config, tmpdir, use_installed) + inputs = + if debug >= 1 then + printf "supermin: prepare: %s\n%!" (String.concat " " inputs); + + if inputs = [] then ( + eprintf "supermin: prepare: no input packages specified\n"; + exit 1; + ); + + let ph = get_package_handler () in + + (* Resolve the package names supplied by the user. Since + * ph_package_of_string returns None if a package is not installed, + * filter_map will return only packages which are installed. + *) + let packages = filter_map ph.ph_package_of_string inputs in + if packages = [] then ( + eprintf "supermin: prepare: none of the packages listed on the command line seem to be installed\n"; + exit 1; + ); + + if debug >= 1 then ( + printf "supermin: packages specified on the command line:\n"; + List.iter (printf " - %s\n") (List.map ph.ph_package_to_string packages); + flush stdout + ); + + (* Resolve the dependencies. + * + * Need to add the original list of packages to the dependencies so + * that we have the complete list. + *) + let packages = + let packages = package_set_of_list packages in + let dependencies = ph.ph_get_all_requires packages in + PackageSet.union packages dependencies in + + if debug >= 1 then ( + printf "supermin: after resolving dependencies, the complete list of packages is:\n"; + PackageSet.iter + (fun pkg -> printf " - %s\n" (ph.ph_package_to_string pkg)) + packages; + flush stdout + ); + + (* Write packages. *) + let packages_file = outputdir // "packages" in + if debug >= 1 then printf "supermin: writing %s\n%!" packages_file; + let chan = open_out packages_file in + PackageSet.iter ( + fun pkg -> fprintf chan "%s\n" (ph.ph_package_name pkg) + ) packages; + close_out chan; + + (* List the files in each package. *) + let packages = + PackageSet.fold ( + fun pkg pkgs -> + let files = ph.ph_get_files pkg in + (pkg, files) :: pkgs + ) packages [] in + + if debug >= 2 then ( + List.iter ( + fun (pkg, files) -> + printf "supermin: files in '%s':\n" (ph.ph_package_to_string pkg); + List.iter + (fun { ft_path = path; ft_config = config } -> + printf " - %s%s\n" path (if config then " [config]" else "")) + files + ) packages; + flush stdout + ); + + let dir = + if not use_installed then ( + (* For packages that contain any config files, we have to download + * the original package, in order to construct the base image. We + * can skip packages that have no config files. + *) + let dir = tmpdir // "prepare.d" in + Unix.mkdir dir 0o755; + + let () = + let dl_packages = filter_map ( + fun (pkg, files) -> + let has_config_files = + List.exists (fun { ft_config = config } -> config) files in + if has_config_files then Some pkg else None + ) packages in + let dl_packages = package_set_of_list dl_packages in + ph.ph_download_all_packages dl_packages dir in + + dir + ) + else (* --use-installed *) "/" in + + (* Get the list of config files, which are the files we will place + * into base. We have to check the files exist too, since they can + * be missing either from the package or from the filesystem (the + * latter case with --use-installed). + *) + let files_from = + let config_files = + List.map ( + fun (_, files) -> + filter_map ( + function + | { ft_config = true; ft_path = path } -> Some path + | { ft_config = false } -> None + ) files + ) packages in + let config_files = List.flatten config_files in + + let config_files = List.filter ( + fun path -> + try close_in (open_in (dir // path)); true + with Sys_error _ -> false + ) config_files in + + (* Put the list of config files into a file, for tar to read. *) + let files_from = tmpdir // "files-from.txt" in + let chan = open_out files_from in + List.iter (fprintf chan ".%s\n") config_files; (* "./filename" *) + close_out chan; + + files_from in + + (* Write base.tar.gz. *) + let base = outputdir // "base.tar.gz" in + if debug >= 1 then printf "supermin: writing %s\n%!" base; + let cmd = + sprintf "tar -C %s -zcf %s -T %s" + (quote dir) (quote base) (quote files_from) in + run_command cmd; diff --git a/src/realpath-c.c b/src/realpath-c.c new file mode 100644 index 0000000..3713c7e --- /dev/null +++ b/src/realpath-c.c @@ -0,0 +1,44 @@ +/* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <config.h> + +#include <stdlib.h> +#include <limits.h> +#include <errno.h> + +#include <caml/alloc.h> +#include <caml/memory.h> +#include <caml/mlvalues.h> +#include <caml/unixsupport.h> + +value +supermin_realpath (value pathv) +{ + CAMLparam1 (pathv); + CAMLlocal1 (rv); + char *r; + + r = realpath (String_val (pathv), NULL); + if (r == NULL) + unix_error (errno, (char *) "realpath", pathv); + + rv = caml_copy_string (r); + free (r); + CAMLreturn (rv); +} diff --git a/src/realpath.ml b/src/realpath.ml new file mode 100644 index 0000000..a9a5ed0 --- /dev/null +++ b/src/realpath.ml @@ -0,0 +1,19 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +external realpath : string -> string = "supermin_realpath" diff --git a/src/realpath.mli b/src/realpath.mli new file mode 100644 index 0000000..309afe2 --- /dev/null +++ b/src/realpath.mli @@ -0,0 +1,19 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +val realpath : string -> string diff --git a/src/rpm.ml b/src/rpm.ml new file mode 100644 index 0000000..51fec08 --- /dev/null +++ b/src/rpm.ml @@ -0,0 +1,192 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +open Printf + +open Utils +open Package_handler + +let rpm_detect () = + Config.rpm <> "no" && + Config.yumdownloader <> "no" && + (file_exists "/etc/redhat-release" || + file_exists "/etc/fedora-release") + +let settings = ref no_settings + +let rpm_init s = settings := s + +type rpm_t = { + name : string; + epoch : int32; + version : string; + release : string; + arch : string; +} + +(* Memo from package type to internal rpm_t. *) +let rpm_of_pkg, pkg_of_rpm = get_memo_functions () + +let rpm_package_of_string str = + (* Parse an RPM name into the fields like name and version. Since + * the package is installed (see check below), it's easier to use RPM + * itself to do this parsing rather than haphazardly parsing it + * ourselves. *) + let parse_rpm str = + let cmd = + sprintf "rpm -q --qf '%%{name} %%{epoch} %%{version} %%{release} %%{arch}\\n' %s" + (quote str) in + let lines = run_command_get_lines cmd in + let lines = List.map (string_split " ") lines in + let rpms = filter_map ( + function + | [ name; ("0"|"(none)"); version; release; arch ] -> + Some { name = name; + epoch = 0_l; + version = version; release = release; arch = arch } + | [ name; epoch; version; release; arch ] -> + Some { name = name; + epoch = Int32.of_string epoch; + version = version; release = release; arch = arch } + | xs -> + (* grrr, RPM doesn't send errors to stderr *) + None + ) lines in + + if rpms = [] then ( + eprintf "supermin: no output from rpm command could be parsed when searching for '%s'\nThe command was:\n %s\n" + str cmd; + exit 1 + ); + + (* RPM will return multiple hits when either multiple versions or + * multiple arches are installed at the same time. We are only + * interested in the highest version with the best + * architecture. + *) + let cmp { version = v1; arch = a1 } { version = v2; arch = a2 } = + let i = compare_version v2 v1 in + if i <> 0 then i + else compare_architecture a2 a1 + in + let rpms = List.sort cmp rpms in + List.hd rpms + + (* Check if an RPM is installed. *) + and check_rpm_installed name = + let cmd = sprintf "rpm -q %s >/dev/null" (quote name) in + 0 = Sys.command cmd + in + + if check_rpm_installed str then ( + let rpm = parse_rpm str in + Some (pkg_of_rpm rpm) + ) + else None + +let rpm_package_to_string pkg = + let rpm = rpm_of_pkg pkg in + if rpm.epoch = 0_l then + sprintf "%s-%s-%s.%s" rpm.name rpm.version rpm.release rpm.arch + else + sprintf "%s-%ld:%s-%s.%s" + rpm.name rpm.epoch rpm.version rpm.release rpm.arch + +let rpm_package_name pkg = + let rpm = rpm_of_pkg pkg in + rpm.name + +let quoted_names pkgs = + let pkgs = PackageSet.elements pkgs in + let pkgs = List.map rpm_package_to_string pkgs in + quoted_list pkgs + +let rpm_get_all_requires pkgs = + let cmd = sprintf "\ +rpm -qR %s | +awk '{print $1}' | +xargs rpm -q --qf '%%{name}\\n' --whatprovides | +grep -v 'no package provides' | +sort -u" (quoted_names pkgs) in + let lines = run_command_get_lines cmd in + let lines = filter_map rpm_package_of_string lines in + package_set_of_list lines + +let rpm_get_requires pkg = rpm_get_all_requires (PackageSet.singleton pkg) + +let rpm_get_all_files pkgs = + let cmd = sprintf "\ +rpm -q --qf '[%%{FILENAMES} %%{FILEFLAGS:fflags}\\n]' %s | +grep '^/' | +sort -u" (quoted_names pkgs) in + let lines = run_command_get_lines cmd in + let lines = List.map (string_split " ") lines in + List.map ( + function + | [ path; flags ] -> + let config = String.contains flags 'c' in + { ft_path = path; ft_config = config } + | _ -> assert false + ) lines + +let rpm_get_files pkg = rpm_get_all_files (PackageSet.singleton pkg) + +let rpm_download_all_packages pkgs dir = + let cmd = + sprintf "%s%s%s --destdir %s %s" + Config.yumdownloader + (if !settings.debug >= 1 then "" else " --quiet") + (match !settings.packager_config with + | None -> "" + | Some filename -> sprintf " -c %s" (quote filename)) + (quote !settings.tmpdir) + (quoted_names pkgs) in + run_command cmd; + + (* Unpack each downloaded package. *) + PackageSet.iter ( + fun pkg -> + let rpm = rpm_of_pkg pkg in + (* yumdownloader doesn't include epoch in the filename. *) + let filename = + sprintf "%s/%s-%s-%s.%s.rpm" !settings.tmpdir rpm.name rpm.version + rpm.release rpm.arch in + let cmd = + sprintf "umask 0000; rpm2cpio %s | (cd %s && cpio --quiet -id)" + (quote filename) (quote dir) in + run_command cmd + ) pkgs + +let rpm_download_package pkg dir = + rpm_download_all_packages (PackageSet.singleton pkg) dir + +let () = + let ph = { + ph_detect = rpm_detect; + ph_init = rpm_init; + ph_package_of_string = rpm_package_of_string; + ph_package_to_string = rpm_package_to_string; + ph_package_name = rpm_package_name; + ph_get_requires = rpm_get_requires; + ph_get_all_requires = rpm_get_all_requires; + ph_get_files = rpm_get_files; + ph_get_all_files = rpm_get_all_files; + ph_download_package = rpm_download_package; + ph_download_all_packages = rpm_download_all_packages; + } in + register_package_handler "rpm" ph diff --git a/src/supermin.ml b/src/supermin.ml index 995f9d8..240abd9 100644 --- a/src/supermin.ml +++ b/src/supermin.ml @@ -1,5 +1,5 @@ -(* supermin 4 - * Copyright (C) 2009-2013 Red Hat Inc. +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,425 +19,169 @@ open Unix open Printf -open Supermin_package_handlers -open Supermin_utils -open Supermin_cmdline +open Types +open Prepare +open Build +open Package_handler -(* Create a temporary directory for use by all the functions in this file. *) -let tmpdir = tmpdir () +type mode = Prepare | Build + +let usage_msg = "\ +supermin - tool for creating supermin appliances +Copyright (C) 2009-2014 Red Hat Inc. + +Usage: + + supermin --prepare LIST OF PACKAGES ... + supermin --build INPUT [INPUT ...] + +For full instructions, read the supermin(1) man page. + +Options: +" + +let main () = + (* Make sure that all the subcommands that we run are printing + * messages in English. Certain package handlers (cough RPM) rely on + * this. + *) + putenv "LANG" "C"; + + (* Create a temporary directory for scratch storage. *) + let tmpdir = + let tmpdir = Filename.temp_file "supermin" ".tmpdir" in + unlink tmpdir; + mkdir tmpdir 0o700; + at_exit + (fun () -> + ignore (Sys.command (sprintf "rm -rf %s" (Filename.quote tmpdir)))); + tmpdir in + + let mode, inputs, args = + let display_version () = + printf "supermin %s\n" Config.package_version; + exit 0 + in + + let add xs s = xs := s :: !xs in + + let copy_kernel = ref false in + let debug = ref 0 in + let dtb_wildcard = ref "" in + let format = ref None in + let host_cpu = ref Config.host_cpu in + let mode = ref None in + let outputdir = ref "." in + let packager_config = ref "" in + let use_installed = ref false in + + let set_debug () = incr debug in + + let set_format = function + | "chroot" | "fs" | "filesystem" -> format := Some Chroot + | "ext2" -> format := Some Ext2 + | s -> + eprintf "supermin: unknown --format option (%s)\n" s; + exit 1 + in + + let rec set_prepare_mode () = + if !mode <> None then + bad_mode (); + mode := Some Prepare + and set_build_mode () = + if !mode <> None then + bad_mode (); + mode := Some Build + and bad_mode () = + eprintf "supermin: you must use --prepare or --build to select the mode\n"; + exit 1 + in + + let ditto = " -\"-" in + let argspec = Arg.align [ + "--build", Arg.Unit set_build_mode, " Build a full appliance"; + "--copy-kernel", Arg.Set copy_kernel, " Copy kernel instead of symlinking"; + "--dtb", Arg.Set_string dtb_wildcard, "WILDCARD Find device tree matching wildcard"; + "--format", Arg.String set_format, "chroot|ext2 Set output format"; + "--host-cpu", Arg.Set_string host_cpu, "ARCH Set host CPU architecture"; + "-o", Arg.Set_string outputdir, "OUTPUTDIR Set output directory"; + "--packager-config", Arg.Set_string packager_config, "CONFIGFILE Set packager config file"; + "--prepare", Arg.Unit set_prepare_mode, " Prepare a supermin appliance"; + "--use-installed", Arg.Set use_installed, " Use installed files instead of accessing network"; + "-v", Arg.Unit set_debug, " Enable debugging messages"; + "--verbose", Arg.Unit set_debug, ditto; + "-V", Arg.Unit display_version, " Display version and exit"; + "--version", Arg.Unit display_version, ditto; + ] in + let inputs = ref [] in + let anon_fun = add inputs in + Arg.parse argspec anon_fun usage_msg; + + let copy_kernel = !copy_kernel in + let debug = !debug in + let dtb_wildcard = match !dtb_wildcard with "" -> None | s -> Some s in + let host_cpu = !host_cpu in + let inputs = List.rev !inputs in + let mode = match !mode with Some x -> x | None -> bad_mode (); Prepare in + let outputdir = !outputdir in + let packager_config = + match !packager_config with "" -> None | s -> Some s in + let use_installed = !use_installed in + + let format = + match mode, !format with + | Prepare, Some _ -> + eprintf "supermin: cannot use --prepare and --format options together\n"; + exit 1 + | Prepare, None -> Chroot (* doesn't matter, prepare doesn't use this *) + | Build, None -> + eprintf "supermin: when using --build, you must specify an output --format\n"; + exit 1 + | Build, Some f -> f in + + mode, inputs, + (copy_kernel, debug, dtb_wildcard, format, host_cpu, outputdir, + packager_config, tmpdir, use_installed) in + + (* Try to find out which package management system we're using. + * This fails with an error if one could not be located. + *) + let () = + let (_, debug, _, _, _, _, packager_config, tmpdir, _) = args in + let settings = { + debug = debug; + tmpdir = tmpdir; + packager_config = packager_config; + } in + check_system settings in + + match mode with + | Prepare -> prepare args inputs + | Build -> build args inputs let () = - debug "%s %s" Config.package_name Config.package_version; - - (* Instead of printing out warnings as we go along, accumulate them - * in lists and print them all out at the end. - *) - let warn_unreadable = ref [] in - - (* Determine which package manager this system uses. *) - check_system (); - let ph = get_package_handler () in - - debug "selected package handler: %s" (get_package_handler_name ()); - - (* Not --names: check files exist. *) - if mode = PkgFiles then ( - List.iter ( - fun pkg -> - if not (file_exists pkg) then ( - eprintf "supermin: %s: no such file (did you miss out the --names option?)\n" pkg; - exit 1 - ) - ) packages - ); - - (* --names: resolve the package list to a full list of package names - * (including dependencies). - *) - let packages = - if mode <> PkgFiles then ( - let packages = ph.ph_resolve_dependencies_and_download packages mode in - debug "resolved packages: %s" (String.concat " " packages); - packages - ) - else packages in - - (* Get the list of files. *) - let files = - List.flatten ( - List.map ( - fun pkg -> - let files = ph.ph_list_files pkg in - List.map (fun (filename, ft) -> filename, ft, pkg) files - ) packages - ) in - - (* Canonicalize the name of directories, so that /a and /a/ are the same. *) - let files = - List.map ( - fun (filename, ft, pkg) -> - let len = String.length filename in - let filename = - if len > 1 (* don't rewrite "/" *) && ft.ft_dir - && filename.[len-1] = '/' then - String.sub filename 0 (len-1) - else - filename in - (filename, ft, pkg) - ) files in - - (* Sort and combine duplicate files. *) - let files = - let files = List.sort compare files in - - let combine (name1, ft1, pkg1) (name2, ft2, pkg2) = - (* Rules for combining files. *) - if ft1.ft_config || ft2.ft_config then ( - (* It's a fairly frequent bug in Fedora for two packages to - * incorrectly list the same config file. Allow this, provided - * the size of both files is 0; or one is a ghost file and the - * other is not. - *) - if ft1.ft_size = 0 && ft2.ft_size = 0 then - (name1, ft1, pkg1) - else if not ft1.ft_ghost && ft2.ft_ghost then - (name1, ft1, pkg1) - else if ft1.ft_ghost && not ft2.ft_ghost then - (name2, ft2, pkg2) - else ( - eprintf "supermin: error: %s is a config file which is listed in two packages (%s, %s)\n" - name1 pkg1 pkg2; - exit 1 - ) - ) - else if (ft1.ft_dir || ft2.ft_dir) && (not (ft1.ft_dir && ft2.ft_dir)) then ( - eprintf "supermin: error: %s appears as both directory and ordinary file (%s, %s)\n" - name1 pkg1 pkg2; - exit 1 - ) - else if ft1.ft_ghost then - (name2, ft2, pkg2) - else - (name1, ft1, pkg1) - in - - let rec loop = function - | [] -> [] - | (name1, _, _ as f1) :: (name2, _, _ as f2) :: fs when name1 = name2 -> - let f = combine f1 f2 in loop (f :: fs) - | f :: fs -> f :: loop fs - in - loop files in - - (* Ignore %ghost non-regular files. RPMs in Fedora 20 contain these. - * It's not clear what they are meant to signify. XXX - *) - let files = List.filter ( - function - | name, { ft_dir = false; ft_ghost = true; ft_mode = mode }, pkg -> - if (mode land 0o170_000) = 0o100_000 then - true - else ( - debug "ignoring ghost non-regular file %s (mode 0%o) from package %s" - name mode pkg; - false - ) - | _ -> true - ) files in - - (* Because we may have excluded some packages, and also because of - * distribution packaging errors, it's not necessarily true that a - * directory is created before each file in that directory. - * Determine those missing directories and add them now. - *) - let files = - let insert_dir, dir_seen = - let h = Hashtbl.create (List.length files) in - let insert_dir dir = Hashtbl.replace h dir true in - let dir_seen dir = Hashtbl.mem h dir in - insert_dir, dir_seen - in - let files = - List.map ( - fun (path, { ft_dir = is_dir }, _ as f) -> - if is_dir then - insert_dir path; - - let rec loop path = - let parent = Filename.dirname path in - if dir_seen parent then [] - else ( - insert_dir parent; - let newdir = (parent, { ft_dir = true; ft_config = false; - ft_ghost = false; ft_mode = 0o40755; - ft_size = 0 }, - "") in - newdir :: loop parent - ) - in - List.rev (f :: loop path) - ) files in - List.flatten files in - - (* Debugging. *) - debug "%d files and directories" (List.length files); - if false then ( - List.iter ( - fun (name, { ft_dir = dir; ft_ghost = ghost; ft_config = config; - ft_mode = mode; ft_size = size }, pkg) -> - printf "%s [%s%s%s%o %d] from %s\n" name - (if dir then "dir " else "") - (if ghost then "ghost " else "") - (if config then "config " else "") - mode size - pkg - ) files - ); - - (* Split the list of files into ones for hostfiles and ones for base image. *) - let p_hmac = Str.regexp "^\\..*\\.hmac$" in - - let hostfiles = ref [] - and baseimgfiles = ref [] in - List.iter ( - fun (path, {ft_dir = dir; ft_ghost = ghost; ft_config = config} ,_ as f) -> - let file = Filename.basename path in - - (* Ignore boot files, kernel, kernel modules. Supermin appliances - * are booted from external kernel and initrd, and - * supermin-helper copies the host kernel modules. - * Note we want to keep the /boot and /lib/modules directory entries. - *) - if string_prefix "/boot/" path then () - else if string_prefix "/lib/modules/" path then () - - (* Always write directory names to both output files. *) - else if dir then ( - hostfiles := f :: !hostfiles; - baseimgfiles := f :: !baseimgfiles; - ) - - (* Timezone configuration is config, but copy it from host system. *) - else if path = "/etc/localtime" then - hostfiles := f :: !hostfiles - - (* Ignore FIPS files (.*.hmac) (RHBZ#654638). *) - else if Str.string_match p_hmac file 0 then () - - (* Ghost files are created empty in the base image. *) - else if ghost then - baseimgfiles := f :: !baseimgfiles - - (* For config files we can't rely on the host-installed copy - * since the admin may have modified then. We have to get the - * original file from the package and put it in the base image. - *) - else if config then - baseimgfiles := f :: !baseimgfiles - - (* Anything else comes from the host. *) - else - hostfiles := f :: !hostfiles - ) files; - let hostfiles = List.rev !hostfiles - and baseimgfiles = List.rev !baseimgfiles in - - (* Write hostfiles. *) - - (* Regexps used below. *) - let p_ld_so = Str.regexp "^ld-[.0-9]+\\.so$" in - let p_libbfd = Str.regexp "^libbfd-.*\\.so$" in - let p_libgcc = Str.regexp "^libgcc_s-.*\\.so\\.\$[0-9]+\$$" in - let p_libntfs3g = Str.regexp "^libntfs-3g\\.so\\..*$" in - let p_lib123so = Str.regexp "^lib\$.*\$-[-.0-9]+\\.so$" in - let p_lib123so123 = - Str.regexp "^lib\$.*\$-[-.0-9]+\\.so\\.\$[0-9]+\$\\." in - let p_libso123 = Str.regexp "^lib\$.*\$\\.so\\.\$[0-9]+\$\\." in - let ntfs3g_once = ref false in - - let chan = open_out (tmpdir // "hostfiles") in - List.iter ( - fun (path, {ft_dir = is_dir; ft_ghost = ghost; ft_config = config; - ft_mode = mode }, _) -> - let dir = Filename.dirname path in - let file = Filename.basename path in - - if is_dir then - fprintf chan "%s\n" path - - (* Warn about hostfiles which are unreadable by non-root. We - * won't be able to add those to the appliance at run time, but - * there's not much else we can do about it except get the - * distros to fix this nonsense. - *) - else if mode land 0o004 = 0 then - warn_unreadable := path :: !warn_unreadable - - (* Replace fixed numbers in some library names by wildcards. *) - else if Str.string_match p_ld_so file 0 then - fprintf chan "%s/ld-*.so\n" dir - - (* Special case for libbfd. *) - else if Str.string_match p_libbfd file 0 then - fprintf chan "%s/libbfd-*.so\n" dir - - (* Special case for libgcc_s-<gccversion>-<date>.so.N *) - else if Str.string_match p_libgcc file 0 then - fprintf chan "%s/libgcc_s-*.so.%s\n" dir (Str.matched_group 1 file) - - (* Special case for libntfs-3g.so.* *) - else if Str.string_match p_libntfs3g file 0 then ( - if not !ntfs3g_once then ( - fprintf chan "%s/libntfs-3g.so.*\n" dir; - ntfs3g_once := true - ) - ) - - (* libfoo-1.2.3.so *) - else if Str.string_match p_lib123so file 0 then - fprintf chan "%s/lib%s-*.so\n" dir (Str.matched_group 1 file) - - (* libfoo-1.2.3.so.123 (but NOT '*.so.N') *) - else if Str.string_match p_lib123so123 file 0 then - fprintf chan "%s/lib%s-*.so.%s.*\n" dir - (Str.matched_group 1 file) (Str.matched_group 2 file) - - (* libfoo.so.1.2.3 (but NOT '*.so.N') *) - else if Str.string_match p_libso123 file 0 then - fprintf chan "%s/lib%s.so.%s.*\n" dir - (Str.matched_group 1 file) (Str.matched_group 2 file) - - (* Anything else comes from the host. *) - else - fprintf chan "%s\n" path - ) hostfiles; - close_out chan; - - (* Write base.img. - * - * We have to create directories and copy files to tmpdir/root - * and then call out to cpio to construct the initrd. - *) - let rootdir = tmpdir // "root" in - mkdir rootdir 0o755; - List.iter ( - fun (path, { ft_dir = is_dir; ft_ghost = ghost; ft_config = config; - ft_mode = mode }, pkg) -> - (* Always write directory names to both output files. *) - if is_dir then ( - (* Directory permissions are fixed up below. *) - if path <> "/" then mkdir (rootdir // path) 0o755 - ) - - (* Ghost files are just touched with the correct perms. *) - else if ghost then ( - let chan = open_out (rootdir // path) in - close_out chan; - chmod (rootdir // path) (mode land 0o777 lor 0o400) - ) - - (* For config files we can't rely on the host-installed copy - * since the admin may have modified it. We have to get the - * original file from the package. - *) - else if config then ( - let outfile = ph.ph_get_file_from_package pkg path in - - (* Note that the output config file might not be a regular file. *) - let statbuf = lstat outfile in - - let destfile = rootdir // path in - - (* Depending on the file type, copy it to destination. *) - match statbuf.st_kind with - | S_REG -> - (* Unreadable files (eg. /etc/gshadow). Make readable. *) - if statbuf.st_perm = 0 then chmod outfile 0o400; - let cmd = - sprintf "cp %s %s" - (Filename.quote outfile) (Filename.quote destfile) in - run_command cmd; - chmod destfile (mode land 0o777 lor 0o400) - | S_LNK -> - let link = readlink outfile in - symlink link destfile - | S_DIR -> assert false - | S_CHR - | S_BLK - | S_FIFO - | S_SOCK -> - eprintf "supermin: error: %s: don't know how to handle this type of file\n" path; - exit 1 - ) - - else - assert false (* should not be reached *) - ) baseimgfiles; - - (* Fix up directory permissions, in reverse order. Since we don't - * want to have a read-only directory that we can't write into above. - *) - List.iter ( - fun (path, { ft_dir = is_dir; ft_mode = mode }, _) -> - if is_dir then chmod (rootdir // path) (mode land 0o3777 lor 0o700) - ) (List.rev baseimgfiles); - - (* Construct the 'base.img' initramfs. Feed in the list of filenames - * partly because we conveniently have them, and partly because - * this results in a nice alphabetical ordering in the cpio file. - *) - (*let cmd = sprintf "ls -lR %s" rootdir in - ignore (Sys.command cmd);*) - let cmd = - sprintf "(cd %s && cpio --quiet -o -0 -H newc) > %s" - rootdir (tmpdir // "base.img") in - let chan = open_process_out cmd in - List.iter (fun (path, _, _) -> fprintf chan ".%s\000" path) baseimgfiles; - let stat = close_process_out chan in - (match stat with - | WEXITED 0 -> () - | WEXITED i -> - eprintf "supermin: command '%s' failed (returned %d), see earlier error messages\n" cmd i; - exit i - | WSIGNALED i -> - eprintf "supermin: command '%s' killed by signal %d" cmd i; - exit 1 - | WSTOPPED i -> - eprintf "supermin: command '%s' stopped by signal %d" cmd i; - exit 1 - ); - - (* Undo directory permissions, because rm -rf can't delete files in - * unreadable directories. - *) - List.iter ( - fun (path, { ft_dir = is_dir; ft_mode = mode }, _) -> - if is_dir then chmod (rootdir // path) 0o755 - ) (List.rev baseimgfiles); - - (* Print warnings. *) - if warnings then ( - (match !warn_unreadable with - | [] -> () - | paths -> - eprintf "supermin: warning: some host files are unreadable by non-root\n"; - eprintf "supermin: warning: get your distro to fix these files:\n"; - List.iter - (fun path -> eprintf "\t%s\n%!" path) - (List.sort compare paths) - ); - ); - - (* Near-atomically copy files to the final output directory. *) - debug "writing %s ..." (outputdir // "base.img"); - let cmd = - sprintf "mv %s %s" - (Filename.quote (tmpdir // "base.img")) - (Filename.quote (outputdir // "base.img")) in - run_command cmd; - debug "writing %s ..." (outputdir // "hostfiles"); - let cmd = - sprintf "mv %s %s" - (Filename.quote (tmpdir // "hostfiles")) - (Filename.quote (outputdir // "hostfiles")) in - run_command cmd + try main () + with + | Unix.Unix_error (code, fname, "") -> (* from a syscall *) + eprintf "supermin: error: %s: %s\n" fname (Unix.error_message code); + exit 1 + | Unix.Unix_error (code, fname, param) -> (* from a syscall *) + eprintf "supermin: error: %s: %s: %s\n" fname (Unix.error_message code) + param; + exit 1 + | Failure msg -> (* from failwith/failwithf *) + eprintf "supermin: failure: %s\n" msg; + exit 1 + | Invalid_argument msg -> (* probably should never happen *) + eprintf "supermin: internal error: invalid argument: %s\n" msg; + exit 1 + | Assert_failure (file, line, char) -> (* should never happen *) + eprintf "supermin: internal error: assertion failed at %s, line %d, char %d\n" file line char; + exit 1 + | Not_found -> (* should never happen *) + eprintf "supermin: internal error: Not_found exception was thrown\n"; + exit 1 + | exn -> (* something not matched above *) + eprintf "supermin: exception: %s\n" (Printexc.to_string exn); + exit 1 diff --git a/src/supermin.pod b/src/supermin.pod index 395d626..c38ee5d 100644 --- a/src/supermin.pod +++ b/src/supermin.pod @@ -2,57 +2,77 @@ =head1 NAME -supermin - Tool for creating supermin appliances +supermin - Tool for creating and building supermin appliances =head1 SYNOPSIS - supermin [-o OUTPUTDIR] --names LIST OF PKGS ... - supermin [-o OUTPUTDIR] PKG FILE NAMES ... + supermin --prepare [-o OUTPUTDIR] PACKAGE [PACKAGE ...] + + supermin --build --format chroot|ext2 INPUT [INPUT ...] =head1 DESCRIPTION -Supermin is a tool for building supermin appliances. These are -tiny appliances (similar to virtual machines), usually around 100KB in +Supermin is a tool for building supermin appliances. These are tiny +appliances (similar to virtual machines), usually around 100KB in size, which get fully instantiated on-the-fly in a fraction of a second when you need to boot one of them. -Originally "fe" in "febootstrap" stood for "Fedora", but this tool is -now distro-independent and can build supermin appliances for several -popular Linux distros, and adding support for others is reasonably -easy. For this reason, starting with version 4, we have renamed the -tool "supermin". - -Note that this manual page documents supermin 4.x which is a complete -rewrite and quite different from febootstrap 2.x. If you are looking -for the febootstrap 2.x tools, then this is not the right place. +This program used to be called febootstrap. This manual page +documents supermin 5.x which is a complete rewrite and quite different +from febootstrap 2.x. If you are looking for the febootstrap 2.x +tools, then this is not the right place. =head2 BASIC OPERATION -There are two modes for using supermin. With the I<--names> -parameter, supermin takes a list of package names and creates a -supermin appliance containing those packages and all dependencies that -those packages require. In this mode supermin usually needs -network access because it may need to consult package repositories in -order to work out dependencies and download packages. - -Without I<--names>, supermin takes a list of packages (ie. -filenames of locally available packages). This package set must be -complete and consistent with no dependencies outside the set of -packages you provide. In this mode supermin does not require any -network access. It works by looking at the package files themselves. - -By "package" we mean the RPM, DEB, (etc.) package. A package name -might be the fully qualified name (eg. C<coreutils-8.5-7.fc14.x86_64>) -or some abbreviation (eg. C<coreutils>). The precise format of the -name and what abbreviations are allowed depends on the package -manager. - -The supermin appliance that supermin writes consists of two files -called C<hostfiles> and C<base.img> (see L</SUPERMIN APPLIANCES> -below). By default these are written to the current directory. If -you specify the I<-o OUTPUTDIR> option then these files are written to -the named directory instead (traditionally this directory is named -C<supermin.d> but you can call it whatever you want). +The supermin tool can be used in two modes, B<preparing> a tiny +supermin appliance, which is done on a build system. And B<building>, +which takes the supermin appliance and constructs a full, bootable +appliance, which is done on the end user's system. + +Supermin does not need to be run as root, and generally I<should not> +be run as root. It does not affect the host system or the packages +installed on the host system. + +=head3 PREPARE MODE + +I<--prepare> creates the tiny supermin appliance in the given output +directory. You give it a list of packages that you want installed, +and supermin will automatically find the dependencies. The list of +packages has to be installed on the host machine. + +For example: + + mkdir supermin.d + supermin --prepare bash coreutils -o supermin.d + +creates a supermin appliance containing the packages C<bash> and +C<coreutils>. Specifically, it creates some files in directory +C<supermin.d>. This directory I<is> the supermin appliance. (See +L</SUPERMIN APPLIANCES> below). + +It is intended that the I<--prepare> step is done on a central build +machine, and the supermin appliance is distributed to end users (which +is easy because supermin appliances are so small). + +=head3 BUILD MODE + +I<--build> (previously a separate program called C<supermin-helper>) +builds the full appliance from the supermin appliance: + + supermin --build --format ext2 supermin.d + +This will create files called C<kernel>, C<root> etc in the current +directory, which is the full sized bootable appliance. + +It is intended that the I<--build> step is done on the end user's +machine at the last second before the appliance is needed. The +packages in the supermin appliance (those specified when the supermin +appliance was prepared) must be installed on the end user's machine. + +=head3 PACKAGES + +By "package" we mean the RPM, Debian, (etc.) package, +eg. C<coreutils>, C<perl>. In all cases supermin can only build a supermin appliance which is identical in distro, version and architecture to the host. It does @@ -66,32 +86,118 @@ I<not> do cross-builds. Display brief command line usage, and exit. -=item B<--exclude REGEXP> +=item B<--build> -After doing dependency resolution, exclude packages which match the -regular expression. +Build the full appliance from the supermin appliance. This used to be +a separate program called C<supermin-helper>. -This option is only used with I<--names>, and it can be given multiple -times on the command line. +=item B<--copy-kernel> -=item B<--names> +(I<--build> mode only) -Provide a list of package names, instead of providing packages -directly. In this mode supermin may require network access. See -L</BASIC OPERATION> above. +Copy the kernel (and device tree, if created) instead of symlinking to +the kernel in C</boot>. -=item B<--no-warnings> +This is fractionally slower, but is necessary if you want to change +the permissions or SELinux label on the kernel or device tree. -Don't print warnings about packaging problems. +=item B<--dtb> WILDCARD -=item B<-o outputdir> +(I<--build> mode only) -Select the output directory where the two supermin appliance files are -written (C<hostfiles> and C<base.img>). The default directory is the -current directory. Note that if this files exist already in the -output directory then they will be overwritten. +If specified, search for a device tree which is compatible with the +selected kernel and the name of which matches the given wildcard. You +can use a wildcard such as C<vexpress-*a9*.dtb> which would match +C<vexpress-v2p-ca9.dtb>. -=item B<--packager-config CONFIGFILE> +Notes: + +=over 4 + +=item * + +You may need to quote the wildcard to prevent it from being expanded +by your shell. + +=item * + +If no I<--dtb> option is given, no device tree will be looked for. + +=item * + +You only need a device tree on architectures such as ARM and PowerPC +which use them. On other architectures, don't use this option. + +=item * + +If you use this option and no compatible device tree can be found, +supermin will exit with an error. + +=back + +=item B<-f> FORMAT + +=item B<--format> FORMAT + +(I<--build> mode only) + +Select the output format for the full appliance. + +There is no default. When using I<--build> you must specify the +I<--format> option. + +Possible formats are: + +=over 4 + +=item chroot + +A directory tree in the host filesystem. + +The filesystem tree is written to C<OUTPUTDIR> (ie. the I<-o> option). + +This is called a C<chroot> because you could literally L<chroot(1)> +into this directory afterwards, although it's a better idea to use a +container technology (LXC, etc.). + +No kernel, initrd or dtb is generated in this mode because it is +assumed that you will be running the appliance using the host kernel. + +=item ext2 + +An ext2 filesystem disk image. + +The output kernel is written to C<OUTPUTDIR/kernel>, the device tree +(if using) to C<OUTPUTDIR/dtb>, a small initramfs which can mount the +appliance to C<OUTPUTDIR/initrd>, and the ext2 filesystem image to +C<OUTPUTDIR/root>. (Where C<OUTPUTDIR> is specified by the I<-o> +option). + +=back + +=item B<--host-cpu> CPU + +(I<--build> mode only) + +Specify the host CPU (eg. C<i686>, C<x86_64>). This is used as a +substring match when searching for compatible kernels. If not +specified, it defaults to the host CPU that supermin was compiled on. + +=item B<-o> OUTPUTDIR + +Select the output directory. + +When using I<--prepare>, this is the directory where the supermin +appliance will be written. When using I<--build>, this is the +directory where the full appliance, kernel etc will be written. In +both cases it defaults to the current directory. + +Note that if files exist already in the output directory then they may +be overwritten. + +=item B<--packager-config> CONFIGFILE + +(I<--prepare> mode only) Set the configuration file for the package manager. This allows you to specify alternate software repositories. @@ -102,13 +208,14 @@ C</etc/pacman.conf>). See L<pacman.conf(5)>. For Yum/RPM distributions, this sets the yum configuration file (default C</etc/yum.conf>). See L<yum.conf(5)>. -=item B<--save-temps> +=item B<--prepare> -Don't remove temporary files and directories on exit. This is useful -for debugging. +Prepare the supermin appliance. =item B<--use-installed> +(I<--prepare> mode only) + If packages are already installed, use the contents (from the local filesystem) instead of downloading them. @@ -117,7 +224,8 @@ changed from what was originally in the package. This is particularly a problem for configuration files. However this option is useful in some controlled situations: for -example when using supermin inside a freshly installed chroot. +example when using supermin inside a freshly installed chroot, or if +you have no network access during the build. =item B<-v> @@ -125,16 +233,31 @@ example when using supermin inside a freshly installed chroot. Enable verbose messages. +You can give this option multiple times to enable even more messages: + +=over 4 + +=item I<-v> + +Debugging of overall stages. + +=item I<-v -v> + +Detailed information within each stage. + +=item I<-v -v -v> + +Massive amounts of debugging (far too much more normal use, but good +if you are trying to diagnose a bug in supermin). + +=back + =item B<-V> =item B<--version> Print the package name and version number, and exit. -=item B<--yum-config CONFIGFILE> - -This is a deprecated alias for I<--packager-config CONFIGFILE>. - =back =head1 SUPERMIN APPLIANCES @@ -146,7 +269,7 @@ appliance share many common files such as C</bin/bash> and C</lib/libc.so> there is no reason to ship these files in the appliance. They can simply be read from the host on demand when the appliance is launched. Therefore to save space we just store the -names of the host files that we want. +packages we want from the host, and copy those in at build time. There are some files which cannot just be copied from the host in this way. These include configuration files which the host admin might @@ -155,15 +278,32 @@ skeleton base image which contains these files and the outline directory structure. Therefore the supermin appliance normally consists of at least two -control files: +control files (C<packages> and C<base.tar.gz>). =over 4 +=item B<packages> + +The list of packages to be copied from the host. Dependencies are +resolved automatically. + +The file is plain text, one package name per line. + +=item B<base.tar> + +=item B<base.tar.gz> + +This tar file (which may be compressed) contains the skeleton +filesystem. Mostly it contains directories and a few configuration +files. + +All paths in the tar file should be relative to the root directory of +the appliance. + =item B<hostfiles> -The list of files that are to be copied from the host. This is a -plain text file with one pathname per line. Directories are included -in this file. +Any other files that are to be copied from the host. This is a plain +text file with one pathname per line. Paths can contain wildcards, which are expanded when the appliance is created, eg: @@ -172,140 +312,127 @@ is created, eg: would copy all of the C<*.repo> files into the appliance. -Each pathname in the file should start with a C</> character. (In -older versions of febootstrap, paths started with C<./> and were -relative to the root directory, but you should not do that in new -files). +Each pathname in the file should start with a C</> character. -=item B<base.img> +Supermin itself does not create hostfiles (although before +S<version 5>, this was the main mechanism used to create the full +appliance). -This uncompressed cpio file contains the skeleton filesystem. Mostly -it contains directories and a few configuration files. +=item B<excludefiles> -All paths in the cpio file should be relative to the root directory of -the appliance. +A list of filenames, directory names, or wildcards prefixed by C<-> +which are excluded from the final appliance. -Note that unlike C<hostfiles>, paths and directories in the base image -don't need to have any relationship to the host filesystem. +This is rather brutal since it just removes things, potentially +breaking packages. However it can be used as a convenient way to +minimize the size of the final appliance. -=item B<base.img.gz> - -Since supermin E<ge> 4.1.4, any cpio image files may be -gzip-compressed to save disk space. C<hostfiles> cannot be -compressed. The supermin program won't create these files. You need -to compress the output yourself, eg by doing: - - gzip -9 supermin.d/*.img +Supermin itself does not create excludefiles. =back +Note that the names above are just suggestions. You can use any names +you want, as supermin detects the contents of each file when it +reconstructs the appliance. You can also have multiple of each type +of file. + =head2 RECONSTRUCTING THE APPLIANCE -The separate tool L<supermin-helper(1)> is used to -reconstruct an appliance from the hostfiles and base image files. +The separate mode C<supermin --build> is used to reconstruct an +appliance from the supermin appliance files. This program in fact iterates recursively over the files and -directories passed to it. A common layout is: +directories passed to it. A common layout could look like this: supermin.d/ - supermin.d/base.img - supermin.d/extra.img - supermin.d/hostfiles - -and then invoking supermin-helper with just the -C<supermin.d> directory path as an argument. + supermin.d/base.tar.gz + supermin.d/extra.tar.gz + supermin.d/packages + supermin.d/zz-hostfiles In this way extra files can be added to the appliance just by creating -another cpio file (C<extra.img> in the example above) and dropping it -into the directory. When the appliance is constructed, the extra -files will appear in the appliance. - -=head3 DIRECTORIES BEFORE FILES - -In order for supermin-helper to run quickly, it does not -know how to create directories automatically. Inside hostfiles and -the cpio files, directories must be specified before any files that -they contain. For example: - - /usr - /usr/sbin - /usr/sbin/serviced - -It is fine to list the same directory name multiple times. - -=head3 LEXICOGRAPHICAL ORDER - -supermin-helper visits the supermin control files in -lexicographical order. Thus in the example above, in the order -C<base.img> -E<gt> C<extra.img> -E<gt> C<hostfiles>. - -This has an important effect: files contained in later cpio files -overwrite earlier files, and directories do not need to be specified -if they have already been created in earlier control files. - -=head3 EXAMPLE OF CREATING EXTRA CPIO FILE - -You can create a file like C<extra.img> very easily using a shell -snippet similar to this one: - - cd $tmpdir - mkdir -p usr/sbin - cp /path/to/serviced usr/sbin/ - echo -e "usr\nusr/sbin\nusr/sbin/serviced" | - cpio --quiet -o -H newc > extra.img - rm -rf usr - -Notice how we instruct cpio to create intermediate directories. +another tar file (C<extra.tar.gz> in the example above) and dropping +it into the directory, and additional host files can be added +(C<zz-hostfiles> in the example above). When the appliance is +constructed, the extra files will appear in the appliance. =head2 MINIMIZING THE SUPERMIN APPLIANCE You may want to "minimize" the supermin appliance in order to save time and space when it is instantiated. Typically you might want to -remove documentation, info files, man pages and locales. We used to -provide a separate tool called C<febootstrap-minimize> for this -purpose, but it is no longer provided. Instead you can post-process -C<hostfiles> yourself to remove any files or directories that you -don't want (by removing lines from the file). Be careful what you -remove because files may be necessary for correct operation of the -appliance. +remove documentation, info files, man pages and locales. -For example: +You can do this by creating an excludefiles that lists files, +directories or wildcards that you don't want to include. They are +skipped when the full appliance is built. - < supermin.d/hostfiles \ - grep -v '^/usr/share/man/' | - grep -v '^/usr/share/doc/' | - grep -v '^/usr/share/info/' > supermin.d/hostfiles-t - mv supermin.d/hostfiles-t supermin.d/hostfiles + -/boot/* + -/lib/modules/* + -/usr/share/man/* + -/usr/share/doc/* + -/usr/share/info/* + +Be careful what you remove because files may be necessary for correct +operation of the appliance. =head2 KERNEL AND KERNEL MODULES Usually the kernel and kernel modules are I<not> included in the -supermin appliance. When the appliance is instantiated, the kernel -modules from the host kernel are copied in, and it is booted using the -host kernel. +supermin appliance. -supermin-helper is able to choose the best host kernel -available to boot the appliance. Users can override this by setting -environment variables (see L<supermin-helper(1)>). +When the full appliance is built, the kernel modules from the host are +copied in, and it is booted using the host kernel. + +Supermin is able to choose the best host kernel available to boot the +appliance. Users can override this by setting environment variables +(see L</ENVIRONMENT VARIABLES> below). =head2 BOOTING AND CACHING THE SUPERMIN APPLIANCE -For fastest boot times you should cache the output of -supermin-helper. See the libguestfs source file -C<src/appliance.c> for an example of how this is done. +For fastest boot times you should cache the output of supermin +I<--build>. See the libguestfs source file C<src/appliance.c> for an +example of how this is done. =head2 ENFORCING AVAILABILITY OF HOSTFILES -supermin-helper builds the appliance by copying in host -files as listed in C<hostfiles>. For this to work those host files -must be available. We usually enforce this by adding requirements -(eg. RPM C<Requires:> lines) on the package that uses the supermin -appliance, so that package cannot be installed without pulling in the -dependent packages and thus making sure the host files are available. +Supermin builds the appliance by copying in host files as listed in +C<hostfiles>. For this to work those host files must be available. +We usually enforce this by adding requirements (eg. RPM C<Requires:> +lines) on the package that uses the supermin appliance, so that +package cannot be installed without pulling in the dependent packages +and thus making sure the host files are available. + +=head1 ENVIRONMENT VARIABLES + +=over 4 + +=item SUPERMIN_KERNEL + +If this environment variable is set, then automatic selection of the +kernel is bypassed and this kernel is used. + +The environment variable should point to a kernel file, +eg. C</boot/vmlinuz-3.0.x86_64> + +=item SUPERMIN_MODULES + +This specifies the kernel modules directory to use. + +The environment variable should point to a module directory, +eg. C</lib/modules/3.0.x86_64/> + +This has no effect if C<SUPERMIN_KERNEL> is not set. + +=item SUPERMIN_DTB + +Force the given device tree file to be used. + +This has no effect if C<SUPERMIN_KERNEL> is not set. + +=back =head1 SEE ALSO -L<supermin-helper(1)>, L<http://people.redhat.com/~rjones/supermin/>, L<guestfs(3)>, L<http://libguestfs.org/>. @@ -320,13 +447,13 @@ Richard W.M. Jones L<http://people.redhat.com/~rjones/> =item * -Matthew Booth L<mbooth(a)redhat.com> +Matthew Booth =back =head1 COPYRIGHT -Copyright (C) 2009-2011 Red Hat Inc. +Copyright (C) 2009-2014 Red Hat Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/supermin_cmdline.ml b/src/supermin_cmdline.ml deleted file mode 100644 index 435dc49..0000000 --- a/src/supermin_cmdline.ml +++ /dev/null @@ -1,120 +0,0 @@ -(* supermin 4 - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - *) - -open Printf - -type mode = (* --names/--names-only flag *) - | PkgFiles (* no flag *) - | PkgNames (* --names *) - | PkgNamesOnly (* --names-only *) - -let excludes = ref [] -let mode = ref PkgFiles -let outputdir = ref "." -let packages = ref [] -let save_temps = ref false -let use_installed = ref false -let verbose = ref false -let warnings = ref true -let packager_config = ref None - -let print_version () = - printf "%s %s\n" Config.package_name Config.package_version; - exit 0 - -let add_exclude re = - excludes := Str.regexp re :: !excludes - -let set_packager_config filename = - (* Need to check that the file exists, and make the path absolute. *) - let filename = - if Filename.is_relative filename then - Filename.concat (Sys.getcwd ()) filename - else filename in - if not (Sys.file_exists filename) then ( - eprintf "supermin: --packager-config: %s: file does not exist\n" - filename; - exit 1 - ); - - packager_config := Some filename - -let argspec = Arg.align [ - "--exclude", Arg.String add_exclude, - "regexp Exclude packages matching regexp"; - "--names", Arg.Unit (fun () -> mode := PkgNames), - " Specify set of root package names on command line"; - "--names-only", Arg.Unit (fun () -> mode := PkgNamesOnly), - " Specify exact set of package names on command line"; - "--no-warnings", Arg.Clear warnings, - " Suppress warnings"; - "-o", Arg.Set_string outputdir, - "outputdir Set output directory (default: \".\")"; - "--packager-config", Arg.String set_packager_config, - "file Set alternate package manager configuration file"; - "--save-temp", Arg.Set save_temps, - " Don't delete temporary files and directories on exit"; - "--save-temps", Arg.Set save_temps, - " Don't delete temporary files and directories on exit"; - "--use-installed", Arg.Set use_installed, - " Use already installed packages for package contents"; - "-v", Arg.Set verbose, - " Enable verbose output"; - "--verbose", Arg.Set verbose, - " Enable verbose output"; - "-V", Arg.Unit print_version, - " Print package name and version, and exit"; - "--version", Arg.Unit print_version, - " Print package name and version, and exit"; - "--yum-config", Arg.String set_packager_config, - "file Deprecated alias for `--packager-config file'"; -] -let anon_fn str = - packages := str :: !packages - -let usage_msg = - "\ -supermin - tool for creating supermin appliances -Copyright (C) 2009-2013 Red Hat Inc. - -Usage: - supermin [-o OUTPUTDIR] --names LIST OF PKGS ... - supermin [-o OUTPUTDIR] PKG FILE NAMES ... - -For full instructions see the supermin(1) man page. - -Options:\n" - -let () = - Arg.parse argspec anon_fn usage_msg; - if !packages = [] then ( - eprintf "supermin: no packages listed on the command line\n"; - exit 1 - ) - -let excludes = List.rev !excludes -let mode = !mode -let outputdir = !outputdir -let packages = List.rev !packages -let save_temps = !save_temps -let use_installed = !use_installed -let verbose = !verbose -let warnings = !warnings -let packager_config = !packager_config - -let debug fs = ksprintf (fun str -> if verbose then print_endline str) fs diff --git a/src/supermin_cmdline.mli b/src/supermin_cmdline.mli deleted file mode 100644 index 7819e05..0000000 --- a/src/supermin_cmdline.mli +++ /dev/null @@ -1,60 +0,0 @@ -(* supermin 4 - * Copyright (C) 2009-2010 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - *) - -(** Command line parsing. *) - -val debug : ('a, unit, string, unit) format4 -> 'a - (** Print string (like printf), but only if --verbose was given on - the command line. *) - -type mode = (** --names/--names-only flag *) - | PkgFiles (** no flag *) - | PkgNames (** --names *) - | PkgNamesOnly (** --names-only *) - -val excludes : Str.regexp list - (** List of package regexps to exclude. *) - -val mode : mode - (** How to interpret {!packages}: - If [mode = PkgFiles] then it's a list of filenames. - If [mode = PkgNames] then it's a list of package names. - If [mode = PkgNamesOnly] then it's a list of package names - and no dependencies are to be loaded. *) - -val outputdir : string - (** Output directory. *) - -val packages : string list - (** List of packages or package names as supplied on the command line. *) - -val save_temps : bool - (** True if [--save-temps] was given on the command line. *) - -val use_installed : bool - (** True if [--use-installed] was given on the command line *) - -val verbose : bool - (** True if [--verbose] was given on the command line. - See also {!debug}. *) - -val warnings : bool - (** If true, print warnings. [--no-warnings] sets this to false. *) - -val packager_config : string option - (** Package manager configuration file. *) diff --git a/src/supermin_debian.ml b/src/supermin_debian.ml deleted file mode 100644 index 3759f9a..0000000 --- a/src/supermin_debian.ml +++ /dev/null @@ -1,231 +0,0 @@ -(* supermin 4 - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - *) - -(* Debian support. *) - -open Unix -open Printf - -open Supermin_package_handlers -open Supermin_utils -open Supermin_cmdline - -(* Create a temporary directory for use by all the functions in this file. *) -let tmpdir = tmpdir () - -let debian_detect () = - file_exists "/etc/debian_version" && - Config.aptitude <> "no" && Config.apt_cache <> "no" && Config.dpkg <> "no" - -let installed_pkgs = ref [] - -let debian_init () = - installed_pkgs := - run_command_get_lines "dpkg-query --show --showformat='${Package}\\n'" - -let get_installed_pkgs () = - match !installed_pkgs with - | [] -> assert false - | pkgs -> pkgs - -(* Select which dependencies will be installed. See apt-cache(8) for - * complete details. Using "-i" means only Depends and Pre-depends - * are installed, which is stricter (fewer packages) than ordinary - * 'apt-get install'. Otherwise, enable everything, then selectively - * disable what you don't want, to make it behave more like - * 'apt-get install'. - *) -let which_dependencies = "-i" -(*let which_dependencies = "--no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances"*) - -let rec debian_resolve_dependencies_and_download names mode = - let which_dependencies = - if mode == PkgNames then which_dependencies ^ " --recurse" - else which_dependencies in - let cmd = - sprintf "%s depends %s %s | grep -v '^[<[:space:]]' | grep -Ev ':\\w+\\b'" - Config.apt_cache which_dependencies - (String.concat " " (List.map Filename.quote names)) in - let pkgs = run_command_get_lines cmd in - let pkgs = - if Config.apt_cache_depends_recurse_broken then - workaround_broken_apt_cache_depends_recurse (sort_uniq pkgs) - else - sort_uniq pkgs in - - (* Exclude packages matching [--exclude] regexps on the command line. *) - let pkgs = - List.filter ( - fun name -> - not (List.exists (fun re -> Str.string_match re name 0) excludes) - ) pkgs in - - let present_pkgs, download_pkgs = - if not use_installed then - [], pkgs - else - List.partition ( - fun pkg -> List.exists ((=) pkg) (get_installed_pkgs ()) - ) pkgs in - - debug "packages already present: %s" (String.concat " " present_pkgs); - debug "wanted packages to download: %s" (String.concat " " download_pkgs); - - (* Download the packages. *) - if (List.length download_pkgs > 0) - then ( - let cmd = - sprintf "umask 0000; cd %s && %s download %s" - (Filename.quote tmpdir) - Config.aptitude - (String.concat " " (List.map Filename.quote download_pkgs)) in - run_command cmd - ); - - (* Find out what aptitude downloaded. *) - let files = Sys.readdir tmpdir in - - let download_pkgs = List.map ( - fun pkg -> - (* Look for 'pkg_*.deb' in the list of files. *) - let pre = pkg ^ "_" in - let r = ref "" in - try - for i = 0 to Array.length files - 1 do - if string_prefix pre files.(i) then ( - r := files.(i); - files.(i) <- ""; - raise Exit - ) - done; - eprintf "supermin: aptitude: error: no file was downloaded corresponding to package %s\n" pkg; - exit 1 - with - Exit -> tmpdir // !r - ) download_pkgs in - - List.sort compare (List.append present_pkgs download_pkgs) - -(* On Ubuntu 10.04 LTS, apt-cache depends --recurse is broken. It - * doesn't return the full list of dependencies. Therefore recurse - * into these dependencies one by one until we reach a fixpoint. - *) -and workaround_broken_apt_cache_depends_recurse names = - debug "workaround for broken 'apt-cache depends --recurse' command:\n %s" - (String.concat " " names); - - let names' = - List.map ( - fun name -> - let cmd = - sprintf "%s depends --recurse -i %s | grep -v '^[<[:space:]]'" - Config.apt_cache (Filename.quote name) in - run_command_get_lines cmd - ) names in - let names' = List.flatten names' in - let names' = sort_uniq names' in - if names <> names' then - workaround_broken_apt_cache_depends_recurse names' - else - names - -let debian_list_files_downloaded pkg = - debug "unpacking %s ..." pkg; - - (* We actually need to extract the file in order to get the - * information about modes etc. - *) - let pkgdir = tmpdir // Filename.basename pkg ^ ".d" in - mkdir pkgdir 0o755; - let cmd = - sprintf "umask 0000; dpkg-deb --fsys-tarfile %s | (cd %s && tar xf -)" - pkg pkgdir in - run_command cmd; - - let cmd = sprintf "cd %s && find ." pkgdir in - let lines = run_command_get_lines cmd in - - let files = List.map ( - fun path -> - assert (path.[0] = '.'); - (* No leading '.' *) - let path = - if path = "." then "/" - else String.sub path 1 (String.length path - 1) in - - (* Find out what it is and get the canonical filename. *) - let statbuf = lstat (pkgdir // path) in - let is_dir = statbuf.st_kind = S_DIR in - - (* No per-file metadata like in RPM, but we can synthesize it - * from the path. - *) - let config = statbuf.st_kind = S_REG && string_prefix "/etc/" path in - - let mode = statbuf.st_perm in - - (path, { ft_dir = is_dir; ft_config = config; ft_mode = mode; - ft_ghost = false; ft_size = statbuf.st_size }) - ) lines in - - files - -let debian_list_files_installed pkg = - debug "using installed package %s ..." pkg; - let cmd = sprintf "dpkg-query --listfiles %s" pkg in - let lines = run_command_get_lines cmd in - (* filter out lines not directly describing fs objects such as - "package diverts others to: /path/to/..." *) - let lines = List.filter ( - fun l -> l.[0] = '/' && l.[1] != '.' - ) lines in - let files = List.map ( - fun path -> - let statbuf = lstat path in - let is_dir = statbuf.st_kind = S_DIR in - let config = statbuf.st_kind = S_REG && string_prefix "/etc/" path in - let mode = statbuf.st_perm in - (path, { ft_dir = is_dir; ft_config = config; ft_mode = mode; - ft_ghost = false; ft_size = statbuf.st_size }) - ) lines in - files - -let debian_list_files pkg = - if use_installed && List.exists ((=) pkg) (get_installed_pkgs ()) then - debian_list_files_installed pkg - else - debian_list_files_downloaded pkg - -(* Easy because we already unpacked the archive above. *) -let debian_get_file_from_package pkg file = - if use_installed && List.exists (fun p -> p = pkg) (get_installed_pkgs ()) - then - file - else - tmpdir // Filename.basename pkg ^ ".d" // file - -let () = - let ph = { - ph_detect = debian_detect; - ph_init = debian_init; - ph_resolve_dependencies_and_download = - debian_resolve_dependencies_and_download; - ph_list_files = debian_list_files; - ph_get_file_from_package = debian_get_file_from_package; - } in - register_package_handler "debian" ph diff --git a/src/supermin_package_handlers.ml b/src/supermin_package_handlers.ml deleted file mode 100644 index 7048f66..0000000 --- a/src/supermin_package_handlers.ml +++ /dev/null @@ -1,81 +0,0 @@ -(* supermin 4 - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - *) - -open Unix -open Printf - -open Supermin_utils -open Supermin_cmdline - -type package_handler = { - ph_detect : unit -> bool; - ph_init : unit -> unit; - ph_resolve_dependencies_and_download : string list -> mode -> string list; - ph_list_files : string -> (string * file_type) list; - ph_get_file_from_package : string -> string -> string -} -and file_type = { - ft_dir : bool; - ft_config : bool; - ft_ghost : bool; - ft_mode : int; - ft_size : int; -} - -let tmpdir = tmpdir () - -let handlers = ref [] - -let register_package_handler name ph = - handlers := (name, ph) :: !handlers - -let handler = ref None - -let rec check_system () = - try - handler := Some ( - List.find ( - fun (_, ph) -> - ph.ph_detect () - ) !handlers - ); - (get_package_handler ()).ph_init () - with Not_found -> - eprintf "\ -supermin: could not detect package manager used by this system or distro. - -If this is a new Linux distro, or not Linux, or a Linux distro that uses -an unusual packaging format then you may need to port supermin. If -you are expecting that supermin should work on this system or distro -then it may be that the package detection code is not working. -"; - exit 1 - -and get_package_handler () = - match !handler with - | Some (_, ph) -> ph - | None -> - check_system (); - get_package_handler () - -and get_package_handler_name () = - match !handler with - | Some (name, _) -> name - | None -> - check_system (); - get_package_handler_name () diff --git a/src/supermin_package_handlers.mli b/src/supermin_package_handlers.mli deleted file mode 100644 index 8ee12dd..0000000 --- a/src/supermin_package_handlers.mli +++ /dev/null @@ -1,73 +0,0 @@ -(* supermin 4 - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - *) - -(** Generic package handler code. *) - -type package_handler = { - ph_detect : unit -> bool; - (** Detect if the current system uses this package manager. This is - called in turn on each package handler, until one returns [true]. *) - - ph_init : unit -> unit; - (** After a package handler is selected, this function is called - which can optionally do any initialization that is required. - This is only called on the package handler if it has returned - [true] from {!ph_detect}. *) - - ph_resolve_dependencies_and_download : string list -> Supermin_cmdline.mode -> string list; - (** [ph_resolve_dependencies_and_download pkgs mode] - Take a list of package names, and using the package manager - resolve those to a list of all the packages that are required - including dependencies. Download the full list of packages and - dependencies into a tmpdir. Return the list of full filenames. - - Note this should also process the [excludes] list. *) - - ph_list_files : string -> (string * file_type) list; - (** [ph_list_files pkg] lists the files and file metadata in the - package called [pkg] (a package file). *) - - ph_get_file_from_package : string -> string -> string; - (** [ph_get_file_from_package pkg file] extracts the - single named file [file] from [pkg]. The path of the - extracted file is returned. *) -} - -(* These file types are inspired by the metadata specifically - * stored by RPM. We should look at what other package formats - * can use too. - *) -and file_type = { - ft_dir : bool; (** Is a directory. *) - ft_config : bool; (** Is a configuration file. *) - ft_ghost : bool; (** Is a ghost (created empty) file. *) - ft_mode : int; (** File mode. *) - ft_size : int; (** File size. *) -} - -val register_package_handler : string -> package_handler -> unit - (** Register a package handler. *) - -val check_system : unit -> unit - (** Check which package manager this system uses. *) - -val get_package_handler : unit -> package_handler - (** Get the selected package manager for this system. *) - -val get_package_handler_name : unit -> string - (** Get the name of the selected package manager for this system. *) diff --git a/src/supermin_pacman.ml b/src/supermin_pacman.ml deleted file mode 100644 index f4d8329..0000000 --- a/src/supermin_pacman.ml +++ /dev/null @@ -1,145 +0,0 @@ -(* supermin 4 - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - *) - -(* ArchLinux support. *) - -open Unix -open Printf - -open Supermin_package_handlers -open Supermin_utils -open Supermin_cmdline - -(* Create a temporary directory for use by all the functions in this file. *) -let tmpdir = tmpdir () - -let pacman_detect () = - file_exists "/etc/arch-release" && - Config.pacman <> "no" - -let pacman_init () = - if use_installed then - failwith "pacman driver doesn't support --use-installed" - -let pacman_resolve_dependencies_and_download names mode = - if mode = PkgNamesOnly then ( - eprintf "supermin: pacman: --names-only flag is not implemented\n"; - exit 1 - ); - - let cmd = - sprintf "(for p in %s; do pactree -u $p; done) | awk '{print $1}' | sort -u" - (String.concat " " (List.map Filename.quote names)) in - let pkgs = run_command_get_lines cmd in - - (* Exclude packages matching [--exclude] regexps on the command line. *) - let pkgs = - List.filter ( - fun name -> - not (List.exists (fun re -> Str.string_match re name 0) excludes) - ) pkgs in - - (* Download the packages. I could use wget `pacman -Sp`, but this - * narrows the pacman -Sy window - *) - - List.iter ( - fun pkg -> - let cmd = - sprintf "umask 0000; cd %s && mkdir -p var/lib/pacman && fakeroot pacman%s -Syw --noconfirm --cachedir=$(pwd) --root=$(pwd) %s" - (Filename.quote tmpdir) - (match packager_config with - | None -> "" - | Some filename -> " --config " ^ filename) - pkg in - if Sys.command cmd <> 0 then ( - (* The package is not in the main repos, check the aur *) - let cmd = - sprintf "umask 0000; cd %s && wget https://aur.archlinux.org/packages/%s/%s/%s.tar.gz && tar xf %s.tar.gz && cd %s && makepkg && mv %s-*.pkg.tar.xz %s" - (Filename.quote tmpdir) - (String.sub pkg 0 2) - pkg - pkg - pkg - pkg - pkg - (Filename.quote tmpdir) in - run_command cmd; - ) - ) pkgs; - - List.sort compare pkgs - -let pacman_list_files pkg = - debug "unpacking %s ..." pkg; - - (* We actually need to extract the file in order to get the - * information about modes etc. - *) - let pkgdir = tmpdir // pkg ^ ".d" in - mkdir pkgdir 0o755; - let cmd = - sprintf "ls -1 %s/%s-*.pkg.* | awk '/\\/%s-[^/-]*-[^/-]*-[^/-]*$/ { print $0 }'" - tmpdir pkg pkg in - let pkgfile = List.hd (run_command_get_lines cmd) in - let cmd = sprintf "umask 0000; fakeroot tar -xf %s -C %s" - (Filename.quote pkgfile) (Filename.quote pkgdir) in - run_command cmd; - - let cmd = sprintf "cd %s && find ." pkgdir in - let lines = run_command_get_lines cmd in - - let files = List.map ( - fun path -> - assert (path.[0] = '.'); - (* No leading '.' *) - let path = - if path = "." then "/" - else String.sub path 1 (String.length path - 1) in - - (* Find out what it is and get the canonical filename. *) - let statbuf = lstat (pkgdir // path) in - let is_dir = statbuf.st_kind = S_DIR in - - (* No per-file metadata like in RPM, but we can synthesize it - * from the path. - *) - let config = statbuf.st_kind = S_REG && string_prefix "/etc/" path in - - let mode = statbuf.st_perm in - - (path, { ft_dir = is_dir; ft_config = config; ft_mode = mode; - ft_ghost = false; ft_size = statbuf.st_size }) - ) lines in - - files - -(* Easy because we already unpacked the archive above. *) -let pacman_get_file_from_package pkg file = - tmpdir // pkg ^ ".d" // file - -let () = - let ph = { - ph_detect = pacman_detect; - ph_init = pacman_init; - ph_resolve_dependencies_and_download = - pacman_resolve_dependencies_and_download; - ph_list_files = pacman_list_files; - ph_get_file_from_package = pacman_get_file_from_package; - } in - register_package_handler "pacman" ph diff --git a/src/supermin_pacman_g2.ml b/src/supermin_pacman_g2.ml deleted file mode 100644 index 9430b5c..0000000 --- a/src/supermin_pacman_g2.ml +++ /dev/null @@ -1,149 +0,0 @@ -(* supermin 4 - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - *) - -(* FrugalWare support. *) - -open Unix -open Printf - -open Supermin_package_handlers -open Supermin_utils -open Supermin_cmdline - -(* Create a temporary directory for use by all the functions in this file. *) -let tmpdir = tmpdir () - -let pacman_g2_detect () = - file_exists "/etc/frugalware-release" && - Config.pacman_g2 <> "no" - -let pacman_g2_init () = - if use_installed then - eprintf "supermin: pacman_g2 driver assumes all packages are already installed when called with option --use-installed.\n%!" - -let pacman_g2_resolve_dependencies_and_download names mode = - debug "resolving deps"; - - debug "filtering deps"; - (* Exclude packages matching [--exclude] regexps on the command line. *) - let pkgs = - List.filter ( - fun name -> - not (List.exists (fun re -> Str.string_match re name 0) excludes) - ) names in - - if mode = PkgNamesOnly then ( - eprintf "supermin: pacman_g2: --names-only flag is not implemented\n"; - exit 1 - ); - - (* Download the packages. I could use wget `pacman -Sp`, but this - * narrows the pacman -Sy window - *) - - List.iter ( - fun pkg -> - let cmd = - sprintf "umask 0000; cd %s && mkdir -p var/cache/pacman-g2/pkg && fakeroot pacman-g2%s -Sy --noconfirm --root=$(pwd) %s" - (Filename.quote tmpdir) - (match packager_config with - | None -> "" - | Some filename -> " --config " ^ filename) - pkg in - run_command cmd; - ) pkgs; - - let cmd = - sprintf "cd %s && fakeroot pacman-g2%s -Q --root=$(pwd)| cut -d ' ' -f 1" - (Filename.quote tmpdir) - (match packager_config with - | None -> "" - | Some filename -> " --config " ^ filename) in - - let pkgs = run_command_get_lines cmd in - - List.sort compare pkgs - -let pacman_g2_list_files pkg = - debug "unpacking %s ..." pkg; - - (* We actually need to extract the file in order to get the - * information about modes etc. - *) - let pkgdir = tmpdir // pkg ^ ".d" in - mkdir pkgdir 0o755; - let cmd = - sprintf "ls -1 %s/var/cache/pacman-g2/pkg/%s-*.fpm" - tmpdir pkg in - let pkgfile = List.hd (run_command_get_lines cmd) in - let cmd = sprintf "umask 0000; fakeroot tar -xf %s -C %s" - (Filename.quote pkgfile) (Filename.quote pkgdir) in - run_command cmd; - - let cmd = sprintf "cd %s && find ." pkgdir in - let lines = run_command_get_lines cmd in - - let excludes = [Str.regexp "./.CHANGELOG"; - Str.regexp "./.FILELIST"; - Str.regexp "./.PKGINFO"; - Str.regexp "./.INSTALL"] in - - let lines = - List.filter ( - fun name -> - not (List.exists (fun re -> Str.string_match re name 0) excludes) - ) lines in - - let files = List.map ( - fun path -> - assert (path.[0] = '.'); - (* No leading '.' *) - let path = - if path = "." then "/" - else String.sub path 1 (String.length path - 1) in - - (* Find out what it is and get the canonical filename. *) - let statbuf = lstat (pkgdir // path) in - let is_dir = statbuf.st_kind = S_DIR in - - (* No per-file metadata like in RPM, but we can synthesize it - * from the path. - *) - let config = statbuf.st_kind = S_REG && string_prefix "/etc/" path in - - let mode = statbuf.st_perm in - - (path, { ft_dir = is_dir; ft_config = config; ft_mode = mode; - ft_ghost = false; ft_size = statbuf.st_size }) - ) lines in - - files - -(* Easy because we already unpacked the archive above. *) -let pacman_g2_get_file_from_package pkg file = - tmpdir // pkg ^ ".d" // file - -let () = - let ph = { - ph_detect = pacman_g2_detect; - ph_init = pacman_g2_init; - ph_resolve_dependencies_and_download = pacman_g2_resolve_dependencies_and_download; - ph_list_files = pacman_g2_list_files; - ph_get_file_from_package = pacman_g2_get_file_from_package; - } in - register_package_handler "pacman-g2" ph diff --git a/src/supermin_urpmi_rpm.ml b/src/supermin_urpmi_rpm.ml deleted file mode 100644 index a598ef5..0000000 --- a/src/supermin_urpmi_rpm.ml +++ /dev/null @@ -1,132 +0,0 @@ -(* supermin 4 - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - *) - -(* URPMI support. *) - -open Unix -open Printf - -open Supermin_package_handlers -open Supermin_utils -open Supermin_cmdline - -(* Create a temporary directory for use by all the functions in this file. *) -let tmpdir = tmpdir () - -let urpmi_rpm_detect () = - (file_exists "/etc/mageia-release") && - Config.urpmi <> "no" && Config.rpm <> "no" - -let urpmi_rpm_init () = - if use_installed then - failwith "urpmi_rpm driver doesn't support --use-installed" - -let urpmi_rpm_resolve_dependencies_and_download names mode = - if mode = PkgNamesOnly then ( - eprintf "supermin: urpmi-rpm: --names-only flag is not implemented\n"; - exit 1 - ); - let cmd = sprintf "urpmq -rd --whatprovides --sources %s" - (String.concat " " names) in - let lines = run_command_get_lines cmd in - (* Return list of package filenames. *) - let g x = - (Filename.concat tmpdir (Filename.basename x)) in - let f x = - let cmd = sprintf "curl %s -o %s" x (g x) in - run_command cmd in - List.iter f lines; - let uf res e = if List.mem e res then res else e::res in - List.fold_left uf [] (List.map g lines) - -let rec urpmi_rpm_list_files pkg = - (* Run rpm -qlp with some extra magic. *) - let cmd = - sprintf "rpm -q --qf '[%%{FILENAMES} %%{FILEFLAGS:fflags} %%{FILEMODES} %%{FILESIZES}\\n]' -p %s" - pkg in - let lines = run_command_get_lines cmd in - - let files = - filter_map ( - fun line -> - match string_split " " line with - | [filename; flags; mode; size] -> - let test_flag = String.contains flags in - let mode = int_of_string mode in - let size = int_of_string size in - if test_flag 'd' then None (* ignore documentation *) - else - Some (filename, { - ft_dir = mode land 0o40000 <> 0; - ft_ghost = test_flag 'g'; ft_config = test_flag 'c'; - ft_mode = mode; ft_size = size; - }) - | _ -> - eprintf "supermin: bad output from rpm command: '%s'" line; - exit 1 - ) lines in - - (* I've never understood why the base packages like 'filesystem' don't - * contain any /dev nodes at all. This leaves every program that - * bootstraps RPMs to create a varying set of device nodes themselves. - * This collection was copied from mock/backend.py. - *) - let files = - let b = Filename.basename pkg in - if string_prefix "filesystem-" b then ( - let dirs = [ "/proc"; "/sys"; "/dev"; "/dev/pts"; "/dev/shm"; - "/dev/mapper" ] in - let dirs = - List.map (fun name -> - name, { ft_dir = true; ft_ghost = false; - ft_config = false; ft_mode = 0o40755; - ft_size = 0 }) dirs in - let devs = [ "/dev/null"; "/dev/full"; "/dev/zero"; "/dev/random"; - "/dev/urandom"; "/dev/tty"; "/dev/console"; - "/dev/ptmx"; "/dev/stdin"; "/dev/stdout"; "/dev/stderr" ] in - (* No need to set the mode because these will go into hostfiles. *) - let devs = - List.map (fun name -> - name, { ft_dir = false; ft_ghost = false; - ft_config = false; ft_mode = 0o644; - ft_size = 0 }) devs in - dirs @ devs @ files - ) else files in - - files - -let urpmi_rpm_get_file_from_package pkg file = - debug "extracting %s from %s ..." file (Filename.basename pkg); - - let outfile = tmpdir // file in - let cmd = - sprintf "umask 0000; rpm2cpio %s | (cd %s && cpio --quiet -id .%s)" - (Filename.quote pkg) (Filename.quote tmpdir) (Filename.quote file) in - run_command cmd; - outfile - -let () = - let ph = { - ph_detect = urpmi_rpm_detect; - ph_init = urpmi_rpm_init; - ph_resolve_dependencies_and_download = - urpmi_rpm_resolve_dependencies_and_download; - ph_list_files = urpmi_rpm_list_files; - ph_get_file_from_package = urpmi_rpm_get_file_from_package; - } in - register_package_handler "urpmi-rpm" ph diff --git a/src/supermin_utils.ml b/src/supermin_utils.ml deleted file mode 100644 index cb8a27e..0000000 --- a/src/supermin_utils.ml +++ /dev/null @@ -1,158 +0,0 @@ -(* supermin 4 - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - *) - -open Unix -open Printf - -let (//) = Filename.concat - -let file_exists name = - try access name [F_OK]; true - with Unix_error _ -> false - -let dir_exists name = - try (stat name).st_kind = S_DIR - with Unix_error _ -> false - -let rec uniq ?(cmp = Pervasives.compare) = function - | [] -> [] - | [x] -> [x] - | x :: y :: xs when cmp x y = 0 -> - uniq ~cmp (x :: xs) - | x :: y :: xs -> - x :: uniq ~cmp (y :: xs) - -let sort_uniq ?(cmp = Pervasives.compare) xs = - let xs = List.sort cmp xs in - let xs = uniq ~cmp xs in - xs - -let rec input_all_lines chan = - try let line = input_line chan in line :: input_all_lines chan - with End_of_file -> [] - -let run_command_get_lines cmd = - let chan = open_process_in cmd in - let lines = input_all_lines chan in - let stat = close_process_in chan in - (match stat with - | WEXITED 0 -> () - | WEXITED i -> - eprintf "supermin: command '%s' failed (returned %d), see earlier error messages\n" cmd i; - exit i - | WSIGNALED i -> - eprintf "supermin: command '%s' killed by signal %d" cmd i; - exit 1 - | WSTOPPED i -> - eprintf "supermin: command '%s' stopped by signal %d" cmd i; - exit 1 - ); - lines - -let run_command cmd = - if Sys.command cmd <> 0 then ( - eprintf "supermin: %s: command failed, see earlier errors\n" cmd; - exit 1 - ) - -let run_shell code args = - let cmd = sprintf "sh -c %s arg0 %s" - (Filename.quote code) - (String.concat " " (List.map Filename.quote args)) in - if Sys.command cmd <> 0 then ( - eprintf "supermin: external shell program failed, see earlier error messages\n"; - exit 1 - ) - -let run_python code args = - let cmd = sprintf "python -c %s %s" - (Filename.quote code) - (String.concat " " (List.map Filename.quote args)) in - if Sys.command cmd <> 0 then ( - eprintf "supermin: external python program failed, see earlier error messages\n"; - exit 1 - ) - -let tmpdir () = - let chan = open_in "/dev/urandom" in - let data = String.create 16 in - really_input chan data 0 (String.length data); - close_in chan; - let data = Digest.to_hex (Digest.string data) in - (* Note this is secure, because if the name already exists, even as a - * symlink, mkdir(2) will fail. - *) - let tmpdir = Filename.temp_dir_name // sprintf "supermin%s.tmp" data in - Unix.mkdir tmpdir 0o700; - - (* Only remove the directory if --save-temps was *not* specified. *) - if not Supermin_cmdline.save_temps then - at_exit - (fun () -> - let cmd = sprintf "rm -rf %s" (Filename.quote tmpdir) in - ignore (Sys.command cmd)); - - tmpdir - -let rec find s sub = - let len = String.length s in - let sublen = String.length sub in - let rec loop i = - if i <= len-sublen then ( - let rec loop2 j = - if j < sublen then ( - if s.[i+j] = sub.[j] then loop2 (j+1) - else -1 - ) else - i (* found *) - in - let r = loop2 0 in - if r = -1 then loop (i+1) else r - ) else - -1 (* not found *) - in - loop 0 - -let rec string_split sep str = - let len = String.length str in - let seplen = String.length sep in - let i = find str sep in - if i = -1 then [str] - else ( - let s' = String.sub str 0 i in - let s'' = String.sub str (i+seplen) (len-i-seplen) in - s' :: string_split sep s'' - ) - -let string_prefix p str = - let len = String.length str in - let plen = String.length p in - len >= plen && String.sub str 0 plen = p - -let path_prefix p path = - let len = String.length path in - let plen = String.length p in - path = p || (len > plen && String.sub path 0 (plen+1) = (p ^ "/")) - -let rec filter_map f = function - | [] -> [] - | x :: xs -> - let x = f x in - match x with - | None -> filter_map f xs - | Some x -> x :: filter_map f xs diff --git a/src/supermin_utils.mli b/src/supermin_utils.mli deleted file mode 100644 index c4a52c6..0000000 --- a/src/supermin_utils.mli +++ /dev/null @@ -1,78 +0,0 @@ -(* supermin 4 - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - *) - -(** Utilities. *) - -val file_exists : string -> bool - (** Return [true] iff file exists. *) - -val dir_exists : string -> bool - (** Return [true] iff dir exists. *) - -val uniq : ?cmp:('a -> 'a -> int) -> 'a list -> 'a list - (** Uniquify a list (the list must be sorted first). *) - -val sort_uniq : ?cmp:('a -> 'a -> int) -> 'a list -> 'a list - (** Sort and uniquify a list. *) - -val input_all_lines : in_channel -> string list - (** Input all lines from a channel, returning a list of lines. *) - -val run_command_get_lines : string -> string list - (** Run the command and read the list of lines that it prints to stdout. *) - -val run_command : string -> unit - (** Run a command using {!Sys.command} and exit if it fails. Be careful - when constructing the command to properly quote any arguments - (using {!Filename.quote}). *) - -val run_shell : string -> string list -> unit - (** [run_shell code args] runs shell [code] with arguments [args]. - This does not return anything, but exits with an error message - if the shell code returns an error. *) - -val run_python : string -> string list -> unit - (** [run_python code args] runs Python [code] with arguments [args]. - This does not return anything, but exits with an error message - if the Python code returns an error. *) - -val tmpdir : unit -> string - (** [tmpdir ()] returns a newly created temporary directory. The - tmp directory is automatically removed when the program exits. - Note that a fresh temporary directory is returned each time you - call this function. *) - -val (//) : string -> string -> string - (** [x // y] concatenates file paths [x] and [y] into a single path. *) - -val find : string -> string -> int -(** [find str sub] searches for [sub] in [str], returning the index - or -1 if not found. *) - -val string_split : string -> string -> string list - (** [string_split sep str] splits [str] at [sep]. *) - -val string_prefix : string -> string -> bool - (** [string_prefix prefix str] returns true iff [str] starts with [prefix]. *) - -val path_prefix : string -> string -> bool - (** [path_prefix prefix path] returns true iff [path] is [prefix] or - [path] starts with [prefix/]. *) - -val filter_map : ('a -> 'b option) -> 'a list -> 'b list - (** map + filter *) diff --git a/src/supermin_yum_rpm.ml b/src/supermin_yum_rpm.ml deleted file mode 100644 index e334d17..0000000 --- a/src/supermin_yum_rpm.ml +++ /dev/null @@ -1,262 +0,0 @@ -(* supermin 4 - * Copyright (C) 2009-2013 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - *) - -(* Yum and RPM support. *) - -open Unix -open Printf - -open Supermin_package_handlers -open Supermin_utils -open Supermin_cmdline - -(* Create a temporary directory for use by all the functions in this file. *) -let tmpdir = tmpdir () - -let yum_rpm_detect () = - (file_exists "/etc/redhat-release" || file_exists "/etc/fedora-release") && - Config.yum <> "no" && Config.rpm <> "no" - -let yum_rpm_init () = - if use_installed then - failwith "yum_rpm driver doesn't support --use-installed" - -let yum_rpm_resolve_dependencies_and_download names mode = - if mode = PkgNamesOnly then ( - eprintf "supermin: yum-rpm: --names-only flag is not implemented\n"; - exit 1 - ); - - (* Liberate this data from python. *) - let tmpfile = tmpdir // "names.tmp" in - let py = sprintf " -import yum -import yum.misc -import sys - -verbose = %d - -if verbose: - print \"supermin_yum_rpm: running python code to query yum and resolve deps\" - -yb = yum.YumBase () -yb.preconf.debuglevel = verbose -yb.preconf.errorlevel = verbose -try: - yb.prerepoconf.multi_progressbar = None -except: - pass -if %s: - yb.preconf.fn = %S -try: - yb.setCacheDir () -except AttributeError: - pass - -if verbose: - print \"supermin_yum_rpm: looking up the base packages from the command line\" -deps = dict () -pkgs = yb.pkgSack.returnPackages (patterns=sys.argv[1:]) -for pkg in pkgs: - deps[pkg] = False - -if verbose: - print \"supermin_yum_rpm: recursively finding all the dependencies\" -stable = False -while not stable: - stable = True - for pkg in deps.keys(): - if deps[pkg] == False: - deps[pkg] = [] - stable = False - if verbose: - print (\"supermin_yum_rpm: examining deps of %%s\" %% - pkg.name) - for r in pkg.requires: - ps = yb.whatProvides (r[0], r[1], r[2]) - best = yb._bestPackageFromList (ps.returnPackages ()) - if best and best.name != pkg.name: - deps[pkg].append (best) - if not deps.has_key (best): - deps[best] = False - deps[pkg] = yum.misc.unique (deps[pkg]) - -# Write it to a file because yum spews garbage on stdout. -f = open (%S, \"w\") -for pkg in deps.keys (): - f.write (\"%%s %%s %%s %%s %%s\\n\" %% - (pkg.name, pkg.epoch, pkg.version, pkg.release, pkg.arch)) -f.close () - -if verbose: - print \"supermin_yum_rpm: finished python code\" -" - (if verbose then 1 else 0) - (match packager_config with None -> "False" | Some _ -> "True") - (match packager_config with None -> "" | Some filename -> filename) - tmpfile in - run_python py names; - let chan = open_in tmpfile in - let lines = input_all_lines chan in - close_in chan; - - (* Get fields. *) - let pkgs = - List.map ( - fun line -> - match string_split " " line with - | [name; epoch; version; release; arch] -> - name, int_of_string epoch, version, release, arch - | _ -> - eprintf "supermin: bad output from python script: '%s'" line; - exit 1 - ) lines in - - (* Something of a hack for x86_64: exclude all i[3456]86 packages. *) - let pkgs = - if Config.host_cpu = "x86_64" then ( - List.filter ( - function (_, _, _, _, ("i386"|"i486"|"i586"|"i686")) -> false - | _ -> true - ) pkgs - ) - else pkgs in - - (* Exclude packages matching [--exclude] regexps on the command line. *) - let pkgs = - List.filter ( - fun (name, _, _, _, _) -> - not (List.exists (fun re -> Str.string_match re name 0) excludes) - ) pkgs in - - (* Sort the list of packages, and remove duplicates (by name). - * XXX This is not quite right: we really want to keep the latest - * package if duplicates are found, but that would require a full - * version compare function. - *) - let pkgs = List.sort (fun a b -> compare b a) pkgs in - let pkgs = - let cmp (name1, _, _, _, _) (name2, _, _, _, _) = compare name1 name2 in - uniq ~cmp pkgs in - let pkgs = List.sort compare pkgs in - - (* Construct package names. *) - let pkgnames = List.map ( - function - | name, 0, version, release, arch -> - sprintf "%s-%s-%s.%s" name version release arch - | name, epoch, version, release, arch -> - sprintf "%d:%s-%s-%s.%s" epoch name version release arch - ) pkgs in - - if pkgnames = [] then ( - eprintf "supermin: yum-rpm: error: no packages to download\n"; - exit 1 - ); - - let cmd = sprintf "yumdownloader%s%s --destdir %s %s" - (if verbose then "" else " --quiet") - (match packager_config with None -> "" - | Some filename -> sprintf " -c %s" filename) - (Filename.quote tmpdir) - (String.concat " " (List.map Filename.quote pkgnames)) in - run_command cmd; - - (* Return list of package filenames. *) - List.map ( - (* yumdownloader doesn't include epoch in the filename *) - fun (name, _, version, release, arch) -> - sprintf "%s/%s-%s-%s.%s.rpm" tmpdir name version release arch - ) pkgs - -let rec yum_rpm_list_files pkg = - (* Run rpm -qlp with some extra magic. *) - let cmd = - sprintf "rpm -q --qf '[%%{FILENAMES} %%{FILEFLAGS:fflags} %%{FILEMODES} %%{FILESIZES}\\n]' -p %s" - pkg in - let lines = run_command_get_lines cmd in - - let files = - filter_map ( - fun line -> - match string_split " " line with - | [filename; flags; mode; size] -> - let test_flag = String.contains flags in - let mode = int_of_string mode in - let size = int_of_string size in - if test_flag 'd' then None (* ignore documentation *) - else - Some (filename, { - ft_dir = mode land 0o40000 <> 0; - ft_ghost = test_flag 'g'; ft_config = test_flag 'c'; - ft_mode = mode; ft_size = size; - }) - | _ -> - eprintf "supermin: bad output from rpm command: '%s'" line; - exit 1 - ) lines in - - (* I've never understood why the base packages like 'filesystem' don't - * contain any /dev nodes at all. This leaves every program that - * bootstraps RPMs to create a varying set of device nodes themselves. - * This collection was copied from mock/backend.py. - *) - let files = - let b = Filename.basename pkg in - if string_prefix "filesystem-" b then ( - let dirs = [ "/proc"; "/sys"; "/dev"; "/dev/pts"; "/dev/shm"; - "/dev/mapper" ] in - let dirs = - List.map (fun name -> - name, { ft_dir = true; ft_ghost = false; - ft_config = false; ft_mode = 0o40755; - ft_size = 0 }) dirs in - let devs = [ "/dev/null"; "/dev/full"; "/dev/zero"; "/dev/random"; - "/dev/urandom"; "/dev/tty"; "/dev/console"; - "/dev/ptmx"; "/dev/stdin"; "/dev/stdout"; "/dev/stderr" ] in - (* No need to set the mode because these will go into hostfiles. *) - let devs = - List.map (fun name -> - name, { ft_dir = false; ft_ghost = false; - ft_config = false; ft_mode = 0o644; - ft_size = 0 }) devs in - dirs @ devs @ files - ) else files in - - files - -let yum_rpm_get_file_from_package pkg file = - debug "extracting %s from %s ..." file (Filename.basename pkg); - - let outfile = tmpdir // file in - let cmd = - sprintf "umask 0000; rpm2cpio %s | (cd %s && cpio --quiet -id .%s)" - (Filename.quote pkg) (Filename.quote tmpdir) (Filename.quote file) in - run_command cmd; - outfile - -let () = - let ph = { - ph_detect = yum_rpm_detect; - ph_init = yum_rpm_init; - ph_resolve_dependencies_and_download = - yum_rpm_resolve_dependencies_and_download; - ph_list_files = yum_rpm_list_files; - ph_get_file_from_package = yum_rpm_get_file_from_package; - } in - register_package_handler "yum-rpm" ph diff --git a/src/supermin_zypp_rpm.ml b/src/supermin_zypp_rpm.ml deleted file mode 100644 index f28df2b..0000000 --- a/src/supermin_zypp_rpm.ml +++ /dev/null @@ -1,269 +0,0 @@ -(* supermin 4 - * Copyright (C) 2009-2010 Red Hat Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - *) - -(* Zypper and RPM support. *) - -(* - * Theory of operation: - * called as root: - * - without --use-installed: - * ->ph_resolve_dependencies_and_download() returns a list of filenames - * Need to download all packages into an empty --root directory so that - * zypper places all dependencies into --pkg-cache-dir - * - with --use-installed: - * ->ph_resolve_dependencies_and_download() returns a list of package names - * Need to work with an empty --root directory so that zypper can list - * all dependencies of "names". This mode assumes that all required packages - * are installed and the system is consistent. Downloading just the missing - * packages is not implemented. - * called as non-root: - * (Due to the usage of --root zypper does not require root permissions.) - * - without --use-installed: - * Same as above. - * - with --use-installed: - * Same as above. - * - * The usage of --packager-config is tricky: If --root is used zypper assumes - * that every config file is below <rootdir>. So the config has to be parsed - * and relevant files/dirs should be copied into <rootdir> so that zypper can - * use the specified config. - *) -open Unix -open Printf -open Inifiles - -open Supermin_package_handlers -open Supermin_utils -open Supermin_cmdline - - -(* Create a temporary directory for use by all the functions in this file. *) -let tmpdir = tmpdir () - -let get_repos_dir () = - let zypper_default = "/etc/zypp/repos.d" in - let parse_repos_dir path = - let cfg = new inifile path in - let dir = (try cfg#getval "main" "reposdir" with _ -> zypper_default) in - dir - in - let dir = (match packager_config with None -> zypper_default | - Some filename -> (try parse_repos_dir filename with _ -> zypper_default) ) in - dir - -let repos_dir = get_repos_dir () - -let zypp_rpm_detect () = - (file_exists "/etc/SuSE-release") && - Config.zypper <> "no" && Config.rpm <> "no" - -let zypp_rpm_init () = - if use_installed then - eprintf "supermin: zypp_rpm driver assumes all packages are already installed when called with option --use-installed.\n%!" - -let zypp_rpm_resolve_dependencies_and_download_no_installed names = - (* Liberate this data from shell. *) - let tmp_pkg_cache_dir = tmpdir // "pkg_cache_dir" in - let tmp_root = tmpdir // "root" in - let sh = sprintf " -%s -unset LANG ${!LC_*} -tmpdir=%S -cache_dir=\"${tmpdir}/cache-dir\" -pkg_cache_dir=%S -time zypper \ - %s \ - %s \ - --root %S --reposd-dir %S \ - --cache-dir \"${cache_dir}\" \ - --pkg-cache-dir \"${pkg_cache_dir}\" \ - --gpg-auto-import-keys \ - --no-gpg-checks \ - --non-interactive \ - install \ - --auto-agree-with-licenses \ - --download-only \ - $@ -" - (if verbose then "set -x" else "") - tmpdir - tmp_pkg_cache_dir - (if verbose then "--verbose --verbose" else "--quiet") - (match packager_config with None -> "" - | Some filename -> sprintf "--config %s" filename) - tmp_root - repos_dir - in - run_shell sh names; - - (* http://rosettacode.org/wiki/Walk_a_directory/Recursively *) - let walk_directory_tree dir pattern = - let select str = Str.string_match (Str.regexp pattern) str 0 in - let rec walk acc = function - | [] -> (acc) - | dir::tail -> - let contents = Array.to_list (Sys.readdir dir) in - let contents = List.rev_map (Filename.concat dir) contents in - let dirs, files = - List.fold_left (fun (dirs,files) f -> - match (stat f).st_kind with - | S_REG -> (dirs, f::files) (* Regular file *) - | S_DIR -> (f::dirs, files) (* Directory *) - | _ -> (dirs, files) - ) ([],[]) contents - in - let matched = List.filter (select) files in - walk (matched @ acc) (dirs @ tail) - in - walk [] [dir] - in - - let pkgs = walk_directory_tree tmp_pkg_cache_dir ".*\\.rpm" in - - (* Return list of package filenames. *) - pkgs - -let zypp_rpm_resolve_dependencies_and_download_use_installed names = - let cmd = sprintf " -%s -unset LANG ${!LC_*} -zypper \ - %s \ - %s \ - --root %S --reposd-dir %S \ - --cache-dir %S \ - --gpg-auto-import-keys \ - --no-gpg-checks \ - --non-interactive \ - --xml \ - install \ - --auto-agree-with-licenses \ - --dry-run \ - %s | \ - xml sel -t \ - -m \"stream/install-summary/to-install/solvable[@type='package']\" \ - -c \"string(@name)\" -n -" - (if verbose then "set -x" else "") - (if verbose then "--verbose --verbose" else "--quiet") - (match packager_config with None -> "" - | Some filename -> sprintf "--config %s" filename) - tmpdir repos_dir tmpdir (String.concat " " (List.map Filename.quote names)) in - let pkg_names = run_command_get_lines cmd in - - (* Return list of package names, remove empty lines. *) - List.filter (fun s -> s <> "") pkg_names - -let zypp_rpm_resolve_dependencies_and_download names mode = - if mode = PkgNamesOnly then ( - eprintf "supermin: zypp-rpm: --names-only flag is not implemented\n"; - exit 1 - ); - - if use_installed then - zypp_rpm_resolve_dependencies_and_download_use_installed names - else - zypp_rpm_resolve_dependencies_and_download_no_installed names - -let rec zypp_rpm_list_files pkg = - (* Run rpm -qlp with some extra magic. *) - let cmd = - sprintf "rpm -q --qf '[%%{FILENAMES} %%{FILEFLAGS:fflags} %%{FILEMODES} %%{FILESIZES}\\n]' %s %S" - (if use_installed then "" else "-p") - pkg in - let lines = run_command_get_lines cmd in - - let files = - filter_map ( - fun line -> - match string_split " " line with - | [filename; flags; mode; size] -> - let test_flag = String.contains flags in - let mode = int_of_string mode in - let size = int_of_string size in - if test_flag 'd' then None (* ignore documentation *) - else ( - (* Skip unreadable files when called as non-root *) - if Unix.getuid() > 0 && - (try Unix.access filename [Unix.R_OK]; false with - Unix_error _ -> eprintf "supermin: EPERM %s\n%!" filename; true) then None - else - Some (filename, { - ft_dir = mode land 0o40000 <> 0; - ft_ghost = test_flag 'g'; ft_config = test_flag 'c'; - ft_mode = mode; ft_size = size; - }) - ) - | _ -> - eprintf "supermin: bad output from rpm command: '%s'" line; - exit 1 - ) lines in - - (* I've never understood why the base packages like 'filesystem' don't - * contain any /dev nodes at all. This leaves every program that - * bootstraps RPMs to create a varying set of device nodes themselves. - * This collection was copied from mock/backend.py. - *) - let files = - let b = Filename.basename pkg in - if string_prefix "filesystem-" b then ( - let dirs = [ "/proc"; "/sys"; "/dev"; "/dev/pts"; "/dev/shm"; - "/dev/mapper" ] in - let dirs = - List.map (fun name -> - name, { ft_dir = true; ft_ghost = false; - ft_config = false; ft_mode = 0o40755; - ft_size = 0 }) dirs in - let devs = [ "/dev/null"; "/dev/full"; "/dev/zero"; "/dev/random"; - "/dev/urandom"; "/dev/tty"; "/dev/console"; - "/dev/ptmx"; "/dev/stdin"; "/dev/stdout"; "/dev/stderr" ] in - (* No need to set the mode because these will go into hostfiles. *) - let devs = - List.map (fun name -> - name, { ft_dir = false; ft_ghost = false; - ft_config = false; ft_mode = 0o644; - ft_size = 0 }) devs in - dirs @ devs @ files - ) else files in - - files - -let zypp_rpm_get_file_from_package pkg file = - if use_installed then - file - else ( - debug "extracting %s from %s ..." file (Filename.basename pkg); - - let outfile = tmpdir // file in - let cmd = - sprintf "umask 0000; rpm2cpio %s | (cd %s && cpio --quiet -id .%s)" - (Filename.quote pkg) (Filename.quote tmpdir) (Filename.quote file) in - run_command cmd; - outfile - ) - -let () = - let ph = { - ph_detect = zypp_rpm_detect; - ph_init = zypp_rpm_init; - ph_resolve_dependencies_and_download = - zypp_rpm_resolve_dependencies_and_download; - ph_list_files = zypp_rpm_list_files; - ph_get_file_from_package = zypp_rpm_get_file_from_package; - } in - register_package_handler "zypp-rpm" ph diff --git a/src/types.ml b/src/types.ml new file mode 100644 index 0000000..4051773 --- /dev/null +++ b/src/types.ml @@ -0,0 +1,19 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +type format = Chroot | Ext2 diff --git a/src/utils.ml b/src/utils.ml new file mode 100644 index 0000000..b3b1448 --- /dev/null +++ b/src/utils.ml @@ -0,0 +1,194 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +open Unix +open Printf + +let (+^) = Int64.add +let (-^) = Int64.sub +let ( *^ ) = Int64.mul +let (/^) = Int64.div + +let (//) = Filename.concat +let quote = Filename.quote +let quoted_list names = String.concat " " (List.map quote names) + +let file_exists name = + try access name [F_OK]; true + with Unix_error _ -> false + +let dir_exists name = + try (stat name).st_kind = S_DIR + with Unix_error _ -> false + +let rec uniq ?(cmp = Pervasives.compare) = function + | [] -> [] + | [x] -> [x] + | x :: y :: xs when cmp x y = 0 -> + uniq ~cmp (x :: xs) + | x :: y :: xs -> + x :: uniq ~cmp (y :: xs) + +let sort_uniq ?(cmp = Pervasives.compare) xs = + let xs = List.sort cmp xs in + let xs = uniq ~cmp xs in + xs + +let rec input_all_lines chan = + try let line = input_line chan in line :: input_all_lines chan + with End_of_file -> [] + +let run_command_get_lines cmd = + let chan = open_process_in cmd in + let lines = input_all_lines chan in + let stat = close_process_in chan in + (match stat with + | WEXITED 0 -> () + | WEXITED i -> + eprintf "supermin: command '%s' failed (returned %d), see earlier error messages\n" cmd i; + exit i + | WSIGNALED i -> + eprintf "supermin: command '%s' killed by signal %d" cmd i; + exit 1 + | WSTOPPED i -> + eprintf "supermin: command '%s' stopped by signal %d" cmd i; + exit 1 + ); + lines + +let run_command cmd = + if Sys.command cmd <> 0 then ( + eprintf "supermin: %s: command failed, see earlier errors\n" cmd; + exit 1 + ) + +let run_shell code args = + let cmd = sprintf "sh -c %s arg0 %s" + (Filename.quote code) + (String.concat " " (List.map Filename.quote args)) in + if Sys.command cmd <> 0 then ( + eprintf "supermin: external shell program failed, see earlier error messages\n"; + exit 1 + ) + +let run_python code args = + let cmd = sprintf "python -c %s %s" + (Filename.quote code) + (String.concat " " (List.map Filename.quote args)) in + if Sys.command cmd <> 0 then ( + eprintf "supermin: external python program failed, see earlier error messages\n"; + exit 1 + ) + +let rec find s sub = + let len = String.length s in + let sublen = String.length sub in + let rec loop i = + if i <= len-sublen then ( + let rec loop2 j = + if j < sublen then ( + if s.[i+j] = sub.[j] then loop2 (j+1) + else -1 + ) else + i (* found *) + in + let r = loop2 0 in + if r = -1 then loop (i+1) else r + ) else + -1 (* not found *) + in + loop 0 + +let rec string_split sep str = + let len = String.length str in + let seplen = String.length sep in + let i = find str sep in + if i = -1 then [str] + else ( + let s' = String.sub str 0 i in + let s'' = String.sub str (i+seplen) (len-i-seplen) in + s' :: string_split sep s'' + ) + +let string_prefix p str = + let len = String.length str in + let plen = String.length p in + len >= plen && String.sub str 0 plen = p + +let path_prefix p path = + let len = String.length path in + let plen = String.length p in + path = p || (len > plen && String.sub path 0 (plen+1) = (p ^ "/")) + +let rec filter_map f = function + | [] -> [] + | x :: xs -> + let x = f x in + match x with + | None -> filter_map f xs + | Some x -> x :: filter_map f xs + +let rex_numbers = Str.regexp "^\$[0-9]+\$\$.*\$$" +let rex_letters = Str.regexp_case_fold "^\$[a-z]+\$\$.*\$$" + +let rec compare_version v1 v2 = + compare (split_version v1) (split_version v2) + +and split_version = function + | "" -> [] + | str -> + let first, rest = + if Str.string_match rex_numbers str 0 then ( + let n = Str.matched_group 1 str in + let rest = Str.matched_group 2 str in + let n = + try `Number (int_of_string n) + with Failure "int_of_string" -> `String n in + n, rest + ) + else if Str.string_match rex_letters str 0 then + `String (Str.matched_group 1 str), Str.matched_group 2 str + else ( + let len = String.length str in + `Char str.[0], String.sub str 1 (len-1) + ) in + first :: split_version rest + +let compare_architecture a1 a2 = + let index_of_architecture = function + | "i386" | "i486" | "i586" | "i686" | "x86_32" | "x86-32" -> 32 + | "x86_64" | "x86-64" | "amd64" -> 64 + | "armel" | "armhf" -> 32 + | "aarch64" -> 64 + | a when string_prefix "armv5" a -> 32 + | a when string_prefix "armv6" a -> 32 + | a when string_prefix "armv7" a -> 32 + | a when string_prefix "armv8" a -> 64 + | "ppc" | "ppc32" -> 32 + | "ppc64" -> 64 + | "sparc" | "sparc32" -> 32 + | "sparc64" -> 64 + | "ia64" -> 64 + | "s390" -> 32 + | "s390x" -> 64 + | "alpha" -> 64 + | a -> + eprintf "supermin: missing support for architecture '%s'\nIt may need to be added to supermin.\n" a; + exit 1 + in + compare (index_of_architecture a1) (index_of_architecture a2) diff --git a/src/utils.mli b/src/utils.mli new file mode 100644 index 0000000..eee39fa --- /dev/null +++ b/src/utils.mli @@ -0,0 +1,90 @@ +(* supermin 5 + * Copyright (C) 2009-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +(** Utilities. *) + +val (+^) : int64 -> int64 -> int64 +val (-^) : int64 -> int64 -> int64 +val ( *^ ) : int64 -> int64 -> int64 +val (/^) : int64 -> int64 -> int64 + (** Int64 operators. *) + +val file_exists : string -> bool + (** Return [true] iff file exists. *) + +val dir_exists : string -> bool + (** Return [true] iff dir exists. *) + +val uniq : ?cmp:('a -> 'a -> int) -> 'a list -> 'a list + (** Uniquify a list (the list must be sorted first). *) + +val sort_uniq : ?cmp:('a -> 'a -> int) -> 'a list -> 'a list + (** Sort and uniquify a list. *) + +val input_all_lines : in_channel -> string list + (** Input all lines from a channel, returning a list of lines. *) + +val run_command_get_lines : string -> string list + (** Run the command and read the list of lines that it prints to stdout. *) + +val run_command : string -> unit + (** Run a command using {!Sys.command} and exit if it fails. Be careful + when constructing the command to properly quote any arguments + (using {!Filename.quote}). *) + +val run_shell : string -> string list -> unit + (** [run_shell code args] runs shell [code] with arguments [args]. + This does not return anything, but exits with an error message + if the shell code returns an error. *) + +val run_python : string -> string list -> unit + (** [run_python code args] runs Python [code] with arguments [args]. + This does not return anything, but exits with an error message + if the Python code returns an error. *) + +val (//) : string -> string -> string + (** [x // y] concatenates file paths [x] and [y] into a single path. *) + +val quote : string -> string + (** Quote a string to protect it from shell interpretation. *) + +val quoted_list : string list -> string + (** Quote a list of strings to protect them from shell interpretation. *) + +val find : string -> string -> int +(** [find str sub] searches for [sub] in [str], returning the index + or -1 if not found. *) + +val string_split : string -> string -> string list + (** [string_split sep str] splits [str] at [sep]. *) + +val string_prefix : string -> string -> bool + (** [string_prefix prefix str] returns true iff [str] starts with [prefix]. *) + +val path_prefix : string -> string -> bool + (** [path_prefix prefix path] returns true iff [path] is [prefix] or + [path] starts with [prefix/]. *) + +val filter_map : ('a -> 'b option) -> 'a list -> 'b list + (** map + filter *) + +val compare_version : string -> string -> int + (** Compare two version-like strings. *) + +val compare_architecture : string -> string -> int + (** Compare two architecture strings. *) -- 1.8.4.2

10 years, 8 months

1
0
0 / 0

[PATCH] builder: add a mandatory 'arch' key in index files

by Pino Toscano

Introduce a mandatory arch= key in all the entries of index files, to identify which architecture is each. Adapt the long and JSON list outputs to print also this new field. This introduces an incompatibility with index files created with virt-builder < 1.26, as they will be rejected until entries will have the arch= key added (which is ignored by older virt-builder, so adding it will not create backward-compatibility issues). --- builder/index_parser.ml | 9 +++++++++ builder/index_parser.mli | 1 + builder/list_entries.ml | 4 ++++ builder/test-index.in | 7 +++++++ builder/test-virt-builder-list.sh | 14 ++++++++++++++ 5 files changed, 35 insertions(+) diff --git a/builder/index_parser.ml b/builder/index_parser.ml index 2d4a642..de4d72e 100644 --- a/builder/index_parser.ml +++ b/builder/index_parser.ml @@ -27,6 +27,7 @@ and entry = { printable_name : string option; (* the name= field *) osinfo : string option; file_uri : string; + arch : string; signature_uri : string option; (* deprecated, will be removed in 1.26 *) checksum_sha512 : string option; revision : int; @@ -43,6 +44,7 @@ and entry = { let print_entry chan (name, { printable_name = printable_name; file_uri = file_uri; + arch = arch; osinfo = osinfo; signature_uri = signature_uri; checksum_sha512 = checksum_sha512; @@ -65,6 +67,7 @@ let print_entry chan (name, { printable_name = printable_name; | Some id -> fp "osinfo=%s\n" id ); fp "file=%s\n" file_uri; + fp "arch=%s\n" arch; (match signature_uri with | None -> () | Some uri -> fp "sig=%s\n" uri @@ -179,6 +182,11 @@ let get_index ~prog ~debug ~downloader ~sigchecker source = with Not_found -> eprintf (f_"virt-builder: no 'file' (URI) entry for '%s'\n") n; corrupt_file () in + let arch = + try List.assoc ("arch", None) fields + with Not_found -> + eprintf (f_"virt-builder: no 'arch' entry for '%s'\n") n; + corrupt_file () in let signature_uri = try Some (make_absolute_uri (List.assoc ("sig", None) fields)) with Not_found -> None in @@ -245,6 +253,7 @@ let get_index ~prog ~debug ~downloader ~sigchecker source = let entry = { printable_name = printable_name; osinfo = osinfo; file_uri = file_uri; + arch = arch; signature_uri = signature_uri; checksum_sha512 = checksum_sha512; revision = revision; diff --git a/builder/index_parser.mli b/builder/index_parser.mli index 3c679b3..0575dc4 100644 --- a/builder/index_parser.mli +++ b/builder/index_parser.mli @@ -21,6 +21,7 @@ and entry = { printable_name : string option; (* the name= field *) osinfo : string option; file_uri : string; + arch : string; signature_uri : string option; (* deprecated, will be removed in 1.26 *) checksum_sha512 : string option; revision : int; diff --git a/builder/list_entries.ml b/builder/list_entries.ml index 27ea95e..edf7dfb 100644 --- a/builder/list_entries.ml +++ b/builder/list_entries.ml @@ -73,6 +73,7 @@ and list_entries_long ~sources index = List.iter ( fun (name, { Index_parser.printable_name = printable_name; + arch = arch; size = size; compressed_size = compressed_size; notes = notes; @@ -83,6 +84,7 @@ and list_entries_long ~sources index = | None -> () | Some name -> printf "%-24s %s\n" (s_"Full name:") name; ); + printf "%-24s %s\n" (s_"Architecture:") arch; printf "%-24s %s\n" (s_"Minimum/default size:") (human_size size); (match compressed_size with | None -> () @@ -168,6 +170,7 @@ and list_entries_json ~sources index = printf " \"templates\": [\n"; iteri ( fun i (name, { Index_parser.printable_name = printable_name; + arch = arch; size = size; compressed_size = compressed_size; notes = notes; @@ -175,6 +178,7 @@ and list_entries_json ~sources index = printf " {\n"; printf " \"os-version\": \"%s\",\n" name; json_optional_printf_string "full-name" printable_name; + printf " \"arch\": \"%s\",\n" arch; printf " \"size\": %Ld,\n" size; json_optional_printf_int64 "compressed-size" compressed_size; print_notes notes; diff --git a/builder/test-index.in b/builder/test-index.in index 1bca6b8..3efebc4 100644 --- a/builder/test-index.in +++ b/builder/test-index.in @@ -1,5 +1,6 @@ [phony-debian] name=Phony Debian +arch=x86_64 file=debian.xz format=raw size=536870912 @@ -9,6 +10,7 @@ notes=Phony Debian look-alike used for testing. [phony-fedora] name=Phony Fedora +arch=x86_64 file=fedora.xz format=raw size=1073741824 @@ -18,6 +20,7 @@ notes=Phony Fedora look-alike used for testing. [phony-fedora-qcow2] name=Phony Fedora qcow2 +arch=x86_64 file=fedora.qcow2.xz format=qcow2 size=1073741824 @@ -27,6 +30,7 @@ notes=Phony Fedora look-alike used for testing. [phony-fedora-qcow2-uncompressed] name=Phony Fedora qcow2 uncompressed +arch=x86_64 file=fedora.qcow2 format=qcow2 size=1073741824 @@ -36,6 +40,7 @@ notes=Phony Fedora look-alike used for testing. [phony-fedora-no-format] name=Phony Fedora +arch=x86_64 file=fedora.qcow2.xz size=1073741824 expand=/dev/sda2 @@ -44,6 +49,7 @@ notes=Phony Fedora look-alike used for testing. [phony-ubuntu] name=Phony Ubuntu +arch=x86_64 file=ubuntu.xz format=raw size=536870912 @@ -52,6 +58,7 @@ notes=Phony Ubuntu look-alike used for testing. [phony-windows] name=Phony Windows +arch=x86_64 file=windows.xz format=raw size=536870912 diff --git a/builder/test-virt-builder-list.sh b/builder/test-virt-builder-list.sh index 634254d..7174152 100755 --- a/builder/test-virt-builder-list.sh +++ b/builder/test-virt-builder-list.sh @@ -46,6 +46,7 @@ Fingerprint: F777 4FB1 AD07 4A7E 8C87 67EA 9173 8F73 E1B7 68A0 os-version: phony-debian Full name: Phony Debian +Architecture: x86_64 Minimum/default size: 512.0M Notes: @@ -54,6 +55,7 @@ Phony Debian look-alike used for testing. os-version: phony-fedora Full name: Phony Fedora +Architecture: x86_64 Minimum/default size: 1.0G Notes: @@ -62,6 +64,7 @@ Phony Fedora look-alike used for testing. os-version: phony-fedora-qcow2 Full name: Phony Fedora qcow2 +Architecture: x86_64 Minimum/default size: 1.0G Notes: @@ -70,6 +73,7 @@ Phony Fedora look-alike used for testing. os-version: phony-fedora-qcow2-uncompressed Full name: Phony Fedora qcow2 uncompressed +Architecture: x86_64 Minimum/default size: 1.0G Notes: @@ -78,6 +82,7 @@ Phony Fedora look-alike used for testing. os-version: phony-fedora-no-format Full name: Phony Fedora +Architecture: x86_64 Minimum/default size: 1.0G Notes: @@ -86,6 +91,7 @@ Phony Fedora look-alike used for testing. os-version: phony-ubuntu Full name: Phony Ubuntu +Architecture: x86_64 Minimum/default size: 512.0M Notes: @@ -94,6 +100,7 @@ Phony Ubuntu look-alike used for testing. os-version: phony-windows Full name: Phony Windows +Architecture: x86_64 Minimum/default size: 512.0M Notes: @@ -118,6 +125,7 @@ if [ "$json_list" != "{ { \"os-version\": \"phony-debian\", \"full-name\": \"Phony Debian\", + \"arch\": \"x86_64\", \"size\": 536870912, \"notes\": { \"C\": \"Phony Debian look-alike used for testing.\" @@ -127,6 +135,7 @@ if [ "$json_list" != "{ { \"os-version\": \"phony-fedora\", \"full-name\": \"Phony Fedora\", + \"arch\": \"x86_64\", \"size\": 1073741824, \"notes\": { \"C\": \"Phony Fedora look-alike used for testing.\" @@ -136,6 +145,7 @@ if [ "$json_list" != "{ { \"os-version\": \"phony-fedora-qcow2\", \"full-name\": \"Phony Fedora qcow2\", + \"arch\": \"x86_64\", \"size\": 1073741824, \"notes\": { \"C\": \"Phony Fedora look-alike used for testing.\" @@ -145,6 +155,7 @@ if [ "$json_list" != "{ { \"os-version\": \"phony-fedora-qcow2-uncompressed\", \"full-name\": \"Phony Fedora qcow2 uncompressed\", + \"arch\": \"x86_64\", \"size\": 1073741824, \"notes\": { \"C\": \"Phony Fedora look-alike used for testing.\" @@ -154,6 +165,7 @@ if [ "$json_list" != "{ { \"os-version\": \"phony-fedora-no-format\", \"full-name\": \"Phony Fedora\", + \"arch\": \"x86_64\", \"size\": 1073741824, \"notes\": { \"C\": \"Phony Fedora look-alike used for testing.\" @@ -163,6 +175,7 @@ if [ "$json_list" != "{ { \"os-version\": \"phony-ubuntu\", \"full-name\": \"Phony Ubuntu\", + \"arch\": \"x86_64\", \"size\": 536870912, \"notes\": { \"C\": \"Phony Ubuntu look-alike used for testing.\" @@ -172,6 +185,7 @@ if [ "$json_list" != "{ { \"os-version\": \"phony-windows\", \"full-name\": \"Phony Windows\", + \"arch\": \"x86_64\", \"size\": 536870912, \"notes\": { \"C\": \"Phony Windows look-alike used for testing.\" -- 1.8.3.1

10 years, 8 months

2
2
0 / 0

[PATCH] sysprep: use Mkdtemp to create the temporary directory

by Pino Toscano

Use the safer mkdtemp instead of manually creating a path. --- sysprep/Makefile.am | 2 ++ sysprep/sysprep_operation_script.ml | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sysprep/Makefile.am b/sysprep/Makefile.am index 2600477..9c9ab14 100644 --- a/sysprep/Makefile.am +++ b/sysprep/Makefile.am @@ -100,6 +100,8 @@ OBJECTS = \ $(top_builddir)/mllib/timezone.cmx \ $(top_builddir)/mllib/firstboot.cmx \ $(top_builddir)/mllib/config.cmx \ + $(top_builddir)/mllib/mkdtemp-c.o \ + $(top_builddir)/mllib/mkdtemp.cmx \ sysprep_operation.cmx \ $(patsubst %,sysprep_operation_%.cmx,$(operations)) \ main.cmx diff --git a/sysprep/sysprep_operation_script.ml b/sysprep/sysprep_operation_script.ml index 518207e..614910f 100644 --- a/sysprep/sysprep_operation_script.ml +++ b/sysprep/sysprep_operation_script.ml @@ -44,9 +44,7 @@ let rec script_perform (g : Guestfs.guestfs) root side_effects = match !scriptdir with | Some dir -> dir, false | None -> - let tmpdir = Filename.temp_dir_name in - let tmpdir = tmpdir // string_random8 () in - mkdir tmpdir 0o755; + let tmpdir = Mkdtemp.mkdtemp (Filename.temp_dir_name // "virt-sysprep.XXXXXX") in tmpdir, true in (* Mount the directory locally. *) -- 1.8.3.1

10 years, 8 months

2
1
0 / 0

[PATCH] builder: split INI C <-> OCaml glue code in own module

by Pino Toscano

Move in an own module the code which calls the C virt_builder_parse_index and does the array -> list conversion of the result. This way this code can be easily called also in places different than Index_parser without the need to copy the types mapping, etc. Just code motion, no actual behaviour changes. --- builder/Makefile.am | 3 +++ builder/index_parser.ml | 17 +---------------- builder/ini_reader.ml | 38 ++++++++++++++++++++++++++++++++++++++ builder/ini_reader.mli | 24 ++++++++++++++++++++++++ po/POTFILES-ml | 1 + 5 files changed, 67 insertions(+), 16 deletions(-) create mode 100644 builder/ini_reader.ml create mode 100644 builder/ini_reader.mli diff --git a/builder/Makefile.am b/builder/Makefile.am index 8c48be5..a72b7ac 100644 --- a/builder/Makefile.am +++ b/builder/Makefile.am @@ -46,6 +46,8 @@ SOURCES = \ index_parser.mli \ index_parser.ml \ index-parser-c.c \ + ini_reader.mli \ + ini_reader.ml \ list_entries.mli \ list_entries.ml \ paths.ml \ @@ -94,6 +96,7 @@ OBJECTS = \ pxzcat.cmx \ setlocale-c.o \ setlocale.cmx \ + ini_reader.cmx \ paths.cmx \ get_kernel.cmx \ downloader.cmx \ diff --git a/builder/index_parser.ml b/builder/index_parser.ml index de4d72e..9fd9cda 100644 --- a/builder/index_parser.ml +++ b/builder/index_parser.ml @@ -102,15 +102,6 @@ let print_entry chan (name, { printable_name = printable_name; ) notes; if hidden then fp "hidden=true\n" -(* Types returned by the C index parser. *) -type sections = section array -and section = string * fields (* [name] + fields *) -and fields = field array -and field = string * string option * string (* key + subkey + value *) - -(* Calls yyparse in the C code. *) -external parse_index : string -> sections = "virt_builder_parse_index" - let get_index ~prog ~debug ~downloader ~sigchecker source = let corrupt_file () = eprintf (f_"\nThe index file downloaded from '%s' is corrupt.\nYou need to ask the supplier of this file to fix it and upload a fixed version.\n") @@ -128,16 +119,10 @@ let get_index ~prog ~debug ~downloader ~sigchecker source = Sigchecker.verify sigchecker tmpfile; (* Try parsing the file. *) - let sections = parse_index tmpfile in + let sections = Ini_reader.read_ini tmpfile in if delete_tmpfile then (try Unix.unlink tmpfile with _ -> ()); - let sections = Array.to_list sections in - let sections = List.map ( - fun (n, fields) -> - n, Array.to_list fields - ) sections in - (* Check for repeated os-version names. *) let nseen = Hashtbl.create 13 in List.iter ( diff --git a/builder/ini_reader.ml b/builder/ini_reader.ml new file mode 100644 index 0000000..fbd4d2f --- /dev/null +++ b/builder/ini_reader.ml @@ -0,0 +1,38 @@ +(* virt-builder + * Copyright (C) 2013-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +type sections = section list +and section = string * fields (* [name] + fields *) +and fields = field list +and field = string * string option * string (* key + subkey + value *) + +(* Types returned by the C index parser. *) +type c_sections = c_section array +and c_section = string * c_fields (* [name] + fields *) +and c_fields = field array + +(* Calls yyparse in the C code. *) +external parse_index : string -> c_sections = "virt_builder_parse_index" + +let read_ini file = + let sections = parse_index file in + let sections = Array.to_list sections in + List.map ( + fun (n, fields) -> + n, Array.to_list fields + ) sections diff --git a/builder/ini_reader.mli b/builder/ini_reader.mli new file mode 100644 index 0000000..992a1cb --- /dev/null +++ b/builder/ini_reader.mli @@ -0,0 +1,24 @@ +(* virt-builder + * Copyright (C) 2013-2014 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +type sections = section list +and section = string * fields (* [name] + fields *) +and fields = field list +and field = string * string option * string (* key + subkey + value *) + +val read_ini : string -> sections diff --git a/po/POTFILES-ml b/po/POTFILES-ml index d3c8918..2c9f5dc 100644 --- a/po/POTFILES-ml +++ b/po/POTFILES-ml @@ -3,6 +3,7 @@ builder/cmdline.ml builder/downloader.ml builder/get_kernel.ml builder/index_parser.ml +builder/ini_reader.ml builder/list_entries.ml builder/paths.ml builder/pxzcat.ml -- 1.8.3.1

10 years, 8 months

2
1
0 / 0

Scientific Linux template update?

by Adam Huffman

Hello Could the Scientific Linux template for virt-builder be updated to the recent 6.5 release? Thanks, Adam

10 years, 8 months

2
3
0 / 0

[PATCH supermin NOT TO BE APPLIED] Supermin 5 rewrite.

by Richard W.M. Jones

I am rewriting supermin to make it more robust. In particular, instead of storing the list of host files to be copied into the appliance (which is not robust if the host package is updated and new files are added), it now stores the list of packages, and works out the dependencies and full list of files by looking at the current package database (eg. rpm database). This patch is not complete, but it has reached a milestone where it is both able to build a supermin appliance in the new format, and construct a chroot full appliance. You can try it out like this: (1) Build a supermin appliance (in /tmp/supermin.d): ./src/supermin -v --prepare zlib bash coreutils kernel -o /tmp/supermin.d (2) Build a chroot full appliance: mkdir /tmp/chroot ./src/supermin -v --build --format chroot -o /tmp/chroot /tmp/supermin.d (Note that the old 'supermin-helper' program has been combined with the main tool. They are now just two different modes of the supermin tool: --prepare vs --build). There is a lot still to do. Rich.

10 years, 8 months

1
1
0 / 0

[W/ commit: 838a3c9] "inconsistent assumptions over interface Guestfs"

by Kashyap Chamarthy

Just did a git pull to the below commit (to get the latest version 1.25.37) and did a local build, this is what I see: Error: The files ../ocaml/guestfs.cmi and perl_edit.cmi make inconsistent assumptions over interface Guestfs Haven't investigated further, thought I'd post it here first: $ git rev-parse --short HEAD 838a3c9 $ ./autogen.sh $ echo $? 0 $ make -j48 [. . .] findlib: [WARNING] Interface perl_edit.cmi occurs in several directories: ., ../mllib findlib: [WARNING] Interface perl_edit.cmi occurs in several directories: ., ../mllib findlib: [WARNING] Interface perl_edit.cmi occurs in several directories: ., ../mllib CC virt_index_validate-index-scan.o ocamlfind ocamlc -g -warn-error CDEFLMPSUVYZX -package str,unix -I ../src/.libs -I ../ocaml -I ../mllib -package gettext-stub -c list_entries.mli -o list_entries.cmi CC index-scan.o findlib: [WARNING] Interface perl_edit.cmi occurs in several directories: ., ../mllib ocamlfind ocamlopt -g -warn-error CDEFLMPSUVYZX -package str,unix -I ../src/.libs -I ../ocaml -I ../mllib -package gettext-stub -c index_parser.ml -o index_parser.cmx ocamlfind ocamlopt -g -warn-error CDEFLMPSUVYZX -package str,unix -I ../src/.libs -I ../ocaml -I ../mllib -package gettext-stub -c cmdline.ml -o cmdline.cmx findlib: [WARNING] Interface perl_edit.cmi occurs in several directories: ., ../mllib findlib: [WARNING] Interface perl_edit.cmi occurs in several directories: ., ../mllib ocamlfind ocamlopt -g -warn-error CDEFLMPSUVYZX -package str,unix -I ../src/.libs -I ../ocaml -I ../mllib -package gettext-stub -c list_entries.ml -o list_entries.cmx findlib: [WARNING] Interface perl_edit.cmi occurs in several directories: ., ../mllib CCLD virt-index-validate ocamlfind ocamlopt -g -warn-error CDEFLMPSUVYZX -package str,unix -I ../src/.libs -I ../ocaml -I ../mllib -package gettext-stub -c builder.ml -o builder.cmx findlib: [WARNING] Interface perl_edit.cmi occurs in several directories: ., ../mllib File "builder.ml", line 1: Error: The files ../ocaml/guestfs.cmi and perl_edit.cmi make inconsistent assumptions over interface Guestfs make[2]: *** [builder.cmx] Error 2 make[2]: Leaving directory `/home/tuser1/src/libguestfs/builder' make[1]: *** [all-recursive] Error 1 make[1]: Leaving directory `/home/tuser1/src/libguestfs' make: *** [all] Error 2 -- /kashyap

10 years, 8 months

2
2
0 / 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Libguestfs February 2014