[PATCH] New APIs: hopen-device hopen-file hread hwrite hseek hclose hclose-all
by Matthew Booth
These APIs provide handle-based read/write streaming and random access to files
and block devices.
---
daemon/.gitignore | 1 +
daemon/Makefile.am | 1 +
daemon/hfile.c | 196 +++++++++++++++++++++++++++++++++++++++++++++
daemon/m4/gnulib-cache.m4 | 3 +-
po/POTFILES.in | 2 +
regressions/Makefile.am | 1 +
regressions/test-hfile.pl | 102 +++++++++++++++++++++++
src/MAX_PROC_NR | 2 +-
src/generator.ml | 114 ++++++++++++++++++++++++++
9 files changed, 420 insertions(+), 2 deletions(-)
create mode 100644 daemon/hfile.c
create mode 100755 regressions/test-hfile.pl
diff --git a/daemon/.gitignore b/daemon/.gitignore
index e06c684..c9837be 100644
--- a/daemon/.gitignore
+++ b/daemon/.gitignore
@@ -50,6 +50,7 @@ m4/gettime.m4
m4/gettimeofday.m4
m4/getpagesize.m4
m4/getugroups.m4
+m4/gl_list.m4
m4/glibc21.m4
m4/glob.m4
m4/gnulib-common.m4
diff --git a/daemon/Makefile.am b/daemon/Makefile.am
index 0c8be08..c243bb6 100644
--- a/daemon/Makefile.am
+++ b/daemon/Makefile.am
@@ -94,6 +94,7 @@ guestfsd_SOURCES = \
guestfsd.c \
headtail.c \
hexdump.c \
+ hfile.c \
htonl.c \
initrd.c \
inotify.c \
diff --git a/daemon/hfile.c b/daemon/hfile.c
new file mode 100644
index 0000000..fedb502
--- /dev/null
+++ b/daemon/hfile.c
@@ -0,0 +1,196 @@
+/* libguestfs - the guestfsd daemon
+ * Copyright (C) 2010 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <config.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "gl_array_list.h"
+
+#include "daemon.h"
+#include "actions.h"
+
+gl_list_t handles;
+
+static void init_handle_list (void) __attribute__((constructor));
+void
+init_handle_list (void)
+{
+ handles = gl_list_nx_create_empty(GL_ARRAY_LIST, NULL, NULL, NULL, false);
+}
+
+static int
+hopen_check(int fd, const char *path)
+{
+ if (fd < 0) {
+ reply_with_perror_errno(errno, "open %s failed", path);
+ return -1;
+ }
+
+ gl_list_node_t node = gl_list_nx_add_last(handles, (void *)(long)fd);
+ if (NULL == node) {
+ close(fd);
+
+ reply_with_error("gl_list_nx_add_last");
+ return -1;
+ }
+
+ return fd;
+}
+
+int /* RInt */
+do_hopen_file (const char *path, int flags)
+{
+ int fd;
+
+ CHROOT_IN;
+ if (flags && O_CREAT) {
+ fd = open(path, flags, 0777);
+ } else {
+ fd = open(path, flags);
+ }
+ CHROOT_OUT;
+
+ return hopen_check(fd, path);
+}
+
+int /* RInt */
+do_hopen_device (const char *device, int flags)
+{
+ int fd;
+
+ fd = open(device, flags);
+
+ return hopen_check(fd, device);
+}
+
+int /* RErr */
+do_hclose (int handle)
+{
+ gl_list_node_t node = gl_list_search(handles, (void *)(long)handle);
+ if (NULL == node) {
+ reply_with_error("%i does not refer to an open handle", handle);
+ return -1;
+ }
+ gl_list_remove_node(handles, node);
+
+ if (close(handle) < 0) {
+ reply_with_perror_errno(errno, "close handle %i failed", handle);
+ return -1;
+ }
+
+ return 0;
+}
+
+int /* RErr */
+do_hclose_all (void)
+{
+ const void *eltp;
+ gl_list_node_t node;
+ gl_list_iterator_t i = gl_list_iterator(handles);
+
+ char *msg = NULL;
+ size_t msglen = 0;
+
+ bool failed = false;
+ while (gl_list_iterator_next(&i, &eltp, &node)) {
+ gl_list_remove_node(handles, node);
+
+ int fd = (long)eltp;
+ if (close(fd) < 0) {
+ failed = true;
+
+ char *error = NULL;
+ int len = asprintf(&error, "close handle %i failed: %m ", fd);
+ if (len < 0) {
+ reply_with_error("asprintf");
+ }
+
+ msg = realloc(msg, msglen + len);
+ if (NULL == msg) {
+ reply_with_error("realloc");
+ return -1;
+ }
+
+ memcpy(msg + msglen, error, len);
+ msglen += len;
+ free(error);
+ }
+ }
+
+ if (failed) {
+ msg[msglen] = '\0';
+ reply_with_error("%s", msg);
+ return -1;
+ }
+
+ return 0;
+}
+
+char * /* RBufferOut */
+do_hread (int handle, int64_t size, size_t *size_r)
+{
+ char *buf = malloc(size);
+ if (NULL == buf) {
+ reply_with_error("malloc");
+ return NULL;
+ }
+
+ ssize_t in = read(handle, buf, size);
+ if (in < 0) {
+ reply_with_perror_errno(errno, "error reading from handle %i", handle);
+ goto error;
+ }
+
+ *size_r = in;
+ return buf;
+error:
+ free(buf);
+ return NULL;
+}
+
+int /* RErr */
+do_hwrite (int handle, const char *content, size_t content_size)
+{
+ size_t pos = 0;
+ while (pos < content_size) {
+ ssize_t out = write(handle, content + pos, content_size - pos);
+
+ if (out < 0) {
+ reply_with_perror_errno(errno, "error writing to handle %i", handle);
+ return -1;
+ }
+ pos += out;
+ }
+
+ return 0;
+}
+
+int64_t /* RInt64 */
+do_hseek (int handle, int64_t offset, int whence)
+{
+ int64_t ret = lseek(handle, offset, whence);
+ if (ret < 0) {
+ reply_with_perror_errno(errno, "error seeking in handle %i", handle);
+ return -1;
+ }
+
+ return ret;
+}
diff --git a/daemon/m4/gnulib-cache.m4 b/daemon/m4/gnulib-cache.m4
index f026cb3..25ffd27 100644
--- a/daemon/m4/gnulib-cache.m4
+++ b/daemon/m4/gnulib-cache.m4
@@ -15,11 +15,12 @@
# Specification in the form of a command-line invocation:
-# gnulib-tool --import --dir=. --lib=libgnu --source-base=lib --m4-base=m4 --doc-base=doc --tests-base=tests --aux-dir=build-aux --with-tests --no-libtool --macro-prefix=gl byteswap c-ctype connect error fsusage futimens getaddrinfo getline glob hash ignore-value manywarnings mkdtemp netdb openat perror pread read-file readlink select sleep socket strchrnul strndup symlinkat sys_select sys_wait vasprintf warnings
+# gnulib-tool --import --dir=. --lib=libgnu --source-base=lib --m4-base=m4 --doc-base=doc --tests-base=tests --aux-dir=build-aux --with-tests --no-libtool --macro-prefix=gl array-list byteswap c-ctype connect error fsusage futimens getaddrinfo getline glob hash ignore-value manywarnings mkdtemp netdb openat perror pread read-file readlink select sleep socket strchrnul strndup symlinkat sys_select sys_wait vasprintf warnings
# Specification in the form of a few gnulib-tool.m4 macro invocations:
gl_LOCAL_DIR([])
gl_MODULES([
+ array-list
byteswap
c-ctype
connect
diff --git a/po/POTFILES.in b/po/POTFILES.in
index e5fb857..c846e05 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -29,6 +29,7 @@ daemon/grub.c
daemon/guestfsd.c
daemon/headtail.c
daemon/hexdump.c
+daemon/hfile.c
daemon/htonl.c
daemon/initrd.c
daemon/inotify.c
@@ -99,6 +100,7 @@ perl/lib/Sys/Guestfs.pm
perl/lib/Sys/Guestfs/Lib.pm
python/guestfs-py.c
regressions/rhbz501893.c
+regressions/test-hfile.pl
regressions/test-lvm-mapping.pl
regressions/test-noexec-stack.pl
ruby/ext/guestfs/_guestfs.c
diff --git a/regressions/Makefile.am b/regressions/Makefile.am
index ca4292a..f7d9992 100644
--- a/regressions/Makefile.am
+++ b/regressions/Makefile.am
@@ -35,6 +35,7 @@ TESTS = \
test-cancellation-download-librarycancels.sh \
test-cancellation-upload-daemoncancels.sh \
test-find0.sh \
+ test-hfile.pl \
test-luks.sh \
test-lvm-filtering.sh \
test-lvm-mapping.pl \
diff --git a/regressions/test-hfile.pl b/regressions/test-hfile.pl
new file mode 100755
index 0000000..4c23722
--- /dev/null
+++ b/regressions/test-hfile.pl
@@ -0,0 +1,102 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use File::Temp qw(tempfile);
+use Sys::Guestfs;
+
+# Create a new guestfs handle
+my $g = Sys::Guestfs->new();
+
+# Create a 10M image file and add it to the handle
+my ($fh, $path) = tempfile();
+truncate($fh, 1024*1024*10);
+close($fh) or die("close $path failed: $!");
+$g->add_drive($path);
+
+$g->launch();
+
+# As qemu is now running we can unlink the image
+unlink($path) or die("unlink $path failed: $!");
+
+my $string = "abcd";
+
+# Test read and writing directly to a block device
+my $h = $g->hopen_device("/dev/vda", 2);
+$g->hwrite($h, $string);
+$g->hseek($h, 0, 0);
+my $in = $g->hread($h, length($string));
+
+die("device: hread returned $in") unless($in eq $string);
+$g->hclose($h);
+
+# Check $h is no longer valid
+eval {
+ $g->hseek($h, 0, 0);
+};
+die("device: handle wasn't closed") unless($@);
+
+# Make an ext2 filesystem and mount it
+$g->mkfs("ext2", "/dev/vda");
+$g->mount("/dev/vda", "/");
+
+# Open should fail read-only
+eval {
+ $h = $g->hopen_file("/test", 0);
+};
+die("file: read-only open non-existant file") unless($@);
+
+# Open should fail write-only without O_CREAT
+eval {
+ $h = $g->hopen_file("/test", 1);
+};
+die("file: write-only open non-existant file") unless($@);
+
+# Open should fail read-write without O_CREAT
+eval {
+ $h = $g->hopen_file("/test", 2);
+};
+die("file: read-write open non-existant file") unless($@);
+
+# Open the file write only, and create it
+$h = $g->hopen_file("/test", 1 | 64);
+
+# Write data to it
+$g->hwrite($h, $string);
+
+# Verify we can't read from it
+$g->hseek($h, 0, 0);
+eval {
+ $in = $g->hread($h, length($string));
+};
+die("file: read of write-only file") unless($@);
+
+# close and check the contents with cat
+$g->hclose($h);
+$in = $g->cat("/test");
+die("file: hwrite data test") unless($in eq $string);
+
+# Open the file read-only
+$h = $g->hopen_file("/test", 0);
+
+# Read the file in 1 byte chunks until the end
+$in = "";
+my $chunk;
+do {
+ $chunk = $g->hread($h, 1);
+ $in .= $chunk;
+} until(length($chunk) == 0);
+die("file: hread in chunks") unless($in eq $string);
+
+# Read past EOF returns a short read
+$g->hseek($h, 0, 0);
+$in = $g->hread($h, length($string) + 1);
+die("file: read past EOF") unless($in eq $string);
+
+# Test close-all
+$g->hclose_all();
+eval {
+ $g->hseek($h, 0, 0);
+};
+die("file: close-all") unless($@);
diff --git a/src/MAX_PROC_NR b/src/MAX_PROC_NR
index c1d1ffb..305aa98 100644
--- a/src/MAX_PROC_NR
+++ b/src/MAX_PROC_NR
@@ -1 +1 @@
-266
+273
diff --git a/src/generator.ml b/src/generator.ml
index 00caa6a..48010dc 100755
--- a/src/generator.ml
+++ b/src/generator.ml
@@ -5355,6 +5355,120 @@ filesystem can be found.
To find the label of a filesystem, use C<guestfs_vfs_label>.");
+ ("hopen_file", (RInt "handle", [Pathname "path"; Int "flags"]), 267, [],
+ [],
+ "open a file",
+ "\
+This command opens C<path> in the guest and returns a handle
+to it. The returned handle is an opaque int which can be passed
+to C<guestfs_h*> calls.
+
+C<flags> determines how the file is opened. It is a bitwise or of:
+
+=over
+
+=item 0
+
+(O_RDONLY) Open the file read only.
+
+=item 1
+
+(O_WRONLY) Open the file write only.
+
+=item 2
+
+(O_RDWR) Open the file read-write.
+
+=item 64
+
+(O_CREAT) Create the file if it does not already exist. The file
+will be created with a mode of 0777, modified by the daemon's
+umask. See L<UMASK>.
+
+=item 512
+
+(O_TRUNC) Truncate a file opened for write to zero length when
+opening.
+
+=item 1024
+
+(O_APPEND) Always write to the end of the file.
+
+=item 262144
+
+(O_NOATIME) Don't update the access time when reading from a file.
+
+=back");
+
+ ("hopen_device", (RInt "handle", [Device "device"; Int "flags"]), 268, [],
+ [],
+ "open a block device",
+ "\
+This command opens a block device and returns a handle to it.
+The returned handle is an opaque int which can be passed to
+C<guestfs_h*> calls.
+
+See L<guestfs_hopen_file> for a description of C<flags>.");
+
+ ("hclose", (RErr, [Int "handle"]), 269, [],
+ [],
+ "close a file handle",
+ "\
+This command closes C<handle> and releases all resources associated
+with it.");
+
+ ("hclose_all", (RErr, []), 270, [],
+ [],
+ "close all open file handles",
+ "\
+This command closes all file handles which have been opened with
+C<guestfs_hopen_*>.");
+
+ ("hread", (RBufferOut "content", [Int "handle"; Int64 "length"]), 271,
+ [ProtocolLimitWarning], [],
+ "read data from an open handle",
+ "\
+This command reads up to C<length> bytes from C<handle> and returns
+the data in C<content>. It will attempt to read exactly C<length>
+bytes, but may return less. Returned C<content> with size 0
+indicates EOF.");
+
+ ("hwrite", (RErr, [Int "handle"; BufferIn "content"]), 272,
+ [ProtocolLimitWarning], [],
+ "write data to an open handle",
+ "\
+This command writes all the data in C<content> to C<handle>. It
+returns an error if this fails.");
+
+ ("hseek", (RInt64 "newoffset", [Int "handle"; Int64 "offset";
+ Int "whence"]), 273, [],
+ [],
+ "seek on an open handle",
+ "\
+This command updates the current position in C<handle>. This
+affects the C<guestfs_hread> and C<guestfs_hwrite> calls.
+
+C<whence> determines the point of reference for C<offset>. It
+must be one of:
+
+=over
+
+=item 0
+
+C<offset> is relative to the beginning of the file.
+
+=item 1
+
+C<offset> is relative to the current position.
+
+=item 2
+
+C<offset> is relative to the end of the file.
+
+=back
+
+hseek returns the new offset from the beginning of the file.");
+
]
let all_functions = non_daemon_functions @ daemon_functions
--
1.7.2.2
14 years, 2 months
[PATCH v2] Add progress bars
by Richard Jones
This is an updated and extended version of the original patch:
https://www.redhat.com/archives/libguestfs/2010-August/msg00163.html
This adds OCaml and Perl bindings (both tested), support for
progress bars in virt-resize, and adds progress notifications
to a number of the simpler commands.
Still to do is to add progress messages to more commands. There
are still a few commands which would be fairly easy (eg. download and
some t*-out commands). There are many others where implementing
progress messages will be quite a lot more complex.
Note: I have changed the default frequency of messages. They are
now sent after two seconds, and every 1/3rd of a second thereafter.
This makes the progress bars smoother and avoids aliasing effects
when displaying time left in seconds.
Note (2): All known comments about the previous patches have been
incorporated.
Rich.
14 years, 2 months
[PATCH] Update pwrite to write a full buffer
by Matthew Booth
This patch changes pwrite such that it will never return a short write. This
is a backwards compatible API change, as checking for short writes remains
correct, but redundant.
---
daemon/file.c | 20 +++++++++++++-------
src/generator.ml | 6 ------
2 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/daemon/file.c b/daemon/file.c
index da899b6..9ce0783 100644
--- a/daemon/file.c
+++ b/daemon/file.c
@@ -501,7 +501,6 @@ int
do_pwrite (const char *path, const char *content, size_t size, int64_t offset)
{
int fd;
- ssize_t r;
CHROOT_IN;
fd = open (path, O_WRONLY);
@@ -512,11 +511,18 @@ do_pwrite (const char *path, const char *content, size_t size, int64_t offset)
return -1;
}
- r = pwrite (fd, content, size, offset);
- if (r == -1) {
- reply_with_perror ("pwrite: %s", path);
- close (fd);
- return -1;
+ size_t written = 0;
+ while (written < size) {
+ ssize_t r = pwrite (fd, content, size - written, offset);
+
+ if (r == -1) {
+ reply_with_perror ("pwrite: %s", path);
+ close (fd);
+ return -1;
+ }
+
+ offset += r;
+ written += r;
}
if (close (fd) == -1) {
@@ -525,7 +531,7 @@ do_pwrite (const char *path, const char *content, size_t size, int64_t offset)
return -1;
}
- return r;
+ return size;
}
/* This runs the 'file' command. */
diff --git a/src/generator.ml b/src/generator.ml
index dc072ef..7dcd7b4 100755
--- a/src/generator.ml
+++ b/src/generator.ml
@@ -5120,12 +5120,6 @@ file is the string C<content> (which can contain any 8 bit data).");
This command writes to part of a file. It writes the data
buffer C<content> to the file C<path> starting at offset C<offset>.
-This command implements the L<pwrite(2)> system call, and like
-that system call it may not write the full data requested. The
-return value is the number of bytes that were actually written
-to the file. This could even be 0, although short writes are
-unlikely for regular files in ordinary circumstances.
-
See also C<guestfs_pread>.");
("resize2fs_size", (RErr, [Device "device"; Int64 "size"]), 248, [],
--
1.7.2.2
14 years, 3 months
[PATCH] daemon: Set O_CLOEXEC flag on the virtio-serial port.
by Richard W.M. Jones
You can see that currently we leak the virtio-serial file
descriptor into child processes.
><fs> debug fds ''
0 /dev/console
1 /dev/console
2 /dev/console
3 /dev/vport0p1
4 /proc/252/fd
><fs> debug sh 'ls -l /proc/self/fd'
total 0
lr-x------ 1 root root 64 Aug 27 15:14 0 -> /dev/null
l-wx------ 1 root root 64 Aug 27 15:14 1 -> pipe:[5124]
l-wx------ 1 root root 64 Aug 27 15:14 2 -> pipe:[5124]
lrwx------ 1 root root 64 Aug 27 15:14 3 -> /dev/vport0p1
lr-x------ 1 root root 64 Aug 27 15:14 4 -> /proc/271/fd
The attached patch fixes this by setting close-on-exec. I'm using
O_CLOEXEC which will only work with relatively recent versions of
Linux.
Rich.
--
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
virt-top is 'top' for virtual machines. Tiny program with many
powerful monitoring features, net stats, disk stats, logging, etc.
http://et.redhat.com/~rjones/virt-top
14 years, 3 months
Re: [Libguestfs] How to install lib guestfs in CentOS
by Richard W.M. Jones
On Fri, Aug 27, 2010 at 04:37:31PM +0530, Saravanan S wrote:
> On Fri, Aug 27, 2010 at 4:10 PM, Richard W.M. Jones <rjones(a)redhat.com>wrote:
>
> >
> > ><fs> lvs
> > ><fs> file /dev/sda
> >
>
> ><fs> file /dev/sda
> x86 boot sector
What exactly is this image file? Can you post it somewhere so
I can take a closer look?
Rich.
--
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
New in Fedora 11: Fedora Windows cross-compiler. Compile Windows
programs, test, and build Windows installers. Over 70 libraries supprt'd
http://fedoraproject.org/wiki/MinGW http://www.annexia.org/fedora_mingw
14 years, 3 months
How to install lib guestfs in CentOS
by Saravanan S
Hi all,
I tried installing
*yum install '*guestf**' on CentOS 5.5
but this doesn't seem to work because of some dependency issues.
Could you please provide me a solution with yum than with source install?
Thanks
--
S Saravanan
14 years, 3 months
Planned libguestfs versions in Fedora 12-15, EPEL 5
by Richard W.M. Jones
Fedora 15 (Rawhide): Development branch 1.5.x.
As usual, this follows the current development versions (currently
1.5.5).
Fedora 14: Development branch 1.5.x.
When Fedora 14 branched from Rawhide a month back, we inadvertently
branched at a development version (1.5.2). We couldn't go back to
the stable branch (1.4.x) without bumping the RPM Epoch. In the
meantime Fedora 14 got a new kernel (2.6.35) which has a massive
performance regression with a particular qemu feature that we relied
on. We had to do a lot of work last week to work around this in
libguestfs. Since we've fixed this problem, I plan to just keep on
the development branch for Fedora 14. By upgrading to >= 1.5.4 we
work around the performance issue, and get many more people testing
the development version into the bargain.
Fedora 13: Stable branch 1.4.x + backports
Fedora 13 + febootstrap 2.8 + libguestfs git is the sweet spot for
performance right now. Because of the non-broken kernel plus the
performance improvements we've made upstream, the appliance boot
times are under 5 seconds.
Therefore I intend F13 to stay on the stable 1.4.x branch, and
backport some major features from 1.5.x:
- inspection code in C (improves performance of guestfish -i)
- ext2-based appliance (performance fix)
- virtio-serial (faster daemon communications)
and some minor features required by virt-v2v too. This is very
much going to be a test for a possible RHEL 6.1 version.
Fedora 12: Old stable branch 1.2.x.
EPEL 5: Old stable branch 1.2.x.
EPEL 5 is following the old stable 1.2.x branch, and I don't plan to
change that, except to push more stable versions along this branch.
Rich.
--
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
virt-df lists disk usage of guests without needing to install any
software inside the virtual machine. Supports Linux and Windows.
http://et.redhat.com/~rjones/virt-df/
14 years, 3 months
