We have extended collectd virt plugin to extract info about disk usage from a libvirt domain using libguestfs. In addition to my previous mail I am attaching some more infomration about the problem. Currently the collectd plugin works fine and retrieves the required statistics. The problem that I face happens after certain number of cycles (getting disk usage statistics). Collectd is terminated with the following error:
This happens after about 490-500 calls of my statistics read function. Collectd is terminated with signal SIGABRT with the following backtrace:
(gdb) bt #0 0x00007ffff71f2e97 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007ffff71f4801 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007ffff723d897 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007ffff72e8cff in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x00007ffff72e8d21 in __fortify_fail () from /lib/x86_64-linux-gnu/libc.so.6 #5 0x00007ffff72e6a10 in __chk_fail () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x00007ffff72e8c0a in __fdelt_warn () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x00007ffff47ed8ba in loop (cmd=cmd@entry=0x7fffbc02abe0) at ../../../lib/command.c:662 #8 0x00007ffff47ee2f5 in guestfs_int_cmd_run (cmd=0x7fffbc02abe0) at ../../../lib/command.c:767 #9 0x00007ffff47efefc in disk_create_qcow2 (optargs=0x7fffbc008d60, backingfile=0x7fffbc09ef30 "/var/lib/nova/instances/5ca86029-d296-4261-9a67-908bdd6c4eab/disk", size=-1, orig_filename=0x7fffbc0301f0 "/tmp/libguestfszZtmRI/overlay1.qcow2", g=0x7fffbc008d60) at ../../../lib/create.c:348 #10 guestfs_impl_disk_create (g=g@entry=0x7fffbc008d60, filename=filename@entry=0x7fffbc0301f0 "/tmp/libguestfszZtmRI/overlay1.qcow2", format=<optimized out>, format@entry=0x7ffff4843998 "qcow2", size=size@entry=-1, optargs=optargs@entry=0x7fffca7fb1a0) at ../../../lib/create.c:88 #11 0x00007ffff4794ca5 in guestfs_disk_create_argv (g=g@entry=0x7fffbc008d60, filename=filename@entry=0x7fffbc0301f0 "/tmp/libguestfszZtmRI/overlay1.qcow2", format=format@entry=0x7ffff4843998 "qcow2", size=size@entry=-1, optargs=optargs@entry=0x7fffca7fb1a0) at ../../../lib/actions-3.c:224 #12 0x00007ffff4807b18 in create_cow_overlay_direct (g=0x7fffbc008d60, datav=<optimized out>, drv=0x7fffbc03a890) at ../../../lib/launch-direct.c:89 #13 0x00007ffff47f0b44 in create_overlay (g=0x7fffbc008d60, drv=0x7fffbc03a890) at ../../../lib/drives.c:87 #14 0x00007ffff47f0d7b in create_drive_file (g=g@entry=0x7fffbc008d60, data=data@entry=0x7fffca7fb2a0) at ../../../lib/drives.c:119 #15 0x00007ffff47f1c55 in guestfs_impl_add_drive_opts (g=g@entry=0x7fffbc008d60, filename=<optimized out>, filename@entry=0x7fffbc10d540 "/var/lib/nova/instances/5ca86029-d296-4261-9a67-908bdd6c4eab/disk", optargs=optargs@entry=0x7fffca7fb420) at ../../../lib/drives.c:826 #16 0x00007ffff4784927 in guestfs_add_drive_opts_argv (g=g@entry=0x7fffbc008d60, filename=filename@entry=0x7fffbc10d540 "/var/lib/nova/instances/5ca86029-d296-4261-9a67-908bdd6c4eab/disk", optargs=optargs@entry=0x7fffca7fb420) at ../../../lib/actions-2.c:180 #17 0x00007ffff48128e0 in add_disk (g=g@entry=0x7fffbc008d60, filename=0x7fffbc10d540 "/var/lib/nova/instances/5ca86029-d296-4261-9a67-908bdd6c4eab/disk", format=<optimized out>, readonly_in_xml=<optimized out>, protocol=<optimized out>, server=<optimized out>, username=0x0, secret=0x0, datavp=0x7fffca7fb610) at ../../../lib/libvirt-domain.c:396 #18 0x00007ffff4813cd6 in for_each_disk (f=0x7ffff48127e0 <add_disk>, data=0x7fffca7fb610, doc=0x7fffbc10cf80, conn=0x7fffbc01fa10, g=0x7fffbc008d60) at ../../../lib/libvirt-domain.c:782 #19 guestfs_impl_add_libvirt_dom (g=g@entry=0x7fffbc008d60, domvp=domvp@entry=0x7fffbc064000, optargs=optargs@entry=0x7fffca7fb790) at ../../../lib/libvirt-domain.c:323 #20 0x00007ffff47ab2c3 in guestfs_add_libvirt_dom_argv (g=g@entry=0x7fffbc008d60, dom=dom@entry=0x7fffbc064000, optargs=optargs@entry=0x7fffca7fb790) at ../../../lib/actions-4.c:174 #21 0x00007ffff4812cf6 in guestfs_impl_add_domain (g=g@entry=0x7fffbc008d60, domain_name=domain_name@entry=0x7fffbc0338e0 "tve50:00000013", optargs=optargs@entry=0x7fffca7fb8c0) at ../../../lib/libvirt-domain.c:163 #22 0x00007ffff4760368 in guestfs_add_domain_argv (g=0x7fffbc008d60, dom=0x7fffbc0338e0 "tve50:00000013", optargs=optargs@entry=0x7fffca7fb8c0) at ../../../lib/actions-0.c:139 #23 0x00007ffff47dfc38 in guestfs_add_domain_va (g=<optimized out>, dom=<optimized out>, args=args@entry=0x7fffca7fb920) at ../../../lib/actions-variants.c:107 #24 0x00007ffff47dfee4 in guestfs_add_domain (g=g@entry=0x7fffbc008d60, dom=dom@entry=0x7fffbc0338e0 "tve50:00000013") at ../../../lib/actions-variants.c:45 #25 0x00007ffff4a78bec in refresh_lists (inst=inst@entry=0x7ffff4c7f940 <lv_read_user_data>) at src/virt.c:2049 #26 0x00007ffff4a7a327 in lv_read (ud=<optimized out>) at src/virt.c:1656 #27 0x0000555555564a1c in plugin_read_thread (args=<optimized out>) at src/daemon/plugin.c:540 #28 0x00007ffff79b66db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #29 0x00007ffff72d588f in clone () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) quit
Deeper analysis shows the error is related to file descriptors handling using FD_* macros in loop() function, most probably trying to access fd which is outside the valid range. Any hints how to avoid or handle this situation would be highly appreciated.