Hi guys, During the last few weeks, I've been looking at how to use SPDK as QEMU's bootable rootfs disk. I managed to boot up a SPDK rootfs disk by using OVMF UEFI boot-loader. And in order to deploy the guest OS before start-up(which has a unrecognizable filesystem to the host), I have written a small patch for the libguestfs. It was based on Redhat's CentOS libguestfs-1.36.10 RPM, hopefully somebody could add this SPDK support into the mainline code. What I'm asking here is a new problem I encountered yesterday. When I increased the guest memory to more than 8GB, the guest kernel would crash at boot-up (see attached screen-shot). This issue would not be reproduced if I only use SPDK as the data disk. Maybe there is something wrong with the boot-loader? Thanks, Bob <0083-Add-SPDK-drive-support.patch> ------------------------------------------------------ From: Bob Chen <a175818323(a)gmail.com&gt; Date: Fri, 27 Jul 2018 12:55:28 +0800 Subject: [PATCH 1/1] Add SPDK drive support --- lib/drives.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ lib/guestfs-internal.h | 2 ++ lib/launch-direct.c | 20 ++++++++++++++++++-- lib/launch-libvirt.c | 3 +++ lib/qemu.c | 3 +++ 5 files changed, 71 insertions(+), 2 deletions(-) diff --git a/lib/drives.c b/lib/drives.c index d3887c1..780b1b2 100644 --- a/lib/drives.c +++ b/lib/drives.c @@ -396,6 +396,45 @@ create_drive_iscsi (guestfs_h *g, } #endif /* DISABLED IN RHEL 7 */ +static struct drive * +create_drive_spdk (guestfs_h *g, + const struct drive_create_data *data) +{ + if (data->username != NULL) { + error (g, _("spdk: you cannot specify a username with this protocol")); + return NULL; + } + if (data->secret != NULL) { + error (g, _("spdk: you cannot specify a secret with this protocol")); + return NULL; + } + + if (data->nr_servers != 1) { + error (g, _("spdk: you must specify exactly one server")); + return NULL; + } + + if (data->servers[0].transport != drive_transport_unix) { + error (g, _("spdk: only unix transport is supported")); + return NULL; + } + + return create_drive_non_file (g, data); +} + +bool +guestfs_int_has_spdk_drive (guestfs_h *g) +{ + size_t i; + struct drive *drv; + ITER_DRIVES(g, i, drv) { + if (drv->src.protocol == drive_protocol_spdk) { + return true; + } + } + return false; +} + /** * Create the special F</dev/null> drive. * @@ -494,6 +533,7 @@ guestfs_int_drive_protocol_to_string (enum drive_protocol protocol) case drive_protocol_sheepdog: return "sheepdog"; case drive_protocol_ssh: return "ssh"; case drive_protocol_tftp: return "tftp"; + case drive_protocol_spdk: return "spdk"; } abort (); } @@ -878,6 +918,11 @@ guestfs_impl_add_drive_opts (guestfs_h *g, const char *filename, drv = create_drive_curl (g, &data); } #endif /* DISABLED IN RHEL 7 */ + else if (STREQ (protocol, "spdk")) { + data.protocol = drive_protocol_spdk; + data.iface = safe_strdup(g, "vhost-user-scsi-pci"); + drv = create_drive_spdk(g, &data); + } else { error (g, _("unknown protocol '%s'"), protocol); drv = NULL; /*FALLTHROUGH*/ diff --git a/lib/guestfs-internal.h b/lib/guestfs-internal.h index 3bae02b..a0e5573 100644 --- a/lib/guestfs-internal.h +++ b/lib/guestfs-internal.h @@ -199,6 +199,7 @@ enum drive_protocol { drive_protocol_sheepdog, drive_protocol_ssh, drive_protocol_tftp, + drive_protocol_spdk, }; enum drive_transport { @@ -829,6 +830,7 @@ extern void guestfs_int_rollback_drives (guestfs_h *g, size_t); extern void guestfs_int_add_dummy_appliance_drive (guestfs_h *g); extern void guestfs_int_free_drives (guestfs_h *g); extern const char *guestfs_int_drive_protocol_to_string (enum drive_protocol protocol); +extern bool guestfs_int_has_spdk_drive (guestfs_h *g); /* appliance.c */ extern int guestfs_int_build_appliance (guestfs_h *g, char **kernel, char **initrd, char **appliance); diff --git a/lib/launch-direct.c b/lib/launch-direct.c index 3d6e72e..f157324 100644 --- a/lib/launch-direct.c +++ b/lib/launch-direct.c @@ -409,8 +409,17 @@ launch_direct (guestfs_h *g, void *datav, const char *arg) ADD_CMDLINE_PRINTF ("%d", g->smp); } - ADD_CMDLINE ("-m"); - ADD_CMDLINE_PRINTF ("%d", g->memsize); + if (guestfs_int_has_spdk_drive(g)) { + ADD_CMDLINE ("-m"); + ADD_CMDLINE ("1G"); + ADD_CMDLINE ("-object"); + ADD_CMDLINE ("memory-backend-file,id=mem0,size=1G,mem-path=/dev/hugepages,share=on"); + ADD_CMDLINE ("-numa"); + ADD_CMDLINE ("node,memdev=mem0"); + } else { + ADD_CMDLINE ("-m"); + ADD_CMDLINE_PRINTF ("%d", g->memsize); + } /* Force exit instead of reboot on panic */ ADD_CMDLINE ("-no-reboot"); @@ -567,6 +576,13 @@ launch_direct (guestfs_h *g, void *datav, const char *arg) goto cleanup0; } #endif + else if (drv->iface && STREQ(drv->iface, "vhost-user-scsi-pci")) { + /* SPDK */ + ADD_CMDLINE ("-chardev"); + ADD_CMDLINE_PRINTF ("socket,id=vhost,path=%s", escaped_file); + ADD_CMDLINE ("-device"); + ADD_CMDLINE ("vhost-user-scsi-pci,chardev=vhost"); + } else if (drv->iface) { ADD_CMDLINE ("-drive"); ADD_CMDLINE_PRINTF ("%s,if=%s", param, drv->iface); diff --git a/lib/launch-libvirt.c b/lib/launch-libvirt.c index 05c398b..468ece9 100644 --- a/lib/launch-libvirt.c +++ b/lib/launch-libvirt.c @@ -1548,6 +1548,9 @@ construct_libvirt_xml_disk (guestfs_h *g, case drive_protocol_tftp: error (g, _("libvirt does not support the qemu curl driver protocols (ftp, http, etc.); try setting LIBGUESTFS_BACKEND=direct")); return -1; + case drive_protocol_spdk: + error (g, _("libvirt does not support SPDK driver protocol yet; try setting LIBGUESTFS_BACKEND=direct")); + return -1; } if (construct_libvirt_xml_disk_target (g, xo, drv_index) == -1) diff --git a/lib/qemu.c b/lib/qemu.c index 887e31b..a77ea87 100644 --- a/lib/qemu.c +++ b/lib/qemu.c @@ -942,6 +942,8 @@ guestfs_int_drive_source_qemu_param (guestfs_h *g, case drive_protocol_tftp: return make_uri (g, "tftp", src->username, src->secret, &src->servers[0], src->u.exportname); + case drive_protocol_spdk: + return safe_strdup(g, src->servers[0].u.socket); } abort (); @@ -1016,6 +1018,7 @@ guestfs_int_discard_possible (guestfs_h *g, struct drive *drv, case drive_protocol_https: case drive_protocol_ssh: case drive_protocol_tftp: + case drive_protocol_spdk: NOT_SUPPORTED (g, -1, _("discard cannot be enabled on this drive: " "protocol '%s' does not support discard"), -- 1.8.3.1