During the last few weeks, I've been looking at how to use SPDK as QEMU's bootable rootfs disk.
I managed to boot up a SPDK rootfs disk by using OVMF UEFI boot-loader. And in order to deploy the guest OS before start-up(which has a unrecognizable filesystem to the host), I have written a small patch for the libguestfs. It was based on Redhat's CentOS libguestfs-1.36.10 RPM, hopefully somebody could add this SPDK support into the mainline code.
What I'm asking here is a new problem I encountered yesterday. When I increased the guest memory to more than 8GB, the guest kernel would crash at boot-up (see attached screen-shot). This issue would not be reproduced if I only use SPDK as the data disk.
From 7927b0fa080db923989cef7181531dfaa09ccf6c Mon Sep 17 00:00:00 2001
Date: Fri, 27 Jul 2018 12:55:28 +0800
Subject: [PATCH 1/1] Add SPDK drive support
---
lib/drives.c | 45 +++++++++++++++++++++++++++++++++++++++++++++
lib/guestfs-internal.h | 2 ++
lib/launch-direct.c | 20 ++++++++++++++++++--
lib/launch-libvirt.c | 3 +++
lib/qemu.c | 3 +++
5 files changed, 71 insertions(+), 2 deletions(-)
diff --git a/lib/drives.c b/lib/drives.c
index d3887c1..780b1b2 100644
--- a/lib/drives.c
+++ b/lib/drives.c
@@ -396,6 +396,45 @@ create_drive_iscsi (guestfs_h *g,
}
#endif /* DISABLED IN RHEL 7 */
+static struct drive *
+create_drive_spdk (guestfs_h *g,
+ const struct drive_create_data *data)
+{
+ if (data->username != NULL) {
+ error (g, _("spdk: you cannot specify a username with this protocol"));
+ return NULL;
+ }
+ if (data->secret != NULL) {
+ error (g, _("spdk: you cannot specify a secret with this protocol"));
+ return NULL;
+ }
+
+ if (data->nr_servers != 1) {
+ error (g, _("spdk: you must specify exactly one server"));
+ return NULL;
+ }
+
+ if (data->servers[0].transport != drive_transport_unix) {
+ error (g, _("spdk: only unix transport is supported"));
+ return NULL;
+ }
+
+ return create_drive_non_file (g, data);
+}
+
+bool
+guestfs_int_has_spdk_drive (guestfs_h *g)
+{
+ size_t i;
+ struct drive *drv;
+ ITER_DRIVES(g, i, drv) {
+ if (drv->src.protocol == drive_protocol_spdk) {
+ return true;
+ }
+ }
+ return false;
+}
+
/**
* Create the special F</dev/null> drive.
*
@@ -494,6 +533,7 @@ guestfs_int_drive_protocol_to_string (enum drive_protocol protocol)
case drive_protocol_sheepdog: return "sheepdog";
case drive_protocol_ssh: return "ssh";
case drive_protocol_tftp: return "tftp";
+ case drive_protocol_spdk: return "spdk";
}
abort ();
}
@@ -878,6 +918,11 @@ guestfs_impl_add_drive_opts (guestfs_h *g, const char *filename,
drv = create_drive_curl (g, &data);
}
#endif /* DISABLED IN RHEL 7 */
+ else if (STREQ (protocol, "spdk")) {
+ data.protocol = drive_protocol_spdk;
+ data.iface = safe_strdup(g, "vhost-user-scsi-pci");
+ drv = create_drive_spdk(g, &data);
+ }
else {
error (g, _("unknown protocol '%s'"), protocol);
drv = NULL; /*FALLTHROUGH*/
diff --git a/lib/guestfs-internal.h b/lib/guestfs-internal.h
index 3bae02b..a0e5573 100644
--- a/lib/guestfs-internal.h
+++ b/lib/guestfs-internal.h
@@ -199,6 +199,7 @@ enum drive_protocol {
drive_protocol_sheepdog,
drive_protocol_ssh,
drive_protocol_tftp,
+ drive_protocol_spdk,
};
enum drive_transport {
@@ -829,6 +830,7 @@ extern void guestfs_int_rollback_drives (guestfs_h *g, size_t);
extern void guestfs_int_add_dummy_appliance_drive (guestfs_h *g);
extern void guestfs_int_free_drives (guestfs_h *g);
extern const char *guestfs_int_drive_protocol_to_string (enum drive_protocol protocol);
+extern bool guestfs_int_has_spdk_drive (guestfs_h *g);
/* appliance.c */
extern int guestfs_int_build_appliance (guestfs_h *g, char **kernel, char **initrd, char **appliance);
diff --git a/lib/launch-direct.c b/lib/launch-direct.c
index 3d6e72e..f157324 100644
--- a/lib/launch-direct.c
+++ b/lib/launch-direct.c
@@ -409,8 +409,17 @@ launch_direct (guestfs_h *g, void *datav, const char *arg)
ADD_CMDLINE_PRINTF ("%d", g->smp);
}
- ADD_CMDLINE ("-m");
- ADD_CMDLINE_PRINTF ("%d", g->memsize);
+ if (guestfs_int_has_spdk_drive(g)) {
+ ADD_CMDLINE ("-m");
+ ADD_CMDLINE ("1G");
+ ADD_CMDLINE ("-object");
+ ADD_CMDLINE ("memory-backend-file,id=mem0,size=1G,mem-path=/dev/hugepages,share=on");
+ ADD_CMDLINE ("-numa");
+ ADD_CMDLINE ("node,memdev=mem0");
+ } else {
+ ADD_CMDLINE ("-m");
+ ADD_CMDLINE_PRINTF ("%d", g->memsize);
+ }
/* Force exit instead of reboot on panic */
ADD_CMDLINE ("-no-reboot");
@@ -567,6 +576,13 @@ launch_direct (guestfs_h *g, void *datav, const char *arg)
goto cleanup0;
}
#endif
+ else if (drv->iface && STREQ(drv->iface, "vhost-user-scsi-pci")) {
+ /* SPDK */
+ ADD_CMDLINE ("-chardev");
+ ADD_CMDLINE_PRINTF ("socket,id=vhost,path=%s", escaped_file);
+ ADD_CMDLINE ("-device");
+ ADD_CMDLINE ("vhost-user-scsi-pci,chardev=vhost");
+ }
else if (drv->iface) {
ADD_CMDLINE ("-drive");
ADD_CMDLINE_PRINTF ("%s,if=%s", param, drv->iface);
diff --git a/lib/launch-libvirt.c b/lib/launch-libvirt.c
index 05c398b..468ece9 100644
--- a/lib/launch-libvirt.c
+++ b/lib/launch-libvirt.c
@@ -1548,6 +1548,9 @@ construct_libvirt_xml_disk (guestfs_h *g,
case drive_protocol_tftp:
error (g, _("libvirt does not support the qemu curl driver protocols (ftp, http, etc.); try setting LIBGUESTFS_BACKEND=direct"));
return -1;
+ case drive_protocol_spdk:
+ error (g, _("libvirt does not support SPDK driver protocol yet; try setting LIBGUESTFS_BACKEND=direct"));
+ return -1;
}
if (construct_libvirt_xml_disk_target (g, xo, drv_index) == -1)
diff --git a/lib/qemu.c b/lib/qemu.c
index 887e31b..a77ea87 100644
--- a/lib/qemu.c
+++ b/lib/qemu.c
@@ -942,6 +942,8 @@ guestfs_int_drive_source_qemu_param (guestfs_h *g,
case drive_protocol_tftp:
return make_uri (g, "tftp", src->username, src->secret,
&src->servers[0], src->u.exportname);
+ case drive_protocol_spdk:
+ return safe_strdup(g, src->servers[0].u.socket);
}
abort ();
@@ -1016,6 +1018,7 @@ guestfs_int_discard_possible (guestfs_h *g, struct drive *drv,
case drive_protocol_https:
case drive_protocol_ssh:
case drive_protocol_tftp:
+ case drive_protocol_spdk:
NOT_SUPPORTED (g, -1,
_("discard cannot be enabled on this drive: "
"protocol '%s' does not support discard"),
--
1.8.3.1