[PATCH libnbd 0/4] lib: Implement TCP fastopen
Since NBD makes several TCP connections (especially if using multi-conn, as you should), it may benefit from using TFO on platforms which support it. This commit simply adds new APIs for enabling, disabling and fetching the flag, and updates the documentation. It does not (yet) implement the feature. I found these articles about TFO interesting: https://blog.apnic.net/2021/07/05/tcp-fast-open-not-so-fast/ https://candrews.integralblue.com/2019/03/the-sad-story-of-tcp-fast-open/ --- docs/libnbd.pod | 3 +++ generator/API.ml | 48 +++++++++++++++++++++++++++++++++++++++++++++--- lib/handle.c | 15 +++++++++++++++ lib/internal.h | 1 + 4 files changed, 64 insertions(+), 3 deletions(-) diff --git a/docs/libnbd.pod b/docs/libnbd.pod index af6a10008..0c8027b82 100644 --- a/docs/libnbd.pod +++ b/docs/libnbd.pod @@ -770,6 +770,9 @@ If you are issuing multiple in-flight requests (see above) and limiting the number, then the limit should be applied to each individual NBD connection. +If opening multiple connections over TCP, TCP fastopen (TFO) +may be beneficial in some circumstances. See L<nbd_set_tcp_fastopen(3)>. + =head1 ENCRYPTION AND AUTHENTICATION The NBD protocol and libnbd supports TLS (sometimes incorrectly called diff --git a/generator/API.ml b/generator/API.ml index 8ee1843ae..a19e193d0 100644 --- a/generator/API.ml +++ b/generator/API.ml @@ -2204,7 +2204,8 @@ "connect_tcp", { as C<\"nbd\">, or it may be a port number as a string such as C<\"10809\">. " ^ blocking_connect_call_description; - see_also = [Link "aio_connect_tcp"; Link "set_opt_mode"]; + see_also = [Link "aio_connect_tcp"; Link "set_opt_mode"; + Link "set_tcp_fastopen"]; }; "connect_socket", { @@ -2516,7 +2517,7 @@ "can_multi_conn", { required." ^ non_blocking_test_call_description; see_also = [SectionLink "Flag calls"; SectionLink "Multi-conn"; - Link "opt_info"]; + Link "opt_info"; Link "set_tcp_fastopen"]; example = Some "examples/server-flags.c"; }; @@ -3307,7 +3308,8 @@ "aio_connect_tcp", { Begin connecting to the NBD server listening on C<hostname:port>. Parameters behave as documented in L<nbd_connect_tcp(3)>. " ^ async_connect_call_description; - see_also = [ Link "connect_tcp"; Link "set_opt_mode" ]; + see_also = [ Link "connect_tcp"; Link "set_opt_mode"; + Link "set_tcp_fastopen" ]; }; "aio_connect_socket", { @@ -4335,6 +4337,44 @@ "is_uri", { see_also = [Link "connect_uri"; Link "aio_connect_uri"; Link "supports_uri"; Link "get_uri"]; }; + + "set_tcp_fastopen", { + default_call with + args = [ Bool "enable" ]; ret = RErr; + permitted_states = [ Created ]; + shortdesc = "enable or disable TCP fastopen (TFO)"; + longdesc = "\ +Enable or disable TCP fastopen (TFO). You should call this +before L<nbd_connect_tcp(3)>. When enabled, this allows faster +connections over TCP by omitting part of the TCP 3-way handshake. + +It is not supported for non-IP sockets (like Unix domain sockets). +Furthermore the server must also support TFO, which only +L<nbdkit(1)> does at the time of writing. There are several +other restrictions which may prevent TFO from working — see the articles +linked below. In the case where TFO is not supported, setting this +flag does nothing. + +Note the default is not defined and may vary depending on +the platform and may change in future. You can read the +default by opening the handle and calling L<nbd_get_tcp_fastopen(3)>."; + see_also = [Link "get_tcp_fastopen"; + URLLink "https://blog.apnic.net/2021/07/05/tcp-fast-open-not-so-fast/"; + URLLink "https://candrews.integralblue.com/2019/03/the-sad-story-of-tcp-fast-open/"]; + }; + + "get_tcp_fastopen", { + default_call with + args = []; ret = RBool; + may_set_error = false; + shortdesc = "get the TCP fastopen (TFO) setting"; + longdesc = "\ +Get the TCP fastopen (TFO) setting. + +Note the default is not defined and may vary depending on +the platform and may change in future."; + see_also = [Link "set_tcp_fastopen"]; + }; ] (* The first stable version that the symbol appeared in, for @@ -4514,6 +4554,8 @@ let first_version = "get_tls_hostname", (1, 22); "is_uri", (1, 22); "get_subprocess_pid", (1, 22); + "set_tcp_fastopen", (1, 22); + "get_tcp_fastopen", (1, 22); (* These calls are proposed for a future version of libnbd, but * have not been added to any released version so far. diff --git a/lib/handle.c b/lib/handle.c index a263cc4c6..f297b78c3 100644 --- a/lib/handle.c +++ b/lib/handle.c @@ -69,6 +69,7 @@ nbd_create (void) } h->unique = 1; + h->fastopen = false; h->tls_verify_peer = true; h->request_eh = true; h->request_sr = true; @@ -697,3 +698,17 @@ nbd_unlocked_stats_chunks_received (struct nbd_handle *h) { return h->chunks_received; } + +int +nbd_unlocked_set_tcp_fastopen (struct nbd_handle *h, bool enable) +{ + h->fastopen = enable; + return 0; +} + +/* NB: may_set_error = false. */ +int +nbd_unlocked_get_tcp_fastopen (struct nbd_handle *h) +{ + return h->fastopen; +} diff --git a/lib/internal.h b/lib/internal.h index d1fc971f0..7e76ee674 100644 --- a/lib/internal.h +++ b/lib/internal.h @@ -111,6 +111,7 @@ struct nbd_handle { char *export_name; /* Export name, never NULL. */ char *sact_name; /* Socket activation name, can be NULL. */ + bool fastopen; /* TCP fastopen (TFO). */ /* TLS settings. */ int tls; /* 0 = disable, 1 = enable, 2 = require */ -- 2.46.0
We use the simpler (Linux only) implementation where you just set the TCP_FASTOPEN_CONNECT socket option. The alternative method of using sendmsg + MSG_FASTOPEN requires more complicated userspace changes. --- generator/states-connect.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/generator/states-connect.c b/generator/states-connect.c index 07d4e234e..00d771d3a 100644 --- a/generator/states-connect.c +++ b/generator/states-connect.c @@ -63,6 +63,23 @@ disable_sigpipe (int sock) #endif } +/* Set TCP fastopen (TFO) on connect if supported, but don't fail. */ +static void +set_tcp_fastopen (struct nbd_handle *h, int family, int sock) +{ +#ifdef TCP_FASTOPEN_CONNECT + const int flag = 1; + + if (!h->fastopen) return; + if (family != AF_INET && family != AF_INET6) return; + + if (setsockopt (sock, IPPROTO_TCP, TCP_FASTOPEN_CONNECT, + &flag, sizeof flag) == -1) + debug (h, "ignoring error trying to request TCP fastopen: %s", + strerror (errno)); +#endif /* TCP_FASTOPEN_CONNECT */ +} + STATE_MACHINE { CONNECT.START: sa_family_t family; @@ -84,6 +101,7 @@ CONNECT.START: disable_nagle (fd); disable_sigpipe (fd); + set_tcp_fastopen (h, family, fd); r = connect (fd, (struct sockaddr *)&h->connaddr, h->connaddrlen); if (r == 0 || (r == -1 && errno == EINPROGRESS)) @@ -197,6 +215,7 @@ CONNECT_TCP.CONNECT: disable_nagle (fd); disable_sigpipe (fd); + set_tcp_fastopen (h, h->rp->ai_family, fd); if (connect (fd, h->rp->ai_addr, h->rp->ai_addrlen) == -1) { if (errno != EINPROGRESS) { -- 2.46.0
If TCP_FASTOPEN_CONNECT is defined it's a sign that we're being compiled on recent Linux. Try, experimentally, enabling TFO always on this platform. Note that the documentation intentionally does not define the default for TFO, so we can revert this later if it causes issues. Also note that even on Linux you won't get TFO unless you change sysctl net.ipv4.tcp_fastopen = 1 and use a server which supports it (ie. nbdkit). --- lib/handle.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/handle.c b/lib/handle.c index f297b78c3..cf486e556 100644 --- a/lib/handle.c +++ b/lib/handle.c @@ -29,6 +29,7 @@ #include <netdb.h> #include <sys/types.h> #include <sys/wait.h> +#include <netinet/tcp.h> #include "ascii-ctype.h" #include "internal.h" @@ -69,7 +70,11 @@ nbd_create (void) } h->unique = 1; +#ifdef TCP_FASTOPEN_CONNECT + h->fastopen = true; +#else h->fastopen = false; +#endif h->tls_verify_peer = true; h->request_eh = true; h->request_sr = true; -- 2.46.0
--- .gitignore | 1 + examples/Makefile.am | 16 +++++ examples/connect-benchmark.c | 112 +++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+) diff --git a/.gitignore b/.gitignore index 0d4bef92c..7bc0a6778 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,7 @@ Makefile.in /dump/nbddump.1 /examples/aio-connect-read /examples/batched-read-write +/examples/connect-benchmark /examples/connect-command /examples/connect-uri /examples/copy-libev diff --git a/examples/Makefile.am b/examples/Makefile.am index bad22cb11..b52996a24 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -22,6 +22,7 @@ EXTRA_DIST = LICENSE-FOR-EXAMPLES noinst_PROGRAMS = \ aio-connect-read \ batched-read-write \ + connect-benchmark \ connect-command \ connect-uri \ encryption \ @@ -62,6 +63,21 @@ batched_read_write_LDADD = \ $(top_builddir)/lib/libnbd.la \ $(NULL) +connect_benchmark_SOURCES = \ + connect-benchmark.c \ + $(NULL) +connect_benchmark_CPPFLAGS = \ + -I$(top_srcdir)/include \ + $(NULL) +connect_benchmark_CFLAGS = \ + $(WARNINGS_CFLAGS) \ + $(PTHREAD_CFLAGS) \ + $(NULL) +connect_benchmark_LDADD = \ + $(top_builddir)/lib/libnbd.la \ + $(PTHREAD_LIBS) \ + $(NULL) + connect_command_SOURCES = \ connect-command.c \ $(NULL) diff --git a/examples/connect-benchmark.c b/examples/connect-benchmark.c new file mode 100644 index 000000000..1e153b915 --- /dev/null +++ b/examples/connect-benchmark.c @@ -0,0 +1,112 @@ +/* The example benchmarks multiple parallel connections to an NBD + * server. You can use it to time how long it takes to connect under + * various circumstances (eg. TLS vs no TLS, old vs newstyle, with or + * without TCP fastopen). It only does the NBD handshake and then + * immediately disconnects. + * + * Typical usage: + * ./examples/connect-benchmark nbd://localhost 16 1000 + * where the parameters are: + * URI of the server + * number of threads + * total number of connections to make (across all threads) + * + * Or: + * nbdkit null --run './examples/connect-benchmark "$uri" 16 1000' + * + * Use 'hyperfine' to time multiple runs. + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <stdint.h> +#include <inttypes.h> +#include <errno.h> + +#include <pthread.h> + +#include <libnbd.h> + +static const char *uri; +static unsigned nr_threads, nr_connections; +static _Atomic unsigned count = 0; + +static void *start_thread (void *arg); + +int +main (int argc, char *argv[]) +{ + pthread_t *threads; + unsigned i; + int err; + + if (argc != 4) { + fprintf (stderr, "%s URI threads connections\n", argv[0]); + exit (EXIT_FAILURE); + } + uri = argv[1]; + if (sscanf (argv[2], "%u", &nr_threads) != 1 || + sscanf (argv[3], "%u", &nr_connections) != 1) { + fprintf (stderr, "%s: cannot read number of threads or connections\n", + argv[0]); + exit (EXIT_FAILURE); + } + + threads = calloc (nr_threads, sizeof threads[0]); + if (threads == NULL) { + perror ("calloc"); + exit (EXIT_FAILURE); + } + + /* Start the worker threads, one per connection. */ + for (i = 0; i < nr_threads; ++i) { + err = pthread_create (&threads[i], NULL, start_thread, NULL); + if (err != 0) { + errno = err; + perror ("pthread_create"); + exit (EXIT_FAILURE); + } + } + + /* Wait for the threads to exit. */ + for (i = 0; i < nr_threads; ++i) { + err = pthread_join (threads[i], NULL); + if (err != 0) { + errno = err; + perror ("pthread_join"); + exit (EXIT_FAILURE); + } + } + + exit (EXIT_SUCCESS); +} + +static void * +start_thread (void *arg) +{ + struct nbd_handle *nbd; + unsigned i; + + for (;;) { + i = count++; + if (i >= nr_connections) break; + + nbd = nbd_create (); + if (nbd == NULL) { + fprintf (stderr, "%s\n", nbd_get_error ()); + exit (EXIT_FAILURE); + } + + if (nbd_connect_uri (nbd, uri) == -1) { + fprintf (stderr, "%s\n", nbd_get_error ()); + exit (EXIT_FAILURE); + } + + nbd_close (nbd); + } + + pthread_exit (NULL); +} -- 2.46.0
participants (1)
-
Richard W.M. Jones