This rather complex feature solves a problem for certain web services that require a cookie or token for access, especially one which must be periodically renewed. For motivation on this see the included documentation, and item (1)(b) here: https://www.redhat.com/archives/libguestfs/2020-July/msg00069.html --- plugins/curl/nbdkit-curl-plugin.pod | 120 +++++++++++ plugins/curl/Makefile.am | 2 + plugins/curl/curldefs.h | 74 +++++++ plugins/curl/auth-script.c | 311 ++++++++++++++++++++++++++++ plugins/curl/curl.c | 87 +++++--- 5 files changed, 561 insertions(+), 33 deletions(-) diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod index d9b2d275..1e0c26d8 100644 --- a/plugins/curl/nbdkit-curl-plugin.pod +++ b/plugins/curl/nbdkit-curl-plugin.pod @@ -36,6 +36,18 @@ ports and protocols used to serve NBD see L<nbdkit(1)>). =over 4 +=item B<auth-script=>SCRIPT + +Run C<SCRIPT> (a command or shell script fragment) to fetch a header +or cookie which is added to requests. This is useful for HTTP/HTTPS +services which need an authorization token, see +L</AUTHORIZATION SCRIPT> below. + +=item B<auth-script-renew=>SECONDS + +If the authorization header printed by C<auth-script> expires after a +certain number of seconds, set this parameter less than that. + =item B<cainfo=>FILENAME (nbdkit E<ge> 1.18) @@ -224,6 +236,114 @@ user-agent header. =back +=head1 AUTHORIZATION SCRIPT + +The C<auth-script> and C<auth-script-renew> parameters allow you to +access HTTP/HTTPS services which require an authorization token. +C<auth-script> should be a command or shell script fragment which +fetches the token and prints extra HTTP header(s). + +In the following example, an imaginary web service requires +authentication using a token fetched from a separate login server. +The token expires after 60 seconds, so we also tell the plugin that it +must renew the token (by re-running the script) if more than 50 +seconds have elapsed: + + nbdkit curl https://service.example.com/disk.img \ + auth-script=' + echo -n "Authorization: Bearer " + curl -s -X POST https://auth.example.com/login | + jq -r .token + ' \ + auth-script-renew=50 + +The script prints zero or more headers, one per line, which are added +to outgoing HTTP/HTTPS requests. The headers are added to those +already specified by the C<header> and C<cookie> parameters. + +If C<auth-script> is used without C<auth-script-renew> then the script +is called just once, before the plugin makes the first request. + +Within the C<auth-script> the following shell variable is available: + +=over 4 + +=item C<$url> + +The URL as passed to the plugin. + +=back + +=head2 VMware ESXi cookies + +VMware ESXi’s web server can expose both VMDK and raw format disk +images. This requires you to log in using HTTP Basic Authentication. +While you can use the C<user> and C<password> parameters to send HTTP +Basic Authentication headers in every request, tests have shown that +it is faster to accept the cookie which the server returns and send +that instead. (It is not clear why it is faster, but one theory is +that VMware has to do a more expensive username and password check +each time.) + +The web server can be accessed as below. Since the cookie expires +after a certain period of time, we use C<auth-script-renew>, and +because the server uses a self-signed certificate we must use +I<--insecure> and C<sslverify=false>. + + SERVER=esx.example.com + DCPATH=data + DS=datastore1 + GUEST=guest-name + URL="https://$SERVER/folder/$GUEST/$GUEST-flat.vmdk?dcPath=$DCPATH&dsName=$DS" + + nbdkit curl "$URL" \ + auth-script=' + curl --head -s --insecure -u root:password "$url" | + sed -ne '{ s/^Set-Cookie: \([^;]*\);.*/Cookie: \1/ip }' + ' \ + auth-script-renew=500 \ + sslverify=false + +=head2 Docker Hub authorization tokens + +Accessing objects like container layers from Docker Hub requires that +you first fetch an authorization token, even for anonymous access. + +You will need this authorization script (F</tmp/auth.sh>): + + #!/bin/sh - + IMAGE=library/fedora + curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE:pull" | + jq -r .token + +You will also need this script to get the blobSum of the layer +(F</tmp/blobsum.sh>): + + #!/bin/sh - + TOKEN=`/tmp/auth.sh` + IMAGE=library/fedora + curl -s -X GET -H "Authorization: Bearer $TOKEN" \ + "https://registry-1.docker.io/v2/$IMAGE/manifests/latest" | + jq -r '.fsLayers[0].blobSum' + +Both scripts must be executable, and both can be run on their own to +check they are working. + +Note C<auth-script-renew> is used because the tokens expire by default +after about 5 minutes (300 seconds). + + IMAGE=library/fedora + BLOBSUM=`/tmp/blobsum.sh` + URL="https://registry-1.docker.io/v2/$IMAGE/blobs/$BLOBSUM" + + nbdkit curl "$URL" \ + auth-script=' echo -n "Authorization: Bearer "; /tmp/auth.sh ' \ + auth-script-renew=200 \ + --filter=gzip + +Note that this exposes a tar file over NBD. See also +L<nbdkit-tar-filter(1)>. + =head1 DEBUG FLAG =over 4 diff --git a/plugins/curl/Makefile.am b/plugins/curl/Makefile.am index ddf1a215..2083ba66 100644 --- a/plugins/curl/Makefile.am +++ b/plugins/curl/Makefile.am @@ -38,6 +38,8 @@ if HAVE_CURL plugin_LTLIBRARIES = nbdkit-curl-plugin.la nbdkit_curl_plugin_la_SOURCES = \ + curldefs.h \ + auth-script.c \ curl.c \ $(top_srcdir)/include/nbdkit-plugin.h \ $(NULL) diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h new file mode 100644 index 00000000..8b8a339e --- /dev/null +++ b/plugins/curl/curldefs.h @@ -0,0 +1,74 @@ +/* nbdkit + * Copyright (C) 2014-2020 Red Hat Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Red Hat nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef NBDKIT_CURLDEFS_H +#define NBDKIT_CURLDEFS_H + +extern const char *url; + +extern const char *auth_script; +extern unsigned auth_script_renew; +extern const char *cainfo; +extern const char *capath; +extern char *cookie; +extern struct curl_slist *headers; +extern char *password; +extern long protocols; +extern const char *proxy; +extern char *proxy_password; +extern const char *proxy_user; +extern bool sslverify; +extern bool tcp_keepalive; +extern bool tcp_nodelay; +extern uint32_t timeout; +extern const char *unix_socket_path; +extern const char *user; +extern const char *user_agent; + +/* The per-connection handle. */ +struct curl_handle { + CURL *c; + bool accept_range; + int64_t exportsize; + char errbuf[CURL_ERROR_SIZE]; + char *write_buf; + uint32_t write_count; + const char *read_buf; + uint32_t read_count; + struct curl_slist *auth_headers; +}; + +/* auth-script.c */ +extern int do_auth_script (struct curl_handle *h); +extern void auth_script_unload (void); + +#endif /* NBDKIT_CURLDEFS_H */ diff --git a/plugins/curl/auth-script.c b/plugins/curl/auth-script.c new file mode 100644 index 00000000..6839cd85 --- /dev/null +++ b/plugins/curl/auth-script.c @@ -0,0 +1,311 @@ +/* nbdkit + * Copyright (C) 2014-2020 Red Hat Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Red Hat nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* Authorization script. */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <string.h> +#include <unistd.h> +#include <time.h> +#include <assert.h> +#include <pthread.h> + +#include <curl/curl.h> + +#include <nbdkit-plugin.h> + +#include "cleanup.h" +#include "utils.h" +#include "vector.h" + +#include "curldefs.h" + +/* This lock protects internal state in this file. */ +static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; + +/* Last time auth-script was run. */ +static time_t last = 0; +static bool auth_script_has_run = false; + +/* List of extra headers and cookies from the output of auth-script. */ +DEFINE_VECTOR_TYPE(string_vector, char *); +static string_vector script_headers = empty_vector; +static string_vector script_cookies = empty_vector; + +/* Called from plugin curl_unload(). */ +void +auth_script_unload (void) +{ + string_vector_iter (&script_headers, (void *) free); + string_vector_iter (&script_cookies, (void *) free); + free (script_headers.ptr); + free (script_cookies.ptr); +} + +static int run_auth_script (struct curl_handle *); +static int set_headers (struct curl_handle *); +static int set_cookies (struct curl_handle *); + +/* This is called from any thread just before we make a curl request. + * The caller checks that auth_script != NULL before calling. + * + * The job of this is two-fold: (1) If we need to run the auth_script + * (either for the first time, or because auth_script_renew has + * elapsed since last time), then it runs it, blocking all threads + * while that happens. (2) Whether or not we ran auth_script, this + * must set up the headers and/or cookies in the CURL handle. + * + * Number (2) is complicated. We cannot simply call + * CURLOPT_HTTPHEADER and CURLOPT_COOKIE a second time because + * curl doesn't work like that. Instead we have to generate + * a new list of headers and new cookie string and set those. + * + * We have to set these every time because the auth-script might have + * been re-run in another thread, and also the auth-script might have + * removed headers/cookies. + * + * When calling CURLOPT_HTTPHEADER we have to keep the list around + * because unfortunately curl doesn't take a copy. Since we don't + * know which other threads might be using it, we must create and use + * this list per handle. For CURLOPT_COOKIE curl internally takes a + * copy. + * + * Because the thread model is NBDKIT_THREAD_MODEL_SERIALIZE_REQUESTS + * we can be assured of exclusive access to curl_handle here. + */ +int +do_auth_script (struct curl_handle *h) +{ + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); + time_t now; + + /* Run or re-run auth-script if we need to. */ + time (&now); + if (!auth_script_has_run || + (auth_script_renew > 0 && now - last >= auth_script_renew)) { + if (run_auth_script (h) == -1) + return -1; + last = now; + auth_script_has_run = true; + } + + /* Set the headers and cookies in the curl handle. */ + if (set_headers (h) == -1 || set_cookies (h) == -1) + return -1; + + return 0; +} + +/* This is called with the lock held when we must run or re-run the + * auth script. + */ +static int +run_auth_script (struct curl_handle *h) +{ + int fd; + char tmpfile[] = "/tmp/errorsXXXXXX"; + FILE *fp; + CLEANUP_FREE char *cmd = NULL, *line = NULL; + size_t len = 0, linelen = 0; + + assert (auth_script != NULL); /* checked by caller */ + + /* Reset the list of headers and cookies. */ + string_vector_iter (&script_headers, (void *) free); + string_vector_iter (&script_cookies, (void *) free); + string_vector_reset (&script_headers); + string_vector_reset (&script_cookies); + + /* Create a temporary file for the errors so we can redirect them + * into nbdkit_error. + */ + fd = mkstemp (tmpfile); + if (fd == -1) { + nbdkit_error ("mkstemp"); + return -1; + } + close (fd); + + /* Generate the full script with the local $url variable. */ + fp = open_memstream (&cmd, &len); + if (fp == NULL) { + nbdkit_error ("open_memstream: %m"); + return -1; + } + fprintf (fp, "exec </dev/null\n"); /* Avoid stdin leaking (nbdkit -s). */ + fprintf (fp, "exec 2>%s\n", tmpfile); /* Catch errors to a temporary file. */ + fprintf (fp, "url="); /* Set the shell variable. */ + shell_quote (url, fp); + putc ('\n', fp); + putc ('\n', fp); + fprintf (fp, "%s", auth_script); /* The script or command. */ + if (fclose (fp) == EOF) { + nbdkit_error ("memstream failed"); + return -1; + } + + /* Run the script and read the headers/cookies. */ + nbdkit_debug ("curl: running authorization script"); + fp = popen (cmd, "r"); + if (fp == NULL) { + nbdkit_error ("popen: %m"); + return -1; + } + while ((len = getline (&line, &linelen, fp)) != -1) { + char *p; + + if (len > 0 && line[len-1] == '\n') + line[len-1] = '\0'; + + if (strncasecmp (line, "cookie:", 7) == 0) { + p = strdup (&line[7]); + if (p == NULL || string_vector_append (&script_cookies, p) == -1) { + nbdkit_error ("malloc"); + pclose (fp); + return -1; + } + } + else { + p = strdup (line); + if (p == NULL || string_vector_append (&script_headers, p) == -1) { + nbdkit_error ("malloc"); + pclose (fp); + return -1; + } + } + } + + /* If the command failed, this should return EOF and the error + * message should be in the temporary file (but we only read the + * first line). + */ + if (pclose (fp) == EOF) { + fp = fopen (tmpfile, "r"); + if ((len = getline (&line, &linelen, fp)) >= 0) { + if (len > 0 && line[len-1] == '\n') + line[len-1] = '\0'; + nbdkit_error ("authorization script failed: %s", line); + } + else + nbdkit_error ("authorization script failed"); + return -1; + } + + nbdkit_debug ("authorization script returned %zu header(s) and %zu cookie(s)", + script_headers.size, script_cookies.size); + + return 0; +} + +static int +set_headers (struct curl_handle *h) +{ + struct curl_slist *p; + size_t i; + + /* Curl does not save a copy of the headers passed to + * CURLOPT_HTTPHEADER so we have to store it in the handle ourselves + * and be careful to unset it in the Curl handle before we free the + * list. + */ + if (h->auth_headers) { + curl_easy_setopt (h->c, CURLOPT_HTTPHEADER, NULL); + curl_slist_free_all (h->auth_headers); + h->auth_headers = NULL; + } + + /* Copy the header=... parameters. */ + for (p = headers; p != NULL; p = p->next) { + h->auth_headers = curl_slist_append (h->auth_headers, p->data); + if (h->auth_headers == NULL) { + nbdkit_error ("curl_slist_append: %m"); + return -1; + } + } + + /* Copy the headers output by the script. */ + for (i = 0; i < script_headers.size; ++i) { + h->auth_headers = + curl_slist_append (h->auth_headers, script_headers.ptr[i]); + if (h->auth_headers == NULL) { + nbdkit_error ("curl_slist_append: %m"); + return -1; + } + } + + /* Set them in the handle. */ + curl_easy_setopt (h->c, CURLOPT_HTTPHEADER, h->auth_headers); + return 0; +} + +static int +set_cookies (struct curl_handle *h) +{ + CLEANUP_FREE char *s = NULL; + size_t i; + + /* For cookies we have to append the cookies from the command line + * with the cookies from the auth script. Either might be empty. + */ + if (cookie != NULL) { + s = strdup (cookie); + if (s == NULL) { + nbdkit_error ("strdup: %m"); + return -1; + } + } + + for (i = 0; i < script_cookies.size; ++i) { + char *ns; + + if (asprintf (&ns, "%s%s%s", + s ? s : "", + s ? " ; " : "", + script_cookies.ptr[i]) == -1) { + nbdkit_error ("asprintf: %m"); + return -1; + } + s = ns; + } + + /* Curl saves a copy of this string in the handle so it's OK to free + * it after calling this. + */ + if (s) + curl_easy_setopt (h->c, CURLOPT_COOKIE, s); + + return 0; +} diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c index 50eef1a8..fe2fc3ac 100644 --- a/plugins/curl/curl.c +++ b/plugins/curl/curl.c @@ -48,9 +48,11 @@ #include <nbdkit-plugin.h> -#include "cleanup.h" #include "ascii-ctype.h" #include "ascii-string.h" +#include "cleanup.h" + +#include "curldefs.h" /* Macro CURL_AT_LEAST_VERSION was added in 2015 (Curl 7.43) so if the * macro isn't present then Curl is very old. @@ -61,24 +63,27 @@ #endif #endif -static const char *url = NULL; /* required */ +/* Plugin configuration. */ +const char *url = NULL; /* required */ -static const char *cainfo = NULL; -static const char *capath = NULL; -static char *cookie = NULL; -static struct curl_slist *headers = NULL; -static char *password = NULL; -static long protocols = CURLPROTO_ALL; -static const char *proxy = NULL; -static char *proxy_password = NULL; -static const char *proxy_user = NULL; -static bool sslverify = true; -static bool tcp_keepalive = false; -static bool tcp_nodelay = true; -static uint32_t timeout = 0; -static const char *unix_socket_path = NULL; -static const char *user = NULL; -static const char *user_agent = NULL; +const char *auth_script = NULL; +unsigned auth_script_renew = 0; +const char *cainfo = NULL; +const char *capath = NULL; +char *cookie = NULL; +struct curl_slist *headers = NULL; +char *password = NULL; +long protocols = CURLPROTO_ALL; +const char *proxy = NULL; +char *proxy_password = NULL; +const char *proxy_user = NULL; +bool sslverify = true; +bool tcp_keepalive = false; +bool tcp_nodelay = true; +uint32_t timeout = 0; +const char *unix_socket_path = NULL; +const char *user = NULL; +const char *user_agent = NULL; /* Use '-D curl.verbose=1' to set. */ int curl_debug_verbose = 0; @@ -98,11 +103,14 @@ curl_load (void) static void curl_unload (void) { - free (password); - free (proxy_password); free (cookie); if (headers) curl_slist_free_all (headers); + free (password); + free (proxy_password); + + auth_script_unload (); + curl_global_cleanup (); } @@ -188,7 +196,17 @@ curl_config (const char *key, const char *value) { int r; - if (strcmp (key, "cainfo") == 0) { + if (strcmp (key, "auth-script") == 0) { + auth_script = value; + } + + else if (strcmp (key, "auth-script-renew") == 0) { + if (nbdkit_parse_unsigned ("auth-script-renew", value, + &auth_script_renew) == -1) + return -1; + } + + else if (strcmp (key, "cainfo") == 0) { cainfo = value; } @@ -322,18 +340,6 @@ curl_config_complete (void) "user=<USER> The user to log in as.\n" \ "user-agent=<USER-AGENT> Send user-agent header for HTTP/HTTPS." -/* The per-connection handle. */ -struct curl_handle { - CURL *c; - bool accept_range; - int64_t exportsize; - char errbuf[CURL_ERROR_SIZE]; - char *write_buf; - uint32_t write_count; - const char *read_buf; - uint32_t read_count; -}; - /* Translate CURLcode to nbdkit_error. */ #define display_curl_error(h, r, fs, ...) \ do { \ @@ -450,7 +456,12 @@ curl_open (int readonly) /* Get the file size and also whether the remote HTTP server * supports byte ranges. + * + * We must run the auth-script if necessary and set headers in the + * handle. */ + if (auth_script && do_auth_script (h) == -1) + goto err; h->accept_range = false; curl_easy_setopt (h->c, CURLOPT_NOBODY, 1); /* No Body, not nobody! */ curl_easy_setopt (h->c, CURLOPT_HEADERFUNCTION, header_cb); @@ -608,6 +619,8 @@ curl_close (void *handle) struct curl_handle *h = handle; curl_easy_cleanup (h->c); + if (h->auth_headers) + curl_slist_free_all (h->auth_headers); free (h); } @@ -638,6 +651,10 @@ curl_pread (void *handle, void *buf, uint32_t count, uint64_t offset) CURLcode r; char range[128]; + /* Run the auth-script if necessary and set headers in the handle. */ + if (auth_script && do_auth_script (h) == -1) + return -1; + /* Tell the write_cb where we want the data to be written. write_cb * will update this if the data comes in multiple sections. */ @@ -699,6 +716,10 @@ curl_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset) CURLcode r; char range[128]; + /* Run the auth-script if necessary and set headers in the handle. */ + if (auth_script && do_auth_script (h) == -1) + return -1; + /* Tell the read_cb where we want the data to be read from. read_cb * will update this if the data comes in multiple sections. */ -- 2.27.0