The yara_load API allows to load a set of Yara rules contained within a file on the host. Rules can be in binary format, as when compiled with yarac command, or in source code format. In the latter case, the rules will be first compiled and then loaded. Subsequent calls of the yara_load API will result in the discard of the previously loaded rules. Signed-off-by: Matteo Cafasso <noxdafox(a)gmail.com&gt; --- daemon/Makefile.am | 1 + daemon/cleanups.c | 28 +++++++ daemon/cleanups.h | 9 ++ daemon/yara.c | 227 +++++++++++++++++++++++++++++++++++++++++++++++++++ generator/actions.ml | 18 ++++ src/MAX_PROC_NR | 2 +- 6 files changed, 284 insertions(+), 1 deletion(-) create mode 100644 daemon/yara.c diff --git a/daemon/Makefile.am b/daemon/Makefile.am index 3a25f43..c385edc 100644 --- a/daemon/Makefile.am +++ b/daemon/Makefile.am @@ -200,6 +200,7 @@ guestfsd_SOURCES = \ wc.c \ xattr.c \ xfs.c \ + yara.c \ zero.c \ zerofree.c diff --git a/daemon/cleanups.c b/daemon/cleanups.c index 092e493..a02e521 100644 --- a/daemon/cleanups.c +++ b/daemon/cleanups.c @@ -24,6 +24,12 @@ #include <augeas.h> +#ifdef HAVE_YARA + +#include <yara.h> + +#endif /* !HAVE_YARA */ + #include "cleanups.h" /* Use by the CLEANUP_* macros. Do not call these directly. */ @@ -62,6 +68,15 @@ cleanup_close (void *ptr) } void +cleanup_fclose (void *ptr) +{ + FILE *f = * (FILE **) ptr; + + if (f) + fclose (f); +} + +void cleanup_aug_close (void *ptr) { augeas *aug = * (augeas **) ptr; @@ -78,3 +93,16 @@ cleanup_free_stringsbuf (void *ptr) { free_stringsbuf ((struct stringsbuf *) ptr); } + +#ifdef HAVE_YARA + +void +cleanup_destroy_yara_compiler (void *ptr) +{ + YR_COMPILER *compiler = * (YR_COMPILER **) ptr; + + if (compiler != NULL) + yr_compiler_destroy (compiler); +} + +#endif /* !HAVE_YARA */ diff --git a/daemon/cleanups.h b/daemon/cleanups.h index 6746e27..c61f646 100644 --- a/daemon/cleanups.h +++ b/daemon/cleanups.h @@ -26,8 +26,12 @@ extern void cleanup_free (void *ptr); extern void cleanup_free_string_list (void *ptr); extern void cleanup_unlink_free (void *ptr); extern void cleanup_close (void *ptr); +extern void cleanup_fclose (void *ptr); extern void cleanup_aug_close (void *ptr); extern void cleanup_free_stringsbuf (void *ptr); +#ifdef HAVE_YARA +extern void cleanup_destroy_yara_compiler (void *ptr); +#endif /* !HAVE_YARA */ #ifdef HAVE_ATTRIBUTE_CLEANUP #define CLEANUP_FREE __attribute__((cleanup(cleanup_free))) @@ -35,8 +39,13 @@ extern void cleanup_free_stringsbuf (void *ptr); __attribute__((cleanup(cleanup_free_string_list))) #define CLEANUP_UNLINK_FREE __attribute__((cleanup(cleanup_unlink_free))) #define CLEANUP_CLOSE __attribute__((cleanup(cleanup_close))) +#define CLEANUP_FCLOSE __attribute__((cleanup(cleanup_fclose))) #define CLEANUP_AUG_CLOSE __attribute__((cleanup(cleanup_aug_close))) #define CLEANUP_FREE_STRINGSBUF __attribute__((cleanup(cleanup_free_stringsbuf))) +#ifdef HAVE_YARA +#define CLEANUP_DESTROY_YARA_COMPILER \ + __attribute__((cleanup(cleanup_destroy_yara_compiler))) +#endif /* !HAVE_YARA */ #else #define CLEANUP_FREE #define CLEANUP_FREE_STRING_LIST diff --git a/daemon/yara.c b/daemon/yara.c new file mode 100644 index 0000000..be53322 --- /dev/null +++ b/daemon/yara.c @@ -0,0 +1,227 @@ +/* libguestfs - the guestfsd daemon + * Copyright (C) 2016 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <config.h> + +#include <stdio.h> +#include <fcntl.h> +#include <stdlib.h> +#include <inttypes.h> +#include <string.h> +#include <unistd.h> +#include <stdbool.h> +#include <rpc/xdr.h> +#include <rpc/types.h> +#include <sys/types.h> +#include <sys/stat.h> + +#include "daemon.h" +#include "actions.h" +#include "optgroups.h" +#include "guestfs_protocol.h" + +#ifdef HAVE_YARA + +#include <yara.h> + +struct write_callback_data { + int fd; + uint64_t written; +}; + +/* Yara compiled rules. */ +static YR_RULES *rules = NULL; +static bool initialized = false; + +static int upload_rules_file (char *); +static int compile_rules_file (const char *); +static int write_callback (void *, const void *, size_t); +static void compile_error_callback (int, const char *, int, const char *, void *); + +/* Has one FileIn parameter. */ +int +do_yara_load (void) +{ + int ret = 0; + char tmpfile[] = "/tmp/yaraXXXXXX"; + + ret = upload_rules_file (tmpfile); + if (ret < 0) + return -1; + + /* Initialize yara only once. */ + if (!initialized) { + ret = yr_initialize (); + if (ret != ERROR_SUCCESS) { + reply_with_error ("failed initializing yara"); + return -1; + } + + initialized = true; + } + + /* Destroy previously loaded rules. */ + if (rules != NULL) { + yr_rules_destroy (rules); + rules = NULL; + } + + /* Try to load the rules as compiled. + * If their are in source code format, compile them first. + */ + ret = yr_rules_load (tmpfile, &rules); + if (ret == ERROR_INVALID_FILE) + ret = compile_rules_file (tmpfile); + + unlink (tmpfile); + + return (ret == ERROR_SUCCESS) ? 0 : -1; +} + +/* Upload rules file on a temporary file. + * Return 0 on success, -1 on error. + */ +static int +upload_rules_file (char *rules_path) +{ + int ret = 0; + CLEANUP_CLOSE int fd = 0; + struct write_callback_data data = { .written = 0 }; + + data.fd = mkstemp (rules_path); + if (data.fd == -1) { + reply_with_perror ("mkstemp"); + return -1; + } + + ret = receive_file (write_callback, &data); + if (ret == -1) { + /* Write error. */ + cancel_receive (); + reply_with_error ("write error: %s", rules_path); + return -1; + } + if (ret == -2) { + /* Cancellation from library */ + reply_with_error ("file upload cancelled"); + return -1; + } + + return 0; +} + +/* Compile source code rules and load them. + * Return ERROR_SUCCESS on success, Yara error code type on error. + */ +static int +compile_rules_file (const char *rules_path) +{ + int ret = 0; + CLEANUP_FCLOSE FILE *rule_file = NULL; + CLEANUP_DESTROY_YARA_COMPILER YR_COMPILER *compiler = NULL; + + ret = yr_compiler_create (&compiler); + if (ret != ERROR_SUCCESS) { + reply_with_error ("yr_compiler_create"); + return ret; + } + + yr_compiler_set_callback (compiler, compile_error_callback, NULL); + + rule_file = fopen (rules_path, "r"); + if (rule_file == NULL) { + reply_with_error ("unable to open rules file"); + return ret; + } + + ret = yr_compiler_add_file (compiler, rule_file, NULL, rules_path); + if (ret > 0) + return ret; + + ret = yr_compiler_get_rules (compiler, &rules); + if (ret != ERROR_SUCCESS) + reply_with_error ("yr_compiler_get_rules"); + + return ret; +} + +/* File upload callback, called by receive_file. + * Return 0 on success, -1 on error. + */ +static int +write_callback (void *data_vp, const void *buf, size_t len) +{ + int ret; + struct write_callback_data *data = data_vp; + + ret = xwrite (data->fd, buf, len); + if (ret == -1) + return -1; + + data->written += len; + + if (progress_hint > 0) + notify_progress (data->written, progress_hint); + + return 0; +} + +/* Yara compilation error callback. + * Reports back the compilation error message. + * Prints compilation warnings if verbose. + */ +static void +compile_error_callback(int level, const char *name, int line, + const char *message, void *data) +{ + if (level == YARA_ERROR_LEVEL_ERROR) + reply_with_error ("(%d): Yara error: %s", line, message); + else + fprintf (stderr, "(%d): Yara warning: %s\n", line, message); +} + +/* Clean up yara handle on daemon exit. */ +void yara_finalize (void) __attribute__((destructor)); +void +yara_finalize (void) +{ + int ret = 0; + + if (rules != NULL) { + yr_rules_destroy (rules); + rules = NULL; + } + + ret = yr_finalize (); + if (ret != ERROR_SUCCESS) + perror ("yr_finalize"); + + initialized = false; +} + +int +optgroup_libyara_available (void) +{ + return 1; +} + +#else /* !HAVE_YARA */ + +OPTGROUP_YARA_NOT_AVAILABLE + +#endif /* !HAVE_YARA */ diff --git a/generator/actions.ml b/generator/actions.ml index 91a1819..f69564a 100644 --- a/generator/actions.ml +++ b/generator/actions.ml @@ -13253,6 +13253,24 @@ is removed." }; shortdesc = "search the entries associated to the given inode"; longdesc = "Internal function for find_inode." }; + { defaults with + name = "yara_load"; added = (1, 35, 15); + style = RErr, [FileIn "filename";], []; + proc_nr = Some 471; + progress = true; cancellable = true; + optional = Some "libyara"; + shortdesc = "load yara rules within libguestfs"; + longdesc = "\ +Load a set of Yara rules from F<filename> within libguestfs appliance. + +Rules can be in binary format, as when compiled with yarac command, or +in source code format. In the latter case, the rules will be first +compiled and then loaded. + +Rules in source code format cannot include external files. In such cases, +it is recommended to compile them first. + +Previously loaded rules will be destroyed." }; ] (* Non-API meta-commands available only in guestfish. diff --git a/src/MAX_PROC_NR b/src/MAX_PROC_NR index 5f476b6..c305aa5 100644 --- a/src/MAX_PROC_NR +++ b/src/MAX_PROC_NR @@ -1 +1 @@ -470 +471 -- 2.10.2

On 21/11/16 18:27, Pino Toscano wrote: Just one question: shall the changes in upload.c be in a separate commit "expose XYZ"? What is the preferred way? This question applies also for the notes in PATCH 5.

The yara_destroy API allows to claim resources back via the removal of the previously loaded Yara rules. Signed-off-by: Matteo Cafasso <noxdafox(a)gmail.com&gt; --- daemon/yara.c | 14 ++++++++++++++ generator/actions.ml | 9 +++++++++ src/MAX_PROC_NR | 2 +- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/daemon/yara.c b/daemon/yara.c index be53322..fe1f69a 100644 --- a/daemon/yara.c +++ b/daemon/yara.c @@ -93,6 +93,20 @@ do_yara_load (void) return (ret == ERROR_SUCCESS) ? 0 : -1; } +int +do_yara_destroy (void) +{ + if (rules == NULL) { + reply_with_error ("no yara rules loaded"); + return -1; + } + + yr_rules_destroy (rules); + rules = NULL; + + return 0; +} + /* Upload rules file on a temporary file. * Return 0 on success, -1 on error. */ diff --git a/generator/actions.ml b/generator/actions.ml index f69564a..152c651 100644 --- a/generator/actions.ml +++ b/generator/actions.ml @@ -13271,6 +13271,15 @@ Rules in source code format cannot include external files. In such cases, it is recommended to compile them first. Previously loaded rules will be destroyed." }; + + { defaults with + name = "yara_destroy"; added = (1, 35, 15); + style = RErr, [], []; + proc_nr = Some 472; + optional = Some "libyara"; + shortdesc = "destroy previously loaded yara rules"; + longdesc = "\ +Destroy previously loaded Yara rules in order to free libguestfs resources." }; ] (* Non-API meta-commands available only in guestfish. diff --git a/src/MAX_PROC_NR b/src/MAX_PROC_NR index c305aa5..68cfb10 100644 --- a/src/MAX_PROC_NR +++ b/src/MAX_PROC_NR @@ -1 +1 @@ -471 +472 -- 2.10.2

On Wednesday, 9 November 2016 22:38:55 CET Matteo Cafasso wrote: This must be initialized as -1, otherwise the CLEANUP_CLOSE handler will close the fd 0, which is stdin of the daemon. I don't think that's the right place for yr_finalize. fname is not used here -- I suggest passing --enable-werror to autogen.sh or configure, so any compiler warning is turned to error. Typo in comment. yara_load supports loading rules already compiled, which could have a namespace set -- I guess it should be reported here as well. That triggers another question: should the yara support allow to load more rules one after each other (with namespaces as well), instead of just one? Thanks, -- Pino Toscano

The yara_scan API parses the file generated by the daemon counterpart function and returns the list of yara_detection structs to the user. It writes the daemon's command output on a temporary file and parses it, deserialising the XDR formatted yara_detection structs. It returns to the caller the list of yara_detection structs generated by the internal_yara_scan command. Signed-off-by: Matteo Cafasso <noxdafox(a)gmail.com&gt; --- generator/actions.ml | 25 +++++++++ src/Makefile.am | 1 + src/yara.c | 140 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 src/yara.c diff --git a/generator/actions.ml b/generator/actions.ml index d9006f2..8595b38 100644 --- a/generator/actions.ml +++ b/generator/actions.ml @@ -3729,6 +3729,31 @@ Searches all the entries associated with the given inode. For each entry, a C<tsk_dirent> structure is returned. See C<filesystem_walk> for more information about C<tsk_dirent> structures." }; + { defaults with + name = "yara_scan"; added = (1, 35, 15); + style = RStructList ("detections", "yara_detection"), [Pathname "path";], []; + optional = Some "libyara"; + progress = true; cancellable = true; + shortdesc = "scan a file with the loaded yara rules"; + longdesc = "\ +Scan a file with the previously loaded Yara rules. + +For each matching rule, a C<yara_detection> structure is returned. + +The C<tsk_dirent> structure contains the following fields. + +=over 4 + +=item 'name' + +Path of the file matching a Yara rule. + +=item 'rule' + +Identifier of the Yara rule which matched against the given file. + +=back" }; + ] (* daemon_functions are any functions which cause some action diff --git a/src/Makefile.am b/src/Makefile.am index 8150d99..812ffbb 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -133,6 +133,7 @@ libguestfs_la_SOURCES = \ wait.c \ whole-file.c \ version.c \ + yara.c \ libguestfs.syms libguestfs_la_CPPFLAGS = \ diff --git a/src/yara.c b/src/yara.c new file mode 100644 index 0000000..0b924a2 --- /dev/null +++ b/src/yara.c @@ -0,0 +1,140 @@ +/* libguestfs + * Copyright (C) 2016 Red Hat Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <string.h> +#include <rpc/xdr.h> +#include <rpc/types.h> + +#include "guestfs.h" +#include "guestfs_protocol.h" +#include "guestfs-internal.h" +#include "guestfs-internal-all.h" +#include "guestfs-internal-actions.h" + +static struct guestfs_yara_detection_list *parse_yara_detection_file (guestfs_h *, const char *); +static int deserialise_yara_detection_list (guestfs_h *, FILE *, struct guestfs_yara_detection_list *); +static char *make_temp_file (guestfs_h *, const char *); + +struct guestfs_yara_detection_list * +guestfs_impl_yara_scan (guestfs_h *g, const char *path) +{ + int ret = 0; + CLEANUP_UNLINK_FREE char *tmpfile = NULL; + + tmpfile = make_temp_file (g, "yara_scan"); + if (tmpfile == NULL) + return NULL; + + ret = guestfs_internal_yara_scan (g, path, tmpfile); + if (ret < 0) + return NULL; + + return parse_yara_detection_file (g, tmpfile); /* caller frees */ +} + +/* Parse the file content and return detections list. + * Return a list of yara_detection on success, NULL on error. + */ +static struct guestfs_yara_detection_list * +parse_yara_detection_file (guestfs_h *g, const char *tmpfile) +{ + int ret = 0; + CLEANUP_FCLOSE FILE *fp = NULL; + struct guestfs_yara_detection_list *detections = NULL; + + fp = fopen (tmpfile, "r"); + if (fp == NULL) { + perrorf (g, "fopen: %s", tmpfile); + return NULL; + } + + /* Initialise results array. */ + detections = safe_malloc (g, sizeof (*detections)); + detections->len = 8; + detections->val = safe_malloc (g, detections->len * + sizeof (*detections->val)); + + /* Deserialise buffer into detection list. */ + ret = deserialise_yara_detection_list (g, fp, detections); + if (ret < 0) { + guestfs_free_yara_detection_list (detections); + return NULL; + } + + return detections; +} + +/* Deserialise the file content and populate the detection list. + * Return the number of deserialised detections, -1 on error. + */ +static int +deserialise_yara_detection_list (guestfs_h *g, FILE *fp, + struct guestfs_yara_detection_list *detections) +{ + XDR xdr; + int ret = 0; + uint32_t index = 0; + struct stat statbuf; + + ret = fstat (fileno(fp), &statbuf); + if (ret == -1) + return -1; + + xdrstdio_create (&xdr, fp, XDR_DECODE); + + for (index = 0; xdr_getpos (&xdr) < statbuf.st_size; index++) { + if (index == detections->len) { + detections->len = 2 * detections->len; + detections->val = safe_realloc (g, detections->val, + detections->len * + sizeof (*detections->val)); + } + + /* Clear the entry so xdr logic will allocate necessary memory. */ + memset (&detections->val[index], 0, sizeof (*detections->val)); + ret = xdr_guestfs_int_yara_detection (&xdr, (guestfs_int_yara_detection *) + &detections->val[index]); + if (ret == 0) + break; + } + + xdr_destroy (&xdr); + detections->len = index; + + return ret ? 0 : -1; +} + +static char * +make_temp_file (guestfs_h *g, const char *name) +{ + int ret = 0; + + ret = guestfs_int_lazy_make_tmpdir (g); + if (ret < 0) + return NULL; + + return safe_asprintf (g, "%s/%s%d", g->tmpdir, name, ++g->unique); +} -- 2.10.2

Signed-off-by: Matteo Cafasso <noxdafox(a)gmail.com&gt; --- configure.ac | 1 + tests/yara/Makefile.am | 26 ++++++++++++++++ tests/yara/test-yara-scan.sh | 72 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 tests/yara/Makefile.am create mode 100755 tests/yara/test-yara-scan.sh diff --git a/configure.ac b/configure.ac index 45bb935..0681010 100644 --- a/configure.ac +++ b/configure.ac @@ -285,6 +285,7 @@ AC_CONFIG_FILES([Makefile tests/tsk/Makefile tests/xfs/Makefile tests/xml/Makefile + tests/yara/Makefile tools/Makefile utils/boot-analysis/Makefile utils/boot-benchmark/Makefile diff --git a/tests/yara/Makefile.am b/tests/yara/Makefile.am new file mode 100644 index 0000000..e23d94e --- /dev/null +++ b/tests/yara/Makefile.am @@ -0,0 +1,26 @@ +# libguestfs +# Copyright (C) 2016 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +include $(top_srcdir)/subdir-rules.mk + +TESTS = \ + test-yara-scan.sh + +TESTS_ENVIRONMENT = $(top_builddir)/run --test + +EXTRA_DIST = \ + $(TESTS) diff --git a/tests/yara/test-yara-scan.sh b/tests/yara/test-yara-scan.sh new file mode 100755 index 0000000..a899e33 --- /dev/null +++ b/tests/yara/test-yara-scan.sh @@ -0,0 +1,72 @@ +#!/bin/bash - +# libguestfs +# Copyright (C) 2016 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Test the yara_scan command. + +set -e + +if [ -n "$SKIP_TEST_YARA_SCAN_SH" ]; then + echo "$0: test skipped because environment variable is set." + exit 77 +fi + +rm -f test-yara-rules.yar + +# Skip if Yara is not supported by the appliance. +if ! guestfish add /dev/null : run : available "libyara"; then + echo "$0: skipped because Yara is not available in the appliance" + exit 77 +fi + +if [ ! -s ../../test-data/phony-guests/blank-fs.img ]; then + echo "$0: skipped because blank-fs.img is zero-sized" + exit 77 +fi + +/bin/cat << EOF > test-yara-rules.yar +rule TestRule +{ + strings: + \$my_text_string = "some text" + + condition: + \$my_text_string +} +EOF + +output=$( +guestfish --ro -a ../../test-data/phony-guests/blank-fs.img <<EOF +run +mount /dev/sda1 / +write /text.txt "some text" +yara-load test-yara-rules.yar +yara-scan /text.txt +umount / +yara-destroy +EOF +) + +echo $output | grep -zq '{ name: /text.txt rule: TestRule }' +if [ $? != 0 ]; then + echo "$0: TestRule not found in detections list." + echo "Detections list:" + echo $output + exit 1 +fi + +rm -f test-yara-rules.yar -- 2.10.2