This uses Linus's technique described here:
https://stackoverflow.com/a/3756466
Before this commit:
$ rm /var/tmp/pattern
$ time ./run nbdcopy [ nbdkit pattern 32G ] /var/tmp/pattern
real 0m32.148s
user 0m18.015s
sys 0m33.216s
$ cachestats /var/tmp/pattern
pages in cache: 7066239/8388608 (84.2%) [filesize=33554432.0K, pagesize=4K]
Notice that the newly written file ends up in the cache, thus trashing
the page cache on the host.
After this commit:
$ rm /var/tmp/pattern
$ time ./run nbdcopy [ nbdkit pattern 32G ] /var/tmp/pattern
real 0m39.961s
user 0m19.117s
sys 0m41.238s
$ cachestats /var/tmp/pattern
pages in cache: 8192/8388608 (0.1%) [filesize=33554432.0K, pagesize=4K]
The newly written file does not disturb the page cache. However there
is about a 25% slow down.
---
copy/file-ops.c | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
diff --git a/copy/file-ops.c b/copy/file-ops.c
index 888a388..d285cb9 100644
--- a/copy/file-ops.c
+++ b/copy/file-ops.c
@@ -159,6 +159,36 @@ page_cache_evict (struct rw_file *rwf, uint64_t orig_offset, size_t
orig_len)
len -= n;
}
}
+
+/* Prepare to evict file contents from the page cache when writing.
+ * We cannot do this directly (as for reads above) because we have to
+ * wait for Linux to finish writing the pages to disk. Therefore the
+ * strategy is to (1) tell Linux to begin writing asynchronously, (2)
+ * evict the previous pages, which have hopefully been written already
+ * by the time we get here. We have to maintain a window per thread.
+ *
+ * For more information see
https://stackoverflow.com/a/3756466 and
+ * the links to Linus's advice from that entry.
+ */
+static inline void
+page_cache_evict_for_writes (struct rw_file *rwf, uint64_t offset, size_t len)
+{
+ static __thread uint64_t prev_offset;
+ static __thread size_t prev_len;
+
+ /* Evict the previous window. */
+ if (prev_len > 0) {
+ sync_file_range (rwf->fd, prev_offset, prev_len,
+ SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE|
+ SYNC_FILE_RANGE_WAIT_AFTER);
+ posix_fadvise (rwf->fd, prev_offset, prev_len, POSIX_FADV_DONTNEED);
+ }
+
+ /* Set up the current window. */
+ sync_file_range (rwf->fd, offset, len, SYNC_FILE_RANGE_WRITE);
+ prev_offset = offset;
+ prev_len = len;
+}
#endif
static bool
@@ -365,6 +395,10 @@ file_synch_write (struct rw *rw,
const void *data, size_t len, uint64_t offset)
{
struct rw_file *rwf = (struct rw_file *)rw;
+#ifdef PAGE_CACHE_MAPPING
+ const uint64_t orig_offset = offset;
+ const size_t orig_len = len;
+#endif
ssize_t r;
while (len > 0) {
@@ -377,6 +411,10 @@ file_synch_write (struct rw *rw,
offset += r;
len -= r;
}
+
+#if PAGE_CACHE_MAPPING
+ page_cache_evict_for_writes (rwf, orig_offset, orig_len);
+#endif
}
static inline bool
--
2.29.0.rc2