tests: fault-injection test for ENOSPC on epoll_ctl

For difficult-to-trigger errors, fault injection is necessary for
testing our error handling.  I have confirmed this test fails with
"avoid leaks on epoll/kqueue resources exhaustion" reverted.

7 files changed, 192 insertions, 3 deletions

diff --git a/Makefile.am b/Makefile.am
index dd1e9e3..b2cbd38 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -132,13 +132,14 @@ PERL_LOG_FLAGS = -v
 SLOWRB_LOG_COMPILER = RUBY="$(RUBY)"
 AM_SLOWRB_LOG_FLAGS = top_srcdir="$(top_srcdir)"
 include $(top_srcdir)/test/ruby.mk
-check_PROGRAMS = test/valid-path-1 test/trywrite-1 \
+check_tests = test/valid-path-1 test/trywrite-1 \
   test/cfg-parser-1 test/fdmap-1 test/thrpool-1 \
   test/queue-idle-1 \
   test/http-parser-1 test/chunk-parser-1 \
   test/ioutil-1
+check_PROGRAMS = $(check_tests)
 
-TESTS = $(SLOW_RB_FILES) $(RB_TESTS_FAST) $(check_PROGRAMS) $(PERL_TESTS)
+TESTS = $(SLOW_RB_FILES) $(RB_TESTS_FAST) $(check_tests) $(PERL_TESTS)
 
 # we need TMPDIR to work in a place where iostat(1) gives stats
 test_tmpdir = $(top_builddir)/tmp
@@ -161,6 +162,15 @@ test_http_parser_1_SOURCES = test/http-parser-1.c $(test_COMMON)
 test_chunk_parser_1_SOURCES = test/chunk-parser-1.c $(test_COMMON)
 test_ioutil_1_SOURCES = test/ioutil-1.c $(test_COMMON)
 
+if HAVE_LD_WRAP
+if HAVE_EPOLL
+check_PROGRAMS += test/epoll-wrap
+test_epoll_wrap_SOURCES = $(cmogstored_SOURCES) test/epoll-wrap.c
+test_epoll_wrap_LDFLAGS = $(cmogstored_LDFLAGS) $(AM_LDFLAGS) \
+                          -Wl,--wrap=epoll_ctl -Wl,--wrap=epoll_create
+endif # HAVE_EPOLL
+endif # HAVE_LD_WRAP
+
 HELP2MAN = help2man
 dist_man_MANS = cmogstored.1
 
diff --git a/configure.ac b/configure.ac
index 5cfdc9b..7194216 100644
--- a/configure.ac
+++ b/configure.ac
@@ -36,6 +36,9 @@ dnl gnulib *at functions aren't thread-safe, ask for the real thing
 AC_CHECK_FUNCS([openat renameat mkdirat fstatat unlinkat])
 
 AC_CHECK_FUNCS([epoll_wait epoll_pwait ppoll])
+AC_CHECK_FUNCS([epoll_ctl], [HAVE_EPOLL=1], [HAVE_EPOLL=0])
+AC_SUBST(HAVE_EPOLL)
+AM_CONDITIONAL(HAVE_EPOLL, test "x$HAVE_EPOLL" = "x1")
 
 dnl libkqueue should work in the future
 AC_CHECK_FUNCS([kqueue])
@@ -65,6 +68,7 @@ esac
 ])
 
 CM_SYSTEMTAP
+CM_LD_WRAP
 
 AC_CONFIG_FILES([Makefile lib/Makefile])
 AC_OUTPUT
diff --git a/m4/ld_wrap.m4 b/m4/ld_wrap.m4
new file mode 100644
index 0000000..6b949e2
--- /dev/null
+++ b/m4/ld_wrap.m4
@@ -0,0 +1,21 @@
+dnl check for --wrap support in ld (expected of GNU ld)
+AC_DEFUN([CM_LD_WRAP],[
+AC_CACHE_CHECK([if gcc/ld supports -Wl,--wrap],
+[cm_cv_ld_wrap],
+[if test "$enable_shared" = no
+then
+        cm_cv_ld_wrap="not needed, shared libraries are disabled"
+else
+        cm_ldflags_save="$LDFLAGS"
+        LDFLAGS="-Wl,--wrap=free"
+        AC_LINK_IFELSE([AC_LANG_PROGRAM([
+                #include <stdlib.h>
+                static void __wrap_free(void *ptr) { __real_free(ptr); }
+                ],[ free(NULL); ]
+        )],
+        [cm_cv_ld_wrap=yes],
+        [cm_cv_ld_wrap=no])
+        LDFLAGS="$cm_ldflags_save"
+fi])
+        AM_CONDITIONAL([HAVE_LD_WRAP], test "x$cm_cv_ld_wrap" = "xyes")
+])
diff --git a/test/.gitignore b/test/.gitignore
index 9d883f4..02f2051 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -1,3 +1,4 @@
 /.*
 slow.mk
+epoll-wrap
 *.so
diff --git a/test/epoll-wrap.c b/test/epoll-wrap.c
new file mode 100644
index 0000000..a527e20
--- /dev/null
+++ b/test/epoll-wrap.c
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2012-2013, Eric Wong <normalperson@yhbt.net>
+ * License: GPLv3 or later (see COPYING for details)
+ */
+/*
+ * fault injection wrapper for epoll
+ */
+#include "cmogstored.h"
+#if defined(HAVE_EPOLL_WAIT) && ! MOG_LIBKQUEUE
+static sig_atomic_t epoll_ctl_fail;
+#define EMIT(s) write(STDERR_FILENO, (s), sizeof(s)-1)
+
+/* test/epoll_enospc depends on the following line */
+static const char msg[] = "epoll_ctl failure injection\n";
+
+int __real_epoll_ctl(int epfd, int op, int fd, struct epoll_event *event);
+int __real_epoll_create(int flags);
+
+int __wrap_epoll_ctl(int epfd, int op, int fd, struct epoll_event *event)
+{
+        if (epoll_ctl_fail) {
+                EMIT(msg);
+                errno = epoll_ctl_fail;
+                return -1;
+        }
+
+        return __real_epoll_ctl(epfd, op, fd, event);
+}
+
+static void set_wrap_epoll_ctl(int signum)
+{
+        if (signum == SIGTTIN) {
+                epoll_ctl_fail = ENOSPC;
+                EMIT("epoll_ctl ENOSPC on\n");
+        } else {
+                epoll_ctl_fail = 0;
+                EMIT("epoll_ctl ENOSPC off\n");
+        }
+}
+
+int __wrap_epoll_create(int flags)
+{
+        struct sigaction sa;
+
+        memset(&sa, 0, sizeof(struct sigaction));
+        CHECK(int, 0, sigemptyset(&sa.sa_mask) );
+        sa.sa_handler = set_wrap_epoll_ctl;
+        CHECK(int, 0, sigaction(SIGTTIN, &sa, NULL));
+        CHECK(int, 0, sigaction(SIGTTOU, &sa, NULL));
+
+        return __real_epoll_create(flags);
+}
+#endif /* defined(HAVE_EPOLL_WAIT) && ! MOG_LIBKQUEUE */
diff --git a/test/epoll_enospc.rb b/test/epoll_enospc.rb
new file mode 100644
index 0000000..5c254c0
--- /dev/null
+++ b/test/epoll_enospc.rb
@@ -0,0 +1,100 @@
+#!/usr/bin/env ruby
+# -*- encoding: binary -*-
+# Copyright (C) 2012-2013, Eric Wong <normalperson@yhbt.net>
+# License: GPLv3 or later (see COPYING for details)
+require 'test/test_helper'
+
+TEST_PROG = 'test/epoll-wrap'
+has_epoll = false
+if File.exist?(TEST_PROG) && `which lsof 2>/dev/null`.strip.size > 0
+  s = `strings test/epoll-wrap`.split(/\n/)
+  has_epoll = !!s.grep(/epoll_ctl failure injection/)[0]
+end
+
+class TestEpollEnospc < Test::Unit::TestCase
+  def setup
+    @tmpdir = Dir.mktmpdir('cmogstored-epoll-enospc-test')
+    @to_close = []
+    @host = TEST_HOST
+    srv = TCPServer.new(@host, 0)
+    @port = srv.addr[1]
+    srv.close
+    @err = Tempfile.new("stderr")
+    cmd = [ TEST_PROG, "--docroot=#@tmpdir", "--httplisten=#@host:#@port",
+            "--maxconns=500" ]
+    vg = ENV["VALGRIND"] and cmd = vg.split(/\s+/).concat(cmd)
+    @pid = fork {
+      $stderr.reopen(@err)
+      @err.close
+      exec(*cmd)
+    }
+    @client = get_client
+  end
+
+  def wait_for(sec, reason)
+    stop = Time.now + sec
+    begin
+      return if yield
+      sleep 0.1
+    end while Time.now < stop
+    assert false, reason
+  end
+
+  def test_close_file
+    sparse_file_prepare
+    @client.write "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"
+    buf = @client.readpartial(1000)
+    assert_match(/\r\n\r\n\z/, buf)
+    Process.kill(:TTIN, @pid)
+
+    wait_for(5, "ENOSPC injection signal") do
+      File.readlines(@err.path).grep(/ENOSPC on/)[0]
+    end
+    @client.write("GET /dev666/sparse-file.fid HTTP/1.1\r\n" \
+                  "Host: example.com\r\n\r\n")
+    sleep 1
+    bytes = 0
+    buf = ""
+    begin
+      bytes += @client.readpartial(666666, buf).bytesize
+    rescue EOFError
+      break
+    end while true
+
+    wait_for(5, "failure injection message") do
+      File.readlines(@err.path).grep(/epoll_ctl failure injection/)[0]
+    end
+
+    wait_for(5, "sparse file close") do
+      `lsof -p #@pid` !~ /sparse-file\.fid/
+    end
+  end
+
+  def teardown
+    Process.kill(:QUIT, @pid) rescue nil
+    _, status = Process.waitpid2(@pid)
+    @to_close.each { |io| io.close unless io.closed? }
+    FileUtils.rm_rf(@tmpdir)
+    @err.rewind
+    #$stderr.write(@err.read)
+    assert status.success?, status.inspect
+  end
+
+  def sparse_file_prepare(big = nil)
+    Dir.mkdir("#@tmpdir/dev666")
+    if nil == big
+      big = 1024 * 1024 * 500 # only 500M
+      big /= 10 if ENV["VALGRIND"] # valgrind slows us down enough :P
+    end
+    File.open("#@tmpdir/dev666/sparse-file.fid", "w") do |fp|
+      begin
+        fp.seek(big - 1)
+      rescue Errno::EINVAL, Errno::ENOSPC
+        big /= 2
+        warn "trying large file size: #{big}"
+        retry
+      end
+      fp.write('.')
+    end
+  end
+end if has_epoll
diff --git a/test/ruby.mk b/test/ruby.mk
index 320f7ab..c82a1ec 100644
--- a/test/ruby.mk
+++ b/test/ruby.mk
@@ -1,6 +1,6 @@
 RB_TESTS_FAST = test/cmogstored-cfg.rb test/http_dav.rb test/http_range.rb \
   test/http_put.rb test/http_getonly.rb test/inherit.rb test/upgrade.rb \
-  test/http_put6_fail.rb
+  test/http_put6_fail.rb test/epoll_enospc.rb
 RB_TESTS_SLOW = test/mgmt-usage.rb test/mgmt.rb test/mgmt-iostat.rb \
  test/http.rb test/http_put_slow.rb test/http_chunked_put.rb \
  test/graceful_quit.rb test/http_idle_expire.rb