Rack hijack issues EPOLL_CTL_DEL

This saves about 200 bytes of unswappable kernel memory,
so it might matter for systems with many connections
when hijacking.

7 files changed, 26 insertions, 10 deletions

diff --git a/lib/yahns/acceptor.rb b/lib/yahns/acceptor.rb
index 22a58ee..2a212cf 100644
--- a/lib/yahns/acceptor.rb
+++ b/lib/yahns/acceptor.rb
@@ -35,9 +35,10 @@ module Yahns::Acceptor # :nodoc:
     return __ac_quit_done?
   end
 
-  def spawn_acceptor(nr, logger, client_class, queue)
+  def spawn_acceptor(nr, logger, client_class)
     @thrs = nr.times.map do
       Thread.new do
+        queue = client_class.queue
         t = Thread.current
         accept_flags = Kgio::SOCK_NONBLOCK | Kgio::SOCK_CLOEXEC
         qev_flags = client_class.superclass::QEV_FLAGS
diff --git a/lib/yahns/http_client.rb b/lib/yahns/http_client.rb
index 3af18cf..721be04 100644
--- a/lib/yahns/http_client.rb
+++ b/lib/yahns/http_client.rb
@@ -226,7 +226,10 @@ class Yahns::HttpClient < Kgio::Socket # :nodoc:
   end
 
   def hijack_proc(env)
-    proc { env[RACK_HIJACK_IO] = self }
+    proc do
+      self.class.queue.queue_del(self) # EPOLL_CTL_DEL
+      env[RACK_HIJACK_IO] = self
+    end
   end
 
   # called automatically by kgio_write
@@ -251,6 +254,16 @@ class Yahns::HttpClient < Kgio::Socket # :nodoc:
     end while true
   end
 
+  def response_hijacked(fn)
+    # we must issue EPOLL_CTL_DEL before hijacking (if we issue it at all),
+    # because the hijacker may close use before we get back to the epoll worker
+    # loop.  EPOLL_CTL_DEL saves about 200 bytes of unswappable kernel memory,
+    # so it can matter if we have lots of hijacked sockets.
+    self.class.queue.queue_del(self)
+    fn.call(self)
+    :ignore
+  end
+
   # if we get any error, try to write something back to the client
   # assuming we haven't closed the socket, but don't get hung up
   # if the socket is already closed or broken.  We'll always return
diff --git a/lib/yahns/http_context.rb b/lib/yahns/http_context.rb
index 651cb08..605989f 100644
--- a/lib/yahns/http_context.rb
+++ b/lib/yahns/http_context.rb
@@ -15,6 +15,7 @@ module Yahns::HttpContext # :nodoc:
   attr_accessor :persistent_connections # true or false only
   attr_accessor :client_timeout
   attr_accessor :qegg
+  attr_accessor :queue # set right before spawning acceptors
   attr_reader :app
   attr_reader :app_defaults
 
@@ -30,6 +31,7 @@ module Yahns::HttpContext # :nodoc:
     @persistent_connections = true
     @client_timeout = 15
     @qegg = nil
+    @queue = nil
   end
 
   # call this after forking
diff --git a/lib/yahns/http_response.rb b/lib/yahns/http_response.rb
index 6b97038..6ddb1c8 100644
--- a/lib/yahns/http_response.rb
+++ b/lib/yahns/http_response.rb
@@ -146,10 +146,7 @@ module Yahns::HttpResponse # :nodoc:
       end while true
     end
 
-    if hijack
-      hijack.call(self)
-      return :ignore
-    end
+    return response_hijacked(hijack) if hijack
 
     if body.respond_to?(:to_path)
       @state = body = Yahns::StreamFile.new(body, alive, offset, count)
diff --git a/lib/yahns/queue_epoll.rb b/lib/yahns/queue_epoll.rb
index c22a624..3d2e33f 100644
--- a/lib/yahns/queue_epoll.rb
+++ b/lib/yahns/queue_epoll.rb
@@ -27,6 +27,10 @@ class Yahns::Queue < SleepyPenguin::Epoll::IO # :nodoc:
     Thread.current[:yahns_fdmap] = @fdmap
   end
 
+  def queue_del(io)
+    epoll_ctl(Epoll::CTL_DEL, io, 0)
+  end
+
   # returns an array of infinitely running threads
   def worker_thread(logger, max_events)
     Thread.new do
diff --git a/lib/yahns/server.rb b/lib/yahns/server.rb
index 462717b..4bd523f 100644
--- a/lib/yahns/server.rb
+++ b/lib/yahns/server.rb
@@ -344,10 +344,10 @@ class Yahns::Server # :nodoc:
       opts = sock_opts(l)
       ctx = opts[:yahns_app_ctx]
       qegg = ctx.qegg || @config.qeggs[:default]
-      q = queues[qegg] ||= qegg_vivify(qegg, fdmap)
+      ctx.queue = queues[qegg] ||= qegg_vivify(qegg, fdmap)
 
       # acceptors feed the the queues
-      l.spawn_acceptor(opts[:threads] || 1, @logger, ctx, q)
+      l.spawn_acceptor(opts[:threads] || 1, @logger, ctx)
     end
     fdmap
   end
diff --git a/lib/yahns/wbuf_common.rb b/lib/yahns/wbuf_common.rb
index 20f6e18..cfafaa2 100644
--- a/lib/yahns/wbuf_common.rb
+++ b/lib/yahns/wbuf_common.rb
@@ -23,8 +23,7 @@ module Yahns::WbufCommon # :nodoc:
   def wbuf_close_common(client)
     @body.close if @body.respond_to?(:close)
     if @wbuf_persist.respond_to?(:call) # hijack
-      @wbuf_persist.call(client)
-      :ignore
+      client.response_hijacked(@wbuf_persist) # :ignore
     else
       @wbuf_persist # true or false or Yahns::StreamFile
     end