From a6d80703cb7568bb529bac913e2bc52d59a244dd Mon Sep 17 00:00:00 2001 From: zedshaw Date: Fri, 23 Jun 2006 13:22:47 +0000 Subject: Refactor reaping dead threads. Slight change to license header to make it clearer. git-svn-id: svn+ssh://rubyforge.org/var/svn/mongrel/trunk@255 19e92222-5c0b-0410-8929-a290d50e31e9 --- ext/http11/http11.c | 2 +- ext/http11/http11_parser.c | 2 +- ext/http11/http11_parser.h | 2 +- ext/http11/http11_parser.rl | 2 +- lib/mongrel.rb | 81 ++++++++++++++++++++++++--------------------- lib/mongrel/camping.rb | 2 +- lib/mongrel/cgi.rb | 2 +- lib/mongrel/command.rb | 2 +- lib/mongrel/debug.rb | 2 +- lib/mongrel/handlers.rb | 2 +- lib/mongrel/init.rb | 2 +- lib/mongrel/rails.rb | 2 +- lib/mongrel/stats.rb | 2 +- lib/mongrel/tcphack.rb | 2 +- test/test_command.rb | 2 +- test/test_conditional.rb | 2 +- test/test_configurator.rb | 2 +- test/test_debug.rb | 2 +- test/test_handlers.rb | 2 +- test/test_http11.rb | 2 +- test/test_response.rb | 2 +- test/test_stats.rb | 2 +- test/test_uriclassifier.rb | 2 +- test/test_ws.rb | 2 +- tools/trickletest.rb | 48 ++++++++++++++++----------- 25 files changed, 94 insertions(+), 81 deletions(-) diff --git a/ext/http11/http11.c b/ext/http11/http11.c index 5e4e510..88e8e98 100644 --- a/ext/http11/http11.c +++ b/ext/http11/http11.c @@ -1,4 +1,4 @@ -/* Mongrel Web Server - A Mostly Ruby Webserver and Library +/* Mongrel Web Server - A Mostly Ruby HTTP server and Library * * Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com * diff --git a/ext/http11/http11_parser.c b/ext/http11/http11_parser.c index 5e593bb..a57051a 100644 --- a/ext/http11/http11_parser.c +++ b/ext/http11/http11_parser.c @@ -1,5 +1,5 @@ #line 1 "ext/http11/http11_parser.rl" -/* Mongrel Web Server - A Mostly Ruby Webserver and Library +/* Mongrel Web Server - A Mostly Ruby HTTP server and Library * * Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com * diff --git a/ext/http11/http11_parser.h b/ext/http11/http11_parser.h index 7bab703..047829a 100644 --- a/ext/http11/http11_parser.h +++ b/ext/http11/http11_parser.h @@ -1,4 +1,4 @@ -/* Mongrel Web Server - A Mostly Ruby Webserver and Library +/* Mongrel Web Server - A Mostly Ruby HTTP server and Library * * Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com * diff --git a/ext/http11/http11_parser.rl b/ext/http11/http11_parser.rl index 37a3023..1b784dd 100644 --- a/ext/http11/http11_parser.rl +++ b/ext/http11/http11_parser.rl @@ -1,4 +1,4 @@ -/* Mongrel Web Server - A Mostly Ruby Webserver and Library +/* Mongrel Web Server - A Mostly Ruby HTTP server and Library * * Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com * diff --git a/lib/mongrel.rb b/lib/mongrel.rb index 94bc85f..4921c25 100644 --- a/lib/mongrel.rb +++ b/lib/mongrel.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # @@ -212,7 +212,7 @@ module Mongrel clen -= @body.write(@socket.read(clen % Const::CHUNK_SIZE)) # then stream out nothing but perfectly sized chunks - while clen > 0 + while clen > 0 and !@socket.closed? data = @socket.read(Const::CHUNK_SIZE) # have to do it this way since @socket.eof? causes it to block raise "Socket closed or read failure" if not data or data.length != Const::CHUNK_SIZE @@ -224,7 +224,7 @@ module Mongrel @body.rewind rescue Object # any errors means we should delete the file, including if the file is dumped - STDERR.puts "Error reading request: #$!" + @socket.close unless @socket.closed? @body.delete if @body.class == Tempfile @body = nil # signals that there was a problem end @@ -454,7 +454,7 @@ module Mongrel end - # This is the main driver of Mongrel, while the Mognrel::HttpParser and Mongrel::URIClassifier + # This is the main driver of Mongrel, while the Mongrel::HttpParser and Mongrel::URIClassifier # make up the majority of how the server functions. It's a very simple class that just # has a thread accepting connections and a simple HttpServer.process_client function # to do the heavy lifting with the IO and Ruby. @@ -517,7 +517,7 @@ module Mongrel begin parser = HttpParser.new params = {} - + request = nil data = client.readpartial(Const::CHUNK_SIZE) nparsed = 0 @@ -535,6 +535,8 @@ module Mongrel params[Const::PATH_INFO] = path_info params[Const::SCRIPT_NAME] = script_name params[Const::REMOTE_ADDR] = params[Const::HTTP_X_FORWARDED_FOR] || client.peeraddr.last + + # TODO: Find a faster/better way to carve out the range, preferably without copying. data = data[nparsed ... data.length] || "" if handlers[0].request_notify @@ -542,7 +544,6 @@ module Mongrel handlers[0].request_begins(params) end - # TODO: Find a faster/better way to carve out the range, preferably without copying. request = HttpRequest.new(params, data, client) # in the case of large file uploads the user could close the socket, so skip those requests @@ -563,7 +564,6 @@ module Mongrel end else # Didn't find it, return a stock 404 response. - # TODO: Implement customer 404 files (but really they should use a real web server). client.write(Const::ERROR_404_RESPONSE) end @@ -580,27 +580,44 @@ module Mongrel # ignored rescue HttpParserError STDERR.puts "#{Time.now}: BAD CLIENT (#{params[Const::HTTP_X_FORWARDED_FOR] || client.peeraddr.last}): #$!" - rescue => details + rescue Object STDERR.puts "#{Time.now}: ERROR: #$!" - STDERR.puts details.backtrace.join("\n") ensure client.close unless client.closed? + request.body.delete if request and request.body.class == Tempfile end end # Used internally to kill off any worker threads that have taken too long # to complete processing. Only called if there are too many processors - # currently servicing. - def reap_dead_workers(worker_list) - mark = Time.now - worker_list.each do |w| - w[:started_on] = Time.now if not w[:started_on] - - if mark - w[:started_on] > @death_time + @timeout - STDERR.puts "Thread #{w.inspect} is too old, killing." - w.raise(StopServer.new("Timed out thread.")) + # currently servicing. It returns the count of workers still active + # after the reap is done. It only runs if there are workers to reap. + def reap_dead_workers(reason='unknown') + if @workers.list.length > 0 + STDERR.puts "#{Time.now}: Reaping #{@workers.list.length} threads for slow workers because of '#{reason}'" + mark = Time.now + @workers.list.each do |w| + w[:started_on] = Time.now if not w[:started_on] + + if mark - w[:started_on] > @death_time + @timeout + STDERR.puts "Thread #{w.inspect} is too old, killing." + w.raise(StopServer.new("Timed out thread.")) + end end end + + return @workers.list.length + end + + # Performs a wait on all the currently running threads and kills any that take + # too long. Right now it just waits 60 seconds, but will expand this to + # allow setting. The @timeout setting does extend this waiting period by + # that much longer. + def graceful_shutdown + while reap_dead_workers("shutdown") > 0 + STDERR.print "Waiting for #{@workers.list.length} requests to finish, could take #{@death_time + @timeout} seconds." + sleep @death_time / 10 + end end @@ -618,14 +635,12 @@ module Mongrel if worker_list.length >= @num_processors STDERR.puts "Server overloaded with #{worker_list.length} processors (#@num_processors max). Dropping connection." client.close - reap_dead_workers(worker_list) + reap_dead_workers("max processors") else - thread = Thread.new do - process_client(client) - end + thread = Thread.new { process_client(client) } + thread.abort_on_exception = true thread[:started_on] = Time.now - thread.priority=1 @workers.add(thread) sleep @timeout/100 if @timeout > 0 @@ -634,22 +649,12 @@ module Mongrel @socket.close if not @socket.closed? break rescue Errno::EMFILE - STDERR.puts "Too many open files. Try increasing ulimits." + reap_dead_workers("too many open files") sleep 0.5 end end - # troll through the threads that are waiting and kill any that take too long - # TODO: Allow for death time to be set if people ask for it. - @death_time = 10 - shutdown_start = Time.now - - while @workers.list.length > 0 - waited_for = (Time.now - shutdown_start).ceil - STDERR.print "Shutdown waited #{waited_for} for #{@workers.list.length} requests, could take #{@death_time + @timeout} seconds.\r" if @workers.list.length > 0 - sleep 1 - reap_dead_workers(@workers.list) - end + graceful_shutdown end return @acceptor @@ -972,7 +977,7 @@ module Mongrel # This method should actually be called *outside* of the - # Configurator block so that you can control it. In otherwords + # Configurator block so that you can control it. In other words # do it like: config.join. def join @listeners.values.each {|s| s.acceptor.join } @@ -991,9 +996,9 @@ module Mongrel # debug "/", what = [:rails] # # And it will only produce the log/mongrel_debug/rails.log file. - # Available options are: :object, :railes, :files, :threads, :params + # Available options are: :object, :rails, :files, :threads, :params # - # NOTE: Use [:files] to get acccesses dumped to stderr like with WEBrick. + # NOTE: Use [:files] to get accesses dumped to stderr like with WEBrick. def debug(location, what = [:object, :rails, :files, :threads, :params]) require 'mongrel/debug' handlers = { diff --git a/lib/mongrel/camping.rb b/lib/mongrel/camping.rb index 351be57..b8922f7 100644 --- a/lib/mongrel/camping.rb +++ b/lib/mongrel/camping.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/lib/mongrel/cgi.rb b/lib/mongrel/cgi.rb index e13af84..a6ce80e 100644 --- a/lib/mongrel/cgi.rb +++ b/lib/mongrel/cgi.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/lib/mongrel/command.rb b/lib/mongrel/command.rb index 1071944..62fa8a4 100644 --- a/lib/mongrel/command.rb +++ b/lib/mongrel/command.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/lib/mongrel/debug.rb b/lib/mongrel/debug.rb index 2a13ae5..cf532fa 100644 --- a/lib/mongrel/debug.rb +++ b/lib/mongrel/debug.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/lib/mongrel/handlers.rb b/lib/mongrel/handlers.rb index 509d3e9..f7f7e74 100644 --- a/lib/mongrel/handlers.rb +++ b/lib/mongrel/handlers.rb @@ -1,7 +1,7 @@ require 'mongrel/stats' require 'zlib' -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/lib/mongrel/init.rb b/lib/mongrel/init.rb index 55c251d..94b4d13 100644 --- a/lib/mongrel/init.rb +++ b/lib/mongrel/init.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/lib/mongrel/rails.rb b/lib/mongrel/rails.rb index f3b9aef..2be780d 100644 --- a/lib/mongrel/rails.rb +++ b/lib/mongrel/rails.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/lib/mongrel/stats.rb b/lib/mongrel/stats.rb index be78741..05eec6c 100644 --- a/lib/mongrel/stats.rb +++ b/lib/mongrel/stats.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/lib/mongrel/tcphack.rb b/lib/mongrel/tcphack.rb index 1ed78a7..82fe9ec 100644 --- a/lib/mongrel/tcphack.rb +++ b/lib/mongrel/tcphack.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/test/test_command.rb b/test/test_command.rb index 85793b2..a5c0d8b 100644 --- a/test/test_command.rb +++ b/test/test_command.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/test/test_conditional.rb b/test/test_conditional.rb index e6fe053..984dea0 100644 --- a/test/test_conditional.rb +++ b/test/test_conditional.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/test/test_configurator.rb b/test/test_configurator.rb index 8a3fe40..95a8fe3 100644 --- a/test/test_configurator.rb +++ b/test/test_configurator.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/test/test_debug.rb b/test/test_debug.rb index 58cdfdd..e327501 100644 --- a/test/test_debug.rb +++ b/test/test_debug.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/test/test_handlers.rb b/test/test_handlers.rb index 8eaf8dd..16292b4 100644 --- a/test/test_handlers.rb +++ b/test/test_handlers.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/test/test_http11.rb b/test/test_http11.rb index bf8276a..1231453 100644 --- a/test/test_http11.rb +++ b/test/test_http11.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/test/test_response.rb b/test/test_response.rb index 97e42ca..05811fb 100644 --- a/test/test_response.rb +++ b/test/test_response.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/test/test_stats.rb b/test/test_stats.rb index 9a6c8f1..547414d 100644 --- a/test/test_stats.rb +++ b/test/test_stats.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/test/test_uriclassifier.rb b/test/test_uriclassifier.rb index 3369f2b..52f4f64 100644 --- a/test/test_uriclassifier.rb +++ b/test/test_uriclassifier.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/test/test_ws.rb b/test/test_ws.rb index 2da7035..9f4a9ef 100644 --- a/test/test_ws.rb +++ b/test/test_ws.rb @@ -1,4 +1,4 @@ -# Mongrel Web Server - A Mostly Ruby Webserver and Library +# Mongrel Web Server - A Mostly Ruby HTTP server and Library # # Copyright (C) 2005 Zed A. Shaw zedshaw AT zedshaw dot com # diff --git a/tools/trickletest.rb b/tools/trickletest.rb index cf3999b..52fcfac 100644 --- a/tools/trickletest.rb +++ b/tools/trickletest.rb @@ -4,34 +4,42 @@ require 'stringio' def do_test(st, chunk) s = TCPSocket.new('127.0.0.1',ARGV[0].to_i); req = StringIO.new(st) - - while data = req.read(chunk) - puts "write #{data.length}: '#{data}'" - s.write(data) - s.flush - sleep 0.1 + nout = 0 + randstop = rand(st.length / 10) + STDERR.puts "stopping after: #{randstop}" + + begin + while data = req.read(chunk) + nout += s.write(data) + s.flush + sleep 0.1 + if nout > randstop + STDERR.puts "BANG! after #{nout} bytes." + break + end + end + rescue Object + STDERR.puts "ERROR: #$!" + ensure + s.close end - s.close end +content = "-" * (1024 * 240) +st = "GET / HTTP/1.1\r\nHost: www.zedshaw.com\r\nContent-Type: text/plain\r\nContent-Length: #{content.length}\r\n\r\n#{content}" -st = "GET / HTTP/1.1\r\nHost: www.zedshaw.com\r\nContent-Type: text/plain\r\n\r\n" +puts "length: #{content.length}" threads = [] ARGV[1].to_i.times do - threads << Thread.new do - (st.length - 1).times do |chunk| - puts ">>>> #{chunk+1} sized chunks" - do_test(st, chunk+1) - end - - 1000.times do - do_test(st, rand(st.length) + 1) - end - + t = Thread.new do + size = 100 + puts ">>>> #{size} sized chunks" + do_test(st, size) end - sleep(1+rand) + t.abort_on_exception = true + threads << t end -threads.each {|t| t.join} +threads.each {|t| t.join} -- cgit v1.2.3-24-ge0c7