From 71716672752e573ff15002aaefd6e8ba8c6b6cb6 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Thu, 9 Dec 2010 03:39:03 +0000 Subject: allow client_buffer_body_size to be tuned Since modern machines have more memory these days and clients are sending more data, avoiding potentially slow filesystem operations for larger uploads can be useful for some applications. --- lib/unicorn/configurator.rb | 24 +++++++----- lib/unicorn/http_server.rb | 8 ++++ lib/unicorn/tee_input.rb | 14 ++++++- t/t0116-client_body_buffer_size.sh | 80 ++++++++++++++++++++++++++++++++++++++ t/t0116.ru | 16 ++++++++ 5 files changed, 131 insertions(+), 11 deletions(-) create mode 100755 t/t0116-client_body_buffer_size.sh create mode 100644 t/t0116.ru diff --git a/lib/unicorn/configurator.rb b/lib/unicorn/configurator.rb index 3cacb91..a044e5d 100644 --- a/lib/unicorn/configurator.rb +++ b/lib/unicorn/configurator.rb @@ -41,6 +41,7 @@ class Unicorn::Configurator :pid => nil, :preload_app => false, :rewindable_input => true, # for Rack 2.x: (Rack::VERSION[0] <= 1), + :client_body_buffer_size => Unicorn::Const::MAX_BODY, } #:startdoc: @@ -181,11 +182,7 @@ class Unicorn::Configurator # server 192.168.0.9:8080 fail_timeout=0; # } def timeout(seconds) - Numeric === seconds or raise ArgumentError, - "not numeric: timeout=#{seconds.inspect}" - seconds >= 3 or raise ArgumentError, - "too low: timeout=#{seconds.inspect}" - set[:timeout] = seconds + set_int(:timeout, seconds, 3) end # sets the current number of worker_processes to +nr+. Each worker @@ -195,11 +192,7 @@ class Unicorn::Configurator # the rest of your Unicorn configuration. See the SIGNALS document # for more information. def worker_processes(nr) - Integer === nr or raise ArgumentError, - "not an integer: worker_processes=#{nr.inspect}" - nr >= 0 or raise ArgumentError, - "not non-negative: worker_processes=#{nr.inspect}" - set[:worker_processes] = nr + set_int(:worker_processes, nr, 1) end # sets listeners to the given +addresses+, replacing or augmenting the @@ -393,6 +386,12 @@ class Unicorn::Configurator set_bool(:rewindable_input, bool) end + # The maximum size (in +bytes+) to buffer in memory before + # resorting to a temporary file. Default is 112 kilobytes. + def client_body_buffer_size(bytes) + set_int(:client_body_buffer_size, bytes, 0) + end + # Allow redirecting $stderr to a given path. Unlike doing this from # the shell, this allows the unicorn process to know the path its # writing to and rotate the file if it is used for logging. The @@ -471,6 +470,11 @@ class Unicorn::Configurator end private + def set_int(var, n, min) + Integer === n or raise ArgumentError, "not an integer: #{var}=#{n.inspect}" + n >= min or raise ArgumentError, "too low (< #{min}): #{var}=#{n.inspect}" + set[var] = n + end def set_path(var, path) #:nodoc: case path diff --git a/lib/unicorn/http_server.rb b/lib/unicorn/http_server.rb index 0bb4359..29b34d6 100644 --- a/lib/unicorn/http_server.rb +++ b/lib/unicorn/http_server.rb @@ -364,6 +364,14 @@ class Unicorn::HttpServer Unicorn::TeeInput : Unicorn::StreamInput end + def client_body_buffer_size + Unicorn::TeeInput.client_body_buffer_size + end + + def client_body_buffer_size=(bytes) + Unicorn::TeeInput.client_body_buffer_size = bytes + end + private # wait for a signal hander to wake us up and then consume the pipe diff --git a/lib/unicorn/tee_input.rb b/lib/unicorn/tee_input.rb index 53f6ebf..a038a84 100644 --- a/lib/unicorn/tee_input.rb +++ b/lib/unicorn/tee_input.rb @@ -16,12 +16,24 @@ class Unicorn::TeeInput < Unicorn::StreamInput # resorting to a temporary file. Default is 112 kilobytes. @@client_body_buffer_size = Unicorn::Const::MAX_BODY + # sets the maximum size of request bodies to buffer in memory, + # amounts larger than this are buffered to the filesystem + def self.client_body_buffer_size=(bytes) + @@client_body_buffer_size = bytes + end + + # returns the maximum size of request bodies to buffer in memory, + # amounts larger than this are buffered to the filesystem + def self.client_body_buffer_size + @@client_body_buffer_size + end + # Initializes a new TeeInput object. You normally do not have to call # this unless you are writing an HTTP server. def initialize(socket, request) @len = request.content_length super - @tmp = @len && @len < @@client_body_buffer_size ? + @tmp = @len && @len <= @@client_body_buffer_size ? StringIO.new("") : Unicorn::TmpIO.new end diff --git a/t/t0116-client_body_buffer_size.sh b/t/t0116-client_body_buffer_size.sh new file mode 100755 index 0000000..c9e17c7 --- /dev/null +++ b/t/t0116-client_body_buffer_size.sh @@ -0,0 +1,80 @@ +#!/bin/sh +. ./test-lib.sh +t_plan 12 "client_body_buffer_size settings" + +t_begin "setup and start" && { + unicorn_setup + rtmpfiles unicorn_config_tmp one_meg + dd if=/dev/zero bs=1M count=1 of=$one_meg + cat >> $unicorn_config < $unicorn_config_tmp + echo client_body_buffer_size 0 >> $unicorn_config + unicorn -D -c $unicorn_config t0116.ru + unicorn_wait_start + fs_class=Unicorn::TmpIO + mem_class=StringIO + + test x"$(cat $fifo)" = xSTART +} + +t_begin "class for a zero-byte file should be StringIO" && { + > $tmp + test xStringIO = x"$(curl -T $tmp -sSf http://$listen/input_class)" +} + +t_begin "class for a 1 byte file should be filesystem-backed" && { + echo > $tmp + test x$fs_class = x"$(curl -T $tmp -sSf http://$listen/tmp_class)" +} + +t_begin "reload with default client_body_buffer_size" && { + mv $unicorn_config_tmp $unicorn_config + kill -HUP $unicorn_pid + test x"$(cat $fifo)" = xSTART +} + +t_begin "class for a 1 byte file should be memory-backed" && { + echo > $tmp + test x$mem_class = x"$(curl -T $tmp -sSf http://$listen/tmp_class)" +} + +t_begin "class for a random blob file should be filesystem-backed" && { + resp="$(curl -T random_blob -sSf http://$listen/tmp_class)" + test x$fs_class = x"$resp" +} + +t_begin "one megabyte file should be filesystem-backed" && { + resp="$(curl -T $one_meg -sSf http://$listen/tmp_class)" + test x$fs_class = x"$resp" +} + +t_begin "reload with a big client_body_buffer_size" && { + echo "client_body_buffer_size(1024 * 1024)" >> $unicorn_config + kill -HUP $unicorn_pid + test x"$(cat $fifo)" = xSTART +} + +t_begin "one megabyte file should be memory-backed" && { + resp="$(curl -T $one_meg -sSf http://$listen/tmp_class)" + test x$mem_class = x"$resp" +} + +t_begin "one megabyte + 1 byte file should be filesystem-backed" && { + echo >> $one_meg + resp="$(curl -T $one_meg -sSf http://$listen/tmp_class)" + test x$fs_class = x"$resp" +} + +t_begin "killing succeeds" && { + kill $unicorn_pid +} + +t_begin "check stderr" && { + check_stderr +} + +t_done diff --git a/t/t0116.ru b/t/t0116.ru new file mode 100644 index 0000000..fab5fce --- /dev/null +++ b/t/t0116.ru @@ -0,0 +1,16 @@ +#\ -E none +use Rack::ContentLength +use Rack::ContentType, 'text/plain' +app = lambda do |env| + input = env['rack.input'] + case env["PATH_INFO"] + when "/tmp_class" + body = input.instance_variable_get(:@tmp).class.name + when "/input_class" + body = input.class.name + else + return [ 500, {}, [] ] + end + [ 200, {}, [ body ] ] +end +run app -- cgit v1.2.3-24-ge0c7