From 20bf660a3efff9229c81b3b3a0feb6844bc27a7c Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Tue, 30 Jun 2009 17:15:56 -0700 Subject: TrailerParser integration into ChunkedReader Support for the "Trailer:" header and associated Trailer lines should be reasonably well supported now --- TODO | 2 -- lib/unicorn.rb | 1 + lib/unicorn/chunked_reader.rb | 10 ++++++++-- lib/unicorn/const.rb | 1 + lib/unicorn/http_request.rb | 2 +- test/unit/test_chunked_reader.rb | 37 +++++++++++++++++++++++++------------ test/unit/test_upload.rb | 27 +++++++++++++++++++++++++++ 7 files changed, 63 insertions(+), 17 deletions(-) diff --git a/TODO b/TODO index 65b3328..7e36cc2 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,3 @@ -* "Trailers:" support with TE:chunked requests - * Support HTTP/1.1 keepalive if (and probably only if) pipelining. We can do this by testing readability of socket immediately after the response is written. diff --git a/lib/unicorn.rb b/lib/unicorn.rb index 343f762..6e937bf 100644 --- a/lib/unicorn.rb +++ b/lib/unicorn.rb @@ -13,6 +13,7 @@ module Unicorn autoload :Configurator, 'unicorn/configurator' autoload :TeeInput, 'unicorn/tee_input' autoload :ChunkedReader, 'unicorn/chunked_reader' + autoload :TrailerParser, 'unicorn/trailer_parser' autoload :Util, 'unicorn/util' Z = '' # the stock empty string we use everywhere... diff --git a/lib/unicorn/chunked_reader.rb b/lib/unicorn/chunked_reader.rb index f9bd4ec..606e4a6 100644 --- a/lib/unicorn/chunked_reader.rb +++ b/lib/unicorn/chunked_reader.rb @@ -7,8 +7,8 @@ require 'unicorn/http11' module Unicorn class ChunkedReader - def initialize(input, buf) - @input, @buf = input, buf + def initialize(env, input, buf) + @env, @input, @buf = env, input, buf @chunk_left = 0 parse_chunk_header end @@ -71,6 +71,12 @@ module Unicorn @chunk_left = $1.to_i(16) if 0 == @chunk_left # EOF buf.sub!(/\A\r\n(?:\r\n)?/, Z) # cleanup for future requests + if trailer = @env[Const::HTTP_TRAILER] + tp = TrailerParser.new(trailer) + while ! tp.execute!(@env, buf) + buf << @input.readpartial(Const::CHUNK_SIZE) + end + end @input = nil end return @chunk_left diff --git a/lib/unicorn/const.rb b/lib/unicorn/const.rb index 9dcf031..be69753 100644 --- a/lib/unicorn/const.rb +++ b/lib/unicorn/const.rb @@ -32,6 +32,7 @@ module Unicorn REMOTE_ADDR="REMOTE_ADDR".freeze HTTP_X_FORWARDED_FOR="HTTP_X_FORWARDED_FOR".freeze HTTP_EXPECT="HTTP_EXPECT".freeze + HTTP_TRAILER="HTTP_TRAILER".freeze RACK_INPUT="rack.input".freeze STREAM_INPUT="unicorn.stream_input".freeze end diff --git a/lib/unicorn/http_request.rb b/lib/unicorn/http_request.rb index a6ed7c9..3df9120 100644 --- a/lib/unicorn/http_request.rb +++ b/lib/unicorn/http_request.rb @@ -96,7 +96,7 @@ module Unicorn if te = PARAMS[Const::HTTP_TRANSFER_ENCODING] if /\Achunked\z/i =~ te - socket = ChunkedReader.new(socket, body) + socket = ChunkedReader.new(PARAMS, socket, body) length = body = nil end end diff --git a/test/unit/test_chunked_reader.rb b/test/unit/test_chunked_reader.rb index 6a26c8f..67fe43b 100644 --- a/test/unit/test_chunked_reader.rb +++ b/test/unit/test_chunked_reader.rb @@ -8,6 +8,7 @@ require 'digest/sha1' class TestChunkedReader < Test::Unit::TestCase def setup + @env = {} @rd, @wr = IO.pipe @rd.binmode @wr.binmode @@ -27,7 +28,7 @@ class TestChunkedReader < Test::Unit::TestCase end def test_error - cr = bin_reader(@rd, "8\r\nasdfasdf\r\n8\r\nasdfasdfa#{'a' * 1024}") + cr = bin_reader("8\r\nasdfasdf\r\n8\r\nasdfasdfa#{'a' * 1024}") a = nil assert_nothing_raised { a = cr.readpartial(8192) } assert_equal 'asdfasdf', a @@ -37,30 +38,30 @@ class TestChunkedReader < Test::Unit::TestCase end def test_eof1 - cr = bin_reader(@rd, "0\r\n") + cr = bin_reader("0\r\n") assert_raises(EOFError) { cr.readpartial(8192) } end def test_eof2 - cr = bin_reader(@rd, "0\r\n\r\n") + cr = bin_reader("0\r\n\r\n") assert_raises(EOFError) { cr.readpartial(8192) } end def test_readpartial1 - cr = bin_reader(@rd, "4\r\nasdf\r\n0\r\n") + cr = bin_reader("4\r\nasdf\r\n0\r\n") assert_equal 'asdf', cr.readpartial(8192) assert_raises(EOFError) { cr.readpartial(8192) } end def test_gets1 - cr = bin_reader(@rd, "4\r\nasdf\r\n0\r\n") + cr = bin_reader("4\r\nasdf\r\n0\r\n") STDOUT.sync = true assert_equal 'asdf', cr.gets assert_raises(EOFError) { cr.readpartial(8192) } end def test_gets2 - cr = bin_reader(@rd, "4\r\nasd\n\r\n0\r\n\r\n") + cr = bin_reader("4\r\nasd\n\r\n0\r\n\r\n") assert_equal "asd\n", cr.gets assert_nil cr.gets end @@ -70,7 +71,7 @@ class TestChunkedReader < Test::Unit::TestCase str = ('a' * max).freeze first = 5 last = str.size - first - cr = bin_reader(@rd, + cr = bin_reader( "#{'%x' % first}\r\n#{str[0, first]}\r\n" \ "#{'%x' % last}\r\n#{str[-last, last]}\r\n" \ "0\r\n") @@ -83,7 +84,7 @@ class TestChunkedReader < Test::Unit::TestCase str = ('a' * max).freeze first = 5 last = str.size - first - cr = bin_reader(@rd, + cr = bin_reader( "#{'%x' % first}\r\n#{str[0, first]}\r\n" \ "#{'%x' % last}\r\n#{str[-last, last]}\r\n" \ "0\r\n") @@ -101,7 +102,7 @@ class TestChunkedReader < Test::Unit::TestCase str = ("z\n" * max).freeze first = 5 last = str.size - first - cr = bin_reader(@rd, + cr = bin_reader( "#{'%x' % first}\r\n#{str[0, first]}\r\n" \ "#{'%x' % last}\r\n#{str[-last, last]}\r\n" \ "0\r\n") @@ -110,7 +111,7 @@ class TestChunkedReader < Test::Unit::TestCase end def test_dd - cr = bin_reader(@rd, "6\r\nhello\n\r\n") + cr = bin_reader("6\r\nhello\n\r\n") tmp = Tempfile.new('test_dd') tmp.sync = true @@ -157,11 +158,23 @@ class TestChunkedReader < Test::Unit::TestCase assert_equal sha1_file.hexdigest, sha1.hexdigest end + def test_trailer + @env['HTTP_TRAILER'] = 'Content-MD5' + pid = fork { @wr.syswrite("Content-MD5: asdf\r\n") } + cr = bin_reader("8\r\nasdfasdf\r\n8\r\nasdfasdf\r\n0\r\n") + assert_equal 'asdfasdf', cr.readpartial(4096) + assert_equal 'asdfasdf', cr.readpartial(4096) + assert_raises(EOFError) { cr.readpartial(4096) } + pid, status = Process.waitpid2(pid) + assert status.success? + assert_equal 'asdf', @env['HTTP_CONTENT_MD5'] + end + private - def bin_reader(sock, buf) + def bin_reader(buf) buf.force_encoding(Encoding::BINARY) if buf.respond_to?(:force_encoding) - Unicorn::ChunkedReader.new(sock, buf) + Unicorn::ChunkedReader.new(@env, @rd, buf) end end diff --git a/test/unit/test_upload.rb b/test/unit/test_upload.rb index b69ba4e..37baa30 100644 --- a/test/unit/test_upload.rb +++ b/test/unit/test_upload.rb @@ -1,5 +1,6 @@ # Copyright (c) 2009 Eric Wong require 'test/test_helper' +require 'digest/md5' include Unicorn @@ -43,6 +44,7 @@ class UploadTest < Test::Unit::TestCase end end resp[:size] = input.size + resp[:content_md5] = env['HTTP_CONTENT_MD5'] [ 200, @hdr.merge({'X-Resp' => resp.inspect}), [] ] end @@ -69,6 +71,31 @@ class UploadTest < Test::Unit::TestCase assert_equal @sha1.hexdigest, resp[:sha1] end + def test_put_content_md5 + md5 = Digest::MD5.new + start_server(@sha1_app) + sock = TCPSocket.new(@addr, @port) + sock.syswrite("PUT / HTTP/1.0\r\nTransfer-Encoding: chunked\r\n" \ + "Trailer: Content-MD5\r\n\r\n") + @count.times do |i| + buf = @random.sysread(@bs) + @sha1.update(buf) + md5.update(buf) + sock.syswrite("#{'%x' % buf.size}\r\n") + sock.syswrite(buf << "\r\n") + end + sock.syswrite("0\r\n") + + content_md5 = [ md5.digest! ].pack('m').strip.freeze + sock.syswrite("Content-MD5: #{content_md5}\r\n") + read = sock.read.split(/\r\n/) + assert_equal "HTTP/1.1 200 OK", read[0] + resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, '')) + assert_equal length, resp[:size] + assert_equal @sha1.hexdigest, resp[:sha1] + assert_equal content_md5, resp[:content_md5] + end + def test_put_trickle_small @count, @bs = 2, 128 start_server(@sha1_app) -- cgit v1.2.3-24-ge0c7