From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: AS24940 78.46.0.0/15 X-Spam-Status: No, score=-2.1 required=3.0 tests=AWL,BAYES_00,RCVD_IN_XBL shortcircuit=no autolearn=no version=3.3.2 X-Original-To: yahns-public@yhbt.net Received: from 80x24.org (tor-exit1.arbitrary.ch [78.46.51.124]) by dcvr.yhbt.net (Postfix) with ESMTP id 27D921F462 for ; Tue, 7 Apr 2015 03:17:06 +0000 (UTC) From: Eric Wong To: yahns-public@yhbt.net Subject: [PATCH 1/2] proxy_pass: preliminary support for passing trailers Date: Tue, 7 Apr 2015 03:16:57 +0000 Message-Id: <1428376618-13665-2-git-send-email-e@80x24.org> In-Reply-To: <1428376618-13665-1-git-send-email-e@80x24.org> References: <1428376618-13665-1-git-send-email-e@80x24.org> List-Id: Rack apps may (through a round-about way) send HTTP trailers to HTTP/1.1 clients, and we need a way to forward those responses through without losing the trailers. --- lib/yahns/proxy_http_response.rb | 81 ++++++++++++++++++++++++++++++---------- lib/yahns/proxy_pass.rb | 1 + test/test_proxy_pass.rb | 62 +++++++++++++++++++++++++++++- 3 files changed, 123 insertions(+), 21 deletions(-) diff --git a/lib/yahns/proxy_http_response.rb b/lib/yahns/proxy_http_response.rb index 8f4790e..fe0a37b 100644 --- a/lib/yahns/proxy_http_response.rb +++ b/lib/yahns/proxy_http_response.rb @@ -120,6 +120,7 @@ module Yahns::HttpResponse # :nodoc: end until len == 0 else # nasty chunked body + req_res.proxy_trailers = nil # define to avoid warnings for now # Only HTTP/1.1 supports chunked responses, we must translate # otherwise. Otherwise, we must drop the connection to signal @@ -130,9 +131,7 @@ module Yahns::HttpResponse # :nodoc: when String kcar.filter_body(buf, tmp) unless buf.empty? - tmp = rechunk ? [ "#{buf.size.to_s(16)}\r\n", buf, "\r\n".freeze ] - : buf - wbuf = proxy_write(wbuf, tmp, alive) + wbuf = proxy_write(wbuf, rechunk ? chunk_out(buf) : buf, alive) end when nil # premature EOF return proxy_err_response(nil, req_res, nil, wbuf) @@ -142,8 +141,25 @@ module Yahns::HttpResponse # :nodoc: return :wait_readable # self remains in :ignore, wait on upstream end until kcar.body_eof? - # TODO: Trailer support - wbuf = proxy_write(wbuf, "0\r\n\r\n".freeze, alive) if rechunk + buf = tmp + req_res.proxy_trailers = [ buf, tlr = [] ] + rbuf = Thread.current[:yahns_rbuf] = '' + if rechunk + until kcar.trailers(tlr, buf) + case rv = req_res.kgio_tryread(0x2000, rbuf) + when String + buf << rv + when :wait_readable + # for ensure: + wbuf ||= Yahns::Wbuf.new(nil, alive, k.output_buffer_tmpdir, + false) + return :wait_readable + when nil # premature EOF + return proxy_err_response(nil, req_res, nil, wbuf) + end # no loop here + end + wbuf = proxy_write(wbuf, trailer_out(tlr), alive) + end end end @@ -178,22 +194,36 @@ module Yahns::HttpResponse # :nodoc: # Rack/Rails can always send streaming responses. rechunk = @hs.env['HTTP_VERSION'] == 'HTTP/1.1'.freeze buf = '' - case tmp = req_res.kgio_tryread(0x2000, rbuf) - when String - kcar.filter_body(buf, tmp) - unless buf.empty? - tmp = rechunk ? [ "#{buf.size.to_s(16)}\r\n", buf, "\r\n".freeze ] - : buf - wbuf.wbuf_write(self, tmp) - end - when nil # premature EOF - return proxy_err_response(nil, req_res, nil, wbuf) - when :wait_readable - return :wait_readable # self remains in :ignore, wait on upstream - end until kcar.body_eof? - # TODO: Trailer support - wbuf.wbuf_write(self, "0\r\n\r\n".freeze) if rechunk + unless req_res.proxy_trailers + # are we done dechunking the main body, yet? + case tmp = req_res.kgio_tryread(0x2000, rbuf) + when String + kcar.filter_body(buf, tmp) + buf.empty? or wbuf.wbuf_write(self, rechunk ? chunk_out(buf) : buf) + when nil # premature EOF + return proxy_err_response(nil, req_res, nil, wbuf) + when :wait_readable + return :wait_readable # self remains in :ignore, wait on upstream + end until kcar.body_eof? + req_res.proxy_trailers = [ tmp, [] ] # onto trailers! + rbuf = Thread.current[:yahns_rbuf] = '' + end + + buf, tlr = *req_res.proxy_trailers + if rechunk + until kcar.trailers(tlr, buf) + case rv = req_res.kgio_tryread(0x2000, rbuf) + when String + buf << rv + when :wait_readable + return :wait_readable + when nil # premature EOF + return proxy_err_response(nil, req_res, nil, wbuf) + end # no loop here + end + wbuf.wbuf_write(self, trailer_out(tlr)) + end end busy = wbuf.busy and return proxy_busy_mod_blocked(wbuf, busy) @@ -226,4 +256,15 @@ module Yahns::HttpResponse # :nodoc: # no touching self after queue_mod :ignore end + + # n.b.: we can use String#size for optimized dispatch under YARV instead + # of String#bytesize because all the IO read methods return a binary + # string when given a maximum read length + def chunk_out(buf) + [ "#{buf.size.to_s(16)}\r\n", buf, "\r\n".freeze ] + end + + def trailer_out(tlr) + "0\r\n#{tlr.map! do |k,v| "#{k}: #{v}\r\n" end.join}\r\n" + end end diff --git a/lib/yahns/proxy_pass.rb b/lib/yahns/proxy_pass.rb index ec1fbcf..3812dbf 100644 --- a/lib/yahns/proxy_pass.rb +++ b/lib/yahns/proxy_pass.rb @@ -12,6 +12,7 @@ require_relative 'proxy_http_response' class Yahns::ProxyPass # :nodoc: class ReqRes < Kgio::Socket attr_writer :resbuf + attr_accessor :proxy_trailers def req_start(c, req, input, chunked) @hdr = @resbuf = nil diff --git a/test/test_proxy_pass.rb b/test/test_proxy_pass.rb index 47fc231..943fb35 100644 --- a/test/test_proxy_pass.rb +++ b/test/test_proxy_pass.rb @@ -53,6 +53,33 @@ class TestProxyPass < Testcase io = env['rack.hijack'].call io.write(TRUNCATE_HEAD) io.close + when '/response-trailer' + h = [ + %w(Content-Type text/pain), + %w(Transfer-Encoding chunked), + %w(Trailer Foo) + ] + b = [ "3\r\n", "hi\n", "\r\n", "0\r\n", "Foo: bar", "\r\n", "\r\n" ] + case env['HTTP_X_TRAILER'] + when 'fast' + b = [ b.join ] + when 'allslow' + def b.each + size.times do |i| + sleep 0.1 + yield self[i] + end + end + when /\Atlrslow(\d+)/ + b.instance_variable_set(:@yahns_sleep_thresh, $1.to_i) + def b.each + size.times do |i| + sleep(0.1) if i > @yahns_sleep_thresh + yield self[i] + end + end + end + [ 200, h, b ] when '/immediate-EOF' env['rack.hijack'].call.close when %r{\A/chunky-slow-(\d+(?:\.\d+)?)\z} @@ -184,7 +211,7 @@ class TestProxyPass < Testcase @srv.close cfg.instance_eval do app(:rack, ProxiedApp.new) do - listen "#{host2}:#{port2}" + listen "#{host2}:#{port2}", rcvbuf: 4096 client_max_body_size nil end stderr_path err.path @@ -192,6 +219,7 @@ class TestProxyPass < Testcase end check_pipelining(host, port) + check_response_trailer(host, port) gplv3 = File.open('COPYING') @@ -434,4 +462,36 @@ class TestProxyPass < Testcase ensure s.close if s end + + def check_response_trailer(host, port) + thrs = [ + "X-Trailer: fast\r\n", + "X-Trailer: allslow\r\n", + "X-Trailer: tlrslow1\r\n", + "X-Trailer: tlrslow2\r\n", + "X-Trailer: tlrslow3\r\n", + "X-Trailer: tlrslow4\r\n", + '' + ].map do |x| + Thread.new do + s = TCPSocket.new(host, port) + s.write "GET /response-trailer HTTP/1.1\r\n#{x}" \ + "Host: example.com\r\n\r\n" + res = '' + buf = '' + Timeout.timeout(60) do + until res =~ /Foo: bar\r\n\r\n\z/ + res << s.readpartial(16384, buf) + end + end + assert_match(%r{\r\n0\r\nFoo: bar\r\n\r\n\z}, res) + assert_match(%r{^Trailer: Foo\r\n}, res) + assert_match(%r{^Transfer-Encoding: chunked\r\n}, res) + assert_match(%r{\AHTTP/1\.1 200 OK\r\n}, res) + s.close + :OK + end + end + thrs.each { |t| assert_equal(:OK, t.value) } + end end -- EW