From 3192ec1b4054bcc228dfb88e57d5e1c828682a7b Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 2 Nov 2013 10:54:17 +0000 Subject: add extras for exec_cgi and try_gzip_static These applications are what I'll be using to run on yahns on my personal server. Including them here will be helpful for me to find bugs. I've already found some, the following commits were directly the result of playing with these extras: * stream_file: only close FDs we opened ourselves * worker-less server should not waitpid indiscriminately * http: do not drop Content-Range from response headers --- extras/exec_cgi.rb | 108 +++++++++++++++++++ extras/try_gzip_static.rb | 208 ++++++++++++++++++++++++++++++++++++ test/helper.rb | 2 +- test/test_extras_exec_cgi.rb | 81 ++++++++++++++ test/test_extras_exec_cgi.sh | 35 ++++++ test/test_extras_try_gzip_static.rb | 177 ++++++++++++++++++++++++++++++ 6 files changed, 610 insertions(+), 1 deletion(-) create mode 100644 extras/exec_cgi.rb create mode 100644 extras/try_gzip_static.rb create mode 100644 test/test_extras_exec_cgi.rb create mode 100755 test/test_extras_exec_cgi.sh create mode 100644 test/test_extras_try_gzip_static.rb diff --git a/extras/exec_cgi.rb b/extras/exec_cgi.rb new file mode 100644 index 0000000..083047e --- /dev/null +++ b/extras/exec_cgi.rb @@ -0,0 +1,108 @@ +# -*- encoding: binary -*- +# Copyright (C) 2013, Eric Wong and all contributors +# License: GPLv2 or later (https://www.gnu.org/licenses/gpl-2.0.txt) +class ExecCgi + class MyIO < Kgio::Pipe + attr_writer :my_pid + attr_writer :body_tip + attr_writer :chunked + + def each + buf = @body_tip || "" + if buf.size > 0 + buf = "#{buf.size.to_s(16)}\r\n#{buf}\r\n" if @chunked + yield buf + end + while tmp = kgio_read(8192, buf) + tmp = "#{tmp.size.to_s(16)}\r\n#{tmp}\r\n" if @chunked + yield tmp + end + yield("0\r\n\r\n") if @chunked + self + end + + def close + super + if defined?(@my_pid) && @my_pid + begin + Process.waitpid(@my_pid) + rescue Errno::ECHILD + end + end + nil + end + end + + PASS_VARS = %w( + CONTENT_LENGTH + CONTENT_TYPE + GATEWAY_INTERFACE + AUTH_TYPE + PATH_INFO + PATH_TRANSLATED + QUERY_STRING + REMOTE_ADDR + REMOTE_HOST + REMOTE_IDENT + REMOTE_USER + REQUEST_METHOD + SERVER_NAME + SERVER_PORT + SERVER_PROTOCOL + SERVER_SOFTWARE + ).map(&:freeze) # frozen strings are faster for Hash assignments + + def initialize(*args) + @args = args + first = args[0] or + raise ArgumentError, "need path to executable" + first[0] == ?/ or args[0] = ::File.expand_path(first) + File.executable?(args[0]) or + raise ArgumentError, "#{args[0]} is not executable" + end + + # Calls the app + def call(env) + cgi_env = { "SCRIPT_NAME" => @args[0], "GATEWAY_INTERFACE" => "CGI/1.1" } + PASS_VARS.each { |key| val = env[key] and cgi_env[key] = val } + env.each { |key,val| cgi_env[key] = val if key =~ /\AHTTP_/ } + pipe = MyIO.pipe + pipe[0].my_pid = Process.spawn(cgi_env, *@args, + out: pipe[1], close_others: true) + pipe[1].close + pipe = pipe[0] + + if head = pipe.kgio_read(8192) + until head =~ /\r?\n\r?\n/ + tmp = pipe.kgio_read(8192) or break + head << tmp + end + head, body = head.split(/\r?\n\r?\n/) + pipe.body_tip = body + pipe.chunked = false + + headers = Rack::Utils::HeaderHash.new + prev = nil + head.split(/\r?\n/).each do |line| + case line + when /^([A-Za-z0-9-]+):\s*(.*)$/ then headers[prev = $1] = $2 + when /^[ \t]/ then headers[prev] << "\n#{line}" if prev + end + end + status = headers.delete("Status") || 200 + unless headers.include?("Content-Length") || + headers.include?("Transfer-Encoding") + case env['HTTP_VERSION'] + when 'HTTP/1.0', nil + # server will drop connection anyways + else + headers["Transfer-Encoding"] = "chunked" + pipe.chunked = true + end + end + [ status, headers, pipe ] + else + [ 500, { "Content-Length" => "0", "Content-Type" => "text/plain" }, [] ] + end + end +end diff --git a/extras/try_gzip_static.rb b/extras/try_gzip_static.rb new file mode 100644 index 0000000..efe47f9 --- /dev/null +++ b/extras/try_gzip_static.rb @@ -0,0 +1,208 @@ +# Copyright (C) 2013, Eric Wong and all contributors +# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt) +require 'time' +require 'rack/utils' +require 'rack/mime' +require 'kgio' + +class TryGzipStatic + attr_accessor :root + class KF < Kgio::File + # attr_writer :sf_range + + # only used if the server does not handle #to_path, + # yahns should never hit this + def each + raise "we should never get here in yahns" + buf = "" + rsize = 8192 + if @sf_range + file.seek(@sf_range.begin) + sf_count = @sf_range.end - @sf_range.begin + 1 + while sf_count > 0 + read(sf_count > rsize ? rsize : sf_count, buf) or break + sf_count -= buf.size + yield buf + end + raise "file truncated" if sf_count != 0 + else + yield(buf) while read(rsize, buf) + end + end + end + + def initialize(root, default_type = 'text/plain') + @root = root + @default_type = default_type + end + + def fspath(env) + path_info = Rack::Utils.unescape(env["PATH_INFO"]) + path_info =~ /\.\./ ? nil : "#@root#{path_info}" + end + + def get_range(env, path, st) + if ims = env["HTTP_IF_MODIFIED_SINCE"] + return [ 304, {}, [] ] if st.mtime.httpdate == ims + end + + size = st.size + ranges = Rack::Utils.byte_ranges(env, size) + if ranges.nil? || ranges.length > 1 + [ 200 ] # serve the whole thing, possibly with static gzip \o/ + elsif ranges.empty? + res = r(416) + res[1]["Content-Range"] = "bytes */#{size}" + res + else # partial response, no using static gzip file + range = ranges[0] + len = range.end - range.begin + 1 + h = fheader(env, path, st, nil, len) + h["Content-Range"] = "bytes #{range.begin}-#{range.end}/#{size}" + [ 206, h, range ] + end + end + + def fheader(env, path, st, gz_st = nil, len = nil) + if path =~ /(.[^.]+)\z/ + mime = Rack::Mime.mime_type($1, @default_type) + else + mime = @default_type + end + len ||= (gz_st ? gz_st : st).size + h = { + "Content-Type" => mime, + "Content-Length" => len.to_s, + "Last-Modified" => st.mtime.httpdate, + "Accept-Ranges" => "bytes", + } + h["Content-Encoding"] = "gzip" if gz_st + h + end + + def head_no_gz(res, env, path, st) + res[1] = fheader(env, path, st) + res[2] = [] # empty body + res + end + + def stat_path(env) + path = fspath(env) or return r(403) + begin + st = File.stat(path) + st.file? ? [ path, st ] : r(404) + rescue Errno::ENOENT + r(404) + rescue Errno::EACCES + r(403) + rescue => e + r(500, e.message, env) + end + end + + def head(env) + path, st = res = stat_path(env) + return res if Integer === path # integer status code on failure + + # see if it's a range request, no gzipped version if so + status, _ = res = get_range(env, path, st) + case status + when 206 + res[2] = [] # empty body, headers are all set + res + when 200 # fall through to trying gzipped version + # client requested gzipped path explicitly or did not want gzip + if path =~ /\.gz\z/i || !want_gzip?(env) + head_no_gz(res, env, path, st) + else # try the gzipped version + begin + gz_st = File.stat("#{path}.gz") + if gz_st.mtime == st.mtime + res[1] = fheader(env, path, st, gz_st) + res[2] = [] + res + else + head_no_gz(res, env, path, st) + end + rescue Errno::ENOENT, Errno::EACCES + head_no_gz(res, env, path, st) + rescue => e + r(500, e.message, env) + end + end + else # 416, 304 + res + end + end + + def call(env) + case env["REQUEST_METHOD"] + when "GET" then get(env) + when "HEAD" then head(env) + else r(405) + end + end + + def want_gzip?(env) + env["HTTP_ACCEPT_ENCODING"] =~ /\bgzip\b/i + end + + def get(env) + path, st, _ = res = stat_path(env) + return res if Integer === path # integer status code on failure + + # see if it's a range request, no gzipped version if so + status, _, _ = res = get_range(env, path, st) + case status + when 206 + res[2] = KF.open(path) # stat succeeded + when 200 + # client requested gzipped path explicitly or did not want gzip + if path =~ /\.gz\z/i || !want_gzip?(env) + res[1] = fheader(env, path, st) + res[2] = KF.open(path) + else + case gzbody = KF.tryopen("#{path}.gz") + when KF + gz_st = gzbody.stat + if gz_st.file? && gz_st.mtime == st.mtime + # yay! serve the gzipped version as the regular one + # this should be the most likely code path + res[1] = fheader(env, path, st, gz_st) + res[2] = gzbody + else + gzbody.close + res[1] = fheader(env, path, st) + res[2] = KF.open(path) + end + when :ENOENT, :EACCES + res[1] = fheader(env, path, st) + res[2] = KF.open(path) + else + res = r(500, gzbody.to_s, env) + end + end + end + res + rescue Errno::ENOENT # could get here from a race + r(404) + rescue Errno::EACCES # could get here from a race + r(403) + rescue => e + r(500, e.message, env) + end + + def r(code, msg = nil, env = nil) + if env && logger = env["rack.logger"] + logger.warn("#{env['REQUEST_METHOD']} #{env['PATH_INFO']} " \ + "#{code} #{msg.inspect}") + end + + if Rack::Utils::STATUS_WITH_NO_ENTITY_BODY.include?(code) + [ code, {}, [] ] + else + h = { 'Content-Type' => 'text/plain', 'Content-Length' => '0' } + [ code, h, [] ] + end + end +end diff --git a/test/helper.rb b/test/helper.rb index 5e417ae..ae22e7f 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -16,7 +16,7 @@ GTL = Mutex.new # fork-aware coverage data gatherer, see also test/covshow.rb if ENV["COVERAGE"] require "coverage" - COVMATCH = %r{/lib/yahns\b.*rb\z} + COVMATCH = %r{(/lib/yahns\b|extras/).*rb\z} COVDUMPFILE = File.expand_path("coverage.dump") def __covmerge diff --git a/test/test_extras_exec_cgi.rb b/test/test_extras_exec_cgi.rb new file mode 100644 index 0000000..403925b --- /dev/null +++ b/test/test_extras_exec_cgi.rb @@ -0,0 +1,81 @@ +# Copyright (C) 2013, Eric Wong and all contributors +# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt) +require_relative 'server_helper' + +class TestExtrasExecCGI < Testcase + ENV["N"].to_i > 1 and parallelize_me! + include ServerHelper + alias setup server_helper_setup + alias teardown server_helper_teardown + + def test_exec_cgi + err, cfg, host, port = @err, Yahns::Config.new, @srv.addr[3], @srv.addr[1] + runme = "#{Dir.pwd}/test/test_extras_exec_cgi.sh" + assert File.executable?(runme), "run test in project root" + pid = mkserver(cfg) do + require './extras/exec_cgi' + cfg.instance_eval do + app(:rack, ExecCgi.new(runme)) do + listen "#{host}:#{port}" + end + stderr_path err.path + end + end + + Timeout.timeout(30) do # we can chunk + c = get_tcp_client(host, port) + c.write "GET / HTTP/1.1\r\nConnection: close\r\n" \ + "Host: example.com\r\n\r\n" + head, body = c.read.split(/\r\n\r\n/, 2) + assert_match %r{^Transfer-Encoding: chunked\b}, head + assert_equal "5\r\nHIHI\n\r\n0\r\n\r\n", body + c.close + cerr = tmpfile(%w(curl .err)) + assert_equal "HIHI\n", `curl -sSfv 2>#{cerr.path} http://#{host}:#{port}/` + assert_match %r{\bTransfer-Encoding: chunked\b}, cerr.read + cerr.close! + end + + Timeout.timeout(30) do # do not chunk on clients who can't handle chunking + c = get_tcp_client(host, port) + c.write "GET / HTTP/1.0\r\nHost: example.com\r\n\r\n" + head, body = c.read.split(/\r\n\r\n/) + assert_equal "HIHI\n", body + refute_match %r{^Transfer-Encoding: chunked\b}, head + c.close + end + + Timeout.timeout(30) do # sure env is sane + c = get_tcp_client(host, port) + c.write "GET /env\r\n\r\n" + head, body = c.read.split(/\r\n\r\n/) + assert_nil body + assert_match %r{^REQUEST_METHOD=GET$}, head + assert_match %r{^PATH_INFO=/env$}, head + assert_match %r{^QUERY_STRING=$}, head + c.close + end + + Timeout.timeout(30) do # known length should not chunk + c = get_tcp_client(host, port) + c.write "GET /known-length HTTP/1.1\r\nConnection: close\r\n" \ + "Host: example.com\r\n\r\n" + head, body = c.read.split(/\r\n\r\n/, 2) + refute_match %r{^Transfer-Encoding: chunked\b}, head + assert_match %r{^Content-Length: 5\b}, head + assert_equal "HIHI\n", body + c.close + end + + Timeout.timeout(30) do # 404 + c = get_tcp_client(host, port) + c.write "GET /not-found HTTP/1.0\r\n\r\n" + head, body = c.read.split(/\r\n\r\n/) + assert_match %r{\AHTTP/1\.1 404 Not Found}, head + assert_nil body + c.close + end + ensure + quit_wait(pid) + end +end diff --git a/test/test_extras_exec_cgi.sh b/test/test_extras_exec_cgi.sh new file mode 100755 index 0000000..e580773 --- /dev/null +++ b/test/test_extras_exec_cgi.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Copyright (C) 2013, Eric Wong and all contributors +# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt) + +# test CGI program, this remains portable POSIX shell (not bash) +set -e + +stdhead () { + echo Content-Type: text/plain + echo Status: 200 OK + echo +} + +case $PATH_INFO in +/) + stdhead + echo HIHI + ;; +/env) + stdhead + env + ;; +/known-length) + echo Content-Type: text/plain + echo Status: 200 OK + echo Content-Length: 5 + echo + echo HIHI + ;; +*) + echo Content-Type: text/plain + echo Status: 404 Not Found + echo + ;; +esac diff --git a/test/test_extras_try_gzip_static.rb b/test/test_extras_try_gzip_static.rb new file mode 100644 index 0000000..416af71 --- /dev/null +++ b/test/test_extras_try_gzip_static.rb @@ -0,0 +1,177 @@ +# Copyright (C) 2013, Eric Wong and all contributors +# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt) +require_relative 'server_helper' +require 'zlib' +require 'time' + +class TestExtrasTryGzipStatic < Testcase + ENV["N"].to_i > 1 and parallelize_me! + include ServerHelper + GPL_TEXT = IO.binread("COPYING").freeze + + def setup + @tmpdir = Dir.mktmpdir + server_helper_setup + end + + def teardown + server_helper_teardown + FileUtils.rm_rf @tmpdir + end + + def test_gzip_static + err, cfg, host, port = @err, Yahns::Config.new, @srv.addr[3], @srv.addr[1] + tmpdir = @tmpdir + pid = mkserver(cfg) do + require './extras/try_gzip_static' + cfg.instance_eval do + app(:rack, TryGzipStatic.new(tmpdir)) do + listen "#{host}:#{port}" + end + stderr_path err.path + end + end + + begin # setup + gpl = "#{tmpdir}/COPYING" + gplgz = "#{tmpdir}/COPYING.gz" + FileUtils.cp("COPYING", gpl) + _, status = Process.waitpid2(fork do + File.open(gplgz, "w") do |fp| + Zlib::GzipWriter.wrap(fp.dup) { |io| io.write(GPL_TEXT) } + end + exit!(0) + end) + assert status.success?, status.inspect + st = File.stat(gpl) + gz_st = File.stat(gplgz) + assert_equal GPL_TEXT, `zcat #{gplgz}`, "Eric screwed up using zlib" + File.utime(st.atime, st.mtime, gplgz) + end + + check = lambda do |req, &blk| + c = get_tcp_client(host, port) + begin + c.write "#{req}\r\n\r\n" + head, body = c.read.split(/\r\n\r\n/) + blk.call(head) + body + ensure + c.close + end + end + + Timeout.timeout(30) do # basic tests + %w(GET HEAD).each do |m| + body = check.call("#{m} /COPYING HTTP/1.0") do |head| + refute_match %r{^Content-Encoding: gzip\b}, head + assert_match %r{^Content-Type: text/plain\b}, head + assert_match %r{^Content-Length: #{st.size}\b}, head + end + case m + when "GET" then assert_equal GPL_TEXT, body + when "HEAD" then assert_nil body + end + + req = "#{m} /COPYING HTTP/1.0\r\nAccept-Encoding: gzip" + body = check.call(req) do |head| + assert_match %r{^Content-Encoding: gzip\b}, head + assert_match %r{^Content-Type: text/plain\b}, head + assert_match %r{^Content-Length: #{gz_st.size}\b}, head + end + case m + when "GET" + assert_equal GPL_TEXT, Zlib::GzipReader.new(StringIO.new(body)).read + when "HEAD" then assert_nil body + end + end + end + + Timeout.timeout(30) do # range tests + %w(HEAD GET).each do |m| + req = "#{m} /COPYING HTTP/1.0\r\n" \ + "Range: bytes=5-46\r\nAccept-Encoding: gzip" + body = check.call(req) do |head| + assert_match %r{\AHTTP/1\.1 206 Partial Content\r\n}, head + refute_match %r{^Content-Encoding: gzip\b}, head + assert_match %r{^Content-Type: text/plain\b}, head + assert_match %r{^Content-Length: 42\b}, head + assert_match %r{^Content-Range: bytes 5-46/#{st.size}\r\n}, head + end + case m + when "GET" then assert_equal GPL_TEXT[5..46], body + when "HEAD" then assert_nil body + end + + req = "#{m} /COPYING HTTP/1.0\r\n" \ + "Range: bytes=66666666-\r\nAccept-Encoding: gzip" + body = check.call(req) do |head| + assert_match %r{^Content-Range: bytes \*/#{st.size}\r\n}, head + assert_match %r{\AHTTP/1\.1 416 }, head + end + assert_nil body + end + end + + Timeout.timeout(30) do # gzip counterpart is nonexistent + File.link(gpl, "#{gpl}.hardlink") + %w(GET HEAD).each do |m| + req = "#{m} /COPYING.hardlink HTTP/1.0\r\nAccept-Encoding: gzip" + body = check.call(req) do |head| + refute_match %r{^Content-Encoding: gzip\b}, head + assert_match %r{^Content-Type: text/plain\b}, head + assert_match %r{^Content-Length: #{st.size}\b}, head + end + case m + when "GET" then assert_equal GPL_TEXT, body + when "HEAD" then assert_nil body + end + end + end + + Timeout.timeout(30) do # If-Modified-Since + %w(GET HEAD).each do |m| + req = "#{m} /COPYING HTTP/1.0\r\n" \ + "If-Modified-Since: #{st.mtime.httpdate}" + body = check.call(req) do |head| + assert_match %r{\AHTTP/1\.1 304 Not Modified}, head + end + assert_nil body + end + end + + # skew the times of the gzip file, should now fail to use gzipped + Timeout.timeout(30) do + File.utime(Time.at(0), Time.at(0), gplgz) + + %w(GET HEAD).each do |m| + req = "#{m} /COPYING HTTP/1.0\r\nAccept-Encoding: gzip" + body = check.call(req) do |head| + refute_match %r{^Content-Encoding: gzip\b}, head + assert_match %r{^Content-Type: text/plain\b}, head + assert_match %r{^Content-Length: #{st.size}\b}, head + end + case m + when "GET" then assert_equal GPL_TEXT, body + when "HEAD" then assert_nil body + end + end + end + + Timeout.timeout(30) do # 404 + %w(GET HEAD).each do |m| + req = "#{m} /cp-ing HTTP/1.0\r\nAccept-Encoding: gzip" + body = check.call(req) do |head| + assert_match %r{HTTP/1\.1 404 }, head + end + assert_nil body + end + body = check.call("FOO /COPYING HTTP/1.0") do |head| + assert_match %r{HTTP/1\.1 405 }, head + end + assert_nil body + end + ensure + quit_wait(pid) + end +end -- cgit v1.2.3-24-ge0c7