diff options
| -rw-r--r-- | .document | 3 | ||||
| -rw-r--r-- | CHANGELOG | 21 | ||||
| -rw-r--r-- | CONTRIBUTORS | 47 | ||||
| -rw-r--r-- | DESIGN | 3 | ||||
| -rw-r--r-- | Manifest | 4 | ||||
| -rw-r--r-- | README | 76 | ||||
| -rw-r--r-- | SIGNALS | 53 | ||||
| -rwxr-xr-x | bin/unicorn | 27 | ||||
| -rwxr-xr-x | bin/unicorn_rails | 182 | ||||
| -rw-r--r-- | ext/unicorn/http11/http11.c | 33 | ||||
| -rw-r--r-- | lib/unicorn.rb | 120 | ||||
| -rw-r--r-- | lib/unicorn/app/exec_cgi.rb | 150 | ||||
| -rw-r--r-- | lib/unicorn/configurator.rb | 19 | ||||
| -rw-r--r-- | lib/unicorn/const.rb | 2 | ||||
| -rw-r--r-- | lib/unicorn/http_request.rb | 4 | ||||
| -rw-r--r-- | lib/unicorn/http_response.rb | 29 | ||||
| -rw-r--r-- | lib/unicorn/launcher.rb | 33 | ||||
| -rw-r--r-- | test/exec/test_exec.rb | 22 | ||||
| -rw-r--r-- | test/test_helper.rb | 13 | ||||
| -rw-r--r-- | test/unit/test_http_parser.rb | 3 | ||||
| -rw-r--r-- | test/unit/test_request.rb | 82 | ||||
| -rw-r--r-- | test/unit/test_response.rb | 7 | ||||
| -rw-r--r-- | test/unit/test_upload.rb | 28 |
23 files changed, 786 insertions, 175 deletions
@@ -5,7 +5,8 @@ CONTRIBUTORS LICENSE SIGNALS TODO -bin +bin/unicorn +bin/unicorn_rails lib ext/**/*.c ext/**/*.rl @@ -1,19 +1,4 @@ +v0.2.2 - small bug fixes, fix Rack multi-value headers (Set-Cookie:) +v0.2.1 - Fix broken Manifest that cause unicorn_rails to not be bundled +v0.2.0 - unicorn_rails launcher script. v0.1.0 - Unicorn - UNIX-only fork of Mongrel free of threading - -v2.0. (WIP) Rack support. - -v1.1.4. Fix camping handler. Correct treatment of @throttle parameter. - -v1.1.3. Fix security flaw of DirHandler; reported on mailing list. - -v1.1.2. Fix worker termination bug; fix JRuby 1.0.3 load order issue; fix require issue on systems without Rubygems. - -v1.1.1. Fix mongrel_rails restart bug; fix bug with Rack status codes. - -v1.1. Pure Ruby URIClassifier. More modular architecture. JRuby support. Move C URIClassifier into mongrel_experimental project. - -v1.0.4. Backport fixes for versioning inconsistency, mongrel_rails bug, and DirHandler bug. - -v1.0.3. Fix user-switching bug; make people upgrade to the latest from the RC. - -v1.0.2. Signed gem; many minor bugfixes and patches. diff --git a/CONTRIBUTORS b/CONTRIBUTORS index ac47dfb..5a6fa4d 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -1,20 +1,29 @@ -Unicorn would not be possible without Zed and all the contributors to Mongrel. +Unicorn developers: +* Eric Wong +* ... (help wanted) -Eric Wong -Ezra Zygmuntowicz -Zed A. Shaw -Luis Lavena -Wilson Bilkovich -Why the Lucky Stiff -Dan Kubb -MenTaLguY -Filipe Lautert -Rick Olson -Wayne E. Seguin -Kirk Haines -Bradley Taylor -Matt Pelletier -Ry Dahl -Nick Sieger -Evan Weaver -Marc-André Cournoyer +We would like to thank following folks for helping make Unicorn possible: + +* Ezra Zygmuntowicz - for helping Eric decide on a sane configuration + format and reasonable defaults. +* Christian Neukirchen - for Rack, which let us put more focus on the server + and drastically cut down on the amount of code we have to maintain. +* Zed A. Shaw - for Mongrel, without which Unicorn would not be possible + +The original Mongrel contributors: + +* Luis Lavena +* Wilson Bilkovich +* Why the Lucky Stiff +* Dan Kubb +* MenTaLguY +* Filipe Lautert +* Rick Olson +* Wayne E. Seguin +* Kirk Haines +* Bradley Taylor +* Matt Pelletier +* Ry Dahl +* Nick Sieger +* Evan Weaver +* Marc-André Cournoyer @@ -32,7 +32,8 @@ Rack application itself is called only within the worker process (but can be loaded within the master). A copy-on-write friendly garbage collector like Ruby Enterprise Edition can be used to minimize memory - usage along with the "preload_app true" directive. + usage along with the "preload_app true" directive (see + Unicorn::Configurator). * The number of worker processes should be scaled to the number of CPUs, memory or even spindles you have. If you have an existing @@ -11,6 +11,7 @@ Rakefile SIGNALS TODO bin/unicorn +bin/unicorn_rails ext/unicorn/http11/ext_help.h ext/unicorn/http11/extconf.rb ext/unicorn/http11/http11.c @@ -19,10 +20,12 @@ ext/unicorn/http11/http11_parser.h ext/unicorn/http11/http11_parser.rl ext/unicorn/http11/http11_parser_common.rl lib/unicorn.rb +lib/unicorn/app/exec_cgi.rb lib/unicorn/configurator.rb lib/unicorn/const.rb lib/unicorn/http_request.rb lib/unicorn/http_response.rb +lib/unicorn/launcher.rb lib/unicorn/socket.rb lib/unicorn/util.rb setup.rb @@ -36,6 +39,7 @@ test/test_helper.rb test/tools/trickletest.rb test/unit/test_configurator.rb test/unit/test_http_parser.rb +test/unit/test_request.rb test/unit/test_response.rb test/unit/test_server.rb test/unit/test_upload.rb @@ -35,7 +35,7 @@ proxy we know of that meets this requirement. == License Unicorn is copyright 2009 Eric Wong and contributors. -It is based on Mongrel: +It is based on Mongrel and carries the same license: Mongrel is copyright 2007 Zed A. Shaw and contributors. It is licensed under the Ruby license and the GPL2. See the include LICENSE file for @@ -46,28 +46,86 @@ details. The library consists of a C extension so you'll need a C compiler or at least a friend who can build it for you. -Finally, the source includes a setup.rb for those who hate RubyGems. +You may download the tarball from the Mongrel project page on Rubyforge +and run setup.rb after unpacking it: -You can get the source via git via the following locations: +http://rubyforge.org/frs/?group_id=1306 - git://git.bogomips.org/unicorn.git +You may also install it via Rubygems on Rubyforge: + + gem install unicorn + +You can get the latest source via git from the following locations +(these versions may not be stable): + git://git.bogomips.org/unicorn.git http://git.bogomips.org/unicorn.git + git://repo.or.cz/unicorn.git (mirror) + http://repo.or.cz/r/unicorn.git (mirror) + +If you have web browser software for the World Wide Web +(on the Information Superhighway), you may browse the code from +your web browser and download the latest snapshot tarballs here: + +* http://git.bogomips.org/cgit/unicorn.git (this server runs Unicorn!) +* http://repo.or.cz/w/unicorn.git (gitweb mirror) == Usage +=== non-Rails Rack applications + Unicorn will look for the config.ru file used by rackup in APP_ROOT. -Optionally, it can use a config file specified by the --config-file/-c -command-line switch. +Optionally, it can use a config file for unicorn-specific options +specified by the --config-file/-c command-line switch. See +Unicorn::Configurator for the syntax of the unicorn-specific +config options. + +In APP_ROOT, just run: -Unicorn should be capable of running all Rack applications. Since this + unicorn + +Unicorn should be capable of running most Rack applications. Since this is a preforking webserver, you do not have to worry about thread-safety of your application or libraries. However, your Rack application may use threads internally (and should even be able to continue running threads after the request is complete). -== Contact +=== Rack-enabled versions of Rails (v2.3.2+) -Newsgroup and mailing list coming, or it'll be a part of the Mongrel project... +In RAILS_ROOT, run: + + unicorn_rails + +Most command-line options for other Rack applications (above) are also +supported. The unicorn_rails launcher attempts to combine the best +features of the Rails-bundled "script/server" with the "rackup"-like +functionality of the `unicorn' launcher. + +== Disclaimer + +There are only a few instances of Unicorn deployed anywhere in the +world. The only public site known to run Unicorn at this time is +http://git.bogomips.org/cgit which runs Unicorn::App::ExecCgi to +fork()+exec() cgit. + +Be one of the first brave guinea pigs to run it on your production site! +Of course there is NO WARRANTY whatsoever if anything goes wrong, but +let us know and we'll try our best to fix it. Unicorn is still in the +early stages and testing + feedback would be *greatly* appreciated; +maybe you'll get Rainbows as a reward! + +== Known Issues + +* WONTFIX: code reloading with Sinatra 0.3.2 (and likely older + versions) apps is broken. The workaround is to force production + mode to disable code reloading in your Sinatra application: + set :env, :production + Since this is no longer an issue with Sinatra 0.9.x apps and only + affected non-production instances, this will not be fixed on our end. + Also remember we're capable of replacing the running binary without + dropping any connections regardless of framework :) + +== Contact +Newsgroup and mailing list maybe coming... Email Eric Wong at normalperson@yhbt.net for now. @@ -7,6 +7,8 @@ processes are documented here as well. === Master Process * HUP - reload config file and gracefully restart all workers + If preload_app is false (the default), the application code + will be reloaded when workers are restarted as well. * INT/TERM - quick shutdown, kills all workers immediately @@ -20,6 +22,9 @@ processes are documented here as well. should be sent to the original process once the child is verified to be up and running. + * WINCH - gracefully stops workers but keep the master running. + This will only work for daemonized processes. + === Worker Processes Sending signals directly to the worker processes should not normally be @@ -32,3 +37,51 @@ automatically respawned. * USR1 - reopen all logs owned by the worker process See Unicorn::Util.reopen_logs for what is considered a log. + +=== Procedure to replace a running unicorn executable + +You may replace a running instance of unicorn with a new one without +losing any incoming connections. Doing so will reload all of your +application code, Unicorn config, Ruby executable, and all libraries. +The only things that will not change (due to OS limitations) are: + +1. The listener backlog size of already-bound sockets + +2. The path to the unicorn executable script. If you want to change to + a different installation of Ruby, you can modify the shebang + line to point to your alternative interpreter. + +The procedure is exactly like that of nginx: + +1. Send USR2 to the master process + +2. Check your process manager or pid files to see if a new master spawned + successfully. If you're using a pid file, the old process will have + ".oldbin" appended to its path. You should have two master instances + of unicorn running now, both of which will have workers servicing + requests. Your process tree should look something like this: + + unicorn master (old) + \_ unicorn worker[0] + \_ unicorn worker[1] + \_ unicorn worker[2] + \_ unicorn worker[3] + \_ unicorn master + \_ unicorn worker[0] + \_ unicorn worker[1] + \_ unicorn worker[2] + \_ unicorn worker[3] + +4. You can now send WINCH to the old master process so only the new workers + serve requests. If your unicorn process is bound to an interactive + terminal, you can skip this step. Step 5 will be more difficult but + you can also skip it if your process is not daemonized. + +5. You should now ensure that everything is running correctly with the + new workers as the old workers die off. + +6a. If everything seems ok, then send QUIT to the old master. You're done! + +6b. If something is broken, then send HUP to the old master to reload + the config and restart its workers. Then send QUIT to the new master + process. diff --git a/bin/unicorn b/bin/unicorn index ebf57c3..9deb872 100755 --- a/bin/unicorn +++ b/bin/unicorn @@ -1,6 +1,5 @@ #!/home/ew/bin/ruby -$stdin.sync = $stdout.sync = $stderr.sync = true -require 'unicorn' # require this first to populate Unicorn::DEFAULT_START_CTX +require 'unicorn/launcher' require 'optparse' env = "development" @@ -163,27 +162,5 @@ if $DEBUG }) end -# only daemonize if we're not inheriting file descriptors from our parent -if daemonize - - $stdin.reopen("/dev/null") - unless ENV['UNICORN_FD'] - exit if fork - Process.setsid - exit if fork - end - - # We don't do a lot of standard daemonization stuff: - # * $stderr/$stderr can/will be redirected separately - # * umask is whatever was set by the parent process at startup - # and can be set in config.ru and config_file, so making it - # 0000 and potentially exposing sensitive log data can be bad - # policy. - # * Don't bother to chdir here since Unicorn is designed to - # run inside APP_ROOT. Unicorn will also re-chdir() to - # the directory it was started in when being re-executed - # to pickup code changes if the original deployment directory - # is a symlink or otherwise got replaced. -end - +Unicorn::Launcher.daemonize! if daemonize Unicorn.run(app, options) diff --git a/bin/unicorn_rails b/bin/unicorn_rails new file mode 100755 index 0000000..177c109 --- /dev/null +++ b/bin/unicorn_rails @@ -0,0 +1,182 @@ +#!/home/ew/bin/ruby +require 'unicorn/launcher' +require 'optparse' +require 'fileutils' + +rails_pid = File.join(Unicorn::HttpServer::DEFAULT_START_CTX[:cwd], + "/tmp/pids/unicorn.pid") +cmd = File.basename($0) +daemonize = false +listeners = [] +options = { :listeners => listeners } +host, port = Unicorn::Const::DEFAULT_HOST, 3000 +ENV['RAILS_ENV'] ||= "development" +map_path = ENV['RAILS_RELATIVE_URL_ROOT'] + +opts = OptionParser.new("", 24, ' ') do |opts| + opts.banner = "Usage: #{cmd} " \ + "[ruby options] [#{cmd} options] [rackup config file]" + opts.separator "Ruby options:" + + lineno = 1 + opts.on("-e", "--eval LINE", "evaluate a LINE of code") do |line| + eval line, TOPLEVEL_BINDING, "-e", lineno + lineno += 1 + end + + opts.on("-d", "--debug", "set debugging flags (set $DEBUG to true)") do + $DEBUG = true + end + + opts.on("-w", "--warn", "turn warnings on for your script") do + $-w = true + end + + opts.on("-I", "--include PATH", + "specify $LOAD_PATH (may be used more than once)") do |path| + $LOAD_PATH.unshift(*path.split(/:/)) + end + + opts.on("-r", "--require LIBRARY", + "require the library, before executing your script") do |library| + require library + end + + opts.separator "#{cmd} options:" + + # some of these switches exist for rackup command-line compatibility, + + opts.on("-o", "--host HOST", + "listen on HOST (default: #{Unicorn::Const::DEFAULT_HOST})") do |h| + host = h + end + + opts.on("-p", "--port PORT", "use PORT (default: #{port})") do |p| + port = p.to_i + end + + opts.on("-E", "--env ENVIRONMENT", + "use ENVIRONMENT for defaults (default: development)") do |e| + ENV['RAILS_ENV'] = e + end + + opts.on("-D", "--daemonize", "run daemonized in the background") do |d| + daemonize = d ? true : false + end + + # Unicorn-specific stuff + opts.on("-l", "--listen {HOST:PORT|PATH}", + "listen on HOST:PORT or PATH", + "this may be specified multiple times", + "(default: #{Unicorn::Const::DEFAULT_LISTEN})") do |address| + listeners << address + end + + opts.on("-c", "--config-file FILE", "Unicorn-specific config file") do |f| + options[:config_file] = File.expand_path(f) + end + + opts.on("-P", "--path PATH", "Runs Rails app mounted at a specific path.", + "(default: /") do |v| + map_path = v + end + + # I'm avoiding Unicorn-specific config options on the command-line. + # IMNSHO, config options on the command-line are redundant given + # config files and make things unnecessarily complicated with multiple + # places to look for a config option. + + opts.separator "Common options:" + + opts.on_tail("-h", "--help", "Show this message") do + puts opts + exit + end + + opts.on_tail("-v", "--version", "Show version") do + puts " v#{Unicorn::Const::UNICORN_VERSION}" + exit + end + + opts.parse! ARGV +end + +require 'pp' if $DEBUG + +# Loads Rails and the private version of Rack it bundles. Returns a +# lambda of arity==0 that will return *another* lambda of arity==1 +# suitable for using inside Rack::Builder.new block. +rails_loader = lambda do || + begin + require 'config/boot' + defined?(::RAILS_ROOT) or abort "RAILS_ROOT not defined by config/boot" + defined?(::RAILS_ENV) or abort "RAILS_ENV not defined by config/boot" + defined?(::Rails::VERSION::STRING) or + abort "Rails::VERSION::STRING not defined by config/boot" + rescue LoadError + abort "#$0 must be run inside RAILS_ROOT (#{::RAILS_ROOT})" + end + + # return the lambda + config = ::ARGV[0] || (File.exist?('config.ru') ? 'config.ru' : nil) + case config + when nil + lambda do || + require 'config/environment' + ActionController::Dispatcher.new + end + when /\.ru$/ + raw = File.open(config, "rb") { |fp| fp.sysread(fp.stat.size) } + # parse embedded command-line options in config.ru comments + if raw[/^#\\(.*)/] + opts.parse! $1.split(/\s+/) + require 'pp' if $DEBUG + end + lambda { || eval("Rack::Builder.new {(#{raw}\n)}.to_app", nil, config) } + else + lambda do || + require config + Object.const_get(File.basename(config, '.rb').capitalize) + end + end +end + +# this won't run until after forking if preload_app is false +app = lambda do || + inner_app = rails_loader.call + require 'active_support' + require 'action_controller' + ActionController::Base.relative_url_root = map_path if map_path + Rack::Builder.new do + use Rails::Rack::LogTailer unless daemonize + use Rails::Rack::Debugger if $DEBUG + map(map_path || '/') do + use Rails::Rack::Static + run inner_app.call + end + end.to_app +end + +if listeners.empty? + listener = "#{host}:#{port}" + listeners << listener +end + +if $DEBUG + pp({ + :unicorn_options => options, + :app => app, + :daemonize => daemonize, + }) +end + +# ensure Rails standard tmp paths exist +%w(cache pids sessions sockets).each do |dir| + FileUtils.mkdir_p("tmp/#{dir}") +end + +if daemonize + options[:pid] = rails_pid + Unicorn::Launcher.daemonize! +end +Unicorn.run(app, options) diff --git a/ext/unicorn/http11/http11.c b/ext/unicorn/http11/http11.c index d5c364a..0b96099 100644 --- a/ext/unicorn/http11/http11.c +++ b/ext/unicorn/http11/http11.c @@ -28,13 +28,9 @@ static VALUE global_fragment; static VALUE global_query_string; static VALUE global_http_version; static VALUE global_content_length; -static VALUE global_http_content_length; static VALUE global_request_path; static VALUE global_content_type; -static VALUE global_http_content_type; static VALUE global_http_body; -static VALUE global_gateway_interface; -static VALUE global_gateway_interface_value; static VALUE global_server_name; static VALUE global_server_port; static VALUE global_server_protocol; @@ -129,6 +125,7 @@ static int common_field_cmp(const void *a, const void *b) } #endif /* HAVE_QSORT_BSEARCH */ +/* this function is not performance-critical */ static void init_common_fields(void) { int i; @@ -137,8 +134,15 @@ static void init_common_fields(void) memcpy(tmp, HTTP_PREFIX, HTTP_PREFIX_LEN); for(i = 0; i < ARRAY_SIZE(common_http_fields); cf++, i++) { - memcpy(tmp + HTTP_PREFIX_LEN, cf->name, cf->len + 1); - cf->value = rb_obj_freeze(rb_str_new(tmp, HTTP_PREFIX_LEN + cf->len)); + /* Rack doesn't like certain headers prefixed with "HTTP_" */ + if (!strcmp("CONTENT_LENGTH", cf->name) || + !strcmp("CONTENT_TYPE", cf->name)) { + cf->value = rb_str_new(cf->name, cf->len); + } else { + memcpy(tmp + HTTP_PREFIX_LEN, cf->name, cf->len + 1); + cf->value = rb_str_new(tmp, HTTP_PREFIX_LEN + cf->len); + } + cf->value = rb_obj_freeze(cf->value); rb_global_variable(&cf->value); } @@ -275,21 +279,8 @@ static void header_done(void *data, const char *at, size_t length) { VALUE req = (VALUE)data; VALUE temp = Qnil; - VALUE ctype = Qnil; - VALUE clen = Qnil; char *colon = NULL; - clen = rb_hash_aref(req, global_http_content_length); - if(clen != Qnil) { - rb_hash_aset(req, global_content_length, clen); - } - - ctype = rb_hash_aref(req, global_http_content_type); - if(ctype != Qnil) { - rb_hash_aset(req, global_content_type, ctype); - } - - rb_hash_aset(req, global_gateway_interface, global_gateway_interface_value); if((temp = rb_hash_aref(req, global_http_host)) != Qnil) { colon = memchr(RSTRING_PTR(temp), ':', RSTRING_LEN(temp)); if(colon != NULL) { @@ -497,12 +488,8 @@ void Init_http11() DEF_GLOBAL(http_version, "HTTP_VERSION"); DEF_GLOBAL(request_path, "REQUEST_PATH"); DEF_GLOBAL(content_length, "CONTENT_LENGTH"); - DEF_GLOBAL(http_content_length, "HTTP_CONTENT_LENGTH"); DEF_GLOBAL(http_body, "HTTP_BODY"); DEF_GLOBAL(content_type, "CONTENT_TYPE"); - DEF_GLOBAL(http_content_type, "HTTP_CONTENT_TYPE"); - DEF_GLOBAL(gateway_interface, "GATEWAY_INTERFACE"); - DEF_GLOBAL(gateway_interface_value, "CGI/1.2"); DEF_GLOBAL(server_name, "SERVER_NAME"); DEF_GLOBAL(server_port, "SERVER_PORT"); DEF_GLOBAL(server_protocol, "SERVER_PROTOCOL"); diff --git a/lib/unicorn.rb b/lib/unicorn.rb index d442f63..2f86de2 100644 --- a/lib/unicorn.rb +++ b/lib/unicorn.rb @@ -23,7 +23,6 @@ module Unicorn # forked worker children. class HttpServer attr_reader :logger - include Process include ::Unicorn::SocketHelper DEFAULT_START_CTX = { @@ -53,7 +52,7 @@ module Unicorn @start_ctx = DEFAULT_START_CTX.dup @start_ctx.merge!(start_ctx) if start_ctx @app = app - @mode = :idle + @sig_queue = [] @master_pid = $$ @workers = Hash.new @io_purgatory = [] # prevents IO objects in here from being GC-ed @@ -160,33 +159,45 @@ module Unicorn # are trapped. See trap_deferred @rd_sig, @wr_sig = IO.pipe unless (@rd_sig && @wr_sig) @rd_sig.nonblock = @wr_sig.nonblock = true + mode = nil + respawn = true - reset_master + QUEUE_SIGS.each { |sig| trap_deferred(sig) } + trap('CHLD') { |sig_nr| awaken_master } $0 = "unicorn master" - logger.info "master process ready" # test relies on this message + logger.info "master process ready" # test_exec.rb relies on this message begin loop do reap_all_workers - case @mode - when :idle + case (mode = @sig_queue.shift) + when nil murder_lazy_workers - spawn_missing_workers + spawn_missing_workers if respawn + master_sleep when 'QUIT' # graceful shutdown break when 'TERM', 'INT' # immediate shutdown stop(false) break - when 'USR1' # user-defined (probably something like log reopening) - kill_each_worker('USR1') + when 'USR1' # rotate logs + logger.info "master rotating logs..." Unicorn::Util.reopen_logs - reset_master + logger.info "master done rotating logs" + kill_each_worker('USR1') when 'USR2' # exec binary, stay alive in case something went wrong reexec - reset_master + when 'WINCH' + if Process.ppid == 1 || Process.getpgrp != $$ + respawn = false + logger.info "gracefully stopping all workers" + kill_each_worker('QUIT') + else + logger.info "SIGWINCH ignored because we're not daemonized" + end when 'HUP' + respawn = true if @config.config_file load_config! - reset_master redo # immediate reaping since we may have QUIT workers else # exec binary and exit if there's no config file logger.info "config_file not present, reexecuting binary" @@ -194,19 +205,7 @@ module Unicorn break end else - logger.error "master process in unknown mode: #{@mode}, resetting" - reset_master - end - reap_all_workers - - ready = begin - IO.select([@rd_sig], nil, nil, 1) or next - rescue Errno::EINTR # next - end - ready[0] && ready[0][0] or next - begin # just consume the pipe when we're awakened, @mode is set - loop { @rd_sig.sysread(Const::CHUNK_SIZE) } - rescue Errno::EAGAIN, Errno::EINTR # next + logger.error "master process in unknown mode: #{mode}" end end rescue Errno::EINTR @@ -214,7 +213,6 @@ module Unicorn rescue Object => e logger.error "Unhandled master loop exception #{e.inspect}." logger.error e.backtrace.join("\n") - reset_master retry end stop # gracefully shutdown all workers on our way out @@ -241,48 +239,57 @@ module Unicorn private # list of signals we care about and trap in master. - TRAP_SIGS = %w(QUIT INT TERM USR1 USR2 HUP).map { |x| x.freeze }.freeze + QUEUE_SIGS = + %w(WINCH QUIT INT TERM USR1 USR2 HUP).map { |x| x.freeze }.freeze # defer a signal for later processing in #join (master process) def trap_deferred(signal) trap(signal) do |sig_nr| - # we only handle/defer one signal at a time and ignore all others - # until we're ready again. Queueing signals can lead to more bugs, - # and simplicity is the most important thing - TRAP_SIGS.each { |sig| trap(sig, 'IGNORE') } - if Symbol === @mode - @mode = signal - begin - @wr_sig.syswrite('.') # wakeup master process from IO.select - rescue Errno::EAGAIN - rescue Errno::EINTR - retry - end + if @sig_queue.size < 5 + @sig_queue << signal + awaken_master + else + logger.error "ignoring SIG#{signal}, queue=#{@sig_queue.inspect}" end end end + # wait for a signal hander to wake us up and then consume the pipe + # Wake up every second anyways to run murder_lazy_workers + def master_sleep + begin + ready = IO.select([@rd_sig], nil, nil, 1) + ready && ready[0] && ready[0][0] or return + loop { @rd_sig.sysread(Const::CHUNK_SIZE) } + rescue Errno::EAGAIN, Errno::EINTR + end + end - def reset_master - @mode = :idle - TRAP_SIGS.each { |sig| trap_deferred(sig) } + def awaken_master + begin + @wr_sig.syswrite('.') # wakeup master process from IO.select + rescue Errno::EAGAIN # pipe is full, master should wake up anyways + rescue Errno::EINTR + retry + end end # reaps all unreaped workers def reap_all_workers begin loop do - pid = waitpid(-1, WNOHANG) or break + pid, status = Process.waitpid2(-1, Process::WNOHANG) + pid or break if @reexec_pid == pid - logger.error "reaped exec()-ed PID:#{pid} status=#{$?.exitstatus}" + logger.error "reaped #{status.inspect} exec()-ed" @reexec_pid = 0 self.pid = @pid.chomp('.oldbin') if @pid + $0 = "unicorn master" else worker = @workers.delete(pid) worker.tempfile.close rescue nil - logger.info "reaped PID:#{pid} " \ - "worker=#{worker.nr rescue 'unknown'} " \ - "status=#{$?.exitstatus}" + logger.info "reaped #{status.inspect} " \ + "worker=#{worker.nr rescue 'unknown'}" end end rescue Errno::ECHILD @@ -330,6 +337,7 @@ module Unicorn @before_exec.call(self) if @before_exec exec(*cmd) end + $0 = "unicorn master (old)" end # forcibly terminate all workers that haven't checked in in @timeout @@ -352,6 +360,13 @@ module Unicorn return if @workers.size == @worker_processes (0...@worker_processes).each do |worker_nr| @workers.values.include?(worker_nr) and next + begin + Dir.chdir(@start_ctx[:cwd]) + rescue Errno::ENOENT => err + logger.fatal "#{err.inspect} (#{@start_ctx[:cwd]})" + @sig_queue << 'QUIT' # forcibly emulate SIGQUIT + return + end tempfile = Tempfile.new('') # as short as possible to save dir space tempfile.unlink # don't allow other processes to find or see it tempfile.sync = true @@ -389,7 +404,8 @@ module Unicorn # by the user. def init_worker_process(worker) build_app! unless @preload_app - TRAP_SIGS.each { |sig| trap(sig, 'IGNORE') } + @sig_queue.clear + QUEUE_SIGS.each { |sig| trap(sig, 'IGNORE') } trap('CHLD', 'DEFAULT') trap('USR1') do @logger.info "worker=#{worker.nr} rotating logs..." @@ -403,7 +419,7 @@ module Unicorn @workers.values.each { |other| other.tempfile.close rescue nil } @workers.clear @start_ctx.clear - @mode = @start_ctx = @workers = @rd_sig = @wr_sig = nil + @start_ctx = @workers = @rd_sig = @wr_sig = nil @listeners.each { |sock| set_cloexec(sock) } ENV.delete('UNICORN_FD') @after_fork.call(self, worker.nr) if @after_fork @@ -426,7 +442,7 @@ module Unicorn @listeners.each { |sock| sock.close rescue nil } # break IO.select end - while alive && @master_pid == ppid + while alive && @master_pid == Process.ppid # we're a goner in @timeout seconds anyways if tempfile.chmod # breaks, so don't trap the exception. Using fchmod() since # futimes() is not available in base Ruby and I very strongly @@ -492,7 +508,7 @@ module Unicorn # is no longer running. def kill_worker(signal, pid) begin - kill(signal, pid) + Process.kill(signal, pid) rescue Errno::ESRCH worker = @workers.delete(pid) and worker.tempfile.close rescue nil end @@ -514,7 +530,7 @@ module Unicorn def valid_pid?(path) if File.exist?(path) && (pid = File.read(path).to_i) > 1 begin - kill(0, pid) + Process.kill(0, pid) return pid rescue Errno::ESRCH end diff --git a/lib/unicorn/app/exec_cgi.rb b/lib/unicorn/app/exec_cgi.rb new file mode 100644 index 0000000..f5e7db9 --- /dev/null +++ b/lib/unicorn/app/exec_cgi.rb @@ -0,0 +1,150 @@ +require 'unicorn' +require 'rack' + +module Unicorn::App + + # This class is highly experimental (even more so than the rest of Unicorn) + # and has never run anything other than cgit. + class ExecCgi + + CHUNK_SIZE = 16384 + PASS_VARS = %w( + CONTENT_LENGTH + CONTENT_TYPE + GATEWAY_INTERFACE + AUTH_TYPE + PATH_INFO + PATH_TRANSLATED + QUERY_STRING + REMOTE_ADDR + REMOTE_HOST + REMOTE_IDENT + REMOTE_USER + REQUEST_METHOD + SERVER_NAME + SERVER_PORT + SERVER_PROTOCOL + SERVER_SOFTWARE + ).map { |x| x.freeze }.freeze # frozen strings are faster for Hash lookups + + # Intializes the app, example of usage in a config.ru + # map "/cgit" do + # run Unicorn::App::ExecCgi.new("/path/to/cgit.cgi") + # end + def initialize(*args) + @args = args.dup + first = @args[0] or + raise ArgumentError, "need path to executable" + first[0..0] == "/" or @args[0] = ::File.expand_path(first) + File.executable?(@args[0]) or + raise ArgumentError, "#{@args[0]} is not executable" + end + + # Calls the app + def call(env) + out, err = Tempfile.new(''), Tempfile.new('') + out.unlink + err.unlink + inp = force_file_input(env) + inp.sync = out.sync = err.sync = true + pid = fork { run_child(inp, out, err, env) } + inp.close + pid, status = Process.waitpid2(pid) + write_errors(env, err, status) if err.stat.size > 0 + err.close + + return parse_output!(out) if status.success? + out.close + [ 500, { 'Content-Length' => '0', 'Content-Type' => 'text/plain' }, [] ] + end + + private + + def run_child(inp, out, err, env) + PASS_VARS.each do |key| + val = env[key] or next + ENV[key] = val + end + ENV['SCRIPT_NAME'] = @args[0] + ENV['GATEWAY_INTERFACE'] = 'CGI/1.1' + env.keys.grep(/^HTTP_/) { |key| ENV[key] = env[key] } + + IO.new(0).reopen(inp) + IO.new(1).reopen(out) + IO.new(2).reopen(err) + exec(*@args) + end + + # Extracts headers from CGI out, will change the offset of out. + # This returns a standard Rack-compatible return value: + # [ 200, HeadersHash, body ] + def parse_output!(out) + size = out.stat.size + out.sysseek(0) + head = out.sysread(CHUNK_SIZE) + offset = 2 + head, body = head.split(/\n\n/, 2) + if body.nil? + head, body = head.split(/\r\n\r\n/, 2) + offset = 4 + end + offset += head.length + out.instance_variable_set('@unicorn_app_exec_cgi_offset', offset) + size -= offset + + # Allows +out+ to be used as a Rack body. + def out.each + sysseek(@unicorn_app_exec_cgi_offset) + begin + loop { yield(sysread(CHUNK_SIZE)) } + rescue EOFError + end + end + + prev = nil + headers = Rack::Utils::HeaderHash.new + head.split(/\r?\n/).each do |line| + case line + when /^([A-Za-z0-9-]+):\s*(.*)$/ then headers[prev = $1] = $2 + when /^[ \t]/ then headers[prev] << "\n#{line}" if prev + end + end + headers['Content-Length'] = size.to_s + [ 200, headers, out ] + end + + # ensures rack.input is a file handle that we can redirect stdin to + def force_file_input(env) + inp = env['rack.input'] + if inp.respond_to?(:fileno) && Integer === inp.fileno + inp + elsif inp.size == 0 # inp could be a StringIO or StringIO-like object + ::File.open('/dev/null') + else + tmp = Tempfile.new('') + tmp.unlink + tmp.binmode + + # Rack::Lint::InputWrapper doesn't allow sysread :( + while buf = inp.read(CHUNK_SIZE) + tmp.syswrite(buf) + end + tmp.sysseek(0) + tmp + end + end + + # rack.errors this may not be an IO object, so we couldn't + # just redirect the CGI executable to that earlier. + def write_errors(env, err, status) + err.seek(0) + dst = env['rack.errors'] + pid = status.pid + dst.write("#{pid}: #{@args.inspect} status=#{status} stderr:\n") + err.each_line { |line| dst.write("#{pid}: #{line}") } + dst.flush + end + + end + +end diff --git a/lib/unicorn/configurator.rb b/lib/unicorn/configurator.rb index dd9ae3b..b4713c5 100644 --- a/lib/unicorn/configurator.rb +++ b/lib/unicorn/configurator.rb @@ -173,13 +173,14 @@ module Unicorn # worker processes. For per-worker listeners, see the after_fork example def listeners(addresses) Array === addresses or addresses = Array(addresses) + addresses.map! { |addr| expand_addr(addr) } @set[:listeners] = addresses end # adds an +address+ to the existing listener set def listen(address) @set[:listeners] = [] unless Array === @set[:listeners] - @set[:listeners] << address + @set[:listeners] << expand_addr(address) end # sets the +path+ for the PID file of the unicorn master process @@ -194,6 +195,10 @@ module Unicorn # properly close/reopen sockets. Files opened for logging do not # have to be reopened as (unbuffered-in-userspace) files opened with # the File::APPEND flag are written to atomically on UNIX. + # + # In addition to reloading the unicorn-specific config settings, + # SIGHUP will reload application code in the working + # directory/symlink when workers are gracefully restarted. def preload_app(bool) case bool when TrueClass, FalseClass @@ -249,5 +254,17 @@ module Unicorn @set[var] = my_proc end + # expands pathnames of sockets if relative to "~" or "~username" + # expands "*:port and ":port" to "0.0.0.0:port" + def expand_addr(address) #:nodoc + return address unless String === address + if address[0..0] == '~' + return File.expand_path(address) + elsif address =~ %r{\A\*?:(\d+)\z} + return "0.0.0.0:#$1" + end + address + end + end end diff --git a/lib/unicorn/const.rb b/lib/unicorn/const.rb index 46398e5..8f9e978 100644 --- a/lib/unicorn/const.rb +++ b/lib/unicorn/const.rb @@ -68,7 +68,7 @@ module Unicorn REQUEST_URI='REQUEST_URI'.freeze REQUEST_PATH='REQUEST_PATH'.freeze - UNICORN_VERSION="0.1.0".freeze + UNICORN_VERSION="0.2.2".freeze UNICORN_TMP_BASE="unicorn".freeze diff --git a/lib/unicorn/http_request.rb b/lib/unicorn/http_request.rb index ce0e408..411c56c 100644 --- a/lib/unicorn/http_request.rb +++ b/lib/unicorn/http_request.rb @@ -130,8 +130,6 @@ module Unicorn raise "No REQUEST PATH" unless @params[Const::REQUEST_PATH] @params["QUERY_STRING"] ||= '' - @params.delete "HTTP_CONTENT_TYPE" - @params.delete "HTTP_CONTENT_LENGTH" @params.update({ "rack.version" => [0,1], "rack.input" => @body, "rack.errors" => $stderr, @@ -155,7 +153,7 @@ module Unicorn end true # success! rescue Object => e - logger.error "Error reading HTTP body: #{e.inspect}" + @logger.error "Error reading HTTP body: #{e.inspect}" socket.closed? or socket.close rescue nil # Any errors means we should delete the file, including if the file diff --git a/lib/unicorn/http_response.rb b/lib/unicorn/http_response.rb index 7bbb940..c8aa3f9 100644 --- a/lib/unicorn/http_response.rb +++ b/lib/unicorn/http_response.rb @@ -21,35 +21,32 @@ module Unicorn class HttpResponse - # headers we allow duplicates for - ALLOWED_DUPLICATES = { - 'Set-Cookie' => true, - 'Set-Cookie2' => true, - 'Warning' => true, - 'WWW-Authenticate' => true, - }.freeze + # Rack does not set/require a Date: header. We always override the + # Connection: and Date: headers no matter what (if anything) our + # Rack application sent us. + SKIP = { 'connection' => true, 'date' => true }.freeze # writes the rack_response to socket as an HTTP response def self.write(socket, rack_response) status, headers, body = rack_response + out = [ "Date: #{Time.now.httpdate}" ] - # Rack does not set/require Date, but don't worry about Content-Length - # since Rack applications that conform to Rack::Lint enforce that - out = [ "#{Const::DATE}: #{Time.now.httpdate}" ] - sent = { Const::CONNECTION => true, Const::DATE => true } - + # Don't bother enforcing duplicate supression, it's a Hash most of + # the time anyways so just hope our app knows what it's doing headers.each do |key, value| - if ! sent[key] || ALLOWED_DUPLICATES[key] - sent[key] = true - out << "#{key}: #{value}" - end + next if SKIP.include?(key.downcase) + value.split(/\n/).each { |v| out << "#{key}: #{v}" } end + # Rack should enforce Content-Length or chunked transfer encoding, + # so don't worry or care about them. socket_write(socket, "HTTP/1.1 #{status} #{HTTP_STATUS_CODES[status]}\r\n" \ "Connection: close\r\n" \ "#{out.join("\r\n")}\r\n\r\n") body.each { |chunk| socket_write(socket, chunk) } + ensure + body.respond_to?(:close) and body.close rescue nil end private diff --git a/lib/unicorn/launcher.rb b/lib/unicorn/launcher.rb new file mode 100644 index 0000000..8c96059 --- /dev/null +++ b/lib/unicorn/launcher.rb @@ -0,0 +1,33 @@ +$stdin.sync = $stdout.sync = $stderr.sync = true +require 'unicorn' + +class Unicorn::Launcher + + # We don't do a lot of standard daemonization stuff: + # * umask is whatever was set by the parent process at startup + # and can be set in config.ru and config_file, so making it + # 0000 and potentially exposing sensitive log data can be bad + # policy. + # * don't bother to chdir("/") here since unicorn is designed to + # run inside APP_ROOT. Unicorn will also re-chdir() to + # the directory it was started in when being re-executed + # to pickup code changes if the original deployment directory + # is a symlink or otherwise got replaced. + def self.daemonize! + $stdin.reopen("/dev/null") + + # We only start a new process group if we're not being reexecuted + # and inheriting file descriptors from our parent + unless ENV['UNICORN_FD'] + exit if fork + Process.setsid + exit if fork + + # $stderr/$stderr can/will be redirected separately in the Unicorn config + $stdout.reopen("/dev/null", "a") + $stderr.reopen("/dev/null", "a") + end + $stdin.sync = $stdout.sync = $stderr.sync = true + end + +end diff --git a/test/exec/test_exec.rb b/test/exec/test_exec.rb index 712037c..ea9fc7c 100644 --- a/test/exec/test_exec.rb +++ b/test/exec/test_exec.rb @@ -283,6 +283,7 @@ end results = retry_hit(["http://#{@addr}:#{@port}/"]) assert_equal String, results[0].class wait_master_ready(COMMON_TMP.path) + wait_workers_ready(COMMON_TMP.path, 4) bf = File.readlines(COMMON_TMP.path).grep(/\bbefore_fork: worker=/) assert_equal 4, bf.size rotate = Tempfile.new('unicorn_rotate') @@ -299,7 +300,7 @@ end sleep DEFAULT_RES log = File.readlines(rotate.path) end - assert_equal 4, log.grep(/rotating logs\.\.\./).size + assert_equal 4, log.grep(/worker=\d+ rotating logs\.\.\./).size assert_equal 0, log.grep(/done rotating logs/).size tries = DEFAULT_TRIES @@ -308,7 +309,7 @@ end sleep DEFAULT_RES log = File.readlines(COMMON_TMP.path) end - assert_equal 4, log.grep(/done rotating logs/).size + assert_equal 4, log.grep(/worker=\d+ done rotating logs/).size assert_equal 0, log.grep(/rotating logs\.\.\./).size assert_nothing_raised { Process.kill('QUIT', pid) } status = nil @@ -496,6 +497,21 @@ end assert status.success?, "exited successfully" end + def wait_workers_ready(path, nr_workers) + tries = DEFAULT_TRIES + lines = [] + while (tries -= 1) > 0 + begin + lines = File.readlines(path).grep(/worker=\d+ spawned/) + lines.size == nr_workers and return + rescue Errno::ENOENT + end + sleep DEFAULT_RES + end + raise "#{nr_workers} workers never became ready:" \ + "\n\t#{lines.join("\n\t")}\n" + end + def wait_master_ready(master_log) tries = DEFAULT_TRIES while (tries -= 1) > 0 @@ -555,7 +571,7 @@ end while (tries -= 1) > 0 && ! File.exist?(path) sleep DEFAULT_RES end - assert File.exist?(path), "path=#{path} exists" + assert File.exist?(path), "path=#{path} exists #{caller.inspect}" end def xfork(&block) diff --git a/test/test_helper.rb b/test/test_helper.rb index f809af3..4243606 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -41,7 +41,18 @@ def redirect_test_io STDOUT.reopen(orig_out) end end - + +# which(1) exit codes cannot be trusted on some systems +# We use UNIX shell utilities in some tests because we don't trust +# ourselves to write Ruby 100% correctly :) +def which(bin) + ex = ENV['PATH'].split(/:/).detect do |x| + x << "/#{bin}" + File.executable?(x) + end or warn "`#{bin}' not found in PATH=#{ENV['PATH']}" + ex +end + # Either takes a string to do a get request against, or a tuple of [URI, HTTP] where # HTTP is some kind of Net::HTTP request object (POST, HEAD, etc.) def hit(uris) diff --git a/test/unit/test_http_parser.rb b/test/unit/test_http_parser.rb index ca1cd01..fc75990 100644 --- a/test/unit/test_http_parser.rb +++ b/test/unit/test_http_parser.rb @@ -25,11 +25,10 @@ class HttpParserTest < Test::Unit::TestCase assert_equal '/', req['REQUEST_PATH'] assert_equal 'HTTP/1.1', req['HTTP_VERSION'] assert_equal '/', req['REQUEST_URI'] - assert_equal 'CGI/1.2', req['GATEWAY_INTERFACE'] assert_equal 'GET', req['REQUEST_METHOD'] assert_nil req['FRAGMENT'] assert_nil req['QUERY_STRING'] - + parser.reset assert parser.nread == 0, "Number read after reset should be 0" end diff --git a/test/unit/test_request.rb b/test/unit/test_request.rb new file mode 100644 index 0000000..37fbb14 --- /dev/null +++ b/test/unit/test_request.rb @@ -0,0 +1,82 @@ +# Copyright (c) 2009 Eric Wong +# You can redistribute it and/or modify it under the same terms as Ruby. + +if RUBY_VERSION =~ /1\.9/ + warn "#$0 current broken under Ruby 1.9 with Rack" + exit 0 +end + +require 'test/test_helper' +begin + require 'rack' + require 'rack/lint' +rescue LoadError + warn "Unable to load rack, skipping test" + exit 0 +end + +include Unicorn + +class RequestTest < Test::Unit::TestCase + + class MockRequest < StringIO + def unicorn_peeraddr + '666.666.666.666' + end + end + + def setup + @request = HttpRequest.new(Logger.new($stderr)) + @app = lambda do |env| + [ 200, { 'Content-Length' => '0', 'Content-Type' => 'text/plain' }, [] ] + end + @lint = Rack::Lint.new(@app) + end + + def test_rack_lint_get + client = MockRequest.new("GET / HTTP/1.1\r\nHost: foo\r\n\r\n") + res = env = nil + assert_nothing_raised { env = @request.read(client) } + assert_equal '666.666.666.666', env['REMOTE_ADDR'] + assert_nothing_raised { res = @lint.call(env) } + end + + def test_rack_lint_put + client = MockRequest.new( + "PUT / HTTP/1.1\r\n" \ + "Host: foo\r\n" \ + "Content-Length: 5\r\n" \ + "\r\n" \ + "abcde") + res = env = nil + assert_nothing_raised { env = @request.read(client) } + assert_nothing_raised { res = @lint.call(env) } + end + + def test_rack_lint_big_put + count = 100 + bs = 0x10000 + buf = (' ' * bs).freeze + length = bs * count + client = Tempfile.new('big_put') + def client.unicorn_peeraddr + '1.1.1.1' + end + client.syswrite( + "PUT / HTTP/1.1\r\n" \ + "Host: foo\r\n" \ + "Content-Length: #{length}\r\n" \ + "\r\n") + count.times { assert_equal bs, client.syswrite(buf) } + assert_equal 0, client.sysseek(0) + res = env = nil + assert_nothing_raised { env = @request.read(client) } + assert_equal length, env['rack.input'].size + count.times { assert_equal buf, env['rack.input'].read(bs) } + assert_nil env['rack.input'].read(bs) + assert_nothing_raised { env['rack.input'].rewind } + assert_nothing_raised { res = @lint.call(env) } + end + +end + diff --git a/test/unit/test_response.rb b/test/unit/test_response.rb index c30a141..4c7423b 100644 --- a/test/unit/test_response.rb +++ b/test/unit/test_response.rb @@ -41,5 +41,12 @@ class ResponseTest < Test::Unit::TestCase io.rewind assert_match(/.* #{HTTP_STATUS_CODES[code]}$/, io.readline.chomp, "wrong default reason phrase") end + + def test_rack_multivalue_headers + out = StringIO.new + HttpResponse.write(out,[200, {"X-Whatever" => "stuff\nbleh"}, []]) + assert_match(/^X-Whatever: stuff\r\nX-Whatever: bleh\r\n/, out.string) + end + end diff --git a/test/unit/test_upload.rb b/test/unit/test_upload.rb index edc94da..41fc473 100644 --- a/test/unit/test_upload.rb +++ b/test/unit/test_upload.rb @@ -135,6 +135,34 @@ class UploadTest < Test::Unit::TestCase assert_equal resp[:size], new_tmp.stat.size end + # Despite reading numerous articles and inspecting the 1.9.1-p0 C + # source, Eric Wong will never trust that we're always handling + # encoding-aware IO objects correctly. Thus this test uses shell + # utilities that should always operate on files/sockets on a + # byte-level. + def test_uncomfortable_with_onenine_encodings + # POSIX doesn't require all of these to be present on a system + which('curl') or return + which('sha1sum') or return + which('dd') or return + + start_server(@sha1_app) + + tmp = Tempfile.new('dd_dest') + assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}", + "bs=#{@bs}", "count=#{@count}"), + "dd #@random to #{tmp}") + sha1_re = %r!\b([a-f0-9]{40})\b! + sha1_out = `sha1sum #{tmp.path}` + assert $?.success?, 'sha1sum ran OK' + + assert_match(sha1_re, sha1_out) + sha1 = sha1_re.match(sha1_out)[1] + resp = `curl -isSfN -T#{tmp.path} http://#@addr:#@port/` + assert $?.success?, 'curl ran OK' + assert_match(%r!\b#{sha1}\b!, resp) + end + private def length |