From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, WEIRD_PORT shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id B33E01F45F for ; Fri, 10 May 2019 02:44:42 +0000 (UTC) From: Eric Wong To: yahns-public@yhbt.net Subject: [PATCH] proxy_pass: document as a public API Date: Fri, 10 May 2019 02:44:42 +0000 Message-Id: <20190510024442.13224-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Might as well... this has been in use at YHBT.net for ~4 years at this point. And given nginx has new corporate overlords, maybe a decidedly non-enterprisey alternative is worth "marketing" :P Previous discussion from 2016: https://YHBT.net/yahns-public/20160220081619.GA10850@dcvr.yhbt.net/ --- .document | 2 + .olddoc.yml | 8 +++ Documentation/yahns_config.pod | 4 +- Rakefile | 20 +++++++- examples/https_proxy_pass.conf.rb | 36 ++++++++++++++ examples/proxy_pass.ru | 11 +++++ extras/proxy_pass.rb | 9 ++-- lib/yahns.rb | 17 ++++--- lib/yahns/proxy_pass.rb | 82 +++++++++++++++++++++++++------ 9 files changed, 160 insertions(+), 29 deletions(-) create mode 100644 .document create mode 100644 .olddoc.yml create mode 100644 examples/https_proxy_pass.conf.rb create mode 100644 examples/proxy_pass.ru diff --git a/.document b/.document new file mode 100644 index 0000000..1880850 --- /dev/null +++ b/.document @@ -0,0 +1,2 @@ +lib/yahns.rb +lib/yahns/proxy_pass.rb diff --git a/.olddoc.yml b/.olddoc.yml new file mode 100644 index 0000000..7e8d2ad --- /dev/null +++ b/.olddoc.yml @@ -0,0 +1,8 @@ +--- +cgit_url: https://yhbt.net/yahns.git +git_url: https://yhbt.net/yahns.git +rdoc_url: https://yhbt.net/yahns/ +ml_url: https://yhbt.net/yahns-public/ +public_email: yahns-public@yhbt.net +nntp_url: + - nntp://news.public-inbox.org/inbox.comp.lang.ruby.yahns diff --git a/Documentation/yahns_config.pod b/Documentation/yahns_config.pod index 737e085..08c2e27 100644 --- a/Documentation/yahns_config.pod +++ b/Documentation/yahns_config.pod @@ -448,10 +448,10 @@ An example which seems to work is: ) # use defaults provided by Ruby on top of OpenSSL, - # but disable client certificate verification as it is rare: + # but disable client certificate verification as it is rare for servers: ssl_ctx.set_params(verify_mode: OpenSSL::SSL::VERIFY_NONE) - # Built-in session cache (only works if worker_processes is nil or 1) + # Built-in session cache (only useful if worker_processes is nil or 1) ssl_ctx.session_cache_mode = OpenSSL::SSL::SSLContext::SESSION_CACHE_SERVER app(:rack, "/path/to/my/app/config.ru") do diff --git a/Rakefile b/Rakefile index 3eb0219..65862f2 100644 --- a/Rakefile +++ b/Rakefile @@ -3,7 +3,24 @@ require 'tempfile' include Rake::DSL -gendocs = %w(NEWS NEWS.atom.xml) +apidoc = { + 'doc/Yahns.html' => 'lib/yahns.rb', + 'doc/Yahns/ProxyPass.html' => 'lib/yahns/proxy_pass.rb' +} + +task apidoc.keys[0] => apidoc.values do + rdoc = ENV['rdoc'] || 'rdoc' + system("git", "set-file-times", *(apidoc.values)) + sh "#{rdoc} -f dark216" # dark216 requires olddoc 1.7+ + + apidoc.each do |dst, src| + src = File.stat(src) + File.utime(src.atime, src.mtime, dst) + end +end + +gendocs = %W(NEWS NEWS.atom.xml #{apidoc.keys[0]}) +task html: apidoc.keys[0] task rsync_docs: gendocs do dest = ENV["RSYNC_DEST"] || "yhbt.net:/srv/yhbt/yahns/" top = %w(INSTALL HACKING README COPYING) @@ -28,6 +45,7 @@ files = `git ls-files Documentation/*.txt`.split(/\n/) files.concat(top) files.concat(gendocs) + files.concat(%w(doc/Yahns.html)) files.concat(%w(yahns yahns-rackup yahns_config).map! { |x| "Documentation/#{x}.txt" }) diff --git a/examples/https_proxy_pass.conf.rb b/examples/https_proxy_pass.conf.rb new file mode 100644 index 0000000..f2fbc3a --- /dev/null +++ b/examples/https_proxy_pass.conf.rb @@ -0,0 +1,36 @@ +# To the extent possible under law, Eric Wong has waived all copyright and +# related or neighboring rights to this example. +# +# See examples/proxy_pass.ru for the complementary rackup file +# + +# Setup an OpenSSL context: +require 'openssl' +ssl_ctx = OpenSSL::SSL::SSLContext.new +ssl_ctx.cert = OpenSSL::X509::Certificate.new( + File.read('/etc/ssl/certs/example.crt') +) +ssl_ctx.extra_chain_cert = [ + OpenSSL::X509::Certificate.new( + File.read('/etc/ssl/certs/chain.crt') + ) +] +ssl_ctx.key = OpenSSL::PKey::RSA.new( + File.read('/etc/ssl/private/example.key') +) + +# use defaults provided by Ruby on top of OpenSSL, +# but disable client certificate verification as it is rare for servers: +ssl_ctx.set_params(verify_mode: OpenSSL::SSL::VERIFY_NONE) + +# Built-in session cache (only useful if worker_processes is nil or 1) +ssl_ctx.session_cache_mode = OpenSSL::SSL::SSLContext::SESSION_CACHE_SERVER + +worker_processes 1 +app(:rack, "/path/to/proxy_pass.ru", preload: true) do + listen 443, ssl_ctx: ssl_ctx + listen '[::]:443', ipv6only: true, ssl_ctx: ssl_ctx +end + +stdout_path "/path/to/my_logs/out.log" +stderr_path "/path/to/my_logs/err.log" diff --git a/examples/proxy_pass.ru b/examples/proxy_pass.ru new file mode 100644 index 0000000..63ee6d9 --- /dev/null +++ b/examples/proxy_pass.ru @@ -0,0 +1,11 @@ +# To the extent possible under law, Eric Wong has waived all copyright and +# related or neighboring rights to this example. +# +# See examples/https_proxy_pass.conf.rb for the complementary rackup file +# + +# optionally, intercept static requests with Rack::Static middleware: +# use Rack::Static, root: '/path/to/public', gzip: true + +require 'yahns/proxy_pass' +run Yahns::ProxyPass.new('http://127.0.0.1:6081') diff --git a/extras/proxy_pass.rb b/extras/proxy_pass.rb index af6fe01..40bf19a 100644 --- a/extras/proxy_pass.rb +++ b/extras/proxy_pass.rb @@ -10,12 +10,13 @@ require 'thread' require 'timeout' -# Totally synchronous and Rack 1.1-compatible, this will probably be rewritten. -# to take advantage of rack.hijack and use the non-blocking I/O facilities -# in yahns. yahns may have to grow a supported API for that... +# Totally synchronous and Rack 1.1-compatible. See Yahns::ProxyPass for +# the rewritten version which takes advantage of rack.hijack and uses +# the internal non-blocking I/O facilities in yahns. yahns may have to +# grow a supported API for that... +# # For now, we this blocks a worker thread; fortunately threads are reasonably # cheap on GNU/Linux... -# This is totally untested but currently doesn't serve anything important. class ProxyPass # :nodoc: class ConnPool def initialize diff --git a/lib/yahns.rb b/lib/yahns.rb index 08945ef..4cf911e 100644 --- a/lib/yahns.rb +++ b/lib/yahns.rb @@ -1,5 +1,5 @@ -# Copyright (C) 2013-2016 all contributors -# License: GPL-3.0+ (https://www.gnu.org/licenses/gpl-3.0.txt) +# Copyright (C) 2013-2019 all contributors +# License: GPL-3.0+ # frozen_string_literal: true $stdout.sync = $stderr.sync = true @@ -16,12 +16,15 @@ Unicorn.__send__(:remove_const, sym) if Unicorn.const_defined?(sym) end -# yahns exposes no user-visible API outside of the config file. -# See https://yhbt.net/yahns.git/tree/examples/yahns_config.txt -# for the config documentation +# yahns exposes little user-visible API outside of the config file. +# See https://yhbt.net/yahns/yahns_config.txt +# for the config documentation (or yahns_config(5) manpage) # and https://yhbt.net/yahns.git/about/ for the homepage. -# Internals are subject to change. - +# +# Yahns::ProxyPass is currently the only public API. +# +# Documented APIs and options are supported forever, +# internals are subject to change. module Yahns # :stopdoc: # We populate this at startup so we can figure out how to reexecute diff --git a/lib/yahns/proxy_pass.rb b/lib/yahns/proxy_pass.rb index 2a37773..bc902f8 100644 --- a/lib/yahns/proxy_pass.rb +++ b/lib/yahns/proxy_pass.rb @@ -1,24 +1,76 @@ # -*- encoding: binary -*- -# Copyright (C) 2013-2016 all contributors -# License: GPL-3.0+ (https://www.gnu.org/licenses/gpl-3.0.txt) +# Copyright (C) 2013-2019 all contributors +# License: GPL-3.0+ # frozen_string_literal: true require 'socket' require 'rack/request' -require 'timeout' - -# XXX consider this file and the proxy-related stuff in yahns -# unstable and experimental! It has never been documented and -# incompatible changes may still happen. -# -# However, it seems to be proxying for our mail archives well enough: -# https://yhbt.net/yahns-public/ +require 'timeout' # only for Timeout::Error require_relative 'proxy_http_response' require_relative 'req_res' -class Yahns::ProxyPass # :nodoc: - attr_reader :proxy_buffering, :response_headers +# Yahns::ProxyPass is a Rack (hijack) app which allows yahns to +# act as a fully-buffering reverse proxy to protect backends +# from slow HTTP clients. +# +# Yahns::ProxyPass relies on the default behavior of yahns to do +# full input and output buffering. Output buffering is lazy, +# meaning it allows streaming output in the best case and +# will only buffer if the client cannot keep up with the server. +# +# The goal of this reverse proxy is to act as a sponge on the same LAN +# or host to any backend HTTP server not optimized for slow clients. +# Yahns::ProxyPass accomplishes this by handling all the slow clients +# internally within yahns itself to minimize time spent in the backend +# HTTP server waiting on slow clients. +# +# It does not do load balancing (we rely on Varnish for that). +# Here is the exact config we use with Varnish, which uses +# the +:response_headers+ option to hide some Varnish headers +# from clients: +# +# run Yahns::ProxyPass.new('http://127.0.0.1:6081', +# response_headers: { +# 'Age' => :ignore, +# 'X-Varnish' => :ignore, +# 'Via' => :ignore +# }) +# +# This is NOT a generic Rack app and must be run with yahns. +# It uses +rack.hijack+, so compatibility with logging +# middlewares (e.g. Rack::CommonLogger) is not great and +# timing information gets lost. +# +# This provides HTTPS termination for our mail archives: +# https://yhbt.net/yahns-public/ +# +# See https://yhbt.net/yahns.git/tree/examples/https_proxy_pass.conf.rb +# and https://yhbt.net/yahns.git/tree/examples/proxy_pass.ru for examples +class Yahns::ProxyPass + attr_reader :proxy_buffering, :response_headers # :nodoc: - def initialize(dest, opts = {}) + # +dest+ must be an HTTP URL with optional variables prefixed with '$'. + # +dest+ may refer to the path to a Unix domain socket in the form: + # + # unix:/absolute/path/to/socket + # + # Variables which may be used in the +dest+ parameter include: + # + # - $url - the entire URL used to make the request + # - $path - the unescaped PATH_INFO of the HTTP request + # - $fullpath - $path with QUERY_STRING + # - $host - the hostname in the Host: header + # + # For Unix domain sockets, variables may be separated from the + # socket path via: ":/". For example: + # + # unix:/absolute/path/to/socket:/$host/$fullpath + # + # Currently :response_headers is the only +opts+ supported. + # :response_headers is a Hash containing a "from => to" mapping + # of response headers. The special value of +:ignore+ indicates + # the header from the backend HTTP server will be ignored instead + # of being blindly passed on to the client. + def initialize(dest, opts = { response_headers: { 'Server' => :ignore } }) case dest when %r{\Aunix:([^:]+)(?::(/.*))?\z} path = $2 @@ -41,7 +93,7 @@ def initialize(dest, opts = {}) init_path_vars(path) end - def init_path_vars(path) + def init_path_vars(path) # :nodoc: path ||= '$fullpath' # methods from Rack::Request we want: allow = %w(fullpath host_with_port host port url path) @@ -54,7 +106,7 @@ def init_path_vars(path) @path = path.gsub(%r{\A/(\$(?:fullpath|path))}, '\1') end - def call(env) + def call(env) # :nodoc: # 3-way handshake for TCP backends while we generate the request header rr = Yahns::ReqRes.start(@sockaddr) c = env['rack.hijack'].call # Yahns::HttpClient#call -- EW