[PATCH] proxy_pass: document as a public API

[PATCH] proxy_pass: document as a public API

[Eric Wong]

Might as well... this has been in use at YHBT.net for ~4 years
at this point.  And given nginx has new corporate overlords,
maybe a decidedly non-enterprisey alternative is worth
"marketing" :P

Previous discussion from 2016:
https://YHBT.net/yahns-public/20160220081619.GA10850@dcvr.yhbt.net/
---
 .document                         |  2 +
 .olddoc.yml                       |  8 +++
 Documentation/yahns_config.pod    |  4 +-
 Rakefile                          | 20 +++++++-
 examples/https_proxy_pass.conf.rb | 36 ++++++++++++++
 examples/proxy_pass.ru            | 11 +++++
 extras/proxy_pass.rb              |  9 ++--
 lib/yahns.rb                      | 17 ++++---
 lib/yahns/proxy_pass.rb           | 82 +++++++++++++++++++++++++------
 9 files changed, 160 insertions(+), 29 deletions(-)
 create mode 100644 .document
 create mode 100644 .olddoc.yml
 create mode 100644 examples/https_proxy_pass.conf.rb
 create mode 100644 examples/proxy_pass.ru

diff --git a/.document b/.document
new file mode 100644
index 0000000..1880850
--- /dev/null
+++ b/.document
@@ -0,0 +1,2 @@
+lib/yahns.rb
+lib/yahns/proxy_pass.rb
diff --git a/.olddoc.yml b/.olddoc.yml
new file mode 100644
index 0000000..7e8d2ad
--- /dev/null
+++ b/.olddoc.yml
@@ -0,0 +1,8 @@
+---
+cgit_url: https://yhbt.net/yahns.git
+git_url: https://yhbt.net/yahns.git
+rdoc_url: https://yhbt.net/yahns/
+ml_url: https://yhbt.net/yahns-public/
+public_email: yahns-public@yhbt.net
+nntp_url:
+  - nntp://news.public-inbox.org/inbox.comp.lang.ruby.yahns
diff --git a/Documentation/yahns_config.pod b/Documentation/yahns_config.pod
index 737e085..08c2e27 100644
--- a/Documentation/yahns_config.pod
+++ b/Documentation/yahns_config.pod
@@ -448,10 +448,10 @@ An example which seems to work is:
   )
 
   # use defaults provided by Ruby on top of OpenSSL,
-  # but disable client certificate verification as it is rare:
+  # but disable client certificate verification as it is rare for servers:
   ssl_ctx.set_params(verify_mode: OpenSSL::SSL::VERIFY_NONE)
 
-  # Built-in session cache (only works if worker_processes is nil or 1)
+  # Built-in session cache (only useful if worker_processes is nil or 1)
   ssl_ctx.session_cache_mode = OpenSSL::SSL::SSLContext::SESSION_CACHE_SERVER
 
   app(:rack, "/path/to/my/app/config.ru") do
diff --git a/Rakefile b/Rakefile
index 3eb0219..65862f2 100644
--- a/Rakefile
+++ b/Rakefile
@@ -3,7 +3,24 @@
 require 'tempfile'
 include Rake::DSL
 
-gendocs = %w(NEWS NEWS.atom.xml)
+apidoc = {
+  'doc/Yahns.html' => 'lib/yahns.rb',
+  'doc/Yahns/ProxyPass.html' => 'lib/yahns/proxy_pass.rb'
+}
+
+task apidoc.keys[0] => apidoc.values do
+  rdoc = ENV['rdoc'] || 'rdoc'
+  system("git", "set-file-times", *(apidoc.values))
+  sh "#{rdoc} -f dark216" # dark216 requires olddoc 1.7+
+
+  apidoc.each do |dst, src|
+    src = File.stat(src)
+    File.utime(src.atime, src.mtime, dst)
+  end
+end
+
+gendocs = %W(NEWS NEWS.atom.xml #{apidoc.keys[0]})
+task html: apidoc.keys[0]
 task rsync_docs: gendocs do
   dest = ENV["RSYNC_DEST"] || "yhbt.net:/srv/yhbt/yahns/"
   top = %w(INSTALL HACKING README COPYING)
@@ -28,6 +45,7 @@
   files = `git ls-files Documentation/*.txt`.split(/\n/)
   files.concat(top)
   files.concat(gendocs)
+  files.concat(%w(doc/Yahns.html))
   files.concat(%w(yahns yahns-rackup yahns_config).map! { |x|
     "Documentation/#{x}.txt"
   })
diff --git a/examples/https_proxy_pass.conf.rb b/examples/https_proxy_pass.conf.rb
new file mode 100644
index 0000000..f2fbc3a
--- /dev/null
+++ b/examples/https_proxy_pass.conf.rb
@@ -0,0 +1,36 @@
+# To the extent possible under law, Eric Wong has waived all copyright and
+# related or neighboring rights to this example.
+#
+# See examples/proxy_pass.ru for the complementary rackup file
+# <https://yhbt.net/yahns.git/tree/examples/proxy_pass.ru>
+
+# Setup an OpenSSL context:
+require 'openssl'
+ssl_ctx = OpenSSL::SSL::SSLContext.new
+ssl_ctx.cert = OpenSSL::X509::Certificate.new(
+  File.read('/etc/ssl/certs/example.crt')
+)
+ssl_ctx.extra_chain_cert = [
+  OpenSSL::X509::Certificate.new(
+    File.read('/etc/ssl/certs/chain.crt')
+  )
+]
+ssl_ctx.key = OpenSSL::PKey::RSA.new(
+  File.read('/etc/ssl/private/example.key')
+)
+
+# use defaults provided by Ruby on top of OpenSSL,
+# but disable client certificate verification as it is rare for servers:
+ssl_ctx.set_params(verify_mode: OpenSSL::SSL::VERIFY_NONE)
+
+# Built-in session cache (only useful if worker_processes is nil or 1)
+ssl_ctx.session_cache_mode = OpenSSL::SSL::SSLContext::SESSION_CACHE_SERVER
+
+worker_processes 1
+app(:rack, "/path/to/proxy_pass.ru", preload: true) do
+  listen 443, ssl_ctx: ssl_ctx
+  listen '[::]:443', ipv6only: true, ssl_ctx: ssl_ctx
+end
+
+stdout_path "/path/to/my_logs/out.log"
+stderr_path "/path/to/my_logs/err.log"
diff --git a/examples/proxy_pass.ru b/examples/proxy_pass.ru
new file mode 100644
index 0000000..63ee6d9
--- /dev/null
+++ b/examples/proxy_pass.ru
@@ -0,0 +1,11 @@
+# To the extent possible under law, Eric Wong has waived all copyright and
+# related or neighboring rights to this example.
+#
+# See examples/https_proxy_pass.conf.rb for the complementary rackup file
+# <https://yhbt.net/yahns.git/tree/examples/https_proxy_pass.conf.rb>
+
+# optionally, intercept static requests with Rack::Static middleware:
+# use Rack::Static, root: '/path/to/public', gzip: true
+
+require 'yahns/proxy_pass'
+run Yahns::ProxyPass.new('http://127.0.0.1:6081')
diff --git a/extras/proxy_pass.rb b/extras/proxy_pass.rb
index af6fe01..40bf19a 100644
--- a/extras/proxy_pass.rb
+++ b/extras/proxy_pass.rb
@@ -10,12 +10,13 @@
 require 'thread'
 require 'timeout'
 
-# Totally synchronous and Rack 1.1-compatible, this will probably be rewritten.
-# to take advantage of rack.hijack and use the non-blocking I/O facilities
-# in yahns.  yahns may have to grow a supported API for that...
+# Totally synchronous and Rack 1.1-compatible.  See Yahns::ProxyPass for
+# the rewritten version which takes advantage of rack.hijack and uses
+# the internal non-blocking I/O facilities in yahns.  yahns may have to
+# grow a supported API for that...
+#
 # For now, we this blocks a worker thread; fortunately threads are reasonably
 # cheap on GNU/Linux...
-# This is totally untested but currently doesn't serve anything important.
 class ProxyPass # :nodoc:
   class ConnPool
     def initialize
diff --git a/lib/yahns.rb b/lib/yahns.rb
index 08945ef..4cf911e 100644
--- a/lib/yahns.rb
+++ b/lib/yahns.rb
@@ -1,5 +1,5 @@
-# Copyright (C) 2013-2016 all contributors <yahns-public@yhbt.net>
-# License: GPL-3.0+ (https://www.gnu.org/licenses/gpl-3.0.txt)
+# Copyright (C) 2013-2019 all contributors <yahns-public@yhbt.net>
+# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 $stdout.sync = $stderr.sync = true
 
@@ -16,12 +16,15 @@
     Unicorn.__send__(:remove_const, sym) if Unicorn.const_defined?(sym)
 end
 
-# yahns exposes no user-visible API outside of the config file.
-# See https://yhbt.net/yahns.git/tree/examples/yahns_config.txt
-# for the config documentation
+# yahns exposes little user-visible API outside of the config file.
+# See https://yhbt.net/yahns/yahns_config.txt
+# for the config documentation (or yahns_config(5) manpage)
 # and https://yhbt.net/yahns.git/about/ for the homepage.
-# Internals are subject to change.
-
+#
+# Yahns::ProxyPass is currently the only public API.
+#
+# Documented APIs and options are supported forever,
+# internals are subject to change.
 module Yahns
   # :stopdoc:
   # We populate this at startup so we can figure out how to reexecute
diff --git a/lib/yahns/proxy_pass.rb b/lib/yahns/proxy_pass.rb
index 2a37773..bc902f8 100644
--- a/lib/yahns/proxy_pass.rb
+++ b/lib/yahns/proxy_pass.rb
@@ -1,24 +1,76 @@
 # -*- encoding: binary -*-
-# Copyright (C) 2013-2016 all contributors <yahns-public@yhbt.net>
-# License: GPL-3.0+ (https://www.gnu.org/licenses/gpl-3.0.txt)
+# Copyright (C) 2013-2019 all contributors <yahns-public@yhbt.net>
+# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require 'socket'
 require 'rack/request'
-require 'timeout'
-
-# XXX consider this file and the proxy-related stuff in yahns
-# unstable and experimental!  It has never been documented and
-# incompatible changes may still happen.
-#
-# However, it seems to be proxying for our mail archives well enough:
-# https://yhbt.net/yahns-public/
+require 'timeout' # only for Timeout::Error
 require_relative 'proxy_http_response'
 require_relative 'req_res'
 
-class Yahns::ProxyPass # :nodoc:
-  attr_reader :proxy_buffering, :response_headers
+# Yahns::ProxyPass is a Rack (hijack) app which allows yahns to
+# act as a fully-buffering reverse proxy to protect backends
+# from slow HTTP clients.
+#
+# Yahns::ProxyPass relies on the default behavior of yahns to do
+# full input and output buffering. Output buffering is lazy,
+# meaning it allows streaming output in the best case and
+# will only buffer if the client cannot keep up with the server.
+#
+# The goal of this reverse proxy is to act as a sponge on the same LAN
+# or host to any backend HTTP server not optimized for slow clients.
+# Yahns::ProxyPass accomplishes this by handling all the slow clients
+# internally within yahns itself to minimize time spent in the backend
+# HTTP server waiting on slow clients.
+#
+# It does not do load balancing (we rely on Varnish for that).
+# Here is the exact config we use with Varnish, which uses
+# the +:response_headers+ option to hide some Varnish headers
+# from clients:
+#
+#    run Yahns::ProxyPass.new('http://127.0.0.1:6081',
+#            response_headers: {
+#              'Age' => :ignore,
+#              'X-Varnish' => :ignore,
+#              'Via' => :ignore
+#            })
+#
+# This is NOT a generic Rack app and must be run with yahns.
+# It uses +rack.hijack+, so compatibility with logging
+# middlewares (e.g. Rack::CommonLogger) is not great and
+# timing information gets lost.
+#
+# This provides HTTPS termination for our mail archives:
+# https://yhbt.net/yahns-public/
+#
+# See https://yhbt.net/yahns.git/tree/examples/https_proxy_pass.conf.rb
+# and https://yhbt.net/yahns.git/tree/examples/proxy_pass.ru for examples
+class Yahns::ProxyPass
+  attr_reader :proxy_buffering, :response_headers # :nodoc:
 
-  def initialize(dest, opts = {})
+  # +dest+ must be an HTTP URL with optional variables prefixed with '$'.
+  # +dest+ may refer to the path to a Unix domain socket in the form:
+  #
+  #     unix:/absolute/path/to/socket
+  #
+  # Variables which may be used in the +dest+ parameter include:
+  #
+  # - $url - the entire URL used to make the request
+  # - $path - the unescaped PATH_INFO of the HTTP request
+  # - $fullpath - $path with QUERY_STRING
+  # - $host - the hostname in the Host: header
+  #
+  # For Unix domain sockets, variables may be separated from the
+  # socket path via: ":/".  For example:
+  #
+  #     unix:/absolute/path/to/socket:/$host/$fullpath
+  #
+  # Currently :response_headers is the only +opts+ supported.
+  # :response_headers is a Hash containing a "from => to" mapping
+  # of response headers.  The special value of +:ignore+ indicates
+  # the header from the backend HTTP server will be ignored instead
+  # of being blindly passed on to the client.
+  def initialize(dest, opts = { response_headers: { 'Server' => :ignore } })
     case dest
     when %r{\Aunix:([^:]+)(?::(/.*))?\z}
       path = $2
@@ -41,7 +93,7 @@ def initialize(dest, opts = {})
     init_path_vars(path)
   end
 
-  def init_path_vars(path)
+  def init_path_vars(path) # :nodoc:
     path ||= '$fullpath'
     # methods from Rack::Request we want:
     allow = %w(fullpath host_with_port host port url path)
@@ -54,7 +106,7 @@ def init_path_vars(path)
     @path = path.gsub(%r{\A/(\$(?:fullpath|path))}, '\1')
   end
 
-  def call(env)
+  def call(env) # :nodoc:
     # 3-way handshake for TCP backends while we generate the request header
     rr = Yahns::ReqRes.start(@sockaddr)
     c = env['rack.hijack'].call # Yahns::HttpClient#call
-- 
EW

[PATCH 2/1] doc: include Yahns/ directory on website

[Eric Wong]

Otherwise, https://yhbt.net/yahns/Yahns/ProxyPass.html won't
be accessible
---
 Rakefile | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Rakefile b/Rakefile
index 65862f2..769356e 100644
--- a/Rakefile
+++ b/Rakefile
@@ -59,6 +59,11 @@
   examples.concat(gzex)
 
   sh("rsync --chmod=Fugo=r -av #{examples.join(' ')} #{dest}/examples/")
+
+  rdoc = apidoc.keys.grep(%r{\Adoc/Yahns/})
+  gzex = rdoc.map { |txt| do_gzip.call(txt) }
+  examples.concat(gzex)
+  sh("rsync --chmod=Fugo=r -av #{rdoc.join(' ')} #{dest}/Yahns/")
 end
 
 def tags
-- 
EW