[PATCH] support systemd-style socket activation via environment

[PATCH] support systemd-style socket activation via environment

[Eric Wong]

While I have my reservations about systemd, socket activation alone
is a good idea and we already have existing infrastructure for
supporting it in SIGUSR2 upgrades.

We are intentionally avoiding linkage to libsystemd to avoid dealing
with ABI compatibility issues between old and new systems.  This
also allows us to integrate more easily with non-systemd systems
which use the same environment variables as systemd.
---
 cmogstored.x    |  7 +++++++
 inherit.c       | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++----
 test/inherit.rb | 32 ++++++++++++++++++++++++++++++++
 3 files changed, 91 insertions(+), 4 deletions(-)

diff --git a/cmogstored.x b/cmogstored.x
index bd5956d..a6b96e6 100644
--- a/cmogstored.x
+++ b/cmogstored.x
@@ -15,6 +15,13 @@ MOG_IOSTAT_CMD - command-line for invoking iostat(1).  This is used
 by the sidechannel to report disk utilization to trackers.
 Default: "iostat -dx 1 30"
 
+LISTEN_FDS, LISTEN_PID - may be used for systemd-style socket activation
+regardless of whether systemd is on the system.  See sd_listen_fds(3)
+for more information.  Since cmogstored supports two protocols on
+different sockets, the command line or config file must still specify
+which addresses to listen on.  This feature is available in cmogstored
+1.5.0 and later.
+
 See MALLOC TUNING for environment variables which may affect memory
 usage.
 
diff --git a/inherit.c b/inherit.c
index ca7afa7..0229a04 100644
--- a/inherit.c
+++ b/inherit.c
@@ -89,6 +89,55 @@ static bool listener_close_each(void *_l, void *unused)
 	return true;
 }
 
+static void listeners_init(void)
+{
+	if (listeners) return;
+	listeners = hash_initialize(3, NULL, listener_hash, listener_cmp, free);
+	mog_oom_if_null(listeners);
+	atexit(listeners_cleanup);
+}
+
+static unsigned long listen_env(const char *env)
+{
+	const char *e = getenv(env);
+	unsigned long tmp;
+	char *end;
+
+	if (!e) return ULONG_MAX;
+	errno = 0;
+	tmp = strtoul(e, &end, 10);
+	if (errno) die_errno("failed to parse %s: %s", env, e);
+	if (*end) die("trailing byte in %s: %s", env, e);
+
+	return tmp;
+}
+
+/* systemd-style socket activation in the vein of sd_listen_fds(3) */
+static void systemd_inherit_fds(void)
+{
+	const int listen_fds_start = 3; /* SD_LISTEN_FDS_START */
+	int fd, listen_fds_end;
+	unsigned long tmp = listen_env("LISTEN_PID");
+
+	if (getpid() != (pid_t)tmp) goto out;
+
+	tmp = listen_env("LISTEN_FDS");
+	if (tmp > INT_MAX) die("LISTEN_FDS out of range: %lu", tmp);
+
+	listeners_init();
+	listen_fds_end = listen_fds_start + (int)tmp;
+	for (fd = listen_fds_start; fd < listen_fds_end; fd++) {
+		if (mog_set_cloexec(fd, true) == 0)
+			register_listen_fd(fd);
+		else
+			die("inherited out %d of %lu LISTEN_FDS",
+			    fd - listen_fds_start, tmp);
+	}
+out:
+	unsetenv("LISTEN_FDS");
+	unsetenv("LISTEN_PID");
+}
+
 /* close all inherited listeners we do not need */
 void mog_inherit_cleanup(void)
 {
@@ -131,13 +180,12 @@ void mog_inherit_init(void)
 	unsigned long fd;
 	unsigned endbyte;
 
+	systemd_inherit_fds();
+
 	if (orig == NULL)
 		return;
 
-	listeners = hash_initialize(3, NULL, listener_hash, listener_cmp, free);
-	mog_oom_if_null(listeners);
-	atexit(listeners_cleanup);
-
+	listeners_init();
 	fds = xstrdup(orig);
 	tip = fds;
 
diff --git a/test/inherit.rb b/test/inherit.rb
index b342ab0..34aa52f 100644
--- a/test/inherit.rb
+++ b/test/inherit.rb
@@ -142,4 +142,36 @@ def test_inherit_high
     @err.rewind
     assert_match(/failed to parse/, @err.read)
   end
+
+  def test_inherit_systemd
+    # disabled test on old Rubies: https://bugs.ruby-lang.org/issues/11336
+    # [ruby-core:69895] [Bug #11336] fixed by r51576
+    return unless RUBY_VERSION.to_f >= 2.3
+
+    mgmt = TCPServer.new(@host, 0)
+    @to_close << mgmt
+    mport = mgmt.addr[1]
+    cmd = %W(cmogstored --docroot=#@tmpdir --httplisten=#@host:#@port
+             --mgmtlisten=#@host:#{mport} --maxconns=100)
+    @pid = fork do
+      ENV['LISTEN_PID'] = "#$$"
+      ENV['LISTEN_FDS'] = '2'
+      exec(*cmd, 3 => mgmt.fileno, 4 => @srv.fileno)
+    end
+
+    # just ensure HTTP works after being inherited
+    Net::HTTP.start(@host, @port) do |http|
+      [ Net::HTTP::Get, Net::HTTP::Head ].each do |meth|
+        resp = http.request(meth.new("/"))
+        assert_kind_of Net::HTTPOK, resp
+      end
+    end
+
+    # still works since drop is open in _this_ process
+    c = TCPSocket.new(@host, mport)
+    assert_instance_of(TCPSocket, c)
+    @to_close << c
+    c.write "hello\n"
+    assert_match /ERROR: unknown command/, c.gets
+  end
 end
-- 
EW

[PATCH] set TCP listener options on inherited sockets

[Eric Wong]

systemd users may not set the correct TCP socket options for
us, so be sure to set TCP_NODELAY, SO_KEEPALIVE, and use
a sufficiently large listen backlog to avoid hurting performance
for users who bind sockets outside of cmogstored.
---
 bind_listen.c | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/bind_listen.c b/bind_listen.c
index 5535b92..c705576 100644
--- a/bind_listen.c
+++ b/bind_listen.c
@@ -15,15 +15,17 @@
  *   http://labs.apnic.net/blabs/?p=57
  */
 
-static int set_tcp_opts(int fd)
+static int set_tcp_opts(int fd, bool inherited)
 {
 	int val;
 	socklen_t len = sizeof(int);
 	int rc;
 
-	val = 1;
-	rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, len);
-	if (rc < 0) return rc;
+	if (!inherited) {
+		val = 1;
+		rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, len);
+		if (rc < 0) return rc;
+	}
 
 	val = 1;
 	rc = setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, len);
@@ -40,8 +42,11 @@ int mog_bind_listen(struct addrinfo *r)
 {
 	/* see if we inherited the socket, first */
 	int fd = mog_inherit_get(r->ai_addr, r->ai_addrlen);
+	const int backlog = 1024;
 
-	if (fd >= 0)
+	if (fd >= 0 &&
+	    set_tcp_opts(fd, true) == 0 &&
+	    listen(fd, backlog) == 0)
 		return fd;
 
 	for (; r; r = r->ai_next) {
@@ -56,9 +61,9 @@ int mog_bind_listen(struct addrinfo *r)
 		 * everywhere yet (in 2012).
 		 */
 		if (mog_set_cloexec(fd, true) == 0 &&
-		    set_tcp_opts(fd) == 0 &&
+		    set_tcp_opts(fd, false) == 0 &&
 		    bind(fd, r->ai_addr, r->ai_addrlen) == 0 &&
-		    listen(fd, 1024) == 0)
+		    listen(fd, backlog) == 0)
 			break;
 
 		PRESERVE_ERRNO( mog_close(fd) );
-- 
EW

[PATCH] doc: add example systemd config files

[Eric Wong]

Since we'll support systemd, it's not a bad idea to include
reasonable example files for users.
---
 Makefile.am                  |  3 ++-
 examples/cmogstored.socket   | 11 +++++++++++
 examples/cmogstored@.service | 27 +++++++++++++++++++++++++++
 3 files changed, 40 insertions(+), 1 deletion(-)
 create mode 100644 examples/cmogstored.socket
 create mode 100644 examples/cmogstored@.service

diff --git a/Makefile.am b/Makefile.am
index 69b347e..5860f9c 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -224,7 +224,8 @@ EXTRA_DIST = $(RB_TESTS) $(RL_CGEN) $(RL_ALL) $(PERL_TESTS) $(extra_doc) \
   tests/.gitignore \
   $(top_srcdir)/.version $(top_srcdir)/.gnulib-version \
   test/valgrind.supp nostd/README \
-  $(tap_support)
+  $(tap_support) \
+  examples/cmogstored.socket examples/cmogstored@.service
 
 TESTS_ENVIRONMENT = PATH=$(top_builddir):$$PATH TMPDIR=$(test_tmpdir)
 
diff --git a/examples/cmogstored.socket b/examples/cmogstored.socket
new file mode 100644
index 0000000..852fd3a
--- /dev/null
+++ b/examples/cmogstored.socket
@@ -0,0 +1,11 @@
+# ==> /etc/systemd/system/cmogstored.socket <==
+[Unit]
+Description = cmogstored sockets
+
+[Socket]
+ListenStream = 127.0.0.1:7500
+ListenStream = 127.0.0.1:7501
+Service = cmogstored@1.service
+
+[Install]
+WantedBy = sockets.target
diff --git a/examples/cmogstored@.service b/examples/cmogstored@.service
new file mode 100644
index 0000000..2361a98
--- /dev/null
+++ b/examples/cmogstored@.service
@@ -0,0 +1,27 @@
+# ==> /etc/systemd/system/cmogstored@.service <==
+# Since SIGUSR2 upgrades do not work under systemd, this service file
+# allows starting two simultaneous services during upgrade time
+# (e.g. cmogstored@1 cmogstored@2) with the intention that they take
+# turns running in-between upgrades.  This should allow upgrading
+# without downtime
+
+[Unit]
+Description = cmogstored storage server %i
+Wants = cmogstored.socket
+After = cmogstored.socket
+
+[Service]
+ExecStart = /usr/local/bin/cmogstored \
+	    --httplisten=127.0.0.1:7500 \
+	    --mgmtlisten=127.0.0.1:7501 \
+	    --docroot=/var/mogdata
+Sockets = cmogstored.socket
+KillSignal = SIGQUIT
+User = mogstored
+Group = nogroup
+LimitNOFILE = 10000
+# some clients are slow, give them time before SIGKILL
+TimeoutStopSec = 7200
+
+[Install]
+WantedBy = multi-user.target
-- 
EW