[PATCH] http: update in-memory devXX/usage on PUT+DELETE

[PATCH] http: update in-memory devXX/usage on PUT+DELETE

[Eric Wong]

Under heavy write traffic, free space changes constantly, and
the periodic updates every 10 (or MOG_DISK_USAGE_INTERVAL)
seconds can be too far behind.  Since we keep the usage file
contents in-memory now for out-of-FD situations, we can update
that without incurring extra VFS traffic.
---
 cmogstored.h |  2 ++
 dev.c        |  7 +++++++
 http_dav.c   |  6 ++++++
 http_put.c   | 28 ++++++++++++++++++++++------
 4 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/cmogstored.h b/cmogstored.h
index 35ddc73..b5211cf 100644
--- a/cmogstored.h
+++ b/cmogstored.h
@@ -398,6 +398,8 @@ void mog_dev_free(void *devptr);
 bool mog_dev_user_rescale_i(void *devp, void *svcp);
 bool mog_dev_requeue_prepare(void *devp, void *ign);
 void mog_dev_usage_update(struct mog_dev *, struct mog_svc *);
+void mog_dev_usage_refresh(struct mog_dev *, struct mog_svc *,
+				struct statvfs *);
 
 /* valid_path.rl */
 int mog_valid_path(const char *buf, size_t len);
diff --git a/dev.c b/dev.c
index c120736..1f723e5 100644
--- a/dev.c
+++ b/dev.c
@@ -184,6 +184,13 @@ emit_usage(struct mog_dev *dev, struct mog_svc *svc, int fd, struct statvfs *v)
 	return rc;
 }
 
+void
+mog_dev_usage_refresh(struct mog_dev *dev, struct mog_svc *svc,
+			struct statvfs *v)
+{
+	(void)emit_usage(dev, svc, -1, v);
+}
+
 static void
 dev_usage_update(struct mog_dev *dev, struct mog_svc *svc, struct statvfs *v)
 {
diff --git a/http_dav.c b/http_dav.c
index 4b6df9b..c9126ae 100644
--- a/http_dav.c
+++ b/http_dav.c
@@ -28,7 +28,13 @@ void mog_http_delete(struct mog_fd *mfd, char *buf)
 
 	rc = mog_unlink(http->svc, path);
 	if (rc == 0) {
+		struct mog_dev *dev;
+
 		mog_http_resp(mfd, "204 No Content", true);
+		dev = mog_dev_for(http->svc, http->_p.mog_devid, false);
+		if (dev)
+			mog_dev_usage_update(dev, http->svc);
+
 		return;
 	}
 
diff --git a/http_put.c b/http_put.c
index fc36ba7..a3bfee6 100644
--- a/http_put.c
+++ b/http_put.c
@@ -12,16 +12,23 @@ static __thread struct {
 	char state[128];
 } rnd;
 
-static void file_close_null(struct mog_fd *mfd)
+static struct mog_dev *file_close_null(struct mog_fd *mfd, struct statvfs *v)
 {
 	struct mog_http *http = &mfd->as.http;
+	struct mog_dev *dev;
 
 	if (http->forward == NULL)
-		return;
+		return NULL;
 	mog_http_unlink_ftmp(http);
 	TRACE(CMOGSTORED_HTTP_BYTES_XFER(mfd->fd, http->forward->as.file.foff));
+	dev = mog_dev_for(http->svc, http->_p.mog_devid, false);
+	if (dev && fstatvfs(http->forward->fd, v) < 0)
+		syslog(LOG_ERR, "fstatvfs(%s%s): %m",
+		       http->svc->docroot,
+		       http->forward->as.file.tmppath);
 	mog_file_close(http->forward);
 	http->forward = NULL;
+	return dev;
 }
 
 bool mog_http_write_full(struct mog_fd *file_mfd, char *buf, size_t buf_len)
@@ -79,6 +86,9 @@ bool mog_http_write_full(struct mog_fd *file_mfd, char *buf, size_t buf_len)
 MOG_NOINLINE static enum mog_next
 stop0(struct mog_fd *mfd, const char *status, size_t status_len)
 {
+	struct statvfs v;
+	struct mog_dev *dev;
+
 	if (status) {
 		struct iovec iov;
 		union { const char *in; char *out; } deconst;
@@ -89,7 +99,9 @@ stop0(struct mog_fd *mfd, const char *status, size_t status_len)
 
 		mog_http_resp0(mfd, &iov, false);
 	}
-	file_close_null(mfd);
+	dev = file_close_null(mfd, &v);
+	if (dev)
+		mog_dev_usage_refresh(dev, mfd->as.http.svc, &v);
 	return MOG_NEXT_CLOSE;
 }
 
@@ -148,19 +160,23 @@ static bool set_perms_commit(struct mog_http *http)
 static void put_commit_resp(struct mog_fd *mfd)
 {
 	struct mog_http *http = &mfd->as.http;
+	struct mog_dev *dev;
+	struct statvfs v;
 
 	if (md5_ok(http)) { /* true if there's no MD5, too */
 		if (set_perms_commit(http)) {
-			file_close_null(mfd);
+			dev = file_close_null(mfd, &v);
 			mog_http_resp(mfd, "201 Created", true);
 		} else {
-			file_close_null(mfd);
+			dev = file_close_null(mfd, &v);
 			mog_http_resp(mfd, "500 Internal Server Error", false);
 		}
 	} else {
-		file_close_null(mfd);
+		dev = file_close_null(mfd, &v);
 		mog_http_resp(mfd, "400 Bad Request", true);
 	}
+	if (dev)
+		mog_dev_usage_refresh(dev, http->svc, &v);
 }
 
 static enum mog_next http_put_commit(struct mog_fd *mfd)

[PATCH v2] http: update in-memory devXX/usage on PUT+DELETE

[Eric Wong]

Under heavy write traffic, free space changes constantly, and
the periodic updates every 10 (or MOG_DISK_USAGE_INTERVAL)
seconds can be too far behind.  Since we keep the usage file
contents in-memory now for out-of-FD situations, we can update
that without incurring extra VFS traffic.

v2: We no longer try to use fstatvfs(2) and instead pay the cost of
an extra name lookups and just update all usage files.  This was
necessary since calculating free space while a file is still
open can take a long time on some FSes and we need to send the
HTTP response back ASAP to avoid timeouts on the client-side.

This avoids contention in the request worker threads and the
mostly idle main thread to do more work.
---
 http_dav.c | 2 ++
 http_put.c | 1 +
 2 files changed, 3 insertions(+)

diff --git a/http_dav.c b/http_dav.c
index 4b6df9b..43b6c7a 100644
--- a/http_dav.c
+++ b/http_dav.c
@@ -29,6 +29,8 @@ void mog_http_delete(struct mog_fd *mfd, char *buf)
 	rc = mog_unlink(http->svc, path);
 	if (rc == 0) {
 		mog_http_resp(mfd, "204 No Content", true);
+		mog_notify(MOG_NOTIFY_DEVICE_REFRESH);
+
 		return;
 	}
 
diff --git a/http_put.c b/http_put.c
index fc36ba7..11d6342 100644
--- a/http_put.c
+++ b/http_put.c
@@ -153,6 +153,7 @@ static void put_commit_resp(struct mog_fd *mfd)
 		if (set_perms_commit(http)) {
 			file_close_null(mfd);
 			mog_http_resp(mfd, "201 Created", true);
+			mog_notify(MOG_NOTIFY_DEVICE_REFRESH);
 		} else {
 			file_close_null(mfd);
 			mog_http_resp(mfd, "500 Internal Server Error", false);