* [PATCH] http: update in-memory devXX/usage on PUT+DELETE
@ 2020-07-24 6:08 Eric Wong
2020-07-25 2:10 ` [PATCH v2] " Eric Wong
0 siblings, 1 reply; 2+ messages in thread
From: Eric Wong @ 2020-07-24 6:08 UTC (permalink / raw)
To: cmogstored-public
Under heavy write traffic, free space changes constantly, and
the periodic updates every 10 (or MOG_DISK_USAGE_INTERVAL)
seconds can be too far behind. Since we keep the usage file
contents in-memory now for out-of-FD situations, we can update
that without incurring extra VFS traffic.
---
cmogstored.h | 2 ++
dev.c | 7 +++++++
http_dav.c | 6 ++++++
http_put.c | 28 ++++++++++++++++++++++------
4 files changed, 37 insertions(+), 6 deletions(-)
diff --git a/cmogstored.h b/cmogstored.h
index 35ddc73..b5211cf 100644
--- a/cmogstored.h
+++ b/cmogstored.h
@@ -398,6 +398,8 @@ void mog_dev_free(void *devptr);
bool mog_dev_user_rescale_i(void *devp, void *svcp);
bool mog_dev_requeue_prepare(void *devp, void *ign);
void mog_dev_usage_update(struct mog_dev *, struct mog_svc *);
+void mog_dev_usage_refresh(struct mog_dev *, struct mog_svc *,
+ struct statvfs *);
/* valid_path.rl */
int mog_valid_path(const char *buf, size_t len);
diff --git a/dev.c b/dev.c
index c120736..1f723e5 100644
--- a/dev.c
+++ b/dev.c
@@ -184,6 +184,13 @@ emit_usage(struct mog_dev *dev, struct mog_svc *svc, int fd, struct statvfs *v)
return rc;
}
+void
+mog_dev_usage_refresh(struct mog_dev *dev, struct mog_svc *svc,
+ struct statvfs *v)
+{
+ (void)emit_usage(dev, svc, -1, v);
+}
+
static void
dev_usage_update(struct mog_dev *dev, struct mog_svc *svc, struct statvfs *v)
{
diff --git a/http_dav.c b/http_dav.c
index 4b6df9b..c9126ae 100644
--- a/http_dav.c
+++ b/http_dav.c
@@ -28,7 +28,13 @@ void mog_http_delete(struct mog_fd *mfd, char *buf)
rc = mog_unlink(http->svc, path);
if (rc == 0) {
+ struct mog_dev *dev;
+
mog_http_resp(mfd, "204 No Content", true);
+ dev = mog_dev_for(http->svc, http->_p.mog_devid, false);
+ if (dev)
+ mog_dev_usage_update(dev, http->svc);
+
return;
}
diff --git a/http_put.c b/http_put.c
index fc36ba7..a3bfee6 100644
--- a/http_put.c
+++ b/http_put.c
@@ -12,16 +12,23 @@ static __thread struct {
char state[128];
} rnd;
-static void file_close_null(struct mog_fd *mfd)
+static struct mog_dev *file_close_null(struct mog_fd *mfd, struct statvfs *v)
{
struct mog_http *http = &mfd->as.http;
+ struct mog_dev *dev;
if (http->forward == NULL)
- return;
+ return NULL;
mog_http_unlink_ftmp(http);
TRACE(CMOGSTORED_HTTP_BYTES_XFER(mfd->fd, http->forward->as.file.foff));
+ dev = mog_dev_for(http->svc, http->_p.mog_devid, false);
+ if (dev && fstatvfs(http->forward->fd, v) < 0)
+ syslog(LOG_ERR, "fstatvfs(%s%s): %m",
+ http->svc->docroot,
+ http->forward->as.file.tmppath);
mog_file_close(http->forward);
http->forward = NULL;
+ return dev;
}
bool mog_http_write_full(struct mog_fd *file_mfd, char *buf, size_t buf_len)
@@ -79,6 +86,9 @@ bool mog_http_write_full(struct mog_fd *file_mfd, char *buf, size_t buf_len)
MOG_NOINLINE static enum mog_next
stop0(struct mog_fd *mfd, const char *status, size_t status_len)
{
+ struct statvfs v;
+ struct mog_dev *dev;
+
if (status) {
struct iovec iov;
union { const char *in; char *out; } deconst;
@@ -89,7 +99,9 @@ stop0(struct mog_fd *mfd, const char *status, size_t status_len)
mog_http_resp0(mfd, &iov, false);
}
- file_close_null(mfd);
+ dev = file_close_null(mfd, &v);
+ if (dev)
+ mog_dev_usage_refresh(dev, mfd->as.http.svc, &v);
return MOG_NEXT_CLOSE;
}
@@ -148,19 +160,23 @@ static bool set_perms_commit(struct mog_http *http)
static void put_commit_resp(struct mog_fd *mfd)
{
struct mog_http *http = &mfd->as.http;
+ struct mog_dev *dev;
+ struct statvfs v;
if (md5_ok(http)) { /* true if there's no MD5, too */
if (set_perms_commit(http)) {
- file_close_null(mfd);
+ dev = file_close_null(mfd, &v);
mog_http_resp(mfd, "201 Created", true);
} else {
- file_close_null(mfd);
+ dev = file_close_null(mfd, &v);
mog_http_resp(mfd, "500 Internal Server Error", false);
}
} else {
- file_close_null(mfd);
+ dev = file_close_null(mfd, &v);
mog_http_resp(mfd, "400 Bad Request", true);
}
+ if (dev)
+ mog_dev_usage_refresh(dev, http->svc, &v);
}
static enum mog_next http_put_commit(struct mog_fd *mfd)
^ permalink raw reply related [flat|nested] 2+ messages in thread
* [PATCH v2] http: update in-memory devXX/usage on PUT+DELETE
2020-07-24 6:08 [PATCH] http: update in-memory devXX/usage on PUT+DELETE Eric Wong
@ 2020-07-25 2:10 ` Eric Wong
0 siblings, 0 replies; 2+ messages in thread
From: Eric Wong @ 2020-07-25 2:10 UTC (permalink / raw)
To: cmogstored-public
Under heavy write traffic, free space changes constantly, and
the periodic updates every 10 (or MOG_DISK_USAGE_INTERVAL)
seconds can be too far behind. Since we keep the usage file
contents in-memory now for out-of-FD situations, we can update
that without incurring extra VFS traffic.
v2: We no longer try to use fstatvfs(2) and instead pay the cost of
an extra name lookups and just update all usage files. This was
necessary since calculating free space while a file is still
open can take a long time on some FSes and we need to send the
HTTP response back ASAP to avoid timeouts on the client-side.
This avoids contention in the request worker threads and the
mostly idle main thread to do more work.
---
http_dav.c | 2 ++
http_put.c | 1 +
2 files changed, 3 insertions(+)
diff --git a/http_dav.c b/http_dav.c
index 4b6df9b..43b6c7a 100644
--- a/http_dav.c
+++ b/http_dav.c
@@ -29,6 +29,8 @@ void mog_http_delete(struct mog_fd *mfd, char *buf)
rc = mog_unlink(http->svc, path);
if (rc == 0) {
mog_http_resp(mfd, "204 No Content", true);
+ mog_notify(MOG_NOTIFY_DEVICE_REFRESH);
+
return;
}
diff --git a/http_put.c b/http_put.c
index fc36ba7..11d6342 100644
--- a/http_put.c
+++ b/http_put.c
@@ -153,6 +153,7 @@ static void put_commit_resp(struct mog_fd *mfd)
if (set_perms_commit(http)) {
file_close_null(mfd);
mog_http_resp(mfd, "201 Created", true);
+ mog_notify(MOG_NOTIFY_DEVICE_REFRESH);
} else {
file_close_null(mfd);
mog_http_resp(mfd, "500 Internal Server Error", false);
^ permalink raw reply related [flat|nested] 2+ messages in thread
end of thread, other threads:[~2020-07-25 2:10 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-07-24 6:08 [PATCH] http: update in-memory devXX/usage on PUT+DELETE Eric Wong
2020-07-25 2:10 ` [PATCH v2] " Eric Wong
Code repositories for project(s) associated with this public inbox
https://yhbt.net/cmogstored.git/
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).