cmogstored.git  about / heads / tags
alternative mogstored implementation for MogileFS
blob 401236ef975a9ba625423f1961b218ea8bdb0e21 5759 bytes (raw)
$ git show khash:dev.c	# shows this blob on the CLI

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
 
/*
 * Copyright (C) 2012-2013, Eric Wong <normalperson@yhbt.net>
 * License: GPLv3 or later (see COPYING for details)
 */
#include "cmogstored.h"
#include "dev.h"

static int stat_prefix(struct mog_svc *svc, struct stat *sb, uint32_t mog_devid)
{
	char prefix[sizeof("/dev" MOG_STR(MOG_DEVID_MAX) "/")];
	int rc;

	assert(mog_devid <= MOG_DEVID_MAX && "mog_devid not filtered");
	rc = snprintf(prefix, sizeof(prefix), "/dev%u/", mog_devid);
	assert(rc > 0 && rc < (int)sizeof(prefix) && "we suck at snprintf");

	return mog_stat(svc, prefix, sb);
}

static struct mog_dev *mog_dev_new(struct mog_svc *svc, uint32_t mog_devid)
{
	struct mog_dev *dev;
	struct stat sb;

	/* we have no space on stack */
	if (mog_devid > MOG_DEVID_MAX)
		return NULL;

	if (stat_prefix(svc, &sb, mog_devid) < 0)
		return NULL;

	dev = mog_cachealign(sizeof(struct mog_dev));
	dev->devid = mog_devid;
	dev->st_dev = sb.st_dev;
	mog_ioq_init(&dev->fsckq, svc, 1);
	mog_ioq_init(&dev->ioq, svc, svc->thr_per_dev);

	return dev;
}

/* lookup or create mog_dev for a given mog_devid */
struct mog_dev *
mog_dev_for(struct mog_svc *svc, uint32_t mog_devid, bool update)
{
	struct mog_dev finder;
	struct mog_dev *ret;
	bool need_refresh = false;
	khint_t k;

	finder.devid = mog_devid;

	CHECK(int, 0, pthread_mutex_lock(&svc->by_mog_devid_lock));
	k = kh_get(by_mog_devid, svc->by_mog_devid, &finder);
	if (k != kh_end(svc->by_mog_devid)) { /* found */
		struct stat sb;

		ret = kh_key(svc->by_mog_devid, k);
		if (!update)
			goto out;

		/*
		 * devXXX dir existed before, but is no longer readable
		 * Possible FS/device error, it could come back, so do
		 * not remove here.
		 */
		if (stat_prefix(svc, &sb, ret->devid) < 0)
			goto out;

		/* st_dev may change due to remount, update if needed */
		ret->st_dev = sb.st_dev;
	} else { /* create a new dev */
		int rc;

		ret = mog_dev_new(svc, mog_devid);

		if (!ret)
			goto out; /* could not stat */

		kh_put(by_mog_devid, svc->by_mog_devid, ret, &rc);
		switch (rc) {
		case 0:
			assert(0 && "mog_dev existed while adding");
			abort();
		case 1: /* never-used bucket */
		case 2: /* deleted bucket */
			if (!update)
				need_refresh = true;
			break; /* OK, inserted */
		default: mog_oom();
		}
	}
out:
	CHECK(int, 0, pthread_mutex_unlock(&svc->by_mog_devid_lock));

	/* we need to get the notify thread to create new worker threads */
	if (need_refresh)
		mog_notify(MOG_NOTIFY_DEVICE_REFRESH);

	return ret;
}

static int
emit_usage(
const struct mog_dev *dev, struct mog_svc *svc, int fd, struct statvfs *v)
{
	int rc = -1;
	unsigned long long available = v->f_bavail;
	unsigned long long total = v->f_blocks - (v->f_bfree - v->f_bavail);
	unsigned long long used = v->f_blocks - v->f_bfree;
	unsigned use = (used * 100) / total + !!((used * 100) % total);
	long long now = (long long)time(NULL);
	long double mb = v->f_frsize / (long double)1024.0;
	const struct mount_entry *me;

	available *= mb;
	total *= mb;
	used *= mb;

	if (use > 100)
		use = 100;

	me = mog_mnt_acquire(dev->st_dev);
	if (me) {
		static const char usage_fmt[] =
			"available: %llu\n"
			"device: %s\n"
			"disk: %s/dev%u\n"
			"time: %lld\n"
			"total: %llu\n"
			"use: %u%%\n"
			"used: %llu\n";

		errno = 0;
		rc = dprintf(fd, usage_fmt,
			     available, me->me_devname, svc->docroot,
			     (unsigned)dev->devid, now, total, use, used);

		PRESERVE_ERRNO( mog_mnt_release(me) );
		if (rc < 0 || errno == ENOSPC) {
			PRESERVE_ERRNO(do {
				syslog(LOG_ERR, "dprintf(%s/dev%u/usage): %m",
				       svc->docroot, (unsigned)dev->devid);
			} while (0));
		}
	} else {
		syslog(LOG_ERR, "mount entry not found for %s/dev%u",
		       svc->docroot, (unsigned)dev->devid);
		errno = ENODEV;
	}

	return rc;
}

int mog_dev_mkusage(const struct mog_dev *dev, struct mog_svc *svc)
{
	struct statvfs v;
	char *usage_path;
	char *tmp_path;
	int fd = -1;

	if (!svc->mgmt_mfd)
		return 0;

	usage_path = xasprintf("/dev%u/usage", (unsigned)dev->devid);
	tmp_path = xasprintf("%s.%x", usage_path, (unsigned)getpid());

	if (mog_unlink(svc, tmp_path) < 0 && errno != ENOENT) goto out;

	errno = 0;
	fd = mog_open_put(svc, tmp_path, O_EXCL|O_CREAT);
	if (fd < 0) {
		if (mog_open_expire_retry(svc))
			fd = mog_open_put(svc, tmp_path, O_EXCL|O_CREAT);
	}
	if (fd < 0) {
		PRESERVE_ERRNO(do {
			syslog(LOG_ERR, "open(%s%s): %m",
			       svc->docroot, tmp_path);
		} while (0));
		goto out;
	}
	if (fstatvfs(fd, &v) < 0) {
		PRESERVE_ERRNO(do {
			syslog(LOG_ERR, "fstatvfs(%s%s): %m",
			       svc->docroot, tmp_path);
		} while (0));
		goto out;
	}
	if (emit_usage(dev, svc, fd, &v) < 0) goto out;
	if (fchmod(fd, svc->put_perms) < 0) {
		PRESERVE_ERRNO(do {
			syslog(LOG_ERR, "fchmod(%s%s): %m",
			       svc->docroot, tmp_path);
		} while (0));
		goto out;
	}

	/* skip rename on EIO if close() fails */
	if (close(fd) != 0) {
		assert(errno != EBADF && "attempted to close bad FD");
		fd = -1;
		if (errno != EINTR) {
			/* possible EIO */
			PRESERVE_ERRNO(do {
				syslog(LOG_ERR, "close(%s%s) failed: %m",
			               svc->docroot, tmp_path);
			} while (0));
			goto out;
		}
	}

	fd = -1;
	errno = 0;
	if (mog_rename(svc, tmp_path, usage_path) != 0) {
		PRESERVE_ERRNO(do {
			syslog(LOG_ERR, "rename(%s(%s => %s))",
			       svc->docroot, tmp_path, usage_path);
		} while (0));
	}
out:
	PRESERVE_ERRNO(do {
		if (errno)
			(void)mog_unlink(svc, tmp_path);
		if (fd >= 0)
			(void)mog_close(fd);
		free(tmp_path);
		free(usage_path);
	} while (0));
	return errno ? -1 : 0;
}

void mog_dev_free(struct mog_dev *dev)
{
	mog_ioq_destroy(&dev->fsckq);
	mog_ioq_destroy(&dev->ioq);
	free(dev);
}

void mog_dev_requeue_prepare(struct mog_dev *dev)
{
	mog_ioq_requeue_prepare(&dev->ioq);
	mog_ioq_requeue_prepare(&dev->fsckq);
}

git clone https://yhbt.net/cmogstored.git