diff options
author | Eric Wong <normalperson@yhbt.net> | 2012-10-08 22:44:03 +0000 |
---|---|---|
committer | Eric Wong <normalperson@yhbt.net> | 2012-10-09 00:17:16 +0000 |
commit | 436f57af99152d62c8c56fda394cbb2159747fe9 (patch) | |
tree | 3f01ff22e6b4d9c6ca52ec9cd9bb275bb087f1cd /lib/mogilefs | |
parent | ef987fde476bd9d36a6514acd50cf94fc8081063 (diff) | |
download | mogilefs-client-436f57af99152d62c8c56fda394cbb2159747fe9.tar.gz |
This allows fast listing of keys and metadata (length, checksum, devcount, class).
Diffstat (limited to 'lib/mogilefs')
-rw-r--r-- | lib/mogilefs/mogilefs.rb | 51 |
1 files changed, 51 insertions, 0 deletions
diff --git a/lib/mogilefs/mogilefs.rb b/lib/mogilefs/mogilefs.rb index eccca78..426217e 100644 --- a/lib/mogilefs/mogilefs.rb +++ b/lib/mogilefs/mogilefs.rb @@ -88,6 +88,57 @@ class MogileFS::MogileFS < MogileFS::Client nil end + # Enumerates keys and yields a +file_info+ hash for each key matched by + # +prefix+ + def each_file_info(prefix = "", args = nil) + # FIXME: there's a lot of duplicate code from list_keys_verbose here... + raise ArgumentError, "need block" unless block_given? + ordered = ready = nil + on_file_info = lambda do |info| + Hash === info or raise info + file_info_cleanup(info) + + # deal with trackers with multiple queryworkers responding out-of-order + ready[info["key"]] = info + while info = ready.delete(ordered[-1]) + ordered.pop + yield info + end + end + + nr = 0 + opts = { :domain => @domain } + opts[:devices] = 1 if args && args[:devices] + after = args ? args[:after] : nil + limit = args ? args[:limit] : nil + + begin + keys, after = list_keys(prefix, after, limit || 1000) + return nr unless keys && keys[0] + ordered = keys.reverse + ready = {} + nr += keys.size + limit -= keys.size if limit + + keys.each do |key| + opts[:key] = key + @backend.pipeline_dispatch(:file_info, opts, &on_file_info) + end + @backend.pipeline_wait + rescue MogileFS::PipelineError, SystemCallError, + MogileFS::RequestTruncatedError, + MogileFS::UnreadableSocketError, + MogileFS::InvalidResponseError, # truncated response + MogileFS::Timeout + @backend.shutdown + keys = ordered - ready.keys + retry + end while limit == nil || limit > 0 + rescue + @backend.shutdown + raise + end + # Retrieves the contents of +key+. If +dst+ is specified, +dst+ # should be an IO-like object capable of receiving the +write+ method # or a path name. +copy_length+ may be specified to limit the number of |