diff --git a/gcsfs/cli/gcsfuse.py b/gcsfs/cli/gcsfuse.py index 0b05d07f..06f69f49 100644 --- a/gcsfs/cli/gcsfuse.py +++ b/gcsfs/cli/gcsfuse.py @@ -14,10 +14,16 @@ help="Billing Project ID") @click.option('--foreground/--background', default=True, help="Run in the foreground or as a background process") +@click.option('--threads/--no-threads', default=True, + help="Whether to run with threads") +@click.option('--cache_files', type=int, default=10, + help="Number of open files to cache") @click.option('-v', '--verbose', count=True, help="Set logging level. '-v' for 'gcsfuse' logging." "'-v -v' for complete debug logging.") -def main(bucket, mount_point, token, project_id, foreground, verbose): +def main(bucket, mount_point, token, project_id, foreground, threads, + cache_files, verbose): + """ Mount a Google Cloud Storage (GCS) bucket to a local directory """ if verbose == 1: logging.basicConfig(level=logging.INFO) @@ -25,10 +31,17 @@ def main(bucket, mount_point, token, project_id, foreground, verbose): if verbose > 1: logging.basicConfig(level=logging.DEBUG) - """ Mount a Google Cloud Storage (GCS) bucket to a local directory """ + fmt = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' + if verbose == 1: + logging.basicConfig(level=logging.INFO, format=fmt) + logging.getLogger("gcsfs.gcsfuse").setLevel(logging.DEBUG) + if verbose > 1: + logging.basicConfig(level=logging.DEBUG, format=fmt) + print("Mounting bucket %s to directory %s" % (bucket, mount_point)) - FUSE(GCSFS(bucket, token=token, project=project_id), - mount_point, nothreads=True, foreground=foreground) + print('foreground:', foreground, ', nothreads:', not threads) + FUSE(GCSFS(bucket, token=token, project=project_id, nfiles=cache_files), + mount_point, nothreads=not threads, foreground=foreground) if __name__ == '__main__': diff --git a/gcsfs/core.py b/gcsfs/core.py index bfa27b46..c242036e 100644 --- a/gcsfs/core.py +++ b/gcsfs/core.py @@ -30,9 +30,7 @@ import warnings from requests.exceptions import RequestException -from .utils import HtmlError -from .utils import is_retriable -from .utils import read_block +from .utils import HtmlError, is_retriable, read_block PY2 = sys.version_info.major == 2 @@ -264,7 +262,8 @@ class GCSFileSystem(object): default_block_size = DEFAULT_BLOCK_SIZE def __init__(self, project=DEFAULT_PROJECT, access='full_control', - token=None, block_size=None, consistency='none', cache_timeout = None): + token=None, block_size=None, consistency='none', + cache_timeout=None): if access not in self.scopes: raise ValueError('access must be one of {}', self.scopes) if project is None: @@ -279,7 +278,6 @@ def __init__(self, project=DEFAULT_PROJECT, access='full_control', self.session = None self.connect(method=token) - self._singleton[0] = self self.cache_timeout = cache_timeout @@ -417,9 +415,8 @@ def _save_tokens(): except Exception as e: warnings.warn('Saving token cache failed: ' + str(e)) + @_tracemethod def _call(self, method, path, *args, **kwargs): - logger.debug("_call(%s, %s, args=%s, kwargs=%s)", method, path, args, kwargs) - for k, v in list(kwargs.items()): # only pass parameters that have values if v is None: @@ -453,7 +450,6 @@ def buckets(self): """Return list of available project buckets.""" return [b["name"] for b in self._list_buckets()["items"]] - @classmethod def _process_object(self, bucket, object_metadata): """Process object resource into gcsfs object information format. @@ -492,12 +488,11 @@ def _get_object(self, path): # listing. raise FileNotFoundError(path) - result = self._process_object(bucket, self._call('get', 'b/{}/o/{}', bucket, key)) + result = self._process_object(bucket, self._call('get', 'b/{}/o/{}', + bucket, key)) - logger.debug("_get_object result: %s", result) return result - @_tracemethod def _maybe_get_cached_listing(self, path): logger.debug("_maybe_get_cached_listing: %s", path) @@ -506,7 +501,8 @@ def _maybe_get_cached_listing(self, path): cache_age = time.time() - retrieved_time if self.cache_timeout is not None and cache_age > self.cache_timeout: logger.debug( - "expired cache path: %s retrieved_time: %.3f cache_age: %.3f cache_timeout: %.3f", + "expired cache path: %s retrieved_time: %.3f cache_age: " + "%.3f cache_timeout: %.3f", path, retrieved_time, cache_age, self.cache_timeout ) del self._listing_cache[path] @@ -532,7 +528,7 @@ def _list_objects(self, path): @_tracemethod def _do_list_objects(self, path, max_results = None): - """Return depaginated object listing for the given {bucket}/{prefix}/ path.""" + """Object listing for the given {bucket}/{prefix}/ path.""" bucket, prefix = split_path(path) if not prefix: prefix = None @@ -540,7 +536,8 @@ def _do_list_objects(self, path, max_results = None): prefixes = [] items = [] page = self._call( - 'get', 'b/{}/o/', bucket, delimiter="/", prefix=prefix, maxResults=max_results) + 'get', 'b/{}/o/', bucket, delimiter="/", prefix=prefix, + maxResults=max_results) assert page["kind"] == "storage#objects" prefixes.extend(page.get("prefixes", [])) @@ -549,8 +546,8 @@ def _do_list_objects(self, path, max_results = None): while next_page_token is not None: page = self._call( - 'get', 'b/{}/o/', bucket, delimiter="/", prefix=prefix, maxResults=max_results, - pageToken=next_page_token) + 'get', 'b/{}/o/', bucket, delimiter="/", prefix=prefix, + maxResults=max_results, pageToken=next_page_token) assert page["kind"] == "storage#objects" prefixes.extend(page.get("prefixes", [])) @@ -558,20 +555,16 @@ def _do_list_objects(self, path, max_results = None): next_page_token = page.get('nextPageToken', None) result = { - "kind" : "storage#objects", - "prefixes" : prefixes, - "items" : [self._process_object(bucket, i) for i in items], + "kind": "storage#objects", + "prefixes": prefixes, + "items": [self._process_object(bucket, i) for i in items], } - logger.debug("_list_objects result: %s", {k : len(result[k]) for k in ("prefixes", "items")}) - return result + @_tracemethod def _list_buckets(self): """Return list of all buckets under the current project.""" - - logger.debug("_list_buckets") - items = [] page = self._call( 'get', 'b/', project=self.project @@ -590,23 +583,22 @@ def _list_buckets(self): next_page_token = page.get('nextPageToken', None) result = { - "kind" : "storage#buckets", - "items" : items, + "kind": "storage#buckets", + "items": items, } - logger.debug("_list_buckets result: %s", {k : len(result[k]) for k in ("items",)}) - return result @_tracemethod def invalidate_cache(self, path=None): """ - Invalidate listing cache for given path, so that it is reloaded on next use. + Invalidate listing cache for given path, it is reloaded on next use. Parameters ---------- path: string or None - If None, clear all listings cached else listings at or under given path. + If None, clear all listings cached else listings at or under given + path. """ if not path: @@ -614,10 +606,9 @@ def invalidate_cache(self, path=None): self._listing_cache.clear() else: path = norm_path(path) - logger.debug("invalidate_cache prefix: %s", path) - invalid_keys = [k for k in self._listing_cache if k.startswith(path)] - logger.debug("invalidate_cache keys: %s", invalid_keys) + invalid_keys = [k for k in self._listing_cache + if k.startswith(path)] for k in invalid_keys: self._listing_cache.pop(k, None) @@ -658,31 +649,29 @@ def ls(self, path, detail=False): elif path.endswith("/"): return self._ls(path, detail) else: - combined_listing = self._ls(path, detail) + self._ls(path + "/", detail) + combined_listing = self._ls(path, detail) + self._ls(path + "/", + detail) if detail: - combined_entries = dict((l["path"],l) for l in combined_listing ) - combined_entries.pop(path+"/", None) + combined_entries = dict( + (l["path"], l) for l in combined_listing) + combined_entries.pop(path + "/", None) return list(combined_entries.values()) else: return list(set(combined_listing) - {path + "/"}) + @_tracemethod def _ls(self, path, detail=False): listing = self._list_objects(path) bucket, key = split_path(path) if not detail: - result = [] # Convert item listing into list of 'item' and 'subdir/' # entries. Items may be of form "key/", in which case there # will be duplicate entries in prefix and item_names. - item_names = [ - f["name"] for f in listing["items"] if f["name"] - ] + item_names = [f["name"] for f in listing["items"] if f["name"]] prefixes = [p for p in listing["prefixes"]] - logger.debug("path: %s item_names: %s prefixes: %s", path, item_names, prefixes) - return [ posixpath.join(bucket, n) for n in set(item_names + prefixes) ] @@ -712,7 +701,6 @@ def walk(self, path, detail=False): raise ValueError("path must include at least target bucket") if path.endswith('/'): - results = [] listing = self.ls(path, detail=True) files = [l for l in listing if l["storageClass"] != "DIRECTORY"] @@ -799,6 +787,8 @@ def info(self, path): bucket, key = split_path(path) if not key: # Return a pseudo dir for the bucket root + # TODO: check that it exists (either is in bucket list, + # or can list it) return { 'bucket': bucket, 'name': "/", @@ -933,6 +923,7 @@ def touch(self, path): with self.open(path, 'wb'): pass + @_tracemethod def read_block(self, fn, offset, length, delimiter=None): """ Read a block of bytes from a GCS file @@ -1283,8 +1274,8 @@ def _fetch(self, start, end): # First read self.start = start self.end = end + self.blocksize - self.cache = _fetch_range(self.details, self.gcsfs.session, start, - self.end) + self.cache = _fetch_range(self.details, self.gcsfs.session, + self.start, self.end) if start < self.start: if self.end - end > self.blocksize: self.start = start @@ -1292,8 +1283,8 @@ def _fetch(self, start, end): self.cache = _fetch_range(self.details, self.gcsfs.session, self.start, self.end) else: - new = _fetch_range(self.details, self.gcsfs.session, start, - self.start) + new = _fetch_range(self.details, self.gcsfs.session, + start, self.start) self.start = start self.cache = new + self.cache if end > self.end: @@ -1384,6 +1375,7 @@ def __exit__(self, *args): self.close() +@_tracemethod def _fetch_range(obj_dict, session, start=None, end=None): """ Get data from GCS @@ -1392,7 +1384,6 @@ def _fetch_range(obj_dict, session, start=None, end=None): start, end : None or integers if not both None, fetch only given range """ - logger.debug("Fetch: %s, %i-%i", obj_dict['name'], start, end) if start is not None or end is not None: start = start or 0 end = end or 0 diff --git a/gcsfs/gcsfuse.py b/gcsfs/gcsfuse.py index 8f1ee53f..49c09c49 100644 --- a/gcsfs/gcsfuse.py +++ b/gcsfs/gcsfuse.py @@ -1,4 +1,5 @@ from __future__ import print_function +from collections import OrderedDict, MutableMapping import os import logging import decorator @@ -9,27 +10,151 @@ from gcsfs import GCSFileSystem, core from pwd import getpwnam from grp import getgrnam +import time +from threading import Lock + + +@decorator.decorator +def _tracemethod(f, self, *args, **kwargs): + logger.debug("%s(args=%s, kwargs=%s)", f.__name__, args, kwargs) + out = f(self, *args, **kwargs) + return out + logger = logging.getLogger(__name__) + @decorator.decorator def _tracemethod(f, self, *args, **kwargs): - logger.debug("%s(args=%s, kwargs=%s)", f.__name__, args, kwargs) - return f(self, *args, **kwargs) + logger.debug("%s(args=%s, kwargs=%s)", f.__name__, args, kwargs) + return f(self, *args, **kwargs) + def str_to_time(s): t = pd.to_datetime(s) return t.to_datetime64().view('int64') / 1e9 +class LRUDict(MutableMapping): + """A dict that discards least-recently-used items""" + + DEFAULT_SIZE = 128 + + def __init__(self, *args, **kwargs): + """Same arguments as OrderedDict with one additions: + + size: maximum number of entries + """ + self.size = kwargs.pop('size', self.DEFAULT_SIZE) + self.data = OrderedDict(*args, **kwargs) + self.purge() + + def purge(self): + """Removes expired or overflowing entries.""" + # pop until maximum capacity is reached + extra = max(0, len(self.data) - self.size) + for _ in range(extra): + self.data.popitem(last=False) + + def __getitem__(self, key): + if key not in self.data: + raise KeyError(key) + self.data.move_to_end(key) + return self.data[key] + + def __setitem__(self, key, value): + self.data[key] = value + self.data.move_to_end(key) + self.purge() + + def __delitem__(self, key): + del self.data[key] + + def __iter__(self): + return iter(list(self.data)) + + def __len__(self): + return len(self.data) + + +class SmallChunkCacher: + """ + Cache open GCSFiles, and data chunks from small reads + + Parameters + ---------- + gcs : instance of GCSFileSystem + cutoff : int + Will store/fetch data from cache for calls to read() with values smaller + than this. + nfile : int + Number of files to store in LRU cache. + """ + + def __init__(self, gcs, cutoff=10000, nfiles=3): + self.gcs = gcs + self.cache = LRUDict(size=nfiles) + self.cutoff = cutoff + self.nfiles = nfiles + + def read(self, fn, offset, size): + """Reach block from file + + If size is less than cutoff, see if the relevant data is in the cache; + either return data from there, or call read() on underlying file object + and store the resultant block in the cache. + """ + if fn not in self.cache: + self.open(fn) + f, chunks = self.cache[fn] + for chunk in chunks: + if chunk['start'] < offset and chunk['end'] > offset + size: + logger.info('cache hit') + start = offset - chunk['start'] + return chunk['data'][start:start + size] + if size > self.cutoff: + # big reads are likely sequential + with f.lock: + f.seek(offset) + return f.read(size) + logger.info('cache miss') + with f.lock: + bs = f.blocksize + f.blocksize = 2 * 2 ** 20 + f.seek(offset) + out = f.read(size) + chunks.append({'start': f.start, 'end': f.end, 'data': f.cache}) + f.blocksize = bs + + return out + + def open(self, fn): + """Create cache entry, or return existing open file + + May result in the eviction of LRU file object and its data blocks. + """ + if fn not in self.cache: + f = self.gcs.open(fn, 'rb') + chunk = f.read(5 * 2**20) + self.cache[fn] = f, [{'start': 0, 'end': 5 * 2**20, 'data': chunk}] + f.lock = Lock() + logger.info('{} inserted into cache'.format(fn)) + else: + logger.info('{} found in cache'.format(fn)) + return self.cache[fn][0] + + class GCSFS(Operations): - def __init__(self, path='.', gcs=None, **fsargs): + def __init__(self, path='.', gcs=None, nfiles=10, **fsargs): if gcs is None: - self.gcs = GCSFileSystem(**fsargs) + # minimum block size: still read on 5MB boundaries. + self.gcs = GCSFileSystem(block_size=30 * 2 ** 20, + cache_timeout=6000, **fsargs) else: self.gcs = gcs - self.cache = {} + self.cache = SmallChunkCacher(self.gcs, nfiles=nfiles) + self.write_cache = {} self.counter = 0 self.root = path @@ -57,12 +182,12 @@ def getattr(self, path, fh=None): data['st_size'] = info['size'] data['st_blksize'] = 5 * 2**20 data['st_nlink'] = 1 - return data @_tracemethod def readdir(self, path, fh): path = ''.join([self.root, path]) + logger.info("List {}, {}".format(path, fh)) files = self.gcs.ls(path) files = [os.path.basename(f.rstrip('/')) for f in files] return ['.', '..'] + files @@ -85,43 +210,50 @@ def rmdir(self, path): @_tracemethod def read(self, path, size, offset, fh): fn = ''.join([self.root, path]) - f = self.cache[fh] - f.seek(offset) - out = f.read(size) + logger.info('read #{} ({}) offset: {}, size: {}'.format( + fh, fn, offset, size)) + out = self.cache.read(fn, offset, size) return out @_tracemethod def write(self, path, data, offset, fh): - f = self.cache[fh] + fn = ''.join([self.root, path]) + logger.info('write #{} ({}) offset'.format(fh, fn, offset)) + f = self.write_cache[fh] f.write(data) return len(data) @_tracemethod def create(self, path, flags): fn = ''.join([self.root, path]) + logger.info('create {} {}'.format(fn, oct(flags))) self.gcs.touch(fn) # this makes sure directory entry exists - wasteful! # write (but ignore creation flags) f = self.gcs.open(fn, 'wb') - self.cache[self.counter] = f + self.write_cache[self.counter] = f + logger.info('-> fh #{}'.format(self.counter)) self.counter += 1 return self.counter - 1 @_tracemethod def open(self, path, flags): fn = ''.join([self.root, path]) + logger.info('open {} {}'.format(fn, oct(flags))) if flags % 2 == 0: # read - f = self.gcs.open(fn, 'rb') + self.cache.open(fn) else: # write (but ignore creation flags) - f = self.gcs.open(fn, 'wb') - self.cache[self.counter] = f + self.gcs.open(fn, 'wb') + self.write_cache[self.counter] = f + logger.info('-> fh #{}'.format(self.counter)) self.counter += 1 return self.counter - 1 @_tracemethod def truncate(self, path, length, fh=None): fn = ''.join([self.root, path]) + logger.info('truncate #{} ({}) to {}'.format(fh, fn, length)) if length != 0: raise NotImplementedError # maybe should be no-op since open with write sets size to zero anyway @@ -130,6 +262,7 @@ def truncate(self, path, length, fh=None): @_tracemethod def unlink(self, path): fn = ''.join([self.root, path]) + logger.info('delete', fn) try: self.gcs.rm(fn, False) except (IOError, FileNotFoundError): @@ -137,12 +270,16 @@ def unlink(self, path): @_tracemethod def release(self, path, fh): + fn = ''.join([self.root, path]) + logger.info('close #{} ({})'.format(fh, fn)) try: - f = self.cache[fh] - f.close() - self.cache.pop(fh, None) # should release any cache memory + if fh in self.write_cache: + # write mode + f = self.write_cache[fh] + f.close() + self.write_cache.pop(fh, None) except Exception as e: - logger.exception("exception on release") + logger.exception("exception on release:" + str(e)) return 0 @_tracemethod diff --git a/gcsfs/tests/recordings/test_fuse.yaml b/gcsfs/tests/recordings/test_fuse.yaml new file mode 100644 index 00000000..0fea6afe --- /dev/null +++ b/gcsfs/tests/recordings/test_fuse.yaml @@ -0,0 +1,483 @@ +interactions: +- request: + body: client_secret=xxx&refresh_token=xxx&grant_type=refresh_token&client_id=xxx + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + Content-Length: ['208'] + content-type: [application/x-www-form-urlencoded] + method: POST + uri: https://www.googleapis.com/oauth2/v4/token + response: + body: + string: !!binary | + H4sIANYVj1oC/6tWSkxOTi0uji/Jz07NU7JSUKqoqFDSUVBKrSjILEotjs8ECRqbGRgAxTJTMJSB + +fEllQWpIEGn1MSi1CKlWgA253KRVgAAAA== + headers: + Cache-Control: ['no-cache, no-store, max-age=0, must-revalidate'] + Content-Encoding: [gzip] + Content-Type: [application/json; charset=UTF-8] + Pragma: [no-cache] + Server: [GSE] + Transfer-Encoding: [chunked] + Vary: [Origin, X-Origin] + X-Content-Type-Options: [nosniff] + X-Frame-Options: [SAMEORIGIN] + X-XSS-Protection: [1; mode=block] + status: {code: 200, message: OK} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + method: GET + uri: https://www.googleapis.com/storage/v1/b/?project=test_project + response: + body: {string: "{\n \"kind\": \"storage#buckets\",\n \"items\": [\n {\n \"kind\": + \"storage#bucket\",\n \"id\": \"anaconda-enterprise\",\n \"selfLink\": + \"https://www.googleapis.com/storage/v1/b/anaconda-enterprise\",\n \"projectNumber\": + \"586241054156\",\n \"name\": \"anaconda-enterprise\",\n \"timeCreated\": + \"2017-07-05T23:53:06.552Z\",\n \"updated\": \"2017-07-14T17:39:54.178Z\",\n + \ \"metageneration\": \"3\",\n \"location\": \"US\",\n \"storageClass\": + \"MULTI_REGIONAL\",\n \"etag\": \"CAM=\"\n },\n {\n \"kind\": \"storage#bucket\",\n + \ \"id\": \"anaconda-public-data\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/anaconda-public-data\",\n + \ \"projectNumber\": \"586241054156\",\n \"name\": \"anaconda-public-data\",\n + \ \"timeCreated\": \"2017-04-05T20:22:12.865Z\",\n \"updated\": \"2017-07-10T16:32:07.980Z\",\n + \ \"metageneration\": \"2\",\n \"location\": \"US\",\n \"storageClass\": + \"MULTI_REGIONAL\",\n \"etag\": \"CAI=\"\n },\n {\n \"kind\": \"storage#bucket\",\n + \ \"id\": \"artifacts.test_project.appspot.com\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/artifacts.test_project.appspot.com\",\n + \ \"projectNumber\": \"586241054156\",\n \"name\": \"artifacts.test_project.appspot.com\",\n + \ \"timeCreated\": \"2016-05-17T18:29:22.774Z\",\n \"updated\": \"2016-05-17T18:29:22.774Z\",\n + \ \"metageneration\": \"1\",\n \"location\": \"US\",\n \"storageClass\": + \"STANDARD\",\n \"etag\": \"CAE=\"\n },\n {\n \"kind\": \"storage#bucket\",\n + \ \"id\": \"test_project_cloudbuild\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/test_project_cloudbuild\",\n + \ \"projectNumber\": \"586241054156\",\n \"name\": \"test_project_cloudbuild\",\n + \ \"timeCreated\": \"2017-11-03T20:06:49.744Z\",\n \"updated\": \"2017-11-03T20:06:49.744Z\",\n + \ \"metageneration\": \"1\",\n \"location\": \"US\",\n \"storageClass\": + \"STANDARD\",\n \"etag\": \"CAE=\"\n },\n {\n \"kind\": \"storage#bucket\",\n + \ \"id\": \"dataflow-anaconda-compute\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/dataflow-anaconda-compute\",\n + \ \"projectNumber\": \"586241054156\",\n \"name\": \"dataflow-anaconda-compute\",\n + \ \"timeCreated\": \"2017-09-14T18:55:42.848Z\",\n \"updated\": \"2017-09-14T18:55:42.848Z\",\n + \ \"metageneration\": \"1\",\n \"location\": \"US\",\n \"storageClass\": + \"MULTI_REGIONAL\",\n \"etag\": \"CAE=\"\n },\n {\n \"kind\": \"storage#bucket\",\n + \ \"id\": \"gcsfs-test\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-test\",\n + \ \"projectNumber\": \"586241054156\",\n \"name\": \"gcsfs-test\",\n \"timeCreated\": + \"2017-12-02T23:25:23.058Z\",\n \"updated\": \"2018-01-04T14:07:08.519Z\",\n + \ \"metageneration\": \"2\",\n \"location\": \"US\",\n \"storageClass\": + \"MULTI_REGIONAL\",\n \"etag\": \"CAI=\"\n },\n {\n \"kind\": \"storage#bucket\",\n + \ \"id\": \"gcsfs-testing\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing\",\n + \ \"projectNumber\": \"586241054156\",\n \"name\": \"gcsfs-testing\",\n + \ \"timeCreated\": \"2017-12-12T16:52:13.675Z\",\n \"updated\": \"2017-12-12T16:52:13.675Z\",\n + \ \"metageneration\": \"1\",\n \"location\": \"US\",\n \"storageClass\": + \"STANDARD\",\n \"etag\": \"CAE=\"\n }\n ]\n}\n"} + headers: + Cache-Control: ['private, max-age=0, must-revalidate, no-transform'] + Content-Length: ['2944'] + Content-Type: [application/json; charset=UTF-8] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 200, message: OK} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + Content-Length: ['0'] + method: DELETE + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fa + response: + body: {string: "{\n \"error\": {\n \"errors\": [\n {\n \"domain\": \"global\",\n + \ \"reason\": \"notFound\",\n \"message\": \"Not Found\"\n }\n ],\n + \ \"code\": 404,\n \"message\": \"Not Found\"\n }\n}\n"} + headers: + Cache-Control: ['private, max-age=0'] + Content-Length: ['165'] + Content-Type: [application/json; charset=UTF-8] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 404, message: Not Found} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + Content-Length: ['0'] + method: DELETE + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fb + response: + body: {string: "{\n \"error\": {\n \"errors\": [\n {\n \"domain\": \"global\",\n + \ \"reason\": \"notFound\",\n \"message\": \"Not Found\"\n }\n ],\n + \ \"code\": 404,\n \"message\": \"Not Found\"\n }\n}\n"} + headers: + Cache-Control: ['private, max-age=0'] + Content-Length: ['165'] + Content-Type: [application/json; charset=UTF-8] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 404, message: Not Found} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + Content-Length: ['0'] + method: DELETE + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fc + response: + body: {string: "{\n \"error\": {\n \"errors\": [\n {\n \"domain\": \"global\",\n + \ \"reason\": \"notFound\",\n \"message\": \"Not Found\"\n }\n ],\n + \ \"code\": 404,\n \"message\": \"Not Found\"\n }\n}\n"} + headers: + Cache-Control: ['private, max-age=0'] + Content-Length: ['165'] + Content-Type: [application/json; charset=UTF-8] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 404, message: Not Found} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + Content-Length: ['0'] + method: DELETE + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fd + response: + body: {string: "{\n \"error\": {\n \"errors\": [\n {\n \"domain\": \"global\",\n + \ \"reason\": \"notFound\",\n \"message\": \"Not Found\"\n }\n ],\n + \ \"code\": 404,\n \"message\": \"Not Found\"\n }\n}\n"} + headers: + Cache-Control: ['private, max-age=0'] + Content-Length: ['165'] + Content-Type: [application/json; charset=UTF-8] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 404, message: Not Found} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + method: GET + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/.DS_Store + response: + body: {string: "{\n \"error\": {\n \"errors\": [\n {\n \"domain\": \"global\",\n + \ \"reason\": \"notFound\",\n \"message\": \"Not Found\"\n }\n ],\n + \ \"code\": 404,\n \"message\": \"Not Found\"\n }\n}\n"} + headers: + Cache-Control: ['private, max-age=0'] + Content-Length: ['165'] + Content-Type: [application/json; charset=UTF-8] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 404, message: Not Found} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + method: GET + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/?delimiter=%2F + response: + body: {string: "{\n \"kind\": \"storage#objects\"\n}\n"} + headers: + Cache-Control: ['private, max-age=0, must-revalidate, no-transform'] + Content-Length: ['31'] + Content-Type: [application/json; charset=UTF-8] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 200, message: OK} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + method: GET + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/.DS_Store + response: + body: {string: "{\n \"error\": {\n \"errors\": [\n {\n \"domain\": \"global\",\n + \ \"reason\": \"notFound\",\n \"message\": \"Not Found\"\n }\n ],\n + \ \"code\": 404,\n \"message\": \"Not Found\"\n }\n}\n"} + headers: + Cache-Control: ['private, max-age=0'] + Content-Length: ['165'] + Content-Type: [application/json; charset=UTF-8] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 404, message: Not Found} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + method: GET + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/hello + response: + body: {string: "{\n \"error\": {\n \"errors\": [\n {\n \"domain\": \"global\",\n + \ \"reason\": \"notFound\",\n \"message\": \"Not Found\"\n }\n ],\n + \ \"code\": 404,\n \"message\": \"Not Found\"\n }\n}\n"} + headers: + Cache-Control: ['private, max-age=0'] + Content-Length: ['165'] + Content-Type: [application/json; charset=UTF-8] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 404, message: Not Found} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + Content-Length: ['0'] + method: POST + uri: https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?name=hello&uploadType=media + response: + body: {string: "{\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/hello/1519326682064763\",\n + \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/hello\",\n + \"name\": \"hello\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": \"1519326682064763\",\n + \"metageneration\": \"1\",\n \"timeCreated\": \"2018-02-22T19:11:22.058Z\",\n + \"updated\": \"2018-02-22T19:11:22.058Z\",\n \"storageClass\": \"STANDARD\",\n + \"timeStorageClassUpdated\": \"2018-02-22T19:11:22.058Z\",\n \"size\": \"0\",\n + \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/hello?generation=1519326682064763&alt=media\",\n + \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CPvOuvmcutkCEAE=\"\n}\n"} + headers: + Cache-Control: ['no-cache, no-store, max-age=0, must-revalidate'] + Content-Length: ['657'] + Content-Type: [application/json; charset=UTF-8] + ETag: [CPvOuvmcutkCEAE=] + Pragma: [no-cache] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 200, message: OK} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + method: GET + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/hello + response: + body: {string: "{\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/hello/1519326682064763\",\n + \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/hello\",\n + \"name\": \"hello\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": \"1519326682064763\",\n + \"metageneration\": \"1\",\n \"timeCreated\": \"2018-02-22T19:11:22.058Z\",\n + \"updated\": \"2018-02-22T19:11:22.058Z\",\n \"storageClass\": \"STANDARD\",\n + \"timeStorageClassUpdated\": \"2018-02-22T19:11:22.058Z\",\n \"size\": \"0\",\n + \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/hello?generation=1519326682064763&alt=media\",\n + \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CPvOuvmcutkCEAE=\"\n}\n"} + headers: + Cache-Control: ['no-cache, no-store, max-age=0, must-revalidate'] + Content-Length: ['657'] + Content-Type: [application/json; charset=UTF-8] + ETag: [CPvOuvmcutkCEAE=] + Pragma: [no-cache] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 200, message: OK} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + method: GET + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/._hello + response: + body: {string: "{\n \"error\": {\n \"errors\": [\n {\n \"domain\": \"global\",\n + \ \"reason\": \"notFound\",\n \"message\": \"Not Found\"\n }\n ],\n + \ \"code\": 404,\n \"message\": \"Not Found\"\n }\n}\n"} + headers: + Cache-Control: ['private, max-age=0'] + Content-Length: ['165'] + Content-Type: [application/json; charset=UTF-8] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 404, message: Not Found} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + method: GET + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/?delimiter=%2F + response: + body: {string: "{\n \"kind\": \"storage#objects\",\n \"items\": [\n {\n \"kind\": + \"storage#object\",\n \"id\": \"gcsfs-testing/hello/1519326682064763\",\n + \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/hello\",\n + \ \"name\": \"hello\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": + \"1519326682064763\",\n \"metageneration\": \"1\",\n \"timeCreated\": + \"2018-02-22T19:11:22.058Z\",\n \"updated\": \"2018-02-22T19:11:22.058Z\",\n + \ \"storageClass\": \"STANDARD\",\n \"timeStorageClassUpdated\": \"2018-02-22T19:11:22.058Z\",\n + \ \"size\": \"0\",\n \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n \"mediaLink\": + \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/hello?generation=1519326682064763&alt=media\",\n + \ \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CPvOuvmcutkCEAE=\"\n }\n ]\n}\n"} + headers: + Cache-Control: ['private, max-age=0, must-revalidate, no-transform'] + Content-Length: ['740'] + Content-Type: [application/json; charset=UTF-8] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 200, message: OK} +- request: + body: hello + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + Content-Length: ['5'] + method: POST + uri: https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?name=hello&uploadType=media + response: + body: {string: "{\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/hello/1519326683041310\",\n + \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/hello\",\n + \"name\": \"hello\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": \"1519326683041310\",\n + \"metageneration\": \"1\",\n \"timeCreated\": \"2018-02-22T19:11:23.032Z\",\n + \"updated\": \"2018-02-22T19:11:23.032Z\",\n \"storageClass\": \"STANDARD\",\n + \"timeStorageClassUpdated\": \"2018-02-22T19:11:23.032Z\",\n \"size\": \"5\",\n + \"md5Hash\": \"XUFAKrxLKna5cZ2REBfFkg==\",\n \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/hello?generation=1519326683041310&alt=media\",\n + \"crc32c\": \"mnG7TA==\",\n \"etag\": \"CJ6c9vmcutkCEAE=\"\n}\n"} + headers: + Cache-Control: ['no-cache, no-store, max-age=0, must-revalidate'] + Content-Length: ['657'] + Content-Type: [application/json; charset=UTF-8] + ETag: [CJ6c9vmcutkCEAE=] + Pragma: [no-cache] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 200, message: OK} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + method: GET + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/hello + response: + body: {string: "{\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/hello/1519326683041310\",\n + \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/hello\",\n + \"name\": \"hello\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": \"1519326683041310\",\n + \"metageneration\": \"1\",\n \"timeCreated\": \"2018-02-22T19:11:23.032Z\",\n + \"updated\": \"2018-02-22T19:11:23.032Z\",\n \"storageClass\": \"STANDARD\",\n + \"timeStorageClassUpdated\": \"2018-02-22T19:11:23.032Z\",\n \"size\": \"5\",\n + \"md5Hash\": \"XUFAKrxLKna5cZ2REBfFkg==\",\n \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/hello?generation=1519326683041310&alt=media\",\n + \"crc32c\": \"mnG7TA==\",\n \"etag\": \"CJ6c9vmcutkCEAE=\"\n}\n"} + headers: + Cache-Control: ['no-cache, no-store, max-age=0, must-revalidate'] + Content-Length: ['657'] + Content-Type: [application/json; charset=UTF-8] + ETag: [CJ6c9vmcutkCEAE=] + Pragma: [no-cache] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 200, message: OK} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + method: GET + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/?delimiter=%2F + response: + body: {string: "{\n \"kind\": \"storage#objects\",\n \"items\": [\n {\n \"kind\": + \"storage#object\",\n \"id\": \"gcsfs-testing/hello/1519326683041310\",\n + \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/hello\",\n + \ \"name\": \"hello\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": + \"1519326683041310\",\n \"metageneration\": \"1\",\n \"timeCreated\": + \"2018-02-22T19:11:23.032Z\",\n \"updated\": \"2018-02-22T19:11:23.032Z\",\n + \ \"storageClass\": \"STANDARD\",\n \"timeStorageClassUpdated\": \"2018-02-22T19:11:23.032Z\",\n + \ \"size\": \"5\",\n \"md5Hash\": \"XUFAKrxLKna5cZ2REBfFkg==\",\n \"mediaLink\": + \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/hello?generation=1519326683041310&alt=media\",\n + \ \"crc32c\": \"mnG7TA==\",\n \"etag\": \"CJ6c9vmcutkCEAE=\"\n }\n ]\n}\n"} + headers: + Cache-Control: ['private, max-age=0, must-revalidate, no-transform'] + Content-Length: ['740'] + Content-Type: [application/json; charset=UTF-8] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 200, message: OK} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + method: GET + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/hello + response: + body: {string: "{\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/hello/1519326683041310\",\n + \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/hello\",\n + \"name\": \"hello\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": \"1519326683041310\",\n + \"metageneration\": \"1\",\n \"timeCreated\": \"2018-02-22T19:11:23.032Z\",\n + \"updated\": \"2018-02-22T19:11:23.032Z\",\n \"storageClass\": \"STANDARD\",\n + \"timeStorageClassUpdated\": \"2018-02-22T19:11:23.032Z\",\n \"size\": \"5\",\n + \"md5Hash\": \"XUFAKrxLKna5cZ2REBfFkg==\",\n \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/hello?generation=1519326683041310&alt=media\",\n + \"crc32c\": \"mnG7TA==\",\n \"etag\": \"CJ6c9vmcutkCEAE=\"\n}\n"} + headers: + Cache-Control: ['no-cache, no-store, max-age=0, must-revalidate'] + Content-Length: ['657'] + Content-Type: [application/json; charset=UTF-8] + ETag: [CJ6c9vmcutkCEAE=] + Pragma: [no-cache] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 200, message: OK} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + Range: [bytes=0-10485759] + method: GET + uri: https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/hello?alt=media&generation=1519326683041310 + response: + body: {string: hello} + headers: + Cache-Control: ['no-cache, no-store, max-age=0, must-revalidate'] + Content-Disposition: [attachment] + Content-Length: ['5'] + Content-Range: [bytes 0-4/5] + Content-Type: [application/octet-stream] + ETag: [CJ6c9vmcutkCEAE=] + Pragma: [no-cache] + Server: [UploadServer] + Vary: [Origin, X-Origin] + X-Goog-Generation: ['1519326683041310'] + X-Goog-Metageneration: ['1'] + X-Goog-Storage-Class: [STANDARD] + status: {code: 206, message: Partial Content} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + Content-Length: ['0'] + method: DELETE + uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/hello + response: + body: {string: ''} + headers: + Cache-Control: ['no-cache, no-store, max-age=0, must-revalidate'] + Content-Length: ['0'] + Content-Type: [application/json] + Pragma: [no-cache] + Server: [UploadServer] + Vary: [Origin, X-Origin] + status: {code: 204, message: No Content} +version: 1