Skip to content

Commit

Permalink
[Doc] Make doc code snippet testable [10/n] (ray-project#37636)
Browse files Browse the repository at this point in the history 
Remove # doctest: +SKIP

Signed-off-by: Balaji Veeramani <balaji@anyscale.com>
Signed-off-by: Jiajun Yao <jeromeyjj@gmail.com>
Co-authored-by: Balaji Veeramani <bveeramani@berkeley.edu>
Co-authored-by: Balaji Veeramani <balaji@anyscale.com>
Signed-off-by: e428265 <arvind.chandramouli@lmco.com>
  • Loading branch information
@jjyao @bveeramani @arvind-chandra
3 people authored and arvind-chandra committed Aug 31, 2023
1 parent 2d4f82f commit a9f8424
Showing 1 changed file with 119 additions and 47 deletions.
166 changes: 119 additions & 47 deletions python/ray/_private/storage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,22 @@ def get_filesystem() -> ("pyarrow.fs.FileSystem", str):
storage filesystem.
Examples:
# Assume ray.init(storage="s3:/bucket/cluster_1/storage")
>>> fs, path = storage.get_filesystem() # doctest: +SKIP
>>> print(fs) # doctest: +SKIP
<pyarrow._fs.LocalFileSystem object at 0x7fd745dd9830>
>>> print(path) # doctest: +SKIP
cluster_1/storage
.. testcode::
import ray
from ray._private import storage
ray.shutdown()
ray.init(storage="/tmp/storage/cluster_1/storage")
fs, path = storage.get_filesystem()
print(fs)
print(path)
.. testoutput::
<pyarrow._fs.LocalFileSystem object at ...>
/tmp/storage/cluster_1/storage
Returns:
Tuple of pyarrow filesystem instance and the path under which files should
Expand All @@ -63,9 +73,16 @@ def get_client(prefix: str) -> "KVClient":
data will be stored under. All writes will be scoped to this sub-dir.
Examples:
# Assume ray.init(storage="s3:/bucket/cluster_1/storage")
>>> client = storage.get_client("foo") # doctest: +SKIP
>>> client.put("foo", b"bar") # doctest: +SKIP
.. testcode::
import ray
from ray._private import storage
ray.shutdown()
ray.init(storage="/tmp/storage/cluster_1/storage")
client = storage.get_client("foo")
client.put("foo", b"bar")
Returns:
KVClient.
Expand Down Expand Up @@ -107,9 +124,16 @@ def put(self, path: str, value: bytes) -> None:
"""Save a blob in persistent storage at the given path, if possible.
Examples:
# Writes "bar" to <storage_prefix>/my_app/path/foo.txt
>>> client = storage.get_client("my_app") # doctest: +SKIP
>>> client.put("path/foo.txt", b"bar") # doctest: +SKIP
.. testcode::
import ray
from ray._private import storage
ray.shutdown()
ray.init(storage="/tmp/storage/cluster_1/storage")
client = storage.get_client("my_app")
client.put("path/foo.txt", b"bar")
Args:
path: Relative directory of the blobs.
Expand All @@ -130,12 +154,19 @@ def get(self, path: str) -> bytes:
"""Load a blob from persistent storage at the given path, if possible.
Examples:
# Loads value from <storage_prefix>/my_app/path/foo.txt
>>> client = storage.get_client("my_app") # doctest: +SKIP
>>> client.get("path/foo.txt") # doctest: +SKIP
b"bar"
>>> client.get("invalid") # doctest: +SKIP
None
.. testcode::
import ray
from ray._private import storage
ray.shutdown()
ray.init(storage="/tmp/storage/cluster_1/storage")
client = storage.get_client("my_app")
client.put("path/foo.txt", b"bar")
assert client.get("path/foo.txt") == b"bar"
assert client.get("invalid") is None
Args:
path: Relative directory of the blobs.
Expand All @@ -158,10 +189,18 @@ def delete(self, path: str) -> bool:
"""Load the blob from persistent storage at the given path, if possible.
Examples:
# Deletes blob at <storage_prefix>/my_app/path/foo.txt
>>> client = storage.get_client("my_app") # doctest: +SKIP
>>> client.delete("path/foo.txt") # doctest: +SKIP
True
.. testcode::
import ray
from ray._private import storage
ray.shutdown()
ray.init(storage="/tmp/storage/cluster_1/storage")
client = storage.get_client("my_app")
client.put("path/foo.txt", b"bar")
assert client.delete("path/foo.txt")
Args:
path: Relative directory of the blob.
Expand All @@ -184,10 +223,18 @@ def delete_dir(self, path: str) -> bool:
"""Delete a directory and its contents, recursively.
Examples:
# Deletes dir at <storage_prefix>/my_app/path/
>>> client = storage.get_client("my_app") # doctest: +SKIP
>>> client.delete_dir("path") # doctest: +SKIP
True
.. testcode::
import ray
from ray._private import storage
ray.shutdown()
ray.init(storage="/tmp/storage/cluster_1/storage")
client = storage.get_client("my_app")
client.put("path/foo.txt", b"bar")
assert client.delete_dir("path")
Args:
path: Relative directory of the blob.
Expand All @@ -210,15 +257,26 @@ def get_info(self, path: str) -> Optional["pyarrow.fs.FileInfo"]:
"""Get info about the persistent blob at the given path, if possible.
Examples:
# Inspect blob at <storage_prefix>/my_app/path/foo.txt
>>> client = storage.get_client("my_app") # doctest: +SKIP
>>> client.get_info("path/foo.txt") # doctest: +SKIP
<FileInfo for '/tmp/storage/my_app/path/foo.txt': type=FileType.File>
.. testcode::
import ray
from ray._private import storage
ray.shutdown()
ray.init(storage="/tmp/storage/cluster_1/storage")
client = storage.get_client("my_app")
client.put("path/foo.txt", b"bar")
print(client.get_info("path/foo.txt"))
# Non-existent blob.
>>> client.get_info("path/does_not_exist.txt") # doctest: +SKIP
None
<FileInfo for '/tmp/storage/my_app/path/foo.txt': type=FileType.NotFound>
print(client.get_info("path/does_not_exist.txt"))
.. testoutput::
<FileInfo for '.../my_app/path/foo.txt': type=FileType.File, size=3>
None
Args:
path: Relative directory of the blob.
Expand All @@ -241,19 +299,33 @@ def list(
"""List blobs and sub-dirs in the given path, if possible.
Examples:
# List created blobs and dirs at <storage_prefix>/my_app/path
>>> client = storage.get_client("my_app") # doctest: +SKIP
>>> client.list("path") # doctest: +SKIP
[<FileInfo for '/tmp/storage/my_app/path/foo.txt' type=FileType.File>,
<FileInfo for '/tmp/storage/my_app/path/subdir' type=FileType.Directory>]
# Non-existent path.
>>> client.get_info("does_not_exist") # doctest: +SKIP
FileNotFoundError: ...
# Not a directory.
>>> storage.get_info("path/foo.txt") # doctest: +SKIP
NotADirectoryError: ...
>>> import ray
>>> from ray._private import storage
>>> ray.shutdown()
Normal usage.
>>> ray.init(storage="/tmp/storage/cluster_1/storage")
RayContext(...)
>>> client = storage.get_client("my_app")
>>> client.put("path/foo.txt", b"bar")
>>> client.list("path")
[<FileInfo for '.../my_app/path/foo.txt': type=FileType.File, size=3>]
Non-existent path.
>>> client.list("does_not_exist")
Traceback (most recent call last):
...
FileNotFoundError: ... No such file or directory
Not a directory.
>>> client.list("path/foo.txt")
Traceback (most recent call last):
...
NotADirectoryError: ... Not a directory
Args:
path: Relative directory to list from.
Expand Down

0 comments on commit a9f8424

Please sign in to comment.