Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Datasets] Use read stage name for naming Data-read tasks on Ray Dashboard #34341

Merged
merged 5 commits into from
Apr 13, 2023
Merged

[Datasets] Use read stage name for naming Data-read tasks on Ray Dashboard #34341

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
aaab1c7
use read stage name from blocklist
Apr 12, 2023
ca91884
unit test
Apr 13, 2023
9b0212d
Merge branch 'master' into task-name-readsource
Apr 13, 2023
5b30657
tests
Apr 13, 2023
07900c4
test names
Apr 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion python/ray/data/_internal/execution/legacy_compat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,8 +219,13 @@ def do_read(blocks: Iterator[Block], ctx: TaskContext) -> Iterator[Block]:
for read_task in blocks:
yield from read_task()

# If the BlockList's read stage name is available, we assign it
# as the operator's name, which is used as the task name.
task_name = "DoRead"
if isinstance(blocks, LazyBlockList):
task_name = getattr(blocks, "_read_stage_name", task_name)
return MapOperator.create(
do_read, inputs, name="DoRead", ray_remote_args=remote_args
do_read, inputs, name=task_name, ray_remote_args=remote_args
)
else:
output = _block_list_to_bundles(blocks, owns_blocks=owns_blocks)
Expand Down
12 changes: 12 additions & 0 deletions python/ray/data/tests/test_execution_optimizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import pandas as pd

import ray
from ray.data._internal.execution.legacy_compat import _blocks_to_input_buffer
from ray.data._internal.execution.operators.map_operator import MapOperator
from ray.data._internal.execution.operators.all_to_all_operator import AllToAllOperator
from ray.data._internal.execution.operators.zip_operator import ZipOperator
Expand Down Expand Up @@ -1078,6 +1079,17 @@ def test_from_torch_e2e(ray_start_regular_shared, enable_optimizer, tmp_path):
_check_usage_record(["FromItems"])


def test_blocks_to_input_buffer_op_name(
ray_start_regular_shared,
enable_streaming_executor,
):
ds: ray.data.Datastream = ray.data.range(10)
blocks, _, _ = ds._plan._optimize()
assert hasattr(blocks, "_tasks"), blocks
physical_op = _blocks_to_input_buffer(blocks, owns_blocks=False)
assert physical_op.name == "ReadRange"


def test_execute_to_legacy_block_list(
ray_start_regular_shared,
enable_optimizer,
Expand Down