diff --git a/python/ray/data/_internal/delegating_block_builder.py b/python/ray/data/_internal/delegating_block_builder.py
index ab33c1aa4b3ae..4655a8e241485 100644
--- a/python/ray/data/_internal/delegating_block_builder.py
+++ b/python/ray/data/_internal/delegating_block_builder.py
@@ -1,10 +1,8 @@
 import collections
 from typing import Any, Mapping, Optional
 
-from ray.air.util.tensor_extensions.arrow import ArrowConversionError
 from ray.data._internal.arrow_block import ArrowBlockBuilder
 from ray.data._internal.block_builder import BlockBuilder
-from ray.data._internal.pandas_block import PandasBlockBuilder
 from ray.data.block import Block, BlockAccessor, BlockType, DataBatch
 
 
@@ -23,17 +21,8 @@ def _inferred_block_type(self) -> Optional[BlockType]:
     def add(self, item: Mapping[str, Any]) -> None:
         assert isinstance(item, collections.abc.Mapping), item
 
-        import pyarrow
-
         if self._builder is None:
-            try:
-                check = ArrowBlockBuilder()
-                check.add(item)
-                check.build()
-                self._builder = ArrowBlockBuilder()
-            except (TypeError, pyarrow.lib.ArrowInvalid, ArrowConversionError):
-                # Can also handle nested Python objects, which Arrow cannot.
-                self._builder = PandasBlockBuilder()
+            self._builder = ArrowBlockBuilder()
 
         self._builder.add(item)