"not a mach-o file" error when trying to import on M1 Mac

[imadahmad97]

/!\ PLEASE INCLUDE THE FULL STACKTRACE AND CODE SNIPPET

Short description
I have set up a tensorflow-ready environment on my mac, but when I try to import tensorflow-datasets, I get the following output:

---

ImportError Traceback (most recent call last)
Cell In[1], line 1
----> 1 import tensorflow_datasets as tfds

File ~/tensorflow-ready/env/lib/python3.9/site-packages/tensorflow_datasets/init.py:43
41 _TIMESTAMP_IMPORT_STARTS = time.time()
42 from absl import logging
---> 43 import tensorflow_datasets.core.logging as _tfds_logging
44 from tensorflow_datasets.core.logging import call_metadata as _call_metadata
46 _metadata = _call_metadata.CallMetadata()

File ~/tensorflow-ready/env/lib/python3.9/site-packages/tensorflow_datasets/core/init.py:22
18 # Allow to use tfds.core.Path in dataset implementation which seems more
19 # natural than having to import a third party module.
20 from etils.epath import Path
---> 22 from tensorflow_datasets.core import community
23 from tensorflow_datasets.core.dataset_builder import BeamBasedBuilder
24 from tensorflow_datasets.core.dataset_builder import BuilderConfig

File ~/tensorflow-ready/env/lib/python3.9/site-packages/tensorflow_datasets/core/community/init.py:18
1 # coding=utf-8
2 # Copyright 2023 The TensorFlow Datasets Authors.
3 #
(...)
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 """Community dataset API."""
---> 18 from tensorflow_datasets.core.community.huggingface_wrapper import mock_builtin_to_use_gfile
19 from tensorflow_datasets.core.community.huggingface_wrapper import mock_huggingface_import
20 from tensorflow_datasets.core.community.load import builder_cls_from_module

File ~/tensorflow-ready/env/lib/python3.9/site-packages/tensorflow_datasets/core/community/huggingface_wrapper.py:31
28 from unittest import mock
30 from etils import epath
---> 31 from tensorflow_datasets.core import dataset_builder
32 from tensorflow_datasets.core import dataset_info
33 from tensorflow_datasets.core import download

File ~/tensorflow-ready/env/lib/python3.9/site-packages/tensorflow_datasets/core/dataset_builder.py:34
32 from etils import epath
33 from tensorflow_datasets.core import constants
---> 34 from tensorflow_datasets.core import dataset_info
35 from tensorflow_datasets.core import dataset_metadata
36 from tensorflow_datasets.core import decode

File ~/tensorflow-ready/env/lib/python3.9/site-packages/tensorflow_datasets/core/dataset_info.py:47
45 from etils import epath
46 from tensorflow_datasets.core import constants
---> 47 from tensorflow_datasets.core import file_adapters
48 from tensorflow_datasets.core import lazy_imports_lib
49 from tensorflow_datasets.core import naming

File ~/tensorflow-ready/env/lib/python3.9/site-packages/tensorflow_datasets/core/file_adapters.py:29
26 from tensorflow_datasets.core.utils import type_utils
27 from tensorflow_datasets.core.utils.lazy_imports_utils import tensorflow as tf
---> 29 from array_record.python import array_record_module
31 ExamplePositions = List[Any]
34 class FileFormat(enum.Enum):

ImportError: dlopen(/Users/imadahmad/tensorflow-ready/env/lib/python3.9/site-packages/array_record/python/array_record_module.so, 0x0002): tried: '/Users/imadahmad/tensorflow-ready/env/lib/python3.9/site-packages/array_record/python/array_record_module.so' (not a mach-o file), '/System/Volumes/Preboot/Cryptexes/OS/Users/imadahmad/tensorflow-ready/env/lib/python3.9/site-packages/array_record/python/array_record_module.so' (no such file), '/Users/imadahmad/tensorflow-ready/env/lib/python3.9/site-packages/array_record/python/array_record_module.so' (not a mach-o file)

Environment information

• Operating System:
• Python version: <3.9>

[gugurete]

I can confirm this issue on my m1 Mac Studio as well (Ventura 13.2.1)
Python 3.10.9 (main, Dec 15 2022, 17:11:09) [Clang 14.0.0 (clang-1400.0.29.202)] on darwin

import tensorflow_datasets as tfds
Traceback (most recent call last):
File "", line 1, in
File "/Users/robert/tensorflow-metal/lib/python3.10/site-packages/tensorflow_datasets/init.py", line 43, in
import tensorflow_datasets.core.logging as _tfds_logging
File "/Users/robert/tensorflow-metal/lib/python3.10/site-packages/tensorflow_datasets/core/init.py", line 22, in
from tensorflow_datasets.core import community
File "/Users/robert/tensorflow-metal/lib/python3.10/site-packages/tensorflow_datasets/core/community/init.py", line 18, in
from tensorflow_datasets.core.community.huggingface_wrapper import mock_builtin_to_use_gfile
File "/Users/robert/tensorflow-metal/lib/python3.10/site-packages/tensorflow_datasets/core/community/huggingface_wrapper.py", line 31, in
from tensorflow_datasets.core import dataset_builder
File "/Users/robert/tensorflow-metal/lib/python3.10/site-packages/tensorflow_datasets/core/dataset_builder.py", line 34, in
from tensorflow_datasets.core import dataset_info
File "/Users/robert/tensorflow-metal/lib/python3.10/site-packages/tensorflow_datasets/core/dataset_info.py", line 47, in
from tensorflow_datasets.core import file_adapters
File "/Users/robert/tensorflow-metal/lib/python3.10/site-packages/tensorflow_datasets/core/file_adapters.py", line 29, in
from array_record.python import array_record_module
ImportError: dlopen(/Users/robert/tensorflow-metal/lib/python3.10/site-packages/array_record/python/array_record_module.so, 0x0002): tried: '/Users/robert/tensorflow-metal/lib/python3.10/site-packages/array_record/python/array_record_module.so' (not a mach-o file), '/System/Volumes/Preboot/Cryptexes/OS/Users/robert/tensorflow-metal/lib/python3.10/site-packages/array_record/python/array_record_module.so' (no such file), '/Users/robert/tensorflow-metal/lib/python3.10/site-packages/array_record/python/array_record_module.so' (not a mach-o file)

[imadahmad97]

I noticed using an older version of tensorflow-datasets (4.4.0) solved the issue for me, however still not working with 4.9.0

[gugurete]

I am wondering if this is a tensorflow_datasets bug or more likely a tensorflow-metal issue

[r1k]

This isn't just an M1 issue, I get the same problem on an older Intel MacBook Pro, running Ventura 13.3 using python 3.10.4.
Trying the older version mentioned above also worked for me.

[marcenacp]

Hi all, thank you so much for pointing out the issue. This is due to macos/windows platforms not being supported by the ArrayRecord dependency (https://github.com/google/array_record). This dependency is core to one of our new features.

We're actively working on short-term and long-term fixes. I already pushed #4855, which should be merged soon. We'll publish v4.9.1 if we confirm that this does solve the issue.

Thanks for your understanding!

[marcenacp]

TFDS 4.9.1 is out with a fix for the installation on macOS.

pip install --upgrade tensorflow-datasets==4.9.1

We did a post-mortem. The main outputs are:

• We are going to integrate macos-latest and windows-latest to our CI/CD in Github Actions, as we see more users on these platforms.
• We are going to make ArrayRecord compatible with windows/macos in the future releases.

I am closing the issue, but will happily re-open it in case we're missing anything. Thank you all for your understanding!

[bschreck]

Hi this is also a problem in airio, used by t5x

I'm seeing the following error:

File "/Users/ben/.pyenv/versions/vid2seq-flax/lib/python3.10/site-packages/t5x/__init__.py", line 17, in <module>
    import t5x.adafactor
  File "/Users/ben/.pyenv/versions/vid2seq-flax/lib/python3.10/site-packages/t5x/adafactor.py", line 64, in <module>
    from t5x import utils
  File "/Users/ben/.pyenv/versions/vid2seq-flax/lib/python3.10/site-packages/t5x/utils.py", line 33, in <module>
    import airio
  File "/Users/ben/.pyenv/versions/vid2seq-flax/lib/python3.10/site-packages/airio/__init__.py", line 18, in <module>
    from airio.dataset_providers import *
  File "/Users/ben/.pyenv/versions/vid2seq-flax/lib/python3.10/site-packages/airio/dataset_providers.py", line 22, in <module>
    from airio import data_sources
  File "/Users/ben/.pyenv/versions/vid2seq-flax/lib/python3.10/site-packages/airio/data_sources.py", line 21, in <module>
    import grain.python as grain
  File "/Users/ben/.pyenv/versions/vid2seq-flax/lib/python3.10/site-packages/grain/python.py", line 21, in <module>
    from . import python_experimental as experimental
  File "/Users/ben/.pyenv/versions/vid2seq-flax/lib/python3.10/site-packages/grain/python_experimental.py", line 21, in <module>
    from . import python_lazy_dataset as lazy_dataset
  File "/Users/ben/.pyenv/versions/vid2seq-flax/lib/python3.10/site-packages/grain/python_lazy_dataset.py", line 49, in <module>
    from ._src.python.lazy_dataset.transformations.shuffle import ShuffleLazyMapDataset
  File "/Users/ben/.pyenv/versions/vid2seq-flax/lib/python3.10/site-packages/grain/_src/python/lazy_dataset/transformations/shuffle.py", line 18, in <module>
    from grain._src.python.experimental.index_shuffle.python import index_shuffle_module as index_shuffle
ImportError: dlopen(/Users/ben/.pyenv/versions/vid2seq-flax/lib/python3.10/site-packages/grain/_src/python/experimental/index_shuffle/python/index_shuffle_module.so, 0x0002): tried: '/Users/ben/.pyenv/versions/vid2seq-flax/lib/python3.10/site-packages/grain/_src/python/experimental/index_shuffle/python/index_shuffle_module.so' (not a mach-o file), '/System/Volumes/Preboot/Cryptexes/OS/Users/ben/.pyenv/versions/vid2seq-flax/lib/python3.10/site-packages/grain/_src/python/experimental/index_shuffle/python/index_shuffle_module.so' (no such file), '/Users/ben/.pyenv/versions/vid2seq-flax/lib/python3.10/site-packages/grain/_src/python/experimental/index_shuffle/python/index_shuffle_module.so' (not a mach-o file), '/Users/ben/.pyenv/versions/3.10.4/envs/vid2seq-flax/lib/python3.10/site-packages/grain/_src/python/experimental/index_shuffle/python/index_shuffle_module.so' (not a mach-o file), '/System/Volumes/Preboot/Cryptexes/OS/Users/ben/.pyenv/versions/3.10.4/envs/vid2seq-flax/lib/python3.10/site-packages/grain/_src/python/experimental/index_shuffle/python/index_shuffle_module.so' (no such file), '/Users/ben/.pyenv/versions/3.10.4/envs/vid2seq-flax/lib/python3.10/site-packages/grain/_src/python/experimental/index_shuffle/python/index_shuffle_module.so' (not a mach-o file)