resorted

python/dolma/cli/deduper.py

@@ -4,13 +4,14 @@
 from typing import Any, Dict, List, Optional
 
 import smart_open
+from omegaconf import OmegaConf as om
+
 from dolma import deduper
 from dolma.cli import BaseCli, field, print_config
 from dolma.cli.shared import WorkDirConfig, get_path_to_temp_file, make_workdirs
 from dolma.core.errors import DolmaConfigError
 from dolma.core.loggers import get_logger
 from dolma.core.paths import glob_path, is_local
-from omegaconf import OmegaConf as om
 
 
 @dataclass

python/dolma/cli/tagger.py

@@ -2,6 +2,9 @@
 from pstats import SortKey
 from typing import List, Optional
 
+from rich.console import Console
+from rich.table import Table
+
 from dolma.cli import BaseCli, field, print_config
 from dolma.cli.shared import WorkDirConfig, make_workdirs
 from dolma.core.errors import DolmaConfigError
@@ -10,8 +13,6 @@
 from dolma.core.registry import TaggerRegistry
 from dolma.core.runtime import create_and_run_tagger
 from dolma.core.utils import import_modules
-from rich.console import Console
-from rich.table import Table
 
 
 @dataclass

python/dolma/core/runtime.py

@@ -17,6 +17,7 @@
 
 import msgspec
 import smart_open
+
 from dolma.core.taggers import BaseTaggerWithMetadata
 
 from .data_types import (

tests/python/test_binning.py

@@ -1,6 +1,7 @@
 import unittest
 
 import numpy as np
+
 from dolma.core.binning import (
     FixedBucketsValTracker,
     InferBucketsValTracker,

tests/python/test_code.py

@@ -10,6 +10,7 @@
 import unittest
 
 from bs4 import BeautifulSoup
+
 from dolma.core.data_types import Document, DocumentWithMetadata
 from dolma.taggers.code import (
     CodeCopyrightTagger,

tests/python/test_omegaconf.py

@@ -2,9 +2,10 @@
 from dataclasses import dataclass
 from unittest import TestCase
 
-from dolma.cli import field, make_parser, namespace_to_nested_omegaconf
 from omegaconf import MissingMandatoryValue
 
+from dolma.cli import field, make_parser, namespace_to_nested_omegaconf
+
 
 @dataclass
 class _1:

tests/python/test_parallel.py

@@ -7,6 +7,7 @@
 from unittest import TestCase
 
 import smart_open
+
 from dolma.core.parallel import BaseParallelProcessor, QueueType
 
 LOCAL_DATA = Path(__file__).parent.parent / "data"

tests/python/test_repetitions_utils.py

@@ -1,6 +1,7 @@
 from unittest import TestCase
 
 import numpy as np
+
 from dolma.taggers.repetitions.utils import (
     find_end_first_consecutive_true,
     find_periodic_sequences,

tests/python/test_runtime.py

@@ -5,6 +5,7 @@
 from unittest import TestCase
 
 import smart_open
+
 from dolma.core.runtime import (
     _make_paths_from_prefix,
     _make_paths_from_substitution,

tests/python/test_tokenizer.py

@@ -6,9 +6,10 @@
 
 import numpy
 import smart_open
+from tokenizers import Tokenizer as BaseTokenizer
+
 from dolma.cli.__main__ import main
 from dolma.tokenizer import Tokenizer
-from tokenizers import Tokenizer as BaseTokenizer
 
 TEST_DIR = Path(__file__).parent.parent.resolve()
 

tests/python/utils.py

@@ -12,9 +12,10 @@
 
 import boto3
 import smart_open
-from dolma.core.paths import glob_path, mkdir_p
 from smart_open import open
 
+from dolma.core.paths import glob_path, mkdir_p
+
 DOLMA_TESTS_S3_PREFIX_ENV_VAR = "DOLMA_TESTS_S3_PREFIX"
 DOLMA_TESTS_SKIP_AWS_ENV_VAR = "DOLMA_TESTS_SKIP_AWS"
 DOLMA_TESTS_S3_PREFIX_DEFAULT = "s3://dolma-tests"