logging

delphi-suite · May 15, 2024 · a135a4f · a135a4f
1 parent 411756a
commit a135a4f
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 3 deletions.
diff --git a/scripts/tokenize_dataset.py b/scripts/tokenize_dataset.py
@@ -109,7 +109,6 @@
     print(f"Tokenizing split='{args.split}'...")
     split_name = args.split.split("[")[0]
     for chunk_idx, ds_chunk in enumerate(ds_chunks_it):
-        print(f"Processing chunk {chunk_idx}...")
         chunk_name = f"{split_name}-{chunk_idx:05}.parquet"
         if args.out_dir:
             ds_parquet_chunk = Path(args.out_dir) / chunk_name
@@ -125,4 +124,4 @@
                 repo_id=args.out_repo_id,
                 repo_type="dataset",
             )
-        print("Done.")
+        print(f"Done saving/uploading '{chunk_name}'")
diff --git a/src/delphi/dataset/tokenization.py b/src/delphi/dataset/tokenization.py
@@ -1,4 +1,3 @@
-import io
 import itertools
 from collections import deque
 from collections.abc import Iterator