simpler structure: remove eval & dataset dirs

delphi-suite · May 22, 2024 · a5a3999 · a5a3999
1 parent 784c8f2
commit a5a3999
Show file tree

Hide file tree

Showing 20 changed files with 596 additions and 655 deletions.
diff --git a/notebooks/eval_notebook.ipynb b/notebooks/eval_notebook.ipynb
diff --git a/scripts/get_next_logprobs.py b/scripts/get_next_logprobs.py
@@ -9,7 +9,6 @@
 from transformers import AutoModelForCausalLM
 
 from delphi import utils
-from delphi.eval.utils import get_all_and_next_logprobs
 
 torch.set_grad_enabled(False)
 
@@ -61,7 +60,7 @@ def get_logprobs_single_model(
     for i in trange(0, n_seq, batch_size):
         batch_tokens = dataset[i : i + batch_size][feature]
         logprobs[i : i + batch_size, 1:] = (
-            get_all_and_next_logprobs(model, batch_tokens)[1].cpu().numpy()  # type: ignore
+            utils.get_all_and_next_logprobs(model, batch_tokens)[1].cpu().numpy()  # type: ignore
         )
     return Dataset.from_dict({"logprobs": [row for row in logprobs]})
 

diff --git a/scripts/tokenize_dataset.py b/scripts/tokenize_dataset.py
@@ -9,7 +9,7 @@
 from transformers import AutoTokenizer
 
 from delphi import utils
-from delphi.dataset.tokenization import get_tokenized_chunks
+from delphi.tokenization import get_tokenized_chunks
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="", allow_abbrev=False)

diff --git a/src/delphi/dataset/__init__.py b/src/delphi/dataset/__init__.py