From bfd0d5762fe59271244435a52000c156159fe363 Mon Sep 17 00:00:00 2001 From: bclavie Date: Fri, 23 Feb 2024 11:56:02 +0100 Subject: [PATCH 1/6] fix: llamaindex imports --- ...hout_annotations_with_instructor_and_RAGatouille.ipynb | 3 ++- ragatouille/data/preprocessors.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/03-finetuning_without_annotations_with_instructor_and_RAGatouille.ipynb b/examples/03-finetuning_without_annotations_with_instructor_and_RAGatouille.ipynb index a270857..9091807 100644 --- a/examples/03-finetuning_without_annotations_with_instructor_and_RAGatouille.ipynb +++ b/examples/03-finetuning_without_annotations_with_instructor_and_RAGatouille.ipynb @@ -106,6 +106,7 @@ "outputs": [], "source": [ "import instructor\n", + "# If you're using llamaindex 0.10 or above, these need to be imported from llama_index.core instead!\n", "from llama_index import Document\n", "from llama_index.text_splitter import SentenceSplitter\n", "from openai import OpenAI\n", @@ -459,7 +460,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.11.7" } }, "nbformat": 4, diff --git a/ragatouille/data/preprocessors.py b/ragatouille/data/preprocessors.py index 0984aaf..13f872d 100644 --- a/ragatouille/data/preprocessors.py +++ b/ragatouille/data/preprocessors.py @@ -1,5 +1,9 @@ -from llama_index import Document -from llama_index.text_splitter import SentenceSplitter +try: + from llama_index import Document + from llama_index.text_splitter import SentenceSplitter +except ImportError: + from llama_index.core import Document + from llama_index.core.text_splitter import SentenceSplitter def llama_index_sentence_splitter( From 4db672ec86b9264c9cc94cad879767a1f86b0aba Mon Sep 17 00:00:00 2001 From: bclavie Date: Fri, 23 Feb 2024 11:56:08 +0100 Subject: [PATCH 2/6] fix: rare device mismatch --- ragatouille/models/colbert.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ragatouille/models/colbert.py b/ragatouille/models/colbert.py index 469fea0..dc4aa1f 100644 --- a/ragatouille/models/colbert.py +++ b/ragatouille/models/colbert.py @@ -11,7 +11,6 @@ from colbert import Indexer, IndexUpdater, Searcher, Trainer from colbert.infra import ColBERTConfig, Run, RunConfig from colbert.modeling.checkpoint import Checkpoint - from ragatouille.models.base import LateInteractionModel # TODO: Move all bsize related calcs to `_set_bsize()` @@ -751,7 +750,7 @@ def encode( - encodings.shape[1], encodings.shape[2], ) - ), + ).to(device=encodings.device), ], dim=1, ) @@ -765,7 +764,7 @@ def encode( - doc_masks.shape[1], ), -float("inf"), - ), + ).to(device=encodings.device), ], dim=1, ) From d1ac0e442c9c8efc6917e1a16ea011a4953c110f Mon Sep 17 00:00:00 2001 From: bclavie Date: Fri, 23 Feb 2024 11:59:59 +0100 Subject: [PATCH 3/6] chore: use only ruff linting --- .github/workflows/ruff.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index e065f91..9a05f99 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -6,8 +6,4 @@ jobs: steps: - uses: actions/checkout@v3 - name: "Linting & Flaking" - uses: chartboost/ruff-action@v1 - - name: "Formatting" - uses: chartboost/ruff-action@v1 - with: - args: format --check \ No newline at end of file + uses: chartboost/ruff-action@v1 \ No newline at end of file From 874c4c8f6aa2794dc84ce83adc57e735e986994d Mon Sep 17 00:00:00 2001 From: bclavie Date: Fri, 23 Feb 2024 12:02:47 +0100 Subject: [PATCH 4/6] ruff --- .github/workflows/ruff.yml | 6 +++++- pyproject.toml | 5 +---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 9a05f99..e065f91 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -6,4 +6,8 @@ jobs: steps: - uses: actions/checkout@v3 - name: "Linting & Flaking" - uses: chartboost/ruff-action@v1 \ No newline at end of file + uses: chartboost/ruff-action@v1 + - name: "Formatting" + uses: chartboost/ruff-action@v1 + with: + args: format --check \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index b9a1415..58c3e47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,7 +96,4 @@ unfixable = [ "T201", "T203", ] -ignore-init-module-imports = true - -[tool.ruff.lint.isort] -section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"] +ignore-init-module-imports = true \ No newline at end of file From 00e555b51203e9ab76c1405891dba41197b79b93 Mon Sep 17 00:00:00 2001 From: bclavie Date: Fri, 23 Feb 2024 12:05:22 +0100 Subject: [PATCH 5/6] ruff --- ragatouille/models/colbert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ragatouille/models/colbert.py b/ragatouille/models/colbert.py index c10c3ef..303a24c 100644 --- a/ragatouille/models/colbert.py +++ b/ragatouille/models/colbert.py @@ -190,8 +190,8 @@ def add_to_index( bsize=bsize, ) else: - if self.config.index_bsize != bsize: # Update bsize if it's different - self.config.index_bsize = bsize + if self.config.index_bsize != bsize: # Update bsize if it's different + self.config.index_bsize = bsize updater = IndexUpdater( config=self.config, searcher=searcher, checkpoint=self.checkpoint From 9cce3612cd8fa70b7f8e2cea86dcf34120057ee4 Mon Sep 17 00:00:00 2001 From: bclavie Date: Fri, 23 Feb 2024 12:06:38 +0100 Subject: [PATCH 6/6] isort --- ragatouille/models/colbert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ragatouille/models/colbert.py b/ragatouille/models/colbert.py index 303a24c..534a952 100644 --- a/ragatouille/models/colbert.py +++ b/ragatouille/models/colbert.py @@ -11,6 +11,7 @@ from colbert import Indexer, IndexUpdater, Searcher, Trainer from colbert.infra import ColBERTConfig, Run, RunConfig from colbert.modeling.checkpoint import Checkpoint + from ragatouille.models.base import LateInteractionModel # TODO: Move all bsize related calcs to `_set_bsize()`