langchain-ai · hinthornw · Nov 21, 2023 · Nov 20, 2023 · Nov 20, 2023 · Nov 20, 2023
diff --git a/csv-qa/pandas_agent_instruct.py b/csv-qa/pandas_agent_instruct.py
@@ -5,7 +5,6 @@
 )
 from langchain.embeddings import OpenAIEmbeddings
 from langchain.llms import OpenAI
-from langchain.prompts import PromptTemplate
 from langchain.smith import RunEvalConfig, run_on_dataset
 from langchain.tools import PythonAstREPLTool
 from langchain.vectorstores import FAISS

diff --git a/csv-qa/pandas_ai.py b/csv-qa/pandas_ai.py
@@ -1,6 +1,4 @@
 import pandas as pd
-from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
-from langchain.agents.agent_types import AgentType
 from langchain.chat_models import ChatOpenAI
 from langchain.prompts import ChatPromptTemplate
 from langchain.schema.output_parser import StrOutputParser

diff --git a/docs/source/notebooks/rag.ipynb b/docs/source/notebooks/rag.ipynb
diff --git a/langchain-docs-benchmarking/prepare_dataset.py b/langchain-docs-benchmarking/prepare_dataset.py
@@ -1,7 +1,7 @@
 """Copy the public dataset to your own langsmith tenant."""
 from typing import Optional
+
 from langsmith import Client
-from tqdm import tqdm
 
 DATASET_NAME = "LangChain Docs Q&A"
 PUBLIC_DATASET_TOKEN = "452ccafc-18e1-4314-885b-edd735f17b9d"

diff --git a/langchain_benchmarks/extraction/email_task.py b/langchain_benchmarks/extraction/email_task.py
@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Optional, List
+from typing import List, Optional
 
 from langchain.smith import RunEvalConfig
 from pydantic import BaseModel, Field

diff --git a/langchain_benchmarks/rag/.gitignore b/langchain_benchmarks/rag/.gitignore
@@ -0,0 +1 @@
+*.sql
diff --git a/langchain_benchmarks/rag/__init__.py b/langchain_benchmarks/rag/__init__.py
@@ -1,8 +1,4 @@
-"""RAG environments."""
-from langchain_benchmarks.rag.evaluators import RAG_EVALUATION
-from langchain_benchmarks.rag.environments.langchain_docs.task import (
-    LANGCHAIN_DOCS_TASK,
-)
+from langchain_benchmarks.rag.tasks import LANGCHAIN_DOCS_TASK
 
-# Please keep this list sorted!
-__all__ = ["LANGCHAIN_DOCS_TASK", "RAG_EVALUATION"]
+# Please keep this sorted
+__all__ = ["LANGCHAIN_DOCS_TASK"]
diff --git a/langchain_benchmarks/rag/environments/langchain_docs/__init__.py b/langchain_benchmarks/rag/environments/langchain_docs/__init__.py
diff --git a/langchain_benchmarks/rag/environments/langchain_docs/architectures/__init__.py b/langchain_benchmarks/rag/environments/langchain_docs/architectures/__init__.py
diff --git a/langchain_benchmarks/rag/environments/langchain_docs/langchain_docs_retriever/__init__.py b/langchain_benchmarks/rag/environments/langchain_docs/langchain_docs_retriever/__init__.py
diff --git a/langchain_benchmarks/rag/environments/langchain_docs/langchain_docs_retriever/download_db.py b/langchain_benchmarks/rag/environments/langchain_docs/langchain_docs_retriever/download_db.py
diff --git a/langchain_benchmarks/rag/environments/langchain_docs/langchain_docs_retriever/retriever.py b/langchain_benchmarks/rag/environments/langchain_docs/langchain_docs_retriever/retriever.py
diff --git a/langchain_benchmarks/rag/evaluators.py b/langchain_benchmarks/rag/evaluators.py
@@ -77,18 +77,21 @@ def evaluate_run(
 """  # noqa
 }
 
-eval_llm = ChatOpenAI(model="gpt-4", temperature=0.0, model_kwargs={"seed": 42})
-# Use a longer-context LLM to check documents
-faithfulness_eval_llm = ChatOpenAI(
-    model="gpt-4-1106-preview", temperature=0.0, model_kwargs={"seed": 42}
-)
 
-RAG_EVALUATION = RunEvalConfig(
-    evaluators=[
-        RunEvalConfig.LabeledScoreString(
-            criteria=_ACCURACY_CRITERION, llm=eval_llm, normalize_by=10.0
-        ),
-        RunEvalConfig.EmbeddingDistance(),
-    ],
-    custom_evaluators=[FaithfulnessEvaluator(llm=faithfulness_eval_llm)],
-)
+def get_eval_config() -> RunEvalConfig:
+    """Returns the evaluator for the environment."""
+    eval_llm = ChatOpenAI(model="gpt-4", temperature=0.0, model_kwargs={"seed": 42})
+    # Use a longer-context LLM to check documents
+    faithfulness_eval_llm = ChatOpenAI(
+        model="gpt-4-1106-preview", temperature=0.0, model_kwargs={"seed": 42}
+    )
+
+    return RunEvalConfig(
+        evaluators=[
+            RunEvalConfig.LabeledScoreString(
+                criteria=_ACCURACY_CRITERION, llm=eval_llm, normalize_by=10.0
+            ),
+            RunEvalConfig.EmbeddingDistance(),
+        ],
+        custom_evaluators=[FaithfulnessEvaluator(llm=faithfulness_eval_llm)],
+    )
diff --git a/langchain_benchmarks/rag/tasks/__init__.py b/langchain_benchmarks/rag/tasks/__init__.py
@@ -0,0 +1,5 @@
+from langchain_benchmarks.rag.tasks.langchain_docs.task import LANGCHAIN_DOCS_TASK
+
+
+# Please keep this sorted
+__all__ = ["LANGCHAIN_DOCS_TASK"]
diff --git a/...rag/environments/langchain_docs/README.md → ...hmarks/rag/tasks/langchain_docs/README.md b/...rag/environments/langchain_docs/README.md → ...hmarks/rag/tasks/langchain_docs/README.md
diff --git a/langchain_benchmarks/rag/tasks/langchain_docs/__init__.py b/langchain_benchmarks/rag/tasks/langchain_docs/__init__.py
@@ -0,0 +1,8 @@
+from langchain_benchmarks.rag.tasks.langchain_docs import architectures, indexing
+from langchain_benchmarks.rag.tasks.langchain_docs.task import LANGCHAIN_DOCS_TASK
+
+DATASET_ID = (
+    "452ccafc-18e1-4314-885b-edd735f17b9d"  # ID of public LangChain Docs dataset
+)
+
+__all__ = ["architectures", "indexing", "DATASET_ID", "LANGCHAIN_DOCS_TASK"]