Wfh/add semistructured (#38)

Refactoring to share rag things

---------

Co-authored-by: Eugene Yurtsev <[email protected]>

csv-qa/pandas_agent_instruct.py

@@ -5,7 +5,6 @@
 )
 from langchain.embeddings import OpenAIEmbeddings
 from langchain.llms import OpenAI
-from langchain.prompts import PromptTemplate
 from langchain.smith import RunEvalConfig, run_on_dataset
 from langchain.tools import PythonAstREPLTool
 from langchain.vectorstores import FAISS

csv-qa/pandas_ai.py

@@ -1,6 +1,4 @@
 import pandas as pd
-from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
-from langchain.agents.agent_types import AgentType
 from langchain.chat_models import ChatOpenAI
 from langchain.prompts import ChatPromptTemplate
 from langchain.schema.output_parser import StrOutputParser

langchain-docs-benchmarking/packages/openai-functions-agent/openai_functions_agent/agent.py

@@ -82,4 +82,4 @@ def mapper(input: dict):
 
 agent_executor = (mapper | agent_executor | (lambda x: x["output"])).with_types(
     input_type=ChainInput
-)
+)

langchain-docs-benchmarking/prepare_dataset.py

@@ -1,7 +1,7 @@
 """Copy the public dataset to your own langsmith tenant."""
 from typing import Optional
+
 from langsmith import Client
-from tqdm import tqdm
 
 DATASET_NAME = "LangChain Docs Q&A"
 PUBLIC_DATASET_TOKEN = "452ccafc-18e1-4314-885b-edd735f17b9d"

langchain_benchmarks/extraction/email_task.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Optional, List
+from typing import List, Optional
 
 from langchain.smith import RunEvalConfig
 from pydantic import BaseModel, Field

langchain_benchmarks/rag/.gitignore

@@ -0,0 +1 @@
+*.sql

langchain_benchmarks/rag/__init__.py

@@ -1,8 +1,5 @@
-"""RAG environments."""
-from langchain_benchmarks.rag.evaluators import RAG_EVALUATION
-from langchain_benchmarks.rag.environments.langchain_docs.task import (
-    LANGCHAIN_DOCS_TASK,
-)
+from langchain_benchmarks.rag.evaluators import get_eval_config
+from langchain_benchmarks.rag.tasks import LANGCHAIN_DOCS_TASK
 
-# Please keep this list sorted!
-__all__ = ["LANGCHAIN_DOCS_TASK", "RAG_EVALUATION"]
+# Please keep this sorted
+__all__ = ["get_eval_config", "LANGCHAIN_DOCS_TASK"]

langchain_benchmarks/rag/evaluators.py

@@ -77,18 +77,21 @@ def evaluate_run(
 """  # noqa
 }
 
-eval_llm = ChatOpenAI(model="gpt-4", temperature=0.0, model_kwargs={"seed": 42})
-# Use a longer-context LLM to check documents
-faithfulness_eval_llm = ChatOpenAI(
-    model="gpt-4-1106-preview", temperature=0.0, model_kwargs={"seed": 42}
-)
 
-RAG_EVALUATION = RunEvalConfig(
-    evaluators=[
-        RunEvalConfig.LabeledScoreString(
-            criteria=_ACCURACY_CRITERION, llm=eval_llm, normalize_by=10.0
-        ),
-        RunEvalConfig.EmbeddingDistance(),
-    ],
-    custom_evaluators=[FaithfulnessEvaluator(llm=faithfulness_eval_llm)],
-)
+def get_eval_config() -> RunEvalConfig:
+    """Returns the evaluator for the environment."""
+    eval_llm = ChatOpenAI(model="gpt-4", temperature=0.0, model_kwargs={"seed": 42})
+    # Use a longer-context LLM to check documents
+    faithfulness_eval_llm = ChatOpenAI(
+        model="gpt-4-1106-preview", temperature=0.0, model_kwargs={"seed": 42}
+    )
+
+    return RunEvalConfig(
+        evaluators=[
+            RunEvalConfig.LabeledScoreString(
+                criteria=_ACCURACY_CRITERION, llm=eval_llm, normalize_by=10.0
+            ),
+            RunEvalConfig.EmbeddingDistance(),
+        ],
+        custom_evaluators=[FaithfulnessEvaluator(llm=faithfulness_eval_llm)],
+    )

langchain_benchmarks/rag/tasks/__init__.py

@@ -0,0 +1,7 @@
+from langchain_benchmarks.rag.tasks.langchain_docs.task import LANGCHAIN_DOCS_TASK
+from langchain_benchmarks.rag.tasks.semi_structured_earnings.task import (
+    SEMI_STRUCTURED_EARNINGS_TASK,
+)
+
+# Please keep this sorted
+__all__ = ["LANGCHAIN_DOCS_TASK", "SEMI_STRUCTURED_EARNINGS_TASK"]

langchain_benchmarks/rag/environments/langchain_docs/README.md → langchain_benchmarks/rag/tasks/langchain_docs/README.md

@@ -1,6 +1,10 @@
-# LangChain Docs Environment
+# LangChain Docs Task 
 
 This code contains utilities to scrape the LangChain docs (already run) and index them
 using common techniques. The docs were scraped using the code in `_ingest_docs.py` and
 uploaded to gcs. To better compare retrieval techniques, we hold these constant and pull
 from that cache whenever generating different indices.
+
+
+The content in `indexing` composes some common indexing strategies with default paramaters for
+benchmarking on the langchain docs.

langchain_benchmarks/rag/tasks/langchain_docs/__init__.py

@@ -0,0 +1,8 @@
+from langchain_benchmarks.rag.tasks.langchain_docs import architectures, indexing
+from langchain_benchmarks.rag.tasks.langchain_docs.task import LANGCHAIN_DOCS_TASK
+
+DATASET_ID = (
+    "452ccafc-18e1-4314-885b-edd735f17b9d"  # ID of public LangChain Docs dataset
+)
+
+__all__ = ["architectures", "indexing", "DATASET_ID", "LANGCHAIN_DOCS_TASK"]