fix: bunch of small prompt/cost tracking fixes

packages/cli/package.json

@@ -16,9 +16,10 @@
     "inquirer-search-list": "^1.2.6",
     "js-tiktoken": "^1.0.8",
     "llamaindex": "^0.1.2",
+    "modelfusion": "^0.137.0",
     "openai": "^4.20.1",
     "openpipe": "^0.8.0",
-    "prompt-iteration-assistant": "^0.0.34",
+    "prompt-iteration-assistant": "^0.0.37",
     "promptfoo": "^0.28.1",
     "pybridge-zod": "^1.1.0",
     "remeda": "^1.29.0",

packages/cli/src/recommender/prompts/answersQuestion/answersQuestion.ts

@@ -5,6 +5,7 @@ import {
 } from "./schemas/answersQuestionInputSchema";
 import { answersQuestionOutputSchema } from "./schemas/answersQuestionOutputSchema";
 import { answersQuestionPrompt } from "./prompts/answersQuestionPrompt";
+import { DefaultRun } from "modelfusion";
 
 export const ANSWERS_QUESTION = "Answers Question";
 
@@ -23,11 +24,12 @@ class AnswersQuestion extends Prompt<
       output: answersQuestionOutputSchema,
     });
   }
-  async execute(args: AnswersQuestionInput) {
+  async execute(args: AnswersQuestionInput & { run?: DefaultRun }) {
     try {
       return this.run({
         stream: false,
         promptVariables: args,
+        run: args.run,
       });
     } catch (e) {
       console.error(e);

packages/cli/src/recommender/prompts/brainstormSubQuestions/brainstormQuestions.ts

@@ -10,6 +10,7 @@ import {
 } from "./schemas/brainstormQuestionsInputSchema";
 import { brainstormQuestionsOutputSchema } from "./schemas/brainstormQuestionsOutputSchema";
 import { brainstormQuestionsZeroShotPrompt } from "./prompts/zeroShot";
+import { DefaultRun } from "modelfusion";
 
 export const BRAINSTORM_QUESTIONS = "Brainstorm Questions";
 
@@ -29,39 +30,22 @@ export class BrainstormQuestions extends Prompt<
     });
   }
 
-  async execute(args: {
-    query: string;
-    openPipeRequestTags?: RequestTagsWithoutName;
-    enableOpenPipeLogging?: boolean;
-  }) {
+  async execute(args: { query: string; run?: DefaultRun }) {
     const promptVariables: BrainstormQuestionsInput = {
       query: args.query,
     };
-    const candidatePrompt = this.chooseCandidatePrompt(promptVariables);
-    const res = await openpipe.functionCall({
-      function: {
-        name: this.name,
-        description: this.description,
-        input: this.input!,
-        output: this.output!,
-      },
-      vars: promptVariables,
-      prompt: candidatePrompt,
-      body: {
-        max_tokens: this.max_tokens,
-        temperature: this.temperature,
-        model: this.model,
+    try {
+      const res = await this.run({
+        promptVariables,
         stream: false,
-      },
-      openPipeRequestTags: args.openPipeRequestTags
-        ? {
-            ...args.openPipeRequestTags,
-            promptName: formatPromptName(this.name, candidatePrompt.name),
-          }
-        : undefined,
-      enableOpenPipeLogging: args.enableOpenPipeLogging,
-    });
-    return res?.questions || [];
+        run: args.run,
+      });
+
+      return res?.questions || [];
+    } catch (e) {
+      console.error(e);
+      return [];
+    }
   }
 }
 

packages/cli/src/recommender/prompts/brainstormSubQuestions/prompts/zeroShot.ts

@@ -19,7 +19,8 @@ export const brainstormQuestionsZeroShotPrompt =
 - Your task is to brainstorm related questions to research.
 - If the topic is controversial, try to include both sides of the argument.
 - DO NOT USE ACRONYMS OR ABBREVIATIONS, ALWAYS USE THE FULL NAME WITH THE ACRONYM IN PARANTHESES.
-- Brainstorm five questions.
+- Don't include introductory or background questions.
+- Brainstorm three questions.
 `.trim()
         ),
         {
@@ -33,11 +34,9 @@ How can Spaced Repetition Systems (SRS) be improved with AI?
           name: toCamelCase(BRAINSTORM_QUESTIONS),
           arguments: {
             questions: [
-              "What are the current limitations of Spaced Repetition Systems (SRS) for learning?",
               "How has artificial intelligence (AI) been applied to educational systems in other contexts? Can these methods be adapted for Spaced Repetition Systems (SRS)?",
               "What potential improvements could artificial intelligence (AI) bring to Spaced Repetition Systems (SRS) in terms of personalization and effectiveness?",
               "What are the possible risks or drawbacks in integrating artificial intelligence (AI) into Spaced Repetition Systems (SRS)?",
-              "How do professionals in the fields of education and artificial intelligence (AI) view the potential integration of artificial intelligence (AI) and Spaced Repetition Systems (SRS)?",
             ],
           },
         }),

packages/cli/src/recommender/prompts/createQueriesFromProfile/createQueriesFromProfile.ts

@@ -11,6 +11,7 @@ import {
 import { openpipe } from "../../../openpipe/openpipe";
 import { experilearningDataset } from "./datasets/experilearningDataset";
 import { createQueriesFromProfileZeroShotFreeFormPrompt } from "./prompts/zeroShotFreeForm";
+import { DefaultRun } from "modelfusion";
 
 export const CREATE_SEARCH_QUERIES_FROM_PROFILE = "Create Questions";
 
@@ -38,39 +39,24 @@ export class CreateSearchQueriesFromProfile extends Prompt<
     user: string;
     profile: string;
     bio: string;
-    openPipeRequestTags?: RequestTagsWithoutName;
-    enableOpenPipeLogging?: boolean;
+    run?: DefaultRun;
   }) {
     const promptVariables: CreateQueriesFromProfileInput = {
       user: args.user,
       bio: args.bio,
       profile: args.profile,
     };
-    const candidatePrompt = this.chooseCandidatePrompt(promptVariables);
-    const res = await openpipe.functionCall({
-      function: {
-        name: this.name,
-        description: this.description,
-        input: this.input!,
-        output: this.output!,
-      },
-      vars: promptVariables,
-      prompt: candidatePrompt,
-      body: {
-        max_tokens: this.max_tokens,
-        temperature: this.temperature,
-        model: this.model,
+    try {
+      const res = await this.run({
+        promptVariables,
         stream: false,
-      },
-      openPipeRequestTags: args.openPipeRequestTags
-        ? {
-            ...args.openPipeRequestTags,
-            promptName: formatPromptName(this.name, candidatePrompt.name),
-          }
-        : undefined,
-      enableOpenPipeLogging: args.enableOpenPipeLogging,
-    });
-    return res || { queries: [] };
+        run: args.run,
+      });
+      return res || { queries: [] };
+    } catch (e) {
+      console.error(e);
+      return { queries: [] };
+    }
   }
 }
 

packages/cli/src/recommender/prompts/findStartOfAnswer/findStartOfAnswer.ts

@@ -1,21 +1,18 @@
 import { Prompt } from "prompt-iteration-assistant";
-import {
-  RequestTagsWithoutName,
-  formatPromptName,
-} from "../../../openpipe/requestTags";
-import { openpipe } from "../../../openpipe/openpipe";
+import { RequestTagsWithoutName } from "../../../openpipe/requestTags";
 import {
   FindStartOfAnswerInput,
   findStartOfAnswerInputSchema,
 } from "./schemas/findStartOfAnswerOutputSchema";
-import { findStartOfAnswerOutputSchema } from "./schemas/findStartOfAnswerInputSchema";
 import { findStartOfAnswerPrompt } from "./prompts/findStartOfAnswerPrompt";
+import { z } from "zod";
+import { DefaultRun } from "modelfusion";
 
 export const FIND_START_OF_ANSWER = "Find Start Of Answer";
 
 class FindStartOfAnswer extends Prompt<
   typeof findStartOfAnswerInputSchema,
-  typeof findStartOfAnswerOutputSchema
+  z.ZodString
 > {
   constructor() {
     super({
@@ -24,17 +21,15 @@ class FindStartOfAnswer extends Prompt<
       prompts: [findStartOfAnswerPrompt],
       model: "gpt-4",
       input: findStartOfAnswerInputSchema,
-      output: findStartOfAnswerOutputSchema,
       exampleData: [],
     });
   }
 
   async execute(args: {
     question: string;
     text: string;
-    openPipeRequestTags?: RequestTagsWithoutName;
-    enableOpenPipeLogging?: boolean;
-  }) {
+    run?: DefaultRun;
+  }): Promise<string | undefined | null> {
     const promptVariables: FindStartOfAnswerInput = {
       text: args.text,
       question: args.question,
@@ -43,10 +38,11 @@ class FindStartOfAnswer extends Prompt<
       return this.run({
         stream: false,
         promptVariables,
+        run: args.run,
       });
     } catch (e) {
       console.error(e);
-      return { quotedAnswer: null };
+      return null;
     }
   }
 }

packages/cli/src/recommender/prompts/findStartOfAnswer/prompts/findStartOfAnswerPrompt.ts

@@ -12,38 +12,9 @@ export const findStartOfAnswerPrompt =
 - Given a question from the user, evalutate whether the beginning of the answer is in the text.
 - If the beginning of the answer is in the text, quote the beginning of the answer.
 - The answer doesn't need to be complete, just the start of it.
-- Quote the beginning of the answer directly from the text as a JSON string, escaping any literal control characters.
+- Quote the beginning of the answer directly from the text.
 `.trim()
         ),
-        //         ChatMessage.user(
-        //           `
-        // # Text
-
-        // # Question
-        // What is the best way to learn a new language?
-        // `.trim()
-        //         ),
-        //         ChatMessage.assistant<FindStartOfAnswerOutput>(null, {
-        //           name: toCamelCase(FIND_START_OF_ANSWER),
-        //           arguments: {
-        //             answersQuestion: true,
-        //             quotedAnswer: "",
-        //           },
-        //         }),
-        //         ChatMessage.user(
-        //           `
-        // # Text
-
-        // # Question
-        // How does chain of thought prompting work?
-        // `.trim()
-        //         ),
-        //         ChatMessage.assistant<FindStartOfAnswerOutput>(null, {
-        //           name: toCamelCase(FIND_START_OF_ANSWER),
-        //           arguments: {
-        //             answersQuestion: false,
-        //           },
-        //         }),
         ChatMessage.user(
           `
 # Question

packages/cli/src/recommender/prompts/findStartOfAnswerYouTube/findStartOfAnswerYouTube.ts

@@ -6,6 +6,7 @@ import {
   findStartOfAnswerYouTubeInputSchema,
 } from "./schemas/findStartOfAnswerYouTubeInputSchema";
 import { findStartOfAnswerYouTubePrompt } from "./prompts/findStartOfAnswerYouTubePrompt";
+import { DefaultRun } from "modelfusion";
 
 export const FIND_START_OF_ANSWER_YOUTUBE = "Find Start Of Answer Cue";
 
@@ -29,8 +30,7 @@ export class FindStartOfAnswerYouTube extends Prompt<
   async execute(args: {
     question: string;
     cues: { text: string }[];
-    openPipeRequestTags?: RequestTagsWithoutName;
-    enableOpenPipeLogging?: boolean;
+    run?: DefaultRun;
   }) {
     const promptVariables: FindStartOfAnswerYouTubeInput = {
       transcript: args.cues
@@ -47,6 +47,7 @@ ${cue.text}
       return this.run({
         stream: false,
         promptVariables,
+        run: args.run,
       });
     } catch (e) {
       console.error(e);

packages/cli/src/recommender/prompts/recursiveTwitterSummarizer/prompts/zeroShot.ts

@@ -9,13 +9,11 @@ export const zeroShot = new CandidatePrompt<RecursiveTwitterSummarizerInput>({
         `
 # Instructions
 - Act as a user profile writer.
-- You are shown a collection of user data.
 - Given the information from multiple sources and not prior knowledge summarize the user's interests into a profile that explains what topics, people, concepts and ideas interest them.
 - Focus on specific low-level interests, like "the use of large language models (LLMs) in recommender systems", as opposed to generic high-level interests like "AI", "technology" and "innovation".
-- Expand abbreviations and acronyms, eg. "LLM agents" should be written as "large language model (LLM) agents".
-- You must include any technical terms and names of people, places, and things that are relevant to the user.
+- Expand abbreviations and acronyms. For example, "LLM agents" should be written as "Large Language Model (LLM) agents".
+- You must include technical terms and names of people, places, and things that are relevant to the user.
 - If summarizing existing summaries, preserve the technical terms and names of concepts, people, places, ideas and events that are relevant to the user.
-- The summary string should be JSON parsable (escaped quotes, etc).
 - Write a two paragraph summary of around 500 words.
 `.trim()
       ),
@@ -28,7 +26,7 @@ ${this.getVariable("user")}
 # Bio
 ${this.getVariable("bio")}
 
-# User data
+# Raw User Data or Existing Summaries
 ${this.getVariable("tweets")}
 `.trim(),
       },