Finetuning rest

apps/rest/api/hello

@@ -0,0 +1 @@
+s

apps/rest/api/logging.yaml

@@ -0,0 +1,22 @@
+version: 1
+disable_existing_loggers: false
+
+formatters:
+  standard:
+    format: "%(asctime)s - %(levelname)s - %(message)s"
+
+handlers:
+  console:
+    class: logging.StreamHandler
+    formatter: standard
+    stream: ext://sys.stdout
+
+loggers:
+  uvicorn:
+    error:
+      propagate: true
+
+root:
+  level: INFO
+  handlers: [console]
+  propagate: no

apps/rest/api/pyproject.toml

@@ -34,6 +34,7 @@ mail = "^2.1.0"
 python-dotenv = "^1.0.1"
 diskcache = "^5.6.3"
 pydantic = "^2.6.4"
+python-upwork-oauth2 = "^3.1.0"
 
 [tool.poetry.group.dev.dependencies]
 mypy = "^1.7.0"

apps/rest/api/shell.nix

@@ -3,7 +3,6 @@
     name = "pipzone";
     targetPkgs = pkgs: (with pkgs; [
         python312
-        poetry
     ]);
     runScript = "fish";
 }).env

apps/rest/api/src/__init__.py

@@ -1,148 +1,43 @@
-import os
-from dotenv import load_dotenv
-import logging
-import diskcache as dc
-import threading
-from uuid import uuid4
+import uvicorn
 
-from .saving import update_db_with_submission
-from . import mail
-from .reddit_utils import get_subreddits
-from .relevance_bot import evaluate_relevance
-from .interfaces import db
-from . import comment_bot
-from .models import (
-    PublishCommentRequest,
-    GenerateCommentRequest,
-    FalseLead,
-)
-from .reddit_worker import RedditStreamWorker
-
-from fastapi import FastAPI, HTTPException
+from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import RedirectResponse
-
-
-# Relevant subreddits to Startino
-SUBREDDIT_NAMES = (
-    "SaaS+SaaSy+startups+YoungEntrepreneurs+NoCodeSaas+nocode+cofounder+Entrepreneur"
-)
-
-load_dotenv()
-REDDIT_PASSWORD = os.getenv("REDDIT_PASSWORD")
-REDDIT_USERNAME = os.getenv("REDDIT_USERNAME")
-
-if REDDIT_PASSWORD is None:
-    raise ValueError("REDDIT_PASSWORD is not set")
-
-if REDDIT_USERNAME is None:
-    raise ValueError("REDDIT_USERNAME is not set")
-
-logger = logging.getLogger("root")
-
-app = FastAPI()
-workers = {}
-
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=[
-        "http://localhost:5173",
-        "http://localhost:5174",
-        "http://localhost:8000",
-        "http://localhost:8001",
-        "http://localhost:8080",
-        "http://localhost:8081",
-        "http://127.0.0.1:5173",
-        "http://127.0.0.1:5174",
-        "http://127.0.0.1:8000",
-        "http://127.0.0.1:8001",
-        "http://127.0.0.1:8080",
-        "http://127.0.0.1:8081",
-        "https://aiti.no",
-        "https://api.aiti.no",
-    ],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-
-@app.get("/")
-def redirect_to_docs() -> RedirectResponse:
-    return RedirectResponse(url="/docs")
-
-
-@app.get("/test")
-def test():
-    print("TEST")
-
-
-@app.post("/start")
-def start_stream():
-    worker_id = str(uuid4())
-    worker = RedditStreamWorker(SUBREDDIT_NAMES, REDDIT_USERNAME, REDDIT_PASSWORD)
-    thread = threading.Thread(target=worker.start)
-    workers[worker_id] = (worker, thread)
-    thread.start()
-    return {"worker_id": worker_id}
-
-
-@app.post("/stop/{worker_id}")
-def stop_stream(worker_id: str):
-    worker, thread = workers.get(worker_id, (None, None))
-    if worker is None:
-        raise HTTPException(status_code=404, detail="Worker not found")
-    worker.stop()
-    thread.join(timeout=10)  # Waits 10 seconds for the thread to finish
-
-    if thread.is_alive():
-        logger.error(f"Thread for Reddit worker didn't stop in time")
-
-    del workers[worker_id]  # Cleanup
-
-    return {"message": "Stream stopped"}
-
-
-@app.post("/generate-comment")
-def generate_comment(generate_request: GenerateCommentRequest):
-    comment: str = comment_bot.generate_comment(
-        generate_request.title,
-        generate_request.selftext,
+from src.routers import index
+
+def create_app() -> FastAPI:
+    app = FastAPI()
+
+    app.include_router(index.router)
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=[
+            "http://localhost:5173",
+            "http://localhost:5174",
+            "http://localhost:8000",
+            "http://localhost:8001",
+            "http://localhost:8080",
+            "http://localhost:8081",
+            "http://127.0.0.1:5173",
+            "http://127.0.0.1:5174",
+            "http://127.0.0.1:8000",
+            "http://127.0.0.1:8001",
+            "http://127.0.0.1:8080",
+            "http://127.0.0.1:8081",
+            "https://rest.futi.no",
+            "https://api.rest.futi.no",
+        ],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
     )
-    if comment is None:
-        raise HTTPException(404, "comment not found")
-
-    return comment
+    return app
 
 
-@app.post("/mark-lead-as-irrelevant")
-def mark_lead_as_irrelevant(false_lead: FalseLead):
-    # Mark the lead as irrelevant in 'leads' table
-    lead = db.update_lead(id=false_lead.lead_id, status="rejected")
+def run():
+    app = create_app()
 
-    if lead is None:
-        raise HTTPException(404, "lead not found")
-
-    # Mark the submission as irrelevant in 'evaluated_submissions' table as a
-    # human answer and review
-    db.update_human_review_for_submission(
-        id=false_lead.submission_id,
-        human_answer=False,
-        correct_reason=false_lead.correct_reason,
-    )
+    uvicorn.run(app, host="0.0.0.0", port=8000, log_config="logging.yaml")
 
-    return {"status": "success"}
-
-
-@app.post("/publish-comment")
-def publish_comment(publish_request: PublishCommentRequest):
-    updated_content = comment_bot.publish_comment(
-        publish_request.lead_id,
-        publish_request.comment,
-        publish_request.reddit_username,
-        publish_request.reddit_password,
-    )
-    if updated_content is None:
-        raise HTTPException(404, "lead not found")
 
-    return updated_content
+if __name__ == "__main__":
+    run()

apps/rest/api/src/comment_bot.py

@@ -34,12 +34,15 @@ def generate_comment(title: str, selftext: str, instructions: str = "") -> str:
     # Post
     Title: {title}
     Content: {selftext}
+
+    # Additional instructions
+    {instructions}
     """
     )
 
     prompt = PromptTemplate(
         template=template,
-        input_variables=["title", "selftext"],
+        input_variables=["title", "selftext", "instructions"],
         partial_variables={"format_instructions": parser.get_format_instructions()},
     )
 
@@ -50,6 +53,7 @@ def generate_comment(title: str, selftext: str, instructions: str = "") -> str:
         {
             "title": title,
             "selftext": selftext,
+            "instructions": instructions,
         }
     )
 
@@ -60,7 +64,8 @@ def publish_comment(
     id, text: str, username: str, password: str
 ) -> PublishCommentResponse | None:
     lead = db.get_lead(id)
-    if lead is None:
+    print(lead)
+    if lead is None or lead.status == "subscriber":
         logging.error(f"Lead with id {id} not found")
         return None
 

apps/rest/api/src/interfaces/db.py

@@ -71,8 +71,20 @@ def post_lead(lead: Lead) -> None:
     """
     supabase: Client = create_client(url, key)
     logger.debug(f"Posting lead: {lead}")
+
+    # Check if lead already exists (temporary until supabase-py supports onConflict)
+    existing_lead = (
+        supabase.table("leads")
+        .select("*")
+        .eq("prospect_username", lead.prospect_username)
+        .execute()
+    )
+
+    if len(existing_lead.data) > 0:
+        return
+
     supabase.table("leads").insert(
-        json.loads(json.dumps(lead.model_dump(), default=str))
+        json.loads(json.dumps(lead.dict(), default=str))
     ).execute()
 
 
@@ -83,8 +95,28 @@ def update_lead(
     Update a lead in the database.
     """
     supabase: Client = create_client(url, key)
-    # Create a dictionary with only non-empty values
-    data = {k: v for k, v in {"status": status, "last_event": last_event}.items() if v}
+
+    if status is "subscriber":
+        last_contacted_at = datetime.now()
+        # Create a dictionary with only non-empty values
+        data = {
+            k: v
+            for k, v in {
+                "status": status,
+                "last_event": last_event,
+                "last_contacted_at": str(last_contacted_at),
+            }.items()
+            if v
+        }
+    else:
+        data = {
+            k: v
+            for k, v in {
+                "status": status,
+                "last_event": last_event,
+            }.items()
+            if v
+        }
 
     logger.debug(f"Updating lead with data: {data}")
     response = supabase.table("leads").update(data).eq("id", str(id)).execute()
@@ -125,6 +157,17 @@ def post_evaluated_submission(saved_submission: SavedSubmission) -> None:
     supabase: Client = create_client(url, key)
     logger.debug(f"Posting submission: {saved_submission}")
 
+    # Check if submission already exists (temporary until supabase-py supports onConflict)
+    existing_submission = (
+        supabase.table("evaluated_submissions")
+        .select("*")
+        .eq("body", saved_submission.body)
+        .execute()
+    )
+
+    if len(existing_submission.data) > 0:
+        return
+
     supabase.table("evaluated_submissions").insert(
         json.loads(json.dumps(saved_submission.model_dump(), default=str))
     ).execute()

apps/rest/api/src/models/reddit_comment.py

@@ -2,7 +2,7 @@
 from langchain.prompts import PromptTemplate
 from langchain_core.pydantic_v1 import BaseModel, Field, validator
 from pydantic import BaseModel as PydanticBaseModel
-
+from typing import Optional
 
 class RedditComment(BaseModel):
     comment: str = Field(description="the text of the reddit comment")
@@ -12,3 +12,4 @@ class RedditComment(BaseModel):
 class GenerateCommentRequest(PydanticBaseModel):
     title: str
     selftext: str
+    instructions: str = ""

apps/rest/api/src/prompts/commenting.py

@@ -14,11 +14,12 @@
 ) as file:
     good_examples = file.read()
 
-with open(os.path.join(script_dir, "bad_examples.md"), "r", encoding="utf-8") as file:
+with open(os.path.join(script_dir, "bad_comment_examples.md"), "r", encoding="utf-8") as file:
     bad_examples = file.read()
 
 purpose = """
-Introduce myself and Futino and quickly capture interest with a relevant value proposition, setting the stage for further engagement and relationship building.
+Introduce myself and Startino and quickly capture interest with a relevant value proposition, setting the stage for further engagement and relationship building.
+Sometimes, if they are simply looking for advice, write a valuable comment, rather than trying to sell them. This is because I will later try to sell them after they respond.
 """
 
 context = f"""
@@ -27,8 +28,10 @@
 
 guidance = """
 - Keep the comment concise and to the point.
-- Don't mention about cost effectiveness of Futino.
-- Mention how we gaurantee our dedication through our services for equity model.
+- Don't mention about cost effectiveness of Startino.
+- Do not mention how we gaurantee our dedication through our services for equity model.
+- Do not mention the equity for services model or anything about equity.
+- You may mention that we sometimes take equity if the post is a startup.
 - Do not use more than 50 words.
 """