diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml index 03d9549..c4b954a 100644 --- a/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -2,5 +2,21 @@ \ No newline at end of file diff --git a/server/src/ingest_server.ts b/server/src/ingest_server.ts index ea275b6..bff4561 100644 --- a/server/src/ingest_server.ts +++ b/server/src/ingest_server.ts @@ -1,12 +1,14 @@ import express from 'express'; import { langchainIngestRouter } from './routers/ingest/langchain_ingestion_router'; import 'dotenv/config'; +import { langchainFeedbackRouter } from './routers/ingest/langchain_feedback_router'; const ingest_server = express(); ingest_server.use(express.json({ limit: '50mb' })); ingest_server.use('/api/runs', langchainIngestRouter); +ingest_server.use('/api/feedback', langchainFeedbackRouter); const PORT = process.env.PORT || 1984; ingest_server.listen(PORT, () => { diff --git a/server/src/models/requests/feedback_request.ts b/server/src/models/requests/feedback_request.ts new file mode 100644 index 0000000..f7718ee --- /dev/null +++ b/server/src/models/requests/feedback_request.ts @@ -0,0 +1,17 @@ +export interface CreateFeedback { + id: string; + run_id: string; + key: string; + score?: number | boolean; + value?: string; + comment?: string; + + [key: string]: unknown; +} + +export interface UpdateFeedback { + score?: number | boolean; + value?: string; + correction?: { [key: string]: unknown }; + comment?: string; +} diff --git a/server/src/models/trace_detail_response.ts b/server/src/models/trace_detail_response.ts index 3d27034..38a9326 100644 --- a/server/src/models/trace_detail_response.ts +++ b/server/src/models/trace_detail_response.ts @@ -1,3 +1,5 @@ +import { CreateFeedback } from './requests/feedback_request'; + export interface TraceDetailResponse { run_id: string; name: string; @@ -10,4 +12,5 @@ export interface TraceDetailResponse { run_type: string; parent_run_id: string | null; children: TraceDetailResponse[]; + feedback?: CreateFeedback; } diff --git a/server/src/repositories/langtrace_repository.ts b/server/src/repositories/langtrace_repository.ts index 23a701e..41875d9 100644 --- a/server/src/repositories/langtrace_repository.ts +++ b/server/src/repositories/langtrace_repository.ts @@ -3,6 +3,8 @@ import { TraceData } from '../models/requests/trace_request'; import { TraceDetailResponse } from '../models/trace_detail_response'; import 'dotenv/config'; import { TracePercentile } from '../models/traces_percentiles'; +import { CreateFeedback, UpdateFeedback } from '../models/requests/feedback_request'; + export class LangtraceRepository { private db!: Db; @@ -53,10 +55,12 @@ export class LangtraceRepository { name: 1, start_time: 1, end_time: 1, - latency: 1 + latency: 1, + feedback: 1 } }, - { $sort: { start_time: -1 } } + { $sort: { start_time: -1 } }, + { $limit: 100 } ]; const collection = this.db.collection(this.collectionName); @@ -105,7 +109,7 @@ export class LangtraceRepository { ]; interface TracesPercentilesMongoRecord { - latency_percentiles: number[] + latency_percentiles: number[]; } const result = await collection.aggregate(pipeline).toArray(); @@ -240,4 +244,25 @@ export class LangtraceRepository { }, ]).toArray(); } + + + async insertFeedbackOnTraceByRunId(feedback: CreateFeedback) { + const collection = this.db.collection('traces'); + await collection.updateOne({ run_id: feedback.run_id }, { $set: { feedback } }); + } + + async updateFeedbackOnTraceByFeedbackId( + feedbackId: string, + feedbackData: UpdateFeedback): + Promise { + const collection = this.db.collection('traces'); + + const setOperation: Record = {}; + for (const [key, value] of Object.entries(feedbackData)) { + setOperation[`feedback.${key}`] = value; + } + + return await collection.updateOne({ 'feedback.id': feedbackId }, + { $set: setOperation }); + } } diff --git a/server/src/routers/ingest/langchain_feedback_router.ts b/server/src/routers/ingest/langchain_feedback_router.ts new file mode 100644 index 0000000..c35aec5 --- /dev/null +++ b/server/src/routers/ingest/langchain_feedback_router.ts @@ -0,0 +1,39 @@ +import { Router, Request as ExpressRequest, Response as ExpressResponse } from 'express'; +import { LangchainToLangtraceService } from '../../services/langchain_to_langtrace_service'; +import { CreateFeedback, UpdateFeedback } from '../../models/requests/feedback_request'; + +export const langchainFeedbackRouter = Router(); +const langchainService = new LangchainToLangtraceService(); + +langchainFeedbackRouter.post('/', async (req: ExpressRequest, res: ExpressResponse) => { + console.debug('POST /api/feedback'); + try { + const runData = req.body as CreateFeedback; + await langchainService.createFeedback(runData); + + console.debug(`Created feedback with id ${runData.id} in run ${runData.run_id}`); + + res.status(201).json( + { feedback_id: runData.id } + ); + } catch (error: unknown) { + console.error(error); + if (error instanceof Error) { + return res.status(400).json({ message: error.message }); + } else { + return res.status(400).json({ message: 'Unknown error' }); + } + } +}); + +langchainFeedbackRouter.patch('/:feedbackId', async (req: ExpressRequest, res: ExpressResponse) => { + console.debug('PATCH /api/feedback/:feedbackId'); + const feedbackId = req.params.feedbackId; + const updateData = req.body as UpdateFeedback; + + const success = await langchainService.updateFeedback(feedbackId, updateData); + if (!success) { + return res.status(404).json({ message: 'Feedback not found or update failed' }); + } + return res.status(204).send(); +}); diff --git a/server/src/services/langchain_to_langtrace_service.ts b/server/src/services/langchain_to_langtrace_service.ts index 841acb6..f1508e9 100644 --- a/server/src/services/langchain_to_langtrace_service.ts +++ b/server/src/services/langchain_to_langtrace_service.ts @@ -1,5 +1,6 @@ import { LangtraceRepository } from '../repositories/langtrace_repository'; import { TraceData } from '../models/requests/trace_request'; +import { CreateFeedback, UpdateFeedback } from '../models/requests/feedback_request'; export class LangchainToLangtraceService { private langtraceRepository: LangtraceRepository; @@ -36,4 +37,24 @@ export class LangchainToLangtraceService { const updateResult = await this.langtraceRepository.updateTrace(trace_id, langchainData); return updateResult.matchedCount > 0; } + + async createFeedback(feedback: CreateFeedback) { + if (!feedback.run_id) { + throw new Error('run_id is required in data'); + } + + await this.langtraceRepository.insertFeedbackOnTraceByRunId( + feedback + ); + return feedback.id; + } + + async updateFeedback(feedbackId: string, feedbackData: UpdateFeedback): Promise { + return ( + await this.langtraceRepository.updateFeedbackOnTraceByFeedbackId( + feedbackId, + feedbackData + ) + ).matchedCount > 0; + } } diff --git a/ui/src/components/Breadcrumb/Breadcrumb.module.scss b/ui/src/components/Breadcrumb/Breadcrumb.module.scss index ea98a57..bbfc7e5 100644 --- a/ui/src/components/Breadcrumb/Breadcrumb.module.scss +++ b/ui/src/components/Breadcrumb/Breadcrumb.module.scss @@ -8,7 +8,7 @@ display: flex; align-items: center; font-size: small; - padding-bottom: 8px; + padding-bottom: 16px; } .breadcrumbItem { diff --git a/ui/src/components/FilterPanel/FilterPanel.module.scss b/ui/src/components/FilterPanel/FilterPanel.module.scss index 9015abc..fadef58 100644 --- a/ui/src/components/FilterPanel/FilterPanel.module.scss +++ b/ui/src/components/FilterPanel/FilterPanel.module.scss @@ -1,10 +1,12 @@ @use '../../styles/constants' as c; .filterPanel { - border-left: c.$border-width solid #ddd; - padding-left: 16px; width: 20%; overflow-x: auto; + background-color: c.$panel-background-color; + border-radius: c.$panel-radius; + box-shadow: 0 0 1px 0 rgba(0, 0, 0, 0.2); + padding: 16px; } .filterPanelChips{ diff --git a/ui/src/components/FilterPanel/index.tsx b/ui/src/components/FilterPanel/index.tsx index c4940c8..ce61658 100644 --- a/ui/src/components/FilterPanel/index.tsx +++ b/ui/src/components/FilterPanel/index.tsx @@ -3,12 +3,12 @@ import styles from './FilterPanel.module.scss'; import { TracePercentile } from '@/models/traces_response'; import PercentileChip from '../PercentileChip'; -interface FilterPanelProps { +interface StatsPanelProps { recordsCount: number; latencyPercentiles: TracePercentile[]; } -const FilterPanel: React.FC = ({ latencyPercentiles, recordsCount }) => { +const StatsPanel: React.FC = ({ latencyPercentiles, recordsCount }) => { return (
@@ -32,8 +32,9 @@ const FilterPanel: React.FC = ({ latencyPercentiles, recordsCo

No latency data available

} +

Feedback

); }; -export default FilterPanel; +export default StatsPanel; diff --git a/ui/src/components/LatencyChip/LatencyChip.module.scss b/ui/src/components/LatencyChip/LatencyChip.module.scss index 01fbe60..30935e3 100644 --- a/ui/src/components/LatencyChip/LatencyChip.module.scss +++ b/ui/src/components/LatencyChip/LatencyChip.module.scss @@ -15,21 +15,21 @@ @extend .latencyChip; background-color: c.$temperature-1; color: c.$temperature-1-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-1-text; } .temperature2 { @extend .latencyChip; background-color: c.$temperature-2; color: c.$temperature-2-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-2-text; } .temperature3 { @extend .latencyChip; background-color: c.$temperature-3; color: c.$temperature-3-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-3-text; } @@ -37,5 +37,5 @@ @extend .latencyChip; background-color: c.$temperature-4; color: c.$temperature-4-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-4-text; } diff --git a/ui/src/components/PercentileChip/PercentileChip.module.scss b/ui/src/components/PercentileChip/PercentileChip.module.scss index a110040..768d048 100644 --- a/ui/src/components/PercentileChip/PercentileChip.module.scss +++ b/ui/src/components/PercentileChip/PercentileChip.module.scss @@ -15,21 +15,21 @@ @extend .percentileChip; background-color: c.$temperature-1; color: c.$temperature-1-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-1-text; } .temperature2 { @extend .percentileChip; background-color: c.$temperature-2; color: c.$temperature-2-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-2-text; } .temperature3 { @extend .percentileChip; background-color: c.$temperature-3; color: c.$temperature-3-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-3-text; } @@ -37,5 +37,5 @@ @extend .percentileChip; background-color: c.$temperature-4; color: c.$temperature-4-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-4-text; } diff --git a/ui/src/components/TraceDetailsPanel/traceDetailsPanel.module.scss b/ui/src/components/TraceDetailsPanel/traceDetailsPanel.module.scss index cc2aaf9..e026b42 100644 --- a/ui/src/components/TraceDetailsPanel/traceDetailsPanel.module.scss +++ b/ui/src/components/TraceDetailsPanel/traceDetailsPanel.module.scss @@ -2,7 +2,10 @@ .traceDetailsPanel { width: 70%; - padding-left: 16px; + background-color: c.$panel-background-color; + border-radius: c.$panel-radius; + box-shadow: 0 0 1px 0 rgba(0, 0, 0, 0.2); + padding: 16px; } .title { @@ -11,8 +14,8 @@ } .content { - background-color: c.$background-color; - border: c.$border-width solid c.$border-color; + background-color: c.$panel-background-color; + border: c.$border-width solid rgba(0, 0, 0, 0.2); padding: 8px; border-radius: c.$border-radius; white-space: pre-wrap; diff --git a/ui/src/components/TraceTable/TraceTable.module.scss b/ui/src/components/TraceTable/TraceTable.module.scss index f0cf9b1..59d33d8 100644 --- a/ui/src/components/TraceTable/TraceTable.module.scss +++ b/ui/src/components/TraceTable/TraceTable.module.scss @@ -1,24 +1,27 @@ @use '../../styles/constants' as c; - .tableContainer { width: 80%; overflow-x: auto; + background-color: c.$panel-background-color; + border-radius: c.$panel-radius; + box-shadow: 0 0 1px 0 rgba(0, 0, 0, 0.2); + padding: 16px; } .fullWidthTable { width: 100%; - border: c.$border-width solid c.$border-color; + border: c.$border-width solid c.$border-color-transparent; border-radius: c.$border-radius; border-collapse: separate; border-spacing: 0; th { - border-bottom: c.$border-width solid c.$border-color; + border-bottom: c.$border-width solid c.$border-color-transparent; height: 36px; padding: 8px; text-align: left; - background-color: #f4f4f4; + background-color: c.$background-color; } td { @@ -53,7 +56,7 @@ .dateDropdown { padding: 8px; border-radius: c.$border-radius; - border: c.$border-width solid c.$border-color; + border: c.$border-width solid c.$border-color-transparent; font-family: 'Ubuntu', 'Open Sans', sans-serif; } } diff --git a/ui/src/components/TraceTable/index.tsx b/ui/src/components/TraceTable/index.tsx index 2cbe75c..7ed81aa 100644 --- a/ui/src/components/TraceTable/index.tsx +++ b/ui/src/components/TraceTable/index.tsx @@ -30,6 +30,7 @@ function TraceTable(props: { Name Start Time Latency + Feedback diff --git a/ui/src/components/TraceTree/index.tsx b/ui/src/components/TraceTree/index.tsx index bd17534..587e920 100644 --- a/ui/src/components/TraceTree/index.tsx +++ b/ui/src/components/TraceTree/index.tsx @@ -48,7 +48,7 @@ const TraceTree: React.FC = ({ const traceHeaderClass = isSelected ? `${styles.traceHeader} ${styles.active}` : styles.traceHeader; return ( -
+
handleSelectTrace(trace)} className={styles.traceTitle}> {trace.run_type.toUpperCase()} diff --git a/ui/src/components/TraceTree/traceTree.module.scss b/ui/src/components/TraceTree/traceTree.module.scss index 9ee1e1e..8987df8 100644 --- a/ui/src/components/TraceTree/traceTree.module.scss +++ b/ui/src/components/TraceTree/traceTree.module.scss @@ -2,16 +2,18 @@ .traceTree { width: 30%; - padding-right: 16px; - border-right: c.$border-width solid #ddd; + padding: 0 16px; height: calc(100vh - #{c.$breadcrumb-height} - 34px); + background-color: c.$panel-background-color; + border-radius: c.$panel-radius; + box-shadow: 0 0 1px 0 rgba(0, 0, 0, 0.2); } .runTypeBox { font-size: 12px; color: #666; display: inline-block; - border: c.$border-width solid c.$border-color; + border: c.$border-width solid c.$border-color-transparent; border-radius: c.$border-radius; padding: 2px 4px; margin-right: 8px; @@ -62,7 +64,7 @@ .traceChildren { margin-left: 8px; - border-left: 2px solid #ddd; + border-left: 2px solid c.$border-color-bold; padding-left: 10px; } @@ -74,5 +76,5 @@ .active { border-radius: c.$border-radius; - border: c.$border-width solid #333333; + border: c.$border-width * 2 solid c.$border-color-bold; } diff --git a/ui/src/models/trace_detail_response.ts b/ui/src/models/trace_detail_response.ts index 3a4ff8e..f470bf9 100644 --- a/ui/src/models/trace_detail_response.ts +++ b/ui/src/models/trace_detail_response.ts @@ -5,6 +5,7 @@ export interface TraceTreeNode { end_time: string; outputs: { [key: string]: any }; inputs: { [key: string]: any }; + error: string | null session_name: string; run_type: string; latency: number; @@ -12,4 +13,5 @@ export interface TraceTreeNode { children: TraceTreeNode[]; execution_order?: number; depth?: number; + feedback?: { [key: string]: any }; //TODO Make this a proper type } diff --git a/ui/src/models/tree/trace_detail_response_tree.ts b/ui/src/models/tree/trace_detail_response_tree.ts index 8ce0ed4..1f9d547 100644 --- a/ui/src/models/tree/trace_detail_response_tree.ts +++ b/ui/src/models/tree/trace_detail_response_tree.ts @@ -1,6 +1,7 @@ export interface TraceDetailResponseParent { run_id: string; name: string; + error: string | null; start_time: string; end_time: string; execution_order: number; @@ -17,6 +18,7 @@ export interface TraceDetailResponseChild { depth: number; run_id: string; execution_order: number; + error: string | null; name: string; start_time: string; end_time: string; diff --git a/ui/src/pages/Home.module.scss b/ui/src/pages/Home.module.scss index 8ff142e..f8ac9d5 100644 --- a/ui/src/pages/Home.module.scss +++ b/ui/src/pages/Home.module.scss @@ -8,6 +8,18 @@ height: 100vh; } +.homePanel { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + width: 50%; + padding: 32px; + border-radius: c.$panel-radius; + background-color: c.$panel-background-color; + box-shadow: 0 0 1px 0 rgba(0, 0, 0, 0.2); +} + .button { display: flex; align-items: center; @@ -15,7 +27,8 @@ padding: 8px; margin-top: 32px; border-radius: c.$border-radius; - border: c.$border-width solid c.$border-color; + border: c.$border-width * 2 solid c.$border-color-bold; + box-shadow: 0 0 1px 0 rgba(0, 0, 0, 0.2); background-color: transparent; cursor: pointer; color: c.$primary-color; diff --git a/ui/src/pages/index.tsx b/ui/src/pages/index.tsx index 4c6ed3a..5106aae 100644 --- a/ui/src/pages/index.tsx +++ b/ui/src/pages/index.tsx @@ -5,12 +5,14 @@ import {Hierarchy} from "iconic-react"; const Home = () => { return (
+

LangTrace

View your Langchain data
Traces +
); }; diff --git a/ui/src/pages/traces/Traces.module.scss b/ui/src/pages/traces/Traces.module.scss index 00e131c..2913a1b 100644 --- a/ui/src/pages/traces/Traces.module.scss +++ b/ui/src/pages/traces/Traces.module.scss @@ -29,22 +29,26 @@ @extend .chip; background-color: c.$temperature-1; color: c.$temperature-1-text; + border: 1px solid c.$temperature-1-text; } .inprogress { @extend .chip; background-color: c.$temperature-2; color: c.$temperature-2-text; + border: 1px solid c.$temperature-2-text; } .warning { @extend .chip; background-color: c.$temperature-3; color: c.$temperature-3-text; + border: 1px solid c.$temperature-3-text; } .error { @extend .chip; background-color: c.$temperature-4; color: c.$temperature-4-text; + border: 1px solid c.$temperature-4-text; } .columnIcon { diff --git a/ui/src/pages/traces/index.tsx b/ui/src/pages/traces/index.tsx index 99d12e6..b1475cc 100644 --- a/ui/src/pages/traces/index.tsx +++ b/ui/src/pages/traces/index.tsx @@ -10,27 +10,19 @@ import { import { IconType } from 'react-icons/lib'; import { useRouter } from 'next/router'; import { GetServerSidePropsContext } from 'next'; -import FilterPanel from '@/components/FilterPanel'; +import StatsPanel from '@/components/FilterPanel'; import TraceTable from '../../components/TraceTable'; import { TracePercentile } from '@/models/traces_response'; import LatencyChip from '@/components/LatencyChip'; +import { TraceTreeNode } from '@/models/trace_detail_response'; const breadcrumbItems = [ { name: 'Home', path: '/' }, { name: 'Traces', path: undefined }, ]; -interface Trace { - run_id: string; - name: string; - error?: string; - start_time: string; - end_time?: string; - latency: number; -} - interface TracesProps { - traces: Trace[]; + traces: TraceTreeNode[]; latencyPercentiles: TracePercentile[]; } @@ -62,7 +54,7 @@ const handleRowClick = (run_id: string) => { window.location.href = `/traces/${run_id}`; }; -function getStatusForTrace(trace: Trace): ReactElement { +function getStatusForTrace(trace: TraceTreeNode): ReactElement { if (trace.error) { return
; } else if (trace.end_time) { @@ -134,18 +126,17 @@ const Traces: React.FC = ({ traces, latencyPercentiles }) => {
{ const runDate = convertToDateTime(trace.start_time); - return ( - handleRowClick(trace.run_id)} - className={styles.clickableRow}> - {trace.run_id} - {getStatusForTrace(trace)} - {trace.name} - {runDate.date} @ {runDate.time} - - - ); + return handleRowClick(trace.run_id)} + className={styles.clickableRow}> + {trace.run_id} + {getStatusForTrace(trace)} + {trace.name} + {runDate.date} @ {runDate.time} + + { trace.feedback?.key ? trace.feedback?.key + ": " + trace.feedback?.score : ''} + ; })}/> - +
); diff --git a/ui/src/pages/traces/traceDetailsPage.module.scss b/ui/src/pages/traces/traceDetailsPage.module.scss index 601b27e..4e4e7e5 100644 --- a/ui/src/pages/traces/traceDetailsPage.module.scss +++ b/ui/src/pages/traces/traceDetailsPage.module.scss @@ -1,6 +1,9 @@ +@use '../../styles/constants' as c; + + .traceDetailsContainer { display: flex; - gap: 0; + gap: 16px; } @media (max-width: 768px) { diff --git a/ui/src/styles/constants.scss b/ui/src/styles/constants.scss index b324201..d7a22ab 100644 --- a/ui/src/styles/constants.scss +++ b/ui/src/styles/constants.scss @@ -1,12 +1,15 @@ $black: #000; $white: #fff; -$background-color: $white; +$background-color: #eee; +$panel-background-color: $white; $primary-color: $black; -$border-radius: 4px; -$border-width: 2px; -$border-color: $black; +$border-radius: 8px; +$panel-radius: 12px; +$border-width: 1px; +$border-color-bold: rgba(0, 0, 0); +$border-color-transparent: rgba($border-color-bold, 0.2); $text-color: $black;