From a326c43c4817019a41ef635ada524b8ac7522c73 Mon Sep 17 00:00:00 2001 From: George Herbert Date: Fri, 12 Jan 2024 23:28:52 +0000 Subject: [PATCH 1/3] feat: show feedback on ui --- .idea/inspectionProfiles/Project_Default.xml | 16 ++++ server/.nvmrc | 2 +- server/package-lock.json | 96 +++++++++---------- server/package.json | 6 +- server/src/ingest_server.ts | 2 + .../src/models/requests/feedback_request.ts | 11 +++ server/src/models/trace_detail_response.ts | 3 + .../src/repositories/langtrace_repository.ts | 19 +++- .../ingest/langchain_feedback_router.ts | 36 +++++++ .../langchain_to_langtrace_service.ts | 30 ++++++ ui/.nvmrc | 2 +- ui/Dockerfile | 2 +- ui/package-lock.json | 16 ++-- ui/package.json | 4 +- .../LatencyChip/LatencyChip.module.scss | 8 +- .../PercentileChip/PercentileChip.module.scss | 8 +- ui/src/components/TraceTable/index.tsx | 1 + ui/src/models/trace_detail_response.ts | 2 + ui/src/pages/traces/Traces.module.scss | 4 + ui/src/pages/traces/index.tsx | 33 +++---- 20 files changed, 206 insertions(+), 95 deletions(-) create mode 100644 server/src/models/requests/feedback_request.ts create mode 100644 server/src/routers/ingest/langchain_feedback_router.ts diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml index 03d9549..c4b954a 100644 --- a/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -2,5 +2,21 @@ \ No newline at end of file diff --git a/server/.nvmrc b/server/.nvmrc index d5a1596..8b0beab 100644 --- a/server/.nvmrc +++ b/server/.nvmrc @@ -1 +1 @@ -20.10.0 +20.11.0 diff --git a/server/package-lock.json b/server/package-lock.json index 4dc6218..0e46fc6 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -15,9 +15,9 @@ "devDependencies": { "@types/express": "4.17.21", "@types/jest": "29.5.11", - "@types/node": "20.10.7", - "@typescript-eslint/eslint-plugin": "6.18.0", - "@typescript-eslint/parser": "6.18.0", + "@types/node": "20.11.0", + "@typescript-eslint/eslint-plugin": "6.18.1", + "@typescript-eslint/parser": "6.18.1", "eslint": "8.56.0", "eslint-import-resolver-typescript": "3.6.1", "eslint-plugin-deprecation": "2.0.0", @@ -1681,9 +1681,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "20.10.7", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.7.tgz", - "integrity": "sha512-fRbIKb8C/Y2lXxB5eVMj4IU7xpdox0Lh8bUPEdtLysaylsml1hOOx1+STloRs/B9nf7C6kPRmmg/V7aQW7usNg==", + "version": "20.11.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.0.tgz", + "integrity": "sha512-o9bjXmDNcF7GbM4CNQpmi+TutCgap/K3w1JyKgxAjqx41zp9qlIAVFi0IhCNsJcXolEqLWhbFbEeL0PvYm4pcQ==", "dev": true, "dependencies": { "undici-types": "~5.26.4" @@ -1763,16 +1763,16 @@ "dev": true }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.18.0.tgz", - "integrity": "sha512-3lqEvQUdCozi6d1mddWqd+kf8KxmGq2Plzx36BlkjuQe3rSTm/O98cLf0A4uDO+a5N1KD2SeEEl6fW97YHY+6w==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.18.1.tgz", + "integrity": "sha512-nISDRYnnIpk7VCFrGcu1rnZfM1Dh9LRHnfgdkjcbi/l7g16VYRri3TjXi9Ir4lOZSw5N/gnV/3H7jIPQ8Q4daA==", "dev": true, "dependencies": { "@eslint-community/regexpp": "^4.5.1", - "@typescript-eslint/scope-manager": "6.18.0", - "@typescript-eslint/type-utils": "6.18.0", - "@typescript-eslint/utils": "6.18.0", - "@typescript-eslint/visitor-keys": "6.18.0", + "@typescript-eslint/scope-manager": "6.18.1", + "@typescript-eslint/type-utils": "6.18.1", + "@typescript-eslint/utils": "6.18.1", + "@typescript-eslint/visitor-keys": "6.18.1", "debug": "^4.3.4", "graphemer": "^1.4.0", "ignore": "^5.2.4", @@ -1821,15 +1821,15 @@ "dev": true }, "node_modules/@typescript-eslint/parser": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.18.0.tgz", - "integrity": "sha512-v6uR68SFvqhNQT41frCMCQpsP+5vySy6IdgjlzUWoo7ALCnpaWYcz/Ij2k4L8cEsL0wkvOviCMpjmtRtHNOKzA==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.18.1.tgz", + "integrity": "sha512-zct/MdJnVaRRNy9e84XnVtRv9Vf91/qqe+hZJtKanjojud4wAVy/7lXxJmMyX6X6J+xc6c//YEWvpeif8cAhWA==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "6.18.0", - "@typescript-eslint/types": "6.18.0", - "@typescript-eslint/typescript-estree": "6.18.0", - "@typescript-eslint/visitor-keys": "6.18.0", + "@typescript-eslint/scope-manager": "6.18.1", + "@typescript-eslint/types": "6.18.1", + "@typescript-eslint/typescript-estree": "6.18.1", + "@typescript-eslint/visitor-keys": "6.18.1", "debug": "^4.3.4" }, "engines": { @@ -1872,13 +1872,13 @@ "dev": true }, "node_modules/@typescript-eslint/scope-manager": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.18.0.tgz", - "integrity": "sha512-o/UoDT2NgOJ2VfHpfr+KBY2ErWvCySNUIX/X7O9g8Zzt/tXdpfEU43qbNk8LVuWUT2E0ptzTWXh79i74PP0twA==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.18.1.tgz", + "integrity": "sha512-BgdBwXPFmZzaZUuw6wKiHKIovms97a7eTImjkXCZE04TGHysG+0hDQPmygyvgtkoB/aOQwSM/nWv3LzrOIQOBw==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.18.0", - "@typescript-eslint/visitor-keys": "6.18.0" + "@typescript-eslint/types": "6.18.1", + "@typescript-eslint/visitor-keys": "6.18.1" }, "engines": { "node": "^16.0.0 || >=18.0.0" @@ -1889,13 +1889,13 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.18.0.tgz", - "integrity": "sha512-ZeMtrXnGmTcHciJN1+u2CigWEEXgy1ufoxtWcHORt5kGvpjjIlK9MUhzHm4RM8iVy6dqSaZA/6PVkX6+r+ChjQ==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.18.1.tgz", + "integrity": "sha512-wyOSKhuzHeU/5pcRDP2G2Ndci+4g653V43gXTpt4nbyoIOAASkGDA9JIAgbQCdCkcr1MvpSYWzxTz0olCn8+/Q==", "dev": true, "dependencies": { - "@typescript-eslint/typescript-estree": "6.18.0", - "@typescript-eslint/utils": "6.18.0", + "@typescript-eslint/typescript-estree": "6.18.1", + "@typescript-eslint/utils": "6.18.1", "debug": "^4.3.4", "ts-api-utils": "^1.0.1" }, @@ -1939,9 +1939,9 @@ "dev": true }, "node_modules/@typescript-eslint/types": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.18.0.tgz", - "integrity": "sha512-/RFVIccwkwSdW/1zeMx3hADShWbgBxBnV/qSrex6607isYjj05t36P6LyONgqdUrNLl5TYU8NIKdHUYpFvExkA==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.18.1.tgz", + "integrity": "sha512-4TuMAe+tc5oA7wwfqMtB0Y5OrREPF1GeJBAjqwgZh1lEMH5PJQgWgHGfYufVB51LtjD+peZylmeyxUXPfENLCw==", "dev": true, "engines": { "node": "^16.0.0 || >=18.0.0" @@ -1952,13 +1952,13 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.18.0.tgz", - "integrity": "sha512-klNvl+Ql4NsBNGB4W9TZ2Od03lm7aGvTbs0wYaFYsplVPhr+oeXjlPZCDI4U9jgJIDK38W1FKhacCFzCC+nbIg==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.18.1.tgz", + "integrity": "sha512-fv9B94UAhywPRhUeeV/v+3SBDvcPiLxRZJw/xZeeGgRLQZ6rLMG+8krrJUyIf6s1ecWTzlsbp0rlw7n9sjufHA==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.18.0", - "@typescript-eslint/visitor-keys": "6.18.0", + "@typescript-eslint/types": "6.18.1", + "@typescript-eslint/visitor-keys": "6.18.1", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", @@ -2003,17 +2003,17 @@ "dev": true }, "node_modules/@typescript-eslint/utils": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.18.0.tgz", - "integrity": "sha512-wiKKCbUeDPGaYEYQh1S580dGxJ/V9HI7K5sbGAVklyf+o5g3O+adnS4UNJajplF4e7z2q0uVBaTdT/yLb4XAVA==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.18.1.tgz", + "integrity": "sha512-zZmTuVZvD1wpoceHvoQpOiewmWu3uP9FuTWo8vqpy2ffsmfCE8mklRPi+vmnIYAIk9t/4kOThri2QCDgor+OpQ==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", "@types/json-schema": "^7.0.12", "@types/semver": "^7.5.0", - "@typescript-eslint/scope-manager": "6.18.0", - "@typescript-eslint/types": "6.18.0", - "@typescript-eslint/typescript-estree": "6.18.0", + "@typescript-eslint/scope-manager": "6.18.1", + "@typescript-eslint/types": "6.18.1", + "@typescript-eslint/typescript-estree": "6.18.1", "semver": "^7.5.4" }, "engines": { @@ -2028,12 +2028,12 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.18.0.tgz", - "integrity": "sha512-1wetAlSZpewRDb2h9p/Q8kRjdGuqdTAQbkJIOUMLug2LBLG+QOjiWoSj6/3B/hA9/tVTFFdtiKvAYoYnSRW/RA==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.18.1.tgz", + "integrity": "sha512-/kvt0C5lRqGoCfsbmm7/CwMqoSkY3zzHLIjdhHZQW3VFrnz7ATecOHR7nb7V+xn4286MBxfnQfQhAmCI0u+bJA==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.18.0", + "@typescript-eslint/types": "6.18.1", "eslint-visitor-keys": "^3.4.1" }, "engines": { diff --git a/server/package.json b/server/package.json index 8685851..162f476 100644 --- a/server/package.json +++ b/server/package.json @@ -20,9 +20,9 @@ "devDependencies": { "@types/express": "4.17.21", "@types/jest": "29.5.11", - "@types/node": "20.10.7", - "@typescript-eslint/eslint-plugin": "6.18.0", - "@typescript-eslint/parser": "6.18.0", + "@types/node": "20.11.0", + "@typescript-eslint/eslint-plugin": "6.18.1", + "@typescript-eslint/parser": "6.18.1", "eslint": "8.56.0", "eslint-import-resolver-typescript": "3.6.1", "eslint-plugin-deprecation": "2.0.0", diff --git a/server/src/ingest_server.ts b/server/src/ingest_server.ts index ea275b6..bff4561 100644 --- a/server/src/ingest_server.ts +++ b/server/src/ingest_server.ts @@ -1,12 +1,14 @@ import express from 'express'; import { langchainIngestRouter } from './routers/ingest/langchain_ingestion_router'; import 'dotenv/config'; +import { langchainFeedbackRouter } from './routers/ingest/langchain_feedback_router'; const ingest_server = express(); ingest_server.use(express.json({ limit: '50mb' })); ingest_server.use('/api/runs', langchainIngestRouter); +ingest_server.use('/api/feedback', langchainFeedbackRouter); const PORT = process.env.PORT || 1984; ingest_server.listen(PORT, () => { diff --git a/server/src/models/requests/feedback_request.ts b/server/src/models/requests/feedback_request.ts new file mode 100644 index 0000000..a9dddae --- /dev/null +++ b/server/src/models/requests/feedback_request.ts @@ -0,0 +1,11 @@ +export interface CreateFeedback { + // TODO Validate that feedback_id is a UUID + feedback_id: string; + run_id?: string; + key: string; + score?: number | boolean; + value?: string; + comment?: string; + + [key: string]: unknown; +} diff --git a/server/src/models/trace_detail_response.ts b/server/src/models/trace_detail_response.ts index 3d27034..38a9326 100644 --- a/server/src/models/trace_detail_response.ts +++ b/server/src/models/trace_detail_response.ts @@ -1,3 +1,5 @@ +import { CreateFeedback } from './requests/feedback_request'; + export interface TraceDetailResponse { run_id: string; name: string; @@ -10,4 +12,5 @@ export interface TraceDetailResponse { run_type: string; parent_run_id: string | null; children: TraceDetailResponse[]; + feedback?: CreateFeedback; } diff --git a/server/src/repositories/langtrace_repository.ts b/server/src/repositories/langtrace_repository.ts index 23a701e..f178ed5 100644 --- a/server/src/repositories/langtrace_repository.ts +++ b/server/src/repositories/langtrace_repository.ts @@ -3,6 +3,8 @@ import { TraceData } from '../models/requests/trace_request'; import { TraceDetailResponse } from '../models/trace_detail_response'; import 'dotenv/config'; import { TracePercentile } from '../models/traces_percentiles'; +import { CreateFeedback } from '../models/requests/feedback_request'; + export class LangtraceRepository { private db!: Db; @@ -53,7 +55,8 @@ export class LangtraceRepository { name: 1, start_time: 1, end_time: 1, - latency: 1 + latency: 1, + feedback: 1 } }, { $sort: { start_time: -1 } } @@ -105,7 +108,7 @@ export class LangtraceRepository { ]; interface TracesPercentilesMongoRecord { - latency_percentiles: number[] + latency_percentiles: number[]; } const result = await collection.aggregate(pipeline).toArray(); @@ -240,4 +243,16 @@ export class LangtraceRepository { }, ]).toArray(); } + + + async insertFeedbackOnTraceByRunId(runId: string, feedback: CreateFeedback) { + const collection = this.db.collection('traces'); + await collection.updateOne({ run_id: runId }, { $set: { feedback } }); + } + + async updateFeedbackOnTraceByRunId(feedbackData: CreateFeedback): Promise { + const collection = this.db.collection('traces'); + return await collection.updateOne({ 'feedback.feedback_id': feedbackData.feedback_id }, + { $set: { feedback: feedbackData } }); + } } diff --git a/server/src/routers/ingest/langchain_feedback_router.ts b/server/src/routers/ingest/langchain_feedback_router.ts new file mode 100644 index 0000000..f19e738 --- /dev/null +++ b/server/src/routers/ingest/langchain_feedback_router.ts @@ -0,0 +1,36 @@ +import { Router, Request as ExpressRequest, Response as ExpressResponse } from 'express'; +import { LangchainToLangtraceService } from '../../services/langchain_to_langtrace_service'; + +export const langchainFeedbackRouter = Router(); +const langchainService = new LangchainToLangtraceService(); + +langchainFeedbackRouter.post('/', async (req: ExpressRequest, res: ExpressResponse) => { + console.debug('POST /api/feedback'); + try { + const runData = req.body; + const feedbackId = await langchainService.createFeedback(runData); + console.debug(`Created feedback with id ${feedbackId} in run ${runData.run_id}`); + res.status(201).json( + { feedback_id: feedbackId } + ); + } catch (error: unknown) { + console.error(error); + if (error instanceof Error) { + return res.status(400).json({ message: error.message }); + } else { + return res.status(400).json({ message: 'Unknown error' }); + } + } +}); + +langchainFeedbackRouter.patch('/:feedbackId', async (req: ExpressRequest, res: ExpressResponse) => { + console.debug('PATCH /api/feedback/:feedbackId'); + const feedbackId = req.params.feedbackId; + const updateData = req.body; + + const success = await langchainService.updateFeedback(feedbackId, updateData); + if (!success) { + return res.status(404).json({ message: 'Feedback not found or update failed' }); + } + return res.status(204).send(); +}); diff --git a/server/src/services/langchain_to_langtrace_service.ts b/server/src/services/langchain_to_langtrace_service.ts index 841acb6..8b10db0 100644 --- a/server/src/services/langchain_to_langtrace_service.ts +++ b/server/src/services/langchain_to_langtrace_service.ts @@ -1,5 +1,7 @@ import { LangtraceRepository } from '../repositories/langtrace_repository'; import { TraceData } from '../models/requests/trace_request'; +import { CreateFeedback } from '../models/requests/feedback_request'; +import { randomUUID } from 'node:crypto'; export class LangchainToLangtraceService { private langtraceRepository: LangtraceRepository; @@ -36,4 +38,32 @@ export class LangchainToLangtraceService { const updateResult = await this.langtraceRepository.updateTrace(trace_id, langchainData); return updateResult.matchedCount > 0; } + + async createFeedback(feedback: CreateFeedback) { + if (!feedback.run_id) { + throw new Error('run_id is required in data'); + } + + if (!feedback.feedback_id) { + feedback.feedback_id = randomUUID(); + } + + const runId = feedback.run_id; + delete feedback.run_id; + + await this.langtraceRepository.insertFeedbackOnTraceByRunId( + runId, + feedback + ); + return feedback.feedback_id; + } + + async updateFeedback(feedbackId: string, feedbackData: CreateFeedback): Promise { + feedbackData.feedback_id = feedbackId; + return ( + await this.langtraceRepository.updateFeedbackOnTraceByRunId( + feedbackData + ) + ).matchedCount > 0; + } } diff --git a/ui/.nvmrc b/ui/.nvmrc index d5a1596..8b0beab 100644 --- a/ui/.nvmrc +++ b/ui/.nvmrc @@ -1 +1 @@ -20.10.0 +20.11.0 diff --git a/ui/Dockerfile b/ui/Dockerfile index 69ed132..05ccae9 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20.10.0-alpine3.19 AS base +FROM node:20.11.0-alpine3.19 AS base ARG langtrace_api_url ENV LANGTRACE_API_URL=$langtrace_api_url diff --git a/ui/package-lock.json b/ui/package-lock.json index 35d2ee3..f3c5801 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -16,13 +16,13 @@ "zod": "3.22.4" }, "devDependencies": { - "@types/node": "20.10.7", + "@types/node": "20.11.0", "@types/react": "18.2.47", "@types/react-dom": "18.2.18", "autoprefixer": "10.4.16", "eslint": "8.56.0", "eslint-config-next": "14.0.4", - "mini-css-extract-plugin": "2.7.6", + "mini-css-extract-plugin": "2.7.7", "postcss": "8.4.33", "sass": "1.69.7", "tailwindcss": "3.4.1", @@ -504,9 +504,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "20.10.7", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.7.tgz", - "integrity": "sha512-fRbIKb8C/Y2lXxB5eVMj4IU7xpdox0Lh8bUPEdtLysaylsml1hOOx1+STloRs/B9nf7C6kPRmmg/V7aQW7usNg==", + "version": "20.11.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.0.tgz", + "integrity": "sha512-o9bjXmDNcF7GbM4CNQpmi+TutCgap/K3w1JyKgxAjqx41zp9qlIAVFi0IhCNsJcXolEqLWhbFbEeL0PvYm4pcQ==", "dev": true, "dependencies": { "undici-types": "~5.26.4" @@ -3369,9 +3369,9 @@ } }, "node_modules/mini-css-extract-plugin": { - "version": "2.7.6", - "resolved": "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-2.7.6.tgz", - "integrity": "sha512-Qk7HcgaPkGG6eD77mLvZS1nmxlao3j+9PkrT9Uc7HAE1id3F41+DdBRYRYkbyfNRGzm8/YWtzhw7nVPmwhqTQw==", + "version": "2.7.7", + "resolved": "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-2.7.7.tgz", + "integrity": "sha512-+0n11YGyRavUR3IlaOzJ0/4Il1avMvJ1VJfhWfCn24ITQXhRr1gghbhhrda6tgtNcpZaWKdSuwKq20Jb7fnlyw==", "dev": true, "dependencies": { "schema-utils": "^4.0.0" diff --git a/ui/package.json b/ui/package.json index e82f7d9..e5b42b7 100644 --- a/ui/package.json +++ b/ui/package.json @@ -17,13 +17,13 @@ "zod": "3.22.4" }, "devDependencies": { - "@types/node": "20.10.7", + "@types/node": "20.11.0", "@types/react": "18.2.47", "@types/react-dom": "18.2.18", "autoprefixer": "10.4.16", "eslint": "8.56.0", "eslint-config-next": "14.0.4", - "mini-css-extract-plugin": "2.7.6", + "mini-css-extract-plugin": "2.7.7", "postcss": "8.4.33", "sass": "1.69.7", "tailwindcss": "3.4.1", diff --git a/ui/src/components/LatencyChip/LatencyChip.module.scss b/ui/src/components/LatencyChip/LatencyChip.module.scss index 01fbe60..30935e3 100644 --- a/ui/src/components/LatencyChip/LatencyChip.module.scss +++ b/ui/src/components/LatencyChip/LatencyChip.module.scss @@ -15,21 +15,21 @@ @extend .latencyChip; background-color: c.$temperature-1; color: c.$temperature-1-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-1-text; } .temperature2 { @extend .latencyChip; background-color: c.$temperature-2; color: c.$temperature-2-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-2-text; } .temperature3 { @extend .latencyChip; background-color: c.$temperature-3; color: c.$temperature-3-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-3-text; } @@ -37,5 +37,5 @@ @extend .latencyChip; background-color: c.$temperature-4; color: c.$temperature-4-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-4-text; } diff --git a/ui/src/components/PercentileChip/PercentileChip.module.scss b/ui/src/components/PercentileChip/PercentileChip.module.scss index a110040..768d048 100644 --- a/ui/src/components/PercentileChip/PercentileChip.module.scss +++ b/ui/src/components/PercentileChip/PercentileChip.module.scss @@ -15,21 +15,21 @@ @extend .percentileChip; background-color: c.$temperature-1; color: c.$temperature-1-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-1-text; } .temperature2 { @extend .percentileChip; background-color: c.$temperature-2; color: c.$temperature-2-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-2-text; } .temperature3 { @extend .percentileChip; background-color: c.$temperature-3; color: c.$temperature-3-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-3-text; } @@ -37,5 +37,5 @@ @extend .percentileChip; background-color: c.$temperature-4; color: c.$temperature-4-text; - border: c.$border-width solid transparent; + border: 1px solid c.$temperature-4-text; } diff --git a/ui/src/components/TraceTable/index.tsx b/ui/src/components/TraceTable/index.tsx index 2cbe75c..7ed81aa 100644 --- a/ui/src/components/TraceTable/index.tsx +++ b/ui/src/components/TraceTable/index.tsx @@ -30,6 +30,7 @@ function TraceTable(props: { Name Start Time Latency + Feedback diff --git a/ui/src/models/trace_detail_response.ts b/ui/src/models/trace_detail_response.ts index 3a4ff8e..f470bf9 100644 --- a/ui/src/models/trace_detail_response.ts +++ b/ui/src/models/trace_detail_response.ts @@ -5,6 +5,7 @@ export interface TraceTreeNode { end_time: string; outputs: { [key: string]: any }; inputs: { [key: string]: any }; + error: string | null session_name: string; run_type: string; latency: number; @@ -12,4 +13,5 @@ export interface TraceTreeNode { children: TraceTreeNode[]; execution_order?: number; depth?: number; + feedback?: { [key: string]: any }; //TODO Make this a proper type } diff --git a/ui/src/pages/traces/Traces.module.scss b/ui/src/pages/traces/Traces.module.scss index 00e131c..2913a1b 100644 --- a/ui/src/pages/traces/Traces.module.scss +++ b/ui/src/pages/traces/Traces.module.scss @@ -29,22 +29,26 @@ @extend .chip; background-color: c.$temperature-1; color: c.$temperature-1-text; + border: 1px solid c.$temperature-1-text; } .inprogress { @extend .chip; background-color: c.$temperature-2; color: c.$temperature-2-text; + border: 1px solid c.$temperature-2-text; } .warning { @extend .chip; background-color: c.$temperature-3; color: c.$temperature-3-text; + border: 1px solid c.$temperature-3-text; } .error { @extend .chip; background-color: c.$temperature-4; color: c.$temperature-4-text; + border: 1px solid c.$temperature-4-text; } .columnIcon { diff --git a/ui/src/pages/traces/index.tsx b/ui/src/pages/traces/index.tsx index 99d12e6..2a574b3 100644 --- a/ui/src/pages/traces/index.tsx +++ b/ui/src/pages/traces/index.tsx @@ -14,23 +14,15 @@ import FilterPanel from '@/components/FilterPanel'; import TraceTable from '../../components/TraceTable'; import { TracePercentile } from '@/models/traces_response'; import LatencyChip from '@/components/LatencyChip'; +import { TraceTreeNode } from '@/models/trace_detail_response'; const breadcrumbItems = [ { name: 'Home', path: '/' }, { name: 'Traces', path: undefined }, ]; -interface Trace { - run_id: string; - name: string; - error?: string; - start_time: string; - end_time?: string; - latency: number; -} - interface TracesProps { - traces: Trace[]; + traces: TraceTreeNode[]; latencyPercentiles: TracePercentile[]; } @@ -62,7 +54,7 @@ const handleRowClick = (run_id: string) => { window.location.href = `/traces/${run_id}`; }; -function getStatusForTrace(trace: Trace): ReactElement { +function getStatusForTrace(trace: TraceTreeNode): ReactElement { if (trace.error) { return
; } else if (trace.end_time) { @@ -134,16 +126,15 @@ const Traces: React.FC = ({ traces, latencyPercentiles }) => {
{ const runDate = convertToDateTime(trace.start_time); - return ( - handleRowClick(trace.run_id)} - className={styles.clickableRow}> - {trace.run_id} - {getStatusForTrace(trace)} - {trace.name} - {runDate.date} @ {runDate.time} - - - ); + return handleRowClick(trace.run_id)} + className={styles.clickableRow}> + {trace.run_id} + {getStatusForTrace(trace)} + {trace.name} + {runDate.date} @ {runDate.time} + + { trace.feedback?.key ? trace.feedback?.key + ": " + trace.feedback?.score : ''} + ; })}/>
From fdff77357fc2ff0d141276b10bbd68aecd3ef866 Mon Sep 17 00:00:00 2001 From: George Herbert Date: Sun, 21 Jan 2024 10:52:47 +0000 Subject: [PATCH 2/3] fix: limit records on /traces --- server/src/repositories/langtrace_repository.ts | 3 ++- ui/src/models/tree/trace_detail_response_tree.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/repositories/langtrace_repository.ts b/server/src/repositories/langtrace_repository.ts index f178ed5..d51c85d 100644 --- a/server/src/repositories/langtrace_repository.ts +++ b/server/src/repositories/langtrace_repository.ts @@ -59,7 +59,8 @@ export class LangtraceRepository { feedback: 1 } }, - { $sort: { start_time: -1 } } + { $sort: { start_time: -1 } }, + { $limit: 100 } ]; const collection = this.db.collection(this.collectionName); diff --git a/ui/src/models/tree/trace_detail_response_tree.ts b/ui/src/models/tree/trace_detail_response_tree.ts index 8ce0ed4..1f9d547 100644 --- a/ui/src/models/tree/trace_detail_response_tree.ts +++ b/ui/src/models/tree/trace_detail_response_tree.ts @@ -1,6 +1,7 @@ export interface TraceDetailResponseParent { run_id: string; name: string; + error: string | null; start_time: string; end_time: string; execution_order: number; @@ -17,6 +18,7 @@ export interface TraceDetailResponseChild { depth: number; run_id: string; execution_order: number; + error: string | null; name: string; start_time: string; end_time: string; From 5433e79a4e4078d0d59774e0d168dc5268f6837a Mon Sep 17 00:00:00 2001 From: George Herbert Date: Tue, 23 Jan 2024 17:13:48 +0000 Subject: [PATCH 3/3] fix: endpoints for /feedback patch --- .../src/models/requests/feedback_request.ts | 12 ++++++++--- .../src/repositories/langtrace_repository.ts | 21 +++++++++++++------ .../ingest/langchain_feedback_router.ts | 13 +++++++----- .../langchain_to_langtrace_service.ts | 19 +++++------------ .../Breadcrumb/Breadcrumb.module.scss | 2 +- .../FilterPanel/FilterPanel.module.scss | 6 ++++-- ui/src/components/FilterPanel/index.tsx | 7 ++++--- .../traceDetailsPanel.module.scss | 9 +++++--- .../TraceTable/TraceTable.module.scss | 13 +++++++----- ui/src/components/TraceTree/index.tsx | 2 +- .../TraceTree/traceTree.module.scss | 12 ++++++----- ui/src/pages/Home.module.scss | 15 ++++++++++++- ui/src/pages/index.tsx | 2 ++ ui/src/pages/traces/index.tsx | 4 ++-- .../pages/traces/traceDetailsPage.module.scss | 5 ++++- ui/src/styles/constants.scss | 11 ++++++---- 16 files changed, 97 insertions(+), 56 deletions(-) diff --git a/server/src/models/requests/feedback_request.ts b/server/src/models/requests/feedback_request.ts index a9dddae..f7718ee 100644 --- a/server/src/models/requests/feedback_request.ts +++ b/server/src/models/requests/feedback_request.ts @@ -1,7 +1,6 @@ export interface CreateFeedback { - // TODO Validate that feedback_id is a UUID - feedback_id: string; - run_id?: string; + id: string; + run_id: string; key: string; score?: number | boolean; value?: string; @@ -9,3 +8,10 @@ export interface CreateFeedback { [key: string]: unknown; } + +export interface UpdateFeedback { + score?: number | boolean; + value?: string; + correction?: { [key: string]: unknown }; + comment?: string; +} diff --git a/server/src/repositories/langtrace_repository.ts b/server/src/repositories/langtrace_repository.ts index d51c85d..41875d9 100644 --- a/server/src/repositories/langtrace_repository.ts +++ b/server/src/repositories/langtrace_repository.ts @@ -3,7 +3,7 @@ import { TraceData } from '../models/requests/trace_request'; import { TraceDetailResponse } from '../models/trace_detail_response'; import 'dotenv/config'; import { TracePercentile } from '../models/traces_percentiles'; -import { CreateFeedback } from '../models/requests/feedback_request'; +import { CreateFeedback, UpdateFeedback } from '../models/requests/feedback_request'; export class LangtraceRepository { @@ -246,14 +246,23 @@ export class LangtraceRepository { } - async insertFeedbackOnTraceByRunId(runId: string, feedback: CreateFeedback) { + async insertFeedbackOnTraceByRunId(feedback: CreateFeedback) { const collection = this.db.collection('traces'); - await collection.updateOne({ run_id: runId }, { $set: { feedback } }); + await collection.updateOne({ run_id: feedback.run_id }, { $set: { feedback } }); } - async updateFeedbackOnTraceByRunId(feedbackData: CreateFeedback): Promise { + async updateFeedbackOnTraceByFeedbackId( + feedbackId: string, + feedbackData: UpdateFeedback): + Promise { const collection = this.db.collection('traces'); - return await collection.updateOne({ 'feedback.feedback_id': feedbackData.feedback_id }, - { $set: { feedback: feedbackData } }); + + const setOperation: Record = {}; + for (const [key, value] of Object.entries(feedbackData)) { + setOperation[`feedback.${key}`] = value; + } + + return await collection.updateOne({ 'feedback.id': feedbackId }, + { $set: setOperation }); } } diff --git a/server/src/routers/ingest/langchain_feedback_router.ts b/server/src/routers/ingest/langchain_feedback_router.ts index f19e738..c35aec5 100644 --- a/server/src/routers/ingest/langchain_feedback_router.ts +++ b/server/src/routers/ingest/langchain_feedback_router.ts @@ -1,5 +1,6 @@ import { Router, Request as ExpressRequest, Response as ExpressResponse } from 'express'; import { LangchainToLangtraceService } from '../../services/langchain_to_langtrace_service'; +import { CreateFeedback, UpdateFeedback } from '../../models/requests/feedback_request'; export const langchainFeedbackRouter = Router(); const langchainService = new LangchainToLangtraceService(); @@ -7,11 +8,13 @@ const langchainService = new LangchainToLangtraceService(); langchainFeedbackRouter.post('/', async (req: ExpressRequest, res: ExpressResponse) => { console.debug('POST /api/feedback'); try { - const runData = req.body; - const feedbackId = await langchainService.createFeedback(runData); - console.debug(`Created feedback with id ${feedbackId} in run ${runData.run_id}`); + const runData = req.body as CreateFeedback; + await langchainService.createFeedback(runData); + + console.debug(`Created feedback with id ${runData.id} in run ${runData.run_id}`); + res.status(201).json( - { feedback_id: feedbackId } + { feedback_id: runData.id } ); } catch (error: unknown) { console.error(error); @@ -26,7 +29,7 @@ langchainFeedbackRouter.post('/', async (req: ExpressRequest, res: ExpressRespon langchainFeedbackRouter.patch('/:feedbackId', async (req: ExpressRequest, res: ExpressResponse) => { console.debug('PATCH /api/feedback/:feedbackId'); const feedbackId = req.params.feedbackId; - const updateData = req.body; + const updateData = req.body as UpdateFeedback; const success = await langchainService.updateFeedback(feedbackId, updateData); if (!success) { diff --git a/server/src/services/langchain_to_langtrace_service.ts b/server/src/services/langchain_to_langtrace_service.ts index 8b10db0..f1508e9 100644 --- a/server/src/services/langchain_to_langtrace_service.ts +++ b/server/src/services/langchain_to_langtrace_service.ts @@ -1,7 +1,6 @@ import { LangtraceRepository } from '../repositories/langtrace_repository'; import { TraceData } from '../models/requests/trace_request'; -import { CreateFeedback } from '../models/requests/feedback_request'; -import { randomUUID } from 'node:crypto'; +import { CreateFeedback, UpdateFeedback } from '../models/requests/feedback_request'; export class LangchainToLangtraceService { private langtraceRepository: LangtraceRepository; @@ -44,24 +43,16 @@ export class LangchainToLangtraceService { throw new Error('run_id is required in data'); } - if (!feedback.feedback_id) { - feedback.feedback_id = randomUUID(); - } - - const runId = feedback.run_id; - delete feedback.run_id; - await this.langtraceRepository.insertFeedbackOnTraceByRunId( - runId, feedback ); - return feedback.feedback_id; + return feedback.id; } - async updateFeedback(feedbackId: string, feedbackData: CreateFeedback): Promise { - feedbackData.feedback_id = feedbackId; + async updateFeedback(feedbackId: string, feedbackData: UpdateFeedback): Promise { return ( - await this.langtraceRepository.updateFeedbackOnTraceByRunId( + await this.langtraceRepository.updateFeedbackOnTraceByFeedbackId( + feedbackId, feedbackData ) ).matchedCount > 0; diff --git a/ui/src/components/Breadcrumb/Breadcrumb.module.scss b/ui/src/components/Breadcrumb/Breadcrumb.module.scss index ea98a57..bbfc7e5 100644 --- a/ui/src/components/Breadcrumb/Breadcrumb.module.scss +++ b/ui/src/components/Breadcrumb/Breadcrumb.module.scss @@ -8,7 +8,7 @@ display: flex; align-items: center; font-size: small; - padding-bottom: 8px; + padding-bottom: 16px; } .breadcrumbItem { diff --git a/ui/src/components/FilterPanel/FilterPanel.module.scss b/ui/src/components/FilterPanel/FilterPanel.module.scss index 9015abc..fadef58 100644 --- a/ui/src/components/FilterPanel/FilterPanel.module.scss +++ b/ui/src/components/FilterPanel/FilterPanel.module.scss @@ -1,10 +1,12 @@ @use '../../styles/constants' as c; .filterPanel { - border-left: c.$border-width solid #ddd; - padding-left: 16px; width: 20%; overflow-x: auto; + background-color: c.$panel-background-color; + border-radius: c.$panel-radius; + box-shadow: 0 0 1px 0 rgba(0, 0, 0, 0.2); + padding: 16px; } .filterPanelChips{ diff --git a/ui/src/components/FilterPanel/index.tsx b/ui/src/components/FilterPanel/index.tsx index c4940c8..ce61658 100644 --- a/ui/src/components/FilterPanel/index.tsx +++ b/ui/src/components/FilterPanel/index.tsx @@ -3,12 +3,12 @@ import styles from './FilterPanel.module.scss'; import { TracePercentile } from '@/models/traces_response'; import PercentileChip from '../PercentileChip'; -interface FilterPanelProps { +interface StatsPanelProps { recordsCount: number; latencyPercentiles: TracePercentile[]; } -const FilterPanel: React.FC = ({ latencyPercentiles, recordsCount }) => { +const StatsPanel: React.FC = ({ latencyPercentiles, recordsCount }) => { return (
@@ -32,8 +32,9 @@ const FilterPanel: React.FC = ({ latencyPercentiles, recordsCo

No latency data available

} +

Feedback

); }; -export default FilterPanel; +export default StatsPanel; diff --git a/ui/src/components/TraceDetailsPanel/traceDetailsPanel.module.scss b/ui/src/components/TraceDetailsPanel/traceDetailsPanel.module.scss index cc2aaf9..e026b42 100644 --- a/ui/src/components/TraceDetailsPanel/traceDetailsPanel.module.scss +++ b/ui/src/components/TraceDetailsPanel/traceDetailsPanel.module.scss @@ -2,7 +2,10 @@ .traceDetailsPanel { width: 70%; - padding-left: 16px; + background-color: c.$panel-background-color; + border-radius: c.$panel-radius; + box-shadow: 0 0 1px 0 rgba(0, 0, 0, 0.2); + padding: 16px; } .title { @@ -11,8 +14,8 @@ } .content { - background-color: c.$background-color; - border: c.$border-width solid c.$border-color; + background-color: c.$panel-background-color; + border: c.$border-width solid rgba(0, 0, 0, 0.2); padding: 8px; border-radius: c.$border-radius; white-space: pre-wrap; diff --git a/ui/src/components/TraceTable/TraceTable.module.scss b/ui/src/components/TraceTable/TraceTable.module.scss index f0cf9b1..59d33d8 100644 --- a/ui/src/components/TraceTable/TraceTable.module.scss +++ b/ui/src/components/TraceTable/TraceTable.module.scss @@ -1,24 +1,27 @@ @use '../../styles/constants' as c; - .tableContainer { width: 80%; overflow-x: auto; + background-color: c.$panel-background-color; + border-radius: c.$panel-radius; + box-shadow: 0 0 1px 0 rgba(0, 0, 0, 0.2); + padding: 16px; } .fullWidthTable { width: 100%; - border: c.$border-width solid c.$border-color; + border: c.$border-width solid c.$border-color-transparent; border-radius: c.$border-radius; border-collapse: separate; border-spacing: 0; th { - border-bottom: c.$border-width solid c.$border-color; + border-bottom: c.$border-width solid c.$border-color-transparent; height: 36px; padding: 8px; text-align: left; - background-color: #f4f4f4; + background-color: c.$background-color; } td { @@ -53,7 +56,7 @@ .dateDropdown { padding: 8px; border-radius: c.$border-radius; - border: c.$border-width solid c.$border-color; + border: c.$border-width solid c.$border-color-transparent; font-family: 'Ubuntu', 'Open Sans', sans-serif; } } diff --git a/ui/src/components/TraceTree/index.tsx b/ui/src/components/TraceTree/index.tsx index bd17534..587e920 100644 --- a/ui/src/components/TraceTree/index.tsx +++ b/ui/src/components/TraceTree/index.tsx @@ -48,7 +48,7 @@ const TraceTree: React.FC = ({ const traceHeaderClass = isSelected ? `${styles.traceHeader} ${styles.active}` : styles.traceHeader; return ( -
+
handleSelectTrace(trace)} className={styles.traceTitle}> {trace.run_type.toUpperCase()} diff --git a/ui/src/components/TraceTree/traceTree.module.scss b/ui/src/components/TraceTree/traceTree.module.scss index 9ee1e1e..8987df8 100644 --- a/ui/src/components/TraceTree/traceTree.module.scss +++ b/ui/src/components/TraceTree/traceTree.module.scss @@ -2,16 +2,18 @@ .traceTree { width: 30%; - padding-right: 16px; - border-right: c.$border-width solid #ddd; + padding: 0 16px; height: calc(100vh - #{c.$breadcrumb-height} - 34px); + background-color: c.$panel-background-color; + border-radius: c.$panel-radius; + box-shadow: 0 0 1px 0 rgba(0, 0, 0, 0.2); } .runTypeBox { font-size: 12px; color: #666; display: inline-block; - border: c.$border-width solid c.$border-color; + border: c.$border-width solid c.$border-color-transparent; border-radius: c.$border-radius; padding: 2px 4px; margin-right: 8px; @@ -62,7 +64,7 @@ .traceChildren { margin-left: 8px; - border-left: 2px solid #ddd; + border-left: 2px solid c.$border-color-bold; padding-left: 10px; } @@ -74,5 +76,5 @@ .active { border-radius: c.$border-radius; - border: c.$border-width solid #333333; + border: c.$border-width * 2 solid c.$border-color-bold; } diff --git a/ui/src/pages/Home.module.scss b/ui/src/pages/Home.module.scss index 8ff142e..f8ac9d5 100644 --- a/ui/src/pages/Home.module.scss +++ b/ui/src/pages/Home.module.scss @@ -8,6 +8,18 @@ height: 100vh; } +.homePanel { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + width: 50%; + padding: 32px; + border-radius: c.$panel-radius; + background-color: c.$panel-background-color; + box-shadow: 0 0 1px 0 rgba(0, 0, 0, 0.2); +} + .button { display: flex; align-items: center; @@ -15,7 +27,8 @@ padding: 8px; margin-top: 32px; border-radius: c.$border-radius; - border: c.$border-width solid c.$border-color; + border: c.$border-width * 2 solid c.$border-color-bold; + box-shadow: 0 0 1px 0 rgba(0, 0, 0, 0.2); background-color: transparent; cursor: pointer; color: c.$primary-color; diff --git a/ui/src/pages/index.tsx b/ui/src/pages/index.tsx index 4c6ed3a..5106aae 100644 --- a/ui/src/pages/index.tsx +++ b/ui/src/pages/index.tsx @@ -5,12 +5,14 @@ import {Hierarchy} from "iconic-react"; const Home = () => { return (
+

LangTrace

View your Langchain data
Traces +
); }; diff --git a/ui/src/pages/traces/index.tsx b/ui/src/pages/traces/index.tsx index 2a574b3..b1475cc 100644 --- a/ui/src/pages/traces/index.tsx +++ b/ui/src/pages/traces/index.tsx @@ -10,7 +10,7 @@ import { import { IconType } from 'react-icons/lib'; import { useRouter } from 'next/router'; import { GetServerSidePropsContext } from 'next'; -import FilterPanel from '@/components/FilterPanel'; +import StatsPanel from '@/components/FilterPanel'; import TraceTable from '../../components/TraceTable'; import { TracePercentile } from '@/models/traces_response'; import LatencyChip from '@/components/LatencyChip'; @@ -136,7 +136,7 @@ const Traces: React.FC = ({ traces, latencyPercentiles }) => { { trace.feedback?.key ? trace.feedback?.key + ": " + trace.feedback?.score : ''} ; })}/> - +
); diff --git a/ui/src/pages/traces/traceDetailsPage.module.scss b/ui/src/pages/traces/traceDetailsPage.module.scss index 601b27e..4e4e7e5 100644 --- a/ui/src/pages/traces/traceDetailsPage.module.scss +++ b/ui/src/pages/traces/traceDetailsPage.module.scss @@ -1,6 +1,9 @@ +@use '../../styles/constants' as c; + + .traceDetailsContainer { display: flex; - gap: 0; + gap: 16px; } @media (max-width: 768px) { diff --git a/ui/src/styles/constants.scss b/ui/src/styles/constants.scss index b324201..d7a22ab 100644 --- a/ui/src/styles/constants.scss +++ b/ui/src/styles/constants.scss @@ -1,12 +1,15 @@ $black: #000; $white: #fff; -$background-color: $white; +$background-color: #eee; +$panel-background-color: $white; $primary-color: $black; -$border-radius: 4px; -$border-width: 2px; -$border-color: $black; +$border-radius: 8px; +$panel-radius: 12px; +$border-width: 1px; +$border-color-bold: rgba(0, 0, 0); +$border-color-transparent: rgba($border-color-bold, 0.2); $text-color: $black;