From dc8737a870f8573ea75cbf44f18de82e04e28581 Mon Sep 17 00:00:00 2001 From: Danilo Pejakovic Date: Thu, 9 Jan 2025 07:56:49 +0100 Subject: [PATCH] adding youtube tool --- .env.example | 4 + api/app/clients/tools/index.js | 2 + api/app/clients/tools/manifest.json | 13 ++ api/app/clients/tools/structured/YouTube.js | 218 ++++++++++++++++++++ api/app/clients/tools/util/handleTools.js | 3 + api/package.json | 2 + package-lock.json | 23 +++ 7 files changed, 265 insertions(+) create mode 100644 api/app/clients/tools/structured/YouTube.js diff --git a/.env.example b/.env.example index f2a51198f42..bafdadad58f 100644 --- a/.env.example +++ b/.env.example @@ -261,6 +261,10 @@ AZURE_AI_SEARCH_SEARCH_OPTION_SELECT= GOOGLE_SEARCH_API_KEY= GOOGLE_CSE_ID= +# YOUTUBE +#----------------- +YOUTUBE_API_KEY= + # SerpAPI #----------------- SERPAPI_API_KEY= diff --git a/api/app/clients/tools/index.js b/api/app/clients/tools/index.js index a8532d4581f..0960970bed5 100644 --- a/api/app/clients/tools/index.js +++ b/api/app/clients/tools/index.js @@ -8,6 +8,7 @@ const StructuredSD = require('./structured/StableDiffusion'); const GoogleSearchAPI = require('./structured/GoogleSearch'); const TraversaalSearch = require('./structured/TraversaalSearch'); const TavilySearchResults = require('./structured/TavilySearchResults'); +const YouTubeTool = require('./structured/YouTube'); module.exports = { availableTools, @@ -19,4 +20,5 @@ module.exports = { TraversaalSearch, StructuredWolfram, TavilySearchResults, + YouTubeTool, }; diff --git a/api/app/clients/tools/manifest.json b/api/app/clients/tools/manifest.json index d2748cdea11..bf7c57002b5 100644 --- a/api/app/clients/tools/manifest.json +++ b/api/app/clients/tools/manifest.json @@ -30,6 +30,19 @@ } ] }, + { + "name": "YouTube", + "pluginKey": "youtube", + "description": "Get YouTube video information, retrieve comments, analyze transcripts and search for videos.", + "icon": "https://www.youtube.com/s/desktop/7449ebf7/img/favicon_144x144.png", + "authConfig": [ + { + "authField": "YOUTUBE_API_KEY", + "label": "YouTube API Key", + "description": "Your YouTube Data API v3 key." + } + ] + }, { "name": "Wolfram", "pluginKey": "wolfram", diff --git a/api/app/clients/tools/structured/YouTube.js b/api/app/clients/tools/structured/YouTube.js new file mode 100644 index 00000000000..46b8cbf80d9 --- /dev/null +++ b/api/app/clients/tools/structured/YouTube.js @@ -0,0 +1,218 @@ +const { Tool } = require('@langchain/core/tools'); +const { z } = require('zod'); +const { youtube } = require('@googleapis/youtube'); +const { YoutubeTranscript } = require('youtube-transcript'); +const { logger } = require('~/config'); + +class YouTubeTool extends Tool { + constructor(fields) { + super(); + this.name = 'youtube'; + /** @type {boolean} Used to initialize the Tool without necessary variables. */ + this.override = fields.override ?? false; + let apiKey = fields.YOUTUBE_API_KEY ?? this.getApiKey(); + this.apiKey = apiKey; + this.description = 'Tool for interacting with YouTube content and data.'; + + this.description_for_model = `// YouTube Content Tool - READ CAREFULLY +// This tool has four SEPARATE operations that must follow these rules: + +// 1. SEARCH VIDEOS: +// - Action: search_videos +// - Required: query (your search term) +// - Optional: maxResults (between 1-50, default: 5) +// - Usecases: finding relevant videos, exploring new content, or getting recommendations +// - Example command: search for "cooking pasta" with max 5 results + +// 2. GET VIDEO INFO: +// - Action: get_video_info +// - Required: videoUrl (full YouTube URL or video ID) +// - Example command: get info for video "https://youtube.com/watch?v=123" + +// 3. GET COMMENTS: +// - Action: get_comments +// - Required: videoUrl (full YouTube URL or video ID) +// - Optional: maxResults (between 1-50, default: 10) +// - Usecases: analyzing user feedback, identifying common issues, or understanding viewer sentiment +// - Example command: get 10 comments from video "https://youtube.com/watch?v=123" + +// 4. GET TRANSCRIPT: +// - Action: get_video_transcript +// - Required: videoUrl (full YouTube URL or video ID) +// - Optional: Usecases: in-depth analysis, summarization, or translation of a video +// - Example command: get transcript for video "https://youtube.com/watch?v=123" + +// CRITICAL RULES: +// - One action per request ONLY +// - Never mix different operations +// - maxResults must be between 1-50 +// - Video URLs can be full links or video IDs +// - All responses are JSON strings +// - Keep requests focused on one action at a time`; + + this.schema = z.object({ + action: z + .enum(['search_videos', 'get_video_info', 'get_comments', 'get_video_transcript']) + .describe('The action to perform. Choose one of the available YouTube operations.'), + query: z + .string() + .optional() + .describe('Required for search_videos: The search term to find videos.'), + videoUrl: z + .string() + .optional() + .describe( + 'Required for video_info, comments, and transcript: The YouTube video URL or ID.', + ), + maxResults: z + .number() + .int() + .min(1) + .max(50) + .optional() + .describe( + 'Optional: Number of results to return. Default: 5 for search, 10 for comments. Max: 50.', + ), + }); + + this.youtubeClient = youtube({ + version: 'v3', + auth: this.apiKey, + }); + } + + getApiKey() { + const apiKey = process.env.YOUTUBE_API_KEY ?? ''; + if (!apiKey && !this.override) { + throw new Error('Missing YOUTUBE_API_KEY environment variable.'); + } + return apiKey; + } + + async _call(input) { + try { + const data = typeof input === 'string' ? JSON.parse(input) : input; + + switch (data.action) { + case 'search_videos': + return JSON.stringify(await this.searchVideos(data.query, data.maxResults)); + case 'get_video_info': + return JSON.stringify(await this.getVideoInfo(data.videoUrl)); + case 'get_comments': + return JSON.stringify(await this.getComments(data.videoUrl, data.maxResults)); + case 'get_video_transcript': + return JSON.stringify(await this.getVideoTranscript(data.videoUrl)); + default: + throw new Error(`Unknown action: ${data.action}`); + } + } catch (error) { + logger.error('[YouTubeTool] Error:', error); + return JSON.stringify({ error: error.message }); + } + } + + async searchVideos(query, maxResults = 5) { + const response = await this.youtubeClient.search.list({ + part: 'snippet', + q: query, + type: 'video', + maxResults, + }); + + return response.data.items.map((item) => ({ + title: item.snippet.title, + description: item.snippet.description, + url: `https://www.youtube.com/watch?v=${item.id.videoId}`, + })); + } + + async getVideoInfo(videoUrl) { + const videoId = this.extractVideoId(videoUrl); + const response = await this.youtubeClient.videos.list({ + part: 'snippet,statistics', + id: videoId, + }); + + const video = response.data.items[0]; + return { + title: video.snippet.title, + description: video.snippet.description, + views: video.statistics.viewCount, + likes: video.statistics.likeCount, + comments: video.statistics.commentCount, + }; + } + + async getComments(videoUrl, maxResults = 10) { + const videoId = this.extractVideoId(videoUrl); + const response = await this.youtubeClient.commentThreads.list({ + part: 'snippet', + videoId: videoId, + maxResults: maxResults, + }); + + return response.data.items.map((item) => ({ + author: item.snippet.topLevelComment.snippet.authorDisplayName, + text: item.snippet.topLevelComment.snippet.textDisplay, + likes: item.snippet.topLevelComment.snippet.likeCount, + })); + } + + async getVideoTranscript(videoUrl) { + const videoId = this.extractVideoId(videoUrl); + try { + // Try to fetch English transcript (most common language) + try { + const englishTranscript = await YoutubeTranscript.fetchTranscript(videoId, { lang: 'en' }); + return { + language: 'English', + transcript: englishTranscript, + }; + } catch (error) { + console.log('English transcript not available, trying German...'); + } + + // If English is not available, try German (very common other language if English is not available) + try { + const germanTranscript = await YoutubeTranscript.fetchTranscript(videoId, { lang: 'de' }); + return { + language: 'German', + transcript: germanTranscript, + }; + } catch (error) { + console.log('German transcript not available, fetching default transcript...'); + } + + // If neither is available, try fetch any transcript (which will be alpabetically first in the list) + const defaultTranscript = await YoutubeTranscript.fetchTranscript(videoId); + return { + language: 'Unknown', + transcript: defaultTranscript, + }; + } catch (error) { + console.error('Error fetching transcript:', error); + return { + error: `Error fetching transcript: ${error.message}`, + }; + } + } + + extractVideoId(url) { + // First, try to match a raw video ID (11 characters) + const rawIdRegex = /^[a-zA-Z0-9_-]{11}$/; + if (rawIdRegex.test(url)) { + return url; + } + + // Then try various URL formats + + const regex = new RegExp( + '(?:youtu\\.be/|youtube(?:\\.com)?/(?:(?:watch\\?v=)|(?:embed/)|' + + '(?:shorts/)|(?:live/)|(?:v/)|(?:/))?)([a-zA-Z0-9_-]{11})(?:\\S+)?$', + ); + const match = url.match(regex); + return match ? match[1] : null; + } +} + +module.exports = YouTubeTool; diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.js index a8ee50c3d4d..61272f20717 100644 --- a/api/app/clients/tools/util/handleTools.js +++ b/api/app/clients/tools/util/handleTools.js @@ -14,6 +14,7 @@ const { TraversaalSearch, StructuredWolfram, TavilySearchResults, + YouTubeTool, } = require('../'); const { primeFiles: primeCodeFiles } = require('~/server/services/Files/Code/process'); const { createFileSearchTool, primeFiles: primeSearchFiles } = require('./fileSearch'); @@ -178,6 +179,7 @@ const loadTools = async ({ 'azure-ai-search': StructuredACS, traversaal_search: TraversaalSearch, tavily_search_results_json: TavilySearchResults, + youtube: YouTubeTool, }; const customConstructors = { @@ -214,6 +216,7 @@ const loadTools = async ({ serpapi: { location: 'Austin,Texas,United States', hl: 'en', gl: 'us' }, dalle: imageGenOptions, 'stable-diffusion': imageGenOptions, + youtube: { YOUTUBE_API_KEY: process.env.YOUTUBE_API_KEY }, }; const toolAuthFields = {}; diff --git a/api/package.json b/api/package.json index e364b68eb9e..979e0819831 100644 --- a/api/package.json +++ b/api/package.json @@ -37,6 +37,7 @@ "@anthropic-ai/sdk": "^0.32.1", "@azure/search-documents": "^12.0.0", "@google/generative-ai": "^0.21.0", + "@googleapis/youtube": "^20.0.0", "@keyv/mongo": "^2.1.8", "@keyv/redis": "^2.8.1", "@langchain/community": "^0.3.14", @@ -103,6 +104,7 @@ "ua-parser-js": "^1.0.36", "winston": "^3.11.0", "winston-daily-rotate-file": "^4.7.1", + "youtube-transcript": "^1.2.1", "zod": "^3.22.4" }, "devDependencies": { diff --git a/package-lock.json b/package-lock.json index 21bd22666be..5e668486e9a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46,6 +46,7 @@ "@anthropic-ai/sdk": "^0.32.1", "@azure/search-documents": "^12.0.0", "@google/generative-ai": "^0.21.0", + "@googleapis/youtube": "^20.0.0", "@keyv/mongo": "^2.1.8", "@keyv/redis": "^2.8.1", "@langchain/community": "^0.3.14", @@ -112,6 +113,7 @@ "ua-parser-js": "^1.0.36", "winston": "^3.11.0", "winston-daily-rotate-file": "^4.7.1", + "youtube-transcript": "^1.2.1", "zod": "^3.22.4" }, "devDependencies": { @@ -8439,6 +8441,18 @@ "node": ">=18.0.0" } }, + "node_modules/@googleapis/youtube": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@googleapis/youtube/-/youtube-20.0.0.tgz", + "integrity": "sha512-wdt1J0JoKYhvpoS2XIRHX0g/9ul/B0fQeeJAhuuBIdYINuuLt6/oZYZZCBmkuhtkA3IllXgqgAXOjLtLRAnR2g==", + "license": "Apache-2.0", + "dependencies": { + "googleapis-common": "^7.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/@grpc/grpc-js": { "version": "1.9.15", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.9.15.tgz", @@ -36259,6 +36273,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/youtube-transcript": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/youtube-transcript/-/youtube-transcript-1.2.1.tgz", + "integrity": "sha512-TvEGkBaajKw+B6y91ziLuBLsa5cawgowou+Bk0ciGpjELDfAzSzTGXaZmeSSkUeknCPpEr/WGApOHDwV7V+Y9Q==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/zod": { "version": "3.23.8", "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz",