From 0a72ea7794f8d8a0ee87f1b93b74fe2f32629f00 Mon Sep 17 00:00:00 2001 From: Ethan Palm <56270045+ethanpalm@users.noreply.github.com> Date: Thu, 14 Nov 2024 10:45:07 -0800 Subject: [PATCH 1/3] Update `codeowners-content-strategy` workflow (#53107) --- ...-content-strategy.yml => codeowners-content-systems.yml} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename .github/workflows/{codeowners-content-strategy.yml => codeowners-content-systems.yml} (86%) diff --git a/.github/workflows/codeowners-content-strategy.yml b/.github/workflows/codeowners-content-systems.yml similarity index 86% rename from .github/workflows/codeowners-content-strategy.yml rename to .github/workflows/codeowners-content-systems.yml index f89f1f85368a..9f1a630d914a 100644 --- a/.github/workflows/codeowners-content-strategy.yml +++ b/.github/workflows/codeowners-content-systems.yml @@ -11,14 +11,14 @@ on: - 'content/contributing/**.md' jobs: - codeowners-content-strategy: + codeowners-content-systems: if: ${{ github.repository == 'github/docs-internal' }} runs-on: ubuntu-latest steps: - name: Check out repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Add Content Strategy as a reviewer + - name: Add Content Systems as a reviewer env: GH_TOKEN: ${{ secrets.DOCS_BOT_PAT_WRITEORG_PROJECT }} PR: ${{ github.event.pull_request.html_url }} @@ -29,5 +29,5 @@ jobs: ) if ! $has_reviewer then - gh pr edit $PR --add-reviewer github/docs-content-strategy + gh pr edit $PR --add-reviewer github/docs-content-systems fi From d35127dfa9d34415fa71fdad34c6551db9cf2534 Mon Sep 17 00:00:00 2001 From: Kevin Heis Date: Thu, 14 Nov 2024 11:17:43 -0800 Subject: [PATCH 2/3] Typescript src/assets (#53110) --- package-lock.json | 27 ++++++++++--------- package.json | 4 +-- src/assets/lib/image-density.d.ts | 1 + ...nt-1.js => deleted-assets-pr-comment-1.ts} | 15 +++++++---- ...omment.js => deleted-assets-pr-comment.ts} | 22 ++++++++++----- ...aned-assets.js => find-orphaned-assets.ts} | 14 +++++++--- ...ist-image-sizes.js => list-image-sizes.ts} | 22 ++++++--------- ...set-images.js => validate-asset-images.ts} | 21 +++++++++------ .../{static-assets.js => static-assets.ts} | 2 +- tsconfig.json | 12 ++------- 10 files changed, 78 insertions(+), 62 deletions(-) create mode 100644 src/assets/lib/image-density.d.ts rename src/assets/scripts/{deleted-assets-pr-comment-1.js => deleted-assets-pr-comment-1.ts} (74%) rename src/assets/scripts/{deleted-assets-pr-comment.js => deleted-assets-pr-comment.ts} (76%) rename src/assets/scripts/{find-orphaned-assets.js => find-orphaned-assets.ts} (95%) rename src/assets/scripts/{list-image-sizes.js => list-image-sizes.ts} (62%) rename src/assets/scripts/{validate-asset-images.js => validate-asset-images.ts} (90%) rename src/assets/tests/{static-assets.js => static-assets.ts} (98%) diff --git a/package-lock.json b/package-lock.json index 44c695262210..601463118080 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,7 +38,7 @@ "express": "4.21.1", "express-rate-limit": "7.4.0", "fastest-levenshtein": "1.0.16", - "file-type": "19.4.1", + "file-type": "19.6.0", "flat": "^6.0.1", "github-slugger": "^2.0.0", "glob": "11.0.0", @@ -6880,12 +6880,13 @@ } }, "node_modules/file-type": { - "version": "19.4.1", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-19.4.1.tgz", - "integrity": "sha512-RuWzwF2L9tCHS76KR/Mdh+DwJZcFCzrhrPXpOw6MlEfl/o31fjpTikzcKlYuyeV7e7ftdCGVJTNOCzkYD/aLbw==", + "version": "19.6.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-19.6.0.tgz", + "integrity": "sha512-VZR5I7k5wkD0HgFnMsq5hOsSc710MJMu5Nc5QYsbe38NN5iPV/XTObYLc/cpttRTf6lX538+5uO1ZQRhYibiZQ==", + "license": "MIT", "dependencies": { "get-stream": "^9.0.1", - "strtok3": "^8.1.0", + "strtok3": "^9.0.1", "token-types": "^6.0.0", "uint8array-extras": "^1.3.0" }, @@ -11546,9 +11547,10 @@ } }, "node_modules/peek-readable": { - "version": "5.1.4", - "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.1.4.tgz", - "integrity": "sha512-E7mY2VmKqw9jYuXrSWGHFuPCW2SLQenzXLF3amGaY6lXXg4/b3gj5HVM7h8ZjCO/nZS9ICs0Cz285+32FvNd/A==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.3.1.tgz", + "integrity": "sha512-GVlENSDW6KHaXcd9zkZltB7tCLosKB/4Hg0fqBJkAoBgYG2Tn1xtMgXtSUuMU9AK/gCm/tTdT8mgAeF4YNeeqw==", + "license": "MIT", "engines": { "node": ">=14.16" }, @@ -13836,12 +13838,13 @@ "license": "MIT" }, "node_modules/strtok3": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-8.1.0.tgz", - "integrity": "sha512-ExzDvHYPj6F6QkSNe/JxSlBxTh3OrI6wrAIz53ulxo1c4hBJ1bT9C/JrAthEKHWG9riVH3Xzg7B03Oxty6S2Lw==", + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-9.0.1.tgz", + "integrity": "sha512-ERPW+XkvX9W2A+ov07iy+ZFJpVdik04GhDA4eVogiG9hpC97Kem2iucyzhFxbFRvQ5o2UckFtKZdp1hkGvnrEw==", + "license": "MIT", "dependencies": { "@tokenizer/token": "^0.3.0", - "peek-readable": "^5.1.4" + "peek-readable": "^5.3.1" }, "engines": { "node": ">=16" diff --git a/package.json b/package.json index 569dfee80c0b..e44d8053a5de 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "delete-orphan-translation-files": "tsx src/workflows/delete-orphan-translation-files.ts", "deleted-features-pr-comment": "tsx src/data-directory/scripts/deleted-features-pr-comment.ts", "dev": "cross-env npm start", - "find-orphaned-assets": "node src/assets/scripts/find-orphaned-assets.js", + "find-orphaned-assets": "tsx src/assets/scripts/find-orphaned-assets.ts", "find-orphaned-features": "tsx src/data-directory/scripts/find-orphaned-features/index.ts", "find-past-built-pr": "tsx src/workflows/find-past-built-pr.ts", "find-unused-variables": "tsx src/content-linter/scripts/find-unsed-variables.ts", @@ -259,7 +259,7 @@ "express": "4.21.1", "express-rate-limit": "7.4.0", "fastest-levenshtein": "1.0.16", - "file-type": "19.4.1", + "file-type": "19.6.0", "flat": "^6.0.1", "github-slugger": "^2.0.0", "glob": "11.0.0", diff --git a/src/assets/lib/image-density.d.ts b/src/assets/lib/image-density.d.ts new file mode 100644 index 000000000000..dbcbd318b1a3 --- /dev/null +++ b/src/assets/lib/image-density.d.ts @@ -0,0 +1 @@ +export const IMAGE_DENSITY: Record diff --git a/src/assets/scripts/deleted-assets-pr-comment-1.js b/src/assets/scripts/deleted-assets-pr-comment-1.ts similarity index 74% rename from src/assets/scripts/deleted-assets-pr-comment-1.js rename to src/assets/scripts/deleted-assets-pr-comment-1.ts index 0adc48c7264d..206143821033 100755 --- a/src/assets/scripts/deleted-assets-pr-comment-1.js +++ b/src/assets/scripts/deleted-assets-pr-comment-1.ts @@ -18,14 +18,19 @@ // [end-readme] import { program } from 'commander' -import main from './deleted-assets-pr-comment.js' +import main from './deleted-assets-pr-comment' program .description('If applicable, print a snippet of Markdown about deleted assets') - .arguments('owner repo base_sha head_sha', 'Simulate what the Actions workflow does') + .arguments('owner repo base_sha head_sha') .parse(process.argv) -const opts = program.opts() -const args = program.args +type MainArgs = { + owner: string + repo: string + baseSHA: string + headSHA: string +} +const opts = program.opts() as MainArgs -console.log(await main(...args, { ...opts })) +console.log(await main(opts)) diff --git a/src/assets/scripts/deleted-assets-pr-comment.js b/src/assets/scripts/deleted-assets-pr-comment.ts similarity index 76% rename from src/assets/scripts/deleted-assets-pr-comment.js rename to src/assets/scripts/deleted-assets-pr-comment.ts index 3aa9352478f1..561fb461b0c6 100755 --- a/src/assets/scripts/deleted-assets-pr-comment.js +++ b/src/assets/scripts/deleted-assets-pr-comment.ts @@ -13,16 +13,22 @@ if (!GITHUB_TOKEN) { // When this file is invoked directly from action as opposed to being imported if (import.meta.url.endsWith(process.argv[1])) { const owner = context.repo.owner - const repo = context.payload.repository.name - const baseSHA = context.payload.pull_request.base.sha - const headSHA = context.payload.pull_request.head.sha + const repo = context.payload.repository?.name || '' + const baseSHA = context.payload.pull_request?.base.sha + const headSHA = context.payload.pull_request?.head.sha - const markdown = await main(owner, repo, baseSHA, headSHA) + const markdown = await main({ owner, repo, baseSHA, headSHA }) core.setOutput('markdown', markdown) } -async function main(owner, repo, baseSHA, headSHA) { - const octokit = github.getOctokit(GITHUB_TOKEN) +type MainArgs = { + owner: string + repo: string + baseSHA: string + headSHA: string +} +async function main({ owner, repo, baseSHA, headSHA }: MainArgs) { + const octokit = github.getOctokit(GITHUB_TOKEN as string) // get the list of file changes from the PR const response = await octokit.rest.repos.compareCommitsWithBasehead({ owner, @@ -32,6 +38,10 @@ async function main(owner, repo, baseSHA, headSHA) { const { files } = response.data + if (!files) { + throw new Error('No files found in the PR') + } + const oldFilenames = [] for (const file of files) { const { filename, status } = file diff --git a/src/assets/scripts/find-orphaned-assets.js b/src/assets/scripts/find-orphaned-assets.ts similarity index 95% rename from src/assets/scripts/find-orphaned-assets.js rename to src/assets/scripts/find-orphaned-assets.ts index bf266fe17312..07885edf5f38 100755 --- a/src/assets/scripts/find-orphaned-assets.js +++ b/src/assets/scripts/find-orphaned-assets.ts @@ -32,7 +32,7 @@ const EXCEPTIONS = new Set([ 'assets/images/site/apple-touch-icon-76x76.png', ]) -function isExceptionPath(imagePath) { +function isExceptionPath(imagePath: string) { // We also check for .DS_Store because any macOS user that has opened // a folder with images will have this on disk. It won't get added // to git anyway thanks to our .DS_Store. @@ -53,9 +53,15 @@ program .option('--exclude-translations', "Don't search in translations/") .parse(process.argv) -main(program.opts(), program.args) +type MainOptions = { + json: boolean + verbose: boolean + exit: boolean + excludeTranslations: boolean +} +main(program.opts()) -async function main(opts) { +async function main(opts: MainOptions) { const { json, verbose, exit, excludeTranslations } = opts const walkOptions = { @@ -164,7 +170,7 @@ async function main(opts) { } } -function getTotalDiskSize(filePaths) { +function getTotalDiskSize(filePaths: Set) { let sum = 0 for (const filePath of filePaths) { sum += fs.statSync(filePath).size diff --git a/src/assets/scripts/list-image-sizes.js b/src/assets/scripts/list-image-sizes.ts similarity index 62% rename from src/assets/scripts/list-image-sizes.js rename to src/assets/scripts/list-image-sizes.ts index 974c748dd021..e343dacdd4b1 100755 --- a/src/assets/scripts/list-image-sizes.js +++ b/src/assets/scripts/list-image-sizes.ts @@ -10,32 +10,26 @@ import { fileURLToPath } from 'url' import path from 'path' import walk from 'walk-sync' import sharp from 'sharp' -import { chain } from 'lodash-es' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const imagesPath = path.join(__dirname, '../assets/images') const imagesExtensions = ['.jpg', '.jpeg', '.png', '.gif'] -const files = chain(walk(imagesPath, { directories: false })).filter((relativePath) => { +const files = walk(imagesPath, { directories: false }).filter((relativePath) => { return imagesExtensions.includes(path.extname(relativePath.toLowerCase())) }) -const infos = await Promise.all( +const images = await Promise.all( files.map(async (relativePath) => { const fullPath = path.join(imagesPath, relativePath) const image = sharp(fullPath) const { width, height } = await image.metadata() - const size = width * height + const size = (width || 0) * (height || 0) return { relativePath, width, height, size } }), ) -const images = files - .map((relativePath, i) => { - return { relativePath, ...infos[i] } +images + .sort((a, b) => b.size - a.size) + .forEach((image) => { + const { relativePath, width, height } = image + console.log(`${width} x ${height} - ${relativePath}`) }) - .orderBy('size', 'desc') - .value() - -images.forEach((image) => { - const { relativePath, width, height } = image - console.log(`${width} x ${height} - ${relativePath}`) -}) diff --git a/src/assets/scripts/validate-asset-images.js b/src/assets/scripts/validate-asset-images.ts similarity index 90% rename from src/assets/scripts/validate-asset-images.js rename to src/assets/scripts/validate-asset-images.ts index 1481ca3b407c..4e9052488b1d 100755 --- a/src/assets/scripts/validate-asset-images.js +++ b/src/assets/scripts/validate-asset-images.ts @@ -19,6 +19,7 @@ import path from 'path' import { program } from 'commander' import chalk from 'chalk' import cheerio from 'cheerio' +// @ts-ignore see https://github.com/sindresorhus/file-type/issues/652 import { fileTypeFromFile } from 'file-type' import walk from 'walk-sync' import isSVG from 'is-svg' @@ -43,7 +44,7 @@ const EXPECT = { '.ico': 'image/x-icon', '.pdf': 'application/pdf', '.webp': 'image/webp', -} +} as Record const CRITICAL = 'critical' const WARNING = 'warning' @@ -56,7 +57,7 @@ program main(program.opts()) -async function main(opts) { +async function main(opts: { dryRun: boolean; verbose: boolean }) { let errors = 0 const files = walk(ASSETS_ROOT, { includeBasePath: true, directories: false }).filter( @@ -71,7 +72,11 @@ async function main(opts) { ) }, ) - const results = (await Promise.all(files.map(checkFile))).filter(Boolean) + const results = (await Promise.all(files.map(checkFile))).filter(Boolean) as [ + level: string, + filePath: string, + error: string, + ][] for (const [level, filePath, error] of results) { console.log( level === CRITICAL ? chalk.red(level) : chalk.yellow(level), @@ -94,7 +99,7 @@ async function main(opts) { process.exitCode = errors } -async function checkFile(filePath) { +async function checkFile(filePath: string) { const ext = path.extname(filePath) const { size } = await fs.stat(filePath) @@ -113,7 +118,7 @@ async function checkFile(filePath) { } try { checkSVGContent(content) - } catch (error) { + } catch (error: any) { return [CRITICAL, filePath, error.message] } } else if (EXPECT[ext]) { @@ -135,15 +140,15 @@ async function checkFile(filePath) { // All is well. Nothing to complain about. } -function checkSVGContent(content) { +function checkSVGContent(content: string) { const $ = cheerio.load(content) const disallowedTagNames = new Set(['script', 'object', 'iframe', 'embed']) $('*').each((i, element) => { - const { tagName } = element + const { tagName } = $(element).get(0) if (disallowedTagNames.has(tagName)) { throw new Error(`contains a <${tagName}> tag`) } - for (const key in element.attribs) { + for (const key in $(element).get(0).attribs) { // Looks for suspicious event handlers on tags. // For example ` Date: Thu, 14 Nov 2024 14:29:34 -0500 Subject: [PATCH 3/3] Implementation of API between Docs and CSE Copilot (#52892) Co-authored-by: Evan Bonsignori Co-authored-by: Evan Bonsignori --- .env.example | 7 +- package-lock.json | 11 +- src/frame/middleware/api.ts | 18 +++ src/search/lib/ai-search-proxy.ts | 78 +++++++++ .../lib/helpers/cse-copilot-docs-versions.ts | 44 ++++++ .../lib/helpers/get-cse-copilot-auth.ts | 24 +++ src/search/middleware/ai-search.ts | 20 +++ src/search/tests/api-ai-search.ts | 148 ++++++++++++++++++ src/tests/mocks/cse-copilot-mock.ts | 71 +++++++++ src/tests/mocks/start-mock-server.ts | 73 +++++++++ src/tests/vitest.setup.ts | 2 + 11 files changed, 490 insertions(+), 6 deletions(-) create mode 100644 src/search/lib/ai-search-proxy.ts create mode 100644 src/search/lib/helpers/cse-copilot-docs-versions.ts create mode 100644 src/search/lib/helpers/get-cse-copilot-auth.ts create mode 100644 src/search/middleware/ai-search.ts create mode 100644 src/search/tests/api-ai-search.ts create mode 100644 src/tests/mocks/cse-copilot-mock.ts create mode 100644 src/tests/mocks/start-mock-server.ts diff --git a/.env.example b/.env.example index 2387db4d6922..4a05a2416fb7 100644 --- a/.env.example +++ b/.env.example @@ -20,4 +20,9 @@ BUILD_RECORDS_MAX_CONCURRENT=100 BUILD_RECORDS_MIN_TIME= # Set to true to enable the /fastly-cache-test route for debugging Fastly headers -ENABLE_FASTLY_TESTING= \ No newline at end of file +ENABLE_FASTLY_TESTING= + +# Needed to auth for AI search +CSE_COPILOT_SECRET= +CSE_COPILOT_ENDPOINT=https://cse-copilot-staging.service.iad.github.net + diff --git a/package-lock.json b/package-lock.json index 601463118080..f293871e6357 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11759,9 +11759,10 @@ "license": "MIT" }, "node_modules/punycode": { - "version": "2.1.1", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", "dev": true, - "license": "MIT", "engines": { "node": ">=6" } @@ -14272,9 +14273,9 @@ } }, "node_modules/type-fest": { - "version": "4.23.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.23.0.tgz", - "integrity": "sha512-ZiBujro2ohr5+Z/hZWHESLz3g08BBdrdLMieYFULJO+tWc437sn8kQsWLJoZErY8alNhxre9K4p3GURAG11n+w==", + "version": "4.26.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.26.1.tgz", + "integrity": "sha512-yOGpmOAL7CkKe/91I5O3gPICmJNLJ1G4zFYVAsRHg7M64biSnPtRj0WNQt++bRkjYOqjWXrhnUw1utzmVErAdg==", "engines": { "node": ">=16" }, diff --git a/src/frame/middleware/api.ts b/src/frame/middleware/api.ts index 62d77ef61916..33ff9c8bb097 100644 --- a/src/frame/middleware/api.ts +++ b/src/frame/middleware/api.ts @@ -3,6 +3,7 @@ import { createProxyMiddleware } from 'http-proxy-middleware' import events from '@/events/middleware.js' import anchorRedirect from '@/rest/api/anchor-redirect.js' +import aiSearch from '@/search/middleware/ai-search' import search from '@/search/middleware/search-routes.js' import pageInfo from '@/pageinfo/middleware' import pageList from '@/pagelist/middleware' @@ -23,6 +24,23 @@ router.use('/pagelist', pageList) // local laptop, they don't have an Elasticsearch. Neither a running local // server or the known credentials to a remote Elasticsearch. Whenever // that's the case, they can just HTTP proxy to the production server. +if (process.env.CSE_COPILOT_ENDPOINT || process.env.NODE_ENV === 'test') { + router.use('/ai-search', aiSearch) +} else { + console.log( + 'Proxying AI Search requests to docs.github.com. To use the cse-copilot endpoint, set the CSE_COPILOT_ENDPOINT environment variable.', + ) + router.use( + '/ai-search', + createProxyMiddleware({ + target: 'https://docs.github.com', + changeOrigin: true, + pathRewrite: function (path, req: ExtendedRequest) { + return req.originalUrl + }, + }), + ) +} if (process.env.ELASTICSEARCH_URL) { router.use('/search', search) } else { diff --git a/src/search/lib/ai-search-proxy.ts b/src/search/lib/ai-search-proxy.ts new file mode 100644 index 000000000000..9ddfdecb02a8 --- /dev/null +++ b/src/search/lib/ai-search-proxy.ts @@ -0,0 +1,78 @@ +import { Request, Response } from 'express' +import got from 'got' +import { getHmacWithEpoch } from '@/search/lib/helpers/get-cse-copilot-auth' +import { getCSECopilotSource } from '#src/search/lib/helpers/cse-copilot-docs-versions.js' + +export const aiSearchProxy = async (req: Request, res: Response) => { + const { query, version, language } = req.body + const errors = [] + + // Validate request body + if (!query) { + errors.push({ message: `Missing required key 'query' in request body` }) + } else if (typeof query !== 'string') { + errors.push({ message: `Invalid 'query' in request body. Must be a string` }) + } + if (!version) { + errors.push({ message: `Missing required key 'version' in request body` }) + } + if (!language) { + errors.push({ message: `Missing required key 'language' in request body` }) + } + + let docsSource = '' + try { + docsSource = getCSECopilotSource(version, language) + } catch (error: any) { + errors.push({ message: error?.message || 'Invalid version or language' }) + } + + if (errors.length) { + res.status(400).json({ errors }) + return + } + + const body = { + chat_context: 'defaults', + docs_source: docsSource, + query, + stream: true, + } + + try { + const stream = got.post(`${process.env.CSE_COPILOT_ENDPOINT}/answers`, { + json: body, + headers: { + Authorization: getHmacWithEpoch(), + 'Content-Type': 'application/json', + }, + isStream: true, + }) + + // Set response headers + res.setHeader('Content-Type', 'application/x-ndjson') + res.flushHeaders() + + // Pipe the got stream directly to the response + stream.pipe(res) + + // Handle stream errors + stream.on('error', (error) => { + console.error('Error streaming from cse-copilot:', error) + // Only send error response if headers haven't been sent + if (!res.headersSent) { + res.status(500).json({ errors: [{ message: 'Internal server error' }] }) + } else { + res.end() + } + }) + + // Ensure response ends when stream ends + stream.on('end', () => { + res.end() + }) + } catch (error) { + console.error('Error posting /answers to cse-copilot:', error) + res.status(500).json({ errors: [{ message: 'Internal server error' }] }) + } +} diff --git a/src/search/lib/helpers/cse-copilot-docs-versions.ts b/src/search/lib/helpers/cse-copilot-docs-versions.ts new file mode 100644 index 000000000000..9b96aa9ddcf5 --- /dev/null +++ b/src/search/lib/helpers/cse-copilot-docs-versions.ts @@ -0,0 +1,44 @@ +// Versions used by cse-copilot +import { allVersions } from '@/versions/lib/all-versions' +const CSE_COPILOT_DOCS_VERSIONS = ['dotcom', 'ghec', 'ghes'] + +// Languages supported by cse-copilot +const DOCS_LANGUAGES = ['en'] +export function supportedCSECopilotLanguages() { + return DOCS_LANGUAGES +} + +export function getCSECopilotSource( + version: (typeof CSE_COPILOT_DOCS_VERSIONS)[number], + language: (typeof DOCS_LANGUAGES)[number], +) { + const cseCopilotDocsVersion = getMiscBaseNameFromVersion(version) + if (!CSE_COPILOT_DOCS_VERSIONS.includes(cseCopilotDocsVersion)) { + throw new Error( + `Invalid 'version' in request body: '${version}'. Must be one of: ${CSE_COPILOT_DOCS_VERSIONS.join(', ')}`, + ) + } + if (!DOCS_LANGUAGES.includes(language)) { + throw new Error( + `Invalid 'language' in request body '${language}'. Must be one of: ${DOCS_LANGUAGES.join(', ')}`, + ) + } + return `docs_${version}_${language}` +} + +function getMiscBaseNameFromVersion(Version: string): string { + const miscBaseName = + Object.values(allVersions).find( + (info) => + info.shortName === Version || + info.plan === Version || + info.miscVersionName === Version || + info.currentRelease === Version, + )?.miscBaseName || '' + + if (!miscBaseName) { + return '' + } + + return miscBaseName +} diff --git a/src/search/lib/helpers/get-cse-copilot-auth.ts b/src/search/lib/helpers/get-cse-copilot-auth.ts new file mode 100644 index 000000000000..a636852452e7 --- /dev/null +++ b/src/search/lib/helpers/get-cse-copilot-auth.ts @@ -0,0 +1,24 @@ +import crypto from 'crypto' + +// github/cse-copilot's API requires an HMAC-SHA256 signature with each request +export function getHmacWithEpoch() { + const epochTime = getEpochTime().toString() + // CSE_COPILOT_SECRET needs to be set for the api-ai-search tests to work + if (process.env.NODE_ENV === 'test') { + process.env.CSE_COPILOT_SECRET = 'mock-secret' + } + if (!process.env.CSE_COPILOT_SECRET) { + throw new Error('CSE_COPILOT_SECRET is not defined') + } + const hmac = generateHmacSha256(process.env.CSE_COPILOT_SECRET, epochTime) + return `${epochTime}.${hmac}` +} + +// In seconds +function getEpochTime(): number { + return Math.floor(Date.now() / 1000) +} + +function generateHmacSha256(key: string, data: string): string { + return crypto.createHmac('sha256', key).update(data).digest('hex') +} diff --git a/src/search/middleware/ai-search.ts b/src/search/middleware/ai-search.ts new file mode 100644 index 000000000000..f2cf89fbc724 --- /dev/null +++ b/src/search/middleware/ai-search.ts @@ -0,0 +1,20 @@ +import express, { Request, Response } from 'express' + +import catchMiddlewareError from '#src/observability/middleware/catch-middleware-error.js' +import { aiSearchProxy } from '../lib/ai-search-proxy' + +const router = express.Router() + +router.post( + '/v1', + catchMiddlewareError(async (req: Request, res: Response) => { + await aiSearchProxy(req, res) + }), +) + +// Redirect to most recent version +router.post('/', (req, res) => { + res.redirect(307, req.originalUrl.replace('/ai-search', '/ai-search/v1')) +}) + +export default router diff --git a/src/search/tests/api-ai-search.ts b/src/search/tests/api-ai-search.ts new file mode 100644 index 000000000000..9b66f2c6db30 --- /dev/null +++ b/src/search/tests/api-ai-search.ts @@ -0,0 +1,148 @@ +import { expect, test, describe, beforeAll, afterAll } from 'vitest' + +import { post } from 'src/tests/helpers/e2etest.js' +import { startMockServer, stopMockServer } from '@/tests/mocks/start-mock-server' + +describe('AI Search Routes', () => { + beforeAll(() => { + startMockServer() + }) + afterAll(() => stopMockServer()) + + test('/api/ai-search/v1 should handle a successful response', async () => { + let apiBody = { query: 'How do I create a Repository?', language: 'en', version: 'dotcom' } + + const response = await fetch('http://localhost:4000/api/ai-search/v1', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(apiBody), + }) + + expect(response.ok).toBe(true) + expect(response.headers.get('content-type')).toBe('application/x-ndjson') + expect(response.headers.get('transfer-encoding')).toBe('chunked') + + if (!response.body) { + throw new Error('ReadableStream not supported in this environment.') + } + + const decoder = new TextDecoder('utf-8') + const reader = response.body.getReader() + let done = false + const chunks = [] + + while (!done) { + const { value, done: readerDone } = await reader.read() + done = readerDone + + if (value) { + // Decode the Uint8Array chunk into a string + const chunkStr = decoder.decode(value, { stream: true }) + chunks.push(chunkStr) + } + } + + // Combine all chunks into a single string + const fullResponse = chunks.join('') + // Split the response into individual chunk lines + const chunkLines = fullResponse.split('\n').filter((line) => line.trim() !== '') + + // Assertions: + + // 1. First chunk should be the SOURCES chunk + expect(chunkLines.length).toBeGreaterThan(0) + const firstChunkMatch = chunkLines[0].match(/^Chunk: (.+)$/) + expect(firstChunkMatch).not.toBeNull() + + const sourcesChunk = JSON.parse(firstChunkMatch?.[1] || '') + expect(sourcesChunk).toHaveProperty('chunkType', 'SOURCES') + expect(sourcesChunk).toHaveProperty('sources') + expect(Array.isArray(sourcesChunk.sources)).toBe(true) + expect(sourcesChunk.sources.length).toBe(3) + + // 2. Subsequent chunks should be MESSAGE_CHUNKs + for (let i = 1; i < chunkLines.length; i++) { + const line = chunkLines[i] + const messageChunk = JSON.parse(line) + expect(messageChunk).toHaveProperty('chunkType', 'MESSAGE_CHUNK') + expect(messageChunk).toHaveProperty('text') + expect(typeof messageChunk.text).toBe('string') + } + + // 3. Verify the complete message is expected + const expectedMessage = + 'Creating a repository on GitHub is something you should already know how to do :shrug:' + const receivedMessage = chunkLines + .slice(1) + .map((line) => JSON.parse(line).text) + .join('') + expect(receivedMessage).toBe(expectedMessage) + }) + + test('should handle validation errors: query missing', async () => { + let body = { language: 'en', version: 'dotcom' } + const response = await post('/api/ai-search/v1', { + body: JSON.stringify(body), + headers: { 'Content-Type': 'application/json' }, + }) + + const responseBody = JSON.parse(response.body) + + expect(response.ok).toBe(false) + expect(responseBody['errors']).toEqual([ + { message: `Missing required key 'query' in request body` }, + ]) + }) + + test('should handle validation errors: language missing', async () => { + let body = { query: 'example query', version: 'dotcom' } + const response = await post('/api/ai-search/v1', { + body: JSON.stringify(body), + headers: { 'Content-Type': 'application/json' }, + }) + + const responseBody = JSON.parse(response.body) + + expect(response.ok).toBe(false) + expect(responseBody['errors']).toEqual([ + { message: `Missing required key 'language' in request body` }, + { message: `Invalid 'language' in request body 'undefined'. Must be one of: en` }, + ]) + }) + + test('should handle validation errors: version missing', async () => { + let body = { query: 'example query', language: 'en' } + const response = await post('/api/ai-search/v1', { + body: JSON.stringify(body), + headers: { 'Content-Type': 'application/json' }, + }) + + const responseBody = JSON.parse(response.body) + + expect(response.ok).toBe(false) + expect(responseBody['errors']).toEqual([ + { message: `Missing required key 'version' in request body` }, + { + message: `Invalid 'version' in request body: 'undefined'. Must be one of: dotcom, ghec, ghes`, + }, + ]) + }) + + test('should handle multiple validation errors: query missing, invalid language and version', async () => { + let body = { language: 'fr', version: 'fpt' } + const response = await post('/api/ai-search/v1', { + body: JSON.stringify(body), + headers: { 'Content-Type': 'application/json' }, + }) + + const responseBody = JSON.parse(response.body) + + expect(response.ok).toBe(false) + expect(responseBody['errors']).toEqual([ + { message: `Missing required key 'query' in request body` }, + { + message: `Invalid 'language' in request body 'fr'. Must be one of: en`, + }, + ]) + }) +}) diff --git a/src/tests/mocks/cse-copilot-mock.ts b/src/tests/mocks/cse-copilot-mock.ts new file mode 100644 index 000000000000..e2b75a17c39a --- /dev/null +++ b/src/tests/mocks/cse-copilot-mock.ts @@ -0,0 +1,71 @@ +import { Request, Response } from 'express' + +// Prefix used for mocking. This can be any value +export const CSE_COPILOT_PREFIX = 'cse-copilot' + +export function cseCopilotPostAnswersMock(req: Request, res: Response) { + // Set headers for chunked transfer and encoding + res.setHeader('Content-Type', 'application/json; charset=utf-8') + res.setHeader('Transfer-Encoding', 'chunked') + + // Define the SOURCES chunk + const sourcesChunk = { + chunkType: 'SOURCES', + sources: [ + { + title: 'Creating a new repository', + url: 'https://docs.github.com/en/repositories/creating-and-managing-repositories/creating-a-new-repository', + index: '/en/repositories/creating-and-managing-repositories/creating-a-new-repository', + }, + { + title: 'Creating and managing repositories', + url: 'https://docs.github.com/en/repositories/creating-and-managing-repositories', + index: '/en/repositories/creating-and-managing-repositories', + }, + { + title: 'GitHub Terms of Service', + url: 'https://docs.github.com/en/site-policy/github-terms/github-terms-of-service', + index: '/en/site-policy/github-terms/github-terms-of-service', + }, + ], + } + + // Function to send a chunk with proper encoding + const sendEncodedChunk = (data: any, isLast = false) => { + const prefix = isLast ? '' : '\n' // Optionally, add delimiters if needed + const buffer = Buffer.from(prefix + data, 'utf-8') + res.write(buffer) + } + + // Send the SOURCES chunk + sendEncodedChunk(`Chunk: ${JSON.stringify(sourcesChunk)}\n\n`) + + // Define the message to be sent in chunks + const message = + 'Creating a repository on GitHub is something you should already know how to do :shrug:' + + // Split the message into words (or adjust the splitting logic as needed) + const words = message.split(' ') + + let index = 0 + + const sendChunk = () => { + if (index < words.length) { + const word = words[index] + const isLastWord = index === words.length - 1 + const chunk = { + chunkType: 'MESSAGE_CHUNK', + text: word + (isLastWord ? '' : ' '), // Add space if not the last word + } + sendEncodedChunk(`${JSON.stringify(chunk)}\n`) + index++ + sendChunk() // Adjust the delay as needed + } else { + // End the response after all chunks are sent + res.end() + } + } + + // Start sending MESSAGE_CHUNKs + sendChunk() +} diff --git a/src/tests/mocks/start-mock-server.ts b/src/tests/mocks/start-mock-server.ts new file mode 100644 index 000000000000..93c01f5ba2a8 --- /dev/null +++ b/src/tests/mocks/start-mock-server.ts @@ -0,0 +1,73 @@ +/* When testing API routes via an integration test, e.g. + +const res = await post('/api/', { + body: JSON.stringify(api_body), + headers: { 'Content-Type': 'application/json' }, +}) + +expect(res.status).toBe(200) + +The `api/` may call an external URL. + +We are unable to use `nock` in this circumstance since we run the server in a separate instance. + +Instead, we can use the `startMockServer` helper to start a mock server that will intercept the request and return a canned response. + +In order for this to work you MUST use a process.env variable for the URL you are calling, + +e.g. `process.env.CSE_COPILOT_ENDPOINT` + +You should override the variable in the overrideEnvForTesting function in this file. +*/ + +import express from 'express' +import { CSE_COPILOT_PREFIX, cseCopilotPostAnswersMock } from './cse-copilot-mock' + +// Define the default port for the mock server +const MOCK_SERVER_PORT = 3012 + +// Construct the server URL using the defined port +const serverUrl = `http://localhost:${MOCK_SERVER_PORT}` + +// Variable to hold the server instance +let server: any = null + +// Override environment variables for testing purposes +export function overrideEnvForTesting() { + process.env.CSE_COPILOT_ENDPOINT = `${serverUrl}/${CSE_COPILOT_PREFIX}` +} + +// Function to start the mock server +export function startMockServer(port = MOCK_SERVER_PORT) { + const app = express() + app.use(express.json()) + + // Define your mock routes here + app.post(`/${CSE_COPILOT_PREFIX}/answers`, cseCopilotPostAnswersMock) + + // Start the server and store the server instance + server = app.listen(port, () => { + console.log(`Mock server is running on port ${port}`) + }) +} + +// Function to stop the mock server +export function stopMockServer(): Promise { + return new Promise((resolve, reject) => { + if (server) { + server.close((err: any) => { + if (err) { + console.error('Error stopping the mock server:', err) + reject(err) + } else { + console.log('Mock server has been stopped.') + server = null + resolve() + } + }) + } else { + console.warn('Mock server is not running.') + resolve() + } + }) +} diff --git a/src/tests/vitest.setup.ts b/src/tests/vitest.setup.ts index f0d68a978f59..3e7ce07708d4 100644 --- a/src/tests/vitest.setup.ts +++ b/src/tests/vitest.setup.ts @@ -1,4 +1,5 @@ import { main } from 'src/frame/start-server' +import { overrideEnvForTesting } from './mocks/start-mock-server' let teardownHappened = false type PromiseType> = T extends Promise ? U : never @@ -7,6 +8,7 @@ type Server = PromiseType> let server: Server | undefined export async function setup() { + overrideEnvForTesting() server = await main() }