From a106c0c90c246f659b874154546c0a1b6e8f5281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Rame=CC=81?= Date: Mon, 19 Feb 2024 17:14:55 +0100 Subject: [PATCH] chore: make some cli operations available through a maintenance endpoint --- src/cli/index.ts | 4 ++ src/{cli.ts => cli/program.ts} | 4 +- src/models/entities/errors.ts | 4 ++ src/pages/api/maintenance/cli.ts | 71 ++++++++++++++++++++++++ src/pages/api/maintenance/jobs/replay.ts | 20 ++----- src/utils/maintenance.ts | 18 ++++++ 6 files changed, 103 insertions(+), 18 deletions(-) create mode 100644 src/cli/index.ts rename src/{cli.ts => cli/program.ts} (99%) create mode 100644 src/pages/api/maintenance/cli.ts create mode 100644 src/utils/maintenance.ts diff --git a/src/cli/index.ts b/src/cli/index.ts new file mode 100644 index 0000000..e5b717f --- /dev/null +++ b/src/cli/index.ts @@ -0,0 +1,4 @@ +import { program } from '@etabli/src/cli/program'; + +// This would break imports from Next.js so isolating it to be run only by CLI +program.parse(); diff --git a/src/cli.ts b/src/cli/program.ts similarity index 99% rename from src/cli.ts rename to src/cli/program.ts index 95dd440..81be141 100644 --- a/src/cli.ts +++ b/src/cli/program.ts @@ -6,7 +6,7 @@ import { cleanLlmSystem, ingestInitiativeListToLlmSystem, ingestToolListToLlmSys import { enhanceRepositoriesIntoDatabase, formatRepositoriesIntoDatabase, saveRepositoryListFile } from '@etabli/src/features/repository'; import { enhanceToolsIntoDatabase, formatToolsIntoDatabase, saveToolCsvFile } from '@etabli/src/features/tool'; -const program = new Command(); +export const program = new Command(); program.name('etabli').description('CLI to some deal with Établi project').version('0.0.0'); @@ -179,5 +179,3 @@ cache .action(async () => { console.log('cache.clear'); }); - -program.parse(); diff --git a/src/models/entities/errors.ts b/src/models/entities/errors.ts index 223445f..78ca4e0 100644 --- a/src/models/entities/errors.ts +++ b/src/models/entities/errors.ts @@ -55,6 +55,10 @@ export const unexpectedErrorError = new UnexpectedError('unexpectedError', 'unex // Validations export const unexpectedDomainRedirectionError = new BusinessError('unexpectedDomainRedirection', 'unexpected domain redirection'); +export const unexpectedCliMaintenanceCommandError = new BusinessError( + 'unexpectedCliMaintenanceCommand', + 'unexpected command passed for maintenance through cli' +); // LLM export const tokensReachTheLimitError = new BusinessError('tokensReachTheLimit', 'too many tokens according to the model limit'); diff --git a/src/pages/api/maintenance/cli.ts b/src/pages/api/maintenance/cli.ts new file mode 100644 index 0000000..d9747e8 --- /dev/null +++ b/src/pages/api/maintenance/cli.ts @@ -0,0 +1,71 @@ +import { Command } from '@commander-js/extra-typings'; +import { NextApiRequest, NextApiResponse } from 'next'; +import { z } from 'zod'; + +import { program } from '@etabli/src/cli/program'; +import { unexpectedCliMaintenanceCommandError } from '@etabli/src/models/entities/errors'; +import { apiHandlerWrapper } from '@etabli/src/utils/api'; +import { assertMaintenanceOperationAuthenticated } from '@etabli/src/utils/maintenance'; + +export const HandlerBodySchema = z + .object({ + command: z.string().min(1).max(1000), + }) + .strict(); +export type HandlerBodySchemaType = z.infer; + +const commandsPrefixesWhitelist: string[] = [ + 'domain fetch', + 'domain format', + 'domain enhance', + 'repository fetch', + 'repository format', + 'repository enhance', + 'tool fetch', + 'tool format', + 'tool enhance', + 'llm initialize', + 'llm initiatives', + 'llm tools', + 'initiative infer', + 'initiative feed', +]; + +// This endpoint allows running cron job commands (that are defined in the CLI) without connecting to the host (note also it would imply from us to make another build in addition to the Next.js build) +// (it avoids maintaining 2 implementations when changing parameters and so on) +export async function handler(req: NextApiRequest, res: NextApiResponse) { + assertMaintenanceOperationAuthenticated(req); + + const body = HandlerBodySchema.parse(req.body); + + // It could be risky to expose the whole CLI due to future implementations, so whitelisting commands we allow + // Note: `commander.js` was not made to reuse easily subcommands to create another program, so we hack a bit we filter based on raw strings + if (!commandsPrefixesWhitelist.some((commandPrefix) => body.command.startsWith(commandPrefix))) { + throw unexpectedCliMaintenanceCommandError; + } + + // [WORKAROUND] We cannot check if it matches a command or not, in case none it terminates the running process + // We can add a catcher but we cannot custom the response since global to the instance... so using a clone to wait for it + let commandFound = true; + + const cloneProgram = new Command(); + for (const command of program.commands) { + cloneProgram.addCommand(command); + } + + cloneProgram.on('command:*', () => { + commandFound = false; + }); + + await cloneProgram.parseAsync(body.command.split(' '), { from: 'user' }); + + if (!commandFound) { + throw unexpectedCliMaintenanceCommandError; + } + + res.send(`the following command has been executed with success:\n${body.command}`); +} + +export default apiHandlerWrapper(handler, { + restrictMethods: ['POST'], +}); diff --git a/src/pages/api/maintenance/jobs/replay.ts b/src/pages/api/maintenance/jobs/replay.ts index b9c1095..40d3113 100644 --- a/src/pages/api/maintenance/jobs/replay.ts +++ b/src/pages/api/maintenance/jobs/replay.ts @@ -1,4 +1,3 @@ -import createHttpError from 'http-errors'; import { NextApiRequest, NextApiResponse } from 'next'; import { JobWithMetadata } from 'pg-boss'; import { z } from 'zod'; @@ -7,16 +6,10 @@ import { BusinessError } from '@etabli/src/models/entities/errors'; import { MaintenanceDataSchemaType, MaintenanceWrapperDataSchema } from '@etabli/src/models/jobs/maintenance'; import { getBossClientInstance } from '@etabli/src/server/queueing/client'; import { apiHandlerWrapper } from '@etabli/src/utils/api'; - -const maintenanceApiKey = process.env.MAINTENANCE_API_KEY; +import { assertMaintenanceOperationAuthenticated } from '@etabli/src/utils/maintenance'; export const replayableJobStates: JobWithMetadata['state'][] = ['completed', 'expired', 'cancelled', 'failed']; -export function isAuthenticated(apiKeyHeader?: string): boolean { - // If the maintenance api key is not defined on the server we prevent executing operations - return !!maintenanceApiKey && maintenanceApiKey === apiKeyHeader; -} - export const HandlerBodySchema = z .object({ jobId: z.string().uuid(), @@ -25,12 +18,7 @@ export const HandlerBodySchema = z export type HandlerBodySchemaType = z.infer; export async function handler(req: NextApiRequest, res: NextApiResponse) { - // Check the originator has the maintenance secret - if (!isAuthenticated((req.headers as any)['x-api-key'])) { - console.log('someone is trying to trigger a job replay without being authenticated'); - - throw new createHttpError.Unauthorized(`invalid api key`); - } + assertMaintenanceOperationAuthenticated(req); const body = HandlerBodySchema.parse(req.body); @@ -85,4 +73,6 @@ export async function handler(req: NextApiRequest, res: NextApiResponse) { res.send(`The job has been replayed with the id "${newJobId}"`); } -export default apiHandlerWrapper(handler); +export default apiHandlerWrapper(handler, { + restrictMethods: ['POST'], +}); diff --git a/src/utils/maintenance.ts b/src/utils/maintenance.ts new file mode 100644 index 0000000..a82d058 --- /dev/null +++ b/src/utils/maintenance.ts @@ -0,0 +1,18 @@ +import createHttpError from 'http-errors'; +import { NextApiRequest } from 'next'; + +const maintenanceApiKey = process.env.MAINTENANCE_API_KEY; + +export function isAuthenticated(apiKeyHeader?: string): boolean { + // If the maintenance api key is not defined on the server we prevent executing operations + return !!maintenanceApiKey && maintenanceApiKey === apiKeyHeader; +} + +// Check the originator has the maintenance secret +export function assertMaintenanceOperationAuthenticated(req: NextApiRequest) { + if (!isAuthenticated((req.headers as any)['x-api-key'])) { + console.log('someone is trying to trigger a maintenance operation without being authenticated'); + + throw new createHttpError.Unauthorized(`invalid api key`); + } +}