diff --git a/targets/export-elasticsearch/.eslintrc.js b/targets/export-elasticsearch/.eslintrc.js new file mode 100644 index 000000000..cb80d1625 --- /dev/null +++ b/targets/export-elasticsearch/.eslintrc.js @@ -0,0 +1,11 @@ +module.exports = { + extends: ["@shared/eslint-config"], + rules: { + "@typescript-eslint/naming-convention": "off", + }, + parserOptions: { + project: "tsconfig.json", + sourceType: "module", + tsconfigRootDir: __dirname, + }, +}; diff --git a/targets/export-elasticsearch/.eslintrc.json b/targets/export-elasticsearch/.eslintrc.json deleted file mode 100644 index 58073a649..000000000 --- a/targets/export-elasticsearch/.eslintrc.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": [ - "@shared/eslint-config" - ], - "rules": { - "@typescript-eslint/naming-convention": "off" - } -} diff --git a/targets/export-elasticsearch/request.http b/targets/export-elasticsearch/request.http index d26a7ab86..8e820aace 100644 --- a/targets/export-elasticsearch/request.http +++ b/targets/export-elasticsearch/request.http @@ -14,8 +14,8 @@ POST http://localhost:8787/export content-type: application/json { - "environment": "production", - "userId": "148a6a2b-c565-4ce2-a093-ad1e74a1c722" + "environment": "preproduction", + "userId": "8babda41-6001-4665-96f5-c430fddb0c53" } ### diff --git a/targets/export-elasticsearch/src/controllers/__test__/export.test.ts b/targets/export-elasticsearch/src/controllers/__test__/export.test.ts index b2e306109..7e11bf7d6 100644 --- a/targets/export-elasticsearch/src/controllers/__test__/export.test.ts +++ b/targets/export-elasticsearch/src/controllers/__test__/export.test.ts @@ -85,14 +85,8 @@ describe("ExportController /export", () => { environment: Environment.preproduction, userId: "890ca91b-f150-4957-9bb2-8500940815f0", }); - expect(res.statusCode).toEqual(202); expect(res.body).toEqual({ - created_at: "2022-03-24T10:09:10.000Z", - environment: Environment.preproduction, - id: "1", - status: Status.running, - updated_at: "2022-03-24T10:09:10.000Z", - user_id: "890ca91b-f150-4957-9bb2-8500940815f0", + isRunning: true, }); }); diff --git a/targets/export-elasticsearch/src/controllers/__test__/fake/export.ts b/targets/export-elasticsearch/src/controllers/__test__/fake/export.ts index 6d9ae1922..9edf1d7c0 100644 --- a/targets/export-elasticsearch/src/controllers/__test__/fake/export.ts +++ b/targets/export-elasticsearch/src/controllers/__test__/fake/export.ts @@ -48,4 +48,18 @@ export class FakeExportService { }, ]; } + + async getRunningExport(): Promise { + await wait(100); + return []; + } + + async verifyAndCleanPreviousExport( + runningResult: ExportEsStatus[], + environment: Environment, + minutes: number + ): Promise { + await wait(100); + return; + } } diff --git a/targets/export-elasticsearch/src/controllers/export.ts b/targets/export-elasticsearch/src/controllers/export.ts index b26a0f8f8..6ffc50285 100644 --- a/targets/export-elasticsearch/src/controllers/export.ts +++ b/targets/export-elasticsearch/src/controllers/export.ts @@ -1,6 +1,5 @@ -import type { ExportEsStatus } from "@socialgouv/cdtn-types"; -import { Environment } from "@socialgouv/cdtn-types"; -import { Request, Response } from "express"; +import { ExportEsStatus, Environment } from "@socialgouv/cdtn-types"; +import { Request } from "express"; import { inject } from "inversify"; import type { interfaces } from "inversify-express-utils"; import { @@ -9,7 +8,6 @@ import { httpPost, queryParam, request, - response, } from "inversify-express-utils"; import { ExportService } from "../services/export"; @@ -25,13 +23,18 @@ export class ExportController implements interfaces.Controller { ) {} @httpPost("/", getName(ExportEsRunMiddleware)) - async run( - @request() req: Request, - @response() res: Response - ): Promise { + async run(@request() req: Request): Promise<{ isRunning: true }> { const body: ValidatorCreateExportEsStatusType = req.body; - res.status(202); - return this.service.runExport(body.userId, body.environment); + const runningResult = await this.service.getRunningExport(); + await this.service.verifyAndCleanPreviousExport( + runningResult, + body.environment, + process.env.DISABLE_LIMIT_EXPORT ? 0 : 15 + ); + this.service.runExport(body.userId, body.environment); + return { + isRunning: true, + }; } @httpGet("/") diff --git a/targets/export-elasticsearch/src/ingester/types/GraphQL.ts b/targets/export-elasticsearch/src/ingester/types/GraphQL.ts index 9d04cede3..9d5181c67 100644 --- a/targets/export-elasticsearch/src/ingester/types/GraphQL.ts +++ b/targets/export-elasticsearch/src/ingester/types/GraphQL.ts @@ -1,14 +1,14 @@ -export type GraphQLResponseRoot = { +export interface GraphQLResponseRoot { data?: Data; errors?: GraphQLResponseError[]; -}; +} -export type GraphQLResponseError = { +export interface GraphQLResponseError { message: string; locations?: GraphQLResponseErrorLocation[]; -}; +} -export type GraphQLResponseErrorLocation = { +export interface GraphQLResponseErrorLocation { line: number; column: number; -}; +} diff --git a/targets/export-elasticsearch/src/services/__test__/export.test.ts b/targets/export-elasticsearch/src/services/__test__/export.test.ts index b2f0fc09e..49c9644eb 100644 --- a/targets/export-elasticsearch/src/services/__test__/export.test.ts +++ b/targets/export-elasticsearch/src/services/__test__/export.test.ts @@ -11,6 +11,8 @@ import { SitemapService } from "../sitemap"; import { FakeCopyService } from "./fake/copy"; import { FakeExportRepository } from "./fake/export"; import { FakeSitemapService } from "./fake/sitemap"; +import { FakeAgreementsService } from "./fake/agreements"; +import { AgreementsService } from "../agreements"; jest.mock("../../workers", () => { return { @@ -40,6 +42,10 @@ describe("ExportService", () => { .bind(getName(SitemapService)) .to(FakeSitemapService) .inSingletonScope(); + container + .bind(getName(AgreementsService)) + .to(FakeAgreementsService) + .inSingletonScope(); service = container.get(getName(ExportService)); mockRepository = container.get(getName(ExportRepository)); jest.clearAllMocks(); @@ -92,25 +98,11 @@ describe("ExportService", () => { .runExport("ABC", Environment.preproduction) .catch((e: Error) => { // eslint-disable-next-line jest/no-conditional-expect - expect(e.message).toBe("There is already a running job"); + expect(e.message).toBe( + "Il y a déjà un export en cours qui a été lancé il y a moins de 15 minutes..." + ); }); }); - - it("should not clean previous job", async () => { - const date = new Date(); - const spy = jest.spyOn(mockRepository, "updateOne"); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - await service.cleanPreviousExport({ - created_at: date, - environment: Environment.preproduction, - id: "1", - status: Status.running, - updated_at: date, - user_id: "userId", - }); - expect(spy).toHaveBeenCalledTimes(0); - }); }); describe("Outdated job has run", () => { @@ -129,29 +121,60 @@ describe("ExportService", () => { ); const spy = jest.spyOn(mockRepository, "updateOne"); await service.runExport("ABC", Environment.preproduction); - expect(spy).toHaveBeenCalledTimes(2); + expect(spy).toHaveBeenCalledTimes(1); expect(runWorkerIngesterPreproduction).toHaveBeenCalledTimes(1); }); - it("should clean previous job because launched > 1h", async () => { + it("should clean previous job because launched > 15min", async () => { const oldDate = new Date(); const expiryDate = new Date( - new Date().setHours(new Date().getHours() + 2) + new Date().setHours(new Date().getMinutes() + 20) ); timekeeper.travel(expiryDate); const spy = jest.spyOn(mockRepository, "updateOne"); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - await service.cleanPreviousExport({ - created_at: oldDate, - environment: Environment.preproduction, - id: "1", - status: Status.running, - updated_at: oldDate, - user_id: "userId", - }); + await service.verifyAndCleanPreviousExport( + [ + { + created_at: oldDate, + environment: Environment.preproduction, + id: "1", + status: Status.running, + updated_at: oldDate, + user_id: "userId", + }, + ], + Environment.preproduction, + 15 + ); expect(spy).toHaveBeenCalledTimes(1); }); + + it("should cancel because there are already an export in an other env", async () => { + const spy = jest.spyOn(mockRepository, "updateOne"); + await service + .verifyAndCleanPreviousExport( + [ + { + created_at: new Date("2020-01-01"), + environment: Environment.preproduction, + id: "1", + status: Status.running, + updated_at: new Date("2020-01-01"), + user_id: "userId", + }, + ], + Environment.production, + 15 + ) + .catch((e: Error) => { + // eslint-disable-next-line jest/no-conditional-expect + expect(e.message).toBe( + "Il y a déjà un export en cours sur un autre environnement..." + ); + }); + + expect(spy).toHaveBeenCalledTimes(0); + }); }); }); }); diff --git a/targets/export-elasticsearch/src/services/__test__/fake/agreements.ts b/targets/export-elasticsearch/src/services/__test__/fake/agreements.ts new file mode 100644 index 000000000..e7fec457f --- /dev/null +++ b/targets/export-elasticsearch/src/services/__test__/fake/agreements.ts @@ -0,0 +1,15 @@ +import { injectable } from "inversify"; + +import { wait } from "../../../utils"; +import { Environment } from "@socialgouv/cdtn-types"; + +@injectable() +export class FakeAgreementsService { + async uploadAgreements( + environment: Environment, + destinationFolder: string, + destinationName: string + ): Promise { + await wait(100); + } +} diff --git a/targets/export-elasticsearch/src/services/__test__/fake/sitemap.ts b/targets/export-elasticsearch/src/services/__test__/fake/sitemap.ts index 308e9b9b9..0dc8c7f32 100644 --- a/targets/export-elasticsearch/src/services/__test__/fake/sitemap.ts +++ b/targets/export-elasticsearch/src/services/__test__/fake/sitemap.ts @@ -1,13 +1,15 @@ import { injectable } from "inversify"; import { wait } from "../../../utils"; +import { Environment } from "@socialgouv/cdtn-types"; @injectable() export class FakeSitemapService { async uploadSitemap( + environment: Environment, sitemapEndpoint: string, - destinationContainer: string, - destinationName: string + destinationFolder: string, + sitemapName: string ): Promise { await wait(100); } diff --git a/targets/export-elasticsearch/src/services/export.ts b/targets/export-elasticsearch/src/services/export.ts index de6c74d4d..a08223664 100644 --- a/targets/export-elasticsearch/src/services/export.ts +++ b/targets/export-elasticsearch/src/services/export.ts @@ -33,78 +33,68 @@ export class ExportService { environment: Environment ): Promise { logger.info(`[${userId}] run export for ${environment}`); - let isReadyToRun = false; - const runningResult = await this.getRunningExport(); - if (runningResult.length > 0) { - isReadyToRun = await this.cleanPreviousExport( - runningResult[0], - process.env.DISABLE_LIMIT_EXPORT ? 0 : 1 - ); // we can avoid to do that with a queue system (e.g. RabbitMQ, Kafka, etc.) - } - if (runningResult.length === 0 || isReadyToRun) { - const id = randomUUID(); - const exportEs = await this.exportRepository.create( - id, - userId, - environment, - Status.running - ); - try { - if (!process.env.DISABLE_INGESTER) { - if (environment === Environment.preproduction) { - await sendMattermostMessage( - `**Préproduction:** mise à jour lancée par *${exportEs.user?.name}* 😎`, - process.env.MATTERMOST_CHANNEL_EXPORT - ); - await runWorkerIngesterPreproduction(); - const exportEsDone = await await this.exportRepository.getOne(id); - await sendMattermostMessage( - `**Préproduction:** mise à jour terminée (${exportEsDone.documentsCount?.total} documents) 😁`, - process.env.MATTERMOST_CHANNEL_EXPORT - ); - } else { - await sendMattermostMessage( - `**Production:** mise à jour lancée par *${exportEs.user?.name}* 🚀`, - process.env.MATTERMOST_CHANNEL_EXPORT - ); - await runWorkerIngesterProduction(); - const exportEsDone = await this.exportRepository.getOne(id); - await sendMattermostMessage( - `**Production:** mise à jour terminée (${exportEsDone.documentsCount?.total} documents) 🎉`, - process.env.MATTERMOST_CHANNEL_EXPORT - ); - } - } - if (!process.env.DISABLE_SITEMAP) { - await this.sitemapService.uploadSitemap(environment); - } - if (!process.env.DISABLE_AGREEMENTS) { - await this.exportAgreementsService.uploadAgreements(environment); - } - if (!process.env.DISABLE_COPY) { - await this.copyContainerService.runCopy(environment); + const id = randomUUID(); + const exportEs = await this.exportRepository.create( + id, + userId, + environment, + Status.running + ); + try { + if (!process.env.DISABLE_INGESTER) { + if (environment === Environment.preproduction) { + await sendMattermostMessage( + `**Préproduction:** mise à jour lancée par *${exportEs.user?.name}* 😎`, + process.env.MATTERMOST_CHANNEL_EXPORT + ); + await runWorkerIngesterPreproduction(); + } else { + await sendMattermostMessage( + `**Production:** mise à jour lancée par *${exportEs.user?.name}* 🚀`, + process.env.MATTERMOST_CHANNEL_EXPORT + ); + await runWorkerIngesterProduction(); } - return await this.exportRepository.updateOne( - id, - Status.completed, - new Date() - ); - } catch (e: any) { + } + if (!process.env.DISABLE_SITEMAP) { + await this.sitemapService.uploadSitemap(environment); + } + if (!process.env.DISABLE_AGREEMENTS) { + await this.exportAgreementsService.uploadAgreements(environment); + } + if (!process.env.DISABLE_COPY) { + await this.copyContainerService.runCopy(environment); + } + const exportEsDone = await this.exportRepository.getOne(id); + if (environment === Environment.preproduction) { await sendMattermostMessage( - environment === Environment.preproduction - ? " La mise à jour de la préproduction a échouée. 😢" - : "La mise à jour de la production a échouée. 😭", + `**Production:** mise à jour terminée (${exportEsDone.documentsCount?.total} documents) 🎉`, process.env.MATTERMOST_CHANNEL_EXPORT ); - return await this.exportRepository.updateOne( - id, - Status.failed, - new Date(), - e.message + } else { + await sendMattermostMessage( + `**Préproduction:** mise à jour terminée (${exportEsDone.documentsCount?.total} documents) 😁`, + process.env.MATTERMOST_CHANNEL_EXPORT ); } - } else { - throw new Error("There is already a running job"); + return await this.exportRepository.updateOne( + id, + Status.completed, + new Date() + ); + } catch (e: any) { + await sendMattermostMessage( + environment === Environment.preproduction + ? " La mise à jour de la préproduction a échouée. 😢" + : "La mise à jour de la production a échouée. 😭", + process.env.MATTERMOST_CHANNEL_EXPORT + ); + return await this.exportRepository.updateOne( + id, + Status.failed, + new Date(), + e.message + ); } } @@ -131,25 +121,35 @@ export class ExportService { }; } - private async getRunningExport(): Promise { + async getRunningExport(): Promise { return this.exportRepository.getByStatus(Status.running); } - private async cleanPreviousExport( - runningResult: ExportEsStatus, - hour = 1 // job created 1 hour ago - ): Promise { - if ( - new Date(runningResult.created_at).getTime() < - new Date(Date.now() - 1000 * 60 * 60 * hour).getTime() - ) { - await this.exportRepository.updateOne( - runningResult.id, - Status.timeout, - new Date() - ); - return true; + async verifyAndCleanPreviousExport( + runningResult: ExportEsStatus[], + environment: Environment, + minutes: number + ): Promise { + if (runningResult.length > 0) { + if (runningResult[0].environment !== environment) { + throw new Error( + "Il y a déjà un export en cours sur un autre environnement..." + ); + } + if ( + new Date(runningResult[0].created_at).getTime() < + new Date(Date.now() - 1000 * 60 * minutes).getTime() + ) { + await this.exportRepository.updateOne( + runningResult[0].id, + Status.timeout, + new Date() + ); + } else { + throw new Error( + `Il y a déjà un export en cours qui a été lancé il y a moins de ${minutes} minutes...` + ); + } } - return false; } } diff --git a/targets/frontend/src/modules/export/components/export.tsx b/targets/frontend/src/modules/export/components/export.tsx index bf9afc1e0..a55c86993 100644 --- a/targets/frontend/src/modules/export/components/export.tsx +++ b/targets/frontend/src/modules/export/components/export.tsx @@ -6,7 +6,6 @@ import { Status, } from "src/components/export-es"; import { Table, Td, Th, Tr } from "src/components/table"; -import { useExportEs } from "src/hooks/exportEs"; import { useSession } from "next-auth/react"; import { Button, @@ -16,6 +15,7 @@ import { } from "@mui/material"; import { ShowDocumentsToUpdateModal } from "./ShowDocumentsToUpdateModal"; import { FixedSnackBar } from "src/components/utils/SnackBar"; +import { useExportEs } from "../hooks/export"; export function Export(): JSX.Element { const [validateExportPreprodModal, setValidateExportPreprodModal] = @@ -29,14 +29,25 @@ export function Export(): JSX.Element { const user = data?.user; const onTrigger = (env: Environment) => { - if (!user) throw new Error("Utilisateur non connecté"); + if (!user) { + alert("Vous devez être connecté pour effectuer cette action"); + return; + } runExportEs(env, user); }; useEffect(() => { - getExportEs(); + getExportEs(false); }, []); + useEffect(() => { + const interval = setInterval(() => { + getExportEs(true); + }, 20000); // 20000 milliseconds = 20 seconds + + return () => clearInterval(interval); + }, [getExportEs]); + return ( <> {exportEsState.error && ( @@ -76,8 +87,8 @@ export function Export(): JSX.Element { Environnement Utilisateur - Crée le - Complété le + Début + Fin Statut Informations diff --git a/targets/frontend/src/hooks/exportEs.ts b/targets/frontend/src/modules/export/hooks/export.ts similarity index 78% rename from targets/frontend/src/hooks/exportEs.ts rename to targets/frontend/src/modules/export/hooks/export.ts index 0441cd2e2..7d9380bba 100644 --- a/targets/frontend/src/hooks/exportEs.ts +++ b/targets/frontend/src/modules/export/hooks/export.ts @@ -1,10 +1,11 @@ import { Environment, ExportEsStatus, Status } from "@socialgouv/cdtn-types"; import { Session } from "next-auth"; import { useState } from "react"; -import { serializeError } from "serialize-error"; +import { serializeError, ErrorObject } from "serialize-error"; +import { generateInitialId } from "@shared/utils"; type ExportEsState = { - error: Error | null; + error: ErrorObject | null; exportData: ExportEsStatus[]; hasGetExportError: boolean; isGetExportLoading: boolean; @@ -14,7 +15,7 @@ type ExportEsState = { export function useExportEs(): [ ExportEsState, - () => void, + (hideLoader: boolean) => void, (environment: Environment, user: Session["user"]) => void, (env: Environment) => Date ] { @@ -27,18 +28,19 @@ export function useExportEs(): [ latestExportProduction: null, }); - const getExportEs = () => { + const getExportEs = (hideLoader: boolean) => { setState((state) => ({ ...state, error: null, hasGetExportError: false, - isGetExportLoading: true, + isGetExportLoading: hideLoader ? false : true, })); fetch("/api/export") - .then(async (response) => { - return response.status === 500 - ? Promise.reject(await response.json()) - : response.json(); + .then((response) => { + if (!response.ok) { + return Promise.reject("Error fetching export data"); + } + return response.json(); }) .then((data) => { setState((state) => ({ @@ -47,6 +49,7 @@ export function useExportEs(): [ isGetExportLoading: false, latestExportPreproduction: data[1].preproduction, latestExportProduction: data[1].production, + error: null, })); }) .catch((error) => { @@ -68,10 +71,11 @@ export function useExportEs(): [ }; const runExportEs = (environment: Environment, user: Session["user"]) => { + const randomId = generateInitialId(); const newExportEs: ExportEsStatus = { created_at: new Date(), environment, - id: "0", + id: randomId, status: Status.running, updated_at: new Date(), user: { @@ -102,21 +106,11 @@ export function useExportEs(): [ method: "POST", }) .then(async (response) => { - return response.status === 500 - ? Promise.reject(await response.json()) - : response.json(); - }) - .then((data) => { - setState((state) => ({ - ...state, - ...Object.assign( - {}, - environment === Environment.preproduction - ? { latestExportPreproduction: data } - : { latestExportProduction: data } - ), - exportData: [data, ...state.exportData].filter((x) => x.id !== "0"), - })); + if (!response.ok) { + const errors = await response.json(); + return Promise.reject(errors.error); + } + return response.json(); }) .catch((error) => { setState((state) => ({ @@ -139,7 +133,7 @@ export function useExportEs(): [ } ), exportData: state.exportData.map((x) => - x.id === "0" ? { ...x, status: Status.failed } : x + x.id === randomId ? { ...x, status: Status.failed } : x ), })); }); diff --git a/targets/frontend/src/pages/api/export.ts b/targets/frontend/src/pages/api/export.ts index 31c03e64e..151d50016 100644 --- a/targets/frontend/src/pages/api/export.ts +++ b/targets/frontend/src/pages/api/export.ts @@ -19,9 +19,7 @@ const main = (req: NextApiRequest, res: NextApiResponse) => { }); } }); - res.status(200).json({ - ...data, - }); + res.status(200).json(data); }) .catch((error) => { res.status(500).json({ @@ -42,8 +40,24 @@ const main = (req: NextApiRequest, res: NextApiResponse) => { "Content-Type": "application/json", }, method: "POST", - }); - res.status(201); + }) + .then((res) => { + return res.json(); + }) + .then((data) => { + if (data.errors) { + res.status(500).json({ + error: data.errors, + }); + } else { + res.status(200).json(data); + } + }) + .catch((error) => { + res.status(500).json({ + error, + }); + }); } };