feat(alerts): ajout d'un feature de notification pour les alertes de …

…la dares (#1004) * fix: alert * fix: alert * fix: alert * feat: ajout de la 1979 * fix: alert * fix: autoscroll
SocialGouv · Sep 22, 2023 · 530043d · 530043d
1 parent c55bd4c
commit 530043d
Show file tree

Hide file tree

Showing 18 changed files with 324 additions and 5 deletions.
diff --git a/shared/types/index.ts b/shared/types/index.ts
@@ -265,7 +265,8 @@ export type AlertInfo = { id: string };
 export type AlertChanges =
   | DilaAlertChanges
   | TravailDataAlertChanges
-  | VddAlertChanges;
+  | VddAlertChanges
+  | DaresAlertChanges;
 
 /** Dila alert changes */
 export type DilaAlertChanges = DilaChanges & {
@@ -382,6 +383,34 @@ export type VddAlertChanges = VddChanges & {
   date: Date;
 };
 
+export type DaresAlertChanges = {
+  type: "dares";
+  title: string;
+  ref: string;
+  date: Date;
+  modified: [];
+  added: {
+    name: string;
+    num: number;
+  }[];
+  removed: {
+    name: string;
+    num: number;
+  }[];
+  documents: [];
+};
+
+export type DaresAlert = {
+  id: string;
+  info: {
+    id: string | number; // idcc number
+  };
+  status: "doing" | "done" | "rejected" | "todo";
+  repository: "dares";
+  ref: string;
+  changes: DaresAlertChanges;
+};
+
 export type VddChanges = {
   modified: FicheVddInfoWithDiff[];
   removed: FicheVddInfo[];

diff --git a/targets/alert-cli/jest.config.js b/targets/alert-cli/jest.config.js
@@ -2,7 +2,7 @@ const config = {
   preset: "ts-jest/presets/js-with-ts-esm",
   testMatch: ["**/__tests__/**/?(*.)+(spec|test).+(ts|tsx|js)"],
   transformIgnorePatterns: [
-    "node_modules/(?!(unist-util-select|zwitch|unist-util-is)/)",
+    "node_modules/(?!(unist-util-select|zwitch|unist-util-is|axios)/)",
   ],
 };
 

diff --git a/targets/alert-cli/package.json b/targets/alert-cli/package.json
@@ -13,8 +13,10 @@
     "@socialgouv/cdtn-slugify": "4.52.1",
     "@socialgouv/cdtn-sources": "4.52.1",
     "@socialgouv/dila-api-client": "1.2.4",
+    "axios": "^1.5.0",
     "diff": "^5.1.0",
     "memoizee": "0.4.15",
+    "node-xlsx": "^0.23.0",
     "p-map": "4",
     "semver": "7.3.5",
     "simple-git": "^3.19.1",

diff --git a/targets/alert-cli/src/dares/__tests__/scrapping.test.ts b/targets/alert-cli/src/dares/__tests__/scrapping.test.ts
@@ -0,0 +1,34 @@
+import axios from "axios";
+import { extractXlsxFromUrl } from "../scrapping";
+
+jest.mock("axios");
+
+describe("extractXlsxFromUrl", () => {
+  it("should extract xlsx file from url", async () => {
+    const url = "https://example.com/files";
+    const html = `
+      <html>
+        <body>
+          <a href="file1.xlsx">File 1</a>
+          <a href="file2.xlsx">File 2</a>
+        </body>
+      </html>
+    `;
+    (axios.get as jest.Mock).mockResolvedValueOnce({ data: html });
+    const result = await extractXlsxFromUrl(url);
+    expect(result).toBe("file1.xlsx");
+  });
+
+  it("should throw error if no xlsx file found", async () => {
+    const url = "https://example.com/files";
+    const html = `
+      <html>
+        <body>
+          <a href="file1.pdf">File 1</a>
+        </body>
+      </html>
+    `;
+    (axios.get as jest.Mock).mockResolvedValueOnce({ data: html });
+    await expect(extractXlsxFromUrl(url)).rejects.toThrow("No xlsx file found");
+  });
+});
diff --git a/targets/alert-cli/src/dares/config.ts b/targets/alert-cli/src/dares/config.ts
@@ -0,0 +1,4 @@
+export const URL_SCRAPING =
+  "https://code.travail.gouv.fr/fiche-ministere-travail/conventions-collectives-nomenclatures";
+export const URL_KALI =
+  "https://raw.githubusercontent.com/SocialGouv/kali-data/master/data/index.json";
diff --git a/targets/alert-cli/src/dares/difference.ts b/targets/alert-cli/src/dares/difference.ts
@@ -0,0 +1,46 @@
+import xlsx from "node-xlsx";
+import { Diff, Agreement } from "./types";
+import fs from "fs";
+
+export function getDifferenceBetweenIndexAndDares(
+  pathDares: string,
+  pathIndex: string
+): Diff {
+  const workSheetsFromFile = xlsx.parse(pathDares);
+
+  const supportedCcXlsx: Agreement[] = [];
+
+  workSheetsFromFile[0].data.forEach((row: string[]) => {
+    const ccNumber = parseInt(row[0]);
+    const ccName = row[1];
+    if (ccNumber && ccName) {
+      const ccNameWithoutParenthesis = ccName
+        .replace(/\(.*annexée.*\)/gi, "")
+        .trim();
+      supportedCcXlsx.push({
+        name: ccNameWithoutParenthesis,
+        num: ccNumber,
+      });
+    }
+  });
+
+  const dataJson = JSON.parse(fs.readFileSync(pathIndex, "utf8"));
+
+  const supportedCcIndexJson: Agreement[] = dataJson.map((cc: any) => {
+    return {
+      name: cc.title,
+      num: cc.num,
+    };
+  });
+
+  const missingAgreementsFromDares: Agreement[] = supportedCcXlsx.filter(
+    (ccIndex) =>
+      !supportedCcIndexJson.find((ccXlsx) => ccXlsx.num === ccIndex.num)
+  );
+
+  const exceedingAgreementsFromKali = supportedCcIndexJson.filter(
+    (ccXlsx) => !supportedCcXlsx.find((ccIndex) => ccIndex.num === ccXlsx.num)
+  );
+
+  return { missingAgreementsFromDares, exceedingAgreementsFromKali };
+}
diff --git a/targets/alert-cli/src/dares/download.ts b/targets/alert-cli/src/dares/download.ts
@@ -0,0 +1,21 @@
+import axios from "axios";
+import fs from "fs";
+import os from "os";
+import path from "path";
+
+export async function downloadFileInTempFolder(
+  url: string,
+  nameOfFile: string
+): Promise<string> {
+  const tempDir = os.tmpdir();
+  const filePath = path.join(tempDir, nameOfFile);
+
+  const response = await axios.get(url, { responseType: "stream" });
+  const writer = fs.createWriteStream(filePath);
+  response.data.pipe(writer);
+
+  return new Promise((resolve, reject) => {
+    writer.on("finish", () => resolve(filePath));
+    writer.on("error", reject);
+  });
+}
diff --git a/targets/alert-cli/src/dares/index.ts b/targets/alert-cli/src/dares/index.ts
@@ -0,0 +1,13 @@
+import { URL_KALI, URL_SCRAPING } from "./config";
+import { getDifferenceBetweenIndexAndDares } from "./difference";
+import { downloadFileInTempFolder } from "./download";
+import { saveDiff } from "./save";
+import { extractXlsxFromUrl } from "./scrapping";
+
+export const runDares = async () => {
+  const xlsxUrl = await extractXlsxFromUrl(URL_SCRAPING);
+  const xlsxPath = await downloadFileInTempFolder(xlsxUrl, "dares.xlsx");
+  const indexPath = await downloadFileInTempFolder(URL_KALI, "index.json");
+  const diff = await getDifferenceBetweenIndexAndDares(xlsxPath, indexPath);
+  await saveDiff(diff);
+};
diff --git a/targets/alert-cli/src/dares/save.ts b/targets/alert-cli/src/dares/save.ts
@@ -0,0 +1,70 @@
+import { AlertRepository } from "../repositories/AlertRepository";
+import { DaresAlertInsert, Diff } from "./types";
+import { client } from "@shared/graphql-client";
+
+export const saveDiff = async (diff: Diff) => {
+  const alertRepository = new AlertRepository(client);
+
+  const alertsRemovedToSave: DaresAlertInsert[] =
+    diff.exceedingAgreementsFromKali.map((agreement) => ({
+      info: {
+        id: agreement.num,
+      },
+      status: "todo",
+      repository: "dares",
+      ref: "v0",
+      changes: {
+        type: "dares",
+        title: agreement.name,
+        ref: agreement.num.toString(),
+        date: new Date(),
+        modified: [],
+        removed: [
+          {
+            name: agreement.name,
+            num: agreement.num,
+          },
+        ],
+        added: [],
+        documents: [],
+      },
+    }));
+
+  const alertsAddedToSave: DaresAlertInsert[] =
+    diff.missingAgreementsFromDares.map((agreement) => ({
+      info: {
+        id: agreement.num,
+      },
+      status: "todo",
+      repository: "dares",
+      ref: "v0",
+      changes: {
+        type: "dares",
+        title: agreement.name,
+        ref: agreement.num.toString(),
+        date: new Date(),
+        modified: [],
+        added: [
+          {
+            name: agreement.name,
+            num: agreement.num,
+          },
+        ],
+        removed: [],
+        documents: [],
+      },
+    }));
+
+  const alertsToSave = [...alertsAddedToSave, ...alertsRemovedToSave];
+
+  const inserts = await Promise.allSettled(
+    alertsToSave.map((alert) => alertRepository.saveAlertDares(alert))
+  );
+
+  inserts.forEach((insert) => {
+    if (insert.status === "fulfilled") {
+      const { ref, repository: repo, info } = insert.value;
+      console.log(`insert alert for ${ref} on ${repo} (${info.id})`);
+    }
+  });
+};
diff --git a/targets/alert-cli/src/dares/scrapping.ts b/targets/alert-cli/src/dares/scrapping.ts
@@ -0,0 +1,12 @@
+import axios from "axios";
+
+export const extractXlsxFromUrl = async (url: string) => {
+  const response = await axios.get(url);
+  const html = response.data;
+  const regex = /href="([^"]*\.xlsx)"/g;
+  const match = regex.exec(html);
+  if (!match) {
+    throw new Error("No xlsx file found");
+  }
+  return match[1];
+};
diff --git a/targets/alert-cli/src/dares/types.ts b/targets/alert-cli/src/dares/types.ts
@@ -0,0 +1,13 @@
+import { DaresAlert } from "@shared/types";
+
+export interface Diff {
+  missingAgreementsFromDares: Agreement[];
+  exceedingAgreementsFromKali: Agreement[];
+}
+
+export interface Agreement {
+  name: string;
+  num: number;
+}
+
+export type DaresAlertInsert = Omit<DaresAlert, "id">;
diff --git a/targets/alert-cli/src/index.ts b/targets/alert-cli/src/index.ts
@@ -4,6 +4,7 @@ import { SourcesRepository } from "./repositories/SourcesRepository";
 import { AlertRepository } from "./repositories/AlertRepository";
 import { AlertDetector } from "./diff";
 import { FicheSPRepository } from "./repositories/FicheSPRepository";
+import { runDares } from "./dares";
 
 export * from "./types";
 
@@ -51,6 +52,7 @@ export async function run(
 }
 
 async function main() {
+  await runDares();
   const githubToken = process.env.GITHUB_TOKEN;
   if (!githubToken) {
     throw new Error("GITHUB_TOKEN is not defined");

diff --git a/targets/alert-cli/src/repositories/AlertRepository.ts b/targets/alert-cli/src/repositories/AlertRepository.ts
@@ -1,6 +1,7 @@
 import { Client } from "@urql/core/dist/types/client";
 import { AlertChanges, AlertInfo, HasuraAlert } from "@shared/types";
 import { batchPromises } from "../batchPromises";
+import { DaresAlertInsert } from "../dares/types";
 
 const insertAlertsMutation = `
 mutation insert_alert($alert: alerts_insert_input!) {
@@ -9,7 +10,7 @@ mutation insert_alert($alert: alerts_insert_input!) {
     update_columns: [changes]
   }) {
     repository,
-    ref
+    ref,
     info
   }
 }
@@ -30,6 +31,19 @@ export class AlertRepository {
     this.client = client;
   }
 
+  async saveAlertDares(data: DaresAlertInsert) {
+    const result = await this.client
+      .mutation<InsertAlertData>(insertAlertsMutation, {
+        alert: data,
+      })
+      .toPromise();
+    if (result.error || !result.data) {
+      console.error(result.error);
+      throw new Error("insertAlert");
+    }
+    return result.data.alert;
+  }
+
   async saveAlertChanges(repository: string, alertChanges: AlertChanges[]) {
     const inserts = await batchPromises(
       alertChanges,

diff --git a/targets/alert-cli/src/repositories/SourcesRepository.ts b/targets/alert-cli/src/repositories/SourcesRepository.ts
@@ -54,7 +54,8 @@ export class SourcesRepository {
       console.error(result.error);
       throw new Error("getSources");
     }
-    return result.data.sources;
+    const sources = result.data.sources;
+    return sources.filter((source) => source.repository !== "dares");
   }
 
   async updateSource(repository: string, tag: string): Promise<Source> {