diff --git a/shared/types/index.ts b/shared/types/index.ts
index fe14efe88..3ffa5835c 100644
--- a/shared/types/index.ts
+++ b/shared/types/index.ts
@@ -265,7 +265,8 @@ export type AlertInfo = { id: string };
export type AlertChanges =
| DilaAlertChanges
| TravailDataAlertChanges
- | VddAlertChanges;
+ | VddAlertChanges
+ | DaresAlertChanges;
/** Dila alert changes */
export type DilaAlertChanges = DilaChanges & {
@@ -382,6 +383,34 @@ export type VddAlertChanges = VddChanges & {
date: Date;
};
+export type DaresAlertChanges = {
+ type: "dares";
+ title: string;
+ ref: string;
+ date: Date;
+ modified: [];
+ added: {
+ name: string;
+ num: number;
+ }[];
+ removed: {
+ name: string;
+ num: number;
+ }[];
+ documents: [];
+};
+
+export type DaresAlert = {
+ id: string;
+ info: {
+ id: string | number; // idcc number
+ };
+ status: "doing" | "done" | "rejected" | "todo";
+ repository: "dares";
+ ref: string;
+ changes: DaresAlertChanges;
+};
+
export type VddChanges = {
modified: FicheVddInfoWithDiff[];
removed: FicheVddInfo[];
diff --git a/targets/alert-cli/jest.config.js b/targets/alert-cli/jest.config.js
index 9cf3d8144..68311fc78 100644
--- a/targets/alert-cli/jest.config.js
+++ b/targets/alert-cli/jest.config.js
@@ -2,7 +2,7 @@ const config = {
preset: "ts-jest/presets/js-with-ts-esm",
testMatch: ["**/__tests__/**/?(*.)+(spec|test).+(ts|tsx|js)"],
transformIgnorePatterns: [
- "node_modules/(?!(unist-util-select|zwitch|unist-util-is)/)",
+ "node_modules/(?!(unist-util-select|zwitch|unist-util-is|axios)/)",
],
};
diff --git a/targets/alert-cli/package.json b/targets/alert-cli/package.json
index 53869a950..ab5ce1154 100644
--- a/targets/alert-cli/package.json
+++ b/targets/alert-cli/package.json
@@ -13,8 +13,10 @@
"@socialgouv/cdtn-slugify": "4.52.1",
"@socialgouv/cdtn-sources": "4.52.1",
"@socialgouv/dila-api-client": "1.2.4",
+ "axios": "^1.5.0",
"diff": "^5.1.0",
"memoizee": "0.4.15",
+ "node-xlsx": "^0.23.0",
"p-map": "4",
"semver": "7.3.5",
"simple-git": "^3.19.1",
diff --git a/targets/alert-cli/src/dares/__tests__/scrapping.test.ts b/targets/alert-cli/src/dares/__tests__/scrapping.test.ts
new file mode 100644
index 000000000..c06e531cf
--- /dev/null
+++ b/targets/alert-cli/src/dares/__tests__/scrapping.test.ts
@@ -0,0 +1,34 @@
+import axios from "axios";
+import { extractXlsxFromUrl } from "../scrapping";
+
+jest.mock("axios");
+
+describe("extractXlsxFromUrl", () => {
+ it("should extract xlsx file from url", async () => {
+ const url = "https://example.com/files";
+ const html = `
+
+
+ File 1
+ File 2
+
+
+ `;
+ (axios.get as jest.Mock).mockResolvedValueOnce({ data: html });
+ const result = await extractXlsxFromUrl(url);
+ expect(result).toBe("file1.xlsx");
+ });
+
+ it("should throw error if no xlsx file found", async () => {
+ const url = "https://example.com/files";
+ const html = `
+
+
+ File 1
+
+
+ `;
+ (axios.get as jest.Mock).mockResolvedValueOnce({ data: html });
+ await expect(extractXlsxFromUrl(url)).rejects.toThrow("No xlsx file found");
+ });
+});
diff --git a/targets/alert-cli/src/dares/config.ts b/targets/alert-cli/src/dares/config.ts
new file mode 100644
index 000000000..6ab297d75
--- /dev/null
+++ b/targets/alert-cli/src/dares/config.ts
@@ -0,0 +1,4 @@
+export const URL_SCRAPING =
+ "https://code.travail.gouv.fr/fiche-ministere-travail/conventions-collectives-nomenclatures";
+export const URL_KALI =
+ "https://raw.githubusercontent.com/SocialGouv/kali-data/master/data/index.json";
diff --git a/targets/alert-cli/src/dares/difference.ts b/targets/alert-cli/src/dares/difference.ts
new file mode 100644
index 000000000..7ff9dbd6a
--- /dev/null
+++ b/targets/alert-cli/src/dares/difference.ts
@@ -0,0 +1,46 @@
+import xlsx from "node-xlsx";
+import { Diff, Agreement } from "./types";
+import fs from "fs";
+
+export function getDifferenceBetweenIndexAndDares(
+ pathDares: string,
+ pathIndex: string
+): Diff {
+ const workSheetsFromFile = xlsx.parse(pathDares);
+
+ const supportedCcXlsx: Agreement[] = [];
+
+ workSheetsFromFile[0].data.forEach((row: string[]) => {
+ const ccNumber = parseInt(row[0]);
+ const ccName = row[1];
+ if (ccNumber && ccName) {
+ const ccNameWithoutParenthesis = ccName
+ .replace(/\(.*annexée.*\)/gi, "")
+ .trim();
+ supportedCcXlsx.push({
+ name: ccNameWithoutParenthesis,
+ num: ccNumber,
+ });
+ }
+ });
+
+ const dataJson = JSON.parse(fs.readFileSync(pathIndex, "utf8"));
+
+ const supportedCcIndexJson: Agreement[] = dataJson.map((cc: any) => {
+ return {
+ name: cc.title,
+ num: cc.num,
+ };
+ });
+
+ const missingAgreementsFromDares: Agreement[] = supportedCcXlsx.filter(
+ (ccIndex) =>
+ !supportedCcIndexJson.find((ccXlsx) => ccXlsx.num === ccIndex.num)
+ );
+
+ const exceedingAgreementsFromKali = supportedCcIndexJson.filter(
+ (ccXlsx) => !supportedCcXlsx.find((ccIndex) => ccIndex.num === ccXlsx.num)
+ );
+
+ return { missingAgreementsFromDares, exceedingAgreementsFromKali };
+}
diff --git a/targets/alert-cli/src/dares/download.ts b/targets/alert-cli/src/dares/download.ts
new file mode 100644
index 000000000..aadb64f70
--- /dev/null
+++ b/targets/alert-cli/src/dares/download.ts
@@ -0,0 +1,21 @@
+import axios from "axios";
+import fs from "fs";
+import os from "os";
+import path from "path";
+
+export async function downloadFileInTempFolder(
+ url: string,
+ nameOfFile: string
+): Promise {
+ const tempDir = os.tmpdir();
+ const filePath = path.join(tempDir, nameOfFile);
+
+ const response = await axios.get(url, { responseType: "stream" });
+ const writer = fs.createWriteStream(filePath);
+ response.data.pipe(writer);
+
+ return new Promise((resolve, reject) => {
+ writer.on("finish", () => resolve(filePath));
+ writer.on("error", reject);
+ });
+}
diff --git a/targets/alert-cli/src/dares/index.ts b/targets/alert-cli/src/dares/index.ts
new file mode 100644
index 000000000..fb54e3be8
--- /dev/null
+++ b/targets/alert-cli/src/dares/index.ts
@@ -0,0 +1,13 @@
+import { URL_KALI, URL_SCRAPING } from "./config";
+import { getDifferenceBetweenIndexAndDares } from "./difference";
+import { downloadFileInTempFolder } from "./download";
+import { saveDiff } from "./save";
+import { extractXlsxFromUrl } from "./scrapping";
+
+export const runDares = async () => {
+ const xlsxUrl = await extractXlsxFromUrl(URL_SCRAPING);
+ const xlsxPath = await downloadFileInTempFolder(xlsxUrl, "dares.xlsx");
+ const indexPath = await downloadFileInTempFolder(URL_KALI, "index.json");
+ const diff = await getDifferenceBetweenIndexAndDares(xlsxPath, indexPath);
+ await saveDiff(diff);
+};
diff --git a/targets/alert-cli/src/dares/save.ts b/targets/alert-cli/src/dares/save.ts
new file mode 100644
index 000000000..023441603
--- /dev/null
+++ b/targets/alert-cli/src/dares/save.ts
@@ -0,0 +1,70 @@
+import { AlertRepository } from "../repositories/AlertRepository";
+import { DaresAlertInsert, Diff } from "./types";
+import { client } from "@shared/graphql-client";
+
+export const saveDiff = async (diff: Diff) => {
+ const alertRepository = new AlertRepository(client);
+
+ const alertsRemovedToSave: DaresAlertInsert[] =
+ diff.exceedingAgreementsFromKali.map((agreement) => ({
+ info: {
+ id: agreement.num,
+ },
+ status: "todo",
+ repository: "dares",
+ ref: "v0",
+ changes: {
+ type: "dares",
+ title: agreement.name,
+ ref: agreement.num.toString(),
+ date: new Date(),
+ modified: [],
+ removed: [
+ {
+ name: agreement.name,
+ num: agreement.num,
+ },
+ ],
+ added: [],
+ documents: [],
+ },
+ }));
+
+ const alertsAddedToSave: DaresAlertInsert[] =
+ diff.missingAgreementsFromDares.map((agreement) => ({
+ info: {
+ id: agreement.num,
+ },
+ status: "todo",
+ repository: "dares",
+ ref: "v0",
+ changes: {
+ type: "dares",
+ title: agreement.name,
+ ref: agreement.num.toString(),
+ date: new Date(),
+ modified: [],
+ added: [
+ {
+ name: agreement.name,
+ num: agreement.num,
+ },
+ ],
+ removed: [],
+ documents: [],
+ },
+ }));
+
+ const alertsToSave = [...alertsAddedToSave, ...alertsRemovedToSave];
+
+ const inserts = await Promise.allSettled(
+ alertsToSave.map((alert) => alertRepository.saveAlertDares(alert))
+ );
+
+ inserts.forEach((insert) => {
+ if (insert.status === "fulfilled") {
+ const { ref, repository: repo, info } = insert.value;
+ console.log(`insert alert for ${ref} on ${repo} (${info.id})`);
+ }
+ });
+};
diff --git a/targets/alert-cli/src/dares/scrapping.ts b/targets/alert-cli/src/dares/scrapping.ts
new file mode 100644
index 000000000..1072001a5
--- /dev/null
+++ b/targets/alert-cli/src/dares/scrapping.ts
@@ -0,0 +1,12 @@
+import axios from "axios";
+
+export const extractXlsxFromUrl = async (url: string) => {
+ const response = await axios.get(url);
+ const html = response.data;
+ const regex = /href="([^"]*\.xlsx)"/g;
+ const match = regex.exec(html);
+ if (!match) {
+ throw new Error("No xlsx file found");
+ }
+ return match[1];
+};
diff --git a/targets/alert-cli/src/dares/types.ts b/targets/alert-cli/src/dares/types.ts
new file mode 100644
index 000000000..3ca4af77c
--- /dev/null
+++ b/targets/alert-cli/src/dares/types.ts
@@ -0,0 +1,13 @@
+import { DaresAlert } from "@shared/types";
+
+export interface Diff {
+ missingAgreementsFromDares: Agreement[];
+ exceedingAgreementsFromKali: Agreement[];
+}
+
+export interface Agreement {
+ name: string;
+ num: number;
+}
+
+export type DaresAlertInsert = Omit;
diff --git a/targets/alert-cli/src/index.ts b/targets/alert-cli/src/index.ts
index fa8ea4b57..3d79fe2d5 100644
--- a/targets/alert-cli/src/index.ts
+++ b/targets/alert-cli/src/index.ts
@@ -4,6 +4,7 @@ import { SourcesRepository } from "./repositories/SourcesRepository";
import { AlertRepository } from "./repositories/AlertRepository";
import { AlertDetector } from "./diff";
import { FicheSPRepository } from "./repositories/FicheSPRepository";
+import { runDares } from "./dares";
export * from "./types";
@@ -51,6 +52,7 @@ export async function run(
}
async function main() {
+ await runDares();
const githubToken = process.env.GITHUB_TOKEN;
if (!githubToken) {
throw new Error("GITHUB_TOKEN is not defined");
diff --git a/targets/alert-cli/src/repositories/AlertRepository.ts b/targets/alert-cli/src/repositories/AlertRepository.ts
index 57f7f9dd2..1dfdc772b 100644
--- a/targets/alert-cli/src/repositories/AlertRepository.ts
+++ b/targets/alert-cli/src/repositories/AlertRepository.ts
@@ -1,6 +1,7 @@
import { Client } from "@urql/core/dist/types/client";
import { AlertChanges, AlertInfo, HasuraAlert } from "@shared/types";
import { batchPromises } from "../batchPromises";
+import { DaresAlertInsert } from "../dares/types";
const insertAlertsMutation = `
mutation insert_alert($alert: alerts_insert_input!) {
@@ -9,7 +10,7 @@ mutation insert_alert($alert: alerts_insert_input!) {
update_columns: [changes]
}) {
repository,
- ref
+ ref,
info
}
}
@@ -30,6 +31,19 @@ export class AlertRepository {
this.client = client;
}
+ async saveAlertDares(data: DaresAlertInsert) {
+ const result = await this.client
+ .mutation(insertAlertsMutation, {
+ alert: data,
+ })
+ .toPromise();
+ if (result.error || !result.data) {
+ console.error(result.error);
+ throw new Error("insertAlert");
+ }
+ return result.data.alert;
+ }
+
async saveAlertChanges(repository: string, alertChanges: AlertChanges[]) {
const inserts = await batchPromises(
alertChanges,
diff --git a/targets/alert-cli/src/repositories/SourcesRepository.ts b/targets/alert-cli/src/repositories/SourcesRepository.ts
index 2c491e20c..da2c101db 100644
--- a/targets/alert-cli/src/repositories/SourcesRepository.ts
+++ b/targets/alert-cli/src/repositories/SourcesRepository.ts
@@ -54,7 +54,8 @@ export class SourcesRepository {
console.error(result.error);
throw new Error("getSources");
}
- return result.data.sources;
+ const sources = result.data.sources;
+ return sources.filter((source) => source.repository !== "dares");
}
async updateSource(repository: string, tag: string): Promise {
diff --git a/targets/frontend/src/components/changes/ChangeGroup.tsx b/targets/frontend/src/components/changes/ChangeGroup.tsx
index 56d84f51f..d22956b24 100644
--- a/targets/frontend/src/components/changes/ChangeGroup.tsx
+++ b/targets/frontend/src/components/changes/ChangeGroup.tsx
@@ -193,6 +193,23 @@ export function AddedChanges({ changes }: ChangesProps): JSX.Element {
);
}
+
+ case "dares": {
+ return (
+ <>
+ {changes.added.map((change) => (
+
+ Convention collective
+
+ {change.num}
+
+ présente dans le fichier de la DARES mais absente de la base de
+ données du CDTN
+
+ ))}
+
+ );
+ }
}
}
@@ -240,6 +257,22 @@ export function RemovedChanges({ changes }: ChangesProps): JSX.Element {
);
}
+ case "dares": {
+ return (
+ <>
+ {changes.removed.map((change) => (
+
+ Convention collective
+
+ {change.num}
+
+ présente dans la base de données du CDTN mais absente dans le
+ fichier de la DARES
+
+ ))}
+
+ );
+ }
}
}
@@ -372,6 +405,9 @@ export function ModifiedChanges({ changes }: ChangesProps): JSX.Element {
);
}
+ case "dares": {
+ return <>;
+ }
}
}
@@ -379,7 +415,7 @@ function getBadgeColor(etat: string) {
switch (etat) {
case "VIGUEUR":
return theme.colors.positive;
- case "MOIFIE":
+ case "MODIFIE":
return theme.colors.caution;
case "ABROGE":
case "ABROGE_DIFF":
diff --git a/targets/hasura/migrations/default/1693311202850_insert_into_public_sources/down.sql b/targets/hasura/migrations/default/1693311202850_insert_into_public_sources/down.sql
new file mode 100644
index 000000000..d3852e3b2
--- /dev/null
+++ b/targets/hasura/migrations/default/1693311202850_insert_into_public_sources/down.sql
@@ -0,0 +1 @@
+DELETE FROM "public"."sources" WHERE "repository" = 'dares';
diff --git a/targets/hasura/migrations/default/1693311202850_insert_into_public_sources/up.sql b/targets/hasura/migrations/default/1693311202850_insert_into_public_sources/up.sql
new file mode 100644
index 000000000..ca86a4571
--- /dev/null
+++ b/targets/hasura/migrations/default/1693311202850_insert_into_public_sources/up.sql
@@ -0,0 +1 @@
+INSERT INTO "public"."sources"("repository", "label", "tag", "created_at") VALUES (E'dares', E'Liste IDCC (dares)', E'v0', E'2023-08-29T12:13:22.797621+00:00');
diff --git a/yarn.lock b/yarn.lock
index 3314b4454..cfc911f5e 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5776,6 +5776,15 @@ axios@^1.0.0:
form-data "^4.0.0"
proxy-from-env "^1.1.0"
+axios@^1.5.0:
+ version "1.5.0"
+ resolved "https://registry.yarnpkg.com/axios/-/axios-1.5.0.tgz#f02e4af823e2e46a9768cfc74691fdd0517ea267"
+ integrity sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==
+ dependencies:
+ follow-redirects "^1.15.0"
+ form-data "^4.0.0"
+ proxy-from-env "^1.1.0"
+
axobject-query@^3.1.1:
version "3.2.1"
resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-3.2.1.tgz#39c378a6e3b06ca679f29138151e45b2b32da62a"
@@ -13444,6 +13453,13 @@ node-releases@^2.0.12:
resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.12.tgz#35627cc224a23bfb06fb3380f2b3afaaa7eb1039"
integrity sha512-QzsYKWhXTWx8h1kIvqfnC++o0pEmpRQA/aenALsL2F4pqNVr7YzcdMlDij5WBnwftRbJCNJL/O7zdKaxKPHqgQ==
+node-xlsx@^0.23.0:
+ version "0.23.0"
+ resolved "https://registry.yarnpkg.com/node-xlsx/-/node-xlsx-0.23.0.tgz#0c4b642f9457712d68f30e1e30351d640cc37e90"
+ integrity sha512-r3KaSZSsSrK92rbPXnX/vDdxURmPPik0rjJ3A+Pybzpjyrk4G6WyGfj8JIz5dMMEpCmWVpmO4qoVPBxnpLv/8Q==
+ dependencies:
+ xlsx "https://cdn.sheetjs.com/xlsx-0.19.3/xlsx-0.19.3.tgz"
+
nodemailer@^6.6.5:
version "6.9.3"
resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.9.3.tgz#e4425b85f05d83c43c5cd81bf84ab968f8ef5cbe"
@@ -18770,6 +18786,10 @@ xdg-basedir@^4.0.0:
resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13"
integrity sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==
+"xlsx@https://cdn.sheetjs.com/xlsx-0.19.3/xlsx-0.19.3.tgz":
+ version "0.19.3"
+ resolved "https://cdn.sheetjs.com/xlsx-0.19.3/xlsx-0.19.3.tgz#f804c1850e2da5260165db0a059dc2a6099d55f3"
+
xml-name-validator@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"