#124 improve UI with new fields

.gitignore

@@ -4,3 +4,5 @@ node_modules
 .env
 deploy_sill.js
 docker-data
+.dev
+.bruno

api/.env.sh

@@ -23,7 +23,9 @@ export CONFIGURATION=$(cat << EOF
   "githubWebhookSecret": "$SILL_WEBHOOK_SECRET",
   "port": $SILL_API_PORT,
   "isDevEnvironnement": $SILL_IS_DEV_ENVIRONNEMENT,
-  "externalSoftwareDataOrigin": $SILL_EXTERNAL_SOFTWARE_DATA_ORIGIN
+  "externalSoftwareDataOrigin": $SILL_EXTERNAL_SOFTWARE_DATA_ORIGIN,
+  "initializeSoftwareFromSource": $INITSOFTFROMSOURCE,
+  "botAgentEmail": $BOTAGENTEMAIL,
 }
 EOF
 ) 

api/scripts/compile-data.ts

@@ -15,7 +15,9 @@ import { env } from "../src/env";
         "githubPersonalAccessTokenForApiRateLimit": env.githubPersonalAccessTokenForApiRateLimit,
         "doPerPerformPeriodicalCompilation": false,
         "doPerformCacheInitialization": false,
-        "externalSoftwareDataOrigin": env.externalSoftwareDataOrigin
+        "externalSoftwareDataOrigin": env.externalSoftwareDataOrigin,
+        "initializeSoftwareFromSource": env.initializeSoftwareFromSource,
+        "botAgentEmail": env.botAgentEmail
     });
 
     await useCases.fetchAndSaveExternalDataForAllSoftwares();

api/scripts/migration/software.ts

@@ -114,7 +114,7 @@ const zSoftwareRow = z.object({
     "license": z.string(),
     "softwareType": zSoftwareType,
     "catalogNumeriqueGouvFrId": z.string().optional(),
-    "versionMin": z.string(),
+    "versionMin": z.string().optional(),
     "workshopUrls": z.array(z.string()),
     "testUrls": z.array(
         z.object({

api/src/core/adapters/dbApi/kysely/createPgSoftwareExternalDataRepository.ts

@@ -8,6 +8,9 @@ export const createPgSoftwareExternalDataRepository = (db: Kysely<Database>): So
             ...softwareExternalData,
             developers: JSON.stringify(softwareExternalData.developers),
             label: JSON.stringify(softwareExternalData.label),
+            keywords: JSON.stringify(softwareExternalData.keywords),
+            applicationCategory: JSON.stringify(softwareExternalData.applicationCategory),
+            programmingLanguage: JSON.stringify(softwareExternalData.programmingLanguage),
             description: JSON.stringify(softwareExternalData.description)
         };
 

api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts

@@ -65,7 +65,7 @@ export const createPgSoftwareRepository = (db: Kysely<Database>): SoftwareReposi
                     .executeTakeFirstOrThrow();
 
                 console.log(
-                    `inserted software correctly, softwareId is : ${softwareId}, about to insert similars : `,
+                    `inserted software correctly, softwareId is : ${softwareId} (${softwareName}), about to insert similars : `,
                     similarSoftwareExternalDataIds
                 );
 
@@ -178,7 +178,8 @@ export const createPgSoftwareRepository = (db: Kysely<Database>): SoftwareReposi
                         documentationUrl: softwareExternalData?.documentationUrl,
                         comptoirDuLibreServiceProviderCount: software.comptoirDuLibreSoftware?.providers.length ?? 0,
                         testUrl: testUrls[0]?.url,
-                        parentWikidataSoftware: parentExternalData
+                        parentWikidataSoftware: parentExternalData,
+                        keywords: software?.keywords ?? softwareExternalData?.keywords ?? []
                     });
                 }),
         getById: getBySoftwareId,
@@ -222,6 +223,16 @@ export const createPgSoftwareRepository = (db: Kysely<Database>): SoftwareReposi
 
             return builder.execute().then(async softwares => {
                 const userAndReferentCountByOrganization = await getUserAndReferentCountByOrganizationBySoftwareId(db);
+                const dateParser = (str: string | Date | undefined | null) => {
+                    if (str && typeof str === "string") {
+                        const date = new Date(str);
+                        return date.valueOf();
+                    } else if (str && str instanceof Date) {
+                        return str.valueOf();
+                    } else {
+                        return undefined;
+                    }
+                };
                 return softwares.map(
                     ({
                         testUrls,
@@ -248,6 +259,10 @@ export const createPgSoftwareRepository = (db: Kysely<Database>): SoftwareReposi
                             //         isInSill: true // TODO: check if this is true
                             //     })
                             // ) ?? [],
+                            latestVersion: software.latestVersion ?? {
+                                semVer: softwareExternalData?.softwareVersion ?? "",
+                                publicationTime: dateParser(softwareExternalData.publicationTime)
+                            },
                             userAndReferentCountByOrganization:
                                 userAndReferentCountByOrganization[software.softwareId] ?? {},
                             authors: (softwareExternalData?.developers ?? []).map(dev => ({
@@ -266,7 +281,9 @@ export const createPgSoftwareRepository = (db: Kysely<Database>): SoftwareReposi
                             comptoirDuLibreServiceProviderCount:
                                 software.comptoirDuLibreSoftware?.providers.length ?? 0,
                             testUrl: testUrls[0]?.url,
-                            parentWikidataSoftware: parentExternalData ?? undefined
+                            parentWikidataSoftware: parentExternalData ?? undefined,
+                            applicationCategory: softwareExternalData?.applicationCategory ?? [],
+                            programmingLanguage: softwareExternalData?.programmingLanguage ?? []
                         });
                     }
                 );
@@ -398,7 +415,12 @@ const makeGetSoftwareBuilder = (db: Kysely<Database>) =>
                     framaLibreId: ref("ext.framaLibreId"),
                     websiteUrl: ref("ext.websiteUrl"),
                     sourceUrl: ref("ext.sourceUrl"),
-                    documentationUrl: ref("ext.documentationUrl")
+                    documentationUrl: ref("ext.documentationUrl"),
+                    programmingLanguage: ref("ext.programmingLanguage"),
+                    applicationCategory: ref("ext.applicationCategory"),
+                    keywords: ref("ext.keywords"),
+                    softwareVersion: ref("ext.softwareVersion"),
+                    publicationTime: ref("ext.publicationTime")
                 }).as("softwareExternalData"),
             ({ ref, fn }) =>
                 fn

api/src/core/adapters/dbApi/kysely/kysely.database.ts

@@ -77,6 +77,11 @@ type SoftwareExternalDatasTable = {
     sourceUrl: string | null;
     documentationUrl: string | null;
     license: string | null;
+    softwareVersion: string | null;
+    keywords: JSONColumnType<string[]> | null;
+    programmingLanguage: JSONColumnType<string[]> | null;
+    applicationCategory: JSONColumnType<string[]> | null;
+    publicationTime: Date | null;
 };
 
 type SoftwareType =
@@ -110,7 +115,7 @@ type SoftwaresTable = {
     license: string;
     softwareType: JSONColumnType<SoftwareType>;
     catalogNumeriqueGouvFrId: string | null;
-    versionMin: string;
+    versionMin: string | null;
     workshopUrls: JSONColumnType<string[]>;
     testUrls: JSONColumnType<
         {

api/src/core/adapters/dbApi/kysely/migrations/1731574107668_allow_null_for_version.ts

@@ -0,0 +1,33 @@
+import { Kysely } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+    await db.schema
+        .alterTable("softwares")
+        .alterColumn("versionMin", ac => ac.dropNotNull())
+        .execute();
+
+    await db.schema
+        .alterTable("software_external_datas")
+        .addColumn("softwareVersion", "text")
+        .addColumn("publicationTime", "date")
+        .addColumn("keywords", "jsonb")
+        .addColumn("programmingLanguage", "jsonb")
+        .addColumn("applicationCategory", "jsonb")
+        .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+    await db.schema
+        .alterTable("softwares")
+        .alterColumn("versionMin", ac => ac.setNotNull())
+        .execute();
+
+    await db.schema
+        .alterTable("software_external_datas")
+        .dropColumn("softwareVersion")
+        .dropColumn("publicationTime")
+        .dropColumn("keywords")
+        .dropColumn("programmingLanguage")
+        .dropColumn("applicationCategory")
+        .execute();
+}

api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts

@@ -49,7 +49,12 @@ const softwareExternalData: SoftwareExternalData = {
     websiteUrl: "https://example.com",
     sourceUrl: "https://example.com/source",
     documentationUrl: "https://example.com/documentation",
-    license: "MIT"
+    license: "MIT",
+    softwareVersion: "1.0.0",
+    keywords: ["Usefull", "Daily"],
+    programmingLanguage: ["C++"],
+    applicationCategory: ["Software Cat I", "Software Cat II"],
+    publicationTime: new Date(1561566581000)
 };
 
 const similarSoftwareExternalData: SoftwareExternalData = {
@@ -64,7 +69,12 @@ const similarSoftwareExternalData: SoftwareExternalData = {
     websiteUrl: "https://example.similar.com",
     sourceUrl: "https://example.similar.com/source",
     documentationUrl: "https://example.similar.com/documentation",
-    license: "MIT"
+    license: "MIT",
+    softwareVersion: "3.0.2",
+    keywords: ["Infra", "Adminsys"],
+    programmingLanguage: ["Python3"],
+    applicationCategory: ["Software Cat I", "Software Cat II"],
+    publicationTime: new Date(1561566581000)
 };
 
 const parentSoftwareExternalData: SoftwareExternalData = {
@@ -151,6 +161,7 @@ describe("pgDbApi", () => {
                 addedTime: expect.any(Number),
                 updateTime: expect.any(Number),
                 annuaireCnllServiceProviders: undefined,
+                applicationCategory: ["Software Cat I", "Software Cat II"],
                 authors: softwareExternalData.developers.map(dev => ({
                     authorName: dev.name,
                     authorUrl: `https://www.wikidata.org/wiki/${dev.id}`
@@ -164,7 +175,10 @@ describe("pgDbApi", () => {
                 externalDataOrigin: "wikidata",
                 externalId,
                 keywords: ["bob", "l'éponge"],
-                latestVersion: undefined,
+                latestVersion: {
+                    "publicationTime": 1561507200000,
+                    "semVer": "1.0.0"
+                },
                 license: "MIT",
                 logoUrl: "https://example.com/logo.png",
                 officialWebsiteUrl: softwareExternalData.websiteUrl,
@@ -178,6 +192,7 @@ describe("pgDbApi", () => {
                     isFromFrenchPublicServices: false,
                     isPresentInSupportContract: true
                 },
+                programmingLanguage: ["C++"],
                 serviceProviders: [],
                 similarSoftwares: [
                     {
@@ -418,7 +433,10 @@ describe("pgDbApi", () => {
                     ...softExtData,
                     developers: JSON.stringify(softExtData.developers),
                     label: JSON.stringify(softExtData.label),
-                    description: JSON.stringify(softExtData.description)
+                    description: JSON.stringify(softExtData.description),
+                    keywords: JSON.stringify(softwareExternalData.keywords),
+                    applicationCategory: JSON.stringify(softwareExternalData.applicationCategory),
+                    programmingLanguage: JSON.stringify(softwareExternalData.programmingLanguage)
                 }))
             )
             .execute();

api/src/core/adapters/fetchExternalData.test.ts

@@ -175,20 +175,26 @@ describe("fetches software extra data (from different providers)", () => {
             const updatedSoftwareExternalDatas = await db.selectFrom("software_external_datas").selectAll().execute();
             expectToEqual(updatedSoftwareExternalDatas, [
                 {
+                    applicationCategory: null,
                     description: "A framwork for creating react SPA that uses webpack as bundler",
                     developers: [],
                     documentationUrl: null,
                     externalDataOrigin: "wikidata",
                     externalId: craSoftwareFormData.externalId,
                     framaLibreId: null,
                     isLibreSoftware: true,
+                    keywords: [],
                     label: "create-react-app",
                     license: "MIT licence",
                     logoUrl: null,
                     sourceUrl: "https://github.com/facebook/create-react-app",
-                    websiteUrl: "https://create-react-app.dev/"
+                    websiteUrl: "https://create-react-app.dev/",
+                    programmingLanguage: [],
+                    softwareVersion: "5.0.1",
+                    publicationTime: new Date("2022-04-11T22:00:00.000Z")
                 },
                 {
+                    applicationCategory: null,
                     description: "open-source JavaScript module bundler",
                     developers: [
                         {
@@ -201,12 +207,16 @@ describe("fetches software extra data (from different providers)", () => {
                     externalId: "Q111590996",
                     framaLibreId: null,
                     isLibreSoftware: true,
+                    keywords: [],
                     label: "Vite",
                     license: "MIT licence",
                     logoUrl:
                         "//upload.wikimedia.org/wikipedia/commons/thumb/f/f1/Vitejs-logo.svg/220px-Vitejs-logo.svg.png",
                     sourceUrl: "https://github.com/vitejs/vite",
-                    websiteUrl: "https://vitejs.dev/"
+                    websiteUrl: "https://vitejs.dev/",
+                    programmingLanguage: ["JavaScript"],
+                    softwareVersion: "5.4.10",
+                    publicationTime: new Date("2024-10-22T22:00:00.000Z")
                 }
             ]);
 
@@ -234,6 +244,7 @@ describe("fetches software extra data (from different providers)", () => {
             const updatedSoftwareExternalDatas = await db.selectFrom("software_external_datas").selectAll().execute();
             expectToEqual(updatedSoftwareExternalDatas, [
                 {
+                    applicationCategory: null,
                     description: {
                         en: "open-source web server software",
                         fr: "serveur web sous licence libre"
@@ -249,12 +260,16 @@ describe("fetches software extra data (from different providers)", () => {
                     externalId: "Q11354",
                     framaLibreId: null,
                     isLibreSoftware: false,
+                    keywords: [],
                     label: "Apache HTTP Server",
                     license: "Apache License v2.0",
                     logoUrl:
                         "//upload.wikimedia.org/wikipedia/commons/thumb/1/10/Apache_HTTP_server_logo_%282019-present%29.svg/220px-Apache_HTTP_server_logo_%282019-present%29.svg.png",
                     sourceUrl: "https://github.com/apache/httpd",
-                    websiteUrl: "https://httpd.apache.org/"
+                    websiteUrl: "https://httpd.apache.org/",
+                    programmingLanguage: ["C"],
+                    softwareVersion: "2.5.0-alpha",
+                    publicationTime: new Date("2017-11-07T23:00:00.000Z")
                 }
             ]);
 

api/src/core/adapters/hal/getHalSoftware.test.ts

@@ -11,26 +11,31 @@ describe("HAL", () => {
             const result = await getHalSoftware("1715545");
 
             expectToEqual(result, {
-                "description": { "en": "-" },
+                "description": { "en": "-", "fr": undefined },
                 "developers": [
                     {
-                        "id": "gruenpeter,-morane",
-                        "name": "Gruenpeter, Morane"
+                        "id": "morane-gruenpeter",
+                        "name": "Morane Gruenpeter"
                     }
                 ],
-                "documentationUrl": "https://inria.hal.science/hal-01715545v1",
+                "documentationUrl": undefined,
                 "externalId": "1715545",
-                "framaLibreId": "",
+                "framaLibreId": undefined,
                 "isLibreSoftware": true,
                 "label": {
                     "en": "Battleship exercise",
                     "fr": "Battleship exercise"
                 },
                 "license": "MIT License",
-                "logoUrl": "",
+                "logoUrl": undefined,
                 "externalDataOrigin": "HAL",
                 "sourceUrl": "https://github.com/moranegg/Battleship",
-                "websiteUrl": "https://inria.hal.science/hal-01715545"
+                "websiteUrl": "https://inria.hal.science/hal-01715545v1",
+                "softwareVersion": undefined,
+                "keywords": undefined,
+                "programmingLanguage": undefined,
+                "applicationCategory": ["info"],
+                "publicationTime": new Date(1561566581000)
             });
         });
     });

api/src/core/adapters/hal/getHalSoftware.ts

@@ -40,8 +40,6 @@ export async function fetchHalSoftwareById(halDocid: string): Promise<HalRawSoft
         `https://api.archives-ouvertes.fr/search/?q=docid:${halDocid}&wt=json&fl=${halSoftwareFieldsToReturnAsString}&sort=docid%20asc`
     ).catch(() => undefined);
 
-    console.log("Hal response status : ", res?.status);
-
     if (res === undefined) {
         throw new HalFetchError(undefined);
     }
@@ -59,3 +57,26 @@ export async function fetchHalSoftwareById(halDocid: string): Promise<HalRawSoft
 
     return json.response.docs[0];
 }
+
+export async function fetchHalSoftwares(): Promise<Array<HalRawSoftware>> {
+    // Filter only software who have an swhidId to filter clean data on https://hal.science, TODO remove and set it as an option to be generic
+    const url = `https://api.archives-ouvertes.fr/search/?q=docType_s:SOFTWARE&rows=10000&fl=${halSoftwareFieldsToReturnAsString}&fq=swhidId_s:["" TO *]`;
+
+    const res = await fetch(url).catch(err => {
+        console.error(err);
+        throw new HalFetchError(undefined);
+    });
+
+    if (res.status === 429) {
+        await new Promise(resolve => setTimeout(resolve, 100));
+        return fetchHalSoftwares();
+    }
+
+    if (res.status === 404) {
+        throw new HalFetchError(res.status);
+    }
+
+    const json = await res.json();
+
+    return json.response.docs;
+}