From 553c94f951eaa0f02285fbabb57273f37658d662 Mon Sep 17 00:00:00 2001 From: Valentin Dide Date: Thu, 14 Sep 2023 17:58:50 +0300 Subject: [PATCH 1/7] Reuse same object on retry. Signed-off-by: Valentin Dide --- index.js | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/index.js b/index.js index 9b40f2672..1cae35f57 100644 --- a/index.js +++ b/index.js @@ -5268,13 +5268,7 @@ export async function submitBom(args, bomContents) { "Content-Type": "application/json", "user-agent": `@CycloneDX/cdxgen ${_version}` }, - json: { - project: args.projectId, - projectName: args.projectName, - projectVersion: projectVersion, - autoCreate: "true", - bom: encodedBomContents - }, + json: bomPayload, responseType: "json" }).json(); } catch (error) { From 9c47c1def41e61173700050681f627cf9c1b92b0 Mon Sep 17 00:00:00 2001 From: Valentin Dide Date: Sat, 13 Apr 2024 16:31:14 +0300 Subject: [PATCH 2/7] Add license expression handling. Signed-off-by: Valentin Dide --- utils.js | 122 +++++++++++++++++++++++++++++++++++--------------- utils.test.js | 40 +++++++++++++++++ 2 files changed, 127 insertions(+), 35 deletions(-) diff --git a/utils.js b/utils.js index 24f57aeae..fa19960c5 100644 --- a/utils.js +++ b/utils.js @@ -274,6 +274,46 @@ function toBase64(hexString) { return Buffer.from(hexString, "hex").toString("base64"); } +const spdxLicenseExpressionOp = [ + " with ", + " and ", + " or ", // Apache-2.0 OR MIT + "-or-later", // GPL-2.0-or-later + "-only" // GPL-2.0-only +]; + +/** + * Method to determine if a license is a valid SPDX license expression + * + * @param {string} license License string + * @returns {boolean} true if the license is a valid SPDX license expression + * @see https://spdx.dev/learn/handling-license-info/ + **/ +function isSpdxLicenseExpression(license) { + const licenseLoweCase = (license || "").toLowerCase(); + if (!licenseLoweCase) { + return false; + } + + if ( + spdxLicenseExpressionOp.some((op) => { + return licenseLoweCase.includes(op); + }) + ) { + return true; + } + + if (licenseLoweCase.endsWith("+")) { + return true; // GPL-2.0+ means GPL-2.0 or any later version, at the licensee’s option. + } + + if (licenseLoweCase.includes("(") && licenseLoweCase.includes(")")) { + return true; // (MIT) means MIT license. It is unlikely to be used in SPDX license expression but just to be safe. + } + + return false; +} + /** * Performs a lookup + validation of the license specified in the * package. If the license is a valid SPDX license ID, set the 'id' @@ -286,43 +326,55 @@ export function getLicenses(pkg) { if (!Array.isArray(license)) { license = [license]; } - return license - .map((l) => { - let licenseContent = {}; - if (typeof l === "string" || l instanceof String) { - if ( - spdxLicenses.some((v) => { - return l === v; - }) - ) { - licenseContent.id = l; - licenseContent.url = "https://opensource.org/licenses/" + l; - } else if (l.startsWith("http")) { - const knownLicense = getKnownLicense(l, pkg); - if (knownLicense) { - licenseContent.id = knownLicense.id; - licenseContent.name = knownLicense.name; - } - // We always need a name to avoid validation errors - // Issue: #469 - if (!licenseContent.name && !licenseContent.id) { - licenseContent.name = "CUSTOM"; - } - licenseContent.url = l; - } else { - licenseContent.name = l; - } - } else if (Object.keys(l).length) { - licenseContent = l; + const licensesAndExpressions = license.map((l) => { + let licenseContent = {}; + if (typeof l === "string" || l instanceof String) { + if ( + spdxLicenses.some((v) => { + return l === v; + }) + ) { + licenseContent.id = l; + licenseContent.url = "https://opensource.org/licenses/" + l; + } else if (l.startsWith("http")) { + const knownLicense = getKnownLicense(l, pkg); + if (knownLicense) { + licenseContent.id = knownLicense.id; + licenseContent.name = knownLicense.name; + } + // We always need a name to avoid validation errors + // Issue: #469 + if (!licenseContent.name && !licenseContent.id) { + licenseContent.name = "CUSTOM"; + } + licenseContent.url = l; + } else if (isSpdxLicenseExpression(l)) { + licenseContent.expression = l; } else { - return undefined; - } - if (!licenseContent.id) { - addLicenseText(pkg, l, licenseContent); + licenseContent.name = l; } - return licenseContent; - }) - .map((l) => ({ license: l })); + } else if (Object.keys(l).length) { + licenseContent = l; + } else { + return undefined; + } + if (!licenseContent.id) { + addLicenseText(pkg, l, licenseContent); + } + return licenseContent; + }); + + const expressions = licensesAndExpressions.filter((f) => { + return f.expression; + }); + if (expressions.length > 1) { + console.warn("multiple license expressions found", expressions); + return [{ expression: expressions[0].expression }]; + } else if (expressions.length === 1) { + return [{ expression: expressions[0].expression }]; + } else { + return licensesAndExpressions.map((l) => ({ license: l })); + } } else { const knownLicense = getKnownLicense(undefined, pkg); if (knownLicense) { diff --git a/utils.test.js b/utils.test.js index 866398cd1..20ff80641 100644 --- a/utils.test.js +++ b/utils.test.js @@ -2058,6 +2058,46 @@ test("get licenses", () => { } } ]); + + licenses = getLicenses({ + license: "GPL-2.0+" + }); + expect(licenses).toEqual([ + { + license: { + id: "GPL-2.0+", + url: "https://opensource.org/licenses/GPL-2.0+" + } + } + ]); + + licenses = getLicenses({ + license: "(MIT or Apache-2.0)" + }); + expect(licenses).toEqual([ + { + expression: "(MIT or Apache-2.0)" + } + ]); + + // In case this is not a known license in the current build but it is a valid SPDX license expression + licenses = getLicenses({ + license: "NOT-GPL-2.1+" + }); + expect(licenses).toEqual([ + { + expression: "NOT-GPL-2.1+" + } + ]); + + licenses = getLicenses({ + license: "GPL-3.0-only WITH Classpath-exception-2.0" + }); + expect(licenses).toEqual([ + { + expression: "GPL-3.0-only WITH Classpath-exception-2.0" + } + ]); }); test("parsePkgJson", async () => { From 002e54133ea08bfaf5bb9cd06881ff9f4c0a2d36 Mon Sep 17 00:00:00 2001 From: Valentin Dide Date: Sat, 13 Apr 2024 19:51:18 +0300 Subject: [PATCH 3/7] Refactoring Signed-off-by: Valentin Dide --- binary.js | 35 +++++++------- evinser.js | 11 +++-- index.js | 5 +- utils.js | 134 ++++++++++++++++++++++++++++------------------------- 4 files changed, 99 insertions(+), 86 deletions(-) diff --git a/binary.js b/binary.js index 63c23be22..3d5680634 100644 --- a/binary.js +++ b/binary.js @@ -12,7 +12,13 @@ import { import { basename, dirname, join, resolve } from "node:path"; import { spawnSync } from "node:child_process"; import { PackageURL } from "packageurl-js"; -import { DEBUG_MODE, TIMEOUT_MS, findLicenseId } from "./utils.js"; +import { + DEBUG_MODE, + TIMEOUT_MS, + findLicenseId, + adjustLicenseInformation, + isSpdxLicenseExpression +} from "./utils.js"; import { URL, fileURLToPath } from "node:url"; @@ -572,33 +578,28 @@ export function getOSPackages(src) { comp.licenses.length ) { const newLicenses = []; - for (const alic of comp.licenses) { - if (alic.license.name) { - // Licenses array can either be made of expressions or id/name but not both - if ( - comp.licenses.length == 1 && - (alic.license.name.toUpperCase().includes(" AND ") || - alic.license.name.toUpperCase().includes(" OR ")) - ) { - newLicenses.push({ expression: alic.license.name }); + for (const aLic of comp.licenses) { + if (aLic.license.name) { + if (isSpdxLicenseExpression(aLic.license.name)) { + newLicenses.push({ expression: aLic.license.name }); } else { - const possibleId = findLicenseId(alic.license.name); - if (possibleId !== alic.license.name) { + const possibleId = findLicenseId(aLic.license.name); + if (possibleId !== aLic.license.name) { newLicenses.push({ license: { id: possibleId } }); } else { newLicenses.push({ - license: { name: alic.license.name } + license: { name: aLic.license.name } }); } } } else if ( - Object.keys(alic).length && - Object.keys(alic.license).length + Object.keys(aLic).length && + Object.keys(aLic.license).length ) { - newLicenses.push(alic); + newLicenses.push(aLic); } } - comp.licenses = newLicenses; + comp.licenses = adjustLicenseInformation(newLicenses); } // Fix hashes if ( diff --git a/evinser.js b/evinser.js index a920a8b32..a1036529a 100644 --- a/evinser.js +++ b/evinser.js @@ -5,7 +5,8 @@ import { executeAtom, getAllFiles, getGradleCommand, - getMavenCommand + getMavenCommand, + getTimestamp } from "./utils.js"; import { findCryptoAlgos } from "./cbomutils.js"; import { tmpdir } from "node:os"; @@ -1085,7 +1086,7 @@ export const createEvinseFile = (sliceArtefacts, options) => { bomJson.annotations.push({ subjects: [bomJson.serialNumber], annotator: { component: bomJson.metadata.tools.components[0] }, - timestamp: new Date().toISOString(), + timestamp: getTimestamp(), text: fs.readFileSync(usagesSlicesFile, "utf8") }); } @@ -1093,7 +1094,7 @@ export const createEvinseFile = (sliceArtefacts, options) => { bomJson.annotations.push({ subjects: [bomJson.serialNumber], annotator: { component: bomJson.metadata.tools.components[0] }, - timestamp: new Date().toISOString(), + timestamp: getTimestamp(), text: fs.readFileSync(dataFlowSlicesFile, "utf8") }); } @@ -1101,7 +1102,7 @@ export const createEvinseFile = (sliceArtefacts, options) => { bomJson.annotations.push({ subjects: [bomJson.serialNumber], annotator: { component: bomJson.metadata.tools.components[0] }, - timestamp: new Date().toISOString(), + timestamp: getTimestamp(), text: fs.readFileSync(reachablesSlicesFile, "utf8") }); } @@ -1109,7 +1110,7 @@ export const createEvinseFile = (sliceArtefacts, options) => { // Increment the version bomJson.version = (bomJson.version || 1) + 1; // Set the current timestamp to indicate this is newer - bomJson.metadata.timestamp = new Date().toISOString(); + bomJson.metadata.timestamp = getTimestamp(); delete bomJson.signature; fs.writeFileSync(evinseOutFile, JSON.stringify(bomJson, null, null)); if (occEvidencePresent || csEvidencePresent || servicesPresent) { diff --git a/index.js b/index.js index 650e965af..94cdf94ef 100644 --- a/index.js +++ b/index.js @@ -114,7 +114,8 @@ import { parseSwiftJsonTree, parseSwiftResolved, parseYarnLock, - readZipEntry + readZipEntry, + getTimestamp } from "./utils.js"; import { collectEnvInfo, @@ -479,7 +480,7 @@ function addMetadata(parentComponent = {}, options = {}) { const lifecycles = options.specVersion >= 1.5 ? addLifecyclesSection(options) : undefined; const metadata = { - timestamp: new Date().toISOString(), + timestamp: getTimestamp(), tools, authors, supplier: undefined diff --git a/utils.js b/utils.js index fa19960c5..cce0eb47e 100644 --- a/utils.js +++ b/utils.js @@ -274,13 +274,14 @@ function toBase64(hexString) { return Buffer.from(hexString, "hex").toString("base64"); } -const spdxLicenseExpressionOp = [ - " with ", - " and ", - " or ", // Apache-2.0 OR MIT - "-or-later", // GPL-2.0-or-later - "-only" // GPL-2.0-only -]; +/** + * Return the current timestamp in YYYY-MM-DDTHH:MM:SSZ format. + * + * @returns {string} ISO formatted timestamp, without milliseconds. + */ +export function getTimestamp() { + return new Date().toISOString().split(".")[0] + "Z"; +} /** * Method to determine if a license is a valid SPDX license expression @@ -295,11 +296,7 @@ function isSpdxLicenseExpression(license) { return false; } - if ( - spdxLicenseExpressionOp.some((op) => { - return licenseLoweCase.includes(op); - }) - ) { + if (/[(\s]+/gi.test(licenseLoweCase)) { return true; } @@ -307,11 +304,34 @@ function isSpdxLicenseExpression(license) { return true; // GPL-2.0+ means GPL-2.0 or any later version, at the licensee’s option. } - if (licenseLoweCase.includes("(") && licenseLoweCase.includes(")")) { - return true; // (MIT) means MIT license. It is unlikely to be used in SPDX license expression but just to be safe. + return false; +} + +/** + * Convert the array of licenses to a CycloneDX 1.5 compliant license array. + * This should return an array containing: + * - one or more SPDX license if no expression is present + * - the first license expression if at least one is present + * + * @param {Array} licenses Array of licenses + * @returns {Array} CycloneDX 1.5 compliant license array + */ +export function adjustLicenseInformation(licenses) { + if (!licenses || !Array.isArray(licenses)) { + return []; } - return false; + const expressions = licenses.filter((f) => { + return f.expression; + }); + if (expressions.length >= 1) { + if (expressions.length > 1) { + console.warn("multiple license expressions found", expressions); + } + return [{ expression: expressions[0].expression }]; + } else { + return licenses.map((l) => ({ license: l })); + } } /** @@ -326,55 +346,45 @@ export function getLicenses(pkg) { if (!Array.isArray(license)) { license = [license]; } - const licensesAndExpressions = license.map((l) => { - let licenseContent = {}; - if (typeof l === "string" || l instanceof String) { - if ( - spdxLicenses.some((v) => { - return l === v; - }) - ) { - licenseContent.id = l; - licenseContent.url = "https://opensource.org/licenses/" + l; - } else if (l.startsWith("http")) { - const knownLicense = getKnownLicense(l, pkg); - if (knownLicense) { - licenseContent.id = knownLicense.id; - licenseContent.name = knownLicense.name; - } - // We always need a name to avoid validation errors - // Issue: #469 - if (!licenseContent.name && !licenseContent.id) { - licenseContent.name = "CUSTOM"; - } - licenseContent.url = l; - } else if (isSpdxLicenseExpression(l)) { - licenseContent.expression = l; + return adjustLicenseInformation( + license.map((l) => { + let licenseContent = {}; + if (typeof l === "string" || l instanceof String) { + if ( + spdxLicenses.some((v) => { + return l === v; + }) + ) { + licenseContent.id = l; + licenseContent.url = "https://opensource.org/licenses/" + l; + } else if (l.startsWith("http")) { + const knownLicense = getKnownLicense(l, pkg); + if (knownLicense) { + licenseContent.id = knownLicense.id; + licenseContent.name = knownLicense.name; + } + // We always need a name to avoid validation errors + // Issue: #469 + if (!licenseContent.name && !licenseContent.id) { + licenseContent.name = "CUSTOM"; + } + licenseContent.url = l; + } else if (isSpdxLicenseExpression(l)) { + licenseContent.expression = l; + } else { + licenseContent.name = l; + } + } else if (Object.keys(l).length) { + licenseContent = l; } else { - licenseContent.name = l; + return undefined; } - } else if (Object.keys(l).length) { - licenseContent = l; - } else { - return undefined; - } - if (!licenseContent.id) { - addLicenseText(pkg, l, licenseContent); - } - return licenseContent; - }); - - const expressions = licensesAndExpressions.filter((f) => { - return f.expression; - }); - if (expressions.length > 1) { - console.warn("multiple license expressions found", expressions); - return [{ expression: expressions[0].expression }]; - } else if (expressions.length === 1) { - return [{ expression: expressions[0].expression }]; - } else { - return licensesAndExpressions.map((l) => ({ license: l })); - } + if (!licenseContent.id) { + addLicenseText(pkg, l, licenseContent); + } + return licenseContent; + }) + ); } else { const knownLicense = getKnownLicense(undefined, pkg); if (knownLicense) { From c7907fafa603a8ca20b130ff4b199caeb0ca5114 Mon Sep 17 00:00:00 2001 From: Valentin Dide Date: Sat, 13 Apr 2024 19:52:59 +0300 Subject: [PATCH 4/7] Forgot to save, Signed-off-by: Valentin Dide --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index cce0eb47e..36d807edf 100644 --- a/utils.js +++ b/utils.js @@ -290,7 +290,7 @@ export function getTimestamp() { * @returns {boolean} true if the license is a valid SPDX license expression * @see https://spdx.dev/learn/handling-license-info/ **/ -function isSpdxLicenseExpression(license) { +export function isSpdxLicenseExpression(license) { const licenseLoweCase = (license || "").toLowerCase(); if (!licenseLoweCase) { return false; From 9cbb828be4fa41375aeb7b1379c835ea03d66181 Mon Sep 17 00:00:00 2001 From: Valentin Dide Date: Sat, 13 Apr 2024 20:17:44 +0300 Subject: [PATCH 5/7] Improved log message. Signed-off-by: Valentin Dide --- validator.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/validator.js b/validator.js index eba593687..a162a1f8b 100644 --- a/validator.js +++ b/validator.js @@ -50,6 +50,9 @@ export const validateBom = (bomJson) => { ); const isValid = validate(bomJson); if (!isValid) { + console.log( + `Schema validation failed for ${bomJson.metadata.component.name}` + ); console.log(validate.errors); return false; } From c4ef4cb270edfdb53991135075020386da5fd647 Mon Sep 17 00:00:00 2001 From: Valentin Dide Date: Sat, 13 Apr 2024 20:40:53 +0300 Subject: [PATCH 6/7] Fixed failing tests. Signed-off-by: Valentin Dide --- .gitignore | 1 + utils.js | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3c460e0c1..13332ad0d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ tmp/ # BOM generated by CDXGEN during local development bom.json +bomresults # Logs logs diff --git a/utils.js b/utils.js index 36d807edf..f5a209094 100644 --- a/utils.js +++ b/utils.js @@ -330,7 +330,13 @@ export function adjustLicenseInformation(licenses) { } return [{ expression: expressions[0].expression }]; } else { - return licenses.map((l) => ({ license: l })); + return licenses.map((l) => { + if (typeof l.license === "object") { + return l; + } else { + return { license: l }; + } + }); } } From a229a7ca09782ebbc4a508d47460ac8ab2ac7417 Mon Sep 17 00:00:00 2001 From: Valentin Dide Date: Sat, 13 Apr 2024 21:27:05 +0300 Subject: [PATCH 7/7] Small fixes. Signed-off-by: Valentin Dide --- utils.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/utils.js b/utils.js index f5a209094..eb84f40d7 100644 --- a/utils.js +++ b/utils.js @@ -291,16 +291,15 @@ export function getTimestamp() { * @see https://spdx.dev/learn/handling-license-info/ **/ export function isSpdxLicenseExpression(license) { - const licenseLoweCase = (license || "").toLowerCase(); - if (!licenseLoweCase) { + if (!license) { return false; } - if (/[(\s]+/gi.test(licenseLoweCase)) { + if (/[(\s]+/g.test(license)) { return true; } - if (licenseLoweCase.endsWith("+")) { + if (license.endsWith("+")) { return true; // GPL-2.0+ means GPL-2.0 or any later version, at the licensee’s option. }