From 2b8ca74446fd287defdf2ab3f2403bb674a24262 Mon Sep 17 00:00:00 2001 From: Oliver Foster Date: Tue, 12 Nov 2024 16:14:28 +0000 Subject: [PATCH 1/6] New: Added xliff translations (fixes #3624) --- grunt/helpers/Translate.js | 70 ++++++++++++++++++++++++++++++++++++-- package-lock.json | 22 ++++++------ package.json | 1 + 3 files changed, 78 insertions(+), 15 deletions(-) diff --git a/grunt/helpers/Translate.js b/grunt/helpers/Translate.js index 54a38b41e..aa4cd1a2b 100644 --- a/grunt/helpers/Translate.js +++ b/grunt/helpers/Translate.js @@ -2,6 +2,7 @@ const path = require('path'); const _ = require('lodash'); const fs = require('fs-extra'); const csv = require('csv'); +const { XMLParser, XMLBuilder, XMLValidator } = require('fast-xml-parser'); const async = require('async'); const globs = require('globs'); const jschardet = require('jschardet'); @@ -177,7 +178,40 @@ class Translate { const filePath = path.join(outputFolder, 'export.json'); this.log(`Exporting json to ${filePath}`); fs.writeJSONSync(filePath, exportTextData, { spaces: 2 }); - return; + return this; + } + + if (['xliff', 'xlf'].includes(this.format)) { + // create csv for each file + const outputGroupedByFile = exportTextData.reduce((prev, current) => { + if (!prev.hasOwnProperty(current.file)) { + prev[current.file] = []; + } + prev[current.file].push(current); + return prev; + }, {}); + + const output = ` + +${Object.entries(outputGroupedByFile).map(([fileName, entries]) => { + return ` +${entries.map(item => { + if (/[<>]+/.test(item.value)) return null; + return ` + + ${item.value} + ${item.value} + + +`; + }).filter(Boolean).join('')} +`; + }).join('')}`; + const filePath = path.join(outputFolder, 'source.xlf'); + this.log(`Exporting xliff to ${filePath}`); + fs.writeFileSync(filePath, `${output}`); + + return this; } // create csv for each file @@ -246,6 +280,8 @@ class Translate { } format = uniqueFileExtensions[0]; switch (format) { + case 'xlf': + case 'xliff': case 'csv': case 'json': this.log(`Format autodetected as ${format}`); @@ -255,7 +291,10 @@ class Translate { } } + if (format === 'xliff') format = 'xlf'; + // discover import files + console.log(`${inputFolder}/*.${format}`); const langFiles = globs.sync([`${inputFolder}/*.${format}`]); if (langFiles.length === 0) { throw new Error(`No languagefiles found to process in folder ${inputFolder}`); @@ -281,8 +320,32 @@ class Translate { case 'json': importData = fs.readJSONSync(langFiles[0]); break; - case 'csv': - default: + case 'xliff': + case 'xlf': { + importData = []; + await async.each(langFiles, (filename, done) => { + const XMLData = fs.readFileSync(filename); + const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '' + }); + const xml = parser.parse(XMLData); + for (const file of xml.xliff.file) { + for (const unit of file.unit) { + const [ id, ...path ] = unit.id.split('/'); + importData.push({ + file: file.id, + id, + path: path.filter(Boolean).join('/'), + value: unit.segment.target['#text'] + }); + } + } + done(); + }); + break; + } case 'csv': + default: { importData = []; const lines = []; await async.each(langFiles, (filename, done) => { @@ -349,6 +412,7 @@ class Translate { throw new Error(`Error processing CSV files: ${err}`); }); break; + } } // check import validity diff --git a/package-lock.json b/package-lock.json index 75e03d8a9..b908eb7c3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "chalk": "^2.4.1", "columnify": "^1.5.4", "csv": "^5.5.3", + "fast-xml-parser": "^4.5.0", "fs-extra": "^8.1.0", "globs": "^0.1.4", "grunt": "^1.6.1", @@ -8595,9 +8596,9 @@ "dev": true }, "node_modules/fast-xml-parser": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", - "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz", + "integrity": "sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==", "funding": [ { "type": "github", @@ -8608,7 +8609,7 @@ "url": "https://paypal.me/naturalintelligence" } ], - "optional": true, + "license": "MIT", "dependencies": { "strnum": "^1.0.5" }, @@ -20866,8 +20867,7 @@ "node_modules/strnum": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", - "optional": true + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" }, "node_modules/supports-color": { "version": "5.5.0", @@ -28621,10 +28621,9 @@ "dev": true }, "fast-xml-parser": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", - "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", - "optional": true, + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz", + "integrity": "sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==", "requires": { "strnum": "^1.0.5" } @@ -37527,8 +37526,7 @@ "strnum": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", - "optional": true + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" }, "supports-color": { "version": "5.5.0", diff --git a/package.json b/package.json index 770110919..4a050e844 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "chalk": "^2.4.1", "columnify": "^1.5.4", "csv": "^5.5.3", + "fast-xml-parser": "^4.5.0", "fs-extra": "^8.1.0", "globs": "^0.1.4", "grunt": "^1.6.1", From c485c8a20967cd5c1af1d97053d404cb80a875d2 Mon Sep 17 00:00:00 2001 From: Oliver Foster Date: Tue, 12 Nov 2024 16:16:40 +0000 Subject: [PATCH 2/6] Remove console log --- grunt/helpers/Translate.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/grunt/helpers/Translate.js b/grunt/helpers/Translate.js index aa4cd1a2b..bb71b6871 100644 --- a/grunt/helpers/Translate.js +++ b/grunt/helpers/Translate.js @@ -210,7 +210,6 @@ ${entries.map(item => { const filePath = path.join(outputFolder, 'source.xlf'); this.log(`Exporting xliff to ${filePath}`); fs.writeFileSync(filePath, `${output}`); - return this; } @@ -294,7 +293,6 @@ ${entries.map(item => { if (format === 'xliff') format = 'xlf'; // discover import files - console.log(`${inputFolder}/*.${format}`); const langFiles = globs.sync([`${inputFolder}/*.${format}`]); if (langFiles.length === 0) { throw new Error(`No languagefiles found to process in folder ${inputFolder}`); From 28ac5b5090c843b0d19719488dc7c0f41300db47 Mon Sep 17 00:00:00 2001 From: Oliver Foster Date: Tue, 12 Nov 2024 16:29:37 +0000 Subject: [PATCH 3/6] HTML escaping --- grunt/helpers/Translate.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/grunt/helpers/Translate.js b/grunt/helpers/Translate.js index bb71b6871..2136de58a 100644 --- a/grunt/helpers/Translate.js +++ b/grunt/helpers/Translate.js @@ -196,11 +196,13 @@ class Translate { ${Object.entries(outputGroupedByFile).map(([fileName, entries]) => { return ` ${entries.map(item => { - if (/[<>]+/.test(item.value)) return null; + const value = /[<>&"'/]/.test(item.value) + ? `` + : item.value; return ` - ${item.value} - ${item.value} + ${value} + ${value} `; From 8b4faccba621a940091abb928d2be3afc2622eab Mon Sep 17 00:00:00 2001 From: Oliver Foster Date: Tue, 12 Nov 2024 16:30:25 +0000 Subject: [PATCH 4/6] Remove imports --- grunt/helpers/Translate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grunt/helpers/Translate.js b/grunt/helpers/Translate.js index 2136de58a..32519a797 100644 --- a/grunt/helpers/Translate.js +++ b/grunt/helpers/Translate.js @@ -2,7 +2,7 @@ const path = require('path'); const _ = require('lodash'); const fs = require('fs-extra'); const csv = require('csv'); -const { XMLParser, XMLBuilder, XMLValidator } = require('fast-xml-parser'); +const { XMLParser } = require('fast-xml-parser'); const async = require('async'); const globs = require('globs'); const jschardet = require('jschardet'); From d0687de7ab73135df5a107a3f8d92d6b0e943e6f Mon Sep 17 00:00:00 2001 From: Oliver Foster Date: Tue, 12 Nov 2024 16:46:06 +0000 Subject: [PATCH 5/6] Switch to xliff 1.2 --- grunt/helpers/Translate.js | 56 ++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/grunt/helpers/Translate.js b/grunt/helpers/Translate.js index 32519a797..4047738ea 100644 --- a/grunt/helpers/Translate.js +++ b/grunt/helpers/Translate.js @@ -191,22 +191,41 @@ class Translate { return prev; }, {}); + // xliff 2.0 + // const output = ` + // + // ${Object.entries(outputGroupedByFile).map(([fileName, entries]) => { + // return ` + // ${entries.map(item => { + // const value = /[<>&"'/]/.test(item.value) + // ? `` + // : item.value; + // return ` + // + // ${value} + // ${value} + // + // + // `; + // }).filter(Boolean).join('')} + // `; + // }).join('')}`; + + // xliff 1.2 const output = ` - + ${Object.entries(outputGroupedByFile).map(([fileName, entries]) => { - return ` + return ` ${entries.map(item => { const value = /[<>&"'/]/.test(item.value) ? `` : item.value; - return ` - - ${value} - ${value} - - + return ` + ${value} + ${value} + `; - }).filter(Boolean).join('')} + }).filter(Boolean).join('')} `; }).join('')}`; const filePath = path.join(outputFolder, 'source.xlf'); @@ -330,14 +349,27 @@ ${entries.map(item => { attributeNamePrefix: '' }); const xml = parser.parse(XMLData); + // xliff 2.0 + // for (const file of xml.xliff.file) { + // for (const unit of file.unit) { + // const [ id, ...path ] = unit.id.split('/'); + // importData.push({ + // file: file.id, + // id, + // path: path.filter(Boolean).join('/'), + // value: unit.segment.target['#text'] + // }); + // } + // } + // xliff 1.2 for (const file of xml.xliff.file) { - for (const unit of file.unit) { + for (const unit of file.body['trans-unit']) { const [ id, ...path ] = unit.id.split('/'); importData.push({ - file: file.id, + file: file.original, id, path: path.filter(Boolean).join('/'), - value: unit.segment.target['#text'] + value: unit.source }); } } From 6f9d3916664c22208276a92e0c183c08e489161c Mon Sep 17 00:00:00 2001 From: Oliver Foster Date: Thu, 14 Nov 2024 11:54:26 +0000 Subject: [PATCH 6/6] Update grunt/helpers/Translate.js Co-authored-by: Cahir O'Doherty <41006337+cahirodoherty-learningpool@users.noreply.github.com> --- grunt/helpers/Translate.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/grunt/helpers/Translate.js b/grunt/helpers/Translate.js index 4047738ea..00b8b6ad3 100644 --- a/grunt/helpers/Translate.js +++ b/grunt/helpers/Translate.js @@ -376,7 +376,8 @@ ${entries.map(item => { done(); }); break; - } case 'csv': + } + case 'csv': default: { importData = []; const lines = [];