diff --git a/README.md b/README.md index 9adce6d..69997af 100644 --- a/README.md +++ b/README.md @@ -25,12 +25,16 @@ import|i [options] [collections...] Options: ``` --i, --id [field] Field to use for Document IDs (default: doc_id) --a, --auto-id [str] Document ID token specifying auto generated Document ID (default: Auto-ID) +-i, --id [field] Field to use for Document IDs. (default: doc_id) +-a, --auto-id [str] Document ID token specifying auto generated Document ID. (default: Auto-ID) -m, --merge Merge Firestore documents. Default is Replace. -k, --chunk [size] Split upload into batches. Max 500 by Firestore constraints. (default: 500) --p, --coll-prefix [prefix] (Sub-)Collection prefix (default: collection) - +-p, --coll-prefix [prefix] (Sub-)Collection prefix. (default: collection) + +-s, --sheet [#] Single mode XLSX Sheet # to import. + +-T, --truncate Delete all documents from target collections before import. + -d, --dry-run Perform a dry run, without committing data. Implies --verbose. -v, --verbose Output document insert paths -h, --help output usage information @@ -40,8 +44,7 @@ Examples: ``` fire-migrate import --dry-run test.json myCollection fire-migrate import --merge test.INDEX.csv myCollection -fire-migrate i -m --id myDocIdField test.xlsx users posts -fire-migrate i -v firestore-dump.json +fire-migrate i -m --id docid test.xlsx myCollection ``` ## Export Data from Firestore @@ -68,8 +71,7 @@ Options: Examples: ``` -fire-migrate export --verbose --no-subcolls myRootCollection.json myCollection +fire-migrate export --verbose --no-subcolls myCollectionRootLevel.json myCollection fire-migrate export users-posts.json users posts -fire-migrate export path/to/indexed-csv/db.csv -fire-migrate e -v firestore-dump.json +fire-migrate e -v firestore-dump.xlsx ``` \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 9020a23..1baa048 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "firestoremigrate", - "version": "1.0.0", + "version": "0.2.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index bc8ca0f..a98d54e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "firestoremigrate", - "version": "1.0.0", + "version": "0.3.0", "description": "", "main": "index.js", "scripts": { diff --git a/samples/test.simple-with-empty-fields.csv b/samples/test.simple-with-empty-fields.csv new file mode 100644 index 0000000..bd74709 --- /dev/null +++ b/samples/test.simple-with-empty-fields.csv @@ -0,0 +1,4 @@ +Name,Price,SKU,Color,Cat +Lamp,23.34,2123m12mxi,Blue,Lights +Rug,99.98,n232ej23m,Orange, +Mug,5.20,lkj123lke,White,Mugs \ No newline at end of file diff --git a/samples/test.simple-with-empty-fields.xlsx b/samples/test.simple-with-empty-fields.xlsx new file mode 100644 index 0000000..3c8531d Binary files /dev/null and b/samples/test.simple-with-empty-fields.xlsx differ diff --git a/src/importCollection/index.ts b/src/importCollection/index.ts index 674fbfd..2a24504 100644 --- a/src/importCollection/index.ts +++ b/src/importCollection/index.ts @@ -9,14 +9,16 @@ import { encodeDoc, cleanCollectionPath, isCollectionPath, isDocumentPath } from const db = admin.firestore(); let batch = db.batch(); -let batchSetCount = 0; +let batchCount = 0; let totalSetCount = 0; +let totalDelCount = 0; let args; - +let delPaths = []; export const execute = async (file: string, collections: string[], options) => { args = options; if( args.dryRun ) args.verbose = true; + try { if( collections.length === 0 ) { @@ -30,7 +32,7 @@ export const execute = async (file: string, collections: string[], options) => { collections = ['/']; } } - + let data = {}; if (file.endsWith(".json")) { @@ -57,7 +59,8 @@ export const execute = async (file: string, collections: string[], options) => { ? 'Dry-Run complete, Firestore was not updated.' : 'Import success, Firestore updated!' ); - console.log(`Total documents: ${totalSetCount}`); + args.truncate && console.log(`Total documents deleted: ${totalDelCount}`); + console.log(`Total documents written: ${totalSetCount}`); } catch (error) { console.log("Import failed: ", error); @@ -68,24 +71,38 @@ export const execute = async (file: string, collections: string[], options) => { // Firestore Write/Batch Handlers +async function batchDel(ref: FirebaseFirestore.DocumentReference) { + // Log if requested + args.verbose && console.log(`Deleting: ${ref.path}`); + + // Mark for batch delete + ++totalDelCount; + await batch.delete(ref); + + // Commit batch on chunk size + if (++batchCount % args.chunk === 0) { + await batchCommit() + } + +} async function batchSet(ref: FirebaseFirestore.DocumentReference, item, options) { // Log if requested - args.verbose && console.log(ref.path); + args.verbose && console.log(`Writing: ${ref.path}`); // Set the Document Data ++totalSetCount; await batch.set(ref, item, options); // Commit batch on chunk size - if (++batchSetCount % args.chunk === 0) { + if (++batchCount % args.chunk === 0) { await batchCommit() } } async function batchCommit(recycle:boolean = true) { // Nothing to commit - if (!batchSetCount) return; + if (!batchCount) return; // Don't commit on Dry Run if (args.dryRun) return; @@ -98,7 +115,7 @@ async function batchCommit(recycle:boolean = true) { // Get a new batch if (recycle) { batch = db.batch(); - batchSetCount = 0; + batchCount = 0; } } @@ -113,20 +130,27 @@ function writeCollections(data): Promise { } function writeCollection(data:JSON, path: string): Promise { - return new Promise(async (resolve, reject) => { + return new Promise(async (resolve, reject) => { const colRef = db.collection(path); + + if (args.truncate) { + await truncateCollection(colRef); + } + const mode = (data instanceof Array) ? 'array' : 'object'; for ( let [id, item] of Object.entries(data)) { // doc-id preference: object key, invoked --id field, auto-id + if (mode === 'array') { + id = args.autoId; + } if (_.hasIn(item, args.id)) { id = item[args.id].toString(); delete(item[args.id]); } - if (!id || mode === 'array' || (id.toLowerCase() === args.autoId.toLowerCase()) ) { + if (!id || (id.toLowerCase() === args.autoId.toLowerCase()) ) { id = colRef.doc().id; - } - + } // Look for and process sub-collections const subColKeys = Object.keys(item).filter(k => k.startsWith(args.collPrefix+':')); @@ -149,9 +173,30 @@ function writeCollection(data:JSON, path: string): Promise { }); } +async function truncateCollection(colRef: FirebaseFirestore.CollectionReference) { + // TODO: Consider firebase-tools:delete + + const path = colRef.path; + if (delPaths.includes(path)) { + // Collection Path already processed + return; + } + delPaths.push(path); + + await colRef.get().then(async (snap) => { + for (let doc of snap.docs) { + // recurse sub-collections + const subCollPaths = await doc.ref.getCollections(); + for (let subColRef of subCollPaths) { + await truncateCollection(subColRef); + } + // mark doc for deletion + await batchDel(doc.ref); + } + }); +} // File Handling Helpers - function dataFromJSON(json) { _.forEach(json, row => { dot.object(row); @@ -261,6 +306,8 @@ function readCSV(file: string, collections: string[]): Promise { // Single Mode CSV, single collection if (!file.endsWith('INDEX.csv')) { + args.verbose && console.log(`Mode: Single CSV Collection`); + if (collections.length > 1) { reject('Multiple collection import from CSV requires an *.INDEX.csv file.'); return; @@ -279,6 +326,7 @@ function readCSV(file: string, collections: string[]): Promise { // Indexed Mode CSV, selected collections and sub-cols if (collections[0] !== '/') { + args.verbose && console.log(`Mode: Selected collections from Indexed CSV`); collections.forEach(collection => { const colls = index.filter(coll => (coll['Collection'] + '/').startsWith(collection + '/')); if (colls.length) { @@ -301,6 +349,7 @@ function readCSV(file: string, collections: string[]): Promise { // Indexed Mode CSV, all collections if (collections[0] === '/') { + args.verbose && console.log(`Mode: All collections from Indexed CSV`); const collection = collections[0]; _.forEach(index, coll => { const colPath = coll['Collection']; @@ -329,15 +378,25 @@ function readXLSXBook(path, collections: string[]): Promise { const indexSheet = book.Sheets['INDEX']; let data = {}; - // Single Sheet as Collection from Non-Indexed Workbook - if (!indexSheet) { + let sheetNum = args.sheet; + if ((sheetCount === 1) && (sheetNum == undefined)) { + sheetNum = 1; + } + + // Single Sheet as Collection, typically from Non-Indexed Workbook + if (sheetNum !== undefined) { + args.verbose && console.log(`Mode: Single XLSX Sheet #${sheetNum}`); const collection = collections[0]; if(isDocumentPath(collection)) { reject(`Invalid collection path for single collection: ${collection}`); return; } - const sheetName = book.SheetNames[+args.sheet - 1]; + const sheetName = book.SheetNames[+sheetNum - 1]; const sheet = book.Sheets[sheetName]; + if (!sheet) { + reject(`Sheet #${sheetNum} not found in workbook`); + return; + } data[collection] = dataFromSheet(sheet); resolve(data); return; @@ -347,6 +406,7 @@ function readXLSXBook(path, collections: string[]): Promise { // Selected Collections and Sub Colls from Indexed Workbook if (collections[0] !== '/') { + args.verbose && console.log('Mode: Selected Sheets from indexed XLSX Workbook'); collections.forEach(collection => { const colls = index.filter(coll => (coll['Collection'] + '/').startsWith(collection + '/')); if (colls.length) { @@ -367,6 +427,7 @@ function readXLSXBook(path, collections: string[]): Promise { // All Collections from Indexed Workbook if (collections[0] === '/') { + args.verbose && console.log('Mode: All Sheets from indexed XLSX Workbook'); const collection = collections[0]; _.forEach(index, coll => { const sheetName = coll['Sheet Name']; diff --git a/src/index.ts b/src/index.ts index 10b0b01..cbc425f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,8 +12,6 @@ admin.initializeApp({ import * as importCollection from './importCollection'; import * as exportCollection from './exportCollection'; -import { parse } from 'url'; - // Help Descriptions const rootDescription = [ @@ -61,7 +59,7 @@ function parseChunk(v:number) { // Base options -args.version('0.1.0') +args.version('0.3.0') .description(rootDescription) .on('--help', () => { console.log(rootHelp); @@ -73,11 +71,15 @@ args.command('import') .alias('i') .description(importDescription) .arguments(' [collections...]') - .option('-i, --id [field]', 'Field to use for Document IDs', 'doc_id') - .option('-a, --auto-id [str]', 'Document ID token specifying auto generated Document ID', 'Auto-ID') + .option('-i, --id [field]', 'Field to use for Document IDs.', 'doc_id') + .option('-a, --auto-id [str]', 'Document ID token specifying auto generated Document ID.', 'Auto-ID') .option('-m, --merge', 'Merge Firestore documents. Default is Replace.') .option('-k, --chunk [size]', 'Split upload into batches. Max 500 by Firestore constraints.', parseChunk, 500 ) - .option('-p, --coll-prefix [prefix]', '(Sub-)Collection prefix', 'collection') + .option('-p, --coll-prefix [prefix]', '(Sub-)Collection prefix.', 'collection') + .option('') + .option('-s, --sheet [#]', 'Single mode XLSX Sheet # to import.') + .option('') + .option('-T, --truncate', 'Delete all documents from target collections before import.') .option('') .option('-d, --dry-run', 'Perform a dry run, without committing data. Implies --verbose.') .option('-v, --verbose', 'Output document insert paths') diff --git a/src/shared.ts b/src/shared.ts index 257c696..94426d7 100644 --- a/src/shared.ts +++ b/src/shared.ts @@ -103,6 +103,19 @@ class NumberFH extends FieldHandler { isDecodeType = (key, val, doc) => { return (typeof val === 'number'); }; + + public isEncodeType = (key: string, val, doc): boolean => { + // simple numbers, or number-like strings + if (+val === +val) return true; + if (typeof val !== 'string') return false; + return val.startsWith(`{"type":"${this.prefix}"`); + }; + public encode = (key: string, val, doc) => { + if (+val === +val) { + return +val; + } + return this.encodeFn(key, JSON.parse(val), doc); + } } class ReferenceFH extends FieldHandler {