classeur-downloader.js

'use strict'

const _ = require('lodash'),
    ApiClient = require('classeur-api-client'),
    async = require('async'),
    fs = require('fs-extra'),
    pathMod = require('path'),
    pathJoin = _.spread(pathMod.join),
    TreeManipulator = require('tree-manipulator')

// const eyes = require('eyes'), p = eyes.inspect.bind(eyes)

/**
* Module for downloading and listing files and folders stored in [Classeur](http://classeur.io/).
*
* @example <caption>Installation</caption>
* npm install classeur-downloader
* @example <caption>Saving a single file's markdown content</caption>
* const downloader = require('classeur-downloader')
* downloader.saveSingleFile({
*     file: 'some file ID',
*     userId: 'user ID',
*     apiKey: 'api key',
*     path: '/some/path.md',
*     markdown: true
* }, (error) => {
*     if (error) throw error
* })
* @example <caption>Saving directories</caption>
* // Saves all files contained in 'folder1' and 'folder2' in subdirectories of mydir/ with those same names:
* downloader.saveTree({
*     folders: ['folder1', 'folder2' ]
*     userId: 'user ID',
*     apiKey: 'api key',
*     path: 'mydir/',
*     markdown: true
* }, (error) => {
*     if (error) throw error
* })
* @see The [README](index.html) for an overview and more usage examples.
* @see The [source code]{@link https://github.com/zbentley/classeur-downloader} on GitHub.
* @see The [classeur-api-client](http://zbentley.github.io/classeur-api-client/versions/latest) module (which `classeur-downloader` is built around) for a lower-level way to interact with the Classeur API.
* @module classeur-downloader
*/


function getTree(byId, print, items) {
    let props = {
        identifierProperty: byId ? 'id' : 'name',
        nestedNodesProperty: 'files',
    }
    if ( print ) {
        props.valueGetter = function(obj, property) {
            // If we're getting the value of a node, and not its children,
            // stringify it for pretty printing.
            if (property === this.identifierProperty) {
                return APIobjectToString(obj, byId)
            } else {
                return obj[property]
            }
        }
    }

    let tm  = new TreeManipulator(props)
    // If contents are supplied, bind the instance methods used by this script
    // to the contents to prevent having to pass around tree manipulators *and*
    // contents everywhere.
    if ( items !== undefined ) {
        _.mixin(tm, {
            print: _.partial(tm.print, items),
            findNode: _.partialRight(tm.findNode, items)
        })
    }

    return tm
}

function errorIfExists(path, cb) {
    fs.stat(path, (error, result) => {
        if (error && error.errno === -2) { //ENOENT
            cb(null, result)
        } else {
            cb(error || new Error(`File ${path} exists, and --overwrite is not set.`), null)
        }
    })
}

function getWriter(path, options, addExtension, content) {
    const writefunc = _.isString(content) ? fs.outputFile : fs.outputJson
    path = pathJoin(path)
    if ( addExtension ) {
        path += _.isString(content) ? '.md' : '.json'
    }

    return options.overwrite
        ? _.partial(writefunc, path, content)
        : _.partial(async.series, [
            _.partial(errorIfExists, path),
            _.partial(writefunc, path, content),
        ])
}

// For each item in the tree, either download it, or make the folder and recurse.
function makeFolderOrSaveFile(conn, tree, options, id, cb) {
    const found = tree.findNode(id),
        kids = tree.nestedNodesProperty,
        node = found.node,
        parallel = [],
        markdown = options.markdown

    let path = found.path

    if ( _.has(node, kids) ) {
        // Handle creation of folder metadata file only applies in JSON mode,
        // and only applies to non-root nodes.
        if ( options.folderMetadata && path.length > 1 ) {
            path[path.length - 1] += '.folder_metadata.json'
            parallel.push(getWriter(path, options, false, node))
        }

        parallel.push(_.partial(async.each, node[kids], (child, cb) => {
            makeFolderOrSaveFile(conn, tree, options, child[tree.identifierProperty], cb)
        }))
    } else {
        parallel.push(_.partial(async.waterfall,[
            _.bind(conn.getFile, conn, node.id),
            function(result, cb) {
                getWriter(path, options, options.addExtension, markdown ? result.content.text : result)(cb)
            }
        ]))
    }

    async.parallel(parallel, cb)
}


// Print a tree rooted at each folder and file. Printing the top-level tree
// results in leading \____ parent relationships for no reason.
function showTree(items, conn, options, cb) {
    const tm = getTree(options.byId, true)
    // TODO group by files vs. folders.
    _.forEach(_.sortBy(items, tm.identifierProperty), _.bind(tm.print, tm))
    cb(null, null)
}

function saveTree(items, conn, options, cb) {
    const path = options.path
    makeFolderOrSaveFile(conn, getTree(options.byId, false, {
        id: path,
        name: path,
        files: items // top-level folders and manually-specified files
    }), options, path, cb)
}

function APIobjectToString(object, byId) {
    if ( object.id || object.name ) {
        let info = [object.id, object.name]
        if ( byId ) info.reverse()
        return `${info.pop()} (${info.pop()})`
    } else {
        return ''
    }
}

function getFilesAndFolders(options, func, cb) {
    const conn = new ApiClient(options.userId, options.apiKey, options.host)
    async.parallel(
        [
            (cb) => { conn.getFolders(options.folders, cb) },
            (cb) => { conn.getFiles(options.files, cb) },
        ],
        (error, result) => {
            if (error) {
                if ( ! _.isError(error) ) {
                    error = new Error(error)
                }
                cb(error, null)
            } else {
                // _.flatten de-'segments' the array into a single list of files and folders.
                func(_.flatten(result), conn, options, cb)
            }
        }
    )
}

function assertOptions(options, maxFiles, maxFolders) {
    options.folders = options.folders || []
    options.files = options.files || []
    if ( ! _.isUndefined(maxFiles) && options.files.length > maxFiles ) {
        throw new Error(`Got ${options.files.length} files, but expected no more than ${maxFiles}:\n${options.files}`)
    }

    if ( ! _.isUndefined(maxFolders) && options.folders.length > maxFolders ) {
        throw new Error(`Got ${options.folders.length} folders, but expected no more than ${maxFolders}:\n${options.folders}`)
    }

    if ( _.isUndefined(options.addExtension) ) {
        options.addExtension = true
    }

    return options
}

function scrubCallback(cb) {
    return (error, result) => {
        result = _.compact(_.flattenDeep(result))
        cb(error || null, _.isEmpty(result) ? null : result)
    }
}

/**
* Options for configuring `classeur-downloader`.
* @typedef {Object} Options:Global
* @property {String} userId - User ID with which to connect to the Classeur API.
* @property {String} apiKey - API key to use when connecting to the Classeur API. This can be obtained by re-generating your key in the Classeur 'User' preferences pane.
* @property {String[]} [files] - Array of file IDs to operate on.
* - At least one value must be supplied in `options.files` or `options.folders`, otherwise an error will be raised.
* @property {String[]} [folders] - Array of folder IDs to operate on.
* - At least one value must be supplied in `options.files` or `options.folders`, otherwise an error will be raised.
* @property {boolean} [byId=false] - If true, files and folders will be handled (saved or printed) by ID. If false, they will be handled by Classeur human-readable name.
*/

/**
* Options for configuring the bulk download behavior of `classeur-downloader`.
* All options used by {@link module:classeur-downloader~Options:Global} are also accepted.
* @typedef {Object} Options:DownloadFilesAndFolders
* @property {String} path - Destination path to save files and folders from Classeur. `path` must be an existent, writable folder.
* - All files in `options.files` will be saved inside of `path`. All folders in the `options.folders` will be created (with names according to the `byId` property) in `path`, and the files they contain will be created within those folders.
* - If `options.overwrite` is not set and name collisions occur with files being saved into `path`, an error will be raised and save operations will halt. Partial results may exist on the filesystem.
* @property {boolean} [overwrite=false] - If true, items in `path` that already exist will be overwritten.
* @property {boolean} [folderMetadata=false] - If true, generate JSON folder metadata for all folders in `folders`.
* - If `true`, a single JSON file will be created next to every Classeur folder downloaded. That JSON file will be named after the folder, and will end in `.folder_metadata.json`. It will contain the full Classeur API metadata information for the folder. This is usually not useful, unless you are using `classeur-downlaoder` to back up a locally-hosted Classeur instance with the intent of using the generated files for some future restoration process.
* @property {boolean} [markdown=false] - Whether or not to write markdown content for files.
* - If `true`, saved files' content will be the markdown content of Classeur documents.
* - If `false`, files' content will be their full JSON data from Classeur. Full JSON data objects include markdown content and other fields, and will likely not be able to be opened directly in a Markdown editor.
* @property {boolean} [addExtension=true] - Whether or not appropriate extensions should be added to files written.
* - If `true`, files saved with `options.markdown` set to `true` will have the `.md` extension, and files saved outside of `markdown` mode will have the `.json` extension.
* - If false, extensions will not be added to files.
*/

/**
* Options for configuring the single-file download behavior of `classeur-downloader`.
* All options used by {@link module:classeur-downloader~Options:Global} are also accepted.
* @typedef {Object} Options:DownloadSingleFile
* @property {String} path - Destination path to save files and folders from Classeur. Must be either a nonexistent path in a writable directory, or an existent, writable file (if `options.overwrite` is set).
* @property {String} [file=options.files[0]] - Single file ID to download and save. If not provided, `options.files[0]` will be used.
* - This option is mutually exclusive with `options.files`.
* @property {boolean} [overwrite=false] - If true, `path` will be overwritten with the new Classeur file content retrieved.
* @property {boolean} [markdown=false] - Whether or not to write markdown content for `options.file`.
* - If `true`, `options.file`'s content will be that file's markdown content, visible in the Classeur UI.
* - If `false`, `options.file`'s content will be its full JSON data from Classeur. Full JSON data objects include markdown content and other fields, and will likely not be able to be opened directly in a Markdown editor.
*/

/**
* @callback CompletionCallback
* @param {Error?} error - An throwable Error (or subclass thereof) if an error occurrend.
* - For errors in writing files, `error` may be any of the errors raised by the [fs](https://nodejs.org/api/fs.html) module.
* - For errors retrieving data from the Classeur API, `error` may be one of the Error subclasses used by [classeur-api-client](http://zbentley.github.io/classeur-api-client/versions/latest). Errors will be supplied to `CompletionCallback`s in the same way they will be supplied to [ClasseurClient~ScrubbedCallback](http://zbentley.github.io/classeur-api-client/versions/latest/module-classeur-api-client.html#.ScrubbedCallback)s.
* - `error` will always be `null` (not `undefined` or another falsy value) if no error occurred.
* @param {*?} result - Behavior of `result` is not defined it should not be used. Will usually be `null`. May sometimes contain an array of partial result objects.
*/

/**
* @summary Prints out (to the console) a tree structure of the Classeur hierarchy of supplied files and folders.
* @param {module:classeur-downloader~Options:Global} options - Options for which Classeur files and folders to retrieve, and how to display them.
* - Folders in the `options.folders` will be printed as root directories, and all of the files they contain will be printed.
* - If the `options.byId` is `true`, files and folders will be printed out as 'id (name)'. Otherwise, they will be printed as 'name (id)', where 'name' is the human-readable name of an object in the Classeur UI.
* @param {module:classeur-downloader~CompletionCallback} callback - Called with an error, if one occurred, or `null` if all operations were successful.
*/
module.exports.showTree = (options, cb) => {
    getFilesAndFolders(assertOptions(options), showTree, scrubCallback(cb))
}

/**
* Folders in the `options.folders` array will be saved as root directories (with names determined by the presence or absence of `options.byId`), and all of the files they contain will be saved within them. Files in `options.files` will be saved at the top level.
* If the `options.byId` is `true`, files and folders' root names will be their Classeur object IDs. Extensions will be added regardless of `options.byId`, depending on the value of `options.addExtension`.
* @summary Saves Classeur files and folders to a specified path on the local filesystem.
* @param {module:classeur-downloader~Options:DownloadFilesAndFolders} options - Options for which Classeur files and folders to retrieve, and how to save them.
* @param {module:classeur-downloader~CompletionCallback} callback - Called with an error, if one occurred, or `null` if all operations were successful.
*
* @example <caption>Saving files by ID</caption>
* // Assume the folder with ID 'abcd' contains files with the IDs 'foo', 'bar', and 'baz'.
* const downloader = require('classeur-downloader')
* downloader.saveTree({
*     files: [ 'quux' ],
*     folders: [ 'abcde' ],
*     userId: 'user ID',
*     apiKey: 'api key',
*     path: 'mydir/',
*     folderMetadata: true
* }, (error) => { if (error) throw error })
* // 'mydir' will now contain:
* //    quux.json
* //    abcde.folder_metadata.json
* //    abcde
* //        \_ foo.json
* //        \_ bar.json
* //        \_ baz.json
*
* @example <caption>Saving markdown content</caption>
* // Assume the folder with ID 'abcd' has the UI-visible name 'My Folder'.
* // Assume it contains two files with IDs 'foo' and 'bar', and the names 'My Stuff' and 'My Other Stuff'.
* downloader.saveTree({
*     folders: [ 'abcde' ],
*     userId: 'user ID',
*     apiKey: 'api key',
*     path: 'mydir/'
*     markdown: true
* }, (error) => { if (error) throw error })
* // 'mydir' will now contain:
* //    My Folder
* //        \_ My Stuff.md
* //        \_ My Other Stuff.md
*/
module.exports.saveTree = (options, cb) => {
    getFilesAndFolders(assertOptions(options), saveTree, scrubCallback(cb))
}

/**
* This function can be used when you don't need/want to create container folders for your retrieved Classeur content.
* @summary Saves a single Classeur file to a specified path on the local filesystem.
* @param {module:classeur-downloader~Options:DownloadSingleFile} options
* - `options.folders` may not be supplied to this function.
* - `options.byId` is ignored by this function.
* - File content will be saved directly to `options.path` no folders or other metadata files will be created.
* - File extensions (e.g. `.md`) will _not_ be added by SaveSingleFile write the extension you want into `options.path` directly.
* @param {module:classeur-downloader~CompletionCallback} callback - Called with an error, if one occurred, or `null` if all operations were successful.
*
* @example <caption>Saving a single file's markdown content</caption>
* const downloader = require('classeur-downloader')
* downloader.saveSingleFile({
*     files: 'some file id',
*     userId: 'user ID',
*     apiKey: 'api key',
*     path: 'myfile.markdown',
*     markdown: true
* }, (error) => { if (error) throw error })
*/
module.exports.saveSingleFile = (options, cb) => {
    options = assertOptions(options, 1, 0)
    const conn = new ApiClient(options.userId, options.apiKey, options.host),
        ext = pathMod.parse(options.path).ext
    async.waterfall([
        _.bind(conn.getFile, conn, options.file || options.files[0]),
        (result, cb) => {
            getWriter([options.path], options, false, options.markdown ? result.content.text : result)(cb)
        }
    ], scrubCallback(cb))
}