From 156214cad44e2ce99ee2f66dd275135b6f20a5db Mon Sep 17 00:00:00 2001 From: Robert Rossmann Date: Sat, 26 Sep 2015 20:42:12 +0200 Subject: [PATCH] Lo and behold! --- .editorconfig | 24 +++++ .eslintrc | 245 ++++++++++++++++++++++++++++++++++++++++++++++ .gitignore | 15 +++ .gitmodules | 3 + LICENSE | 27 +++++ Makefile | 16 +++ README.md | 175 +++++++++++++++++++++++++++++++++ jsdoc.json | 27 +++++ lib/action.js | 105 ++++++++++++++++++++ lib/adapter.js | 239 ++++++++++++++++++++++++++++++++++++++++++++ lib/index.js | 38 +++++++ lib/uploader.js | 88 +++++++++++++++++ lib/util/local.js | 21 ++++ package.json | 44 +++++++++ targets | 1 + 15 files changed, 1068 insertions(+) create mode 100644 .editorconfig create mode 100644 .eslintrc create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 jsdoc.json create mode 100644 lib/action.js create mode 100644 lib/adapter.js create mode 100644 lib/index.js create mode 100644 lib/uploader.js create mode 100644 lib/util/local.js create mode 100644 package.json create mode 160000 targets diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..ab4d6f6 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,24 @@ +# editorconfig.org + +# This is the project's root directory +root = true + +[*] +# Use spaces for indentation +indent_style = space +# Each indent should contain 2 spaces +indent_size = 2 +# Use Unix line endings +end_of_line = lf +# The files are utf-8 encoded +charset = utf-8 +# No whitespace at the end of line +trim_trailing_whitespace = true +# A file must end with an empty line - this is good for version control systems +insert_final_newline = true +# A line should not have more than this amount of chars (not supported by all plugins) +max_line_length = 100 + +[{Makefile,**.mk}] +# Use tabs for indentation (Makefiles require tabs) +indent_style = tab diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..0664194 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,245 @@ +--- +env: + node: true + es6: true + +globals: + sails: true + +rules: + # Recommended rules + comma-dangle: 2 + no-cond-assign: + - 2 + - except-parens + no-console: 1 + no-constant-condition: 2 + no-control-regex: 2 + no-debugger: 2 + # Strict mode takes care of what this rule tries to avoid, but I am including it here anyway... + no-dupe-args: 2 + no-dupe-keys: 2 + no-duplicate-case: 2 + no-empty-character-class: 1 + no-empty: 2 + no-ex-assign: 2 + no-extra-boolean-cast: 2 + no-extra-parens: 1 + no-extra-semi: 2 + no-func-assign: 2 + no-inner-declarations: 2 + no-invalid-regexp: 1 + no-irregular-whitespace: 2 + no-negated-in-lhs: 2 + no-obj-calls: 2 + no-regex-spaces: 1 + no-sparse-arrays: 2 + no-unreachable: 2 + no-useless-concat: 2 + use-isnan: 2 + valid-jsdoc: + - 1 + - requireReturn: true + requireReturnDescription: false + require-jsdoc: 1 + valid-typeof: 2 + no-unexpected-multiline: 2 + constructor-super: 2 + no-const-assign: 2 + + # Best practices rules + + # If there is a setter, there must be a getter (but not required vice-versa) + accessor-pairs: 2 + callback-return: + - 1 + - + - cb + - callback + - next + - done + # If there are more than this number of linear code paths, issue a warning + complexity: + - 1 + - 8 + consistent-return: 2 + default-case: 2 + dot-notation: 2 + # Dots for object property accessors should be on the same line as the property names + dot-location: + - 2 + - property + # Require strict comparison operators everywhere + eqeqeq: 2 + global-require: 2 + handle-callback-err: + - 2 + - "^.*(e|E)rr(or)?" + no-caller: 2 + no-delete-var: 2 + no-div-regex: 1 + no-else-return: 2 + no-empty-label: 2 + # Eval in Node.js makes no sense... + no-eval: 2 + no-implied-eval: 2 + no-extend-native: 2 + no-extra-bind: 2 + no-fallthrough: 2 + no-floating-decimal: 2 + no-invalid-this: 1 + no-iterator: 2 + no-labels: 2 + no-lone-blocks: 2 + no-loop-func: 2 + no-multi-str: 2 + no-native-reassign: 2 + no-new-func: 2 + no-new-require: 2 + no-new-wrappers: 2 + # Do not allow new to be used only for side-effects + # I.e. if you do not store the result of new: new User(); + no-new: 2 + no-octal-escape: 2 + no-octal: 2 + no-path-concat: 1 + no-process-exit: 1 + # Use Object.getPrototypeOf() + no-proto: 2 + no-redeclare: + - 2 + - builtinGlobals: true + no-return-assign: 2 + no-self-compare: 2 + no-sequences: 2 + no-shadow: + - 2 + - builtinGlobals: true + hoist: all + no-shadow-restricted-names: 2 + no-sync: 1 + # Always throw an instance of Error + no-throw-literal: 2 + no-undef: 2 + no-undef-init: 2 + no-unused-expressions: 2 + no-unused-vars: 2 + no-use-before-define: + - 2 + - nofunc + no-useless-call: 2 + no-with: 2 + # I know this sucks, be you should really use the radix operator for Number.parseInt() + radix: 1 + # Require strict mode in the module level + strict: + - 2 + - global + yoda: 2 + no-class-assign: 2 + no-this-before-super: 2 + no-var: 2 + prefer-const: 2 + require-yield: 2 + + # Personal coding style preferences + arrow-parens: + - 2 + - as-needed + arrow-spacing: 2 + curly: + - 1 + - multi + no-multi-spaces: 1 + no-warning-comments: 1 + # Disabled - for me, handling errors in callbacks takes precedence before var statements + # vars-on-top: 1 + no-undefined: 1 + array-bracket-spacing: + - 1 + - always + brace-style: 1 + camelcase: 1 + comma-spacing: 1 + comma-style: + - 2 + - first + consistent-this: + - 2 + - that + eol-last: 2 + func-style: + - 2 + - declaration + id-length: + - 1 + - min: 2 + max: 25 + exceptions: + - i + - _ + key-spacing: 1 + linebreak-style: + - 2 + - unix + max-nested-callbacks: + - 1 + - 4 + new-cap: 1 + new-parens: 2 + newline-after-var: 1 + no-array-constructor: 2 + no-dupe-class-members: 2 + no-lonely-if: 1 + no-mixed-spaces-and-tabs: 2 + no-multiple-empty-lines: + - 1 + - max: 2 + no-nested-ternary: 2 + no-new-object: 2 + no-spaced-func: 2 + no-trailing-spaces: 2 + no-underscore-dangle: 1 + no-unneeded-ternary: 2 + object-curly-spacing: + - 2 + - always + object-shorthand: + - 1 + - methods + one-var: 1 + operator-assignment: + - 1 + - always + prefer-template: 1 + prefer-arrow-callback: 2 + quote-props: + - 2 + - as-needed + quotes: + - 2 + - single + - avoid-escape + semi: + - 2 + - never + space-after-keywords: 2 + space-before-keywords: 2 + space-before-blocks: 2 + space-before-function-paren: 2 + space-in-parens: 2 + space-infix-ops: 2 + space-return-throw-case: 2 + spaced-comment: 1 + wrap-regex: 1 + # Put the asterisk together with the function keyword: function* generator () {} + generator-star-spacing: + - 2 + - after + max-len: + - 2 + - 100 + - 2 + max-params: + - 1 + - 4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4e82029 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +node_modules +docs + +*.log +npm-debug.log + +*~ +*# +.DS_STORE +.netbeans +nbproject +.idea +.node_history +*.sublime-* + diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..4af9ac9 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "targets"] + path = targets + url = https://github.com/Dreamscapes/makefiles.git diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2eb4627 --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2015, Robert Rossmann +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name skipper-better-s3 nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bfe481e --- /dev/null +++ b/Makefile @@ -0,0 +1,16 @@ +# skipper-better-s3 +# +# @author Robert Rossmann +# @copyright 2015 STRV +# @license http://choosealicense.com/licenses/bsd-3-clause BSD-3-Clause License + +# Default - Run it all! (except for coveralls - that should be run only from Travis) +all: install lint docs + +include targets/nodejs/base.mk +include targets/nodejs/install.mk +include targets/nodejs/lint.mk +include targets/nodejs/docs.mk + +# Project-specific information +lintfiles = lib diff --git a/README.md b/README.md new file mode 100644 index 0000000..aed755e --- /dev/null +++ b/README.md @@ -0,0 +1,175 @@ +# [skipper emblem - face of a ship's captain][project-root] Skipper-Better-S3 + +[![NPM Version][npm-badge]][npm-url] +![Runs on Node][node-badge] +![Built with GNU Make][make-badge] +![Uses ECMAScript 2015][es-badge] + +> A better, modern implementation of Skipper's S3 file adapter + +**This adapter uses ECMAScript 2015 (ES 6) syntax**: you must run at least Node.js v4 in order to use this adapter. + +## Why better? + +Have you ever tried to upload a file and make it publicly readable using the official skipper-s3 adapter? Well you cannot do that, there's just no support for such things. What was even worse, the official adapter kept calculating incorrect md5 hashes of the uploaded files which lead to all kinds of errors when I started verifying the uploaded files' integrity. + +Also, the official adapter's codebase seems to be really complicated, at least to me :smile:, which in long term might discourage potential contributions. + +## Installation + +`$ npm install skipper-better-s3 --save` + +Also make sure you have Skipper [installed as your body parser](http://sailsjs.org/documentation/concepts/middleware#adding-or-overriding-http-middleware). + +> Skipper is installed by default in [Sails](https://github.com/balderdashy/sails) v0.10 and above. + +## Usage + +### File uploads + +You upload the files as usual with any other adapter. See some examples below. + +#### Supported options + +| Option | Type | Details | +|------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| key | String | Your Amazon access key | +| secret | String | Your Amazon secret | +| bucket | String | The S3 bucket to upload the file to | +| s3options | Object | Optional options to be passed to the underlying Amazon SDK library when performing the file upload. This could be any parameter that is supported by the S3's [`putObject()`][s3-putobject] method. | +| onProgress | Function | Marked by Skipper core as experimental. If provided, will be called periodically as the data is being uploaded with current progress information. | + +#### Example usage + +```js +const options = + { // This is the usual stuff + adapter: require('skipper-better-s3') + , key: 'somekeyhere' + , secret: 'dontsharethis' + , bucket: 'my-s3-bucket' + // Let's use the custom s3options to upload this file as publicly + // readable by anyone + , s3options: + { ACL: 'public-read' + } + // And while we are at it, let's monitor the progress of this upload + , onProgress: progress => sails.log.verbose('Upload progress:', progress) + } + +req.file('avatar').upload(options, (err, files) => { + // ... Continue as usual +}) +``` + +### File downloads + +To download a previously uploaded file, you must first get an actual adapter object to use it and +provide some basic configuration, like your S3 credentials and the bucket to read from. + +You can either get a readable stream that you can consume or pipe to the client directly (efficient, +does not require buffering the whole file to memory), or you can provide a callback which will then +receive the full contents of the file being downloaded. + +#### Example usage + +```js +const options = + { key: 'somekeyhere' + , secret: 'dontsharethis' + , bucket: 'my-s3-bucket' + } + // This will give you an adapter instance configured with the + // credentials and bucket defined above + , adapter = require('skipper-better-s3')(options) + +// Now the adapter is ready to interface with your S3 bucket. +// Let's assume you have a file named 'avatar.jpg' stored +// in the root of this bucket... + +// Option 1: get a readable stream of the file +const readableStream = adapter.read('avatar.jpg') +// Option 2: get the full file contents in a callback +adapter.read('avatar.jpg', (err, data) => { + // data is now a Buffer containing the avatar +}) +``` + +### Directory listing + +You can get a list of files at a given path in a bucket. + +#### Example usage + +```js +const options = + { key: 'somekeyhere' + , secret: 'dontsharethis' + , bucket: 'my-s3-bucket' + } + // This will give you an adapter instance configured with the + // credentials and bucket defined above + , adapter = require('skipper-better-s3')(options) + +// Assuming there is a directory named 'avatars' in your bucket root... +adapter.ls('avatars', (err, files) => { + // files is now an array of paths, relative to the given directory name +}) +``` + +### Deleting objects + +Simply call `adapter.rm(fd, done)` on a configured adapter instance (see previous examples on how to get such instance). `fd` should be the path to the object to be deleted, relative to the bucket. + +#### Example usage + +```js +// Assuming you already have an adapter instance... +adapter.rm('avatars/123.jpg', (err, res) => { + // res is whatever S3 SDK returns (honestly no idea what's inside, have a look) +}) +``` + +## Extras + +This adapter comes with some extra functionality not defined in the Skipper adapter specifications. + +### Generating signed URLs for S3 + +Signed URLs are a great way of allowing others to interact with your S3 storage directly. For example, you can generate a file download link that will be valid only for 5 minutes, or a link which will allow someone to upload a file into a predetermined location in your S3 bucket. + +This is great, because it allows your clients to interact with your S3 storage directly, instead of bothering your server with all the network traffic and computational power necessary to upload/download files. + +#### Example usage + +```js +// Assuming you already have an adapter instance... +const url = adapter.url('getObject', { s3options: { Key: 'avatars/123.jpg' } }) +// Give the url to the client - they can read this file directly from there +// Optionally do a redirect (303 - "see other") to this file yourself: +// res.redirect(303, url) +``` + +## Documentation + +Full API documentation with detailed description of all methods is available offline. Clone this repo and do the following within the cloned folder: + +```bash +$ npm install +$ make docs +$ open docs/index.html # If open does not work, just double-click this file +``` + +## License + +This software is licensed under the **BSD-3-Clause License**. See the [LICENSE](LICENSE) file for more information. + + +[npm-badge]: https://img.shields.io/npm/v/skipper-better-s3.svg?style=flat-square +[node-badge]: https://img.shields.io/node/v/skipper-better-s3.svg?style=flat-square +[make-badge]: https://img.shields.io/badge/built%20with-GNU%20Make-brightgreen.svg?style=flat-square +[es-badge]: https://img.shields.io/badge/ECMA-2015-f0db4f.svg?style=flat-square +[npm-url]: https://npmjs.org/package/skipper-better-s3 +[skipper-logo]: http://i.imgur.com/P6gptnI.png +[project-root]: https://github.com/Dreamscapes/skipper-better-s3 +[s3-putobject]: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property diff --git a/jsdoc.json b/jsdoc.json new file mode 100644 index 0000000..162ad40 --- /dev/null +++ b/jsdoc.json @@ -0,0 +1,27 @@ +{ + "source": { + "include": [ + "./lib/", + "./README.md" + ] + }, + "opts": { + "recurse": true + }, + "plugins": ["plugins/markdown"], + "templates": { + "cleverLinks": true, + "monospaceLinks": true, + "default": { + "outputSourceFiles": true + }, + "applicationName": "Dreamscapes\\Skipper-better-S3", + "googleAnalytics": "", + "linenums": true + }, + "markdown": { + "parser": "gfm", + "githubRepoName": "skipper-better-s3", + "githubRepoOwner": "Dreamscapes" + } +} diff --git a/lib/action.js b/lib/action.js new file mode 100644 index 0000000..370aefd --- /dev/null +++ b/lib/action.js @@ -0,0 +1,105 @@ +/** + * skipper-better-s3 + * + * @author Robert Rossmann + * @copyright 2015 Robert Rossmann + * @license http://choosealicense.com/licenses/bsd-3-clause BSD-3-Clause License + */ + +'use strict' + +const AWS = require('aws-sdk') + , merge = require('semantic-merge') + , Uploader = require('./uploader') + , local = require('./util/local') + + +module.exports = class Action { + + constructor (opts) { + + const scope = local(this) + + // Save configuration options and the S3 client to local scope + scope.opts = opts + scope.client = new AWS.S3(opts.service) + } + + upload () { + + return new Uploader(local(this).opts) + } + + download (fd, done) { + + const scope = local(this) + , params = merge({ Key: fd }).and(scope.opts.request).into({}) + + scope.client.getObject(params, (err, data) => err ? done(err, null) : done(null, data.Body)) + } + + stream (fd) { + + const scope = local(this) + , params = merge({ Key: fd }).and(scope.opts.request).into({}) + , request = scope.client.getObject(params) + , stream = request.createReadStream() + + // The request object may emit error events, but since we must return a stream, we should + // somehow forward these errors to the caller + request.on('error', err => stream.emit('error', err)) + + return stream + } + + remove (fd, done) { + + const scope = local(this) + , params = merge({ Key: fd }).and(scope.opts.request).into({}) + + scope.client.deleteObject(params, done) + } + + list (dirname, done) { + + // If no dirname has been provided, list root + dirname = dirname || '' + + const scope = local(this) + , params = merge({ Prefix: dirname }).and(scope.opts.request).into({}) + , objects = [] + + scope.client.listObjects(params).eachPage((err, data) => { + if (err) + return done(err) + + // Data will be null only when the last page has been already processed + if (data) { + for (const item of data.Contents) + objects.push(item.Key) + + // Do not return the results just yet... + return // eslint-disable-line consistent-return + } + + // Okay, we now have the full listings - let's strip the dirname from the paths + if (dirname) + for (const i in objects) + objects[i] = objects[i].replace(`${dirname}/`, '') + + // Aaaand, finally we are done! + return done(null, objects) + }) + } + + url (operation, done) { + + const scope = local(this) + , params = scope.opts.request + + // The S3 client seems to be very sensitive about its callback, even passing undefined will + // trigger an error about it not being a function + return done ? scope.client.getSignedUrl(operation, params, done) + : scope.client.getSignedUrl(operation, params) + } +} diff --git a/lib/adapter.js b/lib/adapter.js new file mode 100644 index 0000000..c73e71b --- /dev/null +++ b/lib/adapter.js @@ -0,0 +1,239 @@ +/** + * skipper-better-s3 + * + * @author Robert Rossmann + * @copyright 2015 Robert Rossmann + * @license http://choosealicense.com/licenses/bsd-3-clause BSD-3-Clause License + */ + +'use strict' + +const merge = require('semantic-merge') + , local = require('./util/local') + , Action = require('./action') + + +module.exports = class Adapter { + + /** + * Parse user-provided options into our own internal structure expected by the service + * + * @method Adapter.parseOptions + * @static + * + * @param {Object} opts Options object to be parsed + * @param {Object?} defaults Default configuration to be used for missing properties + * @return {Object} The parsed object + */ + static parseOptions (opts, defaults) { + + // Normalise the hell out of it... + opts = opts || {} + defaults = defaults || {} + defaults.service = defaults.service || {} + defaults.request = defaults.request || {} + + const config = + { service: + { accessKeyId: opts.key || defaults.service.accessKeyId + , secretAccessKey: opts.secret || defaults.service.secretAccessKey + , apiVersion: '2006-03-01' + } + + , request: + { Bucket: opts.bucket || defaults.request.Bucket + } + } + + config.request = merge(config.request) + .and(opts.s3options || {}) + .into({}) + + return config + } + + + /** + * Create a new Adapter instance + * + * @classdesc The Adapter implements the standard interface that Skipper expects from + * each adapter implementation. + * + * @constructor Adapter + * + * @param {Object?} globals Global options to be used for all subsequent requests + * @param {String?} globals.key S3 access key + * @param {String?} globals.secret S3 access secret + * @param {String?} globals.bucket S3 bucket to use for all requests + * @return {Adapter} + */ + constructor (globals) { + + local(this).globals = Adapter.parseOptions(globals) + } + + /** + * Callback for the `ls()` method + * + * @callback lsDone + * @memberof Adapter + * + * @param {Error?} err Optional error + * @param {String[]?} data An array of string paths + */ + + /** + * Get the contents at given path + * + * @method Adapter#ls + * + * @param {String} dirname The path where to perform the listing + * @param {Adapter.lsDone} done Callback function + * @return {void} + */ + ls (dirname, done) { + + return new Action(local(this).globals).list(dirname, done) + } + + /** + * Callback for the `rm()` method + * + * @callback rmDone + * @memberof Adapter + * + * @param {Error?} err Optional error + * @param {Object?} data Additional data about the request, as returned by S3. + * @see http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#deleteObject-property + */ + + /** + * Remove an item at given path + * + * @method Adapter#rm + * + * @param {String} fd The item to be removed + * @param {Adapter.rmDone} done Callback function + * @return {void} + */ + rm (fd, done) { + + return new Action(local(this).globals).remove(fd, done) + } + + /** + * Callback for the `read()` method + * + * @callback readDone + * @memberof Adapter + * + * @param {Error?} err Optional error + * @param {Buffer?} body The item's contents, as buffer + * @see http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getObject-property + */ + + /** + * Read an item at given path + * + * @method Adapter#read + * + * @param {String} fd The item to be read + * @param {Adapter.readDone?} done Optional callback. If provided, the item's contents will + * be passed to it. If omitted, a stream will be returned + * instead which you can consume. + * @return {ReadableStream|void} + */ + read (fd, done) { + + const action = new Action(local(this).globals) + + return done ? action.download(fd, done) + : action.stream(fd) + } + + /** + * Callback for the `url()` method + * + * @callback urlDone + * @memberof Adapter + * + * @param {Error?} err Optional error + * @param {String?} url The signed URL + * @see http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getSignedUrl-property + */ + + /** + * Generate a signed URL for the given S3 operation + * + * @method Adapter#url + * + * @param {String} operation A valid S3 operation identifier + * @param {Object?} opts Optional additional configuration + * @param {String?} opts.key S3 access key + * @param {String?} opts.secret S3 access secret + * @param {String?} opts.bucket S3 bucket to use for this operation + * @param {Object?} opts.s3options Optional object to provide additional options + * for the signed request. The options can be + * anything that is supported by the given S3's + * operation. + * @param {Adapter.urlDone?} done Optional callback. If omitted, you must already + * have valid credentials saved in this service. + * @return {String|void} If called synchronously, will return the signed + * URL for the specified operation. + * @see http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getSignedUrl-property + */ + url (operation, opts, done) { + + opts = Adapter.parseOptions(opts, local(this).globals) + + return new Action(opts).url(operation, done) + } + + /** + * Upload progress handler + * + * This function, if provided, will be called periodically with status information about the + * ongoing upload request. + * + * @callback onProgress + * @memberof Adapter + * + * @param {Object} data Object with current progress information + * @param {String} data.id Unique ID for the current upload request + * @param {String} data.fd Path to the file being written + * @param {String} data.name Name of the file being written + * @param {Integer} data.written Number of bytes written so far + * @param {Integer?} data.total If known, total number of bytes to be written + * @param {Integer} data.percent Percentage progress. If the total amount of bytes to be + * written is unknown, this will be 0. + */ + + /** + * Upload a stream of files + * + * @method Adapter#receive + * + * @param {Object?} opts Optional additional configuration + * @param {String?} opts.key S3 access key + * @param {String?} opts.secret S3 access secret + * @param {String?} opts.bucket S3 bucket to upload to + * @param {Adapter.onProgress?} opts.onProgress Function to be called repeatedly as the + * upload progresses + * @param {Object?} opts.s3options Optional object to provide additional + * options for the upload request. The options + * can be anything that is supported by S3's + * `putObject()` method. + * + * @see http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property + * @return {Uploader} + */ + receive (opts) { + + opts = merge(Adapter.parseOptions(opts, local(this).globals)) + // Allow the special onProgress event listener to be passed to the upload request + .and({ request: { onProgress: opts.onProgress } }) + .recursively.into({}) + + return new Action(opts).upload() + } +} diff --git a/lib/index.js b/lib/index.js new file mode 100644 index 0000000..3fe888f --- /dev/null +++ b/lib/index.js @@ -0,0 +1,38 @@ +/** + * skipper-better-s3 + * + * @author Robert Rossmann + * @copyright 2015 Robert Rossmann + * @license http://choosealicense.com/licenses/bsd-3-clause BSD-3-Clause License + */ + +'use strict' + +const Adapter = require('./adapter') + + +/** + * Instantiate a new adapter + * + * Usually it is enough to call this function once per project. + * + * @param {Object?} globals Optional global options for this adapter + * @return {Adapter} New instance of the Adapter class + */ +module.exports = function getAdapter (globals) { + + // Due to an issue with current Skipper version, we must export a special wrapper around the + // adapter instance. Once that issue is resolved, we can return a new Adapter instance directly + // See the progress on Github: + // https://github.com/balderdashy/skipper/pull/102 + const adapter = new Adapter(globals) + , wrapper = + { ls: (dirname, done) => adapter.ls(dirname, done) + , rm: (fd, done) => adapter.rm(fd, done) + , read: (fd, done) => adapter.read(fd, done) + , receive: opts => adapter.receive(opts) + , url: (operation, opts, done) => adapter.url(operation, opts, done) + } + + return wrapper +} diff --git a/lib/uploader.js b/lib/uploader.js new file mode 100644 index 0000000..d7878db --- /dev/null +++ b/lib/uploader.js @@ -0,0 +1,88 @@ +/** + * skipper-better-s3 + * + * @author Robert Rossmann + * @copyright 2015 Robert Rossmann + * @license http://choosealicense.com/licenses/bsd-3-clause BSD-3-Clause License + */ + +'use strict' + +const WritableStream = require('stream').Writable + , AWS = require('aws-sdk') + , mime = require('mime') + , merge = require('semantic-merge') + , uuid = require('node-uuid') + , local = require('./util/local') + + +module.exports = class Uploader extends WritableStream { + + constructor (opts) { + + // Initialise this writable stream in object mode + super({ objectMode: true }) + + const scope = local(this) + + // Save configuration options and the S3 client to local scope + scope.opts = opts + scope.client = new AWS.S3(opts.service) + } + + _write (stream, encoding, done) { + + // Fix the content type header on the request in case it was incorrectly detected + // Wrong detection can happen when uploading via curl, for example. + // This will also help the S3 library to properly detect and set the Content-Type in storage. + stream.headers['content-type'] = mime.lookup(stream.fd) + + const scope = local(this) + , params = merge({ Key: stream.fd + , Body: stream + , ContentType: stream.headers['content-type'] + }) + .and(scope.opts.request) + .excluding('onProgress') + .into({}) + , request = scope.client.upload(params, (err, data) => { + // Attach the response data to the stream - Skipper packages this in its response to the + // caller for further reuse + stream.extra = data + + return done(err, data) + }) + + // Attach progress handler + this.trackProgress(request, scope.opts.request.onProgress) + } + + trackProgress (request, handler) { + + // If there's no handler provided, do not attach any listeners + if (typeof handler !== 'function') + return + + // Generate a per-request unique ID + const id = uuid.v4() + + request.on('httpUploadProgress', progress => { + // Prepare Skipper-specific progress structure + const written = progress.loaded + // S3 docs say that progress.total may not always be present, so let's have some fallback + , total = progress.total || request.body.byteCount || null + , data = + { id: id + , fd: request.body.fd + , name: request.body.name + , written: written + , total: total + // If total is null, performing bitwise-or will change the result to 0 + // It also rounds down to an integer, which is kinda nice + , percent: written / total * 100 | 0 + } + + handler(data) + }) + } +} diff --git a/lib/util/local.js b/lib/util/local.js new file mode 100644 index 0000000..29bee99 --- /dev/null +++ b/lib/util/local.js @@ -0,0 +1,21 @@ +/** + * skipper-better-s3 + * + * @author Robert Rossmann + * @copyright 2015 Robert Rossmann + * @license http://choosealicense.com/licenses/bsd-3-clause BSD-3-Clause License + */ + +'use strict' + +const map = new WeakMap() + +module.exports = function local (key) { + + let contents = map.get(key) + + if (! contents) + map.set(key, contents = {}) + + return contents +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..1b8721a --- /dev/null +++ b/package.json @@ -0,0 +1,44 @@ +{ + "name": "skipper-better-s3", + "description": "A better approach to Amazon S3 file management for Skipper", + "version": "1.0.0", + "author": { + "name": "Robert Rossmann", + "email": "rr.rossmann@me.com" + }, + "bugs": { + "url": "https://github.com/Dreamscapes/skipper-better-s3/issues" + }, + "dependencies": { + "aws-sdk": "^2.2.4", + "mime": "^1.3.4", + "node-uuid": "^1.4.3", + "semantic-merge": "^1.0.0" + }, + "devDependencies": { + "eslint": "^1.5.1", + "jsdoc": "github:jsdoc3/jsdoc#master" + }, + "engines": { + "node": ">=4.0.0" + }, + "homepage": "https://github.com/Dreamscapes/skipper-better-s3", + "keywords": [ + "S3", + "amazon", + "file", + "sails", + "skipper", + "storage", + "upload" + ], + "license": "BSD-3-Clause", + "main": "lib", + "repository": { + "type": "git", + "url": "git://github.com/Dreamscapes/skipper-better-s3" + }, + "scripts": { + "lint": "eslint lib" + } +} diff --git a/targets b/targets new file mode 160000 index 0000000..0f67efb --- /dev/null +++ b/targets @@ -0,0 +1 @@ +Subproject commit 0f67efbcb4c681a806b58649e542c868a79b1f0c