[Pkg-javascript-commits] [node-multiparty] 02/08: Imported Upstream version 4.0.0
Andrew Kelley
andrewrk-guest at moszumanska.debian.org
Fri Oct 17 05:45:02 UTC 2014
This is an automated email from the git hooks/post-receive script.
andrewrk-guest pushed a commit to branch master
in repository node-multiparty.
commit 78d2aee1f6d1e6aad71c9654371c84fdcd3c8255
Author: Andrew Kelley <superjoe30 at gmail.com>
Date: Fri Oct 17 05:36:01 2014 +0000
Imported Upstream version 4.0.0
---
.travis.yml | 3 -
CHANGELOG.md | 28 ++
LICENSE | 26 +-
README.md | 55 ++--
examples/s3.js | 76 +++---
index.js | 296 ++++++++++++---------
package.json | 26 +-
.../fixture/http/workarounds/missing-hyphens1.http | 12 -
.../fixture/http/workarounds/missing-hyphens2.http | 12 -
test/fixture/js/workarounds.js | 8 -
test/standalone/test-connection-aborted.js | 3 +
test/standalone/test-emit-order.js | 61 +++++
test/standalone/test-error-listen-after-parse.js | 10 +-
test/standalone/test-error-unpipe.js | 33 ++-
test/standalone/test-issue-36.js | 35 ++-
test/standalone/test-issue-5.js | 40 ---
test/standalone/test-max-files-size.js | 14 +-
test/standalone/test-missing-boundary-end.js | 46 ++++
test/standalone/test-stream-error.js | 36 +++
test/test.js | 30 ++-
20 files changed, 530 insertions(+), 320 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index b1fc4b0..6e5919d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,3 @@
language: node_js
node_js:
- - "0.8"
- "0.10"
-before_script:
- - ulimit -n 500
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3f6ce81..de6139c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,31 @@
+### 4.0.0
+
+ * Andrew Kelley:
+ - 'part' events for fields no longer fire if `autoFields` is on.
+ - 'part' events for files no longer fire if `autoFiles` is on.
+ - 'field', 'file', 'part' events are guaranteed to emit in the correct
+ order - the order that the user places the parts in the request.
+ Each `part` 'end' event is guaranteed to emit before the next 'part'
+ event is emitted.
+ - Drop Node.js 0.8.x support.
+ - Remove support for generating the hash digest of a part. If you want this,
+ do it in your own code.
+ - Now `part` objects emit 'error' events. This makes streaming work better
+ since the part stream will emit an error when it is no longer streaming.
+ - `file` objects no longer have the undocumented `ws` property.
+ - More robust `maxFilesSize` implementation. Before it was possible for
+ race conditions to cause more than `maxFilesSize` bytes to get written
+ to disk. That is now fixed.
+ - More robustly random temp file names. Now using 18 bytes of randomness
+ instead of 8.
+ - Better s3 example code.
+ - Delete some unused legacy code.
+ - Update and clarify documentation.
+
+ * Douglas Christopher Wilson:
+ - Require the close boundary. This makes multiparty more RFC-compliant and
+ makes some invalid requests which used to work, now emit an error instead.
+
### 3.3.2
* Douglas Christopher Wilson:
diff --git a/LICENSE b/LICENSE
index 8488a40..ebd3e8a 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,7 +1,25 @@
-Copyright (C) 2011-2013 Felix Geisendörfer, Andrew Kelley
+The MIT License (Expat)
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+Copyright (c) 2014 Andrew Kelley
+Copyright (c) 2014 Douglas Christopher Wilson
+Copyright (c) 2013 Felix Geisendörfer
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation files
+(the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
index 120e5c4..f398ffb 100644
--- a/README.md
+++ b/README.md
@@ -8,12 +8,12 @@ which may be worth looking into.
### Why the fork?
- * This module uses the Node.js v0.10 streams properly, *even in Node.js v0.8*
+ * This module uses the Node.js v0.10 streams properly
* It will not create a temp file for you unless you want it to.
* Counts bytes and does math to help you figure out the `Content-Length` of
- each part.
- * You can easily stream uploads to s3 with
- [knox](https://github.com/LearnBoost/knox), for [example](examples/s3.js).
+ the final part.
+ * You can stream uploads to s3 with
+ [aws-sdk](https://github.com/aws/aws-sdk-js), for [example](examples/s3.js).
* Less bugs. This code is simpler, has all deprecated functionality removed,
has cleaner tests, and does not try to do anything beyond multipart stream
parsing.
@@ -64,13 +64,15 @@ http.createServer(function(req, res) {
## API
### multiparty.Form
+
```js
var form = new multiparty.Form(options)
```
+
Creates a new form. Options:
* `encoding` - sets encoding for the incoming form fields. Defaults to `utf8`.
- * `maxFieldsSize` - Limits the amount of memory a field (not a file) can
+ * `maxFieldsSize` - Limits the amount of memory all fields (not files) can
allocate in bytes. If this value is exceeded, an `error` event is emitted.
The default size is 2MB.
* `maxFields` - Limits the number of fields that will be parsed before
@@ -79,16 +81,13 @@ Creates a new form. Options:
* `maxFilesSize` - Only relevant when `autoFiles` is `true`. Limits the
total bytes accepted for all files combined. If this value is exceeded,
an `error` event is emitted. The default is `Infinity`.
- * `autoFields` - Enables `field` events. This is automatically set to `true`
- if you add a `field` listener.
- * `autoFiles` - Enables `file` events. This is automatically set to `true`
- if you add a `file` listener.
+ * `autoFields` - Enables `field` events and disables `part` events for fields.
+ This is automatically set to `true` if you add a `field` listener.
+ * `autoFiles` - Enables `file` events and disables `part` events for files.
+ This is automatically set to `true` if you add a `file` listener.
* `uploadDir` - Only relevant when `autoFiles` is `true`. The directory for
placing file uploads in. You can move them later using `fs.rename()`.
Defaults to `os.tmpDir()`.
- * `hash` - Only relevant when `autoFiles` is `true`. If you want checksums
- calculated for incoming files, set this to either `sha1` or `md5`.
- Defaults to off.
#### form.parse(request, [cb])
@@ -100,6 +99,8 @@ var count = 0;
var form = new multiparty.Form();
// Errors may be emitted
+// Note that if you are listening to 'part' events, the same error may be
+// emitted from the `form` and the `part`.
form.on('error', function(err) {
console.log('Error parsing form: ' + err.stack);
});
@@ -123,6 +124,10 @@ form.on('part', function(part) {
// ignore file's content here
part.resume();
}
+
+ part.on('error', function(err) {
+ // decide what to do
+ });
});
// Close emitted after form parsed
@@ -134,14 +139,13 @@ form.on('close', function() {
// Parse req
form.parse(req);
-
```
If `cb` is provided, `autoFields` and `autoFiles` are set to `true` and all
fields and files are collected and passed to the callback, removing the need to
-listen to any events on `form`. This is for convenience when wanted to read
-everything, but be careful as this will write all uploaded files to the disk,
-even ones you may not be interested in.
+listen to any events on `form`. This is for convenience when you want to read
+everything, but be sure to write cleanup code, as this will write all uploaded
+files to the disk, even ones you may not be interested in.
```js
form.parse(req, function(err, fields, files) {
@@ -184,6 +188,9 @@ multipart requests!
Only one 'error' event can ever be emitted, and if an 'error' event is
emitted, then 'close' will not be emitted.
+Note that an 'error' event will be emitted both from the `form` and from the
+current `part`.
+
#### 'part' (part)
Emitted when a part is encountered in the request. `part` is a
@@ -200,6 +207,11 @@ Emitted when a part is encountered in the request. `part` is a
If the part had a `Content-Length` header then that value is used
here instead.
+Parts for fields are not emitted when `autoFields` is on, and likewise parts
+for files are not emitted when `autoFiles` is on.
+
+`part` emits 'error' events! Make sure you handle them.
+
#### 'aborted'
Emitted when the request is aborted. This event will be followed shortly
@@ -210,7 +222,12 @@ by an `error` event. In practice you do not need to handle this event.
#### 'close'
Emitted after all parts have been parsed and emitted. Not emitted if an `error`
-event is emitted. This is typically when you would send your response.
+event is emitted.
+
+If you have `autoFiles` on, this is not fired until all the data has been
+flushed to disk and the file handles have been closed.
+
+This is typically when you would send your response.
#### 'file' (name, file)
@@ -228,11 +245,7 @@ stream uploads to disk for you.
- `headers` - the HTTP headers that were sent along with this file
- `size` - size of the file in bytes
-If you set the `form.hash` option, then `file` will also contain a `hash`
-property which is the checksum of the file.
-
#### 'field' (name, value)
* `name` - field name
* `value` - string field value
-
diff --git a/examples/s3.js b/examples/s3.js
index 60617ba..a3b011d 100644
--- a/examples/s3.js
+++ b/examples/s3.js
@@ -1,15 +1,21 @@
-var http = require('http')
- , util = require('util')
- , multiparty = require('../')
- , knox = require('knox')
- , Batch = require('batch')
- , PORT = process.env.PORT || 27372
+if (!process.env.S3_BUCKET || !process.env.S3_KEY || !process.env.S3_SECRET) {
+ console.log("To run this example, do this:");
+ console.log("npm install aws-sdk");
+ console.log('S3_BUCKET="(your s3 bucket)" S3_KEY="(your s3 key)" S3_SECRET="(your s3 secret) node examples/s3.js"');
+ process.exit(1);
+}
-var s3Client = knox.createClient({
- secure: false,
- key: process.env.S3_KEY,
- secret: process.env.S3_SECRET,
- bucket: process.env.S3_BUCKET,
+var http = require('http');
+var util = require('util');
+var multiparty = require('../');
+var AWS = require('aws-sdk');
+var PORT = process.env.PORT || 27372;
+
+var bucket = process.env.S3_BUCKET;
+var s3Client = new AWS.S3({
+ accessKeyId: process.env.S3_KEY,
+ secretAccessKey: process.env.S3_SECRET,
+ // See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#constructor-property
});
var server = http.createServer(function(req, res) {
@@ -17,47 +23,33 @@ var server = http.createServer(function(req, res) {
res.writeHead(200, {'content-type': 'text/html'});
res.end(
'<form action="/upload" enctype="multipart/form-data" method="post">'+
- '<input type="text" name="path"><br>'+
+ '<input type="text" name="path" placeholder="s3 key here"><br>'+
'<input type="file" name="upload"><br>'+
'<input type="submit" value="Upload">'+
'</form>'
);
} else if (req.url === '/upload') {
- var headers = {
- 'x-amz-acl': 'public-read',
- };
var form = new multiparty.Form();
- var batch = new Batch();
- batch.push(function(cb) {
- form.on('field', function(name, value) {
- if (name === 'path') {
- var destPath = value;
- if (destPath[0] !== '/') destPath = '/' + destPath;
- cb(null, destPath);
- }
- });
+ var destPath;
+ form.on('field', function(name, value) {
+ if (name === 'path') {
+ destPath = value;
+ }
});
- batch.push(function(cb) {
- form.on('part', function(part) {
- if (! part.filename) return;
- cb(null, part);
- });
- });
- batch.end(function(err, results) {
- if (err) throw err;
- form.removeListener('close', onEnd);
- var destPath = results[0]
- , part = results[1];
-
- headers['Content-Length'] = part.byteCount;
- s3Client.putStream(part, destPath, headers, function(err, s3Response) {
+ form.on('part', function(part) {
+ s3Client.putObject({
+ Bucket: bucket,
+ Key: destPath,
+ ACL: 'public-read',
+ Body: part,
+ ContentLength: part.byteCount,
+ }, function(err, data) {
if (err) throw err;
- res.statusCode = s3Response.statusCode;
- s3Response.pipe(res);
- console.log("https://s3.amazonaws.com/" + process.env.S3_BUCKET + destPath);
+ console.log("done", data);
+ res.end("OK");
+ console.log("https://s3.amazonaws.com/" + bucket + '/' + destPath);
});
});
- form.on('close', onEnd);
form.parse(req);
} else {
diff --git a/index.js b/index.js
index dd3c7e0..88fc224 100644
--- a/index.js
+++ b/index.js
@@ -1,41 +1,44 @@
-exports.Form = Form;
-
-var stream = require('readable-stream')
- , util = require('util')
- , fs = require('fs')
- , crypto = require('crypto')
- , path = require('path')
- , os = require('os')
- , StringDecoder = require('string_decoder').StringDecoder
- , StreamCounter = require('stream-counter')
-
-var START = 0
- , START_BOUNDARY = 1
- , HEADER_FIELD_START = 2
- , HEADER_FIELD = 3
- , HEADER_VALUE_START = 4
- , HEADER_VALUE = 5
- , HEADER_VALUE_ALMOST_DONE = 6
- , HEADERS_ALMOST_DONE = 7
- , PART_DATA_START = 8
- , PART_DATA = 9
- , PART_END = 10
- , CLOSE_BOUNDARY = 11
- , END = 12
-
- , LF = 10
- , CR = 13
- , SPACE = 32
- , HYPHEN = 45
- , COLON = 58
- , A = 97
- , Z = 122
+var stream = require('stream');
+var util = require('util');
+var fs = require('fs');
+var crypto = require('crypto');
+var path = require('path');
+var os = require('os');
+var StringDecoder = require('string_decoder').StringDecoder;
+var FdSlicer = require('fd-slicer');
+
+var START = 0;
+var START_BOUNDARY = 1;
+var HEADER_FIELD_START = 2;
+var HEADER_FIELD = 3;
+var HEADER_VALUE_START = 4;
+var HEADER_VALUE = 5;
+var HEADER_VALUE_ALMOST_DONE = 6;
+var HEADERS_ALMOST_DONE = 7;
+var PART_DATA_START = 8;
+var PART_DATA = 9;
+var PART_END = 10;
+var CLOSE_BOUNDARY = 11;
+var END = 12;
+
+var LF = 10;
+var CR = 13;
+var SPACE = 32;
+var HYPHEN = 45;
+var COLON = 58;
+var A = 97;
+var Z = 122;
var CONTENT_TYPE_RE = /^multipart\/(?:form-data|related)(?:;|$)/i;
var CONTENT_TYPE_PARAM_RE = /;\s*([^=]+)=(?:"([^"]+)"|([^;]+))/gi;
var FILE_EXT_RE = /(\.[_\-a-zA-Z0-9]{0,16}).*/;
var LAST_BOUNDARY_SUFFIX_LEN = 4; // --\r\n
+// replace base64 characters with safe-for-filename characters
+var b64Safe = {'/': '_', '+': '-'};
+
+exports.Form = Form;
+
util.inherits(Form, stream.Writable);
function Form(options) {
var self = this;
@@ -44,7 +47,6 @@ function Form(options) {
options = options || {};
self.error = null;
- self.finished = false;
self.autoFields = !!options.autoFields;
self.autoFiles = !!options.autoFiles;
@@ -54,7 +56,6 @@ function Form(options) {
self.maxFilesSize = options.maxFilesSize || Infinity;
self.uploadDir = options.uploadDir || os.tmpDir();
self.encoding = options.encoding || 'utf8';
- self.hash = options.hash || false;
self.bytesReceived = 0;
self.bytesExpected = null;
@@ -68,7 +69,7 @@ function Form(options) {
self.backpressure = false;
self.writeCbs = [];
- if (options.boundary) setUpParser(self, options.boundary);
+ self.emitQueue = [];
self.on('newListener', function(eventName) {
if (eventName === 'file') {
@@ -192,12 +193,12 @@ Form.prototype.parse = function(req, cb) {
self.error = err;
req.removeListener('aborted', onReqAborted);
req.removeListener('end', onReqEnd);
+ if (self.destStream) {
+ self.destStream.emit('error', err);
+ }
}
- self.openedFiles.forEach(function(file) {
- destroyFile(self, file);
- });
- self.openedFiles = [];
+ cleanupOpenFiles(self);
if (first) {
self.emit('error', err);
@@ -213,20 +214,20 @@ Form.prototype.parse = function(req, cb) {
Form.prototype._write = function(buffer, encoding, cb) {
if (this.error) return;
- var self = this
- , i = 0
- , len = buffer.length
- , prevIndex = self.index
- , index = self.index
- , state = self.state
- , lookbehind = self.lookbehind
- , boundary = self.boundary
- , boundaryChars = self.boundaryChars
- , boundaryLength = self.boundary.length
- , boundaryEnd = boundaryLength - 1
- , bufferLength = buffer.length
- , c
- , cl
+ var self = this;
+ var i = 0;
+ var len = buffer.length;
+ var prevIndex = self.index;
+ var index = self.index;
+ var state = self.state;
+ var lookbehind = self.lookbehind;
+ var boundary = self.boundary;
+ var boundaryChars = self.boundaryChars;
+ var boundaryLength = self.boundary.length;
+ var boundaryEnd = boundaryLength - 1;
+ var bufferLength = buffer.length;
+ var c;
+ var cl;
for (i = 0; i < len; i++) {
c = buffer[i];
@@ -511,16 +512,12 @@ Form.prototype.onParseHeadersEnd = function(offset) {
self.boundary.length - LAST_BOUNDARY_SUFFIX_LEN) :
undefined;
- self.emit('part', self.destStream);
if (self.destStream.filename == null && self.autoFields) {
handleField(self, self.destStream);
} else if (self.destStream.filename != null && self.autoFiles) {
handleFile(self, self.destStream);
} else {
- beginFlush(self);
- self.destStream.on('end', function(){
- endFlush(self);
- });
+ handlePart(self, self.destStream);
}
}
@@ -549,76 +546,122 @@ function beginFlush(self) {
function endFlush(self) {
self.flushing -= 1;
+
+ if (self.flushing < 0) {
+ // if this happens this is a critical bug in multiparty and this stack trace
+ // will help us figure it out.
+ self.handleError(new Error("unexpected endFlush"));
+ return;
+ }
+
maybeClose(self);
}
function maybeClose(self) {
- if (!self.flushing && self.finished && !self.error) {
- self.emit('close');
- }
+ if (self.flushing > 0 || self.error) return;
+
+ // go through the emit queue in case any field, file, or part events are
+ // waiting to be emitted
+ holdEmitQueue(self)(function() {
+ // nextTick because the user is listening to part 'end' events and we are
+ // using part 'end' events to decide when to emit 'close'. we add our 'end'
+ // handler before the user gets a chance to add theirs. So we make sure
+ // their 'end' event fires before we emit the 'close' event.
+ // this is covered by test/standalone/test-issue-36
+ process.nextTick(function() {
+ self.emit('close');
+ });
+ });
}
-function destroyFile(self, file) {
- if (!file.ws) return;
- file.ws.removeAllListeners('close');
- file.ws.on('close', function() {
- fs.unlink(file.path, function(err) {
- if (err && !self.error) self.handleError(err);
+function cleanupOpenFiles(self) {
+ self.openedFiles.forEach(function(internalFile) {
+ // since fd slicer autoClose is true, destroying the only write stream
+ // is guaranteed by the API to close the fd
+ internalFile.ws.destroy();
+
+ fs.unlink(internalFile.publicFile.path, function(err) {
+ if (err) self.handleError(err);
});
});
- file.ws.destroy();
+ self.openedFiles = [];
+}
+
+function holdEmitQueue(self) {
+ var o = {cb: null};
+ self.emitQueue.push(o);
+ return function(cb) {
+ o.cb = cb;
+ flushEmitQueue(self);
+ };
+}
+
+function flushEmitQueue(self) {
+ while (self.emitQueue.length > 0 && self.emitQueue[0].cb) {
+ self.emitQueue.shift().cb();
+ }
+}
+
+function handlePart(self, partStream) {
+ beginFlush(self);
+ var emitAndReleaseHold = holdEmitQueue(self);
+ partStream.on('end', function() {
+ endFlush(self);
+ });
+ emitAndReleaseHold(function() {
+ self.emit('part', partStream);
+ });
}
function handleFile(self, fileStream) {
if (self.error) return;
- var file = {
+ var publicFile = {
fieldName: fileStream.name,
originalFilename: fileStream.filename,
path: uploadPath(self.uploadDir, fileStream.filename),
headers: fileStream.headers,
+ size: 0,
+ };
+ var internalFile = {
+ publicFile: publicFile,
+ ws: null,
};
beginFlush(self); // flush to write stream
- file.ws = fs.createWriteStream(file.path);
- self.openedFiles.push(file);
- fileStream.pipe(file.ws);
- var counter = new StreamCounter();
- var seenBytes = 0;
- fileStream.pipe(counter);
- var hashWorkaroundStream
- , hash = null;
- if (self.hash) {
- // workaround stream because https://github.com/joyent/node/issues/5216
- hashWorkaroundStream = stream.Writable();
- hash = crypto.createHash(self.hash);
- hashWorkaroundStream._write = function(buffer, encoding, callback) {
- hash.update(buffer);
- callback();
- };
- fileStream.pipe(hashWorkaroundStream);
- }
- counter.on('progress', function() {
- var deltaBytes = counter.bytes - seenBytes;
- seenBytes += deltaBytes;
- self.totalFileSize += deltaBytes;
- if (self.totalFileSize > self.maxFilesSize) {
- if (hashWorkaroundStream) fileStream.unpipe(hashWorkaroundStream);
- fileStream.unpipe(counter);
- fileStream.unpipe(file.ws);
- self.handleError(new Error("maxFilesSize " + self.maxFilesSize + " exceeded"));
- }
- });
- file.ws.on('error', function(err) {
- if (!self.error) self.handleError(err);
- });
- file.ws.on('close', function() {
- if (hash) file.hash = hash.digest('hex');
- file.size = counter.bytes;
- self.emit('file', fileStream.name, file);
- endFlush(self);
+ var emitAndReleaseHold = holdEmitQueue(self);
+ fileStream.on('error', function(err) {
+ self.handleError(err);
});
- beginFlush(self); // flush from file stream
- fileStream.on('end', function(){
- endFlush(self);
+ fs.open(publicFile.path, 'w', function(err, fd) {
+ if (err) return self.handleError(err);
+ var fdSlicer = new FdSlicer(fd, {autoClose: true});
+
+ // end option here guarantees that no more than that amount will be written
+ // or else an error will be emitted
+ internalFile.ws = fdSlicer.createWriteStream({end: self.maxFilesSize - self.totalFileSize});
+
+ // if an error ocurred while we were waiting for fs.open we handle that
+ // cleanup now
+ self.openedFiles.push(internalFile);
+ if (self.error) return cleanupOpenFiles(self);
+
+ var prevByteCount = 0;
+ internalFile.ws.on('error', function(err) {
+ self.handleError(err);
+ });
+ internalFile.ws.on('progress', function() {
+ publicFile.size = internalFile.ws.bytesWritten;
+ var delta = publicFile.size - prevByteCount;
+ self.totalFileSize += delta;
+ prevByteCount = publicFile.size;
+ });
+ fdSlicer.on('close', function() {
+ if (self.error) return;
+ emitAndReleaseHold(function() {
+ self.emit('file', fileStream.name, publicFile);
+ });
+ endFlush(self);
+ });
+ fileStream.pipe(internalFile.ws);
});
}
@@ -627,6 +670,10 @@ function handleField(self, fieldStream) {
var decoder = new StringDecoder(self.encoding);
beginFlush(self);
+ var emitAndReleaseHold = holdEmitQueue(self);
+ fieldStream.on('error', function(err) {
+ self.handleError(err);
+ });
fieldStream.on('readable', function() {
var buffer = fieldStream.read();
if (!buffer) return;
@@ -640,7 +687,9 @@ function handleField(self, fieldStream) {
});
fieldStream.on('end', function() {
- self.emit('field', fieldStream.name, value);
+ emitAndReleaseHold(function() {
+ self.emit('field', fieldStream.name, value);
+ });
endFlush(self);
});
}
@@ -672,26 +721,35 @@ function setUpParser(self, boundary) {
self.index = null;
self.partBoundaryFlag = false;
+ beginFlush(self);
self.on('finish', function() {
- if ((self.state === HEADER_FIELD_START && self.index === 0) ||
- (self.state === PART_DATA && self.index === self.boundary.length))
- {
- self.onParsePartEnd();
- } else if (self.state !== END) {
+ if (self.state !== END) {
self.handleError(new Error('stream ended unexpectedly'));
}
- self.finished = true;
- maybeClose(self);
+ endFlush(self);
});
}
function uploadPath(baseDir, filename) {
var ext = path.extname(filename).replace(FILE_EXT_RE, '$1');
- var name = process.pid + '-' +
- (Math.random() * 0x100000000 + 1).toString(36) + ext;
+ var name = randoString(18) + ext;
return path.join(baseDir, name);
}
+function randoString(size) {
+ return rando(size).toString('base64').replace(/[\/\+]/g, function(x) {
+ return b64Safe[x];
+ });
+}
+
+function rando(size) {
+ try {
+ return crypto.randomBytes(size);
+ } catch (err) {
+ return crypto.pseudoRandomBytes(size);
+ }
+}
+
function parseFilename(headerValue) {
var m = headerValue.match(/\bfilename="(.*?)"($|; )/i);
if (!m) {
diff --git a/package.json b/package.json
index e0cd4d7..58a6498 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "multiparty",
- "version": "3.3.2",
+ "version": "4.0.0",
"description": "multipart/form-data parser which supports streaming",
"repository": {
"type": "git",
@@ -14,21 +14,29 @@
"s3"
],
"devDependencies": {
- "findit": "~2.0.0",
"mkdirp": "~0.5.0",
- "pend": "~1.1.1",
+ "pend": "~1.1.3",
"rimraf": "~2.2.8",
- "superagent": "~0.18.0"
+ "superagent": "~0.20.0",
+ "findit2": "~2.2.2"
},
"scripts": {
- "test": "ulimit -n 500 && node test/test.js"
+ "test": "node test/test.js"
},
"engines": {
- "node": ">=0.8.0"
+ "node": ">=0.10.0"
},
"license": "MIT",
"dependencies": {
- "readable-stream": "~1.1.9",
- "stream-counter": "~0.2.0"
- }
+ "fd-slicer": "~0.3.2"
+ },
+ "bugs": {
+ "url": "https://github.com/andrewrk/node-multiparty/issues"
+ },
+ "main": "index.js",
+ "directories": {
+ "example": "examples",
+ "test": "test"
+ },
+ "author": "Andrew Kelley <superjoe30 at gmail.com>"
}
diff --git a/test/fixture/http/workarounds/missing-hyphens1.http b/test/fixture/http/workarounds/missing-hyphens1.http
deleted file mode 100644
index 2826890..0000000
--- a/test/fixture/http/workarounds/missing-hyphens1.http
+++ /dev/null
@@ -1,12 +0,0 @@
-POST /upload HTTP/1.1
-Host: localhost:8080
-Content-Type: multipart/form-data; boundary=----TLV0SrKD4z1TRxRhAPUvZ
-Content-Length: 178
-
-------TLV0SrKD4z1TRxRhAPUvZ
-Content-Disposition: form-data; name="upload"; filename="plain.txt"
-Content-Type: text/plain
-
-I am a plain text file
-
-------TLV0SrKD4z1TRxRhAPUvZ
diff --git a/test/fixture/http/workarounds/missing-hyphens2.http b/test/fixture/http/workarounds/missing-hyphens2.http
deleted file mode 100644
index 8e18194..0000000
--- a/test/fixture/http/workarounds/missing-hyphens2.http
+++ /dev/null
@@ -1,12 +0,0 @@
-POST /upload HTTP/1.1
-Host: localhost:8080
-Content-Type: multipart/form-data; boundary=----TLV0SrKD4z1TRxRhAPUvZ
-Content-Length: 180
-
-------TLV0SrKD4z1TRxRhAPUvZ
-Content-Disposition: form-data; name="upload"; filename="plain.txt"
-Content-Type: text/plain
-
-I am a plain text file
-
-------TLV0SrKD4z1TRxRhAPUvZ
diff --git a/test/fixture/js/workarounds.js b/test/fixture/js/workarounds.js
deleted file mode 100644
index e59c5b2..0000000
--- a/test/fixture/js/workarounds.js
+++ /dev/null
@@ -1,8 +0,0 @@
-module.exports['missing-hyphens1.http'] = [
- {type: 'file', name: 'upload', filename: 'plain.txt', fixture: 'plain.txt',
- sha1: 'b31d07bac24ac32734de88b3687dddb10e976872'},
-];
-module.exports['missing-hyphens2.http'] = [
- {type: 'file', name: 'upload', filename: 'plain.txt', fixture: 'plain.txt',
- sha1: 'b31d07bac24ac32734de88b3687dddb10e976872'},
-];
diff --git a/test/standalone/test-connection-aborted.js b/test/standalone/test-connection-aborted.js
index bd83e1d..6448e79 100644
--- a/test/standalone/test-connection-aborted.js
+++ b/test/standalone/test-connection-aborted.js
@@ -16,6 +16,9 @@ var server = http.createServer(function (req, res) {
form.on('end', function () {
throw new Error('Unexpected "end" event');
});
+ form.on('close', function () {
+ throw new Error('Unexpected "close" event');
+ });
form.parse(req);
}).listen(0, 'localhost', function () {
var client = net.connect(server.address().port);
diff --git a/test/standalone/test-emit-order.js b/test/standalone/test-emit-order.js
new file mode 100644
index 0000000..a20869c
--- /dev/null
+++ b/test/standalone/test-emit-order.js
@@ -0,0 +1,61 @@
+var http = require('http');
+var multiparty = require('../../');
+var assert = require('assert');
+var superagent = require('superagent');
+var path = require('path');
+var bigFile = path.join(__dirname, "..", "fixture", "file", "pf1y5.png");
+
+var server = http.createServer(function(req, res) {
+ assert.strictEqual(req.url, '/upload');
+ assert.strictEqual(req.method, 'POST');
+
+ var fieldsInOrder = [
+ 'a',
+ 'b',
+ 'myimage.png',
+ 'c',
+ ];
+
+ var form = new multiparty.Form({
+ autoFields: true,
+ });
+
+ form.on('error', function (err) {
+ assert.ifError(err);
+ });
+
+ form.on('part', function(part) {
+ assert.ok(part.filename);
+ var expectedFieldName = fieldsInOrder.shift();
+ assert.strictEqual(part.name, expectedFieldName);
+ part.resume();
+ });
+
+ form.on('field', function(name, value) {
+ var expectedFieldName = fieldsInOrder.shift();
+ assert.strictEqual(name, expectedFieldName);
+ });
+
+ form.on('close', function() {
+ assert.strictEqual(fieldsInOrder.length, 0);
+ res.end("OK");
+ });
+
+ form.parse(req);
+});
+server.listen(function() {
+ var url = 'http://localhost:' + server.address().port + '/upload';
+ var req = superagent.post(url);
+ req.field('a', 'a-value');
+ req.field('b', 'b-value');
+ req.attach('myimage.png', bigFile);
+ req.field('c', 'hello');
+ req.on('error', function(err) {
+ assert.ifError(err);
+ });
+ req.on('response', function(res) {
+ assert.equal(res.statusCode, 200);
+ server.close();
+ });
+ req.end();
+});
diff --git a/test/standalone/test-error-listen-after-parse.js b/test/standalone/test-error-listen-after-parse.js
index aad1d13..00e1469 100644
--- a/test/standalone/test-error-listen-after-parse.js
+++ b/test/standalone/test-error-listen-after-parse.js
@@ -1,8 +1,8 @@
-var multiparty = require('../../')
- , assert = require('assert')
- , http = require('http')
- , net = require('net')
- , stream = require('readable-stream');
+var multiparty = require('../../');
+var assert = require('assert');
+var http = require('http');
+var net = require('net');
+var stream = require('stream');
var form = new multiparty.Form();
var req = new stream.Readable();
diff --git a/test/standalone/test-error-unpipe.js b/test/standalone/test-error-unpipe.js
index 2271230..0213f35 100644
--- a/test/standalone/test-error-unpipe.js
+++ b/test/standalone/test-error-unpipe.js
@@ -1,10 +1,13 @@
-var multiparty = require('../../')
- , assert = require('assert')
- , http = require('http')
- , net = require('net')
- , stream = require('readable-stream');
+var multiparty = require('../../');
+var assert = require('assert');
+var http = require('http');
+var net = require('net');
+var Pend = require('pend');
+var stream = require('stream');
+var err = null;
var form = new multiparty.Form();
+var pend = new Pend();
var req = new stream.Readable();
var unpiped = false;
@@ -15,17 +18,25 @@ req._read = function(){
this.push(new Buffer('--foo!'));
};
-form.on('error', function(err){
+pend.go(function(){
+ form.on('error', function(e){
+ err = e;
+ });
+});
+
+pend.go(function(){
+ form.on('unpipe', function(){
+ unpiped = true;
+ });
+});
+
+pend.wait(function(){
// verification that error event implies unpipe call
assert.ok(err);
assert.ok(unpiped, 'req was unpiped');
assert.equal(req._readableState.flowing, false, 'req not flowing');
assert.equal(req._readableState.pipesCount, 0, 'req has 0 pipes');
-});
-
-form.on('unpipe', function(){
- unpiped = true;
-});
+})
form.parse(req)
assert.equal(req._readableState.flowing, true, 'req flowing');
diff --git a/test/standalone/test-issue-36.js b/test/standalone/test-issue-36.js
index 6959e7c..3bacdda 100644
--- a/test/standalone/test-issue-36.js
+++ b/test/standalone/test-issue-36.js
@@ -2,6 +2,7 @@ var assert = require('assert');
var http = require('http');
var net = require('net');
var multiparty = require('../../');
+var superagent = require('superagent');
var client;
var server = http.createServer(function(req, res) {
@@ -16,28 +17,22 @@ var server = http.createServer(function(req, res) {
form.on('close', function() {
assert.ok(endCalled);
res.end();
- client.end();
- server.close();
});
form.parse(req);
});
server.listen(function() {
- client = net.connect(server.address().port);
-
- var boundary = "------WebKitFormBoundaryvfUZhxgsZDO7FXLF\r\n";
- var oneAttachment = boundary +
- "Content-Disposition: form-data; name=\"upload\"; filename=\"blah1.txt\"\r\n" +
- "Content-Type: text/plain\r\n" +
- "\r\n" +
- "hi1\r\n" +
- "\r\n";
- var payloadSize = oneAttachment.length + boundary.length;
-
- client.write("POST /upload HTTP/1.1\r\n" +
- "Content-Length: " + payloadSize + "\r\n" +
- "Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryvfUZhxgsZDO7FXLF\r\n" +
- "\r\n");
-
- client.write(oneAttachment);
- client.write(boundary);
+ var url = 'http://localhost:' + server.address().port + '/'
+ var req = superagent.post(url)
+ req.set('Content-Type', 'multipart/form-data; boundary=--WebKitFormBoundaryvfUZhxgsZDO7FXLF')
+ req.set('Content-Length', '186')
+ req.write('----WebKitFormBoundaryvfUZhxgsZDO7FXLF\r\n');
+ req.write('Content-Disposition: form-data; name="upload"; filename="blah1.txt"\r\n');
+ req.write('Content-Type: plain/text\r\n');
+ req.write('\r\n');
+ req.write('hi1\r\n');
+ req.write('\r\n');
+ req.write('----WebKitFormBoundaryvfUZhxgsZDO7FXLF--\r\n');
+ req.end(function(err, resp) {
+ server.close();
+ });
});
diff --git a/test/standalone/test-issue-5.js b/test/standalone/test-issue-5.js
deleted file mode 100644
index 2525280..0000000
--- a/test/standalone/test-issue-5.js
+++ /dev/null
@@ -1,40 +0,0 @@
-var assert = require('assert');
-var fs = require('fs');
-var http = require('http');
-var net = require('net');
-var multiparty = require('../../');
-
-var client;
-var attachmentCount = 510;
-var server = http.createServer(function(req, res) {
- var form = new multiparty.Form({maxFields: 10000});
-
- form.parse(req, function(err) {
- assert.strictEqual(err.code, "EMFILE");
- res.end();
- client.end();
- server.close();
- });
-});
-server.listen(function() {
- client = net.connect(server.address().port);
-
- var boundary = "------WebKitFormBoundaryvfUZhxgsZDO7FXLF\r\n";
- var oneAttachment = boundary +
- "Content-Disposition: form-data; name=\"upload\"; filename=\"blah1.txt\"\r\n" +
- "Content-Type: text/plain\r\n" +
- "\r\n" +
- "hi1\r\n" +
- "\r\n";
- var payloadSize = oneAttachment.length * attachmentCount + boundary.length;
-
- client.write("POST /upload HTTP/1.1\r\n" +
- "Content-Length: " + payloadSize + "\r\n" +
- "Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryvfUZhxgsZDO7FXLF\r\n" +
- "\r\n");
-
- for (var i = 0; i < attachmentCount; i += 1) {
- client.write(oneAttachment);
- }
- client.write(boundary);
-});
diff --git a/test/standalone/test-max-files-size.js b/test/standalone/test-max-files-size.js
index a8e1b51..184f991 100644
--- a/test/standalone/test-max-files-size.js
+++ b/test/standalone/test-max-files-size.js
@@ -1,9 +1,9 @@
-var http = require('http')
- , multiparty = require('../../')
- , assert = require('assert')
- , superagent = require('superagent')
- , path = require('path')
- , fs = require('fs')
+var http = require('http');
+var multiparty = require('../../');
+var assert = require('assert');
+var superagent = require('superagent');
+var path = require('path');
+var fs = require('fs');
var server = http.createServer(function(req, res) {
assert.strictEqual(req.url, '/upload');
@@ -15,7 +15,7 @@ var server = http.createServer(function(req, res) {
form.on('error', function (err) {
assert.ok(first);
first = false;
- assert.ok(/maxFilesSize/.test(err.message));
+ assert.strictEqual(err.code, 'ETOOBIG');
});
var fileCount = 0;
diff --git a/test/standalone/test-missing-boundary-end.js b/test/standalone/test-missing-boundary-end.js
new file mode 100644
index 0000000..3686413
--- /dev/null
+++ b/test/standalone/test-missing-boundary-end.js
@@ -0,0 +1,46 @@
+var superagent = require('superagent')
+ , assert = require('assert')
+ , multiparty = require('../../')
+ , http = require('http')
+
+var server = http.createServer(function(req, resp) {
+ var form = new multiparty.Form();
+
+ var errCount = 0;
+ form.on('error', function (err) {
+ assert.ok(err);
+ assert.equal(err.message, 'stream ended unexpectedly');
+ errCount += 1;
+ resp.end();
+ });
+ form.on('part', function (part) {
+ part.resume();
+ });
+ form.on('close', function () {
+ assert.equal(errCount, 1);
+ })
+
+ form.parse(req);
+});
+server.listen(function() {
+ var url = 'http://localhost:' + server.address().port + '/'
+ var req = superagent.post(url)
+ req.set('Content-Type', 'multipart/form-data; boundary=--WebKitFormBoundaryE19zNvXGzXaLvS5C')
+ req.write('----WebKitFormBoundaryE19zNvXGzXaLvS5C\r\n');
+ req.write('Content-Disposition: form-data; name="a[b]"\r\n');
+ req.write('\r\n');
+ req.write('3\r\n');
+ req.write('----WebKitFormBoundaryE19zNvXGzXaLvS5C\r\n');
+ req.write('Content-Disposition: form-data; name="a[c]"\r\n');
+ req.write('\r\n');
+ req.write('4\r\n');
+ req.write('----WebKitFormBoundaryE19zNvXGzXaLvS5C\r\n');
+ req.write('Content-Disposition: form-data; name="file"; filename="test.txt"\r\n');
+ req.write('Content-Type: plain/text\r\n');
+ req.write('\r\n');
+ req.write('and\r\n');
+ req.write('----WebKitFormBoundaryE19zNvXGzXaLvS5C\r\n');
+ req.end(function(err, resp) {
+ server.close();
+ });
+});
diff --git a/test/standalone/test-stream-error.js b/test/standalone/test-stream-error.js
new file mode 100644
index 0000000..a9a9785
--- /dev/null
+++ b/test/standalone/test-stream-error.js
@@ -0,0 +1,36 @@
+var assert = require('assert');
+var http = require('http');
+var net = require('net');
+var multiparty = require('../../');
+
+var server = http.createServer(function (req, res) {
+ var form = new multiparty.Form();
+ var gotPartErr;
+ form.on('part', function(part) {
+ part.on('error', function(err) {
+ gotPartErr = err;
+ });
+ part.resume();
+ });
+ form.on('error', function () {
+ assert.ok(gotPartErr);
+ server.close();
+ });
+ form.on('close', function () {
+ throw new Error('Unexpected "close" event');
+ });
+ form.parse(req);
+}).listen(0, 'localhost', function () {
+ var client = net.connect(server.address().port);
+ client.write(
+ "POST / HTTP/1.1\r\n" +
+ "Content-Length: 186\r\n" +
+ "Content-Type: multipart/form-data; boundary=--WebKitFormBoundaryvfUZhxgsZDO7FXLF\r\n" +
+ "\r\n" +
+ "----WebKitFormBoundaryvfUZhxgsZDO7FXLF\r\n" +
+ "Content-Disposition: form-data; name=\"upload\"; filename=\"blah1.txt\"\r\n" +
+ "Content-Type: plain/text\r\n" +
+ "\r\n" +
+ "hi1\r\n");
+ client.end();
+});
diff --git a/test/test.js b/test/test.js
index 83f65bc..41ef5d6 100644
--- a/test/test.js
+++ b/test/test.js
@@ -1,5 +1,6 @@
var spawn = require('child_process').spawn;
-var findit = require('findit');
+var crypto = require('crypto');
+var findit = require('findit2');
var path = require('path');
var Pend = require('pend');
var rimraf = require('rimraf');
@@ -122,28 +123,44 @@ function resetTempDir(cb) {
});
}
+function computeSha1(o) {
+ return function(cb) {
+ var file = o.value;
+ var hash = fs.createReadStream(file.path).pipe(crypto.createHash('sha1'));
+ hash.read(); // work around pre-https://github.com/joyent/node/commit/4bf1d1007fbd249d1d07b662278a5a34c6be12fd
+ hash.on('data', function(digest) {
+ fs.unlinkSync(file.path);
+ file.hash = digest.toString('hex');
+ cb();
+ });
+ };
+}
+
function uploadFixture(name, cb) {
server.once('request', function(req, res) {
var parts = [];
- var fileNames = [];
var form = new multiparty.Form({
autoFields: true,
autoFiles: true,
});
form.uploadDir = TMP_PATH;
- form.hash = "sha1";
+ var pend = new Pend();
form.on('error', callback);
form.on('file', function(name, value) {
- parts.push({type: 'file', name: name, value: value});
- fileNames.push(value.path);
+ var o = {type: 'file', name: name, value: value};
+ parts.push(o);
+ pend.go(computeSha1(o));
});
form.on('field', function(name, value) {
parts.push({type: 'field', name: name, value: value});
});
form.on('close', function() {
res.end('OK');
- callback(null, parts);
+ pend.wait(function(err) {
+ if (err) throw err;
+ callback(null, parts);
+ });
});
form.parse(req);
@@ -151,7 +168,6 @@ function uploadFixture(name, cb) {
var realCallback = cb;
cb = function() {};
realCallback.apply(null, arguments);
- cleanFiles(fileNames);
}
});
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-javascript/node-multiparty.git
More information about the Pkg-javascript-commits
mailing list