[Pkg-javascript-commits] [node-klaw] 01/05: New upstream version 2.1.1
Julien Puydt
julien.puydt at laposte.net
Mon Nov 20 08:27:57 UTC 2017
This is an automated email from the git hooks/post-receive script.
jpuydt-guest pushed a commit to branch master
in repository node-klaw.
commit 439ddc503ad6017c357facebce48ae62fc422479
Author: Julien Puydt <julien.puydt at laposte.net>
Date: Mon Nov 20 09:20:24 2017 +0100
New upstream version 2.1.1
---
CHANGELOG.md | 12 ++++
README.md | 129 +++++++++++++++++------------------------
package.json | 5 +-
src/index.js | 12 +++-
tests/walk_depth-limit.test.js | 60 +++++++++++++++++++
5 files changed, 137 insertions(+), 81 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c3c0274..a3a6023 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,15 @@
+2.1.1 / 2017-11-18
+------------------
+
+- Performance optimization [#27](https://github.com/jprichardson/node-klaw/pull/27)
+
+2.1.0 / 2017-08-10
+------------------
+
+### Added
+
+- Added `depthLimit` option to limit how deep to recurse into folders. [#25](https://github.com/jprichardson/node-klaw/pull/25)
+
2.0.0 / 2017-06-23
------------------
diff --git a/README.md b/README.md
index 130f905..9e66207 100644
--- a/README.md
+++ b/README.md
@@ -46,40 +46,35 @@ returns an object with two properties: `path` and `stats`. `path` is the full pa
the following:
- `queueMethod` (`string`, default: `'shift'`): Either `'shift'` or `'pop'`. On `readdir()` array, call either `shift()` or `pop()`.
- `pathSorter` (`function`, default: `undefined`): Sorting [function for Arrays](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort).
- - `fs` (`object`, default: `require('fs')`): Use this to hook into the `fs` methods or to use [`mock-fs`](https://github.com/tschaub/mock-fs)
+ - `fs` (`object`, default: [`graceful-fs`](https://github.com/isaacs/node-graceful-fs)): Use this to hook into the `fs` methods or to use [`mock-fs`](https://github.com/tschaub/mock-fs)
- `filter` (`function`, default: `undefined`): Filtering [function for Arrays](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/filter)
+ - `depthLimit` (`number`, default: `undefined`): The number of times to recurse before stopping. -1 for unlimited.
**Streams 1 (push) example:**
```js
-var klaw = require('klaw')
+const klaw = require('klaw')
-var items = [] // files, directories, symlinks, etc
+const items = [] // files, directories, symlinks, etc
klaw('/some/dir')
- .on('data', function (item) {
- items.push(item.path)
- })
- .on('end', function () {
- console.dir(items) // => [ ... array of files]
- })
+ .on('data', item => items.push(item.path))
+ .on('end', () => console.dir(items)) // => [ ... array of files]
```
**Streams 2 & 3 (pull) example:**
```js
-var klaw = require('klaw')
+const klaw = require('klaw')
-var items = [] // files, directories, symlinks, etc
+const items = [] // files, directories, symlinks, etc
klaw('/some/dir')
.on('readable', function () {
- var item
+ let item
while ((item = this.read())) {
items.push(item.path)
}
})
- .on('end', function () {
- console.dir(items) // => [ ... array of files]
- })
+ .on('end', () => console.dir(items)) // => [ ... array of files]
```
### Error Handling
@@ -89,22 +84,20 @@ Listen for the `error` event.
Example:
```js
-var klaw = require('klaw')
+const klaw = require('klaw')
+
klaw('/some/dir')
.on('readable', function () {
- var item
+ let item
while ((item = this.read())) {
// do something with the file
}
})
- .on('error', function (err, item) {
+ .on('error', (err, item) => {
console.log(err.message)
console.log(item.path) // the file the error occurred on
})
- .on('end', function () {
- console.dir(items) // => [ ... array of files]
- })
-
+ .on('end', () => console.dir(items)) // => [ ... array of files]
```
@@ -125,51 +118,47 @@ Install `through2`:
**Example (skipping directories):**
```js
-var klaw = require('klaw')
-var through2 = require('through2')
+const klaw = require('klaw')
+const through2 = require('through2')
-var excludeDirFilter = through2.obj(function (item, enc, next) {
+const excludeDirFilter = through2.obj(function (item, enc, next) {
if (!item.stats.isDirectory()) this.push(item)
next()
})
-var items = [] // files, directories, symlinks, etc
+const items = [] // files, directories, symlinks, etc
klaw('/some/dir')
.pipe(excludeDirFilter)
- .on('data', function (item) {
- items.push(item.path)
- })
- .on('end', function () {
- console.dir(items) // => [ ... array of files without directories]
- })
-
+ .on('data', item => items.push(item.path))
+ .on('end', () => console.dir(items)) // => [ ... array of files without directories]
```
+
**Example (ignore hidden directories):**
+
```js
-var klaw = require('klaw')
-var path = require('path')
+const klaw = require('klaw')
+const path = require('path')
-var filterFunc = function(item){
- var basename = path.basename(item)
+const filterFunc = item => {
+ const basename = path.basename(item)
return basename === '.' || basename[0] !== '.'
}
-klaw('/some/dir', { filter : filterFunc })
- .on('data', function(item){
+klaw('/some/dir', { filter: filterFunc })
+ .on('data', item => {
// only items of none hidden folders will reach here
})
-
```
**Example (totaling size of PNG files):**
```js
-var klaw = require('klaw')
-var path = require('path')
-var through2 = require('through2')
+const klaw = require('klaw')
+const path = require('path')
+const through2 = require('through2')
-var totalPngsInBytes = 0
-var aggregatePngSize = through2.obj(function (item, enc, next) {
+let totalPngsInBytes = 0
+const aggregatePngSize = through2.obj(function (item, enc, next) {
if (path.extname(item.path) === '.png') {
totalPngsInBytes += item.stats.size
}
@@ -179,23 +168,19 @@ var aggregatePngSize = through2.obj(function (item, enc, next) {
klaw('/some/dir')
.pipe(aggregatePngSize)
- .on('data', function (item) {
- items.push(item.path)
- })
- .on('end', function () {
- console.dir(totalPngsInBytes) // => total of all pngs (bytes)
- })
+ .on('data', item => items.push(item.path))
+ .on('end', () => console.dir(totalPngsInBytes)) // => total of all pngs (bytes)
```
**Example (deleting all .tmp files):**
```js
-var fs = require('fs')
-var klaw = require('klaw')
-var through2 = require('through2')
+const fs = require('fs')
+const klaw = require('klaw')
+const through2 = require('through2')
-var deleteAction = through2.obj(function (item, enc, next) {
+const deleteAction = through2.obj(function (item, enc, next) {
this.push(item)
if (path.extname(item.path) === '.tmp') {
@@ -204,19 +189,17 @@ var deleteAction = through2.obj(function (item, enc, next) {
} else {
item.deleted = false
next()
- }
+ }
})
-var deletedFiles = []
+const deletedFiles = []
klaw('/some/dir')
.pipe(deleteAction)
- .on('data', function (item) {
+ .on('data', item => {
if (!item.deleted) return
deletedFiles.push(item.path)
})
- .on('end', function () {
- console.dir(deletedFiles) // => all deleted files
- })
+ .on('end', () => console.dir(deletedFiles)) // => all deleted files
```
You can even chain a bunch of these filters and aggregators together. By using
@@ -228,9 +211,7 @@ multiple pipes.
klaw('/some/dir')
.pipe(filterCertainFiles)
.pipe(deleteSomeOtherFiles)
- .on('end', function () {
- console.log('all done!')
- })
+ .on('end', () => console.log('all done!'))
```
**Example passing (piping) through errors:**
@@ -239,30 +220,26 @@ Node.js does not `pipe()` errors. This means that the error on one stream, like
`klaw` will not pipe through to the next. If you want to do this, do the following:
```js
-var klaw = require('klaw')
-var through2 = require('through2')
+const klaw = require('klaw')
+const through2 = require('through2')
-var excludeDirFilter = through2.obj(function (item, enc, next) {
+const excludeDirFilter = through2.obj(function (item, enc, next) {
if (!item.stats.isDirectory()) this.push(item)
next()
})
-var items = [] // files, directories, symlinks, etc
+const items = [] // files, directories, symlinks, etc
klaw('/some/dir')
- .on('error', function (err) { excludeDirFilter.emit('error', err) }) // forward the error on
+ .on('error', err => excludeDirFilter.emit('error', err)) // forward the error on
.pipe(excludeDirFilter)
- .on('data', function (item) {
- items.push(item.path)
- })
- .on('end', function () {
- console.dir(items) // => [ ... array of files without directories]
- })
+ .on('data', item => items.push(item.path))
+ .on('end', () => console.dir(items)) // => [ ... array of files without directories]
```
### Searching Strategy
-Pass in options for `queueMethod` and `pathSorter` to affect how the file system
+Pass in options for `queueMethod`, `pathSorter`, and `depthLimit` to affect how the file system
is recursively iterated. See the code for more details, it's less than 50 lines :)
diff --git a/package.json b/package.json
index 9fa4f7d..9a85632 100644
--- a/package.json
+++ b/package.json
@@ -1,10 +1,10 @@
{
"name": "klaw",
- "version": "2.0.0",
+ "version": "2.1.1",
"description": "File system walker with Readable stream interface.",
"main": "./src/index.js",
"scripts": {
- "lint": "standard",
+ "lint": "standard && standard-markdown",
"test": "npm run lint && npm run unit",
"unit": "tape tests/**/*.js | tap-spec"
},
@@ -33,6 +33,7 @@
"mkdirp": "^0.5.1",
"rimraf": "^2.4.3",
"standard": "^10.0.2",
+ "standard-markdown": "^4.0.1",
"tap-spec": "^4.1.1",
"tape": "^4.2.2"
}
diff --git a/src/index.js b/src/index.js
index 0e5f869..0904b64 100644
--- a/src/index.js
+++ b/src/index.js
@@ -6,13 +6,14 @@ var util = require('util')
function Walker (dir, options) {
assert.strictEqual(typeof dir, 'string', '`dir` parameter should be of type string. Got type: ' + typeof dir)
var defaultStreamOptions = { objectMode: true }
- var defaultOpts = { queueMethod: 'shift', pathSorter: undefined, filter: undefined }
+ var defaultOpts = { queueMethod: 'shift', pathSorter: undefined, filter: undefined, depthLimit: undefined }
options = Object.assign(defaultOpts, options, defaultStreamOptions)
Readable.call(this, options)
this.root = path.resolve(dir)
this.paths = [this.root]
this.options = options
+ if (options.depthLimit > -1) this.rootDepth = this.root.split(path.sep).length + 1
this.fs = options.fs || require('graceful-fs')
}
util.inherits(Walker, Readable)
@@ -25,7 +26,11 @@ Walker.prototype._read = function () {
self.fs.lstat(pathItem, function (err, stats) {
var item = { path: pathItem, stats: stats }
if (err) return self.emit('error', err, item)
- if (!stats.isDirectory()) return self.push(item)
+
+ if (!stats.isDirectory() || (self.rootDepth &&
+ pathItem.split(path.sep).length - self.rootDepth >= self.options.depthLimit)) {
+ return self.push(item)
+ }
self.fs.readdir(pathItem, function (err, pathItems) {
if (err) {
@@ -36,7 +41,8 @@ Walker.prototype._read = function () {
pathItems = pathItems.map(function (part) { return path.join(pathItem, part) })
if (self.options.filter) pathItems = pathItems.filter(self.options.filter)
if (self.options.pathSorter) pathItems.sort(self.options.pathSorter)
- pathItems.forEach(function (pi) { self.paths.push(pi) })
+ // faster way to do do incremental batch array pushes
+ self.paths.push.apply(self.paths, pathItems)
self.push(item)
})
diff --git a/tests/walk_depth-limit.test.js b/tests/walk_depth-limit.test.js
new file mode 100644
index 0000000..207362f
--- /dev/null
+++ b/tests/walk_depth-limit.test.js
@@ -0,0 +1,60 @@
+var fs = require('fs')
+var mkdirp = require('mkdirp')
+var path = require('path')
+var test = require('./_test')
+var klaw = require('../')
+var fixtures = require('./fixtures')
+
+test('should honor depthLimit option -1', function (t, testDir) {
+ var expected = ['a', 'a/b', 'a/b/c', 'a/b/c/d.txt', 'a/e.jpg', 'h', 'h/i',
+ 'h/i/j', 'h/i/j/k.txt', 'h/i/l.txt', 'h/i/m.jpg']
+ run(t, testDir, -1, expected)
+})
+
+test('should honor depthLimit option 0', function (t, testDir) {
+ var expected = ['a', 'h']
+ run(t, testDir, 0, expected)
+})
+
+test('should honor depthLimit option 1', function (t, testDir) {
+ var expected = ['a', 'a/b', 'a/e.jpg', 'h', 'h/i']
+ run(t, testDir, 1, expected)
+})
+
+test('should honor depthLimit option 2', function (t, testDir) {
+ var expected = ['a', 'a/b', 'a/b/c', 'a/e.jpg', 'h', 'h/i', 'h/i/j',
+ 'h/i/l.txt', 'h/i/m.jpg']
+ run(t, testDir, 2, expected)
+})
+
+test('should honor depthLimit option 3', function (t, testDir) {
+ var expected = ['a', 'a/b', 'a/b/c', 'a/b/c/d.txt', 'a/e.jpg', 'h', 'h/i',
+ 'h/i/j', 'h/i/j/k.txt', 'h/i/l.txt', 'h/i/m.jpg']
+ run(t, testDir, 3, expected)
+})
+
+function run (t, testDir, depthLimit, expected) {
+ fixtures.forEach(function (f) {
+ f = path.join(testDir, f)
+ var dir = path.dirname(f)
+ mkdirp.sync(dir)
+ fs.writeFileSync(f, path.basename(f, path.extname(f)))
+ })
+
+ var items = []
+ klaw(testDir, { depthLimit: depthLimit })
+ .on('data', function (item) {
+ items.push(item.path)
+ })
+ .on('error', t.end)
+ .on('end', function () {
+ items.sort()
+ expected = expected.map(function (item) {
+ return path.join(path.join(testDir, item))
+ })
+ expected.unshift(testDir)
+
+ t.same(items, expected)
+ t.end()
+ })
+}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-javascript/node-klaw.git
More information about the Pkg-javascript-commits
mailing list