[Pkg-javascript-commits] [node-leveldown] 96/492: fixed snappy compression, added compression tests
Andrew Kelley
andrewrk-guest at moszumanska.debian.org
Sun Jul 6 17:13:48 UTC 2014
This is an automated email from the git hooks/post-receive script.
andrewrk-guest pushed a commit to annotated tag rocksdb-0.10.1
in repository node-leveldown.
commit 8da9356ad0fee901b39a2b0d71813f9c42ea20ba
Author: Rod Vagg <rod at vagg.org>
Date: Thu Dec 13 23:23:06 2012 +1100
fixed snappy compression, added compression tests
---
deps/leveldb/leveldb.gyp | 215 ++++++++++++++++++++++-------------------------
package.json | 1 +
test/compression-test.js | 52 ++++++++++++
3 files changed, 152 insertions(+), 116 deletions(-)
diff --git a/deps/leveldb/leveldb.gyp b/deps/leveldb/leveldb.gyp
index 7da6d34..6ff42d9 100644
--- a/deps/leveldb/leveldb.gyp
+++ b/deps/leveldb/leveldb.gyp
@@ -1,13 +1,18 @@
-# Originally taken from the Chromium source leveldatabase.gyp, adapted & simplified for node-levelup
-
-# Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file. See the AUTHORS file for names of contributors.
-
{
- 'target_defaults': {
+ 'targets': [{
+ 'target_name': 'leveldb',
+ 'type': 'static_library',
+ 'dependencies': [
+ "../snappy/snappy.gyp:snappy",
+ ],
+ 'direct_dependent_settings': {
+ 'include_dirs': [
+ 'leveldb-1.5.0/include/',
+ 'leveldb-1.5.0/',
+ ],
+ },
'defines': [
- 'USE_SNAPPY=1',
+ 'SNAPPY=1',
],
'include_dirs': [
'leveldb-1.5.0/',
@@ -52,111 +57,89 @@
],
}],
],
- },
- 'targets': [
- {
- 'target_name': 'leveldb',
- 'type': 'static_library',
- 'dependencies': [
- "../snappy/snappy.gyp:snappy",
- ],
- 'direct_dependent_settings': {
- 'include_dirs': [
- 'leveldb-1.5.0/include/',
- 'leveldb-1.5.0/',
- ],
- 'conditions': [
- ['OS == "win"', {
- 'include_dirs': [
- 'leveldb-1.5.0/port/win',
- ],
- }],
- ],
- },
- 'sources': [
- 'leveldb-1.5.0/db/builder.cc',
- 'leveldb-1.5.0/db/builder.h',
- 'leveldb-1.5.0/db/db_impl.cc',
- 'leveldb-1.5.0/db/db_impl.h',
- 'leveldb-1.5.0/db/db_iter.cc',
- 'leveldb-1.5.0/db/db_iter.h',
- 'leveldb-1.5.0/db/filename.cc',
- 'leveldb-1.5.0/db/filename.h',
- 'leveldb-1.5.0/db/dbformat.cc',
- 'leveldb-1.5.0/db/dbformat.h',
- 'leveldb-1.5.0/db/log_format.h',
- 'leveldb-1.5.0/db/log_reader.cc',
- 'leveldb-1.5.0/db/log_reader.h',
- 'leveldb-1.5.0/db/log_writer.cc',
- 'leveldb-1.5.0/db/log_writer.h',
- 'leveldb-1.5.0/db/memtable.cc',
- 'leveldb-1.5.0/db/memtable.h',
- 'leveldb-1.5.0/db/repair.cc',
- 'leveldb-1.5.0/db/skiplist.h',
- 'leveldb-1.5.0/db/snapshot.h',
- 'leveldb-1.5.0/db/table_cache.cc',
- 'leveldb-1.5.0/db/table_cache.h',
- 'leveldb-1.5.0/db/version_edit.cc',
- 'leveldb-1.5.0/db/version_edit.h',
- 'leveldb-1.5.0/db/version_set.cc',
- 'leveldb-1.5.0/db/version_set.h',
- 'leveldb-1.5.0/db/write_batch.cc',
- 'leveldb-1.5.0/db/write_batch_internal.h',
- 'leveldb-1.5.0/helpers/memenv/memenv.cc',
- 'leveldb-1.5.0/helpers/memenv/memenv.h',
- 'leveldb-1.5.0/include/leveldb/cache.h',
- 'leveldb-1.5.0/include/leveldb/comparator.h',
- 'leveldb-1.5.0/include/leveldb/db.h',
- 'leveldb-1.5.0/include/leveldb/env.h',
- 'leveldb-1.5.0/include/leveldb/filter_policy.h',
- 'leveldb-1.5.0/include/leveldb/iterator.h',
- 'leveldb-1.5.0/include/leveldb/options.h',
- 'leveldb-1.5.0/include/leveldb/slice.h',
- 'leveldb-1.5.0/include/leveldb/status.h',
- 'leveldb-1.5.0/include/leveldb/table.h',
- 'leveldb-1.5.0/include/leveldb/table_builder.h',
- 'leveldb-1.5.0/include/leveldb/write_batch.h',
- 'leveldb-1.5.0/port/port.h',
- 'leveldb-1.5.0/port/port_example.h',
- 'leveldb-1.5.0/port/port_posix.cc',
- 'leveldb-1.5.0/port/port_posix.h',
- 'leveldb-1.5.0/table/block.cc',
- 'leveldb-1.5.0/table/block.h',
- 'leveldb-1.5.0/table/block_builder.cc',
- 'leveldb-1.5.0/table/block_builder.h',
- 'leveldb-1.5.0/table/filter_block.cc',
- 'leveldb-1.5.0/table/filter_block.h',
- 'leveldb-1.5.0/table/format.cc',
- 'leveldb-1.5.0/table/format.h',
- 'leveldb-1.5.0/table/iterator.cc',
- 'leveldb-1.5.0/table/iterator_wrapper.h',
- 'leveldb-1.5.0/table/merger.cc',
- 'leveldb-1.5.0/table/merger.h',
- 'leveldb-1.5.0/table/table.cc',
- 'leveldb-1.5.0/table/table_builder.cc',
- 'leveldb-1.5.0/table/two_level_iterator.cc',
- 'leveldb-1.5.0/table/two_level_iterator.h',
- 'leveldb-1.5.0/util/arena.cc',
- 'leveldb-1.5.0/util/arena.h',
- 'leveldb-1.5.0/util/bloom.cc',
- 'leveldb-1.5.0/util/cache.cc',
- 'leveldb-1.5.0/util/coding.cc',
- 'leveldb-1.5.0/util/coding.h',
- 'leveldb-1.5.0/util/comparator.cc',
- 'leveldb-1.5.0/util/crc32c.cc',
- 'leveldb-1.5.0/util/crc32c.h',
- 'leveldb-1.5.0/util/env.cc',
- 'leveldb-1.5.0/util/env_posix.cc',
- 'leveldb-1.5.0/util/filter_policy.cc',
- 'leveldb-1.5.0/util/hash.cc',
- 'leveldb-1.5.0/util/hash.h',
- 'leveldb-1.5.0/util/logging.cc',
- 'leveldb-1.5.0/util/logging.h',
- 'leveldb-1.5.0/util/mutexlock.h',
- 'leveldb-1.5.0/util/options.cc',
- 'leveldb-1.5.0/util/random.h',
- 'leveldb-1.5.0/util/status.cc',
- ],
- },
- ],
-}
+ 'sources': [
+ 'leveldb-1.5.0/db/builder.cc',
+ 'leveldb-1.5.0/db/builder.h',
+ 'leveldb-1.5.0/db/db_impl.cc',
+ 'leveldb-1.5.0/db/db_impl.h',
+ 'leveldb-1.5.0/db/db_iter.cc',
+ 'leveldb-1.5.0/db/db_iter.h',
+ 'leveldb-1.5.0/db/filename.cc',
+ 'leveldb-1.5.0/db/filename.h',
+ 'leveldb-1.5.0/db/dbformat.cc',
+ 'leveldb-1.5.0/db/dbformat.h',
+ 'leveldb-1.5.0/db/log_format.h',
+ 'leveldb-1.5.0/db/log_reader.cc',
+ 'leveldb-1.5.0/db/log_reader.h',
+ 'leveldb-1.5.0/db/log_writer.cc',
+ 'leveldb-1.5.0/db/log_writer.h',
+ 'leveldb-1.5.0/db/memtable.cc',
+ 'leveldb-1.5.0/db/memtable.h',
+ 'leveldb-1.5.0/db/repair.cc',
+ 'leveldb-1.5.0/db/skiplist.h',
+ 'leveldb-1.5.0/db/snapshot.h',
+ 'leveldb-1.5.0/db/table_cache.cc',
+ 'leveldb-1.5.0/db/table_cache.h',
+ 'leveldb-1.5.0/db/version_edit.cc',
+ 'leveldb-1.5.0/db/version_edit.h',
+ 'leveldb-1.5.0/db/version_set.cc',
+ 'leveldb-1.5.0/db/version_set.h',
+ 'leveldb-1.5.0/db/write_batch.cc',
+ 'leveldb-1.5.0/db/write_batch_internal.h',
+ 'leveldb-1.5.0/helpers/memenv/memenv.cc',
+ 'leveldb-1.5.0/helpers/memenv/memenv.h',
+ 'leveldb-1.5.0/include/leveldb/cache.h',
+ 'leveldb-1.5.0/include/leveldb/comparator.h',
+ 'leveldb-1.5.0/include/leveldb/db.h',
+ 'leveldb-1.5.0/include/leveldb/env.h',
+ 'leveldb-1.5.0/include/leveldb/filter_policy.h',
+ 'leveldb-1.5.0/include/leveldb/iterator.h',
+ 'leveldb-1.5.0/include/leveldb/options.h',
+ 'leveldb-1.5.0/include/leveldb/slice.h',
+ 'leveldb-1.5.0/include/leveldb/status.h',
+ 'leveldb-1.5.0/include/leveldb/table.h',
+ 'leveldb-1.5.0/include/leveldb/table_builder.h',
+ 'leveldb-1.5.0/include/leveldb/write_batch.h',
+ 'leveldb-1.5.0/port/port.h',
+ 'leveldb-1.5.0/port/port_example.h',
+ 'leveldb-1.5.0/port/port_posix.cc',
+ 'leveldb-1.5.0/port/port_posix.h',
+ 'leveldb-1.5.0/table/block.cc',
+ 'leveldb-1.5.0/table/block.h',
+ 'leveldb-1.5.0/table/block_builder.cc',
+ 'leveldb-1.5.0/table/block_builder.h',
+ 'leveldb-1.5.0/table/filter_block.cc',
+ 'leveldb-1.5.0/table/filter_block.h',
+ 'leveldb-1.5.0/table/format.cc',
+ 'leveldb-1.5.0/table/format.h',
+ 'leveldb-1.5.0/table/iterator.cc',
+ 'leveldb-1.5.0/table/iterator_wrapper.h',
+ 'leveldb-1.5.0/table/merger.cc',
+ 'leveldb-1.5.0/table/merger.h',
+ 'leveldb-1.5.0/table/table.cc',
+ 'leveldb-1.5.0/table/table_builder.cc',
+ 'leveldb-1.5.0/table/two_level_iterator.cc',
+ 'leveldb-1.5.0/table/two_level_iterator.h',
+ 'leveldb-1.5.0/util/arena.cc',
+ 'leveldb-1.5.0/util/arena.h',
+ 'leveldb-1.5.0/util/bloom.cc',
+ 'leveldb-1.5.0/util/cache.cc',
+ 'leveldb-1.5.0/util/coding.cc',
+ 'leveldb-1.5.0/util/coding.h',
+ 'leveldb-1.5.0/util/comparator.cc',
+ 'leveldb-1.5.0/util/crc32c.cc',
+ 'leveldb-1.5.0/util/crc32c.h',
+ 'leveldb-1.5.0/util/env.cc',
+ 'leveldb-1.5.0/util/env_posix.cc',
+ 'leveldb-1.5.0/util/filter_policy.cc',
+ 'leveldb-1.5.0/util/hash.cc',
+ 'leveldb-1.5.0/util/hash.h',
+ 'leveldb-1.5.0/util/logging.cc',
+ 'leveldb-1.5.0/util/logging.h',
+ 'leveldb-1.5.0/util/mutexlock.h',
+ 'leveldb-1.5.0/util/options.cc',
+ 'leveldb-1.5.0/util/random.h',
+ 'leveldb-1.5.0/util/status.cc',
+ ],
+ }],
+}
\ No newline at end of file
diff --git a/package.json b/package.json
index 9569fc4..4e81f32 100644
--- a/package.json
+++ b/package.json
@@ -32,6 +32,7 @@
, "slow-stream": ">=0.0.3"
, "delayed": "*"
, "boganipsum": "*"
+ , "du": "*"
}
, "repository": {
"type": "git"
diff --git a/test/compression-test.js b/test/compression-test.js
new file mode 100644
index 0000000..fa15c06
--- /dev/null
+++ b/test/compression-test.js
@@ -0,0 +1,52 @@
+/* Copyright (c) 2012 Rod Vagg <@rvagg> */
+
+var buster = require('buster')
+ , assert = buster.assert
+ , async = require('async')
+ , du = require('du')
+ , delayed = require('delayed').delayed
+ , common = require('./common')
+
+ , compressableData = new Buffer(Array.apply(null, Array(1024 * 100)).map(function () { return 'aaaaaaaaaa' }).join(''))
+ , multiples = 10
+ , dataSize = compressableData.length * multiples
+
+ , verify = function(db, done) {
+ du(db._location, function (err, size) {
+ refute(err)
+ //console.log(Math.round((size / dataSize) * 100) + '% compression ratio (', size, 'b vs', dataSize, 'b)')
+ assert(size < dataSize, 'on-disk size (' + size + ') is less than data size (' + dataSize + ')')
+ done()
+ })
+ }
+
+buster.testCase('Compression', {
+ 'setUp': common.readStreamSetUp
+
+ , 'tearDown': common.commonTearDown
+
+ , 'test data is compressed by default (db.put())': function (done) {
+ this.openTestDatabase(function (db) {
+ async.forEach(
+ Array.apply(null, Array(multiples)).map(function (e, i) {
+ return [ i, compressableData ]
+ })
+ , function (args, callback) {
+ db.put.apply(db, args.concat([callback]))
+ }
+ , delayed(verify.bind(null, db, done), 0.1)
+ )
+ })
+ }
+
+ , '//test data is compressed by default (db.batch())': function (done) {
+ this.openTestDatabase(function (db) {
+ db.batch(
+ Array.apply(null, Array(multiples)).map(function (e, i) {
+ return { type: 'put', key: i, value: compressableData }
+ })
+ , delayed(verify.bind(null, db, done), 0.1)
+ )
+ })
+ }
+})
\ No newline at end of file
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-javascript/node-leveldown.git
More information about the Pkg-javascript-commits
mailing list