[Python-modules-commits] [zict] 01/03: Import zict_0.1.0.orig.tar.gz
Diane Trout
diane at moszumanska.debian.org
Wed Dec 14 06:09:45 UTC 2016
This is an automated email from the git hooks/post-receive script.
diane pushed a commit to branch master
in repository zict.
commit 253bdac907b7388b70a2094f242e70f615e8024c
Author: Diane Trout <diane at ghic.org>
Date: Tue Dec 13 21:21:24 2016 -0800
Import zict_0.1.0.orig.tar.gz
---
LICENSE.txt | 28 +++++++++
MANIFEST.in | 10 ++++
PKG-INFO | 17 ++++++
README.rst | 6 ++
requirements.txt | 1 +
setup.cfg | 5 ++
setup.py | 18 ++++++
zict.egg-info/PKG-INFO | 17 ++++++
zict.egg-info/SOURCES.txt | 29 +++++++++
zict.egg-info/dependency_links.txt | 1 +
zict.egg-info/not-zip-safe | 1 +
zict.egg-info/requires.txt | 1 +
zict.egg-info/top_level.txt | 1 +
zict/__init__.py | 9 +++
zict/buffer.py | 104 ++++++++++++++++++++++++++++++++
zict/common.py | 55 +++++++++++++++++
zict/file.py | 61 +++++++++++++++++++
zict/func.py | 89 +++++++++++++++++++++++++++
zict/lmdb.py | 103 +++++++++++++++++++++++++++++++
zict/lru.py | 105 ++++++++++++++++++++++++++++++++
zict/sieve.py | 104 ++++++++++++++++++++++++++++++++
zict/tests/__init__.py | 0
zict/tests/test_buffer.py | 74 +++++++++++++++++++++++
zict/tests/test_file.py | 69 +++++++++++++++++++++
zict/tests/test_func.py | 46 ++++++++++++++
zict/tests/test_lmdb.py | 69 +++++++++++++++++++++
zict/tests/test_lru.py | 95 +++++++++++++++++++++++++++++
zict/tests/test_sieve.py | 72 ++++++++++++++++++++++
zict/tests/test_zip.py | 84 ++++++++++++++++++++++++++
zict/tests/utils_test.py | 120 +++++++++++++++++++++++++++++++++++++
zict/zip.py | 85 ++++++++++++++++++++++++++
31 files changed, 1479 insertions(+)
diff --git a/LICENSE.txt b/LICENSE.txt
new file mode 100644
index 0000000..8e84973
--- /dev/null
+++ b/LICENSE.txt
@@ -0,0 +1,28 @@
+Copyright (c) 2016 Matthew Rocklin
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ a. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ b. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ c. Neither the name of toolz nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..69b6e72
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,10 @@
+recursive-include zict *.py
+recursive-include docs *.rst
+
+include setup.py
+include README.rst
+include LICENSE.txt
+include MANIFEST.in
+include requirements.txt
+
+prune docs/_build
diff --git a/PKG-INFO b/PKG-INFO
new file mode 100644
index 0000000..6a46334
--- /dev/null
+++ b/PKG-INFO
@@ -0,0 +1,17 @@
+Metadata-Version: 1.0
+Name: zict
+Version: 0.1.0
+Summary: Mutable mapping tools
+Home-page: http://github.com/mrocklin/zict/
+Author: Matthew Rocklin
+Author-email: mrocklin at gmail.com
+License: BSD
+Description: Zict
+ ====
+
+ |Build Status|
+
+ Mutable Mapping interfaces
+
+Keywords: mutable mapping dict
+Platform: UNKNOWN
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..0843012
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,6 @@
+Zict
+====
+
+|Build Status|
+
+Mutable Mapping interfaces
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..c6d4535
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+heapdict
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..861a9f5
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,5 @@
+[egg_info]
+tag_build =
+tag_date = 0
+tag_svn_revision = 0
+
diff --git a/setup.py b/setup.py
new file mode 100755
index 0000000..d770c88
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+import os
+from setuptools import setup
+
+setup(name='zict',
+ version='0.1.0',
+ description='Mutable mapping tools',
+ url='http://github.com/mrocklin/zict/',
+ maintainer='Matthew Rocklin',
+ maintainer_email='mrocklin at gmail.com',
+ license='BSD',
+ keywords='mutable mapping dict',
+ packages=['zict'],
+ install_requires=[open('requirements.txt').read().strip().split('\n')],
+ long_description=(open('README.rst').read() if os.path.exists('README.rst')
+ else ''),
+ zip_safe=False)
diff --git a/zict.egg-info/PKG-INFO b/zict.egg-info/PKG-INFO
new file mode 100644
index 0000000..6a46334
--- /dev/null
+++ b/zict.egg-info/PKG-INFO
@@ -0,0 +1,17 @@
+Metadata-Version: 1.0
+Name: zict
+Version: 0.1.0
+Summary: Mutable mapping tools
+Home-page: http://github.com/mrocklin/zict/
+Author: Matthew Rocklin
+Author-email: mrocklin at gmail.com
+License: BSD
+Description: Zict
+ ====
+
+ |Build Status|
+
+ Mutable Mapping interfaces
+
+Keywords: mutable mapping dict
+Platform: UNKNOWN
diff --git a/zict.egg-info/SOURCES.txt b/zict.egg-info/SOURCES.txt
new file mode 100644
index 0000000..7b06b2c
--- /dev/null
+++ b/zict.egg-info/SOURCES.txt
@@ -0,0 +1,29 @@
+LICENSE.txt
+MANIFEST.in
+README.rst
+requirements.txt
+setup.py
+zict/__init__.py
+zict/buffer.py
+zict/common.py
+zict/file.py
+zict/func.py
+zict/lmdb.py
+zict/lru.py
+zict/sieve.py
+zict/zip.py
+zict.egg-info/PKG-INFO
+zict.egg-info/SOURCES.txt
+zict.egg-info/dependency_links.txt
+zict.egg-info/not-zip-safe
+zict.egg-info/requires.txt
+zict.egg-info/top_level.txt
+zict/tests/__init__.py
+zict/tests/test_buffer.py
+zict/tests/test_file.py
+zict/tests/test_func.py
+zict/tests/test_lmdb.py
+zict/tests/test_lru.py
+zict/tests/test_sieve.py
+zict/tests/test_zip.py
+zict/tests/utils_test.py
\ No newline at end of file
diff --git a/zict.egg-info/dependency_links.txt b/zict.egg-info/dependency_links.txt
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/zict.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/zict.egg-info/not-zip-safe b/zict.egg-info/not-zip-safe
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/zict.egg-info/not-zip-safe
@@ -0,0 +1 @@
+
diff --git a/zict.egg-info/requires.txt b/zict.egg-info/requires.txt
new file mode 100644
index 0000000..c6d4535
--- /dev/null
+++ b/zict.egg-info/requires.txt
@@ -0,0 +1 @@
+heapdict
diff --git a/zict.egg-info/top_level.txt b/zict.egg-info/top_level.txt
new file mode 100644
index 0000000..d7049e0
--- /dev/null
+++ b/zict.egg-info/top_level.txt
@@ -0,0 +1 @@
+zict
diff --git a/zict/__init__.py b/zict/__init__.py
new file mode 100644
index 0000000..5f389cb
--- /dev/null
+++ b/zict/__init__.py
@@ -0,0 +1,9 @@
+from .zip import Zip
+from .file import File
+from .func import Func
+from .lru import LRU
+from .buffer import Buffer
+from .sieve import Sieve
+from .lmdb import LMDB
+
+__version__ = '0.1.0'
diff --git a/zict/buffer.py b/zict/buffer.py
new file mode 100644
index 0000000..da4f56c
--- /dev/null
+++ b/zict/buffer.py
@@ -0,0 +1,104 @@
+from itertools import chain
+
+from .common import ZictBase, close
+from .lru import LRU
+
+
+class Buffer(ZictBase):
+ """ Buffer one dictionary on top of another
+
+ This creates a MutableMapping by combining two MutableMappings, one that
+ feeds into the other when it overflows, based on an LRU mechanism. When
+ the first evicts elements these get placed into the second. When an item
+ is retrieved from the second it is placed back into the first.
+
+ Parameters
+ ----------
+ fast: MutableMapping
+ slow: MutableMapping
+
+ Examples
+ --------
+ >>> fast = dict()
+ >>> slow = Func(dumps, loads, File('storage/')) # doctest: +SKIP
+ >>> def weight(k, v):
+ ... return sys.getsizeof(v)
+ >>> buff = Buffer(fast, slow, 1e8, weight=weight) # doctest: +SKIP
+
+ See Also
+ --------
+ LRU
+ """
+ def __init__(self, fast, slow, n, weight=lambda k, v: 1):
+ self.fast = LRU(n, fast, weight=weight, on_evict=self.fast_to_slow)
+ self.slow = slow
+ self.n = n
+ self.weight = weight
+
+ def fast_to_slow(self, key, value):
+ self.slow[key] = value
+
+ def slow_to_fast(self, key):
+ value = self.slow[key]
+ # Avoid useless movement for heavy values
+ if self.weight(key, value) <= self.n:
+ del self.slow[key]
+ self.fast[key] = value
+ return value
+
+ def __getitem__(self, key):
+ if key in self.fast:
+ return self.fast[key]
+ elif key in self.slow:
+ return self.slow_to_fast(key)
+ else:
+ raise KeyError(key)
+
+ def __setitem__(self, key, value):
+ weight = self.weight(key, value)
+ # Avoid useless movement for heavy values
+ if self.weight(key, value) <= self.n:
+ if key in self.slow:
+ del self.slow[key]
+ self.fast[key] = value
+ else:
+ self.slow[key] = value
+
+ def __delitem__(self, key):
+ if key in self.fast:
+ del self.fast[key]
+ elif key in self.slow:
+ del self.slow[key]
+ else:
+ raise KeyError(key)
+
+ def keys(self):
+ return chain(self.fast.keys(), self.slow.keys())
+
+ def values(self):
+ return chain(self.fast.values(), self.slow.values())
+
+ def items(self):
+ return chain(self.fast.items(), self.slow.items())
+
+ def __len__(self):
+ return len(self.fast) + len(self.slow)
+
+ def __iter__(self):
+ return chain(iter(self.fast), iter(self.slow))
+
+ def __contains__(self, key):
+ return key in self.fast or key in self.slow
+
+ def __str__(self):
+ return 'Buffer<%s, %s>' % (str(self.fast), str(self.slow))
+
+ __repr__ = __str__
+
+ def flush(self):
+ self.fast.flush()
+ self.slow.flush()
+
+ def close(self):
+ close(self.fast)
+ close(self.slow)
diff --git a/zict/common.py b/zict/common.py
new file mode 100644
index 0000000..36adab6
--- /dev/null
+++ b/zict/common.py
@@ -0,0 +1,55 @@
+from __future__ import absolute_import, division, print_function
+
+from collections import Mapping, MutableMapping
+
+
+class ZictBase(MutableMapping):
+ """
+ Base class for zict mappings.
+ """
+
+ def update(*args, **kwds):
+ # Boilerplate for implementing an update() method
+ if not args:
+ raise TypeError("descriptor 'update' of MutableMapping object "
+ "needs an argument")
+ self = args[0]
+ args = args[1:]
+ if len(args) > 1:
+ raise TypeError('update expected at most 1 arguments, got %d' %
+ len(args))
+ items = []
+ if args:
+ other = args[0]
+ if isinstance(other, Mapping) or hasattr(other, "items"):
+ items += other.items()
+ else:
+ # Assuming (key, value) pairs
+ items += other
+ if kwds:
+ items += kwds.items()
+ self._do_update(items)
+
+ def _do_update(self, items):
+ # Default implementation, can be overriden for speed
+ for k, v in items:
+ self[k] = v
+
+ def close(self):
+ """
+ Release any system resources held by this object.
+ """
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.close()
+
+
+def close(z):
+ """
+ Close *z* if possible.
+ """
+ if hasattr(z, "close"):
+ z.close()
diff --git a/zict/file.py b/zict/file.py
new file mode 100644
index 0000000..48cec73
--- /dev/null
+++ b/zict/file.py
@@ -0,0 +1,61 @@
+from __future__ import absolute_import, division, print_function
+
+import errno
+import os
+
+from .common import ZictBase
+
+
+class File(ZictBase):
+ """ Mutable Mapping interface to a directory
+
+ Keys must be strings, values must be bytes
+
+ Parameters
+ ----------
+ directory: string
+ mode: string, ('r', 'w', 'a'), defaults to 'a'
+
+ Examples
+ --------
+ >>> z = File('myfile') # doctest: +SKIP
+ >>> z['x'] = b'123' # doctest: +SKIP
+ >>> z['x'] # doctest: +SKIP
+ b'123'
+ """
+ def __init__(self, directory, mode='a'):
+ self.directory = directory
+ self.mode = mode
+ if not os.path.exists(self.directory):
+ os.mkdir(self.directory)
+
+ def __getitem__(self, key):
+ try:
+ with open(os.path.join(self.directory, key), 'rb') as f:
+ result = f.read()
+ except EnvironmentError as e:
+ if e.args[0] != errno.ENOENT:
+ raise
+ raise KeyError(key)
+ return result
+
+ def __setitem__(self, key, value):
+ with open(os.path.join(self.directory, key), 'wb') as f:
+ f.write(value)
+
+ def keys(self):
+ return iter(os.listdir(self.directory))
+
+ def __iter__(self):
+ return self.keys()
+
+ def __delitem__(self, key):
+ try:
+ os.remove(os.path.join(self.directory, key))
+ except EnvironmentError as e:
+ if e.args[0] != errno.ENOENT:
+ raise
+ raise KeyError(key)
+
+ def __len__(self):
+ return sum(1 for _ in self.keys())
diff --git a/zict/func.py b/zict/func.py
new file mode 100644
index 0000000..6e09db4
--- /dev/null
+++ b/zict/func.py
@@ -0,0 +1,89 @@
+from __future__ import absolute_import, division, print_function
+
+from .common import ZictBase, close
+
+
+class Func(ZictBase):
+ """ Buffer a MutableMapping with a pair of input/output functions
+
+ Parameters
+ ----------
+ dump: callable
+ Function to call on value as we set it into the mapping
+ load: callable
+ Function to call on value as we pull it from the mapping
+ d: MutableMapping
+
+ Examples
+ --------
+ >>> def double(x):
+ ... return x * 2
+
+ >>> def halve(x):
+ ... return x / 2
+
+ >>> d = dict()
+ >>> f = Func(double, halve, d)
+ >>> f['x'] = 10
+ >>> d
+ {'x': 20}
+ >>> f['x']
+ 10.0
+ """
+ def __init__(self, dump, load, d):
+ self.dump = dump
+ self.load = load
+ self.d = d
+
+ def __getitem__(self, key):
+ return self.load(self.d[key])
+
+ def __setitem__(self, key, value):
+ self.d[key] = self.dump(value)
+
+ def __delitem__(self, key):
+ del self.d[key]
+
+ def keys(self):
+ return self.d.keys()
+
+ def values(self):
+ return map(self.load, self.d.values())
+
+ def items(self):
+ return ((k, self.load(v)) for k, v in self.d.items())
+
+ def _do_update(self, items):
+ self.d.update((k, self.dump(v)) for k, v in items)
+
+ def __iter__(self):
+ return iter(self.d)
+
+ def __len__(self):
+ return len(self.d)
+
+ def __str__(self):
+ return '%s<->%s: %s' % (funcname(self.dump),
+ funcname(self.load),
+ str(self.d))
+
+ def __repr__(self):
+ return '%s<->%s: %s' % (funcname(self.dump),
+ funcname(self.load),
+ repr(self.d))
+
+ def flush(self):
+ self.d.flush()
+
+ def close(self):
+ close(self.d)
+
+
+def funcname(func):
+ """Get the name of a function."""
+ while hasattr(func, 'func'):
+ func = func.func
+ try:
+ return func.__name__
+ except:
+ return str(func)
diff --git a/zict/lmdb.py b/zict/lmdb.py
new file mode 100644
index 0000000..2493bf9
--- /dev/null
+++ b/zict/lmdb.py
@@ -0,0 +1,103 @@
+from __future__ import absolute_import, division, print_function
+
+import os
+import sys
+
+from .common import ZictBase
+
+
+if sys.version_info >= (3,):
+ def _encode_key(key):
+ return key.encode('latin1')
+
+ def _decode_key(key):
+ return key.decode('latin1')
+
+else:
+ def _encode_key(key):
+ return key
+
+ def _decode_key(key):
+ return key
+
+
+class LMDB(ZictBase):
+ """ Mutable Mapping interface to a LMDB database.
+
+ Keys must be strings, values must be bytes
+
+ Parameters
+ ----------
+ directory: string
+
+ Examples
+ --------
+ >>> z = LMDB('/tmp/somedir/') # doctest: +SKIP
+ >>> z['x'] = b'123' # doctest: +SKIP
+ >>> z['x'] # doctest: +SKIP
+ b'123'
+ """
+ def __init__(self, directory):
+ import lmdb
+ # map_size is the maximum database size but shouldn't fill up the
+ # virtual address space
+ map_size = (1 << 40 if sys.maxsize >= 2**32 else 1 << 28)
+ # writemap requires sparse file support otherwise the whole
+ # `map_size` may be reserved up front on disk
+ writemap = sys.platform.startswith('linux')
+ self.db = lmdb.open(directory,
+ subdir=True,
+ map_size=map_size,
+ sync=False,
+ writemap=writemap,
+ )
+
+ def __getitem__(self, key):
+ with self.db.begin() as txn:
+ value = txn.get(_encode_key(key))
+ if value is None:
+ raise KeyError(key)
+ return value
+
+ def __setitem__(self, key, value):
+ with self.db.begin(write=True) as txn:
+ txn.put(_encode_key(key), value)
+
+ def __contains__(self, key):
+ with self.db.begin() as txn:
+ return txn.cursor().set_key(_encode_key(key))
+
+ def items(self):
+ cursor = self.db.begin().cursor()
+ return ((_decode_key(k), v)
+ for k, v in cursor.iternext(keys=True, values=True))
+
+ def keys(self):
+ cursor = self.db.begin().cursor()
+ return (_decode_key(k)
+ for k in cursor.iternext(keys=True, values=False))
+
+ def values(self):
+ cursor = self.db.begin().cursor()
+ return cursor.iternext(keys=False, values=True)
+
+ def _do_update(self, items):
+ # Optimized version of update() using a single putmulti() call.
+ items = [(_encode_key(k), v) for k, v in items]
+ with self.db.begin(write=True) as txn:
+ consumed, added = txn.cursor().putmulti(items)
+ assert consumed == added == len(items)
+
+ def __iter__(self):
+ return self.keys()
+
+ def __delitem__(self, key):
+ with self.db.begin(write=True) as txn:
+ if not txn.delete(_encode_key(key)):
+ raise KeyError(key)
+
+ def __len__(self):
+ return self.db.stat()['entries']
+
+ def close(self):
+ self.db.close()
diff --git a/zict/lru.py b/zict/lru.py
new file mode 100644
index 0000000..3b6524f
--- /dev/null
+++ b/zict/lru.py
@@ -0,0 +1,105 @@
+from __future__ import absolute_import, division, print_function
+
+from heapdict import heapdict
+
+from .common import ZictBase, close
+
+
+def do_nothing(k, v):
+ pass
+
+
+class LRU(ZictBase):
+ """ Evict Least Recently Used Elements
+
+ Parameters
+ ----------
+ n: int
+ Number of elements to keep, or total weight if weight= is used
+ d: MutableMapping
+ Dictionary in which to hold elements
+ on_evict: callable
+ Function:: k, v -> action to call on key value pairs prior to eviction
+ weight: callable
+ Function:: k, v -> number to determine the size of keeping the item in
+ the mapping. Defaults to ``(k, v) -> 1``
+
+ Examples
+ --------
+ >>> lru = LRU(2, dict(), on_evict=lambda k, v: print("Lost", k, v))
+ >>> lru['x'] = 1
+ >>> lru['y'] = 2
+ >>> lru['z'] = 3
+ Lost x 1
+ """
+ def __init__(self, n, d, on_evict=do_nothing, weight=lambda k, v: 1):
+ self.d = d
+ self.n = n
+ self.heap = heapdict()
+ self.i = 0
+ self.on_evict = on_evict
+ self.weight = weight
+ self.total_weight = 0
+ self.weights = dict()
+
+ def __getitem__(self, key):
+ result = self.d[key]
+ self.i += 1
+ self.heap[key] = self.i
+ return result
+
+ def __setitem__(self, key, value):
+ if key in self.d:
+ del self[key]
+
+ weight = self.weight(key, value)
+
+ if weight <= self.n:
+ self.d[key] = value
+ self.i += 1
+ self.heap[key] = self.i
+
+ self.weights[key] = weight
+ self.total_weight += weight
+ else:
+ self.on_evict(key, value)
+
+ while self.total_weight > self.n:
+ k, priority = self.heap.popitem()
+ self.total_weight -= self.weights.pop(k)
+ self.on_evict(k, self.d.pop(k))
+
+ def __delitem__(self, key):
+ del self.d[key]
+ del self.heap[key]
+ self.total_weight -= self.weights.pop(key)
+
+ def keys(self):
+ return self.d.keys()
+
+ def values(self):
+ return self.d.values()
+
+ def items(self):
+ return self.d.items()
+
+ def __len__(self):
+ return len(self.d)
+
+ def __iter__(self):
+ return iter(self.d)
+
+ def __contains__(self, key):
+ return key in self.d
+
+ def __str__(self):
+ return 'LRU: %s' % str(self.d)
+
+ def __repr__(self):
+ return 'LRU: %s' % repr(self.d)
+
+ def flush(self):
+ self.d.flush()
+
+ def close(self):
+ close(self.d)
diff --git a/zict/sieve.py b/zict/sieve.py
new file mode 100644
index 0000000..b0ba68f
--- /dev/null
+++ b/zict/sieve.py
@@ -0,0 +1,104 @@
+from __future__ import absolute_import, division, print_function
+
+from collections import defaultdict
+from itertools import chain
+import sys
+
+from .common import ZictBase, close
+
+
+class Sieve(ZictBase):
+ """ Store values in different mappings based on a selector's
+ output.
+
+ This creates a MutableMapping combining several underlying
+ MutableMappings for storage. Items are dispatched based on
+ a selector function provided by the user.
+
+ Parameters
+ ----------
+ mappings: dict of {mapping key: MutableMapping}
+ selector: callable (key, value) -> mapping key
+
+ Examples
+ --------
+ >>> small = {}
+ >>> large = DataBase() # doctest: +SKIP
+ >>> mappings = {True: small, False: large} # doctest: +SKIP
+ >>> def is_small(key, value): # doctest: +SKIP
+ return sys.getsizeof(value) < 10000
+ >>> d = Sieve(mappings, is_small) # doctest: +SKIP
+
+ See Also
+ --------
+ Buffer
+ """
+ def __init__(self, mappings, selector):
+ self.mappings = mappings
+ self.selector = selector
+ self.key_to_mapping = {}
+
+ def __getitem__(self, key):
+ return self.key_to_mapping[key][key]
+
+ def __setitem__(self, key, value):
+ old_mapping = self.key_to_mapping.get(key)
+ mapping = self.mappings[self.selector(key, value)]
+ if old_mapping is not None and old_mapping is not mapping:
+ del old_mapping[key]
+ mapping[key] = value
+ self.key_to_mapping[key] = mapping
+
+ def __delitem__(self, key):
+ del self.key_to_mapping.pop(key)[key]
+
+ def _do_update(self, items):
+ # Optimized update() implementation issuing a single update()
+ # call per underlying mapping.
+ to_delete = []
+ updates = defaultdict(list)
+ mapping_ids = dict((id(m), m) for m in self.mappings.values())
+
+ for key, value in items:
+ old_mapping = self.key_to_mapping.get(key)
+ mapping = self.mappings[self.selector(key, value)]
+ if old_mapping is not None and old_mapping is not mapping:
+ del old_mapping[key]
+ # Can't hash a mutable mapping, so use its id() instead
+ updates[id(mapping)].append((key, value))
+
+ for mid, mitems in updates.items():
+ mapping = mapping_ids[mid]
+ mapping.update(mitems)
+ for key, _ in mitems:
+ self.key_to_mapping[key] = mapping
+
+ def keys(self):
+ return chain.from_iterable(self.mappings.values())
+
+ def values(self):
+ return chain.from_iterable(m.values() for m in self.mappings.values())
+
+ def items(self):
+ return chain.from_iterable(m.items() for m in self.mappings.values())
+
+ def __len__(self):
+ return sum(map(len, self.mappings.values()))
+
+ __iter__ = keys
+
+ def __contains__(self, key):
+ return key in self.key_to_mapping
+
+ def __str__(self):
+ return 'Sieve<%s>' % (str(self.mappings),)
+
+ __repr__ = __str__
+
+ def flush(self):
+ for m in self.mappings.values():
+ m.flush()
+
+ def close(self):
+ for m in self.mappings.values():
+ close(m)
diff --git a/zict/tests/__init__.py b/zict/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/zict/tests/test_buffer.py b/zict/tests/test_buffer.py
new file mode 100644
index 0000000..63d81f9
--- /dev/null
+++ b/zict/tests/test_buffer.py
@@ -0,0 +1,74 @@
+from __future__ import absolute_import, division, print_function
+
+from zict import Buffer
+from . import utils_test
+
+
+def test_simple():
+ a = dict()
+ b = dict()
+ buff = Buffer(a, b, n=10, weight=lambda k, v: v)
+
+ buff['x'] = 1
+ buff['y'] = 2
+
+ assert buff['x'] == 1
+ assert buff['y'] == 2
+ assert a == {'x': 1, 'y': 2}
+ assert buff.fast.total_weight == 3
+
+ buff['z'] = 8
+ assert a == {'y': 2, 'z': 8}
+ assert b == {'x': 1}
+
+ assert buff['x'] == 1
+ assert a == {'x': 1, 'z': 8}
+ assert b == {'y': 2}
+
+ assert 'x' in buff
+ assert 'y' in buff
+ assert 'missing' not in buff
+
+ buff['y'] = 1
+ assert a == {'x': 1, 'y': 1, 'z': 8}
+ assert buff.fast.total_weight == 10
+ assert b == {}
+
+ del buff['z']
+ assert a == {'x': 1, 'y': 1}
+ assert buff.fast.total_weight == 2
+ assert b == {}
+
+ del buff['y']
+ assert a == {'x': 1}
+ assert buff.fast.total_weight == 1
+ assert b == {}
+
+ assert 'y' not in buff
+
+ buff['a'] = 5
+ assert set(buff) == set(buff.keys()) == {'a', 'x'}
+
+ fast_keys = set(buff.fast)
+ slow_keys = set(buff.slow)
+ assert not (fast_keys & slow_keys)
+ assert fast_keys | slow_keys == set(buff)
+
+ # Overweight element stays in slow mapping
+ buff['b'] = 1000
+ assert 'b' in buff.slow
+ assert set(buff.fast) == fast_keys
... 702 lines suppressed ...
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/zict.git
More information about the Python-modules-commits
mailing list