[Python-modules-commits] [zict] 01/03: Import zict_0.1.1.orig.tar.gz
Diane Trout
diane at moszumanska.debian.org
Tue Apr 25 02:59:29 UTC 2017
This is an automated email from the git hooks/post-receive script.
diane pushed a commit to branch master
in repository zict.
commit 973beb21fd68ae29690e01ace4c52796b17031b2
Author: Diane Trout <diane at ghic.org>
Date: Fri Apr 21 16:44:40 2017 -0700
Import zict_0.1.1.orig.tar.gz
---
PKG-INFO | 2 +-
setup.py | 2 +-
zict.egg-info/PKG-INFO | 2 +-
zict/__init__.py | 2 +-
zict/buffer.py | 21 ++++++++++++++--
zict/file.py | 62 ++++++++++++++++++++++++++++++++++-------------
zict/func.py | 16 ++++++------
zict/lru.py | 21 ++++++++++------
zict/tests/test_buffer.py | 37 ++++++++++++++++++++++++++++
zict/tests/test_file.py | 39 +++++++++++++++++++++++++++++
zict/tests/test_func.py | 4 +--
zict/tests/test_lru.py | 23 +++++++++++++++---
12 files changed, 187 insertions(+), 44 deletions(-)
diff --git a/PKG-INFO b/PKG-INFO
index 6a46334..0ed01da 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.0
Name: zict
-Version: 0.1.0
+Version: 0.1.1
Summary: Mutable mapping tools
Home-page: http://github.com/mrocklin/zict/
Author: Matthew Rocklin
diff --git a/setup.py b/setup.py
index d770c88..7fa2bfa 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@ import os
from setuptools import setup
setup(name='zict',
- version='0.1.0',
+ version='0.1.1',
description='Mutable mapping tools',
url='http://github.com/mrocklin/zict/',
maintainer='Matthew Rocklin',
diff --git a/zict.egg-info/PKG-INFO b/zict.egg-info/PKG-INFO
index 6a46334..0ed01da 100644
--- a/zict.egg-info/PKG-INFO
+++ b/zict.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.0
Name: zict
-Version: 0.1.0
+Version: 0.1.1
Summary: Mutable mapping tools
Home-page: http://github.com/mrocklin/zict/
Author: Matthew Rocklin
diff --git a/zict/__init__.py b/zict/__init__.py
index 5f389cb..8d717a6 100644
--- a/zict/__init__.py
+++ b/zict/__init__.py
@@ -6,4 +6,4 @@ from .buffer import Buffer
from .sieve import Sieve
from .lmdb import LMDB
-__version__ = '0.1.0'
+__version__ = '0.1.1'
diff --git a/zict/buffer.py b/zict/buffer.py
index da4f56c..1df21c9 100644
--- a/zict/buffer.py
+++ b/zict/buffer.py
@@ -16,6 +16,12 @@ class Buffer(ZictBase):
----------
fast: MutableMapping
slow: MutableMapping
+ fast_to_slow_callbacks: list of callables
+ These functions run every time data moves from the fast to the slow
+ mapping. They take two arguments, a key and a value
+ slow_to_fast_callbacks: list of callables
+ These functions run every time data moves form the slow to the fast
+ mapping.
Examples
--------
@@ -29,14 +35,23 @@ class Buffer(ZictBase):
--------
LRU
"""
- def __init__(self, fast, slow, n, weight=lambda k, v: 1):
- self.fast = LRU(n, fast, weight=weight, on_evict=self.fast_to_slow)
+ def __init__(self, fast, slow, n, weight=lambda k, v: 1,
+ fast_to_slow_callbacks=None, slow_to_fast_callbacks=None):
+ self.fast = LRU(n, fast, weight=weight, on_evict=[self.fast_to_slow])
self.slow = slow
self.n = n
self.weight = weight
+ if callable(fast_to_slow_callbacks):
+ fast_to_slow_callbacks = [fast_to_slow_callbacks]
+ if callable(slow_to_fast_callbacks):
+ slow_to_fast_callbacks = [slow_to_fast_callbacks]
+ self.fast_to_slow_callbacks = fast_to_slow_callbacks or []
+ self.slow_to_fast_callbacks = slow_to_fast_callbacks or []
def fast_to_slow(self, key, value):
self.slow[key] = value
+ for cb in self.fast_to_slow_callbacks:
+ cb(key, value)
def slow_to_fast(self, key):
value = self.slow[key]
@@ -44,6 +59,8 @@ class Buffer(ZictBase):
if self.weight(key, value) <= self.n:
del self.slow[key]
self.fast[key] = value
+ for cb in self.slow_to_fast_callbacks:
+ cb(key, value)
return value
def __getitem__(self, key):
diff --git a/zict/file.py b/zict/file.py
index 48cec73..0b45752 100644
--- a/zict/file.py
+++ b/zict/file.py
@@ -2,15 +2,37 @@ from __future__ import absolute_import, division, print_function
import errno
import os
+try:
+ from urllib.parse import quote, unquote
+except ImportError:
+ from urllib import quote, unquote
from .common import ZictBase
+def _safe_key(key):
+ """
+ Escape key so as to be usable on all filesystems.
+ """
+ # Even directory separators are unsafe.
+ return quote(key, safe='')
+
+
+def _unsafe_key(key):
+ """
+ Undo the escaping done by _safe_key().
+ """
+ return unquote(key)
+
+
class File(ZictBase):
""" Mutable Mapping interface to a directory
Keys must be strings, values must be bytes
+ Note this shouldn't be used for interprocess persistence, as keys
+ are cached in memory.
+
Parameters
----------
directory: string
@@ -26,36 +48,42 @@ class File(ZictBase):
def __init__(self, directory, mode='a'):
self.directory = directory
self.mode = mode
+ self._keys = set()
if not os.path.exists(self.directory):
os.mkdir(self.directory)
+ else:
+ for n in os.listdir(self.directory):
+ self._keys.add(_unsafe_key(n))
+
+ def __str__(self):
+ return '<File: %s, mode="%s", %d elements>' % (self.directory, self.mode, len(self))
+
+ __repr__ = __str__
def __getitem__(self, key):
- try:
- with open(os.path.join(self.directory, key), 'rb') as f:
- result = f.read()
- except EnvironmentError as e:
- if e.args[0] != errno.ENOENT:
- raise
+ if key not in self._keys:
raise KeyError(key)
- return result
+ with open(os.path.join(self.directory, _safe_key(key)), 'rb') as f:
+ return f.read()
def __setitem__(self, key, value):
- with open(os.path.join(self.directory, key), 'wb') as f:
+ with open(os.path.join(self.directory, _safe_key(key)), 'wb') as f:
f.write(value)
+ self._keys.add(key)
+
+ def __contains__(self, key):
+ return key in self._keys
def keys(self):
- return iter(os.listdir(self.directory))
+ return iter(self._keys)
- def __iter__(self):
- return self.keys()
+ __iter__ = keys
def __delitem__(self, key):
- try:
- os.remove(os.path.join(self.directory, key))
- except EnvironmentError as e:
- if e.args[0] != errno.ENOENT:
- raise
+ if key not in self._keys:
raise KeyError(key)
+ os.remove(os.path.join(self.directory, _safe_key(key)))
+ self._keys.remove(key)
def __len__(self):
- return sum(1 for _ in self.keys())
+ return len(self._keys)
diff --git a/zict/func.py b/zict/func.py
index 6e09db4..9795f21 100644
--- a/zict/func.py
+++ b/zict/func.py
@@ -41,6 +41,9 @@ class Func(ZictBase):
def __setitem__(self, key, value):
self.d[key] = self.dump(value)
+ def __contains__(self, key):
+ return key in self.d
+
def __delitem__(self, key):
del self.d[key]
@@ -63,14 +66,11 @@ class Func(ZictBase):
return len(self.d)
def __str__(self):
- return '%s<->%s: %s' % (funcname(self.dump),
- funcname(self.load),
- str(self.d))
-
- def __repr__(self):
- return '%s<->%s: %s' % (funcname(self.dump),
- funcname(self.load),
- repr(self.d))
+ return '<Func: %s<->%s %s>' % (funcname(self.dump),
+ funcname(self.load),
+ str(self.d))
+
+ __repr__ = __str__
def flush(self):
self.d.flush()
diff --git a/zict/lru.py b/zict/lru.py
index 3b6524f..1f63b35 100644
--- a/zict/lru.py
+++ b/zict/lru.py
@@ -18,7 +18,7 @@ class LRU(ZictBase):
Number of elements to keep, or total weight if weight= is used
d: MutableMapping
Dictionary in which to hold elements
- on_evict: callable
+ on_evict: list of callables
Function:: k, v -> action to call on key value pairs prior to eviction
weight: callable
Function:: k, v -> number to determine the size of keeping the item in
@@ -32,12 +32,14 @@ class LRU(ZictBase):
>>> lru['z'] = 3
Lost x 1
"""
- def __init__(self, n, d, on_evict=do_nothing, weight=lambda k, v: 1):
+ def __init__(self, n, d, on_evict=None, weight=lambda k, v: 1):
self.d = d
self.n = n
self.heap = heapdict()
self.i = 0
- self.on_evict = on_evict
+ if callable(on_evict):
+ on_evict = [on_evict]
+ self.on_evict = on_evict or []
self.weight = weight
self.total_weight = 0
self.weights = dict()
@@ -62,12 +64,15 @@ class LRU(ZictBase):
self.weights[key] = weight
self.total_weight += weight
else:
- self.on_evict(key, value)
+ for cb in self.on_evict:
+ cb(key, value)
while self.total_weight > self.n:
k, priority = self.heap.popitem()
self.total_weight -= self.weights.pop(k)
- self.on_evict(k, self.d.pop(k))
+ v = self.d.pop(k)
+ for cb in self.on_evict:
+ cb(k, v)
def __delitem__(self, key):
del self.d[key]
@@ -93,10 +98,10 @@ class LRU(ZictBase):
return key in self.d
def __str__(self):
- return 'LRU: %s' % str(self.d)
+ sub = str(self.d) if not isinstance(self.d, dict) else 'dict'
+ return '<LRU: %s/%s on %s>' % (self.total_weight, self.n, sub)
- def __repr__(self):
- return 'LRU: %s' % repr(self.d)
+ __repr__ = __str__
def flush(self):
self.d.flush()
diff --git a/zict/tests/test_buffer.py b/zict/tests/test_buffer.py
index 63d81f9..95afad2 100644
--- a/zict/tests/test_buffer.py
+++ b/zict/tests/test_buffer.py
@@ -72,3 +72,40 @@ def test_mapping():
buff = Buffer(a, b, n=2)
utils_test.check_mapping(buff)
utils_test.check_closing(buff)
+
+
+def test_callbacks():
+ f2s = []
+ def f2s_cb(k, v):
+ f2s.append(k)
+
+ s2f = []
+ def s2f_cb(k, v):
+ s2f.append(k)
+
+ a = dict()
+ b = dict()
+ buff = Buffer(a, b, n=10, weight=lambda k, v: v,
+ fast_to_slow_callbacks=f2s_cb,
+ slow_to_fast_callbacks=s2f_cb)
+
+ buff['x'] = 1
+ buff['y'] = 2
+
+ assert buff['x'] == 1
+ assert buff['y'] == 2
+ assert not f2s
+ assert not s2f
+
+ buff['z'] = 8
+
+ assert f2s == ['x']
+ assert s2f == []
+ buff['z']
+
+ assert f2s == ['x']
+ assert s2f == []
+
+ buff['x']
+ assert f2s == ['x', 'y']
+ assert s2f == ['x']
diff --git a/zict/tests/test_file.py b/zict/tests/test_file.py
index 61bcbd6..62fe887 100644
--- a/zict/tests/test_file.py
+++ b/zict/tests/test_file.py
@@ -38,6 +38,16 @@ def test_implementation(fn):
with open(os.path.join(fn, 'x'), 'rb') as f:
assert f.read() == b'123'
+ assert 'x' in z
+
+
+def test_str(fn):
+ z = File(fn)
+ assert fn in str(z)
+ assert fn in repr(z)
+ assert z.mode in str(z)
+ assert z.mode in repr(z)
+
def test_setitem_typeerror(fn):
z = File(fn)
@@ -67,3 +77,32 @@ def test_missing_key(fn):
with pytest.raises(KeyError):
z['x']
+
+
+def test_arbitrary_chars(fn):
+ z = File(fn)
+
+ # Avoid hitting the Windows max filename length
+ chunk = 16
+ for i in range(1, 128, chunk):
+ key = ''.join(['foo_'] + [chr(i) for i in range(i, min(128, i + chunk))])
+ with pytest.raises(KeyError):
+ z[key]
+ z[key] = b'foo'
+ assert z[key] == b'foo'
+ assert list(z) == [key]
+ assert list(z.keys()) == [key]
+ assert list(z.items()) == [(key, b'foo')]
+ assert list(z.values()) == [b'foo']
+
+ zz = File(fn)
+ assert zz[key] == b'foo'
+ assert list(zz) == [key]
+ assert list(zz.keys()) == [key]
+ assert list(zz.items()) == [(key, b'foo')]
+ assert list(zz.values()) == [b'foo']
+ del zz
+
+ del z[key]
+ with pytest.raises(KeyError):
+ z[key]
diff --git a/zict/tests/test_func.py b/zict/tests/test_func.py
index c95bbbe..53524ab 100644
--- a/zict/tests/test_func.py
+++ b/zict/tests/test_func.py
@@ -29,8 +29,8 @@ def test_simple():
assert list(f.values()) == [10]
assert list(f.items()) == [('x', 10)]
- assert all(s in str(f) for s in ['inc', 'dec', 'x'])
- assert all(s in repr(f) for s in ['inc', 'dec', 'x'])
+ assert all(s in str(f) for s in ['inc', 'dec', 'x', 'Func'])
+ assert all(s in repr(f) for s in ['inc', 'dec', 'x', 'Func'])
del f['x']
assert 'x' not in d
diff --git a/zict/tests/test_lru.py b/zict/tests/test_lru.py
index 9f064a3..b9ecef6 100644
--- a/zict/tests/test_lru.py
+++ b/zict/tests/test_lru.py
@@ -30,8 +30,20 @@ def test_simple():
lru['a'] = 5
assert set(lru.keys()) == set(['z', 'a'])
- assert 'a' in str(lru) and '5' in str(lru)
- assert 'a' in repr(lru) and '5' in repr(lru)
+
+def test_str():
+ d = dict()
+ lru = LRU(2, d)
+
+ lru['x'] = 1
+ lru['y'] = 2
+
+ assert str(lru.total_weight) in str(lru)
+ assert str(lru.total_weight) in repr(lru)
+ assert str(lru.n) in str(lru)
+ assert str(lru.n) in repr(lru)
+ assert 'dict' in str(lru)
+ assert 'dict' in repr(lru)
def test_mapping():
@@ -61,15 +73,20 @@ def test_overwrite():
def test_callbacks():
+ count = [0]
+ def cb(k, v):
+ count[0] += 1
+
L = list()
d = dict()
- lru = LRU(2, d, on_evict=lambda k, v: L.append((k, v)))
+ lru = LRU(2, d, on_evict=[lambda k, v: L.append((k, v)), cb])
lru['x'] = 1
lru['y'] = 2
lru['z'] = 3
assert L == [('x', 1)]
+ assert count[0] == len(L)
def test_weight():
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/zict.git
More information about the Python-modules-commits
mailing list