[Python-modules-commits] [pylibmc] 02/18: Import pylibmc_1.5.1.orig.tar.gz
Michael Fladischer
fladi at moszumanska.debian.org
Mon Jul 24 13:23:56 UTC 2017
This is an automated email from the git hooks/post-receive script.
fladi pushed a commit to branch master
in repository pylibmc.
commit d0a53287b0569ae242ee13f312f542270878c1f6
Author: Michael Fladischer <FladischerMichael at fladi.at>
Date: Fri Apr 1 21:31:50 2016 +0200
Import pylibmc_1.5.1.orig.tar.gz
---
LICENSE | 27 ++
PKG-INFO | 18 +-
README.rst | 16 +-
docs/_themes/sap/layout.html | 2 +-
docs/_themes/sap/static/main.css_t | 2 +-
docs/_themes/sap/theme.conf | 2 +-
docs/behaviors.rst | 11 +-
docs/changelog.rst | 10 +-
docs/coders.rst | 11 +
docs/conf.py | 2 +-
docs/reference.rst | 33 ++-
setup.py | 2 +-
src/_pylibmcmodule.c | 567 ++++++++++++++++++++++++++-----------
src/_pylibmcmodule.h | 30 +-
src/pylibmc-version.h | 2 +-
tests/__init__.py | 14 +-
tests/test_refcounts.py | 178 ++++++++++++
tests/test_serialization.py | 223 +++++++++++++++
18 files changed, 964 insertions(+), 186 deletions(-)
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..84549d3
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,27 @@
+Copyright (c) 2008, Ludvig Ericson
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ - Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+ - Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ - Neither the name of the author nor the names of the contributors may
+ be used to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/PKG-INFO b/PKG-INFO
index 4d9381b..9f33970 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: pylibmc
-Version: 1.5.0
+Version: 1.5.1
Summary: Quick and small memcached client for Python
Home-page: http://sendapatch.se/projects/pylibmc/
Author: Ludvig Ericson
@@ -19,7 +19,16 @@ Description: `pylibmc` is a Python client for `memcached <http://memcached.org/>
====================
This release fixes critical memory leaks in common code paths introduced in
- 1.4.2. Highly recommended.
+ 1.4.2. Also fixes a critical bug in a corner of the zlib inflation code, where
+ prior memory errors would trigger a double free. Thank you to everybody
+ involved in the making of this release, and especially `Eau de Web`__, without
+ their contributions, this release and the bug fixes it contains wouldn't have
+ been so expedient.
+
+ __ http://www.eaudeweb.ro/
+
+ .. comment: 1.5.x should have been an extension to 1.4.x, therefore it's best
+ to keep the 1.4.x release announcement below.
New in version 1.4.0
====================
@@ -42,6 +51,11 @@ Description: `pylibmc` is a Python client for `memcached <http://memcached.org/>
__ http://static.adzerk.net/Advertisers/5af77cf0094d4303bb308b955dd05992.jpg
__ bitcoin:12dveKhqiJWCY8zXT4kaHdHELXPeGAUo9h
+ License
+ =======
+
+ Released under the BSD 3-clause license; see `LICENSE <LICENSE>`_ for details.
+
Maintainer
==========
diff --git a/README.rst b/README.rst
index 3fcff93..986b58c 100644
--- a/README.rst
+++ b/README.rst
@@ -11,7 +11,16 @@ New in version 1.5.0
====================
This release fixes critical memory leaks in common code paths introduced in
-1.4.2. Highly recommended.
+1.4.2. Also fixes a critical bug in a corner of the zlib inflation code, where
+prior memory errors would trigger a double free. Thank you to everybody
+involved in the making of this release, and especially `Eau de Web`__, without
+their contributions, this release and the bug fixes it contains wouldn't have
+been so expedient.
+
+__ http://www.eaudeweb.ro/
+
+.. comment: 1.5.x should have been an extension to 1.4.x, therefore it's best
+ to keep the 1.4.x release announcement below.
New in version 1.4.0
====================
@@ -34,6 +43,11 @@ address `12dveKhqiJWCY8zXT4kaHdHELXPeGAUo9h`__.
__ http://static.adzerk.net/Advertisers/5af77cf0094d4303bb308b955dd05992.jpg
__ bitcoin:12dveKhqiJWCY8zXT4kaHdHELXPeGAUo9h
+License
+=======
+
+Released under the BSD 3-clause license; see `LICENSE <LICENSE>`_ for details.
+
Maintainer
==========
diff --git a/docs/_themes/sap/layout.html b/docs/_themes/sap/layout.html
index 1ae2d31..5378505 100644
--- a/docs/_themes/sap/layout.html
+++ b/docs/_themes/sap/layout.html
@@ -1,3 +1,3 @@
-{% extends "default/layout.html" %}
+{% extends "classic/layout.html" %}
{%- block relbar2 %}<!-- I hate crud -->{% endblock %}
diff --git a/docs/_themes/sap/static/main.css_t b/docs/_themes/sap/static/main.css_t
index 68cf490..f4d8d9c 100644
--- a/docs/_themes/sap/static/main.css_t
+++ b/docs/_themes/sap/static/main.css_t
@@ -9,7 +9,7 @@
*
*/
- at import url("default.css");
+ at import url("classic.css");
/* -- page layout ----------------------------------------------------------- */
diff --git a/docs/_themes/sap/theme.conf b/docs/_themes/sap/theme.conf
index 0fd8a9d..791afe5 100644
--- a/docs/_themes/sap/theme.conf
+++ b/docs/_themes/sap/theme.conf
@@ -1,5 +1,5 @@
[theme]
-inherit = default
+inherit = classic
stylesheet = main.css
pygments_style = sphinx
diff --git a/docs/behaviors.rst b/docs/behaviors.rst
index 619011d..8af506c 100644
--- a/docs/behaviors.rst
+++ b/docs/behaviors.rst
@@ -120,16 +120,17 @@ libmemcached behavior constants.
.. _remove_failed:
``"remove_failed"``
- Remove a server from the server list after operations on it have failed for
- the specified number of times in a row. See the section on Failover_.
+ If set, a server will be removed from the server list after this many
+ operations on it in a row have failed. See the section on Failover_.
.. _failure_limit:
``"failure_limit"`` : deprecated
- Use ``"remove_failed"`` if at all possible.
+ Use ``"remove_failed"`` if at all possible, which has the same meaning but
+ uses newer behaviour.
- Remove a server from the server list after operations on it have failed for
- the specified number of times in a row.
+ If set, a server will be removed from the server list after this many
+ operations on it in a row have failed.
.. _auto_eject:
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 68059d2..8709676 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -2,10 +2,16 @@ Change Log
==========
New in version 1.5.0
-====================
+--------------------
This release fixes critical memory leaks in common code paths introduced in
-1.4.2. Highly recommended.
+1.4.2. Also fixes a critical bug in a corner of the zlib inflation code, where
+prior memory errors would trigger a double free. Thank you to everybody
+involved in the making of this release, and especially `Eau de Web`__, without
+their contributions, this release and the bug fixes it contains wouldn't have
+been so expedient.
+
+__ http://www.eaudeweb.ro/
New in version 1.4.0
--------------------
diff --git a/docs/coders.rst b/docs/coders.rst
index eab5abf..0150de5 100644
--- a/docs/coders.rst
+++ b/docs/coders.rst
@@ -7,22 +7,33 @@ The List of Honored People
* Nikola Borisov <nikola.borisof at gmail.com>
* Rick Branson <rick at diodeware.com>
* Otto Bretz <otto.bretz at gmail.com>
+* Misha Brukman <mbrukman at google.com>
* James Brown <jbrown at yelp.com>
+* Adam Chainz <adam at adamj.eu>
+* Mika Eloranta <mel at ohmu.fi>
* Ludvig Ericson <ludvig at lericson.se>
+* Harvey Falcic <harvey.falcic at gmail.com>
+* Michael Fladischer <FladischerMichael at fladi.at>
+* Marius Gedminas <marius at gedmin.as>
* Joe Hansche <jhansche at myyearbook.com>
* Keli Hlodversson <keli at hapti.co>
* ketralnis <ketralnis at reddit.com>
* Paweł Kowalak <pawel.kowalak at gmail.com>
* Hiroki Kumzaki <hiroki.kumazaki at gmail.com>
+* Patrick Lewis <patrick.lewis at eventure.com>
+* Shivaram Lingamneni <slingamn at cs.stanford.edu>
* Andrew McFague <amcfague at wgen.net>
* Remoun Metyas <remoun.metyas at gmail.com>
+* Ed Morley <emorley at mozilla.com>
* Rudá Moura <ruda.moura at gmail.com>
* Muneyuki Noguchi <nogu.dev at gmail.com>
+* Sergey Pashinin <sergey at pashinin.com>
* Michael Schurter <schmichael at urbanairship.com>
* Radek Senfeld <rush at logic.cz>
* Noah Silas <noah at mahalo.com>
* Jari Sukanen <jari.sukanen at f-secure.com>
* John Watson <johnw at mahalo.com>
+* John Whitlock <john-whitlock at ieee.org>
* Neil Williams <neil at reddit.com>
* Josh Wright <jshwright at gmail.com>
* Kelly Wong <kelly at bluejeans.com>
diff --git a/docs/conf.py b/docs/conf.py
index b45a985..3c7c328 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -34,7 +34,7 @@ import pylibmc
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+needs_sphinx = '1.3'
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
diff --git a/docs/reference.rst b/docs/reference.rst
index 06d9034..f223b3e 100644
--- a/docs/reference.rst
+++ b/docs/reference.rst
@@ -2,7 +2,7 @@
Reference
===========
-.. class:: pylibmc.Client(servers[, binary=False, behaviors=None])
+.. class:: pylibmc.Client(servers[, binary=False, username=None, password=None, behaviors=None])
Interface to a set of memcached servers.
@@ -11,6 +11,9 @@
*binary* specifies whether or not to use the binary protocol to talk to the
memcached servers.
+ *username* and *password* are credentials for SASL authentication. It requires support
+ in libmemcached, and binary=True. Test for local support with pylibmc.support_sasl.
+
*behaviors*, if given, is passed to :meth:`Client.set_behaviors` after
initialization.
@@ -218,6 +221,34 @@
not a key exists depends on the version of libmemcached and memcached
used.
+ .. method:: serialize(value) -> bytestring, flag
+
+ Serialize a Python value to bytes *bytestring* and an integer *flag* field
+ for storage in memcached. The default implementation has special cases
+ for bytes, ints/longs, and bools, and falls back to pickle for all other
+ objects. Override this method to use a custom serialization format, or
+ otherwise modify the behavior.
+
+ *flag* is exposed by the memcached protocol. It adds flexibility
+ in terms of encoding schemes: for example, objects *a* and *b* of
+ different types may coincidentally encode to the same *bytestring*,
+ just so long as they encode with different values of *flag*. If distinct
+ values always encode to different byte strings (for example, when
+ serializing all values with pickle), *flag* can simply be set to a
+ constant.
+
+ .. method:: deserialize(bytestring, flag) -> value
+
+ Deserialize *bytestring*, stored with *flag*, back to a Python object.
+ Override this method (in concert with ``serialize``) to use a custom
+ serialization format, or otherwise modify the behavior.
+
+ Raise ``CacheMiss`` in order to simulate a cache miss for the relevant
+ key, i.e., ``get`` will return None and ``get_multi`` will omit the key
+ from the returned mapping. This can be used to recover gracefully from
+ version skew (e.g., retrieving a value that was pickled by a different,
+ incompatible code version).
+
.. data:: behaviors
The behaviors used by the underlying libmemcached object. See
diff --git a/setup.py b/setup.py
index 12688ad..7de01aa 100644
--- a/setup.py
+++ b/setup.py
@@ -74,7 +74,7 @@ if sys.platform == "darwin" and not os.environ.get("ARCHFLAGS"):
# There's a bug in <py3 with Py_True/False that will propagate with GCC's
# strict aliasing rules. Let's skip this flag for now.
-cflags = ["-fno-strict-aliasing", ]
+cflags = ["-fno-strict-aliasing", "-std=c99"]
## Extension definitions
diff --git a/src/_pylibmcmodule.c b/src/_pylibmcmodule.c
index b2d2b5c..1372144 100644
--- a/src/_pylibmcmodule.c
+++ b/src/_pylibmcmodule.c
@@ -79,11 +79,13 @@
#define PyBytes_GET_SIZE PyString_GET_SIZE
#define PyBytes_Size PyString_Size
#define PyLong_AS_LONG PyInt_AS_LONG
-#define PyNumber_Long PyNumber_Int
#define _PyBytes_Resize _PyString_Resize
#define PyObject_Bytes PyObject_Str
#endif
+/* Cache the values of {cP,p}ickle.{load,dump}s */
+static PyObject *_PylibMC_pickle_loads = NULL;
+static PyObject *_PylibMC_pickle_dumps = NULL;
/* {{{ Type methods */
static PylibMC_Client *PylibMC_ClientType_new(PyTypeObject *type,
@@ -102,6 +104,28 @@ static PylibMC_Client *PylibMC_ClientType_new(PyTypeObject *type,
return self;
}
+/* Helper: detect whether the serialize and deserialize methods were overridden. */
+static int _PylibMC_method_is_overridden(PylibMC_Client *self, const char *method) {
+ /* `self.__class__.serialize is _pylibmc.client.serialize`, in C */
+ PyObject *base_method = NULL, *current_class = NULL, *current_method = NULL;
+
+ base_method = PyObject_GetAttrString((PyObject *) &PylibMC_ClientType, method);
+ current_class = PyObject_GetAttrString((PyObject *) self, "__class__");
+ if (current_class != NULL) {
+ current_method = PyObject_GetAttrString(current_class, method);
+ }
+
+ Py_XDECREF(base_method);
+ Py_XDECREF(current_class);
+ Py_XDECREF(current_method);
+
+ if (base_method && current_class && current_method) {
+ return base_method == current_method;
+ } else {
+ return -1;
+ }
+}
+
static void PylibMC_ClientType_dealloc(PylibMC_Client *self) {
if (self->mc != NULL) {
#if LIBMEMCACHED_WITH_SASL_SUPPORT
@@ -180,6 +204,18 @@ static int PylibMC_Client_init(PylibMC_Client *self, PyObject *args,
}
}
+ /* Detect whether we should dispatch to user-modified Python
+ * serialization implementations. */
+ int native_serialization, native_deserialization;
+ if ((native_serialization = _PylibMC_method_is_overridden(self, "serialize")) == -1) {
+ goto error;
+ }
+ self->native_serialization = (uint8_t) native_serialization;
+ if ((native_deserialization = _PylibMC_method_is_overridden(self, "deserialize")) == -1) {
+ goto error;
+ }
+ self->native_deserialization = (uint8_t) native_deserialization;
+
while ((c_srv = PyIter_Next(srvs_it)) != NULL) {
unsigned char stype;
char *hostname;
@@ -397,16 +433,14 @@ static int _PylibMC_Inflate(char *value, size_t size,
}
/* Fall-through */
case Z_OK:
- if((tryrealloc = realloc(out, rvalsz << 1)) == NULL || errno == ENOMEM) {
+ if ((tryrealloc = realloc(out, rvalsz << 1)) == NULL) {
*failure_reason = "realloc";
rc = Z_MEM_ERROR;
goto zerror;
}
-
out = tryrealloc;
/* Wind forward */
-
strm.next_out = (unsigned char*)(out + rvalsz);
strm.avail_out = rvalsz;
rvalsz = rvalsz << 1;
@@ -422,15 +456,11 @@ static int _PylibMC_Inflate(char *value, size_t size,
goto error;
}
- tryrealloc = realloc(out, strm.total_out);
-
- if(tryrealloc == NULL || errno == ENOMEM) {
- /* we failed to *shrink* the value? */
+ if ((tryrealloc = realloc(out, strm.total_out)) == NULL) {
*failure_reason = "realloc";
rc = Z_MEM_ERROR;
goto error;
}
-
out = tryrealloc;
*result = out;
@@ -442,7 +472,7 @@ zerror:
inflateEnd(&strm);
error:
- if(out != NULL) {
+ if (out != NULL) {
free(out);
}
*result = NULL;
@@ -451,12 +481,19 @@ error:
#endif
/* }}} */
-/* {{{ str/bytes key functionality (unicode/bytes on Python 2) */
-static PyObject *_PylibMC_map_str_keys(PyObject *keys) {
+/* Helper for multiset / multiget:
+ 1. Take the iterable `keys` and build a map of UTF-8 encoded bytestrings
+ to Unicode keys.
+ 2. If `key_array` and `nkeys` are not NULL, additionally copy *new*
+ references to everything in the iterable into `key_array`. Store
+ the actual number of items in `nkeys`.
+*/
+static PyObject *_PylibMC_map_str_keys(PyObject *keys, PyObject **key_array, Py_ssize_t *nkeys) {
PyObject *key_str_map = NULL;
PyObject *iter = NULL;
PyObject *key = NULL;
PyObject *key_bytes = NULL;
+ Py_ssize_t i = 0;
key_str_map = PyDict_New();
if (key_str_map == NULL)
@@ -473,13 +510,26 @@ static PyObject *_PylibMC_map_str_keys(PyObject *keys) {
PyDict_SetItem(key_str_map, key_bytes, key);
Py_DECREF(key_bytes);
}
- Py_DECREF(key);
+ /* stash our owned reference to `key` in this array: */
+ if (key_array != NULL && i < *nkeys) {
+ key_array[i++] = key;
+ } else {
+ Py_DECREF(key);
+ }
+ }
+ if (nkeys != NULL) {
+ *nkeys = i;
}
Py_DECREF(iter);
return key_str_map;
cleanup:
+ if (key_array != NULL) {
+ for (Py_ssize_t j = 0; j < i; j++) {
+ Py_DECREF(key_array[j]);
+ }
+ }
Py_XDECREF(key);
Py_XDECREF(iter);
Py_XDECREF(key_str_map);
@@ -491,11 +541,9 @@ static void _PylibMC_cleanup_str_key_mapping(PyObject *key_str_map) {
}
/* }}} */
-static PyObject *_PylibMC_parse_memcached_value(char *value, size_t size,
- uint32_t flags) {
+static PyObject *_PylibMC_parse_memcached_value(PylibMC_Client *self,
+ char *value, size_t size, uint32_t flags) {
PyObject *retval = NULL;
- PyObject *tmp = NULL;
- uint32_t dtype = flags & PYLIBMC_FLAG_TYPES;
#if USE_ZLIB
PyObject *inflated = NULL;
@@ -551,65 +599,123 @@ static PyObject *_PylibMC_parse_memcached_value(char *value, size_t size,
}
#endif
+ if (self->native_deserialization) {
+ retval = _PylibMC_deserialize_native(self, NULL, value, size, flags);
+ } else {
+#if PY_MAJOR_VERSION >= 3
+ retval = PyObject_CallMethod((PyObject *)self, "deserialize", "y#I", value, size, (unsigned int) flags);
+#else
+ retval = PyObject_CallMethod((PyObject *)self, "deserialize", "s#I", value, size, (unsigned int) flags);
+#endif
+ }
+
+#if USE_ZLIB
+ Py_XDECREF(inflated);
+#endif
+
+
+ return retval;
+}
+
+/** Helper because PyLong_FromString requires a null-terminated string. */
+static PyObject *_PyLong_FromStringAndSize(char *value, size_t size, char **pend, int base) {
+ PyObject *retval;
+ char *tmp;
+ if ((tmp = malloc(size+1)) == NULL) {
+ return PyErr_NoMemory();
+ }
+ strncpy(tmp, value, size);
+ tmp[size] = '\0';
+ retval = PyLong_FromString(tmp, pend, base);
+ free(tmp);
+ return retval;
+}
+
+/** C implementation of deserialization.
+
+ This either takes a Python bytestring as `value`, or else `value` is NULL and
+ the value to be deserialized is a byte array `value_str` of length
+ `value_size`.
+ */
+static PyObject *_PylibMC_deserialize_native(PylibMC_Client *self, PyObject *value, char *value_str, size_t value_size, uint32_t flags) {
+ assert(value || value_str);
+ PyObject *retval = NULL;
+
+ uint32_t dtype = flags & PYLIBMC_FLAG_TYPES;
+
switch (dtype) {
case PYLIBMC_FLAG_PICKLE:
- retval = _PylibMC_Unpickle(value, size);
+ retval = value ? _PylibMC_Unpickle_Bytes(value) : _PylibMC_Unpickle(value_str, value_size);
break;
case PYLIBMC_FLAG_INTEGER:
case PYLIBMC_FLAG_LONG:
case PYLIBMC_FLAG_BOOL:
- /* PyInt_FromString doesn't take a length param and we're
- not NULL-terminated, so we'll have to make an
- intermediate Python string out of it */
- tmp = PyBytes_FromStringAndSize(value, size);
- if(tmp == NULL) {
- goto cleanup;
+ if (value) {
+ retval = PyLong_FromString(PyBytes_AS_STRING(value), NULL, 10);
+ } else {
+ retval = _PyLong_FromStringAndSize(value_str, value_size, NULL, 10);;
}
- retval = PyLong_FromString(PyBytes_AS_STRING(tmp), NULL, 10);
- if(retval != NULL && dtype == PYLIBMC_FLAG_BOOL) {
- Py_DECREF(tmp);
- tmp = retval;
- retval = PyBool_FromLong(PyLong_AS_LONG(tmp));
+ if (retval != NULL && dtype == PYLIBMC_FLAG_BOOL) {
+ PyObject *bool_retval = PyBool_FromLong(PyLong_AS_LONG(retval));
+ Py_DECREF(retval);
+ retval = bool_retval;
}
break;
case PYLIBMC_FLAG_NONE:
- retval = PyBytes_FromStringAndSize(value, (Py_ssize_t)size);
+ if (value) {
+ /* acquire an additional reference for parity */
+ Py_INCREF(value);
+ retval = value;
+ } else {
+ retval = PyBytes_FromStringAndSize(value_str, value_size);
+ }
break;
default:
PyErr_Format(PylibMCExc_Error,
- "unknown memcached key flags %u", flags);
+ "unknown memcached key flags %u", dtype);
}
-cleanup:
-
-#if USE_ZLIB
- Py_XDECREF(inflated);
-#endif
-
- Py_XDECREF(tmp);
-
return retval;
}
-static PyObject *_PylibMC_parse_memcached_result(memcached_result_st *res) {
- return _PylibMC_parse_memcached_value((char *)memcached_result_value(res),
+static PyObject *PylibMC_Client_deserialize(PylibMC_Client *self, PyObject *args) {
+ PyObject *value;
+ unsigned int flags;
+ if (!PyArg_ParseTuple(args, "OI", &value, &flags)) {
+ return NULL;
+ }
+ return _PylibMC_deserialize_native(self, value, NULL, 0, flags);
+}
+
+static PyObject *_PylibMC_parse_memcached_result(PylibMC_Client *self, memcached_result_st *res) {
+ return _PylibMC_parse_memcached_value(
+ self,
+ (char *)memcached_result_value(res),
memcached_result_length(res),
memcached_result_flags(res));
}
+/* Helper to call after _PylibMC_parse_memcached_value;
+ determines whether the deserialized value should be ignored
+ and treated as a miss.
+*/
+static int _PylibMC_cache_miss_simulated(PyObject *r) {
+ if (r == NULL && PyErr_Occurred() && PyErr_ExceptionMatches(PylibMCExc_CacheMiss)) {
+ PyErr_Clear();
+ return 1;
+ }
+ return 0;
+}
+
static PyObject *PylibMC_Client_get(PylibMC_Client *self, PyObject *arg) {
char *mc_val;
size_t val_size;
uint32_t flags;
memcached_return error;
- Py_INCREF(arg);
-
if (!_key_normalized_obj(&arg)) {
- Py_DECREF(arg);
return NULL;
} else if (!PySequence_Length(arg) ) {
- Py_DECREF(arg);
Py_RETURN_NONE;
}
@@ -622,8 +728,13 @@ static PyObject *PylibMC_Client_get(PylibMC_Client *self, PyObject *arg) {
Py_DECREF(arg);
if (mc_val != NULL) {
- PyObject *r = _PylibMC_parse_memcached_value(mc_val, val_size, flags);
+ PyObject *r = _PylibMC_parse_memcached_value(self, mc_val, val_size, flags);
free(mc_val);
+ if (_PylibMC_cache_miss_simulated(r)) {
+ /* Since python-memcache returns None when the key doesn't exist,
+ * so shall we. */
+ Py_RETURN_NONE;
+ }
return r;
} else if (error == MEMCACHED_SUCCESS) {
/* This happens for empty values, and so we fake an empty string. */
@@ -646,10 +757,7 @@ static PyObject *PylibMC_Client_gets(PylibMC_Client *self, PyObject *arg) {
memcached_return rc;
PyObject* ret = NULL;
- Py_INCREF(arg);
-
if (!_key_normalized_obj(&arg)) {
- Py_DECREF(arg);
return NULL;
} else if (!PySequence_Length(arg)) {
return Py_BuildValue("(OO)", Py_None, Py_None);
@@ -674,25 +782,37 @@ static PyObject *PylibMC_Client_gets(PylibMC_Client *self, PyObject *arg) {
Py_END_ALLOW_THREADS;
+ int miss = 0;
+ int fail = 0;
if (rc == MEMCACHED_SUCCESS && res != NULL) {
- ret = Py_BuildValue("(NL)",
- _PylibMC_parse_memcached_result(res),
- memcached_result_cas(res));
+ PyObject *val = _PylibMC_parse_memcached_result(self, res);
+ if (_PylibMC_cache_miss_simulated(val)) {
+ miss = 1;
+ } else {
+ ret = Py_BuildValue("(NL)",
+ val,
+ memcached_result_cas(res));
+ }
/* we have to fetch the last result from the mget cursor */
if (NULL != memcached_fetch_result(self->mc, NULL, &rc)) {
memcached_quit(self->mc);
Py_DECREF(ret);
ret = NULL;
+ fail = 1;
PyErr_SetString(PyExc_RuntimeError, "fetch not done");
}
} else if (rc == MEMCACHED_END || rc == MEMCACHED_NOTFOUND) {
- /* Key not found => (None, None) */
- ret = Py_BuildValue("(OO)", Py_None, Py_None);
+ miss = 1;
} else {
ret = PylibMC_ErrFromMemcached(self, "memcached_gets", rc);
}
+ if (miss && !fail) {
+ /* Key not found => (None, None) */
+ ret = Py_BuildValue("(OO)", Py_None, Py_None);
+ }
+
if (res != NULL) {
memcached_result_free(res);
}
@@ -768,7 +888,7 @@ static PyObject *_PylibMC_RunSetCommandSingle(PylibMC_Client *self,
*/
key = PyBytes_FromStringAndSize(key_raw, keylen);
- success = _PylibMC_SerializeValue(key, NULL, value, time, &serialized);
+ success = _PylibMC_SerializeValue(self, key, NULL, value, time, &serialized);
if (!success)
goto cleanup;
@@ -837,7 +957,7 @@ static PyObject *_PylibMC_RunSetCommandMulti(PylibMC_Client *self,
nkeys = (size_t)PyDict_Size(keys);
- key_str_map = _PylibMC_map_str_keys(keys);
+ key_str_map = _PylibMC_map_str_keys(keys, NULL, NULL);
if (key_str_map == NULL) {
goto cleanup;
}
@@ -852,7 +972,7 @@ static PyObject *_PylibMC_RunSetCommandMulti(PylibMC_Client *self,
}
for (i = 0, idx = 0; PyDict_Next(keys, &i, &curr_key, &curr_value); idx++) {
- int success = _PylibMC_SerializeValue(curr_key, key_prefix,
+ int success = _PylibMC_SerializeValue(self, curr_key, key_prefix,
curr_value, time,
&serialized[idx]);
@@ -933,7 +1053,7 @@ static PyObject *_PylibMC_RunCasCommand(PylibMC_Client *self,
/* TODO: because it's RunSetCommand that does the zlib
compression, cas can't currently use compressed values. */
- success = _PylibMC_SerializeValue(key, NULL, value, time, &mset);
+ success = _PylibMC_SerializeValue(self, key, NULL, value, time, &mset);
if (!success || PyErr_Occurred() != NULL) {
goto cleanup;
@@ -979,25 +1099,21 @@ static void _PylibMC_FreeMset(pylibmc_mset *mset) {
mset->value_obj = NULL;
}
-static int _PylibMC_SerializeValue(PyObject* key_obj,
+static int _PylibMC_SerializeValue(PylibMC_Client *self,
+ PyObject* key_obj,
PyObject* key_prefix,
PyObject* value_obj,
time_t time,
pylibmc_mset* serialized) {
- PyObject* store_val = NULL;
/* first zero the whole structure out */
memset((void *)serialized, 0x0, sizeof(pylibmc_mset));
serialized->time = time;
serialized->success = false;
- serialized->flags = PYLIBMC_FLAG_NONE;
-
- /* Make an owned reference to key_obj */
- Py_INCREF(key_obj);
+ serialized->value_obj = NULL;
if (!_key_normalized_obj(&key_obj)) {
- Py_DECREF(key_obj);
return false;
}
@@ -1012,10 +1128,7 @@ static int _PylibMC_SerializeValue(PyObject* key_obj,
/* Check the key_prefix */
if (key_prefix != NULL) {
- Py_INCREF(key_prefix);
-
if (!_key_normalized_obj(&key_prefix)) {
- Py_DECREF(key_prefix);
return false;
}
@@ -1052,32 +1165,101 @@ static int _PylibMC_SerializeValue(PyObject* key_obj,
serialized->prefixed_key_obj = prefixed_key_obj;
}
- /* Build store_val, a Python str/bytes object */
+ int success;
+ /* Build serialized->value_obj, a Python str/bytes object. */
+ if (self->native_serialization) {
+ success = _PylibMC_serialize_native(self, value_obj, &(serialized->value_obj), &(serialized->flags));
+ } else {
+ success = _PylibMC_serialize_user(self, value_obj, &(serialized->value_obj), &(serialized->flags));
+ }
+
+ if (!success) {
+ return false;
+ }
+
+ if (PyBytes_AsStringAndSize(serialized->value_obj, &serialized->value,
+ &serialized->value_len) == -1) {
+ return false;
+ }
+
+ return true;
+}
+
+static int _PylibMC_serialize_user(PylibMC_Client *self, PyObject *value_obj, PyObject **dest, uint32_t *flags) {
+ PyObject *serval_and_flags = PyObject_CallMethod((PyObject *)self, "serialize", "(O)", value_obj);
+ if (serval_and_flags == NULL) {
+ return false;
+ }
+
+ if (PyTuple_Check(serval_and_flags)) {
+ PyObject *flags_obj = PyTuple_GetItem(serval_and_flags, 1);
+ if (flags_obj != NULL) {
+#if PY_MAJOR_VERSION >= 3
+ if (PyLong_Check(flags_obj)) {
+ *flags = (uint32_t) PyLong_AsLong(flags_obj);
+ *dest = PyTuple_GetItem(serval_and_flags, 0);
+ }
+#else
+ if (PyInt_Check(flags_obj)) {
+ *flags = (uint32_t) PyInt_AsLong(flags_obj);
+ *dest = PyTuple_GetItem(serval_and_flags, 0);
+ }
+#endif
+ }
+ }
+
+ if (*dest == NULL) {
+ /* PyErr_SetObject(PyExc_ValueError, serval_and_flags); */
+ PyErr_SetString(PyExc_ValueError, "serialize() must return (bytes, flags)");
+ Py_DECREF(serval_and_flags);
+ return false;
+ } else {
+ /* We're getting rid of serval_and_flags, which owns the only new
+ reference to value_obj. However, we can't deallocate value_obj
+ until we're done with value and value_len (after the set
+ operation). Therefore, take possession of a new reference to it
+ before cleaning up the tuple: */
+ Py_INCREF(*dest);
+ Py_DECREF(serval_and_flags);
+ }
+
+ return true;
+}
+
+/** C implementation of serialization.
+
+ This either returns true and stores the serialized bytestring in *dest
+ and the flags in *flags, or it returns false with an exception set.
+*/
+static int _PylibMC_serialize_native(PylibMC_Client *self, PyObject *value_obj, PyObject **dest, uint32_t *flags) {
+ PyObject *store_val = NULL;
+ uint32_t store_flags = PYLIBMC_FLAG_NONE;
if (PyBytes_Check(value_obj)) {
+ store_flags = PYLIBMC_FLAG_NONE;
/* Make store_val an owned reference */
store_val = value_obj;
Py_INCREF(store_val);
-#if PY_MAJOR_VERSION >= 3
} else if (PyBool_Check(value_obj)) {
- serialized->flags |= PYLIBMC_FLAG_BOOL;
- store_val = PyBytes_FromFormat("%ld", PyLong_AsLong(value_obj));
+ store_flags |= PYLIBMC_FLAG_BOOL;
+ /* bool cannot be subclassed; there are only two singleton values,
+ Py_True and Py_False */
+ const char *value_str = (value_obj == Py_True) ? "1" : "0";
+ store_val = PyBytes_FromString(value_str);
+#if PY_MAJOR_VERSION >= 3
} else if (PyLong_Check(value_obj)) {
- serialized->flags |= PYLIBMC_FLAG_LONG;
- store_val = PyBytes_FromFormat("%ld", PyLong_AsLong(value_obj));
-#else
- } else if (PyBool_Check(value_obj)) {
- serialized->flags |= PYLIBMC_FLAG_BOOL;
- PyObject* tmp = PyNumber_Long(value_obj);
- store_val = PyObject_Bytes(tmp);
+ store_flags |= PYLIBMC_FLAG_LONG;
+ PyObject *tmp = PyObject_Str(value_obj);
+ store_val = PyUnicode_AsEncodedString(tmp, "ascii", "strict");
Py_DECREF(tmp);
+#else
} else if (PyInt_Check(value_obj)) {
- serialized->flags |= PYLIBMC_FLAG_INTEGER;
+ store_flags |= PYLIBMC_FLAG_INTEGER;
PyObject* tmp = PyNumber_Int(value_obj);
store_val = PyObject_Bytes(tmp);
Py_DECREF(tmp);
} else if (PyLong_Check(value_obj)) {
- serialized->flags |= PYLIBMC_FLAG_LONG;
+ store_flags |= PYLIBMC_FLAG_LONG;
PyObject* tmp = PyNumber_Long(value_obj);
store_val = PyObject_Bytes(tmp);
Py_DECREF(tmp);
@@ -1085,7 +1267,7 @@ static int _PylibMC_SerializeValue(PyObject* key_obj,
} else if (value_obj != NULL) {
/* we have no idea what it is, so we'll store it pickled */
Py_INCREF(value_obj);
- serialized->flags |= PYLIBMC_FLAG_PICKLE;
+ store_flags |= PYLIBMC_FLAG_PICKLE;
store_val = _PylibMC_Pickle(value_obj);
Py_DECREF(value_obj);
}
@@ -1094,16 +1276,19 @@ static int _PylibMC_SerializeValue(PyObject* key_obj,
return false;
}
- /* store_val is an owned reference; released when we're done with value and
- * value_len (i.e. not here.) */
- serialized->value_obj = store_val;
+ *dest = store_val;
+ *flags = store_flags;
+ return true;
+}
- if (PyBytes_AsStringAndSize(store_val, &serialized->value,
- &serialized->value_len) == -1) {
- return false;
+static PyObject *PylibMC_Client_serialize(PylibMC_Client *self, PyObject *value_obj) {
+ PyObject *serialized_val;
+ uint32_t flags;
+ if (!_PylibMC_serialize_native(self, value_obj, &serialized_val, &flags)) {
+ return NULL;
}
-
- return true;
+ /* we own a reference to serialized_val. "give" it to the tuple return value: */
+ return Py_BuildValue("(NI)", serialized_val, flags);
}
/* {{{ Set commands (set, replace, add, prepend, append) */
@@ -1562,17 +1747,17 @@ memcached_return pylibmc_memcached_fetch_multi(memcached_st *mc, pylibmc_mget_re
static PyObject *PylibMC_Client_get_multi(
PylibMC_Client *self, PyObject *args, PyObject *kwds) {
- PyObject *key_seq, **key_objs, *retval = NULL;
+ PyObject *key_seq, **key_objs, **orig_key_objs, *retval = NULL;
char **keys, *prefix = NULL;
char *err_func = NULL;
memcached_result_st *res, *results = NULL;
Py_ssize_t prefix_len = 0;
Py_ssize_t i;
- PyObject *key_it, *ckey;
PyObject *key_str_map = NULL;
PyObject *temp_key_obj;
size_t *key_lens;
- size_t nkeys, nresults = 0;
+ Py_ssize_t nkeys = 0, orig_nkeys = 0;
+ size_t nresults = 0;
memcached_return rc;
pylibmc_mget_req req;
@@ -1582,46 +1767,50 @@ static PyObject *PylibMC_Client_get_multi(
&key_seq, &prefix, &prefix_len))
return NULL;
- if ((nkeys = (size_t)PySequence_Length(key_seq)) == -1)
+ if ((orig_nkeys = PySequence_Length(key_seq)) == -1)
return NULL;
/* Populate keys and key_lens. */
- keys = PyMem_New(char *, nkeys);
- key_lens = PyMem_New(size_t, nkeys);
- key_objs = PyMem_New(PyObject *, nkeys);
- if (!keys || !key_lens || !key_objs) {
- PyMem_Free(keys);
- PyMem_Free(key_lens);
... 887 lines suppressed ...
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/pylibmc.git
More information about the Python-modules-commits
mailing list