[Python-modules-commits] [django-q] 02/10: New upstream version 0.9.1
Pierre-Elliott Bécue
peb-guest at moszumanska.debian.org
Tue Feb 6 08:37:55 UTC 2018
This is an automated email from the git hooks/post-receive script.
peb-guest pushed a commit to branch master
in repository django-q.
commit 8555cfae6458bae7980204b6bd673a44999b86ae
Author: Pierre-Elliott Bécue <becue at crans.org>
Date: Mon Feb 5 23:29:15 2018 +0100
New upstream version 0.9.1
---
.travis.yml | 5 ++-
README.rst | 2 +-
django_q/__init__.py | 7 ++--
django_q/brokers/aws_sqs.py | 9 +++--
django_q/cluster.py | 4 +--
django_q/conf.py | 13 ++++---
django_q/core_signing.py | 76 ++++++++++++++++++++++++++++++++++++++++
django_q/models.py | 8 +++--
django_q/queues.py | 69 ++++++++++++++++++++++++++++++++++++
django_q/signing.py | 2 +-
django_q/tasks.py | 3 +-
django_q/tests/settings.py | 5 ++-
django_q/tests/test_admin.py | 7 ++--
django_q/tests/test_cached.py | 3 +-
django_q/tests/test_cluster.py | 3 +-
django_q/tests/test_scheduler.py | 3 +-
django_q/tests/urls.py | 4 +--
docs/conf.py | 6 ++--
docs/configure.rst | 8 ++---
docs/index.rst | 4 +--
docs/install.rst | 7 ++--
requirements.txt | 16 ++++-----
setup.py | 6 ++--
23 files changed, 217 insertions(+), 53 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index a0a5579..269d24f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,12 +5,11 @@ services:
- mongodb
python:
- - "2.7"
- "3.6"
env:
- - DJANGO=1.11.6
- - DJANGO=1.10.8
+ - DJANGO=2.0
+ - DJANGO=1.11.9
- DJANGO=1.8.18
sudo: false
diff --git a/README.rst b/README.rst
index d84f2de..5048359 100644
--- a/README.rst
+++ b/README.rst
@@ -29,7 +29,7 @@ Requirements
- `Arrow <https://github.com/crsmithdev/arrow>`__
- `Blessed <https://github.com/jquast/blessed>`__
-Tested with: Python 2.7 & 3.6. Django 1.8.18, 1.10.8 and 1.11.6
+Tested with: Python 2.7 & 3.6. Django 1.8.18, 1.11.9 and 2.0
Brokers
~~~~~~~
diff --git a/django_q/__init__.py b/django_q/__init__.py
index 4f3b689..e25d49a 100644
--- a/django_q/__init__.py
+++ b/django_q/__init__.py
@@ -1,18 +1,17 @@
import os
import sys
-from django import get_version
+import django
myPath = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, myPath)
-VERSION = (0, 8, 0)
+VERSION = (0, 9, 1)
default_app_config = 'django_q.apps.DjangoQConfig'
# root imports will slowly be deprecated.
# please import from the relevant sub modules
-split_version = get_version().split('.')
-if split_version[1] not in ('9', '10', '11'):
+if django.VERSION[:2] < (1, 9):
from .tasks import async, schedule, result, result_group, fetch, fetch_group, count_group, delete_group, queue_size
from .models import Task, Schedule, Success, Failure
from .cluster import Cluster
diff --git a/django_q/brokers/aws_sqs.py b/django_q/brokers/aws_sqs.py
index 111e631..d33800f 100644
--- a/django_q/brokers/aws_sqs.py
+++ b/django_q/brokers/aws_sqs.py
@@ -51,9 +51,12 @@ class Sqs(Broker):
@staticmethod
def get_connection(list_key=Conf.PREFIX):
- return Session(aws_access_key_id=Conf.SQS['aws_access_key_id'],
- aws_secret_access_key=Conf.SQS['aws_secret_access_key'],
- region_name=Conf.SQS['aws_region'])
+ config = Conf.SQS
+ if 'aws_region' in config:
+ config['region_name'] = config['aws_region']
+ del(config['aws_region'])
+ return Session(**config)
+
def get_queue(self):
self.sqs = self.connection.resource('sqs')
diff --git a/django_q/cluster.py b/django_q/cluster.py
index 23db963..4dd61f6 100644
--- a/django_q/cluster.py
+++ b/django_q/cluster.py
@@ -10,7 +10,7 @@ import signal
import socket
import ast
from time import sleep
-from multiprocessing import Queue, Event, Process, Value, current_process
+from multiprocessing import Event, Process, Value, current_process
# external
import arrow
@@ -30,7 +30,7 @@ from django_q.models import Task, Success, Schedule
from django_q.status import Stat, Status
from django_q.brokers import get_broker
from django_q.signals import pre_execute
-
+from django_q.queues import Queue
class Cluster(object):
diff --git a/django_q/conf.py b/django_q/conf.py
index 2bb6c98..e8afac1 100644
--- a/django_q/conf.py
+++ b/django_q/conf.py
@@ -1,7 +1,7 @@
import logging
from copy import deepcopy
from signal import signal
-from multiprocessing import cpu_count, Queue
+from multiprocessing import cpu_count
# django
from django.utils.translation import ugettext_lazy as _
@@ -11,6 +11,9 @@ from django.conf import settings
import os
import pkg_resources
+# local
+from django_q.queues import Queue
+
# optional
try:
import psutil
@@ -216,10 +219,10 @@ if Conf.ERROR_REPORTER:
# iterate through the configured error reporters,
# and instantiate an ErrorReporter using the provided config
for name, conf in error_conf.items():
- Reporter = pkg_resources.iter_entry_points(
- 'djangoq.errorreporters', name).load()
- e = Reporter(**conf)
- reporters.append(e)
+ for entry in pkg_resources.iter_entry_points(
+ 'djangoq.errorreporters', name):
+ Reporter = entry.load()
+ reporters.append(Reporter(**conf))
error_reporter = ErrorReporter(reporters)
except ImportError:
error_reporter = None
diff --git a/django_q/core_signing.py b/django_q/core_signing.py
new file mode 100644
index 0000000..1790a0b
--- /dev/null
+++ b/django_q/core_signing.py
@@ -0,0 +1,76 @@
+from __future__ import unicode_literals
+
+import datetime
+import time
+import zlib
+
+from django.utils import baseconv
+from django.utils.crypto import constant_time_compare
+from django.utils.encoding import force_bytes, force_str, force_text
+from django.core.signing import BadSignature, SignatureExpired, b64_decode, JSONSerializer, \
+ Signer as Sgnr, TimestampSigner as TsS, dumps
+
+dumps = dumps
+
+
+"""
+The loads function is the same as the `django.core.signing.loads` function
+The difference is that `this` loads function calls `TimestampSigner` and `Signer`
+"""
+def loads(s, key=None, salt='django.core.signing', serializer=JSONSerializer, max_age=None):
+ """
+ Reverse of dumps(), raise BadSignature if signature fails.
+
+ The serializer is expected to accept a bytestring.
+ """
+ # TimestampSigner.unsign() returns str but base64 and zlib compression
+ # operate on bytes.
+ base64d = force_bytes(TimestampSigner(key, salt=salt).unsign(s, max_age=max_age))
+ decompress = False
+ if base64d[:1] == b'.':
+ # It's compressed; uncompress it first
+ base64d = base64d[1:]
+ decompress = True
+ data = b64_decode(base64d)
+ if decompress:
+ data = zlib.decompress(data)
+ return serializer().loads(data)
+
+
+class Signer(Sgnr):
+
+ def unsign(self, signed_value):
+ # force_str is removed in Django 2.0
+ signed_value = force_str(signed_value)
+ if self.sep not in signed_value:
+ raise BadSignature('No "%s" found in value' % self.sep)
+ value, sig = signed_value.rsplit(self.sep, 1)
+ if constant_time_compare(sig, self.signature(value)):
+ # force_text is removed in Django 2.0
+ return force_text(value)
+ raise BadSignature('Signature "%s" does not match' % sig)
+
+
+"""
+TimestampSigner is also the same as `django.core.signing.TimestampSigner` but is
+calling `this` Signer.
+"""
+class TimestampSigner(Signer, TsS):
+
+ def unsign(self, value, max_age=None):
+ """
+ Retrieve original value and check it wasn't signed more
+ than max_age seconds ago.
+ """
+ result = super().unsign(value)
+ value, timestamp = result.rsplit(self.sep, 1)
+ timestamp = baseconv.base62.decode(timestamp)
+ if max_age is not None:
+ if isinstance(max_age, datetime.timedelta):
+ max_age = max_age.total_seconds()
+ # Check timestamp is not older than max_age
+ age = time.time() - timestamp
+ if age > max_age:
+ raise SignatureExpired(
+ 'Signature age %s > %s seconds' % (age, max_age))
+ return value
diff --git a/django_q/models.py b/django_q/models.py
index 5de4f1e..b43f81c 100644
--- a/django_q/models.py
+++ b/django_q/models.py
@@ -1,5 +1,9 @@
from django import get_version
-from django.core.urlresolvers import reverse
+try:
+ from django.urls import reverse
+except ImportError: # Django < 1.10
+ from django.core.urlresolvers import reverse
+from django.utils.html import format_html
from django.utils.translation import ugettext_lazy as _
from django.db import models
from django.utils import timezone
@@ -162,7 +166,7 @@ class Schedule(models.Model):
url = reverse('admin:django_q_success_change', args=(task.id,))
else:
url = reverse('admin:django_q_failure_change', args=(task.id,))
- return '<a href="{}">[{}]</a>'.format(url, task.name)
+ return format_html('<a href="{}">[{}]</a>'.format(url, task.name))
return None
def __unicode__(self):
diff --git a/django_q/queues.py b/django_q/queues.py
new file mode 100644
index 0000000..ad729a1
--- /dev/null
+++ b/django_q/queues.py
@@ -0,0 +1,69 @@
+"""
+The code is derived from https://github.com/althonos/pronto/commit/3384010dfb4fc7c66a219f59276adef3288a886b
+"""
+
+import multiprocessing
+import multiprocessing.queues
+
+
+class SharedCounter(object):
+ """ A synchronized shared counter.
+
+ The locking done by multiprocessing.Value ensures that only a single
+ process or thread may read or write the in-memory ctypes object. However,
+ in order to do n += 1, Python performs a read followed by a write, so a
+ second process may read the old value before the new one is written by
+ the first process. The solution is to use a multiprocessing.Lock to
+ guarantee the atomicity of the modifications to Value.
+
+ This class comes almost entirely from Eli Bendersky's blog:
+ http://eli.thegreenplace.net/2012/01/04/shared-counter-with-pythons-multiprocessing/
+ """
+
+ def __init__(self, n=0):
+ self.count = multiprocessing.Value('i', n)
+
+ def increment(self, n=1):
+ """ Increment the counter by n (default = 1) """
+ with self.count.get_lock():
+ self.count.value += n
+
+ @property
+ def value(self):
+ """ Return the value of the counter """
+ return self.count.value
+
+
+class Queue(multiprocessing.queues.Queue):
+ """ A portable implementation of multiprocessing.Queue.
+
+ Because of multithreading / multiprocessing semantics, Queue.qsize() may
+ raise the NotImplementedError exception on Unix platforms like Mac OS X
+ where sem_getvalue() is not implemented. This subclass addresses this
+ problem by using a synchronized shared counter (initialized to zero) and
+ increasing / decreasing its value every time the put() and get() methods
+ are called, respectively. This not only prevents NotImplementedError from
+ being raised, but also allows us to implement a reliable version of both
+ qsize() and empty().
+ """
+
+ def __init__(self, *args, **kwargs):
+ super(Queue, self).__init__(*args, ctx=multiprocessing.get_context(), **kwargs)
+ self.size = SharedCounter(0)
+
+ def put(self, *args, **kwargs):
+ super(Queue, self).put(*args, **kwargs)
+ self.size.increment(1)
+
+ def get(self, *args, **kwargs):
+ x = super(Queue, self).get(*args, **kwargs)
+ self.size.increment(-1)
+ return x
+
+ def qsize(self):
+ """ Reliable implementation of multiprocessing.Queue.qsize() """
+ return self.size.value
+
+ def empty(self):
+ """ Reliable implementation of multiprocessing.Queue.empty() """
+ return not self.qsize() > 0
diff --git a/django_q/signing.py b/django_q/signing.py
index 4ee8bf3..2460aaa 100644
--- a/django_q/signing.py
+++ b/django_q/signing.py
@@ -4,7 +4,7 @@ try:
except ImportError:
import pickle
-from django.core import signing
+from django_q import core_signing as signing
from django_q.conf import Conf
diff --git a/django_q/tasks.py b/django_q/tasks.py
index 5f1c46b..e64bfc9 100644
--- a/django_q/tasks.py
+++ b/django_q/tasks.py
@@ -1,5 +1,5 @@
"""Provides task functionality."""
-from multiprocessing import Queue, Value
+from multiprocessing import Value
# django
from django.db import IntegrityError
@@ -14,6 +14,7 @@ from django_q.models import Schedule, Task
from django_q.humanhash import uuid
from django_q.brokers import get_broker
from django_q.signals import pre_enqueue
+from django_q.queues import Queue
def async(func, *args, **kwargs):
diff --git a/django_q/tests/settings.py b/django_q/tests/settings.py
index 8bad66c..b544963 100644
--- a/django_q/tests/settings.py
+++ b/django_q/tests/settings.py
@@ -1,4 +1,5 @@
import os
+import django
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -28,16 +29,18 @@ INSTALLED_APPS = (
'django_redis'
)
+
MIDDLEWARE_CLASSES = (
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
- 'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
)
+MIDDLEWARE = MIDDLEWARE_CLASSES
+
ROOT_URLCONF = 'tests.urls'
TEMPLATES = [
diff --git a/django_q/tests/test_admin.py b/django_q/tests/test_admin.py
index 97a4389..7c2acef 100644
--- a/django_q/tests/test_admin.py
+++ b/django_q/tests/test_admin.py
@@ -1,6 +1,9 @@
-from django.core.urlresolvers import reverse
-
+try:
+ from django.urls import reverse
+except ImportError: # Django < 1.10
+ from django.core.urlresolvers import reverse
from django.utils import timezone
+
import pytest
from django_q.tasks import schedule
diff --git a/django_q/tests/test_cached.py b/django_q/tests/test_cached.py
index 1cf9980..e32c60c 100644
--- a/django_q/tests/test_cached.py
+++ b/django_q/tests/test_cached.py
@@ -1,4 +1,4 @@
-from multiprocessing import Event, Queue, Value
+from multiprocessing import Event, Value
import pytest
@@ -8,6 +8,7 @@ from django_q.conf import Conf
from django_q.tasks import async, result, fetch, count_group, result_group, fetch_group, delete_group, delete_cached, \
async_iter, Chain, async_chain, Iter, Async
from django_q.brokers import get_broker
+from django_q.queues import Queue
@pytest.fixture
diff --git a/django_q/tests/test_cluster.py b/django_q/tests/test_cluster.py
index d16d0ac..15461d0 100644
--- a/django_q/tests/test_cluster.py
+++ b/django_q/tests/test_cluster.py
@@ -1,6 +1,6 @@
import sys
import threading
-from multiprocessing import Queue, Event, Value
+from multiprocessing import Event, Value
from time import sleep
from django.utils import timezone
@@ -19,6 +19,7 @@ from django_q.conf import Conf
from django_q.status import Stat
from django_q.brokers import get_broker
from django_q.tests.tasks import multiply
+from django_q.queues import Queue
class WordClass(object):
diff --git a/django_q/tests/test_scheduler.py b/django_q/tests/test_scheduler.py
index 47a0860..5160776 100644
--- a/django_q/tests/test_scheduler.py
+++ b/django_q/tests/test_scheduler.py
@@ -1,5 +1,5 @@
from datetime import timedelta
-from multiprocessing import Queue, Event, Value
+from multiprocessing import Event, Value
import arrow
import pytest
@@ -10,6 +10,7 @@ from django_q.brokers import get_broker
from django_q.cluster import pusher, worker, monitor, scheduler
from django_q.conf import Conf
from django_q.tasks import Schedule, fetch, schedule as create_schedule
+from django_q.queues import Queue
@pytest.fixture
diff --git a/django_q/tests/urls.py b/django_q/tests/urls.py
index eb91d28..ede2ec9 100644
--- a/django_q/tests/urls.py
+++ b/django_q/tests/urls.py
@@ -1,6 +1,6 @@
-from django.conf.urls import include, url
+from django.conf.urls import url
from django.contrib import admin
urlpatterns = [
- url(r'^admin/', include(admin.site.urls)),
+ url(r'^admin/', admin.site.urls),
]
diff --git a/docs/conf.py b/docs/conf.py
index d30c473..1b4fa5e 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -62,7 +62,7 @@ master_doc = 'index'
# General information about the project.
project = 'Django Q'
-copyright = '2015, Ilan Steemers'
+copyright = '2015-2018, Ilan Steemers'
author = 'Ilan Steemers'
# The version info for the project you're documenting, acts as replacement for
@@ -70,9 +70,9 @@ author = 'Ilan Steemers'
# built documents.
#
# The short X.Y version.
-version = '0.8'
+version = '0.9'
# The full version, including alpha/beta/rc tags.
-release = '0.8.0'
+release = '0.9.1'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/docs/configure.rst b/docs/configure.rst
index 66136f4..9dddc1e 100644
--- a/docs/configure.rst
+++ b/docs/configure.rst
@@ -229,7 +229,7 @@ All connection keywords are supported. See the `iron-mq <https://github.com/iron
sqs
~~~
-To use Amazon SQS as a broker you need to provide the AWS region and credentials::
+To use Amazon SQS as a broker you need to provide the AWS region and credentials either via the config, or any other boto3 configuration method::
# example SQS broker connection
@@ -241,9 +241,9 @@ To use Amazon SQS as a broker you need to provide the AWS region and credentials
'queue_limit': 100,
'bulk': 5,
'sqs': {
- 'aws_region': 'us-east-1',
- 'aws_access_key_id': 'ac-Idr.....YwflZBaaxI',
- 'aws_secret_access_key': '500f7b....b0f302e9'
+ 'aws_region': 'us-east-1', # optional
+ 'aws_access_key_id': 'ac-Idr.....YwflZBaaxI', # optional
+ 'aws_secret_access_key': '500f7b....b0f302e9' # optional
}
}
diff --git a/docs/index.rst b/docs/index.rst
index e4c83a3..df5959e 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -21,10 +21,10 @@ Features
- PaaS compatible with multiple instances
- Multi cluster monitor
- Redis, Disque, IronMQ, SQS, MongoDB or ORM
-- Rollbar support
+- Rollbar and Sentry support
-Django Q is tested with: Python 2.7 & 3.6. Django 1.8.18 LTS, 1.10.8 and 1.11.6
+Django Q is tested with: Python 2.7 & 3.6. Django 1.8.18 LTS, 1.11.9 and 2.0
Contents:
diff --git a/docs/install.rst b/docs/install.rst
index 9f4621f..360319b 100644
--- a/docs/install.rst
+++ b/docs/install.rst
@@ -32,7 +32,7 @@ Django Q is tested for Python 2.7 and 3.6
- `Django <https://www.djangoproject.com>`__
Django Q aims to use as much of Django's standard offerings as possible
- The code is tested against Django versions `1.8.18 LTS`, `1.10.8` and `1.11.6`.
+ The code is tested against Django versions `1.8.18 LTS`, `1.11.9` and `2.0`.
- `Django-picklefield <https://github.com/gintas/django-picklefield>`__
@@ -92,6 +92,7 @@ Extras
- `django-q-rollbar <https://github.com/danielwelch/django-q-rollbar>`__ is a Rollbar error reporter::
$ pip install django-q[rollbar]
+
- `django-q-sentry <https://github.com/danielwelch/django-q-sentry>`__ is a Sentry error reporter::
$ pip install django-q[sentry]
@@ -138,9 +139,9 @@ You can reference the `requirements <https://github.com/Koed00/django-q/blob/mas
Django
~~~~~~
We strive to be compatible with last two major version of Django.
-At the moment this means we support the 1.8.18 LTS, 1.10.8 and 1.11.6 releases.
+At the moment this means we support the 1.8.18 LTS, 1.11.9 and 2.0 releases.
-You might find that Django Q still works fine with Django 1.7 and 1.9, but new releases are no longer tested for it.
+You might find that Django Q still works fine with Django 1.7,1.9 and 1.10, but new releases are no longer tested for it.
diff --git a/requirements.txt b/requirements.txt
index 08ed52f..111e9f9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,11 +4,11 @@
#
# pip-compile --output-file requirements.txt requirements.in
#
-arrow==0.10.0
+arrow==0.12.1
blessed==1.14.2
-boto3==1.4.7
-botocore==1.7.27 # via boto3, s3transfer
-certifi==2017.7.27.1 # via requests
+boto3==1.5.22
+botocore==1.8.36 # via boto3, s3transfer
+certifi==2018.1.18 # via requests
chardet==3.0.4 # via requests
django-picklefield==1.0.0
django-redis==4.8.0
@@ -18,13 +18,13 @@ idna==2.6 # via requests
iron-core==1.2.0 # via iron-mq
iron-mq==0.9
jmespath==0.9.3 # via boto3, botocore
-psutil==5.4.0
-pymongo==3.5.1
+psutil==5.4.3
+pymongo==3.6.0
python-dateutil==2.6.1 # via arrow, botocore, iron-core
redis==2.10.6
requests==2.18.4 # via iron-core, rollbar
-rollbar==0.13.13
-s3transfer==0.1.11 # via boto3
+rollbar==0.13.17
+s3transfer==0.1.12 # via boto3
six==1.11.0 # via blessed, python-dateutil, rollbar
urllib3==1.22 # via requests
wcwidth==0.1.7 # via blessed
diff --git a/setup.py b/setup.py
index 1c4ef11..a4a9f37 100644
--- a/setup.py
+++ b/setup.py
@@ -26,7 +26,7 @@ class PyTest(Command):
setup(
name='django-q',
- version='0.8.0',
+ version='0.9.1',
author='Ilan Steemers',
author_email='koed0 at gmail.com',
keywords='django distributed task queue worker scheduler cron redis disque ironmq sqs orm mongodb multiprocessing rollbar',
@@ -59,8 +59,8 @@ setup(
],
entry_points={
'djangoq.errorreporters': [
- 'rollbar = django_q_rollbar.Rollbar',
- 'sentry = django_q_sentry.Sentry',
+ 'rollbar = django_q_rollbar:Rollbar',
+ 'sentry = django_q_sentry:Sentry',
]
},
extras_require={
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/django-q.git
More information about the Python-modules-commits
mailing list