[Python-modules-commits] [django-q] 02/11: Import django-q_0.8.1.orig.tar.gz
Pierre-Elliott Bécue
peb-guest at moszumanska.debian.org
Tue Dec 12 13:58:44 UTC 2017
This is an automated email from the git hooks/post-receive script.
peb-guest pushed a commit to branch master
in repository django-q.
commit 0c068a9f88c176063d9b8bf58b511cdd96737fe2
Author: Pierre-Elliott Bécue <becue at crans.org>
Date: Tue Dec 12 14:03:01 2017 +0100
Import django-q_0.8.1.orig.tar.gz
---
.travis.yml | 4 +--
MANIFEST.in | 3 +-
README.rst | 4 +--
django_q/admin.py | 6 ++--
django_q/apps.py | 4 +--
django_q/cluster.py | 14 ++++-----
django_q/compat.py | 14 +++++++++
django_q/conf.py | 39 +++++++++++++++++++++++++-
django_q/migrations/0009_auto_20171009_0915.py | 19 +++++++++++++
django_q/models.py | 2 +-
django_q/tests/test_brokers.py | 1 +
django_q/tests/test_cached.py | 1 +
django_q/tests/test_cluster.py | 3 +-
django_q/tests/test_monitor.py | 1 +
docs/configure.rst | 24 ++++++++++++++++
docs/errors.rst | 9 ++++++
docs/index.rst | 3 +-
docs/install.rst | 16 +++++++++--
requirements.in | 1 -
requirements.txt | 33 ++++++++++++----------
setup.py | 14 +++++++--
21 files changed, 173 insertions(+), 42 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 128ef22..a0a5579 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,8 +9,8 @@ python:
- "3.6"
env:
- - DJANGO=1.11
- - DJANGO=1.10.7
+ - DJANGO=1.11.6
+ - DJANGO=1.10.8
- DJANGO=1.8.18
sudo: false
diff --git a/MANIFEST.in b/MANIFEST.in
index b3169f4..bb8daf3 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -3,4 +3,5 @@ include README.rst
include django_q/management/*.py
include django_q/management/commands/*.py
include django_q/migrations/*.py
-include django_q/brokers/*.py
\ No newline at end of file
+include django_q/brokers/*.py
+include django_q/tests/*.py
diff --git a/README.rst b/README.rst
index a7b9a72..2f326ff 100644
--- a/README.rst
+++ b/README.rst
@@ -21,7 +21,7 @@ Features
- PaaS compatible with multiple instances
- Multi cluster monitor
- Redis, Disque, IronMQ, SQS, MongoDB or ORM
-- Rollbar support
+- Rollbar and Sentry support
Requirements
~~~~~~~~~~~~
@@ -31,7 +31,7 @@ Requirements
- `Arrow <https://github.com/crsmithdev/arrow>`__
- `Blessed <https://github.com/jquast/blessed>`__
-Tested with: Python 2.7 & 3.6. Django 1.8.18, 1.10.7 and 1.11
+Tested with: Python 2.7 & 3.6. Django 1.8.18, 1.10.8 and 1.11.6
Brokers
~~~~~~~
diff --git a/django_q/admin.py b/django_q/admin.py
index 36f70a1..f8511ad 100644
--- a/django_q/admin.py
+++ b/django_q/admin.py
@@ -2,9 +2,9 @@
from django.contrib import admin
from django.utils.translation import ugettext_lazy as _
-from .tasks import async
-from .models import Success, Failure, Schedule, OrmQ
-from .conf import Conf
+from django_q.tasks import async
+from django_q.models import Success, Failure, Schedule, OrmQ
+from django_q.conf import Conf
class TaskAdmin(admin.ModelAdmin):
diff --git a/django_q/apps.py b/django_q/apps.py
index d53a1e9..4943288 100644
--- a/django_q/apps.py
+++ b/django_q/apps.py
@@ -1,6 +1,6 @@
from django.apps import AppConfig
-from .conf import Conf
+from django_q.conf import Conf
class DjangoQConfig(AppConfig):
@@ -8,4 +8,4 @@ class DjangoQConfig(AppConfig):
verbose_name = Conf.LABEL
def ready(self):
- from django_q.signals import call_hook
\ No newline at end of file
+ from django_q.signals import call_hook
diff --git a/django_q/cluster.py b/django_q/cluster.py
index 4b6c44d..23db963 100644
--- a/django_q/cluster.py
+++ b/django_q/cluster.py
@@ -4,12 +4,6 @@ from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
-from builtins import range
-
-from future import standard_library
-
-standard_library.install_aliases()
-
# Standard
import importlib
import signal
@@ -30,13 +24,15 @@ from django import db
import signing
import tasks
-from django_q.conf import Conf, logger, psutil, get_ppid, rollbar
+from django_q.compat import range
+from django_q.conf import Conf, logger, psutil, get_ppid, error_reporter, rollbar
from django_q.models import Task, Success, Schedule
from django_q.status import Stat, Status
from django_q.brokers import get_broker
from django_q.signals import pre_execute
+
class Cluster(object):
def __init__(self, broker=None):
self.broker = broker or get_broker()
@@ -369,6 +365,8 @@ def worker(task_queue, result_queue, timer, timeout=Conf.TIMEOUT):
f = getattr(m, func)
except (ValueError, ImportError, AttributeError) as e:
result = (e, False)
+ if error_reporter:
+ error_reporter.report()
if rollbar:
rollbar.report_exc_info()
# We're still going
@@ -384,6 +382,8 @@ def worker(task_queue, result_queue, timer, timeout=Conf.TIMEOUT):
result = (res, True)
except Exception as e:
result = ('{}'.format(e), False)
+ if error_reporter:
+ error_reporter.report()
if rollbar:
rollbar.report_exc_info()
# Process result
diff --git a/django_q/compat.py b/django_q/compat.py
new file mode 100644
index 0000000..0ba80c4
--- /dev/null
+++ b/django_q/compat.py
@@ -0,0 +1,14 @@
+from __future__ import absolute_import
+"""
+Compatibility layer.
+
+Intentionally replaces use of python-future
+"""
+
+# https://github.com/Koed00/django-q/issues/4
+
+try:
+ range = xrange
+except NameError:
+ range = range
+
diff --git a/django_q/conf.py b/django_q/conf.py
index 9bfbe71..2bb6c98 100644
--- a/django_q/conf.py
+++ b/django_q/conf.py
@@ -9,6 +9,7 @@ from django.conf import settings
# external
import os
+import pkg_resources
# optional
try:
@@ -141,9 +142,12 @@ class Conf(object):
# The redis stats key
Q_STAT = 'django_q:{}:cluster'.format(PREFIX)
- # Optional Rollbar key
+ # Optional rollbar key
ROLLBAR = conf.get('rollbar', {})
+ # Optional error reporting setup
+ ERROR_REPORTER = conf.get('error_reporter', {})
+
# OSX doesn't implement qsize because of missing sem_getvalue()
try:
QSIZE = Queue().qsize() == 0
@@ -177,6 +181,7 @@ if not logger.handlers:
handler.setFormatter(formatter)
logger.addHandler(handler)
+
# rollbar
if Conf.ROLLBAR:
rollbar_conf = deepcopy(Conf.ROLLBAR)
@@ -190,6 +195,38 @@ else:
rollbar = None
+# Error Reporting Interface
+class ErrorReporter(object):
+
+ # initialize with iterator of reporters (better name, targets?)
+ def __init__(self, reporters):
+ self.targets = [target for target in reporters]
+
+ # report error to all configured targets
+ def report(self):
+ for t in self.targets:
+ t.report()
+
+
+# error reporting setup (sentry or rollbar)
+if Conf.ERROR_REPORTER:
+ error_conf = deepcopy(Conf.ERROR_REPORTER)
+ try:
+ reporters = []
+ # iterate through the configured error reporters,
+ # and instantiate an ErrorReporter using the provided config
+ for name, conf in error_conf.items():
+ Reporter = pkg_resources.iter_entry_points(
+ 'djangoq.errorreporters', name).load()
+ e = Reporter(**conf)
+ reporters.append(e)
+ error_reporter = ErrorReporter(reporters)
+ except ImportError:
+ error_reporter = None
+else:
+ error_reporter = None
+
+
# get parent pid compatibility
def get_ppid():
if hasattr(os, 'getppid'):
diff --git a/django_q/migrations/0009_auto_20171009_0915.py b/django_q/migrations/0009_auto_20171009_0915.py
new file mode 100644
index 0000000..1059126
--- /dev/null
+++ b/django_q/migrations/0009_auto_20171009_0915.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('django_q', '0008_auto_20160224_1026'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='schedule',
+ name='repeats',
+ field=models.IntegerField(default=-1, help_text='n = n times, -1 = forever', verbose_name='Repeats'),
+ ),
+ ]
diff --git a/django_q/models.py b/django_q/models.py
index a4b0223..5de4f1e 100644
--- a/django_q/models.py
+++ b/django_q/models.py
@@ -147,7 +147,7 @@ class Schedule(models.Model):
schedule_type = models.CharField(max_length=1, choices=TYPE, default=TYPE[0][0], verbose_name=_('Schedule Type'))
minutes = models.PositiveSmallIntegerField(null=True, blank=True,
help_text=_('Number of minutes for the Minutes type'))
- repeats = models.SmallIntegerField(default=-1, verbose_name=_('Repeats'), help_text=_('n = n times, -1 = forever'))
+ repeats = models.IntegerField(default=-1, verbose_name=_('Repeats'), help_text=_('n = n times, -1 = forever'))
next_run = models.DateTimeField(verbose_name=_('Next Run'), default=timezone.now, null=True)
task = models.CharField(max_length=100, null=True, editable=False)
diff --git a/django_q/tests/test_brokers.py b/django_q/tests/test_brokers.py
index 05265e9..6ef7cd4 100644
--- a/django_q/tests/test_brokers.py
+++ b/django_q/tests/test_brokers.py
@@ -5,6 +5,7 @@ import pytest
import redis
from django_q.brokers import get_broker, Broker
+from django_q.compat import range
from django_q.conf import Conf
from django_q.humanhash import uuid
diff --git a/django_q/tests/test_cached.py b/django_q/tests/test_cached.py
index d2b4141..1cf9980 100644
--- a/django_q/tests/test_cached.py
+++ b/django_q/tests/test_cached.py
@@ -3,6 +3,7 @@ from multiprocessing import Event, Queue, Value
import pytest
from django_q.cluster import pusher, worker, monitor
+from django_q.compat import range
from django_q.conf import Conf
from django_q.tasks import async, result, fetch, count_group, result_group, fetch_group, delete_group, delete_cached, \
async_iter, Chain, async_chain, Iter, Async
diff --git a/django_q/tests/test_cluster.py b/django_q/tests/test_cluster.py
index 2e33d26..d16d0ac 100644
--- a/django_q/tests/test_cluster.py
+++ b/django_q/tests/test_cluster.py
@@ -11,13 +11,14 @@ myPath = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, myPath + '/../')
from django_q.cluster import Cluster, Sentinel, pusher, worker, monitor, save_task
+from django_q.compat import range
from django_q.humanhash import DEFAULT_WORDLIST, uuid
from django_q.tasks import fetch, fetch_group, async, result, result_group, count_group, delete_group, queue_size
from django_q.models import Task, Success
from django_q.conf import Conf
from django_q.status import Stat
from django_q.brokers import get_broker
-from .tasks import multiply
+from django_q.tests.tasks import multiply
class WordClass(object):
diff --git a/django_q/tests/test_monitor.py b/django_q/tests/test_monitor.py
index d07fe07..0d43b7f 100644
--- a/django_q/tests/test_monitor.py
+++ b/django_q/tests/test_monitor.py
@@ -3,6 +3,7 @@ import pytest
from django_q.tasks import async
from django_q.brokers import get_broker
from django_q.cluster import Cluster
+from django_q.compat import range
from django_q.monitor import monitor, info
from django_q.status import Stat
from django_q.conf import Conf
diff --git a/docs/configure.rst b/docs/configure.rst
index 08e94ab..66136f4 100644
--- a/docs/configure.rst
+++ b/docs/configure.rst
@@ -353,6 +353,26 @@ scheduler
You can disable the scheduler by setting this option to ``False``. This will reduce a little overhead if you're not using schedules, but is most useful if you want to temporarily disable all schedules.
Defaults to ``True``
+.. _error_reporter:
+
+error_reporter
+~~~~~~~~~~~~~~
+You can redirect worker exceptions directly to various error reporters (for example `Rollbar <https://rollbar.com/>`__ or `Sentry <https://docs.sentry.io/>`__) by installing Django Q with the necessary `extras <https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies>`__.
+
+To enable installed error reporters, you must provide the configuration settings required by an error reporter extension::
+
+ # error_reporter config--rollbar example
+ Q_CLUSTER = {
+ 'error_reporter': {
+ 'rollbar': {
+ 'access_token': '32we33a92a5224jiww8982',
+ 'environment': 'Django-Q'
+ }
+ }
+ }
+
+For more information on error reporters and developing error reporting plugins for Django Q, see :doc:`errors<errors>`.
+
rollbar
~~~~~~~
You can redirect worker exceptions directly to your `Rollbar <https://rollbar.com/>`__ dashboard by installing the python notifier with ``pip install rollbar`` and adding this configuration dictionary to your config::
@@ -368,6 +388,10 @@ You can redirect worker exceptions directly to your `Rollbar <https://rollbar.co
Please check the Pyrollbar `configuration reference <https://github.com/rollbar/pyrollbar#configuration-reference>`__ for more options.
Note that you will need a `Rollbar <https://rollbar.com/>`__ account and access token to use this feature.
+
+.. note::
+ The ``rollbar`` setting is included for backwards compatibility, for those who utilized rollbar configuration before the ``error_reporter`` interface was introduced. Note that Rollbar support can be configured either via the ``rollbar`` setting, or via the ``django-q-rollbar`` package and enabled via the ``error_reporter`` setting above.
+
cpu_affinity
~~~~~~~~~~~~
diff --git a/docs/errors.rst b/docs/errors.rst
new file mode 100644
index 0000000..dd95eec
--- /dev/null
+++ b/docs/errors.rst
@@ -0,0 +1,9 @@
+Errors
+------
+.. py:currentmodule:: django_q
+
+Django Q uses a pluggable error reporter system based upon python `extras <https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies>`__, allowing anyone to develop plugins for their favorite error reporting and monitoring integration. Currently implemented examples include `Rollbar <https://rollbar.com/>`__ and `Sentry <https://docs.sentry.io/>`__.
+
+Error reporting plugins register a class which implements a ``report`` method, which is invoked when a Django Q cluster encounters an error, passing information to the particular service. Error reporters must be :ref:`configured<error_reporter>` via the ``Q_CLUSTER`` dictionary in your :file:`settings.py`. These settings are passed as kwargs upon initiation of the Error Reporter. Therefore, in order to implement a new plugin, a package must expose a class which will be instantiated with [...]
+
+For example implementations, see `django-q-rollbar <https://github.com/danielwelch/django-q-rollbar>`__ and `django-q-sentry <https://github.com/danielwelch/django-q-sentry>`__
diff --git a/docs/index.rst b/docs/index.rst
index ee4c3f0..e4c83a3 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -24,7 +24,7 @@ Features
- Rollbar support
-Django Q is tested with: Python 2.7 & 3.6. Django 1.8.18 LTS, 1.10.7 and 1.11
+Django Q is tested with: Python 2.7 & 3.6. Django 1.8.18 LTS, 1.10.8 and 1.11.6
Contents:
@@ -42,6 +42,7 @@ Contents:
Cluster <cluster>
Monitor <monitor>
Admin <admin>
+ Errors <errors>
Signals <signals>
Architecture <architecture>
Examples <examples>
diff --git a/docs/install.rst b/docs/install.rst
index 424fa95..9f4621f 100644
--- a/docs/install.rst
+++ b/docs/install.rst
@@ -32,7 +32,7 @@ Django Q is tested for Python 2.7 and 3.6
- `Django <https://www.djangoproject.com>`__
Django Q aims to use as much of Django's standard offerings as possible
- The code is tested against Django versions `1.8.18 LTS`, `1.10.7` and `1.11`.
+ The code is tested against Django versions `1.8.18 LTS`, `1.10.8` and `1.11.6`.
- `Django-picklefield <https://github.com/gintas/django-picklefield>`__
@@ -86,6 +86,16 @@ Optional
$ pip install rollbar
+
+Extras
+------
+- `django-q-rollbar <https://github.com/danielwelch/django-q-rollbar>`__ is a Rollbar error reporter::
+
+ $ pip install django-q[rollbar]
+- `django-q-sentry <https://github.com/danielwelch/django-q-sentry>`__ is a Sentry error reporter::
+
+ $ pip install django-q[sentry]
+
Compatibility
-------------
Django Q is still a young project. If you do find any incompatibilities please submit an issue on `github <https://github.com/Koed00/django-q>`__.
@@ -112,7 +122,7 @@ Other known issues are:
Python
~~~~~~
The code is always tested against the latest version of Python 2 and Python 3 and we try to stay compatible with the last two versions of each.
-Current tests are performed with Python 2.7.12 and 3.6.1
+Current tests are performed with Python 2.7.14 and 3.6.3
If you do encounter any regressions with earlier versions, please submit an issue on `github <https://github.com/Koed00/django-q>`__
.. note::
@@ -128,7 +138,7 @@ You can reference the `requirements <https://github.com/Koed00/django-q/blob/mas
Django
~~~~~~
We strive to be compatible with last two major version of Django.
-At the moment this means we support the 1.8.18 LTS, 1.10.7 and 1.11 releases.
+At the moment this means we support the 1.8.18 LTS, 1.10.8 and 1.11.6 releases.
You might find that Django Q still works fine with Django 1.7 and 1.9, but new releases are no longer tested for it.
diff --git a/requirements.in b/requirements.in
index 9105803..1246e52 100644
--- a/requirements.in
+++ b/requirements.in
@@ -1,7 +1,6 @@
arrow
blessed
django-picklefield
-future
hiredis
redis
psutil
diff --git a/requirements.txt b/requirements.txt
index 4995402..08ed52f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,22 +6,25 @@
#
arrow==0.10.0
blessed==1.14.2
-boto3==1.4.4
-botocore==1.5.35 # via boto3, s3transfer
-django-picklefield==0.3.2
-django-redis==4.7.0
-docutils==0.13.1 # via botocore
-future==0.16.0
+boto3==1.4.7
+botocore==1.7.27 # via boto3, s3transfer
+certifi==2017.7.27.1 # via requests
+chardet==3.0.4 # via requests
+django-picklefield==1.0.0
+django-redis==4.8.0
+docutils==0.14 # via botocore
hiredis==0.2.0
+idna==2.6 # via requests
iron-core==1.2.0 # via iron-mq
iron-mq==0.9
-jmespath==0.9.2 # via boto3, botocore
-psutil==5.2.1
-pymongo==3.4.0
-python-dateutil==2.6.0 # via arrow, botocore, iron-core
-redis==2.10.5
-requests==2.13.0 # via iron-core, rollbar
-rollbar==0.13.11
-s3transfer==0.1.10 # via boto3
-six==1.10.0 # via blessed, python-dateutil, rollbar
+jmespath==0.9.3 # via boto3, botocore
+psutil==5.4.0
+pymongo==3.5.1
+python-dateutil==2.6.1 # via arrow, botocore, iron-core
+redis==2.10.6
+requests==2.18.4 # via iron-core, rollbar
+rollbar==0.13.13
+s3transfer==0.1.11 # via boto3
+six==1.11.0 # via blessed, python-dateutil, rollbar
+urllib3==1.22 # via requests
wcwidth==0.1.7 # via blessed
diff --git a/setup.py b/setup.py
index 6541dd4..1c4ef11 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,7 @@ setup(
license='MIT',
description='A multiprocessing distributed task queue for Django',
long_description=README,
- install_requires=['django>=1.8', 'django-picklefield', 'blessed', 'arrow', 'future'],
+ install_requires=['django>=1.8', 'django-picklefield', 'blessed', 'arrow'],
test_requires=['pytest', 'pytest-django', ],
cmdclass={'test': PyTest},
classifiers=[
@@ -56,5 +56,15 @@ setup(
'Programming Language :: Python :: 3.6',
'Topic :: Internet :: WWW/HTTP',
'Topic :: Software Development :: Libraries :: Python Modules',
- ]
+ ],
+ entry_points={
+ 'djangoq.errorreporters': [
+ 'rollbar = django_q_rollbar.Rollbar',
+ 'sentry = django_q_sentry.Sentry',
+ ]
+ },
+ extras_require={
+ 'rollbar': ["django-q-rollbar>=0.1"],
+ 'sentry': ["django-q-sentry>=0.1"],
+ }
)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/django-q.git
More information about the Python-modules-commits
mailing list