[Python-modules-commits] [celery-haystack] 01/02: Imported Upstream version 0.9
Michael Fladischer
fladi at moszumanska.debian.org
Tue Aug 11 11:21:07 UTC 2015
This is an automated email from the git hooks/post-receive script.
fladi pushed a commit to branch master
in repository celery-haystack.
commit 4cc4716890b59d0e1b764ea7051522ed2f62ff49
Author: Michael Fladischer <FladischerMichael at fladi.at>
Date: Tue Aug 11 12:02:19 2015 +0200
Imported Upstream version 0.9
---
.gitignore | 11 ++
.travis.yml | 48 +++++++
AUTHORS | 5 +
LICENSE | 27 ++++
MANIFEST.in | 3 +
README.rst | 130 ++++++++++++++++++
celery_haystack/__init__.py | 5 +
celery_haystack/conf.py | 63 +++++++++
celery_haystack/indexes.py | 53 ++++++++
celery_haystack/models.py | 0
celery_haystack/signals.py | 44 ++++++
celery_haystack/tasks.py | 172 ++++++++++++++++++++++++
celery_haystack/test_settings.py | 52 ++++++++
celery_haystack/tests/__init__.py | 0
celery_haystack/tests/models.py | 8 ++
celery_haystack/tests/search_indexes.py | 24 ++++
celery_haystack/tests/search_sites.py | 3 +
celery_haystack/tests/tests.py | 74 +++++++++++
celery_haystack/utils.py | 43 ++++++
docs/Makefile | 130 ++++++++++++++++++
docs/changelog.rst | 167 +++++++++++++++++++++++
docs/conf.py | 228 ++++++++++++++++++++++++++++++++
docs/index.rst | 8 ++
docs/make.bat | 170 ++++++++++++++++++++++++
requirements/v1.txt | 7 +
requirements/v2.txt | 7 +
setup.cfg | 44 ++++++
setup.py | 4 +
28 files changed, 1530 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..dcbc81b
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,11 @@
+.DS_Store
+*.pyc
+celery_haystack/tests/whoosh_index
+*.egg
+*.egg-info
+.coverage
+docs/_build
+build/
+dist/
+.eggs/
+MANIFEST
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..28f6a9d
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,48 @@
+language: python
+python:
+ - "2.6"
+ - "2.7"
+ - "3.3"
+ - "3.4"
+before_install:
+ - export DJANGO_SETTINGS_MODULE=celery_haystack.test_settings
+install:
+ - pip install -e .
+ - pip install -r requirements/$HAYSTACK.txt $DJANGO
+before_script:
+ - flake8 celery_haystack --ignore=E501
+script:
+ - coverage run --branch --source=celery_haystack `which django-admin.py` test celery_haystack
+ - coverage report --omit=celery_haystack/test*
+env:
+ - DJANGO="Django==1.4.20" HAYSTACK=v1
+ - DJANGO="Django==1.4.20" HAYSTACK=v2
+ - DJANGO="Django==1.7.8" HAYSTACK=v1
+ - DJANGO="Django==1.7.8" HAYSTACK=v2
+ - DJANGO="Django==1.8.2" HAYSTACK=v2
+matrix:
+ exclude:
+ - env: DJANGO="Django==1.4.20" HAYSTACK=v1
+ python: "3.3"
+ - env: DJANGO="Django==1.4.20" HAYSTACK=v2
+ python: "3.3"
+ - env: DJANGO="Django==1.4.20" HAYSTACK=v1
+ python: "3.4"
+ - env: DJANGO="Django==1.4.20" HAYSTACK=v2
+ python: "3.4"
+ - env: DJANGO="Django==1.7.8" HAYSTACK=v1
+ python: "2.6"
+ - env: DJANGO="Django==1.7.8" HAYSTACK=v2
+ python: "2.6"
+ - env: DJANGO="Django==1.8.2" HAYSTACK=v1
+ python: "2.6"
+ - env: DJANGO="Django==1.8.2" HAYSTACK=v2
+ python: "2.6"
+ - env: DJANGO="Django==1.7.8" HAYSTACK=v1
+ python: "3.3"
+ - env: DJANGO="Django==1.7.8" HAYSTACK=v1
+ python: "3.4"
+ - env: DJANGO="Django==1.8.2" HAYSTACK=v1
+ python: "3.3"
+ - env: DJANGO="Django==1.8.2" HAYSTACK=v1
+ python: "3.4"
diff --git a/AUTHORS b/AUTHORS
new file mode 100644
index 0000000..bcf1972
--- /dev/null
+++ b/AUTHORS
@@ -0,0 +1,5 @@
+Josh Bohde
+Germán M. Bravo
+Jannis Leidel <jannis at leidel.info>
+Daniel Lindsley
+Stefan Wehrmeyer
\ No newline at end of file
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..a06d88e
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,27 @@
+Copyright (c) 2011-2013, Jannis Leidel and contributors.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ 3. Neither the name of celery-haystack nor the names of its contributors may
+ be used to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..7c985df
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,3 @@
+include AUTHORS
+include LICENSE
+include README.rst
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..cd62304
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,130 @@
+===============
+celery-haystack
+===============
+
+.. image:: https://secure.travis-ci.org/django-haystack/celery-haystack.png?branch=develop
+ :alt: Build Status
+ :target: http://travis-ci.org/django-haystack/celery-haystack
+
+This Django app allows you to utilize Celery for automatically updating and
+deleting objects in a Haystack_ search index.
+
+Requirements
+------------
+
+* Django 1.4+
+* Haystack_ `1.2.X`_ *or* `2.X`_
+* Celery_ 3.X
+
+You also need to install your choice of one of the supported search engines
+for Haystack and one of the supported backends for Celery.
+
+
+.. _Haystack: http://haystacksearch.org
+.. _`1.2.X`: http://pypi.python.org/pypi/django-haystack/1.2.5
+.. _`2.X`: https://github.com/toastdriven/django-haystack/tree/master
+
+Installation
+------------
+
+Use your favorite Python package manager to install the app from PyPI, e.g.::
+
+ pip install celery-haystack
+
+By default a few dependencies will automatically be installed:
+
+- django-appconf_ -- An app to gracefully handle application settings.
+
+- `django-celery-transactions`_ -- An app that "holds on to Celery tasks
+ until the current database transaction is committed, avoiding potential
+ race conditions as described in `Celery's user guide`_."
+
+.. _django-appconf: http://pypi.python.org/pypi/django-appconf
+.. _`django-celery-transactions`: https://github.com/chrisdoble/django-celery-transactions
+.. _`Celery's user guide`: http://celery.readthedocs.org/en/latest/userguide/tasks.html#database-transactions
+
+Usage
+-----
+
+Haystack 1.X
+~~~~~~~~~~~~
+
+1. Add ``'celery_haystack'`` to the ``INSTALLED_APPS`` setting
+
+ .. code:: python
+
+ INSTALLED_APPS = [
+ # ..
+ 'celery_haystack',
+ ]
+
+2. Alter all of your ``SearchIndex`` subclasses to inherit from
+ ``celery_haystack.indexes.CelerySearchIndex``
+
+ .. code:: python
+
+ from haystack import site, indexes
+ from celery_haystack.indexes import CelerySearchIndex
+ from myapp.models import Note
+
+ class NoteIndex(CelerySearchIndex):
+ text = indexes.CharField(document=True, model_attr='content')
+
+ site.register(Note, NoteIndex)
+
+3. Ensure your Celery instance is running.
+
+Haystack 2.X
+~~~~~~~~~~~~
+
+1. Add ``'celery_haystack'`` to the ``INSTALLED_APPS`` setting
+
+ .. code:: python
+
+ INSTALLED_APPS = [
+ # ..
+ 'celery_haystack',
+ ]
+
+2. Enable the celery-haystack signal processor in the settings
+
+ .. code:: python
+
+ HAYSTACK_SIGNAL_PROCESSOR = 'celery_haystack.signals.CelerySignalProcessor'
+
+3. Alter all of your ``SearchIndex`` subclasses to inherit from
+ ``celery_haystack.indexes.CelerySearchIndex`` and
+ ``haystack.indexes.Indexable``
+
+ .. code:: python
+
+ from haystack import indexes
+ from celery_haystack.indexes import CelerySearchIndex
+ from myapp.models import Note
+
+ class NoteIndex(CelerySearchIndex, indexes.Indexable):
+ text = indexes.CharField(document=True, model_attr='content')
+
+ def get_model(self):
+ return Note
+
+4. Ensure your Celery instance is running.
+
+Thanks
+------
+
+This app is a blatant rip-off of Daniel Lindsley's queued_search_
+app but uses Ask Solem Hoel's Celery_ instead of the equally awesome
+queues_ library by Matt Croyden.
+
+.. _queued_search: https://github.com/toastdriven/queued_search/
+.. _Celery: http://celeryproject.org/
+.. _queues: http://code.google.com/p/queues/
+
+Issues
+------
+
+Please use the `Github issue tracker`_ for any bug reports or feature
+requests.
+
+.. _`Github issue tracker`: https://github.com/django-haystack/celery-haystack/issues
diff --git a/celery_haystack/__init__.py b/celery_haystack/__init__.py
new file mode 100644
index 0000000..76de8d7
--- /dev/null
+++ b/celery_haystack/__init__.py
@@ -0,0 +1,5 @@
+__version__ = '0.9'
+
+
+def version_hook(config):
+ config['metadata']['version'] = __version__
diff --git a/celery_haystack/conf.py b/celery_haystack/conf.py
new file mode 100644
index 0000000..4298ae9
--- /dev/null
+++ b/celery_haystack/conf.py
@@ -0,0 +1,63 @@
+from django.conf import settings # noqa
+from django.core.exceptions import ImproperlyConfigured
+from haystack import constants, __version__ as haystack_version
+from haystack.management.commands import update_index as cmd
+from appconf import AppConf
+
+
+class CeleryHaystack(AppConf):
+ #: The default alias to
+ DEFAULT_ALIAS = None
+ #: The delay (in seconds) before task will be executed (Celery countdown)
+ COUNTDOWN = 0
+ #: The delay (in seconds) after which a failed index is retried
+ RETRY_DELAY = 5 * 60
+ #: The number of retries that are done
+ MAX_RETRIES = 1
+ #: The default Celery task class
+ DEFAULT_TASK = 'celery_haystack.tasks.CeleryHaystackSignalHandler'
+ #: The name of the celery queue to use, or None for default
+ QUEUE = None
+ #: Whether the task should be handled transaction safe
+ TRANSACTION_SAFE = True
+
+ #: The batch size used by the CeleryHaystackUpdateIndex task
+ COMMAND_BATCH_SIZE = None
+ #: The max age of items used by the CeleryHaystackUpdateIndex task
+ COMMAND_AGE = None
+ #: Wehther to remove items from the index that aren't in the DB anymore
+ COMMAND_REMOVE = False
+ #: The number of multiprocessing workers used by the CeleryHaystackUpdateIndex task
+ COMMAND_WORKERS = 0
+ #: The names of apps to run update_index for
+ COMMAND_APPS = []
+ #: The verbosity level of the update_index call
+ COMMAND_VERBOSITY = 1
+
+ def configure_default_alias(self, value):
+ return value or getattr(constants, 'DEFAULT_ALIAS', None)
+
+ def configure_command_batch_size(self, value):
+ return value or getattr(cmd, 'DEFAULT_BATCH_SIZE', None)
+
+ def configure_command_age(self, value):
+ return value or getattr(cmd, 'DEFAULT_AGE', None)
+
+ def configure(self):
+ data = {}
+ for name, value in self.configured_data.items():
+ if name in ('RETRY_DELAY', 'MAX_RETRIES',
+ 'COMMAND_WORKERS', 'COMMAND_VERBOSITY'):
+ value = int(value)
+ data[name] = value
+ return data
+
+
+signal_processor = getattr(settings, 'HAYSTACK_SIGNAL_PROCESSOR', None)
+
+
+if haystack_version[0] >= 2 and signal_processor is None:
+ raise ImproperlyConfigured("When using celery-haystack with Haystack 2.X "
+ "the HAYSTACK_SIGNAL_PROCESSOR setting must be "
+ "set. Use 'celery_haystack.signals."
+ "CelerySignalProcessor' as default.")
diff --git a/celery_haystack/indexes.py b/celery_haystack/indexes.py
new file mode 100644
index 0000000..4899c06
--- /dev/null
+++ b/celery_haystack/indexes.py
@@ -0,0 +1,53 @@
+from django.db.models import signals
+
+from haystack import indexes
+
+from .utils import enqueue_task
+
+
+class CelerySearchIndex(indexes.SearchIndex):
+ """
+ A ``SearchIndex`` subclass that enqueues updates/deletes for later
+ processing using Celery.
+ """
+ # We override the built-in _setup_* methods to connect the enqueuing
+ # operation.
+ def _setup_save(self, model):
+ signals.post_save.connect(self.enqueue_save,
+ sender=model,
+ dispatch_uid=CelerySearchIndex)
+
+ def _setup_delete(self, model):
+ signals.post_delete.connect(self.enqueue_delete,
+ sender=model,
+ dispatch_uid=CelerySearchIndex)
+
+ def _teardown_save(self, model):
+ signals.post_save.disconnect(self.enqueue_save,
+ sender=model,
+ dispatch_uid=CelerySearchIndex)
+
+ def _teardown_delete(self, model):
+ signals.post_delete.disconnect(self.enqueue_delete,
+ sender=model,
+ dispatch_uid=CelerySearchIndex)
+
+ def enqueue_save(self, instance, **kwargs):
+ if not getattr(instance, 'skip_indexing', False):
+ return self.enqueue('update', instance)
+
+ def enqueue_delete(self, instance, **kwargs):
+ if not getattr(instance, 'skip_indexing', False):
+ return self.enqueue('delete', instance)
+
+ def enqueue(self, action, instance):
+ """
+ Shoves a message about how to update the index into the queue.
+
+ This is a standardized string, resembling something like::
+
+ ``notes.note.23``
+ # ...or...
+ ``weblog.entry.8``
+ """
+ return enqueue_task(action, instance)
diff --git a/celery_haystack/models.py b/celery_haystack/models.py
new file mode 100644
index 0000000..e69de29
diff --git a/celery_haystack/signals.py b/celery_haystack/signals.py
new file mode 100644
index 0000000..f952f99
--- /dev/null
+++ b/celery_haystack/signals.py
@@ -0,0 +1,44 @@
+from django.db.models import signals
+
+from haystack.signals import BaseSignalProcessor
+from haystack.exceptions import NotHandled
+
+from .utils import enqueue_task
+from .indexes import CelerySearchIndex
+
+
+class CelerySignalProcessor(BaseSignalProcessor):
+
+ def setup(self):
+ signals.post_save.connect(self.enqueue_save)
+ signals.post_delete.connect(self.enqueue_delete)
+
+ def teardown(self):
+ signals.post_save.disconnect(self.enqueue_save)
+ signals.post_delete.disconnect(self.enqueue_delete)
+
+ def enqueue_save(self, sender, instance, **kwargs):
+ return self.enqueue('update', instance, sender, **kwargs)
+
+ def enqueue_delete(self, sender, instance, **kwargs):
+ return self.enqueue('delete', instance, sender, **kwargs)
+
+ def enqueue(self, action, instance, sender, **kwargs):
+ """
+ Given an individual model instance, determine if a backend
+ handles the model, check if the index is Celery-enabled and
+ enqueue task.
+ """
+ using_backends = self.connection_router.for_write(instance=instance)
+
+ for using in using_backends:
+ try:
+ connection = self.connections[using]
+ index = connection.get_unified_index().get_index(sender)
+ except NotHandled:
+ continue # Check next backend
+
+ if isinstance(index, CelerySearchIndex):
+ if action == 'update' and not index.should_update(instance):
+ continue
+ enqueue_task(action, instance)
diff --git a/celery_haystack/tasks.py b/celery_haystack/tasks.py
new file mode 100644
index 0000000..34fac4e
--- /dev/null
+++ b/celery_haystack/tasks.py
@@ -0,0 +1,172 @@
+from django.core.exceptions import ImproperlyConfigured
+from django.core.management import call_command
+from django.db.models.loading import get_model
+
+from .conf import settings
+
+try:
+ from haystack import connections, connection_router
+ from haystack.exceptions import NotHandled as IndexNotFoundException
+ legacy = False
+except ImportError:
+ try:
+ from haystack import site
+ from haystack.exceptions import NotRegistered as IndexNotFoundException # noqa
+ legacy = True
+ except ImportError as e:
+ raise ImproperlyConfigured("Haystack couldn't be imported: %s" % e)
+
+if settings.CELERY_HAYSTACK_TRANSACTION_SAFE and not getattr(settings, 'CELERY_ALWAYS_EAGER', False):
+ from djcelery_transactions import PostTransactionTask as Task
+else:
+ from celery.task import Task # noqa
+
+from celery.utils.log import get_task_logger
+
+logger = get_task_logger(__name__)
+
+
+class CeleryHaystackSignalHandler(Task):
+ using = settings.CELERY_HAYSTACK_DEFAULT_ALIAS
+ max_retries = settings.CELERY_HAYSTACK_MAX_RETRIES
+ default_retry_delay = settings.CELERY_HAYSTACK_RETRY_DELAY
+
+ def split_identifier(self, identifier, **kwargs):
+ """
+ Break down the identifier representing the instance.
+
+ Converts 'notes.note.23' into ('notes.note', 23).
+ """
+ bits = identifier.split('.')
+
+ if len(bits) < 2:
+ logger.error("Unable to parse object "
+ "identifer '%s'. Moving on..." % identifier)
+ return (None, None)
+
+ pk = bits[-1]
+ # In case Django ever handles full paths...
+ object_path = '.'.join(bits[:-1])
+ return (object_path, pk)
+
+ def get_model_class(self, object_path, **kwargs):
+ """
+ Fetch the model's class in a standarized way.
+ """
+ bits = object_path.split('.')
+ app_name = '.'.join(bits[:-1])
+ classname = bits[-1]
+ model_class = get_model(app_name, classname)
+
+ if model_class is None:
+ raise ImproperlyConfigured("Could not load model '%s'." %
+ object_path)
+ return model_class
+
+ def get_instance(self, model_class, pk, **kwargs):
+ """
+ Fetch the instance in a standarized way.
+ """
+ instance = None
+ try:
+ instance = model_class._default_manager.get(pk=pk)
+ except model_class.DoesNotExist:
+ logger.error("Couldn't load %s.%s.%s. Somehow it went missing?" %
+ (model_class._meta.app_label.lower(),
+ model_class._meta.object_name.lower(), pk))
+ except model_class.MultipleObjectsReturned:
+ logger.error("More than one object with pk %s. Oops?" % pk)
+ return instance
+
+ def get_indexes(self, model_class, **kwargs):
+ """
+ Fetch the model's registered ``SearchIndex`` in a standarized way.
+ """
+ try:
+ if legacy:
+ index_holder = site
+ yield index_holder.get_index(model_class), self.using
+ else:
+ using_backends = connection_router.for_write(**{'models': [model_class]})
+ for using in using_backends:
+ index_holder = connections[using].get_unified_index()
+ yield index_holder.get_index(model_class), using
+ except IndexNotFoundException:
+ raise ImproperlyConfigured("Couldn't find a SearchIndex for %s." %
+ model_class)
+
+ def run(self, action, identifier, **kwargs):
+ """
+ Trigger the actual index handler depending on the
+ given action ('update' or 'delete').
+ """
+ # First get the object path and pk (e.g. ('notes.note', 23))
+ object_path, pk = self.split_identifier(identifier, **kwargs)
+ if object_path is None or pk is None:
+ msg = "Couldn't handle object with identifier %s" % identifier
+ logger.error(msg)
+ raise ValueError(msg)
+
+ # Then get the model class for the object path
+ model_class = self.get_model_class(object_path, **kwargs)
+ for current_index, using in self.get_indexes(model_class, **kwargs):
+ current_index_name = ".".join([current_index.__class__.__module__,
+ current_index.__class__.__name__])
+
+ if action == 'delete':
+ # If the object is gone, we'll use just the identifier
+ # against the index.
+ try:
+ current_index.remove_object(identifier, using=using)
+ except Exception as exc:
+ logger.exception(exc)
+ self.retry(exc=exc)
+ else:
+ msg = ("Deleted '%s' (with %s)" %
+ (identifier, current_index_name))
+ logger.debug(msg)
+ elif action == 'update':
+ # and the instance of the model class with the pk
+ instance = self.get_instance(model_class, pk, **kwargs)
+ if instance is None:
+ logger.debug("Failed updating '%s' (with %s)" %
+ (identifier, current_index_name))
+ raise ValueError("Couldn't load object '%s'" % identifier)
+
+ # Call the appropriate handler of the current index and
+ # handle exception if neccessary
+ try:
+ current_index.update_object(instance, using=using)
+ except Exception as exc:
+ logger.exception(exc)
+ self.retry(exc=exc)
+ else:
+ msg = ("Updated '%s' (with %s)" %
+ (identifier, current_index_name))
+ logger.debug(msg)
+ else:
+ logger.error("Unrecognized action '%s'. Moving on..." % action)
+ raise ValueError("Unrecognized action %s" % action)
+
+
+class CeleryHaystackUpdateIndex(Task):
+ """
+ A celery task class to be used to call the update_index management
+ command from Celery.
+ """
+ def run(self, apps=None, **kwargs):
+ defaults = {
+ 'batchsize': settings.CELERY_HAYSTACK_COMMAND_BATCH_SIZE,
+ 'age': settings.CELERY_HAYSTACK_COMMAND_AGE,
+ 'remove': settings.CELERY_HAYSTACK_COMMAND_REMOVE,
+ 'using': [settings.CELERY_HAYSTACK_DEFAULT_ALIAS],
+ 'workers': settings.CELERY_HAYSTACK_COMMAND_WORKERS,
+ 'verbosity': settings.CELERY_HAYSTACK_COMMAND_VERBOSITY,
+ }
+ defaults.update(kwargs)
+ if apps is None:
+ apps = settings.CELERY_HAYSTACK_COMMAND_APPS
+ # Run the update_index management command
+ logger.info("Starting update index")
+ call_command('update_index', *apps, **defaults)
+ logger.info("Finishing update index")
diff --git a/celery_haystack/test_settings.py b/celery_haystack/test_settings.py
new file mode 100644
index 0000000..8dbfb62
--- /dev/null
+++ b/celery_haystack/test_settings.py
@@ -0,0 +1,52 @@
+import os
+
+import django
+
+from celery import Celery
+
+app = Celery('celery_haystack')
+app.config_from_object('django.conf:settings')
+
+
+DEBUG = True
+
+TEST_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), 'tests'))
+
+INSTALLED_APPS = [
+ 'haystack',
+ 'djcelery',
+ 'celery_haystack',
+ 'celery_haystack.tests',
+]
+
+DATABASES = {
+ 'default': {
+ 'ENGINE': 'django.db.backends.sqlite3',
+ 'NAME': ':memory:',
+ }
+}
+
+SECRET_KEY = 'really-not-secret'
+
+BROKER_TRANSPORT = "memory"
+CELERY_ALWAYS_EAGER = True
+CELERY_IGNORE_RESULT = True
+CELERYD_LOG_LEVEL = "DEBUG"
+CELERY_DEFAULT_QUEUE = "celery-haystack"
+
+if django.VERSION < (1, 6):
+ TEST_RUNNER = 'discover_runner.DiscoverRunner'
+
+if os.environ.get('HAYSTACK') == 'v1':
+ HAYSTACK_SITECONF = 'celery_haystack.tests.search_sites'
+ HAYSTACK_SEARCH_ENGINE = 'whoosh'
+ HAYSTACK_WHOOSH_PATH = os.path.join(TEST_ROOT, 'whoosh_index')
+
+elif os.environ.get('HAYSTACK') == 'v2':
+ HAYSTACK_CONNECTIONS = {
+ 'default': {
+ 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
+ 'PATH': os.path.join(TEST_ROOT, 'whoosh_index'),
+ }
+ }
+ HAYSTACK_SIGNAL_PROCESSOR = 'celery_haystack.signals.CelerySignalProcessor'
diff --git a/celery_haystack/tests/__init__.py b/celery_haystack/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/celery_haystack/tests/models.py b/celery_haystack/tests/models.py
new file mode 100644
index 0000000..4aecfdc
--- /dev/null
+++ b/celery_haystack/tests/models.py
@@ -0,0 +1,8 @@
+from django.db import models
+
+
+class Note(models.Model):
+ content = models.TextField()
+
+ def __unicode__(self):
+ return self.content
diff --git a/celery_haystack/tests/search_indexes.py b/celery_haystack/tests/search_indexes.py
new file mode 100644
index 0000000..7aa4fbd
--- /dev/null
+++ b/celery_haystack/tests/search_indexes.py
@@ -0,0 +1,24 @@
+from haystack import indexes, __version__ as haystack_version
+
+from .models import Note
+from ..indexes import CelerySearchIndex
+
+if haystack_version[:2] < (2, 0):
+ from haystack import site
+
+ class Indexable(object):
+ pass
+ indexes.Indexable = Indexable
+else:
+ site = None # noqa
+
+
+# Simplest possible subclass that could work.
+class NoteIndex(CelerySearchIndex, indexes.Indexable):
+ text = indexes.CharField(document=True, model_attr='content')
+
+ def get_model(self):
+ return Note
+
+if site:
+ site.register(Note, NoteIndex)
diff --git a/celery_haystack/tests/search_sites.py b/celery_haystack/tests/search_sites.py
new file mode 100644
index 0000000..59580c7
--- /dev/null
+++ b/celery_haystack/tests/search_sites.py
@@ -0,0 +1,3 @@
+import haystack
+
+haystack.autodiscover()
diff --git a/celery_haystack/tests/tests.py b/celery_haystack/tests/tests.py
new file mode 100644
index 0000000..43f2384
--- /dev/null
+++ b/celery_haystack/tests/tests.py
@@ -0,0 +1,74 @@
+from django.core.management import call_command
+from django.test import TestCase
+
+from haystack.query import SearchQuerySet
+
+from .models import Note
+
+
+class QueuedSearchIndexTestCase(TestCase):
+
+ def assertSearchResultLength(self, count):
+ self.assertEqual(count, len(SearchQuerySet()))
+
+ def assertSearchResultContains(self, pk, text):
+ results = SearchQuerySet().filter(id='tests.note.%s' % pk)
+ self.assertTrue(results)
+ self.assertTrue(text in results[0].text)
+
+ def setUp(self):
+ # Nuke the index.
+ call_command('clear_index', interactive=False, verbosity=0)
+
+ # Throw away all Notes
+ Note.objects.all().delete()
+
+ def test_update(self):
+ self.assertSearchResultLength(0)
+ note1 = Note.objects.create(content='Because everyone loves tests.')
+ self.assertSearchResultLength(1)
+ self.assertSearchResultContains(note1.pk, 'loves')
+
+ note2 = Note.objects.create(content='More test data.')
+ self.assertSearchResultLength(2)
+ self.assertSearchResultContains(note2.pk, 'More')
+
+ note3 = Note.objects.create(content='The test data. All done.')
+ self.assertSearchResultLength(3)
+ self.assertSearchResultContains(note3.pk, 'All done')
+
+ note3.content = 'Final test note FOR REAL'
+ note3.save()
+ self.assertSearchResultLength(3)
+ self.assertSearchResultContains(note3.pk, 'FOR REAL')
+
+ def test_delete(self):
+ note1 = Note.objects.create(content='Because everyone loves tests.')
+ note2 = Note.objects.create(content='More test data.')
+ note3 = Note.objects.create(content='The test data. All done.')
+ self.assertSearchResultLength(3)
+ note1.delete()
+ self.assertSearchResultLength(2)
+ note2.delete()
+ self.assertSearchResultLength(1)
+ note3.delete()
+ self.assertSearchResultLength(0)
+
+ def test_complex(self):
+ note1 = Note.objects.create(content='Because everyone loves test.')
+ self.assertSearchResultLength(1)
+
+ Note.objects.create(content='More test data.')
+ self.assertSearchResultLength(2)
+ note1.delete()
+ self.assertSearchResultLength(1)
+
+ note3 = Note.objects.create(content='The test data. All done.')
+ self.assertSearchResultLength(2)
+
+ note3.title = 'Final test note FOR REAL'
+ note3.save()
+ self.assertSearchResultLength(2)
+
+ note3.delete()
+ self.assertSearchResultLength(1)
diff --git a/celery_haystack/utils.py b/celery_haystack/utils.py
new file mode 100644
index 0000000..e351b2e
--- /dev/null
+++ b/celery_haystack/utils.py
@@ -0,0 +1,43 @@
+from django.core.exceptions import ImproperlyConfigured
+from django.utils.importlib import import_module
+from django.db import connection
+
+from haystack.utils import get_identifier
+
+from .conf import settings
+
+
+def get_update_task(task_path=None):
+ import_path = task_path or settings.CELERY_HAYSTACK_DEFAULT_TASK
+ module, attr = import_path.rsplit('.', 1)
+ try:
+ mod = import_module(module)
+ except ImportError as e:
+ raise ImproperlyConfigured('Error importing module %s: "%s"' %
+ (module, e))
+ try:
+ Task = getattr(mod, attr)
+ except AttributeError:
+ raise ImproperlyConfigured('Module "%s" does not define a "%s" '
+ 'class.' % (module, attr))
+ return Task()
+
+
+def enqueue_task(action, instance):
+ """
+ Common utility for enqueing a task for the given action and
+ model instance.
+ """
+ identifier = get_identifier(instance)
+ kwargs = {}
+ if settings.CELERY_HAYSTACK_QUEUE:
+ kwargs['queue'] = settings.CELERY_HAYSTACK_QUEUE
+ if settings.CELERY_HAYSTACK_COUNTDOWN:
+ kwargs['countdown'] = settings.CELERY_HAYSTACK_COUNTDOWN
+ task = get_update_task()
+ if hasattr(connection, 'on_commit'):
+ connection.on_commit(
+ lambda: task.apply_async((action, identifier), {}, **kwargs)
+ )
+ else:
+ task.apply_async((action, identifier), {}, **kwargs)
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000..6e92558
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,130 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+PAPER =
+BUILDDIR = _build
+
+# Internal variables.
+PAPEROPT_a4 = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest
+
+help:
+ @echo "Please use \`make <target>' where <target> is one of"
+ @echo " html to make standalone HTML files"
+ @echo " dirhtml to make HTML files named index.html in directories"
+ @echo " singlehtml to make a single large HTML file"
+ @echo " pickle to make pickle files"
+ @echo " json to make JSON files"
+ @echo " htmlhelp to make HTML files and a HTML help project"
+ @echo " qthelp to make HTML files and a qthelp project"
+ @echo " devhelp to make HTML files and a Devhelp project"
+ @echo " epub to make an epub"
+ @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+ @echo " latexpdf to make LaTeX files and run them through pdflatex"
+ @echo " text to make text files"
+ @echo " man to make manual pages"
+ @echo " changes to make an overview of all changed/added/deprecated items"
+ @echo " linkcheck to check all external links for integrity"
+ @echo " doctest to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+ -rm -rf $(BUILDDIR)/*
+
+html:
+ $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+ $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+ $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+ @echo
+ @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+ $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+ @echo
+ @echo "Build finished; now you can process the pickle files."
+
+json:
+ $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+ @echo
+ @echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+ $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+ @echo
+ @echo "Build finished; now you can run HTML Help Workshop with the" \
+ ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+ $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+ @echo
+ @echo "Build finished; now you can run "qcollectiongenerator" with the" \
+ ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+ @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/celery-haystack.qhcp"
+ @echo "To view the help file:"
+ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/celery-haystack.qhc"
+
+devhelp:
+ $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+ @echo
+ @echo "Build finished."
+ @echo "To view the help file:"
+ @echo "# mkdir -p $$HOME/.local/share/devhelp/celery-haystack"
... 729 lines suppressed ...
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/celery-haystack.git
More information about the Python-modules-commits
mailing list