[Python-modules-commits] [python-scrapy] 01/06: Import python-scrapy_1.3.3.orig.tar.gz
Michael Fladischer
fladi at moszumanska.debian.org
Fri Mar 17 08:37:26 UTC 2017
This is an automated email from the git hooks/post-receive script.
fladi pushed a commit to branch master
in repository python-scrapy.
commit e393c0a03fdd919e9ff3129345c7abd4724b7d1b
Author: Michael Fladischer <FladischerMichael at fladi.at>
Date: Thu Mar 16 20:38:18 2017 +0100
Import python-scrapy_1.3.3.orig.tar.gz
---
.bumpversion.cfg | 2 +-
docs/news.rst | 14 +++++++++++++-
docs/topics/settings.rst | 23 +++++++++++++++++++++++
scrapy/VERSION | 2 +-
scrapy/commands/runspider.py | 1 +
scrapy/commands/settings.py | 3 ++-
scrapy/commands/startproject.py | 5 +++--
scrapy/commands/version.py | 3 ++-
scrapy/settings/default_settings.py | 1 +
scrapy/spiderloader.py | 12 ++++++++----
tests/test_spiderloader/__init__.py | 9 ++++++++-
11 files changed, 63 insertions(+), 12 deletions(-)
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index b95e0ba..ed7aa0d 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 1.3.2
+current_version = 1.3.3
commit = True
tag = True
tag_name = {new_version}
diff --git a/docs/news.rst b/docs/news.rst
index ff1e4ce..305e431 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,13 +3,25 @@
Release notes
=============
+Scrapy 1.3.3 (2017-03-10)
+-------------------------
+
+Bug fixes
+~~~~~~~~~
+
+- Make ``SpiderLoader`` raise ``ImportError`` again by default for missing
+ dependencies and wrong :setting:`SPIDER_MODULES`.
+ These exceptions were silenced as warnings since 1.3.0.
+ A new setting is introduced to toggle between warning or exception if needed ;
+ see :setting:`SPIDER_LOADER_WARN_ONLY` for details.
+
Scrapy 1.3.2 (2017-02-13)
-------------------------
Bug fixes
~~~~~~~~~
-- Preserve crequest class when converting to/from dicts (utils.reqser) (:issue:`2510`).
+- Preserve request class when converting to/from dicts (utils.reqser) (:issue:`2510`).
- Use consistent selectors for author field in tutorial (:issue:`2551`).
- Fix TLS compatibility in Twisted 17+ (:issue:`2558`)
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0515a9e..dee2aed 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1116,6 +1116,29 @@ Default: ``'scrapy.spiderloader.SpiderLoader'``
The class that will be used for loading spiders, which must implement the
:ref:`topics-api-spiderloader`.
+.. setting:: SPIDER_LOADER_WARN_ONLY
+
+SPIDER_LOADER_WARN_ONLY
+-----------------------
+
+.. versionadded:: 1.3.3
+
+Default: ``False``
+
+By default, when scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
+it will fail loudly if there is any ``ImportError`` exception.
+But you can choose to silence this exception and turn it into a simple
+warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.
+
+.. note::
+ Some :ref:`scrapy commands <topics-commands>` run with this setting to ``True``
+ already (i.e. they will only issue a warning and will not fail)
+ since they do not actually need to load spider classes to work:
+ :command:`scrapy runspider <runspider>`,
+ :command:`scrapy settings <settings>`,
+ :command:`scrapy startproject <startproject>`,
+ :command:`scrapy version <version>`.
+
.. setting:: SPIDER_MIDDLEWARES
SPIDER_MIDDLEWARES
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 1892b92..31e5c84 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.3.2
+1.3.3
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 1da09e4..a98033d 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -28,6 +28,7 @@ def _import_file(filepath):
class Command(ScrapyCommand):
requires_project = False
+ default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
def syntax(self):
return "[options] <spider_file>"
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index bce4e60..bee52f0 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -7,7 +7,8 @@ from scrapy.settings import BaseSettings
class Command(ScrapyCommand):
requires_project = False
- default_settings = {'LOG_ENABLED': False}
+ default_settings = {'LOG_ENABLED': False,
+ 'SPIDER_LOADER_WARN_ONLY': True}
def syntax(self):
return "[options]"
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 5941066..c17aaf4 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -26,7 +26,8 @@ IGNORE = ignore_patterns('*.pyc', '.svn')
class Command(ScrapyCommand):
requires_project = False
- default_settings = {'LOG_ENABLED': False}
+ default_settings = {'LOG_ENABLED': False,
+ 'SPIDER_LOADER_WARN_ONLY': True}
def syntax(self):
return "<project_name> [project_dir]"
@@ -118,4 +119,4 @@ class Command(ScrapyCommand):
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
join(scrapy.__path__[0], 'templates')
return join(_templates_base_dir, 'project')
-
+
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index a9954ed..e22f98f 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -11,7 +11,8 @@ from scrapy.commands import ScrapyCommand
class Command(ScrapyCommand):
- default_settings = {'LOG_ENABLED': False}
+ default_settings = {'LOG_ENABLED': False,
+ 'SPIDER_LOADER_WARN_ONLY': True}
def syntax(self):
return "[-v]"
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 24714a7..e8016ed 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -241,6 +241,7 @@ SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'
SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
+SPIDER_LOADER_WARN_ONLY = False
SPIDER_MIDDLEWARES = {}
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index d4f0f66..1b96026 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -18,6 +18,7 @@ class SpiderLoader(object):
"""
def __init__(self, settings):
self.spider_modules = settings.getlist('SPIDER_MODULES')
+ self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')
self._spiders = {}
self._load_all_spiders()
@@ -31,10 +32,13 @@ class SpiderLoader(object):
for module in walk_modules(name):
self._load_spiders(module)
except ImportError as e:
- msg = ("\n{tb}Could not load spiders from module '{modname}'. "
- "Check SPIDER_MODULES setting".format(
- modname=name, tb=traceback.format_exc()))
- warnings.warn(msg, RuntimeWarning)
+ if self.warn_only:
+ msg = ("\n{tb}Could not load spiders from module '{modname}'. "
+ "See above traceback for details.".format(
+ modname=name, tb=traceback.format_exc()))
+ warnings.warn(msg, RuntimeWarning)
+ else:
+ raise
@classmethod
def from_settings(cls, settings):
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index b2ad93b..fe1acdf 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -91,11 +91,18 @@ class SpiderLoaderTest(unittest.TestCase):
self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
self.assertEqual(crawler.spidercls.name, 'spider1')
+ def test_bad_spider_modules_exception(self):
+
+ module = 'tests.test_spiderloader.test_spiders.doesnotexist'
+ settings = Settings({'SPIDER_MODULES': [module]})
+ self.assertRaises(ImportError, SpiderLoader.from_settings, settings)
+
def test_bad_spider_modules_warning(self):
with warnings.catch_warnings(record=True) as w:
module = 'tests.test_spiderloader.test_spiders.doesnotexist'
- settings = Settings({'SPIDER_MODULES': [module]})
+ settings = Settings({'SPIDER_MODULES': [module],
+ 'SPIDER_LOADER_WARN_ONLY': True})
spider_loader = SpiderLoader.from_settings(settings)
self.assertIn("Could not load spiders from module", str(w[0].message))
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/python-scrapy.git
More information about the Python-modules-commits
mailing list