[Git][debian-gis-team/pywps][master] 5 commits: New upstream version 4.2.8
Bas Couwenberg
gitlab at salsa.debian.org
Wed Sep 16 16:59:02 BST 2020
Bas Couwenberg pushed to branch master at Debian GIS Project / pywps
Commits:
aac10d81 by Bas Couwenberg at 2020-09-16T17:31:06+02:00
New upstream version 4.2.8
- - - - -
886e5a6b by Bas Couwenberg at 2020-09-16T17:31:08+02:00
Update upstream source from tag 'upstream/4.2.8'
Update to upstream version '4.2.8'
with Debian dir 1ebc635753db4f00657d4132af3362b602d1499a
- - - - -
56246c80 by Bas Couwenberg at 2020-09-16T17:32:26+02:00
New upstream release.
- - - - -
a218c061 by Bas Couwenberg at 2020-09-16T17:44:59+02:00
Update lintian overrides.
- - - - -
933e1614 by Bas Couwenberg at 2020-09-16T17:44:59+02:00
Set distribution to unstable.
- - - - -
12 changed files:
- VERSION.txt
- debian/changelog
- − debian/source/lintian-overrides
- docs/configuration.rst
- pywps/__init__.py
- pywps/app/Process.py
- pywps/app/Service.py
- pywps/app/exceptions.py
- pywps/configuration.py
- pywps/inout/basic.py
- pywps/processing/scheduler.py
- + tests/test_app_exceptions.py
Changes:
=====================================
VERSION.txt
=====================================
@@ -1 +1 @@
-4.2.7
+4.2.8
=====================================
debian/changelog
=====================================
@@ -1,3 +1,10 @@
+pywps (4.2.8-1) unstable; urgency=medium
+
+ * New upstream release.
+ * Update lintian overrides.
+
+ -- Bas Couwenberg <sebastic at debian.org> Wed, 16 Sep 2020 17:36:38 +0200
+
pywps (4.2.7-1) unstable; urgency=medium
* New upstream release.
=====================================
debian/source/lintian-overrides deleted
=====================================
@@ -1,6 +0,0 @@
-# Not worth the effort
-testsuite-autopkgtest-missing
-
-# False positive
-redundant-globbing-patterns *
-
=====================================
docs/configuration.rst
=====================================
@@ -95,7 +95,7 @@ configuration file <https://docs.pycsw.org/en/latest/configuration.html>`_.
the URL of the WPS service endpoint
:language:
- a comma-separated list of ISO 639-1 language and ISO 3166-1 alpha2 country
+ a comma-separated list of ISO 639-1 language and ISO 3166-1 alpha2 country
code of the service
(e.g. ``en-CA``, ``fr-CA``, ``en-US``)
@@ -110,14 +110,14 @@ configuration file <https://docs.pycsw.org/en/latest/configuration.html>`_.
of cores in the processor of the hosting machine. As well, speed and
response time of hard drives impact ultimate processing performance. A
reasonable number of parallel running processes is not higher than the
- number of processor cores.
+ number of processor cores. -1 for no limit.
:maxrequestsize:
maximal request size. 0 for no limit
:maxprocesses:
maximal number of requests being stored in queue, waiting till they can be
- processed (see ``parallelprocesses`` configuration option).
+ processed (see ``parallelprocesses`` configuration option). -1 for no limit.
:workdir:
a directory to store all temporary files (which should be always deleted,
@@ -170,6 +170,11 @@ configuration file <https://docs.pycsw.org/en/latest/configuration.html>`_.
the `scheduler` backend and is by default set automatically:
`os.path.dirname(os.path.realpath(sys.argv[0]))`
+:drmaa_native_specification:
+ option to set the DRMAA native specification, for example to limit number of
+ CPUs and memory usage. Example: `--cpus-per-task=1 --mem=1024`.
+ See DRMAA docs for details: https://github.com/natefoo/slurm-drmaa
+
[logging]
---------
@@ -280,4 +285,3 @@ Sample file
prefix=appname/coolapp/
public=true
encrypt=false
-
=====================================
pywps/__init__.py
=====================================
@@ -9,7 +9,7 @@ import os
from lxml.builder import ElementMaker
-__version__ = '4.2.7'
+__version__ = '4.2.8'
LOGGER = logging.getLogger('PYWPS')
LOGGER.debug('setting core variables')
=====================================
pywps/app/Process.py
=====================================
@@ -208,7 +208,7 @@ class Process(object):
# try to store for later usage
else:
maxprocesses = int(config.get_config_value('server', 'maxprocesses'))
- if stored >= maxprocesses:
+ if stored >= maxprocesses and maxprocesses != -1:
raise ServerBusy('Maximum number of processes in queue reached. Please try later.')
LOGGER.debug("Store process in job queue, uuid={}".format(self.uuid))
dblog.store_process(self.uuid, wps_request)
=====================================
pywps/app/Service.py
=====================================
@@ -378,7 +378,7 @@ def _validate_file_input(href):
file_path = os.path.abspath(file_path)
# build allowed paths list
inputpaths = config.get_config_value('server', 'allowedinputpaths')
- allowed_paths = [os.path.abspath(p.strip()) for p in inputpaths.split(':') if p.strip()]
+ allowed_paths = [os.path.abspath(p.strip()) for p in inputpaths.split(os.pathsep) if p.strip()]
for allowed_path in allowed_paths:
if file_path.startswith(allowed_path):
LOGGER.debug("Accepted file url as input.")
=====================================
pywps/app/exceptions.py
=====================================
@@ -7,42 +7,57 @@
Process exceptions raised intentionally in processes to provide information for users.
"""
+import re
+
+DEFAULT_ALLOWED_CHARS = ".:!?=,;-_/"
+
+import logging
+
+LOGGER = logging.getLogger('PYWPS')
+
+
+def format_message(text, min_length=3, max_length=300, allowed_chars=None):
+ allowed_chars = allowed_chars or DEFAULT_ALLOWED_CHARS
+ special = re.escape(allowed_chars)
+ pattern = rf'[\w{special}]+'
+ msg = ' '.join(re.findall(pattern, text))
+ msg.strip()
+ if len(msg) >= min_length:
+ msg = msg[:max_length]
+ else:
+ msg = ''
+ return msg
+
class ProcessError(Exception):
""":class:`pywps.app.exceptions.ProcessError` is an :class:`Exception`
you can intentionally raise in a process
to provide a user-friendly error message.
- The error message gets validated (3<= message length <=144) and only
+ The error message gets formatted (3<= message length <=300) and only
alpha numeric characters and a few special characters are allowed.
- The special characters are: `.`, `:`, `!`, `?`, `=`, `,`, `-`.
"""
- min_msg_length = 3
- max_msg_length = 144
- allowed_chars = ['.', ':', '!', '?', '=', ',', '-']
default_msg = 'Sorry, process failed. Please check server error log.'
- def __init__(self, msg=None):
+ def __init__(self, msg=None, min_length=3, max_length=300, allowed_chars=None):
self.msg = msg
+ self.min_length = min_length
+ self.max_length = max_length
+ self.allowed_chars = allowed_chars or DEFAULT_ALLOWED_CHARS
def __str__(self):
return self.message
- def _validate_message(self):
- valid = False
- if self.msg and self.min_msg_length <= len(self.msg) <= self.max_msg_length:
- # remove spaces
- test_str = self.msg.replace(' ', '')
- # remove allowed non alpha-numeric chars
- for char in self.allowed_chars:
- test_str = test_str.replace(char, '')
- # only alpha numeric string accepted
- valid = test_str.isalnum()
- return valid
-
@property
def message(self):
- if self._validate_message():
- new_msg = "{}".format(self.msg)
- else:
- new_msg = self.default_msg
- return new_msg
+ try:
+ msg = format_message(
+ self.msg,
+ min_length=self.min_length,
+ max_length=self.max_length,
+ allowed_chars=self.allowed_chars)
+ except Exception as e:
+ LOGGER.warning(f"process error formatting failed: {e}")
+ msg = None
+ if not msg:
+ msg = self.default_msg
+ return msg
=====================================
pywps/configuration.py
=====================================
@@ -100,6 +100,8 @@ def load_configuration(cfgfiles=None):
CONFIG.add_section('processing')
CONFIG.set('processing', 'mode', 'default')
CONFIG.set('processing', 'path', os.path.dirname(os.path.realpath(sys.argv[0])))
+ # https://github.com/natefoo/slurm-drmaa
+ CONFIG.set('processing', 'drmaa_native_specification', '')
CONFIG.add_section('logging')
CONFIG.set('logging', 'file', '')
=====================================
pywps/inout/basic.py
=====================================
@@ -932,7 +932,7 @@ class ComplexInput(BasicIO, BasicComplex, IOHandler):
file_path = os.path.abspath(file_path)
# build allowed paths list
inputpaths = config.get_config_value('server', 'allowedinputpaths')
- allowed_paths = [os.path.abspath(p.strip()) for p in inputpaths.split(':') if p.strip()]
+ allowed_paths = [os.path.abspath(p.strip()) for p in inputpaths.split(os.pathsep) if p.strip()]
for allowed_path in allowed_paths:
if file_path.startswith(allowed_path):
LOGGER.debug("Accepted file url as input.")
=====================================
pywps/processing/scheduler.py
=====================================
@@ -53,6 +53,9 @@ class Scheduler(Processing):
jt.args = ['-c', cfg_file, dump_filename]
else:
jt.args = [dump_filename]
+ drmaa_native_specification = config.get_config_value('processing', 'drmaa_native_specification')
+ if drmaa_native_specification:
+ jt.nativeSpecification = drmaa_native_specification
jt.joinFiles = True
jt.outputPath = ":{}".format(os.path.join(self.job.workdir, "job-output.txt"))
# run job
=====================================
tests/test_app_exceptions.py
=====================================
@@ -0,0 +1,35 @@
+##################################################################
+# Copyright 2018 Open Source Geospatial Foundation and others #
+# licensed under MIT, Please consult LICENSE.txt for details #
+##################################################################
+
+import unittest
+from pywps.app.exceptions import format_message, ProcessError, DEFAULT_ALLOWED_CHARS
+
+
+class AppExceptionsTest(unittest.TestCase):
+
+ def setUp(self):
+ pass
+
+ def test_format_message(self):
+ assert format_message('no data available') == 'no data available'
+ assert format_message(' no data available! ') == 'no data available!'
+ assert format_message('no') == ''
+ assert format_message('no data available', max_length=7) == 'no data'
+ assert format_message('no &data% available') == 'no data available'
+ assert format_message(DEFAULT_ALLOWED_CHARS) == DEFAULT_ALLOWED_CHARS
+
+ def test_process_error(self):
+ assert ProcessError(' no &data available!').message == 'no data available!'
+ assert ProcessError('no', min_length=2).message == 'no'
+ assert ProcessError('0 data available', max_length=6).message == '0 data'
+ assert ProcessError('no data? not available!', allowed_chars='?').message == 'no data? not available'
+ assert ProcessError('').message == 'Sorry, process failed. Please check server error log.'
+ assert ProcessError(1234).message == 'Sorry, process failed. Please check server error log.'
+ try:
+ raise ProcessError('no data!!')
+ except ProcessError as e:
+ assert f"{e}" == 'no data!!'
+ else:
+ assert False
View it on GitLab: https://salsa.debian.org/debian-gis-team/pywps/-/compare/7181e4ddcbf1b2edf7d41656ed932b02890a2032...933e1614fac6ecbdd793b163da7724152b55955d
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/pywps/-/compare/7181e4ddcbf1b2edf7d41656ed932b02890a2032...933e1614fac6ecbdd793b163da7724152b55955d
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20200916/a85ed137/attachment-0001.html>
More information about the Pkg-grass-devel
mailing list