[Git][debian-gis-team/pywps][upstream] New upstream version 4.2.8

Bas Couwenberg gitlab at salsa.debian.org
Wed Sep 16 16:59:07 BST 2020



Bas Couwenberg pushed to branch upstream at Debian GIS Project / pywps


Commits:
aac10d81 by Bas Couwenberg at 2020-09-16T17:31:06+02:00
New upstream version 4.2.8
- - - - -


11 changed files:

- VERSION.txt
- debian/changelog
- docs/configuration.rst
- pywps/__init__.py
- pywps/app/Process.py
- pywps/app/Service.py
- pywps/app/exceptions.py
- pywps/configuration.py
- pywps/inout/basic.py
- pywps/processing/scheduler.py
- + tests/test_app_exceptions.py


Changes:

=====================================
VERSION.txt
=====================================
@@ -1 +1 @@
-4.2.7
+4.2.8


=====================================
debian/changelog
=====================================
@@ -1,10 +1,18 @@
+pywps (4.2.8) trusty; urgency=medium
+
+  * update scheduler with drmaa config (#547).
+  * process error with formatting (#546).
+  * allow path list separation also on Windows (#545).
+  * allow inputpaths to accept full windows paths (#544).
+
+ -- Carsten Ehbrecht <ehbrecht at dkrz.de>  Web, 15 Sep 2020 18:00:00 +0000
+
 pywps (4.2.7) trusty; urgency=medium
 
   * ext_autodoc: support RST anonymous link (#542).
 
  -- Carsten Ehbrecht <ehbrecht at dkrz.de>  Tue, 04 Aug 2020 18:00:00 +0000
 
-
 pywps (4.2.6) trusty; urgency=medium
 
   * Fixed tests on travis (#541).


=====================================
docs/configuration.rst
=====================================
@@ -95,7 +95,7 @@ configuration file <https://docs.pycsw.org/en/latest/configuration.html>`_.
     the URL of the WPS service endpoint
 
 :language:
-    a comma-separated list of ISO 639-1 language and ISO 3166-1 alpha2 country 
+    a comma-separated list of ISO 639-1 language and ISO 3166-1 alpha2 country
     code of the service
     (e.g. ``en-CA``, ``fr-CA``, ``en-US``)
 
@@ -110,14 +110,14 @@ configuration file <https://docs.pycsw.org/en/latest/configuration.html>`_.
     of cores  in the processor of the hosting machine. As well, speed and
     response time of hard drives impact ultimate processing performance. A
     reasonable number of parallel running processes is not higher than the
-    number of processor cores.
+    number of processor cores. -1 for no limit.
 
 :maxrequestsize:
     maximal request size. 0 for no limit
 
 :maxprocesses:
     maximal number of requests being stored in queue, waiting till they can be
-    processed (see ``parallelprocesses`` configuration option).
+    processed (see ``parallelprocesses`` configuration option). -1 for no limit.
 
 :workdir:
     a directory to store all temporary files (which should be always deleted,
@@ -170,6 +170,11 @@ configuration file <https://docs.pycsw.org/en/latest/configuration.html>`_.
     the `scheduler` backend and is by default set automatically:
     `os.path.dirname(os.path.realpath(sys.argv[0]))`
 
+:drmaa_native_specification:
+    option to set the DRMAA native specification, for example to limit number of
+    CPUs and memory usage. Example: `--cpus-per-task=1 --mem=1024`.
+    See DRMAA docs for details: https://github.com/natefoo/slurm-drmaa
+
 [logging]
 ---------
 
@@ -280,4 +285,3 @@ Sample file
   prefix=appname/coolapp/
   public=true
   encrypt=false
-


=====================================
pywps/__init__.py
=====================================
@@ -9,7 +9,7 @@ import os
 
 from lxml.builder import ElementMaker
 
-__version__ = '4.2.7'
+__version__ = '4.2.8'
 
 LOGGER = logging.getLogger('PYWPS')
 LOGGER.debug('setting core variables')


=====================================
pywps/app/Process.py
=====================================
@@ -208,7 +208,7 @@ class Process(object):
             # try to store for later usage
             else:
                 maxprocesses = int(config.get_config_value('server', 'maxprocesses'))
-                if stored >= maxprocesses:
+                if stored >= maxprocesses and maxprocesses != -1:
                     raise ServerBusy('Maximum number of processes in queue reached. Please try later.')
                 LOGGER.debug("Store process in job queue, uuid={}".format(self.uuid))
                 dblog.store_process(self.uuid, wps_request)


=====================================
pywps/app/Service.py
=====================================
@@ -378,7 +378,7 @@ def _validate_file_input(href):
     file_path = os.path.abspath(file_path)
     # build allowed paths list
     inputpaths = config.get_config_value('server', 'allowedinputpaths')
-    allowed_paths = [os.path.abspath(p.strip()) for p in inputpaths.split(':') if p.strip()]
+    allowed_paths = [os.path.abspath(p.strip()) for p in inputpaths.split(os.pathsep) if p.strip()]
     for allowed_path in allowed_paths:
         if file_path.startswith(allowed_path):
             LOGGER.debug("Accepted file url as input.")


=====================================
pywps/app/exceptions.py
=====================================
@@ -7,42 +7,57 @@
 Process exceptions raised intentionally in processes to provide information for users.
 """
 
+import re
+
+DEFAULT_ALLOWED_CHARS = ".:!?=,;-_/"
+
+import logging
+
+LOGGER = logging.getLogger('PYWPS')
+
+
+def format_message(text, min_length=3, max_length=300, allowed_chars=None):
+    allowed_chars = allowed_chars or DEFAULT_ALLOWED_CHARS
+    special = re.escape(allowed_chars)
+    pattern = rf'[\w{special}]+'
+    msg = ' '.join(re.findall(pattern, text))
+    msg.strip()
+    if len(msg) >= min_length:
+        msg = msg[:max_length]
+    else:
+        msg = ''
+    return msg
+
 
 class ProcessError(Exception):
     """:class:`pywps.app.exceptions.ProcessError` is an :class:`Exception`
     you can intentionally raise in a process
     to provide a user-friendly error message.
-    The error message gets validated (3<= message length <=144) and only
+    The error message gets formatted (3<= message length <=300) and only
     alpha numeric characters and a few special characters are allowed.
-    The special characters are: `.`, `:`, `!`, `?`, `=`, `,`, `-`.
     """
-    min_msg_length = 3
-    max_msg_length = 144
-    allowed_chars = ['.', ':', '!', '?', '=', ',', '-']
     default_msg = 'Sorry, process failed. Please check server error log.'
 
-    def __init__(self, msg=None):
+    def __init__(self, msg=None, min_length=3, max_length=300, allowed_chars=None):
         self.msg = msg
+        self.min_length = min_length
+        self.max_length = max_length
+        self.allowed_chars = allowed_chars or DEFAULT_ALLOWED_CHARS
 
     def __str__(self):
         return self.message
 
-    def _validate_message(self):
-        valid = False
-        if self.msg and self.min_msg_length <= len(self.msg) <= self.max_msg_length:
-            # remove spaces
-            test_str = self.msg.replace(' ', '')
-            # remove allowed non alpha-numeric chars
-            for char in self.allowed_chars:
-                test_str = test_str.replace(char, '')
-            # only alpha numeric string accepted
-            valid = test_str.isalnum()
-        return valid
-
     @property
     def message(self):
-        if self._validate_message():
-            new_msg = "{}".format(self.msg)
-        else:
-            new_msg = self.default_msg
-        return new_msg
+        try:
+            msg = format_message(
+                self.msg,
+                min_length=self.min_length,
+                max_length=self.max_length,
+                allowed_chars=self.allowed_chars)
+        except Exception as e:
+            LOGGER.warning(f"process error formatting failed: {e}")
+            msg = None
+        if not msg:
+            msg = self.default_msg
+        return msg


=====================================
pywps/configuration.py
=====================================
@@ -100,6 +100,8 @@ def load_configuration(cfgfiles=None):
     CONFIG.add_section('processing')
     CONFIG.set('processing', 'mode', 'default')
     CONFIG.set('processing', 'path', os.path.dirname(os.path.realpath(sys.argv[0])))
+    # https://github.com/natefoo/slurm-drmaa
+    CONFIG.set('processing', 'drmaa_native_specification', '')
 
     CONFIG.add_section('logging')
     CONFIG.set('logging', 'file', '')


=====================================
pywps/inout/basic.py
=====================================
@@ -932,7 +932,7 @@ class ComplexInput(BasicIO, BasicComplex, IOHandler):
         file_path = os.path.abspath(file_path)
         # build allowed paths list
         inputpaths = config.get_config_value('server', 'allowedinputpaths')
-        allowed_paths = [os.path.abspath(p.strip()) for p in inputpaths.split(':') if p.strip()]
+        allowed_paths = [os.path.abspath(p.strip()) for p in inputpaths.split(os.pathsep) if p.strip()]
         for allowed_path in allowed_paths:
             if file_path.startswith(allowed_path):
                 LOGGER.debug("Accepted file url as input.")


=====================================
pywps/processing/scheduler.py
=====================================
@@ -53,6 +53,9 @@ class Scheduler(Processing):
                 jt.args = ['-c', cfg_file, dump_filename]
             else:
                 jt.args = [dump_filename]
+            drmaa_native_specification = config.get_config_value('processing', 'drmaa_native_specification')
+            if drmaa_native_specification:
+                jt.nativeSpecification = drmaa_native_specification
             jt.joinFiles = True
             jt.outputPath = ":{}".format(os.path.join(self.job.workdir, "job-output.txt"))
             # run job


=====================================
tests/test_app_exceptions.py
=====================================
@@ -0,0 +1,35 @@
+##################################################################
+# Copyright 2018 Open Source Geospatial Foundation and others    #
+# licensed under MIT, Please consult LICENSE.txt for details     #
+##################################################################
+
+import unittest
+from pywps.app.exceptions import format_message, ProcessError, DEFAULT_ALLOWED_CHARS
+
+
+class AppExceptionsTest(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def test_format_message(self):
+        assert format_message('no data available') == 'no data available'
+        assert format_message(' no data available! ') == 'no data available!'
+        assert format_message('no') == ''
+        assert format_message('no data available', max_length=7) == 'no data'
+        assert format_message('no &data% available') == 'no data available'
+        assert format_message(DEFAULT_ALLOWED_CHARS) == DEFAULT_ALLOWED_CHARS
+
+    def test_process_error(self):
+        assert ProcessError(' no &data available!').message == 'no data available!'
+        assert ProcessError('no', min_length=2).message == 'no'
+        assert ProcessError('0 data available', max_length=6).message == '0 data'
+        assert ProcessError('no data? not available!', allowed_chars='?').message == 'no data? not available'
+        assert ProcessError('').message == 'Sorry, process failed. Please check server error log.'
+        assert ProcessError(1234).message == 'Sorry, process failed. Please check server error log.'
+        try:
+            raise ProcessError('no data!!')
+        except ProcessError as e:
+            assert f"{e}" == 'no data!!'
+        else:
+            assert False



View it on GitLab: https://salsa.debian.org/debian-gis-team/pywps/-/commit/aac10d81bf416dc6035eda1021ae87f71983e2ca

-- 
View it on GitLab: https://salsa.debian.org/debian-gis-team/pywps/-/commit/aac10d81bf416dc6035eda1021ae87f71983e2ca
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20200916/84bb2731/attachment-0001.html>


More information about the Pkg-grass-devel mailing list