[Git][debian-gis-team/pywps][experimental] 6 commits: New upstream version 4.4.3
Bas Couwenberg (@sebastic)
gitlab at salsa.debian.org
Tue May 11 09:40:21 BST 2021
Bas Couwenberg pushed to branch experimental at Debian GIS Project / pywps
Commits:
e421c0d4 by Bas Couwenberg at 2021-05-11T08:41:54+02:00
New upstream version 4.4.3
- - - - -
2cda0f6e by Bas Couwenberg at 2021-05-11T08:41:55+02:00
Update upstream source from tag 'upstream/4.4.3'
Update to upstream version '4.4.3'
with Debian dir 422fcb9ff84d6078b60f470a461271ced8877161
- - - - -
767637e2 by Bas Couwenberg at 2021-05-11T08:42:07+02:00
New upstream release.
- - - - -
5517a76f by Bas Couwenberg at 2021-05-11T08:44:35+02:00
Add python3-pytest to build dependencies.
- - - - -
c5894037 by Bas Couwenberg at 2021-05-11T08:50:03+02:00
Refresh patches.
- - - - -
e52207f1 by Bas Couwenberg at 2021-05-11T08:51:19+02:00
Set distribution to experimental.
- - - - -
29 changed files:
- .github/workflows/main.yml
- CONTRIBUTORS.md
- VERSION.txt
- debian/changelog
- debian/control
- debian/patches/offline-tests.patch
- − docs/metalinkprocess.py
- docs/process.rst
- pywps/__init__.py
- pywps/app/WPSRequest.py
- pywps/configuration.py
- pywps/inout/basic.py
- pywps/inout/outputs.py
- pywps/inout/storage/file.py
- pywps/response/execute.py
- pywps/tests.py
- + pywps/util.py
- pywps/validator/__init__.py
- requirements-dev.txt
- setup.cfg
- tests/processes/__init__.py
- − tests/processes/metalinkprocess.py
- + tests/processes/metalinkprocess.py
- tests/test_describe.py
- tests/test_execute.py
- tests/test_filestorage.py
- tests/test_inout.py
- tests/test_ows.py
- tests/validator/test_complexvalidators.py
Changes:
=====================================
.github/workflows/main.yml
=====================================
@@ -15,7 +15,7 @@ jobs:
- uses: actions/checkout at v2
- name: Install packages
run: |
- sudo apt-get -y install gdal-bin libgdal-dev libnetcdf-dev libhdf5-dev
+ sudo apt-get update && sudo apt-get -y install gdal-bin libgdal-dev libnetcdf-dev libhdf5-dev
- uses: actions/setup-python at v2
name: Setup Python ${{ matrix.python-version }}
with:
@@ -28,7 +28,7 @@ jobs:
pip3 install -r requirements-extra.txt
pip3 install -r requirements-gdal.txt
- name: run tests ⚙️
- run: python3 -m unittest tests
+ run: pytest -v tests
- name: run coveralls ⚙️
run: coveralls
if: matrix.python-version == 3.6
=====================================
CONTRIBUTORS.md
=====================================
@@ -9,6 +9,7 @@
* @SiggyF Fedor Baart
* @jonas-eberle Jonas Eberle
* @cehbrecht Carsten Ehbrecht
+* @idanmiara Idan Miara
# Contributor to older versions of PyWPS (< 4.x)
@@ -28,5 +29,5 @@
# NOTE
-This file is keeped manually. Feel free to contact us, if your contribution is
+This file is kept manually. Feel free to contact us, if your contribution is
missing here.
=====================================
VERSION.txt
=====================================
@@ -1 +1 @@
-4.4.2
+4.4.3
=====================================
debian/changelog
=====================================
@@ -1,3 +1,11 @@
+pywps (4.4.3-1~exp1) experimental; urgency=medium
+
+ * New upstream release.
+ * Add python3-pytest to build dependencies.
+ * Refresh patches.
+
+ -- Bas Couwenberg <sebastic at debian.org> Tue, 11 May 2021 08:51:06 +0200
+
pywps (4.4.2-1~exp1) experimental; urgency=medium
* New upstream release.
=====================================
debian/control
=====================================
@@ -18,6 +18,7 @@ Build-Depends: debhelper (>= 10~),
python3-lxml,
python3-netcdf4,
python3-owslib,
+ python3-pytest,
python3-requests,
python3-setuptools,
python3-sphinx,
=====================================
debian/patches/offline-tests.patch
=====================================
@@ -22,7 +22,7 @@ Forwarded: not-needed
self.skipTest('GRASS lib not found')
--- a/tests/validator/test_complexvalidators.py
+++ b/tests/validator/test_complexvalidators.py
-@@ -79,7 +79,8 @@ class ValidateTest(unittest.TestCase):
+@@ -80,7 +80,8 @@ class ValidateTest(unittest.TestCase):
self.assertTrue(validategml(gml_input, MODE.SIMPLE), 'SIMPLE validation')
if WITH_GDAL:
self.assertTrue(validategml(gml_input, MODE.STRICT), 'STRICT validation')
@@ -32,27 +32,29 @@ Forwarded: not-needed
gml_input.stream.close()
def test_json_validator(self):
-@@ -140,6 +141,7 @@ class ValidateTest(unittest.TestCase):
+@@ -141,6 +142,8 @@ class ValidateTest(unittest.TestCase):
else:
self.assertFalse(validatenetcdf(netcdf_input, MODE.STRICT), 'STRICT validation')
+ @unittest.skipIf('OFFLINE_TESTS' in os.environ, "offline tests only")
++ @pytest.mark.skipif('OFFLINE_TESTS' in os.environ, reason="offline tests only")
+ @pytest.mark.xfail(reason="test.opendap.org is offline")
def test_dods_validator(self):
opendap_input = ComplexInput('dods', 'opendap test', [FORMATS.DODS,])
- opendap_input.url = "http://test.opendap.org:80/opendap/netcdf/examples/sresa1b_ncar_ccsm3_0_run1_200001.nc"
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
-@@ -230,6 +230,7 @@ def get_output(doc):
+@@ -231,6 +231,8 @@ def get_output(doc):
class ExecuteTest(unittest.TestCase):
"""Test for Exeucte request KVP request"""
+ @unittest.skipIf('OFFLINE_TESTS' in os.environ, "offline tests only")
++ @pytest.mark.skipif('OFFLINE_TESTS' in os.environ, reason="offline tests only")
+ @pytest.mark.xfail(reason="test.opendap.org is offline")
def test_dods(self):
if not WITH_NC4:
- self.skipTest('netCDF4 not installed')
--- a/tests/test_inout.py
+++ b/tests/test_inout.py
-@@ -131,6 +131,7 @@ class IOHandlerTest(unittest.TestCase):
+@@ -132,6 +132,7 @@ class IOHandlerTest(unittest.TestCase):
with self.assertRaises(TypeError):
self.iohandler[0].data = '5'
@@ -60,7 +62,7 @@ Forwarded: not-needed
def test_url(self):
if not service_ok('https://demo.mapserver.org'):
self.skipTest("mapserver is unreachable")
-@@ -521,6 +522,7 @@ class ComplexOutputTest(unittest.TestCas
+@@ -515,6 +516,7 @@ class ComplexOutputTest(unittest.TestCas
b = self.complex_out.base64
self.assertEqual(base64.b64decode(b).decode(), self.data)
=====================================
docs/metalinkprocess.py deleted
=====================================
@@ -1,43 +0,0 @@
-from pywps import Process, LiteralInput, ComplexOutput, FORMATS
-from pywps.inout.outputs import MetaLink4, MetaFile
-
-
-class MultipleOutputs(Process):
- def __init__(self):
- inputs = [
- LiteralInput('count', 'Number of output files',
- abstract='The number of generated output files.',
- data_type='integer',
- default=2)]
- outputs = [
- ComplexOutput('output', 'Metalink4 output',
- abstract='A metalink file storing URIs to multiple files',
- as_reference=True,
- supported_formats=[FORMATS.META4])
- ]
-
- super(MultipleOutputs, self).__init__(
- self._handler,
- identifier='multiple-outputs',
- title='Multiple Outputs',
- abstract='Produces multiple files and returns a document'
- ' with references to these files.',
- inputs=inputs,
- outputs=outputs,
- store_supported=True,
- status_supported=True
- )
-
- def _handler(self, request, response):
- max_outputs = request.inputs['count'][0].data
-
- ml = MetaLink4('test-ml-1', 'MetaLink with links to text files.', workdir=self.workdir)
- for i in range(max_outputs):
- # Create a MetaFile instance, which instantiates a ComplexOutput object.
- mf = MetaFile('output_{}'.format(i), 'Test output', format=FORMATS.TEXT)
- mf.data = 'output: {}'.format(i) # or mf.file = <path to file> or mf.url = <url>
- ml.append(mf)
-
- # The `xml` property of the Metalink4 class returns the metalink content.
- response.outputs['output'].data = ml.xml
- return response
=====================================
docs/process.rst
=====================================
@@ -385,7 +385,7 @@ instance.
Example process
---------------
-.. literalinclude:: metalinkprocess.py
+.. literalinclude:: ../tests/processes/metalinkprocess.py
:language: python
Process Exceptions
=====================================
pywps/__init__.py
=====================================
@@ -9,7 +9,7 @@ import os
from lxml.builder import ElementMaker
-__version__ = "4.4.2"
+__version__ = "4.4.3"
LOGGER = logging.getLogger('PYWPS')
LOGGER.debug('setting core variables')
=====================================
pywps/app/WPSRequest.py
=====================================
@@ -18,6 +18,7 @@ from pywps import configuration
from pywps import get_version_from_ns
import json
+from urllib.parse import unquote
LOGGER = logging.getLogger("PYWPS")
@@ -509,15 +510,15 @@ def get_data_from_kvp(data, part=None):
# First field is identifier and its value
(identifier, val) = fields[0].split("=")
io['identifier'] = identifier
- io['data'] = val
+ io['data'] = unquote(val)
# Get the attributes of the data
for attr in fields[1:]:
(attribute, attr_val) = attr.split('=', 1)
if attribute == 'xlink:href':
- io['href'] = attr_val
+ io['href'] = unquote(attr_val)
else:
- io[attribute] = attr_val
+ io[attribute] = unquote(attr_val)
# Add the input/output with all its attributes and values to the
# dictionary
=====================================
pywps/configuration.py
=====================================
@@ -17,6 +17,8 @@ import configparser
__author__ = "Calin Ciociu"
+from pywps.util import file_uri
+
RAW_OPTIONS = [('logging', 'format'), ]
CONFIG = None
@@ -76,7 +78,7 @@ def load_configuration(cfgfiles=None):
CONFIG.set('server', 'temp_path', tempfile.gettempdir())
CONFIG.set('server', 'processes_path', '')
outputpath = tempfile.gettempdir()
- CONFIG.set('server', 'outputurl', 'file://{}'.format(outputpath))
+ CONFIG.set('server', 'outputurl', file_uri(outputpath))
CONFIG.set('server', 'outputpath', outputpath)
# list of allowed input paths (file url input) seperated by ':'
CONFIG.set('server', 'allowedinputpaths', '')
=====================================
pywps/inout/basic.py
=====================================
@@ -2,6 +2,7 @@
# Copyright 2018 Open Source Geospatial Foundation and others #
# licensed under MIT, Please consult LICENSE.txt for details #
##################################################################
+from pathlib import PurePath
from pywps.translations import lower_case_dict
from io import StringIO
@@ -32,6 +33,8 @@ from copy import deepcopy
from io import BytesIO
import humanize
+import weakref
+
_SOURCE_TYPE = namedtuple('SOURCE_TYPE', 'MEMORY, FILE, STREAM, DATA, URL')
SOURCE_TYPE = _SOURCE_TYPE(0, 1, 2, 3, 4)
@@ -54,13 +57,6 @@ def _is_textfile(filename):
return is_text
-def extend_instance(obj, cls):
- """Apply mixins to a class instance after creation."""
- base_cls = obj.__class__
- base_cls_name = obj.__class__.__name__
- obj.__class__ = type(base_cls_name, (cls, base_cls), {})
-
-
class UOM(object):
"""
:param uom: unit of measure
@@ -82,12 +78,66 @@ class UOM(object):
return self.uom == other.uom
+class NoneIOHandler(object):
+ """Base class for implementation of IOHandler internal"""
+
+ prop = None
+
+ def __init__(self, ref):
+ self._ref = weakref.ref(ref)
+
+ @property
+ def file(self):
+ """Return filename."""
+ return None
+
+ @property
+ def data(self):
+ """Read file and return content."""
+ return None
+
+ @property
+ def base64(self):
+ """Return base64 encoding of data."""
+ return None
+
+ @property
+ def stream(self):
+ """Return stream object."""
+ return None
+
+ @property
+ def mem(self):
+ """Return memory object."""
+ return None
+
+ @property
+ def url(self):
+ """Return url to file."""
+ return None
+
+ @property
+ def size(self):
+ """Length of the linked content in octets."""
+ return None
+
+ @property
+ def post_data(self):
+ raise NotImplementedError
+
+ # Will raise an error if used on invalid object
+ @post_data.setter
+ def post_data(self, value):
+ raise NotImplementedError
+
+
class IOHandler(object):
- """Base IO handling class subclassed by specialized versions: FileHandler, UrlHandler, DataHandler, etc.
+ """Base IO handling class that handle multple IO types
- If the specialized handling class is not known when the object is created, instantiate the object with IOHandler.
- The first time the `file`, `url` or `data` attribute is set, the associated subclass will be automatically
- registered. Once set, the specialized subclass cannot be switched.
+ This class is created with NoneIOHandler that have no data
+ inside. To initialise data you can set the `file`, `url`, `data` or
+ `stream` attribute. If reset one of this attribute old data are lost and
+ replaced by the new one.
:param workdir: working directory, to save temporal file objects in.
:param mode: ``MODE`` validation mode.
@@ -119,7 +169,6 @@ class IOHandler(object):
>>>
>>> # testing file object on input
>>> ioh_file.file = fileobj.name
- >>> assert isinstance(ioh_file, FileHandler
>>> assert ioh_file.file == fileobj.name
>>> assert isinstance(ioh_file.stream, RawIOBase)
>>> # skipped assert isinstance(ioh_file.memory_object, POSH)
@@ -128,16 +177,16 @@ class IOHandler(object):
>>> ioh_stream = IOHandler(workdir=tmp)
>>> assert ioh_stream.workdir == tmp
>>> ioh_stream.stream = FileIO(fileobj.name,'r')
- >>> assert isinstance(ioh_stream, StreamHandler)
>>> assert open(ioh_stream.file).read() == ioh_file.stream.read()
>>> assert isinstance(ioh_stream.stream, RawIOBase)
"""
- prop = None
def __init__(self, workdir=None, mode=MODE.NONE):
+
+ self._iohandler = NoneIOHandler(self)
+
# Internal defaults for class and subclass properties.
self._workdir = None
- self._reset_cache()
# Set public defaults
self.workdir = workdir
@@ -149,21 +198,6 @@ class IOHandler(object):
self.uuid = None # request identifier
self.data_set = False
- # This creates dummy property setters and getters for `file`, `data`, `url`, `stream` that
- # 1. register the subclass methods according to the given property
- # 2. replace the property setter by the subclass property setter
- # 3. set the property
- self._create_fset_properties()
-
- def _reset_cache(self):
- """Sets all internal objects to None."""
- self._file = None
- self._data = None
- self._post_data = None
- self._stream = None
- self._url = None
- self.data_set = False
-
def _check_valid(self):
"""Validate this input using given validator
"""
@@ -183,6 +217,7 @@ class IOHandler(object):
@workdir.setter
def workdir(self, path):
"""Set working temporary directory for files to be stored in."""
+
if path is not None:
if not os.path.exists(path):
os.makedirs(path)
@@ -253,58 +288,94 @@ class IOHandler(object):
"""
return deepcopy(self)
- @staticmethod
- def _create_fset_properties():
- """Create properties that when set for the first time, will determine
- the instance's handler class.
-
- Example
- -------
- >>> h = IOHandler()
- >>> isinstance(h, DataHandler)
- False
- >>> h.data = 1 # Mixes the DataHandler class to IOHandler. h inherits DataHandler methods.
- >>> isinstance(h, DataHandler)
- True
-
- Note that trying to set another attribute (e.g. `h.file = 'a.txt'`) will raise an AttributeError.
+ @property
+ def base64(self):
+ """Return raw data
+ WARNING: may be bytes or str"""
+ return self._iohandler.base64
+
+ @property
+ def size(self):
+ """Return object size in bytes.
"""
- for cls in (FileHandler, DataHandler, StreamHandler, UrlHandler):
- def fset(s, value, kls=cls):
- """Assign the handler class and set the value to the attribute.
+ return self._iohandler.size
- This function will only be called once. The next `fset` will
- use the subclass' property.
- """
- # Add cls methods to this instance.
- extend_instance(s, kls)
+ @property
+ def file(self):
+ """Return a file name"""
+ return self._iohandler.file
- # Set the attribute value through the associated cls property.
- setattr(s, kls.prop, value)
+ @file.setter
+ def file(self, value):
+ self._iohandler = FileHandler(value, self)
+ self._check_valid()
- setattr(IOHandler, cls.prop, property(fget=lambda x: None, fset=fset))
+ @property
+ def data(self):
+ """Return raw data
+ WARNING: may be bytes or str"""
+ return self._iohandler.data
+ @data.setter
+ def data(self, value):
+ self._iohandler = DataHandler(value, self)
+ self._check_valid()
-class FileHandler(IOHandler):
+ @property
+ def stream(self):
+ """Return stream of data
+ WARNING: may be FileIO or StringIO"""
+ return self._iohandler.stream
+
+ @stream.setter
+ def stream(self, value):
+ self._iohandler = StreamHandler(value, self)
+ self._check_valid()
+
+ @property
+ def url(self):
+ """Return the url of data"""
+ return self._iohandler.url
+
+ @url.setter
+ def url(self, value):
+ self._iohandler = UrlHandler(value, self)
+ self._check_valid()
+
+ # FIXME: post_data is only related to url, this should be initialize with url setter
+ @property
+ def post_data(self):
+ return self._iohandler.post_data
+
+ # Will raise an arror if used on invalid object
+ @post_data.setter
+ def post_data(self, value):
+ self._iohandler.post_data = value
+
+ @property
+ def prop(self):
+ return self._iohandler.prop
+
+
+class FileHandler(NoneIOHandler):
prop = 'file'
+ def __init__(self, value, ref):
+ self._ref = weakref.ref(ref)
+ self._data = None
+ self._stream = None
+ self._file = os.path.abspath(value)
+
@property
def file(self):
"""Return filename."""
return self._file
- @file.setter
- def file(self, value):
- """Set file name"""
- self._reset_cache()
- self._file = os.path.abspath(value)
- self._check_valid()
-
@property
def data(self):
"""Read file and return content."""
if self._data is None:
- openmode = self._openmode()
+ openmode = self._openmode(self._ref())
kwargs = {} if 'b' in openmode else {'encoding': 'utf8'}
with open(self.file, mode=openmode, **kwargs) as fh:
self._data = fh.read()
@@ -320,38 +391,33 @@ class FileHandler(IOHandler):
def stream(self):
"""Return stream object."""
from io import FileIO
- if getattr(self, '_stream', None) and not self._stream.closed:
+ if self._stream and not self._stream.closed:
self._stream.close()
self._stream = FileIO(self.file, mode='r', closefd=True)
return self._stream
- @property
- def mem(self):
- """Return memory object."""
- raise NotImplementedError
-
@property
def url(self):
"""Return url to file."""
- import pathlib
- return pathlib.PurePosixPath(self.file).as_uri()
+ result = PurePath(self.file).as_uri()
+ return result
@property
def size(self):
"""Length of the linked content in octets."""
return os.stat(self.file).st_size
- def _openmode(self, data=None):
+ def _openmode(self, base, data=None):
openmode = 'r'
# in Python 3 we need to open binary files in binary mode.
checked = False
- if hasattr(self, 'data_format'):
- if self.data_format.encoding == 'base64':
+ if hasattr(base, 'data_format'):
+ if base.data_format.encoding == 'base64':
# binary, when the data is to be encoded to base64
openmode += 'b'
checked = True
- elif 'text/' in self.data_format.mime_type:
+ elif 'text/' in base.data_format.mime_type:
# not binary, when mime_type is 'text/'
checked = True
# when we can't guess it from the mime_type, we need to check the file.
@@ -364,6 +430,12 @@ class FileHandler(IOHandler):
class DataHandler(FileHandler):
prop = 'data'
+ def __init__(self, value, ref):
+ self._ref = weakref.ref(ref)
+ self._file = None
+ self._stream = None
+ self._data = value
+
def _openmode(self, data=None):
openmode = 'w'
if isinstance(data, bytes):
@@ -375,13 +447,7 @@ class DataHandler(FileHandler):
@property
def data(self):
"""Return data."""
- return getattr(self, '_data', None)
-
- @data.setter
- def data(self, value):
- self._reset_cache()
- self._data = value
- self._check_valid()
+ return self._data
@property
def file(self):
@@ -390,7 +456,7 @@ class DataHandler(FileHandler):
Requesting the file attributes writes the data to a temporary file on disk.
"""
if self._file is None:
- self._file = self._build_file_name()
+ self._file = self._ref()._build_file_name()
openmode = self._openmode(self.data)
kwargs = {} if 'b' in openmode else {'encoding': 'utf8'}
with open(self._file, openmode, **kwargs) as fh:
@@ -410,18 +476,17 @@ class DataHandler(FileHandler):
class StreamHandler(DataHandler):
prop = 'stream'
+ def __init__(self, value, ref):
+ self._ref = weakref.ref(ref)
+ self._file = None
+ self._data = None
+ self._stream = value
+
@property
def stream(self):
"""Return the stream."""
return self._stream
- @stream.setter
- def stream(self, value):
- """Set the stream."""
- self._reset_cache()
- self._stream = value
- self._check_valid()
-
@property
def data(self):
"""Return the data from the stream."""
@@ -433,18 +498,19 @@ class StreamHandler(DataHandler):
class UrlHandler(FileHandler):
prop = 'url'
+ def __init__(self, value, ref):
+ self._ref = weakref.ref(ref)
+ self._file = None
+ self._data = None
+ self._stream = None
+ self._url = value
+ self._post_data = None
+
@property
def url(self):
"""Return the URL."""
return self._url
- @url.setter
- def url(self, value):
- """Set the URL value."""
- self._reset_cache()
- self._url = value
- self._check_valid()
-
@property
def file(self):
"""Downloads URL and return file pointer.
@@ -453,7 +519,7 @@ class UrlHandler(FileHandler):
if self._file is not None:
return self._file
- self._file = self._build_file_name(href=self.url)
+ self._file = self._ref()._build_file_name(href=self.url)
max_byte_size = self.max_size()
@@ -465,7 +531,7 @@ class UrlHandler(FileHandler):
raise NoApplicableCode('File reference error: {}'.format(e))
error_message = 'File size for input "{}" exceeded. Maximum allowed: {}'.format(
- self.inpt.get('identifier', '?'), humanize.naturalsize(max_byte_size))
+ self._ref().inpt.get('identifier', '?'), humanize.naturalsize(max_byte_size))
if int(max_byte_size) > 0:
if int(data_size) > int(max_byte_size):
@@ -529,7 +595,7 @@ class UrlHandler(FileHandler):
return byte_size
-class SimpleHandler(DataHandler):
+class SimpleHandler(IOHandler):
"""Data handler for Literal In- and Outputs
>>> class Int_type(object):
@@ -552,19 +618,19 @@ class SimpleHandler(DataHandler):
"""
def __init__(self, workdir=None, data_type=None, mode=MODE.NONE):
- DataHandler.__init__(self, workdir=workdir, mode=mode)
+ IOHandler.__init__(self, workdir=workdir, mode=mode)
if data_type not in LITERAL_DATA_TYPES:
raise ValueError('data_type {} not in {}'.format(data_type, LITERAL_DATA_TYPES))
self.data_type = data_type
- @DataHandler.data.setter
+ @IOHandler.data.setter
def data(self, value):
"""Set data value. Inputs are converted into target format.
"""
if self.data_type and value is not None:
value = convert(self.data_type, value)
- DataHandler.data.fset(self, value)
+ IOHandler.data.fset(self, value)
class BasicIO:
@@ -825,7 +891,7 @@ class LiteralOutput(BasicIO, BasicLiteral, SimpleHandler):
return validate_anyvalue
-class BBoxInput(BasicIO, BasicBoundingBox, DataHandler):
+class BBoxInput(BasicIO, BasicBoundingBox, IOHandler):
"""Basic Bounding box input abstract class
"""
@@ -845,7 +911,7 @@ class BBoxInput(BasicIO, BasicBoundingBox, DataHandler):
translations=translations,
)
BasicBoundingBox.__init__(self, crss, dimensions)
- DataHandler.__init__(self, workdir=workdir, mode=mode)
+ IOHandler.__init__(self, workdir=workdir, mode=mode)
if default_type != SOURCE_TYPE.DATA:
raise InvalidParameterValue("Source types other than data are not supported.")
@@ -856,7 +922,7 @@ class BBoxInput(BasicIO, BasicBoundingBox, DataHandler):
self._set_default_value(default, default_type)
-class BBoxOutput(BasicIO, BasicBoundingBox, DataHandler):
+class BBoxOutput(BasicIO, BasicBoundingBox, IOHandler):
"""Basic BoundingBox output class
"""
@@ -864,7 +930,7 @@ class BBoxOutput(BasicIO, BasicBoundingBox, DataHandler):
dimensions=None, workdir=None, mode=MODE.NONE, translations=None):
BasicIO.__init__(self, identifier, title, abstract, keywords, translations=translations)
BasicBoundingBox.__init__(self, crss, dimensions)
- DataHandler.__init__(self, workdir=workdir, mode=mode)
+ IOHandler.__init__(self, workdir=workdir, mode=mode)
self._storage = None
@property
@@ -909,8 +975,6 @@ class ComplexInput(BasicIO, BasicComplex, IOHandler):
def file_handler(self, inpt):
"""<wps:Reference /> handler.
Used when href is a file url."""
- extend_instance(self, FileHandler)
-
# check if file url is allowed
self._validate_file_input(href=inpt.get('href'))
# save the file reference input in workdir
=====================================
pywps/inout/outputs.py
=====================================
@@ -8,6 +8,7 @@ WPS Output classes
import lxml.etree as etree
import os
+import re
from pywps.app.Common import Metadata
from pywps.exceptions import InvalidParameterValue
from pywps.inout import basic
@@ -145,11 +146,6 @@ class ComplexOutput(basic.ComplexOutput):
'translations': self.translations,
}
- if self.as_reference:
- data = self._json_reference(data)
- else:
- data = self._json_data(data)
-
if self.data_format:
if self.data_format.mime_type:
data['mimetype'] = self.data_format.mime_type
@@ -158,6 +154,11 @@ class ComplexOutput(basic.ComplexOutput):
if self.data_format.schema:
data['schema'] = self.data_format.schema
+ if self.as_reference:
+ data = self._json_reference(data)
+ else:
+ data = self._json_data(data)
+
return data
@classmethod
@@ -209,6 +210,8 @@ class ComplexOutput(basic.ComplexOutput):
def _json_data(self, data):
"""Return Data node
"""
+ # Match only data that are safe CDATA pattern.
+ CDATA_PATTERN = re.compile(r'^<!\[CDATA\[((?!\]\]>).)*\]\]>$')
data["type"] = "complex"
@@ -222,15 +225,39 @@ class ComplexOutput(basic.ComplexOutput):
else:
if self.data_format.encoding == 'base64':
data["data"] = self.base64.decode('utf-8')
-
else:
# Otherwise we assume all other formats are unsafe and need to be enclosed in a CDATA tag.
if isinstance(self.data, bytes):
- out = self.data.encode(self.data_format.encoding or 'utf-8')
+ # Try to inline data as text but if fail encode is in base64
+ if self.data_format.encoding == 'utf-8':
+ out = self.data.decode('utf-8')
+ # If data is already enclosed with CDATA pattern, do not add it twice
+ if CDATA_PATTERN.match(out):
+ data["data"] = out
+ else:
+ # Check if the data does not contain ]]> patern if is safe to use CDATA
+ # other wise we fallback to base64 encoding.
+ if not re.search('\\]\\]>', out):
+ data["data"] = '<![CDATA[{}]]>'.format(out)
+ else:
+ data['encoding'] = 'base64' # override the unsafe encoding
+ data["data"] = self.base64.decode('utf-8')
+ else:
+ data['encoding'] = 'base64' # override the unsafe encoding
+ data["data"] = self.base64.decode('utf-8')
else:
- out = self.data
-
- data["data"] = '<![CDATA[{}]]>'.format(out)
+ out = str(self.data)
+ # If data is already enclose with CDATApatern do not add it twise
+ if CDATA_PATTERN.match(out):
+ data["data"] = out
+ else:
+ # Check if the data does not contain ]]> patern if is safe to use CDATA
+ # other wise we fallback to base64 encoding.
+ if not re.search('\\]\\]>', out):
+ data["data"] = '<![CDATA[{}]]>'.format(out)
+ else:
+ data['encoding'] = 'base64' # override the unsafe encoding
+ data["data"] = self.base64.decode('utf-8')
return data
=====================================
pywps/inout/storage/file.py
=====================================
@@ -157,7 +157,7 @@ class FileStorage(CachedStorage):
if isinstance(destination, IOHandler):
output_name, _ = _build_output_name(destination)
just_file_name = os.path.basename(output_name)
- dst = "{}/{}".format(destination.uuid, just_file_name)
+ dst = f"{destination.uuid}/{just_file_name}"
else:
dst = destination
=====================================
pywps/response/execute.py
=====================================
@@ -14,6 +14,7 @@ from werkzeug.wrappers import Response
from pywps.response.status import WPS_STATUS
from pywps.response import WPSResponse
from pywps.inout.formats import FORMATS
+from pywps.inout.outputs import ComplexOutput
import urllib.parse as urlparse
from urllib.parse import urlencode
@@ -202,8 +203,14 @@ class ExecuteResponse(WPSResponse):
wps_output_value = self.outputs[wps_output_identifier]
if wps_output_value.source_type is None:
return NoApplicableCode("Expected output was not generated")
+ suffix = ''
+ if isinstance(wps_output_value, ComplexOutput):
+ if wps_output_value.data_format.extension is not None:
+ suffix = wps_output_value.data_format.extension
return Response(wps_output_value.data,
- mimetype=self.wps_request.outputs[wps_output_identifier].get('mimetype', None))
+ mimetype=wps_output_value.data_format.mime_type,
+ headers={'Content-Disposition': 'attachment; filename="{}"'
+ .format(wps_output_identifier + suffix)})
else:
if not self.doc:
return NoApplicableCode("Output was not generated")
=====================================
pywps/tests.py
=====================================
@@ -2,6 +2,8 @@
# Copyright 2018 Open Source Geospatial Foundation and others #
# licensed under MIT, Please consult LICENSE.txt for details #
##################################################################
+import tempfile
+from pathlib import Path
import lxml.etree
import requests
@@ -172,5 +174,6 @@ def assert_wps_version(response, version="1.0.0"):
'/ows:ServiceTypeVersion')
found_version = elem[0].text
assert version == found_version
- with open("/tmp/out.xml", "wb") as out:
+ tmp = Path(tempfile.mkdtemp())
+ with open(tmp / "out.xml", "wb") as out:
out.writelines(response.response)
=====================================
pywps/util.py
=====================================
@@ -0,0 +1,26 @@
+##################################################################
+# Copyright 2018 Open Source Geospatial Foundation and others #
+# licensed under MIT, Please consult LICENSE.txt for details #
+##################################################################
+
+import platform
+from typing import Union
+
+from pathlib import Path
+from urllib.parse import urlparse
+
+is_windows = platform.system() == 'Windows'
+
+
+def file_uri(path: Union[str, Path]) -> str:
+ path = Path(path)
+ path = path.as_uri()
+ return str(path)
+
+
+def uri_to_path(uri) -> str:
+ p = urlparse(uri)
+ path = p.path
+ if is_windows:
+ path = str(Path(path)).lstrip('\\')
+ return path
=====================================
pywps/validator/__init__.py
=====================================
@@ -15,7 +15,7 @@ from pywps.validator.base import emptyvalidator
LOGGER = logging.getLogger('PYWPS')
_VALIDATORS = {
- 'application/vnd.geo+json': validategeojson,
+ 'application/geo+json': validategeojson,
'application/json': validatejson,
'application/x-zipped-shp': validateshapefile,
'application/gml+xml': validategml,
=====================================
requirements-dev.txt
=====================================
@@ -1,8 +1,9 @@
coverage
coveralls
+pytest
flake8
pylint
-Sphinx
+Sphinx<4.0
twine
wheel
-bump2version
\ No newline at end of file
+bump2version
=====================================
setup.cfg
=====================================
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 4.4.2
+current_version = 4.4.3
commit = False
tag = False
parse = (?P<major>\d+)\.(?P<minor>\d+).(?P<patch>\d+)
=====================================
tests/processes/__init__.py
=====================================
@@ -42,7 +42,7 @@ class Greeter(Process):
@staticmethod
def greeter(request, response):
name = request.inputs['name'][0].data
- assert type(name) is text_type
+ assert type(name) is str
response.outputs['message'].data = "Hello {}!".format(name)
return response
=====================================
tests/processes/metalinkprocess.py deleted
=====================================
@@ -1 +0,0 @@
-../../docs/metalinkprocess.py
\ No newline at end of file
=====================================
tests/processes/metalinkprocess.py
=====================================
@@ -0,0 +1,43 @@
+from pywps import Process, LiteralInput, ComplexOutput, FORMATS
+from pywps.inout.outputs import MetaLink4, MetaFile
+
+
+class MultipleOutputs(Process):
+ def __init__(self):
+ inputs = [
+ LiteralInput('count', 'Number of output files',
+ abstract='The number of generated output files.',
+ data_type='integer',
+ default=2)]
+ outputs = [
+ ComplexOutput('output', 'Metalink4 output',
+ abstract='A metalink file storing URIs to multiple files',
+ as_reference=True,
+ supported_formats=[FORMATS.META4])
+ ]
+
+ super(MultipleOutputs, self).__init__(
+ self._handler,
+ identifier='multiple-outputs',
+ title='Multiple Outputs',
+ abstract='Produces multiple files and returns a document'
+ ' with references to these files.',
+ inputs=inputs,
+ outputs=outputs,
+ store_supported=True,
+ status_supported=True
+ )
+
+ def _handler(self, request, response):
+ max_outputs = request.inputs['count'][0].data
+
+ ml = MetaLink4('test-ml-1', 'MetaLink with links to text files.', workdir=self.workdir)
+ for i in range(max_outputs):
+ # Create a MetaFile instance, which instantiates a ComplexOutput object.
+ mf = MetaFile('output_{}'.format(i), 'Test output', format=FORMATS.TEXT)
+ mf.data = 'output: {}'.format(i) # or mf.file = <path to file> or mf.url = <url>
+ ml.append(mf)
+
+ # The `xml` property of the Metalink4 class returns the metalink content.
+ response.outputs['output'].data = ml.xml
+ return response
=====================================
tests/test_describe.py
=====================================
@@ -4,6 +4,7 @@
##################################################################
import unittest
+import pytest
from collections import namedtuple
from pywps import Process, Service, LiteralInput, ComplexInput, BoundingBoxInput
from pywps import LiteralOutput, ComplexOutput, BoundingBoxOutput
@@ -335,6 +336,7 @@ class InputDescriptionTest(unittest.TestCase):
class OutputDescriptionTest(unittest.TestCase):
+ @pytest.mark.skip(reason="not working")
def test_literal_output(self):
literal = LiteralOutput('literal', 'Literal foo', abstract='Description',
keywords=['kw1', 'kw2'], uoms=['metre'])
@@ -360,6 +362,7 @@ class OutputDescriptionTest(unittest.TestCase):
assert default_uom.attrib['{{{}}}reference'.format(NAMESPACES['ows'])] == OGCUNIT['metre']
assert len(supported_uoms) == 1
+ @pytest.mark.skip(reason="not working")
def test_complex_output(self):
complexo = ComplexOutput('complex', 'Complex foo', [Format('GML')], keywords=['kw1', 'kw2'])
doc = complexo.describe_xml()
@@ -375,6 +378,7 @@ class OutputDescriptionTest(unittest.TestCase):
assert keywords is not None
assert len(kws) == 2
+ @pytest.mark.skip(reason="not working")
def test_bbox_output(self):
bbox = BoundingBoxOutput('bbox', 'BBox foo', keywords=['kw1', 'kw2'],
crss=["EPSG:4326"])
@@ -398,5 +402,6 @@ def load_tests(loader=None, tests=None, pattern=None):
loader.loadTestsFromTestCase(DescribeProcessInputTest),
loader.loadTestsFromTestCase(InputDescriptionTest),
loader.loadTestsFromTestCase(DescribeProcessTranslationsTest),
+ loader.loadTestsFromTestCase(OutputDescriptionTest),
]
return unittest.TestSuite(suite_list)
=====================================
tests/test_execute.py
=====================================
@@ -4,6 +4,7 @@
##################################################################
import unittest
+import pytest
import lxml.etree
import json
import tempfile
@@ -230,6 +231,7 @@ def get_output(doc):
class ExecuteTest(unittest.TestCase):
"""Test for Exeucte request KVP request"""
+ @pytest.mark.xfail(reason="test.opendap.org is offline")
def test_dods(self):
if not WITH_NC4:
self.skipTest('netCDF4 not installed')
=====================================
tests/test_filestorage.py
=====================================
@@ -2,10 +2,12 @@
# Copyright 2018 Open Source Geospatial Foundation and others #
# licensed under MIT, Please consult LICENSE.txt for details #
##################################################################
+from pathlib import Path
from pywps.inout.storage.file import FileStorageBuilder, FileStorage, _build_output_name
from pywps.inout.storage import STORE_TYPE
from pywps.inout.basic import ComplexOutput
+from pywps.util import file_uri
from pywps import configuration, FORMATS
from urllib.parse import urlparse
@@ -26,7 +28,7 @@ class FileStorageTests(unittest.TestCase):
output = ComplexOutput('testme', 'Test', supported_formats=[FORMATS.TEXT], workdir=self.tmp_dir)
output.data = "Hello World!"
output_name, suffix = _build_output_name(output)
- self.assertEqual(output.file, self.tmp_dir + '/input.txt')
+ self.assertEqual(output.file, str(Path(self.tmp_dir) / 'input.txt'))
self.assertEqual(output_name, 'input.txt')
self.assertEqual(suffix, '.txt')
@@ -40,44 +42,46 @@ class FileStorageTests(unittest.TestCase):
self.assertEqual(store_type, STORE_TYPE.PATH)
self.assertEqual(store_str, 'input.txt')
- with open(self.tmp_dir + '/' + store_str) as f:
+ with open(Path(self.tmp_dir) / store_str) as f:
self.assertEqual(f.read(), "Hello World!")
def test_write(self):
configuration.CONFIG.set('server', 'outputpath', self.tmp_dir)
- configuration.CONFIG.set('server', 'outputurl', 'file://' + self.tmp_dir)
+ configuration.CONFIG.set('server', 'outputurl', file_uri(self.tmp_dir))
storage = FileStorageBuilder().build()
output = ComplexOutput('testme', 'Test', supported_formats=[FORMATS.TEXT], workdir=self.tmp_dir)
output.data = "Hello World!"
url = storage.write(output.data, 'foo.txt')
- self.assertEqual(url, 'file://' + self.tmp_dir + '/foo.txt')
- with open(self.tmp_dir + '/foo.txt') as f:
+ fname = Path(self.tmp_dir) / 'foo.txt'
+ self.assertEqual(url, file_uri(fname))
+ with open(fname) as f:
self.assertEqual(f.read(), "Hello World!")
def test_url(self):
configuration.CONFIG.set('server', 'outputpath', self.tmp_dir)
- configuration.CONFIG.set('server', 'outputurl', 'file://' + self.tmp_dir)
+ configuration.CONFIG.set('server', 'outputurl', file_uri(self.tmp_dir))
storage = FileStorageBuilder().build()
output = ComplexOutput('testme', 'Test', supported_formats=[FORMATS.TEXT], workdir=self.tmp_dir)
output.data = "Hello World!"
output.uuid = '595129f0-1a6c-11ea-a30c-acde48001122'
url = storage.url(output)
- self.assertEqual('file://' + self.tmp_dir + '/595129f0-1a6c-11ea-a30c-acde48001122' + '/input.txt', url)
+ fname = Path(self.tmp_dir) / '595129f0-1a6c-11ea-a30c-acde48001122' / 'input.txt'
+ self.assertEqual(file_uri(fname), url)
file_name = 'test.txt'
url = storage.url(file_name)
-
- self.assertEqual('file://' + self.tmp_dir + '/test.txt', url)
+ fname = Path(self.tmp_dir) / 'test.txt'
+ self.assertEqual(file_uri(fname), url)
def test_location(self):
configuration.CONFIG.set('server', 'outputpath', self.tmp_dir)
storage = FileStorageBuilder().build()
file_name = 'test.txt'
loc = storage.location(file_name)
-
- self.assertEqual(self.tmp_dir + '/test.txt', loc)
+ fname = Path(self.tmp_dir) / 'test.txt'
+ self.assertEqual(str(fname), loc)
def load_tests(loader=None, tests=None, pattern=None):
=====================================
tests/test_inout.py
=====================================
@@ -22,6 +22,7 @@ from pywps.app.Common import Metadata
from pywps.validator import get_validator
from pywps.inout.basic import IOHandler, SOURCE_TYPE, SimpleHandler, BBoxInput, BBoxOutput, \
ComplexInput, ComplexOutput, LiteralOutput, LiteralInput, _is_textfile
+from pywps.util import uri_to_path
from pywps.inout.literaltypes import convert, AllowedValue, AnyValue
from pywps.inout.outputs import MetaFile, MetaLink, MetaLink4
from io import StringIO
@@ -259,15 +260,6 @@ class SerializationComplexInputTest(unittest.TestCase):
self.assertEqual(complex_1.as_reference, complex_2.as_reference)
self.assertEqual(complex_1.translations, complex_2.translations)
- self.assertEqual(complex_1.prop, complex_2.prop)
-
- if complex_1.prop != 'url':
- # don't download the file when running tests
- self.assertEqual(complex_1.file, complex_2.file)
- self.assertEqual(complex_1.data, complex_2.data)
-
- self.assertEqual(complex_1.url, complex_2.url)
-
def test_complex_input_file(self):
complex = self.make_complex_input()
some_file = os.path.join(self.tmp_dir, "some_file.txt")
@@ -277,6 +269,9 @@ class SerializationComplexInputTest(unittest.TestCase):
complex2 = inout.inputs.ComplexInput.from_json(complex.json)
self.assert_complex_equals(complex, complex2)
self.assertEqual(complex.prop, 'file')
+ self.assertEqual(complex2.prop, 'file')
+ self.assertEqual(complex.file, complex2.file)
+ self.assertEqual(complex.data, complex2.data)
def test_complex_input_data(self):
complex = self.make_complex_input()
@@ -285,11 +280,11 @@ class SerializationComplexInputTest(unittest.TestCase):
assert complex.json['data'] == '<![CDATA[some data]]>'
# dump to json and load it again
complex2 = inout.inputs.ComplexInput.from_json(complex.json)
- # it's expected that the file path changed
- complex._file = complex2.file
self.assert_complex_equals(complex, complex2)
self.assertEqual(complex.prop, 'data')
+ self.assertEqual(complex2.prop, 'data')
+ self.assertEqual(complex.data, complex2.data)
def test_complex_input_stream(self):
complex = self.make_complex_input()
@@ -299,12 +294,10 @@ class SerializationComplexInputTest(unittest.TestCase):
# dump to json and load it again
complex2 = inout.inputs.ComplexInput.from_json(complex.json)
# the serialized stream becomes a data type
- # we hard-code it for the testing comparison
- complex.prop = 'data'
- # it's expected that the file path changed
- complex._file = complex2.file
-
+ self.assertEqual(complex.prop, 'stream')
+ self.assertEqual(complex2.prop, 'data')
self.assert_complex_equals(complex, complex2)
+ self.assertEqual(complex.data, complex2.data)
def test_complex_input_url(self):
complex = self.make_complex_input()
@@ -504,7 +497,8 @@ class ComplexOutputTest(unittest.TestCase):
with self.complex_out.stream as s:
self.assertEqual(s.read(), bytes(self.data, encoding='utf8'))
- with open(urlparse(self.complex_out.url).path) as f:
+ path = uri_to_path(self.complex_out.url)
+ with open(path) as f:
self.assertEqual(f.read(), self.data)
def test_file_handler_netcdf(self):
=====================================
tests/test_ows.py
=====================================
@@ -29,7 +29,7 @@ def create_feature():
def feature(request, response):
input = request.inputs['input'][0].file
# What do we need to assert a Complex input?
- # assert type(input) is text_type
+ # assert type(input) is str
# open the input file
try:
@@ -83,7 +83,7 @@ def create_sum_one():
def sum_one(request, response):
input = request.inputs['input'][0].file
# What do we need to assert a Complex input?
- # assert type(input) is text_type
+ # assert type(input) is str
import grass.script as grass
=====================================
tests/validator/test_complexvalidators.py
=====================================
@@ -7,6 +7,7 @@
"""
import unittest
+import pytest
import sys
from pywps.validator.complexvalidator import *
from pywps.inout.formats import FORMATS
@@ -140,6 +141,7 @@ class ValidateTest(unittest.TestCase):
else:
self.assertFalse(validatenetcdf(netcdf_input, MODE.STRICT), 'STRICT validation')
+ @pytest.mark.xfail(reason="test.opendap.org is offline")
def test_dods_validator(self):
opendap_input = ComplexInput('dods', 'opendap test', [FORMATS.DODS,])
opendap_input.url = "http://test.opendap.org:80/opendap/netcdf/examples/sresa1b_ncar_ccsm3_0_run1_200001.nc"
View it on GitLab: https://salsa.debian.org/debian-gis-team/pywps/-/compare/ab96ab186f044b48d0b4354dbfbede1566d39587...e52207f195881455745f0415c8de216eba1dad1a
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/pywps/-/compare/ab96ab186f044b48d0b4354dbfbede1566d39587...e52207f195881455745f0415c8de216eba1dad1a
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20210511/2278ae18/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list