[Git][debian-gis-team/pywps][upstream] New upstream version 4.4.1
Bas Couwenberg
gitlab at salsa.debian.org
Mon Mar 22 04:49:28 GMT 2021
Bas Couwenberg pushed to branch upstream at Debian GIS Project / pywps
Commits:
f6a9f37c by Bas Couwenberg at 2021-03-22T05:38:08+01:00
New upstream version 4.4.1
- - - - -
14 changed files:
- .github/workflows/main.yml
- VERSION.txt
- debian/changelog
- docs/configuration.rst
- pywps/app/Process.py
- pywps/app/Service.py
- pywps/configuration.py
- pywps/dependencies.py
- pywps/exceptions.py
- pywps/inout/basic.py
- pywps/inout/outputs.py
- pywps/inout/storage/file.py
- requirements.txt
- tests/test_inout.py
Changes:
=====================================
.github/workflows/main.yml
=====================================
@@ -4,7 +4,7 @@ on: [ push, pull_request ]
jobs:
main:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.6, 3.7, 3.8, 3.9]
=====================================
VERSION.txt
=====================================
@@ -1 +1 @@
-4.4.0
+4.4.1
=====================================
debian/changelog
=====================================
@@ -1,3 +1,13 @@
+pywps (4.4.1) trusty; urgency=medium
+
+ * Added option `storage_copy_function` (#584).
+ * Quick-fix to avoid import ogr exception when running tests without gdal (#583).
+ * Fixed issues with metalink URL outputs (#582, #581, #580, #571).
+ * Fixed issue with stored requests (#579).
+ * Fixed incorrect use of `self.__class__` in super (#578).
+
+ -- Carsten Ehbrecht <ehbrecht at dkrz.de> Sun, 21 Mar 2021 18:00:00 +0000
+
pywps (4.4.0) trusty; urgency=medium
* Dropping support for Python 2.x (#574).
=====================================
docs/configuration.rst
=====================================
@@ -113,7 +113,10 @@ configuration file <https://docs.pycsw.org/en/latest/configuration.html>`_.
number of processor cores. -1 for no limit.
:maxrequestsize:
- maximal request size. 0 for no limit
+ maximal request size. 0 for no limit.
+
+:maxsingleinputsize:
+ maximal request size for a single input. 0 for no limit.
:maxprocesses:
maximal number of requests being stored in queue, waiting till they can be
@@ -153,6 +156,15 @@ configuration file <https://docs.pycsw.org/en/latest/configuration.html>`_.
:storagetype:
The type of storage to use when storing status and results. Possible values are: ``file``, ``s3``. Defaults to ``file``.
+:storage_copy_function:
+ When using file storage you can choose the copy function. Possible values are:
+
+ * ``copy``: using ``shutil.copy2``,
+ * ``move``: using ``shutil.move``,
+ * ``link``: using ``os.link`` (hardlink).
+
+ Default: ``copy``.
+
[processing]
------------
=====================================
pywps/app/Process.py
=====================================
@@ -20,9 +20,11 @@ from pywps.inout.inputs import input_from_json
from pywps.inout.outputs import output_from_json
import pywps.configuration as config
from pywps.exceptions import (StorageNotSupported, OperationNotSupported,
- ServerBusy, NoApplicableCode)
+ ServerBusy, NoApplicableCode,
+ InvalidParameterValue)
from pywps.app.exceptions import ProcessError
from pywps.inout.storage.builder import StorageBuilder
+from pywps.inout.outputs import ComplexOutput
import importlib
@@ -311,6 +313,7 @@ class Process(object):
process._set_uuid(uuid)
process._setup_status_storage()
process.async_ = True
+ process.setup_outputs_from_wps_request(new_wps_request)
new_wps_response = ExecuteResponse(new_wps_request, process=process, uuid=uuid)
new_wps_response.store_status_file = True
process._run_async(new_wps_request, new_wps_response)
@@ -446,3 +449,31 @@ class Process(object):
LOGGER.debug('GISRC {}, GISBASE {}, GISDBASE {}, LOCATION {}, MAPSET {}'.format(
os.environ.get('GISRC'), os.environ.get('GISBASE'),
dbase, location, os.path.basename(mapset_name)))
+
+ def setup_outputs_from_wps_request(self, wps_request):
+ # set as_reference to True for all the outputs specified as reference
+ # if the output is not required to be raw
+ if not wps_request.raw:
+ for wps_outpt in wps_request.outputs:
+
+ is_reference = wps_request.outputs[wps_outpt].get('asReference', 'false')
+ mimetype = wps_request.outputs[wps_outpt].get('mimetype', '')
+ if is_reference.lower() == 'true':
+ # check if store is supported
+ if self.store_supported == 'false':
+ raise StorageNotSupported(
+ 'The storage of data is not supported for this process.')
+
+ is_reference = True
+ else:
+ is_reference = False
+
+ for outpt in self.outputs:
+ if outpt.identifier == wps_outpt:
+ outpt.as_reference = is_reference
+ if isinstance(outpt, ComplexOutput) and mimetype != '':
+ data_format = [f for f in outpt.supported_formats if f.mime_type == mimetype]
+ if len(data_format) == 0:
+ raise InvalidParameterValue(
+ 'MimeType ' + mimetype + ' not valid')
+ outpt.data_format = data_format[0]
=====================================
pywps/app/Service.py
=====================================
@@ -13,7 +13,6 @@ import pywps.configuration as config
from pywps.exceptions import MissingParameterValue, NoApplicableCode, InvalidParameterValue, FileSizeExceeded, \
StorageNotSupported, FileURLNotSupported
from pywps.inout.inputs import ComplexInput, LiteralInput, BoundingBoxInput
-from pywps.inout.outputs import ComplexOutput
from pywps.dblog import log_request, store_status
from pywps import response
from pywps.response.status import WPS_STATUS
@@ -138,32 +137,7 @@ class Service(object):
wps_request.inputs = data_inputs
- # set as_reference to True for all the outputs specified as reference
- # if the output is not required to be raw
- if not wps_request.raw:
- for wps_outpt in wps_request.outputs:
-
- is_reference = wps_request.outputs[wps_outpt].get('asReference', 'false')
- mimetype = wps_request.outputs[wps_outpt].get('mimetype', '')
- if is_reference.lower() == 'true':
- # check if store is supported
- if process.store_supported == 'false':
- raise StorageNotSupported(
- 'The storage of data is not supported for this process.')
-
- is_reference = True
- else:
- is_reference = False
-
- for outpt in process.outputs:
- if outpt.identifier == wps_outpt:
- outpt.as_reference = is_reference
- if isinstance(outpt, ComplexOutput) and mimetype != '':
- data_format = [f for f in outpt.supported_formats if f.mime_type == mimetype]
- if len(data_format) == 0:
- raise InvalidParameterValue(
- 'MimeType ' + mimetype + ' not valid')
- outpt.data_format = data_format[0]
+ process.setup_outputs_from_wps_request(wps_request)
wps_response = process.execute(wps_request, uuid)
return wps_response
=====================================
pywps/configuration.py
=====================================
@@ -89,6 +89,10 @@ def load_configuration(cfgfiles=None):
# after process has finished.
CONFIG.set('server', 'cleantempdir', 'true')
CONFIG.set('server', 'storagetype', 'file')
+ # File storage outputs can be copied, moved or linked
+ # from the workdir to the output folder.
+ # Allowed functions: "copy", "move", "link" (default "copy")
+ CONFIG.set('server', 'storage_copy_function', 'copy')
CONFIG.add_section('processing')
CONFIG.set('processing', 'mode', 'default')
=====================================
pywps/dependencies.py
=====================================
@@ -10,6 +10,7 @@ try:
from osgeo import gdal, ogr
except ImportError:
warnings.warn('Complex validation requires GDAL/OGR support.')
+ ogr = None
try:
import netCDF4
=====================================
pywps/exceptions.py
=====================================
@@ -115,7 +115,13 @@ class StorageNotSupported(NoApplicableCode):
class NotEnoughStorage(NoApplicableCode):
- """Storage not supported exception implementation
+ """Not enough storage exception implementation
+ """
+ code = 400
+
+
+class FileStorageError(NoApplicableCode):
+ """File storage exception implementation
"""
code = 400
=====================================
pywps/inout/basic.py
=====================================
@@ -30,6 +30,7 @@ import base64
from collections import namedtuple
from copy import deepcopy
from io import BytesIO
+import humanize
_SOURCE_TYPE = namedtuple('SOURCE_TYPE', 'MEMORY, FILE, STREAM, DATA, URL')
@@ -336,6 +337,11 @@ class FileHandler(IOHandler):
import pathlib
return pathlib.PurePosixPath(self.file).as_uri()
+ @property
+ def size(self):
+ """Length of the linked content in octets."""
+ return os.stat(self.file).st_size
+
def _openmode(self, data=None):
openmode = 'r'
# in Python 3 we need to open binary files in binary mode.
@@ -441,12 +447,15 @@ class UrlHandler(FileHandler):
@property
def file(self):
+ """Downloads URL and return file pointer.
+ Checks if size is allowed before download.
+ """
if self._file is not None:
return self._file
self._file = self._build_file_name(href=self.url)
- max_byte_size = self.max_input_size()
+ max_byte_size = self.max_size()
# Create request
try:
@@ -455,21 +464,24 @@ class UrlHandler(FileHandler):
except Exception as e:
raise NoApplicableCode('File reference error: {}'.format(e))
- error_message = 'File size for input "{}" exceeded. Maximum allowed: {} megabytes'.format(
- self.inpt.get('identifier', '?'), max_byte_size)
+ error_message = 'File size for input "{}" exceeded. Maximum allowed: {}'.format(
+ self.inpt.get('identifier', '?'), humanize.naturalsize(max_byte_size))
- if int(data_size) > int(max_byte_size):
- raise FileSizeExceeded(error_message)
+ if int(max_byte_size) > 0:
+ if int(data_size) > int(max_byte_size):
+ raise FileSizeExceeded(error_message)
try:
with open(self._file, 'wb') as f:
data_size = 0
for chunk in reference_file.iter_content(chunk_size=1024):
data_size += len(chunk)
- if int(data_size) > int(max_byte_size):
- raise FileSizeExceeded(error_message)
+ if int(max_byte_size) > 0:
+ if int(data_size) > int(max_byte_size):
+ raise FileSizeExceeded(error_message)
f.write(chunk)
-
+ except FileSizeExceeded:
+ raise
except Exception as e:
raise NoApplicableCode(e)
@@ -483,6 +495,16 @@ class UrlHandler(FileHandler):
def post_data(self, value):
self._post_data = value
+ @property
+ def size(self):
+ """Get content-length of URL without download"""
+ req = self._openurl(self.url)
+ if req.ok:
+ size = int(req.headers.get('content-length', '0'))
+ else:
+ size = 0
+ return size
+
@staticmethod
def _openurl(href, data=None):
"""Open given href.
@@ -496,14 +518,15 @@ class UrlHandler(FileHandler):
return req
@staticmethod
- def max_input_size():
+ def max_size():
"""Calculates maximal size for input file based on configuration
and units.
:return: maximum file size in bytes
"""
ms = config.get_config_value('server', 'maxsingleinputsize')
- return config.get_size_mb(ms) * 1024**2
+ byte_size = config.get_size_mb(ms) * 1024**2
+ return byte_size
class SimpleHandler(DataHandler):
=====================================
pywps/inout/outputs.py
=====================================
@@ -320,6 +320,7 @@ class MetaFile:
The metalink document is created by a `MetaLink` instance, which
holds a number of `MetaFile` instances.
"""
+ self._size = None
self._output = ComplexOutput(
identifier=identity or '',
title=description or '',
@@ -360,8 +361,15 @@ class MetaFile:
@property
def size(self):
- """Length of the linked content in octets."""
- return os.stat(self.file).st_size
+ """Size of the linked content in bytes."""
+ if self._size is None:
+ self._size = self._output.size
+ return self._size
+
+ @size.setter
+ def size(self, value):
+ """Set size to avoid size calculation."""
+ self._size = int(value)
@property
def urls(self):
=====================================
pywps/inout/storage/file.py
=====================================
@@ -6,7 +6,7 @@
import logging
import os
from urllib.parse import urljoin
-from pywps.exceptions import NotEnoughStorage
+from pywps.exceptions import NotEnoughStorage, FileStorageError
from pywps import configuration as config
from pywps.inout.basic import IOHandler
@@ -22,7 +22,8 @@ class FileStorageBuilder(StorageImplementationBuilder):
def build(self):
file_path = config.get_config_value('server', 'outputpath')
base_url = config.get_config_value('server', 'outputurl')
- return FileStorage(file_path, base_url)
+ copy_function = config.get_config_value('server', 'storage_copy_function')
+ return FileStorage(file_path, base_url, copy_function=copy_function)
def _build_output_name(output):
@@ -59,17 +60,17 @@ class FileStorage(CachedStorage):
True
"""
- def __init__(self, output_path, output_url):
+ def __init__(self, output_path, output_url, copy_function=None):
"""
"""
CachedStorage.__init__(self)
self.target = output_path
self.output_url = output_url
+ self.copy_function = copy_function
def _do_store(self, output):
import platform
import math
- import shutil
import tempfile
import uuid
@@ -103,8 +104,12 @@ class FileStorage(CachedStorage):
dir=target)[1]
full_output_name = os.path.join(target, output_name)
- LOGGER.info('Storing file output to {}'.format(full_output_name))
- shutil.copy2(output.file, full_output_name)
+ LOGGER.info(f'Storing file output to {full_output_name} ({self.copy_function}).')
+ try:
+ self.copy(output.file, full_output_name, self.copy_function)
+ except Exception:
+ LOGGER.exception(f"Could not copy {output_name}.")
+ raise FileStorageError("Could not copy output file.")
just_file_name = os.path.basename(output_name)
@@ -113,6 +118,27 @@ class FileStorage(CachedStorage):
return (STORE_TYPE.PATH, output_name, url)
+ @staticmethod
+ def copy(src, dst, copy_function=None):
+ """Copy file from source to destination using `copy_function`.
+
+ Values of `copy_function` (default=`copy`):
+ * copy: using `shutil.copy2`
+ * move: using `shutil.move`
+ * link: using `os.link` (hardlink)
+ """
+ import shutil
+ if copy_function == 'move':
+ shutil.move(src, dst)
+ elif copy_function == 'link':
+ try:
+ os.link(src, dst)
+ except Exception:
+ LOGGER.warn("Could not create hardlink. Fallback to copy.")
+ FileStorage.copy(src, dst)
+ else:
+ shutil.copy2(src, dst)
+
def write(self, data, destination, data_format=None):
"""
Write data to self.target
=====================================
requirements.txt
=====================================
@@ -6,3 +6,4 @@ python-dateutil
requests
SQLAlchemy
werkzeug
+humanize
=====================================
tests/test_inout.py
=====================================
@@ -772,6 +772,12 @@ class TestMetaLink(unittest.TestCase):
mf._set_workdir(self.tmp_dir)
return mf
+ def metafile_with_url(self):
+ mf = MetaFile('identifier', 'title', fmt=FORMATS.JSON)
+ mf.url = "https://pywps.org/"
+ mf._set_workdir(self.tmp_dir)
+ return mf
+
def test_metafile(self):
mf = self.metafile()
self.assertEqual('identifier', mf.identity)
@@ -821,6 +827,25 @@ class TestMetaLink(unittest.TestCase):
ml4.checksums = True
assert 'hash' in ml4.xml
+ def test_size(self):
+ ml4 = self.metalink4()
+ ml4.append(self.metafile_with_url())
+ assert 'size' in ml4.xml
+
+ def test_no_size(self):
+ ml4 = self.metalink4()
+ mf = self.metafile_with_url()
+ mf.size = 0
+ ml4.files = [mf]
+ assert 'size' not in ml4.xml
+
+ def test_set_size(self):
+ ml4 = self.metalink4()
+ mf = self.metafile_with_url()
+ mf.size = 100
+ ml4.files = [mf]
+ assert '<size>100</size>' in ml4.xml
+
def load_tests(loader=None, tests=None, pattern=None):
if not loader:
View it on GitLab: https://salsa.debian.org/debian-gis-team/pywps/-/commit/f6a9f37ced9e966f98b1eec34ac5bd7a183d48d9
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/pywps/-/commit/f6a9f37ced9e966f98b1eec34ac5bd7a183d48d9
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20210322/fbc75d10/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list