[mapproxy] 01/07: New upstream version 1.11.0
Bas Couwenberg
sebastic at debian.org
Mon Nov 20 16:28:15 UTC 2017
This is an automated email from the git hooks/post-receive script.
sebastic pushed a commit to branch master
in repository mapproxy.
commit e3c9905fdd9f73e167a30cf485d436af6bd3be66
Author: Bas Couwenberg <sebastic at xs4all.nl>
Date: Mon Nov 20 16:42:39 2017 +0100
New upstream version 1.11.0
---
.travis.yml | 17 +-
AUTHORS.txt | 2 +-
CHANGES.txt | 34 ++
doc/caches.rst | 59 +-
doc/conf.py | 4 +-
doc/configuration.rst | 36 +-
doc/deployment.rst | 6 +-
doc/install_osgeo4w.rst | 13 +-
doc/mapproxy_util.rst | 51 +-
doc/sources.rst | 31 +-
mapproxy/__init__.py | 1 -
mapproxy/cache/compact.py | 622 ++++++++++++++++-----
mapproxy/cache/couchdb.py | 12 +-
mapproxy/cache/file.py | 8 +-
mapproxy/cache/geopackage.py | 6 +-
mapproxy/cache/legend.py | 1 -
mapproxy/cache/mbtiles.py | 8 +-
mapproxy/cache/redis.py | 2 +-
mapproxy/cache/renderd.py | 1 -
mapproxy/cache/riak.py | 20 +-
mapproxy/cache/s3.py | 4 +-
mapproxy/cache/sqlite.py | 413 --------------
mapproxy/cache/tile.py | 2 -
mapproxy/client/http.py | 187 +++----
mapproxy/client/wms.py | 1 -
mapproxy/compat/__init__.py | 7 +-
mapproxy/config/config.py | 1 -
mapproxy/config/loader.py | 41 +-
mapproxy/config/validator.py | 2 +-
.../config_template/base_config/full_example.yaml | 16 +
mapproxy/image/__init__.py | 9 +-
mapproxy/image/merge.py | 1 -
mapproxy/image/transform.py | 234 ++++++--
mapproxy/layer.py | 31 +-
mapproxy/multiapp.py | 1 -
mapproxy/request/wms/__init__.py | 1 -
mapproxy/request/wmts.py | 2 +-
mapproxy/script/defrag.py | 184 ++++++
mapproxy/script/export.py | 8 +-
mapproxy/script/util.py | 23 +-
mapproxy/script/wms_capabilities.py | 6 +-
mapproxy/seed/cachelock.py | 1 -
mapproxy/seed/cleanup.py | 4 +-
mapproxy/seed/script.py | 1 +
mapproxy/seed/seeder.py | 2 +
mapproxy/service/demo.py | 13 +-
mapproxy/service/template_helper.py | 22 +-
mapproxy/service/templates/wms100capabilities.xml | 2 +-
mapproxy/service/templates/wms110capabilities.xml | 2 +-
mapproxy/service/templates/wms111capabilities.xml | 2 +-
mapproxy/service/templates/wms130capabilities.xml | 2 +-
mapproxy/service/templates/wmts100capabilities.xml | 10 +
mapproxy/service/tile.py | 2 +-
mapproxy/service/wms.py | 59 +-
mapproxy/source/__init__.py | 5 +-
mapproxy/source/mapnik.py | 4 +-
mapproxy/source/wms.py | 26 +-
mapproxy/srs.py | 12 +-
mapproxy/test/image.py | 2 +
mapproxy/test/system/__init__.py | 2 +-
mapproxy/test/system/fixture/layer.yaml | 11 +-
mapproxy/test/system/fixture/wms_srs_extent.yaml | 20 +-
mapproxy/test/system/test_arcgis.py | 2 +-
mapproxy/test/system/test_auth.py | 2 +-
mapproxy/test/system/test_behind_proxy.py | 2 +-
mapproxy/test/system/test_cache_band_merge.py | 1 -
mapproxy/test/system/test_cache_geopackage.py | 2 +-
mapproxy/test/system/test_cache_grid_names.py | 1 -
mapproxy/test/system/test_cache_mbtiles.py | 2 +-
mapproxy/test/system/test_cache_s3.py | 2 +-
mapproxy/test/system/test_cache_source.py | 1 -
mapproxy/test/system/test_combined_sources.py | 2 +-
mapproxy/test/system/test_coverage.py | 2 +-
mapproxy/test/system/test_disable_storage.py | 1 -
mapproxy/test/system/test_formats.py | 2 +-
mapproxy/test/system/test_kml.py | 1 -
mapproxy/test/system/test_layergroups.py | 2 +-
mapproxy/test/system/test_legendgraphic.py | 2 +-
mapproxy/test/system/test_mapnik.py | 2 +-
mapproxy/test/system/test_mapserver.py | 2 +-
mapproxy/test/system/test_mixed_mode_format.py | 2 +-
mapproxy/test/system/test_multi_cache_layers.py | 2 +-
mapproxy/test/system/test_multiapp.py | 2 +-
mapproxy/test/system/test_renderd_client.py | 1 -
mapproxy/test/system/test_scalehints.py | 2 +-
mapproxy/test/system/test_seed.py | 1 -
mapproxy/test/system/test_seed_only.py | 2 +-
mapproxy/test/system/test_sld.py | 2 +-
mapproxy/test/system/test_source_errors.py | 2 +-
mapproxy/test/system/test_tilesource_minmax_res.py | 1 -
mapproxy/test/system/test_tms.py | 1 -
mapproxy/test/system/test_tms_origin.py | 1 -
mapproxy/test/system/test_util_conf.py | 1 -
mapproxy/test/system/test_util_export.py | 1 -
mapproxy/test/system/test_util_grids.py | 1 -
mapproxy/test/system/test_util_wms_capabilities.py | 1 -
mapproxy/test/system/test_watermark.py | 2 +-
mapproxy/test/system/test_wms.py | 12 +-
mapproxy/test/system/test_wms_srs_extent.py | 56 +-
mapproxy/test/system/test_wms_version.py | 2 +-
mapproxy/test/system/test_wmsc.py | 2 +-
mapproxy/test/system/test_wmts.py | 9 +-
mapproxy/test/system/test_wmts_dimensions.py | 2 +-
mapproxy/test/system/test_wmts_restful.py | 5 +-
mapproxy/test/system/test_xslt_featureinfo.py | 2 +-
mapproxy/test/unit/test_cache.py | 2 -
mapproxy/test/unit/test_cache_compact.py | 197 ++++++-
mapproxy/test/unit/test_cache_couchdb.py | 1 -
mapproxy/test/unit/test_cache_geopackage.py | 2 +-
mapproxy/test/unit/test_cache_redis.py | 1 -
mapproxy/test/unit/test_cache_riak.py | 1 -
mapproxy/test/unit/test_cache_tile.py | 4 +-
mapproxy/test/unit/test_client.py | 119 ++--
mapproxy/test/unit/test_client_cgi.py | 1 -
mapproxy/test/unit/test_concat_legends.py | 1 -
mapproxy/test/unit/test_conf_loader.py | 6 +-
mapproxy/test/unit/test_conf_validator.py | 15 +-
mapproxy/test/unit/test_config.py | 1 -
mapproxy/test/unit/test_featureinfo.py | 1 -
mapproxy/test/unit/test_geom.py | 9 +-
mapproxy/test/unit/test_grid.py | 3 +-
mapproxy/test/unit/test_image.py | 91 ++-
mapproxy/test/unit/test_multiapp.py | 1 -
mapproxy/test/unit/test_seed.py | 2 +-
mapproxy/test/unit/test_seed_cachelock.py | 1 -
mapproxy/test/unit/test_tiled_source.py | 1 -
mapproxy/test/unit/test_utils.py | 1 -
mapproxy/test/unit/test_wms_layer.py | 2 +-
mapproxy/util/async.py | 1 -
mapproxy/util/coverage.py | 1 -
mapproxy/util/ext/dictspec/validator.py | 1 -
mapproxy/util/ext/serving.py | 278 +--------
mapproxy/util/fs.py | 20 +-
mapproxy/util/geom.py | 8 +-
mapproxy/util/lock.py | 1 -
mapproxy/util/py.py | 1 -
mapproxy/util/yaml.py | 2 +-
mapproxy/wsgiapp.py | 10 +-
release.py | 18 +-
requirements-tests.txt | 1 +
requirements-travis.txt | 3 +-
setup.py | 9 +-
142 files changed, 1963 insertions(+), 1312 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 9d991c7..0335113 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,6 +5,7 @@ python:
- "3.3"
- "3.4"
- "3.5"
+ - "3.6"
services:
- couchdb
@@ -32,19 +33,31 @@ env:
global:
- MAPPROXY_TEST_COUCHDB=http://127.0.0.1:5984
- MAPPROXY_TEST_REDIS=127.0.0.1:6379
+ - MAPPROXY_TEST_RIAK_HTTP=http://localhost:8098
+ - MAPPROXY_TEST_RIAK_PBC=pbc://localhost:8087
# do not load /etc/boto.cfg with Python 3 incompatible plugin
# https://github.com/travis-ci/travis-ci/issues/5246#issuecomment-166460882
- BOTO_CONFIG=/doesnotexist
+matrix:
+ # Test 2.7 and 3.6 also with latest Pillow version
+ include:
+ - python: "2.7"
+ env: USE_LATEST_PILLOW=1
+ - python: "3.6"
+ env: USE_LATEST_PILLOW=1
+
cache:
directories:
- $HOME/.cache/pip
+before_install:
+ - echo -n "ulimit -n 4096" | sudo tee /etc/default/riak && sudo service riak restart # default open file limit is too low for riak
+
install:
- # riak packages are not compatible with Python 3
- - "if [[ $TRAVIS_PYTHON_VERSION = '2.7' ]]; then pip install protobuf>=2.4.1 riak==2.2 riak_pb>=2.0; export MAPPROXY_TEST_RIAK_PBC=pbc://localhost:8087; fi"
- "pip install -r requirements-tests.txt"
+ - "if [[ $USE_LATEST_PILLOW = '1' ]]; then pip install -U Pillow; fi"
- "pip freeze"
script: nosetests mapproxy
diff --git a/AUTHORS.txt b/AUTHORS.txt
index bef029e..1223f18 100644
--- a/AUTHORS.txt
+++ b/AUTHORS.txt
@@ -25,7 +25,7 @@ Patches and Suggestions
- Matt Walker
- Miloslav Kmeť
- Paul Norman
-- Ramūnas
+- Ramūnas Dronga
- Richard Duivenvoorde
- Stephan Holl
- Steven D. Lander
diff --git a/CHANGES.txt b/CHANGES.txt
index 092c72e..5a2a5cb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,37 @@
+1.11.0 2017-11-xx
+~~~~~~~~~~~~~~~~~
+
+Improvements:
+
+- Improve reprojection performance and accuracy.
+- ArcGIS compact cache: Support for version 2.
+- ArcGIS compact cache: Improve performance for version 1.
+- ArcGIS compact cache: Add ``mapproxy-util defrag`` to reduce bundle size
+ after tiles were removed/updated.
+- ArcGIS REST source: Support opts.map and seed_only.
+- Use systems CA certs by default and fix ssl_no_cert_checks
+ for Python >=2.7.9 and >=3.4
+- WMS: Improve Bounding Boxes in Capabilities.
+- Mapserver: Find mapserv binary in PATH environment.
+
+Fixes:
+
+- Seed: Always show last log line (100%).
+- Fix saving transparent PNGs for some versions of Pillow
+ (workaround for Pillow bug #2633)
+- SQLite: Fix possible errors on first request after start.
+- Demo: Fix demo client with `use_grid_names`.
+- serve-develop: Fix header encoding for Python 3.
+- Seed: Fix --interactive for Python 3.
+- Support tagged layers for sources with colons in name.
+- Support # character in Basis Authentication password.
+- Fix import error with shapely>=1.6
+- Fix duplicate level caches when using WMTS KVP with MBtile/SQLite/CouchDB.
+
+Other:
+
+- Remove support for Python 2.6
+
1.10.4 2017-08-17
~~~~~~~~~~~~~~~~~
diff --git a/doc/caches.rst b/doc/caches.rst
index 01ff791..4dabd42 100644
--- a/doc/caches.rst
+++ b/doc/caches.rst
@@ -286,7 +286,7 @@ This backend is good for very large caches which can be distributed over many no
Requirements
------------
-You will need the `Python Riak client <https://pypi.python.org/pypi/riak>`_ version 2.0 or newer. You can install it in the usual way, for example with ``pip install riak``. Environments with older version must be upgraded with ``pip install -U riak``.
+You will need the `Python Riak client <https://pypi.python.org/pypi/riak>`_ version 2.4.2 or older. You can install it in the usual way, for example with ``pip install riak==2.4.2``. Environments with older version must be upgraded with ``pip install -U riak==2.4.2``. Python library depends on packages `python-dev`, `libffi-dev` and `libssl-dev`.
Configuration
-------------
@@ -294,13 +294,13 @@ Configuration
Available options:
``nodes``:
- A list of riak nodes. Each node needs a ``host`` and optionally a ``pb_port`` and an ``http_port`` if the ports differ from the default. A single localhost node is used if you don't configure any nodes.
+ A list of riak nodes. Each node needs a ``host`` and optionally a ``pb_port`` and an ``http_port`` if the ports differ from the default. Defaults to single localhost node.
``protocol``:
Communication protocol. Allowed options is ``http``, ``https`` and ``pbc``. Defaults to ``pbc``.
``bucket``:
- The name of the bucket MapProxy uses for this cache. The bucket is the namespace for the tiles and needs to be unique for each cache. Defaults to cache name suffixed with grid name (e.g. ``mycache_webmercator``).
+ The name of the bucket MapProxy uses for this cache. The bucket is the namespace for the tiles and must be unique for each cache. Defaults to cache name suffixed with grid name (e.g. ``mycache_webmercator``).
``default_ports``:
Default ``pb`` and ``http`` ports for ``pbc`` and ``http`` protocols. Will be used as the default for each defined node.
@@ -313,20 +313,21 @@ Example
::
- myriakcache:
- sources: [mywms]
- grids: [mygrid]
- type: riak
- nodes:
- - host: 1.example.org
- pb_port: 9999
- - host: 1.example.org
- - host: 1.example.org
- protocol: pbc
- bucket: myriakcachetiles
- default_ports:
- pb: 8087
- http: 8098
+ myriakcache:
+ sources: [mywms]
+ grids: [mygrid]
+ cache:
+ type: riak
+ nodes:
+ - host: 1.example.org
+ pb_port: 9999
+ - host: 1.example.org
+ - host: 1.example.org
+ protocol: pbc
+ bucket: myriakcachetiles
+ default_ports:
+ pb: 8087
+ http: 8098
.. _cache_redis:
@@ -488,10 +489,16 @@ Example
===========
.. versionadded:: 1.10.0
+ Support for format version 1
+
+.. versionadded:: 1.11.0
+ Support for format version 2
+
+Store tiles in ArcGIS compatible compact cache files. A single compact cache ``.bundle`` file stores up to about 16,000 tiles.
-Store tiles in ArcGIS compatible compact cache files. A single compact cache ``.bundle`` file stores up to about 16,000 tiles. There is one additional ``.bundlx`` index file for each ``.bundle`` data file.
+Version 1 of the compact cache format is compatible with ArcGIS 10.0 and the default version of ArcGIS 10.0-10.2. Version 2 is supported by ArcGIS 10.3 or higher.
+Version 1 stores is one additional ``.bundlx`` index file for each ``.bundle`` data file.
-Only version 1 of the compact cache format (ArcGIS 10.0-10.2) is supported. Version 2 (ArcGIS 10.3 or higher) is not supported at the moment.
Available options:
@@ -499,7 +506,7 @@ Available options:
Directory where MapProxy should store the level directories. This will not add the cache name or grid name to the path. You can use this option to point MapProxy to an existing compact cache.
``version``:
- The version of the ArcGIS compact cache format. This option is required.
+ The version of the ArcGIS compact cache format. This option is required. Either ``1`` or ``2``.
You can set the ``sources`` to an empty list, if you use an existing compact cache files and do not have a source.
@@ -515,16 +522,22 @@ The following configuration will load tiles from ``/path/to/cache/L00/R0000C0000
grids: [webmercator]
cache:
type: compact
- version: 1
+ version: 2
directory: /path/to/cache
.. note::
+ MapProxy does not support reading and writiting of the ``conf.cdi`` and ``conf.xml`` files. You need to configure a compatible MapProxy grid when you want to reuse exsting ArcGIS compact caches in MapProxy. You need to create or modify existing ``conf.cdi`` and ``conf.xml`` files when you want to use compact caches created with MapProxy in ArcGIS.
+
+
+.. note::
+
The compact cache format does not include any timestamps for each tile and the seeding function is limited therefore. If you include any ``refresh_before`` time in a seed task, all tiles will be recreated regardless of the value. The cleanup process does not support any ``remove_before`` times for compact caches and it always removes all tiles.
Use the ``--summary`` option of the ``mapproxy-seed`` tool.
.. note::
- The compact cache format is append-only to allow parallel read and write operations. Removing or refreshing tiles with ``mapproxy-seed`` does not reduce the size of the cache files. Therefore, this format is not suitable for caches that require frequent updates.
-
+ The compact cache format is append-only to allow parallel read and write operations.
+ Removing or refreshing tiles with ``mapproxy-seed`` does not reduce the size of the cache files.
+ You can use the :ref:`defrag-compact-cache <mapproxy_defrag_compact_cache>` util to reduce the file size of existing bundle files.
diff --git a/doc/conf.py b/doc/conf.py
index 96968a8..57769b4 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -49,9 +49,9 @@ copyright = u'Oliver Tonnhofer, Omniscale'
# built documents.
#
# The short X.Y version.
-version = '1.10'
+version = '1.11'
# The full version, including alpha/beta/rc tags.
-release = '1.10.4'
+release = '1.11.0'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/doc/configuration.rst b/doc/configuration.rst
index 5bf27e4..616a188 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -127,7 +127,7 @@ The old syntax to configure each layer as a dictionary with the key as the name
layers:
mylayer:
title: My Layer
- source: [mysoruce]
+ source: [mysource]
should become
@@ -136,7 +136,7 @@ should become
layers:
- name: mylayer
title: My Layer
- source: [mysoruce]
+ source: [mysource]
The mixed format where the layers are a list (``-``) but each layer is still a dictionary is no longer supported (e.g. ``- mylayer:`` becomes ``- name: mylayer``).
@@ -614,7 +614,7 @@ Requests with 1500, 1000 or 701m/px resolution will use the first level, request
The extent of your grid. You can use either a list or a string with the lower left and upper right coordinates. You can set the SRS of the coordinates with the ``bbox_srs`` option. If that option is not set the ``srs`` of the grid will be used.
-MapProxy always expects your BBOX coordinates order to be east, south, west, north, regardless of your SRS :ref:`axis order <axis_order>`.
+MapProxy always expects your BBOX coordinates order to be west, south, east, north regardless of your SRS :ref:`axis order <axis_order>`.
::
@@ -900,14 +900,13 @@ The following options define how tiles are created and stored. Most options can
HTTP related options.
-Secure HTTPS Connections (HTTPS)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. note:: You need Python 2.6 or the `SSL module <http://pypi.python.org/pypi/ssl>`_ for this feature.
+Secure HTTP Connections (HTTPS)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
MapProxy supports access to HTTPS servers. Just use ``https`` instead of ``http`` when
-defining the URL of a source. MapProxy needs a file that contains the root and CA
-certificates. If the server certificate is signed by a "standard" root certificate (i.e. your browser does not warn you), then you can use a cert file that is distributed with your system. On Debian based systems you can use ``/etc/ssl/certs/ca-certificates.crt``.
+defining the URL of a source.
+
+MapProxy verifies the SSL/TLS connections against your systems "certification authority" (CA) certificates. You can provide your own set of root certificates with the ``ssl_ca_certs`` option.
See the `Python SSL documentation <http://docs.python.org/dev/library/ssl.html#ssl-certificates>`_ for more information about the format.
::
@@ -915,11 +914,26 @@ See the `Python SSL documentation <http://docs.python.org/dev/library/ssl.html#s
http:
ssl_ca_certs: /etc/ssl/certs/ca-certificates.crt
-If you want to use SSL but do not need certificate verification, then you can disable it with the ``ssl_no_cert_checks`` option. You can also disable this check on a source level, see :ref:`WMS source options <wms_source_ssl_no_cert_checks>`.
+
+.. versionadded:: 1.11.0
+
+ MapProxy uses the systems CA files by default, if you use Python >=2.7.9 or >=3.4.
+
+
+.. note::
+
+ You need to supply a CA file that includes the root certificates if you use older MapProxy or older Python versions. Otherwise MapProxy will fail to establish the connection. You can set the ``http.ssl_no_cert_checks`` options to ``true`` to disable this verification.
+
+
+``ssl_no_cert_checks``
+
+If you want to use SSL/TLS but do not need certificate verification, then you can disable it with the ``ssl_no_cert_checks`` option. You can also disable this check on a source level.
+
::
http:
- ssl_no_cert_checks: True
+ ssl_no_cert_checks: true
+
``client_timeout``
^^^^^^^^^^^^^^^^^^
diff --git a/doc/deployment.rst b/doc/deployment.rst
index 0063af8..4d4628f 100644
--- a/doc/deployment.rst
+++ b/doc/deployment.rst
@@ -145,7 +145,7 @@ You need a server script that creates the MapProxy application (see :ref:`above
To start MapProxy with the Gunicorn web server with four processes, the eventlet worker and our server script (without ``.py``)::
cd /path/of/config.py/
- gunicorn -k eventlet -w 4 -b :8080 config:application
+ gunicorn -k eventlet -w 4 -b :8080 config:application --no-sendfile
An example upstart script (``/etc/init/mapproxy.conf``) might look like::
@@ -160,7 +160,9 @@ An example upstart script (``/etc/init/mapproxy.conf``) might look like::
chdir /etc/opt/mapproxy
- exec /opt/mapproxy/bin/gunicorn -k eventlet -w 8 -b :8080 application \
+ exec /opt/mapproxy/bin/gunicorn -k eventlet -w 8 -b :8080 \
+ --no-sendfile \
+ application \
>>/var/log/mapproxy/gunicorn.log 2>&1
diff --git a/doc/install_osgeo4w.rst b/doc/install_osgeo4w.rst
index c2d5319..cffe726 100644
--- a/doc/install_osgeo4w.rst
+++ b/doc/install_osgeo4w.rst
@@ -18,22 +18,19 @@ Please refer to the `OSGeo4W installer FAQ <http://trac.osgeo.org/osgeo4w/wiki/F
At this point, you should see an OSGeo4W shell icon on your desktop and/or start menu. Right-click that, and *run as administrator*.
-As happens with the standard Windows installation, you need to `install the distribute package <http://pypi.python.org/pypi/distribute#distribute-setup-py>`_ to get the ``easy_install`` command. Run this in your administrator OSGeo4W shell, e.g.::
+In the OSGeo4W window, run::
- C:\OSGeo4W> python C:\Users\MyUsername\Downloads\distribute-setup.py
-
-Once ``easy_install`` is working within the OSGeo4W python environment, run::
-
- C:\OSGeo4W> easy_install mapproxy
+ C:\OSGeo4W> pip install mapproxy
and
::
- C:\OSGeo4W> easy_install pyproj
+ C:\OSGeo4W> pip install pyproj
-If these three last commands didn't print out any errors, your installation of MapProxy is successful. You can now close the OSGeo4W shell with administrator privileges, as it is no longer needed.
+If these last two commands didn't print out any errors, your installation of MapProxy is successful. You can now close the OSGeo4W shell with administrator privileges, as it is no longer needed.
+In older versions of OSGeo4W ``pip`` may not recognized. In such a case, please follow the instructions for `installing pip with get-pip.py <https://pip.pypa.io/en/stable/installing/#installing-with-get-pip-py>`_ and rerty the above ``pip install`` commands.
Check installation
------------------
diff --git a/doc/mapproxy_util.rst b/doc/mapproxy_util.rst
index 2c5a0df..f1f7a03 100644
--- a/doc/mapproxy_util.rst
+++ b/doc/mapproxy_util.rst
@@ -31,6 +31,7 @@ The current sub-commands are:
- :ref:`mapproxy_util_wms_capabilities`
- :ref:`mapproxy_util_grids`
- :ref:`mapproxy_util_export`
+- :ref:`mapproxy_defrag_compact_cache`
- ``autoconfig`` (see :ref:`mapproxy_util_autoconfig`)
@@ -54,7 +55,7 @@ This sub-command creates example configurations for you. There are templates for
.. cmdoption:: -f <mapproxy.yaml>, --mapproxy-conf <mapproxy.yaml>
- The path to the MapProxy configuration. Required for some templates.
+ The path of the MapProxy configuration. Required for some templates.
.. cmdoption:: --force
@@ -457,7 +458,7 @@ Required arguments:
.. cmdoption:: -f, --mapproxy-conf
- The path to the MapProxy configuration of the source cache.
+ The path of the MapProxy configuration of the source cache.
.. cmdoption:: --source
@@ -547,3 +548,49 @@ Export tiles into an MBTiles file using a custom grid definition.
--grid "srs='EPSG:4326' bbox=[5,50,10,60] tile_size=[512,512]" \
--source osm_cache --dest osm.mbtiles --type mbtile \
+
+
+.. _mapproxy_defrag_compact_cache:
+
+``defrag-compact-cache``
+========================
+
+
+The ArcGIS compact cache format version 1 and 2 are append only. Updating existing tiles will increase the file size. Bundle files become larger and fragmented with time. The ``defrag-compact-cache`` sub-command compacts existing bundle files by rewriting and reorganizing each bundle file.
+
+
+.. program:: mapproxy-util defrag-compact-cache
+
+
+Required arguments:
+
+.. cmdoption:: -f, --mapproxy-conf
+
+ The path of the MapProxy configuration with the configured compact caches.
+
+Optional arguments:
+
+.. cmdoption:: --caches
+
+ Comma separated list of caches to defragment. By default all configured compact caches will be defragmented.
+
+.. cmdoption:: --min-percent, --min-mb
+
+ Bundle files with only a minmal fragmentation are skipped. You can define this threshold with ``--min-percent`` as the required minimal percentage of unused space and ``--min-mb`` as the minimal required unused space in megabytes. Both thresholds must be exceeded. Defaults to 10% and 1MB.
+
+.. option:: -n, --dry-run
+
+ This will simulate the defragmentation process.
+
+
+Examples
+--------
+
+Defragment bundle files from ``map1_cache`` and ``map2_cache`` when they have more than 20% and 5MB of unused space. E.g. a 20 MB bundle file only gets rewritten if it becomes smaller then 15MB after defragmentation; a 500MB bundle file only gets rewritten if it becomes smaller then 400MB after defragmentation.
+
+::
+
+ mapproxy-util defrag-compact-cache -f mapproxy.yaml \
+ --min-percent 20 \
+ --min-mb 5 \
+ --caches map1_cache,map2_cache
diff --git a/doc/sources.rst b/doc/sources.rst
index 7c825c2..c93fd0c 100644
--- a/doc/sources.rst
+++ b/doc/sources.rst
@@ -178,16 +178,10 @@ You can configure the following HTTP related options for this source:
- ``headers``
- ``client_timeout``
- ``ssl_ca_certs``
-- ``ssl_no_cert_checks`` (see below)
+- ``ssl_no_cert_checks``
See :ref:`HTTP Options <http_ssl>` for detailed documentation.
-.. _wms_source_ssl_no_cert_checks:
-
-``ssl_no_cert_checks``
-
- MapProxy checks the SSL server certificates for any ``req.url`` that use HTTPS. You need to supply a file (see) that includes that certificate, otherwise MapProxy will fail to establish the connection. You can set the ``http.ssl_no_cert_checks`` options to ``true`` to disable this verification.
-
.. _tagged_source_names:
Tagged source names
@@ -274,7 +268,10 @@ This describes the ArcGIS source. The only required option is ``url``. You need
``opts``
^^^^^^^^
-.. versionadded: 1.10.0
+.. versionadded:: 1.10.0
+.. versionadded:: 1.11.0
+ ``map`` option
+
This option affects what request MapProxy sends to the source ArcGIS server.
@@ -287,6 +284,19 @@ This option affects what request MapProxy sends to the source ArcGIS server.
``featureinfo_tolerance``
Tolerance in pixel within the ArcGIS server should identify features.
+
+``map``
+ If this is set to ``false``, MapProxy will not request images from this source. You can use this option in combination with ``featureinfo: true`` to create a source that is only used for feature info requests.
+
+
+
+``seed_only``
+^^^^^^^^^^^^^
+
+.. versionadded:: 1.11.0
+
+See :ref:`seed_only <wms_seed_only>`
+
Example configuration
^^^^^^^^^^^^^^^^^^^^^
@@ -375,7 +385,7 @@ You can configure the following HTTP related options for this source:
- ``headers``
- ``client_timeout``
- ``ssl_ca_certs``
-- ``ssl_no_cert_checks`` (:ref:`see above <wms_source_ssl_no_cert_checks>`)
+- ``ssl_no_cert_checks``
See :ref:`HTTP Options <http_ssl>` for detailed documentation.
@@ -469,6 +479,9 @@ You can also set these options in the :ref:`globals-conf-label` section.
Path where the Mapserver should be executed from. It should be the directory where any relative paths in your mapfile are based on.
+.. versionadded:: 1.11.0
+ The ``mapserv`` binary is searched in all directories of the ``PATH`` environment, if ``binary`` is not set.
+
Example configuration
^^^^^^^^^^^^^^^^^^^^^
diff --git a/mapproxy/__init__.py b/mapproxy/__init__.py
index b0d6433..e69de29 100644
--- a/mapproxy/__init__.py
+++ b/mapproxy/__init__.py
@@ -1 +0,0 @@
-__import__('pkg_resources').declare_namespace(__name__)
\ No newline at end of file
diff --git a/mapproxy/cache/compact.py b/mapproxy/cache/compact.py
index 3a82877..bc9e787 100644
--- a/mapproxy/cache/compact.py
+++ b/mapproxy/cache/compact.py
@@ -1,5 +1,5 @@
# This file is part of the MapProxy project.
-# Copyright (C) 2016 Omniscale <http://omniscale.de>
+# Copyright (C) 2016-2017 Omniscale <http://omniscale.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
+import contextlib
import errno
import hashlib
import os
@@ -30,23 +30,28 @@ import logging
log = logging.getLogger(__name__)
-class CompactCacheV1(TileCacheBase):
+class CompactCacheBase(TileCacheBase):
supports_timestamp = False
+ bundle_class = None
def __init__(self, cache_dir):
self.lock_cache_id = 'compactcache-' + hashlib.md5(cache_dir.encode('utf-8')).hexdigest()
self.cache_dir = cache_dir
- def _get_bundle(self, tile_coord):
+ def _get_bundle_fname_and_offset(self, tile_coord):
x, y, z = tile_coord
level_dir = os.path.join(self.cache_dir, 'L%02d' % z)
- c = x // BUNDLEX_GRID_WIDTH * BUNDLEX_GRID_WIDTH
- r = y // BUNDLEX_GRID_HEIGHT * BUNDLEX_GRID_HEIGHT
+ c = x // BUNDLEX_V1_GRID_WIDTH * BUNDLEX_V1_GRID_WIDTH
+ r = y // BUNDLEX_V1_GRID_HEIGHT * BUNDLEX_V1_GRID_HEIGHT
basename = 'R%04xC%04x' % (r, c)
- return Bundle(os.path.join(level_dir, basename), offset=(c, r))
+ return os.path.join(level_dir, basename), (c, r)
+
+ def _get_bundle(self, tile_coord):
+ bundle_fname, offset = self._get_bundle_fname_and_offset(tile_coord)
+ return self.bundle_class(bundle_fname, offset=offset)
def is_cached(self, tile):
if tile.coord is None:
@@ -62,12 +67,53 @@ class CompactCacheV1(TileCacheBase):
return self._get_bundle(tile.coord).store_tile(tile)
+ def store_tiles(self, tiles):
+ if len(tiles) > 1:
+ # Check if all tiles are from a single bundle.
+ bundle_files = set()
+ tile_coord = None
+ for t in tiles:
+ if t.stored:
+ continue
+ bundle_files.add(self._get_bundle_fname_and_offset(t.coord)[0])
+ tile_coord = t.coord
+ if len(bundle_files) == 1:
+ return self._get_bundle(tile_coord).store_tiles(tiles)
+
+ # Tiles are across multiple bundles
+ failed = False
+ for tile in tiles:
+ if not self.store_tile(tile):
+ failed = True
+ return not failed
+
+
def load_tile(self, tile, with_metadata=False):
if tile.source or tile.coord is None:
return True
return self._get_bundle(tile.coord).load_tile(tile)
+ def load_tiles(self, tiles, with_metadata=False):
+ if len(tiles) > 1:
+ # Check if all tiles are from a single bundle.
+ bundle_files = set()
+ tile_coord = None
+ for t in tiles:
+ if t.source or t.coord is None:
+ continue
+ bundle_files.add(self._get_bundle_fname_and_offset(t.coord)[0])
+ tile_coord = t.coord
+ if len(bundle_files) == 1:
+ return self._get_bundle(tile_coord).load_tiles(tiles)
+
+ # No support_bulk_load or tiles are across multiple bundles
+ missing = False
+ for tile in tiles:
+ if not self.load_tile(tile, with_metadata=with_metadata):
+ missing = True
+ return not missing
+
def remove_tile(self, tile):
if tile.coord is None:
return True
@@ -86,9 +132,9 @@ class CompactCacheV1(TileCacheBase):
return False
BUNDLE_EXT = '.bundle'
-BUNDLEX_EXT = '.bundlx'
+BUNDLEX_V1_EXT = '.bundlx'
-class Bundle(object):
+class BundleV1(object):
def __init__(self, base_filename, offset):
self.base_filename = base_filename
self.lock_filename = base_filename + '.lck'
@@ -96,81 +142,133 @@ class Bundle(object):
def _rel_tile_coord(self, tile_coord):
return (
- tile_coord[0] % BUNDLEX_GRID_WIDTH,
- tile_coord[1] % BUNDLEX_GRID_HEIGHT,
+ tile_coord[0] % BUNDLEX_V1_GRID_WIDTH,
+ tile_coord[1] % BUNDLEX_V1_GRID_HEIGHT,
)
+ def data(self):
+ return BundleDataV1(self.base_filename + BUNDLE_EXT, self.offset)
+
+ def index(self):
+ return BundleIndexV1(self.base_filename + BUNDLEX_V1_EXT)
+
def is_cached(self, tile):
if tile.source or tile.coord is None:
return True
- idx = BundleIndex(self.base_filename + BUNDLEX_EXT)
- x, y = self._rel_tile_coord(tile.coord)
- offset = idx.tile_offset(x, y)
- if offset == 0:
- return False
+ with self.index().readonly() as idx:
+ if not idx:
+ return False
+ x, y = self._rel_tile_coord(tile.coord)
+ offset = idx.tile_offset(x, y)
+ if offset == 0:
+ return False
- bundle = BundleData(self.base_filename + BUNDLE_EXT, self.offset)
- size = bundle.read_size(offset)
+ with self.data().readonly() as bundle:
+ size = bundle.read_size(offset)
return size != 0
def store_tile(self, tile):
if tile.stored:
return True
+ return self.store_tiles([tile])
- with tile_buffer(tile) as buf:
- data = buf.read()
+ def store_tiles(self, tiles):
+ tiles_data = []
+ for t in tiles:
+ if t.stored:
+ continue
+ with tile_buffer(t) as buf:
+ data = buf.read()
+ tiles_data.append((t.coord, data))
with FileLock(self.lock_filename):
- bundle = BundleData(self.base_filename + BUNDLE_EXT, self.offset)
- idx = BundleIndex(self.base_filename + BUNDLEX_EXT)
- x, y = self._rel_tile_coord(tile.coord)
- offset = idx.tile_offset(x, y)
- offset, size = bundle.append_tile(data, prev_offset=offset)
- idx.update_tile_offset(x, y, offset=offset, size=size)
+ with self.data().readwrite() as bundle:
+ with self.index().readwrite() as idx:
+ for tile_coord, data in tiles_data:
+ x, y = self._rel_tile_coord(tile_coord)
+ offset = idx.tile_offset(x, y)
+ offset, size = bundle.append_tile(data, prev_offset=offset)
+ idx.update_tile_offset(x, y, offset=offset, size=size)
return True
+
def load_tile(self, tile, with_metadata=False):
if tile.source or tile.coord is None:
return True
+ return self.load_tiles([tile], with_metadata)
- idx = BundleIndex(self.base_filename + BUNDLEX_EXT)
- x, y = self._rel_tile_coord(tile.coord)
- offset = idx.tile_offset(x, y)
- if offset == 0:
- return False
-
- bundle = BundleData(self.base_filename + BUNDLE_EXT, self.offset)
- data = bundle.read_tile(offset)
- if not data:
- return False
- tile.source = ImageSource(BytesIO(data))
+ def load_tiles(self, tiles, with_metadata=False):
+ missing = False
- return True
+ with self.index().readonly() as idx:
+ if not idx:
+ return False
+ with self.data().readonly() as bundle:
+ for t in tiles:
+ if t.source or t.coord is None:
+ continue
+ x, y = self._rel_tile_coord(t.coord)
+ offset = idx.tile_offset(x, y)
+ if offset == 0:
+ missing = True
+ continue
+
+ data = bundle.read_tile(offset)
+ if not data:
+ missing = True
+ continue
+ t.source = ImageSource(BytesIO(data))
+
+ return not missing
def remove_tile(self, tile):
if tile.coord is None:
return True
with FileLock(self.lock_filename):
- idx = BundleIndex(self.base_filename + BUNDLEX_EXT)
- x, y = self._rel_tile_coord(tile.coord)
- idx.remove_tile_offset(x, y)
+ with self.index().readwrite() as idx:
+ x, y = self._rel_tile_coord(tile.coord)
+ idx.remove_tile_offset(x, y)
return True
+ def size(self):
+ total_size = 0
+
+ with self.index().readonly() as idx:
+ if not idx:
+ return 0, 0
+
+ with self.data().readonly() as bundle:
+ for y in range(BUNDLEX_V1_GRID_HEIGHT):
+ for x in range(BUNDLEX_V1_GRID_WIDTH):
+ offset = idx.tile_offset(x, y)
+ if not offset:
+ continue
+ size = bundle.read_size(offset)
+ if not size:
+ continue
+ total_size += size + 4
+
+ actual_size = os.path.getsize(bundle.filename)
+ return total_size + BUNDLE_V1_HEADER_SIZE + (BUNDLEX_V1_GRID_HEIGHT * BUNDLEX_V1_GRID_WIDTH * 4), actual_size
+
+
+BUNDLEX_V1_GRID_WIDTH = 128
+BUNDLEX_V1_GRID_HEIGHT = 128
+BUNDLEX_V1_HEADER_SIZE = 16
+BUNDLEX_V1_HEADER = b'\x03\x00\x00\x00\x10\x00\x00\x00\x00\x40\x00\x00\x05\x00\x00\x00'
+BUNDLEX_V1_FOOTER_SIZE = 16
+BUNDLEX_V1_FOOTER = b'\x00\x00\x00\x00\x10\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00'
-BUNDLEX_GRID_WIDTH = 128
-BUNDLEX_GRID_HEIGHT = 128
-BUNDLEX_HEADER_SIZE = 16
-BUNDLEX_HEADER = b'\x03\x00\x00\x00\x10\x00\x00\x00\x00\x40\x00\x00\x05\x00\x00\x00'
-BUNDLEX_FOOTER_SIZE = 16
-BUNDLEX_FOOTER = b'\x00\x00\x00\x00\x10\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00'
+INT64LE = struct.Struct('<Q')
-class BundleIndex(object):
+class BundleIndexV1(object):
def __init__(self, filename):
self.filename = filename
+ self._fh = None
# defer initialization to update/remove calls to avoid
# index creation on is_cached (prevents new files in read-only caches)
self._initialized = False
@@ -181,122 +279,386 @@ class BundleIndex(object):
return
ensure_directory(self.filename)
buf = BytesIO()
- buf.write(BUNDLEX_HEADER)
- for i in range(BUNDLEX_GRID_WIDTH * BUNDLEX_GRID_HEIGHT):
- buf.write(struct.pack('<Q', (i*4)+BUNDLE_HEADER_SIZE)[:5])
- buf.write(BUNDLEX_FOOTER)
+ buf.write(BUNDLEX_V1_HEADER)
+
+ for i in range(BUNDLEX_V1_GRID_WIDTH * BUNDLEX_V1_GRID_HEIGHT):
+ buf.write(INT64LE.pack((i*4)+BUNDLE_V1_HEADER_SIZE)[:5])
+ buf.write(BUNDLEX_V1_FOOTER)
write_atomic(self.filename, buf.getvalue())
- def _tile_offset(self, x, y):
- return BUNDLEX_HEADER_SIZE + (x * BUNDLEX_GRID_HEIGHT + y) * 5
+ def _tile_index_offset(self, x, y):
+ return BUNDLEX_V1_HEADER_SIZE + (x * BUNDLEX_V1_GRID_HEIGHT + y) * 5
def tile_offset(self, x, y):
- idx_offset = self._tile_offset(x, y)
+ if self._fh is None:
+ raise RuntimeError('not called within readonly/readwrite context')
+ idx_offset = self._tile_index_offset(x, y)
+ self._fh.seek(idx_offset)
+ offset = INT64LE.unpack(self._fh.read(5) + b'\x00\x00\x00')[0]
+ return offset
+
+ def update_tile_offset(self, x, y, offset, size):
+ if self._fh is None:
+ raise RuntimeError('not called within readwrite context')
+ idx_offset = self._tile_index_offset(x, y)
+ offset = INT64LE.pack(offset)[:5]
+ self._fh.seek(idx_offset, os.SEEK_SET)
+ self._fh.write(offset)
+
+ def remove_tile_offset(self, x, y):
+ if self._fh is None:
+ raise RuntimeError('not called within readwrite context')
+ idx_offset = self._tile_index_offset(x, y)
+ self._fh.seek(idx_offset)
+ self._fh.write(b'\x00' * 5)
+
+ @contextlib.contextmanager
+ def readonly(self):
try:
- with open(self.filename, 'rb') as f:
- f.seek(idx_offset)
- offset = struct.unpack('<Q', f.read(5) + b'\x00\x00\x00')[0]
- return offset
+ with open(self.filename, 'rb') as fh:
+ b = BundleIndexV1(self.filename)
+ b._fh = fh
+ yield b
except IOError as ex:
if ex.errno == errno.ENOENT:
- # mising bundle file -> missing tile
- return 0
- raise
+ # missing bundle file -> missing tile
+ yield None
+ else:
+ raise ex
- def update_tile_offset(self, x, y, offset, size):
+ @contextlib.contextmanager
+ def readwrite(self):
self._init_index()
- idx_offset = self._tile_offset(x, y)
- offset = struct.pack('<Q', offset)[:5]
- with open(self.filename, 'r+b') as f:
- f.seek(idx_offset, os.SEEK_SET)
- f.write(offset)
+ with open(self.filename, 'r+b') as fh:
+ b = BundleIndexV1(self.filename)
+ b._fh = fh
+ yield b
- def remove_tile_offset(self, x, y):
- self._init_index()
- idx_offset = self._tile_offset(x, y)
- with open(self.filename, 'r+b') as f:
- f.seek(idx_offset)
- f.write(b'\x00' * 5)
-
-# The bundle file has a header with 15 little-endian long values (60 bytes).
-# NOTE: the fixed values might be some flags for image options (format, aliasing)
-# all files available for testing had the same values however.
-BUNDLE_HEADER_SIZE = 60
-BUNDLE_HEADER = [
+
+
+ # The bundle file has a header with 15 little-endian long values (60 bytes).
+ # NOTE: the fixed values might be some flags for image options (format, aliasing)
+ # all files available for testing had the same values however.
+BUNDLE_V1_HEADER_SIZE = 60
+BUNDLE_V1_HEADER = [
3 , # 0, fixed
16384 , # 1, max. num of tiles 128*128 = 16384
16 , # 2, size of largest tile
5 , # 3, fixed
0 , # 4, num of tiles in bundle (*4)
- 0 , # 5, fixed
- 60+65536 , # 6, bundle size
- 0 , # 7, fixed
- 40 , # 8 fixed
- 0 , # 9, fixed
- 16 , # 10, fixed
- 0 , # 11, y0
- 127 , # 12, y1
- 0 , # 13, x0
- 127 , # 14, x1
+ 60+65536 , # 5, bundle size
+ 40 , # 6 fixed
+ 16 , # 7, fixed
+ 0 , # 8, y0
+ 127 , # 9, y1
+ 0 , # 10, x0
+ 127 , # 11, x1
]
-BUNDLE_HEADER_STRUCT_FORMAT = '<lllllllllllllll'
+BUNDLE_V1_HEADER_STRUCT_FORMAT = '<4I3Q5I'
-class BundleData(object):
+class BundleDataV1(object):
def __init__(self, filename, tile_offsets):
self.filename = filename
self.tile_offsets = tile_offsets
+ self._fh = None
if not os.path.exists(self.filename):
self._init_bundle()
def _init_bundle(self):
ensure_directory(self.filename)
- header = list(BUNDLE_HEADER)
- header[13], header[11] = self.tile_offsets
- header[14], header[12] = header[13]+127, header[11]+127
+ header = list(BUNDLE_V1_HEADER)
+ header[10], header[8] = self.tile_offsets
+ header[11], header[9] = header[10]+127, header[8]+127
write_atomic(self.filename,
- struct.pack(BUNDLE_HEADER_STRUCT_FORMAT, *header) +
+ struct.pack(BUNDLE_V1_HEADER_STRUCT_FORMAT, *header) +
# zero-size entry for each tile
- (b'\x00' * (BUNDLEX_GRID_HEIGHT * BUNDLEX_GRID_WIDTH * 4)))
+ (b'\x00' * (BUNDLEX_V1_GRID_HEIGHT * BUNDLEX_V1_GRID_WIDTH * 4)))
+
+
+ @contextlib.contextmanager
+ def readonly(self):
+ with open(self.filename, 'rb') as fh:
+ b = BundleDataV1(self.filename, self.tile_offsets)
+ b._fh = fh
+ yield b
+
+ @contextlib.contextmanager
+ def readwrite(self):
+ with open(self.filename, 'r+b') as fh:
+ b = BundleDataV1(self.filename, self.tile_offsets)
+ b._fh = fh
+ yield b
def read_size(self, offset):
- with open(self.filename, 'rb') as f:
- f.seek(offset)
- return struct.unpack('<L', f.read(4))[0]
+ if self._fh is None:
+ raise RuntimeError('not called within readonly/readwrite context')
+ self._fh.seek(offset)
+ return struct.unpack('<L', self._fh.read(4))[0]
def read_tile(self, offset):
- with open(self.filename, 'rb') as f:
- f.seek(offset)
- size = struct.unpack('<L', f.read(4))[0]
- if size <= 0:
- return False
- return f.read(size)
+ if self._fh is None:
+ raise RuntimeError('not called within readonly/readwrite context')
+ self._fh.seek(offset)
+ size = struct.unpack('<L', self._fh.read(4))[0]
+ if size <= 0:
+ return False
+ return self._fh.read(size)
def append_tile(self, data, prev_offset):
+ if self._fh is None:
+ raise RuntimeError('not called within readwrite context')
size = len(data)
is_new_tile = True
- with open(self.filename, 'r+b') as f:
- if prev_offset:
- f.seek(prev_offset, os.SEEK_SET)
- if f.tell() == prev_offset:
- if struct.unpack('<L', f.read(4))[0] > 0:
- is_new_tile = False
-
- f.seek(0, os.SEEK_END)
- offset = f.tell()
- if offset == 0:
- f.write(b'\x00' * 16) # header
- offset = 16
- f.write(struct.pack('<L', size))
- f.write(data)
-
- # update header
- f.seek(0, os.SEEK_SET)
- header = list(struct.unpack(BUNDLE_HEADER_STRUCT_FORMAT, f.read(60)))
- header[2] = max(header[2], size)
- header[6] += size + 4
- if is_new_tile:
- header[4] += 4
- f.seek(0, os.SEEK_SET)
- f.write(struct.pack(BUNDLE_HEADER_STRUCT_FORMAT, *header))
+ if prev_offset:
+ self._fh.seek(prev_offset, os.SEEK_SET)
+ if self._fh.tell() == prev_offset:
+ if struct.unpack('<L', self._fh.read(4))[0] > 0:
+ is_new_tile = False
+
+ self._fh.seek(0, os.SEEK_END)
+ offset = self._fh.tell()
+ if offset == 0:
+ self._fh.write(b'\x00' * 16) # header
+ offset = 16
+ self._fh.write(struct.pack('<L', size))
+ self._fh.write(data)
+
+ # update header
+ self._fh.seek(0, os.SEEK_SET)
+ header = list(struct.unpack(BUNDLE_V1_HEADER_STRUCT_FORMAT, self._fh.read(60)))
+ header[2] = max(header[2], size)
+ header[5] += size + 4
+ if is_new_tile:
+ header[4] += 4
+ self._fh.seek(0, os.SEEK_SET)
+ self._fh.write(struct.pack(BUNDLE_V1_HEADER_STRUCT_FORMAT, *header))
+
+ return offset, size
+
+
+BUNDLE_V2_GRID_WIDTH = 128
+BUNDLE_V2_GRID_HEIGHT = 128
+BUNDLE_V2_TILES = BUNDLE_V2_GRID_WIDTH * BUNDLE_V2_GRID_HEIGHT
+BUNDLE_V2_INDEX_SIZE = BUNDLE_V2_TILES * 8
+
+BUNDLE_V2_HEADER = (
+ 3, # Version
+ BUNDLE_V2_TILES, # numRecords
+ 0, # maxRecord Size
+ 5, # Offset Size
+ 0, # Slack Space
+ 64 + BUNDLE_V2_INDEX_SIZE, # File Size
+ 40, # User Header Offset
+ 20 + BUNDLE_V2_INDEX_SIZE, # User Header Size
+ 3, # Legacy 1
+ 16, # Legacy 2 0?
+ BUNDLE_V2_TILES, # Legacy 3
+ 5, # Legacy 4
+ BUNDLE_V2_INDEX_SIZE # Index Size
+)
+BUNDLE_V2_HEADER_STRUCT_FORMAT = '<4I3Q6I'
+BUNDLE_V2_HEADER_SIZE = 64
+
+
+class BundleV2(object):
+ def __init__(self, base_filename, offset=None):
+ # offset not used by V2
+ self.filename = base_filename + '.bundle'
+ self.lock_filename = base_filename + '.lck'
+
+ # defer initialization to update/remove calls to avoid
+ # index creation on is_cached (prevents new files in read-only caches)
+ self._initialized = False
+
+ def _init_index(self):
+ self._initialized = True
+ if os.path.exists(self.filename):
+ return
+ ensure_directory(self.filename)
+ buf = BytesIO()
+ buf.write(struct.pack(BUNDLE_V2_HEADER_STRUCT_FORMAT, *BUNDLE_V2_HEADER))
+ # Empty index (ArcGIS stores an offset of 4 and size of 0 for missing tiles)
+ buf.write(struct.pack('<%dQ' % BUNDLE_V2_TILES, *(4, ) * BUNDLE_V2_TILES))
+ write_atomic(self.filename, buf.getvalue())
+
+ def _tile_idx_offset(self, x, y):
+ return BUNDLE_V2_HEADER_SIZE + (x + BUNDLE_V2_GRID_HEIGHT * y) * 8
+ def _rel_tile_coord(self, tile_coord):
+ return (tile_coord[0] % BUNDLE_V2_GRID_WIDTH,
+ tile_coord[1] % BUNDLE_V2_GRID_HEIGHT, )
+
+ def _tile_offset_size(self, fh, x, y):
+ idx_offset = self._tile_idx_offset(x, y)
+ fh.seek(idx_offset)
+ val = INT64LE.unpack(fh.read(8))[0]
+ # Index contains 8 bytes per tile.
+ # Size is stored in 24 most significant bits.
+ # Offset in the least significant 40 bits.
+ size = val >> 40
+ if size == 0:
+ return 0, 0
+ offset = val - (size << 40)
return offset, size
+
+ def _load_tile(self, fh, tile):
+ if tile.source or tile.coord is None:
+ return True
+
+ x, y = self._rel_tile_coord(tile.coord)
+ offset, size = self._tile_offset_size(fh, x, y)
+ if not size:
+ return False
+
+ fh.seek(offset)
+ data = fh.read(size)
+
+ tile.source = ImageSource(BytesIO(data))
+ return True
+
+ def load_tile(self, tile, with_metadata=False):
+ if tile.source or tile.coord is None:
+ return True
+
+ return self.load_tiles([tile], with_metadata)
+
+ def load_tiles(self, tiles, with_metadata=False):
+ missing = False
+
+ with self._readonly() as fh:
+ if not fh:
+ return False
+
+ for t in tiles:
+ if t.source or t.coord is None:
+ continue
+ if not self._load_tile(fh, t):
+ missing = True
+
+ return not missing
+
+ def is_cached(self, tile):
+ with self._readonly() as fh:
+ if not fh:
+ return False
+
+ x, y = self._rel_tile_coord(tile.coord)
+ _, size = self._tile_offset_size(fh, x, y)
+ if not size:
+ return False
+ return True
+
+ def _update_tile_offset(self, fh, x, y, offset, size):
+ idx_offset = self._tile_idx_offset(x, y)
+ val = offset + (size << 40)
+
+ fh.seek(idx_offset, os.SEEK_SET)
+ fh.write(INT64LE.pack(val))
+
+ def _append_tile(self, fh, data):
+ # Write tile size first, then tile data.
+ # Offset points to actual tile data.
+ fh.seek(0, os.SEEK_END)
+ fh.write(struct.pack('<L', len(data)))
+ offset = fh.tell()
+ fh.write(data)
+ return offset
+
+ def _update_metadata(self, fh, filesize, tilesize):
+ # Max record/tile size
+ fh.seek(8)
+ old_tilesize = struct.unpack('<I', fh.read(4))[0]
+ if tilesize > old_tilesize:
+ fh.seek(8)
+ fh.write(struct.pack('<I', tilesize))
+
+ # Complete file size
+ fh.seek(24)
+ fh.write(struct.pack("<Q", filesize))
+
+ def _store_tile(self, fh, tile_coord, data):
+ size = len(data)
+ x, y = self._rel_tile_coord(tile_coord)
+ offset = self._append_tile(fh, data)
+ self._update_tile_offset(fh, x, y, offset, size)
+
+ filesize = offset + size
+ self._update_metadata(fh, filesize, size)
+
+ def store_tile(self, tile):
+ if tile.stored:
+ return True
+
+ return self.store_tiles([tile])
+
+ def store_tiles(self, tiles):
+ self._init_index()
+
+ tiles_data = []
+ for t in tiles:
+ if t.stored:
+ continue
+ with tile_buffer(t) as buf:
+ data = buf.read()
+ tiles_data.append((t.coord, data))
+
+ with FileLock(self.lock_filename):
+ with self._readwrite() as fh:
+ for tile_coord, data in tiles_data:
+ self._store_tile(fh, tile_coord, data)
+
+ return True
+
+
+ def remove_tile(self, tile):
+ if tile.coord is None:
+ return True
+
+ self._init_index()
+ with FileLock(self.lock_filename):
+ with self._readwrite() as fh:
+ x, y = self._rel_tile_coord(tile.coord)
+ self._update_tile_offset(fh, x, y, 0, 0)
+
+ return True
+
+ def size(self):
+ total_size = 0
+ with self._readonly() as fh:
+ if not fh:
+ return 0, 0
+ for y in range(BUNDLE_V2_GRID_HEIGHT):
+ for x in range(BUNDLE_V2_GRID_WIDTH):
+ _, size = self._tile_offset_size(fh, x, y)
+ if size:
+ total_size += size + 4
+ fh.seek(0, os.SEEK_END)
+ actual_size = fh.tell()
+ return total_size + 64 + BUNDLE_V2_INDEX_SIZE, actual_size
+
+ @contextlib.contextmanager
+ def _readonly(self):
+ try:
+ with open(self.filename, 'rb') as fh:
+ yield fh
+ except IOError as ex:
+ if ex.errno == errno.ENOENT:
+ # missing bundle file -> missing tile
+ yield None
+ else:
+ raise ex
+
+ @contextlib.contextmanager
+ def _readwrite(self):
+ self._init_index()
+ with open(self.filename, 'r+b') as fh:
+ yield fh
+
+
+
+class CompactCacheV1(CompactCacheBase):
+ bundle_class = BundleV1
+
+class CompactCacheV2(CompactCacheBase):
+ bundle_class = BundleV2
+
diff --git a/mapproxy/cache/couchdb.py b/mapproxy/cache/couchdb.py
index 659f384..7847ff0 100644
--- a/mapproxy/cache/couchdb.py
+++ b/mapproxy/cache/couchdb.py
@@ -13,10 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import codecs
import datetime
+import json
import socket
import time
import hashlib
@@ -37,13 +37,6 @@ try:
except ImportError:
requests = None
-try:
- import simplejson as json
-except ImportError:
- try:
- import json
- except ImportError:
- json = None
import logging
log = logging.getLogger(__name__)
@@ -59,9 +52,6 @@ class CouchDBCache(TileCacheBase):
if requests is None:
raise ImportError("CouchDB backend requires 'requests' package.")
- if json is None:
- raise ImportError("CouchDB backend requires 'simplejson' package or Python 2.6+.")
-
self.lock_cache_id = 'couchdb-' + hashlib.md5((url + db_name).encode('utf-8')).hexdigest()
self.file_ext = file_ext
self.tile_grid = tile_grid
diff --git a/mapproxy/cache/file.py b/mapproxy/cache/file.py
index 51fe297..b8a04f3 100644
--- a/mapproxy/cache/file.py
+++ b/mapproxy/cache/file.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
import errno
import hashlib
@@ -164,11 +163,8 @@ class FileCache(TileCacheBase):
if os.path.exists(tile_loc) or os.path.islink(tile_loc):
os.unlink(tile_loc)
- # Use relative path for the symlink if os.path.relpath is available
- # (only supported with >= Python 2.6)
- if hasattr(os.path, 'relpath'):
- real_tile_loc = os.path.relpath(real_tile_loc,
- os.path.dirname(tile_loc))
+ # Use relative path for the symlink
+ real_tile_loc = os.path.relpath(real_tile_loc, os.path.dirname(tile_loc))
try:
os.symlink(real_tile_loc, tile_loc)
diff --git a/mapproxy/cache/geopackage.py b/mapproxy/cache/geopackage.py
index 4e9fc69..7b5831f 100644
--- a/mapproxy/cache/geopackage.py
+++ b/mapproxy/cache/geopackage.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import hashlib
import logging
@@ -274,9 +273,10 @@ class GeopackageCache(TileCacheBase):
"""
PROJCS["WGS 84 / Pseudo-Mercator",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,\
AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],\
-UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","9122"]]AUTHORITY["EPSG","4326"]],\
+UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]],\
PROJECTION["Mercator_1SP"],PARAMETER["central_meridian",0],PARAMETER["scale_factor",1],PARAMETER["false_easting",0],\
-PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["X",EAST],AXIS["Y",NORTH]\
+PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["X",EAST],AXIS["Y",NORTH],\
+AUTHORITY["EPSG","3857"]]\
"""
),
(4326, 'epsg', 4326, 'WGS 84',
diff --git a/mapproxy/cache/legend.py b/mapproxy/cache/legend.py
index 7322b2d..6181344 100644
--- a/mapproxy/cache/legend.py
+++ b/mapproxy/cache/legend.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
import hashlib
diff --git a/mapproxy/cache/mbtiles.py b/mapproxy/cache/mbtiles.py
index 5af49f6..f47b100 100644
--- a/mapproxy/cache/mbtiles.py
+++ b/mapproxy/cache/mbtiles.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
+import glob
import hashlib
import os
import sqlite3
@@ -29,6 +29,10 @@ from mapproxy.compat import BytesIO, PY2, itertools
import logging
log = logging.getLogger(__name__)
+if not hasattr(glob, 'escape'):
+ import re
+ glob.escape = lambda pathname: re.sub(r'([*?[])', r'[\1]', pathname)
+
def sqlite_datetime_to_timestamp(datetime):
if datetime is None:
return None
@@ -381,6 +385,8 @@ class MBTilesLevelCache(TileCacheBase):
if timestamp == 0:
level_cache.cleanup()
os.unlink(level_cache.mbtile_file)
+ for file in glob.glob("%s-*" % glob.escape(level_cache.mbtile_file)):
+ os.unlink(file)
return True
else:
return level_cache.remove_level_tiles_before(level, timestamp)
diff --git a/mapproxy/cache/redis.py b/mapproxy/cache/redis.py
index abb9ce1..9a78cb1 100644
--- a/mapproxy/cache/redis.py
+++ b/mapproxy/cache/redis.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, absolute_import
+from __future__ import absolute_import
import hashlib
diff --git a/mapproxy/cache/renderd.py b/mapproxy/cache/renderd.py
index 16dd7f6..82d2b81 100644
--- a/mapproxy/cache/renderd.py
+++ b/mapproxy/cache/renderd.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import time
import hashlib
diff --git a/mapproxy/cache/riak.py b/mapproxy/cache/riak.py
index eaec17c..be01ef2 100644
--- a/mapproxy/cache/riak.py
+++ b/mapproxy/cache/riak.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, absolute_import
+from __future__ import absolute_import
import threading
import hashlib
@@ -36,14 +36,14 @@ class UnexpectedResponse(CacheBackendError):
pass
class RiakCache(TileCacheBase):
- def __init__(self, nodes, protocol, bucket, tile_grid, use_secondary_index=False):
+ def __init__(self, nodes, protocol, bucket, tile_grid, use_secondary_index=False, timeout=60):
if riak is None:
raise ImportError("Riak backend requires 'riak' package.")
self.nodes = nodes
self.protocol = protocol
- self.lock_cache_id = 'riak-' + hashlib.md5(nodes[0]['host'] + bucket).hexdigest()
- self.request_timeout = 10000 # 10s, TODO make configurable
+ self.lock_cache_id = 'riak-' + hashlib.md5(bucket.encode('utf-8')).hexdigest()
+ self.request_timeout = timeout * 1000
self.bucket_name = bucket
self.tile_grid = tile_grid
self.use_secondary_index = use_secondary_index
@@ -57,7 +57,9 @@ class RiakCache(TileCacheBase):
@property
def bucket(self):
- return self.connection.bucket(self.bucket_name)
+ if not getattr(self._db_conn_cache, 'bucket', None):
+ self._db_conn_cache.bucket = self.connection.bucket(self.bucket_name)
+ return self._db_conn_cache.bucket
def _get_object(self, coord):
(x, y, z) = coord
@@ -82,9 +84,7 @@ class RiakCache(TileCacheBase):
return 0.0
def is_cached(self, tile):
- if tile.source:
- return True
- return self.load_tile(tile)
+ return self.load_tile(tile, True)
def _store_bulk(self, tiles):
for tile in tiles:
@@ -101,7 +101,7 @@ class RiakCache(TileCacheBase):
res.add_index('tile_coord_bin', '%02d-%07d-%07d' % (z, x, y))
try:
- res.store(return_body=False, timeout=self.request_timeout)
+ res.store(w=1, dw=1, pw=1, return_body=False, timeout=self.request_timeout)
except riak.RiakError as ex:
log.warn('unable to store tile: %s', ex)
return False
@@ -126,6 +126,8 @@ class RiakCache(TileCacheBase):
self.load_tile(tile, True)
def load_tile(self, tile, with_metadata=False):
+ if tile.timestamp is None:
+ tile.timestamp = 0
if tile.source or tile.coord is None:
return True
diff --git a/mapproxy/cache/s3.py b/mapproxy/cache/s3.py
index 1bbd1d8..a46b041 100644
--- a/mapproxy/cache/s3.py
+++ b/mapproxy/cache/s3.py
@@ -13,8 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
+import calendar
import hashlib
import sys
import threading
@@ -93,7 +93,7 @@ class S3Cache(TileCacheBase):
def _set_metadata(self, response, tile):
if 'LastModified' in response:
- tile.timestamp = float(response['LastModified'].strftime('%s'))
+ tile.timestamp = calendar.timegm(response['LastModified'].timetuple())
if 'ContentLength' in response:
tile.size = response['ContentLength']
diff --git a/mapproxy/cache/sqlite.py b/mapproxy/cache/sqlite.py
deleted file mode 100644
index d41f5cd..0000000
--- a/mapproxy/cache/sqlite.py
+++ /dev/null
@@ -1,413 +0,0 @@
-from __future__ import print_function, division
-
-import errno
-import os
-import sqlite3
-import time
-from contextlib import contextmanager
-from functools import partial
-from mapproxy.compat.itertools import groupby
-from six.moves import map
-from six.moves import zip
-
-
-from mapproxy.compat import BytesIO
-from mapproxy.image import ImageSource, is_single_color_image
-from mapproxy.cache.base import tile_buffer
-from mapproxy.cache.base import TileCacheBase
-
-
-class Metadata(object):
- def __init__(self, db, layer_name, tile_grid, file_ext):
- self.layer_id = layer_name
- self.grid = tile_grid
- self.file_ext = file_ext
- self.db = db
- self.table_name = "metadata"
-
- def _create_metadata_table(self):
- cursor = self.db.cursor()
- stmt = """CREATE TABLE IF NOT EXISTS %s (
- layer_id TEXT NOT NULL,
- matrix_id TEXT NOT NULL,
- matrix_set_id TEXT NOT NULL,
- table_name TEXT NOT NULL,
- bbox TEXT,
- srs TEXT,
- format TEXT,
- min_tile_col INTEGER,
- max_tile_col INTEGER,
- min_tile_row INTEGER,
- max_tile_row INTEGER,
- tile_width INTEGER,
- tile_height INTEGER,
- matrix_width INTEGER,
- matrix_height INTEGER,
- CONSTRAINT unique_rows UNIQUE (layer_id, matrix_id, matrix_set_id, table_name)
- )""" % (self.table_name)
- cursor.execute(stmt)
- self.db.commit()
-
- def store(self, tile_set):
- self.tile_set_table_name(tile_set)
- params = self._tile_set_params_dict(tile_set)
- cursor = self.db.cursor()
- stmt = """CREATE TABLE IF NOT EXISTS %s (
- x INTEGER,
- y INTEGER,
- data BLOB,
- date_added INTEGER,
- unique_tile TEXT
- )""" % (tile_set.table_name)
- cursor.execute(stmt)
- stmt = """CREATE UNIQUE INDEX IF NOT EXISTS idx_%s_xy ON %s (x, y)""" % (tile_set.table_name, tile_set.table_name)
- cursor.execute(stmt)
- stmt = """INSERT INTO %s (layer_id, bbox, srs, format, min_tile_col, max_tile_col, min_tile_row,
- max_tile_row, tile_width, tile_height, matrix_width, matrix_height, matrix_id, matrix_set_id, table_name)
- VALUES (:layer_id, :bbox, :srs, :format, :min_tile_col, :max_tile_col, :min_tile_row, :max_tile_row,
- :tile_width, :tile_height, :matrix_width, :matrix_height, :matrix_id,
- :matrix_set_name, :table_name)""" % (self.table_name)
- try:
- cursor.execute(stmt, params)
- except sqlite3.IntegrityError as e:
- pass
- self.db.commit()
-
- def _tile_set_params_dict(self, tile_set):
- level = tile_set.level
- tile_width, tile_height = self.grid.tile_size
- matrix_width, matrix_height = self.grid.grid_sizes[level]
- params = {
- 'layer_id' : self.layer_id,
- 'bbox' : ', '.join(map(str, [v for v in self.grid.bbox])),
- 'srs' : self.grid.srs.srs_code,
- 'format' : self.file_ext,
- 'min_tile_col' : tile_set.grid[0],
- 'max_tile_col' : tile_set.grid[2],
- 'min_tile_row' : tile_set.grid[1],
- 'max_tile_row' : tile_set.grid[3],
- 'tile_width' : tile_width,
- 'tile_height' : tile_height,
- 'matrix_width' : matrix_width,
- 'matrix_height' : matrix_height,
- 'matrix_id' : level,
- 'matrix_set_name' : self.grid.name,
- 'table_name' : tile_set.table_name
- }
- return params
-
- def tile_set_table_name(self, tile_set):
- if tile_set.table_name:
- return tile_set.table_name
-
- level = tile_set.level
- grid = tile_set.grid
- query = " AND ".join(["layer_id = ?",
- "min_tile_col = ?",
- "min_tile_row = ?",
- "max_tile_col = ?",
- "max_tile_row = ?",
- "matrix_id = ?",
- "matrix_set_id = ?"])
- cursor = self.db.cursor()
- stmt = "SELECT table_name FROM %s WHERE %s" % (self.table_name, query)
- cursor.execute(stmt, (self.layer_id, grid[0], grid[1], grid[2], grid[3], level, self.grid.name))
- result = cursor.fetchone()
- if result is not None:
- table_name = result['table_name']
- else:
- x0, y0, x1, y1 = tile_set.grid
- table_name = "tileset_%s_%s_%d_%d_%d_%d_%d" % (self.layer_id, self.grid.name, tile_set.level, x0, y0, x1, y1)
-
- tile_set.table_name = table_name
- return table_name
-
-
-class TileSet(object):
-
- def __init__(self, db, level, file_ext, grid, unique_tiles):
- self.db = db
- self.table_name = None
- self.file_ext = file_ext
- self._metadata_stored = False
- self.grid = grid
- self.level = level
- self.unique_tiles = unique_tiles
-
- def get_tiles(self, tiles):
- stmt = "SELECT x, y, data, date_added, unique_tile FROM %s WHERE " % (self.table_name)
- stmt += ' OR '.join(['(x = ? AND y = ?)'] * len(tiles))
-
- coords = []
- for tile in tiles:
- x, y, level = tile.coord
- coords.append(x)
- coords.append(y)
-
- cursor = self.db.cursor()
- try:
- cursor.execute(stmt, coords)
- except sqlite3.OperationalError as e:
- print(e)
-
- #associate the right tiles with the cursor
- tile_dict = {}
- for tile in tiles:
- x, y, level = tile.coord
- tile_dict[(x, y)] = tile
-
- for row in cursor:
- tile = tile_dict[(row['x'], row['y'])]
- #TODO get unique tiles if row['data'] == null
- data = row['data'] if row['data'] is not None else self.unique_tiles.get_data(row['unique_tile'])
- tile.timestamp = row['date_added']
- tile.size = len(data)
- tile.source = ImageSource(BytesIO(data), size=tile.size)
- cursor.close()
- return tiles
-
- def is_cached(self, tile):
- x, y, level = tile.coord
- cursor = self.db.cursor()
- stmt = "SELECT date_added from %s WHERE x = ? AND y = ?" % (self.table_name)
- try:
- cursor.execute(stmt, (x, y))
- except sqlite3.OperationalError as e:
- #table does not exist
- #print e
- pass
- result = cursor.fetchone()
- if result is not None:
- return True
- return False
-
- def set_tile(self, tile):
- x, y, z = tile.coord
- assert self.grid[0] <= x < self.grid[2]
- assert self.grid[1] <= y < self.grid[3]
-
-
- color = is_single_color_image(tile.source.as_image())
-
- with tile_buffer(tile) as buf:
- _data = buffer(buf.read())
-
- if color:
- data = None
- _color = ''.join('%02x' % v for v in color)
- self.unique_tiles.set_data(_data, _color)
- else:
- #get value of cStringIO-Object and store it to a buffer
- data = _data
- _color = None
-
- timestamp = int(time.time())
- cursor = self.db.cursor()
- stmt = "INSERT INTO %s (x, y, data, date_added, unique_tile) VALUES (?,?,?,?,?)" % (self.table_name)
- try:
- cursor.execute(stmt, (x, y, data, timestamp, _color))
- except (sqlite3.IntegrityError, sqlite3.OperationalError) as e:
- #tile is already present, updating data
- stmt = "UPDATE %s SET data = ?, date_added = ?, unique_tile = ? WHERE x = ? AND y = ?" % (self.table_name)
- try:
- cursor.execute(stmt, (data, timestamp, _color, x, y))
- except sqlite3.OperationalError as e:
- #database is locked
- print(e)
- return False
- return True
-
- def set_tiles(self, tiles):
- result = all([self.set_tile(t) for t in tiles])
- self.db.commit()
- return result
-
- def remove_tiles(self, tiles):
- cursor = self.db.cursor()
- stmt = "DELETE FROM %s WHERE" % (self.table_name)
- stmt += ' OR '.join(['(x = ? AND y = ?)'] * len(tiles))
- coords = []
- for t in tiles:
- x, y, level = t.coord
- coords.append(x)
- coords.append(y)
- try:
- cursor.execute(stmt, coords)
- self.db.commit()
- except sqlite3.OperationalError as e:
- #no such table
- #print e
- pass
- if cursor.rowcount < 1:
- return False
- return True
-
-
-class TileSetManager(object):
- def __init__(self, db, subgrid_size, file_ext, metadata, unique_tiles):
- self.db = db
- self.subgrid_size = subgrid_size
- self.file_ext = file_ext
- self.md = metadata
- self.unique_tiles = unique_tiles
- self._tile_sets = {}
-
- def store(self, tiles):
- return all(tile_set.set_tiles(list(tile_set_tiles))
- for tile_set, tile_set_tiles
- in groupby(tiles, partial(self._get_tile_set, create_db_entry=True)))
-
- def load(self, tiles):
- for tile_set, tile_set_tiles in groupby(tiles, self._get_tile_set):
- tile_set.get_tiles(list(tile_set_tiles))
-
- for tile in tiles:
- if tile.source is None:
- return False
- return True
-
- def remove(self, tiles):
- return all(tile_set.remove_tiles(list(tile_set_tiles))
- for tile_set, tile_set_tiles in groupby(tiles, self._get_tile_set))
-
- def is_cached(self, tile):
- tile_set = self._get_tile_set(tile)
- return tile_set.is_cached(tile)
-
- def _get_tile_set(self, tile, create_db_entry=False):
- x, y, level = tile.coord
- x0, y0 = self.subgrid_size
- x_pos = x//x0
- y_pos = y//y0
- key = (x0 * x_pos, y0 * y_pos)
- if key not in self._tile_sets:
- grid = [x0 * x_pos, y0 * y_pos, x0 * x_pos + x0, y0 * y_pos + y0]
- tile_set = TileSet(self.db, level, self.file_ext, grid, self.unique_tiles)
- tile_set.table_name = self.md.tile_set_table_name(tile_set)
- self._tile_sets[key] = tile_set
- else:
- tile_set = self._tile_sets[key]
- if create_db_entry and not tile_set._metadata_stored:
- self.md.store(tile_set)
- tile_set._metadata_stored = True
- return tile_set
-
-class UniqueTiles(object):
- def __init__(self, db):
- self.db = db
- self.table_name = "unique_tiles"
-
- def _create_unique_tiles_table(self):
- cursor = self.db.cursor()
- stmt = """CREATE TABLE IF NOT EXISTS %s (
- id TEXT PRIMARY KEY,
- data BLOB
- ) """ % (self.table_name)
- cursor.execute(stmt)
- self.db.commit()
-
- def set_data(self, data, color):
- cursor = self.db.cursor()
- stmt = "INSERT INTO %s (id, data) VALUES (?, ?)" % (self.table_name)
- try:
- cursor.execute(stmt, (color, data))
- except sqlite3.IntegrityError as e:
- #data already present
- pass
- #TODO check entry point for commit()
- self.db.commit()
-
- def get_data(self, color):
- #_color = tile.unique_tile
- cursor = self.db.cursor()
- stmt = "SELECT data FROM %s WHERE id = ?" % (self.table_name)
- try:
- cursor.execute(stmt, (color,))
- except sqlite3.OperationalError as e:
- #tile not present
- pass
- row = cursor.fetchone()
- #returning raw data here - an ImageSource-Object will be created within the TileSet-Class
- return row['data']
-
-class DatabaseStore(TileCacheBase):
- def __init__(self, path, layer_name, grid, sub_grid, file_ext):
- self.cache_path = path
- self.layer_name = layer_name
- self._subgrid_size = sub_grid
- self.grid = grid
- self.file_ext = file_ext
- self._tile_set_mgr = {}
- self._db = sqlite3.connect(self.cache_path, timeout=3)
- self._db.row_factory = sqlite3.Row
- self._metadata = Metadata(self._db, self.layer_name, self.grid, self.file_ext)
- self._metadata._create_metadata_table()
- self._unique_tiles = UniqueTiles(self._db)
- self._unique_tiles._create_unique_tiles_table()
-
- @property
- def db(self):
- return self._db
-
- def _get_tile_set_mgr(self, tile):
- x, y, level = tile.coord
- if level not in self._tile_set_mgr:
- size = self.grid.grid_sizes[level]
- size = min(size[0], self._subgrid_size[0]), min(size[1], self._subgrid_size[1])
- self._tile_set_mgr[level] = TileSetManager(self.db, size, self.file_ext, self._metadata, self._unique_tiles)
- return self._tile_set_mgr[level]
-
- def store_tile(self, tile):
- mgr = self._get_tile_set_mgr(tile)
- return mgr.store([tile])
-
- def store_tiles(self, tiles):
- first_tile = None
- for t in tiles:
- if t.coord is not None:
- first_tile = t
- break
- else:
- return True
- mgr = self._get_tile_set_mgr(first_tile)
- return mgr.store(tiles)
-
- def load_tile(self, tile, with_metadata=False):
- if tile.coord is None:
- return True
-
- mgr = self._get_tile_set_mgr(tile)
- return mgr.load([tile])
-
- #TODO implement metadata query
- def load_tiles(self, tiles, with_metadata=False):
- first_tile = None
- for t in tiles:
- if t.coord is not None:
- first_tile = t
- break
- else:
- return True
- mgr = self._get_tile_set_mgr(first_tile)
- return mgr.load([t for t in tiles if t.source is None and t.coord is not None])
-
- def remove_tile(self, tile):
- mgr = self._get_tile_set_mgr(tile)
- return mgr.remove([tile])
-
- def remove_tiles(self, tiles):
- first_tile = None
- for t in tiles:
- if t.coord is not None:
- first_tile = t
- break
- else:
- return True
- mgr = self._get_tile_set_mgr(first_tile)
- return mgr.remove(tiles)
-
- def is_cached(self, tile):
- if tile.coord is None:
- return True
- mgr = self._get_tile_set_mgr(tile)
- return mgr.is_cached(tile)
diff --git a/mapproxy/cache/tile.py b/mapproxy/cache/tile.py
index 560c415..0aa4037 100644
--- a/mapproxy/cache/tile.py
+++ b/mapproxy/cache/tile.py
@@ -35,7 +35,6 @@ Tile caching (creation, caching and retrieval of tiles).
"""
-from __future__ import with_statement
from functools import partial
from contextlib import contextmanager
@@ -72,7 +71,6 @@ class TileManager(object):
self.sources = sources
self.minimize_meta_requests = minimize_meta_requests
self._expire_timestamp = None
- self.transparent = self.sources[0].transparent
self.pre_store_filter = pre_store_filter or []
self.concurrent_tile_creators = concurrent_tile_creators
self.tile_creator_class = tile_creator_class or TileCreator
diff --git a/mapproxy/client/http.py b/mapproxy/client/http.py
index 7bc7202..9b02719 100644
--- a/mapproxy/client/http.py
+++ b/mapproxy/client/http.py
@@ -1,5 +1,5 @@
# This file is part of the MapProxy project.
-# Copyright (C) 2010 Omniscale <http://omniscale.de>
+# Copyright (C) 2010-2017 Omniscale <http://omniscale.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -37,35 +37,87 @@ else:
from http import client as httplib
import socket
+import ssl
+
+supports_ssl_default_context = False
+if hasattr(ssl, 'create_default_context'):
+ # Python >=2.7.9 and >=3.4.0
+ supports_ssl_default_context = True
class HTTPClientError(Exception):
def __init__(self, arg, response_code=None):
Exception.__init__(self, arg)
self.response_code = response_code
-if sys.version_info >= (2, 6):
- _urllib2_has_timeout = True
-else:
- _urllib2_has_timeout = False
-_max_set_timeout = None
+def build_https_handler(ssl_ca_certs, insecure):
+ if supports_ssl_default_context:
+ # python >=2.7.9 and >=3.4 supports ssl context in
+ # HTTPSHandler use this
+ if insecure:
+ ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+ ctx.verify_mode = ssl.CERT_NONE
+ elif ssl_ca_certs:
+ ctx = ssl.create_default_context(cafile=ssl_ca_certs)
+ else:
+ ctx = ssl.create_default_context()
+ return urllib2.HTTPSHandler(context=ctx)
+ else:
+ if insecure:
+ return None
+ else:
+ connection_class = verified_https_connection_with_ca_certs(
+ ssl_ca_certs)
+ return VerifiedHTTPSHandler(connection_class=connection_class)
-try:
- import ssl
- ssl # prevent pyflakes warnings
-except ImportError:
- ssl = None
+class VerifiedHTTPSConnection(httplib.HTTPSConnection):
+ def __init__(self, *args, **kw):
+ self._ca_certs = kw.pop('ca_certs', None)
+ httplib.HTTPSConnection.__init__(self, *args, **kw)
-def _set_global_socket_timeout(timeout):
- global _max_set_timeout
- if _max_set_timeout is None:
- _max_set_timeout = timeout
- elif _max_set_timeout != timeout:
- _max_set_timeout = max(_max_set_timeout, timeout)
- warnings.warn("Python >=2.6 required for individual HTTP timeouts. Setting global timeout to %.1f." %
- _max_set_timeout)
- socket.setdefaulttimeout(_max_set_timeout)
+ def connect(self):
+ # overrides the version in httplib so that we do
+ # certificate verification
+
+ if hasattr(socket, 'create_connection') and hasattr(self, 'source_address'):
+ sock = socket.create_connection((self.host, self.port),
+ self.timeout, self.source_address)
+ else:
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.connect((self.host, self.port))
+
+ if hasattr(self, '_tunnel_host') and self._tunnel_host:
+ # for Python >= 2.6 with proxy support
+ self.sock = sock
+ self._tunnel()
+
+ # wrap the socket using verification with the root
+ # certs in self.ca_certs_path
+ self.sock = ssl.wrap_socket(sock,
+ self.key_file,
+ self.cert_file,
+ cert_reqs=ssl.CERT_REQUIRED,
+ ca_certs=self._ca_certs)
+
+
+def verified_https_connection_with_ca_certs(ca_certs):
+ """
+ Creates VerifiedHTTPSConnection classes with given ca_certs file.
+ """
+ def wrapper(*args, **kw):
+ kw['ca_certs'] = ca_certs
+ return VerifiedHTTPSConnection(*args, **kw)
+ return wrapper
+
+
+class VerifiedHTTPSHandler(urllib2.HTTPSHandler):
+ def __init__(self, connection_class=VerifiedHTTPSConnection):
+ self.specialized_conn_class = connection_class
+ urllib2.HTTPSHandler.__init__(self)
+
+ def https_open(self, req):
+ return self.do_open(self.specialized_conn_class, req)
class _URLOpenerCache(object):
@@ -78,12 +130,12 @@ class _URLOpenerCache(object):
def __init__(self):
self._opener = {}
- def __call__(self, ssl_ca_certs, url, username, password):
- if ssl_ca_certs not in self._opener:
+ def __call__(self, ssl_ca_certs, url, username, password, insecure=False):
+ cache_key = (ssl_ca_certs, insecure)
+ if cache_key not in self._opener:
handlers = []
- if ssl_ca_certs:
- connection_class = verified_https_connection_with_ca_certs(ssl_ca_certs)
- https_handler = VerifiedHTTPSHandler(connection_class=connection_class)
+ https_handler = build_https_handler(ssl_ca_certs, insecure)
+ if https_handler:
handlers.append(https_handler)
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
authhandler = urllib2.HTTPBasicAuthHandler(passman)
@@ -92,11 +144,12 @@ class _URLOpenerCache(object):
handlers.append(authhandler)
opener = urllib2.build_opener(*handlers)
+
opener.addheaders = [('User-agent', 'MapProxy-%s' % (version,))]
- self._opener[ssl_ca_certs] = (opener, passman)
+ self._opener[cache_key] = (opener, passman)
else:
- opener, passman = self._opener[ssl_ca_certs]
+ opener, passman = self._opener[cache_key]
if url is not None and username is not None and password is not None:
passman.add_password(None, url, username, password)
@@ -108,24 +161,15 @@ create_url_opener = _URLOpenerCache()
class HTTPClient(object):
def __init__(self, url=None, username=None, password=None, insecure=False,
ssl_ca_certs=None, timeout=None, headers=None):
- if _urllib2_has_timeout:
- self._timeout = timeout
- else:
- self._timeout = None
- _set_global_socket_timeout(timeout)
+ self._timeout = timeout
if url and url.startswith('https'):
if insecure:
ssl_ca_certs = None
- else:
- if ssl is None:
- raise ImportError('No ssl module found. SSL certificate '
- 'verification requires Python 2.6 or ssl module. Upgrade '
- 'or disable verification with http.ssl_no_cert_checks option.')
- if ssl_ca_certs is None:
+ elif ssl_ca_certs is None and not supports_ssl_default_context:
raise HTTPClientError('No ca_certs file set (http.ssl_ca_certs). '
'Set file or disable verification with http.ssl_no_cert_checks option.')
- self.opener = create_url_opener(ssl_ca_certs, url, username, password)
+ self.opener = create_url_opener(ssl_ca_certs, url, username, password, insecure=insecure)
self.header_list = headers.items() if headers else []
def open(self, url, data=None):
@@ -149,7 +193,7 @@ class HTTPClient(object):
reraise_exception(HTTPClientError('HTTP Error "%s": %d'
% (url, e.code), response_code=code), sys.exc_info())
except URLError as e:
- if ssl and isinstance(e.reason, ssl.SSLError):
+ if isinstance(e.reason, ssl.SSLError):
e = HTTPClientError('Could not verify connection to URL "%s": %s'
% (url, e.reason.args[1]))
reraise_exception(e, sys.exc_info())
@@ -192,17 +236,18 @@ def auth_data_from_url(url):
('http://localhost/bar', ('bar', 'b:az@'))
>>> auth_data_from_url('http://bar foo; foo at bar:b:az@@localhost/bar')
('http://localhost/bar', ('bar foo; foo at bar', 'b:az@'))
+ >>> auth_data_from_url('https://bar:foo#;%$@localhost/bar')
+ ('https://localhost/bar', ('bar', 'foo#;%$'))
"""
username = password = None
if '@' in url:
- scheme, host, path, query, frag = urlparse.urlsplit(url)
- if '@' in host:
- auth_data, host = host.rsplit('@', 1)
- url = url.replace(auth_data+'@', '', 1)
- if ':' in auth_data:
- username, password = auth_data.split(':', 1)
- else:
- username = auth_data
+ head, url = url.rsplit('@', 1)
+ schema, auth_data = head.split('//', 1)
+ url = schema + '//' + url
+ if ':' in auth_data:
+ username, password = auth_data.split(':', 1)
+ else:
+ username = auth_data
return url, (username, password)
@@ -224,49 +269,3 @@ def retrieve_image(url, client=None):
raise HTTPClientError('response is not an image: (%s)' % (resp.read()))
return ImageSource(resp)
-
-class VerifiedHTTPSConnection(httplib.HTTPSConnection):
- def __init__(self, *args, **kw):
- self._ca_certs = kw.pop('ca_certs', None)
- httplib.HTTPSConnection.__init__(self, *args, **kw)
-
- def connect(self):
- # overrides the version in httplib so that we do
- # certificate verification
-
- if hasattr(socket, 'create_connection') and hasattr(self, 'source_address'):
- sock = socket.create_connection((self.host, self.port),
- self.timeout, self.source_address)
- else:
- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- sock.connect((self.host, self.port))
-
- if hasattr(self, '_tunnel_host') and self._tunnel_host:
- # for Python >= 2.6 with proxy support
- self.sock = sock
- self._tunnel()
-
- # wrap the socket using verification with the root
- # certs in self.ca_certs_path
- self.sock = ssl.wrap_socket(sock,
- self.key_file,
- self.cert_file,
- cert_reqs=ssl.CERT_REQUIRED,
- ca_certs=self._ca_certs)
-
-def verified_https_connection_with_ca_certs(ca_certs):
- """
- Creates VerifiedHTTPSConnection classes with given ca_certs file.
- """
- def wrapper(*args, **kw):
- kw['ca_certs'] = ca_certs
- return VerifiedHTTPSConnection(*args, **kw)
- return wrapper
-
-class VerifiedHTTPSHandler(urllib2.HTTPSHandler):
- def __init__(self, connection_class=VerifiedHTTPSConnection):
- self.specialized_conn_class = connection_class
- urllib2.HTTPSHandler.__init__(self)
-
- def https_open(self, req):
- return self.do_open(self.specialized_conn_class, req)
\ No newline at end of file
diff --git a/mapproxy/client/wms.py b/mapproxy/client/wms.py
index caa462e..c0ae45a 100644
--- a/mapproxy/client/wms.py
+++ b/mapproxy/client/wms.py
@@ -16,7 +16,6 @@
"""
WMS clients for maps and information.
"""
-from __future__ import with_statement
from mapproxy.compat import text_type
from mapproxy.request.base import split_mime_type
from mapproxy.layer import InfoQuery
diff --git a/mapproxy/compat/__init__.py b/mapproxy/compat/__init__.py
index fdb594f..0ccf442 100644
--- a/mapproxy/compat/__init__.py
+++ b/mapproxy/compat/__init__.py
@@ -38,4 +38,9 @@ if PY2:
except ImportError:
from StringIO import StringIO as BytesIO
else:
- from io import BytesIO
\ No newline at end of file
+ from io import BytesIO
+
+if PY2:
+ raw_input = raw_input
+else:
+ raw_input = input
diff --git a/mapproxy/config/config.py b/mapproxy/config/config.py
index 620e145..451ce77 100644
--- a/mapproxy/config/config.py
+++ b/mapproxy/config/config.py
@@ -16,7 +16,6 @@
"""
System-wide configuration.
"""
-from __future__ import with_statement
import os
import copy
import contextlib
diff --git a/mapproxy/config/loader.py b/mapproxy/config/loader.py
index 033853e..c0ad82d 100644
--- a/mapproxy/config/loader.py
+++ b/mapproxy/config/loader.py
@@ -16,7 +16,7 @@
"""
Configuration loading and system initializing.
"""
-from __future__ import with_statement, division
+from __future__ import division
import os
import sys
@@ -34,6 +34,7 @@ from mapproxy.config.spec import validate_options
from mapproxy.util.py import memoize
from mapproxy.util.ext.odict import odict
from mapproxy.util.yaml import load_yaml_file, YAMLError
+from mapproxy.util.fs import find_exec
from mapproxy.compat.modules import urlparse
from mapproxy.compat import string_type, iteritems
@@ -606,6 +607,13 @@ class ArcGISSourceConfiguration(SourceConfiguration):
from mapproxy.srs import SRS
from mapproxy.request.arcgis import create_request
+ if not self.conf.get('opts', {}).get('map', True):
+ return None
+
+ if not self.context.seed and self.conf.get('seed_only'):
+ from mapproxy.source import DummySource
+ return DummySource(coverage=self.coverage())
+
# Get the supported SRS codes and formats from the configuration.
supported_srs = [SRS(code) for code in self.conf.get("supported_srs", [])]
supported_formats = [file_ext(f) for f in self.conf.get("supported_formats", [])]
@@ -835,13 +843,16 @@ class MapServerSourceConfiguration(WMSSourceConfiguration):
WMSSourceConfiguration.__init__(self, conf, context)
self.script = self.context.globals.get_path('mapserver.binary',
self.conf)
+ if not self.script:
+ self.script = find_exec('mapserv')
+
if not self.script or not os.path.isfile(self.script):
raise ConfigurationError('could not find mapserver binary (%r)' %
(self.script, ))
# set url to dummy script name, required as identifier
# for concurrent_request
- self.conf['req']['url'] = 'http://localhost' + self.script
+ self.conf['req']['url'] = 'mapserver://' + self.script
mapfile = self.context.globals.abspath(self.conf['req']['map'])
self.conf['req']['map'] = mapfile
@@ -1181,10 +1192,12 @@ class CacheConfiguration(ConfigurationBase):
bucket = self.conf['name'] + '_' + suffix
use_secondary_index = self.conf['cache'].get('secondary_index', False)
+ timeout = self.context.globals.get_value('http.client_timeout', self.conf)
return RiakCache(nodes=nodes, protocol=protocol, bucket=bucket,
tile_grid=grid_conf.tile_grid(),
use_secondary_index=use_secondary_index,
+ timeout=timeout
)
def _redis_cache(self, grid_conf, file_ext):
@@ -1208,7 +1221,7 @@ class CacheConfiguration(ConfigurationBase):
)
def _compact_cache(self, grid_conf, file_ext):
- from mapproxy.cache.compact import CompactCacheV1
+ from mapproxy.cache.compact import CompactCacheV1, CompactCacheV2
cache_dir = self.cache_dir()
if self.conf.get('cache', {}).get('directory'):
@@ -1221,11 +1234,13 @@ class CacheConfiguration(ConfigurationBase):
else:
cache_dir = os.path.join(cache_dir, self.conf['name'], grid_conf.tile_grid().name)
- if self.conf['cache']['version'] != 1:
- raise ConfigurationError("compact cache only supports version 1")
- return CompactCacheV1(
- cache_dir=cache_dir,
- )
+ version = self.conf['cache']['version']
+ if version == 1:
+ return CompactCacheV1(cache_dir=cache_dir)
+ elif version == 2:
+ return CompactCacheV2(cache_dir=cache_dir)
+
+ raise ConfigurationError("compact cache only supports version 1 or 2")
def _tile_cache(self, grid_conf, file_ext):
if self.conf.get('disable_storage', False):
@@ -1453,7 +1468,7 @@ class CacheConfiguration(ConfigurationBase):
if use_renderd:
from mapproxy.cache.renderd import RenderdTileCreator, has_renderd_support
if not has_renderd_support():
- raise ConfigurationError("renderd requires Python >=2.6 and requests")
+ raise ConfigurationError("renderd requires requests library")
if self.context.seed:
priority = 10
else:
@@ -1528,7 +1543,7 @@ class CacheConfiguration(ConfigurationBase):
if len(caches) == 1:
layer = caches[0][0]
else:
- layer = SRSConditional(caches, caches[0][0].extent, caches[0][0].transparent, opacity=image_opts.opacity)
+ layer = SRSConditional(caches, caches[0][0].extent, opacity=image_opts.opacity)
if 'use_direct_from_level' in self.conf:
self.conf['use_direct_from_res'] = main_grid.resolution(self.conf['use_direct_from_level'])
@@ -1695,7 +1710,7 @@ class LayerConfiguration(ConfigurationBase):
if grid_name_as_path:
md['name_path'] = (md['name'], md['grid_name'])
else:
- md['name_path'] = (self.conf['name'], grid.srs.srs_code.replace(':', '').upper())
+ md['name_path'] = (md['name'], grid.srs.srs_code.replace(':', '').upper())
md['name_internal'] = md['name_path'][0] + '_' + md['name_path'][1]
md['format'] = self.context.caches[cache_name].image_opts().format
md['cache_name'] = cache_name
@@ -1906,9 +1921,11 @@ class ServiceConfiguration(ConfigurationBase):
lyr = layer_conf.wms_layer()
if lyr:
layers[layer_name] = lyr
- tile_layers = self.tile_layers(conf)
image_formats = self.context.globals.get_value('image_formats', conf, global_key='wms.image_formats')
srs = self.context.globals.get_value('srs', conf, global_key='wms.srs')
+ tms_conf = self.context.services.conf.get('tms', {}) or {}
+ use_grid_names = tms_conf.get('use_grid_names', False)
+ tile_layers = self.tile_layers(tms_conf, use_grid_names=use_grid_names)
# WMTS restful template
wmts_conf = self.context.services.conf.get('wmts', {}) or {}
diff --git a/mapproxy/config/validator.py b/mapproxy/config/validator.py
index 74b9757..1919555 100644
--- a/mapproxy/config/validator.py
+++ b/mapproxy/config/validator.py
@@ -108,7 +108,7 @@ class Validator(object):
def _split_tagged_source(self, source_name):
layers = None
if ':' in str(source_name):
- source_name, layers = str(source_name).split(':')
+ source_name, layers = str(source_name).split(':', 1)
layers = layers.split(',') if layers is not None else None
return source_name, layers
diff --git a/mapproxy/config_template/base_config/full_example.yaml b/mapproxy/config_template/base_config/full_example.yaml
index 14bdd62..fb8eff7 100644
--- a/mapproxy/config_template/base_config/full_example.yaml
+++ b/mapproxy/config_template/base_config/full_example.yaml
@@ -329,6 +329,22 @@ caches:
center: '{{wgs_tile_centroid}}'
grids: [GLOBAL_MERCATOR]
sources: [osm_wms]
+ riak_cache:
+ grid: [GLOBAL_MERCATOR]
+ sources: [osm_wms]
+ cache:
+ type: riak
+ bucket: tile_bucket
+ protocol: pbc
+ default_ports:
+ pb: 8087
+ http: 8098
+ nodes:
+ - host: 1.example.com
+ pb_port: 9999
+ - host: 2.example.com
+ - host: 3.example.com
+ http_port: 8888
sources:
# minimal WMS source
diff --git a/mapproxy/image/__init__.py b/mapproxy/image/__init__.py
index 42572c7..57e44f6 100644
--- a/mapproxy/image/__init__.py
+++ b/mapproxy/image/__init__.py
@@ -16,7 +16,6 @@
"""
Image and tile manipulation (transforming, merging, etc).
"""
-from __future__ import with_statement
import io
from io import BytesIO
@@ -196,7 +195,7 @@ def SubImageSource(source, size, offset, image_opts, cacheable=True):
source = ImageSource(source)
subimg = source.as_image()
img.paste(subimg, offset)
- return ImageSource(img, size=size, image_opts=image_opts, cacheable=cacheable)
+ return ImageSource(img, size=size, image_opts=new_image_opts, cacheable=cacheable)
class BlankImageSource(object):
"""
@@ -313,6 +312,12 @@ def img_to_buf(img, image_opts):
defaults['quality'] = image_opts.encoding_options['jpeg_quality']
else:
defaults['quality'] = base_config().image.jpeg_quality
+
+ # unsupported transparency tuple can still be in non-RGB img.infos
+ # see: https://github.com/python-pillow/Pillow/pull/2633
+ if format == 'png' and img.mode != 'RGB' and 'transparency' in img.info and isinstance(img.info['transparency'], tuple):
+ del img.info['transparency']
+
img.save(buf, format, **defaults)
buf.seek(0)
return buf
diff --git a/mapproxy/image/merge.py b/mapproxy/image/merge.py
index cf35ebe..bea132e 100644
--- a/mapproxy/image/merge.py
+++ b/mapproxy/image/merge.py
@@ -16,7 +16,6 @@
"""
Image and tile manipulation (transforming, merging, etc).
"""
-from __future__ import with_statement
from collections import namedtuple
from mapproxy.compat.image import Image, ImageColor, ImageChops, ImageMath
diff --git a/mapproxy/image/transform.py b/mapproxy/image/transform.py
index ba7ec2e..ed4bc4a 100644
--- a/mapproxy/image/transform.py
+++ b/mapproxy/image/transform.py
@@ -30,13 +30,11 @@ class ImageTransformer(object):
The quadrilateral will then be transformed with the source coordinates
into the destination quad (affine).
- This method will perform good transformation results if the number of
- quads is high enough (even transformations with strong distortions).
- Tests on images up to 1500x1500 have shown that meshes beyond 8x8
- will not improve the results.
+ The number of quads is calculated dynamically to keep the deviation in
+ the image transformation below one pixel.
.. _PIL Image.transform:
- http://www.pythonware.com/library/pil/handbook/image.htm#Image.transform
+ http://pillow.readthedocs.io/en/stable/reference/Image.html#PIL.Image.Image.transform
::
@@ -49,20 +47,17 @@ class ImageTransformer(object):
---------------.
large src image large dst image
"""
- def __init__(self, src_srs, dst_srs, mesh_div=8):
+ def __init__(self, src_srs, dst_srs, max_px_err=1):
"""
:param src_srs: the srs of the source image
:param dst_srs: the srs of the target image
:param resampling: the resampling method used for transformation
:type resampling: nearest|bilinear|bicubic
- :param mesh_div: the number of quads in each direction to use
- for transformation (totals to ``mesh_div**2`` quads)
-
"""
self.src_srs = src_srs
self.dst_srs = dst_srs
- self.mesh_div = mesh_div
self.dst_bbox = self.dst_size = None
+ self.max_px_err = max_px_err
def transform(self, src_img, src_bbox, dst_size, dst_bbox, image_opts):
"""
@@ -129,43 +124,39 @@ class ImageTransformer(object):
"""
Do a 'real' transformation with a transformed mesh (see above).
"""
- src_bbox = self.src_srs.align_bbox(src_bbox)
- dst_bbox = self.dst_srs.align_bbox(dst_bbox)
- src_size = src_img.size
- src_quad = (0, 0, src_size[0], src_size[1])
- dst_quad = (0, 0, dst_size[0], dst_size[1])
- to_src_px = make_lin_transf(src_bbox, src_quad)
- to_dst_w = make_lin_transf(dst_quad, dst_bbox)
- meshes = []
# more recent versions of Pillow use center coordinates for
# transformations, we manually need to add half a pixel otherwise
if transform_uses_center():
- px_offset = 0.0
+ use_center_px = False
else:
- px_offset = 0.5
-
- def dst_quad_to_src(quad):
- src_quad = []
- for dst_px in [(quad[0], quad[1]), (quad[0], quad[3]),
- (quad[2], quad[3]), (quad[2], quad[1])]:
- dst_w = to_dst_w((dst_px[0]+px_offset, dst_px[1]+px_offset))
- src_w = self.dst_srs.transform_to(self.src_srs, dst_w)
- src_px = to_src_px(src_w)
- src_quad.extend(src_px)
- return quad, src_quad
-
- mesh_div = self.mesh_div
- while mesh_div > 1 and (dst_size[0] / mesh_div < 10 or dst_size[1] / mesh_div < 10):
- mesh_div -= 1
- for quad in griddify(dst_quad, mesh_div):
- meshes.append(dst_quad_to_src(quad))
+ use_center_px = True
+
+ meshes = transform_meshes(
+ src_size=src_img.size,
+ src_bbox=src_bbox,
+ src_srs=self.src_srs,
+ dst_size=dst_size,
+ dst_bbox=dst_bbox,
+ dst_srs=self.dst_srs,
+ max_px_err=self.max_px_err,
+ use_center_px=use_center_px,
+ )
img = img_for_resampling(src_img.as_image(), image_opts.resampling)
result = img.transform(dst_size, Image.MESH, meshes,
image_filter[image_opts.resampling])
+
+ if False:
+ # draw mesh for debuging
+ from PIL import ImageDraw
+ draw = ImageDraw.Draw(result)
+ for g, _ in meshes:
+ draw.rectangle(g, fill=None, outline=(255, 0, 0))
+
return ImageSource(result, size=dst_size, image_opts=image_opts)
+
def _no_transformation_needed(self, src_size, src_bbox, dst_size, dst_bbox):
"""
>>> src_bbox = (-2504688.5428486541, 1252344.271424327,
@@ -183,6 +174,128 @@ class ImageTransformer(object):
self.src_srs == self.dst_srs and
bbox_equals(src_bbox, dst_bbox, xres/10, yres/10))
+
+def transform_meshes(
+ src_size, src_bbox, src_srs,
+ dst_size, dst_bbox, dst_srs,
+ max_px_err=1,
+ use_center_px=False,
+ ):
+ """
+ transform_meshes creates a list of QUAD transformation parameters for PIL's
+ MESH image transformation.
+
+ Each QUAD is a rectangle in the destination image, like ``(0, 0, 100, 100)`` and
+ a list of four pixel coordinates in the source image that match the destination rectangle.
+ The four points form a quadliteral (i.e. not a rectangle).
+ PIL's image transform uses affine transformation to fill each rectangle in the destination
+ image with data from the source quadliteral.
+
+ The number of QUADs is calculated dynamically to keep the deviation in the image
+ transformation below one pixel. Image transformations for large map scales can be transformed with
+ 1-4 QUADs most of the time. For low scales, transform_meshes can generate a few hundred QUADs.
+
+ It generates a maximum of one QUAD per 50 pixel.
+ """
+ src_bbox = src_srs.align_bbox(src_bbox)
+ dst_bbox = dst_srs.align_bbox(dst_bbox)
+ src_rect = (0, 0, src_size[0], src_size[1])
+ dst_rect = (0, 0, dst_size[0], dst_size[1])
+ to_src_px = make_lin_transf(src_bbox, src_rect)
+ to_src_w = make_lin_transf(src_rect, src_bbox)
+ to_dst_w = make_lin_transf(dst_rect, dst_bbox)
+ meshes = []
+
+ if use_center_px:
+ px_offset = 0.5
+ else:
+ px_offset = 0.0
+
+ def dst_quad_to_src(quad):
+ src_quad = []
+ for dst_px in [(quad[0], quad[1]), (quad[0], quad[3]),
+ (quad[2], quad[3]), (quad[2], quad[1])]:
+ dst_w = to_dst_w(
+ (dst_px[0] + px_offset, dst_px[1] + px_offset))
+ src_w = dst_srs.transform_to(src_srs, dst_w)
+ src_px = to_src_px(src_w)
+ src_quad.extend(src_px)
+
+ return quad, src_quad
+
+ res = (dst_bbox[2] - dst_bbox[0]) / dst_size[0]
+ max_err = max_px_err * res
+
+ def is_good(quad, src_quad):
+ w = quad[2] - quad[0]
+ h = quad[3] - quad[1]
+
+ if w < 50 or h < 50:
+ return True
+
+ xc = quad[0] + w / 2.0 - 0.5
+ yc = quad[1] + h / 2.0 - 0.5
+
+ # coordinate for the center of the quad
+ dst_w = to_dst_w((xc, yc))
+
+ # actual coordinate for the center of the quad
+ src_px = center_quad_transform(quad, src_quad)
+ real_dst_w = src_srs.transform_to(dst_srs, to_src_w(src_px))
+
+ err = max(abs(dst_w[0] - real_dst_w[0]), abs(dst_w[1] - real_dst_w[1]))
+ return err < max_err
+
+
+ # recursively add meshes. divide each quad into four sub quad till
+ # accuracy is good enough.
+ def add_meshes(quads):
+ for quad in quads:
+ quad, src_quad = dst_quad_to_src(quad)
+ if is_good(quad, src_quad):
+ meshes.append((quad, src_quad))
+ else:
+ add_meshes(divide_quad(quad))
+
+ add_meshes([(0, 0, dst_size[0], dst_size[1])])
+ return meshes
+
+
+def center_quad_transform(quad, src_quad):
+ """
+ center_quad_transfrom transforms the center pixel coordinates
+ from ``quad`` to ``src_quad`` by using affine transformation
+ as used by PIL.Image.transform.
+ """
+ w = quad[2] - quad[0]
+ h = quad[3] - quad[1]
+
+ nw = src_quad[0:2]
+ sw = src_quad[2:4]
+ se = src_quad[4:6]
+ ne = src_quad[6:8]
+ x0, y0 = nw
+ As = 1.0 / w
+ At = 1.0 / h
+
+ a0 = x0
+ a1 = (ne[0] - x0) * As
+ a2 = (sw[0] - x0) * At
+ a3 = (se[0] - sw[0] - ne[0] + x0) * As * At
+ a4 = y0
+ a5 = (ne[1] - y0) * As
+ a6 = (sw[1] - y0) * At
+ a7 = (se[1] - sw[1] - ne[1] + y0) * As * At
+
+ x = w / 2.0 - 0.5
+ y = h / 2.0 - 0.5
+
+ return (
+ a0 + a1*x + a2*y + a3*x*y,
+ a4 + a5*x + a6*y + a7*x*y
+ )
+
+
def img_for_resampling(img, resampling):
"""
Convert P images to RGB(A) for non-NEAREST resamplings.
@@ -197,22 +310,41 @@ def img_for_resampling(img, resampling):
img = img.convert('RGBA')
return img
-def griddify(quad, steps):
+
+def divide_quad(quad):
"""
- Divides a box (`quad`) into multiple boxes (``steps x steps``).
+ divide_quad in up to four sub quads. Only divide horizontal if quad is twice as wide then high,
+ and vertical vice versa.
+ PIL.Image.transform expects that the lower-right corner
+ of a quad overlaps by one pixel.
- >>> list(griddify((0, 0, 500, 500), 2))
+ >>> divide_quad((0, 0, 500, 500))
[(0, 0, 250, 250), (250, 0, 500, 250), (0, 250, 250, 500), (250, 250, 500, 500)]
+ >>> divide_quad((0, 0, 2000, 500))
+ [(0, 0, 1000, 500), (1000, 0, 2000, 500)]
+ >>> divide_quad((100, 200, 200, 500))
+ [(100, 200, 200, 350), (100, 350, 200, 500)]
+
"""
- w = quad[2]-quad[0]
- h = quad[3]-quad[1]
- x_step = w / float(steps)
- y_step = h / float(steps)
-
- y = quad[1]
- for _ in range(steps):
- x = quad[0]
- for _ in range(steps):
- yield (int(x), int(y), int(x+x_step), int(y+y_step))
- x += x_step
- y += y_step
+ w = quad[2] - quad[0]
+ h = quad[3] - quad[1]
+ xc = int(quad[0] + w/2)
+ yc = int(quad[1] + h/2)
+
+ if w > 2*h:
+ return [
+ (quad[0], quad[1], xc, quad[3]),
+ (xc, quad[1], quad[2], quad[3]),
+ ]
+ if h > 2*w:
+ return [
+ (quad[0], quad[1], quad[2], yc),
+ (quad[0], yc, quad[2], quad[3]),
+ ]
+
+ return [
+ (quad[0], quad[1], xc, yc),
+ (xc, quad[1], quad[2], yc),
+ (quad[0], yc, xc, quad[3]),
+ (xc, yc, quad[2], quad[3]),
+ ]
diff --git a/mapproxy/layer.py b/mapproxy/layer.py
index e0c7246..42ee718 100644
--- a/mapproxy/layer.py
+++ b/mapproxy/layer.py
@@ -18,7 +18,7 @@
Layers that can get maps/infos from different sources/caches.
"""
-from __future__ import division, with_statement
+from __future__ import division
from mapproxy.grid import NoTiles, GridError, merge_resolution_range, bbox_intersects, bbox_contains
from mapproxy.image import SubImageSource, bbox_position_in_image
from mapproxy.image.opts import ImageOptions
@@ -50,14 +50,6 @@ class MapLayer(object):
def __init__(self, image_opts=None):
self.image_opts = image_opts or ImageOptions()
- def _get_transparent(self):
- return self.image_opts.transparent
-
- def _set_transparent(self, value):
- self.image_opts.transparent = value
-
- transparent = property(_get_transparent, _set_transparent)
-
def _get_opacity(self):
return self.image_opts.opacity
@@ -66,10 +58,17 @@ class MapLayer(object):
opacity = property(_get_opacity, _set_opacity)
- def is_opaque(self):
- if self.opacity is None:
- return not self.transparent
- return self.opacity >= 0.99
+ def is_opaque(self, query):
+ """
+ Whether the query result is opaque.
+
+ This method is used for optimizations: layers below an opaque
+ layer can be skipped. As sources with `transparent: false`
+ still can return transparent images (min_res/max_res/coverages),
+ implementations of this method need to be certain that the image
+ is indeed opaque. is_opaque should return False if in doubt.
+ """
+ return False
def check_res_range(self, query):
if (self.res_range and
@@ -305,8 +304,6 @@ class ResolutionConditional(MapLayer):
self.resolution = resolution
self.srs = srs
- #TODO
- self.transparent = self.one.transparent
self.opacity = opacity
self.extent = extent
@@ -331,9 +328,8 @@ class SRSConditional(MapLayer):
PROJECTED = 'PROJECTED'
GEOGRAPHIC = 'GEOGRAPHIC'
- def __init__(self, layers, extent, transparent=False, opacity=None):
+ def __init__(self, layers, extent, opacity=None):
MapLayer.__init__(self)
- self.transparent = transparent
# TODO geographic/projected fallback
self.srs_map = {}
self.res_range = merge_layer_res_ranges([l[0] for l in layers])
@@ -403,7 +399,6 @@ class CacheMapLayer(MapLayer):
self.grid = tile_manager.grid
self.extent = extent or map_extent_from_grid(self.grid)
self.res_range = merge_layer_res_ranges(self.tile_manager.sources)
- self.transparent = tile_manager.transparent
self.max_tile_limit = max_tile_limit
def get_map(self, query):
diff --git a/mapproxy/multiapp.py b/mapproxy/multiapp.py
index a50763d..7aa3604 100644
--- a/mapproxy/multiapp.py
+++ b/mapproxy/multiapp.py
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
from mapproxy.request import Request
diff --git a/mapproxy/request/wms/__init__.py b/mapproxy/request/wms/__init__.py
index 548be1e..f75f84f 100644
--- a/mapproxy/request/wms/__init__.py
+++ b/mapproxy/request/wms/__init__.py
@@ -16,7 +16,6 @@
"""
Service requests (parsing, handling, etc).
"""
-from __future__ import with_statement
import codecs
from mapproxy.request.wms import exception
from mapproxy.exception import RequestError
diff --git a/mapproxy/request/wmts.py b/mapproxy/request/wmts.py
index 97300cb..ce26ee5 100644
--- a/mapproxy/request/wmts.py
+++ b/mapproxy/request/wmts.py
@@ -152,7 +152,7 @@ class WMTS100TileRequest(WMTSRequest):
self.layer = self.params.layer
self.tilematrixset = self.params.tilematrixset
self.format = self.params.format # TODO
- self.tile = (int(self.params.coord[0]), int(self.params.coord[1]), self.params.coord[2]) # TODO
+ self.tile = (int(self.params.coord[0]), int(self.params.coord[1]), int(self.params.coord[2]))
self.origin = 'nw'
self.dimensions = self.params.dimensions
diff --git a/mapproxy/script/defrag.py b/mapproxy/script/defrag.py
new file mode 100644
index 0000000..ac4dffa
--- /dev/null
+++ b/mapproxy/script/defrag.py
@@ -0,0 +1,184 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2017 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import glob
+import optparse
+import os.path
+import re
+import sys
+from collections import OrderedDict
+
+from mapproxy.cache.compact import CompactCacheV1, CompactCacheV2
+from mapproxy.cache.tile import Tile
+from mapproxy.config import local_base_config
+from mapproxy.config.loader import load_configuration, ConfigurationError
+
+import logging
+log = logging.getLogger('mapproxy.defrag')
+
+def defrag_command(args=None):
+ parser = optparse.OptionParser("%prog defrag-compact [options] -f mapproxy_conf")
+ parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
+ help="MapProxy configuration.")
+
+ parser.add_option("--min-percent", type=float, default=10.0,
+ help="Only defrag if fragmentation is larger (10 means at least 10% of the file does not have to be used)")
+
+ parser.add_option("--min-mb", type=float, default=1.0,
+ help="Only defrag if fragmentation is larger (2 means at least 2MB the file does not have to be used)")
+
+ parser.add_option("--dry-run", "-n", action="store_true",
+ help="Do not de-fragment, only print output")
+
+ parser.add_option("--caches", dest="cache_names", metavar='cache1,cache2,...',
+ help="only defragment the named caches")
+
+
+ from mapproxy.script.util import setup_logging
+ import logging
+ setup_logging(logging.INFO, format="[%(asctime)s] %(msg)s")
+
+ if args:
+ args = args[1:] # remove script name
+
+ (options, args) = parser.parse_args(args)
+ if not options.mapproxy_conf:
+ parser.print_help()
+ sys.exit(1)
+
+ try:
+ proxy_configuration = load_configuration(options.mapproxy_conf)
+ except IOError as e:
+ print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr)
+ sys.exit(2)
+ except ConfigurationError as e:
+ print(e, file=sys.stderr)
+ print('ERROR: invalid configuration (see above)', file=sys.stderr)
+ sys.exit(2)
+
+
+ with local_base_config(proxy_configuration.base_config):
+ available_caches = OrderedDict()
+ for name, cache_conf in proxy_configuration.caches.items():
+ for grid, extent, tile_mgr in cache_conf.caches():
+ if isinstance(tile_mgr.cache, (CompactCacheV1, CompactCacheV2)):
+ available_caches.setdefault(name, []).append(tile_mgr.cache)
+
+ if options.cache_names:
+ defrag_caches = options.cache_names.split(',')
+ missing = set(defrag_caches).difference(available_caches.keys())
+ if missing:
+ print('unknown caches: %s' % (', '.join(missing), ))
+ print('available compact caches: %s' %
+ (', '.join(available_caches.keys()), ))
+ sys.exit(1)
+ else:
+ defrag_caches = None
+
+ for name, caches in available_caches.items():
+ if defrag_caches and name not in defrag_caches:
+ continue
+ for cache in caches:
+ logger = DefragLog(name)
+ defrag_compact_cache(cache,
+ min_percent=options.min_percent/100,
+ min_bytes=options.min_mb*1024*1024,
+ dry_run=options.dry_run,
+ log_progress=logger,
+ )
+
+def bundle_offset(fname):
+ """
+ >>> bundle_offset("path/to/R0000C0000.bundle")
+ (0, 0)
+ >>> bundle_offset("path/to/R0380C1380.bundle")
+ (4992, 896)
+ """
+ match = re.search('R([A-F0-9]{4,})C([A-F0-9]{4,}).bundle$', fname, re.IGNORECASE)
+ if match:
+ r = int(match.group(1), 16)
+ c = int(match.group(2), 16)
+ return c, r
+
+class DefragLog(object):
+ def __init__(self, cache_name):
+ self.cache_name = cache_name
+ def log(self, fname, fragmentation, fragmentation_bytes, num, total, defrag):
+ msg = "%s: %3d/%d (%s) fragmentation is %.1f%% (%dkb)" % (
+ self.cache_name, num, total, fname, fragmentation, fragmentation_bytes/1024
+ )
+ if defrag:
+ msg += " - defragmenting"
+ else:
+ msg += " - skipping"
+ log.info(msg)
+
+def defrag_compact_cache(cache, min_percent=0.1, min_bytes=1024*1024, log_progress=None, dry_run=False):
+ bundles = glob.glob(os.path.join(cache.cache_dir, 'L??', 'R????C????.bundle'))
+
+ for i, bundle_file in enumerate(bundles):
+ offset = bundle_offset(bundle_file)
+ b = cache.bundle_class(bundle_file.rstrip('.bundle'), offset)
+ size, file_size = b.size()
+
+ defrag = 1 - float(size) / file_size
+ defrag_bytes = file_size - size
+
+
+ skip = False
+ if defrag < min_percent or defrag_bytes < min_bytes:
+ skip = True
+
+ if log_progress:
+ log_progress.log(
+ fname=bundle_file,
+ fragmentation=defrag * 100,
+ fragmentation_bytes=defrag_bytes,
+ num=i+1, total=len(bundles),
+ defrag=not skip,
+ )
+
+ if skip or dry_run:
+ continue
+
+ tmp_bundle = os.path.join(cache.cache_dir, 'tmp_defrag')
+ defb = cache.bundle_class(tmp_bundle, offset)
+ stored_tiles = False
+
+ for y in range(128):
+ tiles = [Tile((x, y, 0)) for x in range(128)]
+ b.load_tiles(tiles)
+ tiles = [t for t in tiles if t.source]
+ if tiles:
+ stored_tiles = True
+ defb.store_tiles(tiles)
+
+ # remove first
+ # - in case bundle is empty
+ # - windows does not support rename to existing files
+ if os.path.exists(bundle_file):
+ os.remove(bundle_file)
+ if os.path.exists(bundle_file[:-1] + 'x'):
+ os.remove(bundle_file[:-1] + 'x')
+
+ if stored_tiles:
+ os.rename(tmp_bundle + '.bundle', bundle_file)
+ if os.path.exists(tmp_bundle + '.bundlx'):
+ os.rename(tmp_bundle + '.bundlx', bundle_file[:-1] + 'x')
+ if os.path.exists(tmp_bundle + '.lck'):
+ os.unlink(tmp_bundle + '.lck')
+
diff --git a/mapproxy/script/export.py b/mapproxy/script/export.py
index 4a35207..2db6aee 100644
--- a/mapproxy/script/export.py
+++ b/mapproxy/script/export.py
@@ -224,6 +224,12 @@ def export_command(args=None):
'version': 1,
'directory': options.dest,
}
+ elif options.type == 'compact-v2':
+ cache_conf['cache'] = {
+ 'type': 'compact',
+ 'version': 2,
+ 'directory': options.dest,
+ }
elif options.type in ('tc', 'mapproxy'):
cache_conf['cache'] = {
'type': 'file',
@@ -273,7 +279,7 @@ def export_command(args=None):
print('WARN: grids are incompatible. needs to scale/reproject tiles for export.', file=sys.stderr)
md = dict(name='export', cache_name='cache', grid_name=options.grid, dest=options.dest)
- task = SeedTask(md, mgr, levels, None, seed_coverage)
+ task = SeedTask(md, mgr, levels, 1, seed_coverage)
print(format_export_task(task, custom_grid=custom_grid))
diff --git a/mapproxy/script/util.py b/mapproxy/script/util.py
index 5449aaf..ee75611 100755
--- a/mapproxy/script/util.py
+++ b/mapproxy/script/util.py
@@ -25,23 +25,24 @@ import textwrap
import logging
from mapproxy.compat import iteritems
-from mapproxy.version import version
+from mapproxy.script.conf.app import config_command
+from mapproxy.script.defrag import defrag_command
+from mapproxy.script.export import export_command
+from mapproxy.script.grids import grids_command
from mapproxy.script.scales import scales_command
from mapproxy.script.wms_capabilities import wms_capabilities_command
-from mapproxy.script.grids import grids_command
-from mapproxy.script.export import export_command
-from mapproxy.script.conf.app import config_command
-
+from mapproxy.version import version
-def setup_logging(level=logging.INFO):
+def setup_logging(level=logging.INFO, format=None):
mapproxy_log = logging.getLogger('mapproxy')
mapproxy_log.setLevel(level)
ch = logging.StreamHandler(sys.stdout)
- ch.setLevel(logging.DEBUG)
- formatter = logging.Formatter(
- "[%(asctime)s] %(name)s - %(levelname)s - %(message)s")
+ ch.setLevel(level)
+ if not format:
+ format = "[%(asctime)s] %(name)s - %(levelname)s - %(message)s"
+ formatter = logging.Formatter(format)
ch.setFormatter(formatter)
mapproxy_log.addHandler(ch)
@@ -312,6 +313,10 @@ commands = {
'autoconfig': {
'func': config_command,
'help': 'Create config from WMS capabilities.'
+ },
+ 'defrag-compact-cache': {
+ 'func': defrag_command,
+ 'help': 'De-fragmentate compact caches.'
}
}
diff --git a/mapproxy/script/wms_capabilities.py b/mapproxy/script/wms_capabilities.py
index 1066030..8f968d7 100644
--- a/mapproxy/script/wms_capabilities.py
+++ b/mapproxy/script/wms_capabilities.py
@@ -18,6 +18,8 @@ from __future__ import print_function
import sys
import optparse
+from xml import etree
+
from mapproxy.compat import iteritems, BytesIO
from mapproxy.compat.modules import urlparse
from mapproxy.client.http import open_url, HTTPClientError
@@ -102,9 +104,7 @@ def parse_capabilities(fileobj, version='1.1.1'):
except ValueError as ex:
log_error('%s\n%s\n%s\n%s\nNot a capabilities document: %s', 'Recieved document:', '-'*80, fileobj.getvalue(), '-'*80, ex.args[0])
sys.exit(1)
- except Exception as ex:
- # catch all, etree.ParseError only avail since Python 2.7
- # 2.5 and 2.6 raises exc from underlying implementation like expat
+ except etree.ElementTree.ParseError as ex:
log_error('%s\n%s\n%s\n%s\nCould not parse the document: %s', 'Recieved document:', '-'*80, fileobj.getvalue(), '-'*80, ex.args[0])
sys.exit(1)
diff --git a/mapproxy/seed/cachelock.py b/mapproxy/seed/cachelock.py
index 212a749..c1bbc84 100644
--- a/mapproxy/seed/cachelock.py
+++ b/mapproxy/seed/cachelock.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import errno
import os
diff --git a/mapproxy/seed/cleanup.py b/mapproxy/seed/cleanup.py
index 81c7eea..bfe0ab7 100644
--- a/mapproxy/seed/cleanup.py
+++ b/mapproxy/seed/cleanup.py
@@ -99,9 +99,7 @@ def normpath(path):
if path.startswith('\\'):
return path
- # only supported with >= Python 2.6
- if hasattr(os.path, 'relpath'):
- path = os.path.relpath(path)
+ path = os.path.relpath(path)
if path.startswith('../../'):
path = os.path.abspath(path)
diff --git a/mapproxy/seed/script.py b/mapproxy/seed/script.py
index 64579be..3e84444 100644
--- a/mapproxy/seed/script.py
+++ b/mapproxy/seed/script.py
@@ -34,6 +34,7 @@ from mapproxy.seed.cleanup import cleanup
from mapproxy.seed.util import (format_seed_task, format_cleanup_task,
ProgressLog, ProgressStore)
from mapproxy.seed.cachelock import CacheLocker
+from mapproxy.compat import raw_input
SECONDS_PER_DAY = 60 * 60 * 24
SECONDS_PER_MINUTE = 60
diff --git a/mapproxy/seed/seeder.py b/mapproxy/seed/seeder.py
index 320fc6e..713e36e 100644
--- a/mapproxy/seed/seeder.py
+++ b/mapproxy/seed/seeder.py
@@ -375,6 +375,8 @@ class TileWalker(object):
# check if subtile was already processed. see comment in __init__
if subtile in self.seeded_tiles[current_level]:
+ if not levels:
+ self.seed_progress.step_forward(total_subtiles)
continue
self.seeded_tiles[current_level].appendleft(subtile)
diff --git a/mapproxy/service/demo.py b/mapproxy/service/demo.py
index 2c266bf..fdfb86e 100644
--- a/mapproxy/service/demo.py
+++ b/mapproxy/service/demo.py
@@ -190,7 +190,12 @@ class DemoServer(Server):
def _render_tms_template(self, template, req):
template = get_template(template, default_inherit="demo/static.html")
- tile_layer = self.tile_layers['_'.join([req.args['tms_layer'], req.args['srs'].replace(':','')])]
+ for layer in self.tile_layers.values():
+ if (layer.name == req.args['tms_layer'] and
+ layer.grid.srs.srs_code == req.args['srs']):
+ tile_layer = layer
+ break
+
resolutions = tile_layer.grid.tile_sets
res = []
for level, resolution in resolutions:
@@ -215,7 +220,11 @@ class DemoServer(Server):
def _render_wmts_template(self, template, req):
template = get_template(template, default_inherit="demo/static.html")
- wmts_layer = self.tile_layers['_'.join([req.args['wmts_layer'], req.args['srs'].replace(':','')])]
+ for layer in self.tile_layers.values():
+ if (layer.name == req.args['wmts_layer'] and
+ layer.grid.srs.srs_code == req.args['srs']):
+ wmts_layer = layer
+ break
restful_url = self.restful_template.replace('{Layer}', wmts_layer.name, 1)
if '{Format}' in restful_url:
diff --git a/mapproxy/service/template_helper.py b/mapproxy/service/template_helper.py
index 371b46e..ae07a27 100644
--- a/mapproxy/service/template_helper.py
+++ b/mapproxy/service/template_helper.py
@@ -17,7 +17,7 @@ from cgi import escape
from mapproxy.template import bunch
__all__ = ['escape', 'indent', 'bunch', 'wms100format', 'wms100info_format',
- 'wms111metadatatype', 'limit_llbbox']
+ 'wms111metadatatype']
def indent(text, n=2):
return '\n'.join(' '*n + line for line in text.split('\n'))
@@ -51,23 +51,3 @@ def wms111metadatatype(type):
return 'TC211'
if type == 'FGDC:1998':
return 'FGDC'
-
-def limit_llbbox(bbox):
- """
- Limit the long/lat bounding box to +-180/89.99999999 degrees.
-
- Some clients can't handle +-90 north/south, so we subtract a tiny bit.
-
- >>> ', '.join('%.6f' % x for x in limit_llbbox((-200,-90.0, 180, 90)))
- '-180.000000, -89.999999, 180.000000, 89.999999'
- >>> ', '.join('%.6f' % x for x in limit_llbbox((-20,-9.0, 10, 10)))
- '-20.000000, -9.000000, 10.000000, 10.000000'
- """
- minx, miny, maxx, maxy = bbox
-
- minx = max(-180, minx)
- miny = max(-89.999999, miny)
- maxx = min(180, maxx)
- maxy = min(89.999999, maxy)
-
- return minx, miny, maxx, maxy
\ No newline at end of file
diff --git a/mapproxy/service/templates/wms100capabilities.xml b/mapproxy/service/templates/wms100capabilities.xml
index a0ec8f8..d7b9fb9 100644
--- a/mapproxy/service/templates/wms100capabilities.xml
+++ b/mapproxy/service/templates/wms100capabilities.xml
@@ -73,7 +73,7 @@
{{endif}}
<Title>{{ layer.title }}</Title>
<SRS>{{for s in srs}}{{s}} {{endfor}}</SRS>
- {{py: extent = limit_llbbox(layer.extent.llbbox)}}
+ {{py: extent = layer_llbbox(layer)}}
<LatLonBoundingBox minx="{{ extent[0] }}" miny="{{ extent[1] }}" maxx="{{ extent[2] }}" maxy="{{ extent[3] }}" />
{{for srs_code, bbox in layer_srs_bbox(layer)}}
<BoundingBox SRS="{{srs_code}}" minx="{{ bbox[0] }}" miny="{{ bbox[1] }}" maxx="{{ bbox[2] }}" maxy="{{ bbox[3] }}" />
diff --git a/mapproxy/service/templates/wms110capabilities.xml b/mapproxy/service/templates/wms110capabilities.xml
index 53dce73..4620739 100644
--- a/mapproxy/service/templates/wms110capabilities.xml
+++ b/mapproxy/service/templates/wms110capabilities.xml
@@ -110,7 +110,7 @@
{{if with_srs}}
<SRS>{{for s in srs}}{{s}} {{endfor}}</SRS>
{{endif}}
- {{py: extent = limit_llbbox(layer.extent.llbbox)}}
+ {{py: extent = layer_llbbox(layer)}}
<LatLonBoundingBox minx="{{ extent[0] }}" miny="{{ extent[1] }}" maxx="{{ extent[2] }}" maxy="{{ extent[3] }}" />
{{for srs_code, bbox in layer_srs_bbox(layer)}}
<BoundingBox SRS="{{srs_code}}" minx="{{ bbox[0] }}" miny="{{ bbox[1] }}" maxx="{{ bbox[2] }}" maxy="{{ bbox[3] }}" />
diff --git a/mapproxy/service/templates/wms111capabilities.xml b/mapproxy/service/templates/wms111capabilities.xml
index 4baaa73..24a0268 100644
--- a/mapproxy/service/templates/wms111capabilities.xml
+++ b/mapproxy/service/templates/wms111capabilities.xml
@@ -127,7 +127,7 @@
<SRS>{{s}}</SRS>
{{endfor}}
{{endif}}
- {{py: extent = limit_llbbox(layer.extent.llbbox)}}
+ {{py: extent = layer_llbbox(layer)}}
<LatLonBoundingBox minx="{{ extent[0] }}" miny="{{ extent[1] }}" maxx="{{ extent[2] }}" maxy="{{ extent[3] }}" />
{{for srs_code, bbox in layer_srs_bbox(layer)}}
<BoundingBox SRS="{{srs_code}}" minx="{{ bbox[0] }}" miny="{{ bbox[1] }}" maxx="{{ bbox[2] }}" maxy="{{ bbox[3] }}" />
diff --git a/mapproxy/service/templates/wms130capabilities.xml b/mapproxy/service/templates/wms130capabilities.xml
index b73bb19..fa5a28e 100644
--- a/mapproxy/service/templates/wms130capabilities.xml
+++ b/mapproxy/service/templates/wms130capabilities.xml
@@ -222,7 +222,7 @@
<CRS>{{s}}</CRS>
{{endfor}}
{{endif}}
- {{py: extent = limit_llbbox(layer.extent.llbbox)}}
+ {{py: extent = layer_llbbox(layer)}}
<EX_GeographicBoundingBox>
<westBoundLongitude>{{ extent[0] }}</westBoundLongitude>
<eastBoundLongitude>{{ extent[2] }}</eastBoundLongitude>
diff --git a/mapproxy/service/templates/wmts100capabilities.xml b/mapproxy/service/templates/wmts100capabilities.xml
index a3a7acc..80f4f8d 100644
--- a/mapproxy/service/templates/wmts100capabilities.xml
+++ b/mapproxy/service/templates/wmts100capabilities.xml
@@ -3,6 +3,16 @@
<ows:ServiceIdentification>
<ows:Title>{{service.title}}</ows:Title>
<ows:Abstract>{{service.abstract}}</ows:Abstract>
+ {{if service.keyword_list and len(service.keyword_list) > 0}}
+ <ows:KeywordList>
+ {{for list in service.keyword_list}}
+ {{py: kw=bunch(default='', **list)}}
+ {{for keyword in kw.keywords}}
+ <ows:Keyword{{if kw.vocabulary}} vocabulary="{{kw.vocabulary}}"{{endif}}>{{keyword}}</ows:Keyword>
+ {{endfor}}
+ {{endfor}}
+ </ows:KeywordList>
+ {{endif}}
<ows:ServiceType>OGC WMTS</ows:ServiceType>
<ows:ServiceTypeVersion>1.0.0</ows:ServiceTypeVersion>
<ows:Fees>{{service.get('fees', 'none')}}</ows:Fees>
diff --git a/mapproxy/service/tile.py b/mapproxy/service/tile.py
index 8885d96..01071cc 100644
--- a/mapproxy/service/tile.py
+++ b/mapproxy/service/tile.py
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import division, with_statement
+from __future__ import division
import math
import time
diff --git a/mapproxy/service/wms.py b/mapproxy/service/wms.py
index bc9c2df..e58befd 100644
--- a/mapproxy/service/wms.py
+++ b/mapproxy/service/wms.py
@@ -108,7 +108,7 @@ class WMSServer(Server):
if layer.renders_query(query):
# if layer is not transparent and will be rendered,
# remove already added (then hidden) layers
- if not layer.transparent:
+ if layer.is_opaque(query):
actual_layers = odict()
for layer_name, map_layers in layer.map_layers_for_query(query):
actual_layers[layer_name] = map_layers
@@ -480,23 +480,37 @@ class Capabilities(object):
self.inspire_md = inspire_md
def layer_srs_bbox(self, layer, epsg_axis_order=False):
- layer_srs_code = layer.extent.srs.srs_code
for srs, extent in iteritems(self.srs_extents):
+ # is_default is True when no explicit bbox is defined for this srs
+ # use layer extent
if extent.is_default:
bbox = layer.extent.bbox_for(SRS(srs))
- else:
+ elif layer.extent.is_default:
bbox = extent.bbox_for(SRS(srs))
+ else:
+ # Use intersection of srs_extent and layer.extent.
+ bbox = extent.intersection(layer.extent).bbox_for(SRS(srs))
if epsg_axis_order:
bbox = switch_bbox_epsg_axis_order(bbox, srs)
- yield srs, bbox
+
+ if srs in self.srs:
+ yield srs, bbox
# add native srs
+ layer_srs_code = layer.extent.srs.srs_code
if layer_srs_code not in self.srs_extents:
bbox = layer.extent.bbox
if epsg_axis_order:
bbox = switch_bbox_epsg_axis_order(bbox, layer_srs_code)
- yield layer_srs_code, bbox
+ if layer_srs_code in self.srs:
+ yield layer_srs_code, bbox
+
+ def layer_llbbox(self, layer):
+ if 'EPSG:4326' in self.srs_extents:
+ llbbox = self.srs_extents['EPSG:4326'].intersection(layer.extent).llbbox
+ return limit_llbbox(llbbox)
+ return limit_llbbox(layer.extent.llbbox)
def render(self, _map_request):
return self._render_template(_map_request.capabilities_template)
@@ -513,12 +527,34 @@ class Capabilities(object):
srs=self.srs,
tile_layers=self.tile_layers,
layer_srs_bbox=self.layer_srs_bbox,
+ layer_llbbox=self.layer_llbbox,
inspire_md=inspire_md,
)
# strip blank lines
doc = '\n'.join(l for l in doc.split('\n') if l.rstrip())
return doc
+
+def limit_llbbox(bbox):
+ """
+ Limit the long/lat bounding box to +-180/89.99999999 degrees.
+
+ Some clients can't handle +-90 north/south, so we subtract a tiny bit.
+
+ >>> ', '.join('%.6f' % x for x in limit_llbbox((-200,-90.0, 180, 90)))
+ '-180.000000, -89.999999, 180.000000, 89.999999'
+ >>> ', '.join('%.6f' % x for x in limit_llbbox((-20,-9.0, 10, 10)))
+ '-20.000000, -9.000000, 10.000000, 10.000000'
+ """
+ minx, miny, maxx, maxy = bbox
+
+ minx = max(-180, minx)
+ miny = max(-89.999999, miny)
+ maxx = min(180, maxx)
+ maxy = min(89.999999, maxy)
+
+ return minx, miny, maxx, maxy
+
class LayerRenderer(object):
def __init__(self, layers, query, request, raise_source_errors=True,
concurrent_rendering=1):
@@ -621,8 +657,6 @@ class WMSLayerBase(object):
"True if .info() is supported"
queryable = False
- transparent = False
-
"True is .legend() is supported"
has_legend = False
legend_url = None
@@ -633,9 +667,6 @@ class WMSLayerBase(object):
"MapExtend of the layer"
extent = None
- def is_opaque(self):
- return not self.transparent
-
def map_layers_for_query(self, query):
raise NotImplementedError()
@@ -666,9 +697,11 @@ class WMSLayer(WMSLayerBase):
res_range = merge_layer_res_ranges(map_layers)
self.res_range = res_range
self.queryable = True if info_layers else False
- self.transparent = all(not map_lyr.is_opaque() for map_lyr in self.map_layers)
self.has_legend = True if legend_layers else False
+ def is_opaque(self, query):
+ return any(l.is_opaque(query) for l in self.map_layers)
+
def renders_query(self, query):
if self.res_range and not self.res_range.contains(query.bbox, query.size, query.srs):
return False
@@ -727,13 +760,15 @@ class WMSGroupLayer(WMSLayerBase):
self.md = md or {}
self.is_active = True if this is not None else False
self.layers = layers
- self.transparent = True if this and not this.is_opaque() or all(not l.is_opaque() for l in layers) else False
self.has_legend = True if this and this.has_legend or any(l.has_legend for l in layers) else False
self.queryable = True if this and this.queryable or any(l.queryable for l in layers) else False
all_layers = layers + ([self.this] if self.this else [])
self.extent = merge_layer_extents(all_layers)
self.res_range = merge_layer_res_ranges(all_layers)
+ def is_opaque(self, query):
+ return any(l.is_opaque(query) for l in self.layers)
+
@property
def legend_size(self):
return self.this.legend_size
diff --git a/mapproxy/source/__init__.py b/mapproxy/source/__init__.py
index 1934384..9f50d57 100644
--- a/mapproxy/source/__init__.py
+++ b/mapproxy/source/__init__.py
@@ -45,8 +45,8 @@ class DebugSource(MapLayer):
def __init__(self):
MapLayer.__init__(self)
self.extent = DefaultMapExtent()
- self.transparent = True
self.res_range = None
+
def get_map(self, query):
bbox = query.bbox
w = bbox[2] - bbox[0]
@@ -67,8 +67,9 @@ class DummySource(MapLayer):
"""
def __init__(self, coverage=None):
MapLayer.__init__(self)
+ self.image_opts.transparent = True
self.extent = MapExtent((-180, -90, 180, 90), SRS(4326))
- self.transparent = True
self.extent = MapExtent(coverage.bbox, coverage.srs) if coverage else DefaultMapExtent()
+
def get_map(self, query):
raise BlankImage()
diff --git a/mapproxy/source/mapnik.py b/mapproxy/source/mapnik.py
index de04565..1f140e5 100644
--- a/mapproxy/source/mapnik.py
+++ b/mapproxy/source/mapnik.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, absolute_import
+from __future__ import absolute_import
import sys
import time
@@ -148,4 +148,4 @@ class MapnikSource(MapLayer):
status='200' if data else '500', size=size, method='API', duration=time.time()-start_time)
return ImageSource(BytesIO(data), size=query.size,
- image_opts=ImageOptions(transparent=self.transparent, format=query.format))
+ image_opts=ImageOptions(format=query.format))
diff --git a/mapproxy/source/wms.py b/mapproxy/source/wms.py
index 00f3445..4bce02b 100644
--- a/mapproxy/source/wms.py
+++ b/mapproxy/source/wms.py
@@ -44,7 +44,7 @@ class WMSSource(MapLayer):
self.transparent_color = transparent_color
self.transparent_color_tolerance = transparent_color_tolerance
if self.transparent_color:
- self.transparent = True
+ self.image_opts.transparent = True
self.coverage = coverage
self.res_range = res_range
if self.coverage:
@@ -52,6 +52,30 @@ class WMSSource(MapLayer):
else:
self.extent = DefaultMapExtent()
+ def is_opaque(self, query):
+ """
+ Returns true if we are sure that the image is not transparent.
+ """
+ if self.res_range and not self.res_range.contains(query.bbox, query.size,
+ query.srs):
+ return False
+
+ if self.image_opts.transparent:
+ return False
+
+ if self.opacity is not None and (0.0 < self.opacity < 0.99):
+ return False
+
+ if not self.coverage:
+ # not transparent and no coverage
+ return True
+
+ if self.coverage.contains(query.bbox, query.srs):
+ # not transparent and completely inside coverage
+ return True
+
+ return False
+
def get_map(self, query):
if self.res_range and not self.res_range.contains(query.bbox, query.size,
query.srs):
diff --git a/mapproxy/srs.py b/mapproxy/srs.py
index 996335f..6a870ce 100644
--- a/mapproxy/srs.py
+++ b/mapproxy/srs.py
@@ -345,15 +345,13 @@ def calculate_bbox(points):
"""
points = list(points)
# points can be INF for invalid transformations, filter out
- # INF is not portable for <2.6 so we check against a large value
- MAX = 1e300
try:
- minx = min(p[0] for p in points if p[0] <= MAX)
- miny = min(p[1] for p in points if p[1] <= MAX)
- maxx = max(p[0] for p in points if p[0] <= MAX)
- maxy = max(p[1] for p in points if p[1] <= MAX)
+ minx = min(p[0] for p in points if p[0] != float('inf'))
+ miny = min(p[1] for p in points if p[1] != float('inf'))
+ maxx = max(p[0] for p in points if p[0] != float('inf'))
+ maxy = max(p[1] for p in points if p[1] != float('inf'))
return (minx, miny, maxx, maxy)
- except ValueError: # everything is INF
+ except ValueError: # min/max are called with empty list when everything is inf
raise TransformationError()
def merge_bbox(bbox1, bbox2):
diff --git a/mapproxy/test/image.py b/mapproxy/test/image.py
index 1fbffb3..f1853d2 100644
--- a/mapproxy/test/image.py
+++ b/mapproxy/test/image.py
@@ -172,6 +172,8 @@ def tmp_image(size, format='png', color=None, mode='RGB'):
img = Image.new(mode, size, color=color)
else:
img = create_debug_img(size)
+ if format == 'jpeg':
+ img = img.convert('RGB')
data = BytesIO()
img.save(data, format)
data.seek(0)
diff --git a/mapproxy/test/system/__init__.py b/mapproxy/test/system/__init__.py
index c3bbd71..8563c75 100644
--- a/mapproxy/test/system/__init__.py
+++ b/mapproxy/test/system/__init__.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import os
import tempfile
import shutil
diff --git a/mapproxy/test/system/fixture/layer.yaml b/mapproxy/test/system/fixture/layer.yaml
index dc84b6f..c6af73b 100644
--- a/mapproxy/test/system/fixture/layer.yaml
+++ b/mapproxy/test/system/fixture/layer.yaml
@@ -20,11 +20,11 @@ services:
wmts:
wms:
image_formats: ['image/png', 'image/jpeg', 'png8']
- srs: ['EPSG:4326', 'EPSG:4258', 'CRS:84', 'EPSG:900913', 'EPSG:3857', 'EPSG:31466', 'EPSG:31467', 'EPSG:31468', 'EPSG:25831', 'EPSG:25832', 'EPSG:25833']
+ srs: ['EPSG:4326', 'CRS:84', 'EPSG:900913', 'EPSG:3857', 'EPSG:31466', 'EPSG:31467', 'EPSG:31468', 'EPSG:25831', 'EPSG:25832', 'EPSG:25833']
bbox_srs:
- - bbox: [2750000, 5000000, 4250000, 6500000]
- srs: 'EPSG:31467'
- 'EPSG:3857'
+ - bbox: [-180, -70, 180, 90]
+ srs: 'EPSG:4326'
md:
title: MapProxy test fixture ☃
abstract: This is MapProxy.
@@ -174,6 +174,11 @@ sources:
req:
url: http://localhost:42423/service
layers: bar
+ coverage:
+ # coverage in projection not in wms.srs,
+ # should not be advertised in capabilities #288
+ bbox: [-180, -80, 170, 80]
+ srs: 'EPSG:4258'
wms_cache:
type: wms
supported_srs: ['EPSG:900913', 'EPSG:4326']
diff --git a/mapproxy/test/system/fixture/wms_srs_extent.yaml b/mapproxy/test/system/fixture/wms_srs_extent.yaml
index 2f6c92a..358475a 100644
--- a/mapproxy/test/system/fixture/wms_srs_extent.yaml
+++ b/mapproxy/test/system/fixture/wms_srs_extent.yaml
@@ -1,10 +1,16 @@
services:
wms:
image_formats: ['image/png', 'image/jpeg']
- srs: ['EPSG:4326', 'EPSG:4258', 'CRS:84', 'EPSG:900913', 'EPSG:3857', 'EPSG:31466', 'EPSG:31467', 'EPSG:31468', 'EPSG:25831', 'EPSG:25832', 'EPSG:25833']
+ srs: ['EPSG:4326', 'EPSG:4258', 'CRS:84', 'EPSG:900913', 'EPSG:3857', 'EPSG:31467', 'EPSG:31468', 'EPSG:25831', 'EPSG:25832', 'EPSG:25833']
bbox_srs:
- bbox: [0.0, 3500000.0, 1000000.0, 8500000.0]
srs: 'EPSG:25832'
+ - bbox: [2750000, 5000000, 4250000, 6500000]
+ srs: 'EPSG:31467'
+ - bbox: [2750000, 5000000, 4250000, 6500000]
+ srs: 'EPSG:31466'
+ - 'EPSG:3857'
+
md:
title: MapProxy test fixture ☃
@@ -12,6 +18,9 @@ layers:
- name: direct
title: Direct Layer
sources: [direct]
+ - name: direct_coverage
+ title: Direct Layer with Coverage
+ sources: [direct_coverage]
sources:
direct:
@@ -19,3 +28,12 @@ sources:
req:
url: http://localhost:42423/service
layers: bar
+
+ direct_coverage:
+ type: wms
+ req:
+ url: http://localhost:42423/service
+ layers: bar
+ coverage:
+ bbox: [5, 50, 10, 55]
+ srs: 'EPSG:4326'
diff --git a/mapproxy/test/system/test_arcgis.py b/mapproxy/test/system/test_arcgis.py
index 8e01c3a..e0356d9 100644
--- a/mapproxy/test/system/test_arcgis.py
+++ b/mapproxy/test/system/test_arcgis.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
from io import BytesIO
from mapproxy.request.wms import WMS111FeatureInfoRequest
diff --git a/mapproxy/test/system/test_auth.py b/mapproxy/test/system/test_auth.py
index f2db9e0..028468f 100644
--- a/mapproxy/test/system/test_auth.py
+++ b/mapproxy/test/system/test_auth.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
from mapproxy.test.system import module_setup, module_teardown, SystemTest
from mapproxy.test.image import img_from_buf, create_tmp_image, is_transparent
diff --git a/mapproxy/test/system/test_behind_proxy.py b/mapproxy/test/system/test_behind_proxy.py
index 2e49d9d..baf5459 100644
--- a/mapproxy/test/system/test_behind_proxy.py
+++ b/mapproxy/test/system/test_behind_proxy.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
from mapproxy.test.system import module_setup, module_teardown, SystemTest, make_base_config
diff --git a/mapproxy/test/system/test_cache_band_merge.py b/mapproxy/test/system/test_cache_band_merge.py
index cca22b6..1167d73 100644
--- a/mapproxy/test/system/test_cache_band_merge.py
+++ b/mapproxy/test/system/test_cache_band_merge.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
from mapproxy.request.wms import WMS111MapRequest
from mapproxy.request.wmts import WMTS100CapabilitiesRequest
from mapproxy.test.image import img_from_buf
diff --git a/mapproxy/test/system/test_cache_geopackage.py b/mapproxy/test/system/test_cache_geopackage.py
index c6aa47b..f190fe4 100644
--- a/mapproxy/test/system/test_cache_geopackage.py
+++ b/mapproxy/test/system/test_cache_geopackage.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import os
import shutil
diff --git a/mapproxy/test/system/test_cache_grid_names.py b/mapproxy/test/system/test_cache_grid_names.py
index c9f9ee6..f1428de 100644
--- a/mapproxy/test/system/test_cache_grid_names.py
+++ b/mapproxy/test/system/test_cache_grid_names.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
from mapproxy.test.image import tmp_image
from mapproxy.test.http import mock_httpd
diff --git a/mapproxy/test/system/test_cache_mbtiles.py b/mapproxy/test/system/test_cache_mbtiles.py
index 006447c..13bbe09 100644
--- a/mapproxy/test/system/test_cache_mbtiles.py
+++ b/mapproxy/test/system/test_cache_mbtiles.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import os
import shutil
diff --git a/mapproxy/test/system/test_cache_s3.py b/mapproxy/test/system/test_cache_s3.py
index d84ac6e..c133f89 100644
--- a/mapproxy/test/system/test_cache_s3.py
+++ b/mapproxy/test/system/test_cache_s3.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
from io import BytesIO
diff --git a/mapproxy/test/system/test_cache_source.py b/mapproxy/test/system/test_cache_source.py
index c2b0680..ca3c879 100644
--- a/mapproxy/test/system/test_cache_source.py
+++ b/mapproxy/test/system/test_cache_source.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
from mapproxy.request.wms import WMS111MapRequest
from mapproxy.test.image import tmp_image
diff --git a/mapproxy/test/system/test_combined_sources.py b/mapproxy/test/system/test_combined_sources.py
index b0c1db9..28acdea 100644
--- a/mapproxy/test/system/test_combined_sources.py
+++ b/mapproxy/test/system/test_combined_sources.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
from io import BytesIO
from mapproxy.request.wms import WMS111MapRequest
diff --git a/mapproxy/test/system/test_coverage.py b/mapproxy/test/system/test_coverage.py
index 0b2187d..3f6bbb4 100644
--- a/mapproxy/test/system/test_coverage.py
+++ b/mapproxy/test/system/test_coverage.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
from io import BytesIO
from mapproxy.request.wms import WMS111MapRequest
diff --git a/mapproxy/test/system/test_disable_storage.py b/mapproxy/test/system/test_disable_storage.py
index c00be1d..c7c1c94 100644
--- a/mapproxy/test/system/test_disable_storage.py
+++ b/mapproxy/test/system/test_disable_storage.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
from mapproxy.test.image import is_png, tmp_image
diff --git a/mapproxy/test/system/test_formats.py b/mapproxy/test/system/test_formats.py
index 1f1ea72..5949bcd 100644
--- a/mapproxy/test/system/test_formats.py
+++ b/mapproxy/test/system/test_formats.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import os
from io import BytesIO
from mapproxy.request.wms import WMS111MapRequest, WMS111FeatureInfoRequest
diff --git a/mapproxy/test/system/test_kml.py b/mapproxy/test/system/test_kml.py
index fd161bc..751b91d 100644
--- a/mapproxy/test/system/test_kml.py
+++ b/mapproxy/test/system/test_kml.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
import hashlib
from io import BytesIO
diff --git a/mapproxy/test/system/test_layergroups.py b/mapproxy/test/system/test_layergroups.py
index b32f2b4..57f35fd 100644
--- a/mapproxy/test/system/test_layergroups.py
+++ b/mapproxy/test/system/test_layergroups.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
from mapproxy.test.system import module_setup, module_teardown, SystemTest
from mapproxy.test.system.test_wms import is_111_capa, is_110_capa, is_100_capa, is_130_capa, ns130
diff --git a/mapproxy/test/system/test_legendgraphic.py b/mapproxy/test/system/test_legendgraphic.py
index 8791f9d..5e6016e 100644
--- a/mapproxy/test/system/test_legendgraphic.py
+++ b/mapproxy/test/system/test_legendgraphic.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
from io import BytesIO
from mapproxy.compat.image import Image
diff --git a/mapproxy/test/system/test_mapnik.py b/mapproxy/test/system/test_mapnik.py
index 27fefb6..0431d3d 100644
--- a/mapproxy/test/system/test_mapnik.py
+++ b/mapproxy/test/system/test_mapnik.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import os
from mapproxy.test.system import module_setup, module_teardown, SystemTest
diff --git a/mapproxy/test/system/test_mapserver.py b/mapproxy/test/system/test_mapserver.py
index 4b17ef3..55aaace 100644
--- a/mapproxy/test/system/test_mapserver.py
+++ b/mapproxy/test/system/test_mapserver.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import os
import stat
diff --git a/mapproxy/test/system/test_mixed_mode_format.py b/mapproxy/test/system/test_mixed_mode_format.py
index 18b7f0c..136e6a4 100644
--- a/mapproxy/test/system/test_mixed_mode_format.py
+++ b/mapproxy/test/system/test_mixed_mode_format.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import os
from io import BytesIO
from mapproxy.compat.image import (
diff --git a/mapproxy/test/system/test_multi_cache_layers.py b/mapproxy/test/system/test_multi_cache_layers.py
index 0c70038..df9487a 100644
--- a/mapproxy/test/system/test_multi_cache_layers.py
+++ b/mapproxy/test/system/test_multi_cache_layers.py
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import functools
diff --git a/mapproxy/test/system/test_multiapp.py b/mapproxy/test/system/test_multiapp.py
index c59807f..f2a71c0 100644
--- a/mapproxy/test/system/test_multiapp.py
+++ b/mapproxy/test/system/test_multiapp.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import io
import os
import tempfile
diff --git a/mapproxy/test/system/test_renderd_client.py b/mapproxy/test/system/test_renderd_client.py
index de76b12..d52c853 100644
--- a/mapproxy/test/system/test_renderd_client.py
+++ b/mapproxy/test/system/test_renderd_client.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
try:
diff --git a/mapproxy/test/system/test_scalehints.py b/mapproxy/test/system/test_scalehints.py
index b8d8c5c..6bcebe8 100644
--- a/mapproxy/test/system/test_scalehints.py
+++ b/mapproxy/test/system/test_scalehints.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import math
diff --git a/mapproxy/test/system/test_seed.py b/mapproxy/test/system/test_seed.py
index d5f6abf..b0bbb0f 100644
--- a/mapproxy/test/system/test_seed.py
+++ b/mapproxy/test/system/test_seed.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
import time
import shutil
diff --git a/mapproxy/test/system/test_seed_only.py b/mapproxy/test/system/test_seed_only.py
index 8dbee8c..d75ea73 100644
--- a/mapproxy/test/system/test_seed_only.py
+++ b/mapproxy/test/system/test_seed_only.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
from io import BytesIO
from mapproxy.request.wms import WMS111MapRequest
diff --git a/mapproxy/test/system/test_sld.py b/mapproxy/test/system/test_sld.py
index 7620c71..7439dfd 100644
--- a/mapproxy/test/system/test_sld.py
+++ b/mapproxy/test/system/test_sld.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import os
import tempfile
diff --git a/mapproxy/test/system/test_source_errors.py b/mapproxy/test/system/test_source_errors.py
index b14d392..2dacdbe 100644
--- a/mapproxy/test/system/test_source_errors.py
+++ b/mapproxy/test/system/test_source_errors.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import os
diff --git a/mapproxy/test/system/test_tilesource_minmax_res.py b/mapproxy/test/system/test_tilesource_minmax_res.py
index 9990780..762878f 100644
--- a/mapproxy/test/system/test_tilesource_minmax_res.py
+++ b/mapproxy/test/system/test_tilesource_minmax_res.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
from mapproxy.test.image import tmp_image
from mapproxy.test.http import mock_httpd
from mapproxy.test.system import module_setup, module_teardown, SystemTest, make_base_config
diff --git a/mapproxy/test/system/test_tms.py b/mapproxy/test/system/test_tms.py
index 31f21b2..2212225 100644
--- a/mapproxy/test/system/test_tms.py
+++ b/mapproxy/test/system/test_tms.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
import hashlib
from io import BytesIO
diff --git a/mapproxy/test/system/test_tms_origin.py b/mapproxy/test/system/test_tms_origin.py
index b2712b5..eac17fc 100644
--- a/mapproxy/test/system/test_tms_origin.py
+++ b/mapproxy/test/system/test_tms_origin.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
from mapproxy.test.image import is_jpeg
from mapproxy.test.system import module_setup, module_teardown, SystemTest, make_base_config
from nose.tools import eq_
diff --git a/mapproxy/test/system/test_util_conf.py b/mapproxy/test/system/test_util_conf.py
index bff485b..0233217 100644
--- a/mapproxy/test/system/test_util_conf.py
+++ b/mapproxy/test/system/test_util_conf.py
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
import shutil
diff --git a/mapproxy/test/system/test_util_export.py b/mapproxy/test/system/test_util_export.py
index fc83313..a4a2ff8 100644
--- a/mapproxy/test/system/test_util_export.py
+++ b/mapproxy/test/system/test_util_export.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
import tempfile
import shutil
diff --git a/mapproxy/test/system/test_util_grids.py b/mapproxy/test/system/test_util_grids.py
index fff5000..9365c2c 100644
--- a/mapproxy/test/system/test_util_grids.py
+++ b/mapproxy/test/system/test_util_grids.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
from nose.tools import assert_raises
diff --git a/mapproxy/test/system/test_util_wms_capabilities.py b/mapproxy/test/system/test_util_wms_capabilities.py
index 676cb04..45f12e9 100644
--- a/mapproxy/test/system/test_util_wms_capabilities.py
+++ b/mapproxy/test/system/test_util_wms_capabilities.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
from nose.tools import assert_raises
diff --git a/mapproxy/test/system/test_watermark.py b/mapproxy/test/system/test_watermark.py
index cf4abdf..4771d04 100644
--- a/mapproxy/test/system/test_watermark.py
+++ b/mapproxy/test/system/test_watermark.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
from io import BytesIO
diff --git a/mapproxy/test/system/test_wms.py b/mapproxy/test/system/test_wms.py
index d61fd24..435fc03 100644
--- a/mapproxy/test/system/test_wms.py
+++ b/mapproxy/test/system/test_wms.py
@@ -160,14 +160,16 @@ class TestWMS111(WMSTest):
bboxs = xml.xpath('//Layer/Layer[1]/BoundingBox')
bboxs = dict((e.attrib['SRS'], e) for e in bboxs)
assert_almost_equal_bbox(
- bbox_srs_from_boundingbox(bboxs['EPSG:31467']),
- [2750000.0, 5000000.0, 4250000.0, 6500000.0])
- assert_almost_equal_bbox(
bbox_srs_from_boundingbox(bboxs['EPSG:3857']),
[-20037508.3428, -15538711.0963, 18924313.4349, 15538711.0963])
assert_almost_equal_bbox(
bbox_srs_from_boundingbox(bboxs['EPSG:4326']),
- [-180.0, -80.0, 170.0, 80.0])
+ [-180.0, -70.0, 170.0, 80.0])
+
+ bbox_srs = xml.xpath('//Layer/Layer/BoundingBox')
+ bbox_srs = set(e.attrib['SRS'] for e in bbox_srs)
+ # we have a coverage in EPSG:4258, but it is not in wms.srs (#288)
+ assert 'EPSG:4258' not in bbox_srs
assert validate_with_dtd(xml, dtd_name='wms/1.1.1/WMS_MS_Capabilities.dtd')
@@ -647,7 +649,7 @@ class TestWMS110(WMSTest):
llbox = xml.xpath('//Capability/Layer/LatLonBoundingBox')[0]
# some clients don't like 90deg north/south
- assert_almost_equal(float(llbox.attrib['miny']), -89.999999, 6)
+ assert_almost_equal(float(llbox.attrib['miny']), -70.0, 6)
assert_almost_equal(float(llbox.attrib['maxy']), 89.999999, 6)
assert_almost_equal(float(llbox.attrib['minx']), -180.0, 6)
assert_almost_equal(float(llbox.attrib['maxx']), 180.0, 6)
diff --git a/mapproxy/test/system/test_wms_srs_extent.py b/mapproxy/test/system/test_wms_srs_extent.py
index 1ccd118..6cd0f7c 100644
--- a/mapproxy/test/system/test_wms_srs_extent.py
+++ b/mapproxy/test/system/test_wms_srs_extent.py
@@ -13,12 +13,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
+from mapproxy.request.wms import WMS111MapRequest, WMS111CapabilitiesRequest
from mapproxy.test.system import module_setup, module_teardown, SystemTest, make_base_config
from mapproxy.test.image import is_png, is_transparent
from mapproxy.test.image import tmp_image, assert_colors_equal, img_from_buf
from mapproxy.test.http import mock_httpd
+
+from mapproxy.test.system.test_wms import bbox_srs_from_boundingbox
+from mapproxy.test.unit.test_grid import assert_almost_equal_bbox
from nose.tools import eq_
test_config = {}
@@ -33,6 +37,54 @@ def teardown_module():
class TestWMSSRSExtentTest(SystemTest):
config = test_config
+ def setup(self):
+ SystemTest.setup(self)
+ self.common_req = WMS111MapRequest(url='/service?', param=dict(service='WMS',
+ version='1.1.1'))
+
+ def test_wms_capabilities(self):
+ req = WMS111CapabilitiesRequest(url='/service?').copy_with_request_params(self.common_req)
+ resp = self.app.get(req)
+ eq_(resp.content_type, 'application/vnd.ogc.wms_xml')
+ xml = resp.lxml
+
+ bboxs = xml.xpath('//Layer/Layer[1]/BoundingBox')
+ bboxs = dict((e.attrib['SRS'], e) for e in bboxs)
+
+ assert_almost_equal_bbox(
+ bbox_srs_from_boundingbox(bboxs['EPSG:31467']),
+ [2750000.0, 5000000.0, 4250000.0, 6500000.0])
+ assert_almost_equal_bbox(
+ bbox_srs_from_boundingbox(bboxs['EPSG:25832']),
+ [0.0, 3500000.0, 1000000.0, 8500000.0])
+
+ assert_almost_equal_bbox(
+ bbox_srs_from_boundingbox(bboxs['EPSG:3857']),
+ [-20037508.3428, -147730762.670, 20037508.3428, 147730758.195])
+ assert_almost_equal_bbox(
+ bbox_srs_from_boundingbox(bboxs['EPSG:4326']),
+ [-180.0, -90.0, 180.0, 90.0])
+
+
+ # bboxes clipped to coverage
+ bboxs = xml.xpath('//Layer/Layer[2]/BoundingBox')
+ bboxs = dict((e.attrib['SRS'], e) for e in bboxs)
+ assert_almost_equal_bbox(
+ bbox_srs_from_boundingbox(bboxs['EPSG:31467']),
+ [3213331.57335, 5540436.91132, 3571769.72263, 6104110.432])
+ assert_almost_equal_bbox(
+ bbox_srs_from_boundingbox(bboxs['EPSG:25832']),
+ [213372.048961, 5538660.64621, 571666.447504, 6102110.74547])
+
+ assert_almost_equal_bbox(
+ bbox_srs_from_boundingbox(bboxs['EPSG:3857']),
+ [556597.453966, 6446275.84102, 1113194.90793, 7361866.11305])
+ assert_almost_equal_bbox(
+ bbox_srs_from_boundingbox(bboxs['EPSG:4326']),
+ [5.0, 50.0, 10.0, 55.0])
+
+
+
def test_out_of_extent(self):
resp = self.app.get('http://localhost/service?SERVICE=WMS&REQUEST=GetMap'
'&LAYERS=direct&STYLES='
@@ -95,4 +147,4 @@ class TestWMSSRSExtentTest(SystemTest):
eq_(resp.content_type, 'image/png')
assert is_png(resp.body)
assert_colors_equal(img_from_buf(resp.body).convert('RGBA'),
- [(50 * 100, [255, 0, 0, 255]), (50 * 100, [0, 255, 0, 255])])
\ No newline at end of file
+ [(50 * 100, [255, 0, 0, 255]), (50 * 100, [0, 255, 0, 255])])
diff --git a/mapproxy/test/system/test_wms_version.py b/mapproxy/test/system/test_wms_version.py
index 72bca92..e682407 100644
--- a/mapproxy/test/system/test_wms_version.py
+++ b/mapproxy/test/system/test_wms_version.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
from mapproxy.test.system import module_setup, module_teardown, SystemTest, make_base_config
from mapproxy.test.system.test_wms import is_110_capa, is_111_capa
diff --git a/mapproxy/test/system/test_wmsc.py b/mapproxy/test/system/test_wmsc.py
index cd2ef11..c21a8d3 100644
--- a/mapproxy/test/system/test_wmsc.py
+++ b/mapproxy/test/system/test_wmsc.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
from io import BytesIO
from mapproxy.request.wms import (
diff --git a/mapproxy/test/system/test_wmts.py b/mapproxy/test/system/test_wmts.py
index 257c3ec..d2e2e9b 100644
--- a/mapproxy/test/system/test_wmts.py
+++ b/mapproxy/test/system/test_wmts.py
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import re
import os
@@ -97,6 +97,13 @@ class TestWMTS(SystemTest):
data = BytesIO(resp.body)
assert is_jpeg(data)
+ # test with integer tilematrix
+ url = str(self.common_tile_req).replace('=01', '=1')
+ resp = self.app.get(url)
+ eq_(resp.content_type, 'image/jpeg')
+ data = BytesIO(resp.body)
+ assert is_jpeg(data)
+
def test_get_tile_flipped_axis(self):
# test default tile lock directory
tiles_lock_dir = os.path.join(test_config['base_dir'], 'cache_data', 'tile_locks')
diff --git a/mapproxy/test/system/test_wmts_dimensions.py b/mapproxy/test/system/test_wmts_dimensions.py
index 84df4ed..d1d4490 100644
--- a/mapproxy/test/system/test_wmts_dimensions.py
+++ b/mapproxy/test/system/test_wmts_dimensions.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import functools
diff --git a/mapproxy/test/system/test_wmts_restful.py b/mapproxy/test/system/test_wmts_restful.py
index 7def075..af027d1 100644
--- a/mapproxy/test/system/test_wmts_restful.py
+++ b/mapproxy/test/system/test_wmts_restful.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import functools
@@ -76,6 +76,9 @@ class TestWMTS(SystemTest):
with serv:
resp = self.app.get('/wmts/myrest/tms_cache_ul/ulgrid/01/0/0.png', status=200)
eq_(resp.content_type, 'image/png')
+ # test without leading 0 in level
+ resp = self.app.get('/wmts/myrest/tms_cache_ul/ulgrid/1/0/0.png', status=200)
+ eq_(resp.content_type, 'image/png')
def test_get_tile_source_error(self):
resp = self.app.get('/wmts/myrest/tms_cache/GLOBAL_MERCATOR/01/0/0.png', status=500)
diff --git a/mapproxy/test/system/test_xslt_featureinfo.py b/mapproxy/test/system/test_xslt_featureinfo.py
index 0cd01c3..12e5bea 100644
--- a/mapproxy/test/system/test_xslt_featureinfo.py
+++ b/mapproxy/test/system/test_xslt_featureinfo.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import os
from mapproxy.request.wms import WMS111FeatureInfoRequest, WMS130FeatureInfoRequest
diff --git a/mapproxy/test/unit/test_cache.py b/mapproxy/test/unit/test_cache.py
index b1cac63..eda8b01 100644
--- a/mapproxy/test/unit/test_cache.py
+++ b/mapproxy/test/unit/test_cache.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
import re
import time
@@ -869,7 +868,6 @@ class MockLayer(object):
class TestResolutionConditionalLayers(object):
def setup(self):
self.low = MockLayer()
- self.low.transparent = False #TODO
self.high = MockLayer()
self.layer = ResolutionConditional(self.low, self.high, 10, SRS(900913),
GLOBAL_GEOGRAPHIC_EXTENT)
diff --git a/mapproxy/test/unit/test_cache_compact.py b/mapproxy/test/unit/test_cache_compact.py
index 405f09a..be4ded6 100644
--- a/mapproxy/test/unit/test_cache_compact.py
+++ b/mapproxy/test/unit/test_cache_compact.py
@@ -13,18 +13,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import os
import time
import struct
+import shutil
+import tempfile
from io import BytesIO
-from mapproxy.cache.compact import CompactCacheV1
+from mapproxy.cache.compact import CompactCacheV1, CompactCacheV2
from mapproxy.cache.tile import Tile
from mapproxy.image import ImageSource
from mapproxy.image.opts import ImageOptions
+from mapproxy.script.defrag import defrag_compact_cache
from mapproxy.test.unit.test_cache_tile import TileCacheTestBase
from nose.tools import eq_
@@ -125,3 +128,193 @@ class TestCompactCacheV1(TileCacheTestBase):
self.cache.store_tile(t)
assert_header([4000 + 4, 6000 + 4 + 3000 + 4, 1000 + 4], 6000) # still contains bytes from overwritten tile
+
+
+class TestCompactCacheV2(TileCacheTestBase):
+
+ always_loads_metadata = True
+
+ def setup(self):
+ TileCacheTestBase.setup(self)
+ self.cache = CompactCacheV2(
+ cache_dir=self.cache_dir,
+ )
+
+ def test_bundle_files(self):
+ assert not os.path.exists(os.path.join(self.cache_dir, 'L00', 'R0000C0000.bundle'))
+ self.cache.store_tile(self.create_tile(coord=(0, 0, 0)))
+ assert os.path.exists(os.path.join(self.cache_dir, 'L00', 'R0000C0000.bundle'))
+
+ assert not os.path.exists(os.path.join(self.cache_dir, 'L12', 'R0000C0000.bundle'))
+ self.cache.store_tile(self.create_tile(coord=(127, 127, 12)))
+ assert os.path.exists(os.path.join(self.cache_dir, 'L12', 'R0000C0000.bundle'))
+
+ assert not os.path.exists(os.path.join(self.cache_dir, 'L12', 'R0100C0080.bundle'))
+ self.cache.store_tile(self.create_tile(coord=(128, 256, 12)))
+ assert os.path.exists(os.path.join(self.cache_dir, 'L12', 'R0100C0080.bundle'))
+
+ def test_bundle_files_not_created_on_is_cached(self):
+ assert not os.path.exists(os.path.join(self.cache_dir, 'L00', 'R0000C0000.bundle'))
+ self.cache.is_cached(Tile(coord=(0, 0, 0)))
+ assert not os.path.exists(os.path.join(self.cache_dir, 'L00', 'R0000C0000.bundle'))
+
+ def test_missing_tiles(self):
+ self.cache.store_tile(self.create_tile(coord=(130, 200, 8)))
+ assert os.path.exists(os.path.join(self.cache_dir, 'L08', 'R0080C0080.bundle'))
+
+ # test that all other tiles in this bundle are missing
+ assert self.cache.is_cached(Tile((130, 200, 8)))
+ for x in range(128, 255):
+ for y in range(128, 255):
+ if x == 130 and y == 200:
+ continue
+ assert not self.cache.is_cached(Tile((x, y, 8))), (x, y)
+ assert not self.cache.load_tile(Tile((x, y, 8))), (x, y)
+
+ def test_remove_level_tiles_before(self):
+ self.cache.store_tile(self.create_tile(coord=(0, 0, 12)))
+ assert os.path.exists(os.path.join(self.cache_dir, 'L12', 'R0000C0000.bundle'))
+
+ # not removed with timestamp
+ self.cache.remove_level_tiles_before(12, time.time())
+ assert os.path.exists(os.path.join(self.cache_dir, 'L12', 'R0000C0000.bundle'))
+
+ # removed with timestamp=0 (remove_all:true in seed.yaml)
+ self.cache.remove_level_tiles_before(12, 0)
+ assert not os.path.exists(os.path.join(self.cache_dir, 'L12'))
+
+
+ def test_bundle_header(self):
+ t = Tile((5000, 1000, 12), ImageSource(BytesIO(b'a' * 4000), image_opts=ImageOptions(format='image/png')))
+ self.cache.store_tile(t)
+ assert os.path.exists(os.path.join(self.cache_dir, 'L12', 'R0380C1380.bundle'))
+
+ def assert_header(tile_bytes_written, max_tile_bytes):
+ with open(os.path.join(self.cache_dir, 'L12', 'R0380C1380.bundle'), 'r+b') as f:
+ header = struct.unpack('<4I3Q6I', f.read(64))
+ eq_(header[0], 3) # version
+ eq_(header[1], 128*128)
+ eq_(header[2], max_tile_bytes)
+ eq_(header[5], 64 + 128*128*8 + sum(tile_bytes_written))
+
+ assert_header([4000 + 4], 4000)
+
+ t = Tile((5000, 1001, 12), ImageSource(BytesIO(b'a' * 6000), image_opts=ImageOptions(format='image/png')))
+ self.cache.store_tile(t)
+ assert_header([4000 + 4, 6000 + 4], 6000)
+
+ t = Tile((4992, 999, 12), ImageSource(BytesIO(b'a' * 1000), image_opts=ImageOptions(format='image/png')))
+ self.cache.store_tile(t)
+ assert_header([4000 + 4, 6000 + 4, 1000 + 4], 6000)
+
+ t = Tile((5000, 1001, 12), ImageSource(BytesIO(b'a' * 3000), image_opts=ImageOptions(format='image/png')))
+ self.cache.store_tile(t)
+ assert_header([4000 + 4, 6000 + 4 + 3000 + 4, 1000 + 4], 6000) # still contains bytes from overwritten tile
+
+
+class mockProgressLog(object):
+ def __init__(self):
+ self.logs = []
+
+ def log(self, fname, fragmentation, fragmentation_bytes, num, total, defrag):
+ self.logs.append({
+ 'fname': fname,
+ 'fragmentation': fragmentation,
+ 'fragmentation_bytes': fragmentation_bytes,
+ 'num': num,
+ 'total': total,
+ 'defrag': defrag,
+ })
+ self.logs.sort(key=lambda x: (x['fname'], 'num'))
+
+class DefragmentationTestBase(object):
+ def setup(self):
+ self.cache_dir = tempfile.mkdtemp()
+
+ def teardown(self):
+ if os.path.exists(self.cache_dir):
+ shutil.rmtree(self.cache_dir)
+
+ def test_defragmentation_empty_bundle(self):
+ cache = self.cache_class(self.cache_dir)
+
+ t = Tile((5000, 1000, 12),
+ ImageSource(BytesIO(b'a' * 60*1024), image_opts=ImageOptions(format='image/png')))
+ cache.store_tile(t)
+ cache.remove_tile(t)
+
+ fname = os.path.join(self.cache_dir, 'L12', 'R0380C1380.bundle')
+ assert os.path.exists(fname)
+
+ logger = mockProgressLog()
+ defrag_compact_cache(cache, min_bytes=50000, log_progress=logger)
+
+ assert not os.path.exists(fname)
+
+ def test_defragmentation_min_bytes(self):
+ cache = self.cache_class(self.cache_dir)
+
+ for _ in range(2):
+ t = Tile((5000, 1000, 12),
+ ImageSource(BytesIO(b'a' * 60*1024), image_opts=ImageOptions(format='image/png')))
+ cache.store_tile(t)
+
+ logger = mockProgressLog()
+ fname = os.path.join(self.cache_dir, 'L12', 'R0380C1380.bundle')
+ before = os.path.getsize(fname)
+ defrag_compact_cache(cache, log_progress=logger)
+ assert len(logger.logs) == 1
+ eq_(logger.logs[0]['defrag'], False)
+ after = os.path.getsize(fname)
+ assert before == after
+
+ logger = mockProgressLog()
+ before = os.path.getsize(fname)
+ defrag_compact_cache(cache, min_bytes=50000, log_progress=logger)
+ assert len(logger.logs) == 1
+ eq_(logger.logs[0]['defrag'], True)
+ after = os.path.getsize(fname)
+ assert after < before
+
+ def test_defragmentation_min_percent(self):
+ cache = self.cache_class(self.cache_dir)
+
+ t = Tile((10000, 2000, 13),
+ ImageSource(
+ BytesIO(b'a' * 120 * 1024),
+ image_opts=ImageOptions(format='image/png')))
+ cache.store_tile(t)
+
+ for x in range(100):
+ for _ in range(2 if x < 10 else 1):
+ t = Tile((5000+x, 1000, 12),
+ ImageSource(
+ BytesIO(b'a' * 120 * 1024),
+ image_opts=ImageOptions(format='image/png')))
+ cache.store_tile(t)
+
+ logger = mockProgressLog()
+ fname = os.path.join(self.cache_dir, 'L12', 'R0380C1380.bundle')
+ before = os.path.getsize(fname)
+ defrag_compact_cache(cache, log_progress=logger)
+ assert len(logger.logs) == 2
+ eq_(logger.logs[0]['defrag'], False)
+ eq_(logger.logs[1]['defrag'], False)
+ after = os.path.getsize(fname)
+ assert before == after
+
+ logger = mockProgressLog()
+ before = os.path.getsize(fname)
+ defrag_compact_cache(cache, min_percent=0.08, log_progress=logger)
+ assert len(logger.logs) == 2
+ eq_(logger.logs[0]['defrag'], True)
+ eq_(logger.logs[1]['defrag'], False)
+ after = os.path.getsize(fname)
+ assert after < before
+
+
+class TestDefragmentationV1(DefragmentationTestBase):
+ cache_class = CompactCacheV1
+
+class TestDefragmentationV2(DefragmentationTestBase):
+ cache_class = CompactCacheV2
diff --git a/mapproxy/test/unit/test_cache_couchdb.py b/mapproxy/test/unit/test_cache_couchdb.py
index 7776b7a..fce1582 100644
--- a/mapproxy/test/unit/test_cache_couchdb.py
+++ b/mapproxy/test/unit/test_cache_couchdb.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import re
import os
diff --git a/mapproxy/test/unit/test_cache_geopackage.py b/mapproxy/test/unit/test_cache_geopackage.py
index baf8321..c87b6a8 100644
--- a/mapproxy/test/unit/test_cache_geopackage.py
+++ b/mapproxy/test/unit/test_cache_geopackage.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import os
import time
diff --git a/mapproxy/test/unit/test_cache_redis.py b/mapproxy/test/unit/test_cache_redis.py
index 4aa3645..211f6c5 100644
--- a/mapproxy/test/unit/test_cache_redis.py
+++ b/mapproxy/test/unit/test_cache_redis.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
try:
import redis
diff --git a/mapproxy/test/unit/test_cache_riak.py b/mapproxy/test/unit/test_cache_riak.py
index eff1b32..769656c 100644
--- a/mapproxy/test/unit/test_cache_riak.py
+++ b/mapproxy/test/unit/test_cache_riak.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
import random
diff --git a/mapproxy/test/unit/test_cache_tile.py b/mapproxy/test/unit/test_cache_tile.py
index 891ccaa..8dcea6c 100644
--- a/mapproxy/test/unit/test_cache_tile.py
+++ b/mapproxy/test/unit/test_cache_tile.py
@@ -13,8 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
+import calendar
import datetime
import os
import shutil
@@ -146,7 +146,7 @@ class TileCacheTestBase(object):
if tile.timestamp:
now = time.time()
if self.uses_utc:
- now = time.mktime(datetime.datetime.utcnow().timetuple())
+ now = calendar.timegm(datetime.datetime.utcnow().timetuple())
assert abs(tile.timestamp - now) <= 10
if tile.size:
assert tile.size == size
diff --git a/mapproxy/test/unit/test_client.py b/mapproxy/test/unit/test_client.py
index f8662e7..60502e8 100644
--- a/mapproxy/test/unit/test_client.py
+++ b/mapproxy/test/unit/test_client.py
@@ -1,5 +1,5 @@
# This file is part of the MapProxy project.
-# Copyright (C) 2010 Omniscale <http://omniscale.de>
+# Copyright (C) 2010-2017 Omniscale <http://omniscale.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,13 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
import time
import sys
-from mapproxy.client.http import HTTPClient, HTTPClientError
+from mapproxy.client.http import HTTPClient, HTTPClientError, supports_ssl_default_context
from mapproxy.client.tile import TMSClient, TileClient, TileURLTemplate
from mapproxy.client.wms import WMSClient, WMSInfoClient
from mapproxy.grid import tile_grid
@@ -86,38 +85,25 @@ class TestHTTPClient(object):
assert False, 'expected HTTPClientError'
@attr('online')
- def test_https_no_ssl_module_error(self):
- from mapproxy.client import http
- old_ssl = http.ssl
+ def test_https_untrusted_root(self):
+ if not supports_ssl_default_context:
+ # old python versions require ssl_ca_certs
+ raise SkipTest()
+ self.client = HTTPClient('https://untrusted-root.badssl.com/')
try:
- http.ssl = None
- try:
- self.client = HTTPClient('https://www.google.com/')
- except ImportError:
- pass
- else:
- assert False, 'no ImportError for missing ssl module'
- finally:
- http.ssl = old_ssl
+ self.client.open('https://untrusted-root.badssl.com/')
+ except HTTPClientError as e:
+ assert_re(e.args[0], r'Could not verify connection to URL')
@attr('online')
- def test_https_no_ssl_module_insecure(self):
- from mapproxy.client import http
- old_ssl = http.ssl
- try:
- http.ssl = None
- self.client = HTTPClient('https://www.google.com/', insecure=True)
- self.client.open('https://www.google.com/')
- finally:
- http.ssl = old_ssl
+ def test_https_insecure(self):
+ self.client = HTTPClient(
+ 'https://untrusted-root.badssl.com/', insecure=True)
+ self.client.open('https://untrusted-root.badssl.com/')
@attr('online')
- def test_https_valid_cert(self):
- try:
- import ssl; ssl
- except ImportError:
- raise SkipTest()
-
+ def test_https_valid_ca_cert_file(self):
+ # verify with fixed ca_certs file
cert_file = '/etc/ssl/certs/ca-certificates.crt'
if os.path.exists(cert_file):
self.client = HTTPClient('https://www.google.com/', ssl_ca_certs=cert_file)
@@ -130,16 +116,23 @@ class TestHTTPClient(object):
self.client.open('https://www.google.com/')
@attr('online')
- def test_https_invalid_cert(self):
- try:
- import ssl; ssl
- except ImportError:
+ def test_https_valid_default_cert(self):
+ # modern python should verify by default
+ if not supports_ssl_default_context:
raise SkipTest()
+ self.client = HTTPClient('https://www.google.com/')
+ self.client.open('https://www.google.com/')
+ @attr('online')
+ def test_https_invalid_cert(self):
+ # load 'wrong' root cert
with TempFile() as tmp:
- self.client = HTTPClient('https://www.google.com/', ssl_ca_certs=tmp)
+ with open(tmp, 'wb') as f:
+ f.write(GOOGLE_ROOT_CERT)
+ self.client = HTTPClient(
+ 'https://untrusted-root.badssl.com/', ssl_ca_certs=tmp)
try:
- self.client.open('https://www.google.com/')
+ self.client.open('https://untrusted-root.badssl.com/')
except HTTPClientError as e:
assert_re(e.args[0], r'Could not verify connection to URL')
@@ -149,15 +142,12 @@ class TestHTTPClient(object):
import mapproxy.client.http
- old_timeout = mapproxy.client.http._max_set_timeout
- mapproxy.client.http._max_set_timeout = None
-
client1 = HTTPClient(timeout=0.1)
client2 = HTTPClient(timeout=0.5)
with mock_httpd(TESTSERVER_ADDRESS, [test_req]):
try:
start = time.time()
- client1.open(TESTSERVER_URL+'/')
+ client1.open(TESTSERVER_URL + '/')
except HTTPClientError as ex:
assert 'timed out' in ex.args[0]
else:
@@ -167,7 +157,7 @@ class TestHTTPClient(object):
with mock_httpd(TESTSERVER_ADDRESS, [test_req]):
try:
start = time.time()
- client2.open(TESTSERVER_URL+'/')
+ client2.open(TESTSERVER_URL + '/')
except HTTPClientError as ex:
assert 'timed out' in ex.args[0]
else:
@@ -178,12 +168,53 @@ class TestHTTPClient(object):
assert 0.1 <= duration1 < 0.5, duration1
assert 0.5 <= duration2 < 0.9, duration2
- mapproxy.client.http._max_set_timeout = old_timeout
-# Equifax Secure Certificate Authority
-# Expires: 2018-08-22
+# root certificates for google.com, if no ca-certificates.cert
+# file is found
GOOGLE_ROOT_CERT = b"""
-----BEGIN CERTIFICATE-----
+MIIDVDCCAjygAwIBAgIDAjRWMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVT
+MRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9i
+YWwgQ0EwHhcNMDIwNTIxMDQwMDAwWhcNMjIwNTIxMDQwMDAwWjBCMQswCQYDVQQG
+EwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UEAxMSR2VvVHJ1c3Qg
+R2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2swYYzD9
+9BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9mOSm9BXiLnTjoBbdq
+fnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIuT8rxh0PBFpVXLVDv
+iS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6cJmTM386DGXHKTubU
+1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmRCw7+OC7RHQWa9k0+
+bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5aszPeE4uwc2hGKceeoW
+MPRfwCvocWvk+QIDAQABo1MwUTAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTA
+ephojYn7qwVkDBF9qn1luMrMTjAfBgNVHSMEGDAWgBTAephojYn7qwVkDBF9qn1l
+uMrMTjANBgkqhkiG9w0BAQUFAAOCAQEANeMpauUvXVSOKVCUn5kaFOSPeCpilKIn
+Z57QzxpeR+nBsqTP3UEaBU6bS+5Kb1VSsyShNwrrZHYqLizz/Tt1kL/6cdjHPTfS
+tQWVYrmm3ok9Nns4d0iXrKYgjy6myQzCsplFAMfOEVEiIuCl6rYVSAlk6l5PdPcF
+PseKUgzbFbS9bZvlxrFUaKnjaZC2mqUPuLk/IH2uSrW4nOQdtqvmlKXBx4Ot2/Un
+hw4EbNX/3aBd7YdStysVAq45pmp06drE57xNNB6pXE0zX5IJL4hmXXeXxx12E6nV
+5fEWCRE11azbJHFwLJhWC9kXtNHjUStedejV0NxPNO3CBWaAocvmMw==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDujCCAqKgAwIBAgILBAAAAAABD4Ym5g0wDQYJKoZIhvcNAQEFBQAwTDEgMB4G
+A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjIxEzARBgNVBAoTCkdsb2JhbFNp
+Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDYxMjE1MDgwMDAwWhcNMjExMjE1
+MDgwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMjETMBEG
+A1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI
+hvcNAQEBBQADggEPADCCAQoCggEBAKbPJA6+Lm8omUVCxKs+IVSbC9N/hHD6ErPL
+v4dfxn+G07IwXNb9rfF73OX4YJYJkhD10FPe+3t+c4isUoh7SqbKSaZeqKeMWhG8
+eoLrvozps6yWJQeXSpkqBy+0Hne/ig+1AnwblrjFuTosvNYSuetZfeLQBoZfXklq
+tTleiDTsvHgMCJiEbKjNS7SgfQx5TfC4LcshytVsW33hoCmEofnTlEnLJGKRILzd
+C9XZzPnqJworc5HGnRusyMvo4KD0L5CLTfuwNhv2GXqF4G3yYROIXJ/gkwpRl4pa
+zq+r1feqCapgvdzZX99yqWATXgAByUr6P6TqBwMhAo6CygPCm48CAwEAAaOBnDCB
+mTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUm+IH
+V2ccHsBqBt5ZtJot39wZhi4wNgYDVR0fBC8wLTAroCmgJ4YlaHR0cDovL2NybC5n
+bG9iYWxzaWduLm5ldC9yb290LXIyLmNybDAfBgNVHSMEGDAWgBSb4gdXZxwewGoG
+3lm0mi3f3BmGLjANBgkqhkiG9w0BAQUFAAOCAQEAmYFThxxol4aR7OBKuEQLq4Gs
+J0/WwbgcQ3izDJr86iw8bmEbTUsp9Z8FHSbBuOmDAGJFtqkIk7mpM0sYmsL4h4hO
+291xNBrBVNpGP+DTKqttVCL1OmLNIG+6KYnX3ZHu01yiPqFbQfXf5WRDLenVOavS
+ot+3i9DAgBkcRcAtjOj4LaR0VknFBbVPFd5uRHg5h6h+u/N5GJG79G+dwfCMNYxd
+AfvDbbnvRG15RjF+Cv6pgsH/76tuIMRQyV+dTZsXjAzlAcmgQWpzU/qlULRuJQ/7
+TBj0/VLZjmmx6BEP3ojY+x1J96relc8geMJgEtslQIxq/H5COEBkEveegeGTLg==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
MIIDIDCCAomgAwIBAgIENd70zzANBgkqhkiG9w0BAQUFADBOMQswCQYDVQQGEwJV
UzEQMA4GA1UEChMHRXF1aWZheDEtMCsGA1UECxMkRXF1aWZheCBTZWN1cmUgQ2Vy
dGlmaWNhdGUgQXV0aG9yaXR5MB4XDTk4MDgyMjE2NDE1MVoXDTE4MDgyMjE2NDE1
diff --git a/mapproxy/test/unit/test_client_cgi.py b/mapproxy/test/unit/test_client_cgi.py
index 878fe2c..cac1aca 100644
--- a/mapproxy/test/unit/test_client_cgi.py
+++ b/mapproxy/test/unit/test_client_cgi.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
import shutil
import stat
diff --git a/mapproxy/test/unit/test_concat_legends.py b/mapproxy/test/unit/test_concat_legends.py
index 150d8b1..0588866 100644
--- a/mapproxy/test/unit/test_concat_legends.py
+++ b/mapproxy/test/unit/test_concat_legends.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
from mapproxy.compat.image import Image
from mapproxy.image import ImageSource
from mapproxy.image.merge import concat_legends
diff --git a/mapproxy/test/unit/test_conf_loader.py b/mapproxy/test/unit/test_conf_loader.py
index b0fdc50..1c78b27 100644
--- a/mapproxy/test/unit/test_conf_loader.py
+++ b/mapproxy/test/unit/test_conf_loader.py
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import division, with_statement
+from __future__ import division
import yaml
import time
from mapproxy.srs import SRS
@@ -948,13 +948,13 @@ class TestLoadCoverage(object):
def test_load_empty_geojson(self):
with TempFile() as tf:
with open(tf, 'wb') as f:
- f.write('{"type": "FeatureCollection", "features": []}')
+ f.write(b'{"type": "FeatureCollection", "features": []}')
conf = {'datasource': tf, 'srs': 'EPSG:4326'}
assert_raises(EmptyGeometryError, load_coverage, conf)
def test_load_empty_geojson_ogr(self):
with TempFile() as tf:
with open(tf, 'wb') as f:
- f.write('{"type": "FeatureCollection", "features": []}')
+ f.write(b'{"type": "FeatureCollection", "features": []}')
conf = {'datasource': tf, 'where': '0 != 1', 'srs': 'EPSG:4326'}
assert_raises(EmptyGeometryError, load_coverage, conf)
diff --git a/mapproxy/test/unit/test_conf_validator.py b/mapproxy/test/unit/test_conf_validator.py
index c914060..27fa668 100644
--- a/mapproxy/test/unit/test_conf_validator.py
+++ b/mapproxy/test/unit/test_conf_validator.py
@@ -219,6 +219,19 @@ class TestValidator(object):
errors = validate_references(conf)
eq_(errors, [])
+ def test_tagged_source_with_colons(self):
+ conf = self._test_conf('''
+ caches:
+ one_cache:
+ grids: [GLOBAL_MERCATOR]
+ sources: ['one_source:ns:foo,ns:bar']
+ ''')
+
+ del conf['sources']['one_source']['req']['layers']
+
+ errors = validate_references(conf)
+ eq_(errors, [])
+
def test_with_grouped_layer(self):
conf = self._test_conf('''
layers:
@@ -374,4 +387,4 @@ class TestValidator(object):
eq_(errors, [
"Source 'missing1' for cache 'one_cache' not found in config",
"Source 'missing2' for cache 'cache_missing_source' not found in config",
- ])
\ No newline at end of file
+ ])
diff --git a/mapproxy/test/unit/test_config.py b/mapproxy/test/unit/test_config.py
index bd0e6e1..fd62398 100644
--- a/mapproxy/test/unit/test_config.py
+++ b/mapproxy/test/unit/test_config.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
from mapproxy.config import Options, base_config, load_base_config
diff --git a/mapproxy/test/unit/test_featureinfo.py b/mapproxy/test/unit/test_featureinfo.py
index 39dff83..3e24942 100644
--- a/mapproxy/test/unit/test_featureinfo.py
+++ b/mapproxy/test/unit/test_featureinfo.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
import tempfile
diff --git a/mapproxy/test/unit/test_geom.py b/mapproxy/test/unit/test_geom.py
index 8d11fea..956c74c 100644
--- a/mapproxy/test/unit/test_geom.py
+++ b/mapproxy/test/unit/test_geom.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import division, with_statement
+from __future__ import division
import os
import tempfile
@@ -47,6 +47,11 @@ from mapproxy.util.coverage import BBOXCoverage
import shapely
import shapely.prepared
+try:
+ # shapely >=1.6
+ from shapely.errors import ReadingError
+except ImportError:
+ from shapely.geos import ReadingError
from nose.tools import eq_, raises
@@ -117,7 +122,7 @@ class TestPolygonLoading(object):
assert polygon.is_valid
eq_(polygon.type, 'MultiPolygon')
- @raises(shapely.geos.ReadingError)
+ @raises(ReadingError)
def test_loading_broken(self):
with TempFile() as fname:
with open(fname, 'wb') as f:
diff --git a/mapproxy/test/unit/test_grid.py b/mapproxy/test/unit/test_grid.py
index e17b780..1cea8df 100644
--- a/mapproxy/test/unit/test_grid.py
+++ b/mapproxy/test/unit/test_grid.py
@@ -1058,8 +1058,7 @@ class TestBBOXContains(object):
def assert_almost_equal_bbox(bbox1, bbox2, places=2):
for coord1, coord2 in zip(bbox1, bbox2):
- assert_almost_equal(coord1, coord2, places)
-
+ assert_almost_equal(coord1, coord2, places, msg='%s != %s' % (bbox1, bbox2))
class TestResolutionRange(object):
def test_meter(self):
diff --git a/mapproxy/test/unit/test_image.py b/mapproxy/test/unit/test_image.py
index 238da1a..9d03adf 100644
--- a/mapproxy/test/unit/test_image.py
+++ b/mapproxy/test/unit/test_image.py
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
from io import BytesIO
@@ -33,10 +32,10 @@ from mapproxy.image import (
from mapproxy.image.merge import merge_images, BandMerger
from mapproxy.image.opts import ImageOptions
from mapproxy.image.tile import TileMerger, TileSplitter
-from mapproxy.image.transform import ImageTransformer
+from mapproxy.image.transform import ImageTransformer, transform_meshes
from mapproxy.test.image import is_png, is_jpeg, is_tiff, create_tmp_image_file, check_format, create_debug_img, create_image
from mapproxy.srs import SRS
-from nose.tools import eq_
+from nose.tools import eq_, assert_almost_equal
from mapproxy.test.image import assert_img_colors_eq
from nose.plugins.skip import SkipTest
@@ -129,6 +128,17 @@ class TestImageSource(object):
eq_(img.mode, 'RGBA')
assert img.getpixel((0, 0)) == (0, 0, 0, 0)
+ def test_save_with_unsupported_transparency(self):
+ # check if encoding of non-RGB image with tuple as transparency
+ # works. workaround for Pillow #2633
+ img = Image.new('P', (100, 100))
+ img.info['transparency'] = (0, 0, 0)
+ image_opts = PNG_FORMAT.copy()
+
+ ir = ImageSource(img, image_opts=image_opts)
+ img = Image.open(ir.as_buffer())
+ eq_(img.mode, 'P')
+
class TestSubImageSource(object):
def test_full(self):
sub_img = create_image((100, 100), color=[100, 120, 130, 140])
@@ -433,23 +443,84 @@ class TestTransform(object):
self.dst_srs = SRS(4326)
self.dst_bbox = (0.2, 45.1, 8.3, 53.2)
self.src_bbox = self.dst_srs.transform_bbox_to(self.src_srs, self.dst_bbox)
- def test_transform(self, mesh_div=4):
- transformer = ImageTransformer(self.src_srs, self.dst_srs, mesh_div=mesh_div)
+ def test_transform(self):
+ transformer = ImageTransformer(self.src_srs, self.dst_srs)
result = transformer.transform(self.src_img, self.src_bbox, self.dst_size, self.dst_bbox,
image_opts=ImageOptions(resampling='nearest'))
assert isinstance(result, ImageSource)
assert result.as_image() != self.src_img.as_image()
assert result.size == (100, 150)
- def _test_compare_mesh_div(self):
+ def _test_compare_max_px_err(self):
"""
Create transformations with different div values.
"""
- for div in [1, 2, 4, 6, 8, 12, 16]:
- transformer = ImageTransformer(self.src_srs, self.dst_srs, mesh_div=div)
+ for err in [0.2, 0.5, 1, 2, 4, 6, 8, 12, 16]:
+ transformer = ImageTransformer(self.src_srs, self.dst_srs, max_px_err=err)
result = transformer.transform(self.src_img, self.src_bbox,
- self.dst_size, self.dst_bbox)
- result.as_image().save('/tmp/transform-%d.png' % (div,))
+ self.dst_size, self.dst_bbox,
+ image_opts=ImageOptions(resampling='nearest'))
+ result.as_image().save('/tmp/transform-%03d.png' % (err*10,))
+
+
+class TestMesh(object):
+
+ def test_mesh_utm(self):
+ meshes = transform_meshes(
+ src_size=(1335, 1531),
+ src_bbox=(3.65, 39.84, 17.00, 55.15),
+ src_srs=SRS(4326),
+ dst_size=(853, 1683),
+ dst_bbox=(158512, 4428236, 1012321, 6111268),
+ dst_srs=SRS(25832),
+ )
+ eq_(len(meshes), 40)
+
+ def test_mesh_none(self):
+ meshes = transform_meshes(
+ src_size=(1000, 1500),
+ src_bbox=(0, 0, 10, 15),
+ src_srs=SRS(4326),
+ dst_size=(1000, 1500),
+ dst_bbox=(0, 0, 10, 15),
+ dst_srs=SRS(4326),
+ )
+
+ eq_(meshes, [((0, 0, 1000, 1500), [0.0, 0.0, 0.0, 1500.0, 1000.0, 1500.0, 1000.0, 0.0])])
+ eq_(len(meshes), 1)
+
+
+ def test_mesh(self):
+ # low map scale -> more meshes
+ # print(SRS(4326).transform_bbox_to(SRS(3857), (5, 50, 10, 55)))
+ meshes = transform_meshes(
+ src_size=(1000, 2000),
+ src_bbox=(556597, 6446275, 1113194, 7361866),
+ src_srs=SRS(3857),
+ dst_size=(1000, 1000),
+ dst_bbox=(5, 50, 10, 55),
+ dst_srs=SRS(4326),
+ )
+ eq_(len(meshes), 16)
+
+ # large map scale -> one meshes
+ # print(SRS(4326).transform_bbox_to(SRS(3857), (5, 50, 5.1, 50.1)))
+ meshes = transform_meshes(
+ src_size=(1000, 2000),
+ src_bbox=(556597.4539663672, 6446275.841017158,
+ 567729.4030456939, 6463612.124257667),
+ src_srs=SRS(3857),
+ dst_size=(1000, 1000),
+ dst_bbox=(5, 50, 5.1, 50.1),
+ dst_srs=SRS(4326),
+ )
+ eq_(len(meshes), 1)
+
+ # quad stretches whole image plus 1 pixel
+ eq_(meshes[0][0], (0, 0, 1000, 1000))
+ for e, a in zip(meshes[0][1], [0.0, 0.0, 0.0, 2000.0, 1000.0, 2000.0, 1000.0, 0.0]):
+ assert_almost_equal(e, a)
+
class TestSingleColorImage(object):
diff --git a/mapproxy/test/unit/test_multiapp.py b/mapproxy/test/unit/test_multiapp.py
index e99c6e7..80e4fd5 100644
--- a/mapproxy/test/unit/test_multiapp.py
+++ b/mapproxy/test/unit/test_multiapp.py
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
import time
import tempfile
diff --git a/mapproxy/test/unit/test_seed.py b/mapproxy/test/unit/test_seed.py
index f4c2c47..f20c9e9 100644
--- a/mapproxy/test/unit/test_seed.py
+++ b/mapproxy/test/unit/test_seed.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
import os
import time
diff --git a/mapproxy/test/unit/test_seed_cachelock.py b/mapproxy/test/unit/test_seed_cachelock.py
index c8ff531..52e7412 100644
--- a/mapproxy/test/unit/test_seed_cachelock.py
+++ b/mapproxy/test/unit/test_seed_cachelock.py
@@ -12,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import multiprocessing
import os
diff --git a/mapproxy/test/unit/test_tiled_source.py b/mapproxy/test/unit/test_tiled_source.py
index 3e32aef..548d5d1 100644
--- a/mapproxy/test/unit/test_tiled_source.py
+++ b/mapproxy/test/unit/test_tiled_source.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
from mapproxy.client.tile import TMSClient
from mapproxy.grid import TileGrid
diff --git a/mapproxy/test/unit/test_utils.py b/mapproxy/test/unit/test_utils.py
index 3190393..c726570 100644
--- a/mapproxy/test/unit/test_utils.py
+++ b/mapproxy/test/unit/test_utils.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import os
import glob
import sys
diff --git a/mapproxy/test/unit/test_wms_layer.py b/mapproxy/test/unit/test_wms_layer.py
index 9bd6bc0..2bbe719 100644
--- a/mapproxy/test/unit/test_wms_layer.py
+++ b/mapproxy/test/unit/test_wms_layer.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, division
+from __future__ import division
from mapproxy.layer import MapQuery, InfoQuery
from mapproxy.srs import SRS
diff --git a/mapproxy/util/async.py b/mapproxy/util/async.py
index 7a63b21..f0e98c0 100644
--- a/mapproxy/util/async.py
+++ b/mapproxy/util/async.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
MAX_MAP_ASYNC_THREADS = 20
diff --git a/mapproxy/util/coverage.py b/mapproxy/util/coverage.py
index 05a83c0..4479d2a 100644
--- a/mapproxy/util/coverage.py
+++ b/mapproxy/util/coverage.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement
import operator
import threading
diff --git a/mapproxy/util/ext/dictspec/validator.py b/mapproxy/util/ext/dictspec/validator.py
index efa03af..9c5e86b 100644
--- a/mapproxy/util/ext/dictspec/validator.py
+++ b/mapproxy/util/ext/dictspec/validator.py
@@ -18,7 +18,6 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
-from __future__ import with_statement
import re
from contextlib import contextmanager
diff --git a/mapproxy/util/ext/serving.py b/mapproxy/util/ext/serving.py
index 50a8d3d..37a6110 100644
--- a/mapproxy/util/ext/serving.py
+++ b/mapproxy/util/ext/serving.py
@@ -1,41 +1,13 @@
# -*- coding: utf-8 -*-
"""
- werkzeug.serving
- ~~~~~~~~~~~~~~~~
-
- There are many ways to serve a WSGI application. While you're developing
- it you usually don't want a full blown webserver like Apache but a simple
- standalone one. From Python 2.5 onwards there is the `wsgiref`_ server in
- the standard library. If you're using older versions of Python you can
- download the package from the cheeseshop.
-
- However there are some caveats. Sourcecode won't reload itself when
- changed and each time you kill the server using ``^C`` you get an
- `KeyboardInterrupt` error. While the latter is easy to solve the first
- one can be a pain in the ass in some situations.
-
- The easiest way is creating a small ``start-myproject.py`` that runs the
- application::
-
- #!/usr/bin/env python
- # -*- coding: utf-8 -*-
- from myproject import make_app
- from werkzeug.serving import run_simple
-
- app = make_app(...)
- run_simple('localhost', 8080, app, use_reloader=True)
-
- You can also pass it a `extra_files` keyword argument with a list of
- additional files (like configuration files) you want to observe.
-
- For bigger applications you should consider using `werkzeug.script`
- instead of a simple start file.
+ WSGI server code for `mapproxy-util serve-develop`.
+ Supports automatic reloading and proper ^C handling.
+ Stripped down version of werkzeug.serving.
:copyright: (c) 2013 by the Werkzeug Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
-from __future__ import with_statement
import os
import socket
@@ -60,10 +32,16 @@ from mapproxy.compat import iteritems, PY2, text_type
from mapproxy.compat.itertools import chain
from mapproxy.util.py import reraise
-def wsgi_encoding_dance(s, charset='utf-8', errors='replace'):
- if isinstance(s, bytes):
- return s
- return s.encode(charset, errors)
+if PY2:
+ def wsgi_encoding_dance(s, charset='utf-8', errors='replace'):
+ if isinstance(s, bytes):
+ return s
+ return s.encode(charset, errors)
+else:
+ def wsgi_encoding_dance(s, charset='utf-8', errors='replace'):
+ if isinstance(s, text_type):
+ s = s.encode(charset)
+ return s.decode('latin1', errors)
try:
from urllib.parse import urlparse as url_parse, unquote as url_unquote
@@ -94,7 +72,7 @@ class WSGIRequestHandler(BaseHTTPRequestHandler, object):
def shutdown_server():
self.server.shutdown_signal = True
- url_scheme = self.server.ssl_context is None and 'http' or 'https'
+ url_scheme = 'http'
path_info = url_unquote(request_url.path)
environ = {
@@ -218,8 +196,7 @@ class WSGIRequestHandler(BaseHTTPRequestHandler, object):
except (socket.error, socket.timeout) as e:
self.connection_dropped(e)
except Exception:
- if self.server.ssl_context is None or not is_ssl_error():
- raise
+ raise
if self.server.shutdown_signal:
self.initiate_shutdown()
return rv
@@ -285,115 +262,6 @@ class WSGIRequestHandler(BaseHTTPRequestHandler, object):
BaseRequestHandler = WSGIRequestHandler
-def generate_adhoc_ssl_pair(cn=None):
- from random import random
- from OpenSSL import crypto
-
- # pretty damn sure that this is not actually accepted by anyone
- if cn is None:
- cn = '*'
-
- cert = crypto.X509()
- cert.set_serial_number(int(random() * sys.maxsize))
- cert.gmtime_adj_notBefore(0)
- cert.gmtime_adj_notAfter(60 * 60 * 24 * 365)
-
- subject = cert.get_subject()
- subject.CN = cn
- subject.O = 'Dummy Certificate'
-
- issuer = cert.get_issuer()
- issuer.CN = 'Untrusted Authority'
- issuer.O = 'Self-Signed'
-
- pkey = crypto.PKey()
- pkey.generate_key(crypto.TYPE_RSA, 768)
- cert.set_pubkey(pkey)
- cert.sign(pkey, 'md5')
-
- return cert, pkey
-
-
-def make_ssl_devcert(base_path, host=None, cn=None):
- """Creates an SSL key for development. This should be used instead of
- the ``'adhoc'`` key which generates a new cert on each server start.
- It accepts a path for where it should store the key and cert and
- either a host or CN. If a host is given it will use the CN
- ``*.host/CN=host``.
-
- For more information see :func:`run_simple`.
-
- .. versionadded:: 0.9
-
- :param base_path: the path to the certificate and key. The extension
- ``.crt`` is added for the certificate, ``.key`` is
- added for the key.
- :param host: the name of the host. This can be used as an alternative
- for the `cn`.
- :param cn: the `CN` to use.
- """
- from OpenSSL import crypto
- if host is not None:
- cn = '*.%s/CN=%s' % (host, host)
- cert, pkey = generate_adhoc_ssl_pair(cn=cn)
-
- cert_file = base_path + '.crt'
- pkey_file = base_path + '.key'
-
- with open(cert_file, 'wb') as f:
- f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
- with open(pkey_file, 'wb') as f:
- f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey))
-
- return cert_file, pkey_file
-
-
-def generate_adhoc_ssl_context():
- """Generates an adhoc SSL context for the development server."""
- from OpenSSL import SSL
- cert, pkey = generate_adhoc_ssl_pair()
- ctx = SSL.Context(SSL.SSLv23_METHOD)
- ctx.use_privatekey(pkey)
- ctx.use_certificate(cert)
- return ctx
-
-
-def load_ssl_context(cert_file, pkey_file):
- """Loads an SSL context from a certificate and private key file."""
- from OpenSSL import SSL
- ctx = SSL.Context(SSL.SSLv23_METHOD)
- ctx.use_certificate_file(cert_file)
- ctx.use_privatekey_file(pkey_file)
- return ctx
-
-
-def is_ssl_error(error=None):
- """Checks if the given error (or the current one) is an SSL error."""
- if error is None:
- error = sys.exc_info()[1]
- from OpenSSL import SSL
- return isinstance(error, SSL.Error)
-
-
-class _SSLConnectionFix(object):
- """Wrapper around SSL connection to provide a working makefile()."""
-
- def __init__(self, con):
- self._con = con
-
- def makefile(self, mode, bufsize):
- return socket._fileobject(self._con, mode, bufsize)
-
- def __getattr__(self, attrib):
- return getattr(self._con, attrib)
-
- def shutdown(self, arg=None):
- try:
- self._con.shutdown()
- except Exception:
- pass
-
-
def select_ip_version(host, port):
"""Returns AF_INET4 or AF_INET6 depending on where to connect to."""
# disabled due to problems with current ipv6 implementations
@@ -420,7 +288,7 @@ class BaseWSGIServer(HTTPServer, object):
request_queue_size = 128
def __init__(self, host, port, app, handler=None,
- passthrough_errors=False, ssl_context=None):
+ passthrough_errors=False):
if handler is None:
handler = WSGIRequestHandler
self.address_family = select_ip_version(host, port)
@@ -429,21 +297,6 @@ class BaseWSGIServer(HTTPServer, object):
self.passthrough_errors = passthrough_errors
self.shutdown_signal = False
- if ssl_context is not None:
- try:
- from OpenSSL import tsafe
- except ImportError:
- raise TypeError('SSL is not available if the OpenSSL '
- 'library is not installed.')
- if isinstance(ssl_context, tuple):
- ssl_context = load_ssl_context(*ssl_context)
- if ssl_context == 'adhoc':
- ssl_context = generate_adhoc_ssl_context()
- self.socket = tsafe.Connection(ssl_context, self.socket)
- self.ssl_context = ssl_context
- else:
- self.ssl_context = None
-
def log(self, type, message, *args):
_log(type, message, *args)
@@ -462,8 +315,6 @@ class BaseWSGIServer(HTTPServer, object):
def get_request(self):
con, info = self.socket.accept()
- if self.ssl_context is not None:
- con = _SSLConnectionFix(con)
return con, info
@@ -517,48 +368,6 @@ def _reloader_stat_loop(extra_files=None, interval=1):
time.sleep(interval)
-def _reloader_inotify(extra_files=None, interval=None):
- # Mutated by inotify loop when changes occur.
- changed = [False]
-
- # Setup inotify watches
- from pyinotify import WatchManager, Notifier
-
- # this API changed at one point, support both
- try:
- from pyinotify import EventsCodes as ec
- ec.IN_ATTRIB
- except (ImportError, AttributeError):
- import pyinotify as ec
-
- wm = WatchManager()
- mask = ec.IN_DELETE_SELF | ec.IN_MOVE_SELF | ec.IN_MODIFY | ec.IN_ATTRIB
-
- def signal_changed(event):
- if changed[0]:
- return
- _log('info', ' * Detected change in %r, reloading' % event.path)
- changed[:] = [True]
-
- for fname in extra_files or ():
- wm.add_watch(fname, mask, signal_changed)
-
- # ... And now we wait...
- notif = Notifier(wm)
- try:
- while not changed[0]:
- # always reiterate through sys.modules, adding them
- for fname in _iter_module_files():
- wm.add_watch(fname, mask, signal_changed)
- notif.process_events()
- if notif.check_events(timeout=interval):
- notif.read_events()
- # TODO Set timeout to something small and check parent liveliness
- finally:
- notif.stop()
- sys.exit(3)
-
-
# currently we always use the stat loop reloader for the simple reason
# that the inotify one does not respond to added files properly. Also
# it's quite buggy and the API is a mess.
@@ -620,7 +429,7 @@ def run_simple(hostname, port, application, use_reloader=False,
use_debugger=False, use_evalex=True,
extra_files=None, reloader_interval=1, threaded=False,
processes=1, request_handler=None, static_files=None,
- passthrough_errors=False, ssl_context=None):
+ passthrough_errors=False):
"""Start an application using wsgiref and with an optional reloader. This
wraps `wsgiref` to fix the wrong default reporting of the multithreaded
WSGI variable and adds optional multithreading and fork support.
@@ -670,30 +479,22 @@ def run_simple(hostname, port, application, use_reloader=False,
:param passthrough_errors: set this to `True` to disable the error catching.
This means that the server will die on errors but
it can be useful to hook debuggers in (pdb etc.)
- :param ssl_context: an SSL context for the connection. Either an OpenSSL
- context, a tuple in the form ``(cert_file, pkey_file)``,
- the string ``'adhoc'`` if the server should
- automatically create one, or `None` to disable SSL
- (which is the default).
"""
if use_debugger:
from werkzeug.debug import DebuggedApplication
application = DebuggedApplication(application, use_evalex)
- if static_files:
- from werkzeug.wsgi import SharedDataMiddleware
- application = SharedDataMiddleware(application, static_files)
def inner():
ThreadedWSGIServer(hostname, port, application, request_handler,
- passthrough_errors, ssl_context).serve_forever()
+ passthrough_errors).serve_forever()
if os.environ.get('WERKZEUG_RUN_MAIN') != 'true':
display_hostname = hostname != '*' and hostname or 'localhost'
if ':' in display_hostname:
display_hostname = '[%s]' % display_hostname
quit_msg = '(Press CTRL+C to quit)'
- _log('info', ' * Running on %s://%s:%d/ %s', ssl_context is None
- and 'http' or 'https', display_hostname, port, quit_msg)
+ _log('info', ' * Running on http://%s:%d/ %s',
+ display_hostname, port, quit_msg)
if use_reloader:
# Create and destroy a socket so that any exceptions are raised before
# we spawn a separate Python interpreter and lose this ability.
@@ -705,42 +506,3 @@ def run_simple(hostname, port, application, use_reloader=False,
run_with_reloader(inner, extra_files, reloader_interval)
else:
inner()
-
-def main():
- '''A simple command-line interface for :py:func:`run_simple`.'''
-
- # in contrast to argparse, this works at least under Python < 2.7
- import optparse
- from werkzeug.utils import import_string
-
- parser = optparse.OptionParser(usage='Usage: %prog [options] app_module:app_object')
- parser.add_option('-b', '--bind', dest='address',
- help='The hostname:port the app should listen on.')
- parser.add_option('-d', '--debug', dest='use_debugger',
- action='store_true', default=False,
- help='Use Werkzeug\'s debugger.')
- parser.add_option('-r', '--reload', dest='use_reloader',
- action='store_true', default=False,
- help='Reload Python process if modules change.')
- options, args = parser.parse_args()
-
- hostname, port = None, None
- if options.address:
- address = options.address.split(':')
- hostname = address[0]
- if len(address) > 1:
- port = address[1]
-
- if len(args) != 1:
- sys.stdout.write('No application supplied, or too much. See --help\n')
- sys.exit(1)
- app = import_string(args[0])
-
- run_simple(
- hostname=(hostname or '127.0.0.1'), port=int(port or 5000),
- application=app, use_reloader=options.use_reloader,
- use_debugger=options.use_debugger
- )
-
-if __name__ == '__main__':
- main()
diff --git a/mapproxy/util/fs.py b/mapproxy/util/fs.py
index 675a543..2cfffb3 100644
--- a/mapproxy/util/fs.py
+++ b/mapproxy/util/fs.py
@@ -16,7 +16,7 @@
"""
File system related utility functions.
"""
-from __future__ import with_statement, absolute_import
+from __future__ import absolute_import
import time
import os
import sys
@@ -124,7 +124,7 @@ def write_atomic(filename, data):
# where file locking does not work (network fs)
path_tmp = filename + '.tmp-' + str(random.randint(0, 99999999))
try:
- fd = os.open(path_tmp, os.O_EXCL | os.O_CREAT | os.O_WRONLY)
+ fd = os.open(path_tmp, os.O_EXCL | os.O_CREAT | os.O_WRONLY, 0o666)
with os.fdopen(fd, 'wb') as f:
f.write(data)
os.rename(path_tmp, filename)
@@ -137,3 +137,19 @@ def write_atomic(filename, data):
else:
with open(filename, 'wb') as f:
f.write(data)
+
+
+def find_exec(executable):
+ """
+ Search executable in PATH environment. Return path if found, None if not.
+ """
+ path = os.environ.get('PATH')
+ if not path:
+ return
+ for p in path.split(os.path.pathsep):
+ p = os.path.join(p, executable)
+ if os.path.exists(p):
+ return p
+ p += '.exe'
+ if os.path.exists(p):
+ return p
diff --git a/mapproxy/util/geom.py b/mapproxy/util/geom.py
index 21ce047..bdc9600 100644
--- a/mapproxy/util/geom.py
+++ b/mapproxy/util/geom.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import division, with_statement
+from __future__ import division
import os
import json
@@ -32,7 +32,11 @@ try:
import shapely.geometry
import shapely.ops
import shapely.prepared
- from shapely.geos import ReadingError
+ try:
+ # shapely >=1.6
+ from shapely.errors import ReadingError
+ except ImportError:
+ from shapely.geos import ReadingError
geom_support = True
except ImportError:
geom_support = False
diff --git a/mapproxy/util/lock.py b/mapproxy/util/lock.py
index 963145b..da5c4c4 100644
--- a/mapproxy/util/lock.py
+++ b/mapproxy/util/lock.py
@@ -16,7 +16,6 @@
"""
Utility methods and classes (file locking, asynchronous execution pools, etc.).
"""
-from __future__ import with_statement
import random
import time
diff --git a/mapproxy/util/py.py b/mapproxy/util/py.py
index 91a410d..b35a309 100644
--- a/mapproxy/util/py.py
+++ b/mapproxy/util/py.py
@@ -16,7 +16,6 @@
"""
Python related helper functions.
"""
-from __future__ import with_statement
from functools import wraps
from mapproxy.compat import PY2
diff --git a/mapproxy/util/yaml.py b/mapproxy/util/yaml.py
index 3fde31a..8d59208 100644
--- a/mapproxy/util/yaml.py
+++ b/mapproxy/util/yaml.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import with_statement, absolute_import
+from __future__ import absolute_import
from mapproxy.compat import string_type
import yaml
diff --git a/mapproxy/wsgiapp.py b/mapproxy/wsgiapp.py
index 7d08b12..5822083 100644
--- a/mapproxy/wsgiapp.py
+++ b/mapproxy/wsgiapp.py
@@ -24,6 +24,13 @@ import time
import threading
import warnings
+try:
+ # time.strptime is thread-safe, but not the first call.
+ # Import _strptime as a workaround. See: http://bugs.python.org/issue7980
+ import _strptime
+except ImportError:
+ pass
+
from mapproxy.compat import iteritems
from mapproxy.request import Request
from mapproxy.response import Response
@@ -93,9 +100,6 @@ def make_wsgi_app(services_conf=None, debug=False, ignore_config_warnings=True,
:param reloader: reload mapproxy.yaml when it changed
"""
- if sys.version_info[0] == 2 and sys.version_info[1] == 5:
- warnings.warn('Support for Python 2.5 is deprecated since 1.7.0 and will be dropped with 1.8.0', FutureWarning)
-
if reloader:
make_app = lambda: make_wsgi_app(services_conf=services_conf, debug=debug,
reloader=False)
diff --git a/release.py b/release.py
index 1878a39..b18d4f7 100644
--- a/release.py
+++ b/release.py
@@ -82,14 +82,22 @@ def upload_test_sdist_command():
date = backtick_('date +%Y%m%d').strip()
print('python setup.py egg_info -R -D -b ".dev%s" register -r testpypi sdist upload -r testpypi' % (date, ))
+def check_uncommited():
+ if sh('git diff-index --quiet HEAD --') != 0:
+ print('ABORT: uncommited changes. please commit (and tag) release version number')
+ sys.exit(1)
+
def upload_final_sdist_command():
- sh('python setup.py egg_info -b "" -D sdist upload')
+ check_uncommited()
+ build_sdist_command()
+ ver = version()
+ sh('twine upload dist/MapProxy-%(ver)s.tar.gz' % locals())
def upload_final_wheel_command():
- sh('python setup.py egg_info -b "" -D bdist_wheel upload')
-
-def register_command():
- sh('python setup.py egg_info -b "" -D register')
+ check_uncommited()
+ build_wheel_command()
+ ver = version()
+ sh('twine upload dist/MapProxy-%(ver)s-py2.py3-none-any.whl' % locals())
def link_latest_command(ver=None):
if ver is None:
diff --git a/requirements-tests.txt b/requirements-tests.txt
index 351bb02..b7c5367 100644
--- a/requirements-tests.txt
+++ b/requirements-tests.txt
@@ -17,6 +17,7 @@ docutils==0.13.1
enum-compat==0.0.2
futures==3.0.5
greenlet==0.4.12
+riak==2.6.1
httpretty==0.8.10
Jinja2==2.9.5
jmespath==0.9.1
diff --git a/requirements-travis.txt b/requirements-travis.txt
index dc47f9b..d06d53e 100644
--- a/requirements-travis.txt
+++ b/requirements-travis.txt
@@ -6,4 +6,5 @@ nose==1.1.2
Shapely==1.2.15
PyYAML==3.10
Pillow==1.7.7
-eventlet==0.9.17
\ No newline at end of file
+eventlet==0.9.17
+riak==2.6.1
diff --git a/setup.py b/setup.py
index 90c527c..f03a5cd 100644
--- a/setup.py
+++ b/setup.py
@@ -54,14 +54,13 @@ def long_description(changelog_releases=10):
setup(
name='MapProxy',
- version="1.10.4",
- description='An accelerating proxy for web map services',
+ version="1.11.0",
+ description='An accelerating proxy for tile and web map services',
long_description=long_description(7),
author='Oliver Tonnhofer',
author_email='olt at omniscale.de',
- url='http://mapproxy.org',
+ url='https://mapproxy.org',
license='Apache Software License 2.0',
- namespace_packages = ['mapproxy'],
packages=find_packages(),
include_package_data=True,
entry_points = {
@@ -86,11 +85,11 @@ setup(
"Development Status :: 5 - Production/Stable",
"License :: OSI Approved :: Apache Software License",
"Operating System :: OS Independent",
- "Programming Language :: Python :: 2.6",
"Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3.3",
"Programming Language :: Python :: 3.4",
"Programming Language :: Python :: 3.5",
+ "Programming Language :: Python :: 3.6",
"Topic :: Internet :: Proxy Servers",
"Topic :: Internet :: WWW/HTTP :: WSGI",
"Topic :: Scientific/Engineering :: GIS",
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/mapproxy.git
More information about the Pkg-grass-devel
mailing list