[python-stetl] 04/12: New upstream version 1.1+ds
Bas Couwenberg
sebastic at debian.org
Tue Nov 7 18:53:09 UTC 2017
This is an automated email from the git hooks/post-receive script.
sebastic pushed a commit to branch master
in repository python-stetl.
commit e85bddc32ea82b7dcf62160221ead35ec77b901b
Author: Bas Couwenberg <sebastic at xs4all.nl>
Date: Tue Nov 7 19:00:51 2017 +0100
New upstream version 1.1+ds
---
.flake8 | 5 +
CHANGES.txt | 22 +-
CONTRIBUTING.md | 147 ++
CREDITS.txt | 12 +-
PKG-INFO | 122 +-
README.md | 84 +-
VERSION.txt | 2 +-
docker/Dockerfile | 44 -
docker/README.md | 38 -
docker/build.sh | 9 -
docker/test/1_copystd/etl.cfg | 11 -
docker/test/1_copystd/etl.sh | 9 -
docs/background.rst | 2 +-
docs/cases.rst | 22 +-
docs/code.rst | 18 +-
docs/conf.py | 4 +-
docs/contact.rst | 2 +-
docs/foss4g13-abstract.txt | 2 +-
docs/index.rst | 5 +-
docs/install.rst | 122 +-
docs/intro.rst | 4 +-
docs/links.rst | 44 +-
docs/using.rst | 214 ++-
examples/README.md | 9 +
.../10_jinja2_templating/output/cities-gjson.gml | 4 +-
examples/basics/12_gdal_ogr/etl.cfg | 5 +-
examples/basics/12_gdal_ogr/etl.sh | 4 +-
examples/basics/12_gdal_ogr/output/cities.dbf | Bin 1432 -> 1413 bytes
examples/basics/12_gdal_ogr/output/cities.gml | 6 +-
examples/basics/15_splitter/cities2gml.xsl | 50 +
examples/basics/15_splitter/etl.cfg | 22 +
examples/basics/{4_validate => 15_splitter}/etl.sh | 2 +-
examples/basics/15_splitter/gmlcities.xsd | 33 +
.../basics/15_splitter}/input/cities.xml | 2 +-
examples/basics/15_splitter/output/gmlcities.gml | 45 +
examples/basics/16_merger/cities2gml.xsl | 50 +
examples/basics/16_merger/etl.cfg | 25 +
examples/basics/{4_validate => 16_merger}/etl.sh | 2 +-
examples/basics/16_merger/gmlcities.xsd | 33 +
.../basics/16_merger/input1}/cities.xml | 2 +-
.../basics/16_merger/input2}/cities.xml | 2 +-
examples/basics/16_merger/output/gmlcities.gml | 45 +
examples/basics/1_copystd/etl.sh | 7 +-
examples/basics/3_shape/output/gmlcities.dbf | Bin 95 -> 96 bytes
examples/basics/3_shape/temp/gmlcities.gfs | 3 +
examples/basics/4_validate/etl.sh | 2 +-
examples/basics/8_wfs/etl.cfg | 9 +-
examples/basics/README.md | 23 +
examples/basics/readme.txt | 33 -
examples/basics/runall.log | 1501 ++++++++++++--------
examples/basics/runall.sh | 9 +-
examples/bgt/README.md | 6 +-
examples/ordnancesurvey/{readme.txt => README.md} | 25 +-
examples/top10nl/{readme.txt => README.md} | 13 +-
requirements-dev.txt | 6 +
requirements-main.txt | 4 +
setup.py | 17 +-
stetl/__init__.py | 2 -
stetl/chain.py | 110 +-
stetl/component.py | 43 +-
stetl/etl.py | 6 +-
stetl/factory.py | 12 +-
stetl/filters/formatconverter.py | 15 +-
stetl/filters/gmlfeatureextractor.py | 2 -
stetl/filters/gmlsplitter.py | 5 +-
stetl/filters/nullfilter.py | 24 +
stetl/filters/packetbuffer.py | 28 +
stetl/filters/packetwriter.py | 6 +-
stetl/filters/stringfilter.py | 5 +-
stetl/filters/templatingfilter.py | 13 +-
stetl/filters/xmlassembler.py | 23 +-
stetl/filters/xmlelementreader.py | 28 +-
stetl/filters/xsltfilter.py | 1 -
stetl/filters/zipfileextractor.py | 14 +-
stetl/input.py | 1 -
stetl/inputs/dbinput.py | 65 +-
stetl/inputs/deegreeinput.py | 53 +-
stetl/inputs/fileinput.py | 135 +-
stetl/inputs/httpinput.py | 88 +-
stetl/inputs/ogrinput.py | 138 +-
stetl/main.py | 19 +-
stetl/merger.py | 128 ++
stetl/output.py | 4 +-
stetl/outputs/dboutput.py | 9 +-
stetl/outputs/deegreeoutput.py | 26 +-
stetl/outputs/execoutput.py | 108 +-
stetl/outputs/fileoutput.py | 23 +-
stetl/outputs/httpoutput.py | 109 +-
stetl/outputs/ogroutput.py | 77 +-
stetl/outputs/standardoutput.py | 5 +-
stetl/outputs/wfsoutput.py | 37 +-
stetl/packet.py | 25 +
stetl/postgis.py | 31 +-
stetl/splitter.py | 107 ++
stetl/util.py | 26 +-
stetl/utils/apachelog.py | 19 +-
stetl/version.py | 2 +-
tests/README.txt | 26 +-
tests/__init__.pyc | Bin 134 -> 0 bytes
.../temp/gmlcities.gfs => tests/data/cities.gfs | 2 +
tests/data/cities.gml | 45 +
tests/data/command.txt | 1 +
tests/data/data_xmlassembler.gml | 271 ++++
tests/data/dummy.gml | 113 ++
tests/data/dummy.sql | 2 +
tests/data/dummy2.gml | 113 ++
tests/data/gfs/top10-funcgebied.gfs | 229 +++
tests/data/keep_dummy.gml | 113 ++
tests/data/menu.json | 11 +
tests/data/pandhoogtes-klein.csv | 10 +
tests/data/pandhoogtes.csv | 432 ++++++
tests/data/stringfileinput_formatargs.txt | 1 +
tests/data/temp/.gitignore | 4 +
tests/data/xslt/dummy.xslt | 55 +
tests/data/zipfileinput.zip | Bin 0 -> 18713 bytes
tests/filters/__init__.py | 0
tests/filters/configs/packetwriter.cfg | 16 +
tests/filters/configs/stringsubstitutionfilter.cfg | 18 +
tests/filters/configs/xmlassembler.cfg | 39 +
tests/filters/configs/xmlelementreader.cfg | 33 +
tests/filters/configs/xsltfilter.cfg | 19 +
tests/filters/configs/zipfileextractor.cfg | 17 +
tests/filters/test_packet_writer.py | 49 +
tests/filters/test_string_substitution_filter.py | 38 +
tests/filters/test_xml_assembler.py | 65 +
tests/filters/test_xml_element_reader.py | 70 +
tests/filters/test_xslt_filter.py | 50 +
tests/filters/test_zip_file_extractor.py | 45 +
tests/inputs/__init__.py | 0
tests/inputs/configs/csvfileinput.cfg | 14 +
tests/inputs/configs/globfileinput.cfg | 16 +
tests/inputs/configs/jsonfileinput.cfg | 12 +
tests/inputs/configs/linestreamerfileinput.cfg | 11 +
tests/inputs/configs/mergermultiinput.cfg | 19 +
tests/inputs/configs/ogrinput.cfg | 16 +
tests/inputs/configs/stringfileinput.cfg | 17 +
.../inputs/configs/xmlelementstreamerfileinput.cfg | 12 +
tests/inputs/configs/zipfileinput.cfg | 17 +
tests/inputs/test_csv_file_input.py | 39 +
tests/inputs/test_glob_file_input.py | 44 +
tests/inputs/test_json_file_input.py | 51 +
tests/inputs/test_line_streamer_file_input.py | 43 +
tests/inputs/test_merger_multi_input.py | 90 ++
tests/inputs/test_ogr_input.py | 37 +
tests/inputs/test_string_file_input.py | 59 +
.../inputs/test_xml_element_streamer_file_input.py | 38 +
tests/inputs/test_zip_file_input.py | 44 +
tests/outputs/__init__.py | 0
tests/outputs/configs/commandexecoutput.cfg | 12 +
tests/outputs/configs/ogr2ogrexecoutput.cfg | 81 ++
tests/outputs/configs/postgresdboutput.cfg | 17 +
tests/outputs/configs/splittermultioutput.cfg | 14 +
tests/outputs/configs/standardoutput.cfg | 11 +
tests/outputs/test_command_exec_output.py | 47 +
tests/outputs/test_ogr2ogr_exec_output.py | 201 +++
tests/outputs/test_postgres_db_output.py | 47 +
tests/outputs/test_split_outputs.py | 43 +
tests/outputs/test_standard_output.py | 43 +
tests/stetl_test_case.py | 59 +
tests/test_chain.py | 245 +---
tests/test_chain.pyc | Bin 994 -> 0 bytes
tests/test_config.py | 23 +
162 files changed, 6150 insertions(+), 1674 deletions(-)
diff --git a/.flake8 b/.flake8
new file mode 100644
index 0000000..cf0a673
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,5 @@
+[flake8]
+ignore =
+max-line-length = 160
+exclude = setup.py,.git,.cache,docs,docker,build,dist,examples,tests
+max-complexity = 38
diff --git a/CHANGES.txt b/CHANGES.txt
index db73902..69fa1a6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,11 +1,29 @@
Changes
=======
+v1.1.1 - november 7, 2017
+-------------------------
+
+Biggest change is that the Stetl repo moved to https://github.com/geopython/stetl/.
+
+See closed issues in Milestone 1.1.1: https://github.com/geopython/stetl/milestone/6?closed=1
+
+Highlights:
+
+- new Component Splitter to split (Filter/Output) data streams within a Chain
+- new Component Merger to combine (Input) data streams within a Chain
+- Splitter and Merger can be combined in single Chain
+- automatic Travis build
+- more Unit tests
+- flake8 for clean Python code
+- move to GDAL v2 (though v1 may still work)
+- new compact Docker Image based on debian:stretch-slim
+- bugfixes XML stream support
v1.0.9 - 17 june 2016
---------------------
-See https://github.com/justb4/stetl/issues?q=milestone%3A%22Version+1.0.9%22+is%3Aclosed
+See https://github.com/geopython/stetl/issues?q=milestone%3A%22Version+1.0.9%22+is%3Aclosed
Highlights:
@@ -16,7 +34,7 @@ Highlights:
v1.0.8 - 2 july 2015
--------------------
-See https://github.com/justb4/stetl/issues?q=milestone%3A%22Version+1.0.8%22
+See https://github.com/geopython/stetl/issues?q=milestone%3A%22Version+1.0.8%22
- generic OgrOutput component
- Apache Log File Input
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..b899812
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,147 @@
+# Contributing to this project
+
+Please take a moment to review this document in order to make the contribution
+process easy and effective for everyone involved.
+
+Following these guidelines helps to communicate that you respect the time of
+the developers managing and developing this open source project. In return,
+they should reciprocate that respect in addressing your issue or assessing
+patches and features.
+
+
+## Using the issue tracker
+
+The issue tracker is the preferred channel for [bug reports](#bugs),
+[features requests](#features) and [submitting pull
+requests](#pull-requests), but please respect the following restrictions:
+
+* Please **do not** use the issue tracker for personal support requests (use
+ [Stack Overflow](http://stackoverflow.com) or IRC).
+
+* Please **do not** derail or troll issues. Keep the discussion on topic and
+ respect the opinions of others.
+
+
+<a name="bugs"></a>
+## Bug reports
+
+A bug is a _demonstrable problem_ that is caused by the code in the repository.
+Good bug reports are extremely helpful - thank you!
+
+Guidelines for bug reports:
+
+1. **Use the GitHub issue search** — check if the issue has already been
+ reported.
+
+2. **Check if the issue has been fixed** — try to reproduce it using the
+ latest `master` or development branch in the repository.
+
+3. **Isolate the problem** — create a [reduced test
+ case](http://css-tricks.com/reduced-test-cases/) and a live example.
+
+A good bug report shouldn't leave others needing to chase you up for more
+information. Please try to be as detailed as possible in your report. What is
+your environment? What steps will reproduce the issue? What browser(s) and OS
+experience the problem? What would you expect to be the outcome? All these
+details will help people to fix any potential bugs.
+
+Example:
+
+> Short and descriptive example bug report title
+>
+> A summary of the issue and the browser/OS environment in which it occurs. If
+> suitable, include the steps required to reproduce the bug.
+>
+> 1. This is the first step
+> 2. This is the second step
+> 3. Further steps, etc.
+>
+> `<url>` - a link to the reduced test case
+>
+> Any other information you want to share that is relevant to the issue being
+> reported. This might include the lines of code that you have identified as
+> causing the bug, and potential solutions (and your opinions on their
+> merits).
+
+
+<a name="features"></a>
+## Feature requests
+
+Feature requests are welcome. But take a moment to find out whether your idea
+fits with the scope and aims of the project. It's up to *you* to make a strong
+case to convince the project's developers of the merits of this feature. Please
+provide as much detail and context as possible.
+
+
+<a name="pull-requests"></a>
+## Pull requests
+
+Good pull requests - patches, improvements, new features - are a fantastic
+help. They should remain focused in scope and avoid containing unrelated
+commits.
+
+**Please ask first** before embarking on any significant pull request (e.g.
+implementing features, refactoring code, porting to a different language),
+otherwise you risk spending a lot of time working on something that the
+project's developers might not want to merge into the project.
+
+Please adhere to the coding conventions used throughout a project (indentation,
+accurate comments, etc.) and any other requirements (such as test coverage).
+
+Follow this process if you'd like your work considered for inclusion in the
+project:
+
+1. [Fork](http://help.github.com/fork-a-repo/) the project, clone your fork,
+ and configure the remotes:
+
+ ```bash
+ # Clone your fork of the repo into the current directory
+ git clone https://github.com/<your-username>/<repo-name>
+ # Navigate to the newly cloned directory
+ cd <repo-name>
+ # Assign the original repo to a remote called "upstream"
+ git remote add upstream https://github.com/<upstream-owner>/<repo-name>
+ ```
+
+2. If you cloned a while ago, get the latest changes from upstream:
+
+ ```bash
+ git checkout <dev-branch>
+ git pull upstream <dev-branch>
+ ```
+
+3. Create a new topic branch (off the main project development branch) to
+ contain your feature, change, or fix:
+
+ ```bash
+ git checkout -b <topic-branch-name>
+ ```
+
+4. Commit your changes in logical chunks. Please adhere to these [git commit
+ message guidelines](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)
+ or your code is unlikely be merged into the main project. Use Git's
+ [interactive rebase](https://help.github.com/articles/interactive-rebase)
+ feature to tidy up your commits before making them public.
+
+5. Locally merge (or rebase) the upstream development branch into your topic branch:
+
+ ```bash
+ git pull [--rebase] upstream <dev-branch>
+ ```
+
+6. Push your topic branch up to your fork:
+
+ ```bash
+ git push origin <topic-branch-name>
+ ```
+
+7. [Open a Pull Request](https://help.github.com/articles/using-pull-requests/)
+ with a clear title and description.
+
+**IMPORTANT**: By submitting a patch, you agree to allow the project owner to
+license your work under the same license as that used by the project.
+
+## Thanks
+
+This doc copied and adapted from original at:
+https://github.com/necolas/issue-guidelines/blob/master/CONTRIBUTING.md
\ No newline at end of file
diff --git a/CREDITS.txt b/CREDITS.txt
index 48ca7d9..ba9274e 100644
--- a/CREDITS.txt
+++ b/CREDITS.txt
@@ -1,15 +1,21 @@
Credits
=======
-Stetl is written by:
+Stetl is developed by:
-* Just van den Broecke (http://www.justobjects.nl)
+* Just van den Broecke (initiator, http://www.justobjects.nl)
+* Frank Steggink
+* Thijs Brentjens
+* and more, see contributors: https://github.com/geopython/stetl/graphs/contributors
+
+Bas Couwenberg is providing Debian/Ubuntu packaging.
This project would not be possible without the great work of Frank Warmerdam and other
GDAL/OGR developers (http://gdal.org).
-Plus the people that brought Python, PostGIS (like Paul Ramsey), lxml and the libs
+Plus the people that brought Python, PostGIS (Paul Ramsey et al.), Jinja2, lxml and the libs
like GEOS, Proj, libxml2 and libxslt.
We are mainly standing on the shoulders of these giants.
+Thanks to Tom Kralidis for helping out to move from personal repo to https://github.com/geopython organization.
diff --git a/PKG-INFO b/PKG-INFO
index c9d6ccd..f54029b 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,56 +1,70 @@
Metadata-Version: 1.1
Name: Stetl
-Version: 1.0.9
+Version: 1.1
Summary: Stetl provides transformation for spatial data
-Home-page: http://github.com/justb4/stetl
+Home-page: http://github.com/geopython/stetl
Author: Just van den Broecke
Author-email: justb4 at gmail.com
License: GNU GPL v3
Description: # Stetl - Streaming ETL
- Stetl, streaming ETL, pronounced "staedl", is a lightweight ETL-framework for the conversion of rich (as GML)
- geospatial data conversion. Stetl is Open Source (GNU GPL v3).
+ Stetl, streaming ETL, pronounced "staedl", is a lightweight ETL-framework for geospatial data conversion.
- ## Documentation
+ [](https://travis-ci.org/geopython/stetl)
+ [](http://stetl.readthedocs.org/en/latest)
+ [](https://gitter.im/justb4/stetl)
- The main website and documentation can be found on http://stetl.org (or http://stetl.readthedocs.org).
- Read a 5-minute introduction here: http://www.slideshare.net/justb4/5-minute-intro-to-setl and a longer presentation
- here: http://www.slideshare.net/justb4/stetl-foss4g20131024v1.
- Stetl was presented at several occasions like the
- FOSS4G 2013 in Nottingham http://2013.foss4g.org.
+ Notice: the Stetl GH repo is now at the [GeoPython GH organization](https://github.com/geopython).
- ## Origins
- Stetl originated in the INSPIRE-FOSS project (www.inspire-foss.org)
- and was created by Just van den Broecke. Since Stetl evolved into a wider use like
- transforming Dutch GML-based datasets such as IMGEO/BGT (Large Scale Topography)
- and IMKAD/BRK (Cadastral Data) it has now a repo of its own.
+ # License
+
+ Stetl is released under a [GNU GPL v3](https://en.wikipedia.org/wiki/GNU_General_Public_License) license
+ (see [LICENSE.txt](LICENSE.txt)).
+
+ ## Documentation
- ## Design
+ The Stetl website and documentation can be found via http://stetl.org.
+ For a quick overview read the [5-minute Stetl-introduction](http://www.slideshare.net/justb4/5-minute-intro-to-setl),
+ or a [more detailed presentation](http://www.slideshare.net/justb4/stetl-foss4g20131024v1).
+ Stetl was presented at several events like the
+ [FOSS4G 2013 in Nottingham](http://2013.foss4g.org) and [GeoPython 2016](http://www.geopython.net).
- Stetl basically glues together existing parsing and transformation tools like GDAL/OGR (ogr2ogr) and XSLT.
- By using native tools like libxml and libxslt (via Python lxml) Stetl is speed-optimized.
+ ## Concepts
- Stetl has a similar design as Spring (Java) and other modern frameworks based on IoC (Inversion of Control, http://en.wikipedia.org/wiki/Inversion_of_Control).
- A configuration file (in Python config format) specifies your chain of ETL steps.
- This chain is formed by a series of Python modules/objects and their parameters. These are
- symbolically specified in the config file. You just invoke etl.py the main program with a config file.
- The config file specifies the input modules (e.g. PostGIS), transformers (e.g. XSLT) and outputs (e.g. a GML file or even
- WFS-T a geospatial protocol to publish GML to a server).
+ Stetl basically glues together existing parsing and transformation tools like [GDAL/OGR](http://gdal.org), Jinja2 and
+ XSLT with custom Python code. By using native libraries like `libxml2` and `libxslt` (via Python `lxml`) Stetl is speed-optimized.
- Stetl has been proven to handle 10's of millions of objects without any memory issues.
+ A configuration file, in Python config `.ini` format, specifies a chained sequence of transformation
+ steps: typically an `Input` connected to one or more `Filters`, and finally to an `Output`.
+ At runtime, this sequence is instantiated and run as a linked series of Python objects. These objects are
+ symbolically specified (by their module/class name) and parameterized in the config file.
+ Via the `stetl -c <config file>` command, the transformation is executed.
+
+ Stetl has been proven to handle 10's of millions of GML objects without any memory issues.
This is achieved through a technique called "streaming and splitting".
- For example: using the OgrPostgisInput module an GML stream can be generated from the database.
- A component called the GmlSplitter can split this stream into managable chunks (like 20000 features)
+ For example: using the `OgrPostgisInput` module an GML stream can be generated from the database.
+ A component called the `GmlSplitter` can split this stream into manageable chunks (like 20000 features)
and feed this upstream into the ETL chain.
- ## Examples
+ ## Use Cases
+
+ Stetl has been found particularly useful for complex GML-related ETL-cases, like those found
+ within [EU INSPIRE](http://inspire.ec.europa.eu/) Data Harmonization and the transformation
+ of GML/XML-based National geo-datasets to for example PostGIS.
+
+ Most of the data conversions within the [Dutch NLExtract Project](https://github.com/nlextract/NLExtract) apply Stetl.
+
+ Stetl also proved to be very effective in [IoT-related transformations involving the SensorWeb/SOS](https://github.com/Geonovum/smartemission).
- Stetl has been found in particularly useful for INSPIRE-related transformations and other complex GML-related ETL.
+ ## Examples
- See examples under the examples dir.
+ Browse all examples under the [examples dir](examples).
+ Best is to start with the [basic examples](examples/basics)
- Another example in http://code.google.com/p/inspire-foss/source/browse/trunk/etl/NL.Kadaster/Addresses
- (Dutch Addresses (BAG) to INSPIRE Addresses)
+ ## Installation
+
+ Stetl can be installed via PyPi `pip install stetl` and recently as a [Stetl Docker image](https://hub.docker.com/r/justb4/stetl).
+ More on [installation in the documentation](http://www.stetl.org/en/latest/install.html).
## Contributing
@@ -61,11 +75,19 @@ Description: # Stetl - Streaming ETL
* [Feature requests](CONTRIBUTING.md#features)
* [Pull requests](CONTRIBUTING.md#pull-requests)
+ ## Origins
+
+ Stetl originated in the INSPIRE-FOSS project: [2009-2013 now archived](https://github.com/justb4/inspire-foss).
+ Since then Stetl evolved into a wider use like
+ transforming [Dutch GML-based Open Datasets](https://github.com/nlextract/NLExtract) such as IMGEO/BGT (Large Scale Topography)
+ and IMKAD/BRK (Cadastral Data).
+
## Finally
The word "stetl" is also an alternative writing for "shtetl":
http://en.wikipedia.org/wiki/Stetl : "...Material things were neither disdained nor
- extremely praised in the shtetl. Learning and education were the ultimate measures of worth in the eyes of the community,
+ extremely praised in the shtetl. Learning and education were the ultimate measures of
+ worth in the eyes of the community,
while money was secondary to status..."
@@ -74,11 +96,29 @@ Description: # Stetl - Streaming ETL
Changes
=======
+ v1.1.1 - november 7, 2017
+ -------------------------
+
+ Biggest change is that the Stetl repo moved to https://github.com/geopython/stetl/.
+
+ See closed issues in Milestone 1.1.1: https://github.com/geopython/stetl/milestone/6?closed=1
+
+ Highlights:
+
+ - new Component Splitter to split (Filter/Output) data streams within a Chain
+ - new Component Merger to combine (Input) data streams within a Chain
+ - Splitter and Merger can be combined in single Chain
+ - automatic Travis build
+ - more Unit tests
+ - flake8 for clean Python code
+ - move to GDAL v2 (though v1 may still work)
+ - new compact Docker Image based on debian:stretch-slim
+ - bugfixes XML stream support
v1.0.9 - 17 june 2016
---------------------
- See https://github.com/justb4/stetl/issues?q=milestone%3A%22Version+1.0.9%22+is%3Aclosed
+ See https://github.com/geopython/stetl/issues?q=milestone%3A%22Version+1.0.9%22+is%3Aclosed
Highlights:
@@ -89,7 +129,7 @@ Description: # Stetl - Streaming ETL
v1.0.8 - 2 july 2015
--------------------
- See https://github.com/justb4/stetl/issues?q=milestone%3A%22Version+1.0.8%22
+ See https://github.com/geopython/stetl/issues?q=milestone%3A%22Version+1.0.8%22
- generic OgrOutput component
- Apache Log File Input
@@ -147,18 +187,24 @@ Description: # Stetl - Streaming ETL
Credits
=======
- Stetl is written by:
+ Stetl is developed by:
+
+ * Just van den Broecke (initiator, http://www.justobjects.nl)
+ * Frank Steggink
+ * Thijs Brentjens
+ * and more, see contributors: https://github.com/geopython/stetl/graphs/contributors
- * Just van den Broecke (http://www.justobjects.nl)
+ Bas Couwenberg is providing Debian/Ubuntu packaging.
This project would not be possible without the great work of Frank Warmerdam and other
GDAL/OGR developers (http://gdal.org).
- Plus the people that brought Python, PostGIS (like Paul Ramsey), lxml and the libs
+ Plus the people that brought Python, PostGIS (Paul Ramsey et al.), Jinja2, lxml and the libs
like GEOS, Proj, libxml2 and libxslt.
We are mainly standing on the shoulders of these giants.
+ Thanks to Tom Kralidis for helping out to move from personal repo to https://github.com/geopython organization.
Keywords: etl xsl gdal gis vector feature data
Platform: UNKNOWN
diff --git a/README.md b/README.md
index e83e536..998ed67 100644
--- a/README.md
+++ b/README.md
@@ -1,48 +1,62 @@
# Stetl - Streaming ETL
-Stetl, streaming ETL, pronounced "staedl", is a lightweight ETL-framework for the conversion of rich (as GML)
-geospatial data conversion. Stetl is Open Source (GNU GPL v3).
+Stetl, streaming ETL, pronounced "staedl", is a lightweight ETL-framework for geospatial data conversion.
-## Documentation
+[](https://travis-ci.org/geopython/stetl)
+[](http://stetl.readthedocs.org/en/latest)
+[](https://gitter.im/justb4/stetl)
-The main website and documentation can be found on http://stetl.org (or http://stetl.readthedocs.org).
-Read a 5-minute introduction here: http://www.slideshare.net/justb4/5-minute-intro-to-setl and a longer presentation
-here: http://www.slideshare.net/justb4/stetl-foss4g20131024v1.
-Stetl was presented at several occasions like the
-FOSS4G 2013 in Nottingham http://2013.foss4g.org.
+Notice: the Stetl GH repo is now at the [GeoPython GH organization](https://github.com/geopython).
-## Origins
-Stetl originated in the INSPIRE-FOSS project (www.inspire-foss.org)
-and was created by Just van den Broecke. Since Stetl evolved into a wider use like
-transforming Dutch GML-based datasets such as IMGEO/BGT (Large Scale Topography)
-and IMKAD/BRK (Cadastral Data) it has now a repo of its own.
+# License
-## Design
+Stetl is released under a [GNU GPL v3](https://en.wikipedia.org/wiki/GNU_General_Public_License) license
+(see [LICENSE.txt](LICENSE.txt)).
+
+## Documentation
-Stetl basically glues together existing parsing and transformation tools like GDAL/OGR (ogr2ogr) and XSLT.
-By using native tools like libxml and libxslt (via Python lxml) Stetl is speed-optimized.
+The Stetl website and documentation can be found via http://stetl.org.
+For a quick overview read the [5-minute Stetl-introduction](http://www.slideshare.net/justb4/5-minute-intro-to-setl),
+or a [more detailed presentation](http://www.slideshare.net/justb4/stetl-foss4g20131024v1).
+Stetl was presented at several events like the
+[FOSS4G 2013 in Nottingham](http://2013.foss4g.org) and [GeoPython 2016](http://www.geopython.net).
-Stetl has a similar design as Spring (Java) and other modern frameworks based on IoC (Inversion of Control, http://en.wikipedia.org/wiki/Inversion_of_Control).
-A configuration file (in Python config format) specifies your chain of ETL steps.
-This chain is formed by a series of Python modules/objects and their parameters. These are
-symbolically specified in the config file. You just invoke etl.py the main program with a config file.
-The config file specifies the input modules (e.g. PostGIS), transformers (e.g. XSLT) and outputs (e.g. a GML file or even
-WFS-T a geospatial protocol to publish GML to a server).
+## Concepts
-Stetl has been proven to handle 10's of millions of objects without any memory issues.
+Stetl basically glues together existing parsing and transformation tools like [GDAL/OGR](http://gdal.org), Jinja2 and
+XSLT with custom Python code. By using native libraries like `libxml2` and `libxslt` (via Python `lxml`) Stetl is speed-optimized.
+
+A configuration file, in Python config `.ini` format, specifies a chained sequence of transformation
+steps: typically an `Input` connected to one or more `Filters`, and finally to an `Output`.
+At runtime, this sequence is instantiated and run as a linked series of Python objects. These objects are
+symbolically specified (by their module/class name) and parameterized in the config file.
+Via the `stetl -c <config file>` command, the transformation is executed.
+
+Stetl has been proven to handle 10's of millions of GML objects without any memory issues.
This is achieved through a technique called "streaming and splitting".
-For example: using the OgrPostgisInput module an GML stream can be generated from the database.
-A component called the GmlSplitter can split this stream into managable chunks (like 20000 features)
+For example: using the `OgrPostgisInput` module an GML stream can be generated from the database.
+A component called the `GmlSplitter` can split this stream into manageable chunks (like 20000 features)
and feed this upstream into the ETL chain.
-## Examples
+## Use Cases
-Stetl has been found in particularly useful for INSPIRE-related transformations and other complex GML-related ETL.
+Stetl has been found particularly useful for complex GML-related ETL-cases, like those found
+within [EU INSPIRE](http://inspire.ec.europa.eu/) Data Harmonization and the transformation
+of GML/XML-based National geo-datasets to for example PostGIS.
-See examples under the examples dir.
+Most of the data conversions within the [Dutch NLExtract Project](https://github.com/nlextract/NLExtract) apply Stetl.
-Another example in http://code.google.com/p/inspire-foss/source/browse/trunk/etl/NL.Kadaster/Addresses
-(Dutch Addresses (BAG) to INSPIRE Addresses)
+Stetl also proved to be very effective in [IoT-related transformations involving the SensorWeb/SOS](https://github.com/Geonovum/smartemission).
+
+## Examples
+
+Browse all examples under the [examples dir](examples).
+Best is to start with the [basic examples](examples/basics)
+
+## Installation
+
+Stetl can be installed via PyPi `pip install stetl` and recently as a [Stetl Docker image](https://hub.docker.com/r/justb4/stetl).
+More on [installation in the documentation](http://www.stetl.org/en/latest/install.html).
## Contributing
@@ -53,11 +67,19 @@ review the [guidelines for contributing](CONTRIBUTING.md).
* [Feature requests](CONTRIBUTING.md#features)
* [Pull requests](CONTRIBUTING.md#pull-requests)
+## Origins
+
+Stetl originated in the INSPIRE-FOSS project: [2009-2013 now archived](https://github.com/justb4/inspire-foss).
+Since then Stetl evolved into a wider use like
+transforming [Dutch GML-based Open Datasets](https://github.com/nlextract/NLExtract) such as IMGEO/BGT (Large Scale Topography)
+and IMKAD/BRK (Cadastral Data).
+
## Finally
The word "stetl" is also an alternative writing for "shtetl":
http://en.wikipedia.org/wiki/Stetl : "...Material things were neither disdained nor
-extremely praised in the shtetl. Learning and education were the ultimate measures of worth in the eyes of the community,
+extremely praised in the shtetl. Learning and education were the ultimate measures of
+worth in the eyes of the community,
while money was secondary to status..."
diff --git a/VERSION.txt b/VERSION.txt
index e5a4a5e..b123147 100644
--- a/VERSION.txt
+++ b/VERSION.txt
@@ -1 +1 @@
-1.0.9
\ No newline at end of file
+1.1
\ No newline at end of file
diff --git a/docker/Dockerfile b/docker/Dockerfile
deleted file mode 100644
index fbb2b6b..0000000
--- a/docker/Dockerfile
+++ /dev/null
@@ -1,44 +0,0 @@
-FROM ubuntu:14.04
-# FROM python:2.7 would be preferred but trickier
-# TODO: use a more lightweight Linux distro than Ubuntu
-
-MAINTAINER just at justobjects.nl
-
-# Configure timezone and locale
-ARG IMAGE_TIMEZONE="Europe/Amsterdam"
-# set time right adn configure timezone and locale
-RUN echo "$IMAGE_TIMEZONE" > /etc/timezone && \
- dpkg-reconfigure -f noninteractive tzdata
-
-# Optional: Use local cached debs from host (saves your bandwidth!)
-# Change ip below to that of your apt-cacher-ng host
-# Or comment this line out if you do not with to use caching
-# ADD 71-apt-cacher-ng /etc/apt/apt.conf.d/71-apt-cacher-ng
-
-RUN export DEBIAN_FRONTEND=noninteractive TERM=linux && \
- echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
-
-RUN apt-get update && \
- apt-get install -y software-properties-common
-
-# Use UbuntuGIS for latest GDAL, Proj.4, GEOS etc
-RUN apt-add-repository ppa:ubuntugis/ubuntugis-unstable
-RUN apt-get update && \
- apt-get install -y git \
- libproj0 \
- libgeos-dev \
- python-lxml \
- python-gdal \
- libgdal-dev \
- build-essential \
- python-dev \
- python-setuptools
-
-# Get latest Stetl from git
-RUN git clone https://github.com/justb4/stetl /opt/stetl
-
-# Use Stetl installer
-RUN cd /opt/stetl; python setup.py install
-
-# Allow docker run
-ENTRYPOINT ["/usr/local/bin/stetl"]
diff --git a/docker/README.md b/docker/README.md
deleted file mode 100644
index 3ac8aa9..0000000
--- a/docker/README.md
+++ /dev/null
@@ -1,38 +0,0 @@
-# Stetl Docker
-
-Docker image to run ETL tool Stetl. See http://stetl.org.
-Public Docker images are available at: https://hub.docker.com/r/justb4/stetl/.
-
-## Status
-
-First version on June 17, 2016. Improvements welcome.
-
-## Building
-
-Use ./build.sh or
-
- docker build -t justb4/stetl:latest .
-
-ARGs: optional --build-arg IMAGE_TIMEZONE="Europe/Amsterdam"
-
-## Running
-
-Mapping volumes (``docker -v``) can be handly to maintain all config and data on the Host, outside the Docker container.
-Note that full paths are required.
-
-Example, running a basic Stetl example:
-
- cd test/1_copystd
- docker run -v `pwd`:`pwd` -w `pwd` -t -i justb4/stetl:latest -c etl.cfg
-
- or
- ./etl.sh
-
-Many Stetl configs require a connection to PostGIS. This can be effected with a linked container: ``--link postgis``, or
-better using Docker networking.
-
-## More
-
-See https://github.com/Geonovum/smartemission/tree/master/etl for a real-world example using a single Stetl Docker image
-for multiple Stetl configurations and linking with a PostGIS Docker container. For example:
-https://github.com/Geonovum/smartemission/blob/master/etl/last.sh
diff --git a/docker/build.sh b/docker/build.sh
deleted file mode 100755
index 1a802e6..0000000
--- a/docker/build.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-#
-# Build Stetl Docker image
-
-# Optional: build with custom/your timezone and forced via --no-cache
-# sudo docker build -t --no-cache --build-arg IMAGE_TIMEZONE="Europe/Amsterdam" justb4/stetl:latest .
-
-sudo docker build -t justb4/stetl:latest .
-
diff --git a/docker/test/1_copystd/etl.cfg b/docker/test/1_copystd/etl.cfg
deleted file mode 100644
index fae6419..0000000
--- a/docker/test/1_copystd/etl.cfg
+++ /dev/null
@@ -1,11 +0,0 @@
-# Trivial example of an ETL Chain that just copies a file to standard output.
-
-[etl]
-chains = input_xml_file|output_std
-
-[input_xml_file]
-class = inputs.fileinput.XmlFileInput
-file_path = input/cities.xml
-
-[output_std]
-class = outputs.standardoutput.StandardXmlOutput
diff --git a/docker/test/1_copystd/etl.sh b/docker/test/1_copystd/etl.sh
deleted file mode 100755
index e3f1402..0000000
--- a/docker/test/1_copystd/etl.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/sh
-#
-# Test Stetl Docker image.
-# Simple ETL for copying a file to standard output.
-#
-#
-sudo docker run -v `pwd`:`pwd` -w `pwd` -t -i justb4/stetl:latest -c etl.cfg
-
-
diff --git a/docs/background.rst b/docs/background.rst
index f8a9555..aa2e7f0 100644
--- a/docs/background.rst
+++ b/docs/background.rst
@@ -115,6 +115,6 @@ open geo-datasets. Stetl development is now (april 2013) in an initial phase and
GitHub. The current version is workable but we hope to present a v1.0 at FOSS4G with more
documentation and as a standard Python Package via PyPi. The main link is:
http://stetl.org (now links to GitHub).
-To get started find some basic examples here: https://github.com/justb4/stetl/tree/master/examples/basics.
+To get started find some basic examples here: https://github.com/geopython/stetl/tree/master/examples/basics.
diff --git a/docs/cases.rst b/docs/cases.rst
index 1f6b77e..d597fb7 100644
--- a/docs/cases.rst
+++ b/docs/cases.rst
@@ -14,31 +14,31 @@ Dutch Open Geo-Datasets, in particular the country wide
Buildings and Addresses (BAG). These datasets are provided as XML/GML. The ETL mostly provides
a transformation to PostGIS. For all Key Registries, except for the BAG, Stetl is used, basically
as-is, without extra (Python) programming. See also the NLExtract GitHub:
-https://github.com/opengeogroep/NLExtract
+https://github.com/nlextract/NLExtract
Topography (BRT/Top10NL)
~~~~~~~~~~~~~~~~~~~~~~~~
-See https://github.com/opengeogroep/NLExtract/tree/master/top10nl/etl and the Stetl conf at
-https://github.com/opengeogroep/NLExtract/tree/master/top10nl/etl/conf/
+See https://github.com/nlextract/NLExtract/tree/master/top10nl/etl and the Stetl conf at
+https://github.com/nlextract/NLExtract/tree/master/top10nl/etl/conf/
Detailed Topography (BGT)
~~~~~~~~~~~~~~~~~~~~~~~~~
-See https://github.com/opengeogroep/NLExtract/tree/master/bgt and the Stetl conf at
-https://github.com/opengeogroep/NLExtract/blob/master/bgt/etl/conf/
+See https://github.com/nlextract/NLExtract/tree/master/bgt and the Stetl conf at
+https://github.com/nlextract/NLExtract/blob/master/bgt/etl/conf/
Cadastral Parcels (BRK)
~~~~~~~~~~~~~~~~~~~~~~~
-See https://github.com/opengeogroep/NLExtract/tree/master/brk/etl
-and the Stetl conf at https://github.com/opengeogroep/NLExtract/tree/master/brk/etl/conf
+See https://github.com/nlextract/NLExtract/tree/master/brk/etl
+and the Stetl conf at https://github.com/nlextract/NLExtract/tree/master/brk/etl/conf
INSPIRE
-------
These were the origins of Stetl. This project was sponsored by Kadaster.
-See http://inspire-foss.org. The ETL involved the transformation of Dutch Key Registries (see above)
+See https://github.com/justb4/inspire-foss. The ETL involved the transformation of Dutch Key Registries (see above)
to harmonized INSPIRE GML according to the Annexes.
Addresses
@@ -53,7 +53,7 @@ Ordnance Survey
A successful Proof-of-Concept to convert Ordnance Survey Mastermap GML to PostGIS:
-https://github.com/justb4/stetl/tree/master/examples/ordnancesurvey
+https://github.com/geopython/stetl/tree/master/examples/ordnancesurvey
SOSPilot
--------
@@ -82,7 +82,9 @@ Smart Emission
--------------
Sensors for air quality, meteo and audio at civilians. Project by University of Nijmegen/Gemeente Nijmegen with participation
-by Geonovum. Stetl is used to transform a low-level sensor API to PostGIS and later on WMS/WFS/SOS.
+by Geonovum. Stetl is used to transform a low-level sensor API to PostGIS and later on WMS/WFS/SOS and the SensorThings API.
+Also InfluxDB output is developed here.
+
This is also an example how to use a Stetl Docker image:
See https://github.com/Geonovum/smartemission/tree/master/etl
diff --git a/docs/code.rst b/docs/code.rst
index 4b3195c..ae03164 100644
--- a/docs/code.rst
+++ b/docs/code.rst
@@ -34,7 +34,8 @@ from an :class:`stetl.input.Input` via zero or more :class:`stetl.filter.Filter`
As a trivial example: an :class:`stetl.input.Input` could be an XML file, a :class:`stetl.filter.Filter` could represent
an XSLT file and an :class:`stetl.output.Output` a PostGIS database. This is effected by specialized classes in
-the subpackages inputs, filters, and outputs.
+the subpackages inputs, filters, and outputs. New in 1.1.0: :class:`stetl.Splitter` to split data to multiple Outputs
+and :class:`stetl.Merger` to combine multiple `Inputs`.
.. automodule:: stetl.factory
:members:
@@ -60,6 +61,13 @@ the subpackages inputs, filters, and outputs.
:members:
:show-inheritance:
+.. automodule:: stetl.splitter
+ :members:
+ :show-inheritance:
+
+.. automodule:: stetl.merger
+ :members:
+ :show-inheritance:
Components: Inputs
------------------
@@ -95,6 +103,10 @@ Components: Filters
:members:
:show-inheritance:
+.. automodule:: stetl.filters.xmlelementreader
+ :members:
+ :show-inheritance:
+
.. automodule:: stetl.filters.xmlvalidator
:members:
:show-inheritance:
@@ -138,6 +150,10 @@ Components: Outputs
:members:
:show-inheritance:
+.. automodule:: stetl.outputs.execoutput
+ :members:
+ :show-inheritance:
+
.. automodule:: stetl.outputs.dboutput
:members:
:show-inheritance:
diff --git a/docs/conf.py b/docs/conf.py
index f580701..41505ba 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -52,9 +52,9 @@ copyright = u'2013+, Just van den Broecke'
# built documents.
#
# The short X.Y version.
-version = '1.0.9'
+version = '1.1'
# The full version, including alpha/beta/rc tags.
-release = '1.0.9'
+release = '1.1'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/docs/contact.rst b/docs/contact.rst
index 62ce585..2fba668 100644
--- a/docs/contact.rst
+++ b/docs/contact.rst
@@ -5,7 +5,7 @@ Contact
The website `stetl.org <http://stetl.org>`_ is the main entry point for all of Stetl.
-All development is done via GitHub: see https://github.com/justb4/stetl.
+All development is done via GitHub: see https://github.com/geopython/stetl.
Contact the main author Just van den Broecke via email at just at justobjects.nl.
diff --git a/docs/foss4g13-abstract.txt b/docs/foss4g13-abstract.txt
index ec3aecc..7e0caf2 100644
--- a/docs/foss4g13-abstract.txt
+++ b/docs/foss4g13-abstract.txt
@@ -73,7 +73,7 @@ open geo-datasets. Stetl development is now (april 2013) in an initial phase and
GitHub. The current version is workable but we hope to present a v1.0 at FOSS4G with more
documentation and as a standard Python Package via PyPi. The main link is:
http://stetl.org (now links to GitHub).
-To get started find some basic examples here: https://github.com/justb4/stetl/tree/master/examples/basics.
+To get started find some basic examples here: https://github.com/geopython/stetl/tree/master/examples/basics.
This presentation will gently introduce the above "GML challenge" and explain the basics of Stetl
starting with some simple examples, building up to more complex cases like INSPIRE
diff --git a/docs/index.rst b/docs/index.rst
index 32036c0..f976765 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -12,9 +12,10 @@ XSLT. Stetl processing is driven from a configuration (.ini) file.
Stetl is written in Python and in particular suited for processing GML.
This is the documentation of the Stetl toolkit.
-The code is on GitHub: https://github.com/justb4/stetl.
+The code is on GitHub: https://github.com/geopython/stetl. Since July 2016 the project is a proud
+member of the `GeoPython GitHub organization <https://github.com/geopython>`_.
-See an `introductory Stetl presentation at Slideshare <http://www.slideshare.net/justb4/stetl-foss4g20131024v1>`_.
+See an `introductory Stetl presentation on Slideshare <http://www.slideshare.net/justb4/geospatial-etl-with-stetl-geopython-2016>`_.
This is document version |release| generated on |today|.
diff --git a/docs/install.rst b/docs/install.rst
index 1116253..69656d1 100644
--- a/docs/install.rst
+++ b/docs/install.rst
@@ -3,6 +3,8 @@
Installation
============
+Stetl currently only runs with Python 2 (2.7+). `Work is underway <https://github.com/geopython/stetl/pull/27>`_ for Python3 support.
+
Easiest is to first install the Stetl-dependencies (see below) and then
install and maintain Stetl on your system as a Python package (pip is preferred). ::
@@ -11,81 +13,107 @@ install and maintain Stetl on your system as a Python package (pip is preferred)
easy_install stetl
Alternatively you can download Stetl from
-Github: by cloning (preferred) or downloading: https://github.com/justb4/stetl/archive/master.zip
+Github: by cloning (preferred) or downloading: https://github.com/geopython/stetl/archive/master.zip
and then install locally ::
(sudo) python setup.py install
Try the examples first. This should work on Linuxes and Mac OSX.
+
Windows installation may be more involved depending on your local Python setup. Platform-specific
installations below.
You may also want to download the complete .tar.gz distro from PyPi:
https://pypi.python.org/pypi/Stetl . This includes the examples and tests.
-**Docker:** Since version 1.0.9 Stetl also can be installed and run via `Docker <http://docker.com>`_. See
+**Docker**
+
+Since version 1.0.9 Stetl also can be installed and run via `Docker <http://docker.com>`_. See
:ref:`install_docker` below.
+**Debian/Ubuntu**
+
+Thanks to Bas Couwenberg, work is performed to provide Stetl as Debian packages on both Debian and Ubuntu, see details:
+https://packages.debian.org/search?keywords=stetl (Debian) and
+https://launchpad.net/ubuntu/+source/python-stetl (Ubuntu, Xenial and later).
+Stetl is split into 2 packages ``python-stetl``, the Python framework and ``stetl`` the command line utility.
+NB the versions of these packages may be older than when installing Stetl via `pip` from PyPi
+or directly from GitHub. Always check this first.
+
Dependencies
------------
Stetl depends on the following Python packages:
-* GDAL bindings for Python
+* GDAL (v2+) bindings for Python
* psycopg2 (PostgreSQL client)
* lxml
+* Jinja2 templating
-``GDAL`` requires the native GDAL/OGR tools to be installed.
+``GDAL`` Python binding requires the native GDAL/OGR libs and tools (version 2+) to be installed.
``lxml`` http://lxml.de/installation.html requires the native (C) libraries:
* libxslt (required by lxml)
-* libxml2 (required by lxml)
+* libxml2 with Python bindings (required by lxml)
When using the ``Jinja2`` templating filter, ``Jinja2TemplatingFilter``, see http://jinja.pocoo.org:
* Python Jinja2 package
+Platform-specific guidelines for dependencies follow next.
+
Linux
------
+~~~~~
+
+For Debian-based distro's like Ubuntu and Debian itself, most packages should be able to be installed via apt-get.
+
+Tip: to get latest versions of GDAL and other Open Source geospatial software, best is
+to add the `UbuntuGIS Repository <https://wiki.ubuntu.com/UbuntuGIS>`_.
+Below a setup that works in Ubuntu 16.04 Xenial using Debian/Ubuntu packages. In some cases you may
+choose to install the same packages via `pip` to have more recent versions like for `lxml`.
-Most packages should be able to be installed by apt-get or Python ``pip`` or ``easy_install``.
+- Python dependencies: ::
+ apt-get install python-setuptools
+ apt-get install python-dev
+ apt-get install python-pip
+ pip install --upgrade pip
+
+- ``libxml2/libxslt`` libs are usually already installed. Together with Python ``lxml``, the total install for ``lxml`` is: ::
-- Optional: Python package dependencies
- ::
+ apt-get install python-libxml2
+ apt-get install python-libxslt1
+ apt-get install libxml2-dev libxslt1-dev lib32z1-dev
+ apt-get install python-lxml
- sudo apt-get install python-setuptools
- sudo apt-get install python-dev
- sudo apt-get install libpq-dev
+- ``GDAL`` (http://gdal.org) version 2+ with Python bindings: ::
-- ``libxml2/libxslt`` libs are usually already installed. Together with Python ``lxml``
- the total install for ``lxml`` is.
- ::
+ # Add UbuntuGIS repo to get latest GDAL, at least v2 on Ubuntu 16.04, Xenial.
+ add-apt-repository ppa:ubuntugis/ubuntugis-unstable
+ apt-get update
+ apt-get install gdal-bin
+ gdalinfo --version
+ # should show something like: GDAL 2.2.1, released 2017/06/23
- apt-get of yum install libxml2
- apt-get of yum install libxslt1.1
- apt-get of yum install python-lxml
+ apt-get install python-gdal
-- GDAL (http://gdal.org) with Python bindings
- ::
+- the PostgreSQL client library for Python ``psycopg2``: ::
- apt-get of yum install gdal-bin
- apt-get of yum install python-gdal
+ apt-get install python-psycopg2
-- the PostgreSQL client library for Python ``psycopg2``
- ::
+- for ``Jinja2``: ::
- sudo easy_install psycopg2
+ apt-get install python-jinja2
-- Python package ``argparse`` if you have Python < 2.7
- ::
- sudo easy_install argparse
+Mac OSX
+~~~~~~~
+Dependencies can best be installed via `Homebrew <http://brew.sh/>`_.
Windows
--------
+~~~~~~~
Best is to install GDAL and python using the OSGeo4W Installer from http://trac.osgeo.org/osgeo4w.
@@ -129,19 +157,47 @@ Try running the examples when running with a downloaded distro. ::
Look for any error messages in your output.
+Run Unit Tests
+--------------
+
+You can run unit tests to completely verify your installation. First install some extra packages: ::
+
+ pip install -r requirements-dev.txt
+
+Then run the tests using `nose2`. ::
+
+ nose2
+
.. _install_docker:
Install with Docker
-------------------
-One of the cleanest ways to use Stetl is via `Docker <http://docker.com>`_. Your environment needs to be
+The fastest way to use Stetl is via `Docker <http://docker.com>`_. The Stetl Docker Image is lightweight,
+compressed just over 100MB, based on a Debian "slim" Docker Image.
+
+Your environment needs to be
setup to use Docker and probably you want to use some tooling like `Vagrant <https://www.vagrantup.com/>`_. The author uses
a combination of VirtualBox with Ubuntu and Vagrant on Mac OSX to run Docker, but this
is a bit out of scope here.
Assuming you have a working Docker environment, there are two ways to install Stetl with Docker:
-* build a Docker image yourself using the Dockerfile in https://github.com/justb4/stetl/tree/master/docker
-* use a prebuilt public Stetl Docker image: https://hub.docker.com/r/justb4/stetl
+* build a Docker image yourself using the Dockerfile in https://github.com/geopython/stetl/blob/master/Dockerfile
+* use a prebuilt public Stetl Docker image from Docker Hub: https://hub.docker.com/r/geopython/stetl
+
+When rebuilding you can add build arguments for your environment, defaults: ::
+
+ ARG TIMEZONE="Europe/Amsterdam"
+ ARG LOCALE="en_US.UTF-8"
+ ARG ADD_PYTHON_DEB_PACKAGES=""
+ ARG ADD_PYTHON_PIP_PACKAGES=""
+
+For example building with extra Python packages: ::
+
+ docker build --build-arg ADD_PYTHON_DEB_PACKAGES="python-requests python-tz" -t geopython/stetl:latest .
+ docker build --build-arg ADD_PYTHON_PIP_PACKAGES="scikit-learn==0.18 influxdb" -t geopython/stetl:latest .
+
+Or you may extend the Stetl Dockerfile with your own Dockerfile.
-For running Stetl using Docker see :ref:`run_docker`.
\ No newline at end of file
+For running Stetl using Docker see :ref:`run_docker`.
diff --git a/docs/intro.rst b/docs/intro.rst
index ffa926c..d4bc89f 100644
--- a/docs/intro.rst
+++ b/docs/intro.rst
@@ -7,7 +7,7 @@ Stetl, streaming ETL, pronounced "staedl", is a lightweight ETL-framework for th
geospatial data. Stetl is Open Source (GNU GPL v3).
Read a 5-minute introduction here: http://www.slideshare.net/justb4/5-minute-intro-to-setl and a longer presentation
-here: http://www.slideshare.net/justb4/stetl-foss4g20131024v1. Plus a presentation of
+here: http://www.slideshare.net/justb4/geospatial-etl-with-stetl-geopython-2016. Plus a presentation of
Stetl for use in INSPIRE transformation: http://www.slideshare.net/justb4/2-stetlinspiretransformv1 with even
a video recording: https://www.youtube.com/watch?v=vjdpYBm4AaM
@@ -15,7 +15,7 @@ Stetl originated in the `INSPIRE-FOSS project <http//www.inspire-foss.org>`_
and was originally created by `Just van den Broecke <http://nl.linkedin.com/in/justb4>`_.
Subsequently, Stetl evolved into a wider use
transforming Dutch GML-based datasets such as Top10NL, IMGEO/BGT (Large Scale Topography)
-and IMKAD/BRK (Kadastral Data). Therefore Stetl now has a repository of its own at `GitHub <https://github.com/justb4/stetl>`_.
+and IMKAD/BRK (Kadastral Data). Therefore Stetl now has a repository of its own at `GitHub <https://github.com/geopython/stetl>`_.
Stetl basically glues together existing parsing and transformation tools like `GDAL/OGR (ogr2ogr) <http://gdal.org>`_ and
`XSLT <http://en.wikipedia.org/wiki/XSLT>`_. By using native tools like `libxml2` and `libxslt` (via `Python lxml <http://lxml.de>`_)
diff --git a/docs/links.rst b/docs/links.rst
index 95537e6..817935e 100644
--- a/docs/links.rst
+++ b/docs/links.rst
@@ -5,11 +5,41 @@ Links
Below links relevant to Stetl.
-.. [GDALOGR] GDAL/OGR, http://gdal.org
-.. [lxml] lxml, http://lxml.de
-.. [Deegree] Deegree, http://www.deegree.org
-.. [INSPIRE] INSPIRE, http://inspire.ec.europa.eu/
-.. [INSPIRE-FOSS] INSPIRE FOSS project, http://inspire-foss.org
-.. [PostGIS] PostGIS/PostgreSQL, http://postgis.refractions.net
-.. [Slideshare] Several presentations on Stetl, http://www.slideshare.net/justb4
+Presentations
+=============
+Below several presentations on Stetl given at various events. The most recent/relevant at the top.
+
+* `GeoPython2016 - Spatial ETL with Stetl <http://www.slideshare.net/justb4/geospatial-etl-with-stetl-geopython-2016>`_
+* `5-minute intro Stetl <http://www.slideshare.net/justb4/5-minute-intro-to-setl>`_
+* `FOSS4G Nottingham 2013 <http://www.slideshare.net/justb4/stetl-foss4g20131024v1>`_
+* `Eurogeographics 2013 - INSPIRE Transform with Stetl: <http://www.slideshare.net/justb4/2-stetlinspiretransformv1>`_
+* `Video recording of Eurogeographics 2013 Stetl pres: <https://www.youtube.com/watch?v=vjdpYBm4AaM>`_
+* `Several presentations on Stetl on SlideShare <http://www.slideshare.net/justb4>`_ (search for 'Stetl')
+
+Stetl Projects/Cases
+====================
+
+Known uses of Stetl. More detail in the chapter on :ref:`cases`.
+
+* `NLExtract <http://nlextract.nl>`_
+* `SOSPilot <http://sospilot.geonovum.nl>`_
+* `Smart Emission <http://smartemission.nl>`_
+* `INSPIRE FOSS Project (Archived) <https://github.com/justb4/inspire-foss>`_
+
+Tools
+=====
+
+Tools/components used by/with Stetl.
+
+* `GDAL/OGR <http://gdal.org>`_
+* `lxml <http://lxml.de>`_
+* `deegree WMS/WFS <http://www.deegree.org>`_
+* `PostGIS/PostgreSQL <http://http://postgis.org/>`_
+* `Jinja2 <http://jinja.pocoo.org/>`_
+
+Other
+=====
+
+* `More Geospatial Python projects, <https://github.com/geopython>`_
+* `INSPIRE, <http://inspire.ec.europa.eu/>`_
diff --git a/docs/using.rst b/docs/using.rst
index 67162cc..862c3a8 100644
--- a/docs/using.rst
+++ b/docs/using.rst
@@ -5,15 +5,15 @@ Using Stetl
This section explains how to use Stetl for your ETL. It assumes Stetl is installed and
you are able to run the examples. It may be useful to study some of the examples,
-especially the core ones found in the `examples/basics directory <https://github.com/justb4/stetl/tree/master/examples/basics>`_.
+especially the core ones found in the `examples/basics directory <https://github.com/geopython/stetl/tree/master/examples/basics>`_.
These examples start numbering from 1, building up more complex ETL cases like `(INSPIRE) transformation using
-Jinja2 Templating <https://github.com/justb4/stetl/tree/master/examples/basics/10_jinja2_templating>`_.
+Jinja2 Templating <https://github.com/geopython/stetl/tree/master/examples/basics/10_jinja2_templating>`_.
In addition there are example cases like the Dutch
-Topo map (Top10NL) ETL in the `examples/top10nl directory <https://github.com/justb4/stetl/tree/master/examples/top10nl>`_ .
+Topo map (Top10NL) ETL in the `examples/top10nl directory <https://github.com/geopython/stetl/tree/master/examples/top10nl>`_ .
The core concepts of Stetl remain pretty simple: an input resource like a file or a database table is
-mapped to an output resource (also a file, a database, etc) via one or more filters.
+mapped to an output resource (also a file, a database, a remote HTTP server etc) via one or more filters.
The input, filters and output are connected in a pipeline called a `processing chain` or Chain.
This is a bit similar to a current in electrical engineering: an input flows
through several filters, that each modify the current. In our case the current is (geospatial) data.
@@ -22,10 +22,10 @@ Stetl design follows the so-called `Pipes and Filters Architectural Pattern <htt
Stetl Config
------------
-Stetl components (inputs, filters, outputs) and their interconnection (the Pipeline/Chain)
+Stetl components (Inputs, Filters, Outputs) and their interconnection (the Pipeline/Chain)
are specified in a Stetl config file. The file format follows the Python ``.ini`` file-format.
-To illustrate, let's look at the example `2_xslt <https://github.com/justb4/stetl/tree/master/examples/basics/2_xslt>`_.
+To illustrate, let's look at the example `2_xslt <https://github.com/geopython/stetl/tree/master/examples/basics/2_xslt>`_.
This example takes the input file ``input/cities.xml`` and transforms this file to a valid GML file called
``output/gmlcities.gml``. The Stetl config file looks as follows. ::
@@ -58,8 +58,25 @@ a the Unix pipe symbol "|".
So the above Chain is ``input_xml_file|transformer_xslt|output_file``. The names
of the component sections like ``[input_xml_file]`` are arbitrary.
+Note: since v1.1.0 a datastream can be split (see below) to multiple ``Outputs`` using ``()`` like : ::
+
+ [etl]
+ chains = input_xml_file|transformer_xslt|(output_gml_file)(output_wfs)
+
+Or multiple Input streams can be combined/merged like: ::
+
+ [etl]
+ chains = (input_http_api_1) (input_http_api_2) | data_transformer | output_db
+
+It is even possible to have both Splitting and Merging together with filtering: ::
+
+ [etl]
+ chains = (input_http_api_1 | cleaner_filter) (input_http_api_2) | data_transformer | (output_db) (output_file)
+
+
Configuring Components
----------------------
+
Most Stetl Components, i.e. inputs, filters, outputs, have properties that can be configured within their
respective [section] in the config file. But what are the possible properties, values and defaults?
This is documented within each Component class using the ``@Config`` decorator much similar to the standard Python
@@ -74,49 +91,48 @@ For class authors: this information is added
via the Python Decorators much similar to ``@property``. The :class:`stetl.component.Config`
is used to define read-only properties for each Component instance. For example, ::
- class FileInput(Input):
- """
- Abstract base class for specific FileInputs, use derived classes.
- """
-
- # Start attribute config meta
- # Applying Decorator pattern with the Config class to provide
- # read-only config values from the configured properties.
-
- @Config(ptype=str, default=None, required=False)
- def file_path(self):
- """
- Path to file or files or URLs: can be a dir or files or URLs
- or even multiple, comma separated. For URLs only JSON is supported now.
-
- Required: True
-
- Default: None
- """
- pass
-
- @Config(ptype=str, default='*.[gxGX][mM][lL]', required=False)
- def filename_pattern(self):
- """
- Filename pattern according to Python glob.glob for example:
- '\*.[gxGX][mM][lL]'
-
- Required: False
-
- Default: '\*.[gxGX][mM][lL]'
- """
- pass
-
- # End attribute config meta
-
- def __init__(self, configdict, section, produces):
- Input.__init__(self, configdict, section, produces)
-
- # Create the list of files to be used as input
- self.file_list = Util.make_file_list(self.file_path, None, self.filename_pattern, self.depth_search)
-
-This defines two configurable properties for the class FileInput.
-Each ``@Config`` has three parameters: ``p_type``, the Python type (``str``, ``list``, ``dict``, ``bool``, ``int``),
+ class FileInput(Input):
+ """
+ Abstract base class for specific FileInputs, use derived classes.
+ """
+
+ # Start attribute config meta
+ # Applying Decorator pattern with the Config class to provide
+ # read-only config values from the configured properties.
+
+ @Config(ptype=str, default=None, required=False)
+ def file_path(self):
+ """
+ Path to file or files or URLs: can be a dir or files or URLs
+ or even multiple, comma separated. For URLs only JSON is supported now.
+ """
+ pass
+
+ @Config(ptype=str, default='*.[gxGX][mM][lL]', required=False)
+ def filename_pattern(self):
+ """
+ Filename pattern according to Python ``glob.glob`` for example:
+ '\\*.[gxGX][mM][lL]'
+ """
+ pass
+
+ @Config(ptype=bool, default=False, required=False)
+ def depth_search(self):
+ """
+ Should we recurse into sub-directories to find files?
+ """
+ pass
+
+ # End attribute config meta
+
+ def __init__(self, configdict, section, produces):
+ Input.__init__(self, configdict, section, produces)
+
+ # Create the list of files to be used as input
+ self.file_list = Util.make_file_list(self.file_path, None, self.filename_pattern, self.depth_search)
+
+This defines three configurable properties for the class FileInput.
+Each ``@Config`` has three parameters: ``ptype``, the Python type (``str``, ``list``, ``dict``, ``bool``, ``int``),
``default`` (default value if not present) and ``required`` (if property in mandatory or optional).
Within the config one can set specific
@@ -144,7 +160,7 @@ with more complex transformations.
Suppose we want to convert the resulting GML to an `ESRI Shapefile`. As we cannot use GDAL ``ogr2ogr`` on the input
file, we need to combine XSLT and `ogr2ogr`. See example
-`3_shape <https://github.com/justb4/stetl/tree/master/examples/basics/3_shape>`_. Now we replace the output
+`3_shape <https://github.com/geopython/stetl/tree/master/examples/basics/3_shape>`_. Now we replace the output
by using `outputs.ogroutput.Ogr2OgrOutput`, which can execute any `ogr2ogr` command, converting
whatever it gets as input from the previous Filter in the Chain. ::
@@ -176,19 +192,19 @@ whatever it gets as input from the previous Filter in the Chain. ::
Using Docker
~~~~~~~~~~~~
-A convenient way to run Stetl is via a Docker image. See the installation instructions at
+The most convenient way to run Stetl is via Docker. See the installation instructions at
:ref:`install_docker`. A full example can be viewed in the Smart Emission project:
https://github.com/Geonovum/smartemission/tree/master/etl.
In the simplest case you run a Stetl Docker container from your own built image or the Dockerhub-provided
-one, `justb4/stetl:latest <https://hub.docker.com/r/justb4/stetl>`_ basically as follows: ::
+one, `geopython/stetl:<version> stetl <https://hub.docker.com/r/geopython/stetl>`_ as follows (`latest` version): ::
- sudo docker run -v <host dir>:<container dir> -w <work dir> justb4/stetl:latest <any Stetl arguments>
+ sudo docker run -v <host dir>:<container dir> -w <work dir> geopython/stetl:latest stetl <any Stetl arguments>
For example within the current directory you may have an ``etl.cfg`` Stetl file: ::
WORK_DIR=`pwd`
- sudo docker run -v ${WORK_DIR}:${WORK_DIR} -w ${WORK_DIR} justb4/stetl:latest -c etl.cfg
+ sudo docker run -v ${WORK_DIR}:${WORK_DIR} -w ${WORK_DIR} geopython/stetl:latest stetl -c etl.cfg
A more advanced setup would be (network-)linking to a PostGIS Docker image
like `kartoza/postgis <https://hub.docker.com/r/kartoza/postgis/>`_: ::
@@ -200,12 +216,15 @@ like `kartoza/postgis <https://hub.docker.com/r/kartoza/postgis/>`_: ::
STETL_ARGS="-c etl.cfg -a local.args"
WORK_DIR="`pwd`"
- sudo docker run --name stetl --link postgis:postgis -v ${WORK_DIR}:${WORK_DIR} -w ${WORK_DIR} justb4/stetl:latest ${STETL_ARGS}
+ sudo docker run --name stetl --link postgis:postgis -v ${WORK_DIR}:${WORK_DIR} -w ${WORK_DIR} geopython/stetl:latest stetl ${STETL_ARGS}
The last example is used within the SmartEmission project. Also with more detail and keeping
-all dynamic data (like PostGIS DB), your Stetl config and results, and logs within the host. For PostGIS see: https://github.com/Geonovum/smartemission/tree/master/services/postgis
+all dynamic data (like PostGIS DB), your Stetl config and results, and logs within the host.
+For PostGIS see: https://github.com/Geonovum/smartemission/tree/master/services/postgis
and Stetl see: https://github.com/Geonovum/smartemission/tree/master/etl.
+Even better is to use `docker-compose`.
+
Stetl Integration
-----------------
@@ -218,7 +237,7 @@ Reusable Stetl Configs
What we saw in the last example is that it is hard to reuse this `etl.cfg` when we have for example a different input file
or want to map to different output files. For this Stetl supports `parameter substitution`. Here command line parameters are substituted
for variables in `etl.cfg`. A variable is declared between curly brackets like `{out_xml}`. See
-example `6_cmdargs <https://github.com/justb4/stetl/tree/master/examples/basics/6_cmdargs>`_. ::
+example `6_cmdargs <https://github.com/geopython/stetl/tree/master/examples/basics/6_cmdargs>`_. ::
[etl]
chains = input_xml_file|transformer_xslt|output_file
@@ -252,7 +271,7 @@ Where the content of the `etl.args` properties file is: ::
out_xml=output/gmlcities.gml
This makes an ETL chain highly reusable. A very elaborate Stetl config with parameter substitution can be seen in the
-`Top10NL ETL <https://github.com/justb4/stetl/blob/master/examples/top10nl/etl-top10nl.cfg>`_.
+`Top10NL ETL <https://github.com/geopython/stetl/blob/master/examples/top10nl/etl-top10nl.cfg>`_.
Connection Compatibility
------------------------
@@ -336,7 +355,8 @@ The syntax: chains are separated by commas (steps are sill separated by pipe sym
Chains are executed in order. We can even reuse the
specified components from within the same file. Each will have a separate instance within a Chain.
-For example in the `Top10NL example <https://github.com/justb4/stetl/blob/master/examples/top10nl/etl-top10nl.cfg>`_ we see three Chains::
+For example in the `Top10NL example <https://github.com/geopython/stetl/blob/master/examples/top10nl/etl-top10nl.cfg>`_
+we see three Chains::
[etl]
chains = input_sql_pre|schema_name_filter|output_postgres,
@@ -346,3 +366,79 @@ For example in the `Top10NL example <https://github.com/justb4/stetl/blob/master
Here the Chain `input_sql_pre|schema_name_filter|output_postgres` sets up a PostgreSQL schema and
creates tables. `input_big_gml_files|xml_assembler|transformer_xslt|output_ogr2ogr` does the actual ETL and
`input_sql_post|schema_name_filter|output_postgres` does some PostgreSQL postprocessing.
+
+Chain Splitting
+---------------
+
+In some cases we may want to split processed data to multiple ``Filters`` or ``Outputs``.
+For example to produce output files in multiple formats like GML, GeoJSON etc
+or to publish converted (Filtered) data to multiple remote services (SOS, SensorThings API)
+or just for simple debugging to a target ``Output`` and ``StandardOutput``.
+
+See issue https://github.com/geopython/stetl/issues/35 and
+the `Chain Split example <https://github.com/geopython/stetl/tree/master/examples/basics/15_splitter>`_.
+
+Here the Chains are split by using ``()`` in the ETL Chain definition: ::
+
+ # Transform input xml to valid GML file using an XSLT filter and pass to multiple outputs.
+ # Below are two Chains: simple Output splitting and splitting to 3 sub-Chains at Filter level.
+
+ [etl]
+ chains = input_xml_file | transformer_xslt |(output_file)(output_std),
+ input_xml_file | (transformer_xslt|output_file) (output_std) (transformer_xslt|output_std)
+
+
+ [input_xml_file]
+ class = inputs.fileinput.XmlFileInput
+ file_path = input/cities.xml
+
+ [transformer_xslt]
+ class = filters.xsltfilter.XsltFilter
+ script = cities2gml.xsl
+
+ [output_file]
+ class = outputs.fileoutput.FileOutput
+ file_path = output/gmlcities.gml
+
+ [output_std]
+ class = outputs.standardoutput.StandardOutput
+
+Chain Merging
+-------------
+
+In some cases we may want to merge (combine, join) multiple input streams.
+
+For example to harvest data from multiple HTTP REST APIs, or to realize a `Filter` that
+integrates data from two data-sources.
+
+See issue https://github.com/geopython/stetl/issues/59 and
+the `Chain Merge example <https://github.com/geopython/stetl/tree/master/examples/basics/16_merger>`_.
+
+Here the Chains are merged by using ``()`` notation in the ETL Chain definition, possibly even combined with Splitting
+Outputs: ::
+
+ # Merge two inputs into single Filter.
+
+ [etl]
+ chains = (input_1) (input_2)|transformer_xslt|output_std,
+ (input_1) (input_2)|transformer_xslt|(output_file)(output_std)
+
+
+ [input_1]
+ class = inputs.fileinput.XmlFileInput
+ file_path = input1/cities.xml
+
+ [input_2]
+ class = inputs.fileinput.XmlFileInput
+ file_path = input2/cities.xml
+
+ [transformer_xslt]
+ class = filters.xsltfilter.XsltFilter
+ script = cities2gml.xsl
+
+ [output_file]
+ class = outputs.fileoutput.FileOutput
+ file_path = output/gmlcities.gml
+
+ [output_std]
+ class = outputs.standardoutput.StandardOutput
diff --git a/examples/README.md b/examples/README.md
new file mode 100644
index 0000000..7275845
--- /dev/null
+++ b/examples/README.md
@@ -0,0 +1,9 @@
+# Stetl Examples
+
+The directories below each show various examples for Stetl.
+Best is to first browse [The Basic Examples](basics)
+
+For more examples and cases see the following projects:
+
+* [NLExtract](https://github.com/nlextract/NLExtract) - conversion of Open Dutch geodata sets
+* [Smart Emission](https://github.com/Geonovum/smartemission/tree/master/etl) - IoT SensorWeb/SOS
diff --git a/examples/basics/10_jinja2_templating/output/cities-gjson.gml b/examples/basics/10_jinja2_templating/output/cities-gjson.gml
index 63d5854..edabf3b 100644
--- a/examples/basics/10_jinja2_templating/output/cities-gjson.gml
+++ b/examples/basics/10_jinja2_templating/output/cities-gjson.gml
@@ -42,7 +42,7 @@
<cities:name>Amsterdam</cities:name>
<cities:population>779808</cities:population>
<cities:geometry>
- <gml:Point srsName="urn:ogc:def:crs:EPSG::4258" gml:id="point-1"><gml:pos>52.373045454545455 4.894836363636363</gml:pos></gml:Point>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG::4258" gml:id="point-1"><gml:pos>52.3730454545455 4.89483636363636</gml:pos></gml:Point>
</cities:geometry>
</cities:City>
</gml:featureMember>
@@ -51,7 +51,7 @@
<cities:name>Bonn</cities:name>
<cities:population>327913</cities:population>
<cities:geometry>
- <gml:Point srsName="urn:ogc:def:crs:EPSG::4258" gml:id="point-2"><gml:pos>50.734554545454543 7.099818181818182</gml:pos></gml:Point>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG::4258" gml:id="point-2"><gml:pos>50.7345545454545 7.09981818181818</gml:pos></gml:Point>
</cities:geometry>
</cities:City>
</gml:featureMember>
diff --git a/examples/basics/12_gdal_ogr/etl.cfg b/examples/basics/12_gdal_ogr/etl.cfg
index eeb38f5..09b0218 100644
--- a/examples/basics/12_gdal_ogr/etl.cfg
+++ b/examples/basics/12_gdal_ogr/etl.cfg
@@ -6,7 +6,10 @@
# input_gml_file|output_ogr_shapefile,
# input_gml_file|ogr_feature_to_geojson_feature|geojson_feature_to_ogr_feature|output_ogr_gmlfile
-chains = input_geojson_file|geojson_coll_to_ogr_feature_arr|output_ogr_gmlfile
+chains = input_geojson_file|geojson_coll_to_ogr_feature_arr|output_ogr_gmlfile,
+ input_gml_file_to_array|ogr_feature_array_to_geojson_collection|tolowercase_filter|output_file,
+ input_gml_file|output_ogr_shapefile
+
#
diff --git a/examples/basics/12_gdal_ogr/etl.sh b/examples/basics/12_gdal_ogr/etl.sh
index 165e2c6..98e371d 100755
--- a/examples/basics/12_gdal_ogr/etl.sh
+++ b/examples/basics/12_gdal_ogr/etl.sh
@@ -5,6 +5,4 @@
# Shortcut to call Stetl main.py with etl config.
#
-# stetl -c etl.cfg
-../../../stetl/main.py -c etl.cfg
-
+stetl -c etl.cfg
diff --git a/examples/basics/12_gdal_ogr/output/cities.dbf b/examples/basics/12_gdal_ogr/output/cities.dbf
index aad1113..dc18814 100644
Binary files a/examples/basics/12_gdal_ogr/output/cities.dbf and b/examples/basics/12_gdal_ogr/output/cities.dbf differ
diff --git a/examples/basics/12_gdal_ogr/output/cities.gml b/examples/basics/12_gdal_ogr/output/cities.gml
index 774b1f6..4c77471 100644
--- a/examples/basics/12_gdal_ogr/output/cities.gml
+++ b/examples/basics/12_gdal_ogr/output/cities.gml
@@ -13,7 +13,7 @@
<gml:featureMember>
<ogr:cities fid="cities.0">
- <ogr:geometryProperty><gml:Point><gml:coordinates>4.894836363636363,52.373045454545455</gml:coordinates></gml:Point></ogr:geometryProperty>
+ <ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>4.89483636363636,52.3730454545455</gml:coordinates></gml:Point></ogr:geometryProperty>
<ogr:id>1359</ogr:id>
<ogr:STATUS>National capital</ogr:STATUS>
<ogr:ObjectID>12386305</ogr:ObjectID>
@@ -31,7 +31,7 @@
</gml:featureMember>
<gml:featureMember>
<ogr:cities fid="cities.1">
- <ogr:geometryProperty><gml:Point><gml:coordinates>7.099818181818182,50.734554545454543</gml:coordinates></gml:Point></ogr:geometryProperty>
+ <ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>7.09981818181818,50.7345545454545</gml:coordinates></gml:Point></ogr:geometryProperty>
<ogr:id>1374</ogr:id>
<ogr:STATUS>Other</ogr:STATUS>
<ogr:ObjectID>12517379</ogr:ObjectID>
@@ -49,7 +49,7 @@
</gml:featureMember>
<gml:featureMember>
<ogr:cities fid="cities.2">
- <ogr:geometryProperty><gml:Point><gml:coordinates>12.52,41.88</gml:coordinates></gml:Point></ogr:geometryProperty>
+ <ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>12.52,41.88</gml:coordinates></gml:Point></ogr:geometryProperty>
<ogr:id>1027</ogr:id>
<ogr:STATUS>National and provincial capital</ogr:STATUS>
<ogr:ObjectID>9306117</ogr:ObjectID>
diff --git a/examples/basics/15_splitter/cities2gml.xsl b/examples/basics/15_splitter/cities2gml.xsl
new file mode 100644
index 0000000..bfa3b0d
--- /dev/null
+++ b/examples/basics/15_splitter/cities2gml.xsl
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Transform plain XML cities XML to valid GML.
+
+Author: Just van den Broecke, Just Objects B.V.
+-->
+<xsl:stylesheet version="1.0"
+ xmlns:ogr="http://ogr.maptools.org/"
+ xmlns:gml="http://www.opengis.net/gml"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+ >
+ <xsl:output method="xml" omit-xml-declaration="no" indent="yes"/>
+ <xsl:strip-space elements="*"/>
+
+ <xsl:template match="/">
+ <ogr:FeatureCollection
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:ogr="http://ogr.maptools.org/"
+ xmlns:gml="http://www.opengis.net/gml"
+ xsi:schemaLocation="http://ogr.maptools.org/ ../gmlcities.xsd http://www.opengis.net/gml http://schemas.opengis.net/gml/2.1.2/feature.xsd"
+ >
+ <gml:boundedBy>
+ <gml:Box>
+ <gml:coord><gml:X>-180.0</gml:X><gml:Y>-90.0</gml:Y></gml:coord>
+ <gml:coord><gml:X>180.0</gml:X><gml:Y>90.0</gml:Y></gml:coord>
+ </gml:Box>
+ </gml:boundedBy>
+ <!-- Loop through all cities. -->
+ <xsl:apply-templates/>
+ </ogr:FeatureCollection>
+ </xsl:template>
+
+ <!-- Make each city an ogr:featureMember. -->
+ <xsl:template match="city">
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>
+ <xsl:value-of select="name"/>
+ </ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates><xsl:value-of select="lat"/>,<xsl:value-of select="lon"/></gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ </xsl:template>
+</xsl:stylesheet>
diff --git a/examples/basics/15_splitter/etl.cfg b/examples/basics/15_splitter/etl.cfg
new file mode 100644
index 0000000..75e7796
--- /dev/null
+++ b/examples/basics/15_splitter/etl.cfg
@@ -0,0 +1,22 @@
+# Transform input xml to valid GML file using an XSLT filter and pass to multiple outputs.
+# Below are two Chains: simple Output splitting and splitting at Filter level.
+
+[etl]
+chains = input_xml_file|transformer_xslt|(output_file)(output_std),
+ input_xml_file |(transformer_xslt|output_file) (output_std) (transformer_xslt|output_std)
+
+
+[input_xml_file]
+class = inputs.fileinput.XmlFileInput
+file_path = input/cities.xml
+
+[transformer_xslt]
+class = filters.xsltfilter.XsltFilter
+script = cities2gml.xsl
+
+[output_file]
+class = outputs.fileoutput.FileOutput
+file_path = output/gmlcities.gml
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/examples/basics/4_validate/etl.sh b/examples/basics/15_splitter/etl.sh
similarity index 84%
copy from examples/basics/4_validate/etl.sh
copy to examples/basics/15_splitter/etl.sh
index e70f79b..2cdcf3a 100755
--- a/examples/basics/4_validate/etl.sh
+++ b/examples/basics/15_splitter/etl.sh
@@ -4,6 +4,6 @@
#
# Author: Just van den Broecke
#
-stetl -c etl.cfg
+stetl -c etl.cfg
diff --git a/examples/basics/15_splitter/gmlcities.xsd b/examples/basics/15_splitter/gmlcities.xsd
new file mode 100644
index 0000000..a7ca448
--- /dev/null
+++ b/examples/basics/15_splitter/gmlcities.xsd
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema targetNamespace="http://ogr.maptools.org/" xmlns:ogr="http://ogr.maptools.org/"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:gml="http://www.opengis.net/gml"
+ elementFormDefault="qualified" version="1.0">
+ <xs:import namespace="http://www.opengis.net/gml"
+ schemaLocation="http://schemas.opengis.net/gml/2.1.2/feature.xsd"/>
+ <xs:element name="FeatureCollection" type="ogr:FeatureCollectionType" substitutionGroup="gml:_FeatureCollection"/>
+ <xs:complexType name="FeatureCollectionType">
+ <xs:complexContent>
+ <xs:extension base="gml:AbstractFeatureCollectionType">
+ <xs:attribute name="lockId" type="xs:string" use="optional"/>
+ <xs:attribute name="scope" type="xs:string" use="optional"/>
+ </xs:extension>
+ </xs:complexContent>
+ </xs:complexType>
+ <xs:element name="City" type="ogr:City_Type" substitutionGroup="gml:_Feature"/>
+ <xs:complexType name="City_Type">
+ <xs:complexContent>
+ <xs:extension base="gml:AbstractFeatureType">
+ <xs:sequence>
+ <xs:element name="name" nillable="false" minOccurs="1" maxOccurs="1">
+ <xs:simpleType>
+ <xs:restriction base="xs:string">
+ <xs:maxLength value="42"/>
+ </xs:restriction>
+ </xs:simpleType>
+ </xs:element>
+ <xs:element name="geometry" type="gml:PointPropertyType" nillable="false" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:extension>
+ </xs:complexContent>
+ </xs:complexType>
+</xs:schema>
diff --git a/docker/test/1_copystd/input/cities.xml b/examples/basics/15_splitter/input/cities.xml
similarity index 97%
copy from docker/test/1_copystd/input/cities.xml
copy to examples/basics/15_splitter/input/cities.xml
index 58721f0..2b89646 100644
--- a/docker/test/1_copystd/input/cities.xml
+++ b/examples/basics/15_splitter/input/cities.xml
@@ -15,4 +15,4 @@
<lat>41.9</lat>
<lon>12.5</lon>
</city>
-</cities>
+</cities>
\ No newline at end of file
diff --git a/examples/basics/15_splitter/output/gmlcities.gml b/examples/basics/15_splitter/output/gmlcities.gml
new file mode 100644
index 0000000..57c76b0
--- /dev/null
+++ b/examples/basics/15_splitter/output/gmlcities.gml
@@ -0,0 +1,45 @@
+<?xml version='1.0' encoding='utf-8'?>
+<ogr:FeatureCollection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ogr="http://ogr.maptools.org/" xmlns:gml="http://www.opengis.net/gml" xsi:schemaLocation="http://ogr.maptools.org/ ../gmlcities.xsd http://www.opengis.net/gml http://schemas.opengis.net/gml/2.1.2/feature.xsd">
+ <gml:boundedBy>
+ <gml:Box>
+ <gml:coord>
+ <gml:X>-180.0</gml:X>
+ <gml:Y>-90.0</gml:Y>
+ </gml:coord>
+ <gml:coord>
+ <gml:X>180.0</gml:X>
+ <gml:Y>90.0</gml:Y>
+ </gml:coord>
+ </gml:Box>
+ </gml:boundedBy>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Amsterdam</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>52.4,4.9</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Bonn</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>50.7,7.1</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Rome</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>41.9,12.5</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+</ogr:FeatureCollection>
diff --git a/examples/basics/16_merger/cities2gml.xsl b/examples/basics/16_merger/cities2gml.xsl
new file mode 100644
index 0000000..bfa3b0d
--- /dev/null
+++ b/examples/basics/16_merger/cities2gml.xsl
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Transform plain XML cities XML to valid GML.
+
+Author: Just van den Broecke, Just Objects B.V.
+-->
+<xsl:stylesheet version="1.0"
+ xmlns:ogr="http://ogr.maptools.org/"
+ xmlns:gml="http://www.opengis.net/gml"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+ >
+ <xsl:output method="xml" omit-xml-declaration="no" indent="yes"/>
+ <xsl:strip-space elements="*"/>
+
+ <xsl:template match="/">
+ <ogr:FeatureCollection
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:ogr="http://ogr.maptools.org/"
+ xmlns:gml="http://www.opengis.net/gml"
+ xsi:schemaLocation="http://ogr.maptools.org/ ../gmlcities.xsd http://www.opengis.net/gml http://schemas.opengis.net/gml/2.1.2/feature.xsd"
+ >
+ <gml:boundedBy>
+ <gml:Box>
+ <gml:coord><gml:X>-180.0</gml:X><gml:Y>-90.0</gml:Y></gml:coord>
+ <gml:coord><gml:X>180.0</gml:X><gml:Y>90.0</gml:Y></gml:coord>
+ </gml:Box>
+ </gml:boundedBy>
+ <!-- Loop through all cities. -->
+ <xsl:apply-templates/>
+ </ogr:FeatureCollection>
+ </xsl:template>
+
+ <!-- Make each city an ogr:featureMember. -->
+ <xsl:template match="city">
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>
+ <xsl:value-of select="name"/>
+ </ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates><xsl:value-of select="lat"/>,<xsl:value-of select="lon"/></gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ </xsl:template>
+</xsl:stylesheet>
diff --git a/examples/basics/16_merger/etl.cfg b/examples/basics/16_merger/etl.cfg
new file mode 100644
index 0000000..a107eec
--- /dev/null
+++ b/examples/basics/16_merger/etl.cfg
@@ -0,0 +1,25 @@
+# Merge two inputs into single Filter.
+
+[etl]
+chains = (input_1) (input_2)|transformer_xslt|output_std,
+ (input_1) (input_2)|transformer_xslt|(output_file)(output_std)
+
+
+[input_1]
+class = inputs.fileinput.XmlFileInput
+file_path = input1/cities.xml
+
+[input_2]
+class = inputs.fileinput.XmlFileInput
+file_path = input2/cities.xml
+
+[transformer_xslt]
+class = filters.xsltfilter.XsltFilter
+script = cities2gml.xsl
+
+[output_file]
+class = outputs.fileoutput.FileOutput
+file_path = output/gmlcities.gml
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/examples/basics/4_validate/etl.sh b/examples/basics/16_merger/etl.sh
similarity index 84%
copy from examples/basics/4_validate/etl.sh
copy to examples/basics/16_merger/etl.sh
index e70f79b..2cdcf3a 100755
--- a/examples/basics/4_validate/etl.sh
+++ b/examples/basics/16_merger/etl.sh
@@ -4,6 +4,6 @@
#
# Author: Just van den Broecke
#
-stetl -c etl.cfg
+stetl -c etl.cfg
diff --git a/examples/basics/16_merger/gmlcities.xsd b/examples/basics/16_merger/gmlcities.xsd
new file mode 100644
index 0000000..a7ca448
--- /dev/null
+++ b/examples/basics/16_merger/gmlcities.xsd
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema targetNamespace="http://ogr.maptools.org/" xmlns:ogr="http://ogr.maptools.org/"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:gml="http://www.opengis.net/gml"
+ elementFormDefault="qualified" version="1.0">
+ <xs:import namespace="http://www.opengis.net/gml"
+ schemaLocation="http://schemas.opengis.net/gml/2.1.2/feature.xsd"/>
+ <xs:element name="FeatureCollection" type="ogr:FeatureCollectionType" substitutionGroup="gml:_FeatureCollection"/>
+ <xs:complexType name="FeatureCollectionType">
+ <xs:complexContent>
+ <xs:extension base="gml:AbstractFeatureCollectionType">
+ <xs:attribute name="lockId" type="xs:string" use="optional"/>
+ <xs:attribute name="scope" type="xs:string" use="optional"/>
+ </xs:extension>
+ </xs:complexContent>
+ </xs:complexType>
+ <xs:element name="City" type="ogr:City_Type" substitutionGroup="gml:_Feature"/>
+ <xs:complexType name="City_Type">
+ <xs:complexContent>
+ <xs:extension base="gml:AbstractFeatureType">
+ <xs:sequence>
+ <xs:element name="name" nillable="false" minOccurs="1" maxOccurs="1">
+ <xs:simpleType>
+ <xs:restriction base="xs:string">
+ <xs:maxLength value="42"/>
+ </xs:restriction>
+ </xs:simpleType>
+ </xs:element>
+ <xs:element name="geometry" type="gml:PointPropertyType" nillable="false" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:extension>
+ </xs:complexContent>
+ </xs:complexType>
+</xs:schema>
diff --git a/docker/test/1_copystd/input/cities.xml b/examples/basics/16_merger/input1/cities.xml
similarity index 97%
copy from docker/test/1_copystd/input/cities.xml
copy to examples/basics/16_merger/input1/cities.xml
index 58721f0..2b89646 100644
--- a/docker/test/1_copystd/input/cities.xml
+++ b/examples/basics/16_merger/input1/cities.xml
@@ -15,4 +15,4 @@
<lat>41.9</lat>
<lon>12.5</lon>
</city>
-</cities>
+</cities>
\ No newline at end of file
diff --git a/docker/test/1_copystd/input/cities.xml b/examples/basics/16_merger/input2/cities.xml
similarity index 97%
rename from docker/test/1_copystd/input/cities.xml
rename to examples/basics/16_merger/input2/cities.xml
index 58721f0..2b89646 100644
--- a/docker/test/1_copystd/input/cities.xml
+++ b/examples/basics/16_merger/input2/cities.xml
@@ -15,4 +15,4 @@
<lat>41.9</lat>
<lon>12.5</lon>
</city>
-</cities>
+</cities>
\ No newline at end of file
diff --git a/examples/basics/16_merger/output/gmlcities.gml b/examples/basics/16_merger/output/gmlcities.gml
new file mode 100644
index 0000000..57c76b0
--- /dev/null
+++ b/examples/basics/16_merger/output/gmlcities.gml
@@ -0,0 +1,45 @@
+<?xml version='1.0' encoding='utf-8'?>
+<ogr:FeatureCollection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ogr="http://ogr.maptools.org/" xmlns:gml="http://www.opengis.net/gml" xsi:schemaLocation="http://ogr.maptools.org/ ../gmlcities.xsd http://www.opengis.net/gml http://schemas.opengis.net/gml/2.1.2/feature.xsd">
+ <gml:boundedBy>
+ <gml:Box>
+ <gml:coord>
+ <gml:X>-180.0</gml:X>
+ <gml:Y>-90.0</gml:Y>
+ </gml:coord>
+ <gml:coord>
+ <gml:X>180.0</gml:X>
+ <gml:Y>90.0</gml:Y>
+ </gml:coord>
+ </gml:Box>
+ </gml:boundedBy>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Amsterdam</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>52.4,4.9</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Bonn</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>50.7,7.1</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Rome</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>41.9,12.5</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+</ogr:FeatureCollection>
diff --git a/examples/basics/1_copystd/etl.sh b/examples/basics/1_copystd/etl.sh
index b3d5a82..5b169e6 100755
--- a/examples/basics/1_copystd/etl.sh
+++ b/examples/basics/1_copystd/etl.sh
@@ -4,5 +4,10 @@
#
# Shortcut to call Stetl main.py with etl config.
#
-stetl -c etl.cfg
+stetl=stetl
+
+# PYTHONPATH=${PYTHONPATH}:../../..
+# stetl=../../../stetl/main.py
+
+$stetl -c etl.cfg
diff --git a/examples/basics/3_shape/output/gmlcities.dbf b/examples/basics/3_shape/output/gmlcities.dbf
index ae8ba32..2cb4d78 100644
Binary files a/examples/basics/3_shape/output/gmlcities.dbf and b/examples/basics/3_shape/output/gmlcities.dbf differ
diff --git a/examples/basics/3_shape/temp/gmlcities.gfs b/examples/basics/3_shape/temp/gmlcities.gfs
index 0f9e11f..d7f2ed0 100644
--- a/examples/basics/3_shape/temp/gmlcities.gfs
+++ b/examples/basics/3_shape/temp/gmlcities.gfs
@@ -2,6 +2,9 @@
<GMLFeatureClass>
<Name>City</Name>
<ElementPath>City</ElementPath>
+ <GeometryName>geometry</GeometryName>
+ <GeometryElementPath>geometry</GeometryElementPath>
+ <!--POINT-->
<GeometryType>1</GeometryType>
<SRSName>GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]</SRSName>
<DatasetSpecificInfo>
diff --git a/examples/basics/4_validate/etl.sh b/examples/basics/4_validate/etl.sh
index e70f79b..2cdcf3a 100755
--- a/examples/basics/4_validate/etl.sh
+++ b/examples/basics/4_validate/etl.sh
@@ -4,6 +4,6 @@
#
# Author: Just van den Broecke
#
-stetl -c etl.cfg
+stetl -c etl.cfg
diff --git a/examples/basics/8_wfs/etl.cfg b/examples/basics/8_wfs/etl.cfg
index eee8fe8..4f48532 100644
--- a/examples/basics/8_wfs/etl.cfg
+++ b/examples/basics/8_wfs/etl.cfg
@@ -5,16 +5,15 @@ chains = input_wfs|output_std
[input_wfs]
class = inputs.httpinput.HttpInput
-url = http://suite.opengeo.org/geoserver/ows
+url = http://geodata.nationaalgeoregister.nl/bag/wfs
parameters = {
'service' : 'WFS',
'version' : '1.1.0',
'request' : 'GetFeature',
- 'srsName' : 'EPSG:4326',
+ 'srsName' : 'EPSG:28992',
'outputFormat' : 'text/xml; subtype=gml/2.1.2',
- 'typename' : 'states',
- 'filter' :'<ogc:Filter xmlns:ogc="http://www.opengis.net/ogc"><ogc:Intersects><ogc:PropertyName/><gml:Polygon xmlns:gml="http://www.opengis.net/gml" srsName="EPSG:4326"><gml:exterior><gml:LinearRing><gml:posList>-101.8671875 32.177734375 -101.8671875 39.6923828125 -91.935546875 39.6923828125 -91.935546875 32.177734375 -101.8671875 32.177734375</gml:posList></gml:LinearRing></gml:exterior></gml:Polygon></ogc:Intersects></ogc:Filter>'
-
+ 'typename' : 'verblijfsobject',
+ 'filter' :'<ogc:Filter xmlns:ogc="http://www.opengis.net/ogc"><ogc:BBOX><gml:Envelope xmlns:gml="http://www.opengis.net/gml" srsName="EPSG:28992"><gml:lowerCorner>183774.83 450577.24</gml:lowerCorner><gml:upperCorner>184277.99 450809.92</gml:upperCorner></gml:Envelope></ogc:BBOX></ogc:Filter>'
}
diff --git a/examples/basics/README.md b/examples/basics/README.md
new file mode 100644
index 0000000..7b3c991
--- /dev/null
+++ b/examples/basics/README.md
@@ -0,0 +1,23 @@
+# Stetl Examples - Basics
+
+The directories below each show the most basic examples for Stetl.
+The examples build up from simple to more complex by directory number prefix.
+
+As a general Stetl-health test you may run all examples using `./runall.sh`.
+
+* 1_copystd - just copy an XML file to standard output
+* 2_xslt - transform an input XML file to a GML file
+* 3_shape - transform an input XML file to a GML in memory etree and output to a Shape file
+* 4_validate - use of the validator filter after generating the GML doc
+* 5_split - split the input XML file before transforming to GML and output to multiple GML files
+* 6_cmdargs - reuse ETL config file by substituting symbolic variables via command line (-a) arguments or properties file
+* 7_mycomponent - adding custom/user-defined Input, Filter and/or Output Components
+* 8_wfs - fetch input data from WFS and process it
+* 9_string_templating - transform using standard Python string template with CSV input vars
+* 10_jinja2_templating - transform using standard Jinja2 http://jinja.pocoo.org Templating
+* 11_formatconvert - adapt incompatible inputs to outputs via the FormatConvertFilter
+* 12_gdal_ogr - direct OgrInput (and later output)
+* 13_dbinput - input from SQL sources, here SLQLite Input
+* 14_logfileinput - input from Apache Logfile
+* 15_splitter - Splitter Component: split Chain over multiple outputs or sub-Chains
+* 16_merger - Merger Component: combines multiple inputs into single input
diff --git a/examples/basics/readme.txt b/examples/basics/readme.txt
deleted file mode 100644
index 15b7bd8..0000000
--- a/examples/basics/readme.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-The directories below each show basic examples for Stetl.
-The examples build up from simple to more complex by directory number prefix.
-
-As a general health test you may run all examples using ./runall.sh
-
-1_copystd - just copy an XML file to standard output
-2_xslt - transform an input XML file to a GML file
-3_shape - transform an input XML file to a GML in memory etree and output to a Shape file
-4_validate - use of the validator filter after generating the GML doc
-5_split - split the input XML file before transforming to GML and output to multiple GML files
-6_cmdargs - reuse ETL config file by substituting symbolic variables via command line (-a) arguments or properties file
-7_mycomponent - adding custom/user-defined Input, Filter and/or Output Components
-8_wfs - fetch input data from WFS and process it
-9_string_templating - transform using standard Python string template with CSV input vars
-10_jinja2_templating - transform using standard Jinja2 http://jinja.pocoo.org Templating
-11_formatconvert - adapt incompatible inputs to outputs via the FormatConvertFilter
-12_gdal_ogr - direct OgrInput (and later output)
-13_dbinput - input from SQL sources, here SLQLite Input
-14_logfileinput - input from Apache Logfile
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/examples/basics/runall.log b/examples/basics/runall.log
index 395777a..d05d0dd 100644
--- a/examples/basics/runall.log
+++ b/examples/basics/runall.log
@@ -1,91 +1,92 @@
+~/project/stetl/git/examples/basics/10_jinja2_templating ~/project/stetl/git/examples/basics
==== running etl.sh for 10_jinja2_templating ====
-2014-11-24 11:04:13,862 util INFO Found cStringIO, good!
-2014-11-24 11:04:14,180 util INFO Found lxml.etree, native XML parsing, fabulous!
-2014-11-24 11:04:15,984 util INFO Found GDAL/OGR Python bindings, super!!
-2014-11-24 11:04:16,106 main INFO Stetl version = 1.0.7
-2014-11-24 11:04:16,108 ETL INFO INIT - Stetl version is 1.0.7
-2014-11-24 11:04:16,108 ETL INFO Config/working dir =/Users/just/project/stetl/git/examples/basics/10_jinja2_templating
-2014-11-24 11:04:16,108 ETL INFO Reading config_file = etl.cfg
-2014-11-24 11:04:16,126 ETL INFO START
-2014-11-24 11:04:16,127 util INFO Timer start: total ETL
-2014-11-24 11:04:16,127 chain INFO Assembling Chain: input_json|filter_template_xml|output_xml_file...
-2014-11-24 11:04:16,387 input INFO cfg = {'class': 'inputs.fileinput.JsonFileInput', 'file_path': 'input/cities.json'}
-2014-11-24 11:04:16,449 fileinput INFO file_list=['input/cities.json']
-2014-11-24 11:04:16,451 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/cities.xml'}
-2014-11-24 11:04:16,451 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/10_jinja2_templating
-2014-11-24 11:04:16,452 chain INFO Running Chain: input_json|filter_template_xml|output_xml_file
-2014-11-24 11:04:16,452 templatingfilter INFO Init: templating
-2014-11-24 11:04:17,283 templatingfilter INFO template file read and template created OK: templates/cities-json2xml.jinja2
-2014-11-24 11:04:17,284 fileinput INFO Read/parse for start for file=input/cities.json....
-2014-11-24 11:04:17,297 fileinput INFO Read/parse ok for file=input/cities.json
-2014-11-24 11:04:17,297 fileinput INFO all files done
-2014-11-24 11:04:17,298 fileoutput INFO writing to file output/cities.xml
-2014-11-24 11:04:17,298 fileoutput INFO written to output/cities.xml
-2014-11-24 11:04:17,298 templatingfilter INFO Exit: templating
-2014-11-24 11:04:17,298 chain INFO DONE - 1 rounds - chain=input_json|filter_template_xml|output_xml_file
-2014-11-24 11:04:17,298 chain INFO Assembling Chain: input_json|filter_template_gml|output_gml_file...
-2014-11-24 11:04:17,298 input INFO cfg = {'class': 'inputs.fileinput.JsonFileInput', 'file_path': 'input/cities.json'}
-2014-11-24 11:04:17,298 fileinput INFO file_list=['input/cities.json']
-2014-11-24 11:04:17,299 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/cities.gml'}
-2014-11-24 11:04:17,299 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/10_jinja2_templating
-2014-11-24 11:04:17,299 chain INFO Running Chain: input_json|filter_template_gml|output_gml_file
-2014-11-24 11:04:17,299 templatingfilter INFO Init: templating
-2014-11-24 11:04:17,299 templatingfilter INFO Read JSON file with globals from: input/globals.json
-2014-11-24 11:04:17,395 templatingfilter INFO template file read and template created OK: templates/cities-json2gml.jinja2
-2014-11-24 11:04:17,396 fileinput INFO Read/parse for start for file=input/cities.json....
-2014-11-24 11:04:17,396 fileinput INFO Read/parse ok for file=input/cities.json
-2014-11-24 11:04:17,396 fileinput INFO all files done
-2014-11-24 11:04:17,422 fileoutput INFO writing to file output/cities.gml
-2014-11-24 11:04:17,423 fileoutput INFO written to output/cities.gml
-2014-11-24 11:04:17,423 templatingfilter INFO Exit: templating
-2014-11-24 11:04:17,423 chain INFO DONE - 1 rounds - chain=input_json|filter_template_gml|output_gml_file
-2014-11-24 11:04:17,423 chain INFO Assembling Chain: input_geojson|filter_template_geojson2gml|output_gml_file2...
-2014-11-24 11:04:17,423 input INFO cfg = {'output_format': 'geojson_collection', 'class': 'inputs.fileinput.JsonFileInput', 'file_path': 'https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json'}
-2014-11-24 11:04:17,423 fileinput INFO file_list=['https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json']
-2014-11-24 11:04:17,423 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/cities-gjson.gml'}
-2014-11-24 11:04:17,424 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/10_jinja2_templating
-2014-11-24 11:04:17,424 chain INFO Running Chain: input_geojson|filter_template_geojson2gml|output_gml_file2
-2014-11-24 11:04:17,424 templatingfilter INFO Init: templating
-2014-11-24 11:04:17,424 templatingfilter INFO Read JSON file with globals from: input/globals.json
-2014-11-24 11:04:17,424 templatingfilter INFO Read JSON file with globals from: https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/more-globals.json
-2014-11-24 11:04:18,374 templatingfilter INFO template file read and template created OK: templates/cities-gjson2gml.jinja2
-2014-11-24 11:04:18,374 fileinput INFO Read/parse for start for file=https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json....
-2014-11-24 11:04:18,768 fileinput INFO Read/parse ok for file=https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json
-2014-11-24 11:04:18,768 fileinput INFO all files done
-2014-11-24 11:04:18,997 fileoutput INFO writing to file output/cities-gjson.gml
-2014-11-24 11:04:18,998 fileoutput INFO written to output/cities-gjson.gml
-2014-11-24 11:04:18,998 templatingfilter INFO Exit: templating
-2014-11-24 11:04:18,998 chain INFO DONE - 1 rounds - chain=input_geojson|filter_template_geojson2gml|output_gml_file2
-2014-11-24 11:04:18,998 chain INFO Assembling Chain: input_geojson|filter_template_geojson2gml|output_std...
-2014-11-24 11:04:18,998 input INFO cfg = {'output_format': 'geojson_collection', 'class': 'inputs.fileinput.JsonFileInput', 'file_path': 'https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json'}
-2014-11-24 11:04:18,998 fileinput INFO file_list=['https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json']
-2014-11-24 11:04:18,999 output INFO cfg = {'class': 'outputs.standardoutput.StandardOutput'}
-2014-11-24 11:04:19,000 chain INFO Running Chain: input_geojson|filter_template_geojson2gml|output_std
-2014-11-24 11:04:19,000 templatingfilter INFO Init: templating
-2014-11-24 11:04:19,000 templatingfilter INFO Read JSON file with globals from: input/globals.json
-2014-11-24 11:04:19,000 templatingfilter INFO Read JSON file with globals from: https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/more-globals.json
-2014-11-24 11:04:19,061 templatingfilter INFO template file read and template created OK: templates/cities-gjson2gml.jinja2
-2014-11-24 11:04:19,061 fileinput INFO Read/parse for start for file=https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json....
-2014-11-24 11:04:19,109 fileinput INFO Read/parse ok for file=https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json
-2014-11-24 11:04:19,109 fileinput INFO all files done
-2014-11-24 11:04:19,121 templatingfilter INFO Exit: templating
-2014-11-24 11:04:19,121 chain INFO DONE - 1 rounds - chain=input_geojson|filter_template_geojson2gml|output_std
-2014-11-24 11:04:19,121 chain INFO Assembling Chain: input_addresses_csv|convert_record_array_to_struct|filter_template_addresses2inspire|output_inspire_addresses...
-2014-11-24 11:04:19,121 input INFO cfg = {'class': 'inputs.fileinput.CsvFileInput', 'file_path': 'input/addresses.csv'}
-2014-11-24 11:04:19,122 fileinput INFO file_list=['input/addresses.csv']
-2014-11-24 11:04:19,138 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/inspire-addresses.gml'}
-2014-11-24 11:04:19,138 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/10_jinja2_templating
-2014-11-24 11:04:19,138 chain INFO Running Chain: input_addresses_csv|convert_record_array_to_struct|filter_template_addresses2inspire|output_inspire_addresses
-2014-11-24 11:04:19,139 fileinput INFO Open CSV file: input/addresses.csv
-2014-11-24 11:04:19,139 templatingfilter INFO Init: templating
-2014-11-24 11:04:19,139 templatingfilter INFO Read JSON file with globals from: input/addresses-globals.json
-2014-11-24 11:04:19,231 templatingfilter INFO template file read and template created OK: templates/addresses2inspire-ad.jinja2
-2014-11-24 11:04:19,290 fileoutput INFO writing to file output/inspire-addresses.gml
-2014-11-24 11:04:19,291 fileoutput INFO written to output/inspire-addresses.gml
-2014-11-24 11:04:19,291 templatingfilter INFO Exit: templating
-2014-11-24 11:04:19,291 chain INFO DONE - 1 rounds - chain=input_addresses_csv|convert_record_array_to_struct|filter_template_addresses2inspire|output_inspire_addresses
-2014-11-24 11:04:19,291 util INFO Timer end: total ETL time=3.0 sec
-2014-11-24 11:04:19,291 ETL INFO ALL DONE
+2017-11-07 15:34:32,338 util INFO Found cStringIO, good!
+2017-11-07 15:34:32,356 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:32,382 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:32,388 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:32,388 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:32,389 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/10_jinja2_templating
+2017-11-07 15:34:32,389 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:32,390 ETL INFO START
+2017-11-07 15:34:32,390 util INFO Timer start: total ETL
+2017-11-07 15:34:32,390 chain INFO Assembling Chain: input_json|filter_template_xml|output_xml_file...
+2017-11-07 15:34:32,395 input INFO cfg = {'class': 'inputs.fileinput.JsonFileInput', 'file_path': 'input/cities.json'}
+2017-11-07 15:34:32,396 fileinput INFO file_list=['input/cities.json']
+2017-11-07 15:34:32,397 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/cities.xml'}
+2017-11-07 15:34:32,397 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/10_jinja2_templating
+2017-11-07 15:34:32,397 chain INFO Running Chain: input_json|filter_template_xml|output_xml_file
+2017-11-07 15:34:32,397 templatingfilter INFO Init: templating
+2017-11-07 15:34:32,427 templatingfilter INFO template file read and template created OK: templates/cities-json2xml.jinja2
+2017-11-07 15:34:32,427 fileinput INFO Read/parse for start for file=input/cities.json....
+2017-11-07 15:34:32,427 fileinput INFO Read/parse ok for file=input/cities.json
+2017-11-07 15:34:32,427 fileinput INFO all files done
+2017-11-07 15:34:32,428 fileoutput INFO writing to file output/cities.xml
+2017-11-07 15:34:32,428 fileoutput INFO written to output/cities.xml
+2017-11-07 15:34:32,428 templatingfilter INFO Exit: templating
+2017-11-07 15:34:32,428 chain INFO DONE - 1 rounds - chain=input_json|filter_template_xml|output_xml_file
+2017-11-07 15:34:32,428 chain INFO Assembling Chain: input_json|filter_template_gml|output_gml_file...
+2017-11-07 15:34:32,428 input INFO cfg = {'class': 'inputs.fileinput.JsonFileInput', 'file_path': 'input/cities.json'}
+2017-11-07 15:34:32,428 fileinput INFO file_list=['input/cities.json']
+2017-11-07 15:34:32,428 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/cities.gml'}
+2017-11-07 15:34:32,428 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/10_jinja2_templating
+2017-11-07 15:34:32,429 chain INFO Running Chain: input_json|filter_template_gml|output_gml_file
+2017-11-07 15:34:32,429 templatingfilter INFO Init: templating
+2017-11-07 15:34:32,429 templatingfilter INFO Read JSON file with globals from: input/globals.json
+2017-11-07 15:34:32,437 templatingfilter INFO template file read and template created OK: templates/cities-json2gml.jinja2
+2017-11-07 15:34:32,437 fileinput INFO Read/parse for start for file=input/cities.json....
+2017-11-07 15:34:32,437 fileinput INFO Read/parse ok for file=input/cities.json
+2017-11-07 15:34:32,437 fileinput INFO all files done
+2017-11-07 15:34:32,449 fileoutput INFO writing to file output/cities.gml
+2017-11-07 15:34:32,450 fileoutput INFO written to output/cities.gml
+2017-11-07 15:34:32,450 templatingfilter INFO Exit: templating
+2017-11-07 15:34:32,450 chain INFO DONE - 1 rounds - chain=input_json|filter_template_gml|output_gml_file
+2017-11-07 15:34:32,450 chain INFO Assembling Chain: input_geojson|filter_template_geojson2gml|output_gml_file2...
+2017-11-07 15:34:32,450 input INFO cfg = {'output_format': 'geojson_collection', 'class': 'inputs.fileinput.JsonFileInput', 'file_path': 'https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json'}
+2017-11-07 15:34:32,450 fileinput INFO file_list=['https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json']
+2017-11-07 15:34:32,451 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/cities-gjson.gml'}
+2017-11-07 15:34:32,451 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/10_jinja2_templating
+2017-11-07 15:34:32,451 chain INFO Running Chain: input_geojson|filter_template_geojson2gml|output_gml_file2
+2017-11-07 15:34:32,451 templatingfilter INFO Init: templating
+2017-11-07 15:34:32,451 templatingfilter INFO Read JSON file with globals from: input/globals.json
+2017-11-07 15:34:32,451 templatingfilter INFO Read JSON file with globals from: https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/more-globals.json
+2017-11-07 15:34:37,691 templatingfilter INFO template file read and template created OK: templates/cities-gjson2gml.jinja2
+2017-11-07 15:34:37,691 fileinput INFO Read/parse for start for file=https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json....
+2017-11-07 15:34:38,268 fileinput INFO Read/parse ok for file=https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json
+2017-11-07 15:34:38,268 fileinput INFO all files done
+2017-11-07 15:34:38,283 fileoutput INFO writing to file output/cities-gjson.gml
+2017-11-07 15:34:38,283 fileoutput INFO written to output/cities-gjson.gml
+2017-11-07 15:34:38,283 templatingfilter INFO Exit: templating
+2017-11-07 15:34:38,283 chain INFO DONE - 1 rounds - chain=input_geojson|filter_template_geojson2gml|output_gml_file2
+2017-11-07 15:34:38,284 chain INFO Assembling Chain: input_geojson|filter_template_geojson2gml|output_std...
+2017-11-07 15:34:38,284 input INFO cfg = {'output_format': 'geojson_collection', 'class': 'inputs.fileinput.JsonFileInput', 'file_path': 'https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json'}
+2017-11-07 15:34:38,284 fileinput INFO file_list=['https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json']
+2017-11-07 15:34:38,284 output INFO cfg = {'class': 'outputs.standardoutput.StandardOutput'}
+2017-11-07 15:34:38,284 chain INFO Running Chain: input_geojson|filter_template_geojson2gml|output_std
+2017-11-07 15:34:38,284 templatingfilter INFO Init: templating
+2017-11-07 15:34:38,284 templatingfilter INFO Read JSON file with globals from: input/globals.json
+2017-11-07 15:34:38,284 templatingfilter INFO Read JSON file with globals from: https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/more-globals.json
+2017-11-07 15:34:38,349 templatingfilter INFO template file read and template created OK: templates/cities-gjson2gml.jinja2
+2017-11-07 15:34:38,349 fileinput INFO Read/parse for start for file=https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json....
+2017-11-07 15:34:38,411 fileinput INFO Read/parse ok for file=https://raw.githubusercontent.com/justb4/stetl/master/examples/basics/10_jinja2_templating/input/cities-gjson.json
+2017-11-07 15:34:38,411 fileinput INFO all files done
+2017-11-07 15:34:38,418 templatingfilter INFO Exit: templating
+2017-11-07 15:34:38,418 chain INFO DONE - 1 rounds - chain=input_geojson|filter_template_geojson2gml|output_std
+2017-11-07 15:34:38,418 chain INFO Assembling Chain: input_addresses_csv|convert_record_array_to_struct|filter_template_addresses2inspire|output_inspire_addresses...
+2017-11-07 15:34:38,419 input INFO cfg = {'class': 'inputs.fileinput.CsvFileInput', 'file_path': 'input/addresses.csv'}
+2017-11-07 15:34:38,419 fileinput INFO file_list=['input/addresses.csv']
+2017-11-07 15:34:38,419 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/inspire-addresses.gml'}
+2017-11-07 15:34:38,419 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/10_jinja2_templating
+2017-11-07 15:34:38,419 chain INFO Running Chain: input_addresses_csv|convert_record_array_to_struct|filter_template_addresses2inspire|output_inspire_addresses
+2017-11-07 15:34:38,419 fileinput INFO Open CSV file: input/addresses.csv
+2017-11-07 15:34:38,419 templatingfilter INFO Init: templating
+2017-11-07 15:34:38,419 templatingfilter INFO Read JSON file with globals from: input/addresses-globals.json
+2017-11-07 15:34:38,445 templatingfilter INFO template file read and template created OK: templates/addresses2inspire-ad.jinja2
+2017-11-07 15:34:38,453 fileoutput INFO writing to file output/inspire-addresses.gml
+2017-11-07 15:34:38,453 fileoutput INFO written to output/inspire-addresses.gml
+2017-11-07 15:34:38,453 templatingfilter INFO Exit: templating
+2017-11-07 15:34:38,453 chain INFO DONE - 1 rounds - chain=input_addresses_csv|convert_record_array_to_struct|filter_template_addresses2inspire|output_inspire_addresses
+2017-11-07 15:34:38,453 util INFO Timer end: total ETL time=6.0 sec
+2017-11-07 15:34:38,453 ETL INFO ALL DONE
<?xml version='1.0' encoding='utf-8'?>
<cities:FeatureCollection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:cities="http://cities.maptools.org/"
@@ -130,7 +131,7 @@
<cities:name>Amsterdam</cities:name>
<cities:population>779808</cities:population>
<cities:geometry>
- <gml:Point srsName="urn:ogc:def:crs:EPSG::4258" gml:id="point-1"><gml:pos>52.373045454545455 4.894836363636363</gml:pos></gml:Point>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG::4258" gml:id="point-1"><gml:pos>52.3730454545455 4.89483636363636</gml:pos></gml:Point>
</cities:geometry>
</cities:City>
</gml:featureMember>
@@ -139,7 +140,7 @@
<cities:name>Bonn</cities:name>
<cities:population>327913</cities:population>
<cities:geometry>
- <gml:Point srsName="urn:ogc:def:crs:EPSG::4258" gml:id="point-2"><gml:pos>50.734554545454543 7.099818181818182</gml:pos></gml:Point>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG::4258" gml:id="point-2"><gml:pos>50.7345545454545 7.09981818181818</gml:pos></gml:Point>
</cities:geometry>
</cities:City>
</gml:featureMember>
@@ -154,274 +155,598 @@
</gml:featureMember>
</cities:FeatureCollection>
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/11_formatconvert ~/project/stetl/git/examples/basics
==== running etl.sh for 11_formatconvert ====
-2014-11-24 11:04:19,609 util INFO Found cStringIO, good!
-2014-11-24 11:04:19,639 util INFO Found lxml.etree, native XML parsing, fabulous!
-2014-11-24 11:04:19,666 util INFO Found GDAL/OGR Python bindings, super!!
-2014-11-24 11:04:19,680 main INFO Stetl version = 1.0.7
-2014-11-24 11:04:19,681 ETL INFO INIT - Stetl version is 1.0.7
-2014-11-24 11:04:19,681 ETL INFO Config/working dir =/Users/just/project/stetl/git/examples/basics/11_formatconvert
-2014-11-24 11:04:19,682 ETL INFO Reading config_file = etl.cfg
-2014-11-24 11:04:19,693 ETL INFO START
-2014-11-24 11:04:19,693 util INFO Timer start: total ETL
-2014-11-24 11:04:19,693 chain INFO Assembling Chain: input_xml_file|convert_xml_to_string|convert_string_to_xml|output_xml_file...
-2014-11-24 11:04:19,695 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
-2014-11-24 11:04:19,696 fileinput INFO file_list=['input/cities.xml']
-2014-11-24 11:04:19,697 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/cities.xml'}
-2014-11-24 11:04:19,698 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/11_formatconvert
-2014-11-24 11:04:19,698 chain INFO Running Chain: input_xml_file|convert_xml_to_string|convert_string_to_xml|output_xml_file
-2014-11-24 11:04:19,698 fileinput INFO Read/parse for start for file=input/cities.xml....
-2014-11-24 11:04:19,796 fileinput INFO Read/parse ok for file=input/cities.xml
-2014-11-24 11:04:19,796 fileinput INFO all files done
-2014-11-24 11:04:19,797 fileoutput INFO writing to file output/cities.xml
-2014-11-24 11:04:19,797 fileoutput INFO written to output/cities.xml
-2014-11-24 11:04:19,797 chain INFO DONE - 1 rounds - chain=input_xml_file|convert_xml_to_string|convert_string_to_xml|output_xml_file
-2014-11-24 11:04:19,797 chain INFO Assembling Chain: input_complex_xml_file|convert_to_json|output_json_file...
-2014-11-24 11:04:19,797 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/building-dutch-bag.xml'}
-2014-11-24 11:04:19,798 fileinput INFO file_list=['input/building-dutch-bag.xml']
-2014-11-24 11:04:19,798 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/building-dutch.json'}
-2014-11-24 11:04:19,798 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/11_formatconvert
-2014-11-24 11:04:19,798 chain INFO Running Chain: input_complex_xml_file|convert_to_json|output_json_file
-2014-11-24 11:04:19,798 fileinput INFO Read/parse for start for file=input/building-dutch-bag.xml....
-2014-11-24 11:04:19,885 fileinput INFO Read/parse ok for file=input/building-dutch-bag.xml
-2014-11-24 11:04:19,885 fileinput INFO all files done
-2014-11-24 11:04:19,926 fileoutput INFO writing to file output/building-dutch.json
-2014-11-24 11:04:19,938 fileoutput INFO written to output/building-dutch.json
-2014-11-24 11:04:19,945 chain INFO DONE - 1 rounds - chain=input_complex_xml_file|convert_to_json|output_json_file
-2014-11-24 11:04:19,945 chain INFO Assembling Chain: input_gml_sf_file|convert_to_geojson|output_geojson_file...
-2014-11-24 11:04:19,945 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.gml'}
-2014-11-24 11:04:19,946 fileinput INFO file_list=['input/cities.gml']
-2014-11-24 11:04:19,946 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/cities-gjson.json'}
-2014-11-24 11:04:19,946 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/11_formatconvert
-2014-11-24 11:04:19,946 chain INFO Running Chain: input_gml_sf_file|convert_to_geojson|output_geojson_file
-2014-11-24 11:04:19,946 fileinput INFO Read/parse for start for file=input/cities.gml....
-2014-11-24 11:04:19,999 fileinput INFO Read/parse ok for file=input/cities.gml
-2014-11-24 11:04:20,000 fileinput INFO all files done
-2014-11-24 11:04:20,002 fileoutput INFO writing to file output/cities-gjson.json
-2014-11-24 11:04:20,002 fileoutput INFO written to output/cities-gjson.json
-2014-11-24 11:04:20,002 chain INFO DONE - 1 rounds - chain=input_gml_sf_file|convert_to_geojson|output_geojson_file
-2014-11-24 11:04:20,002 util INFO Timer end: total ETL time=0.0 sec
-2014-11-24 11:04:20,003 ETL INFO ALL DONE
+2017-11-07 15:34:38,791 util INFO Found cStringIO, good!
+2017-11-07 15:34:38,810 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:38,835 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:38,841 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:38,842 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:38,842 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/11_formatconvert
+2017-11-07 15:34:38,842 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:38,844 ETL INFO START
+2017-11-07 15:34:38,844 util INFO Timer start: total ETL
+2017-11-07 15:34:38,844 chain INFO Assembling Chain: input_xml_file|convert_xml_to_string|convert_string_to_xml|output_xml_file...
+2017-11-07 15:34:38,847 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
+2017-11-07 15:34:38,847 fileinput INFO file_list=['input/cities.xml']
+2017-11-07 15:34:38,848 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/cities.xml'}
+2017-11-07 15:34:38,849 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/11_formatconvert
+2017-11-07 15:34:38,849 chain INFO Running Chain: input_xml_file|convert_xml_to_string|convert_string_to_xml|output_xml_file
+2017-11-07 15:34:38,849 fileinput INFO Read/parse for start for file=input/cities.xml....
+2017-11-07 15:34:38,852 fileinput INFO Read/parse ok for file=input/cities.xml
+2017-11-07 15:34:38,852 fileinput INFO all files done
+2017-11-07 15:34:38,853 fileoutput INFO writing to file output/cities.xml
+2017-11-07 15:34:38,853 fileoutput INFO written to output/cities.xml
+2017-11-07 15:34:38,853 chain INFO DONE - 1 rounds - chain=input_xml_file|convert_xml_to_string|convert_string_to_xml|output_xml_file
+2017-11-07 15:34:38,853 chain INFO Assembling Chain: input_complex_xml_file|convert_to_json|output_json_file...
+2017-11-07 15:34:38,854 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/building-dutch-bag.xml'}
+2017-11-07 15:34:38,854 fileinput INFO file_list=['input/building-dutch-bag.xml']
+2017-11-07 15:34:38,854 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/building-dutch.json'}
+2017-11-07 15:34:38,854 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/11_formatconvert
+2017-11-07 15:34:38,854 chain INFO Running Chain: input_complex_xml_file|convert_to_json|output_json_file
+2017-11-07 15:34:38,854 fileinput INFO Read/parse for start for file=input/building-dutch-bag.xml....
+2017-11-07 15:34:38,855 fileinput INFO Read/parse ok for file=input/building-dutch-bag.xml
+2017-11-07 15:34:38,855 fileinput INFO all files done
+2017-11-07 15:34:38,865 fileoutput INFO writing to file output/building-dutch.json
+2017-11-07 15:34:38,869 fileoutput INFO written to output/building-dutch.json
+2017-11-07 15:34:38,870 chain INFO DONE - 1 rounds - chain=input_complex_xml_file|convert_to_json|output_json_file
+2017-11-07 15:34:38,870 chain INFO Assembling Chain: input_gml_sf_file|convert_to_geojson|output_geojson_file...
+2017-11-07 15:34:38,870 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.gml'}
+2017-11-07 15:34:38,870 fileinput INFO file_list=['input/cities.gml']
+2017-11-07 15:34:38,870 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/cities-gjson.json'}
+2017-11-07 15:34:38,871 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/11_formatconvert
+2017-11-07 15:34:38,871 chain INFO Running Chain: input_gml_sf_file|convert_to_geojson|output_geojson_file
+2017-11-07 15:34:38,871 fileinput INFO Read/parse for start for file=input/cities.gml....
+2017-11-07 15:34:38,871 fileinput INFO Read/parse ok for file=input/cities.gml
+2017-11-07 15:34:38,872 fileinput INFO all files done
+2017-11-07 15:34:38,874 fileoutput INFO writing to file output/cities-gjson.json
+2017-11-07 15:34:38,874 fileoutput INFO written to output/cities-gjson.json
+2017-11-07 15:34:38,875 chain INFO DONE - 1 rounds - chain=input_gml_sf_file|convert_to_geojson|output_geojson_file
+2017-11-07 15:34:38,875 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:38,875 ETL INFO ALL DONE
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/12_gdal_ogr ~/project/stetl/git/examples/basics
==== running etl.sh for 12_gdal_ogr ====
-2014-11-24 11:04:20,561 util INFO Found cStringIO, good!
-2014-11-24 11:04:20,642 util INFO Found lxml.etree, native XML parsing, fabulous!
-2014-11-24 11:04:20,667 util INFO Found GDAL/OGR Python bindings, super!!
-2014-11-24 11:04:20,725 main INFO Stetl version = 1.0.7
-2014-11-24 11:04:20,727 ETL INFO INIT - Stetl version is 1.0.7
-2014-11-24 11:04:20,727 ETL INFO Config/working dir =/Users/just/project/stetl/git/examples/basics/12_gdal_ogr
-2014-11-24 11:04:20,727 ETL INFO Reading config_file = etl.cfg
-2014-11-24 11:04:20,749 ETL INFO START
-2014-11-24 11:04:20,749 util INFO Timer start: total ETL
-2014-11-24 11:04:20,749 chain INFO Assembling Chain: input_gml_file|convert_to_geojson_feature|output_std...
-2014-11-24 11:04:20,937 util INFO Found cStringIO, good!
-2014-11-24 11:04:20,937 util INFO Found lxml.etree, native XML parsing, fabulous!
-2014-11-24 11:04:20,937 util INFO Found GDAL/OGR Python bindings, super!!
-2014-11-24 11:04:20,938 input INFO cfg = {'source_format': 'GML', 'data_source': 'input/cities.gml', 'output_format': 'ogr_feature', 'class': 'inputs.ogrinput.OgrInput', 'source_options': "{'GDAL_CACHEMAX': '64', 'CPL_DEBUG': 'OFF'}"}
-2014-11-24 11:04:21,011 output INFO cfg = {'class': 'outputs.standardoutput.StandardOutput'}
-2014-11-24 11:04:21,011 chain INFO Running Chain: input_gml_file|convert_to_geojson_feature|output_std
-2014-11-24 11:04:21,011 ogrinput INFO Using GDAL/OGR version: 1100100
-2014-11-24 11:04:21,093 ogrinput INFO Opened OGR source ok: input/cities.gml layer count=1
-2014-11-24 11:04:21,094 ogrinput INFO Start reading from OGR Source: input/cities.gml, Layer: heronfeat
-2014-11-24 11:04:21,121 ogrinput INFO End reading from Layer: heronfeat
-2014-11-24 11:04:21,121 ogrinput INFO Closing OGR source: input/cities.gml
-2014-11-24 11:04:21,121 chain INFO DONE - 7 rounds - chain=input_gml_file|convert_to_geojson_feature|output_std
-2014-11-24 11:04:21,121 chain INFO Assembling Chain: input_gml_file_to_array|convert_to_geojson_collection|tolowercase_filter|output_file...
-2014-11-24 11:04:21,121 input INFO cfg = {'source_format': 'GML', 'data_source': 'input/cities.gml', 'output_format': 'ogr_feature_array', 'class': 'inputs.ogrinput.OgrInput'}
-2014-11-24 11:04:21,172 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/cities.geojson'}
-2014-11-24 11:04:21,172 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/12_gdal_ogr
-2014-11-24 11:04:21,172 chain INFO Running Chain: input_gml_file_to_array|convert_to_geojson_collection|tolowercase_filter|output_file
-2014-11-24 11:04:21,173 ogrinput INFO Using GDAL/OGR version: 1100100
-2014-11-24 11:04:21,173 ogrinput INFO Opened OGR source ok: input/cities.gml layer count=1
-2014-11-24 11:04:21,174 ogrinput INFO Start reading from OGR Source: input/cities.gml, Layer: heronfeat
-2014-11-24 11:04:21,174 ogrinput INFO End reading all features from Layer: heronfeat count=5
-2014-11-24 11:04:21,178 fileoutput INFO writing to file output/cities.geojson
-2014-11-24 11:04:21,179 fileoutput INFO written to output/cities.geojson
-2014-11-24 11:04:21,179 ogrinput INFO Closing OGR source: input/cities.gml
-2014-11-24 11:04:21,179 chain INFO DONE - 2 rounds - chain=input_gml_file_to_array|convert_to_geojson_collection|tolowercase_filter|output_file
-2014-11-24 11:04:21,179 util INFO Timer end: total ETL time=0.0 sec
-2014-11-24 11:04:21,179 ETL INFO ALL DONE
-{
- "geometry": {
- "type": "Point",
- "coordinates": [
- 4.632,
- 52.387
- ]
- },
- "type": "Feature",
- "properties": {
- "STATUS": "Provincial capital",
- "ObjectID": 12386304,
- "POP_RANK": 5,
- "CNTRY_NAME": "Netherlands",
- "GMI_ADMIN": "NLD-NHL",
- "gml_id": "cities.1358",
- "ADMIN_NAME": "Noord-Holland",
- "LABEL_FLAG": 0,
- "CITY_NAME": "Haarlem",
- "POP_CLASS": "100,000 to 250,000",
- "PORT_ID": 0,
- "FIPS_CNTRY": "NL"
- },
- "id": 1358
-}
-{
- "geometry": {
- "type": "Point",
- "coordinates": [
- 4.89483636,
- 52.37304545
- ]
- },
- "type": "Feature",
- "properties": {
- "STATUS": "National capital",
- "ObjectID": 12386305,
- "POP_RANK": 3,
- "CNTRY_NAME": "Netherlands",
- "GMI_ADMIN": "NLD-NHL",
- "gml_id": "cities.1359",
- "ADMIN_NAME": "Noord-Holland",
- "LABEL_FLAG": 0,
- "CITY_NAME": "Amsterdam",
- "POP_CLASS": "500,000 to 1,000,000",
- "PORT_ID": 31060,
- "FIPS_CNTRY": "NL"
- },
- "id": 1359
-}
-{
- "geometry": {
- "type": "Point",
- "coordinates": [
- 5.112,
- 52.1
- ]
- },
- "type": "Feature",
- "properties": {
- "STATUS": "Provincial capital",
- "ObjectID": 12386306,
- "POP_RANK": 5,
- "CNTRY_NAME": "Netherlands",
- "GMI_ADMIN": "NLD-UTR",
- "gml_id": "cities.1360",
- "ADMIN_NAME": "Utrecht",
- "LABEL_FLAG": 1,
- "CITY_NAME": "Utrecht",
- "POP_CLASS": "100,000 to 250,000",
- "PORT_ID": 0,
- "FIPS_CNTRY": "NL"
- },
- "id": 1360
-}
-{
- "geometry": {
- "type": "Point",
- "coordinates": [
- 4.281,
- 52.076
- ]
- },
- "type": "Feature",
- "properties": {
- "STATUS": "Provincial capital",
- "ObjectID": 12386307,
- "POP_RANK": 4,
- "CNTRY_NAME": "Netherlands",
- "GMI_ADMIN": "NLD-ZHL",
- "gml_id": "cities.1361",
- "ADMIN_NAME": "Zuid-Holland",
- "LABEL_FLAG": 0,
- "CITY_NAME": "The Hague",
- "POP_CLASS": "250,000 to 500,000",
- "PORT_ID": 0,
- "FIPS_CNTRY": "NL"
- },
- "id": 1361
-}
-{
- "geometry": {
- "type": "Point",
- "coordinates": [
- 4.48515455,
- 51.92559091
- ]
- },
- "type": "Feature",
- "properties": {
- "STATUS": "Other",
- "ObjectID": 12386308,
- "POP_RANK": 2,
- "CNTRY_NAME": "Netherlands",
- "GMI_ADMIN": "NLD-ZHL",
- "gml_id": "cities.1362",
- "ADMIN_NAME": "Zuid-Holland",
- "LABEL_FLAG": 0,
- "CITY_NAME": "Rotterdam",
- "POP_CLASS": "1,000,000 to 5,000,000",
- "PORT_ID": 31140,
- "FIPS_CNTRY": "NL"
- },
- "id": 1362
-}
+2017-11-07 15:34:39,190 util INFO Found cStringIO, good!
+2017-11-07 15:34:39,209 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:39,234 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:39,241 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:39,241 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:39,242 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/12_gdal_ogr
+2017-11-07 15:34:39,242 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:39,243 ETL INFO START
+2017-11-07 15:34:39,243 util INFO Timer start: total ETL
+2017-11-07 15:34:39,243 chain INFO Assembling Chain: input_geojson_file|geojson_coll_to_ogr_feature_arr|output_ogr_gmlfile...
+2017-11-07 15:34:39,247 input INFO cfg = {'output_format': 'geojson_collection', 'class': 'inputs.fileinput.JsonFileInput', 'file_path': 'input/cities.geojson'}
+2017-11-07 15:34:39,247 fileinput INFO file_list=['input/cities.geojson']
+2017-11-07 15:34:39,250 output INFO cfg = {'input_format': 'ogr_feature_array', 'dest_data_source': 'output/cities.gml', 'dest_format': 'GML', 'new_layer_name': 'cities', 'class': 'outputs.ogroutput.OgrOutput', 'overwrite': 'True'}
+2017-11-07 15:34:39,250 chain INFO Running Chain: input_geojson_file|geojson_coll_to_ogr_feature_arr|output_ogr_gmlfile
+2017-11-07 15:34:39,250 ogroutput INFO Using GDAL/OGR version: 2020200
+2017-11-07 15:34:39,255 ogroutput INFO Opened OGR dest ok: output/cities.gml
+2017-11-07 15:34:39,255 fileinput INFO Read/parse for start for file=input/cities.geojson....
+2017-11-07 15:34:39,256 fileinput INFO Read/parse ok for file=input/cities.geojson
+2017-11-07 15:34:39,256 fileinput INFO all files done
+2017-11-07 15:34:39,257 ogroutput INFO End writing to: output/cities.gml
+2017-11-07 15:34:39,258 chain INFO DONE - 1 rounds - chain=input_geojson_file|geojson_coll_to_ogr_feature_arr|output_ogr_gmlfile
+2017-11-07 15:34:39,258 chain INFO Assembling Chain: input_gml_file_to_array|ogr_feature_array_to_geojson_collection|tolowercase_filter|output_file...
+2017-11-07 15:34:39,258 input INFO cfg = {'source_format': 'GML', 'data_source': 'input/cities.gml', 'output_format': 'ogr_feature_array', 'class': 'inputs.ogrinput.OgrInput'}
+2017-11-07 15:34:39,259 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/cities.geojson'}
+2017-11-07 15:34:39,259 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/12_gdal_ogr
+2017-11-07 15:34:39,259 chain INFO Running Chain: input_gml_file_to_array|ogr_feature_array_to_geojson_collection|tolowercase_filter|output_file
+2017-11-07 15:34:39,259 ogrinput INFO Using GDAL/OGR version: 2020200
+2017-11-07 15:34:39,261 ogrinput INFO Opened OGR source ok: input/cities.gml layer count=1
+2017-11-07 15:34:39,261 ogrinput INFO Start reading from OGR Source: input/cities.gml, Layer: heronfeat
+2017-11-07 15:34:39,261 ogrinput INFO End reading all features from Layer: heronfeat count=5
+2017-11-07 15:34:39,264 fileoutput INFO writing to file output/cities.geojson
+2017-11-07 15:34:39,265 fileoutput INFO written to output/cities.geojson
+2017-11-07 15:34:39,265 ogrinput INFO Closing OGR source: input/cities.gml
+2017-11-07 15:34:39,266 chain INFO DONE - 2 rounds - chain=input_gml_file_to_array|ogr_feature_array_to_geojson_collection|tolowercase_filter|output_file
+2017-11-07 15:34:39,266 chain INFO Assembling Chain: input_gml_file|output_ogr_shapefile...
+2017-11-07 15:34:39,266 input INFO cfg = {'source_format': 'GML', 'data_source': 'input/cities.gml', 'output_format': 'ogr_feature', 'class': 'inputs.ogrinput.OgrInput', 'source_options': "{'GDAL_CACHEMAX': '64', 'CPL_DEBUG': 'OFF'}"}
+2017-11-07 15:34:39,266 output INFO cfg = {'input_format': 'ogr_feature', 'dest_data_source': 'output/cities.shp', 'dest_format': 'ESRI Shapefile', 'new_layer_name': 'cities', 'class': 'outputs.ogroutput.OgrOutput', 'overwrite': 'True'}
+2017-11-07 15:34:39,266 chain INFO Running Chain: input_gml_file|output_ogr_shapefile
+2017-11-07 15:34:39,266 ogrinput INFO Using GDAL/OGR version: 2020200
+2017-11-07 15:34:39,267 ogrinput INFO Opened OGR source ok: input/cities.gml layer count=1
+2017-11-07 15:34:39,267 ogroutput INFO Using GDAL/OGR version: 2020200
+2017-11-07 15:34:39,270 ogroutput INFO Opened OGR dest ok: output/cities.shp
+2017-11-07 15:34:39,270 ogrinput INFO Start reading from OGR Source: input/cities.gml, Layer: heronfeat
+2017-11-07 15:34:39,270 ogrinput INFO End reading from Layer: heronfeat
+2017-11-07 15:34:39,271 ogroutput INFO End writing to: output/cities.shp
+2017-11-07 15:34:39,271 ogrinput INFO Closing OGR source: input/cities.gml
+2017-11-07 15:34:39,271 ogroutput INFO End writing to: output/cities.shp
+2017-11-07 15:34:39,271 chain INFO DONE - 7 rounds - chain=input_gml_file|output_ogr_shapefile
+2017-11-07 15:34:39,271 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:39,271 ETL INFO ALL DONE
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/13_dbinput ~/project/stetl/git/examples/basics
==== running etl.sh for 13_dbinput ====
-2014-11-24 11:04:21,344 util INFO Found cStringIO, good!
-2014-11-24 11:04:21,373 util INFO Found lxml.etree, native XML parsing, fabulous!
-2014-11-24 11:04:21,409 util INFO Found GDAL/OGR Python bindings, super!!
-2014-11-24 11:04:21,422 main INFO Stetl version = 1.0.7
-2014-11-24 11:04:21,423 ETL INFO INIT - Stetl version is 1.0.7
-2014-11-24 11:04:21,423 ETL INFO Config/working dir =/Users/just/project/stetl/git/examples/basics/13_dbinput
-2014-11-24 11:04:21,423 ETL INFO Reading config_file = etl.cfg
-2014-11-24 11:04:21,458 ETL INFO START
-2014-11-24 11:04:21,458 util INFO Timer start: total ETL
-2014-11-24 11:04:21,458 chain INFO Assembling Chain: input_sqlite_all|convert_records_to_json|output_cities_file...
-2014-11-24 11:04:21,796 input INFO cfg = {'database_name': 'input/cities.sdb', 'class': 'inputs.dbinput.SqliteDbInput', 'read_once': 'True', 'table': 'cities', 'query': 'select * from cities', 'output_format': 'record_array'}
-2014-11-24 11:04:22,462 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/cities.json'}
-2014-11-24 11:04:22,462 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/13_dbinput
-2014-11-24 11:04:22,462 chain INFO Running Chain: input_sqlite_all|convert_records_to_json|output_cities_file
-2014-11-24 11:04:22,462 dbinput INFO Connect to SQLite DB: input/cities.sdb
-2014-11-24 11:04:22,791 dbinput INFO Connect to SQLite DB: input/cities.sdb
-2014-11-24 11:04:22,792 dbinput INFO 3 records read
-2014-11-24 11:04:22,792 dbinput INFO Nothing to do. All file_records done
-2014-11-24 11:04:22,799 fileoutput INFO writing to file output/cities.json
-2014-11-24 11:04:22,800 fileoutput INFO written to output/cities.json
-2014-11-24 11:04:22,800 chain INFO DONE - 1 rounds - chain=input_sqlite_all|convert_records_to_json|output_cities_file
-2014-11-24 11:04:22,800 chain INFO Assembling Chain: input_sqlite_single|convert_record_to_json|output_city_file...
-2014-11-24 11:04:22,800 input INFO cfg = {'database_name': 'input/cities.sdb', 'class': 'inputs.dbinput.SqliteDbInput', 'read_once': 'True', 'table': 'cities', 'query': "select * from cities where name = 'Rome'", 'output_format': 'record'}
-2014-11-24 11:04:22,801 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/city.json'}
-2014-11-24 11:04:22,801 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/13_dbinput
-2014-11-24 11:04:22,801 chain INFO Running Chain: input_sqlite_single|convert_record_to_json|output_city_file
-2014-11-24 11:04:22,801 dbinput INFO Connect to SQLite DB: input/cities.sdb
-2014-11-24 11:04:22,802 dbinput INFO Connect to SQLite DB: input/cities.sdb
-2014-11-24 11:04:22,802 dbinput INFO 1 records read
-2014-11-24 11:04:22,802 dbinput INFO Nothing to do. All file_records done
-2014-11-24 11:04:22,803 fileoutput INFO writing to file output/city.json
-2014-11-24 11:04:22,803 fileoutput INFO written to output/city.json
-2014-11-24 11:04:22,803 chain INFO DONE - 1 rounds - chain=input_sqlite_single|convert_record_to_json|output_city_file
-2014-11-24 11:04:22,803 util INFO Timer end: total ETL time=1.0 sec
-2014-11-24 11:04:22,803 ETL INFO ALL DONE
+2017-11-07 15:34:39,636 util INFO Found cStringIO, good!
+2017-11-07 15:34:39,656 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:39,681 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:39,688 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:39,689 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:39,689 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/13_dbinput
+2017-11-07 15:34:39,690 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:39,691 ETL INFO START
+2017-11-07 15:34:39,691 util INFO Timer start: total ETL
+2017-11-07 15:34:39,691 chain INFO Assembling Chain: input_sqlite_all|convert_records_to_json|output_cities_file...
+2017-11-07 15:34:39,715 input INFO cfg = {'database_name': 'input/cities.sdb', 'class': 'inputs.dbinput.SqliteDbInput', 'read_once': 'True', 'table': 'cities', 'query': 'select * from cities', 'output_format': 'record_array'}
+2017-11-07 15:34:39,720 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/cities.json'}
+2017-11-07 15:34:39,720 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/13_dbinput
+2017-11-07 15:34:39,721 chain INFO Running Chain: input_sqlite_all|convert_records_to_json|output_cities_file
+2017-11-07 15:34:39,721 dbinput INFO Connect to SQLite DB: input/cities.sdb
+2017-11-07 15:34:39,726 dbinput INFO Connect to SQLite DB: input/cities.sdb
+2017-11-07 15:34:39,726 dbinput INFO 3 records read
+2017-11-07 15:34:39,726 dbinput INFO Nothing to do. All file_records done
+2017-11-07 15:34:39,727 fileoutput INFO writing to file output/cities.json
+2017-11-07 15:34:39,727 fileoutput INFO written to output/cities.json
+2017-11-07 15:34:39,727 chain INFO DONE - 1 rounds - chain=input_sqlite_all|convert_records_to_json|output_cities_file
+2017-11-07 15:34:39,727 chain INFO Assembling Chain: input_sqlite_single|convert_record_to_json|output_city_file...
+2017-11-07 15:34:39,727 input INFO cfg = {'database_name': 'input/cities.sdb', 'class': 'inputs.dbinput.SqliteDbInput', 'read_once': 'True', 'table': 'cities', 'query': "select * from cities where name = 'Rome'", 'output_format': 'record'}
+2017-11-07 15:34:39,727 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/city.json'}
+2017-11-07 15:34:39,727 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/13_dbinput
+2017-11-07 15:34:39,727 chain INFO Running Chain: input_sqlite_single|convert_record_to_json|output_city_file
+2017-11-07 15:34:39,728 dbinput INFO Connect to SQLite DB: input/cities.sdb
+2017-11-07 15:34:39,728 dbinput INFO Connect to SQLite DB: input/cities.sdb
+2017-11-07 15:34:39,728 dbinput INFO 1 records read
+2017-11-07 15:34:39,728 dbinput INFO Nothing to do. All file_records done
+2017-11-07 15:34:39,728 fileoutput INFO writing to file output/city.json
+2017-11-07 15:34:39,728 fileoutput INFO written to output/city.json
+2017-11-07 15:34:39,729 chain INFO DONE - 1 rounds - chain=input_sqlite_single|convert_record_to_json|output_city_file
+2017-11-07 15:34:39,729 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:39,729 ETL INFO ALL DONE
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/14_logfileinput ~/project/stetl/git/examples/basics
+==== running etl.sh for 14_logfileinput ====
+2017-11-07 15:34:40,032 util INFO Found cStringIO, good!
+2017-11-07 15:34:40,049 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:40,075 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:40,081 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:40,082 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:40,082 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/14_logfileinput
+2017-11-07 15:34:40,082 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:40,084 ETL INFO START
+2017-11-07 15:34:40,084 util INFO Timer start: total ETL
+2017-11-07 15:34:40,084 chain INFO Assembling Chain: input_apache_log|to_record_array|output_std...
+2017-11-07 15:34:40,086 input INFO cfg = {'file_path': 'input', 'log_format': '%h %l %u %t \\"%r\\" %>s %b \\"%{Referer}i\\" \\"%{User-agent}i\\" %D', 'filename_pattern': '*.log', 'class': 'inputs.fileinput.ApacheLogFileInput', 'key_map': '{\'%>s\': \'status\', \'%D\': \'deltat\', \'%{User-agent}i\': \'agent\', \'%b\': \'bytes\', \'%{Referer}i\': \'referer\', \'%t\': \'time\', "\'%h": \'host\', \'%r\': \'request\'}'}
+2017-11-07 15:34:40,086 fileinput INFO file_list=['input/apache.log', 'input/apache2.log']
+2017-11-07 15:34:40,090 output INFO cfg = {'class': 'outputs.standardoutput.StandardOutput'}
+2017-11-07 15:34:40,090 chain INFO Running Chain: input_apache_log|to_record_array|output_std
+2017-11-07 15:34:40,091 fileinput INFO file opened : input/apache.log
+2017-11-07 15:34:40,091 fileinput INFO EOF file
+2017-11-07 15:34:40,091 fileinput INFO file opened : input/apache2.log
+[{'status': 200, 'deltat': 2580, 'request': '/map/gast/tms/1.0.0/opentopo/EPSG28992/6/32/40.png', 'bytes': 32085, 'agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko', 'referer': 'http://217.21.192.135/sekaarten/?id=baggeren', 'key': 'fbc25dbc7cc817a75f0b1bee87958480', 'time': 20150420084459}, {'status': 200, 'deltat': 2939, 'request': '/map/gast/tms/1.0.0/opentopo/EPSG28992/6/32/41.png', 'bytes': 10891, 'agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0 [...]
+[{'status': 500, 'deltat': 251096, 'request': '/map/gast/wmts', 'bytes': 469, 'agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36', 'referer': None, 'key': 'a026db25148bb54e94eda0e2cba28578', 'time': 20150421123333}, {'status': 301, 'deltat': 240, 'request': '/map/gast', 'bytes': 297, 'agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.3 [...]
+[{'status': 500, 'deltat': 3407, 'request': '/map/gast/wmts/1.0.0', 'bytes': 475, 'agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36', 'referer': None, 'key': '1c7ce8547891e786a08d45064e3c6882', 'time': 20150421123420}, {'status': 500, 'deltat': 3203, 'request': '/map/gast/wmts/1.0.0?SERVICE=WMS&REQUEST=GetCapabilities', 'bytes': 475, 'agent': 'Mozilla/5.0 QGIS/2.2.0-Valmiera', 'referer': None, 'key': 'ac29dfd [...]
+[{'status': 200, 'deltat': 243456, 'request': '/map/gast/service?SERVICE=WMS&REQUEST=GetCapabilities', 'bytes': 11637, 'agent': 'Mozilla/5.0 QGIS/2.2.0-Valmiera', 'referer': None, 'key': 'de593d9eecf62db260435229e2e46315', 'time': 20150421123639}, {'status': 200, 'deltat': 18184, 'request': '/map/gast/service?SERVICE=WMS&REQUEST=GetCapabilities', 'bytes': 11637, 'agent': 'Mozilla/5.0 QGIS/2.2.0-Valmiera', 'referer': None, 'key': '7ffbfd35d514a9680ac4ff9c2622a3ad', 'time': 20150421123657} [...]
+[{'status': 200, 'deltat': 2593224, 'request': '/map/gast/service?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&BBOX=4.87428421837786807,52.30826341961707726,4.94064551776737115,52.32733496592636868&SRS=EPSG:4326&WIDTH=2306&HEIGHT=663&LAYERS=opentopo&STYLES=&FORMAT=image/jpeg&DPI=72&MAP_RESOLUTION=72&FORMAT_OPTIONS=dpi:72', 'bytes': 682877, 'agent': 'Mozilla/5.0 QGIS/2.2.0-Valmiera', 'referer': None, 'key': '60ebd56e37c277497c5f240d8bf3daa7', 'time': 20150421123846}, {'status': 200, 'deltat': [...]
+[{'status': 200, 'deltat': 1008772, 'request': '/map/gast/service?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&BBOX=4.65634629858872362,52.116047279264194,5.71784019987099157,52.41501733832054555&SRS=EPSG:4326&WIDTH=2308&HEIGHT=650&LAYERS=opentopo&STYLES=&FORMAT=image/jpeg&DPI=72&MAP_RESOLUTION=72&FORMAT_OPTIONS=dpi:72', 'bytes': 687988, 'agent': 'Mozilla/5.0 QGIS/2.2.0-Valmiera', 'referer': None, 'key': 'a6fa14558bdaaa9176546374ddb6be05', 'time': 20150421123906}, {'status': 200, 'deltat': 2 [...]
+[{'status': 200, 'deltat': 96065, 'request': '/map/gast/service?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&BBOX=4.85085928333482297,52.31989516121636541,4.8529353590157136,52.32049405290776178&SRS=EPSG:4326&WIDTH=2310&HEIGHT=666&LAYERS=opentopo&STYLES=&FORMAT=image/jpeg&DPI=72&MAP_RESOLUTION=72&FORMAT_OPTIONS=dpi:72', 'bytes': 160121, 'agent': 'Mozilla/5.0 QGIS/2.2.0-Valmiera', 'referer': None, 'key': '5ca5425f30d7a1fcf5ccf56f3fc0691a', 'time': 20150421123945}, {'status': 200, 'deltat': 39 [...]
+[{'status': 500, 'deltat': 1937, 'request': '/map/gast/service?LAYERS=opentopo&FORMAT=image%2Fpng&SRS=EPSG%3A4326&EXCEPTIONS=application%2Fvnd.ogc.se_inimage&TRANSPARENT=TRUE&SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&STYLES=&BBOX=3.1235377115391,50.892941613156,5.4511713191977,52.087620966958&WIDTH=1169&HEIGHT=600', 'bytes': 602, 'agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36', 'referer': 'http://s.test.map [...]
+[{'status': 200, 'deltat': 365184, 'request': '/map/gast/service?LAYERS=opentopo&FORMAT=image%2Fpng&SRS=EPSG%3A4326&EXCEPTIONS=application%2Fvnd.ogc.se_inimage&TRANSPARENT=TRUE&SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&STYLES=&BBOX=3.1235377115391,50.892941613156,5.4511713191977,52.087620966958&WIDTH=1169&HEIGHT=600', 'bytes': 334352, 'agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36', 'referer': 'http://s.tes [...]
+[{'status': 200, 'deltat': 340430, 'request': '/map/gast/service?LAYERS=opentopo&FORMAT=image%2Fpng&SRS=EPSG%3A4326&EXCEPTIONS=application%2Fvnd.ogc.se_inimage&TRANSPARENT=TRUE&SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&STYLES=&BBOX=3.1235377115391,50.892941613156,5.4511713191977,52.087620966958&WIDTH=1169&HEIGHT=600', 'bytes': 334352, 'agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36', 'referer': 'http://s.tes [...]
+2017-11-07 15:34:40,096 fileinput INFO EOF file list
+2017-11-07 15:34:40,096 chain INFO DONE - 112 rounds - chain=input_apache_log|to_record_array|output_std
+2017-11-07 15:34:40,096 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:40,096 ETL INFO ALL DONE
+ Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36', 'referer': 'http://s.test.map5.nl/map/gast/service?LAYERS=opentopo&FORMAT=image%2Fpng&SRS=EPSG%3A4326&EXCEPTIONS=application%2Fvnd.ogc.se_inimage&TRANSPARENT=TRUE&SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&STYLES=&BBOX=3.1235377115391,50.892941613156,5.4511713191977,52.087620966958&WIDTH=1169&HEIGHT=600', 'key': 'e161bedea500f0d2201be628b14b8edf', 'time': 20150421124923}]
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/15_splitter ~/project/stetl/git/examples/basics
+==== running etl.sh for 15_splitter ====
+2017-11-07 15:34:40,404 util INFO Found cStringIO, good!
+2017-11-07 15:34:40,423 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:40,449 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:40,456 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:40,457 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:40,457 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/15_splitter
+2017-11-07 15:34:40,457 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:40,457 ETL INFO START
+2017-11-07 15:34:40,457 util INFO Timer start: total ETL
+2017-11-07 15:34:40,457 chain INFO Assembling Chain: input_xml_file|transformer_xslt|(output_file)(output_std)...
+2017-11-07 15:34:40,459 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
+2017-11-07 15:34:40,459 fileinput INFO file_list=['input/cities.xml']
+2017-11-07 15:34:40,463 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/gmlcities.gml'}
+2017-11-07 15:34:40,463 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/15_splitter
+2017-11-07 15:34:40,464 output INFO cfg = {'class': 'outputs.standardoutput.StandardOutput'}
+2017-11-07 15:34:40,464 chain INFO Running Chain: input_xml_file|transformer_xslt|(output_file)(output_std)
+2017-11-07 15:34:40,464 fileinput INFO Read/parse for start for file=input/cities.xml....
+2017-11-07 15:34:40,464 fileinput INFO Read/parse ok for file=input/cities.xml
+2017-11-07 15:34:40,464 fileinput INFO all files done
+2017-11-07 15:34:40,465 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:40,465 fileoutput INFO writing to file output/gmlcities.gml
+2017-11-07 15:34:40,465 fileoutput INFO written to output/gmlcities.gml
+2017-11-07 15:34:40,465 chain INFO DONE - 1 rounds - chain=input_xml_file|transformer_xslt|(output_file)(output_std)
+2017-11-07 15:34:40,465 chain INFO Assembling Chain: input_xml_file |(transformer_xslt|output_file) (output_std) (transformer_xslt|output_std)...
+2017-11-07 15:34:40,465 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
+2017-11-07 15:34:40,466 fileinput INFO file_list=['input/cities.xml']
+2017-11-07 15:34:40,466 chain INFO Assembling Chain: transformer_xslt|output_file...
+2017-11-07 15:34:40,466 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/gmlcities.gml'}
+2017-11-07 15:34:40,466 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/15_splitter
+2017-11-07 15:34:40,466 output INFO cfg = {'class': 'outputs.standardoutput.StandardOutput'}
+2017-11-07 15:34:40,466 chain INFO Assembling Chain: transformer_xslt|output_std...
+2017-11-07 15:34:40,467 output INFO cfg = {'class': 'outputs.standardoutput.StandardOutput'}
+2017-11-07 15:34:40,467 chain INFO Running Chain: input_xml_file |(transformer_xslt|output_file) (output_std) (transformer_xslt|output_std)
+2017-11-07 15:34:40,467 fileinput INFO Read/parse for start for file=input/cities.xml....
+2017-11-07 15:34:40,467 fileinput INFO Read/parse ok for file=input/cities.xml
+2017-11-07 15:34:40,467 fileinput INFO all files done
+2017-11-07 15:34:40,467 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:40,467 fileoutput INFO writing to file output/gmlcities.gml
+2017-11-07 15:34:40,468 fileoutput INFO written to output/gmlcities.gml
+2017-11-07 15:34:40,468 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:40,468 chain INFO DONE - 1 rounds - chain=input_xml_file |(transformer_xslt|output_file) (output_std) (transformer_xslt|output_std)
+2017-11-07 15:34:40,468 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:40,468 ETL INFO ALL DONE
+<?xml version='1.0' encoding='utf-8'?>
+<ogr:FeatureCollection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ogr="http://ogr.maptools.org/" xmlns:gml="http://www.opengis.net/gml" xsi:schemaLocation="http://ogr.maptools.org/ ../gmlcities.xsd http://www.opengis.net/gml http://schemas.opengis.net/gml/2.1.2/feature.xsd">
+ <gml:boundedBy>
+ <gml:Box>
+ <gml:coord>
+ <gml:X>-180.0</gml:X>
+ <gml:Y>-90.0</gml:Y>
+ </gml:coord>
+ <gml:coord>
+ <gml:X>180.0</gml:X>
+ <gml:Y>90.0</gml:Y>
+ </gml:coord>
+ </gml:Box>
+ </gml:boundedBy>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Amsterdam</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>52.4,4.9</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Bonn</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>50.7,7.1</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Rome</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>41.9,12.5</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+</ogr:FeatureCollection>
+
+<?xml version='1.0' encoding='utf-8'?>
+<cities>
+ <city>
+ <name>Amsterdam</name>
+ <lat>52.4</lat>
+ <lon>4.9</lon>
+ </city>
+ <city>
+ <name>Bonn</name>
+ <lat>50.7</lat>
+ <lon>7.1</lon>
+ </city>
+ <city>
+ <name>Rome</name>
+ <lat>41.9</lat>
+ <lon>12.5</lon>
+ </city>
+</cities>
+
+<?xml version='1.0' encoding='utf-8'?>
+<ogr:FeatureCollection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ogr="http://ogr.maptools.org/" xmlns:gml="http://www.opengis.net/gml" xsi:schemaLocation="http://ogr.maptools.org/ ../gmlcities.xsd http://www.opengis.net/gml http://schemas.opengis.net/gml/2.1.2/feature.xsd">
+ <gml:boundedBy>
+ <gml:Box>
+ <gml:coord>
+ <gml:X>-180.0</gml:X>
+ <gml:Y>-90.0</gml:Y>
+ </gml:coord>
+ <gml:coord>
+ <gml:X>180.0</gml:X>
+ <gml:Y>90.0</gml:Y>
+ </gml:coord>
+ </gml:Box>
+ </gml:boundedBy>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Amsterdam</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>52.4,4.9</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Bonn</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>50.7,7.1</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Rome</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>41.9,12.5</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+</ogr:FeatureCollection>
+
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/16_merger ~/project/stetl/git/examples/basics
+==== running etl.sh for 16_merger ====
+2017-11-07 15:34:40,839 util INFO Found cStringIO, good!
+2017-11-07 15:34:40,857 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:40,884 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:40,890 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:40,891 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:40,891 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/16_merger
+2017-11-07 15:34:40,892 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:40,893 ETL INFO START
+2017-11-07 15:34:40,893 util INFO Timer start: total ETL
+2017-11-07 15:34:40,893 chain INFO Assembling Chain: (input_1) (input_2)|transformer_xslt|output_std...
+2017-11-07 15:34:40,895 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input1/cities.xml'}
+2017-11-07 15:34:40,895 fileinput INFO file_list=['input1/cities.xml']
+2017-11-07 15:34:40,895 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input2/cities.xml'}
+2017-11-07 15:34:40,895 fileinput INFO file_list=['input2/cities.xml']
+2017-11-07 15:34:40,898 output INFO cfg = {'class': 'outputs.standardoutput.StandardOutput'}
+2017-11-07 15:34:40,899 chain INFO Running Chain: (input_1) (input_2)|transformer_xslt|output_std
+2017-11-07 15:34:40,899 fileinput INFO Read/parse for start for file=input1/cities.xml....
+2017-11-07 15:34:40,899 fileinput INFO Read/parse ok for file=input1/cities.xml
+2017-11-07 15:34:40,899 fileinput INFO all files done
+2017-11-07 15:34:40,899 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:40,899 fileinput INFO Read/parse for start for file=input2/cities.xml....
+2017-11-07 15:34:40,900 fileinput INFO Read/parse ok for file=input2/cities.xml
+2017-11-07 15:34:40,900 fileinput INFO all files done
+2017-11-07 15:34:40,900 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:40,900 chain INFO DONE - 1 rounds - chain=(input_1) (input_2)|transformer_xslt|output_std
+2017-11-07 15:34:40,900 chain INFO Assembling Chain: (input_1) (input_2)|transformer_xslt|(output_file)(output_std)...
+2017-11-07 15:34:40,900 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input1/cities.xml'}
+2017-11-07 15:34:40,900 fileinput INFO file_list=['input1/cities.xml']
+2017-11-07 15:34:40,900 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input2/cities.xml'}
+2017-11-07 15:34:40,901 fileinput INFO file_list=['input2/cities.xml']
+2017-11-07 15:34:40,901 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/gmlcities.gml'}
+2017-11-07 15:34:40,901 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/16_merger
+2017-11-07 15:34:40,901 output INFO cfg = {'class': 'outputs.standardoutput.StandardOutput'}
+2017-11-07 15:34:40,901 chain INFO Running Chain: (input_1) (input_2)|transformer_xslt|(output_file)(output_std)
+2017-11-07 15:34:40,902 fileinput INFO Read/parse for start for file=input1/cities.xml....
+2017-11-07 15:34:40,902 fileinput INFO Read/parse ok for file=input1/cities.xml
+2017-11-07 15:34:40,902 fileinput INFO all files done
+2017-11-07 15:34:40,902 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:40,902 fileoutput INFO writing to file output/gmlcities.gml
+2017-11-07 15:34:40,902 fileoutput INFO written to output/gmlcities.gml
+<?xml version='1.0' encoding='utf-8'?>
+<ogr:FeatureCollection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ogr="http://ogr.maptools.org/" xmlns:gml="http://www.opengis.net/gml" xsi:schemaLocation="http://ogr.maptools.org/ ../gmlcities.xsd http://www.opengis.net/gml http://schemas.opengis.net/gml/2.1.2/feature.xsd">
+ <gml:boundedBy>
+ <gml:Box>
+ <gml:coord>
+ <gml:X>-180.0</gml:X>
+ <gml:Y>-90.0</gml:Y>
+ </gml:coord>
+ <gml:coord>
+ <gml:X>180.0</gml:X>
+ <gml:Y>90.0</gml:Y>
+ </gml:coord>
+ </gml:Box>
+ </gml:boundedBy>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Amsterdam</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>52.4,4.9</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Bonn</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>50.7,7.1</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Rome</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>41.9,12.5</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+</ogr:FeatureCollection>
+
+<?xml version='1.0' encoding='utf-8'?>
+<ogr:FeatureCollection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ogr="http://ogr.maptools.org/" xmlns:gml="http://www.opengis.net/gml" xsi:schemaLocation="http://ogr.maptools.org/ ../gmlcities.xsd http://www.opengis.net/gml http://schemas.opengis.net/gml/2.1.2/feature.xsd">
+ <gml:boundedBy>
+ <gml:Box>
+ <gml:coord>
+ <gml:X>-180.0</gml:X>
+ <gml:Y>-90.0</gml:Y>
+ </gml:coord>
+ <gml:coord>
+ <gml:X>180.0</gml:X>
+ <gml:Y>90.0</gml:Y>
+ </gml:coord>
+ </gml:Box>
+ </gml:boundedBy>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Amsterdam</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>52.4,4.9</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Bonn</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>50.7,7.1</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Rome</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>41.9,12.5</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+</ogr:FeatureCollection>
+
+<?xml version='1.0' encoding='utf-8'?>
+<ogr:FeatureCollection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ogr="http://ogr.maptools.org/" xmlns:gml="http://www.opengis.net/gml" xsi:schemaLocation="http://ogr.maptools.org/ ../gmlcities.xsd http://www.opengis.net/gml http://schemas.opengis.net/gml/2.1.2/feature.xsd">
+ <gml:boundedBy>
+ <gml:Box>
+ <gml:coord>
+ <gml:X>-180.0</gml:X>
+ <gml:Y>-90.0</gml:Y>
+ </gml:coord>
+ <gml:coord>
+ <gml:X>180.0</gml:X>
+ <gml:Y>90.0</gml:Y>
+ </gml:coord>
+ </gml:Box>
+ </gml:boundedBy>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Amsterdam</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>52.4,4.9</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Bonn</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>50.7,7.1</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <g2017-11-07 15:34:40,902 fileinput INFO Read/parse for start for file=input2/cities.xml....
+2017-11-07 15:34:40,902 fileinput INFO Read/parse ok for file=input2/cities.xml
+2017-11-07 15:34:40,902 fileinput INFO all files done
+2017-11-07 15:34:40,902 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:40,902 fileoutput INFO writing to file output/gmlcities.gml
+2017-11-07 15:34:40,903 fileoutput INFO written to output/gmlcities.gml
+2017-11-07 15:34:40,903 chain INFO DONE - 1 rounds - chain=(input_1) (input_2)|transformer_xslt|(output_file)(output_std)
+2017-11-07 15:34:40,903 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:40,903 ETL INFO ALL DONE
+ml:featureMember>
+ <ogr:City>
+ <ogr:name>Rome</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>41.9,12.5</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+</ogr:FeatureCollection>
+
+<?xml version='1.0' encoding='utf-8'?>
+<ogr:FeatureCollection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ogr="http://ogr.maptools.org/" xmlns:gml="http://www.opengis.net/gml" xsi:schemaLocation="http://ogr.maptools.org/ ../gmlcities.xsd http://www.opengis.net/gml http://schemas.opengis.net/gml/2.1.2/feature.xsd">
+ <gml:boundedBy>
+ <gml:Box>
+ <gml:coord>
+ <gml:X>-180.0</gml:X>
+ <gml:Y>-90.0</gml:Y>
+ </gml:coord>
+ <gml:coord>
+ <gml:X>180.0</gml:X>
+ <gml:Y>90.0</gml:Y>
+ </gml:coord>
+ </gml:Box>
+ </gml:boundedBy>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Amsterdam</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>52.4,4.9</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Bonn</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>50.7,7.1</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Rome</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>41.9,12.5</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+</ogr:FeatureCollection>
+
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/1_copystd ~/project/stetl/git/examples/basics
==== running etl.sh for 1_copystd ====
-2014-11-24 11:04:23,717 util INFO Found cStringIO, good!
-2014-11-24 11:04:23,765 util INFO Found lxml.etree, native XML parsing, fabulous!
-2014-11-24 11:04:25,344 util INFO Found GDAL/OGR Python bindings, super!!
-2014-11-24 11:04:25,357 main INFO Stetl version = 1.0.7
-2014-11-24 11:04:25,359 ETL INFO INIT - Stetl version is 1.0.7
-2014-11-24 11:04:25,359 ETL INFO Config/working dir =/Users/just/project/stetl/git/examples/basics/1_copystd
-2014-11-24 11:04:25,360 ETL INFO Reading config_file = etl.cfg
-2014-11-24 11:04:25,377 ETL INFO START
-2014-11-24 11:04:25,377 util INFO Timer start: total ETL
-2014-11-24 11:04:25,377 chain INFO Assembling Chain: input_xml_file|output_std...
-2014-11-24 11:04:25,380 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
-2014-11-24 11:04:25,381 fileinput INFO file_list=['input/cities.xml']
-2014-11-24 11:04:25,381 output INFO cfg = {'class': 'outputs.standardoutput.StandardXmlOutput'}
-2014-11-24 11:04:25,381 chain INFO Running Chain: input_xml_file|output_std
-2014-11-24 11:04:25,382 fileinput INFO Read/parse for start for file=input/cities.xml....
-2014-11-24 11:04:25,424 fileinput INFO Read/parse ok for file=input/cities.xml
-2014-11-24 11:04:25,424 fileinput INFO all files done
-2014-11-24 11:04:25,424 chain INFO DONE - 1 rounds - chain=input_xml_file|output_std
-2014-11-24 11:04:25,424 util INFO Timer end: total ETL time=0.0 sec
-2014-11-24 11:04:25,424 ETL INFO ALL DONE
+2017-11-07 15:34:41,242 util INFO Found cStringIO, good!
+2017-11-07 15:34:41,261 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:41,289 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:41,295 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:41,296 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:41,296 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/1_copystd
+2017-11-07 15:34:41,296 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:41,298 ETL INFO START
+2017-11-07 15:34:41,298 util INFO Timer start: total ETL
+2017-11-07 15:34:41,298 chain INFO Assembling Chain: input_xml_file|output_std...
+2017-11-07 15:34:41,300 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
+2017-11-07 15:34:41,300 fileinput INFO file_list=['input/cities.xml']
+2017-11-07 15:34:41,302 output INFO cfg = {'class': 'outputs.standardoutput.StandardXmlOutput'}
+2017-11-07 15:34:41,302 chain INFO Running Chain: input_xml_file|output_std
+2017-11-07 15:34:41,302 fileinput INFO Read/parse for start for file=input/cities.xml....
+2017-11-07 15:34:41,302 fileinput INFO Read/parse ok for file=input/cities.xml
+2017-11-07 15:34:41,303 fileinput INFO all files done
+2017-11-07 15:34:41,303 chain INFO DONE - 1 rounds - chain=input_xml_file|output_std
+2017-11-07 15:34:41,303 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:41,303 ETL INFO ALL DONE
<?xml version='1.0' encoding='utf-8'?>
<cities>
<city>
@@ -441,131 +766,139 @@
</city>
</cities>
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/2_xslt ~/project/stetl/git/examples/basics
==== running etl.sh for 2_xslt ====
-2014-11-24 11:04:25,650 util INFO Found cStringIO, good!
-2014-11-24 11:04:25,682 util INFO Found lxml.etree, native XML parsing, fabulous!
-2014-11-24 11:04:25,738 util INFO Found GDAL/OGR Python bindings, super!!
-2014-11-24 11:04:25,761 main INFO Stetl version = 1.0.7
-2014-11-24 11:04:25,762 ETL INFO INIT - Stetl version is 1.0.7
-2014-11-24 11:04:25,762 ETL INFO Config/working dir =/Users/just/project/stetl/git/examples/basics/2_xslt
-2014-11-24 11:04:25,763 ETL INFO Reading config_file = etl.cfg
-2014-11-24 11:04:25,763 ETL INFO START
-2014-11-24 11:04:25,764 util INFO Timer start: total ETL
-2014-11-24 11:04:25,764 chain INFO Assembling Chain: input_xml_file|transformer_xslt|output_file...
-2014-11-24 11:04:25,766 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
-2014-11-24 11:04:25,767 fileinput INFO file_list=['input/cities.xml']
-2014-11-24 11:04:26,028 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/gmlcities.gml'}
-2014-11-24 11:04:26,028 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/2_xslt
-2014-11-24 11:04:26,029 chain INFO Running Chain: input_xml_file|transformer_xslt|output_file
-2014-11-24 11:04:26,029 fileinput INFO Read/parse for start for file=input/cities.xml....
-2014-11-24 11:04:26,049 fileinput INFO Read/parse ok for file=input/cities.xml
-2014-11-24 11:04:26,049 fileinput INFO all files done
-2014-11-24 11:04:26,149 xsltfilter INFO XSLT Transform OK
-2014-11-24 11:04:26,149 fileoutput INFO writing to file output/gmlcities.gml
-2014-11-24 11:04:26,150 fileoutput INFO written to output/gmlcities.gml
-2014-11-24 11:04:26,150 chain INFO DONE - 1 rounds - chain=input_xml_file|transformer_xslt|output_file
-2014-11-24 11:04:26,150 util INFO Timer end: total ETL time=0.0 sec
-2014-11-24 11:04:26,150 ETL INFO ALL DONE
+2017-11-07 15:34:41,622 util INFO Found cStringIO, good!
+2017-11-07 15:34:41,640 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:41,665 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:41,671 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:41,671 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:41,672 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/2_xslt
+2017-11-07 15:34:41,672 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:41,673 ETL INFO START
+2017-11-07 15:34:41,673 util INFO Timer start: total ETL
+2017-11-07 15:34:41,673 chain INFO Assembling Chain: input_xml_file|transformer_xslt|output_file...
+2017-11-07 15:34:41,675 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
+2017-11-07 15:34:41,675 fileinput INFO file_list=['input/cities.xml']
+2017-11-07 15:34:41,679 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/gmlcities.gml'}
+2017-11-07 15:34:41,679 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/2_xslt
+2017-11-07 15:34:41,679 chain INFO Running Chain: input_xml_file|transformer_xslt|output_file
+2017-11-07 15:34:41,679 fileinput INFO Read/parse for start for file=input/cities.xml....
+2017-11-07 15:34:41,680 fileinput INFO Read/parse ok for file=input/cities.xml
+2017-11-07 15:34:41,680 fileinput INFO all files done
+2017-11-07 15:34:41,680 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:41,680 fileoutput INFO writing to file output/gmlcities.gml
+2017-11-07 15:34:41,680 fileoutput INFO written to output/gmlcities.gml
+2017-11-07 15:34:41,680 chain INFO DONE - 1 rounds - chain=input_xml_file|transformer_xslt|output_file
+2017-11-07 15:34:41,680 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:41,680 ETL INFO ALL DONE
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/3_shape ~/project/stetl/git/examples/basics
==== running etl.sh for 3_shape ====
-2014-11-24 11:04:27,238 util INFO Found cStringIO, good!
-2014-11-24 11:04:27,402 util INFO Found lxml.etree, native XML parsing, fabulous!
-2014-11-24 11:04:27,510 util INFO Found GDAL/OGR Python bindings, super!!
-2014-11-24 11:04:27,547 main INFO Stetl version = 1.0.7
-2014-11-24 11:04:27,557 ETL INFO INIT - Stetl version is 1.0.7
-2014-11-24 11:04:27,557 ETL INFO Config/working dir =/Users/just/project/stetl/git/examples/basics/3_shape
-2014-11-24 11:04:27,557 ETL INFO Reading config_file = etl.cfg
-2014-11-24 11:04:27,594 ETL INFO START
-2014-11-24 11:04:27,595 util INFO Timer start: total ETL
-2014-11-24 11:04:27,595 chain INFO Assembling Chain: input_xml_file|transformer_xslt|output_ogr_shape...
-2014-11-24 11:04:27,598 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
-2014-11-24 11:04:27,598 fileinput INFO file_list=['input/cities.xml']
-2014-11-24 11:04:27,630 output INFO cfg = {'ogr2ogr_cmd': 'ogr2ogr\n-overwrite\n-f "ESRI Shapefile"\n-a_srs epsg:4326\noutput/gmlcities.shp\ntemp/gmlcities.gml', 'class': 'outputs.ogroutput.Ogr2OgrOutput', 'temp_file': 'temp/gmlcities.gml'}
-2014-11-24 11:04:27,630 chain INFO Running Chain: input_xml_file|transformer_xslt|output_ogr_shape
-2014-11-24 11:04:27,630 fileinput INFO Read/parse for start for file=input/cities.xml....
-2014-11-24 11:04:27,631 fileinput INFO Read/parse ok for file=input/cities.xml
-2014-11-24 11:04:27,631 fileinput INFO all files done
-2014-11-24 11:04:27,631 xsltfilter INFO XSLT Transform OK
-2014-11-24 11:04:27,631 ogroutput INFO writing to file temp/gmlcities.gml
-2014-11-24 11:04:27,632 ogroutput INFO written to temp/gmlcities.gml
-2014-11-24 11:04:27,632 ogroutput INFO executing cmd=ogr2ogr -overwrite -f "ESRI Shapefile" -a_srs epsg:4326 output/gmlcities.shp temp/gmlcities.gml
-2014-11-24 11:04:32,400 ogroutput INFO execute done
-2014-11-24 11:04:32,401 chain INFO DONE - 1 rounds - chain=input_xml_file|transformer_xslt|output_ogr_shape
-2014-11-24 11:04:32,401 util INFO Timer end: total ETL time=5.0 sec
-2014-11-24 11:04:32,401 ETL INFO ALL DONE
+2017-11-07 15:34:42,028 util INFO Found cStringIO, good!
+2017-11-07 15:34:42,053 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:42,086 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:42,092 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:42,093 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:42,093 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/3_shape
+2017-11-07 15:34:42,094 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:42,095 ETL INFO START
+2017-11-07 15:34:42,095 util INFO Timer start: total ETL
+2017-11-07 15:34:42,095 chain INFO Assembling Chain: input_xml_file|transformer_xslt|output_ogr_shape...
+2017-11-07 15:34:42,097 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
+2017-11-07 15:34:42,097 fileinput INFO file_list=['input/cities.xml']
+2017-11-07 15:34:42,104 output INFO cfg = {'ogr2ogr_cmd': 'ogr2ogr\n-overwrite\n-f "ESRI Shapefile"\n-a_srs epsg:4326\noutput/gmlcities.shp\ntemp/gmlcities.gml', 'class': 'outputs.ogroutput.Ogr2OgrOutput', 'temp_file': 'temp/gmlcities.gml'}
+2017-11-07 15:34:42,104 chain INFO Running Chain: input_xml_file|transformer_xslt|output_ogr_shape
+2017-11-07 15:34:42,104 fileinput INFO Read/parse for start for file=input/cities.xml....
+2017-11-07 15:34:42,104 fileinput INFO Read/parse ok for file=input/cities.xml
+2017-11-07 15:34:42,104 fileinput INFO all files done
+2017-11-07 15:34:42,105 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:42,105 ogroutput INFO writing to file temp/gmlcities.gml
+2017-11-07 15:34:42,105 ogroutput INFO written to temp/gmlcities.gml
+2017-11-07 15:34:42,105 ogroutput INFO executing cmd=ogr2ogr -overwrite -f "ESRI Shapefile" -a_srs epsg:4326 output/gmlcities.shp temp/gmlcities.gml
+2017-11-07 15:34:42,134 ogroutput INFO execute done
+2017-11-07 15:34:42,134 chain INFO DONE - 1 rounds - chain=input_xml_file|transformer_xslt|output_ogr_shape
+2017-11-07 15:34:42,134 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:42,134 ETL INFO ALL DONE
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/4_validate ~/project/stetl/git/examples/basics
==== running etl.sh for 4_validate ====
-2014-11-24 11:04:32,819 util INFO Found cStringIO, good!
-2014-11-24 11:04:32,969 util INFO Found lxml.etree, native XML parsing, fabulous!
-2014-11-24 11:04:34,348 util INFO Found GDAL/OGR Python bindings, super!!
-2014-11-24 11:04:34,445 main INFO Stetl version = 1.0.7
-2014-11-24 11:04:34,447 ETL INFO INIT - Stetl version is 1.0.7
-2014-11-24 11:04:34,447 ETL INFO Config/working dir =/Users/just/project/stetl/git/examples/basics/4_validate
-2014-11-24 11:04:34,447 ETL INFO Reading config_file = etl.cfg
-2014-11-24 11:04:34,476 ETL INFO START
-2014-11-24 11:04:34,476 util INFO Timer start: total ETL
-2014-11-24 11:04:34,476 chain INFO Assembling Chain: input_xml_file|transformer_xslt|xml_schema_validator|output_file...
-2014-11-24 11:04:34,733 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
-2014-11-24 11:04:34,733 fileinput INFO file_list=['input/cities.xml']
-2014-11-24 11:04:34,915 xmlvalidator INFO Building the Schema once with (GML XSD) dependencies for schema=gmlcities.xsd (be patient...)
-2014-11-24 11:04:51,992 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/gmlcities.gml'}
-2014-11-24 11:04:51,993 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/4_validate
-2014-11-24 11:04:51,993 chain INFO Running Chain: input_xml_file|transformer_xslt|xml_schema_validator|output_file
-2014-11-24 11:04:51,993 fileinput INFO Read/parse for start for file=input/cities.xml....
-2014-11-24 11:04:52,001 fileinput INFO Read/parse ok for file=input/cities.xml
-2014-11-24 11:04:52,002 fileinput INFO all files done
-2014-11-24 11:04:52,002 xsltfilter INFO XSLT Transform OK
-2014-11-24 11:04:52,002 xmlvalidator INFO Validating doc against schema=gmlcities.xsd ...
-2014-11-24 11:04:52,003 xmlvalidator INFO Validation result: True
-2014-11-24 11:04:52,003 fileoutput INFO writing to file output/gmlcities.gml
-2014-11-24 11:04:52,017 fileoutput INFO written to output/gmlcities.gml
-2014-11-24 11:04:52,017 chain INFO DONE - 1 rounds - chain=input_xml_file|transformer_xslt|xml_schema_validator|output_file
-2014-11-24 11:04:52,018 util INFO Timer end: total ETL time=18.0 sec
-2014-11-24 11:04:52,018 ETL INFO ALL DONE
+2017-11-07 15:34:42,436 util INFO Found cStringIO, good!
+2017-11-07 15:34:42,454 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:42,481 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:42,486 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:42,487 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:42,487 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/4_validate
+2017-11-07 15:34:42,487 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:42,489 ETL INFO START
+2017-11-07 15:34:42,489 util INFO Timer start: total ETL
+2017-11-07 15:34:42,489 chain INFO Assembling Chain: input_xml_file|transformer_xslt|xml_schema_validator|output_file...
+2017-11-07 15:34:42,491 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
+2017-11-07 15:34:42,491 fileinput INFO file_list=['input/cities.xml']
+2017-11-07 15:34:42,494 xmlvalidator INFO Building the Schema once with (GML XSD) dependencies for schema=gmlcities.xsd (be patient...)
+2017-11-07 15:34:43,949 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/gmlcities.gml'}
+2017-11-07 15:34:43,949 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/4_validate
+2017-11-07 15:34:43,949 chain INFO Running Chain: input_xml_file|transformer_xslt|xml_schema_validator|output_file
+2017-11-07 15:34:43,949 fileinput INFO Read/parse for start for file=input/cities.xml....
+2017-11-07 15:34:43,950 fileinput INFO Read/parse ok for file=input/cities.xml
+2017-11-07 15:34:43,950 fileinput INFO all files done
+2017-11-07 15:34:43,950 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:43,950 xmlvalidator INFO Validating doc against schema=gmlcities.xsd ...
+2017-11-07 15:34:43,950 xmlvalidator INFO Validation result: True
+2017-11-07 15:34:43,950 fileoutput INFO writing to file output/gmlcities.gml
+2017-11-07 15:34:43,951 fileoutput INFO written to output/gmlcities.gml
+2017-11-07 15:34:43,951 chain INFO DONE - 1 rounds - chain=input_xml_file|transformer_xslt|xml_schema_validator|output_file
+2017-11-07 15:34:43,951 util INFO Timer end: total ETL time=1.0 sec
+2017-11-07 15:34:43,951 ETL INFO ALL DONE
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/5_split ~/project/stetl/git/examples/basics
==== running etl.sh for 5_split ====
-2014-11-24 11:04:52,234 util INFO Found cStringIO, good!
-2014-11-24 11:04:52,302 util INFO Found lxml.etree, native XML parsing, fabulous!
-2014-11-24 11:04:52,668 util INFO Found GDAL/OGR Python bindings, super!!
-2014-11-24 11:04:52,683 main INFO Stetl version = 1.0.7
-2014-11-24 11:04:52,685 ETL INFO INIT - Stetl version is 1.0.7
-2014-11-24 11:04:52,685 ETL INFO Config/working dir =/Users/just/project/stetl/git/examples/basics/5_split
-2014-11-24 11:04:52,685 ETL INFO Reading config_file = etl.cfg
-2014-11-24 11:04:52,698 ETL INFO START
-2014-11-24 11:04:52,698 util INFO Timer start: total ETL
-2014-11-24 11:04:52,698 chain INFO Assembling Chain: input_big_xml_file|xml_assembler|transformer_xslt|multi_output_file...
-2014-11-24 11:04:52,700 input INFO cfg = {'element_tags': 'city', 'class': 'inputs.fileinput.XmlElementStreamerFileInput', 'file_path': 'input'}
-2014-11-24 11:04:52,701 fileinput INFO file_list=['input/cities.xml']
-2014-11-24 11:04:52,701 fileinput INFO Element tags to be matched: ['city']
-2014-11-24 11:04:52,716 xmlassembler INFO cfg = {'max_elements': '2', 'element_container_tag': 'cities', 'class': 'filters.xmlassembler.XmlAssembler', 'container_doc': "<?xml version='1.0' encoding='utf-8'?>\n<cities>\n</cities>"}
-2014-11-24 11:04:52,736 output INFO cfg = {'class': 'outputs.fileoutput.MultiFileOutput', 'file_path': 'output/gmlcities-%03d.gml'}
-2014-11-24 11:04:52,736 chain INFO Running Chain: input_big_xml_file|xml_assembler|transformer_xslt|multi_output_file
-2014-11-24 11:04:52,736 fileinput INFO file opened : input/cities.xml
-2014-11-24 11:04:52,792 xmlassembler INFO xmldoc ready: elms=2 total_elms=2
-2014-11-24 11:04:52,793 xsltfilter INFO XSLT Transform OK
-2014-11-24 11:04:52,793 fileoutput INFO writing to file output/gmlcities-001.gml
-2014-11-24 11:04:52,793 fileoutput INFO written to output/gmlcities-001.gml
-2014-11-24 11:04:52,793 fileinput INFO End of doc: input/cities.xml elem_count=3
-2014-11-24 11:04:52,794 fileinput INFO End of stream
-2014-11-24 11:04:52,794 xmlassembler INFO xmldoc ready: elms=1 total_elms=3
-2014-11-24 11:04:52,794 xsltfilter INFO XSLT Transform OK
-2014-11-24 11:04:52,794 fileoutput INFO writing to file output/gmlcities-002.gml
-2014-11-24 11:04:52,794 fileoutput INFO written to output/gmlcities-002.gml
-2014-11-24 11:04:52,794 chain INFO DONE - 26 rounds - chain=input_big_xml_file|xml_assembler|transformer_xslt|multi_output_file
-2014-11-24 11:04:52,795 chain INFO Assembling Chain: input_big_xml_file|xml_assembler|transformer_xslt|output_std...
-2014-11-24 11:04:52,795 input INFO cfg = {'element_tags': 'city', 'class': 'inputs.fileinput.XmlElementStreamerFileInput', 'file_path': 'input'}
-2014-11-24 11:04:52,795 fileinput INFO file_list=['input/cities.xml']
-2014-11-24 11:04:52,795 fileinput INFO Element tags to be matched: ['city']
-2014-11-24 11:04:52,795 xmlassembler INFO cfg = {'max_elements': '2', 'element_container_tag': 'cities', 'class': 'filters.xmlassembler.XmlAssembler', 'container_doc': "<?xml version='1.0' encoding='utf-8'?>\n<cities>\n</cities>"}
-2014-11-24 11:04:52,796 output INFO cfg = {'class': 'outputs.standardoutput.StandardXmlOutput'}
-2014-11-24 11:04:52,797 chain INFO Running Chain: input_big_xml_file|xml_assembler|transformer_xslt|output_std
-2014-11-24 11:04:52,797 fileinput INFO file opened : input/cities.xml
-2014-11-24 11:04:52,797 xmlassembler INFO xmldoc ready: elms=2 total_elms=2
-2014-11-24 11:04:52,798 xsltfilter INFO XSLT Transform OK
-2014-11-24 11:04:52,798 fileinput INFO End of doc: input/cities.xml elem_count=3
-2014-11-24 11:04:52,798 fileinput INFO End of stream
-2014-11-24 11:04:52,798 xmlassembler INFO xmldoc ready: elms=1 total_elms=3
-2014-11-24 11:04:52,798 xsltfilter INFO XSLT Transform OK
-2014-11-24 11:04:52,798 chain INFO DONE - 26 rounds - chain=input_big_xml_file|xml_assembler|transformer_xslt|output_std
-2014-11-24 11:04:52,798 util INFO Timer end: total ETL time=0.0 sec
-2014-11-24 11:04:52,799 ETL INFO ALL DONE
+2017-11-07 15:34:44,245 util INFO Found cStringIO, good!
+2017-11-07 15:34:44,263 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:44,290 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:44,297 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:44,297 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:44,298 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/5_split
+2017-11-07 15:34:44,298 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:44,299 ETL INFO START
+2017-11-07 15:34:44,299 util INFO Timer start: total ETL
+2017-11-07 15:34:44,299 chain INFO Assembling Chain: input_big_xml_file|xml_assembler|transformer_xslt|multi_output_file...
+2017-11-07 15:34:44,301 input INFO cfg = {'element_tags': 'city', 'class': 'inputs.fileinput.XmlElementStreamerFileInput', 'file_path': 'input'}
+2017-11-07 15:34:44,303 fileinput INFO file_list=['input/cities.xml']
+2017-11-07 15:34:44,304 fileinput INFO Element tags to be matched: ['city']
+2017-11-07 15:34:44,304 xmlassembler INFO cfg = {'max_elements': '2', 'element_container_tag': 'cities', 'class': 'filters.xmlassembler.XmlAssembler', 'container_doc': "<?xml version='1.0' encoding='utf-8'?>\n<cities>\n</cities>"}
+2017-11-07 15:34:44,306 output INFO cfg = {'class': 'outputs.fileoutput.MultiFileOutput', 'file_path': 'output/gmlcities-%03d.gml'}
+2017-11-07 15:34:44,307 chain INFO Running Chain: input_big_xml_file|xml_assembler|transformer_xslt|multi_output_file
+2017-11-07 15:34:44,307 fileinput INFO file opened : input/cities.xml
+2017-11-07 15:34:44,309 xmlassembler INFO xmldoc ready: elms=2 total_elms=2
+2017-11-07 15:34:44,309 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:44,309 fileoutput INFO writing to file output/gmlcities-001.gml
+2017-11-07 15:34:44,310 fileoutput INFO written to output/gmlcities-001.gml
+2017-11-07 15:34:44,310 fileinput INFO End of doc: input/cities.xml elem_count=3
+2017-11-07 15:34:44,311 fileinput INFO End of stream
+2017-11-07 15:34:44,311 xmlassembler INFO xmldoc ready: elms=1 total_elms=3
+2017-11-07 15:34:44,311 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:44,311 fileoutput INFO writing to file output/gmlcities-002.gml
+2017-11-07 15:34:44,311 fileoutput INFO written to output/gmlcities-002.gml
+2017-11-07 15:34:44,311 chain INFO DONE - 26 rounds - chain=input_big_xml_file|xml_assembler|transformer_xslt|multi_output_file
+2017-11-07 15:34:44,311 chain INFO Assembling Chain: input_big_xml_file|xml_assembler|transformer_xslt|output_std...
+2017-11-07 15:34:44,312 input INFO cfg = {'element_tags': 'city', 'class': 'inputs.fileinput.XmlElementStreamerFileInput', 'file_path': 'input'}
+2017-11-07 15:34:44,312 fileinput INFO file_list=['input/cities.xml']
+2017-11-07 15:34:44,312 fileinput INFO Element tags to be matched: ['city']
+2017-11-07 15:34:44,312 xmlassembler INFO cfg = {'max_elements': '2', 'element_container_tag': 'cities', 'class': 'filters.xmlassembler.XmlAssembler', 'container_doc': "<?xml version='1.0' encoding='utf-8'?>\n<cities>\n</cities>"}
+2017-11-07 15:34:44,313 output INFO cfg = {'class': 'outputs.standardoutput.StandardXmlOutput'}
+2017-11-07 15:34:44,313 chain INFO Running Chain: input_big_xml_file|xml_assembler|transformer_xslt|output_std
+2017-11-07 15:34:44,313 fileinput INFO file opened : input/cities.xml
+2017-11-07 15:34:44,313 xmlassembler INFO xmldoc ready: elms=2 total_elms=2
+2017-11-07 15:34:44,313 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:44,313 fileinput INFO End of doc: input/cities.xml elem_count=3
+2017-11-07 15:34:44,313 fileinput INFO End of stream
+2017-11-07 15:34:44,314 xmlassembler INFO xmldoc ready: elms=1 total_elms=3
+2017-11-07 15:34:44,314 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:44,314 chain INFO DONE - 26 rounds - chain=input_big_xml_file|xml_assembler|transformer_xslt|output_std
+2017-11-07 15:34:44,314 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:44,314 ETL INFO ALL DONE
<?xml version='1.0' encoding='utf-8'?>
<ogr:FeatureCollection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ogr="http://ogr.maptools.org/" xmlns:gml="http://www.opengis.net/gml" xsi:schemaLocation="http://ogr.maptools.org/ ../gmlcities.xsd http://www.opengis.net/gml http://schemas.opengis.net/gml/2.1.2/feature.xsd">
<gml:boundedBy>
@@ -628,55 +961,86 @@
</gml:featureMember>
</ogr:FeatureCollection>
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/6_cmdargs ~/project/stetl/git/examples/basics
==== running etl.sh for 6_cmdargs ====
-2014-11-24 11:04:52,990 util INFO Found cStringIO, good!
-2014-11-24 11:04:53,018 util INFO Found lxml.etree, native XML parsing, fabulous!
-2014-11-24 11:04:53,044 util INFO Found GDAL/OGR Python bindings, super!!
-2014-11-24 11:04:53,057 main INFO Stetl version = 1.0.7
-2014-11-24 11:04:53,059 ETL INFO INIT - Stetl version is 1.0.7
-2014-11-24 11:04:53,059 ETL INFO Config/working dir =/Users/just/project/stetl/git/examples/basics/6_cmdargs
-2014-11-24 11:04:53,059 ETL INFO Reading config_file = etl.cfg
-2014-11-24 11:04:53,059 ETL INFO Substituting 3 args in config file from args_dict: {'out_xml': 'output/gmlcities.gml', 'in_xsl': 'cities2gml.xsl', 'in_xml': 'input/cities.xml'}
-2014-11-24 11:04:53,097 ETL INFO Substituting args OK
-2014-11-24 11:04:53,098 ETL INFO START
-2014-11-24 11:04:53,098 util INFO Timer start: total ETL
-2014-11-24 11:04:53,098 chain INFO Assembling Chain: input_xml_file|transformer_xslt|output_file...
-2014-11-24 11:04:53,103 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
-2014-11-24 11:04:53,103 fileinput INFO file_list=['input/cities.xml']
-2014-11-24 11:04:53,123 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/gmlcities.gml'}
-2014-11-24 11:04:53,123 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/6_cmdargs
-2014-11-24 11:04:53,123 chain INFO Running Chain: input_xml_file|transformer_xslt|output_file
-2014-11-24 11:04:53,123 fileinput INFO Read/parse for start for file=input/cities.xml....
-2014-11-24 11:04:53,124 fileinput INFO Read/parse ok for file=input/cities.xml
-2014-11-24 11:04:53,124 fileinput INFO all files done
-2014-11-24 11:04:53,124 xsltfilter INFO XSLT Transform OK
-2014-11-24 11:04:53,124 fileoutput INFO writing to file output/gmlcities.gml
-2014-11-24 11:04:53,141 fileoutput INFO written to output/gmlcities.gml
-2014-11-24 11:04:53,142 chain INFO DONE - 1 rounds - chain=input_xml_file|transformer_xslt|output_file
-2014-11-24 11:04:53,142 util INFO Timer end: total ETL time=0.0 sec
-2014-11-24 11:04:53,142 ETL INFO ALL DONE
+2017-11-07 15:34:44,611 util INFO Found cStringIO, good!
+2017-11-07 15:34:44,632 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:44,658 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:44,664 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:44,666 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:44,666 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/6_cmdargs
+2017-11-07 15:34:44,666 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:44,666 ETL INFO Substituting 3 args in config file from args_dict: {'out_xml': 'output/gmlcities.gml', 'in_xsl': 'cities2gml.xsl', 'in_xml': 'input/cities.xml'}
+2017-11-07 15:34:44,667 ETL INFO Substituting args OK
+2017-11-07 15:34:44,667 ETL INFO START
+2017-11-07 15:34:44,667 util INFO Timer start: total ETL
+2017-11-07 15:34:44,668 chain INFO Assembling Chain: input_xml_file|transformer_xslt|output_file...
+2017-11-07 15:34:44,670 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
+2017-11-07 15:34:44,670 fileinput INFO file_list=['input/cities.xml']
+2017-11-07 15:34:44,673 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/gmlcities.gml'}
+2017-11-07 15:34:44,673 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/6_cmdargs
+2017-11-07 15:34:44,674 chain INFO Running Chain: input_xml_file|transformer_xslt|output_file
+2017-11-07 15:34:44,674 fileinput INFO Read/parse for start for file=input/cities.xml....
+2017-11-07 15:34:44,674 fileinput INFO Read/parse ok for file=input/cities.xml
+2017-11-07 15:34:44,674 fileinput INFO all files done
+2017-11-07 15:34:44,674 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:44,674 fileoutput INFO writing to file output/gmlcities.gml
+2017-11-07 15:34:44,675 fileoutput INFO written to output/gmlcities.gml
+2017-11-07 15:34:44,675 chain INFO DONE - 1 rounds - chain=input_xml_file|transformer_xslt|output_file
+2017-11-07 15:34:44,675 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:44,675 ETL INFO ALL DONE
+2017-11-07 15:34:45,022 util INFO Found cStringIO, good!
+2017-11-07 15:34:45,045 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:45,074 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:45,079 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:45,080 main INFO Found args file at: etl.args
+2017-11-07 15:34:45,081 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:45,081 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/6_cmdargs
+2017-11-07 15:34:45,081 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:45,081 ETL INFO Substituting 4 args in config file from args_dict: OrderedDict([('__name__', 'asection'), ('in_xml', 'input/cities.xml'), ('in_xsl', 'cities2gml.xsl'), ('out_xml', 'output/gmlcities.gml')])
+2017-11-07 15:34:45,081 ETL INFO Substituting args OK
+2017-11-07 15:34:45,081 ETL INFO START
+2017-11-07 15:34:45,081 util INFO Timer start: total ETL
+2017-11-07 15:34:45,081 chain INFO Assembling Chain: input_xml_file|transformer_xslt|output_file...
+2017-11-07 15:34:45,084 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
+2017-11-07 15:34:45,084 fileinput INFO file_list=['input/cities.xml']
+2017-11-07 15:34:45,088 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/gmlcities.gml'}
+2017-11-07 15:34:45,088 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/6_cmdargs
+2017-11-07 15:34:45,088 chain INFO Running Chain: input_xml_file|transformer_xslt|output_file
+2017-11-07 15:34:45,088 fileinput INFO Read/parse for start for file=input/cities.xml....
+2017-11-07 15:34:45,088 fileinput INFO Read/parse ok for file=input/cities.xml
+2017-11-07 15:34:45,088 fileinput INFO all files done
+2017-11-07 15:34:45,089 xsltfilter INFO XSLT Transform OK
+2017-11-07 15:34:45,089 fileoutput INFO writing to file output/gmlcities.gml
+2017-11-07 15:34:45,089 fileoutput INFO written to output/gmlcities.gml
+2017-11-07 15:34:45,089 chain INFO DONE - 1 rounds - chain=input_xml_file|transformer_xslt|output_file
+2017-11-07 15:34:45,089 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:45,089 ETL INFO ALL DONE
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/7_mycomponent ~/project/stetl/git/examples/basics
==== running etl.sh for 7_mycomponent ====
-2014-11-24 11:04:53,303 util INFO Found cStringIO, good!
-2014-11-24 11:04:53,333 util INFO Found lxml.etree, native XML parsing, fabulous!
-2014-11-24 11:04:53,359 util INFO Found GDAL/OGR Python bindings, super!!
-2014-11-24 11:04:53,372 main INFO Stetl version = 1.0.7
-2014-11-24 11:04:53,373 ETL INFO INIT - Stetl version is 1.0.7
-2014-11-24 11:04:53,374 ETL INFO Config/working dir =/Users/just/project/stetl/git/examples/basics/7_mycomponent
-2014-11-24 11:04:53,374 ETL INFO Reading config_file = etl.cfg
-2014-11-24 11:04:53,417 ETL INFO START
-2014-11-24 11:04:53,417 util INFO Timer start: total ETL
-2014-11-24 11:04:53,417 chain INFO Assembling Chain: input_xml_file|my_filter|output_std...
-2014-11-24 11:04:53,420 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
-2014-11-24 11:04:53,490 fileinput INFO file_list=['input/cities.xml']
-2014-11-24 11:04:53,608 output INFO cfg = {'class': 'outputs.standardoutput.StandardXmlOutput'}
-2014-11-24 11:04:53,608 chain INFO Running Chain: input_xml_file|my_filter|output_std
-2014-11-24 11:04:53,608 fileinput INFO Read/parse for start for file=input/cities.xml....
-2014-11-24 11:04:53,624 fileinput INFO Read/parse ok for file=input/cities.xml
-2014-11-24 11:04:53,624 fileinput INFO all files done
-2014-11-24 11:04:53,625 myfilter INFO CALLING MyFilter OK!!!!
-2014-11-24 11:04:53,625 chain INFO DONE - 1 rounds - chain=input_xml_file|my_filter|output_std
-2014-11-24 11:04:53,625 util INFO Timer end: total ETL time=0.0 sec
-2014-11-24 11:04:53,625 ETL INFO ALL DONE
+2017-11-07 15:34:45,391 util INFO Found cStringIO, good!
+2017-11-07 15:34:45,408 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:45,434 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:45,440 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:45,441 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:45,441 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/7_mycomponent
+2017-11-07 15:34:45,441 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:45,448 ETL INFO START
+2017-11-07 15:34:45,448 util INFO Timer start: total ETL
+2017-11-07 15:34:45,448 chain INFO Assembling Chain: input_xml_file|my_filter|output_std...
+2017-11-07 15:34:45,451 input INFO cfg = {'class': 'inputs.fileinput.XmlFileInput', 'file_path': 'input/cities.xml'}
+2017-11-07 15:34:45,451 fileinput INFO file_list=['input/cities.xml']
+2017-11-07 15:34:45,460 output INFO cfg = {'class': 'outputs.standardoutput.StandardXmlOutput'}
+2017-11-07 15:34:45,460 chain INFO Running Chain: input_xml_file|my_filter|output_std
+2017-11-07 15:34:45,460 fileinput INFO Read/parse for start for file=input/cities.xml....
+2017-11-07 15:34:45,467 fileinput INFO Read/parse ok for file=input/cities.xml
+2017-11-07 15:34:45,467 fileinput INFO all files done
+2017-11-07 15:34:45,467 myfilter INFO CALLING MyFilter OK!!!!
+2017-11-07 15:34:45,467 chain INFO DONE - 1 rounds - chain=input_xml_file|my_filter|output_std
+2017-11-07 15:34:45,467 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:45,467 ETL INFO ALL DONE
<?xml version='1.0' encoding='utf-8'?>
<cities>
<city>
@@ -696,50 +1060,55 @@
</city>
</cities>
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/8_wfs ~/project/stetl/git/examples/basics
==== running etl.sh for 8_wfs ====
-2014-11-24 11:04:53,819 util INFO Found cStringIO, good!
-2014-11-24 11:04:53,847 util INFO Found lxml.etree, native XML parsing, fabulous!
-2014-11-24 11:04:53,873 util INFO Found GDAL/OGR Python bindings, super!!
-2014-11-24 11:04:53,887 main INFO Stetl version = 1.0.7
-2014-11-24 11:04:53,888 ETL INFO INIT - Stetl version is 1.0.7
-2014-11-24 11:04:53,888 ETL INFO Config/working dir =/Users/just/project/stetl/git/examples/basics/8_wfs
-2014-11-24 11:04:53,889 ETL INFO Reading config_file = etl.cfg
-2014-11-24 11:04:53,935 ETL INFO START
-2014-11-24 11:04:53,935 util INFO Timer start: total ETL
-2014-11-24 11:04:53,935 chain INFO Assembling Chain: input_wfs|output_std...
-2014-11-24 11:04:55,186 input INFO cfg = {'url': 'http://suite.opengeo.org/geoserver/ows', 'class': 'inputs.httpinput.HttpInput', 'parameters': '{\n\'service\' : \'WFS\',\n\'version\' : \'1.1.0\',\n\'request\' : \'GetFeature\',\n\'srsName\' : \'EPSG:4326\',\n\'outputFormat\' : \'text/xml; subtype=gml/2.1.2\',\n\'typename\' : \'states\',\n\'filter\' :\'<ogc:Filter xmlns:ogc="http://www.opengis.net/ogc"><ogc:Intersects><ogc:PropertyName/><gml:Polygon xmlns:gml="http://www.opengis.net/gml" [...]
-2014-11-24 11:04:55,201 httpinput INFO url=http://suite.opengeo.org/geoserver/ows parameters={'srsName': 'EPSG:4326', 'outputFormat': 'text/xml; subtype=gml/2.1.2', 'service': 'WFS', 'request': 'GetFeature', 'filter': '<ogc:Filter xmlns:ogc="http://www.opengis.net/ogc"><ogc:Intersects><ogc:PropertyName/><gml:Polygon xmlns:gml="http://www.opengis.net/gml" srsName="EPSG:4326"><gml:exterior><gml:LinearRing><gml:posList>-101.8671875 32.177734375 -101.8671875 39.6923828125 -91.935546875 39.69 [...]
-2014-11-24 11:04:55,201 output INFO cfg = {'class': 'outputs.standardoutput.StandardOutput'}
-2014-11-24 11:04:55,202 chain INFO Running Chain: input_wfs|output_std
-<?xml version="1.0" encoding="UTF-8"?><wfs:FeatureCollection xmlns="http://www.opengis.net/wfs" xmlns:wfs="http://www.opengis.net/wfs" xmlns:usa="http://census.gov" xmlns:gml="http://www.opengis.net/gml" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://census.gov http://suite.opengeo.org/geoserver/wfs?service=WFS&version=1.0.0&request=DescribeFeatureType&typeName=usa%3Astates http://www.opengis.net/wfs http://suite.opengeo.org/geoserver/schemas [...]
-2014-11-24 11:04:55,843 chain INFO DONE - 2 rounds - chain=input_wfs|output_std
-2014-11-24 11:04:55,843 util INFO Timer end: total ETL time=2.0 sec
-2014-11-24 11:04:55,843 ETL INFO ALL DONE
-731378 -106.45143,31.764427 -106.490549,31.748924 -106.54714,31.807299 -106.602806,31.825018 -106.602043,31.844406 -106.635922,31.866237 -106.629191,31.883699 -106.645584,31.895328 -106.611847,31.920003 -106.629742,31.926567 -106.619569,31.971577 -106.639536,31.980343 -106.618486,32.000495 -103.064422,32.000517 -103.064657,32.959097</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></gml:polygonMember></gml:MultiPolygon></usa:the_geom><usa:GEOID10>48</usa:GEOID10><usa [...]
+2017-11-07 15:34:45,778 util INFO Found cStringIO, good!
+2017-11-07 15:34:45,796 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:45,823 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:45,829 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:45,830 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:45,830 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/8_wfs
+2017-11-07 15:34:45,830 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:45,832 ETL INFO START
+2017-11-07 15:34:45,832 util INFO Timer start: total ETL
+2017-11-07 15:34:45,832 chain INFO Assembling Chain: input_wfs|output_std...
+2017-11-07 15:34:45,838 input INFO cfg = {'url': 'http://geodata.nationaalgeoregister.nl/bag/wfs', 'class': 'inputs.httpinput.HttpInput', 'parameters': '{\n\'service\' : \'WFS\',\n\'version\' : \'1.1.0\',\n\'request\' : \'GetFeature\',\n\'srsName\' : \'EPSG:28992\',\n\'outputFormat\' : \'text/xml; subtype=gml/2.1.2\',\n\'typename\' : \'verblijfsobject\',\n\'filter\' :\'<ogc:Filter xmlns:ogc="http://www.opengis.net/ogc"><ogc:BBOX><gml:Envelope xmlns:gml="http://www.opengis.net/gml" srsNam [...]
+2017-11-07 15:34:45,839 httpinput INFO url=http://geodata.nationaalgeoregister.nl/bag/wfs parameters={'srsName': 'EPSG:28992', 'outputFormat': 'text/xml; subtype=gml/2.1.2', 'service': 'WFS', 'request': 'GetFeature', 'filter': '<ogc:Filter xmlns:ogc="http://www.opengis.net/ogc"><ogc:BBOX><gml:Envelope xmlns:gml="http://www.opengis.net/gml" srsName="EPSG:28992"><gml:lowerCorner>183774.83 450577.24</gml:lowerCorner><gml:upperCorner>184277.99 450809.92</gml:upperCorner></gml:Envelope></ogc: [...]
+2017-11-07 15:34:45,839 output INFO cfg = {'class': 'outputs.standardoutput.StandardOutput'}
+2017-11-07 15:34:45,839 chain INFO Running Chain: input_wfs|output_std
+2017-11-07 15:34:45,910 httpinput INFO EOF URL reading done
+2017-11-07 15:34:45,910 chain INFO DONE - 2 rounds - chain=input_wfs|output_std
+2017-11-07 15:34:45,910 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:45,910 ETL INFO ALL DONE
+<?xml version="1.0" encoding="UTF-8"?><wfs:FeatureCollection xmlns="http://www.opengis.net/wfs" xmlns:wfs="http://www.opengis.net/wfs" xmlns:gml="http://www.opengis.net/gml" xmlns:bag="http://bag.geonovum.nl" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/wfs http://schemas.opengis.net/wfs/1.0.0/WFS-basic.xsd http://bag.geonovum.nl https://geodata.nationaalgeoregister.nl/bag/wfs?service=WFS&version=1.0.0&request=DescribeFeatureTyp [...]
+~/project/stetl/git/examples/basics
+~/project/stetl/git/examples/basics/9_string_templating ~/project/stetl/git/examples/basics
==== running etl.sh for 9_string_templating ====
-2014-11-24 11:04:56,006 util INFO Found cStringIO, good!
-2014-11-24 11:04:56,035 util INFO Found lxml.etree, native XML parsing, fabulous!
-2014-11-24 11:04:56,064 util INFO Found GDAL/OGR Python bindings, super!!
-2014-11-24 11:04:56,076 main INFO Stetl version = 1.0.7
-2014-11-24 11:04:56,077 ETL INFO INIT - Stetl version is 1.0.7
-2014-11-24 11:04:56,077 ETL INFO Config/working dir =/Users/just/project/stetl/git/examples/basics/9_string_templating
-2014-11-24 11:04:56,078 ETL INFO Reading config_file = etl.cfg
-2014-11-24 11:04:56,126 ETL INFO START
-2014-11-24 11:04:56,127 util INFO Timer start: total ETL
-2014-11-24 11:04:56,127 chain INFO Assembling Chain: input_csv|filter_template|output_file...
-2014-11-24 11:04:56,130 input INFO cfg = {'output_format': 'record', 'class': 'inputs.fileinput.CsvFileInput', 'file_path': 'input/city.csv'}
-2014-11-24 11:04:56,130 fileinput INFO file_list=['input/city.csv']
-2014-11-24 11:04:56,152 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/city.xml'}
-2014-11-24 11:04:56,152 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/9_string_templating
-2014-11-24 11:04:56,152 chain INFO Running Chain: input_csv|filter_template|output_file
-2014-11-24 11:04:56,152 fileinput INFO Open CSV file: input/city.csv
-2014-11-24 11:04:56,152 templatingfilter INFO Init: templating
-2014-11-24 11:04:56,152 templatingfilter INFO Init: reading template file %s ..." % self.template_file
-2014-11-24 11:04:56,197 templatingfilter INFO template file read OK: city_template.xml
-2014-11-24 11:04:56,218 fileinput INFO CSV row nr 1 read: {'lat': '52.4', 'city': 'amsterdam', 'lon': '4.9'}
-2014-11-24 11:04:56,218 fileoutput INFO writing to file output/city.xml
-2014-11-24 11:04:56,251 fileoutput INFO written to output/city.xml
-2014-11-24 11:04:56,251 templatingfilter INFO Exit: templating
-2014-11-24 11:04:56,251 chain INFO DONE - 2 rounds - chain=input_csv|filter_template|output_file
-2014-11-24 11:04:56,251 util INFO Timer end: total ETL time=0.0 sec
-2014-11-24 11:04:56,251 ETL INFO ALL DONE
+2017-11-07 15:34:46,296 util INFO Found cStringIO, good!
+2017-11-07 15:34:46,315 util INFO Found lxml.etree, native XML parsing, fabulous!
+2017-11-07 15:34:46,339 util INFO Found GDAL/OGR Python bindings, super!!
+2017-11-07 15:34:46,346 main INFO Stetl version = 1.1.0
+2017-11-07 15:34:46,347 ETL INFO INIT - Stetl version is 1.1.0
+2017-11-07 15:34:46,347 ETL INFO Config/working dir = /Users/just/project/stetl/git/examples/basics/9_string_templating
+2017-11-07 15:34:46,347 ETL INFO Reading config_file = etl.cfg
+2017-11-07 15:34:46,349 ETL INFO START
+2017-11-07 15:34:46,349 util INFO Timer start: total ETL
+2017-11-07 15:34:46,349 chain INFO Assembling Chain: input_csv|filter_template|output_file...
+2017-11-07 15:34:46,351 input INFO cfg = {'output_format': 'record', 'class': 'inputs.fileinput.CsvFileInput', 'file_path': 'input/city.csv'}
+2017-11-07 15:34:46,351 fileinput INFO file_list=['input/city.csv']
+2017-11-07 15:34:46,354 output INFO cfg = {'class': 'outputs.fileoutput.FileOutput', 'file_path': 'output/city.xml'}
+2017-11-07 15:34:46,354 fileoutput INFO working dir /Users/just/project/stetl/git/examples/basics/9_string_templating
+2017-11-07 15:34:46,354 chain INFO Running Chain: input_csv|filter_template|output_file
+2017-11-07 15:34:46,354 fileinput INFO Open CSV file: input/city.csv
+2017-11-07 15:34:46,354 templatingfilter INFO Init: templating
+2017-11-07 15:34:46,354 templatingfilter INFO Init: reading template file %s ..." % self.template_file
+2017-11-07 15:34:46,354 templatingfilter INFO template file read OK: city_template.xml
+2017-11-07 15:34:46,355 fileinput INFO CSV row nr 1 read: {'lat': '52.4', 'city': 'amsterdam', 'lon': '4.9'}
+2017-11-07 15:34:46,355 fileoutput INFO writing to file output/city.xml
+2017-11-07 15:34:46,355 fileoutput INFO written to output/city.xml
+2017-11-07 15:34:46,355 templatingfilter INFO Exit: templating
+2017-11-07 15:34:46,355 chain INFO DONE - 2 rounds - chain=input_csv|filter_template|output_file
+2017-11-07 15:34:46,355 util INFO Timer end: total ETL time=0.0 sec
+2017-11-07 15:34:46,355 ETL INFO ALL DONE
+~/project/stetl/git/examples/basics
diff --git a/examples/basics/runall.sh b/examples/basics/runall.sh
index 5ad1d99..7f997d0 100755
--- a/examples/basics/runall.sh
+++ b/examples/basics/runall.sh
@@ -1,14 +1,11 @@
-#!/bin/sh
+#!/bin/bash
#
# Shortcut to run all basic examples.
#
for dir in `echo [0-9]*`; do
- cd $dir
+ pushd $dir
echo "==== running etl.sh for $dir ===="
./etl.sh
- cd ..
+ popd
done
-
-
-
diff --git a/examples/bgt/README.md b/examples/bgt/README.md
index 6a6686b..c730274 100644
--- a/examples/bgt/README.md
+++ b/examples/bgt/README.md
@@ -1,4 +1,6 @@
-In this example Dutch large-scale topo-data (BGT) to a flattened GML feature collection.
-Needs some finalization.
+# Dutch large-scale topo-data (BGT) to GML
+In this example Dutch large-scale topo-data (BGT) to a flattened GML feature collection.
+**NB this example is outdated.**
+**For a production version see https://github.com/nlextract/NLExtract/tree/master/bgt/etl**
diff --git a/examples/ordnancesurvey/readme.txt b/examples/ordnancesurvey/README.md
similarity index 65%
rename from examples/ordnancesurvey/readme.txt
rename to examples/ordnancesurvey/README.md
index d813b6c..59d6295 100644
--- a/examples/ordnancesurvey/readme.txt
+++ b/examples/ordnancesurvey/README.md
@@ -1,15 +1,19 @@
+# UK Ordnance Survey Mastermap GML to PostGIS
+
This example is a proof of concept to load UK Ordnance Survey Mastermap GML into PostGIS using
-Astun Technology Loader config and GML manipulation unaltered. A small sample of UK Mastermap data has been extracted from
+Astun Technology Loader config and GML manipulation unaltered.
+
+A small sample of UK Mastermap data has been extracted from
a sample download provided by Ordnance Survey. The full file has also been
-sucessfully loaded and is called: 58116-SX9192-2c1.gml.
+sucessfully loaded and is called: `58116-SX9192-2c1.gml`.
Files:
-- input/osmm-topo-small.gml the input file
+- `input/osmm-topo-small.gml` the input file
- stetl config: stetl.cfg
-- Stetl Filter: python/osmmfilter.py, calling the unaltered prep_osmgml.py from Astun Tech
+- Stetl Filter: `python/osmmfilter.py`, calling the unaltered `prep_osmgml.py` from Astun Tech
- command (Linux/Mac): etl.sh
-- output: to PostGIS and into the file output/osmm_topo_prepared.gml
+- output: to PostGIS and into the file `output/osmm_topo_prepared.gml`
- gfs/ determines the mapping to PostGIS of the prepared data
- postgres/*.sql - scripts to setup/init the PG DB (called by Stetl)
@@ -17,20 +21,21 @@ How this works:
On calling etl.sh Stetl will read the config file stetl.cfg,
substituting any commandline parameters (-a option) passed to it.
Stetl will then assemble and execute the specified Chains under the
-[etl] section:
+`[etl]` section:
-The Chain "input_sql_pre|schema_name_filter|output_postgres" will initialize the DB and setup
+The Chain `input_sql_pre|schema_name_filter|output_postgres` will initialize the DB and setup
all required tables/indexes from the files under postgres/*.sql.
-The Chain "input_big_gml_files|osmm_filter|xml_assembler|output_file" will read the input file(s) from
+The Chain `input_big_gml_files|osmm_filter|xml_assembler|output_file` will read the input file(s) from
the directory input/, stream the feature elements to its output. The "osmm_filter" will
receive each feature element, calling the configured "preparer" in prep_osmgml.py. Finally
the "output_file" component will write the result to the file under output/.
-The Chain "input_big_gml_files|osmm_filter|xml_assembler|output_ogr2ogr" does the samen, only it will
+The Chain `input_big_gml_files|osmm_filter|xml_assembler|output_ogr2ogr` does the samen, only it will
write the results to PostGIS using the gfs/ mapping file. NB: in Stetl 1.0.4 still with a temp file
but we will change this.
-Links:
+## Links
+
- Astuntech Loader - https://github.com/AstunTechnology/Loader
- Ordnance Survey UK - http://www.ordnancesurvey.co.uk/
\ No newline at end of file
diff --git a/examples/top10nl/readme.txt b/examples/top10nl/README.md
similarity index 80%
rename from examples/top10nl/readme.txt
rename to examples/top10nl/README.md
index 5828ea2..99988fe 100644
--- a/examples/top10nl/readme.txt
+++ b/examples/top10nl/README.md
@@ -1,17 +1,20 @@
-Note: this example was the base for the full implementation in the NLExtract project where
+# Dutch topo-data (BRT/Top10NL) to PostGIS
+
+In this example Dutch topodata (BRT/Top10NL) is converted to PostGIS.
+This example was the base for the full implementation in the NLExtract project where
it will be maintained more actively.
-See: https://github.com/opengeogroep/NLExtract/tree/master/top10nl/etl
-USE NLExtract FOR REAL TOP10NL EXTRACT!!!
+**This example is outdated.**
+**For a production version see https://github.com/nlextract/NLExtract/tree/master/top10nl/etl**
-=============================
+## Notes - kept for historic reasons
This example shows how to convert the Dutch national topographic dataset Top10NL (v1.1.1) GML to
PostGIS.
NB this is an example. The official and actively maintained
Top10NL extract prog is now part of the NLExtract project and located at:
-https://github.com/opengeogroep/NLExtract/tree/master/top10nl/etl !!
+https://github.com/nlextract/NLExtract/tree/master/top10nl/etl !!
The entire ETL is driven through the file etl-top10nl.cfg without any
programming (except for the XSLT step). The script etl-top10nl.sh is a shortcut to
diff --git a/requirements-dev.txt b/requirements-dev.txt
new file mode 100644
index 0000000..240bdc9
--- /dev/null
+++ b/requirements-dev.txt
@@ -0,0 +1,6 @@
+nose2
+cov-core
+mock
+Sphinx
+flake8
+pylint
diff --git a/requirements-main.txt b/requirements-main.txt
new file mode 100644
index 0000000..15d05be
--- /dev/null
+++ b/requirements-main.txt
@@ -0,0 +1,4 @@
+psycopg2>=2.6
+lxml>=3.7
+Jinja2>=2.8
+GDAL>=2
diff --git a/setup.py b/setup.py
index 1be4e58..d4f2a92 100644
--- a/setup.py
+++ b/setup.py
@@ -1,16 +1,16 @@
import logging
-import sys
from setuptools import setup, find_packages
# To publish: python setup.py sdist upload -r pypi
# Have to do this after importing setuptools, which monkey patches distutils.
-from distutils.extension import Extension
+# from distutils.extension import Extension
logging.basicConfig()
log = logging.getLogger()
# Parse the version from the stetl module.
+version = ''
with open('stetl/version.py', 'r') as f:
for line in f:
if line.find("__version__") >= 0:
@@ -32,15 +32,8 @@ with open('CREDITS.txt', 'r') as f:
with open('CHANGES.txt', 'r') as f:
changes = f.read()
-requirements = [
- 'psycopg2',
- 'lxml',
- 'GDAL<2.0',
- 'Jinja2'
-]
-
-if sys.version_info < (2, 7):
- requirements.append('argparse')
+with open('requirements-main.txt') as f:
+ requirements = f.read().splitlines()
setup(
name='Stetl',
@@ -52,7 +45,7 @@ setup(
author_email='justb4 at gmail.com',
maintainer='Just van den Broecke',
maintainer_email='justb4 at gmail.com',
- url='http://github.com/justb4/stetl',
+ url='http://github.com/geopython/stetl',
long_description=readme + "\n" + changes + "\n" + credits,
packages=find_packages(exclude=['tests']),
namespace_packages=['stetl'],
diff --git a/stetl/__init__.py b/stetl/__init__.py
index aef71ed..1d0527d 100644
--- a/stetl/__init__.py
+++ b/stetl/__init__.py
@@ -10,5 +10,3 @@ __import__('pkg_resources').declare_namespace(__name__)
# from packet import *
# from postgis import *
# from util import *
-
-
diff --git a/stetl/chain.py b/stetl/chain.py
index 102ff34..04d6176 100644
--- a/stetl/chain.py
+++ b/stetl/chain.py
@@ -8,6 +8,8 @@
from factory import factory
from packet import Packet
from util import Util
+from splitter import Splitter
+from merger import Merger
log = Util.get_log('chain')
@@ -26,7 +28,7 @@ class Chain:
self.first_comp = None
self.cur_comp = None
self.config_dict = config_dict
- self.chain_str = chain_str
+ self.chain_str = chain_str.strip()
def assemble(self):
"""
@@ -36,10 +38,55 @@ class Chain:
log.info('Assembling Chain: %s...' % self.chain_str)
# Create linked list of input/filter/output (ETL Component) objects
- chain_str_arr = self.chain_str.split('|')
- for etl_section_name in chain_str_arr:
- # Create the ETL component by name and properties
- etl_comp = factory.create_obj(self.config_dict, etl_section_name)
+ chain_str = self.chain_str
+ sub_comps = []
+ while chain_str:
+ chain_str = chain_str.strip()
+
+ # Check and handle Splitter construct
+ # e.g. input_xml_file |(transformer_xslt|output_file) (output_std) (transformer_xslt|output_std)
+ if chain_str.startswith('('):
+ etl_section_name, chain_str = chain_str.split(')', 1)
+ etl_section_name = etl_section_name.strip('(')
+
+ # Check for subchain (split at Filter level)
+ if '|' in etl_section_name:
+ # Have subchain: use Chain to assemble
+ sub_chain = Chain(etl_section_name, self.config_dict)
+ sub_chain.assemble()
+ child_comp = sub_chain.first_comp
+ else:
+ # Single component (Output) to split
+ child_comp = factory.create_obj(self.config_dict, etl_section_name.strip())
+
+ # Assemble Components (can be subchains) for Splitter later
+ sub_comps.append(child_comp)
+ if '(' in chain_str:
+ # Still components (subchains) to assemble for Splitter
+ continue
+
+ if len(sub_comps) > 0:
+ if chain_str.startswith('|'):
+ # Next component is Merger with children
+ etl_comp = Merger(self.config_dict, sub_comps)
+ dummy, chain_str = chain_str.split('|', 1)
+ else:
+ # Next component is Splitter with children
+ etl_comp = Splitter(self.config_dict, sub_comps)
+ sub_comps = []
+ else:
+
+ # "Normal" case: regular Components piped in Chain
+ if '|' in chain_str:
+ # More than one component in remaining Chain
+ etl_section_name, chain_str = chain_str.split('|', 1)
+ else:
+ # Last element, we're done!
+ etl_section_name = chain_str
+ chain_str = None
+
+ # Create the ETL component by name and properties
+ etl_comp = factory.create_obj(self.config_dict, etl_section_name.strip())
# Add component to end of Chain
self.add(etl_comp)
@@ -59,6 +106,53 @@ class Chain:
# Remember current
self.cur_comp = etl_comp
+ def get_by_class(self, clazz):
+ """
+ Get Component instance from Chain by class, mainly for testing.
+ :param clazz:
+ :return Component:
+ """
+ cur_comp = self.first_comp
+ while cur_comp:
+ if cur_comp.__class__ == clazz:
+ return cur_comp
+
+ # Try next in Chain
+ cur_comp = cur_comp.next
+
+ return None
+
+ def get_by_id(self, id):
+ """
+ Get Component instance from Chain, mainly for testing.
+ :param name:
+ :return Component:
+ """
+ cur_comp = self.first_comp
+ while cur_comp:
+ if cur_comp.get_id() == id:
+ return cur_comp
+
+ # Try next in Chain
+ cur_comp = cur_comp.next
+
+ return None
+
+ def get_by_index(self, index):
+ """
+ Get Component instance from Chain by position/index in Chain, mainly for testing.
+ :param clazz:
+ :return Component:
+ """
+ cur_comp = self.first_comp
+ i = 0
+ while cur_comp and i < index:
+ # Try next in Chain
+ cur_comp = cur_comp.next
+ i += 1
+
+ return cur_comp
+
def run(self):
"""
Run the ETL Chain.
@@ -74,14 +168,14 @@ class Chain:
rounds = 0
try:
while not packet.is_end_of_stream():
- # try:
+ # try:
# Invoke the first component to start the chain
packet.init()
packet = self.first_comp.process(packet)
rounds += 1
# except (Exception), e:
- # log.error("Fatal Error in ETL: %s"% str(e))
- # break
+ # log.error("Fatal Error in ETL: %s"% str(e))
+ # break
finally:
# Always one time exit for entire Chain
self.first_comp.do_exit()
diff --git a/stetl/component.py b/stetl/component.py
index f3fa887..cf6ead8 100644
--- a/stetl/component.py
+++ b/stetl/component.py
@@ -20,6 +20,7 @@ class Config(object):
Each property is defined by @Config(type, default, required).
Basic idea comes from: https://wiki.python.org/moin/PythonDecoratorLibrary#Cached_Properties
"""
+
def __init__(self, ptype=str, default=None, required=False):
"""
If there are no decorator arguments, the function
@@ -30,7 +31,7 @@ class Config(object):
self.default = default
self.required = required
- def __call__(self, fget, doc=None):
+ def __call__(self, fget, doc=''):
"""
The __call__ method is not called until the
decorated function is called. self is returned such that __get__ below is called
@@ -41,10 +42,24 @@ class Config(object):
self.property_name = fget.__name__
# print "Inside __call__() name=%s" % self.property_name
- # For Spinx documention build we need the original function with docstring.
+ # For Spinx documentation build we need the original function with docstring.
IS_SPHINX_BUILD = bool(os.getenv('SPHINX_BUILD'))
if IS_SPHINX_BUILD:
- fget.__doc__ = '``CONFIG`` - %s' % fget.__doc__
+ doc = doc.strip()
+ # TODO more detail, example below
+ # doc = '``Parameter`` - %s\n\n' % doc
+ doc += '* type: %s\n' % str(self.ptype).split("'")[1]
+ doc += '* required: %s\n' % self.required
+ doc += '* default: %s\n' % self.default
+
+ # if self.value:
+ # doc += '* value: %s\n' % self.value
+ # else:
+ # doc += '* required: %s\n' % self.required
+ # doc += '* default: %s\n' % self.default
+ # doc += '* value_range: %s\n' % self.value_range
+
+ fget.__doc__ = '``CONFIG`` %s\n%s' % (fget.__doc__, doc)
return fget
else:
return self
@@ -79,7 +94,7 @@ class Config(object):
return comp_inst.cfg_vals[self.property_name]
-class Component:
+class Component(object):
"""
Abstract Base class for all Input, Filter and Output Components.
@@ -89,8 +104,6 @@ class Component:
def input_format(self):
"""
The specific input format if the consumes parameter is a list or the format to be converted to the output_format.
- Required: False
- Default: None
"""
pass
@@ -98,8 +111,6 @@ class Component:
def output_format(self):
"""
The specific output format if the produces parameter is a list or the format to which the input format is converted.
- Required: False
- Default: None
"""
pass
@@ -110,6 +121,7 @@ class Component:
# The actual typed values as populated within Config Decorator
self.cfg_vals = dict()
self.next = None
+ self.section = section
# First assume single output provided by derived class
self._output_format = produces
@@ -142,8 +154,22 @@ class Component:
raise ValueError(
'Incompatible components are linked: %s and %s' % (str(self), str(self.next)))
+ # Get our id: currently the [section] name
+ def get_id(self):
+ return self.section
+
+ # Get last Component in Chain
+ def get_last(self):
+ last = self
+ while last.next:
+ last = last.next
+ if isinstance(last, list):
+ last = last[0]
+ return last
+
# Check our compatibility with the next Component in the Chain
def is_compatible(self):
+
# Ok, nothing next in Chain
if self.next is None or self._output_format is FORMAT.none or self.next._input_format == FORMAT.any:
return True
@@ -232,4 +258,3 @@ class Component:
Allows derived Components to perform a one-time exit/cleanup.
"""
pass
-
diff --git a/stetl/etl.py b/stetl/etl.py
index bf1c28a..3d11e05 100755
--- a/stetl/etl.py
+++ b/stetl/etl.py
@@ -14,6 +14,7 @@ import StringIO
log = Util.get_log('ETL')
+
class ETL:
"""The main class: builds ETL Chains with connected Components from a config and let them run.
@@ -70,10 +71,9 @@ class ETL:
else:
# Parse config file directly
self.configdict.read(config_file)
- except Exception, e:
+ except Exception as e:
log.error("Fatal Error reading config file: err=%s" % str(e))
-
def run(self):
# The main ETL processing
log.info("START")
@@ -94,7 +94,7 @@ class ETL:
chains_str_arr = chains_str.split(',')
for chain_str in chains_str_arr:
# Build single Chain of components and let it run
- chain = Chain(chain_str.strip(), self.configdict)
+ chain = Chain(chain_str, self.configdict)
chain.assemble()
# Run the ETL for this Chain
diff --git a/stetl/factory.py b/stetl/factory.py
index dd48701..817c207 100755
--- a/stetl/factory.py
+++ b/stetl/factory.py
@@ -4,6 +4,7 @@ from util import Util
log = Util.get_log('factory')
+
class Factory:
"""
Object and class Factory (Pattern).
@@ -52,7 +53,6 @@ class Factory:
return class_obj
-
def new_instance(self, class_obj, configdict, section):
"""Returns object instance from class instance.
@@ -66,11 +66,11 @@ class Factory:
factory = Factory()
-#bar="bar"
-#foo="foo"
+# bar="bar"
+# foo="foo"
# x = mkinst("factory.Foo", bar, 0, 4, disc="bust")
-#y = mkinst("Bar", foo, batman="robin")
+# y = mkinst("Bar", foo, batman="robin")
# = import_class("foo.Foo", foo)
#
-#o = x(foo)
-#x.p()
+# o = x(foo)
+# x.p()
diff --git a/stetl/filters/formatconverter.py b/stetl/filters/formatconverter.py
index 8a1e1fd..3a1e0dc 100644
--- a/stetl/filters/formatconverter.py
+++ b/stetl/filters/formatconverter.py
@@ -32,12 +32,6 @@ class FormatConverter(Filter):
def converter_args(self):
"""
Custom converter-specific arguments.
-
- Type: dictionary
-
- Required: False
-
- Default: None
"""
pass
@@ -71,7 +65,7 @@ class FormatConverter(Filter):
# OGR feature def
self.feat_def = None
-
+
def invoke(self, packet):
if packet.data is None:
@@ -140,7 +134,8 @@ class FormatConverter(Filter):
return packet
@staticmethod
- def etree_doc2struct(packet, strip_space=True, strip_ns=True, sub=False, attr_prefix='', gml2ogr=True, ogr2json=True):
+ def etree_doc2struct(packet, strip_space=True, strip_ns=True, sub=False, attr_prefix='', gml2ogr=True,
+ ogr2json=True):
"""
:param packet:
:param strip_space:
@@ -168,7 +163,8 @@ class FormatConverter(Filter):
return packet
@staticmethod
- def etree_elem2struct(packet, strip_space=True, strip_ns=True, sub=False, attr_prefix='', gml2ogr=True, ogr2json=True):
+ def etree_elem2struct(packet, strip_space=True, strip_ns=True, sub=False, attr_prefix='', gml2ogr=True,
+ ogr2json=True):
"""
:param packet:
:param strip_space:
@@ -357,6 +353,7 @@ class FormatConverter(Filter):
packet.data = feature
return packet
+
# 'xml_line_stream', 'etree_doc', 'etree_element', 'etree_feature_array', 'xml_doc_as_string',
# 'string', 'record', 'record_array', 'geojson_collection', geojson_feature', 'struct',
# 'ogr_feature', 'ogr_feature_array', 'any'
diff --git a/stetl/filters/gmlfeatureextractor.py b/stetl/filters/gmlfeatureextractor.py
index 9d05535..b7f5f8b 100644
--- a/stetl/filters/gmlfeatureextractor.py
+++ b/stetl/filters/gmlfeatureextractor.py
@@ -50,5 +50,3 @@ class GmlFeatureExtractor(Filter):
self.total_features += len(packet.data)
log.info('extracted %d features from GML etree doc (total = %d)' % (len(packet.data), self.total_features))
return packet
-
-
diff --git a/stetl/filters/gmlsplitter.py b/stetl/filters/gmlsplitter.py
index 8fe9fea..ea66666 100755
--- a/stetl/filters/gmlsplitter.py
+++ b/stetl/filters/gmlsplitter.py
@@ -12,6 +12,7 @@ from stetl.packet import FORMAT
log = Util.get_log('gmlsplitter')
+
class GmlSplitter(Filter):
"""
Split a stream of text XML lines into documents
@@ -20,6 +21,7 @@ class GmlSplitter(Filter):
consumes=FORMAT.xml_line_stream, produces=FORMAT.etree_doc
"""
+
def __init__(self, configdict, section='gml_splitter'):
Filter.__init__(self, configdict, section, consumes=FORMAT.xml_line_stream, produces=FORMAT.etree_doc)
@@ -126,7 +128,7 @@ class GmlSplitter(Filter):
try:
# print '[' + self.buffer.getvalue() + ']'
packet.data = etree.parse(self.buffer, self.xml_parser)
- # print buffer.getvalue()
+ # print buffer.getvalue()
except Exception, e:
bufStr = self.buffer.getvalue()
if not bufStr:
@@ -179,4 +181,3 @@ class GmlSplitter(Filter):
# Still within feature
return False
-
diff --git a/stetl/filters/nullfilter.py b/stetl/filters/nullfilter.py
new file mode 100644
index 0000000..0e43d5e
--- /dev/null
+++ b/stetl/filters/nullfilter.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+#
+# Filter that does noting, just passes any data through.
+#
+# Author:Just van den Broecke
+
+from stetl.util import Util
+from stetl.filter import Filter
+from stetl.packet import FORMAT
+
+log = Util.get_log("nullfilter")
+
+
+class NullFilter(Filter):
+ """
+ Pass-through Filter, does nothing. Mainly used in Test Cases.
+ """
+
+ # Constructor
+ def __init__(self, configdict, section, consumes=FORMAT.any, produces=FORMAT.any):
+ Filter.__init__(self, configdict, section, consumes, produces)
+
+ def invoke(self, packet):
+ return packet
diff --git a/stetl/filters/packetbuffer.py b/stetl/filters/packetbuffer.py
new file mode 100644
index 0000000..7be94ff
--- /dev/null
+++ b/stetl/filters/packetbuffer.py
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+#
+# Packet buffering.
+#
+# Author:Just van den Broecke
+
+import copy
+from stetl.util import Util
+from stetl.filter import Filter
+from stetl.packet import FORMAT
+
+log = Util.get_log("packetbuffer")
+
+
+class PacketBuffer(Filter):
+ """
+ Buffers all incoming Packets, main use is unit-testing to inspect Packets after ETL is done.
+ """
+
+ # Constructor
+ def __init__(self, configdict, section):
+ Filter.__init__(self, configdict, section, consumes=FORMAT.any, produces=FORMAT.any)
+ self.packet_list = []
+
+ def invoke(self, packet):
+ # Buffer Packet and pass-through, we need a deep copy as Packets may be cleared/reused
+ self.packet_list.append(copy.copy(packet))
+ return packet
diff --git a/stetl/filters/packetwriter.py b/stetl/filters/packetwriter.py
index a390ddc..68c85f1 100644
--- a/stetl/filters/packetwriter.py
+++ b/stetl/filters/packetwriter.py
@@ -27,10 +27,6 @@ class PacketWriter(Filter):
def file_path(self):
"""
File path to write content to.
-
- Required: True
-
- Default: None
"""
pass
@@ -56,6 +52,6 @@ class PacketWriter(Filter):
out_file.close()
log.info("written to %s" % file_path)
-
+
packet.data = file_path
return packet
diff --git a/stetl/filters/stringfilter.py b/stetl/filters/stringfilter.py
index 8e505e1..c260bea 100644
--- a/stetl/filters/stringfilter.py
+++ b/stetl/filters/stringfilter.py
@@ -10,6 +10,7 @@ from stetl.packet import FORMAT
log = Util.get_log("stringfilter")
+
class StringFilter(Filter):
"""
Base class for any string filtering
@@ -53,7 +54,3 @@ class StringSubstitutionFilter(StringFilter):
# String substitution based on Python String.format()
packet.data = packet.data.format(**self.format_args_dict)
return packet
-
-
-
-
diff --git a/stetl/filters/templatingfilter.py b/stetl/filters/templatingfilter.py
index e9cfe05..3adc90d 100644
--- a/stetl/filters/templatingfilter.py
+++ b/stetl/filters/templatingfilter.py
@@ -36,8 +36,6 @@ class TemplatingFilter(Filter):
def template_file(self):
"""
Path to template file. One of template_file or template_string needs to be configured.
- Required: False
- Default: None
"""
pass
@@ -45,8 +43,6 @@ class TemplatingFilter(Filter):
def template_string(self):
"""
Template string. One of template_file or template_string needs to be configured.
- Required: False
- Default: None
"""
pass
@@ -119,7 +115,7 @@ class StringTemplatingFilter(TemplatingFilter):
packet.data = [self.template.substitute(item) for item in packet.data]
else:
packet.data = self.template.substitute(packet.data)
-
+
return packet
@@ -147,8 +143,6 @@ class Jinja2TemplatingFilter(TemplatingFilter):
def template_search_paths(self):
"""
List of directories where to search for templates, default is current working directory only.
- Required: False
- Default: [os.getcwd()]
"""
pass
@@ -157,8 +151,6 @@ class Jinja2TemplatingFilter(TemplatingFilter):
"""
One or more JSON files or URLs with global variables that can be used anywhere in template.
Multiple files will be merged into one globals dictionary
- Required: False
- Default: None
"""
pass
@@ -174,7 +166,7 @@ class Jinja2TemplatingFilter(TemplatingFilter):
def create_template(self):
try:
from jinja2 import Environment, FileSystemLoader
- except Exception, e:
+ except Exception as e:
log.error(
'Cannot import modules from Jinja2, err= %s; You probably need to install Jinja2 first, see http://jinja.pocoo.org',
str(e))
@@ -307,4 +299,3 @@ class Jinja2TemplatingFilter(TemplatingFilter):
log.error(gml_str)
return gml_str
-
diff --git a/stetl/filters/xmlassembler.py b/stetl/filters/xmlassembler.py
index 60c794f..e83735b 100644
--- a/stetl/filters/xmlassembler.py
+++ b/stetl/filters/xmlassembler.py
@@ -11,6 +11,7 @@ from stetl.packet import FORMAT
log = Util.get_log('xmlassembler')
+
class XmlAssembler(Filter):
"""
Split a stream of etree DOM XML elements (usually Features) into etree DOM docs.
@@ -39,6 +40,10 @@ class XmlAssembler(Filter):
# Valid element: consume and handle
self.consume_element(packet)
+ # Document is obviously not finished, reset EoD/EoS in packet
+ packet.set_end_of_stream(False)
+ packet.set_end_of_doc(False)
+
if packet.is_end_of_stream() or packet.is_end_of_doc() or len(self.element_arr) >= self.max_elements:
# EOF but still data in buffer: make doc
# log.info("Flush doc")
@@ -50,23 +55,23 @@ class XmlAssembler(Filter):
# Always move the data (element) from packet
element = packet.consume()
- if element is None or packet.is_end_of_stream() is True:
- return packet
-
- self.total_element_count += 1
+ if element is not None:
+ self.total_element_count += 1
+ self.element_arr.append(element)
- self.element_arr.append(element)
return packet
def flush_elements(self, packet):
+ packet.set_end_of_doc()
+
if len(self.element_arr) == 0:
return packet
# Start new doc (TODO clone)
try:
etree_doc = etree.fromstring(self.container_doc, self.xml_parser)
- except Exception, e:
- log.error("new container doc not OK")
+ except Exception as e:
+ log.error('new container doc not OK: %s' % str(e))
return packet
parent_element = etree_doc.xpath(self.element_container_xpath)
@@ -76,9 +81,7 @@ class XmlAssembler(Filter):
for element in self.element_arr:
parent_element.append(element)
- log.info("xmldoc ready: elms=%d total_elms=%d" % (len(self.element_arr), self.total_element_count))
+ log.info('xmldoc ready: elms=%d total_elms=%d' % (len(self.element_arr), self.total_element_count))
packet.data = etree_doc
self.element_arr = []
return packet
-
-
diff --git a/stetl/filters/xmlelementreader.py b/stetl/filters/xmlelementreader.py
index 4a16497..108752f 100644
--- a/stetl/filters/xmlelementreader.py
+++ b/stetl/filters/xmlelementreader.py
@@ -5,6 +5,8 @@
#
# Author: Frank Steggink
#
+from copy import deepcopy
+
from stetl.component import Config
from stetl.filter import Filter
from stetl.util import Util, etree
@@ -28,10 +30,6 @@ class XmlElementReader(Filter):
"""
Comma-separated string of XML (feature) element tag names of the elements that should be extracted
and added to the output element stream.
-
- Required: True
-
- Default: None
"""
pass
@@ -39,10 +37,6 @@ class XmlElementReader(Filter):
def strip_namespaces(self):
"""
should namespaces be removed from the input document and thus not be present in the output element stream?
-
- Required: False
-
- Default: False
"""
pass
@@ -67,7 +61,7 @@ class XmlElementReader(Filter):
event = None
packet.data = None
-
+
if self.context is None:
# Open file
fd = open(self.cur_file_path)
@@ -82,8 +76,8 @@ class XmlElementReader(Filter):
return packet
def process_xml(self, packet):
- while not self.context is None:
- #while not packet.is_end_of_doc():
+ while self.context is not None:
+ # while not packet.is_end_of_doc():
try:
event, elem = self.context.next()
except (etree.XMLSyntaxError, StopIteration):
@@ -110,20 +104,18 @@ class XmlElementReader(Filter):
if tag in self.element_tags:
if event == "start":
- # TODO check if deepcopy is the right thing to do here.
- # packet.data = elem
pass
- # self.root.remove(elem)
elif event == "end":
- # Delete the element from the tree
- # self.root.clear()
- packet.data = elem
+ packet.data = deepcopy(elem)
self.elem_count += 1
- self.root.remove(elem)
if self.strip_namespaces:
packet.data = Util.stripNamespaces(elem).getroot()
+ # Clear the root element, since iterparse still builds a tree
+ # See http://effbot.org/zone/element-iterparse.htm
+ self.root.clear()
+
# If there is a next component, let it process
if self.next:
# Hand-over data (line, doc whatever) to the next component
diff --git a/stetl/filters/xsltfilter.py b/stetl/filters/xsltfilter.py
index f98bc05..00e225d 100755
--- a/stetl/filters/xsltfilter.py
+++ b/stetl/filters/xsltfilter.py
@@ -39,4 +39,3 @@ class XsltFilter(Filter):
packet.data = self.xslt_obj(packet.data)
log.info("XSLT Transform OK")
return packet
-
diff --git a/stetl/filters/zipfileextractor.py b/stetl/filters/zipfileextractor.py
index dd01cd9..680640b 100644
--- a/stetl/filters/zipfileextractor.py
+++ b/stetl/filters/zipfileextractor.py
@@ -26,10 +26,6 @@ class ZipFileExtractor(Filter):
def file_path(self):
"""
File name to write the extracted file to.
-
- Required: True
-
- Default: None
"""
pass
@@ -41,13 +37,11 @@ class ZipFileExtractor(Filter):
self.cur_file_path = self.cfg.get('file_path')
def invoke(self, packet):
- event = None
-
+
if packet.data is None:
log.info("No file name given")
return packet
-
- import os
+
import zipfile
with zipfile.ZipFile(packet.data['file_path']) as z:
@@ -61,10 +55,10 @@ class ZipFileExtractor(Filter):
packet.data = self.cur_file_path
return packet
-
+
def after_chain_invoke(self, packet):
import os.path
if os.path.isfile(self.cur_file_path):
os.remove(self.cur_file_path)
-
+
return True
diff --git a/stetl/input.py b/stetl/input.py
index 02b50b0..fd446f7 100755
--- a/stetl/input.py
+++ b/stetl/input.py
@@ -26,4 +26,3 @@ class Input(Component):
def read(self, packet):
return packet
-
diff --git a/stetl/inputs/dbinput.py b/stetl/inputs/dbinput.py
index 0d75492..1de5cf8 100644
--- a/stetl/inputs/dbinput.py
+++ b/stetl/inputs/dbinput.py
@@ -65,25 +65,41 @@ class SqlDbInput(DbInput):
The query (string) to fire.
"""
pass
+
# End attribute config meta
- def __init__(self, configdict, section, produces):
- DbInput.__init__(self, configdict, section, produces=produces)
- self.columns = self.column_names
+ def __init__(self, configdict, section):
+ DbInput.__init__(self, configdict, section, produces=[FORMAT.record_array, FORMAT.record])
+ self.columns = None
+ if self.column_names is not None:
+ self.columns = self.column_names.split(',')
self.select_all = "select * from %s" % self.table
- def result_to_output(self, db_records):
+ def tuples_to_records(self, db_tuples, columns=None):
"""
- Convert DB-specific record iterator to single Python record (dict) or record array (list of dict).
+ Convert tuple array (list of tuple) to list of records (list of dict's) using list of column names.
"""
+ if columns is None:
+ columns = self.columns
+
# record is Python list of Python dict (multiple records)
records = list()
# Convert list of lists to list of dict using column_names
- for db_record in db_records:
- records.append(dict(zip(self.columns, db_record)))
+ for db_tuple in db_tuples:
+ records.append(dict(zip(columns, db_tuple)))
+
+ return records
+
+ def result_to_output(self, db_tuples):
+ """
+ Convert DB-specific record tuples to single Python record (dict) or record array (list of dict).
+
+ """
+
+ records = self.tuples_to_records(db_tuples)
# We may have specified a single record output_format in rare cases
if self.output_format == FORMAT.record:
@@ -99,11 +115,11 @@ class SqlDbInput(DbInput):
DB-neutral query returning Python record list.
"""
- # Perform DB-specific query
- db_records = self.raw_query(query_str)
+ # Perform DB-specific query (gets result as list of values as tuples)
+ db_tuples = self.raw_query(query_str)
# Convert query result to record_array
- return self.result_to_output(db_records)
+ return self.result_to_output(db_tuples)
def raw_query(self, query_str):
"""
@@ -145,7 +161,7 @@ class PostgresDbInput(SqlDbInput):
@Config(ptype=str, required=False, default='5432')
def port(self):
"""
- port for host, defaults to `'5432'
+ port for host, defaults to '5432'
"""
pass
@@ -169,22 +185,31 @@ class PostgresDbInput(SqlDbInput):
The postgres schema name, defaults to 'public'
"""
pass
+
# End attribute config meta
def __init__(self, configdict, section):
- SqlDbInput.__init__(self, configdict, section, produces=[FORMAT.record_array, FORMAT.record])
+ SqlDbInput.__init__(self, configdict, section)
self.db = None
+ def init_columns(self):
+ if self.columns is not None:
+ # Already initialized, reset columns_names to re-init
+ return
+
+ if self.column_names is None:
+ # If no explicit column names given, get all columns from DB meta info
+ self.columns = self.db.get_column_names(self.cfg.get('table'), self.cfg.get('schema'))
+ else:
+ # Columns provided: make list
+ self.columns = self.column_names.split(',')
+
def init(self):
# Connect only once to DB
log.info('Init: connect to DB')
self.db = PostGIS(self.cfg.get_dict())
self.db.connect()
-
- # If no explicit column names given, get from DB meta info
- self.columns = self.column_names
- if self.column_names is None:
- self.columns = self.db.get_column_names(self.cfg.get('table'), self.cfg.get('schema'))
+ self.init_columns()
def exit(self):
# Disconnect from DB when done
@@ -193,6 +218,7 @@ class PostgresDbInput(SqlDbInput):
self.db.disconnect()
def raw_query(self, query_str):
+ self.init_columns()
self.db.execute(query_str)
@@ -201,6 +227,7 @@ class PostgresDbInput(SqlDbInput):
return db_records
+
class SqliteDbInput(SqlDbInput):
"""
Input by querying records from a SQLite database.
@@ -211,7 +238,7 @@ class SqliteDbInput(SqlDbInput):
"""
def __init__(self, configdict, section):
- SqlDbInput.__init__(self, configdict, section, produces=[FORMAT.record_array, FORMAT.record])
+ SqlDbInput.__init__(self, configdict, section)
self.db = None
import sqlite3
@@ -241,4 +268,4 @@ class SqliteDbInput(SqlDbInput):
log.info('%d records read' % len(db_records))
conn.close()
- return db_records
\ No newline at end of file
+ return db_records
diff --git a/stetl/inputs/deegreeinput.py b/stetl/inputs/deegreeinput.py
index d128937..a75f284 100644
--- a/stetl/inputs/deegreeinput.py
+++ b/stetl/inputs/deegreeinput.py
@@ -8,6 +8,7 @@
import codecs
import re
+from stetl.component import Config
from stetl.postgis import PostGIS
from stetl.input import Input
from stetl.util import Util, etree, StringIO
@@ -15,6 +16,7 @@ from stetl.packet import FORMAT
log = Util.get_log('deegreeinput')
+
class DeegreeBlobstoreInput(Input):
"""
Read features from deegree Blobstore DB into an etree doc.
@@ -22,13 +24,47 @@ class DeegreeBlobstoreInput(Input):
produces=FORMAT.etree_doc
"""
+ # Start attribute config meta
+
+ @Config(ptype=int, required=False, default=10000)
+ def max_features_per_doc(self):
+ """
+ Max features to read from input feature GML stream per internal document.
+ """
+ pass
+
+ @Config(ptype=str, required=True, default=None)
+ def start_container(self):
+ """
+ Tag that starts container.
+ """
+ pass
+
+ @Config(ptype=str, required=True, default=None)
+ def end_container(self):
+ """
+ Tag that ends container.
+ """
+ pass
+
+ @Config(ptype=str, required=False, default=False)
+ def start_feature_tag(self):
+ """
+ XML tag that starts Feature.
+ """
+ pass
+
+ @Config(ptype=str, required=False, default=None)
+ def end_feature_tag(self):
+ """
+ XML tag that ends Feature.
+ """
+ pass
+
+ # End attribute config meta
+
def __init__(self, configdict, section):
Input.__init__(self, configdict, section, produces=FORMAT.etree_doc)
- self.max_features_per_doc = self.cfg.get_int('max_features_per_doc', 10000)
- self.start_container = self.cfg.get('start_container')
- self.end_container = self.cfg.get('end_container')
- self.start_feature_tag = self.cfg.get('start_feature_tag')
- self.end_feature_tag = self.cfg.get('end_feature_tag')
self.cur_feature_blob = None
self.rowcount = 0
@@ -154,15 +190,14 @@ class DeegreeBlobstoreInput(Input):
try:
# print '[' + self.buffer.getvalue() + ']'
packet.data = etree.parse(self.buffer, self.xml_parser)
- # print buffer.getvalue()
- except Exception, e:
+ # print buffer.getvalue()
+ except Exception as e:
bufStr = self.buffer.getvalue()
if not bufStr:
log.info("parse buffer empty: content=[%s]" % bufStr)
else:
log.error("error in buffer parsing %s" % str(e))
- print bufStr
+ # print(bufStr)
raise
self.buffer.close()
self.buffer = None
-
diff --git a/stetl/inputs/fileinput.py b/stetl/inputs/fileinput.py
index 26b5e0a..642c2c1 100644
--- a/stetl/inputs/fileinput.py
+++ b/stetl/inputs/fileinput.py
@@ -30,10 +30,6 @@ class FileInput(Input):
"""
Path to file or files or URLs: can be a dir or files or URLs
or even multiple, comma separated. For URLs only JSON is supported now.
-
- Required: True
-
- Default: None
"""
pass
@@ -41,11 +37,7 @@ class FileInput(Input):
def filename_pattern(self):
"""
Filename pattern according to Python ``glob.glob`` for example:
- '\*.[gxGX][mM][lL]'
-
- Required: False
-
- Default: '\*.[gxGX][mM][lL]'
+ '\\*.[gxGX][mM][lL]'
"""
pass
@@ -53,10 +45,6 @@ class FileInput(Input):
def depth_search(self):
"""
Should we recurse into sub-directories to find files?
-
- Required: False
-
- Default: False
"""
pass
@@ -114,10 +102,6 @@ class StringFileInput(FileInput):
Formatting of content according to Python String.format()
Input file should have substitutable values like {schema} {foo}
format_args should be of the form ``format_args = schema:test foo:bar``
-
- Required: False
-
- Default: None
"""
pass
@@ -130,40 +114,24 @@ class StringFileInput(FileInput):
# Optional formatting of content according to Python String.format()
# Input file should have substitutable values like {schema} {foo}
# format_args should be of the form format_args = schema:test foo:bar
-
if self.format_args:
# Convert string to dict: http://stackoverflow.com/a/1248990
self.format_args = Util.string_to_dict(self.format_args, ':')
- def read(self, packet):
- # No more files left and done with current file ?
- if not len(self.file_list) and self.file is None:
- packet.set_end_of_stream()
- log.info("EOF file list, all files done")
- return packet
-
- # Done with current file or first file ?
- if self.file is None:
- self.cur_file_path = self.file_list.pop(0)
- self.file = open(self.cur_file_path, 'r')
- log.info("file opened : %s" % self.cur_file_path)
-
- # Assume valid file content
- file_content = self.file.read()
-
- # Optional: string substitution based on Python String.format()
- # But you can also use StringSubstitutionFilter from filters.
- if self.format_args:
- file_content = file_content.format(**self.format_args)
-
- # Cleanup
- self.file.close()
- self.file = None
+ def read_file(self, file_path):
+ """
+ Overridden from base class.
+ """
- log.info("file read : %s size=%d" % (self.cur_file_path, len(file_content)))
+ file_content = None
+ with open(file_path, 'r') as f:
+ file_content = f.read()
+ # Optional: string substitution based on Python String.format()
+ # But you can also use StringSubstitutionFilter from filters.
+ if self.format_args:
+ file_content = file_content.format(**self.format_args)
- packet.data = file_content
- return packet
+ return file_content
class XmlFileInput(FileInput):
@@ -182,8 +150,8 @@ class XmlFileInput(FileInput):
data = None
try:
data = etree.parse(file_path)
- except Exception, e:
- log.info("file read and parsed NOT OK : %s" % file_path)
+ except Exception as e:
+ log.info('file read and parsed NOT OK : %s, err=%s' % (file_path, str(e)))
return data
@@ -203,10 +171,6 @@ class XmlElementStreamerFileInput(FileInput):
"""
Comma-separated string of XML (feature) element tag names of the elements that should be extracted
and added to the output element stream.
-
- Required: True
-
- Default: None
"""
pass
@@ -214,10 +178,6 @@ class XmlElementStreamerFileInput(FileInput):
def strip_namespaces(self):
"""
should namespaces be removed from the input document and thus not be present in the output element stream?
-
- Required: False
-
- Default: False
"""
pass
@@ -385,21 +345,14 @@ class CsvFileInput(FileInput):
def delimiter(self):
"""
A one-character string used to separate fields. It defaults to ','.
-
- Required: False
-
- Default: ',' (comma)
"""
pass
@Config(ptype=str, default='"', required=False)
def quote_char(self):
"""
- A one-character string used to quote fields containing special characters, such as the delimiter or quotechar, or which contain new-line characters. It defaults to '"'
-
- Required: False
-
- Default: "
+ A one-character string used to quote fields containing special characters, such as the delimiter or quotechar,
+ or which contain new-line characters. It defaults to '"'.
"""
pass
@@ -427,7 +380,7 @@ class CsvFileInput(FileInput):
packet.data = self.csv_reader.next()
log.info("CSV row nr %d read: %s" % (self.csv_reader.line_num - 1, packet.data))
- except Exception, e:
+ except Exception:
if self._output_format == FORMAT.record_array:
packet.data = self.arr
@@ -465,7 +418,7 @@ class JsonFileInput(FileInput):
with open(file_path) as data_file:
file_data = json.load(data_file)
- except Exception, e:
+ except Exception as e:
log.error('Cannot read JSON from %s, err= %s' % (file_path, str(e)))
raise e
@@ -481,20 +434,13 @@ class ApacheLogFileInput(FileInput):
"""
- @Config(ptype=dict, default=
- {'%l': 'logname', '%>s': 'status', '%D': 'deltat', '%{User-agent}i': 'agent', '%b': 'bytes',
- '%{Referer}i': 'referer', '%u': 'user', '%t': 'time', "'%h": 'host', '%r': 'request'}
- , required=False)
+ @Config(ptype=dict, default={'%l': 'logname', '%>s': 'status', '%D': 'deltat',
+ '%{User-agent}i': 'agent', '%b': 'bytes', '%{Referer}i': 'referer',
+ '%u': 'user', '%t': 'time', "'%h": 'host', '%r': 'request'}, required=False)
def key_map(self):
"""
Map of cryptic %-field names to readable keys in record.
- Type: dictionary
-
- Required: False
-
- Default: {'%l': 'logname', '%>s': 'status', '%D': 'deltat', '%{User-agent}i': 'agent', '%b': 'bytes', '%{Referer}i': 'referer', '%u': 'user', '%t': 'time', "'%h": 'host', '%r': 'request'}
-
"""
pass
@@ -502,10 +448,6 @@ class ApacheLogFileInput(FileInput):
def log_format(self):
"""
Log format according to Apache CLF
-
- Required: False
-
- Default: '%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"'
"""
pass
@@ -572,10 +514,6 @@ class ZipFileInput(FileInput):
def name_filter(self):
"""
Regular "glob.glob" expression for filtering out filenames from the ZIP archive.
-
- Required: False
-
- Default: * (all files in zip-archive)
"""
pass
@@ -616,3 +554,32 @@ class ZipFileInput(FileInput):
self.file_content = None
return packet
+
+
+class GlobFileInput(FileInput):
+ """
+ Returns file names based on the glob.glob pattern given as filename_filter.
+
+ produces=FORMAT.string or FORMAT.line_stream
+ """
+
+ def __init__(self, configdict, section, produces=[FORMAT.string, FORMAT.line_stream]):
+ FileInput.__init__(self, configdict, section, produces)
+
+ def read(self, packet):
+ if not len(self.file_list):
+ return packet
+
+ file_path = self.file_list.pop(0)
+
+ # TODO: os.path.join?
+ packet.data = file_path
+
+ # One-time read: we're all done
+ packet.set_end_of_doc()
+ if not len(self.file_list):
+ log.info("all files done")
+ packet.set_end_of_stream()
+
+ self.file_list_done.append(file_path)
+ return packet
diff --git a/stetl/inputs/httpinput.py b/stetl/inputs/httpinput.py
index 7c59324..72d3b97 100644
--- a/stetl/inputs/httpinput.py
+++ b/stetl/inputs/httpinput.py
@@ -7,6 +7,7 @@
import re
from urllib2 import Request, urlopen, URLError, HTTPError
import urllib
+import base64
from stetl.component import Config
from stetl.input import Input
@@ -33,14 +34,45 @@ class HttpInput(Input):
def url(self):
"""
The HTTP URL string.
+ """
+ pass
+
+ @Config(ptype=dict, default=None, required=False)
+ def auth(self):
+ """
+ Authentication data: Flat JSON-like struct dependent on auth type/schema.
+ Only the `type` field is required, other fields depend on auth schema.
+ Supported values : ::
+
+ type: basic|token
+
+ If the type is ``basic`` (HTTP Basic Authentication) two additional fields ``user``
+ and ``password`` are required.
+ If the type is ``token`` (HTTP Token) additional two additional fields ``keyword``
+ and ``token`` are required.
+
+ Any required Base64 encoding is provided by ``HttpInput``.
- Required: True
+ Examples: ::
- Default: None
+ # Basic Auth
+ url = https://some.rest.api.com
+ auth = {
+ type: basic,
+ user: myname
+ password: mypassword
+ }
+
+ # Token Auth
+ url = https://some.rest.api.com
+ auth = {
+ type: token,
+ keyword: Bearer
+ token: mytoken
+ }
"""
pass
-
@Config(ptype=dict, default=None, required=False)
def parameters(self):
"""
@@ -57,10 +89,6 @@ class HttpInput(Input):
outputFormat : text/xml; subtype=gml/2.1.2,
typename : natura2000
}
-
- Required: False
-
- Default: None
"""
pass
@@ -71,9 +99,33 @@ class HttpInput(Input):
log.info("url=%s parameters=%s" % (self.url, self.parameters))
+ def add_authorization(self, request):
+ """
+ Add authorization from config data. Authorization scheme-specific.
+ May be extended or overloaded for additional schemes.
+
+ :param request: the HTTP Request
+ :return:
+ """
+ auth_creds = self.auth
+ auth_type = auth_creds['type']
+ auth_val = None
+ if auth_type == 'basic':
+ # Basic auth: http://mozgovipc.blogspot.nl/2012/06/python-http-basic-authentication-with.html
+ # base64 encode username and password
+ # write the Authorization header like: 'Basic base64encode(username + ':' + password)
+ auth_val = base64.encodestring('%s:%s' % (auth_creds['user'], auth_creds['password']))
+ auth_val = "Basic %s" % auth_val
+ elif auth_type == 'token':
+ # Bearer Type, see eg. https://tools.ietf.org/html/rfc6750
+ auth_val = "%s %s" % (auth_creds['keyword'], auth_creds['token'])
+
+ request.add_header("Authorization", auth_val.replace('\n', ''))
+
def read_from_url(self, url, parameters=None):
"""
Read the data from the URL.
+
:param url: the url to fetch
:param parameters: optional dict of query parameters
:return:
@@ -87,6 +139,10 @@ class HttpInput(Input):
if parameters:
query_string = urllib.urlencode(parameters)
+ # Add optional Authorization
+ if self.auth:
+ self.add_authorization(req)
+
response = urlopen(req, query_string)
except HTTPError as e:
log.error('HTTPError fetching from URL %s: code=%d e=%s' % (url, e.code, e))
@@ -95,12 +151,13 @@ class HttpInput(Input):
log.error('URLError fetching from URL %s: reason=%s e=%s' % (url, e.reason, e))
raise e
- # everything is fine
+ # Everything is fine
return response.read()
def read(self, packet):
"""
Read the data from the URL.
+
:param packet:
:return:
"""
@@ -131,6 +188,13 @@ class ApacheDirInput(HttpInput):
date time is too fragile over different Apache servers).
"""
+ @Config(ptype=str, default='xml', required=False)
+ def file_ext(self):
+ """
+ The file extension for target files in Apache dir.
+ """
+ pass
+
def __init__(self, configdict, section, produces=FORMAT.record):
HttpInput.__init__(self, configdict, section, produces)
# look for a link + a timestamp + a size ('-' for dir)
@@ -138,8 +202,7 @@ class ApacheDirInput(HttpInput):
# This appeared to be too fragile, e.g. different date formats per apache server
# default file extension to filter
- self.file_ext = self.cfg.get('file_ext', 'xml')
- # default regular expresion for file
+ # default regular expression for file
self.file_reg_exp = self.cfg.get('file_reg_exp', 'href="([^"]*%s)"' % self.file_ext)
self.parse_re = re.compile(self.file_reg_exp)
self.file_list = None
@@ -161,6 +224,7 @@ class ApacheDirInput(HttpInput):
def next_file(self):
"""
Return a tuple (name, date, size) with next file info.
+
:return tuple:
"""
@@ -178,6 +242,7 @@ class ApacheDirInput(HttpInput):
def no_more_files(self):
"""
More files left?.
+
:return Boolean:
"""
return self.file_index == len(self.file_list) - 1
@@ -185,6 +250,7 @@ class ApacheDirInput(HttpInput):
def read(self, packet):
"""
Read the data from the URL.
+
:param packet:
:return:
"""
@@ -213,8 +279,8 @@ class ApacheDirInput(HttpInput):
def filter_file(self, file_name):
"""
Filter the file_name, e.g. to suppress reading, default: return file_name.
+
:param file_name:
:return string or None:
"""
return file_name
-
diff --git a/stetl/inputs/ogrinput.py b/stetl/inputs/ogrinput.py
index 69ac2d3..2cad7bb 100644
--- a/stetl/inputs/ogrinput.py
+++ b/stetl/inputs/ogrinput.py
@@ -36,10 +36,6 @@ class OgrInput(Input):
"""
String denoting the OGR datasource. Usually a path to a file like "path/rivers.shp" or connection string
to PostgreSQL like "PG: host=localhost dbname='rivers' user='postgres'".
-
- Required: True
-
- Default: None
"""
pass
@@ -50,10 +46,6 @@ class OgrInput(Input):
many standard formats that are self-describing like ESRI Shapefile.
Examples: 'PostgreSQL', 'GeoJSON' etc
-
- Required: False
-
- Default: None
"""
pass
@@ -61,12 +53,6 @@ class OgrInput(Input):
def source_options(self):
"""
Custom datasource-specific options. Used in gdal.SetConfigOption().
-
- Type: dictionary
-
- Required: False
-
- Default: None
"""
pass
@@ -74,10 +60,6 @@ class OgrInput(Input):
def sql(self):
"""
String with SQL query. Mandatory for PostgreSQL OGR source.
-
- Required: False (True for PostgreSQL OGR source)
-
- Default: None
"""
pass
@@ -207,6 +189,101 @@ class OgrPostgisInput(Input):
produces=FORMAT.xml_line_stream
"""
+ # Start attribute config meta
+ @Config(ptype=str, required=False, default='localhost')
+ def in_pg_host(self):
+ """
+ Host of input DB.
+ """
+ pass
+
+ @Config(ptype=str, required=False, default='5432')
+ def in_pg_port(self):
+ """
+ Port of input DB.
+ """
+ pass
+
+ @Config(ptype=str, required=True, default=None)
+ def in_pg_db(self):
+ """
+ Database name input DB.
+
+ """
+ pass
+
+ @Config(ptype=str, required=False, default=None)
+ def in_pg_schema(self):
+ """
+ DB Schema name input DB.
+ """
+ pass
+
+ @Config(ptype=str, required=False, default='postgres')
+ def in_pg_user(self):
+ """
+ User input DB.
+ """
+ pass
+
+ @Config(ptype=str, required=False, default='postgres')
+ def in_pg_password(self):
+ """
+ Password input DB.
+ """
+ pass
+
+ @Config(ptype=str, required=False, default=None)
+ def in_srs(self):
+ """
+ SRS (projection) (ogr2ogr -s_srs) input DB e.g. 'EPSG:28992'.
+ """
+ pass
+
+ @Config(ptype=str, required=False, default=None)
+ def in_pg_sql(self):
+ """
+ The input query (string) to fire.
+ """
+ pass
+
+ @Config(ptype=str, required=False, default=None)
+ def out_srs(self):
+ """
+ Target SRS (ogr2ogr -t_srs) code output stream.
+ """
+ pass
+
+ @Config(ptype=str, required=False, default='2')
+ def out_dimension(self):
+ """
+ Dimension (OGR: DIM=N) of features in output stream.
+ """
+ pass
+
+ @Config(ptype=str, required=False, default=None)
+ def out_gml_format(self):
+ """
+ GML format OGR name in output stream, e.g. 'GML3'.
+ """
+ pass
+
+ @Config(ptype=str, required=False, default=None)
+ def out_layer_name(self):
+ """
+ New Layer name (ogr2ogr -nln) output stream, e.g. 'address'.
+ """
+ pass
+
+ @Config(ptype=str, required=False, default=None)
+ def out_geotype(self):
+ """
+ OGR Geometry type new layer in output stream, e.g. POINT.
+ """
+ pass
+
+ # End attribute config meta
+
# TODO make this template configurable so we can have generic ogr2ogr input....
pg_conn_tmpl = "PG:host=%s dbname=%s active_schema=%s user=%s password=%s port=%s"
cmd_tmpl = 'ogr2ogr|-t_srs|%s|-s_srs|%s|-f|GML|%s|-dsco|FORMAT=%s|-lco|DIM=%s|%s|-SQL|%s|-nln|%s|%s'
@@ -219,33 +296,19 @@ class OgrPostgisInput(Input):
self.ogr_process = None
self.eof_stdout = False
self.eof_stderr = False
-
- in_pg_host = self.cfg.get('in_pg_host', 'localhost')
- in_pg_db = self.cfg.get('in_pg_db')
- in_pg_schema = self.cfg.get('in_pg_schema', 'public')
- in_pg_user = self.cfg.get('in_pg_user', 'postgres')
- in_pg_password = self.cfg.get('in_pg_password', 'postgres')
- in_pg_port = self.cfg.get('in_pg_port', '5432')
- in_srs = self.cfg.get('in_srs')
- in_pg_sql = self.cfg.get('in_pg_sql')
-
- out_srs = self.cfg.get('out_srs')
- out_file = '/vsistdout/'
- out_gml_format = self.cfg.get('out_gml_format')
- out_dimension = self.cfg.get('out_dimension', '2')
- out_layer_name = self.cfg.get('out_layer_name')
- out_geotype = self.cfg.get('out_geotype', '')
+ self.out_file = '/vsistdout/'
#
# Build ogr2ogr command line
#
# PostGIS PG: options
self.pg = OgrPostgisInput.pg_conn_tmpl % (
- in_pg_host, in_pg_db, in_pg_schema, in_pg_user, in_pg_password, in_pg_port)
+ self.in_pg_host, self.in_pg_db, self.in_pg_schema, self.in_pg_user, self.in_pg_password, self.in_pg_port)
# Entire ogr2ogr command line
self.cmd = OgrPostgisInput.cmd_tmpl % (
- out_srs, in_srs, out_file, out_gml_format, out_dimension, self.pg, in_pg_sql, out_layer_name, out_geotype)
+ self.out_srs, self.in_srs, self.out_file, self.out_gml_format, self.out_dimension, self.pg, self.in_pg_sql,
+ self.out_layer_name, self.out_geotype)
# Make array to make it easy for Popen with quotes etc
self.cmd = self.cmd.split('|')
@@ -307,6 +370,3 @@ class OgrPostgisInput(Input):
log.info('EOF ogr2ogr output')
return packet
-
-
-
diff --git a/stetl/main.py b/stetl/main.py
index 84a15d2..02eb552 100755
--- a/stetl/main.py
+++ b/stetl/main.py
@@ -9,11 +9,13 @@ from etl import ETL
from factory import factory
from util import Util
from version import __version__
-import argparse #apt-get install python-argparse
+import argparse # apt-get install python-argparse
import inspect
import os
+
log = Util.get_log('main')
+
def parse_args():
log.info("Stetl version = %s" % __version__)
@@ -29,8 +31,8 @@ def parse_args():
dest='config_args', required=False)
argparser.add_argument('-d ', '--doc', type=str,
- help='Get component documentation like its configuration parameters, e.g. stetl doc stetl.inputs.fileinput.FileInput',
- dest='doc_args', required=False)
+ help='Get component documentation like its configuration parameters, e.g. stetl doc stetl.inputs.fileinput.FileInput',
+ dest='doc_args', required=False)
args = argparser.parse_args()
@@ -44,9 +46,10 @@ def parse_args():
return args
+
# DEPRECATED, now using @Config which also documents with Sphinx
def print_config_attrs(clazz):
- skip =['Filter', 'Input', 'Output', 'Component']
+ skip = ['Filter', 'Input', 'Output', 'Component']
for base in clazz.__bases__:
if base.__name__ not in skip:
@@ -54,7 +57,6 @@ def print_config_attrs(clazz):
"""Print documentation for Attr object"""
-
module_name, _, class_name = clazz.__name__.rpartition('.')
# print 'From class: %s' % class_name
@@ -73,13 +75,14 @@ def print_config_attrs(clazz):
if attr_count == 0:
print ('No config attributes or class not yet documented')
+
def print_classes(package):
# is_module = inspect.ismodule(class_name)
import inputs
import pkgutil
- package=inputs
+ package = inputs
for importer, modname, ispkg in pkgutil.walk_packages(path=package.__path__,
- prefix=package.__name__+'.',
+ prefix=package.__name__ + '.',
onerror=lambda x: None):
print('stetl.' + modname)
for name, data in inspect.getmembers(modname, inspect.isclass):
@@ -107,6 +110,7 @@ def print_doc(class_name):
log.error("cannot print info class named '%s' e=%s - you made a typo?" % (class_name, str(e)))
raise e
+
def main():
"""The `main` function, to be called from commandline, like `python src/main.py -c etl.cfg`.
@@ -130,5 +134,6 @@ def main():
else:
print('Unknown option, try stetl -h for help')
+
if __name__ == "__main__":
main()
diff --git a/stetl/merger.py b/stetl/merger.py
new file mode 100644
index 0000000..8699350
--- /dev/null
+++ b/stetl/merger.py
@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+#
+# Merger Component base class for ETL.
+#
+# Author: Just van den Broecke
+#
+
+import random
+from util import Util
+from component import Component
+
+log = Util.get_log('merger')
+
+
+class Merger(Component):
+ """
+ Component that merges multiple Input Components into a single Component.
+ Use this for example to combine multiple input streams like API endpoints.
+ The Merger will embed Child Components to which actions are delegated.
+ A Child Component may be a sub-Chain e.g. (Input|Filter|Filter..) sequence.
+ Hence the "next" should be coupled to the last Component in that sub-Chain with
+ the degenerate case where the sub-Chain is a single (Input) Component.
+ NB this Component can only be used for Inputs.
+ """
+
+ def __init__(self, config_dict, child_list):
+ # Assemble child list
+ self.children = []
+ section_name = ''
+ for child in child_list:
+ section_name += '-%s_%d' % (child.get_id(), random.randrange(0, 100000))
+
+ # A Child can be a sub-Chain: each child is tuple: [0] is first
+ # [1] is last in sub-Chain. child[0] === child[1] if child is single Component.
+ # Need to remember both first and last in order to link/unlink subchain.
+ # So we store the Child as a tuple of (first, last).
+ self.children.append((child, child.get_last()))
+
+ # Add ourselves to config for compat with Component
+ config_dict.add_section(section_name)
+
+ # We use the in/out formats of first child, will be compat-checked later
+ Component.__init__(self, config_dict, section_name, consumes=self.first(self.children[0])._input_format,
+ produces=self.last(self.children[0])._output_format)
+
+ self.end_count = len(self.children)
+
+ def add_next(self, next_component):
+ for child in self.children:
+ # Couple Child Component's last .next directly to our next
+ self.last(child).add_next(next_component)
+
+ # Remember next
+ self.next = next_component
+
+ def first(self, child):
+ """
+ Get first Component in Child sub-Chain.
+ :param child:
+ :return: first Component
+ """
+ return child[0]
+
+ def last(self, child):
+ """
+ Get last Component in Child sub-Chain.
+ :param child:
+ :return: last Component
+ """
+ return child[1]
+
+ # Check compatibility with our child Components
+ def is_compatible(self):
+ for child in self.children:
+ # Last in subchain must be compatible
+ if not self.last(child).is_compatible():
+ return False
+ return True
+
+ def process(self, packet):
+ # Defer processing to children
+ # and track of End-of-Stream Packets
+
+ for child in self.children:
+ # Skip inactive child Components
+ if not self.last(child).next:
+ continue
+
+ # Defer to child
+ self.first(child).process(packet)
+
+ # Keep track of End-of-Stream
+ if packet.is_end_of_stream():
+ # deactivate Child by unlinking
+ # otherwise we'll keep getting EoS
+ self.last(child).next = None
+ self.end_count -= 1
+
+ # Re-init to start afresh again
+ packet.init()
+
+ # Only if all children have End-of-Stream
+ # declare the Packet returned EoS.
+ if self.end_count == 0:
+ packet.set_end_of_stream()
+
+ return packet
+
+ def do_init(self):
+ for child in self.children:
+ # Only init the child, without
+ # initing upstream Components via Chain
+ self.last(child).next = None
+ self.first(child).do_init()
+ self.last(child).next = self.next
+
+ # init upstream Components once
+ self.next.do_init()
+
+ def do_exit(self):
+ for child in self.children:
+ # Only exit the child, without
+ # exiting upstream Components via Chain
+ self.last(child).next = None
+ self.first(child).do_exit()
+
+ # exit upstream Components once
+ self.next.do_exit()
diff --git a/stetl/output.py b/stetl/output.py
index cdb8516..4eacb32 100755
--- a/stetl/output.py
+++ b/stetl/output.py
@@ -5,15 +5,17 @@
# Author: Just van den Broecke
#
from component import Component
-from util import Util, etree
+from util import Util
log = Util.get_log('output')
+
class Output(Component):
"""
Abstract Base class for all Output Components.
"""
+
def __init__(self, configdict, section, consumes):
Component.__init__(self, configdict, section, consumes=consumes, produces=None)
log.info("cfg = %s" % self.cfg.to_string())
diff --git a/stetl/outputs/dboutput.py b/stetl/outputs/dboutput.py
index d7bc894..0b98a99 100644
--- a/stetl/outputs/dboutput.py
+++ b/stetl/outputs/dboutput.py
@@ -24,6 +24,7 @@ class DbOutput(Output):
def write(self, packet):
return packet
+
class PostgresDbOutput(DbOutput):
"""
Output to PostgreSQL database.
@@ -32,6 +33,7 @@ class PostgresDbOutput(DbOutput):
consumes=FORMAT.string
"""
+
# Start attribute config meta
@Config(ptype=str, required=True, default=None)
def database(self):
@@ -69,6 +71,7 @@ class PostgresDbOutput(DbOutput):
pass
# End attribute config meta
+
def __init__(self, configdict, section):
DbOutput.__init__(self, configdict, section, consumes=FORMAT.string)
@@ -82,6 +85,7 @@ class PostgresDbOutput(DbOutput):
log.info('executed SQL, rowcount=%d' % rowcount)
return packet
+
class PostgresInsertOutput(PostgresDbOutput):
"""
Output by inserting single record into Postgres database.
@@ -95,6 +99,7 @@ class PostgresInsertOutput(PostgresDbOutput):
consumes=FORMAT.record
"""
+
# Start attribute config meta
@Config(ptype=str, required=False, default='public')
def table(self):
@@ -116,6 +121,7 @@ class PostgresInsertOutput(PostgresDbOutput):
The key column name of the table, required when replacing records.
"""
pass
+
# End attribute config meta
def __init__(self, configdict, section, consumes=FORMAT.record):
@@ -138,7 +144,8 @@ class PostgresInsertOutput(PostgresDbOutput):
# We assume that all records do the same INSERT key/values
# See http://grokbase.com/t/postgresql/psycopg/12735bvkmv/insert-into-with-a-dictionary-or-generally-with-a-variable-number-of-columns
# e.g. INSERT INTO lml_files ("file_name", "file_data") VALUES (%s,%s)
- query = "INSERT INTO %s (%s) VALUES (%s)" % (self.cfg.get('table'), ",".join(['%s' % k for k in record]), ",".join(["%s",]*len(record.keys())))
+ query = "INSERT INTO %s (%s) VALUES (%s)" % (
+ self.cfg.get('table'), ",".join(['%s' % k for k in record]), ",".join(["%s", ] * len(record.keys())))
log.info('query is %s', query)
return query
diff --git a/stetl/outputs/deegreeoutput.py b/stetl/outputs/deegreeoutput.py
index 627e6d7..0455646 100644
--- a/stetl/outputs/deegreeoutput.py
+++ b/stetl/outputs/deegreeoutput.py
@@ -15,12 +15,14 @@ import os
log = Util.get_log('deegreeoutput')
+
class DeegreeBlobstoreOutput(Output):
"""
Insert features into deegree Blobstore from an etree doc.
consumes=FORMAT.etree_doc
"""
+
def __init__(self, configdict, section):
Output.__init__(self, configdict, section, consumes=FORMAT.etree_doc)
self.overwrite = self.cfg.get_bool('overwrite')
@@ -75,27 +77,27 @@ class DeegreeBlobstoreOutput(Output):
gml_ns = None
for childNode in featureMembers:
if gml_ns is None:
- if childNode.nsmap.has_key('gml'):
+ if 'gml' in childNode.nsmap:
gml_ns = childNode.nsmap['gml']
- else:
- if childNode.nsmap.has_key('GML'):
- gml_ns = childNode.nsmap['GML']
+ elif 'GML' in childNode.nsmap:
+ gml_ns = childNode.nsmap['GML']
gml_id = childNode.get('{%s}id' % gml_ns)
feature_type_id = self.feature_type_ids[childNode.tag]
# Find a GML geometry in the GML NS
- ogrGeomWKT = None
+ # ogrGeomWKT = None
# gmlMembers = childNode.xpath(".//gml:Point|.//gml:Curve|.//gml:Surface|.//gml:MultiSurface", namespaces=NS)
gmlMembers = childNode.xpath(
- ".//*[local-name() = 'Point']|.//*[local-name() = 'Polygon']|.//*[local-name() = 'Curve']|.//*[local-name() = 'Surface']|.//*[local-name() = 'MultiSurface']")
+ ".//*[local-name() = 'Point']|.//*[local-name() = 'Polygon']|.//*[local-name() =\
+ 'Curve']|.//*[local-name() = 'Surface']|.//*[local-name() = 'MultiSurface']")
geom_str = None
for gmlMember in gmlMembers:
if geom_str is None:
geom_str = etree.tostring(gmlMember)
- # no need for GDAL Python bindings for now, maybe when we'll optimize with COPY iso INSERT
- # ogrGeom = ogr.CreateGeometryFromGML(str(gmlStr))
+ # no need for GDAL Python bindings for now, maybe when we'll optimize with COPY iso INSERT
+ # ogrGeom = ogr.CreateGeometryFromGML(str(gmlStr))
# if ogrGeom is not None:
# ogrGeomWKT = ogrGeom.ExportToWkt()
# if ogrGeomWKT is not None:
@@ -132,6 +134,7 @@ class DeegreeBlobstoreOutput(Output):
log.info("inserted %s features" % count)
return packet
+
#
class DeegreeFSLoaderOutput(Output):
"""
@@ -152,7 +155,6 @@ class DeegreeFSLoaderOutput(Output):
if packet.data is None:
return packet
- gml_doc = packet.data
d3tools_path = self.cfg.get('d3tools_path')
workspace_path = self.cfg.get('workspace_path')
feature_store = self.cfg.get('feature_store')
@@ -170,8 +172,6 @@ class DeegreeFSLoaderOutput(Output):
p.stdin.write(packet.to_string())
- result = p.communicate()[0]
- return packet
-
+ # result = p.communicate()[0]
# print(result)
-
+ return packet
diff --git a/stetl/outputs/execoutput.py b/stetl/outputs/execoutput.py
index 1fe7b55..b0286eb 100644
--- a/stetl/outputs/execoutput.py
+++ b/stetl/outputs/execoutput.py
@@ -35,7 +35,24 @@ class ExecOutput(Output):
subprocess.call(cmd, shell=use_shell)
log.info("execute done")
-
+
+class CommandExecOutput(ExecOutput):
+ """
+ Executes an arbitrary command.
+
+ consumes=FORMAT.string
+ """
+
+ def __init__(self, configdict, section):
+ ExecOutput.__init__(self, configdict, section, consumes=FORMAT.string)
+
+ def write(self, packet):
+ if packet.data is not None:
+ self.execute_cmd(packet.data)
+
+ return packet
+
+
class Ogr2OgrExecOutput(ExecOutput):
"""
Executes an Ogr2Ogr command.
@@ -54,10 +71,6 @@ class Ogr2OgrExecOutput(ExecOutput):
"""
String denoting the OGR data destination. Usually a path to a file like "path/rivers.shp" or connection string
to PostgreSQL like "PG: host=localhost dbname='rivers' user='postgres'".
-
- Required: True
-
- Default: None
"""
pass
@@ -68,37 +81,21 @@ class Ogr2OgrExecOutput(ExecOutput):
many standard formats that are self-describing like ESRI Shapefile.
Examples: 'PostgreSQL', 'GeoJSON' etc
-
- Required: False
-
- Default: None
"""
pass
- @Config(ptype=list, default=[], required=False)
+ @Config(ptype=str, default=None, required=False)
def lco(self):
"""
Options for newly created layer (-lco).
-
- Type: list
-
- Required: False
-
- Default: []
"""
pass
- @Config(ptype=list, default=None, required=False)
+ @Config(ptype=str, default=None, required=False)
def spatial_extent(self):
"""
Spatial extent (-spat), to pass as xmin ymin xmax ymax
-
- Type: list
-
- Required: False
-
- Default: []
"""
pass
@@ -107,21 +104,13 @@ class Ogr2OgrExecOutput(ExecOutput):
"""
Name of GFS template file to use during loading. Passed to ogr2ogr as
--config GML_GFS_TEMPLATE <name>
-
- Required: False
-
- Default: None
"""
pass
- @Config(ptype=list, default=None, required=False)
+ @Config(ptype=str, default=None, required=False)
def options(self):
"""
Miscellaneous options to pass to ogr2ogr.
-
- Required: False
-
- Default: None
"""
pass
@@ -129,12 +118,6 @@ class Ogr2OgrExecOutput(ExecOutput):
def cleanup_input(self):
"""
Flag to indicate whether the input file to ogr2ogr should be cleaned up.
-
- Type: boolean
-
- Required: False
-
- Default: False
"""
pass
@@ -144,31 +127,13 @@ class Ogr2OgrExecOutput(ExecOutput):
def __init__(self, configdict, section):
ExecOutput.__init__(self, configdict, section, consumes=FORMAT.string)
- # For creating tables the GFS file needs to be newer than
- # the .gml file. -lco GML_GFS_TEMPLATE somehow does not work
- # so we copy the .gfs file each time with the .gml file with
- # the same base name
- self.lco = self.cfg.get('lco')
- self.gfs_template = self.cfg.get('gfs_template')
- spatial_extent = self.cfg.get('spatial_extent')
- options = self.cfg.get('options')
-
- dest_format = self.cfg.get('dest_format')
- dest_data_source = self.cfg.get('dest_data_source')
-
- if not dest_format:
- raise ValueError('Verplichte parameter dest_format ontbreekt')
- if not dest_data_source:
- raise ValueError('Verplichte parameter dest_data_source ontbreekt')
-
- self.ogr2ogr_cmd = 'ogr2ogr -f ' + dest_format + ' ' + dest_data_source
-
- if spatial_extent:
- self.ogr2ogr_cmd += ' -spat ' + spatial_extent
- if options:
- self.ogr2ogr_cmd += ' ' + options
- self.cleanup_input = self.cfg.get('cleanup_input')
-
+ self.ogr2ogr_cmd = 'ogr2ogr -f ' + self.dest_format + ' ' + self.dest_data_source
+
+ if self.spatial_extent:
+ self.ogr2ogr_cmd += ' -spat ' + self.spatial_extent
+ if self.options:
+ self.ogr2ogr_cmd += ' ' + self.options
+
self.first_run = True
def write(self, packet):
@@ -180,8 +145,6 @@ class Ogr2OgrExecOutput(ExecOutput):
if self.lco and self.first_run is True:
ogr2ogr_cmd += ' ' + self.lco
self.first_run = False
-
- import os.path
if type(packet.data) is list:
for item in packet.data:
@@ -190,20 +153,25 @@ class Ogr2OgrExecOutput(ExecOutput):
self.execute(ogr2ogr_cmd, packet.data)
return packet
-
+
def execute(self, ogr2ogr_cmd, file_path):
+
+ # For creating tables the GFS file needs to be newer than
+ # the .gml file. -lco GML_GFS_TEMPLATE somehow does not work
+ # so we copy the .gfs file each time with the .gml file with
+ # the same base name
# Copy the .gfs file if required, use the same base name
# so ogr2ogr will pick it up.
+ gfs_path = None
if self.gfs_template:
file_ext = os.path.splitext(file_path)
gfs_path = file_ext[0] + '.gfs'
shutil.copy(self.gfs_template, gfs_path)
-
+
# Append file name to command as last argument
self.execute_cmd(ogr2ogr_cmd + ' ' + file_path)
-
+
if self.cleanup_input:
os.remove(file_path)
- if self.gfs_template:
+ if gfs_path:
os.remove(gfs_path)
-
diff --git a/stetl/outputs/fileoutput.py b/stetl/outputs/fileoutput.py
index d33639c..a269d24 100644
--- a/stetl/outputs/fileoutput.py
+++ b/stetl/outputs/fileoutput.py
@@ -7,6 +7,7 @@
from stetl.output import Output
from stetl.util import Util
from stetl.packet import FORMAT
+from stetl.component import Config
import os
@@ -20,6 +21,17 @@ class FileOutput(Output):
consumes=FORMAT.any
"""
+ # Start attribute config meta
+
+ @Config(ptype=str, default=None, required=True)
+ def file_path(self):
+ """
+ Path to file, for MultiFileOutput can be of the form like: gmlcities-%03d.gml
+ """
+ pass
+
+ # End attribute config meta
+
def __init__(self, configdict, section):
Output.__init__(self, configdict, section, consumes=FORMAT.any)
log.info("working dir %s" % os.getcwd())
@@ -28,8 +40,7 @@ class FileOutput(Output):
if packet.data is None:
return packet
- file_path = self.cfg.get('file_path')
- return self.write_file(packet, file_path)
+ return self.write_file(packet, self.file_path)
def write_file(self, packet, file_path):
log.info('writing to file %s' % file_path)
@@ -44,10 +55,11 @@ class FileOutput(Output):
class MultiFileOutput(FileOutput):
"""
- Print to multiple files from subsequent packets like strings or etree docs.
+ Print to multiple files from subsequent packets like strings or etree docs, file_path must be of a form like: gmlcities-%03d.gml.
consumes=FORMAT.any
"""
+
def __init__(self, configdict, section):
Output.__init__(self, configdict, section, consumes=FORMAT.any)
self.file_num = 1
@@ -56,8 +68,7 @@ class MultiFileOutput(FileOutput):
if packet.data is None:
return packet
- # file_path can be of the form: gmlcities-%03d.gml
- file_path = self.cfg.get('file_path')
- file_path %= self.file_num
+ # file_path must be of the form: gmlcities-%03d.gml
+ file_path = self.file_path % self.file_num
self.file_num += 1
return self.write_file(packet, file_path)
diff --git a/stetl/outputs/httpoutput.py b/stetl/outputs/httpoutput.py
index cc568ec..ecebb14 100644
--- a/stetl/outputs/httpoutput.py
+++ b/stetl/outputs/httpoutput.py
@@ -1,46 +1,101 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
-# Output classes for ETL.
+# Base classes for HTTP output like WFS-T and SOS-T or any other HTTP writing service.
#
# Author: Just van den Broecke
#
from stetl.output import Output
from stetl.util import Util
from stetl.packet import FORMAT
+from stetl.component import Config
import httplib
import base64
log = Util.get_log('httpoutput')
+
class HttpOutput(Output):
"""
Output via HTTP protocol, usually via POST.
- consumes=FORMAT.etree_doc
+ consumes=FORMAT.any
"""
+
+ @Config(ptype=str, default=None, required=True)
+ def host(self):
+ """
+ The hostname/IP addr for target request.
+ """
+ pass
+
+ @Config(ptype=int, default=80, required=False)
+ def port(self):
+ """
+ The port number for target request.
+ """
+ pass
+
+ @Config(ptype=str, default='/', required=False)
+ def path(self):
+ """
+ The path number for target request.
+ """
+ pass
+
+ @Config(ptype=str, default='POST', required=False)
+ def method(self):
+ """
+ The HTTP method for target request.
+ """
+ pass
+
+ @Config(ptype=str, default='text/xml', required=False)
+ def content_type(self):
+ """
+ The HTTP ContentType request header for target request.
+ """
+ pass
+
+ @Config(ptype=str, default=None, required=False)
+ def user(self):
+ """
+ The Username for HTTP basic auth for target request.
+ """
+ pass
+
+ @Config(ptype=str, default=None, required=False)
+ def password(self):
+ """
+ The Password for HTTP basic auth for target request.
+ """
+ pass
+
+ @Config(ptype=bool, default=True, required=False)
+ def list_fanout(self):
+ """
+ If we consume a list(), should we create a HTTP req for each member?
+ """
+ pass
+
def __init__(self, configdict, section, consumes=FORMAT.any):
Output.__init__(self, configdict, section, consumes)
- self.host = self.cfg.get('host')
- self.port = self.cfg.get('port', '80')
- self.path = self.cfg.get('path')
- self.method = self.cfg.get('method', 'POST')
- self.user = self.cfg.get('user', None)
- self.password = self.cfg.get('password', None)
- self.content_type = self.cfg.get('content_type', 'text/xml')
# self.accept_type = self.cfg.get('accept_type', self.content_type)
-
- # If we receive a list(), should we create a HTTP req for each member?
- self.list_fanout = self.cfg.get_bool('list_fanout', True)
self.req_nr = 0
def create_payload(self, packet):
+ """
+ Create a HTTP body payload like for POST of an XML or JSON message.
+ Subclasses like WFS and SOS override.
+ :param packet:
+ :return payload as string:
+ """
return packet.data
def post(self, packet, payload):
self.req_nr += 1
- webservice = httplib.HTTP(self.host)
+ webservice = httplib.HTTP(self.host, self.port)
# write your headers
webservice.putrequest(self.method, self.path)
webservice.putheader("Host", self.host)
@@ -62,31 +117,29 @@ class HttpOutput(Output):
# get the response
statuscode, statusmessage, header = webservice.getreply()
log.info("Req nr %d - response status: code=%d msg=%s" % (self.req_nr, statuscode, statusmessage))
- if statuscode != 200:
+ if statuscode == 200:
+ res = webservice.getfile().read()
+ elif statuscode == 204:
+ res = ''
+ else:
log.error("Headers: %s" % str(header))
res = webservice.getfile().read()
log.info('Content: %s' % res)
- # conn = httplib.HTTPConnection(self.host, self.port)
- # conn.request(self.method, self.path, payload, headers)
-
- # response = conn.getresponse()
- # log.info('status=%s msg=%s' % (response.status, response.msg))
- # log.info('response=%s' % response.read(1024))
- # conn.close()
- return packet
+ return statuscode, statusmessage, res
def write(self, packet):
if packet.data is None:
return packet
if type(packet.data) is list and self.list_fanout is True:
- # Multiple records in list, save original
- original_data = packet.data
- for data_elm in original_data:
- packet.data = data_elm
- self.post(packet, self.create_payload(packet))
- packet.data = original_data
+ # Multiple records in list, save original
+ original_data = packet.data
+ for data_elm in original_data:
+ packet.data = data_elm
+ self.post(packet, self.create_payload(packet))
+
+ packet.data = original_data
else:
# Regular, single data element or list_fanout is False
diff --git a/stetl/outputs/ogroutput.py b/stetl/outputs/ogroutput.py
index 556a34e..7b65533 100644
--- a/stetl/outputs/ogroutput.py
+++ b/stetl/outputs/ogroutput.py
@@ -34,12 +34,6 @@ class OgrOutput(Output):
def append(self):
"""
Add to destination destination if it extists (ogr2ogr -append option).
-
- Type: boolean
-
- Required: False
-
- Default: False
"""
pass
@@ -49,10 +43,6 @@ class OgrOutput(Output):
"""
String denoting the OGR data destination. Usually a path to a file like "path/rivers.shp" or connection string
to PostgreSQL like "PG: host=localhost dbname='rivers' user='postgres'".
-
- Required: True
-
- Default: None
"""
pass
@@ -63,10 +53,6 @@ class OgrOutput(Output):
many standard formats that are self-describing like ESRI Shapefile.
Examples: 'PostgreSQL', 'GeoJSON' etc
-
- Required: False
-
- Default: None
"""
pass
@@ -76,10 +62,6 @@ class OgrOutput(Output):
Creation options.
Examples: ..
-
- Required: False
-
- Default: []
"""
pass
@@ -87,26 +69,13 @@ class OgrOutput(Output):
def dest_options(self):
"""
Custom data destination-specific options. Used in gdal.SetConfigOption().
-
- Type: dictionary
-
- Required: False
-
- Default: None
"""
pass
-
@Config(ptype=list, default=[], required=False)
def layer_create_options(self):
"""
Options for newly created layer (-lco).
-
- Type: list
-
- Required: True
-
- Default: []
"""
pass
@@ -116,10 +85,6 @@ class OgrOutput(Output):
"""
Layer name for layer created in the destination source.
- Type: string
-
- Required: True
-
"""
pass
@@ -128,12 +93,6 @@ class OgrOutput(Output):
def overwrite(self):
"""
Overwrite destination if it extists (ogr2ogr -overwrite option).
-
- Type: boolean
-
- Required: False
-
- Default: False
"""
pass
@@ -142,12 +101,6 @@ class OgrOutput(Output):
def target_srs(self):
"""
SRS (projection) for the target.
-
- Type: string
-
- Required: False
-
- Default: None (take from Input)
"""
pass
@@ -155,10 +108,6 @@ class OgrOutput(Output):
def sql(self):
"""
String with SQL query. Mandatory for PostgreSQL OGR dest.
-
- Required: False (True for PostgreSQL OGR dest)
-
- Default: None
"""
pass
@@ -168,7 +117,6 @@ class OgrOutput(Output):
def __init__(self, configdict, section):
Output.__init__(self, configdict, section, consumes=[FORMAT.ogr_feature, FORMAT.ogr_feature_array])
-
def init(self):
self.ogr = ogr
@@ -208,9 +156,9 @@ class OgrOutput(Output):
# Loosely based on https://github.com/OSGeo/gdal/blob/trunk/gdal/swig/python/samples/ogr2ogr.py
- #/* -------------------------------------------------------------------- */
- #/* Try opening the output data source as an existing, writable */
- #/* -------------------------------------------------------------------- */
+ # /* -------------------------------------------------------------------- */
+ # /* Try opening the output data source as an existing, writable */
+ # /* -------------------------------------------------------------------- */
if self.update:
# Try opening in update mode
self.dest_fd = ogr.Open(self.dest_data_source, True)
@@ -226,9 +174,9 @@ class OgrOutput(Output):
self.dest_driver = None
self.update = False
- #/* -------------------------------------------------------------------- */
- #/* Find the output driver. */
- #/* -------------------------------------------------------------------- */
+ # /* -------------------------------------------------------------------- */
+ # /* Find the output driver. */
+ # /* -------------------------------------------------------------------- */
if self.dest_driver is None:
# Open OGR data dest in write-only mode.
@@ -247,18 +195,18 @@ class OgrOutput(Output):
log.error("%s driver does not support data source creation." % self.dest_format)
raise Exception()
- #/* -------------------------------------------------------------------- */
- #/* Create the output data source. */
- #/* -------------------------------------------------------------------- */
+ # /* -------------------------------------------------------------------- */
+ # /* Create the output data source. */
+ # /* -------------------------------------------------------------------- */
if self.dest_fd is None:
self.dest_fd = self.dest_driver.CreateDataSource(self.dest_data_source, options=self.dest_create_options)
if self.dest_fd is None:
log.error("%s driver failed to create %s" % (self.dest_format, self.dest_data_source))
raise Exception()
- #/* -------------------------------------------------------------------- */
- #/* Parse the output SRS definition if possible. */
- #/* -------------------------------------------------------------------- */
+ # /* -------------------------------------------------------------------- */
+ # /* Parse the output SRS definition if possible. */
+ # /* -------------------------------------------------------------------- */
output_srs_ref = None
if self.target_srs is not None:
output_srs_ref = osr.SpatialReference()
@@ -391,4 +339,3 @@ class Ogr2OgrOutput(Output):
self.execute_cmd(ogr2ogr_cmd)
return packet
-
diff --git a/stetl/outputs/standardoutput.py b/stetl/outputs/standardoutput.py
index 18f13ac..9d992bf 100644
--- a/stetl/outputs/standardoutput.py
+++ b/stetl/outputs/standardoutput.py
@@ -10,6 +10,7 @@ from stetl.packet import FORMAT
log = Util.get_log('standardoutput')
+
#
class StandardOutput(Output):
"""
@@ -17,6 +18,7 @@ class StandardOutput(Output):
consumes=FORMAT.any
"""
+
def __init__(self, configdict, section):
Output.__init__(self, configdict, section, consumes=FORMAT.any)
@@ -28,12 +30,14 @@ class StandardOutput(Output):
print(packet.to_string())
return packet
+
class StandardXmlOutput(Output):
"""
Pretty print XML from etree doc to standard output. OBSOLETE, can be done with StandardOutput
consumes=FORMAT.etree_doc
"""
+
def __init__(self, configdict, section):
Output.__init__(self, configdict, section, consumes=FORMAT.etree_doc)
@@ -44,4 +48,3 @@ class StandardXmlOutput(Output):
# Default: print to stdout
print(packet.to_string())
return packet
-
diff --git a/stetl/outputs/wfsoutput.py b/stetl/outputs/wfsoutput.py
index e0bd01b..0557421 100644
--- a/stetl/outputs/wfsoutput.py
+++ b/stetl/outputs/wfsoutput.py
@@ -5,6 +5,7 @@
#
# Author: Just van den Broecke
#
+from stetl.component import Config
from stetl.output import Output
from stetl.util import Util
from stetl.packet import FORMAT
@@ -12,6 +13,7 @@ import httplib
log = Util.get_log('wfsoutput')
+
class WFSTOutput(Output):
"""
Insert features via WFS-T (WFS Transaction) OGC protocol from an etree doc.
@@ -19,6 +21,37 @@ class WFSTOutput(Output):
consumes=FORMAT.etree_doc
"""
+ # Start attribute config meta
+ @Config(ptype=str, required=True, default=None)
+ def wfs_host(self):
+ """
+ Hostname-part of URL e.g. geodata.ngr.nl.
+ """
+ pass
+
+ @Config(ptype=str, required=False, default='80')
+ def wfs_port(self):
+ """
+ Port-part of URL.
+ """
+ pass
+
+ @Config(ptype=str, required=True, default=None)
+ def wfs_path(self):
+ """
+ Path-part of URL e.g. '/bag/wfs'.
+ """
+ pass
+
+ @Config(ptype=str, required=False, default='GenerateNew')
+ def idgen(self):
+ """
+ Mode that WFS server generates new Id's for incoming Features.
+ """
+ pass
+
+ # End attribute config meta
+
wfst_req = '''<?xml version="1.0" encoding="UTF-8"?>
<wfs:Transaction version="1.1.0" service="WFS"
xmlns:wfs="http://www.opengis.net/wfs"
@@ -34,10 +67,6 @@ class WFSTOutput(Output):
def __init__(self, configdict, section):
Output.__init__(self, configdict, section, consumes=FORMAT.etree_doc)
- self.wfs_host = self.cfg.get('host')
- self.wfs_port = self.cfg.get('port', '80')
- self.wfs_path = self.cfg.get('path')
- self.idgen = self.cfg.get('idgen', 'GenerateNew')
def write(self, packet):
if packet.data is None:
diff --git a/stetl/packet.py b/stetl/packet.py
index 280771c..e88ed04 100644
--- a/stetl/packet.py
+++ b/stetl/packet.py
@@ -7,6 +7,7 @@
from util import etree
import json
+
class Packet:
"""
Represents units of (any) data and status passed along Chain of Components.
@@ -56,6 +57,7 @@ class Packet:
s = str(self.data)
return s
+
# Simple enum emulation NOT ANY MORE: TOO INVOLVED AND INFLEXIBLE: use Strings with predefined ones in FORMAT.*
# See http://stackoverflow.com/questions/1969005/enumerations-in-python
# class Enum(object):
@@ -68,6 +70,29 @@ class Packet:
class FORMAT:
+ """
+ Format of Packet (enumeration).
+
+ Current possible values:
+
+ * 'none'
+ * 'xml_line_stream'
+ * 'line_stream'
+ * 'etree_doc'
+ * 'etree_element'
+ * 'etree_feature_array'
+ * 'xml_doc_as_string'
+ * 'string'
+ * 'record'
+ * 'record_array'
+ * 'struct'
+ * 'geojson_feature'
+ * 'geojson_collection'
+ * 'ogr_feature'
+ * 'ogr_feature_array'
+ * 'any'
+
+ """
none = 'none'
xml_line_stream = 'xml_line_stream'
line_stream = 'line_stream'
diff --git a/stetl/postgis.py b/stetl/postgis.py
index b250396..44a19b3 100755
--- a/stetl/postgis.py
+++ b/stetl/postgis.py
@@ -18,8 +18,9 @@ except ImportError:
class PostGIS:
def __init__(self, config):
- # Lees de configuratie
+ # Lees de configuratie
self.config = config
+ self.e = None
def initialiseer(self, bestand):
log.info('Connecting...')
@@ -31,7 +32,7 @@ class PostGIS:
self.cursor.execute(script)
self.connection.commit()
log.info('script executed')
- except psycopg2.DatabaseError, e:
+ except psycopg2.DatabaseError as e:
log.warn("error '%s' from script '%s'" % (str(e), str(bestand)))
def connect(self, initdb=False):
@@ -47,14 +48,14 @@ class PostGIS:
self.set_schema()
log.debug("Connected to database %s" % (self.config['database']))
- except Exception, e:
- log.error("Cannot connect to database '%s'" % (self.config['database']))
+ except Exception as e:
+ log.error("Cannot connect to database '%s' e=%s" % (self.config['database'], str(e)))
def disconnect(self):
self.e = None
try:
self.connection.close()
- except (Exception), e:
+ except Exception as e:
self.e = e
log.error("error %s in close" % (str(e)))
@@ -66,13 +67,12 @@ class PostGIS:
self.connection.commit()
if close is True:
self.connection.close()
- except (Exception), e:
+ except Exception as e:
self.e = e
log.error("error %s in commit" % (str(e)))
return self.e
-
def create_schema(self):
# Public schema: no further action required
if self.config['schema'] != 'public':
@@ -81,7 +81,6 @@ class PostGIS:
self.execute('''CREATE SCHEMA %s;''' % self.config['schema'])
self.connection.commit()
-
def set_schema(self):
# Non-public schema set search path
if self.config['schema'] != 'public':
@@ -89,26 +88,22 @@ class PostGIS:
self.execute('SET search_path TO %s,public' % self.config['schema'])
self.connection.commit()
-
def log_action(self, action, bestand="n.v.t", bericht='geen', error=False):
sql = "INSERT INTO stetl_log(actie, bestand, error, bericht) VALUES (%s, %s, %s, %s)"
parameters = (action, bestand, error, bericht)
self.tx_execute(sql, parameters)
-
def log_meta(self, key, value):
sql = "INSERT INTO stetl_info(sleutel, waarde) VALUES (%s, %s)"
parameters = (key, value)
self.tx_execute(sql, parameters)
-
def make_bytea(self, blob):
return psycopg2.Binary(blob)
def get_column_names(self, table, schema='public'):
- column_names = []
-
- self.cursor.execute("select column_name from information_schema.columns where table_schema = '%s' and table_name='%s'" % (schema, table))
+ self.cursor.execute(
+ "select column_name from information_schema.columns where table_schema = '%s' and table_name='%s'" % (schema, table))
column_names = [row[0] for row in self.cursor]
return column_names
@@ -120,14 +115,13 @@ class PostGIS:
self.cursor.execute(sql)
# log.debug(self.cursor.statusmessage)
- except (Exception), e:
+ except Exception as e:
log.error("error %s in query: %s with params: %s" % (str(e), str(sql), str(parameters)))
# self.log_actie("uitvoeren_db", "n.v.t", "fout=%s" % str(e), True)
return -1
return self.cursor.rowcount
-
def file_execute(self, sqlfile):
self.e = None
try:
@@ -139,12 +133,11 @@ class PostGIS:
self.connection.commit()
f.close()
log.info("SQL executed OK")
- except (Exception), e:
+ except Exception as e:
self.e = e
self.log_action("file_execute", "n.v.t", "fout=%s" % str(e), True)
log.warn("can't execute SQL script, error: %s" % (str(e)))
-
def tx_execute(self, sql, parameters=None):
self.e = None
try:
@@ -154,7 +147,7 @@ class PostGIS:
self.connection.close()
# log.debug(self.cursor.statusmessage)
- except (Exception), e:
+ except Exception as e:
self.e = e
log.error("error %s in transaction: %s with parms: %s" % (str(e), str(sql), str(parameters)))
diff --git a/stetl/splitter.py b/stetl/splitter.py
new file mode 100644
index 0000000..957e9e4
--- /dev/null
+++ b/stetl/splitter.py
@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+#
+# Splitter Component base class for ETL.
+#
+# Author: Just van den Broecke
+#
+
+import random
+from util import Util
+from component import Component
+
+log = Util.get_log('splitter')
+
+
+class Splitter(Component):
+ """
+ Component that splits a single input to multiple output Components.
+ Use this for example to produce multiple output file formats (GML, GeoJSON etc)
+ or to publish to multiple remote services (SOS, SensorThings API) or for simple
+ debugging: target Output and StandardOutput.
+ """
+
+ def __init__(self, config_dict, child_list):
+ # Assemble child list
+ children = []
+ section_name = ''
+ for child in child_list:
+ section_name += '-%s_%d' % (child.get_id(), random.randrange(0, 100000))
+ children.append(child)
+
+ # Add ourselves to config for compat with Component
+ config_dict.add_section(section_name)
+
+ # We use the in/out formats of first child, will be compat chcked later
+ Component.__init__(self, config_dict, section_name, consumes=children[0]._input_format,
+ produces=children[0]._output_format)
+
+ # Component sets self.next to None...
+ self.next = children
+
+ def add_next(self, next_component):
+ # We use child list, maybe to be used later
+ pass
+
+ # Check our compatibility with our child Components
+ def is_compatible(self):
+ for comp in self.next:
+ if not comp.is_compatible():
+ return False
+ return True
+
+ def process(self, packet):
+ # Defer processing to our child Components
+ data = packet.data
+ for comp in self.next:
+ packet.data = data
+ comp.process(packet)
+ return packet
+
+ def do_init(self):
+ for comp in self.next:
+ comp.do_init()
+
+ def do_exit(self):
+ # Notify all child comps that we exit
+ for comp in self.next:
+ comp.do_exit()
+
+ def before_invoke(self, packet):
+ """
+ Called just before Component invoke.
+ """
+ for comp in self.next:
+ if not comp.before_invoke(packet):
+ return False
+ return True
+
+ def after_invoke(self, packet):
+ """
+ Called right after Component invoke.
+ """
+ for comp in self.next:
+ if not comp.after_invoke(packet):
+ return False
+ return True
+
+ def after_chain_invoke(self, packet):
+ """
+ Called right after entire Component Chain invoke.
+ """
+ for comp in self.next:
+ if not comp.after_chain_invoke(packet):
+ return False
+ return True
+
+ def invoke(self, packet):
+ for comp in self.next:
+ packet = comp.invoke(packet)
+ return packet
+
+ def init(self):
+ for comp in self.next:
+ comp.init()
+
+ def exit(self):
+ for comp in self.next:
+ comp.exit()
diff --git a/stetl/util.py b/stetl/util.py
index c1f031e..4ae7809 100755
--- a/stetl/util.py
+++ b/stetl/util.py
@@ -4,13 +4,17 @@
#
# Author:Just van den Broecke
-import logging, os, glob, sys, types
-from time import *
+import logging
+import os
+import glob
+import types
+from time import time
from ConfigParser import ConfigParser
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(name)s %(levelname)s %(message)s')
+
# Static utility methods
class Util:
# http://wiki.tei-c.org/index.php/Remove-Namespaces.xsl
@@ -190,7 +194,7 @@ class Util:
result = dict(prepend.items() + {"$": parseChildren(x.getchildren())}.items())
if tag in final:
- if type(final[tag]) is not types.ListType:
+ if not isinstance(final[tag], types.ListType):
final[tag] = [final[tag]]
final[tag].append(result)
@@ -288,7 +292,7 @@ class Util:
# Handy for e.g. XPath expressions
@staticmethod
def stripNamespaces(node):
- if not Util.xslt_strip_ns_doc:
+ if Util.xslt_strip_ns_doc is not None:
Util.xslt_strip_ns_doc = etree.fromstring(Util.xslt_strip_ns)
transform = etree.XSLT(Util.xslt_strip_ns_doc)
@@ -329,7 +333,7 @@ class Util:
elem = elem[x]
except ValueError:
elem = elem.get(x)
- except:
+ except Exception:
pass
return elem
@@ -342,10 +346,10 @@ try:
from cStringIO import StringIO
log.info("Found cStringIO, good!")
-except:
+except Exception:
from StringIO import StringIO
- log.warning("Found StringIO (this is suboptimal, try cStringIO")
+ log.warning("Found %s - this is suboptimal, try cStringIO" % str(type(StringIO)))
try:
from lxml import etree
@@ -386,16 +390,19 @@ try:
from osgeo import ogr
from osgeo import osr
except ImportError:
+ log.warn("No osgeo.gdal|ogr|osr packages found, trying direct import...")
try:
import gdal
import ogr
import osr
except ImportError:
- log.warn("No GDAL/OGR Python bindings. Ok if not using GDAL/OGR functions. See https://pypi.python.org/pypi/GDAL")
+ log.warn(
+ "No GDAL/OGR Python bindings. Ok if not using GDAL/OGR functions. See https://pypi.python.org/pypi/GDAL")
if ogr and gdal and osr:
log.info("Found GDAL/OGR Python bindings, super!!")
+
class ConfigSection():
def __init__(self, config_section):
self.config_dict = dict(config_section)
@@ -404,7 +411,7 @@ class ConfigSection():
return name in self.config_dict
def get(self, name, default=None):
- if not self.config_dict.has_key(name):
+ if name not in self.config_dict:
return default
return self.config_dict[name]
@@ -469,4 +476,3 @@ class ConfigSection():
def to_string(self):
return repr(self.config_dict)
-
diff --git a/stetl/utils/apachelog.py b/stetl/utils/apachelog.py
index 7422101..b5ac02e 100644
--- a/stetl/utils/apachelog.py
+++ b/stetl/utils/apachelog.py
@@ -153,7 +153,8 @@ class parser:
for element in format.split(' '):
hasquotes = 0
- if findquotes.search(element): hasquotes = 1
+ if findquotes.search(element):
+ hasquotes = 1
if hasquotes:
element = lstripquotes.sub('', element)
@@ -201,12 +202,12 @@ class parser:
if k in ['%>s', '%b', '%D']:
try:
v = int(v)
- except:
+ except Exception:
v = 0
elif k == '%t':
try:
v = int(parse_date(v)[0])
- except:
+ except Exception:
v = 0
elif v == '-':
v = None
@@ -222,7 +223,7 @@ class parser:
if self._options['request_path_only']:
try:
v = v.split(' ')[1]
- except:
+ except Exception:
v = ''
# JvdB map %-like keys to readable names using key map
@@ -390,7 +391,6 @@ if __name__ == '__main__':
msg='Line 1 %{User-Agent}i'
)
-
def testline2(self):
data = self.p.parse(self.line2)
self.assertEqual(data['%h'], '212.74.15.68', msg='Line 2 %h')
@@ -431,8 +431,8 @@ if __name__ == '__main__':
)
self.assertEqual(
data['%r'],
- r'GET /core/listing/pl_boat_detail.jsp?&units=Feet&checked_boats=' \
- r'1176818&slim=broker&&hosturl=giffordmarine&&ywo=giffordmarine& ' \
+ r'GET /core/listing/pl_boat_detail.jsp?&units=Feet&checked_boats='
+ r'1176818&slim=broker&&hosturl=giffordmarine&&ywo=giffordmarine& '
r'HTTP/1.1',
msg='Line 3 %r'
)
@@ -440,18 +440,17 @@ if __name__ == '__main__':
self.assertEqual(data['%b'], '2888', msg='Line 3 %b')
self.assertEqual(
data['%{Referer}i'],
- r'http://search.yahoo.com/bin/search?p=\"grady%20white%20306' \
+ r'http://search.yahoo.com/bin/search?p=\"grady%20white%20306'
r'%20bimini\"',
msg='Line 3 %{Referer}i'
)
self.assertEqual(
data['%{User-Agent}i'],
- 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; YPC 3.0.3; ' \
+ 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; YPC 3.0.3; '
'yplus 4.0.00d)',
msg='Line 3 %{User-Agent}i'
)
-
def testjunkline(self):
self.assertRaises(ApacheLogParserError, self.p.parse, 'foobar')
diff --git a/stetl/version.py b/stetl/version.py
index 39e0411..f901408 100644
--- a/stetl/version.py
+++ b/stetl/version.py
@@ -1 +1 @@
-__version__ = "1.0.9"
+__version__ = "1.1"
diff --git a/tests/README.txt b/tests/README.txt
index f4a101e..0e0070e 100644
--- a/tests/README.txt
+++ b/tests/README.txt
@@ -1,10 +1,24 @@
-Testing is dan with Python unittest and run with 'nose',
-see https://nose.readthedocs.org/en/latest/
+Testing is done with Python unittest and run with 'nose2',
+see http://nose2.readthedocs.io/en/latest/
-To install and run with nose:
-pip install nose
+To install and run with nose2:
+pip install nose2
-# from root dir (one above this dir) run
-nosetests [options] tests
+To execute nose2:
+nose2 [options]
+Coverage plugin:
+pip install cov-core
+Execution with coverage (from root dir):
+nose2 --with-coverage
+nose2 --with-coverage -coverage-report html
+The test coverage is written to the dir htmlcov.
+
+Mock objects: mock
+pip install mock
+More information: http://www.voidspace.org.uk/python/mock/
+
+When executing nose2, all directories are scanned for tests. See the nose2 documentation how this is done.
+Regarding the coverage: if you specify the tests directory, only the coverage of the unit tests themselves is reported.
+If you would like to see coverage for the source files as well, you need to execute nose2 from the Stetl root dir.
diff --git a/tests/__init__.pyc b/tests/__init__.pyc
deleted file mode 100644
index 7b7264c..0000000
Binary files a/tests/__init__.pyc and /dev/null differ
diff --git a/examples/basics/3_shape/temp/gmlcities.gfs b/tests/data/cities.gfs
similarity index 89%
copy from examples/basics/3_shape/temp/gmlcities.gfs
copy to tests/data/cities.gfs
index 0f9e11f..68a013f 100644
--- a/examples/basics/3_shape/temp/gmlcities.gfs
+++ b/tests/data/cities.gfs
@@ -2,6 +2,8 @@
<GMLFeatureClass>
<Name>City</Name>
<ElementPath>City</ElementPath>
+ <GeometryName>geometry</GeometryName>
+ <GeometryElementPath>geometry</GeometryElementPath>
<GeometryType>1</GeometryType>
<SRSName>GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]</SRSName>
<DatasetSpecificInfo>
diff --git a/tests/data/cities.gml b/tests/data/cities.gml
new file mode 100644
index 0000000..57c76b0
--- /dev/null
+++ b/tests/data/cities.gml
@@ -0,0 +1,45 @@
+<?xml version='1.0' encoding='utf-8'?>
+<ogr:FeatureCollection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ogr="http://ogr.maptools.org/" xmlns:gml="http://www.opengis.net/gml" xsi:schemaLocation="http://ogr.maptools.org/ ../gmlcities.xsd http://www.opengis.net/gml http://schemas.opengis.net/gml/2.1.2/feature.xsd">
+ <gml:boundedBy>
+ <gml:Box>
+ <gml:coord>
+ <gml:X>-180.0</gml:X>
+ <gml:Y>-90.0</gml:Y>
+ </gml:coord>
+ <gml:coord>
+ <gml:X>180.0</gml:X>
+ <gml:Y>90.0</gml:Y>
+ </gml:coord>
+ </gml:Box>
+ </gml:boundedBy>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Amsterdam</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>52.4,4.9</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Bonn</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>50.7,7.1</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+ <gml:featureMember>
+ <ogr:City>
+ <ogr:name>Rome</ogr:name>
+ <ogr:geometry>
+ <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
+ <gml:coordinates>41.9,12.5</gml:coordinates>
+ </gml:Point>
+ </ogr:geometry>
+ </ogr:City>
+ </gml:featureMember>
+</ogr:FeatureCollection>
diff --git a/tests/data/command.txt b/tests/data/command.txt
new file mode 100644
index 0000000..cb8805c
--- /dev/null
+++ b/tests/data/command.txt
@@ -0,0 +1 @@
+echo "Dit is een test"
diff --git a/tests/data/data_xmlassembler.gml b/tests/data/data_xmlassembler.gml
new file mode 100644
index 0000000..bf1ab1e
--- /dev/null
+++ b/tests/data/data_xmlassembler.gml
@@ -0,0 +1,271 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<top10nl:FeatureCollectionT10NL xmlns:top10nl="http://register.geostandaarden.nl/gmlapplicatieschema/top10nl/1.2.0" xmlns:brt="http://register.geostandaarden.nl/gmlapplicatieschema/brt-algemeen/1.2.0" xmlns:gml="http://www.opengis.net/gml/3.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xlink="http://www.w3.org/1999/xlink" xsi:schemaLocation="http://register.geostandaarden.nl/gmlapplicatieschema/top10nl/1.2.0 top10nl.xsd" gml:id="NL.TOP10NL_FC1">
+ <!--Dit GML-bestand is gegenereerd met make-gml.xsl-->
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101010">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101010</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>arboretum</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied arboretum</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied arboretum</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:puntGeometrie>
+ <gml:Point gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:pos>101014.5 400104.5</gml:pos>
+ </gml:Point>
+ </brt:puntGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101011">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101011</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>arboretum</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied arboretum</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied arboretum</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:vlakGeometrie>
+ <gml:Polygon gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:exterior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101010 400110 101019 400110 101019 400119 101010 400119 101010 400110</gml:posList>
+ </gml:LinearRing>
+ </gml:exterior>
+ </gml:Polygon>
+ </brt:vlakGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101012">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101012</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>arboretum</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied arboretum</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied arboretum</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:vlakGeometrie>
+ <gml:Polygon gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:exterior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101010 400120 101019 400120 101019 400129 101010 400129 101010 400120</gml:posList>
+ </gml:LinearRing>
+ </gml:exterior>
+ <gml:interior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101013 400123 101013 400126 101016 400126 101016 400123 101013 400123</gml:posList>
+ </gml:LinearRing>
+ </gml:interior>
+ </gml:Polygon>
+ </brt:vlakGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101013">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101013</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>arboretum</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied arboretum</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied arboretum</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:multivlakGeometrie>
+ <gml:MultiSurface gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:surfaceMember>
+ <gml:Polygon gml:id="x">
+ <gml:exterior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101010 400130 101019 400130 101019 400134 101010 400134 101010 400130</gml:posList>
+ </gml:LinearRing>
+ </gml:exterior>
+ </gml:Polygon>
+ </gml:surfaceMember>
+ <gml:surfaceMember>
+ <gml:Polygon gml:id="x">
+ <gml:exterior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101010 400135 101019 400135 101019 400139 101010 400139 101010 400135</gml:posList>
+ </gml:LinearRing>
+ </gml:exterior>
+ </gml:Polygon>
+ </gml:surfaceMember>
+ </gml:MultiSurface>
+ </brt:multivlakGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101020">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101020</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>attractiepark</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied attractiepark</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied attractiepark</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:puntGeometrie>
+ <gml:Point gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:pos>101024.5 400104.5</gml:pos>
+ </gml:Point>
+ </brt:puntGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101021">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101021</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>attractiepark</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied attractiepark</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied attractiepark</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:vlakGeometrie>
+ <gml:Polygon gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:exterior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101020 400110 101029 400110 101029 400119 101020 400119 101020 400110</gml:posList>
+ </gml:LinearRing>
+ </gml:exterior>
+ </gml:Polygon>
+ </brt:vlakGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101022">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101022</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>attractiepark</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied attractiepark</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied attractiepark</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:vlakGeometrie>
+ <gml:Polygon gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:exterior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101020 400120 101029 400120 101029 400129 101020 400129 101020 400120</gml:posList>
+ </gml:LinearRing>
+ </gml:exterior>
+ <gml:interior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101023 400123 101023 400126 101026 400126 101026 400123 101023 400123</gml:posList>
+ </gml:LinearRing>
+ </gml:interior>
+ </gml:Polygon>
+ </brt:vlakGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+</top10nl:FeatureCollectionT10NL>
diff --git a/tests/data/dummy.gml b/tests/data/dummy.gml
new file mode 100644
index 0000000..0c3f66c
--- /dev/null
+++ b/tests/data/dummy.gml
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<top10nl:FeatureCollectionT10NL xmlns:top10nl="http://register.geostandaarden.nl/gmlapplicatieschema/top10nl/1.2.0" xmlns:brt="http://register.geostandaarden.nl/gmlapplicatieschema/brt-algemeen/1.2.0" xmlns:gml="http://www.opengis.net/gml/3.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xlink="http://www.w3.org/1999/xlink" xsi:schemaLocation="http://register.geostandaarden.nl/gmlapplicatieschema/top10nl/1.2.0 top10nl.xsd" gml:id="NL.TOP10NL_FC1">
+ <!--Dit GML-bestand is gegenereerd met make-gml.xsl-->
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101010">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101010</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>arboretum</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied arboretum</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied arboretum</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:puntGeometrie>
+ <gml:Point gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:pos>101014.5 400104.5</gml:pos>
+ </gml:Point>
+ </brt:puntGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101011">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101011</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>arboretum</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied arboretum</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied arboretum</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:vlakGeometrie>
+ <gml:Polygon gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:exterior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101010 400110 101019 400110 101019 400119 101010 400119 101010 400110</gml:posList>
+ </gml:LinearRing>
+ </gml:exterior>
+ </gml:Polygon>
+ </brt:vlakGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101012">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101012</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>arboretum</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied arboretum</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied arboretum</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:vlakGeometrie>
+ <gml:Polygon gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:exterior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101010 400120 101019 400120 101019 400129 101010 400129 101010 400120</gml:posList>
+ </gml:LinearRing>
+ </gml:exterior>
+ <gml:interior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101013 400123 101013 400126 101016 400126 101016 400123 101013 400123</gml:posList>
+ </gml:LinearRing>
+ </gml:interior>
+ </gml:Polygon>
+ </brt:vlakGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+</top10nl:FeatureCollectionT10NL>
diff --git a/tests/data/dummy.sql b/tests/data/dummy.sql
new file mode 100644
index 0000000..47e515f
--- /dev/null
+++ b/tests/data/dummy.sql
@@ -0,0 +1,2 @@
+create schema dummy;
+create table dummy.sample (id serial, value text);
\ No newline at end of file
diff --git a/tests/data/dummy2.gml b/tests/data/dummy2.gml
new file mode 100644
index 0000000..0c3f66c
--- /dev/null
+++ b/tests/data/dummy2.gml
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<top10nl:FeatureCollectionT10NL xmlns:top10nl="http://register.geostandaarden.nl/gmlapplicatieschema/top10nl/1.2.0" xmlns:brt="http://register.geostandaarden.nl/gmlapplicatieschema/brt-algemeen/1.2.0" xmlns:gml="http://www.opengis.net/gml/3.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xlink="http://www.w3.org/1999/xlink" xsi:schemaLocation="http://register.geostandaarden.nl/gmlapplicatieschema/top10nl/1.2.0 top10nl.xsd" gml:id="NL.TOP10NL_FC1">
+ <!--Dit GML-bestand is gegenereerd met make-gml.xsl-->
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101010">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101010</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>arboretum</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied arboretum</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied arboretum</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:puntGeometrie>
+ <gml:Point gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:pos>101014.5 400104.5</gml:pos>
+ </gml:Point>
+ </brt:puntGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101011">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101011</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>arboretum</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied arboretum</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied arboretum</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:vlakGeometrie>
+ <gml:Polygon gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:exterior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101010 400110 101019 400110 101019 400119 101010 400119 101010 400110</gml:posList>
+ </gml:LinearRing>
+ </gml:exterior>
+ </gml:Polygon>
+ </brt:vlakGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101012">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101012</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>arboretum</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied arboretum</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied arboretum</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:vlakGeometrie>
+ <gml:Polygon gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:exterior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101010 400120 101019 400120 101019 400129 101010 400129 101010 400120</gml:posList>
+ </gml:LinearRing>
+ </gml:exterior>
+ <gml:interior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101013 400123 101013 400126 101016 400126 101016 400123 101013 400123</gml:posList>
+ </gml:LinearRing>
+ </gml:interior>
+ </gml:Polygon>
+ </brt:vlakGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+</top10nl:FeatureCollectionT10NL>
diff --git a/tests/data/gfs/top10-funcgebied.gfs b/tests/data/gfs/top10-funcgebied.gfs
new file mode 100644
index 0000000..4e5fff4
--- /dev/null
+++ b/tests/data/gfs/top10-funcgebied.gfs
@@ -0,0 +1,229 @@
+<GMLFeatureClassList>
+ <!--Dit GFS-bestand is gegenereerd met make-gfs.xsl-->
+ <GMLFeatureClass>
+ <Name>FunctioneelGebied_Punt</Name>
+ <ElementPath>FunctioneelGebied_Punt</ElementPath>
+ <GeometryType>1</GeometryType>
+ <SRSName>EPSG:28992</SRSName>
+ <DatasetSpecificInfo>
+ <ExtentXMin>10000</ExtentXMin>
+ <ExtentXMax>280000</ExtentXMax>
+ <ExtentYMin>300000</ExtentYMin>
+ <ExtentYMax>625000</ExtentYMax>
+ </DatasetSpecificInfo>
+ <!--Begin properties van Top10Object-->
+ <PropertyDefn>
+ <Name>namespace</Name>
+ <ElementPath>identificatie|NEN3610ID|namespace</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>lokaalID</Name>
+ <ElementPath>identificatie|NEN3610ID|lokaalID</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>brontype</Name>
+ <ElementPath>brontype</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>bronactualiteit</Name>
+ <ElementPath>bronactualiteit</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>bronbeschrijving</Name>
+ <ElementPath>bronbeschrijving</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>bronnauwkeurigheid</Name>
+ <ElementPath>bronnauwkeurigheid</ElementPath>
+ <Type>Real</Type>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>objectBeginTijd</Name>
+ <ElementPath>objectBeginTijd</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>objectEindTijd</Name>
+ <ElementPath>objectEindTijd</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>tijdstipRegistratie</Name>
+ <ElementPath>tijdstipRegistratie</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>eindRegistratie</Name>
+ <ElementPath>eindRegistratie</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>tdnCode</Name>
+ <ElementPath>tdnCode</ElementPath>
+ <Type>Integer</Type>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>visualisatieCode</Name>
+ <ElementPath>visualisatieCode</ElementPath>
+ <Type>Integer</Type>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>mutatieType</Name>
+ <ElementPath>mutatieType</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <!--Einde properties van Top10Object-->
+ <!--Begin properties van FunctioneelGebied-->
+ <PropertyDefn>
+ <Name>typeFunctioneelGebied</Name>
+ <ElementPath>typeFunctioneelGebied</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>soortnaam</Name>
+ <ElementPath>soortnaam</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>naamNL</Name>
+ <ElementPath>naamNL</ElementPath>
+ <Type>StringList</Type>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>naamFries</Name>
+ <ElementPath>naamFries</ElementPath>
+ <Type>StringList</Type>
+ </PropertyDefn>
+ <!--Einde properties van FunctioneelGebied-->
+ </GMLFeatureClass>
+ <GMLFeatureClass>
+ <Name>FunctioneelGebied_Vlak</Name>
+ <ElementPath>FunctioneelGebied_Vlak</ElementPath>
+ <GeometryType>6</GeometryType>
+ <SRSName>EPSG:28992</SRSName>
+ <DatasetSpecificInfo>
+ <ExtentXMin>10000</ExtentXMin>
+ <ExtentXMax>280000</ExtentXMax>
+ <ExtentYMin>300000</ExtentYMin>
+ <ExtentYMax>625000</ExtentYMax>
+ </DatasetSpecificInfo>
+ <!--Begin properties van Top10Object-->
+ <PropertyDefn>
+ <Name>namespace</Name>
+ <ElementPath>identificatie|NEN3610ID|namespace</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>lokaalID</Name>
+ <ElementPath>identificatie|NEN3610ID|lokaalID</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>brontype</Name>
+ <ElementPath>brontype</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>bronactualiteit</Name>
+ <ElementPath>bronactualiteit</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>bronbeschrijving</Name>
+ <ElementPath>bronbeschrijving</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>bronnauwkeurigheid</Name>
+ <ElementPath>bronnauwkeurigheid</ElementPath>
+ <Type>Real</Type>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>objectBeginTijd</Name>
+ <ElementPath>objectBeginTijd</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>objectEindTijd</Name>
+ <ElementPath>objectEindTijd</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>tijdstipRegistratie</Name>
+ <ElementPath>tijdstipRegistratie</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>eindRegistratie</Name>
+ <ElementPath>eindRegistratie</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>tdnCode</Name>
+ <ElementPath>tdnCode</ElementPath>
+ <Type>Integer</Type>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>visualisatieCode</Name>
+ <ElementPath>visualisatieCode</ElementPath>
+ <Type>Integer</Type>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>mutatieType</Name>
+ <ElementPath>mutatieType</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <!--Einde properties van Top10Object-->
+ <!--Begin properties van FunctioneelGebied-->
+ <PropertyDefn>
+ <Name>typeFunctioneelGebied</Name>
+ <ElementPath>typeFunctioneelGebied</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>soortnaam</Name>
+ <ElementPath>soortnaam</ElementPath>
+ <Type>String</Type>
+ <Width>250</Width>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>naamNL</Name>
+ <ElementPath>naamNL</ElementPath>
+ <Type>StringList</Type>
+ </PropertyDefn>
+ <PropertyDefn>
+ <Name>naamFries</Name>
+ <ElementPath>naamFries</ElementPath>
+ <Type>StringList</Type>
+ </PropertyDefn>
+ <!--Einde properties van FunctioneelGebied-->
+ </GMLFeatureClass>
+</GMLFeatureClassList>
diff --git a/tests/data/keep_dummy.gml b/tests/data/keep_dummy.gml
new file mode 100644
index 0000000..0c3f66c
--- /dev/null
+++ b/tests/data/keep_dummy.gml
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<top10nl:FeatureCollectionT10NL xmlns:top10nl="http://register.geostandaarden.nl/gmlapplicatieschema/top10nl/1.2.0" xmlns:brt="http://register.geostandaarden.nl/gmlapplicatieschema/brt-algemeen/1.2.0" xmlns:gml="http://www.opengis.net/gml/3.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xlink="http://www.w3.org/1999/xlink" xsi:schemaLocation="http://register.geostandaarden.nl/gmlapplicatieschema/top10nl/1.2.0 top10nl.xsd" gml:id="NL.TOP10NL_FC1">
+ <!--Dit GML-bestand is gegenereerd met make-gml.xsl-->
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101010">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101010</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>arboretum</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied arboretum</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied arboretum</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:puntGeometrie>
+ <gml:Point gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:pos>101014.5 400104.5</gml:pos>
+ </gml:Point>
+ </brt:puntGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101011">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101011</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>arboretum</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied arboretum</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied arboretum</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:vlakGeometrie>
+ <gml:Polygon gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:exterior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101010 400110 101019 400110 101019 400119 101010 400119 101010 400110</gml:posList>
+ </gml:LinearRing>
+ </gml:exterior>
+ </gml:Polygon>
+ </brt:vlakGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+ <top10nl:FeatureMember>
+ <top10nl:FunctioneelGebied gml:id="nl.top10nl.101012">
+ <top10nl:identificatie>
+ <brt:NEN3610ID>
+ <brt:namespace>NL.TOP10NL</brt:namespace>
+ <brt:lokaalID>101012</brt:lokaalID>
+ </brt:NEN3610ID>
+ </top10nl:identificatie>
+ <top10nl:brontype>Luchtfoto</top10nl:brontype>
+ <top10nl:bronactualiteit>2015-01-01</top10nl:bronactualiteit>
+ <top10nl:bronbeschrijving>Een orthogerectificeerde fotografische opname van een deel van het aardoppervlak. Gemaakt vanuit een vliegtuig.</top10nl:bronbeschrijving>
+ <top10nl:bronnauwkeurigheid>0.4</top10nl:bronnauwkeurigheid>
+ <top10nl:objectBeginTijd>2015-08-15</top10nl:objectBeginTijd>
+ <top10nl:tijdstipRegistratie>2015-08-15</top10nl:tijdstipRegistratie>
+ <top10nl:tdnCode>999</top10nl:tdnCode>
+ <top10nl:visualisatieCode>19160</top10nl:visualisatieCode>
+ <top10nl:mutatieType>TEST</top10nl:mutatieType>
+ <top10nl:typeFunctioneelGebied>arboretum</top10nl:typeFunctioneelGebied>
+ <!--Het optionele attribuut soortnaam wordt overgeslagen-->
+ <top10nl:naamNL>Test typeFunctioneelGebied arboretum</top10nl:naamNL>
+ <top10nl:naamFries>Test typeFunctioneelGebied arboretum</top10nl:naamFries>
+ <top10nl:geometrie>
+ <brt:BRTVlakMultivlakOfPunt>
+ <brt:vlakGeometrie>
+ <gml:Polygon gml:id="x" srsName="urn:ogc:def:crs:EPSG::28992">
+ <gml:exterior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101010 400120 101019 400120 101019 400129 101010 400129 101010 400120</gml:posList>
+ </gml:LinearRing>
+ </gml:exterior>
+ <gml:interior>
+ <gml:LinearRing>
+ <gml:posList srsDimension="2" count="5">101013 400123 101013 400126 101016 400126 101016 400123 101013 400123</gml:posList>
+ </gml:LinearRing>
+ </gml:interior>
+ </gml:Polygon>
+ </brt:vlakGeometrie>
+ </brt:BRTVlakMultivlakOfPunt>
+ </top10nl:geometrie>
+ </top10nl:FunctioneelGebied>
+ </top10nl:FeatureMember>
+</top10nl:FeatureCollectionT10NL>
diff --git a/tests/data/menu.json b/tests/data/menu.json
new file mode 100644
index 0000000..fe6c37d
--- /dev/null
+++ b/tests/data/menu.json
@@ -0,0 +1,11 @@
+{"menu": {
+ "id": "file",
+ "value": "File",
+ "popup": {
+ "menuitem": [
+ {"value": "New", "onclick": "CreateNewDoc()"},
+ {"value": "Open", "onclick": "OpenDoc()"},
+ {"value": "Close", "onclick": "CloseDoc()"}
+ ]
+ }
+}}
\ No newline at end of file
diff --git a/tests/data/pandhoogtes-klein.csv b/tests/data/pandhoogtes-klein.csv
new file mode 100644
index 0000000..c395f55
--- /dev/null
+++ b/tests/data/pandhoogtes-klein.csv
@@ -0,0 +1,10 @@
+ID, height_min, height_avg, height_max
+"373100000011010",11.741,17.224,18.362
+"373100000011011",11.759,14.053,14.734
+"373100000011014",8.785,13.642,18.333
+"373100000011015",8.464,11.506,12.780
+"373100000011016",7.360,12.080,15.054
+"373100000011018",11.417,15.628,16.325
+"373100000011021",11.536,15.679,18.843
+"373100000011022",11.678,13.423,14.002
+"373100000011028",8.694,14.160,18.309
diff --git a/tests/data/pandhoogtes.csv b/tests/data/pandhoogtes.csv
new file mode 100644
index 0000000..3f2f0d1
--- /dev/null
+++ b/tests/data/pandhoogtes.csv
@@ -0,0 +1,432 @@
+ID, height_min, height_avg, height_max
+"373100000011010",11.741,17.224,18.362
+"373100000011011",11.759,14.053,14.734
+"373100000011014",8.785,13.642,18.333
+"373100000011015",8.464,11.506,12.780
+"373100000011016",7.360,12.080,15.054
+"373100000011018",11.417,15.628,16.325
+"373100000011021",11.536,15.679,18.843
+"373100000011022",11.678,13.423,14.002
+"373100000011028",8.694,14.160,18.309
+"373100000011032",5.703,10.157,14.371
+"373100000011041",5.894,13.833,16.599
+"373100000011200",7.553,12.963,15.433
+"373100000011582",7.587,16.624,18.038
+"373100000011600",14.487,18.617,22.296
+"373100000011601",10.269,12.515,13.174
+"373100000011603",5.968,14.573,18.750
+"373100000011604",5.408,7.581,8.015
+"373100000011606",14.257,18.710,21.578
+"373100000011607",9.950,12.781,13.696
+"373100000011608",12.358,16.879,18.809
+"373100000011609",15.903,21.406,23.163
+"373100000011610",5.520,9.358,13.372
+"373100000011611",5.354,9.139,11.573
+"373100000011615",9.350,13.579,17.346
+"373100000011624",5.858,10.880,14.572
+"373100000011626",5.765,9.948,14.906
+"373100000011697",10.725,16.089,19.876
+"373100000011702",15.432,20.538,21.632
+"373100000011703",5.511,10.280,14.434
+"373100000011704",4.949,10.848,13.340
+"373100000011712",5.528,8.427,10.855
+"373100000011717",5.889,8.588,12.221
+"373100000011718",5.458,11.211,12.320
+"373100000011719",7.685,10.044,10.346
+"373100000011720",5.585,10.122,14.938
+"373100000011731",8.781,12.815,17.163
+"373100000011732",8.680,11.325,12.343
+"373100000011771",7.561,13.026,14.477
+"373100000011789",5.496,10.987,14.480
+"373100000011796",5.848,8.729,10.172
+"373100000011797",6.027,8.532,11.090
+"373100000011798",5.372,9.678,12.679
+"373100000011802",7.496,13.131,16.356
+"373100000011803",6.502,8.725,9.159
+"373100000011805",8.932,15.608,19.132
+"373100000011806",11.109,18.841,22.773
+"373100000011807",7.843,14.592,16.989
+"373100000011821",9.916,14.428,16.432
+"373100000011825",8.467,11.510,13.355
+"373100000011833",8.771,11.674,13.367
+"373100000011834",5.387,9.818,12.198
+"373100000011838",5.478,8.680,11.680
+"373100000011841",8.453,11.573,14.659
+"373100000011842",5.712,11.505,14.207
+"373100000011843",8.519,13.985,18.307
+"373100000011903",7.326,12.685,15.017
+"373100000011905",5.777,8.810,11.225
+"373100000011908",5.327,11.290,14.401
+"373100000011909",5.394,8.367,9.227
+"373100000011912",5.470,9.227,12.950
+"373100000011913",7.150,11.248,13.162
+"373100000011915",15.839,19.134,21.115
+"373100000011916",7.834,12.002,14.558
+"373100000011917",12.945,16.542,18.588
+"373100000011919",10.703,14.593,16.281
+"373100000011920",5.968,9.116,13.423
+"373100000011921",7.786,12.517,13.887
+"373100000011922",9.694,16.089,18.835
+"373100000011938",9.971,14.868,16.165
+"373100000011949",8.581,14.324,17.551
+"373100000012010",7.989,11.044,12.480
+"373100000012011",13.059,18.942,21.885
+"373100000012013",8.360,13.694,16.692
+"373100000012016",5.529,10.713,12.191
+"373100000012017",5.576,8.203,12.815
+"373100000012018",6.353,11.478,13.682
+"373100000012019",7.803,13.881,16.604
+"373100000012020",8.446,10.878,11.399
+"373100000012023",6.937,11.896,13.420
+"373100000012024",5.267,10.550,16.669
+"373100000012025",6.316,11.082,13.861
+"373100000012026",10.909,15.793,19.650
+"373100000012027",10.807,13.380,13.944
+"373100000012028",5.402,9.781,13.964
+"373100000012029",5.396,9.193,10.959
+"373100000012030",12.776,19.412,22.264
+"373100000012031",5.860,10.471,12.999
+"373100000012036",8.658,14.038,16.916
+"373100000012038",8.085,11.969,15.681
+"373100000012039",11.123,14.539,15.836
+"373100000012045",6.008,9.295,12.364
+"373100000012116",5.330,11.090,14.075
+"373100000012117",5.458,8.581,9.337
+"373100000012120",9.309,16.292,17.787
+"373100000012121",8.469,10.998,12.007
+"373100000012125",8.144,11.071,12.699
+"373100000012128",10.052,16.101,19.072
+"373100000012129",12.349,14.558,15.158
+"373100000012131",7.865,13.121,16.588
+"373100000012132",6.204,14.157,17.671
+"373100000012136",5.839,10.107,13.263
+"373100000012137",5.625,8.975,11.367
+"373100000012138",7.356,13.158,16.090
+"373100000012139",11.694,16.497,17.769
+"373100000012145",7.643,12.084,15.994
+"373100000012146",5.867,17.555,19.001
+"373100000012152",5.340,12.040,15.283
+"373100000012153",5.431,8.381,9.905
+"373100000012154",5.364,7.664,7.737
+"373100000012155",5.369,10.512,14.603
+"373100000012156",5.352,7.403,7.691
+"373100000012216",11.067,14.641,14.946
+"373100000012220",5.211,9.904,12.205
+"373100000012221",5.321,7.813,8.477
+"373100000012235",11.633,18.569,21.815
+"373100000012236",9.047,12.172,13.294
+"373100000012237",6.300,11.909,14.778
+"373100000012238",6.388,8.515,8.944
+"373100000012239",12.116,16.161,19.532
+"373100000012240",12.152,14.529,15.740
+"373100000012241",9.198,11.404,11.768
+"373100000012242",7.948,13.922,16.633
+"373100000012246",5.967,9.546,12.459
+"373100000012248",5.580,11.527,15.751
+"373100000012249",8.519,12.238,16.548
+"373100000012250",8.534,11.555,13.064
+"373100000012252",10.911,15.977,18.118
+"373100000012273",5.447,12.768,17.305
+"373100000012295",5.877,16.712,19.220
+"373100000012309",5.181,9.923,12.610
+"373100000012313",8.627,12.069,15.540
+"373100000012336",7.810,17.531,20.826
+"373100000012337",9.443,16.968,21.137
+"373100000012340",5.279,9.882,11.634
+"373100000012348",8.339,13.510,15.967
+"373100000012349",8.258,18.042,19.310
+"373100000012358",11.033,16.182,18.919
+"373100000012359",8.785,14.299,17.605
+"373100000012369",7.508,11.132,16.599
+"373100000012373",7.647,12.735,16.526
+"373100000012374",7.643,12.741,16.341
+"373100000012375",7.689,12.898,16.637
+"373100000012377",7.667,12.114,16.042
+"373100000012379",7.598,12.892,16.472
+"373100000012383",6.031,10.555,12.872
+"373100000012389",5.382,8.839,13.348
+"373100000012391",5.176,12.350,14.707
+"373100000012392",5.328,7.981,8.219
+"373100000012406",5.179,11.694,14.622
+"373100000012407",5.211,9.199,11.272
+"373100000012434",8.097,16.566,18.046
+"373100000012442",5.664,10.942,14.813
+"373100000012443",11.527,15.125,17.271
+"373100000012448",11.313,14.593,17.302
+"373100000012456",10.003,15.874,19.513
+"373100000012466",11.410,18.494,21.579
+"373100000012467",11.552,14.746,16.418
+"373100000012494",7.671,11.976,13.403
+"373100000012498",6.069,10.994,14.017
+"373100000012499",5.759,8.671,11.912
+"373100000012501",5.691,13.697,15.613
+"373100000012559",6.066,12.891,17.480
+"373100000012577",5.912,11.117,15.761
+"373100000012579",8.610,14.724,17.587
+"373100000012580",4.984,7.289,7.525
+"373100000012583",8.770,12.972,14.968
+"373100000012599",7.494,11.370,15.086
+"373100000012607",11.513,16.142,18.051
+"373100000012608",11.601,14.665,15.712
+"373100000012672",11.892,16.536,18.341
+"373100000012676",4.731,10.705,14.650
+"373100000012679",7.816,13.757,17.266
+"373100000012691",3.612,11.534,13.336
+"373100000012694",11.000,16.832,20.482
+"373100000012695",5.104,12.226,16.667
+"373100000013028",8.057,12.265,14.691
+"373100000013029",5.448,8.068,12.112
+"373100000013030",8.070,10.937,12.225
+"373100000013037",5.623,10.869,13.155
+"373100000013038",5.567,12.620,16.137
+"373100000013045",5.726,14.509,19.854
+"373100000013046",5.695,8.855,10.473
+"373100000013058",15.542,23.204,27.817
+"373100000013067",13.567,20.714,25.117
+"373100000013074",12.750,20.328,25.277
+"373100000013082",4.555,8.157,9.889
+"373100000013100",5.248,10.415,15.718
+"373100000013106",11.939,17.310,20.711
+"373100000013111",5.977,14.162,21.718
+"373100000013114",5.409,13.290,17.715
+"373100000013115",5.937,10.083,11.098
+"373100000013116",5.962,8.290,9.286
+"373100000013117",6.269,10.066,11.108
+"373100000013126",12.955,22.419,27.313
+"373100000013127",7.486,12.838,17.011
+"373100000013128",7.487,12.503,14.393
+"373100000013129",17.317,24.478,27.706
+"373100000013133",9.851,15.947,18.607
+"373100000013134",12.173,17.317,18.720
+"373100000013135",13.479,16.018,17.315
+"373100000013136",19.658,23.673,25.143
+"373100000013139",5.490,10.679,13.566
+"373100000013149",14.339,18.778,21.230
+"373100000013162",12.958,21.547,25.927
+"373100000013166",13.589,18.757,20.755
+"373100000013167",16.033,18.667,19.300
+"373100000013168",9.585,12.799,14.741
+"373100000013169",16.380,20.235,22.772
+"373100000013170",16.517,24.904,27.195
+"373100000013171",8.959,15.071,18.281
+"373100000013175",11.427,17.662,18.468
+"373100000013181",12.607,17.166,19.308
+"373100000013184",4.630,11.074,16.606
+"373100000013187",5.797,9.167,12.068
+"373100000013188",5.606,7.955,9.109
+"373100000013197",10.525,15.195,17.106
+"373100000013198",18.539,24.621,30.019
+"373100000013199",18.002,21.056,22.862
+"373100000013200",13.543,19.008,21.359
+"373100000013203",9.413,14.210,15.380
+"373100000014638",5.379,5.483,5.654
+"373100000014639",5.851,12.473,14.757
+"373100000014642",5.840,12.574,14.585
+"373100000014692",5.070,7.671,8.092
+"373100000014717",5.314,5.374,5.438
+"373100000014793",11.576,13.200,13.693
+"373100000014805",7.988,10.190,10.446
+"373100000014821",5.254,7.947,10.548
+"373100000014873",3.906,4.010,4.337
+"373100000014882",9.919,11.867,12.179
+"373100000014884",6.076,8.473,8.908
+"373100000014893",8.161,9.223,9.651
+"373100000014896",8.569,10.135,13.716
+"373100000014899",7.000,9.162,11.299
+"373100000014903",11.964,13.570,14.034
+"373100000014909",8.399,9.030,9.091
+"373100000014910",12.027,12.037,12.047
+"373100000014912",7.132,12.913,15.008
+"373100000014913",5.038,9.984,15.435
+"373100000014914",28.068,31.945,33.185
+"373100000014915",7.754,12.907,16.749
+"373100000014916",8.015,13.841,17.361
+"373100000014917",5.963,12.060,16.789
+"373100000014918",5.377,11.111,15.568
+"373100000014919",7.899,13.618,16.146
+"373100000014920",16.440,21.295,24.199
+"373100000014921",5.475,11.437,13.680
+"373100000014922",8.272,16.264,22.212
+"373100000014923",8.306,15.204,20.774
+"373100000014924",8.401,11.211,13.062
+"373100000014925",8.117,12.622,16.029
+"373100000014926",6.335,17.060,20.448
+"373100000014927",5.848,16.099,19.733
+"373100000014928",7.873,13.828,17.738
+"373100000014982",9.556,20.889,28.431
+"373100000014992",7.352,13.987,17.571
+"373100000015115",5.829,11.381,14.560
+"373100000015122",5.817,12.035,14.788
+"373100000015125",8.276,14.095,16.279
+"373100000015187",11.928,16.553,18.387
+"373100000015207",8.961,11.390,12.306
+"373100000015280",6.194,20.619,23.316
+"373100000015290",10.681,14.273,16.563
+"373100000015295",10.047,13.663,16.069
+"373100000015296",11.175,14.845,16.580
+"373100000015297",8.767,12.445,14.744
+"373100000015336",6.506,9.384,9.612
+"373100000015347",6.707,9.213,9.843
+"373100000015361",10.954,13.247,14.976
+"373100000015369",5.124,5.337,5.420
+"373100000015393",13.065,15.920,16.646
+"373100000015426",9.418,13.058,15.353
+"373100000015458",5.277,9.038,10.778
+"373100000015478",5.437,6.607,10.533
+"373100000015483",8.528,11.915,13.140
+"373100000015543",10.470,14.099,15.885
+"373100000015561",8.746,12.181,14.762
+"373100000015601",5.702,8.761,9.895
+"373100000015609",8.610,14.502,16.961
+"373100000015613",11.664,13.811,16.438
+"373100000015616",7.892,11.499,13.332
+"373100000015675",8.300,10.679,13.672
+"373100000015705",5.025,7.931,8.818
+"373100000015725",4.483,6.758,7.570
+"373100000015729",8.311,10.622,11.482
+"373100000015740",4.432,6.509,9.537
+"373100000015769",5.797,8.695,9.037
+"373100000015785",8.573,10.564,10.908
+"373100000015840",6.460,14.042,15.916
+"373100000015868",8.531,10.591,11.249
+"373100000015870",5.308,5.368,5.464
+"373100000015885",8.474,12.532,19.400
+"373100000015889",7.902,10.333,10.565
+"373100000015890",9.213,11.396,11.781
+"373100000015935",9.898,13.560,16.002
+"373100000015942",5.372,8.400,8.739
+"373100000015971",6.545,10.230,11.823
+"373100000015995",8.315,11.408,12.526
+"373100000016051",8.734,10.748,11.051
+"373100000016054",6.935,9.485,10.668
+"373100000016061",5.854,9.617,13.578
+"373100000016070",9.359,12.891,15.292
+"373100000016093",6.052,13.468,17.699
+"373100000016103",9.002,14.416,16.035
+"373100000016104",8.998,14.397,14.878
+"373100000016105",9.024,14.419,16.007
+"373100000016106",9.012,14.427,15.998
+"373100000016107",8.991,14.410,15.259
+"373100000016111",9.037,14.448,15.965
+"373100000016112",8.720,14.386,15.977
+"373100000016113",9.046,14.363,15.999
+"373100000016217",5.371,5.454,5.528
+"373100000016257",8.542,10.975,11.834
+"373100000016372",5.410,5.511,5.575
+"373100000016467",7.813,10.273,10.800
+"373100000016493",5.361,5.456,5.569
+"373100000016503",5.866,12.441,14.574
+"373100000016504",5.852,12.374,14.585
+"373100000016505",5.851,11.391,14.576
+"373100000016509",5.393,5.500,5.594
+"373100000016510",5.344,5.477,5.629
+"373100000016511",5.835,12.496,14.563
+"373100000016547",7.653,10.204,10.449
+"373100000016579",5.148,5.366,7.438
+"373100000016679",6.252,11.583,14.615
+"373100000016680",10.514,18.695,19.649
+"373100000016686",8.076,13.978,16.511
+"373100000016687",5.295,10.409,13.517
+"373100000016690",4.778,8.418,11.332
+"373100000016691",7.985,13.680,16.324
+"373100000016697",8.440,12.936,16.070
+"373100000016704",5.765,12.569,14.882
+"373100000016719",7.734,10.840,12.929
+"373100000016721",6.954,12.029,14.937
+"373100000016723",7.029,14.933,16.989
+"373100000016725",13.696,18.485,20.847
+"373100000016731",5.918,12.928,14.737
+"373100000016732",6.309,14.267,16.190
+"373100000016733",5.641,10.304,13.891
+"373100000016736",8.711,12.004,13.278
+"373100000016749",12.306,19.542,21.512
+"373100000016759",7.633,12.199,16.621
+"373100000016764",11.753,18.996,19.508
+"373100000016776",6.138,17.042,19.762
+"373100000016792",4.022,6.254,8.169
+"373100000016793",3.662,6.681,8.464
+"373100000016825",6.073,12.197,13.905
+"373100000016826",6.062,12.236,13.962
+"373100000016828",13.069,15.923,18.452
+"373100000016837",6.072,15.116,18.129
+"373100000016840",5.961,12.120,15.007
+"373100000016855",5.401,8.185,9.674
+"373100000017049",6.095,11.522,14.147
+"373100000017062",8.376,16.957,18.720
+"373100000017404",27.884,40.410,56.319
+"373100000017450",8.349,11.385,11.459
+"373100000017477",8.485,11.914,12.641
+"373100000017632",6.679,17.380,21.110
+"373100000017709",8.175,16.777,18.310
+"373100000017725",18.923,24.755,29.192
+"373100000017727",5.909,9.950,10.498
+"373100000017737",8.526,26.697,32.795
+"373100000017840",6.251,11.093,18.499
+"373100000017844",5.625,10.318,13.402
+"373100000017846",6.918,11.479,16.077
+"373100000017850",8.883,14.000,14.172
+"373100000017851",7.659,12.708,15.646
+"373100000017949",6.194,11.585,14.083
+"373100000017954",4.912,6.037,6.955
+"373100000017957",12.960,15.612,17.006
+"373100000017958",9.516,14.780,17.406
+"373100000017959",7.528,12.599,16.519
+"373100000017961",3.822,9.026,12.117
+"373100000017963",5.410,10.497,14.891
+"373100000018053",8.402,14.487,17.610
+"373100000018059",8.383,16.937,18.339
+"373100000018061",8.343,16.978,18.341
+"373100000018069",5.024,11.307,16.048
+"373100000018072",9.226,15.710,19.889
+"373100000018075",6.083,17.268,19.138
+"373100000018076",5.915,11.780,15.542
+"373100000018081",4.600,9.428,13.864
+"373100000018086",8.458,12.791,15.537
+"373100000018087",9.509,17.184,22.023
+"373100000018176",9.756,17.569,21.864
+"373100000018181",9.419,14.411,16.773
+"373100000018182",5.943,13.892,18.080
+"373100000018187",7.437,11.909,16.189
+"373100000018195",8.015,12.735,15.876
+"373100000018293",6.067,16.131,19.431
+"373100000018296",6.153,17.491,19.218
+"373100000018299",4.975,10.286,13.902
+"373100000018301",6.841,12.754,15.266
+"373100000018304",4.732,10.744,13.072
+"373100000018311",11.426,17.022,20.071
+"373100000018314",6.196,14.114,18.097
+"373100000018410",6.450,18.210,20.797
+"373100000018415",5.849,10.950,13.346
+"373100000018416",6.328,13.322,18.619
+"373100000018421",6.837,13.662,15.898
+"373100000018425",4.972,8.888,11.382
+"373100000018426",6.876,12.708,16.137
+"373100000018428",6.047,11.918,15.487
+"373100000018429",5.530,12.363,14.560
+"373100000024746",11.441,15.330,16.344
+"373100000024747",7.599,11.655,14.102
+"373100000024770",14.955,23.284,27.174
+"373100000025404",14.306,18.597,26.573
+"373100000025470",12.785,15.135,15.237
+"373100000025541",6.240,9.442,13.002
+"373100000025762",6.132,8.785,10.255
+"373100000025953",5.349,7.553,8.019
+"373100000026070",10.789,12.704,13.055
+"373100000026196",7.901,9.934,10.293
+"373100000026197",5.700,8.123,11.995
+"373100000026198",5.792,9.490,14.405
+"373100000026199",5.545,8.648,12.477
+"373100000026200",5.955,6.972,10.234
+"373100000026201",5.497,7.924,10.910
+"373100000026202",5.244,8.074,10.777
+"373100000026203",11.130,13.686,13.952
+"373100000026204",7.632,9.829,10.647
+"373100000026205",12.725,15.063,17.107
+"373100000026206",14.829,16.035,19.635
+"373100000026207",6.493,8.694,10.758
+"373100000026208",8.436,10.410,16.618
+"373100000026209",8.030,8.167,8.450
+"373100000026210",5.481,8.284,9.326
+"373100000026211",9.448,11.526,12.327
+"373100000027107",5.476,7.855,8.527
+"373100000027108",5.813,9.950,15.591
\ No newline at end of file
diff --git a/tests/data/stringfileinput_formatargs.txt b/tests/data/stringfileinput_formatargs.txt
new file mode 100644
index 0000000..d1f86f1
--- /dev/null
+++ b/tests/data/stringfileinput_formatargs.txt
@@ -0,0 +1 @@
+{greeting} {name}!
\ No newline at end of file
diff --git a/tests/data/temp/.gitignore b/tests/data/temp/.gitignore
new file mode 100644
index 0000000..86d0cb2
--- /dev/null
+++ b/tests/data/temp/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file
diff --git a/tests/data/xslt/dummy.xslt b/tests/data/xslt/dummy.xslt
new file mode 100644
index 0000000..bbd4cae
--- /dev/null
+++ b/tests/data/xslt/dummy.xslt
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+Voorbewerking Top10NL GML Objecten
+
+Auteur: Just van den Broecke
+
+Aangepast voor Top10NL versie 1.1.1 en 1.2 door Frank Steggink
+
+Dit XSLT script doet een voorbewerking op de ruwe Top10NL GML zoals door Het Kadaster
+geleverd. Dit is nodig omdat GDAL ogr2ogr niet alle mogelijkheden van GML goed aankan
+en omdat met ingang van versie 1.2 de plaats van de geometrie in een feature is gewijzigd.
+
+Voornamelijk gaat het om meerdere geometrie-attributen per Top10 Object. Het interne
+GDAL model kent maar 1 geometrie per feature. Daarnaast is het bij visualiseren bijv.
+met SLDs voor een WMS vaak het handigst om 1 geometrie per laag te hebben. Dit geldt ook
+als we bijvoorbeeld een OGR conversie naar ESRI Shapefile willen doen met ogr2ogr.
+
+Dit script splitst objecten uit Top10NL uit in geometrie-specifieke objecten.
+Bijvoorbeeld een weg (objecttype Wegdeel) heeft twee geometrie-attributen. Het attribuut
+hoofdGeometrie kan weer een vlak, lijn of punt bevatten en het attribuut hartGeometrie kan
+een lijn of punt bevatten. Na uitsplitsen ontstaan max. 5 verschillende objecttypen, namelijk
+Wegdeel_Vlak, Wegdeel_Lijn, Wegdeel_hartLijn etc. Ieder van deze objecten bevat slechts een
+geometrie.
+
+-->
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+ xmlns:xalan="http://xml.apache.org/xalan" exclude-result-prefixes="xalan"
+ xmlns:top10nl="http://register.geostandaarden.nl/gmlapplicatieschema/top10nl/1.2.0"
+ xmlns:brt="http://register.geostandaarden.nl/gmlapplicatieschema/brt-algemeen/1.2.0"
+ xmlns:gml="http://www.opengis.net/gml/3.2"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ xmlns:smil20="http://www.w3.org/2001/SMIL20/"
+ xmlns:smil20lang="http://www.w3.org/2001/SMIL20/Language">
+ <xsl:output method="xml" version="1.0" encoding="utf-8" indent="yes"/>
+ <xsl:strip-space elements="*"/>
+
+ <!-- Start: output omhullende FeatureCollection -->
+ <xsl:template match="/">
+ <dummy>
+ <xsl:apply-templates/>
+ </dummy>
+ </xsl:template>
+
+ <xsl:template match="top10nl:FeatureMember">
+ <xsl:for-each select="top10nl:FunctioneelGebied">
+ <funcgeb>
+ <xsl:attribute name="id">
+ <xsl:value-of select="@gml:id"/>
+ </xsl:attribute>
+ </funcgeb>
+ </xsl:for-each>
+ </xsl:template>
+
+</xsl:stylesheet>
diff --git a/tests/data/zipfileinput.zip b/tests/data/zipfileinput.zip
new file mode 100644
index 0000000..87b275d
Binary files /dev/null and b/tests/data/zipfileinput.zip differ
diff --git a/tests/filters/__init__.py b/tests/filters/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/filters/configs/packetwriter.cfg b/tests/filters/configs/packetwriter.cfg
new file mode 100644
index 0000000..9825b52
--- /dev/null
+++ b/tests/filters/configs/packetwriter.cfg
@@ -0,0 +1,16 @@
+# Config file for unit testing PacketWriter.
+
+[etl]
+chains = parse_file_input|packet_writer|output_std
+
+[parse_file_input]
+class = inputs.fileinput.StringFileInput
+file_path = tests/data/dummy.gml
+
+# Writes the payload of a packet as a string to a file
+[packet_writer]
+class = filters.packetwriter.PacketWriter
+file_path = tests/data/temp/tempfile.gml
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/filters/configs/stringsubstitutionfilter.cfg b/tests/filters/configs/stringsubstitutionfilter.cfg
new file mode 100644
index 0000000..954aa9e
--- /dev/null
+++ b/tests/filters/configs/stringsubstitutionfilter.cfg
@@ -0,0 +1,18 @@
+# Config file for unit testing StringSubstitutionFilter.
+
+[etl]
+chains = input_string_file|string_sub_filter|packet_buffer|output_std
+
+[input_string_file]
+class = inputs.fileinput.StringFileInput
+file_path = tests/data/stringfileinput_formatargs.txt
+
+[string_sub_filter]
+class = filters.stringfilter.StringSubstitutionFilter
+format_args = greeting:Hello name:NLExtract
+
+[packet_buffer]
+class = filters.packetbuffer.PacketBuffer
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/filters/configs/xmlassembler.cfg b/tests/filters/configs/xmlassembler.cfg
new file mode 100644
index 0000000..641f4d0
--- /dev/null
+++ b/tests/filters/configs/xmlassembler.cfg
@@ -0,0 +1,39 @@
+# Config file for unit testing XmlAssembler.
+
+[etl]
+# chains = input_glob_file|parse_xml_file_filter|xml_assembler|output_std
+chains = parse_xml_file_input|xml_assembler|packet_buffer|output_std
+
+[input_glob_file]
+class = inputs.fileinput.GlobFileInput
+file_path = tests/data/dummy.gml
+
+# The source input file producing XML elements
+[parse_xml_file_filter]
+class = filters.xmlelementreader.XmlElementReader
+element_tags = FeatureMember
+
+[parse_xml_file_input]
+class = inputs.fileinput.XmlElementStreamerFileInput
+element_tags = FeatureMember
+file_path = tests/data/data_xmlassembler.gml
+
+# Assembles etree docs gml:featureMember elements, each with "max_elements" elements
+[xml_assembler]
+class = filters.xmlassembler.XmlAssembler
+max_elements = 2
+container_doc = <?xml version="1.0" encoding="UTF-8"?>
+ <gml:FeatureCollectionT10NL
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:top10nl="http://www.kadaster.nl/schemas/imbrt/top10nl/1.2"
+ xmlns:brt="http://www.kadaster.nl/schemas/imbrt/brt-alg/1.0"
+ xmlns:gml="http://www.opengis.net/gml/3.2"
+ xsi:schemaLocation="http://www.kadaster.nl/schemas/imbrt/top10nl/1.2 http://www.kadaster.nl/schemas/top10nl/vyyyymmdd/TOP10NL_1_2.xsd">
+ </gml:FeatureCollectionT10NL >
+element_container_tag = FeatureCollectionT10NL
+
+[packet_buffer]
+class = filters.packetbuffer.PacketBuffer
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/filters/configs/xmlelementreader.cfg b/tests/filters/configs/xmlelementreader.cfg
new file mode 100644
index 0000000..ca03b72
--- /dev/null
+++ b/tests/filters/configs/xmlelementreader.cfg
@@ -0,0 +1,33 @@
+# Config file for unit testing XmlElementReader.
+
+[etl]
+chains = input_glob_file|parse_xml_file|output_std,
+ input_glob_file|parse_xml_file_strip_namespace|output_std,
+ input_glob_file_no_namespace|parse_xml_file_no_namespace|output_std
+
+[input_glob_file]
+class = inputs.fileinput.GlobFileInput
+file_path = tests/data/dummy.gml
+
+# The source input file producing XML elements
+[parse_xml_file]
+class = filters.xmlelementreader.XmlElementReader
+element_tags = FeatureMember
+
+# Strip namespace from XML elements
+[parse_xml_file_strip_namespace]
+class = filters.xmlelementreader.XmlElementReader
+element_tags = FeatureMember
+strip_namespaces = True
+
+# XML file without namespace
+[input_glob_file_no_namespace]
+class = inputs.fileinput.GlobFileInput
+file_path = tests/data/cities.xml
+
+[parse_xml_file_no_namespace]
+class = filters.xmlelementreader.XmlElementReader
+element_tags = city
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/filters/configs/xsltfilter.cfg b/tests/filters/configs/xsltfilter.cfg
new file mode 100644
index 0000000..d6218fd
--- /dev/null
+++ b/tests/filters/configs/xsltfilter.cfg
@@ -0,0 +1,19 @@
+# Config file for unit testing XsltFilter.
+
+[etl]
+chains = parse_xml_file_input|transformer_xslt|packet_buffer|output_std
+
+[parse_xml_file_input]
+class = inputs.fileinput.XmlFileInput
+file_path = tests/data/dummy.gml
+
+# Transforms into simple/flat feature data (single geometry per feature type, single attrs)
+[transformer_xslt]
+class = filters.xsltfilter.XsltFilter
+script = tests/data/xslt/dummy.xslt
+
+[packet_buffer]
+class = filters.packetbuffer.PacketBuffer
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/filters/configs/zipfileextractor.cfg b/tests/filters/configs/zipfileextractor.cfg
new file mode 100644
index 0000000..38ebdc8
--- /dev/null
+++ b/tests/filters/configs/zipfileextractor.cfg
@@ -0,0 +1,17 @@
+# Config file for unit testing ZipFileExtractor.
+
+[etl]
+chains = input_zip_file|extract_zip_file|output_std
+
+[input_zip_file]
+class = inputs.fileinput.ZipFileInput
+file_path = tests/data/zipfileinput.zip
+name_filter = *.[gG][mM][lL]
+
+# Filter to extract a ZIP file one by one to a temporary location
+[extract_zip_file]
+class=filters.zipfileextractor.ZipFileExtractor
+file_path = tests/data/temp/tempfile.gml
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/filters/test_packet_writer.py b/tests/filters/test_packet_writer.py
new file mode 100644
index 0000000..e8872c6
--- /dev/null
+++ b/tests/filters/test_packet_writer.py
@@ -0,0 +1,49 @@
+import os
+
+from stetl.etl import ETL
+from stetl.filters.packetwriter import PacketWriter
+from tests.stetl_test_case import StetlTestCase
+
+class PacketWriterTest(StetlTestCase):
+ """Unit tests for PacketWriter"""
+
+ def setUp(self):
+ super(PacketWriterTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/packetwriter.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain, 1)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('filters.packetwriter.PacketWriter', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.get_by_index(1), PacketWriter))
+
+ def test_execute(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.run()
+
+ # Check if temp file exists
+ section = StetlTestCase.get_section(chain, 1)
+ file_path = self.etl.configdict.get(section, 'file_path')
+ self.assertTrue(os.path.exists(file_path))
+
+ # Compare temp file to input file byte by byte
+ section = StetlTestCase.get_section(chain)
+ orig_file_path = self.etl.configdict.get(section, 'file_path')
+ with open(file_path, 'rb') as f:
+ file = f.read()
+ with open(orig_file_path, 'rb') as f:
+ orig_file = f.read()
+ self.assertEqual(file, orig_file)
+
+ # Remove temp file
+ os.remove(file_path)
diff --git a/tests/filters/test_string_substitution_filter.py b/tests/filters/test_string_substitution_filter.py
new file mode 100644
index 0000000..cb8d626
--- /dev/null
+++ b/tests/filters/test_string_substitution_filter.py
@@ -0,0 +1,38 @@
+import os
+
+from stetl.etl import ETL
+from stetl.filters.packetbuffer import PacketBuffer
+from stetl.filters.stringfilter import StringSubstitutionFilter
+from tests.stetl_test_case import StetlTestCase
+
+class StringSubstitutionFilterTest(StetlTestCase):
+ """Unit tests for StringSubstitutionFilter"""
+
+ def setUp(self):
+ super(StringSubstitutionFilterTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/stringsubstitutionfilter.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain, 1)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('filters.stringfilter.StringSubstitutionFilter', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.get_by_index(1), StringSubstitutionFilter))
+
+ def test_execute(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.run()
+
+ buffer_filter = chain.get_by_class(PacketBuffer)
+ packet_list = buffer_filter.packet_list
+
+ self.assertEqual(packet_list[0].data, 'Hello NLExtract!')
diff --git a/tests/filters/test_xml_assembler.py b/tests/filters/test_xml_assembler.py
new file mode 100755
index 0000000..440b89f
--- /dev/null
+++ b/tests/filters/test_xml_assembler.py
@@ -0,0 +1,65 @@
+import os
+
+from stetl.etl import ETL
+from stetl.filters.xmlassembler import XmlAssembler
+from stetl.filters.packetbuffer import PacketBuffer
+from tests.stetl_test_case import StetlTestCase
+
+class XmlAssemblerTest(StetlTestCase):
+ """Unit tests for XmlAssembler"""
+
+ def setUp(self):
+ super(XmlAssemblerTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/xmlassembler.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain, 1)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('filters.xmlassembler.XmlAssembler', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.get_by_index(1), XmlAssembler))
+
+ def test_execute(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.run()
+
+ buffer_filter = chain.get_by_class(PacketBuffer)
+ packet_list = buffer_filter.packet_list
+
+ # most Packets are empty, but we need to find 2 filled with etree docs
+ doc_packet_list = []
+ for packet in packet_list:
+ if packet.data is not None:
+ doc_packet_list.append(packet)
+
+ # Assertion: we need to see 2 documents
+ self.assertEqual(len(doc_packet_list), 4)
+ namespaces={'gml': 'http://www.opengis.net/gml/3.2', 'top10nl': 'http://register.geostandaarden.nl/gmlapplicatieschema/top10nl/1.2.0'}
+
+ # Assertion: first doc has two FeatureMember elements with proper Namespaces
+ xml_doc1 = doc_packet_list[0].data
+ feature_elms = xml_doc1.xpath('/gml:FeatureCollectionT10NL/top10nl:FeatureMember', namespaces=namespaces)
+ self.assertEqual(len(feature_elms), 2)
+
+ # Assertion: last doc has one FeatureMember with proper Namespaces
+ last = len(doc_packet_list) - 1
+ xml_doc2 = doc_packet_list[last].data
+ feature_elms = xml_doc2.xpath('/gml:FeatureCollectionT10NL/top10nl:FeatureMember', namespaces=namespaces)
+ self.assertEqual(len(feature_elms), 1)
+
+ # Assertion: first doc has end_of_doc but not end_of_stream set
+ self.assertTrue(doc_packet_list[0].end_of_doc, msg='doc1: end_of_doc if False')
+ self.assertFalse(doc_packet_list[0].end_of_stream, msg='doc1: end_of_stream is True')
+
+ # Assertion: last doc has end_of_doc and end_of_stream set
+ self.assertTrue(doc_packet_list[last].end_of_doc, msg='doc2: end_of_doc if False')
+ self.assertTrue(doc_packet_list[last].end_of_stream, msg='doc2: end_of_stream if False')
diff --git a/tests/filters/test_xml_element_reader.py b/tests/filters/test_xml_element_reader.py
new file mode 100644
index 0000000..7e7c0f1
--- /dev/null
+++ b/tests/filters/test_xml_element_reader.py
@@ -0,0 +1,70 @@
+import os
+import re
+import sys
+
+from stetl.etl import ETL
+from stetl.filters.xmlelementreader import XmlElementReader
+from tests.stetl_test_case import StetlTestCase
+
+class XmlElementReaderTest(StetlTestCase):
+ """Unit tests for XmlElementReader"""
+
+ def setUp(self):
+ super(XmlElementReaderTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/xmlelementreader.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain, 1)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('filters.xmlelementreader.XmlElementReader', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.first_comp.next, XmlElementReader))
+
+ def check_execution_result(self, chain, result, with_namespace):
+ section = StetlTestCase.get_section(chain, 1)
+ element_tags = self.etl.configdict.get(section, 'element_tags')
+ pattern = "<Element %s%s " % ('{[^}]+}' if with_namespace else '', element_tags)
+ for elem in result:
+ self.assertIsNotNone(re.match(pattern, elem))
+
+ def test_execute(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.run()
+
+ # Check the number of elements
+ result = sys.stdout.getvalue().strip().split('\n')
+ self.assertEqual(len(result), 3)
+
+ # Check the actual elements
+ self.check_execution_result(chain, result, True)
+
+ def test_strip_namespaces(self):
+ chain = StetlTestCase.get_chain(self.etl, 1)
+ chain.run()
+
+ # Check the number of elements
+ result = sys.stdout.getvalue().strip().split('\n')
+ self.assertEqual(len(result), 3)
+
+ # Check the actual elements
+ self.check_execution_result(chain, result, False)
+
+ def test_no_namespace(self):
+ chain = StetlTestCase.get_chain(self.etl, 2)
+ chain.run()
+
+ # Check the number of elements
+ result = sys.stdout.getvalue().strip().split('\n')
+ self.assertEqual(len(result), 3)
+
+ # Check the actual elements
+ self.check_execution_result(chain, result, False)
diff --git a/tests/filters/test_xslt_filter.py b/tests/filters/test_xslt_filter.py
new file mode 100644
index 0000000..4bdefd1
--- /dev/null
+++ b/tests/filters/test_xslt_filter.py
@@ -0,0 +1,50 @@
+import os
+import re
+import sys
+
+from stetl.etl import ETL
+from stetl.filters.xsltfilter import XsltFilter
+from stetl.filters.packetbuffer import PacketBuffer
+from tests.stetl_test_case import StetlTestCase
+
+class XsltFilterTest(StetlTestCase):
+ """Unit tests for XsltFilter"""
+
+ def setUp(self):
+ super(XsltFilterTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/xsltfilter.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain, 1)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('filters.xsltfilter.XsltFilter', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.get_by_index(1), XsltFilter))
+
+ def test_execute(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.run()
+
+ buffer_filter = chain.get_by_class(PacketBuffer)
+ packet_list = buffer_filter.packet_list
+ self.assertEqual(len(packet_list), 1)
+
+ # Inspect the result
+ result = sys.stdout.getvalue().strip().split('\n')
+ self.assertEqual(len(result), 6)
+
+ pattern = r'<funcgeb id="[^"]+"/>'
+ for i in range(2, 5):
+ self.assertIsNotNone(re.match(pattern, result[i].strip()))
+ self.assertTrue(result[1].strip().startswith('<dummy '))
+ self.assertEqual(result[5].strip(), '</dummy>')
+
\ No newline at end of file
diff --git a/tests/filters/test_zip_file_extractor.py b/tests/filters/test_zip_file_extractor.py
new file mode 100644
index 0000000..9635db4
--- /dev/null
+++ b/tests/filters/test_zip_file_extractor.py
@@ -0,0 +1,45 @@
+import mock
+import os
+
+from stetl.etl import ETL
+from stetl.filters.zipfileextractor import ZipFileExtractor
+from tests.stetl_test_case import StetlTestCase
+
+class ZipFileExtractorTest(StetlTestCase):
+ """Unit tests for ZipFileExtractor"""
+
+ def setUp(self):
+ super(ZipFileExtractorTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/zipfileextractor.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain, 1)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('filters.zipfileextractor.ZipFileExtractor', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.first_comp.next, ZipFileExtractor))
+
+ @mock.patch('stetl.filters.zipfileextractor.ZipFileExtractor.after_chain_invoke', autospec=True)
+ def test_execute(self, mock_after_chain_invoke):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.run()
+
+ # ZIP file contains two GML files, both should be extracted; count is 3 because of final
+ # call, so the ZipFileExtractor can indicate that no more files can be found.
+ self.assertTrue(mock_after_chain_invoke.called)
+ self.assertEqual(3, mock_after_chain_invoke.call_count)
+
+ # Check if temp file exists
+ section = StetlTestCase.get_section(chain, 1)
+ file_path = self.etl.configdict.get(section, 'file_path')
+ self.assertTrue(os.path.exists(file_path))
+ os.remove(file_path)
diff --git a/tests/inputs/__init__.py b/tests/inputs/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/inputs/configs/csvfileinput.cfg b/tests/inputs/configs/csvfileinput.cfg
new file mode 100644
index 0000000..a1f33bc
--- /dev/null
+++ b/tests/inputs/configs/csvfileinput.cfg
@@ -0,0 +1,14 @@
+# Trivial example of an ETL Chain that just copies a file to standard output.
+
+[etl]
+chains = input_csv_file|output_std
+
+[input_csv_file]
+class = inputs.fileinput.CsvFileInput
+file_path = tests/data/pandhoogtes.csv
+output_format = record
+delimiter = ,
+quote_char = "
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/inputs/configs/globfileinput.cfg b/tests/inputs/configs/globfileinput.cfg
new file mode 100644
index 0000000..e97b4b1
--- /dev/null
+++ b/tests/inputs/configs/globfileinput.cfg
@@ -0,0 +1,16 @@
+# Trivial example of an ETL Chain that just copies the names of files matching a glob pattern to standard output.
+
+[etl]
+chains = input_glob_file|output_std,
+ input_glob_file_wildcard|output_std
+
+[input_glob_file]
+class = inputs.fileinput.GlobFileInput
+file_path = tests/data/dummy.gml
+
+[input_glob_file_wildcard]
+class = inputs.fileinput.GlobFileInput
+file_path = tests/data/dummy*.gml
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/inputs/configs/jsonfileinput.cfg b/tests/inputs/configs/jsonfileinput.cfg
new file mode 100644
index 0000000..83c546d
--- /dev/null
+++ b/tests/inputs/configs/jsonfileinput.cfg
@@ -0,0 +1,12 @@
+# Trivial example of an ETL Chain that just copies a file to standard output.
+
+[etl]
+chains = input_csv_file|output_std
+
+[input_csv_file]
+class = inputs.fileinput.JsonFileInput
+file_path = tests/data/menu.json
+output_format = struct
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/inputs/configs/linestreamerfileinput.cfg b/tests/inputs/configs/linestreamerfileinput.cfg
new file mode 100644
index 0000000..412eeac
--- /dev/null
+++ b/tests/inputs/configs/linestreamerfileinput.cfg
@@ -0,0 +1,11 @@
+# Trivial example of an ETL Chain that just copies a file to standard output.
+
+[etl]
+chains = input_text_file|output_std
+
+[input_text_file]
+class = inputs.fileinput.LineStreamerFileInput
+file_path = tests/data/cities.xml
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/inputs/configs/mergermultiinput.cfg b/tests/inputs/configs/mergermultiinput.cfg
new file mode 100644
index 0000000..9087ab3
--- /dev/null
+++ b/tests/inputs/configs/mergermultiinput.cfg
@@ -0,0 +1,19 @@
+# ETL Chain that merges multiple input files to single standard output.
+
+[etl]
+chains = (input_file_1) (input_file_2)|output_std,
+ (input_file_1 | filter_null) (input_file_2 | filter_null)|output_std
+
+[input_file_1]
+class = inputs.fileinput.LineStreamerFileInput
+file_path = tests/data/cities.xml
+
+[input_file_2]
+class = inputs.fileinput.LineStreamerFileInput
+file_path = tests/data/pandhoogtes-klein.csv
+
+[filter_null]
+class = filters.nullfilter.NullFilter
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/inputs/configs/ogrinput.cfg b/tests/inputs/configs/ogrinput.cfg
new file mode 100644
index 0000000..61c728d
--- /dev/null
+++ b/tests/inputs/configs/ogrinput.cfg
@@ -0,0 +1,16 @@
+# Trivial example of an ETL Chain that just copies a file to standard output.
+
+[etl]
+chains = input_ogr|output_std
+
+[input_ogr]
+class = inputs.ogrinput.OgrInput
+data_source = tests/data/cities.gml
+source_format = GML
+source_options = {'GDAL_CACHEMAX': '64', 'CPL_DEBUG': 'OFF'}
+output_format = ogr_feature
+file_path = tests/data/cities.xml
+element_tags = city
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/inputs/configs/stringfileinput.cfg b/tests/inputs/configs/stringfileinput.cfg
new file mode 100644
index 0000000..ea3e60a
--- /dev/null
+++ b/tests/inputs/configs/stringfileinput.cfg
@@ -0,0 +1,17 @@
+# Trivial example of an ETL Chain that just copies a file to standard output.
+
+[etl]
+chains = input_string_file|output_std,
+ input_string_file_format_args|output_std
+
+[input_string_file]
+class = inputs.fileinput.StringFileInput
+file_path = tests/data/cities.xml
+
+[input_string_file_format_args]
+class = inputs.fileinput.StringFileInput
+file_path = tests/data/stringfileinput_formatargs.txt
+format_args = greeting:Hello name:NLExtract
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/inputs/configs/xmlelementstreamerfileinput.cfg b/tests/inputs/configs/xmlelementstreamerfileinput.cfg
new file mode 100644
index 0000000..8c5a830
--- /dev/null
+++ b/tests/inputs/configs/xmlelementstreamerfileinput.cfg
@@ -0,0 +1,12 @@
+# Trivial example of an ETL Chain that just copies a file to standard output.
+
+[etl]
+chains = input_xml_file|output_std
+
+[input_xml_file]
+class = inputs.fileinput.XmlElementStreamerFileInput
+file_path = tests/data/cities.xml
+element_tags = city
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/inputs/configs/zipfileinput.cfg b/tests/inputs/configs/zipfileinput.cfg
new file mode 100644
index 0000000..980d903
--- /dev/null
+++ b/tests/inputs/configs/zipfileinput.cfg
@@ -0,0 +1,17 @@
+# Trivial example of an ETL Chain that just copies the names of files within a zip file to standard output.
+
+[etl]
+chains = input_zip_file|output_std,
+ input_zip_file_name_filter|output_std
+
+[input_zip_file]
+class = inputs.fileinput.ZipFileInput
+file_path = tests/data/zipfileinput.zip
+
+[input_zip_file_name_filter]
+class = inputs.fileinput.ZipFileInput
+file_path = tests/data/zipfileinput.zip
+name_filter = *.[gG][mM][lL]
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/inputs/test_csv_file_input.py b/tests/inputs/test_csv_file_input.py
new file mode 100644
index 0000000..1bb0f07
--- /dev/null
+++ b/tests/inputs/test_csv_file_input.py
@@ -0,0 +1,39 @@
+import os
+import sys
+
+from stetl.etl import ETL
+from stetl.packet import Packet
+from stetl.inputs.fileinput import CsvFileInput
+from tests.stetl_test_case import StetlTestCase
+
+class CsvFileInputTest(StetlTestCase):
+ """Unit tests for CsvFileInput"""
+ pass
+
+ def setUp(self):
+ super(CsvFileInputTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/csvfileinput.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('inputs.fileinput.CsvFileInput', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.first_comp, CsvFileInput))
+
+ def test_execute(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.run()
+
+ result = sys.stdout.getvalue().strip().split('\n')
+
+ self.assertEqual(len(result), 431)
diff --git a/tests/inputs/test_glob_file_input.py b/tests/inputs/test_glob_file_input.py
new file mode 100644
index 0000000..b8a9ded
--- /dev/null
+++ b/tests/inputs/test_glob_file_input.py
@@ -0,0 +1,44 @@
+import os
+import sys
+
+from stetl.etl import ETL
+from stetl.packet import Packet
+from stetl.inputs.fileinput import GlobFileInput
+from tests.stetl_test_case import StetlTestCase
+
+class GlobFileInputTest(StetlTestCase):
+ """Unit tests for GlobFileInput"""
+
+ def setUp(self):
+ super(GlobFileInputTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/globfileinput.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('inputs.fileinput.GlobFileInput', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.first_comp, GlobFileInput))
+
+ def test_execute(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.run()
+
+ result = sys.stdout.getvalue().strip().split('\n')
+ self.assertEqual(len(result), 1)
+
+ def test_wildcard(self):
+ chain = StetlTestCase.get_chain(self.etl, 1)
+ chain.run()
+
+ result = sys.stdout.getvalue().strip().split('\n')
+ self.assertEqual(len(result), 2)
diff --git a/tests/inputs/test_json_file_input.py b/tests/inputs/test_json_file_input.py
new file mode 100644
index 0000000..f53df44
--- /dev/null
+++ b/tests/inputs/test_json_file_input.py
@@ -0,0 +1,51 @@
+import os
+import sys
+
+from stetl.etl import ETL
+from stetl.packet import Packet
+from stetl.inputs.fileinput import JsonFileInput
+from tests.stetl_test_case import StetlTestCase
+
+class JsonFileInputTest(StetlTestCase):
+ """Unit tests for JsonFileInput"""
+ pass
+
+ def setUp(self):
+ super(JsonFileInputTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/jsonfileinput.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('inputs.fileinput.JsonFileInput', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.first_comp, JsonFileInput))
+
+ def test_execute(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.first_comp.do_init()
+ packet = Packet()
+ packet.init()
+ packet.component = chain.first_comp
+ chain.first_comp.before_invoke(packet)
+ packet = chain.first_comp.invoke(packet)
+
+ self.assertIsNotNone(packet.data)
+ self.assertIsInstance(packet.data, dict)
+ self.assertTrue('menu' in packet.data)
+ self.assertIsNotNone(packet.data['menu'])
+
+ mydict = packet.data['menu']
+ self.assertEqual(len(mydict), 3)
+ self.assertTrue('id' in mydict)
+ self.assertTrue('value' in mydict)
+ self.assertTrue('popup' in mydict)
diff --git a/tests/inputs/test_line_streamer_file_input.py b/tests/inputs/test_line_streamer_file_input.py
new file mode 100644
index 0000000..b725391
--- /dev/null
+++ b/tests/inputs/test_line_streamer_file_input.py
@@ -0,0 +1,43 @@
+import os
+import sys
+
+from stetl.etl import ETL
+from stetl.packet import Packet
+from stetl.inputs.fileinput import LineStreamerFileInput
+from tests.stetl_test_case import StetlTestCase
+
+class LineStreamerFileInputTest(StetlTestCase):
+ """Unit tests for LineStreamerFileInput"""
+ pass
+
+ def setUp(self):
+ super(LineStreamerFileInputTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/linestreamerfileinput.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('inputs.fileinput.LineStreamerFileInput', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.first_comp, LineStreamerFileInput))
+
+ def test_execute(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.run()
+
+ result = sys.stdout.getvalue().split('\n')
+
+ # The total number of lines written to stdout is twice the number of lines in the text file,
+ # because the print statement is used. This causes an extra linebreak to written. The number
+ # to assert, is even one higher, because of the split statement above. The last "line" is an
+ # empty string.
+ self.assertEqual(len(result), 37)
diff --git a/tests/inputs/test_merger_multi_input.py b/tests/inputs/test_merger_multi_input.py
new file mode 100644
index 0000000..bde0696
--- /dev/null
+++ b/tests/inputs/test_merger_multi_input.py
@@ -0,0 +1,90 @@
+import os
+import sys
+
+from stetl.etl import ETL
+from stetl.inputs.fileinput import LineStreamerFileInput
+from stetl.filters.nullfilter import NullFilter
+from stetl.outputs.standardoutput import StandardOutput
+from stetl.merger import Merger
+from tests.stetl_test_case import StetlTestCase
+
+
+class MergerMultiInputTest(StetlTestCase):
+ """Unit tests for Merger"""
+ pass
+
+ def setUp(self):
+ super(MergerMultiInputTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/mergermultiinput.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_instance(self):
+ # Chain #1 - Simple Case
+ chain = StetlTestCase.get_chain(self.etl)
+
+ merger_comp = chain.first_comp
+ self.assertTrue(isinstance(merger_comp, Merger))
+ self.assertEqual(len(merger_comp.children), 2)
+ self.assertTrue(isinstance(merger_comp.children[0][0], LineStreamerFileInput),
+ "Next is not LineStreamerFileInput")
+ self.assertTrue(isinstance(merger_comp.children[1][0], LineStreamerFileInput),
+ "Next is not LineStreamerFileInput")
+ self.assertTrue(isinstance(merger_comp.children[0][1], LineStreamerFileInput),
+ "Next is not LineStreamerFileInput")
+ self.assertTrue(isinstance(merger_comp.children[1][1], LineStreamerFileInput),
+ "Next is not LineStreamerFileInput")
+ self.assertTrue(isinstance(merger_comp.children[0][1].next, StandardOutput),
+ "Next is not StandardOutput")
+ self.assertTrue(isinstance(merger_comp.children[1][1].next, StandardOutput),
+ "Next is not StandardOutput")
+
+ # Flag for End-of-Stream 2 subcomps
+ self.assertEqual(merger_comp.end_count, 2)
+
+ # Chain #2 - SubChain Case
+ chain = StetlTestCase.get_chain(self.etl, index=1)
+
+ merger_comp = chain.first_comp
+ children = merger_comp.children
+ self.assertTrue(isinstance(merger_comp, Merger))
+ self.assertEqual(len(merger_comp.children), 2, "Child count is not 2")
+ self.assertTrue(isinstance(merger_comp.first(children[0]), LineStreamerFileInput),
+ "Next is not LineStreamerFileInput")
+ self.assertTrue(isinstance(merger_comp.first(children[1]), LineStreamerFileInput),
+ "Next is not LineStreamerFileInput")
+ self.assertTrue(isinstance(merger_comp.children[0][0].next, NullFilter),
+ "Next is not NullFilter")
+ self.assertTrue(isinstance(merger_comp.first(children[1]).next, NullFilter),
+ "Next is not NullFilter")
+ self.assertTrue(isinstance(merger_comp.last(children[1]).next, StandardOutput),
+ "Next is not StandardOutput")
+ self.assertTrue(isinstance(merger_comp.children[1][1].next, StandardOutput),
+ "Next is not StandardOutput")
+
+ # Flag for End-of-Stream 2 subcomps
+ self.assertEqual(merger_comp.end_count, 2)
+
+ def test_execute(self):
+ run_count = 0
+ for index in [0, 1]:
+ chain = StetlTestCase.get_chain(self.etl, index=index)
+ chain.run()
+
+ merger_comp = chain.first_comp
+
+ # Flag for End-of-Stream 2 subcomps
+ self.assertEqual(merger_comp.end_count, 0)
+
+ # Result should be merged lines from both files
+ result = sys.stdout.getvalue().split('\n')
+
+ # Strip empty lines
+ result = [s for s in result if (s or len(s) > 0)]
+
+ # Total should be sum of linecount non-empty lines in input files
+ # each run increases line count hence multiply by run_count
+ run_count += 1
+ self.assertEqual(len(result), 28 * run_count)
diff --git a/tests/inputs/test_ogr_input.py b/tests/inputs/test_ogr_input.py
new file mode 100644
index 0000000..535e2c1
--- /dev/null
+++ b/tests/inputs/test_ogr_input.py
@@ -0,0 +1,37 @@
+import os
+import sys
+
+from stetl.etl import ETL
+from stetl.inputs.ogrinput import OgrInput
+from tests.stetl_test_case import StetlTestCase
+
+class OgrInputTest(StetlTestCase):
+ """Unit tests for OgrInput"""
+ pass
+
+ def setUp(self):
+ super(OgrInputTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/ogrinput.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('inputs.ogrinput.OgrInput', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.first_comp, OgrInput))
+
+ def test_execute(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.run()
+
+ result = sys.stdout.getvalue().strip().split('\n')
+ self.assertEqual(len(result), 3)
diff --git a/tests/inputs/test_string_file_input.py b/tests/inputs/test_string_file_input.py
new file mode 100644
index 0000000..d1cf939
--- /dev/null
+++ b/tests/inputs/test_string_file_input.py
@@ -0,0 +1,59 @@
+import os
+import sys
+
+from stetl.etl import ETL
+from stetl.packet import Packet
+from stetl.inputs.fileinput import StringFileInput
+from tests.stetl_test_case import StetlTestCase
+
+class StringFileInputTest(StetlTestCase):
+ """Unit tests for StringFileInput"""
+
+ def setUp(self):
+ super(StringFileInputTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/stringfileinput.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('inputs.fileinput.StringFileInput', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.first_comp, StringFileInput))
+
+ def test_execute(self):
+ # Read content of input file
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain)
+ fn = self.etl.configdict.get(section, 'file_path')
+ with open(fn, 'r') as f:
+ contents = f.read()
+
+ # Invoke first component of chain
+ chain.first_comp.do_init()
+ packet = Packet()
+ packet.init()
+ packet.component = chain.first_comp
+ chain.first_comp.before_invoke(packet)
+ packet = chain.first_comp.invoke(packet)
+
+ self.assertEqual(packet.data, contents)
+
+ def test_format_args(self):
+ chain = StetlTestCase.get_chain(self.etl, 1)
+ chain.first_comp.do_init()
+ packet = Packet()
+ packet.init()
+ packet.component = chain.first_comp
+ chain.first_comp.before_invoke(packet)
+ packet = chain.first_comp.invoke(packet)
+
+ self.assertEqual(packet.data, 'Hello NLExtract!')
diff --git a/tests/inputs/test_xml_element_streamer_file_input.py b/tests/inputs/test_xml_element_streamer_file_input.py
new file mode 100644
index 0000000..9552ff4
--- /dev/null
+++ b/tests/inputs/test_xml_element_streamer_file_input.py
@@ -0,0 +1,38 @@
+import os
+import sys
+
+from stetl.etl import ETL
+from stetl.packet import Packet
+from stetl.inputs.fileinput import XmlElementStreamerFileInput
+from tests.stetl_test_case import StetlTestCase
+
+class XmlElementStreamerFileInputTest(StetlTestCase):
+ """Unit tests for XmlElementStreamerFileInput"""
+ pass
+
+ def setUp(self):
+ super(XmlElementStreamerFileInputTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/xmlelementstreamerfileinput.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('inputs.fileinput.XmlElementStreamerFileInput', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.first_comp, XmlElementStreamerFileInput))
+
+ def test_execute(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.run()
+
+ result = sys.stdout.getvalue().strip().split('\n')
+ self.assertEqual(len(result), 3)
diff --git a/tests/inputs/test_zip_file_input.py b/tests/inputs/test_zip_file_input.py
new file mode 100644
index 0000000..ff07972
--- /dev/null
+++ b/tests/inputs/test_zip_file_input.py
@@ -0,0 +1,44 @@
+import os
+import sys
+
+from stetl.etl import ETL
+from stetl.packet import Packet
+from stetl.inputs.fileinput import ZipFileInput
+from tests.stetl_test_case import StetlTestCase
+
+class ZipFileInputTest(StetlTestCase):
+ """Unit tests for ZipFileInput"""
+
+ def setUp(self):
+ super(ZipFileInputTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/zipfileinput.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('inputs.fileinput.ZipFileInput', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.first_comp, ZipFileInput))
+
+ def test_execute(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.run()
+
+ result = sys.stdout.getvalue().strip().split('\n')
+ self.assertEqual(len(result), 6)
+
+ def test_name_filter(self):
+ chain = StetlTestCase.get_chain(self.etl, 1)
+ chain.run()
+
+ result = sys.stdout.getvalue().strip().split('\n')
+ self.assertEqual(len(result), 2)
diff --git a/tests/outputs/__init__.py b/tests/outputs/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/outputs/configs/commandexecoutput.cfg b/tests/outputs/configs/commandexecoutput.cfg
new file mode 100644
index 0000000..a07b7ca
--- /dev/null
+++ b/tests/outputs/configs/commandexecoutput.cfg
@@ -0,0 +1,12 @@
+# Trivial example of an ETL Chain that just executes a command.
+
+[etl]
+chains = input_string_file|output_exec
+
+[input_string_file]
+class = inputs.fileinput.StringFileInput
+file_path = tests/data/command.txt
+
+[output_exec]
+class = outputs.execoutput.CommandExecOutput
+input_format = string
diff --git a/tests/outputs/configs/ogr2ogrexecoutput.cfg b/tests/outputs/configs/ogr2ogrexecoutput.cfg
new file mode 100644
index 0000000..e60b059
--- /dev/null
+++ b/tests/outputs/configs/ogr2ogrexecoutput.cfg
@@ -0,0 +1,81 @@
+# Trivial example of an ETL Chain that just executes a command.
+
+[etl]
+chains = input_glob_file|output_ogr2ogr,
+ input_glob_file|output_ogr2ogr_lco,
+ input_glob_file|output_ogr2ogr_spat,
+ input_glob_file|output_ogr2ogr_options,
+ input_glob_file|output_ogr2ogr_gfs,
+ input_glob_file_temp|output_ogr2ogr_cleanup,
+ input_glob_file_temp|output_ogr2ogr_cleanup_gfs,
+ input_glob_file_keep|output_ogr2ogr_no_cleanup
+
+[input_glob_file]
+class = inputs.fileinput.GlobFileInput
+file_path = tests/data/dummy.gml
+
+[input_glob_file_temp]
+class = inputs.fileinput.GlobFileInput
+file_path = tests/data/dummy_temp.gml
+
+[input_glob_file_keep]
+class = inputs.fileinput.GlobFileInput
+file_path = tests/data/keep_dummy.gml
+
+[output_ogr2ogr]
+class = outputs.execoutput.Ogr2OgrExecOutput
+# destination format: OGR vector format name
+dest_format = PostgreSQL
+# destination datasource: name of datasource
+dest_data_source = "PG:dbname=dummydb host=dummyhost port=dummyport user=dummyuser password=dummypassword active_schema=dummyschema"
+
+[output_ogr2ogr_lco]
+class = outputs.execoutput.Ogr2OgrExecOutput
+dest_format = PostgreSQL
+dest_data_source = "PG:dbname=dummydb host=dummyhost port=dummyport user=dummyuser password=dummypassword active_schema=dummyschema"
+# layer creation options will only be added to ogr2ogr on first run
+lco = -lco LAUNDER=YES -lco PRECISION=NO
+
+[output_ogr2ogr_spat]
+class = outputs.execoutput.Ogr2OgrExecOutput
+dest_format = PostgreSQL
+dest_data_source = "PG:dbname=dummydb host=dummyhost port=dummyport user=dummyuser password=dummypassword active_schema=dummyschema"
+# spatial_extent, translates to -spat xmin ymin xmax ymax
+spatial_extent = 140000 450000 150000 460000
+
+[output_ogr2ogr_options]
+class = outputs.execoutput.Ogr2OgrExecOutput
+dest_format = PostgreSQL
+dest_data_source = "PG:dbname=dummydb host=dummyhost port=dummyport user=dummyuser password=dummypassword active_schema=dummyschema"
+# miscellaneous ogr2ogr options
+options = -append -gt 65536 {multi_opts} --config PG_USE_COPY NO
+
+[output_ogr2ogr_gfs]
+class = outputs.execoutput.Ogr2OgrExecOutput
+dest_format = PostgreSQL
+dest_data_source = "PG:dbname=dummydb host=dummyhost port=dummyport user=dummyuser password=dummypassword active_schema=dummyschema"
+# gfs template
+gfs_template = tests/data/gfs/top10-funcgebied.gfs
+
+[output_ogr2ogr_cleanup]
+class = outputs.execoutput.Ogr2OgrExecOutput
+dest_format = PostgreSQL
+dest_data_source = "PG:dbname=dummydb host=dummyhost port=dummyport user=dummyuser password=dummypassword active_schema=dummyschema"
+cleanup_input = True
+
+[output_ogr2ogr_cleanup_gfs]
+class = outputs.execoutput.Ogr2OgrExecOutput
+dest_format = PostgreSQL
+dest_data_source = "PG:dbname=dummydb host=dummyhost port=dummyport user=dummyuser password=dummypassword active_schema=dummyschema"
+# gfs template
+gfs_template = tests/data/gfs/top10-funcgebied.gfs
+cleanup_input = True
+
+[output_ogr2ogr_no_cleanup]
+class = outputs.execoutput.Ogr2OgrExecOutput
+# destination format: OGR vector format name
+dest_format = PostgreSQL
+# destination datasource: name of datasource
+dest_data_source = "PG:dbname=dummydb host=dummyhost port=dummyport user=dummyuser password=dummypassword active_schema=dummyschema"
+# cleanup input?
+cleanup_input = False
diff --git a/tests/outputs/configs/postgresdboutput.cfg b/tests/outputs/configs/postgresdboutput.cfg
new file mode 100644
index 0000000..18c8705
--- /dev/null
+++ b/tests/outputs/configs/postgresdboutput.cfg
@@ -0,0 +1,17 @@
+# Trivial example of an ETL Chain that just executes an SQL script in a Postgres database.
+
+[etl]
+chains = input_string_file|output_postgres
+
+[input_string_file]
+class = inputs.fileinput.StringFileInput
+file_path = tests/data/dummy.sql
+
+[output_postgres]
+class = outputs.dboutput.PostgresDbOutput
+database = {database}
+host = {host}
+port = {port}
+user = {user}
+password = {password}
+schema = {schema}
diff --git a/tests/outputs/configs/splittermultioutput.cfg b/tests/outputs/configs/splittermultioutput.cfg
new file mode 100644
index 0000000..253b459
--- /dev/null
+++ b/tests/outputs/configs/splittermultioutput.cfg
@@ -0,0 +1,14 @@
+# ETL Chain that splits a file to multiple outputs.
+
+[etl]
+chains = input_file | (output_std_1) (output_std_2)
+
+[input_file]
+class = inputs.fileinput.LineStreamerFileInput
+file_path = tests/data/cities.xml
+
+[output_std_1]
+class = outputs.standardoutput.StandardOutput
+
+[output_std_2]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/outputs/configs/standardoutput.cfg b/tests/outputs/configs/standardoutput.cfg
new file mode 100644
index 0000000..4edc9c0
--- /dev/null
+++ b/tests/outputs/configs/standardoutput.cfg
@@ -0,0 +1,11 @@
+# Trivial example of an ETL Chain that just copies a file to standard output.
+
+[etl]
+chains = input_string_file|output_std
+
+[input_string_file]
+class = inputs.fileinput.StringFileInput
+file_path = tests/data/cities.xml
+
+[output_std]
+class = outputs.standardoutput.StandardOutput
diff --git a/tests/outputs/test_command_exec_output.py b/tests/outputs/test_command_exec_output.py
new file mode 100644
index 0000000..8a555b2
--- /dev/null
+++ b/tests/outputs/test_command_exec_output.py
@@ -0,0 +1,47 @@
+import mock
+import os
+import sys
+
+from stetl.etl import ETL
+from stetl.outputs.execoutput import CommandExecOutput
+from tests.stetl_test_case import StetlTestCase
+
+class CommandExecOutputTest(StetlTestCase):
+ """Unit tests for CommandExecOutput"""
+
+ def setUp(self):
+ super(CommandExecOutputTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/commandexecoutput.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain, -1)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('outputs.execoutput.CommandExecOutput', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.cur_comp, CommandExecOutput))
+
+ @mock.patch('subprocess.call', autospec=True)
+ def test_execute(self, mock_call):
+ # Read content of input file
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain)
+ fn = self.etl.configdict.get(section, 'file_path')
+ with open(fn, 'r') as f:
+ contents = f.read()
+
+ self.etl.run()
+
+ self.assertTrue(mock_call.called)
+ self.assertEqual(1, mock_call.call_count)
+ args, kwargs = mock_call.call_args
+ self.assertEqual(contents, args[0])
+
\ No newline at end of file
diff --git a/tests/outputs/test_ogr2ogr_exec_output.py b/tests/outputs/test_ogr2ogr_exec_output.py
new file mode 100644
index 0000000..2e1c105
--- /dev/null
+++ b/tests/outputs/test_ogr2ogr_exec_output.py
@@ -0,0 +1,201 @@
+import mock
+import os
+import shutil
+import sys
+
+from stetl.etl import ETL
+from stetl.outputs.execoutput import Ogr2OgrExecOutput
+from tests.stetl_test_case import StetlTestCase
+
+class Ogr2OgrExecOutputTest(StetlTestCase):
+ """Unit tests for Ogr2OgrExecOutput"""
+
+ def setUp(self):
+ super(Ogr2OgrExecOutputTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/ogr2ogrexecoutput.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain, -1)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('outputs.execoutput.Ogr2OgrExecOutput', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.cur_comp, Ogr2OgrExecOutput))
+
+ def parse_command(self, command):
+ """
+ Parses the command line. With this regex the string is split at whitespace, except when
+ whitespace is occurring between quotes. Because split keeps the capturing groups this way,
+ every second item is removed from the list.
+ """
+
+ import re
+ list = re.split(r"\s+(?=([^\"]*\"[^\"]*\")*[^\"]*$)", command)
+ return list[0::2]
+
+ @mock.patch('subprocess.call', autospec=True)
+ def test_execute(self, mock_call):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.run()
+
+ self.assertTrue(mock_call.called)
+ self.assertEqual(1, mock_call.call_count)
+
+ # Check command line
+ args, kwargs = mock_call.call_args
+ list = self.parse_command(args[0])
+ self.assertEqual(len(list), 5)
+ self.assertEqual(list[0], 'ogr2ogr')
+
+ # Compare command line arguments with config
+ section = StetlTestCase.get_section(chain, -1)
+ file_path = self.etl.configdict.get(StetlTestCase.get_section(chain), 'file_path')
+
+ # Certain options should not occur
+ self.assertFalse('-spat' in list)
+ self.assertFalse('-lco' in list)
+
+ # Destination format
+ self.assertTrue('-f' in list)
+ f_idx = list.index('-f')
+ dest_format = self.etl.configdict.get(section, 'dest_format')
+ self.assertEqual(list[f_idx + 1], dest_format)
+
+ # Destination datasource
+ dest_data_source = self.etl.configdict.get(section, 'dest_data_source')
+ self.assertEqual(list[f_idx + 2], dest_data_source)
+
+ # Source datasource
+ self.assertEqual(list[-1], file_path)
+
+ @mock.patch('subprocess.call', autospec=True)
+ def test_execute_lco(self, mock_call):
+ chain = StetlTestCase.get_chain(self.etl, 1)
+ chain.run()
+
+ # Check command line
+ args, kwargs = mock_call.call_args
+ list = self.parse_command(args[0])
+ self.assertEqual(len(list), 9)
+
+ # Check layer creation options
+ self.assertTrue('-lco' in list)
+ lco_indices = [i for i, x in enumerate(list) if x == '-lco']
+ self.assertEqual(len(lco_indices), 2)
+ self.assertEqual(list[lco_indices[0] + 1], 'LAUNDER=YES')
+ self.assertEqual(list[lco_indices[1] + 1], 'PRECISION=NO')
+
+ @mock.patch('subprocess.call', autospec=True)
+ def test_execute_extent(self, mock_call):
+ chain = StetlTestCase.get_chain(self.etl, 2)
+ chain.run()
+
+ # Check command line
+ args, kwargs = mock_call.call_args
+ list = self.parse_command(args[0])
+ self.assertEqual(len(list), 10)
+
+ # Check spatial extent
+ section = StetlTestCase.get_section(chain, -1)
+ self.assertTrue('-spat' in list)
+ spat_idx = list.index('-spat')
+ spatial_extent = self.etl.configdict.get(section, 'spatial_extent')
+ self.assertEqual(spatial_extent.split(), list[spat_idx + 1:spat_idx + 5])
+
+ @mock.patch('subprocess.call', autospec=True)
+ def test_execute_options(self, mock_call):
+ chain = StetlTestCase.get_chain(self.etl, 3)
+ chain.run()
+
+ # Check command line
+ args, kwargs = mock_call.call_args
+ list = self.parse_command(args[0])
+ self.assertEqual(len(list), 12)
+
+ # Check spatial extent
+ self.assertTrue('-append' in list)
+ self.assertTrue('-gt' in list)
+ self.assertTrue('--config' in list)
+
+ @mock.patch('subprocess.call', autospec=True)
+ def test_execute_gfs(self, mock_call):
+ chain = StetlTestCase.get_chain(self.etl, 4)
+ chain.run()
+
+ # Check command line
+ args, kwargs = mock_call.call_args
+ list = self.parse_command(args[0])
+ self.assertEqual(len(list), 5)
+
+ # Check if GFS file exists, and clean it up
+ file_path = self.etl.configdict.get(StetlTestCase.get_section(chain), 'file_path')
+ file_ext = os.path.splitext(file_path)
+ gfs_path = file_ext[0] + '.gfs'
+ self.assertTrue(os.path.exists(gfs_path))
+
+ os.remove(gfs_path)
+ self.assertFalse(os.path.exists(gfs_path))
+
+ @mock.patch('subprocess.call', autospec=True)
+ def test_execute_cleanup(self, mock_call):
+ # Copy test file to temporary location, because it will be cleaned up
+ orig_file_path = self.etl.configdict.get(StetlTestCase.get_section(StetlTestCase.get_chain(self.etl)), 'file_path')
+ orig_file_ext = os.path.splitext(orig_file_path)
+ temp_file_path = orig_file_ext[0] + "_temp" + orig_file_ext[1]
+ shutil.copy(orig_file_path, temp_file_path)
+
+ chain = StetlTestCase.get_chain(self.etl, 5)
+ chain.run()
+
+ # Check command line
+ args, kwargs = mock_call.call_args
+ list = self.parse_command(args[0])
+ self.assertEqual(len(list), 5)
+
+ # Check if temp file has been removed
+ self.assertFalse(os.path.exists(temp_file_path))
+
+ @mock.patch('subprocess.call', autospec=True)
+ def test_execute_cleanup_gfs(self, mock_call):
+ # Copy test file to temporary location, because it will be cleaned up
+ orig_file_path = self.etl.configdict.get(StetlTestCase.get_section(StetlTestCase.get_chain(self.etl)), 'file_path')
+ orig_file_ext = os.path.splitext(orig_file_path)
+ temp_file_path = orig_file_ext[0] + "_temp" + orig_file_ext[1]
+ shutil.copy(orig_file_path, temp_file_path)
+
+ chain = StetlTestCase.get_chain(self.etl, 6)
+ chain.run()
+
+ # Check command line
+ args, kwargs = mock_call.call_args
+ list = self.parse_command(args[0])
+ self.assertEqual(len(list), 5)
+
+ # Check if temp file has been removed
+ self.assertFalse(os.path.exists(temp_file_path))
+
+ # Check if GFS file has already been removed
+ gfs_path = orig_file_ext[0] + "_temp.gfs"
+ self.assertFalse(os.path.exists(gfs_path))
+
+ @mock.patch('subprocess.call', autospec=True)
+ def test_execute_no_cleanup(self, mock_call):
+ chain = StetlTestCase.get_chain(self.etl, 7)
+ file_path = self.etl.configdict.get(StetlTestCase.get_section(chain), 'file_path')
+ chain.run()
+
+ # Check command line
+ args, kwargs = mock_call.call_args
+ list = self.parse_command(args[0])
+ self.assertEqual(len(list), 5)
+
+ # Check if input file still exists
+ self.assertTrue(os.path.exists(file_path))
diff --git a/tests/outputs/test_postgres_db_output.py b/tests/outputs/test_postgres_db_output.py
new file mode 100644
index 0000000..d10fc83
--- /dev/null
+++ b/tests/outputs/test_postgres_db_output.py
@@ -0,0 +1,47 @@
+import mock
+import os
+import sys
+
+from stetl.etl import ETL
+from stetl.outputs.dboutput import PostgresDbOutput
+from tests.stetl_test_case import StetlTestCase
+
+class PostgresDbOutputTest(StetlTestCase):
+ """Unit tests for PostgresDbOutput"""
+
+ def setUp(self):
+ super(PostgresDbOutputTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/postgresdboutput.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain, -1)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('outputs.dboutput.PostgresDbOutput', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.cur_comp, PostgresDbOutput))
+
+ @mock.patch('stetl.postgis.PostGIS.tx_execute', autospec=True)
+ def test_execute(self, mock_tx_execute):
+ # Read content of input file
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain)
+ fn = self.etl.configdict.get(section, 'file_path')
+ with open(fn, 'r') as f:
+ contents = f.read()
+
+ self.etl.run()
+
+ self.assertTrue(mock_tx_execute.called)
+ self.assertEqual(1, mock_tx_execute.call_count)
+ args, kwargs = mock_tx_execute.call_args
+ self.assertEqual(contents, args[1])
+
\ No newline at end of file
diff --git a/tests/outputs/test_split_outputs.py b/tests/outputs/test_split_outputs.py
new file mode 100644
index 0000000..ae9f186
--- /dev/null
+++ b/tests/outputs/test_split_outputs.py
@@ -0,0 +1,43 @@
+import os
+import sys
+
+from stetl.etl import ETL
+from stetl.outputs.standardoutput import StandardOutput
+from stetl.splitter import Splitter
+from tests.stetl_test_case import StetlTestCase
+
+class SplitterMultiOutputTest(StetlTestCase):
+ """Unit tests for Splitter"""
+ pass
+
+ def setUp(self):
+ super(SplitterMultiOutputTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/splittermultioutput.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ splitter_comp = chain.first_comp.next
+ self.assertTrue(isinstance(splitter_comp, Splitter))
+
+ # The next is a list of multiple Outputs
+ self.assertEqual(len(splitter_comp.next), 2)
+ self.assertTrue(isinstance(splitter_comp.next[0], StandardOutput))
+ self.assertTrue(isinstance(splitter_comp.next[1], StandardOutput))
+
+ def test_execute(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ chain.run()
+
+ # Result should be merged lines from both files
+ result = sys.stdout.getvalue().split('\n')
+
+ # Strip empty lines
+ result = [s for s in result if (s or len(s) > 0)]
+
+ # Total should be twice of linecount non-empty lines in input file
+ self.assertEqual(len(result), 36)
diff --git a/tests/outputs/test_standard_output.py b/tests/outputs/test_standard_output.py
new file mode 100644
index 0000000..1f03f8d
--- /dev/null
+++ b/tests/outputs/test_standard_output.py
@@ -0,0 +1,43 @@
+import os
+import sys
+
+from stetl.etl import ETL
+from stetl.outputs.standardoutput import StandardOutput
+from tests.stetl_test_case import StetlTestCase
+
+class StandardOutputTest(StetlTestCase):
+ """Unit tests for StandardOutput"""
+
+ def setUp(self):
+ super(StandardOutputTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/standardoutput.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_class(self):
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain, -1)
+ class_name = self.etl.configdict.get(section, 'class')
+
+ self.assertEqual('outputs.standardoutput.StandardOutput', class_name)
+
+ def test_instance(self):
+ chain = StetlTestCase.get_chain(self.etl)
+
+ self.assertTrue(isinstance(chain.cur_comp, StandardOutput))
+
+ def test_execute(self):
+ # Read content of input file
+ chain = StetlTestCase.get_chain(self.etl)
+ section = StetlTestCase.get_section(chain)
+ fn = self.etl.configdict.get(section, 'file_path')
+ with open(fn, 'r') as f:
+ contents = f.read()
+
+ self.etl.run()
+
+ self.assertGreater(sys.stdout.getvalue(), 0)
+ # Assert includes last linebreak from stdout, due to print function
+ self.assertEqual(sys.stdout.getvalue(), contents + '\n')
diff --git a/tests/stetl_test_case.py b/tests/stetl_test_case.py
new file mode 100644
index 0000000..629b50d
--- /dev/null
+++ b/tests/stetl_test_case.py
@@ -0,0 +1,59 @@
+import logging
+import os
+import sys
+import unittest
+
+from stetl.chain import Chain
+from stetl.util import Util
+from StringIO import StringIO
+
+logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+
+class StetlTestCase(unittest.TestCase):
+ """Base class for Stetl test cases"""
+
+ def setUp(self):
+ # Replace logger method
+ self._old_get_log = Util.get_log
+
+ @staticmethod
+ def get_log_new(name):
+ log = logging.getLogger(name)
+ log.setLevel(logging.WARN)
+ return log
+
+ Util.get_log = get_log_new
+
+ # Disable info logging
+ logging.disable(logging.WARN)
+
+ # Replace stdout
+ self._saved_stdout = sys.stdout
+ sys.stdout = StringIO()
+
+ # Replace work dir
+ self._cwd = os.getcwd()
+ os.chdir(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
+
+ def tearDown(self):
+ # Restore old enviroment
+ sys.stdout = self._saved_stdout
+ logging.disable(logging.NOTSET)
+ Util.get_log = self._old_get_log
+ os.chdir(self._cwd)
+
+ @classmethod
+ def get_chain(cls, etl, index=0, assemble=True):
+
+ chains_str = etl.configdict.get('etl', 'chains')
+ chain_strs = chains_str.split(',')
+ chain = Chain(chain_strs[index].strip(), etl.configdict)
+
+ if assemble:
+ chain.assemble()
+
+ return chain
+
+ @classmethod
+ def get_section(cls, chain, index=0):
+ return chain.chain_str.split('|')[index]
diff --git a/tests/test_chain.py b/tests/test_chain.py
index 70ce15a..32aed2f 100644
--- a/tests/test_chain.py
+++ b/tests/test_chain.py
@@ -1,213 +1,40 @@
# testing: to be called by nosetests
-import logging
-import sys
-import unittest
+import os
from stetl.etl import ETL
-
-logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
-
-
-class ConfigTest(unittest.TestCase):
- """Basic configuration tests"""
-
- def test_type(self):
- cfg_dict = {'config_file': 'tests/configs/copy_in_out.cfg'}
- etl = ETL(cfg_dict)
- self.failUnlessEqual(etl.configdict.get('etl', 'chains'), 'input_xml_file|output_std')
-
-
-# etl.run()
-#
-# def main():
-# # Test file listing
-# cand_file = sys.argv[1]
-# file_list = Util.make_file_list(cand_file)
-# print(str(file_list))
-#
-#
-# if __name__ == "__main__":
-# main()
-#
-
-# def geometry_wkb(wkb):
-# return GeomBuilder().build_wkb(wkb)
-#
-#
-# class OGRBuilderExceptionsTest(unittest.TestCase):
-# def test(self):
-# geom = {'type': "Bogus", 'coordinates': None}
-# self.assertRaises(ValueError, geometryRT, geom)
-
-# The round tripping tests are defined in this not to be run base class.
-#
-# class RoundTripping(object):
-# """Derive type specific classes from this."""
-# def test_type(self):
-# self.failUnlessEqual(
-# geometryRT(self.geom)['type'], self.geom['type'])
-# def test_coordinates(self):
-# self.failUnlessEqual(
-# geometryRT(self.geom)['coordinates'], self.geom['coordinates'])
-#
-# # All these get their tests from the RoundTripping class.
-# #
-# class PointRoundTripTest(unittest.TestCase, RoundTripping):
-# def setUp(self):
-# self.geom = {'type': "Point", 'coordinates': (0.0, 0.0)}
-#
-# class LineStringRoundTripTest(unittest.TestCase, RoundTripping):
-# def setUp(self):
-# self.geom = {
-# 'type': "LineString",
-# 'coordinates': [(0.0, 0.0), (1.0, 1.0)]}
-#
-# class PolygonRoundTripTest1(unittest.TestCase, RoundTripping):
-# """An explicitly closed polygon."""
-# def setUp(self):
-# self.geom = {
-# 'type': "Polygon",
-# 'coordinates': [
-# [(0.0, 0.0), (0.0, 1.0), (1.0, 1.0), (1.0, 0.0), (0.0, 0.0)]]}
-#
-# class PolygonRoundTripTest2(unittest.TestCase, RoundTripping):
-# """An implicitly closed polygon."""
-# def setUp(self):
-# self.geom = {
-# 'type': "Polygon",
-# 'coordinates': [
-# [(0.0, 0.0), (0.0, 1.0), (1.0, 1.0), (1.0, 0.0)]]}
-# def test_coordinates(self):
-# self.failUnlessEqual(
-# [geometryRT(self.geom)['coordinates'][0][:-1]],
-# self.geom['coordinates'])
-#
-# class MultiPointRoundTripTest(unittest.TestCase, RoundTripping):
-# def setUp(self):
-# self.geom = {
-# 'type': "MultiPoint", 'coordinates': [(0.0, 0.0), (1.0, 1.0)]}
-#
-# class MultiLineStringRoundTripTest(unittest.TestCase, RoundTripping):
-# def setUp(self):
-# self.geom = {
-# 'type': "MultiLineString",
-# 'coordinates': [[(0.0, 0.0), (1.0, 1.0)]]}
-#
-# class MultiPolygonRoundTripTest1(unittest.TestCase, RoundTripping):
-# def setUp(self):
-# # This is an explicitly closed polygon.
-# self.geom = {
-# 'type': "MultiPolygon",
-# 'coordinates': [[
-# [(0.0, 0.0), (0.0, 1.0), (1.0, 1.0), (1.0, 0.0), (0.0, 0.0)]
-# ]]}
-#
-# class MultiPolygonRoundTripTest2(unittest.TestCase, RoundTripping):
-# def setUp(self):
-# # This is an implicitly closed polygon.
-# self.geom = {
-# 'type': "MultiPolygon",
-# 'coordinates':
-# [[[(0.0, 0.0), (0.0, 1.0), (1.0, 1.0), (1.0, 0.0)]]]}
-# def test_coordinates(self):
-# self.failUnlessEqual(
-# [[geometryRT(self.geom)['coordinates'][0][0][:-1]]],
-# self.geom['coordinates'])
-#
-# class GeometryCollectionRoundTripTest(unittest.TestCase):
-# def setUp(self):
-# self.geom = {
-# 'type': "GeometryCollection",
-# 'geometries': [
-# {'type': "Point", 'coordinates': (0.0, 0.0)}, {
-# 'type': "LineString",
-# 'coordinates': [(0.0, 0.0), (1.0, 1.0)]}]}
-# def test_len(self):
-# result = geometryRT(self.geom)
-# self.failUnlessEqual(len(result['geometries']), 2)
-# def test_type(self):
-# result = geometryRT(self.geom)
-# self.failUnlessEqual(
-# [g['type'] for g in result['geometries']],
-# ['Point', 'LineString'])
-#
-# class PointTest(unittest.TestCase):
-# def test_point(self):
-# # Hex-encoded Point (0 0)
-# try:
-# wkb = bytes.fromhex("010100000000000000000000000000000000000000")
-# except:
-# wkb = "010100000000000000000000000000000000000000".decode('hex')
-# geom = geometry_wkb(wkb)
-# self.failUnlessEqual(geom['type'], "Point")
-# self.failUnlessEqual(geom['coordinates'], (0.0, 0.0))
-#
-# class LineStringTest(unittest.TestCase):
-# def test_line(self):
-# # Hex-encoded LineString (0 0, 1 1)
-# try:
-# wkb = bytes.fromhex("01020000000200000000000000000000000000000000000000000000000000f03f000000000000f03f")
-# except:
-# wkb = "01020000000200000000000000000000000000000000000000000000000000f03f000000000000f03f".decode('hex')
-# geom = geometry_wkb(wkb)
-# self.failUnlessEqual(geom['type'], "LineString")
-# self.failUnlessEqual(geom['coordinates'], [(0.0, 0.0), (1.0, 1.0)])
-#
-# class PolygonTest(unittest.TestCase):
-# def test_polygon(self):
-# # 1 x 1 box (0, 0, 1, 1)
-# try:
-# wkb = bytes.fromhex("01030000000100000005000000000000000000f03f0000000000000000000000000000f03f000000000000f03f0000000000000000000000000000f03f00000000000000000000000000000000000000000000f03f0000000000000000")
-# except:
-# wkb = "01030000000100000005000000000000000000f03f0000000000000000000000000000f03f000000000000f03f0000000000000000000000000000f03f00000000000000000000000000000000000000000000f03f0000000000000000".decode('hex')
-# geom = geometry_wkb(wkb)
-# self.failUnlessEqual(geom['type'], "Polygon")
-# self.failUnlessEqual(len(geom['coordinates']), 1)
-# self.failUnlessEqual(len(geom['coordinates'][0]), 5)
-# x, y = zip(*geom['coordinates'][0])
-# self.failUnlessEqual(min(x), 0.0)
-# self.failUnlessEqual(min(y), 0.0)
-# self.failUnlessEqual(max(x), 1.0)
-# self.failUnlessEqual(max(y), 1.0)
-#
-# class MultiPointTest(unittest.TestCase):
-# def test_multipoint(self):
-# try:
-# wkb = bytes.fromhex("0104000000020000000101000000000000000000000000000000000000000101000000000000000000f03f000000000000f03f")
-# except:
-# wkb = "0104000000020000000101000000000000000000000000000000000000000101000000000000000000f03f000000000000f03f".decode('hex')
-# geom = geometry_wkb(wkb)
-# self.failUnlessEqual(geom['type'], "MultiPoint")
-# self.failUnlessEqual(geom['coordinates'], [(0.0, 0.0), (1.0, 1.0)])
-#
-# class MultiLineStringTest(unittest.TestCase):
-# def test_multilinestring(self):
-# # Hex-encoded LineString (0 0, 1 1)
-# try:
-# wkb = bytes.fromhex("01050000000100000001020000000200000000000000000000000000000000000000000000000000f03f000000000000f03f")
-# except:
-# wkb = "01050000000100000001020000000200000000000000000000000000000000000000000000000000f03f000000000000f03f".decode('hex')
-# geom = geometry_wkb(wkb)
-# self.failUnlessEqual(geom['type'], "MultiLineString")
-# self.failUnlessEqual(len(geom['coordinates']), 1)
-# self.failUnlessEqual(len(geom['coordinates'][0]), 2)
-# self.failUnlessEqual(geom['coordinates'][0], [(0.0, 0.0), (1.0, 1.0)])
-#
-# class MultiPolygonTest(unittest.TestCase):
-# def test_multipolygon(self):
-# # [1 x 1 box (0, 0, 1, 1)]
-# try:
-# wkb = bytes.fromhex("01060000000100000001030000000100000005000000000000000000f03f0000000000000000000000000000f03f000000000000f03f0000000000000000000000000000f03f00000000000000000000000000000000000000000000f03f0000000000000000")
-# except:
-# wkb = "01060000000100000001030000000100000005000000000000000000f03f0000000000000000000000000000f03f000000000000f03f0000000000000000000000000000f03f00000000000000000000000000000000000000000000f03f0000000000000000".decode('hex')
-# geom = geometry_wkb(wkb)
-# self.failUnlessEqual(geom['type'], "MultiPolygon")
-# self.failUnlessEqual(len(geom['coordinates']), 1)
-# self.failUnlessEqual(len(geom['coordinates'][0]), 1)
-# self.failUnlessEqual(len(geom['coordinates'][0][0]), 5)
-# x, y = zip(*geom['coordinates'][0][0])
-# self.failUnlessEqual(min(x), 0.0)
-# self.failUnlessEqual(min(y), 0.0)
-# self.failUnlessEqual(max(x), 1.0)
-# self.failUnlessEqual(max(y), 1.0)
+from stetl.chain import Chain
+from tests.stetl_test_case import StetlTestCase
+
+class ChainTest(StetlTestCase):
+ """Basic chain tests"""
+
+ def setUp(self):
+ super(ChainTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/copy_in_out.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_chain(self):
+ chain = StetlTestCase.get_chain(self.etl, assemble=False)
+
+ self.assertIsNone(chain.first_comp)
+ self.assertIsNone(chain.cur_comp)
+
+ def test_chain_assembly(self):
+ chain = StetlTestCase.get_chain(self.etl, assemble=False)
+ chain.assemble()
+
+ self.assertIsNotNone(chain.first_comp)
+ self.assertIsNotNone(chain.cur_comp)
+
+ self.assertIsNotNone(chain.first_comp.next)
+ self.assertIsNone(chain.cur_comp.next)
+
+ comp = chain.first_comp
+ while comp.next is not None:
+ comp = comp.next
+
+ self.assertIs(comp, chain.cur_comp)
diff --git a/tests/test_chain.pyc b/tests/test_chain.pyc
deleted file mode 100644
index 8a3c06f..0000000
Binary files a/tests/test_chain.pyc and /dev/null differ
diff --git a/tests/test_config.py b/tests/test_config.py
new file mode 100644
index 0000000..e4e622c
--- /dev/null
+++ b/tests/test_config.py
@@ -0,0 +1,23 @@
+# testing: to be called by nosetests
+
+import os
+
+from stetl.etl import ETL
+from tests.stetl_test_case import StetlTestCase
+
+class ConfigTest(StetlTestCase):
+ """Basic configuration tests"""
+
+ def setUp(self):
+ super(ConfigTest, self).setUp()
+
+ # Initialize Stetl
+ curr_dir = os.path.dirname(os.path.realpath(__file__))
+ cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/copy_in_out.cfg')}
+ self.etl = ETL(cfg_dict)
+
+ def test_type(self):
+ self.assertEqual(self.etl.configdict.get('etl', 'chains'), 'input_xml_file|output_std')
+
+ def test_run(self):
+ self.etl.run()
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/python-stetl.git
More information about the Pkg-grass-devel
mailing list