[med-svn] [biomaj3-core] 01/04: New upstream version 3.0.9

Olivier Sallou osallou at debian.org
Thu Aug 17 07:44:19 UTC 2017


This is an automated email from the git hooks/post-receive script.

osallou pushed a commit to branch master
in repository biomaj3-core.

commit 38666e248d0cf8eff8d7a2fa3d4aefc3a82f05c4
Author: Olivier Sallou <osallou at debian.org>
Date:   Thu Aug 17 07:20:40 2017 +0000

    New upstream version 3.0.9
---
 .gitignore                     |  72 +++++
 .travis.yml                    |  31 ++
 CHANGES.txt                    |  22 ++
 LICENSE                        | 662 +++++++++++++++++++++++++++++++++++++++++
 MANIFEST.in                    |   2 +
 README.md                      |  10 +
 biomaj_core/__init__.py        |   0
 biomaj_core/bmajindex.py       | 213 +++++++++++++
 biomaj_core/config.py          | 482 ++++++++++++++++++++++++++++++
 biomaj_core/mimes-bio.txt      |  18 ++
 biomaj_core/utils.py           | 315 ++++++++++++++++++++
 requirements.txt               |   5 +
 setup.cfg                      |   2 +
 setup.py                       |  53 ++++
 tests/alu.properties           |  43 +++
 tests/bank/process/test.sh     |  11 +
 tests/bank/test.fasta.gz       | Bin 0 -> 45 bytes
 tests/bank/test2.fasta         |   2 +
 tests/bank/test_100.txt        |   1 +
 tests/biomaj_tests.py          | 297 ++++++++++++++++++
 tests/computed.properties      |  44 +++
 tests/computed2.properties     |  45 +++
 tests/computederror.properties |  43 +++
 tests/directhttp.properties    |  41 +++
 tests/error.properties         |  43 +++
 tests/global.properties        | 123 ++++++++
 tests/local.properties         |  41 +++
 tests/locallist.properties     |  44 +++
 tests/localprocess.properties  | 100 +++++++
 tests/multi.properties         |  60 ++++
 tests/noargs.properties        |  48 +++
 tests/noexe.properties         |  48 +++
 tests/noname.properties        |  49 +++
 tests/prenoargs.properties     |  45 +++
 tests/prenoexe.properties      |  45 +++
 tests/prenoname.properties     |  45 +++
 tests/rmnoargs.properties      |  45 +++
 tests/rmnoexe.properties       |  45 +++
 tests/rmnoname.properties      |  45 +++
 tests/sub1.properties          |  43 +++
 tests/sub2.properties          |  41 +++
 tests/testhttp.properties      |  43 +++
 42 files changed, 3367 insertions(+)

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..754c2a7
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,72 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# Coveralls
+.coveralls.yml
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# PyCharm
+.idea
+
+# Vim
+.viminfo
+# Less history
+.lesshst
+
+.dbshell
+.emacs*
+.ipython
+.mongo*
+#*.properties
+
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..5f2c44e
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,31 @@
+language: python
+sudo: false
+python:
+  - "2.7"
+  - "3.4"
+  - "3.5"
+# Apply only on main branches
+branches:
+  except:
+    - /^feature.*$/
+# command to install dependencies
+#before_install:
+#  - "sudo apt-get update -qq"
+#  - "sudo apt-get install -qq libldap2-dev libsasl2-dev"
+install:
+  - "pip install flake8"
+  - "pip install -r requirements.txt"
+  - "pip install coverage"
+  - "pip install python-coveralls"
+  - "python setup.py -q install"
+#  - "echo data_file=$TRAVIS_BUILD_DIR/.coverage >> .coveragerc"
+# command to run tests
+#before_script:
+#  - sleep 10
+#script: nosetests --with-coverage --cover-package=biomaj -a '!network'
+#script: nosetests --with-coverage --cover-package=biomaj
+script:
+   - python setup.py test
+   - flake8 --ignore E501,E123 biomaj_core/*.py
+#  - coveralls
+
diff --git a/CHANGES.txt b/CHANGES.txt
new file mode 100644
index 0000000..6964b4f
--- /dev/null
+++ b/CHANGES.txt
@@ -0,0 +1,22 @@
+3.0.9:
+  #1 Add checks on bank properties
+3.0.8:
+  Add DOCKER_URL, MONGO_URL, MONGO_DB env variable in override
+  Do not fail on elasticsearch index creation, just log
+  Add rsync check
+3.0.7:
+  set logging only if necessary
+3.0.6:
+  Fix configparser import for python 2 vs 3
+3.0.5:
+  Add config override with env variables
+3.0.4:
+  Code cleanup
+3.0.3:
+  Put index management in core
+3.0.2:
+  Add missing deps
+3.0.1:
+  Add README etc..
+3.0.0:
+  separation of biomaj and biomaj_core
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..cebe035
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,662 @@
+GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<http://www.gnu.org/licenses/>.
+
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..5dcc15a
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,2 @@
+include *.txt *.md
+recursive-include biomaj_core *.txt
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..e6d7650
--- /dev/null
+++ b/README.md
@@ -0,0 +1,10 @@
+# Biomaj
+
+Biomaj core library
+
+
+Properties can be overriden by environment variables with pattern BIOMAJ_X_Y_Z for property x.y.z
+
+Example:
+
+use BIOMAJ_LDAP_HOST for ldap.host
diff --git a/biomaj_core/__init__.py b/biomaj_core/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/biomaj_core/bmajindex.py b/biomaj_core/bmajindex.py
new file mode 100644
index 0000000..a6c68bb
--- /dev/null
+++ b/biomaj_core/bmajindex.py
@@ -0,0 +1,213 @@
+from builtins import str
+from builtins import object
+import logging
+import copy
+from elasticsearch import Elasticsearch
+
+
+class BmajIndex(object):
+    """
+    ElasticSearch indexation and search
+    """
+
+    """
+    ElasticSearch server
+    """
+    es = None
+
+    """
+    Index name
+    """
+    index = 'biomaj'
+
+    """
+    Do indexing
+    """
+    do_index = False
+
+    """
+    Skip if failure (tests)
+    """
+    skip_if_failure = False
+
+    @staticmethod
+    def load(hosts=None, index='biomaj', do_index=True):
+        """
+        Initialize index
+
+        :param hosts: List of elastic search nodes to connect to
+        :type hosts: list
+        :param do_index: index data or not
+        :type do_index: bool
+        """
+        if hosts is None:
+            hosts = ['localhost']
+        if not do_index:
+            return
+        BmajIndex.index = index
+        BmajIndex.do_index = do_index
+        if BmajIndex.es is None:
+            BmajIndex.es = Elasticsearch(hosts)
+
+            mapping = {
+                "mappings": {
+                    "production": {
+                        "date_detection": False
+                    },
+                    "releasestats": {
+                        "date_detection": False,
+                    }
+                }
+            }
+            try:
+                if not BmajIndex.es.indices.exists(index=BmajIndex.index):
+                    BmajIndex.es.indices.create(index=BmajIndex.index, body=mapping)
+            except Exception as e:
+                logging.error('ElasticSearch connection error, check server is running and configuration: ' + str(e))
+
+    @staticmethod
+    def _bulk_delete(query, flush=True):
+        try:
+            page = BmajIndex.es.search(
+                index=BmajIndex.index,
+                doc_type='production',
+                search_type="query_then_fetch",
+                size=1000,
+                body={
+                    'query': {
+                        'match': {
+                            'bank': query['bank']
+                        }
+                    }
+                })
+
+            if page is None:
+                return
+            bulk_delete = ''
+            for del_hit in page['hits']['hits']:
+                if ('release' in query and query['release'] == del_hit['_source']['release']) or 'release' not in query:
+                    bulk_delete += "{ \"delete\" : {\"_index\":\"" + BmajIndex.index + "\",\"_type\":\"production\", \"_id\" : \"" + del_hit['_id'] + "\" } }\n"
+            if bulk_delete:
+                BmajIndex.es.bulk(body=bulk_delete)
+                if flush:
+                    BmajIndex.es.indices.flush(index=BmajIndex.index, force=True)
+        except Exception as e:
+            if BmajIndex.skip_if_failure:
+                BmajIndex.do_index = False
+            else:
+                logging.error('Failed to delete bank index: ' + str(e))
+
+    @staticmethod
+    def delete_all_bank(bank_name):
+        """
+        Delete complete index for a bank
+        """
+        if not BmajIndex.do_index:
+            return
+        BmajIndex._bulk_delete({"bank": bank_name}, True)
+
+    @staticmethod
+    def remove(bank_name, release):
+        """
+        Remove a production release
+
+        :param bank_name: Name of the bank
+        :type bank_name: str
+        :param release: production release
+        :type release: str
+        """
+        if not BmajIndex.do_index:
+            return
+        BmajIndex._bulk_delete({"release": release, "bank": bank_name})
+        """
+        try:
+            query = {
+              "query" : {
+                "term" : {"release" : release, "bank": bank_name}
+                }
+              }
+            BmajIndex.es.delete_by_query(index=BmajIndex.index, body=query)
+        except Exception as e:
+            logging.error('Index:Remove:'+bank_name+'_'+str(release)+':Exception:'+str(e))
+            if BmajIndex.skip_if_failure:
+                BmajIndex.do_index = False
+        """
+
+    @staticmethod
+    def search(query):
+        if not BmajIndex.do_index:
+            return None
+        res = BmajIndex.es.search(index=BmajIndex.index,
+                                  doc_type='production',
+                                  search_type="query_then_fetch",
+                                  body=query)
+        return res['hits']['hits']
+
+    @staticmethod
+    def searchq(query, size=1000):
+        """
+        Lucene syntax search
+
+        :param query: Lucene search string
+        :type query: str
+        :param size: number of results
+        :type size: int
+        :return: list of matches
+        """
+        if not BmajIndex.do_index:
+            return None
+        res = BmajIndex.es.search(index=BmajIndex.index, doc_type='production', q=query, size=size)
+        return res['hits']['hits']
+
+    @staticmethod
+    def add_stat(stat_id, stat):
+        """
+        Add some statistics, must contain release and bank properties.
+        """
+        if not BmajIndex.do_index:
+            return
+        if stat['release'] is None or stat['bank'] is None:
+            return False
+        try:
+            BmajIndex.es.index(index=BmajIndex.index, doc_type='releasestats', id=stat_id, body=stat)
+        except Exception:
+            if BmajIndex.skip_if_failure:
+                BmajIndex.do_index = False
+            else:
+                return False
+        return True
+
+    @staticmethod
+    def add(bank_name, prod, flush=False):
+        """
+        Index a production release
+
+        :param bank_name: Name of the bank
+        :type bank_name: str
+        :param prod: session release object
+        :type prod: dict
+        :param flush: Force flushing
+        :type flush: bool
+        """
+        if not BmajIndex.do_index:
+            return
+        obj = copy.deepcopy(prod)
+        if obj['release'] is None:
+            return
+        obj['bank'] = bank_name
+        formats = obj['formats']
+        try:
+            for fkey, fvalue in formats.items():
+                for elt in fvalue:
+                    elt['format'] = fkey
+                    elt['bank'] = bank_name
+                    elt['release'] = obj['release']
+                    if 'status' in obj:
+                        elt['status'] = obj['status']
+                    BmajIndex.es.index(index=BmajIndex.index, doc_type='production', body=elt)
+            if flush:
+                BmajIndex.es.indices.flush(index=BmajIndex.index, force=True)
+        except Exception as e:
+            logging.error('Index:Add:' + bank_name + '_' + str(obj['release']) + ':Exception:' + str(e))
+            if BmajIndex.skip_if_failure:
+                BmajIndex.do_index = False
diff --git a/biomaj_core/config.py b/biomaj_core/config.py
new file mode 100644
index 0000000..9f60c5d
--- /dev/null
+++ b/biomaj_core/config.py
@@ -0,0 +1,482 @@
+from __future__ import print_function
+from future import standard_library
+from builtins import str
+from builtins import object
+import logging
+import logging.config
+import os
+import time
+import sys
+
+from biomaj_core.bmajindex import BmajIndex
+
+if sys.version < '3':
+    import ConfigParser as configparser
+else:
+    import configparser
+
+standard_library.install_aliases()
+
+
+class BiomajConfig(object):
+    """
+    Manage Biomaj configuration
+    """
+
+    DEFAULTS = {
+        'http.parse.dir.line': r'<img[\s]+src="[\S]+"[\s]+alt="\[DIR\]"[\s]*/?>[\s]*<a[\s]+href="([\S]+)/"[\s]*>.*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})',
+        'http.parse.file.line': r'<img[\s]+src="[\S]+"[\s]+alt="\[[\s]+\]"[\s]*/?>[\s]<a[\s]+href="([\S]+)".*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})[\s]+([\d\.]+[MKG]{0,1})',
+        'http.group.dir.name': 1,
+        'http.group.dir.date': 2,
+        'http.group.file.name': 1,
+        'http.group.file.date': 2,
+        'http.group.file.size': 3,
+        'visibility.default': 'public',
+        'historic.logfile.level': 'INFO',
+        'bank.num.threads': 2,
+        'files.num.threads': 4,
+        'use_elastic': 0,
+        'use_drmaa': 0,
+        'db.type': '',
+        'db.formats': '',
+        'keep.old.version': 1,
+        'docker.sudo': '1',
+        'auto_publish': 0
+    }
+
+    # Old biomaj level compatibility
+    LOGLEVEL = {
+        'DEBUG': logging.DEBUG,
+        'VERBOSE': logging.INFO,
+        'INFO': logging.INFO,
+        'WARN': logging.WARNING,
+        'ERR': logging.ERROR
+    }
+
+    """
+    Global configuration file
+    """
+    global_config = None
+
+    """
+    Per use global configuration file, overriding global_config
+    """
+    user_config = None
+    allow_user_config = False
+
+    @staticmethod
+    def load_config(config_file=None, allow_user_config=True):
+        """
+        Loads general config
+
+        :param config_file: global.properties file path
+        :type config_file: str
+        :param allow_user_config: use ~/.biomaj.cfg if present
+        :type allow_user_config: bool
+        """
+        BiomajConfig.allow_user_config = allow_user_config
+
+        if config_file is None:
+            env_file = os.environ.get('BIOMAJ_CONF')
+            if env_file is not None and os.path.exists(env_file):
+                config_file = env_file
+            else:
+                env_file = 'global.properties'
+                if os.path.exists(env_file):
+                    config_file = env_file
+
+        if config_file is None or not os.path.exists(config_file):
+            raise Exception('Missing global configuration file')
+
+        BiomajConfig.config_file = os.path.abspath(config_file)
+
+        BiomajConfig.global_config = configparser.ConfigParser()
+
+        # if allow_user_config and os.path.exists(os.path.expanduser('~/.biomaj.cfg')):
+        #    BiomajConfig.user_config_file = os.path.expanduser('~/.biomaj.cfg')
+        #    BiomajConfig.user_config = configparser.ConfigParser()
+        #    BiomajConfig.user_config.read([os.path.expanduser('~/.biomaj.cfg')])
+        # else:
+        #    BiomajConfig.user_config_file = None
+
+        BiomajConfig.global_config.read([config_file])
+
+        # ElasticSearch indexation support
+        do_index = False
+        if BiomajConfig.global_config.get('GENERAL', 'use_elastic') and \
+                BiomajConfig.global_config.get('GENERAL', 'use_elastic') == "1":
+            do_index = True
+        if do_index:
+            if BiomajConfig.global_config.get('GENERAL', 'elastic_nodes'):
+                elastic_hosts = BiomajConfig.global_config.get('GENERAL',
+                                                               'elastic_nodes')\
+                    .split(',')
+            else:
+                elastic_hosts = ['localhost']
+            elastic_index = BiomajConfig.global_config.get('GENERAL', 'elastic_index')
+            if elastic_index is None:
+                elastic_index = 'biomaj'
+
+            if BiomajConfig.global_config.has_option('GENERAL', 'test') and \
+                    BiomajConfig.global_config.get('GENERAL', 'test') == "1":
+                # Test connection to elasticsearch, if not available skip indexing for tests
+                BmajIndex.skip_if_failure = True
+
+            BmajIndex.load(index=elastic_index,
+                           hosts=elastic_hosts,
+                           do_index=do_index)
+
+    def __init__(self, bank, options=None):
+        """
+        Loads bank configuration
+
+        :param bank: bank name
+        :type bank: str
+        :param options: bank options
+        :type options: argparse
+        """
+        self.log_level = logging.INFO
+        self.name = bank
+        if BiomajConfig.global_config is None:
+            BiomajConfig.load_config()
+        self.config_bank = configparser.ConfigParser()
+        conf_dir = BiomajConfig.global_config.get('GENERAL', 'conf.dir')
+        if not os.path.exists(os.path.join(conf_dir, bank + '.properties')):
+            logging.error('Bank configuration file does not exists')
+            raise Exception('Configuration file ' + bank +
+                            '.properties does not exists')
+        try:
+            config_files = [BiomajConfig.config_file]
+            self.user_config_file = None
+            if self.allow_user_config and hasattr(options, 'user')\
+                    and options.user\
+                    and os.path.exists(os.path.expanduser('~' + options.user + '/.biomaj.cfg')):
+                self.user_config_file = os.path.expanduser('~/' + options.user +
+                                                           '.biomaj.cfg')
+                self.user_config = configparser.ConfigParser()
+                self.user_config.read([os.path.expanduser('~/' + options.user +
+                                                          '.biomaj.cfg')])
+
+            if self.user_config_file is not None:
+                config_files.append(self.user_config_file)
+            config_files.append(os.path.join(conf_dir, bank + '.properties'))
+            self.config_bank.read(config_files)
+        except Exception as e:
+            logging.error("Configuration file error " + str(e))
+            raise Exception("Configuration file error " + str(e))
+
+        self.last_modified = int(os.stat(os.path.join(conf_dir, bank +
+                                                      '.properties')).st_mtime)
+
+        do_log = False
+        if options is None:
+            do_log = True
+        elif hasattr(options, 'no_log') and not options.no_log:
+            do_log = True
+        elif type(options) is dict and 'no_log' in options and not options['no_log']:
+            do_log = True
+
+        if do_log:
+            if os.path.exists(os.path.expanduser('~/.biomaj.cfg')):
+                logging.config.fileConfig(os.path.expanduser('~/.biomaj.cfg'))
+            else:
+                logging.config.fileConfig(BiomajConfig.config_file)
+
+            logger = logging.getLogger()
+            bank_log_dir = os.path.join(self.get('log.dir'), bank, str(time.time()))
+            if not os.path.exists(bank_log_dir):
+                os.makedirs(bank_log_dir)
+            hdlr = logging.FileHandler(os.path.join(bank_log_dir, bank + '.log'))
+            self.log_file = os.path.join(bank_log_dir, bank + '.log')
+            if options is not None and options.get_option('log') is not None:
+                hdlr.setLevel(BiomajConfig.LOGLEVEL[options.get_option('log')])
+                self.log_level = BiomajConfig.LOGLEVEL[options.get_option('log')]
+            else:
+                hdlr.setLevel(BiomajConfig.LOGLEVEL[self.get('historic.logfile.level')])
+                self.log_level = BiomajConfig.LOGLEVEL[self.get('historic.logfile.level')]
+            formatter = logging.Formatter('%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s')
+            hdlr.setFormatter(formatter)
+            logger.addHandler(hdlr)
+            '''
+            # For monolithic, update default logger/handlers?
+            existing_logger_names = list(logging.getLogger().manager.loggerDict.keys())
+            for existing_logger in existing_logger_names:
+                if existing_logger.startswith('biomaj'):
+                    tmp_logger = logging.getLogger(existing_logger)
+                    tmp_logger.setLevel(self.log_level)
+                    for handler in tmp_logger.handlers:
+                        handler.setLevel(self.log_level)
+            '''
+        else:
+            self.log_file = 'none'
+
+        cache_dir = self.get('cache.dir')
+        if cache_dir is None:
+            print("Configuration file error: cache.dir empty")
+            logging.error("cache.dir is not defined")
+            sys.exit(1)
+
+        if not os.path.exists(cache_dir):
+            os.makedirs(cache_dir)
+
+        process_dir = self.get('process.dir')
+        if process_dir is None:
+            print("Configuration file error: process.dir empty")
+            logging.error("process.dir is not defined")
+            sys.exit(1)
+
+        if not os.path.exists(process_dir):
+            os.makedirs(process_dir)
+
+        data_dir = self.get('data.dir')
+        if data_dir is None:
+            print("Configuration file error: data.dir empty")
+            logging.error("data.dir is not defined")
+            sys.exit(1)
+
+        if not os.path.exists(data_dir):
+            os.makedirs(data_dir)
+
+        lock_dir = self.get('lock.dir')
+        if lock_dir is None:
+            print("Configuration file error: lock.dir empty")
+            logging.error("lock.dir is not defined")
+            sys.exit(1)
+
+        if not os.path.exists(lock_dir):
+            os.makedirs(lock_dir)
+
+    def reset_logger(self):
+        '''
+        Reconfigure logging
+        '''
+        if self.log_file == 'none':
+            return
+        if os.path.exists(os.path.expanduser('~/.biomaj.cfg')):
+            logging.config.fileConfig(os.path.expanduser('~/.biomaj.cfg'))
+        else:
+            logging.config.fileConfig(BiomajConfig.config_file)
+
+        logger = logging.getLogger()
+        hdlr = logging.FileHandler(self.log_file)
+        hdlr.setLevel(self.log_level)
+        formatter = logging.Formatter('%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s')
+        hdlr.setFormatter(formatter)
+        logger.addHandler(hdlr)
+
+    def set(self, prop, value, section='GENERAL'):
+        self.config_bank.set(section, prop, value)
+
+    def _in_env(self, prop):
+        '''
+        Search for env variable BIOMAJ_X_Y_Z for property X.Y.Z
+        '''
+        env_prop = 'BIOMAJ_' + prop.upper().replace('.', '_')
+        if env_prop in os.environ:
+            return os.environ[env_prop]
+        else:
+            return None
+
+    def get_bool(self, prop, section='GENERAL', escape=True, default=None):
+        """
+        Get a boolean property from bank or general configration. Optionally in section.
+        """
+        value = None
+
+        if self._in_env(prop):
+            value = self._in_env(prop)
+        else:
+            value = self.get(prop, section, escape, default)
+
+        if value is None:
+            return False
+        if value is True or value == 'true' or value == '1':
+            return True
+        else:
+            return False
+
+    def get(self, prop, section='GENERAL', escape=True, default=None):
+        """
+        Get a property from bank or general configration. Optionally in section.
+        """
+        if self._in_env(prop):
+            return self._in_env(prop)
+        # Compatibility fields
+        if prop == 'depends':
+            depend = self.get('db.source', section, escape, None)
+            if depend:
+                return depend
+
+        if self.config_bank.has_option(section, prop):
+            val = self.config_bank.get(section, prop)
+            if prop == 'remote.dir' and not val.endswith('/'):
+                val = val + '/'
+            # If regexp, escape backslashes
+            if escape and (prop == 'local.files' or prop == 'remote.files' or
+                                   prop == 'http.parse.dir.line' or
+                                   prop == 'http.parse.file.line'):
+                val = val.replace('\\\\', '\\')
+            return val
+
+        if self.user_config is not None:
+            if self.user_config.has_option(section, prop):
+                return self.user_config.get(section, prop)
+
+        if BiomajConfig.global_config.has_option(section, prop):
+            return BiomajConfig.global_config.get(section, prop)
+
+        if prop in BiomajConfig.DEFAULTS:
+            return BiomajConfig.DEFAULTS[prop]
+
+        return default
+
+    def get_time(self):
+        """
+        Return last modification time of config files
+        """
+        return self.last_modified
+
+    def check(self):
+        """
+        Check configuration
+        """
+        self.set('localrelease', '')
+        self.set('remoterelease', '')
+
+        status = True
+        if not self.get('data.dir'):
+            logging.error('data.dir is not set')
+            status = False
+        if not self.get('conf.dir'):
+            logging.error('conf.dir is not set')
+            status = False
+        if not self.get('log.dir'):
+            logging.error('log.dir is not set')
+            status = False
+        if not self.get('process.dir'):
+            logging.error('process.dir is not set')
+            status = False
+        if not self.get('lock.dir'):
+            logging.error('lock.dir is not set')
+            status = False
+        if not self.get('cache.dir'):
+            logging.error('cache.dir is not set')
+            status = False
+        if not self.get('db.fullname'):
+            logging.warn('db.fullname is not set')
+        if not self.get('db.formats'):
+            logging.warn('db.formats is not set')
+        if self.get('use_ldap'):
+            if not self.get('ldap.host') or not self.get('ldap.port') or \
+                    not self.get('ldap.dn'):
+                logging.error('use_ldap set to 1 but missing configuration')
+                status = False
+        if self.get('use_elastic'):
+            if not self.get('elastic_nodes') or not self.get('elastic_index'):
+                logging.error('use_elastic set to 1 but missing configuration')
+                status = False
+
+        # if not self.get('celery.queue') or not self.get('celery.broker'):
+        #    logging.warn('celery config is not set, that\'s fine if you do not use Celery for background tasks')
+
+        if not self.get('mail.smtp.host'):
+            logging.warn('SMTP mail config not set, you will not be able to send emails')
+        if self.get('mail.smtp.host') and not self.get('mail.from'):
+            logging.error('Mail origin mail.from not set')
+            status = False
+
+        if not self.get('offline.dir.name'):
+            logging.error('offline.dir.name is not set')
+            status = False
+        elif self.get('offline.dir.name').startswith('/'):
+            logging.error('offline dir must be relative to data.dir and should not start with a /')
+            status = False
+        if not self.get('dir.version'):
+            logging.error('dir.version is not set')
+            status = False
+        elif self.get('dir.version').startswith('/'):
+            logging.error('dir.version must be relative to data.dir and should not start with a /')
+            status = False
+        if not self.get('protocol'):
+            logging.error('protocol is not set')
+            status = False
+        else:
+            protocol = self.get('protocol')
+            allowed_protocols = ['none', 'multi', 'local', 'ftp', 'sftp', 'http',
+                                 'https', 'directftp', 'directhttp', 'directhttps', 'rsync']
+            if protocol not in allowed_protocols:
+                logging.error('Protocol not supported: ' + protocol)
+                status = False
+            if protocol not in ['multi', 'none']:
+                if protocol != 'local' and not self.get('server'):
+                    logging.error('server not set')
+                    status = False
+                if not self.get('remote.dir'):
+                    logging.error('remote.dir not set')
+                    status = False
+                elif not self.get('remote.dir').endswith('/'):
+                    logging.error('remote.dir must end with a /')
+                    return False
+                if protocol not in ['direcftp', 'directhttp', 'directhttps'] and\
+                        not self.get('remote.files') and\
+                        not self.get('remote.list'):
+                    logging.error('remote.files not set')
+                    status = False
+        if not self.get('local.files'):
+            logging.error('local.files is not set')
+            status = False
+        # Remove processes
+        processes = ['db.remove.process', 'db.pre.process']
+        for process in processes:
+            if self.get(process):
+                metas = self.get(process).split(',')
+                for meta in metas:
+                    if not self.get(meta):
+                        logging.error('Metaprocess ' + meta + ' not defined')
+                        status = False
+                    else:
+                        procs = self.get(meta).split(',')
+                        for proc in procs:
+                            if not self.get(proc + '.name'):
+                                logging.error('Process ' + proc + ' not defined')
+                                status = False
+                            else:
+                                if not self.get(proc + '.exe'):
+                                    logging.error('Process exe for ' + proc +
+                                                  ' not defined')
+                                    status = False
+                                if not self.get(proc + '.args'):
+                                    logging.error(
+                                        'Process args for ' + proc +
+                                        ' not defined')
+                                    status = False
+        # Check blocks
+        if self.get('BLOCKS'):
+            blocks = self.get('BLOCKS').split(',')
+            for block in blocks:
+                if not self.get(block + '.db.post.process'):
+                    logging.error('Block ' + block + ' not defined')
+                    status = False
+                else:
+                    metas = self.get(block + '.db.post.process').split(',')
+                    for meta in metas:
+                        if not self.get(meta):
+                            logging.error('Metaprocess ' + meta + ' not defined')
+                            status = False
+                        else:
+                            procs = self.get(meta).split(',')
+                            for proc in procs:
+                                if not self.get(proc + '.name'):
+                                    logging.error('Process ' + proc +
+                                                  ' not defined')
+                                    status = False
+                                else:
+                                    if not self.get(proc + '.exe'):
+                                        logging.error('Process exe for ' +
+                                                      proc + ' not defined')
+                                        status = False
+                                    if not self.get(proc + '.args'):
+                                        logging.error('Process args for ' +
+                                                      proc + ' not defined')
+                                        status = False
+        return status
diff --git a/biomaj_core/mimes-bio.txt b/biomaj_core/mimes-bio.txt
new file mode 100644
index 0000000..c794bf6
--- /dev/null
+++ b/biomaj_core/mimes-bio.txt
@@ -0,0 +1,18 @@
+# Biological file mime types
+application/fasta         fasta fa fsa
+application/bam           bam bai
+application/gff           gff gff3
+application/bed           bed
+application/fastq         fastq
+application/gtf           gtf
+application/octet-stream  ab1 scf
+application/axt           axt
+application/csFasta       csfasta
+application/FasttqSolexa  fastqsolexa
+application/Interval      interval
+application/Laj           laj
+application/Lav           lav
+application/Maf           maf
+application/QualityScore  qual
+application/BlastXml      blastxml
+application/Wiggle        wig
diff --git a/biomaj_core/utils.py b/biomaj_core/utils.py
new file mode 100644
index 0000000..56f8eea
--- /dev/null
+++ b/biomaj_core/utils.py
@@ -0,0 +1,315 @@
+import os
+import re
+import logging
+import shutil
+import datetime
+import time
+import subprocess
+from subprocess import CalledProcessError
+import socket
+
+from mimetypes import MimeTypes
+
+
+class Utils(object):
+    """
+    Utility classes
+    """
+
+    mime = None
+
+    @staticmethod
+    def service_config_override(config):
+        if 'RABBITMQ_HOST' in os.environ and os.environ['RABBITMQ_HOST']:
+            config['rabbitmq']['host'] = os.environ['RABBITMQ_HOST']
+        if 'RABBITMQ_PORT' in os.environ and os.environ['RABBITMQ_PORT']:
+            config['rabbitmq']['port'] = int(os.environ['RABBITMQ_PORT'])
+        if 'RABBITMQ_USER' in os.environ and os.environ['RABBITMQ_USER']:
+            config['rabbitmq']['user'] = os.environ['RABBITMQ_USER']
+        if 'RABBITMQ_PASSWORD' in os.environ and os.environ['RABBITMQ_PASSWORD']:
+            config['rabbitmq']['password'] = os.environ['RABBITMQ_PASSWORD']
+        if 'RABBITMQ_VHOST' in os.environ and os.environ['RABBITMQ_VHOST']:
+            config['rabbitmq']['virtual_host'] = os.environ['RABBITMQ_VHOST']
+        if 'consul' not in config:
+            config['consul'] = {'host': None, 'id': None}
+        if not config['consul']['id']:
+            if 'HOSTNAME' in os.environ and os.environ['HOSTNAME']:
+                config['consul']['id'] = os.environ['HOSTNAME']
+            else:
+                config['consul']['id'] = socket.gethostname()
+
+        if 'web' not in config:
+            config['web'] = {}
+        if 'BIOMAJ_HOSTNAME' in os.environ and os.environ['BIOMAJ_HOSTNAME']:
+            config['web']['hostname'] = os.environ['BIOMAJ_HOSTNAME']
+        else:
+            if 'HOSTNAME' in os.environ and os.environ['HOSTNAME']:
+                config['web']['hostname'] = os.environ['HOSTNAME']
+            else:
+                config['web']['hostname'] = socket.gethostname()
+
+        if 'docker' not in config:
+            config['docker'] = {}
+        if 'DOCKER_URL' in os.environ and os.environ['DOCKER_URL']:
+            config['docker']['url'] = os.environ['DOCKER_URL']
+        else:
+            config['docker']['url'] = None
+
+        if 'REDIS_HOST' in os.environ and os.environ['REDIS_HOST']:
+            config['redis']['host'] = os.environ['REDIS_HOST']
+        if 'REDIS_PORT' in os.environ and os.environ['REDIS_PORT']:
+            config['redis']['port'] = int(os.environ['REDIS_PORT'])
+        if 'REDIS_PREFIX' in os.environ and os.environ['REDIS_PREFIX']:
+            config['redis']['prefix'] = os.environ['REDIS_PREFIX']
+        if 'CONSUL_HOST' in os.environ and os.environ['CONSUL_HOST']:
+            config['consul']['host'] = os.environ['CONSUL_HOST']
+        if 'CONSUL_ID' in os.environ and os.environ['CONSUL_ID']:
+            config['consul']['id'] = os.environ['CONSUL_ID']
+        if 'WEB_PORT' in os.environ and os.environ['WEB_PORT']:
+            config['web']['port'] = int(os.environ['WEB_PORT'])
+        if 'WEB_LOCAL_ENDPOINT' in os.environ and os.environ['WEB_LOCAL_ENDPOINT']:
+            config['web']['local_endpoint'] = os.environ['WEB_LOCAL_ENDPOINT']
+        if 'mongo' not in config:
+            config['mongo'] = {'url': None, 'db': 'biomaj'}
+        if 'MONGO_URL' in os.environ and os.environ['MONGO_URL']:
+            config['mongo']['url'] = os.environ['MONGO_URL']
+        if 'MONGO_DB' in os.environ and os.environ['MONGO_DB']:
+            config['mongo']['db'] = os.environ['MONGO_DB']
+
+    @staticmethod
+    def get_folder_size(folder):
+        """
+        Get directory path full size
+
+        :param folder: directory path
+        :type folder: str
+        """
+        if not os.path.exists(folder):
+            return -1
+        folder_size = 0
+        for (path, dirs, files) in os.walk(folder):
+            for ffile in files:
+                filename = os.path.join(path, ffile)
+                folder_size += os.path.getsize(filename)
+        return folder_size
+
+    @staticmethod
+    def detect_format(filename):
+        """
+        try to detect file format by extension
+        """
+        if Utils.mime is None:
+            Utils.mime = MimeTypes()
+            mimesfile = os.path.join(os.path.dirname(__file__), 'mimes-bio.txt')
+            Utils.mime.read(mimesfile, True)
+        return Utils.mime.guess_type(filename, True)
+
+    @staticmethod
+    def get_more_recent_file(files):
+        """
+        Return the date of the most recent file in list.
+
+        Each file is a dict like with (at least) parameters: year, month, day
+        """
+        release = None
+        for rfile in files:
+            if release is None:
+                release = {'year': rfile['year'], 'month': rfile['month'], 'day': rfile['day']}
+            else:
+                rel_date = datetime.date(int(release['year']), int(release['month']), int(release['day']))
+                file_date = datetime.date(int(rfile['year']), int(rfile['month']), int(rfile['day']))
+                if file_date > rel_date:
+                    release['year'] = rfile['year']
+                    release['month'] = rfile['month']
+                    release['day'] = rfile['day']
+        return release
+
+    @staticmethod
+    def month_to_num(date):
+        return {
+            'Jan': 1,
+            'Feb': 2,
+            'Mar': 3,
+            'Apr': 4,
+            'May': 5,
+            'Jun': 6,
+            'Jul': 7,
+            'Aug': 8,
+            'Sep': 9,
+            'Oct': 10,
+            'Nov': 11,
+            'Dec': 12,
+            '01': 1,
+            '02': 2,
+            '03': 3,
+            '04': 4,
+            '05': 5,
+            '06': 6,
+            '07': 7,
+            '08': 8,
+            '09': 9,
+            '10': 10,
+            '11': 11,
+            '12': 12
+            }[date]
+
+    @staticmethod
+    def copy_files(files_to_copy, to_dir, move=False, lock=None):
+        """
+        Copy or move files to to_dir, keeping directory structure.
+
+        Copy keeps the original file stats.
+        Files should have attributes name and root:
+        - root: root directory
+        - name: relative path of file in root directory
+
+        /root/file/file1 will be copied in to_dir/file/file1
+
+        :param files_to_copy: list of files to copy
+        :type files_to_copy: list
+        :param to_dir: destination directory
+        :type to_dir: str
+        :param move: move instead of copy
+        :type move: bool
+        :param lock: thread lock object for multi-threads
+        :type lock: Lock
+        """
+        logger = logging.getLogger('biomaj')
+        nb_files = len(files_to_copy)
+        cur_files = 1
+        for file_to_copy in files_to_copy:
+            logger.debug(str(cur_files) + '/' + str(nb_files) + ' copy file ' + file_to_copy['name'])
+            cur_files += 1
+            from_file = file_to_copy['root'] + '/' + file_to_copy['name']
+            to_file = to_dir + '/' + file_to_copy['name']
+            if lock is not None:
+                lock.acquire()
+                try:
+                    if not os.path.exists(os.path.dirname(to_file)):
+                        os.makedirs(os.path.dirname(to_file))
+                except Exception as e:
+                    logger.error(e)
+                finally:
+                    lock.release()
+
+            else:
+                if not os.path.exists(os.path.dirname(to_file)):
+                    try:
+                        os.makedirs(os.path.dirname(to_file))
+                    except Exception as e:
+                        logger.error(e)
+            if move:
+                shutil.move(from_file, to_file)
+            else:
+                start_time = datetime.datetime.now()
+                start_time = time.mktime(start_time.timetuple())
+                shutil.copyfile(from_file, to_file)
+                end_time = datetime.datetime.now()
+                end_time = time.mktime(end_time.timetuple())
+                file_to_copy['download_time'] = end_time - start_time
+                shutil.copystat(from_file, to_file)
+
+    @staticmethod
+    def copy_files_with_regexp(from_dir, to_dir, regexps, move=False, lock=None):
+        """
+        Copy or move files from from_dir to to_dir matching regexps.
+        Copy keeps the original file stats.
+
+        :param from_dir: origin directory
+        :type from_dir: str
+        :param to_dir: destination directory
+        :type to_dir: str
+        :param regexps: list of regular expressions that files in from_dir should match to be copied
+        :type regexps: list
+        :param move: move instead of copy
+        :type move: bool
+        :param lock: thread lock object for multi-threads
+        :type lock: Lock
+        :return: list of copied files with their size
+        """
+        logger = logging.getLogger('biomaj')
+        files_to_copy = []
+        for root, dirs, files in os.walk(from_dir, topdown=True):
+            for name in files:
+                for reg in regexps:
+                    file_relative_path = os.path.join(root, name).replace(from_dir, '')
+                    if file_relative_path.startswith('/'):
+                        file_relative_path = file_relative_path.replace('/', '', 1)
+                    if reg == "**/*":
+                        files_to_copy.append({'name': file_relative_path})
+                        continue
+                    if re.match(reg, file_relative_path):
+                        files_to_copy.append({'name': file_relative_path})
+                        continue
+
+        for file_to_copy in files_to_copy:
+            from_file = from_dir + '/' + file_to_copy['name']
+            to_file = to_dir + '/' + file_to_copy['name']
+
+            if lock is not None:
+                lock.acquire()
+                try:
+                    if not os.path.exists(os.path.dirname(to_file)):
+                        os.makedirs(os.path.dirname(to_file))
+                except Exception as e:
+                    logger.error(e)
+                finally:
+                    lock.release()
+            else:
+                if not os.path.exists(os.path.dirname(to_file)):
+                    os.makedirs(os.path.dirname(to_file))
+            if move:
+                shutil.move(from_file, to_file)
+            else:
+                shutil.copyfile(from_file, to_file)
+                shutil.copystat(from_file, to_file)
+            file_to_copy['size'] = os.path.getsize(to_file)
+            f_stat = datetime.datetime.fromtimestamp(os.path.getmtime(to_file))
+            file_to_copy['year'] = str(f_stat.year)
+            file_to_copy['month'] = str(f_stat.month)
+            file_to_copy['day'] = str(f_stat.day)
+            (file_format, encoding) = Utils.detect_format(to_file)
+            file_to_copy['format'] = file_format
+        return files_to_copy
+
+    @staticmethod
+    def uncompress(archivefile, remove=True):
+        """
+        Test if file is an archive, and uncompress it
+        Remove archive file if specified
+
+        :param file: full path to file to check and uncompress
+        :type file: str
+        :param remove: remove archive if present
+        :type remove: bool
+        :return: True if ok, False if an error occured
+        """
+        is_archive = False
+        logger = logging.getLogger('biomaj')
+        try:
+            if archivefile.endswith('.tar.gz'):
+                subprocess.check_call("tar xfz " + archivefile + " --overwrite -C " + os.path.dirname(archivefile), shell=True)
+                is_archive = True
+            elif archivefile.endswith('.tar'):
+                subprocess.check_call("tar xf " + archivefile + " --overwrite -C " + os.path.dirname(archivefile), shell=True)
+                is_archive = True
+            elif archivefile.endswith('.bz2'):
+                subprocess.check_call("tar xjf " + archivefile + " --overwrite -C " + os.path.dirname(archivefile), shell=True)
+                is_archive = True
+            elif archivefile.endswith('.gz'):
+                subprocess.check_call("gunzip -f " + archivefile, shell=True)
+                is_archive = True
+            elif archivefile.endswith('.zip'):
+                subprocess.check_call("unzip -o " + archivefile + " -d " + os.path.dirname(archivefile), shell=True)
+                is_archive = True
+        except CalledProcessError as uncompresserror:
+            logger.error("Uncompress error of %s: %s" % (archivefile, str(uncompresserror)))
+            return False
+
+        if is_archive:
+            logger.debug('Uncompress:uncompress:' + archivefile)
+
+        if is_archive and remove and os.path.exists(archivefile):
+            os.remove(archivefile)
+
+        return True
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..89deaa9
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,5 @@
+nose
+mock
+future
+elasticsearch
+
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..3c6e79c
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal=1
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..6adbbc1
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,53 @@
+try:
+    from setuptools import setup, find_packages
+except ImportError:
+    from distutils.core import setup
+
+from distutils.command.install import install
+import os
+
+
+here = os.path.abspath(os.path.dirname(__file__))
+with open(os.path.join(here, 'README.md')) as f:
+    README = f.read()
+with open(os.path.join(here, 'CHANGES.txt')) as f:
+    CHANGES = f.read()
+
+
+config = {
+    'description': 'BioMAJ core library',
+    'long_description': README + '\n\n' + CHANGES,
+    'author': 'Olivier Sallou',
+    'url': 'http://biomaj.genouest.org',
+    'download_url': 'http://biomaj.genouest.org',
+    'author_email': 'olivier.sallou at irisa.fr',
+    'version': '3.0.9',
+     'classifiers': [
+        'Development Status :: 5 - Production/Stable',
+        'Environment :: Console',
+        'Natural Language :: English',
+        'Operating System :: POSIX :: Linux',
+        # Indicate who your project is intended for
+        'Intended Audience :: Science/Research',
+        'Topic :: Scientific/Engineering :: Bio-Informatics',
+        # Pick your license as you wish (should match "license" above)
+        'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
+        # Specify the Python versions you support here. In particular, ensure
+        # that you indicate whether you support Python 2, Python 3 or both.
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4'
+    ],
+    'install_requires': [
+                         'future',
+                         'elasticsearch'
+                        ],
+    'tests_require': ['nose', 'mock'],
+    'test_suite': 'nose.collector',
+    'packages': find_packages(),
+    'include_package_data': True,
+    'name': 'biomaj_core'
+}
+
+setup(**config)
diff --git a/tests/alu.properties b/tests/alu.properties
new file mode 100644
index 0000000..0e729e0
--- /dev/null
+++ b/tests/alu.properties
@@ -0,0 +1,43 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="alu.n : alu repeat element. alu.a : translation of alu.n repeats"
+db.name=alu
+db.type=nucleic_protein
+
+offline.dir.name=offline/ncbi/blast/alu_tmp
+dir.version=ncbi/blast/alu
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=ftp.ncbi.nih.gov
+remote.dir=/blast/db/FASTA/
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.files=^alu.*\.gz$
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^alu\.(a|n).*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/bank/process/test.sh b/tests/bank/process/test.sh
new file mode 100755
index 0000000..2d510e2
--- /dev/null
+++ b/tests/bank/process/test.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo "Testing a process"
+
+echo "test meta data"
+echo "##BIOMAJ#blast#nucleic#organism:hg19,chr:chr1#blast/chr1/chr1db"
+echo "##BIOMAJ#blast#nucleic#organism:hg19,chr:chr2#blast/chr2/chr2db"
+
+echo "test meta data 2"
+
+echo "##BIOMAJ#fasta#nucleic#organism:hg19#fasta/chr1.fa,fasta/chr2.fa"
diff --git a/tests/bank/test.fasta.gz b/tests/bank/test.fasta.gz
new file mode 100644
index 0000000..666d6f2
Binary files /dev/null and b/tests/bank/test.fasta.gz differ
diff --git a/tests/bank/test2.fasta b/tests/bank/test2.fasta
new file mode 100644
index 0000000..410ca0f
--- /dev/null
+++ b/tests/bank/test2.fasta
@@ -0,0 +1,2 @@
+>test2
+gcgcgcgcgcgcgcgccgcgcgcgcgcgcgcggc
diff --git a/tests/bank/test_100.txt b/tests/bank/test_100.txt
new file mode 100644
index 0000000..c7f7c3b
--- /dev/null
+++ b/tests/bank/test_100.txt
@@ -0,0 +1 @@
+This is a sample file to extract Release 103 from a text file
diff --git a/tests/biomaj_tests.py b/tests/biomaj_tests.py
new file mode 100644
index 0000000..a8a245d
--- /dev/null
+++ b/tests/biomaj_tests.py
@@ -0,0 +1,297 @@
+from nose.tools import *
+from nose.plugins.attrib import attr
+
+import shutil
+import os
+import tempfile
+import stat
+
+from biomaj_core.config import BiomajConfig
+from biomaj_core.utils import Utils
+from biomaj_core.bmajindex import BmajIndex
+
+import unittest
+
+
+class UtilsForTest:
+    """
+    Copy properties files to a temp directory and update properties to
+    use a temp directory
+    """
+
+    def __init__(self):
+        """
+        Setup the temp dirs and files.
+        """
+        self.global_properties = None
+        self.bank_properties = None
+
+        self.test_dir = tempfile.mkdtemp('biomaj')
+
+        self.conf_dir = os.path.join(self.test_dir, 'conf')
+        if not os.path.exists(self.conf_dir):
+            os.makedirs(self.conf_dir)
+        self.data_dir = os.path.join(self.test_dir, 'data')
+        if not os.path.exists(self.data_dir):
+            os.makedirs(self.data_dir)
+        self.log_dir = os.path.join(self.test_dir, 'log')
+        if not os.path.exists(self.log_dir):
+            os.makedirs(self.log_dir)
+        self.process_dir = os.path.join(self.test_dir, 'process')
+        if not os.path.exists(self.process_dir):
+            os.makedirs(self.process_dir)
+        self.lock_dir = os.path.join(self.test_dir, 'lock')
+        if not os.path.exists(self.lock_dir):
+            os.makedirs(self.lock_dir)
+        self.cache_dir = os.path.join(self.test_dir, 'cache')
+        if not os.path.exists(self.cache_dir):
+            os.makedirs(self.cache_dir)
+
+        if self.global_properties is None:
+            self.__copy_global_properties()
+
+        if self.bank_properties is None:
+            self.__copy_test_bank_properties()
+
+    def clean(self):
+        """
+        Deletes temp directory
+        """
+        shutil.rmtree(self.test_dir)
+
+    def __copy_test_bank_properties(self):
+        if self.bank_properties is not None:
+            return
+        self.bank_properties = ['alu', 'local', 'testhttp', 'directhttp',
+                                'noname', 'noexe', 'noargs', 'prenoname',
+                                'prenoexe', 'prenoargs', 'rmnoname', 'rmnoexe',
+                                'rmnoargs']
+        curdir = os.path.dirname(os.path.realpath(__file__))
+        for b in self.bank_properties:
+            from_file = os.path.join(curdir, b+'.properties')
+            to_file = os.path.join(self.conf_dir, b+'.properties')
+            shutil.copyfile(from_file, to_file)
+
+        self.bank_process = ['test.sh']
+        curdir = os.path.dirname(os.path.realpath(__file__))
+        procdir = os.path.join(curdir, 'bank/process')
+        for proc in self.bank_process:
+            from_file = os.path.join(procdir, proc)
+            to_file = os.path.join(self.process_dir, proc)
+            shutil.copyfile(from_file, to_file)
+            os.chmod(to_file, stat.S_IRWXU)
+
+        # Manage local bank test, use bank test subdir as remote
+        properties = ['multi.properties', 'computederror.properties',
+                      'error.properties', 'local.properties',
+                      'localprocess.properties', 'testhttp.properties',
+                      'computed.properties', 'computed2.properties',
+                      'sub1.properties', 'sub2.properties']
+        for prop in properties:
+            from_file = os.path.join(curdir, prop)
+            to_file = os.path.join(self.conf_dir, prop)
+            fout = open(to_file,'w')
+            with open(from_file,'r') as fin:
+                for line in fin:
+                    if line.startswith('remote.dir'):
+                        fout.write("remote.dir="+os.path.join(curdir,
+                                                              'bank') + "\n")
+                    elif line.startswith('remote.files'):
+                        fout.write(line.replace('/tmp', os.path.join(curdir,
+                                                                     'bank')))
+                    else:
+                        fout.write(line)
+            fout.close()
+
+    def __copy_global_properties(self):
+        if self.global_properties is not None:
+            return
+        self.global_properties = os.path.join(self.conf_dir, 'global.properties')
+        curdir = os.path.dirname(os.path.realpath(__file__))
+        global_template = os.path.join(curdir, 'global.properties')
+        fout = open(self.global_properties, 'w')
+        with open(global_template,'r') as fin:
+            for line in fin:
+                if line.startswith('conf.dir'):
+                    fout.write("conf.dir="+self.conf_dir+"\n")
+                elif line.startswith('log.dir'):
+                    fout.write("log.dir="+self.log_dir+"\n")
+                elif line.startswith('data.dir'):
+                    fout.write("data.dir="+self.data_dir+"\n")
+                elif line.startswith('process.dir'):
+                    fout.write("process.dir="+self.process_dir+"\n")
+                elif line.startswith('lock.dir'):
+                    fout.write("lock.dir="+self.lock_dir+"\n")
+                else:
+                    fout.write(line)
+        fout.close()
+
+
+class TestBiomajUtils(unittest.TestCase):
+
+    def setUp(self):
+        self.utils = UtilsForTest()
+
+    def tearDown(self):
+        self.utils.clean()
+
+    def test_properties_override(self):
+        BiomajConfig.load_config(self.utils.global_properties,
+                                 allow_user_config=False)
+        config = BiomajConfig('local')
+        ldap_host = config.get('ldap.host')
+        self.assertTrue(ldap_host == 'localhost')
+        os.environ['BIOMAJ_LDAP_HOST'] = 'someserver'
+        ldap_host = config.get('ldap.host')
+        self.assertTrue(ldap_host == 'someserver')
+
+    def test_service_config_override(self):
+        config = {
+            'rabbitmq': { 'host': '1.2.3.4'}
+        }
+        Utils.service_config_override(config)
+        self.assertTrue(config['rabbitmq']['host'] == '1.2.3.4')
+        os.environ['RABBITMQ_HOST'] = '4.3.2.1'
+        Utils.service_config_override(config)
+        self.assertTrue(config['rabbitmq']['host'] == '4.3.2.1')
+
+    def test_mimes(self):
+        fasta_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                                  'bank/test2.fasta')
+        (mime, encoding) = Utils.detect_format(fasta_file)
+        self.assertTrue('application/fasta' == mime)
+
+    @attr('compress')
+    def test_uncompress(self):
+        from_file = { 'root': os.path.dirname(os.path.realpath(__file__)),
+                      'name': 'bank/test.fasta.gz'
+                      }
+
+        to_dir = self.utils.data_dir
+        Utils.copy_files([from_file], to_dir)
+        Utils.uncompress(os.path.join(to_dir, from_file['name']))
+        self.assertTrue(os.path.exists(to_dir+'/bank/test.fasta'))
+
+    def test_copy_with_regexp(self):
+        from_dir = os.path.dirname(os.path.realpath(__file__))
+        to_dir = self.utils.data_dir
+        Utils.copy_files_with_regexp(from_dir, to_dir, ['.*\.py'])
+        self.assertTrue(os.path.exists(to_dir+'/biomaj_tests.py'))
+
+    def test_copy(self):
+        from_dir = os.path.dirname(os.path.realpath(__file__))
+        local_file = 'biomaj_tests.py'
+        files_to_copy = [ {'root': from_dir, 'name': local_file}]
+        to_dir = self.utils.data_dir
+        Utils.copy_files(files_to_copy, to_dir)
+        self.assertTrue(os.path.exists(to_dir+'/biomaj_tests.py'))
+
+    @attr('check')
+    def test_check_method(self):
+        """Check .name, .exe and .args are well check during bank configuration
+        checking"""
+        BiomajConfig.load_config(self.utils.global_properties)
+        for conf in ['noname', 'noexe', 'noargs', 'prenoname', 'prenoexe',
+                     'prenoargs', 'rmnoname', 'rmnoexe', 'rmnoargs']:
+            config = BiomajConfig(conf)
+            self.assertFalse(config.check())
+
+
+ at attr('elastic')
+class TestElastic(unittest.TestCase):
+    """
+    test indexing and search
+    """
+
+    def setUp(self):
+        BmajIndex.es = None
+        self.utils = UtilsForTest()
+        curdir = os.path.dirname(os.path.realpath(__file__))
+        BiomajConfig.load_config(self.utils.global_properties,
+                                 allow_user_config=False)
+        if BmajIndex.do_index == False:
+            self.skipTest("Skipping indexing tests due to elasticsearch not available")
+        # Delete all banks
+        b = Bank('local')
+        b.banks.remove({})
+        BmajIndex.delete_all_bank('local')
+
+        self.config = BiomajConfig('local')
+        data_dir = self.config.get('data.dir')
+        lock_file = os.path.join(data_dir,'local.lock')
+        if os.path.exists(lock_file):
+          os.remove(lock_file)
+
+    def tearDown(self):
+        data_dir = self.config.get('data.dir')
+        lock_file = os.path.join(data_dir,'local.lock')
+        if os.path.exists(lock_file):
+          os.remove(lock_file)
+        self.utils.clean()
+        BmajIndex.delete_all_bank('test')
+
+    def test_index(self):
+        BmajIndex.do_index = True
+        prod = {
+    			"data_dir" : "/tmp/test/data",
+    			"formats" : {
+    				"fasta" : [
+    					{
+    						"files" : [
+    							"fasta/chr1.fa",
+    							"fasta/chr2.fa"
+    						],
+    						"types" : [
+    							"nucleic"
+    						],
+    						"tags" : {
+    							"organism" : "hg19"
+    						}
+    					}
+    				],
+    				"blast": [
+    					{
+    						"files" : [
+    							"blast/chr1/chr1db"
+    						],
+    						"types" : [
+    							"nucleic"
+    						],
+    						"tags" : {
+    							"chr" : "chr1",
+    							"organism" : "hg19"
+    						}
+    					}
+    				]
+
+    			},
+    			"freeze" : False,
+    			"session" : 1416229253.930908,
+    			"prod_dir" : "alu-2003-11-26",
+    			"release" : "2003-11-26",
+    			"types" : [
+    				"nucleic"
+    			]
+    		}
+
+        BmajIndex.add('test',prod, True)
+
+        query = {
+          'query' : {
+            'match' : {'bank': 'test'}
+            }
+          }
+        res = BmajIndex.search(query)
+        self.assertTrue(len(res)==2)
+
+
+    def test_remove_all(self):
+        self.test_index()
+        query = {
+          'query' : {
+            'match' : {'bank': 'test'}
+            }
+          }
+        BmajIndex.delete_all_bank('test')
+        res = BmajIndex.search(query)
+        self.assertTrue(len(res)==0)
diff --git a/tests/computed.properties b/tests/computed.properties
new file mode 100644
index 0000000..214baf4
--- /dev/null
+++ b/tests/computed.properties
@@ -0,0 +1,44 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="computed local system bank test"
+db.name=local0
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/local0_tmp
+dir.version=test/local0
+
+depends=sub1
+sub1.files.move=flat/test_.*
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/tmp/
+remote.files=^test.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/computed2.properties b/tests/computed2.properties
new file mode 100644
index 0000000..2768000
--- /dev/null
+++ b/tests/computed2.properties
@@ -0,0 +1,45 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="computed local system bank test"
+db.name=local0
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/local0_tmp
+dir.version=test/local0
+
+depends=sub1
+
+ref.release=sub1
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/tmp/
+remote.files=^test.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/computederror.properties b/tests/computederror.properties
new file mode 100644
index 0000000..ce4bae1
--- /dev/null
+++ b/tests/computederror.properties
@@ -0,0 +1,43 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="computed error local system bank test"
+db.name=computederror
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/computederror_tmp
+dir.version=test/computederror
+
+depends=sub2,error
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/tmp/
+remote.files=^test.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/directhttp.properties b/tests/directhttp.properties
new file mode 100644
index 0000000..30f673d
--- /dev/null
+++ b/tests/directhttp.properties
@@ -0,0 +1,41 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="directhttp system bank test"
+db.name=directhttp
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/local_tmp
+dir.version=test/directhttp
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=directhttp
+server=ftp2.fr.debian.org
+
+release.protocol=directhttp
+release.server=ftp2.fr.debian.org
+release.remote.dir=/debian/README
+release.file=README
+release.regexp=([0-9.]+),
+release.file.compressed=
+
+#remote.dir=common/downloads/Current_Release/Pfalciparum3D7/fasta/data/PlasmoDB-25_Pfalciparum3D7_Genome.fasta
+#plasmo/communityDownload.do?fname=Atg3_alignment.txt
+remote.dir=/debian/README.html
+remote.files=
+
+local.files=debian/README.html
+
+## Post Process  ##  The files should be located in the projectfiles/process
+BLOCKS=
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/error.properties b/tests/error.properties
new file mode 100644
index 0000000..2e50f00
--- /dev/null
+++ b/tests/error.properties
@@ -0,0 +1,43 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="error local system bank test"
+db.name=error
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/error_tmp
+dir.version=test/error
+
+depends=sub2
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/error/
+remote.files=^error.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^error.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/global.properties b/tests/global.properties
new file mode 100644
index 0000000..75cb02c
--- /dev/null
+++ b/tests/global.properties
@@ -0,0 +1,123 @@
+[GENERAL]
+test=1
+conf.dir=/tmp/biomaj/config
+log.dir=/tmp/biomaj/log
+process.dir=/tmp/biomaj/process
+#The root directory where all databases are stored.
+#If your data is not stored under one directory hirearchy
+#you can override this value in the database properties file.
+data.dir=/tmp/biomaj/
+lock.dir=/tmp/biomaj/lock
+cache.dir=/tmp/biomaj/cache
+
+db.url=mongodb://localhost:27017
+db.name=biomaj_test
+
+use_ldap=1
+ldap.host=localhost
+ldap.port=389
+ldap.dn=nodomain
+
+# Use ElasticSearch for index/search capabilities
+use_elastic=0
+#Comma separated list of elasticsearch nodes  host1,host2:port2
+elastic_nodes=localhost
+elastic_index=biomaj_test
+
+celery.queue=biomaj
+celery.broker=mongodb://localhost:27017/biomaj_celery
+
+# Get directory stats (can be time consuming depending on number of files etc...)
+data.stats=1
+
+# List of user admin (linux user id, comma separated)
+admin=
+
+# Auto publish on updates (do not need publish flag, can be ovveriden in bank property file)
+auto_publish=0
+
+########################
+# Global properties file
+
+
+#To override these settings for a specific database go to its
+#properties file and uncomment or add the specific line you want
+#to override.
+
+#----------------
+# Mail Configuration
+#---------------
+#Uncomment thes lines if you want receive mail when the workflow is finished
+
+mail.smtp.host=
+mail.admin=
+mail.from=
+
+#---------------------
+#Proxy authentification
+#---------------------
+#proxyHost=
+#proxyPort=
+#proxyUser=
+#proxyPassword=
+
+#Number of thread for processes
+bank.num.threads=2
+
+#Number of threads to use for downloading
+files.num.threads=4
+
+#to keep more than one release increase this value
+keep.old.version=0
+
+#----------------------
+# Release configuration
+#----------------------
+release.separator=_
+
+#The historic log file is generated in log/
+#define level information for output : DEBUG,INFO,WARN,ERR
+historic.logfile.level=DEBUG
+
+#http.parse.dir.line=<a[\s]+href="([\S]+)/".*alt="\[DIR\]">.*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})
+http.parse.dir.line=<img[\s]+src="[\S]+"[\s]+alt="\[DIR\]"[\s]*/?>[\s]*<a[\s]+href="([\S]+)/"[\s]*>.*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})
+http.parse.file.line=<img[\s]+src="[\S]+"[\s]+alt="\[[\s]+\]"[\s]*/?>[\s]<a[\s]+href="([\S]+)".*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})[\s]+([\d\.]+[MKG]{0,1})
+
+http.group.dir.name=1
+http.group.dir.date=2
+http.group.file.name=1
+http.group.file.date=2
+http.group.file.size=3
+
+
+# Bank default access
+visibility.default=public
+
+
+[loggers]
+keys = root, biomaj
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = INFO
+handlers = console
+
+[logger_biomaj]
+level = DEBUG
+handlers = console
+qualname = biomaj
+propagate=0
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = DEBUG
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
diff --git a/tests/local.properties b/tests/local.properties
new file mode 100644
index 0000000..7f6f5fd
--- /dev/null
+++ b/tests/local.properties
@@ -0,0 +1,41 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=local
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/local_tmp
+dir.version=test/local
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/tmp/
+remote.files=^test.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/locallist.properties b/tests/locallist.properties
new file mode 100644
index 0000000..a901b2c
--- /dev/null
+++ b/tests/locallist.properties
@@ -0,0 +1,44 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=locallist
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/localprocess.properties b/tests/localprocess.properties
new file mode 100644
index 0000000..7166186
--- /dev/null
+++ b/tests/localprocess.properties
@@ -0,0 +1,100 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=local
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/local_tmp
+dir.version=test/local
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/tmp/
+remote.files=^test.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Pre process
+db.pre.process=META0
+
+## Remove process
+db.remove.process=META0
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+BLOCKS=BLOCK1,BLOCK2
+BLOCK1.db.post.process=META0
+BLOCK2.db.post.process=META1,META2,META3
+META0=PROC0
+META1=PROC1,PROC2
+META2=PROC3
+META3=PROC4,PROC5
+
+
+PROC0.name=test0
+PROC0.desc=sample test
+PROC0.cluster=false
+PROC0.type=test
+PROC0.exe=echo
+PROC0.args=test $datadir
+
+PROC1.name=test1
+PROC1.desc=sample test
+PROC1.cluster=false
+PROC1.type=test
+PROC1.exe=touch
+PROC1.args=$datadir/$dirversion/$localrelease/proc1.txt
+
+PROC2.name=test2
+PROC2.desc=sample test
+PROC2.cluster=false
+PROC2.type=test
+PROC2.exe=touch
+PROC2.args=$datadir/$dirversion/$localrelease/proc2.txt
+
+PROC3.name=test3
+PROC3.desc=sample test
+PROC3.cluster=false
+PROC3.type=test
+PROC3.exe=echo
+PROC3.args=test 3
+
+PROC4.name=test4
+PROC4.desc=sample test
+PROC4.cluster=false
+PROC4.type=test
+PROC4.exe=echo
+PROC4.args=test 4
+
+PROC5.name=test5
+PROC5.desc=sample test
+PROC5.cluster=false
+PROC5.type=testmetadata
+PROC5.exe=test.sh
+PROC5.args=
+PROC5.format=test
+PROC5.types=any
+PROC5.tags=chr:chr1,organism:hg19
+# If files is set, then the post-process does not have to print generated files on STDOUT (but can)
+# in this case, the list of files will be extracted from this list with above format/types/tags
+PROC5.files=dir1/file1,dir1/file2,dir1/file3
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/multi.properties b/tests/multi.properties
new file mode 100644
index 0000000..82e08f9
--- /dev/null
+++ b/tests/multi.properties
@@ -0,0 +1,60 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname=test for multi protocol
+db.name=multi
+db.type=test
+
+offline.dir.name=offline/multi_tmp
+dir.version=multi
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=multi
+server=
+remote.dir=
+
+remote.file.0.protocol = directhttp
+remote.file.0.server = httpbin.org
+remote.file.0.path = /get
+remote.file.0.params.keys = key1,key2
+remote.file.0.params.key1 = value1
+remote.file.0.params.key2 = value2
+remote.file.0.name = test1.json
+
+remote.file.1.protocol = directhttp
+remote.file.1.method = POST
+remote.file.1.server = httpbin.org
+remote.file.1.path = /post
+remote.file.1.params.keys = key1,key2
+remote.file.1.params.key1 = value1
+remote.file.1.params.key2 = value2
+remote.file.1.name = test2.json
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.files=^stable/Release$
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/noargs.properties b/tests/noargs.properties
new file mode 100644
index 0000000..36710c8
--- /dev/null
+++ b/tests/noargs.properties
@@ -0,0 +1,48 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=noargs
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+BLOCKS=PROC1
+PROC1.db.post.process=JOB1
+JOB1=job
+
+job.name=job_name
+job.exe=job_exe.sh
+job.desc=job desc
\ No newline at end of file
diff --git a/tests/noexe.properties b/tests/noexe.properties
new file mode 100644
index 0000000..ade982e
--- /dev/null
+++ b/tests/noexe.properties
@@ -0,0 +1,48 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=noexe
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+BLOCKS=PROC1
+PROC1.db.post.process=JOB1
+JOB1=job
+
+job.name=job_name
+job.args=args1 arg2
+job.desc=job desc
\ No newline at end of file
diff --git a/tests/noname.properties b/tests/noname.properties
new file mode 100644
index 0000000..518a0bf
--- /dev/null
+++ b/tests/noname.properties
@@ -0,0 +1,49 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=noname
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+BLOCKS=PROC1
+PROC1.db.post.process=JOB1
+JOB1=job
+
+
+job.exe=job_exe.sh
+job.args=args1 arg2
+job.desc=job desc
\ No newline at end of file
diff --git a/tests/prenoargs.properties b/tests/prenoargs.properties
new file mode 100644
index 0000000..e94c69f
--- /dev/null
+++ b/tests/prenoargs.properties
@@ -0,0 +1,45 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=prenoargs
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.pre.process=PREJOB1
+PREJOB1=prejob
+
+prejob.name=job_name
+prejob.exe=job_exe.sh
+prejob.desc=job desc
\ No newline at end of file
diff --git a/tests/prenoexe.properties b/tests/prenoexe.properties
new file mode 100644
index 0000000..8d0d89c
--- /dev/null
+++ b/tests/prenoexe.properties
@@ -0,0 +1,45 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=prenoexe
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.pre.process=PREJOB1
+PREJOB1=prejob
+
+prejob.name=job_name
+prejob.args=args1 arg2
+prejob.desc=job desc
\ No newline at end of file
diff --git a/tests/prenoname.properties b/tests/prenoname.properties
new file mode 100644
index 0000000..97dd1a5
--- /dev/null
+++ b/tests/prenoname.properties
@@ -0,0 +1,45 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=prenoname
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.pre.process=PREJOB1
+PREJOB1=prejob
+
+prejob.exe=job_exe.sh
+prejob.args=args1 arg2
+prejob.desc=job desc
\ No newline at end of file
diff --git a/tests/rmnoargs.properties b/tests/rmnoargs.properties
new file mode 100644
index 0000000..11b2d45
--- /dev/null
+++ b/tests/rmnoargs.properties
@@ -0,0 +1,45 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=rmnoargs
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.remove.process=RMJOB1
+RMJOB1=rmjob
+
+rmjob.name=job_name
+rmjob.exe=job_exe.sh
+rmjob.desc=job desc
\ No newline at end of file
diff --git a/tests/rmnoexe.properties b/tests/rmnoexe.properties
new file mode 100644
index 0000000..f5dcbaf
--- /dev/null
+++ b/tests/rmnoexe.properties
@@ -0,0 +1,45 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=rmnoexe
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.remove.process=RMJOB1
+RMJOB1=rmjob
+
+rmjob.name=job_name
+rmjob.args=args1 arg2
+rmjob.desc=job desc
\ No newline at end of file
diff --git a/tests/rmnoname.properties b/tests/rmnoname.properties
new file mode 100644
index 0000000..54cc993
--- /dev/null
+++ b/tests/rmnoname.properties
@@ -0,0 +1,45 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=rmnoname
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.remove.process=RMJOB1
+RMJOB1=rmjob
+
+rmjob.exe=job_exe.sh
+rmjob.args=args1 arg2
+rmjob.desc=job desc
\ No newline at end of file
diff --git a/tests/sub1.properties b/tests/sub1.properties
new file mode 100644
index 0000000..8e0c69b
--- /dev/null
+++ b/tests/sub1.properties
@@ -0,0 +1,43 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="sub local system bank test"
+db.name=local1
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/local1_tmp
+dir.version=test/local1
+
+depends=sub2
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/tmp/
+remote.files=^test.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/sub2.properties b/tests/sub2.properties
new file mode 100644
index 0000000..b9d3142
--- /dev/null
+++ b/tests/sub2.properties
@@ -0,0 +1,41 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="sub local system bank test"
+db.name=local2
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/local2_tmp
+dir.version=test/local2
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/tmp/
+remote.files=^test.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/testhttp.properties b/tests/testhttp.properties
new file mode 100644
index 0000000..454753e
--- /dev/null
+++ b/tests/testhttp.properties
@@ -0,0 +1,43 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname=test for http protocol
+db.name=testhttp
+db.type=package
+
+offline.dir.name=offline/testhttp_tmp
+dir.version=testhttp
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=http
+server=ftp2.fr.debian.org
+remote.dir=/debian/dists/
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.files=^stable/Release$
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^.*
+
+## Post Process  ##  The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+
+
+### Deployment ###
+
+keep.old.version=1

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/biomaj3-core.git



More information about the debian-med-commit mailing list