[med-svn] [biomaj3-core] 01/04: New upstream version 3.0.9
Olivier Sallou
osallou at debian.org
Thu Aug 17 07:44:19 UTC 2017
This is an automated email from the git hooks/post-receive script.
osallou pushed a commit to branch master
in repository biomaj3-core.
commit 38666e248d0cf8eff8d7a2fa3d4aefc3a82f05c4
Author: Olivier Sallou <osallou at debian.org>
Date: Thu Aug 17 07:20:40 2017 +0000
New upstream version 3.0.9
---
.gitignore | 72 +++++
.travis.yml | 31 ++
CHANGES.txt | 22 ++
LICENSE | 662 +++++++++++++++++++++++++++++++++++++++++
MANIFEST.in | 2 +
README.md | 10 +
biomaj_core/__init__.py | 0
biomaj_core/bmajindex.py | 213 +++++++++++++
biomaj_core/config.py | 482 ++++++++++++++++++++++++++++++
biomaj_core/mimes-bio.txt | 18 ++
biomaj_core/utils.py | 315 ++++++++++++++++++++
requirements.txt | 5 +
setup.cfg | 2 +
setup.py | 53 ++++
tests/alu.properties | 43 +++
tests/bank/process/test.sh | 11 +
tests/bank/test.fasta.gz | Bin 0 -> 45 bytes
tests/bank/test2.fasta | 2 +
tests/bank/test_100.txt | 1 +
tests/biomaj_tests.py | 297 ++++++++++++++++++
tests/computed.properties | 44 +++
tests/computed2.properties | 45 +++
tests/computederror.properties | 43 +++
tests/directhttp.properties | 41 +++
tests/error.properties | 43 +++
tests/global.properties | 123 ++++++++
tests/local.properties | 41 +++
tests/locallist.properties | 44 +++
tests/localprocess.properties | 100 +++++++
tests/multi.properties | 60 ++++
tests/noargs.properties | 48 +++
tests/noexe.properties | 48 +++
tests/noname.properties | 49 +++
tests/prenoargs.properties | 45 +++
tests/prenoexe.properties | 45 +++
tests/prenoname.properties | 45 +++
tests/rmnoargs.properties | 45 +++
tests/rmnoexe.properties | 45 +++
tests/rmnoname.properties | 45 +++
tests/sub1.properties | 43 +++
tests/sub2.properties | 41 +++
tests/testhttp.properties | 43 +++
42 files changed, 3367 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..754c2a7
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,72 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# Coveralls
+.coveralls.yml
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# PyCharm
+.idea
+
+# Vim
+.viminfo
+# Less history
+.lesshst
+
+.dbshell
+.emacs*
+.ipython
+.mongo*
+#*.properties
+
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..5f2c44e
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,31 @@
+language: python
+sudo: false
+python:
+ - "2.7"
+ - "3.4"
+ - "3.5"
+# Apply only on main branches
+branches:
+ except:
+ - /^feature.*$/
+# command to install dependencies
+#before_install:
+# - "sudo apt-get update -qq"
+# - "sudo apt-get install -qq libldap2-dev libsasl2-dev"
+install:
+ - "pip install flake8"
+ - "pip install -r requirements.txt"
+ - "pip install coverage"
+ - "pip install python-coveralls"
+ - "python setup.py -q install"
+# - "echo data_file=$TRAVIS_BUILD_DIR/.coverage >> .coveragerc"
+# command to run tests
+#before_script:
+# - sleep 10
+#script: nosetests --with-coverage --cover-package=biomaj -a '!network'
+#script: nosetests --with-coverage --cover-package=biomaj
+script:
+ - python setup.py test
+ - flake8 --ignore E501,E123 biomaj_core/*.py
+# - coveralls
+
diff --git a/CHANGES.txt b/CHANGES.txt
new file mode 100644
index 0000000..6964b4f
--- /dev/null
+++ b/CHANGES.txt
@@ -0,0 +1,22 @@
+3.0.9:
+ #1 Add checks on bank properties
+3.0.8:
+ Add DOCKER_URL, MONGO_URL, MONGO_DB env variable in override
+ Do not fail on elasticsearch index creation, just log
+ Add rsync check
+3.0.7:
+ set logging only if necessary
+3.0.6:
+ Fix configparser import for python 2 vs 3
+3.0.5:
+ Add config override with env variables
+3.0.4:
+ Code cleanup
+3.0.3:
+ Put index management in core
+3.0.2:
+ Add missing deps
+3.0.1:
+ Add README etc..
+3.0.0:
+ separation of biomaj and biomaj_core
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..cebe035
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,662 @@
+GNU AFFERO GENERAL PUBLIC LICENSE
+ Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+ A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate. Many developers of free software are heartened and
+encouraged by the resulting cooperation. However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+ The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community. It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server. Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+ An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals. This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU Affero General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Remote Network Interaction; Use with the GNU General Public License.
+
+ Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software. This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time. Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published
+ by the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source. For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code. There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<http://www.gnu.org/licenses/>.
+
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..5dcc15a
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,2 @@
+include *.txt *.md
+recursive-include biomaj_core *.txt
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..e6d7650
--- /dev/null
+++ b/README.md
@@ -0,0 +1,10 @@
+# Biomaj
+
+Biomaj core library
+
+
+Properties can be overriden by environment variables with pattern BIOMAJ_X_Y_Z for property x.y.z
+
+Example:
+
+use BIOMAJ_LDAP_HOST for ldap.host
diff --git a/biomaj_core/__init__.py b/biomaj_core/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/biomaj_core/bmajindex.py b/biomaj_core/bmajindex.py
new file mode 100644
index 0000000..a6c68bb
--- /dev/null
+++ b/biomaj_core/bmajindex.py
@@ -0,0 +1,213 @@
+from builtins import str
+from builtins import object
+import logging
+import copy
+from elasticsearch import Elasticsearch
+
+
+class BmajIndex(object):
+ """
+ ElasticSearch indexation and search
+ """
+
+ """
+ ElasticSearch server
+ """
+ es = None
+
+ """
+ Index name
+ """
+ index = 'biomaj'
+
+ """
+ Do indexing
+ """
+ do_index = False
+
+ """
+ Skip if failure (tests)
+ """
+ skip_if_failure = False
+
+ @staticmethod
+ def load(hosts=None, index='biomaj', do_index=True):
+ """
+ Initialize index
+
+ :param hosts: List of elastic search nodes to connect to
+ :type hosts: list
+ :param do_index: index data or not
+ :type do_index: bool
+ """
+ if hosts is None:
+ hosts = ['localhost']
+ if not do_index:
+ return
+ BmajIndex.index = index
+ BmajIndex.do_index = do_index
+ if BmajIndex.es is None:
+ BmajIndex.es = Elasticsearch(hosts)
+
+ mapping = {
+ "mappings": {
+ "production": {
+ "date_detection": False
+ },
+ "releasestats": {
+ "date_detection": False,
+ }
+ }
+ }
+ try:
+ if not BmajIndex.es.indices.exists(index=BmajIndex.index):
+ BmajIndex.es.indices.create(index=BmajIndex.index, body=mapping)
+ except Exception as e:
+ logging.error('ElasticSearch connection error, check server is running and configuration: ' + str(e))
+
+ @staticmethod
+ def _bulk_delete(query, flush=True):
+ try:
+ page = BmajIndex.es.search(
+ index=BmajIndex.index,
+ doc_type='production',
+ search_type="query_then_fetch",
+ size=1000,
+ body={
+ 'query': {
+ 'match': {
+ 'bank': query['bank']
+ }
+ }
+ })
+
+ if page is None:
+ return
+ bulk_delete = ''
+ for del_hit in page['hits']['hits']:
+ if ('release' in query and query['release'] == del_hit['_source']['release']) or 'release' not in query:
+ bulk_delete += "{ \"delete\" : {\"_index\":\"" + BmajIndex.index + "\",\"_type\":\"production\", \"_id\" : \"" + del_hit['_id'] + "\" } }\n"
+ if bulk_delete:
+ BmajIndex.es.bulk(body=bulk_delete)
+ if flush:
+ BmajIndex.es.indices.flush(index=BmajIndex.index, force=True)
+ except Exception as e:
+ if BmajIndex.skip_if_failure:
+ BmajIndex.do_index = False
+ else:
+ logging.error('Failed to delete bank index: ' + str(e))
+
+ @staticmethod
+ def delete_all_bank(bank_name):
+ """
+ Delete complete index for a bank
+ """
+ if not BmajIndex.do_index:
+ return
+ BmajIndex._bulk_delete({"bank": bank_name}, True)
+
+ @staticmethod
+ def remove(bank_name, release):
+ """
+ Remove a production release
+
+ :param bank_name: Name of the bank
+ :type bank_name: str
+ :param release: production release
+ :type release: str
+ """
+ if not BmajIndex.do_index:
+ return
+ BmajIndex._bulk_delete({"release": release, "bank": bank_name})
+ """
+ try:
+ query = {
+ "query" : {
+ "term" : {"release" : release, "bank": bank_name}
+ }
+ }
+ BmajIndex.es.delete_by_query(index=BmajIndex.index, body=query)
+ except Exception as e:
+ logging.error('Index:Remove:'+bank_name+'_'+str(release)+':Exception:'+str(e))
+ if BmajIndex.skip_if_failure:
+ BmajIndex.do_index = False
+ """
+
+ @staticmethod
+ def search(query):
+ if not BmajIndex.do_index:
+ return None
+ res = BmajIndex.es.search(index=BmajIndex.index,
+ doc_type='production',
+ search_type="query_then_fetch",
+ body=query)
+ return res['hits']['hits']
+
+ @staticmethod
+ def searchq(query, size=1000):
+ """
+ Lucene syntax search
+
+ :param query: Lucene search string
+ :type query: str
+ :param size: number of results
+ :type size: int
+ :return: list of matches
+ """
+ if not BmajIndex.do_index:
+ return None
+ res = BmajIndex.es.search(index=BmajIndex.index, doc_type='production', q=query, size=size)
+ return res['hits']['hits']
+
+ @staticmethod
+ def add_stat(stat_id, stat):
+ """
+ Add some statistics, must contain release and bank properties.
+ """
+ if not BmajIndex.do_index:
+ return
+ if stat['release'] is None or stat['bank'] is None:
+ return False
+ try:
+ BmajIndex.es.index(index=BmajIndex.index, doc_type='releasestats', id=stat_id, body=stat)
+ except Exception:
+ if BmajIndex.skip_if_failure:
+ BmajIndex.do_index = False
+ else:
+ return False
+ return True
+
+ @staticmethod
+ def add(bank_name, prod, flush=False):
+ """
+ Index a production release
+
+ :param bank_name: Name of the bank
+ :type bank_name: str
+ :param prod: session release object
+ :type prod: dict
+ :param flush: Force flushing
+ :type flush: bool
+ """
+ if not BmajIndex.do_index:
+ return
+ obj = copy.deepcopy(prod)
+ if obj['release'] is None:
+ return
+ obj['bank'] = bank_name
+ formats = obj['formats']
+ try:
+ for fkey, fvalue in formats.items():
+ for elt in fvalue:
+ elt['format'] = fkey
+ elt['bank'] = bank_name
+ elt['release'] = obj['release']
+ if 'status' in obj:
+ elt['status'] = obj['status']
+ BmajIndex.es.index(index=BmajIndex.index, doc_type='production', body=elt)
+ if flush:
+ BmajIndex.es.indices.flush(index=BmajIndex.index, force=True)
+ except Exception as e:
+ logging.error('Index:Add:' + bank_name + '_' + str(obj['release']) + ':Exception:' + str(e))
+ if BmajIndex.skip_if_failure:
+ BmajIndex.do_index = False
diff --git a/biomaj_core/config.py b/biomaj_core/config.py
new file mode 100644
index 0000000..9f60c5d
--- /dev/null
+++ b/biomaj_core/config.py
@@ -0,0 +1,482 @@
+from __future__ import print_function
+from future import standard_library
+from builtins import str
+from builtins import object
+import logging
+import logging.config
+import os
+import time
+import sys
+
+from biomaj_core.bmajindex import BmajIndex
+
+if sys.version < '3':
+ import ConfigParser as configparser
+else:
+ import configparser
+
+standard_library.install_aliases()
+
+
+class BiomajConfig(object):
+ """
+ Manage Biomaj configuration
+ """
+
+ DEFAULTS = {
+ 'http.parse.dir.line': r'<img[\s]+src="[\S]+"[\s]+alt="\[DIR\]"[\s]*/?>[\s]*<a[\s]+href="([\S]+)/"[\s]*>.*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})',
+ 'http.parse.file.line': r'<img[\s]+src="[\S]+"[\s]+alt="\[[\s]+\]"[\s]*/?>[\s]<a[\s]+href="([\S]+)".*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})[\s]+([\d\.]+[MKG]{0,1})',
+ 'http.group.dir.name': 1,
+ 'http.group.dir.date': 2,
+ 'http.group.file.name': 1,
+ 'http.group.file.date': 2,
+ 'http.group.file.size': 3,
+ 'visibility.default': 'public',
+ 'historic.logfile.level': 'INFO',
+ 'bank.num.threads': 2,
+ 'files.num.threads': 4,
+ 'use_elastic': 0,
+ 'use_drmaa': 0,
+ 'db.type': '',
+ 'db.formats': '',
+ 'keep.old.version': 1,
+ 'docker.sudo': '1',
+ 'auto_publish': 0
+ }
+
+ # Old biomaj level compatibility
+ LOGLEVEL = {
+ 'DEBUG': logging.DEBUG,
+ 'VERBOSE': logging.INFO,
+ 'INFO': logging.INFO,
+ 'WARN': logging.WARNING,
+ 'ERR': logging.ERROR
+ }
+
+ """
+ Global configuration file
+ """
+ global_config = None
+
+ """
+ Per use global configuration file, overriding global_config
+ """
+ user_config = None
+ allow_user_config = False
+
+ @staticmethod
+ def load_config(config_file=None, allow_user_config=True):
+ """
+ Loads general config
+
+ :param config_file: global.properties file path
+ :type config_file: str
+ :param allow_user_config: use ~/.biomaj.cfg if present
+ :type allow_user_config: bool
+ """
+ BiomajConfig.allow_user_config = allow_user_config
+
+ if config_file is None:
+ env_file = os.environ.get('BIOMAJ_CONF')
+ if env_file is not None and os.path.exists(env_file):
+ config_file = env_file
+ else:
+ env_file = 'global.properties'
+ if os.path.exists(env_file):
+ config_file = env_file
+
+ if config_file is None or not os.path.exists(config_file):
+ raise Exception('Missing global configuration file')
+
+ BiomajConfig.config_file = os.path.abspath(config_file)
+
+ BiomajConfig.global_config = configparser.ConfigParser()
+
+ # if allow_user_config and os.path.exists(os.path.expanduser('~/.biomaj.cfg')):
+ # BiomajConfig.user_config_file = os.path.expanduser('~/.biomaj.cfg')
+ # BiomajConfig.user_config = configparser.ConfigParser()
+ # BiomajConfig.user_config.read([os.path.expanduser('~/.biomaj.cfg')])
+ # else:
+ # BiomajConfig.user_config_file = None
+
+ BiomajConfig.global_config.read([config_file])
+
+ # ElasticSearch indexation support
+ do_index = False
+ if BiomajConfig.global_config.get('GENERAL', 'use_elastic') and \
+ BiomajConfig.global_config.get('GENERAL', 'use_elastic') == "1":
+ do_index = True
+ if do_index:
+ if BiomajConfig.global_config.get('GENERAL', 'elastic_nodes'):
+ elastic_hosts = BiomajConfig.global_config.get('GENERAL',
+ 'elastic_nodes')\
+ .split(',')
+ else:
+ elastic_hosts = ['localhost']
+ elastic_index = BiomajConfig.global_config.get('GENERAL', 'elastic_index')
+ if elastic_index is None:
+ elastic_index = 'biomaj'
+
+ if BiomajConfig.global_config.has_option('GENERAL', 'test') and \
+ BiomajConfig.global_config.get('GENERAL', 'test') == "1":
+ # Test connection to elasticsearch, if not available skip indexing for tests
+ BmajIndex.skip_if_failure = True
+
+ BmajIndex.load(index=elastic_index,
+ hosts=elastic_hosts,
+ do_index=do_index)
+
+ def __init__(self, bank, options=None):
+ """
+ Loads bank configuration
+
+ :param bank: bank name
+ :type bank: str
+ :param options: bank options
+ :type options: argparse
+ """
+ self.log_level = logging.INFO
+ self.name = bank
+ if BiomajConfig.global_config is None:
+ BiomajConfig.load_config()
+ self.config_bank = configparser.ConfigParser()
+ conf_dir = BiomajConfig.global_config.get('GENERAL', 'conf.dir')
+ if not os.path.exists(os.path.join(conf_dir, bank + '.properties')):
+ logging.error('Bank configuration file does not exists')
+ raise Exception('Configuration file ' + bank +
+ '.properties does not exists')
+ try:
+ config_files = [BiomajConfig.config_file]
+ self.user_config_file = None
+ if self.allow_user_config and hasattr(options, 'user')\
+ and options.user\
+ and os.path.exists(os.path.expanduser('~' + options.user + '/.biomaj.cfg')):
+ self.user_config_file = os.path.expanduser('~/' + options.user +
+ '.biomaj.cfg')
+ self.user_config = configparser.ConfigParser()
+ self.user_config.read([os.path.expanduser('~/' + options.user +
+ '.biomaj.cfg')])
+
+ if self.user_config_file is not None:
+ config_files.append(self.user_config_file)
+ config_files.append(os.path.join(conf_dir, bank + '.properties'))
+ self.config_bank.read(config_files)
+ except Exception as e:
+ logging.error("Configuration file error " + str(e))
+ raise Exception("Configuration file error " + str(e))
+
+ self.last_modified = int(os.stat(os.path.join(conf_dir, bank +
+ '.properties')).st_mtime)
+
+ do_log = False
+ if options is None:
+ do_log = True
+ elif hasattr(options, 'no_log') and not options.no_log:
+ do_log = True
+ elif type(options) is dict and 'no_log' in options and not options['no_log']:
+ do_log = True
+
+ if do_log:
+ if os.path.exists(os.path.expanduser('~/.biomaj.cfg')):
+ logging.config.fileConfig(os.path.expanduser('~/.biomaj.cfg'))
+ else:
+ logging.config.fileConfig(BiomajConfig.config_file)
+
+ logger = logging.getLogger()
+ bank_log_dir = os.path.join(self.get('log.dir'), bank, str(time.time()))
+ if not os.path.exists(bank_log_dir):
+ os.makedirs(bank_log_dir)
+ hdlr = logging.FileHandler(os.path.join(bank_log_dir, bank + '.log'))
+ self.log_file = os.path.join(bank_log_dir, bank + '.log')
+ if options is not None and options.get_option('log') is not None:
+ hdlr.setLevel(BiomajConfig.LOGLEVEL[options.get_option('log')])
+ self.log_level = BiomajConfig.LOGLEVEL[options.get_option('log')]
+ else:
+ hdlr.setLevel(BiomajConfig.LOGLEVEL[self.get('historic.logfile.level')])
+ self.log_level = BiomajConfig.LOGLEVEL[self.get('historic.logfile.level')]
+ formatter = logging.Formatter('%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s')
+ hdlr.setFormatter(formatter)
+ logger.addHandler(hdlr)
+ '''
+ # For monolithic, update default logger/handlers?
+ existing_logger_names = list(logging.getLogger().manager.loggerDict.keys())
+ for existing_logger in existing_logger_names:
+ if existing_logger.startswith('biomaj'):
+ tmp_logger = logging.getLogger(existing_logger)
+ tmp_logger.setLevel(self.log_level)
+ for handler in tmp_logger.handlers:
+ handler.setLevel(self.log_level)
+ '''
+ else:
+ self.log_file = 'none'
+
+ cache_dir = self.get('cache.dir')
+ if cache_dir is None:
+ print("Configuration file error: cache.dir empty")
+ logging.error("cache.dir is not defined")
+ sys.exit(1)
+
+ if not os.path.exists(cache_dir):
+ os.makedirs(cache_dir)
+
+ process_dir = self.get('process.dir')
+ if process_dir is None:
+ print("Configuration file error: process.dir empty")
+ logging.error("process.dir is not defined")
+ sys.exit(1)
+
+ if not os.path.exists(process_dir):
+ os.makedirs(process_dir)
+
+ data_dir = self.get('data.dir')
+ if data_dir is None:
+ print("Configuration file error: data.dir empty")
+ logging.error("data.dir is not defined")
+ sys.exit(1)
+
+ if not os.path.exists(data_dir):
+ os.makedirs(data_dir)
+
+ lock_dir = self.get('lock.dir')
+ if lock_dir is None:
+ print("Configuration file error: lock.dir empty")
+ logging.error("lock.dir is not defined")
+ sys.exit(1)
+
+ if not os.path.exists(lock_dir):
+ os.makedirs(lock_dir)
+
+ def reset_logger(self):
+ '''
+ Reconfigure logging
+ '''
+ if self.log_file == 'none':
+ return
+ if os.path.exists(os.path.expanduser('~/.biomaj.cfg')):
+ logging.config.fileConfig(os.path.expanduser('~/.biomaj.cfg'))
+ else:
+ logging.config.fileConfig(BiomajConfig.config_file)
+
+ logger = logging.getLogger()
+ hdlr = logging.FileHandler(self.log_file)
+ hdlr.setLevel(self.log_level)
+ formatter = logging.Formatter('%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s')
+ hdlr.setFormatter(formatter)
+ logger.addHandler(hdlr)
+
+ def set(self, prop, value, section='GENERAL'):
+ self.config_bank.set(section, prop, value)
+
+ def _in_env(self, prop):
+ '''
+ Search for env variable BIOMAJ_X_Y_Z for property X.Y.Z
+ '''
+ env_prop = 'BIOMAJ_' + prop.upper().replace('.', '_')
+ if env_prop in os.environ:
+ return os.environ[env_prop]
+ else:
+ return None
+
+ def get_bool(self, prop, section='GENERAL', escape=True, default=None):
+ """
+ Get a boolean property from bank or general configration. Optionally in section.
+ """
+ value = None
+
+ if self._in_env(prop):
+ value = self._in_env(prop)
+ else:
+ value = self.get(prop, section, escape, default)
+
+ if value is None:
+ return False
+ if value is True or value == 'true' or value == '1':
+ return True
+ else:
+ return False
+
+ def get(self, prop, section='GENERAL', escape=True, default=None):
+ """
+ Get a property from bank or general configration. Optionally in section.
+ """
+ if self._in_env(prop):
+ return self._in_env(prop)
+ # Compatibility fields
+ if prop == 'depends':
+ depend = self.get('db.source', section, escape, None)
+ if depend:
+ return depend
+
+ if self.config_bank.has_option(section, prop):
+ val = self.config_bank.get(section, prop)
+ if prop == 'remote.dir' and not val.endswith('/'):
+ val = val + '/'
+ # If regexp, escape backslashes
+ if escape and (prop == 'local.files' or prop == 'remote.files' or
+ prop == 'http.parse.dir.line' or
+ prop == 'http.parse.file.line'):
+ val = val.replace('\\\\', '\\')
+ return val
+
+ if self.user_config is not None:
+ if self.user_config.has_option(section, prop):
+ return self.user_config.get(section, prop)
+
+ if BiomajConfig.global_config.has_option(section, prop):
+ return BiomajConfig.global_config.get(section, prop)
+
+ if prop in BiomajConfig.DEFAULTS:
+ return BiomajConfig.DEFAULTS[prop]
+
+ return default
+
+ def get_time(self):
+ """
+ Return last modification time of config files
+ """
+ return self.last_modified
+
+ def check(self):
+ """
+ Check configuration
+ """
+ self.set('localrelease', '')
+ self.set('remoterelease', '')
+
+ status = True
+ if not self.get('data.dir'):
+ logging.error('data.dir is not set')
+ status = False
+ if not self.get('conf.dir'):
+ logging.error('conf.dir is not set')
+ status = False
+ if not self.get('log.dir'):
+ logging.error('log.dir is not set')
+ status = False
+ if not self.get('process.dir'):
+ logging.error('process.dir is not set')
+ status = False
+ if not self.get('lock.dir'):
+ logging.error('lock.dir is not set')
+ status = False
+ if not self.get('cache.dir'):
+ logging.error('cache.dir is not set')
+ status = False
+ if not self.get('db.fullname'):
+ logging.warn('db.fullname is not set')
+ if not self.get('db.formats'):
+ logging.warn('db.formats is not set')
+ if self.get('use_ldap'):
+ if not self.get('ldap.host') or not self.get('ldap.port') or \
+ not self.get('ldap.dn'):
+ logging.error('use_ldap set to 1 but missing configuration')
+ status = False
+ if self.get('use_elastic'):
+ if not self.get('elastic_nodes') or not self.get('elastic_index'):
+ logging.error('use_elastic set to 1 but missing configuration')
+ status = False
+
+ # if not self.get('celery.queue') or not self.get('celery.broker'):
+ # logging.warn('celery config is not set, that\'s fine if you do not use Celery for background tasks')
+
+ if not self.get('mail.smtp.host'):
+ logging.warn('SMTP mail config not set, you will not be able to send emails')
+ if self.get('mail.smtp.host') and not self.get('mail.from'):
+ logging.error('Mail origin mail.from not set')
+ status = False
+
+ if not self.get('offline.dir.name'):
+ logging.error('offline.dir.name is not set')
+ status = False
+ elif self.get('offline.dir.name').startswith('/'):
+ logging.error('offline dir must be relative to data.dir and should not start with a /')
+ status = False
+ if not self.get('dir.version'):
+ logging.error('dir.version is not set')
+ status = False
+ elif self.get('dir.version').startswith('/'):
+ logging.error('dir.version must be relative to data.dir and should not start with a /')
+ status = False
+ if not self.get('protocol'):
+ logging.error('protocol is not set')
+ status = False
+ else:
+ protocol = self.get('protocol')
+ allowed_protocols = ['none', 'multi', 'local', 'ftp', 'sftp', 'http',
+ 'https', 'directftp', 'directhttp', 'directhttps', 'rsync']
+ if protocol not in allowed_protocols:
+ logging.error('Protocol not supported: ' + protocol)
+ status = False
+ if protocol not in ['multi', 'none']:
+ if protocol != 'local' and not self.get('server'):
+ logging.error('server not set')
+ status = False
+ if not self.get('remote.dir'):
+ logging.error('remote.dir not set')
+ status = False
+ elif not self.get('remote.dir').endswith('/'):
+ logging.error('remote.dir must end with a /')
+ return False
+ if protocol not in ['direcftp', 'directhttp', 'directhttps'] and\
+ not self.get('remote.files') and\
+ not self.get('remote.list'):
+ logging.error('remote.files not set')
+ status = False
+ if not self.get('local.files'):
+ logging.error('local.files is not set')
+ status = False
+ # Remove processes
+ processes = ['db.remove.process', 'db.pre.process']
+ for process in processes:
+ if self.get(process):
+ metas = self.get(process).split(',')
+ for meta in metas:
+ if not self.get(meta):
+ logging.error('Metaprocess ' + meta + ' not defined')
+ status = False
+ else:
+ procs = self.get(meta).split(',')
+ for proc in procs:
+ if not self.get(proc + '.name'):
+ logging.error('Process ' + proc + ' not defined')
+ status = False
+ else:
+ if not self.get(proc + '.exe'):
+ logging.error('Process exe for ' + proc +
+ ' not defined')
+ status = False
+ if not self.get(proc + '.args'):
+ logging.error(
+ 'Process args for ' + proc +
+ ' not defined')
+ status = False
+ # Check blocks
+ if self.get('BLOCKS'):
+ blocks = self.get('BLOCKS').split(',')
+ for block in blocks:
+ if not self.get(block + '.db.post.process'):
+ logging.error('Block ' + block + ' not defined')
+ status = False
+ else:
+ metas = self.get(block + '.db.post.process').split(',')
+ for meta in metas:
+ if not self.get(meta):
+ logging.error('Metaprocess ' + meta + ' not defined')
+ status = False
+ else:
+ procs = self.get(meta).split(',')
+ for proc in procs:
+ if not self.get(proc + '.name'):
+ logging.error('Process ' + proc +
+ ' not defined')
+ status = False
+ else:
+ if not self.get(proc + '.exe'):
+ logging.error('Process exe for ' +
+ proc + ' not defined')
+ status = False
+ if not self.get(proc + '.args'):
+ logging.error('Process args for ' +
+ proc + ' not defined')
+ status = False
+ return status
diff --git a/biomaj_core/mimes-bio.txt b/biomaj_core/mimes-bio.txt
new file mode 100644
index 0000000..c794bf6
--- /dev/null
+++ b/biomaj_core/mimes-bio.txt
@@ -0,0 +1,18 @@
+# Biological file mime types
+application/fasta fasta fa fsa
+application/bam bam bai
+application/gff gff gff3
+application/bed bed
+application/fastq fastq
+application/gtf gtf
+application/octet-stream ab1 scf
+application/axt axt
+application/csFasta csfasta
+application/FasttqSolexa fastqsolexa
+application/Interval interval
+application/Laj laj
+application/Lav lav
+application/Maf maf
+application/QualityScore qual
+application/BlastXml blastxml
+application/Wiggle wig
diff --git a/biomaj_core/utils.py b/biomaj_core/utils.py
new file mode 100644
index 0000000..56f8eea
--- /dev/null
+++ b/biomaj_core/utils.py
@@ -0,0 +1,315 @@
+import os
+import re
+import logging
+import shutil
+import datetime
+import time
+import subprocess
+from subprocess import CalledProcessError
+import socket
+
+from mimetypes import MimeTypes
+
+
+class Utils(object):
+ """
+ Utility classes
+ """
+
+ mime = None
+
+ @staticmethod
+ def service_config_override(config):
+ if 'RABBITMQ_HOST' in os.environ and os.environ['RABBITMQ_HOST']:
+ config['rabbitmq']['host'] = os.environ['RABBITMQ_HOST']
+ if 'RABBITMQ_PORT' in os.environ and os.environ['RABBITMQ_PORT']:
+ config['rabbitmq']['port'] = int(os.environ['RABBITMQ_PORT'])
+ if 'RABBITMQ_USER' in os.environ and os.environ['RABBITMQ_USER']:
+ config['rabbitmq']['user'] = os.environ['RABBITMQ_USER']
+ if 'RABBITMQ_PASSWORD' in os.environ and os.environ['RABBITMQ_PASSWORD']:
+ config['rabbitmq']['password'] = os.environ['RABBITMQ_PASSWORD']
+ if 'RABBITMQ_VHOST' in os.environ and os.environ['RABBITMQ_VHOST']:
+ config['rabbitmq']['virtual_host'] = os.environ['RABBITMQ_VHOST']
+ if 'consul' not in config:
+ config['consul'] = {'host': None, 'id': None}
+ if not config['consul']['id']:
+ if 'HOSTNAME' in os.environ and os.environ['HOSTNAME']:
+ config['consul']['id'] = os.environ['HOSTNAME']
+ else:
+ config['consul']['id'] = socket.gethostname()
+
+ if 'web' not in config:
+ config['web'] = {}
+ if 'BIOMAJ_HOSTNAME' in os.environ and os.environ['BIOMAJ_HOSTNAME']:
+ config['web']['hostname'] = os.environ['BIOMAJ_HOSTNAME']
+ else:
+ if 'HOSTNAME' in os.environ and os.environ['HOSTNAME']:
+ config['web']['hostname'] = os.environ['HOSTNAME']
+ else:
+ config['web']['hostname'] = socket.gethostname()
+
+ if 'docker' not in config:
+ config['docker'] = {}
+ if 'DOCKER_URL' in os.environ and os.environ['DOCKER_URL']:
+ config['docker']['url'] = os.environ['DOCKER_URL']
+ else:
+ config['docker']['url'] = None
+
+ if 'REDIS_HOST' in os.environ and os.environ['REDIS_HOST']:
+ config['redis']['host'] = os.environ['REDIS_HOST']
+ if 'REDIS_PORT' in os.environ and os.environ['REDIS_PORT']:
+ config['redis']['port'] = int(os.environ['REDIS_PORT'])
+ if 'REDIS_PREFIX' in os.environ and os.environ['REDIS_PREFIX']:
+ config['redis']['prefix'] = os.environ['REDIS_PREFIX']
+ if 'CONSUL_HOST' in os.environ and os.environ['CONSUL_HOST']:
+ config['consul']['host'] = os.environ['CONSUL_HOST']
+ if 'CONSUL_ID' in os.environ and os.environ['CONSUL_ID']:
+ config['consul']['id'] = os.environ['CONSUL_ID']
+ if 'WEB_PORT' in os.environ and os.environ['WEB_PORT']:
+ config['web']['port'] = int(os.environ['WEB_PORT'])
+ if 'WEB_LOCAL_ENDPOINT' in os.environ and os.environ['WEB_LOCAL_ENDPOINT']:
+ config['web']['local_endpoint'] = os.environ['WEB_LOCAL_ENDPOINT']
+ if 'mongo' not in config:
+ config['mongo'] = {'url': None, 'db': 'biomaj'}
+ if 'MONGO_URL' in os.environ and os.environ['MONGO_URL']:
+ config['mongo']['url'] = os.environ['MONGO_URL']
+ if 'MONGO_DB' in os.environ and os.environ['MONGO_DB']:
+ config['mongo']['db'] = os.environ['MONGO_DB']
+
+ @staticmethod
+ def get_folder_size(folder):
+ """
+ Get directory path full size
+
+ :param folder: directory path
+ :type folder: str
+ """
+ if not os.path.exists(folder):
+ return -1
+ folder_size = 0
+ for (path, dirs, files) in os.walk(folder):
+ for ffile in files:
+ filename = os.path.join(path, ffile)
+ folder_size += os.path.getsize(filename)
+ return folder_size
+
+ @staticmethod
+ def detect_format(filename):
+ """
+ try to detect file format by extension
+ """
+ if Utils.mime is None:
+ Utils.mime = MimeTypes()
+ mimesfile = os.path.join(os.path.dirname(__file__), 'mimes-bio.txt')
+ Utils.mime.read(mimesfile, True)
+ return Utils.mime.guess_type(filename, True)
+
+ @staticmethod
+ def get_more_recent_file(files):
+ """
+ Return the date of the most recent file in list.
+
+ Each file is a dict like with (at least) parameters: year, month, day
+ """
+ release = None
+ for rfile in files:
+ if release is None:
+ release = {'year': rfile['year'], 'month': rfile['month'], 'day': rfile['day']}
+ else:
+ rel_date = datetime.date(int(release['year']), int(release['month']), int(release['day']))
+ file_date = datetime.date(int(rfile['year']), int(rfile['month']), int(rfile['day']))
+ if file_date > rel_date:
+ release['year'] = rfile['year']
+ release['month'] = rfile['month']
+ release['day'] = rfile['day']
+ return release
+
+ @staticmethod
+ def month_to_num(date):
+ return {
+ 'Jan': 1,
+ 'Feb': 2,
+ 'Mar': 3,
+ 'Apr': 4,
+ 'May': 5,
+ 'Jun': 6,
+ 'Jul': 7,
+ 'Aug': 8,
+ 'Sep': 9,
+ 'Oct': 10,
+ 'Nov': 11,
+ 'Dec': 12,
+ '01': 1,
+ '02': 2,
+ '03': 3,
+ '04': 4,
+ '05': 5,
+ '06': 6,
+ '07': 7,
+ '08': 8,
+ '09': 9,
+ '10': 10,
+ '11': 11,
+ '12': 12
+ }[date]
+
+ @staticmethod
+ def copy_files(files_to_copy, to_dir, move=False, lock=None):
+ """
+ Copy or move files to to_dir, keeping directory structure.
+
+ Copy keeps the original file stats.
+ Files should have attributes name and root:
+ - root: root directory
+ - name: relative path of file in root directory
+
+ /root/file/file1 will be copied in to_dir/file/file1
+
+ :param files_to_copy: list of files to copy
+ :type files_to_copy: list
+ :param to_dir: destination directory
+ :type to_dir: str
+ :param move: move instead of copy
+ :type move: bool
+ :param lock: thread lock object for multi-threads
+ :type lock: Lock
+ """
+ logger = logging.getLogger('biomaj')
+ nb_files = len(files_to_copy)
+ cur_files = 1
+ for file_to_copy in files_to_copy:
+ logger.debug(str(cur_files) + '/' + str(nb_files) + ' copy file ' + file_to_copy['name'])
+ cur_files += 1
+ from_file = file_to_copy['root'] + '/' + file_to_copy['name']
+ to_file = to_dir + '/' + file_to_copy['name']
+ if lock is not None:
+ lock.acquire()
+ try:
+ if not os.path.exists(os.path.dirname(to_file)):
+ os.makedirs(os.path.dirname(to_file))
+ except Exception as e:
+ logger.error(e)
+ finally:
+ lock.release()
+
+ else:
+ if not os.path.exists(os.path.dirname(to_file)):
+ try:
+ os.makedirs(os.path.dirname(to_file))
+ except Exception as e:
+ logger.error(e)
+ if move:
+ shutil.move(from_file, to_file)
+ else:
+ start_time = datetime.datetime.now()
+ start_time = time.mktime(start_time.timetuple())
+ shutil.copyfile(from_file, to_file)
+ end_time = datetime.datetime.now()
+ end_time = time.mktime(end_time.timetuple())
+ file_to_copy['download_time'] = end_time - start_time
+ shutil.copystat(from_file, to_file)
+
+ @staticmethod
+ def copy_files_with_regexp(from_dir, to_dir, regexps, move=False, lock=None):
+ """
+ Copy or move files from from_dir to to_dir matching regexps.
+ Copy keeps the original file stats.
+
+ :param from_dir: origin directory
+ :type from_dir: str
+ :param to_dir: destination directory
+ :type to_dir: str
+ :param regexps: list of regular expressions that files in from_dir should match to be copied
+ :type regexps: list
+ :param move: move instead of copy
+ :type move: bool
+ :param lock: thread lock object for multi-threads
+ :type lock: Lock
+ :return: list of copied files with their size
+ """
+ logger = logging.getLogger('biomaj')
+ files_to_copy = []
+ for root, dirs, files in os.walk(from_dir, topdown=True):
+ for name in files:
+ for reg in regexps:
+ file_relative_path = os.path.join(root, name).replace(from_dir, '')
+ if file_relative_path.startswith('/'):
+ file_relative_path = file_relative_path.replace('/', '', 1)
+ if reg == "**/*":
+ files_to_copy.append({'name': file_relative_path})
+ continue
+ if re.match(reg, file_relative_path):
+ files_to_copy.append({'name': file_relative_path})
+ continue
+
+ for file_to_copy in files_to_copy:
+ from_file = from_dir + '/' + file_to_copy['name']
+ to_file = to_dir + '/' + file_to_copy['name']
+
+ if lock is not None:
+ lock.acquire()
+ try:
+ if not os.path.exists(os.path.dirname(to_file)):
+ os.makedirs(os.path.dirname(to_file))
+ except Exception as e:
+ logger.error(e)
+ finally:
+ lock.release()
+ else:
+ if not os.path.exists(os.path.dirname(to_file)):
+ os.makedirs(os.path.dirname(to_file))
+ if move:
+ shutil.move(from_file, to_file)
+ else:
+ shutil.copyfile(from_file, to_file)
+ shutil.copystat(from_file, to_file)
+ file_to_copy['size'] = os.path.getsize(to_file)
+ f_stat = datetime.datetime.fromtimestamp(os.path.getmtime(to_file))
+ file_to_copy['year'] = str(f_stat.year)
+ file_to_copy['month'] = str(f_stat.month)
+ file_to_copy['day'] = str(f_stat.day)
+ (file_format, encoding) = Utils.detect_format(to_file)
+ file_to_copy['format'] = file_format
+ return files_to_copy
+
+ @staticmethod
+ def uncompress(archivefile, remove=True):
+ """
+ Test if file is an archive, and uncompress it
+ Remove archive file if specified
+
+ :param file: full path to file to check and uncompress
+ :type file: str
+ :param remove: remove archive if present
+ :type remove: bool
+ :return: True if ok, False if an error occured
+ """
+ is_archive = False
+ logger = logging.getLogger('biomaj')
+ try:
+ if archivefile.endswith('.tar.gz'):
+ subprocess.check_call("tar xfz " + archivefile + " --overwrite -C " + os.path.dirname(archivefile), shell=True)
+ is_archive = True
+ elif archivefile.endswith('.tar'):
+ subprocess.check_call("tar xf " + archivefile + " --overwrite -C " + os.path.dirname(archivefile), shell=True)
+ is_archive = True
+ elif archivefile.endswith('.bz2'):
+ subprocess.check_call("tar xjf " + archivefile + " --overwrite -C " + os.path.dirname(archivefile), shell=True)
+ is_archive = True
+ elif archivefile.endswith('.gz'):
+ subprocess.check_call("gunzip -f " + archivefile, shell=True)
+ is_archive = True
+ elif archivefile.endswith('.zip'):
+ subprocess.check_call("unzip -o " + archivefile + " -d " + os.path.dirname(archivefile), shell=True)
+ is_archive = True
+ except CalledProcessError as uncompresserror:
+ logger.error("Uncompress error of %s: %s" % (archivefile, str(uncompresserror)))
+ return False
+
+ if is_archive:
+ logger.debug('Uncompress:uncompress:' + archivefile)
+
+ if is_archive and remove and os.path.exists(archivefile):
+ os.remove(archivefile)
+
+ return True
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..89deaa9
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,5 @@
+nose
+mock
+future
+elasticsearch
+
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..3c6e79c
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal=1
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..6adbbc1
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,53 @@
+try:
+ from setuptools import setup, find_packages
+except ImportError:
+ from distutils.core import setup
+
+from distutils.command.install import install
+import os
+
+
+here = os.path.abspath(os.path.dirname(__file__))
+with open(os.path.join(here, 'README.md')) as f:
+ README = f.read()
+with open(os.path.join(here, 'CHANGES.txt')) as f:
+ CHANGES = f.read()
+
+
+config = {
+ 'description': 'BioMAJ core library',
+ 'long_description': README + '\n\n' + CHANGES,
+ 'author': 'Olivier Sallou',
+ 'url': 'http://biomaj.genouest.org',
+ 'download_url': 'http://biomaj.genouest.org',
+ 'author_email': 'olivier.sallou at irisa.fr',
+ 'version': '3.0.9',
+ 'classifiers': [
+ 'Development Status :: 5 - Production/Stable',
+ 'Environment :: Console',
+ 'Natural Language :: English',
+ 'Operating System :: POSIX :: Linux',
+ # Indicate who your project is intended for
+ 'Intended Audience :: Science/Research',
+ 'Topic :: Scientific/Engineering :: Bio-Informatics',
+ # Pick your license as you wish (should match "license" above)
+ 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
+ # Specify the Python versions you support here. In particular, ensure
+ # that you indicate whether you support Python 2, Python 3 or both.
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.4'
+ ],
+ 'install_requires': [
+ 'future',
+ 'elasticsearch'
+ ],
+ 'tests_require': ['nose', 'mock'],
+ 'test_suite': 'nose.collector',
+ 'packages': find_packages(),
+ 'include_package_data': True,
+ 'name': 'biomaj_core'
+}
+
+setup(**config)
diff --git a/tests/alu.properties b/tests/alu.properties
new file mode 100644
index 0000000..0e729e0
--- /dev/null
+++ b/tests/alu.properties
@@ -0,0 +1,43 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="alu.n : alu repeat element. alu.a : translation of alu.n repeats"
+db.name=alu
+db.type=nucleic_protein
+
+offline.dir.name=offline/ncbi/blast/alu_tmp
+dir.version=ncbi/blast/alu
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=ftp.ncbi.nih.gov
+remote.dir=/blast/db/FASTA/
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.files=^alu.*\.gz$
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^alu\.(a|n).*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/bank/process/test.sh b/tests/bank/process/test.sh
new file mode 100755
index 0000000..2d510e2
--- /dev/null
+++ b/tests/bank/process/test.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo "Testing a process"
+
+echo "test meta data"
+echo "##BIOMAJ#blast#nucleic#organism:hg19,chr:chr1#blast/chr1/chr1db"
+echo "##BIOMAJ#blast#nucleic#organism:hg19,chr:chr2#blast/chr2/chr2db"
+
+echo "test meta data 2"
+
+echo "##BIOMAJ#fasta#nucleic#organism:hg19#fasta/chr1.fa,fasta/chr2.fa"
diff --git a/tests/bank/test.fasta.gz b/tests/bank/test.fasta.gz
new file mode 100644
index 0000000..666d6f2
Binary files /dev/null and b/tests/bank/test.fasta.gz differ
diff --git a/tests/bank/test2.fasta b/tests/bank/test2.fasta
new file mode 100644
index 0000000..410ca0f
--- /dev/null
+++ b/tests/bank/test2.fasta
@@ -0,0 +1,2 @@
+>test2
+gcgcgcgcgcgcgcgccgcgcgcgcgcgcgcggc
diff --git a/tests/bank/test_100.txt b/tests/bank/test_100.txt
new file mode 100644
index 0000000..c7f7c3b
--- /dev/null
+++ b/tests/bank/test_100.txt
@@ -0,0 +1 @@
+This is a sample file to extract Release 103 from a text file
diff --git a/tests/biomaj_tests.py b/tests/biomaj_tests.py
new file mode 100644
index 0000000..a8a245d
--- /dev/null
+++ b/tests/biomaj_tests.py
@@ -0,0 +1,297 @@
+from nose.tools import *
+from nose.plugins.attrib import attr
+
+import shutil
+import os
+import tempfile
+import stat
+
+from biomaj_core.config import BiomajConfig
+from biomaj_core.utils import Utils
+from biomaj_core.bmajindex import BmajIndex
+
+import unittest
+
+
+class UtilsForTest:
+ """
+ Copy properties files to a temp directory and update properties to
+ use a temp directory
+ """
+
+ def __init__(self):
+ """
+ Setup the temp dirs and files.
+ """
+ self.global_properties = None
+ self.bank_properties = None
+
+ self.test_dir = tempfile.mkdtemp('biomaj')
+
+ self.conf_dir = os.path.join(self.test_dir, 'conf')
+ if not os.path.exists(self.conf_dir):
+ os.makedirs(self.conf_dir)
+ self.data_dir = os.path.join(self.test_dir, 'data')
+ if not os.path.exists(self.data_dir):
+ os.makedirs(self.data_dir)
+ self.log_dir = os.path.join(self.test_dir, 'log')
+ if not os.path.exists(self.log_dir):
+ os.makedirs(self.log_dir)
+ self.process_dir = os.path.join(self.test_dir, 'process')
+ if not os.path.exists(self.process_dir):
+ os.makedirs(self.process_dir)
+ self.lock_dir = os.path.join(self.test_dir, 'lock')
+ if not os.path.exists(self.lock_dir):
+ os.makedirs(self.lock_dir)
+ self.cache_dir = os.path.join(self.test_dir, 'cache')
+ if not os.path.exists(self.cache_dir):
+ os.makedirs(self.cache_dir)
+
+ if self.global_properties is None:
+ self.__copy_global_properties()
+
+ if self.bank_properties is None:
+ self.__copy_test_bank_properties()
+
+ def clean(self):
+ """
+ Deletes temp directory
+ """
+ shutil.rmtree(self.test_dir)
+
+ def __copy_test_bank_properties(self):
+ if self.bank_properties is not None:
+ return
+ self.bank_properties = ['alu', 'local', 'testhttp', 'directhttp',
+ 'noname', 'noexe', 'noargs', 'prenoname',
+ 'prenoexe', 'prenoargs', 'rmnoname', 'rmnoexe',
+ 'rmnoargs']
+ curdir = os.path.dirname(os.path.realpath(__file__))
+ for b in self.bank_properties:
+ from_file = os.path.join(curdir, b+'.properties')
+ to_file = os.path.join(self.conf_dir, b+'.properties')
+ shutil.copyfile(from_file, to_file)
+
+ self.bank_process = ['test.sh']
+ curdir = os.path.dirname(os.path.realpath(__file__))
+ procdir = os.path.join(curdir, 'bank/process')
+ for proc in self.bank_process:
+ from_file = os.path.join(procdir, proc)
+ to_file = os.path.join(self.process_dir, proc)
+ shutil.copyfile(from_file, to_file)
+ os.chmod(to_file, stat.S_IRWXU)
+
+ # Manage local bank test, use bank test subdir as remote
+ properties = ['multi.properties', 'computederror.properties',
+ 'error.properties', 'local.properties',
+ 'localprocess.properties', 'testhttp.properties',
+ 'computed.properties', 'computed2.properties',
+ 'sub1.properties', 'sub2.properties']
+ for prop in properties:
+ from_file = os.path.join(curdir, prop)
+ to_file = os.path.join(self.conf_dir, prop)
+ fout = open(to_file,'w')
+ with open(from_file,'r') as fin:
+ for line in fin:
+ if line.startswith('remote.dir'):
+ fout.write("remote.dir="+os.path.join(curdir,
+ 'bank') + "\n")
+ elif line.startswith('remote.files'):
+ fout.write(line.replace('/tmp', os.path.join(curdir,
+ 'bank')))
+ else:
+ fout.write(line)
+ fout.close()
+
+ def __copy_global_properties(self):
+ if self.global_properties is not None:
+ return
+ self.global_properties = os.path.join(self.conf_dir, 'global.properties')
+ curdir = os.path.dirname(os.path.realpath(__file__))
+ global_template = os.path.join(curdir, 'global.properties')
+ fout = open(self.global_properties, 'w')
+ with open(global_template,'r') as fin:
+ for line in fin:
+ if line.startswith('conf.dir'):
+ fout.write("conf.dir="+self.conf_dir+"\n")
+ elif line.startswith('log.dir'):
+ fout.write("log.dir="+self.log_dir+"\n")
+ elif line.startswith('data.dir'):
+ fout.write("data.dir="+self.data_dir+"\n")
+ elif line.startswith('process.dir'):
+ fout.write("process.dir="+self.process_dir+"\n")
+ elif line.startswith('lock.dir'):
+ fout.write("lock.dir="+self.lock_dir+"\n")
+ else:
+ fout.write(line)
+ fout.close()
+
+
+class TestBiomajUtils(unittest.TestCase):
+
+ def setUp(self):
+ self.utils = UtilsForTest()
+
+ def tearDown(self):
+ self.utils.clean()
+
+ def test_properties_override(self):
+ BiomajConfig.load_config(self.utils.global_properties,
+ allow_user_config=False)
+ config = BiomajConfig('local')
+ ldap_host = config.get('ldap.host')
+ self.assertTrue(ldap_host == 'localhost')
+ os.environ['BIOMAJ_LDAP_HOST'] = 'someserver'
+ ldap_host = config.get('ldap.host')
+ self.assertTrue(ldap_host == 'someserver')
+
+ def test_service_config_override(self):
+ config = {
+ 'rabbitmq': { 'host': '1.2.3.4'}
+ }
+ Utils.service_config_override(config)
+ self.assertTrue(config['rabbitmq']['host'] == '1.2.3.4')
+ os.environ['RABBITMQ_HOST'] = '4.3.2.1'
+ Utils.service_config_override(config)
+ self.assertTrue(config['rabbitmq']['host'] == '4.3.2.1')
+
+ def test_mimes(self):
+ fasta_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+ 'bank/test2.fasta')
+ (mime, encoding) = Utils.detect_format(fasta_file)
+ self.assertTrue('application/fasta' == mime)
+
+ @attr('compress')
+ def test_uncompress(self):
+ from_file = { 'root': os.path.dirname(os.path.realpath(__file__)),
+ 'name': 'bank/test.fasta.gz'
+ }
+
+ to_dir = self.utils.data_dir
+ Utils.copy_files([from_file], to_dir)
+ Utils.uncompress(os.path.join(to_dir, from_file['name']))
+ self.assertTrue(os.path.exists(to_dir+'/bank/test.fasta'))
+
+ def test_copy_with_regexp(self):
+ from_dir = os.path.dirname(os.path.realpath(__file__))
+ to_dir = self.utils.data_dir
+ Utils.copy_files_with_regexp(from_dir, to_dir, ['.*\.py'])
+ self.assertTrue(os.path.exists(to_dir+'/biomaj_tests.py'))
+
+ def test_copy(self):
+ from_dir = os.path.dirname(os.path.realpath(__file__))
+ local_file = 'biomaj_tests.py'
+ files_to_copy = [ {'root': from_dir, 'name': local_file}]
+ to_dir = self.utils.data_dir
+ Utils.copy_files(files_to_copy, to_dir)
+ self.assertTrue(os.path.exists(to_dir+'/biomaj_tests.py'))
+
+ @attr('check')
+ def test_check_method(self):
+ """Check .name, .exe and .args are well check during bank configuration
+ checking"""
+ BiomajConfig.load_config(self.utils.global_properties)
+ for conf in ['noname', 'noexe', 'noargs', 'prenoname', 'prenoexe',
+ 'prenoargs', 'rmnoname', 'rmnoexe', 'rmnoargs']:
+ config = BiomajConfig(conf)
+ self.assertFalse(config.check())
+
+
+ at attr('elastic')
+class TestElastic(unittest.TestCase):
+ """
+ test indexing and search
+ """
+
+ def setUp(self):
+ BmajIndex.es = None
+ self.utils = UtilsForTest()
+ curdir = os.path.dirname(os.path.realpath(__file__))
+ BiomajConfig.load_config(self.utils.global_properties,
+ allow_user_config=False)
+ if BmajIndex.do_index == False:
+ self.skipTest("Skipping indexing tests due to elasticsearch not available")
+ # Delete all banks
+ b = Bank('local')
+ b.banks.remove({})
+ BmajIndex.delete_all_bank('local')
+
+ self.config = BiomajConfig('local')
+ data_dir = self.config.get('data.dir')
+ lock_file = os.path.join(data_dir,'local.lock')
+ if os.path.exists(lock_file):
+ os.remove(lock_file)
+
+ def tearDown(self):
+ data_dir = self.config.get('data.dir')
+ lock_file = os.path.join(data_dir,'local.lock')
+ if os.path.exists(lock_file):
+ os.remove(lock_file)
+ self.utils.clean()
+ BmajIndex.delete_all_bank('test')
+
+ def test_index(self):
+ BmajIndex.do_index = True
+ prod = {
+ "data_dir" : "/tmp/test/data",
+ "formats" : {
+ "fasta" : [
+ {
+ "files" : [
+ "fasta/chr1.fa",
+ "fasta/chr2.fa"
+ ],
+ "types" : [
+ "nucleic"
+ ],
+ "tags" : {
+ "organism" : "hg19"
+ }
+ }
+ ],
+ "blast": [
+ {
+ "files" : [
+ "blast/chr1/chr1db"
+ ],
+ "types" : [
+ "nucleic"
+ ],
+ "tags" : {
+ "chr" : "chr1",
+ "organism" : "hg19"
+ }
+ }
+ ]
+
+ },
+ "freeze" : False,
+ "session" : 1416229253.930908,
+ "prod_dir" : "alu-2003-11-26",
+ "release" : "2003-11-26",
+ "types" : [
+ "nucleic"
+ ]
+ }
+
+ BmajIndex.add('test',prod, True)
+
+ query = {
+ 'query' : {
+ 'match' : {'bank': 'test'}
+ }
+ }
+ res = BmajIndex.search(query)
+ self.assertTrue(len(res)==2)
+
+
+ def test_remove_all(self):
+ self.test_index()
+ query = {
+ 'query' : {
+ 'match' : {'bank': 'test'}
+ }
+ }
+ BmajIndex.delete_all_bank('test')
+ res = BmajIndex.search(query)
+ self.assertTrue(len(res)==0)
diff --git a/tests/computed.properties b/tests/computed.properties
new file mode 100644
index 0000000..214baf4
--- /dev/null
+++ b/tests/computed.properties
@@ -0,0 +1,44 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="computed local system bank test"
+db.name=local0
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/local0_tmp
+dir.version=test/local0
+
+depends=sub1
+sub1.files.move=flat/test_.*
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/tmp/
+remote.files=^test.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/computed2.properties b/tests/computed2.properties
new file mode 100644
index 0000000..2768000
--- /dev/null
+++ b/tests/computed2.properties
@@ -0,0 +1,45 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="computed local system bank test"
+db.name=local0
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/local0_tmp
+dir.version=test/local0
+
+depends=sub1
+
+ref.release=sub1
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/tmp/
+remote.files=^test.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/computederror.properties b/tests/computederror.properties
new file mode 100644
index 0000000..ce4bae1
--- /dev/null
+++ b/tests/computederror.properties
@@ -0,0 +1,43 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="computed error local system bank test"
+db.name=computederror
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/computederror_tmp
+dir.version=test/computederror
+
+depends=sub2,error
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/tmp/
+remote.files=^test.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/directhttp.properties b/tests/directhttp.properties
new file mode 100644
index 0000000..30f673d
--- /dev/null
+++ b/tests/directhttp.properties
@@ -0,0 +1,41 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="directhttp system bank test"
+db.name=directhttp
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/local_tmp
+dir.version=test/directhttp
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=directhttp
+server=ftp2.fr.debian.org
+
+release.protocol=directhttp
+release.server=ftp2.fr.debian.org
+release.remote.dir=/debian/README
+release.file=README
+release.regexp=([0-9.]+),
+release.file.compressed=
+
+#remote.dir=common/downloads/Current_Release/Pfalciparum3D7/fasta/data/PlasmoDB-25_Pfalciparum3D7_Genome.fasta
+#plasmo/communityDownload.do?fname=Atg3_alignment.txt
+remote.dir=/debian/README.html
+remote.files=
+
+local.files=debian/README.html
+
+## Post Process ## The files should be located in the projectfiles/process
+BLOCKS=
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/error.properties b/tests/error.properties
new file mode 100644
index 0000000..2e50f00
--- /dev/null
+++ b/tests/error.properties
@@ -0,0 +1,43 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="error local system bank test"
+db.name=error
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/error_tmp
+dir.version=test/error
+
+depends=sub2
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/error/
+remote.files=^error.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^error.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/global.properties b/tests/global.properties
new file mode 100644
index 0000000..75cb02c
--- /dev/null
+++ b/tests/global.properties
@@ -0,0 +1,123 @@
+[GENERAL]
+test=1
+conf.dir=/tmp/biomaj/config
+log.dir=/tmp/biomaj/log
+process.dir=/tmp/biomaj/process
+#The root directory where all databases are stored.
+#If your data is not stored under one directory hirearchy
+#you can override this value in the database properties file.
+data.dir=/tmp/biomaj/
+lock.dir=/tmp/biomaj/lock
+cache.dir=/tmp/biomaj/cache
+
+db.url=mongodb://localhost:27017
+db.name=biomaj_test
+
+use_ldap=1
+ldap.host=localhost
+ldap.port=389
+ldap.dn=nodomain
+
+# Use ElasticSearch for index/search capabilities
+use_elastic=0
+#Comma separated list of elasticsearch nodes host1,host2:port2
+elastic_nodes=localhost
+elastic_index=biomaj_test
+
+celery.queue=biomaj
+celery.broker=mongodb://localhost:27017/biomaj_celery
+
+# Get directory stats (can be time consuming depending on number of files etc...)
+data.stats=1
+
+# List of user admin (linux user id, comma separated)
+admin=
+
+# Auto publish on updates (do not need publish flag, can be ovveriden in bank property file)
+auto_publish=0
+
+########################
+# Global properties file
+
+
+#To override these settings for a specific database go to its
+#properties file and uncomment or add the specific line you want
+#to override.
+
+#----------------
+# Mail Configuration
+#---------------
+#Uncomment thes lines if you want receive mail when the workflow is finished
+
+mail.smtp.host=
+mail.admin=
+mail.from=
+
+#---------------------
+#Proxy authentification
+#---------------------
+#proxyHost=
+#proxyPort=
+#proxyUser=
+#proxyPassword=
+
+#Number of thread for processes
+bank.num.threads=2
+
+#Number of threads to use for downloading
+files.num.threads=4
+
+#to keep more than one release increase this value
+keep.old.version=0
+
+#----------------------
+# Release configuration
+#----------------------
+release.separator=_
+
+#The historic log file is generated in log/
+#define level information for output : DEBUG,INFO,WARN,ERR
+historic.logfile.level=DEBUG
+
+#http.parse.dir.line=<a[\s]+href="([\S]+)/".*alt="\[DIR\]">.*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})
+http.parse.dir.line=<img[\s]+src="[\S]+"[\s]+alt="\[DIR\]"[\s]*/?>[\s]*<a[\s]+href="([\S]+)/"[\s]*>.*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})
+http.parse.file.line=<img[\s]+src="[\S]+"[\s]+alt="\[[\s]+\]"[\s]*/?>[\s]<a[\s]+href="([\S]+)".*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})[\s]+([\d\.]+[MKG]{0,1})
+
+http.group.dir.name=1
+http.group.dir.date=2
+http.group.file.name=1
+http.group.file.date=2
+http.group.file.size=3
+
+
+# Bank default access
+visibility.default=public
+
+
+[loggers]
+keys = root, biomaj
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = INFO
+handlers = console
+
+[logger_biomaj]
+level = DEBUG
+handlers = console
+qualname = biomaj
+propagate=0
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = DEBUG
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
diff --git a/tests/local.properties b/tests/local.properties
new file mode 100644
index 0000000..7f6f5fd
--- /dev/null
+++ b/tests/local.properties
@@ -0,0 +1,41 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=local
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/local_tmp
+dir.version=test/local
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/tmp/
+remote.files=^test.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/locallist.properties b/tests/locallist.properties
new file mode 100644
index 0000000..a901b2c
--- /dev/null
+++ b/tests/locallist.properties
@@ -0,0 +1,44 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=locallist
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/localprocess.properties b/tests/localprocess.properties
new file mode 100644
index 0000000..7166186
--- /dev/null
+++ b/tests/localprocess.properties
@@ -0,0 +1,100 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=local
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/local_tmp
+dir.version=test/local
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/tmp/
+remote.files=^test.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Pre process
+db.pre.process=META0
+
+## Remove process
+db.remove.process=META0
+
+## Post Process ## The files should be located in the projectfiles/process directory
+BLOCKS=BLOCK1,BLOCK2
+BLOCK1.db.post.process=META0
+BLOCK2.db.post.process=META1,META2,META3
+META0=PROC0
+META1=PROC1,PROC2
+META2=PROC3
+META3=PROC4,PROC5
+
+
+PROC0.name=test0
+PROC0.desc=sample test
+PROC0.cluster=false
+PROC0.type=test
+PROC0.exe=echo
+PROC0.args=test $datadir
+
+PROC1.name=test1
+PROC1.desc=sample test
+PROC1.cluster=false
+PROC1.type=test
+PROC1.exe=touch
+PROC1.args=$datadir/$dirversion/$localrelease/proc1.txt
+
+PROC2.name=test2
+PROC2.desc=sample test
+PROC2.cluster=false
+PROC2.type=test
+PROC2.exe=touch
+PROC2.args=$datadir/$dirversion/$localrelease/proc2.txt
+
+PROC3.name=test3
+PROC3.desc=sample test
+PROC3.cluster=false
+PROC3.type=test
+PROC3.exe=echo
+PROC3.args=test 3
+
+PROC4.name=test4
+PROC4.desc=sample test
+PROC4.cluster=false
+PROC4.type=test
+PROC4.exe=echo
+PROC4.args=test 4
+
+PROC5.name=test5
+PROC5.desc=sample test
+PROC5.cluster=false
+PROC5.type=testmetadata
+PROC5.exe=test.sh
+PROC5.args=
+PROC5.format=test
+PROC5.types=any
+PROC5.tags=chr:chr1,organism:hg19
+# If files is set, then the post-process does not have to print generated files on STDOUT (but can)
+# in this case, the list of files will be extracted from this list with above format/types/tags
+PROC5.files=dir1/file1,dir1/file2,dir1/file3
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/multi.properties b/tests/multi.properties
new file mode 100644
index 0000000..82e08f9
--- /dev/null
+++ b/tests/multi.properties
@@ -0,0 +1,60 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname=test for multi protocol
+db.name=multi
+db.type=test
+
+offline.dir.name=offline/multi_tmp
+dir.version=multi
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=multi
+server=
+remote.dir=
+
+remote.file.0.protocol = directhttp
+remote.file.0.server = httpbin.org
+remote.file.0.path = /get
+remote.file.0.params.keys = key1,key2
+remote.file.0.params.key1 = value1
+remote.file.0.params.key2 = value2
+remote.file.0.name = test1.json
+
+remote.file.1.protocol = directhttp
+remote.file.1.method = POST
+remote.file.1.server = httpbin.org
+remote.file.1.path = /post
+remote.file.1.params.keys = key1,key2
+remote.file.1.params.key1 = value1
+remote.file.1.params.key2 = value2
+remote.file.1.name = test2.json
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.files=^stable/Release$
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/noargs.properties b/tests/noargs.properties
new file mode 100644
index 0000000..36710c8
--- /dev/null
+++ b/tests/noargs.properties
@@ -0,0 +1,48 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=noargs
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+BLOCKS=PROC1
+PROC1.db.post.process=JOB1
+JOB1=job
+
+job.name=job_name
+job.exe=job_exe.sh
+job.desc=job desc
\ No newline at end of file
diff --git a/tests/noexe.properties b/tests/noexe.properties
new file mode 100644
index 0000000..ade982e
--- /dev/null
+++ b/tests/noexe.properties
@@ -0,0 +1,48 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=noexe
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+BLOCKS=PROC1
+PROC1.db.post.process=JOB1
+JOB1=job
+
+job.name=job_name
+job.args=args1 arg2
+job.desc=job desc
\ No newline at end of file
diff --git a/tests/noname.properties b/tests/noname.properties
new file mode 100644
index 0000000..518a0bf
--- /dev/null
+++ b/tests/noname.properties
@@ -0,0 +1,49 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=noname
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+BLOCKS=PROC1
+PROC1.db.post.process=JOB1
+JOB1=job
+
+
+job.exe=job_exe.sh
+job.args=args1 arg2
+job.desc=job desc
\ No newline at end of file
diff --git a/tests/prenoargs.properties b/tests/prenoargs.properties
new file mode 100644
index 0000000..e94c69f
--- /dev/null
+++ b/tests/prenoargs.properties
@@ -0,0 +1,45 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=prenoargs
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.pre.process=PREJOB1
+PREJOB1=prejob
+
+prejob.name=job_name
+prejob.exe=job_exe.sh
+prejob.desc=job desc
\ No newline at end of file
diff --git a/tests/prenoexe.properties b/tests/prenoexe.properties
new file mode 100644
index 0000000..8d0d89c
--- /dev/null
+++ b/tests/prenoexe.properties
@@ -0,0 +1,45 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=prenoexe
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.pre.process=PREJOB1
+PREJOB1=prejob
+
+prejob.name=job_name
+prejob.args=args1 arg2
+prejob.desc=job desc
\ No newline at end of file
diff --git a/tests/prenoname.properties b/tests/prenoname.properties
new file mode 100644
index 0000000..97dd1a5
--- /dev/null
+++ b/tests/prenoname.properties
@@ -0,0 +1,45 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=prenoname
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.pre.process=PREJOB1
+PREJOB1=prejob
+
+prejob.exe=job_exe.sh
+prejob.args=args1 arg2
+prejob.desc=job desc
\ No newline at end of file
diff --git a/tests/rmnoargs.properties b/tests/rmnoargs.properties
new file mode 100644
index 0000000..11b2d45
--- /dev/null
+++ b/tests/rmnoargs.properties
@@ -0,0 +1,45 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=rmnoargs
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.remove.process=RMJOB1
+RMJOB1=rmjob
+
+rmjob.name=job_name
+rmjob.exe=job_exe.sh
+rmjob.desc=job desc
\ No newline at end of file
diff --git a/tests/rmnoexe.properties b/tests/rmnoexe.properties
new file mode 100644
index 0000000..f5dcbaf
--- /dev/null
+++ b/tests/rmnoexe.properties
@@ -0,0 +1,45 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=rmnoexe
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.remove.process=RMJOB1
+RMJOB1=rmjob
+
+rmjob.name=job_name
+rmjob.args=args1 arg2
+rmjob.desc=job desc
\ No newline at end of file
diff --git a/tests/rmnoname.properties b/tests/rmnoname.properties
new file mode 100644
index 0000000..54cc993
--- /dev/null
+++ b/tests/rmnoname.properties
@@ -0,0 +1,45 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="local system bank test"
+db.name=rmnoname
+db.type=nucleic_protein
+db.formats=fasta
+offline.dir.name=offline/test/locallist_tmp
+dir.version=test/locallist
+mail.smtp.host=nowhere
+mail.from=here
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=nowhere
+
+release.files=
+release.regexp=
+release.file.compressed=
+
+remote.dir=
+remote.files=empty
+remote.files.list=true
+remote.files.1.path=/tmp/test.fasta.gz
+remote.files.2.path=/tmp/test2.fasta
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.remove.process=RMJOB1
+RMJOB1=rmjob
+
+rmjob.exe=job_exe.sh
+rmjob.args=args1 arg2
+rmjob.desc=job desc
\ No newline at end of file
diff --git a/tests/sub1.properties b/tests/sub1.properties
new file mode 100644
index 0000000..8e0c69b
--- /dev/null
+++ b/tests/sub1.properties
@@ -0,0 +1,43 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="sub local system bank test"
+db.name=local1
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/local1_tmp
+dir.version=test/local1
+
+depends=sub2
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/tmp/
+remote.files=^test.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/sub2.properties b/tests/sub2.properties
new file mode 100644
index 0000000..b9d3142
--- /dev/null
+++ b/tests/sub2.properties
@@ -0,0 +1,41 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="sub local system bank test"
+db.name=local2
+db.type=nucleic_protein
+
+offline.dir.name=offline/test/local2_tmp
+dir.version=test/local2
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=local
+server=
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.dir=/tmp/
+remote.files=^test.*
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^test.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+### Deployment ###
+
+keep.old.version=1
diff --git a/tests/testhttp.properties b/tests/testhttp.properties
new file mode 100644
index 0000000..454753e
--- /dev/null
+++ b/tests/testhttp.properties
@@ -0,0 +1,43 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname=test for http protocol
+db.name=testhttp
+db.type=package
+
+offline.dir.name=offline/testhttp_tmp
+dir.version=testhttp
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=http
+server=ftp2.fr.debian.org
+remote.dir=/debian/dists/
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.files=^stable/Release$
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^.*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+
+
+### Deployment ###
+
+keep.old.version=1
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/biomaj3-core.git
More information about the debian-med-commit
mailing list