[Python-modules-commits] [python-bioblend] 01/02: Imported Upstream version 0.7.0
Steffen Möller
moeller at moszumanska.debian.org
Wed Jun 29 20:28:49 UTC 2016
This is an automated email from the git hooks/post-receive script.
moeller pushed a commit to branch master
in repository python-bioblend.
commit 6fc29503567d5deedcc0c1635fc9ff09a3ec2496
Author: Steffen Moeller <moeller at debian.org>
Date: Wed Jun 29 22:28:47 2016 +0200
Imported Upstream version 0.7.0
---
.gitignore | 44 +
.travis.yml | 59 +
ABOUT.rst | 60 +
CHANGELOG.md | 196 +++
MANIFEST.in | 9 +
README.rst | 35 +
bioblend/__init__.py | 84 ++
bioblend/cloudman/__init__.py | 729 +++++++++++
bioblend/cloudman/launch.py | 757 +++++++++++
bioblend/config.py | 64 +
bioblend/galaxy/__init__.py | 91 ++
bioblend/galaxy/client.py | 204 +++
bioblend/galaxy/config/__init__.py | 37 +
bioblend/galaxy/dataset_collections/__init__.py | 86 ++
bioblend/galaxy/datasets/__init__.py | 190 +++
bioblend/galaxy/datatypes/__init__.py | 64 +
bioblend/galaxy/folders/__init__.py | 42 +
bioblend/galaxy/forms/__init__.py | 65 +
bioblend/galaxy/ftpfiles/__init__.py | 20 +
bioblend/galaxy/genomes/__init__.py | 98 ++
bioblend/galaxy/groups/__init__.py | 189 +++
bioblend/galaxy/histories/__init__.py | 526 ++++++++
bioblend/galaxy/jobs/__init__.py | 116 ++
bioblend/galaxy/libraries/__init__.py | 515 ++++++++
bioblend/galaxy/objects/__init__.py | 2 +
bioblend/galaxy/objects/client.py | 421 ++++++
bioblend/galaxy/objects/galaxy_instance.py | 131 ++
bioblend/galaxy/objects/wrappers.py | 1376 ++++++++++++++++++++
bioblend/galaxy/quotas/__init__.py | 64 +
bioblend/galaxy/roles/__init__.py | 52 +
bioblend/galaxy/tool_data/__init__.py | 62 +
bioblend/galaxy/tools/__init__.py | 218 ++++
bioblend/galaxy/tools/inputs.py | 74 ++
bioblend/galaxy/toolshed/__init__.py | 135 ++
bioblend/galaxy/users/__init__.py | 132 ++
bioblend/galaxy/visual/__init__.py | 58 +
bioblend/galaxy/workflows/__init__.py | 587 +++++++++
bioblend/galaxyclient.py | 189 +++
bioblend/toolshed/__init__.py | 39 +
bioblend/toolshed/repositories/__init__.py | 427 ++++++
bioblend/toolshed/tools/__init__.py | 64 +
bioblend/util/__init__.py | 56 +
docs/Makefile | 153 +++
docs/_static/.empty | 0
docs/api_docs/cloudman/all.rst | 21 +
docs/api_docs/cloudman/docs.rst | 134 ++
docs/api_docs/galaxy/all.rst | 164 +++
docs/api_docs/galaxy/docs.rst | 405 ++++++
docs/api_docs/lib_config.rst | 16 +
docs/api_docs/toolshed/all.rst | 17 +
docs/conf.py | 255 ++++
docs/examples/cloudman_basic_usage_scenario.py | 52 +
docs/examples/create_user_get_api_key.py | 24 +
docs/examples/list_data_libraries.py | 30 +
docs/examples/list_histories.py | 32 +
docs/examples/list_workflows.py | 30 +
docs/examples/objects/README.txt | 49 +
docs/examples/objects/common.py | 4 +
docs/examples/objects/list_data_libraries.py | 30 +
docs/examples/objects/list_histories.py | 33 +
docs/examples/objects/list_workflows.py | 30 +
docs/examples/objects/small.ga | 95 ++
docs/examples/objects/small.py | 54 +
docs/examples/objects/w2_bacterial_reseq.py | 62 +
docs/examples/objects/w3_bacterial_denovo.py | 77 ++
docs/examples/objects/w5_galaxy_api.py | 88 ++
docs/examples/objects/w5_metagenomics.py | 58 +
docs/examples/run_imported_workflow.py | 90 ++
docs/examples/start_cloudman.py | 62 +
.../tophat_cufflinks_pairedend_workflow.ga | 151 +++
docs/index.rst | 165 +++
my_unittest2.py | 10 +
run_bioblend_tests.sh | 122 ++
run_galaxy.sh | 139 ++
setup.cfg | 8 +
setup.py | 49 +
teamcity_run_tests.sh | 5 +
tests/CloudmanTestBase.py | 69 +
tests/GalaxyTestBase.py | 42 +
tests/README.TXT | 17 +
tests/TestCloudmanLaunch.py | 48 +
tests/TestCloudmanMock.py | 142 ++
tests/TestCloudmanServices.py | 59 +
tests/TestGalaxyDatasetCollections.py | 138 ++
tests/TestGalaxyDatasets.py | 41 +
tests/TestGalaxyGroups.py | 65 +
tests/TestGalaxyHistories.py | 159 +++
tests/TestGalaxyInstance.py | 40 +
tests/TestGalaxyLibraries.py | 85 ++
tests/TestGalaxyObjects.py | 747 +++++++++++
tests/TestGalaxyRoles.py | 18 +
tests/TestGalaxyToolData.py | 24 +
tests/TestGalaxyToolInputs.py | 45 +
tests/TestGalaxyTools.py | 129 ++
tests/TestGalaxyUsers.py | 55 +
tests/TestGalaxyWorkflows.py | 167 +++
tests/data/1.bed | 65 +
tests/data/paste_columns.ga | 87 ++
tests/data/paste_columns_collections.ga | 94 ++
tests/data/test_workflow_pause.ga | 118 ++
tests/test_util.py | 63 +
tox.ini | 16 +
102 files changed, 13614 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..8ca8a47
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,44 @@
+*.py[co]
+*~
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+.eggs
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+
+#Translations
+*.mo
+
+#Mr Developer
+.mr.developer.cfg
+
+#Vim
+*.swp
+
+#Code coverage
+cover
+
+#eclipse/pydev
+.project
+.pydevproject
+
+# compiled docs
+docs/_build
+
+# Python virtualenv
+.venv
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..2e0f2c4
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,59 @@
+sudo: false
+language: python
+python:
+ - "2.7"
+env:
+ - TOX_ENV=py34 GALAXY_VERSION=dev
+ - TOX_ENV=py33 GALAXY_VERSION=dev
+ - TOX_ENV=py27 GALAXY_VERSION=dev
+ - TOX_ENV=py27 GALAXY_VERSION=release_15.07
+ - TOX_ENV=py27 GALAXY_VERSION=release_15.05
+ - TOX_ENV=py27 GALAXY_VERSION=release_15.03
+ - TOX_ENV=py27 GALAXY_VERSION=release_15.01
+ - TOX_ENV=py27 GALAXY_VERSION=release_14.10
+ - TOX_ENV=py27 GALAXY_VERSION=release_14.08
+ - TOX_ENV=py27 GALAXY_VERSION=release_14.06
+ - TOX_ENV=py27 GALAXY_VERSION=release_14.04
+ - TOX_ENV=py27 GALAXY_VERSION=release_14.02
+ - TOX_ENV=py26 GALAXY_VERSION=dev
+
+install:
+ - python setup.py install
+ - cp run_galaxy.sh $HOME
+ - pip install "tox>=1.8.0"
+
+before_script:
+ # Install Galaxy
+ - cd $HOME
+ - wget https://github.com/galaxyproject/galaxy/archive/${GALAXY_VERSION}.tar.gz
+ - tar xvzf ${GALAXY_VERSION}.tar.gz | tail
+ # Releases/dev branch are named differently
+ - cd galaxy-${GALAXY_VERSION}/
+ # Create a PostgreSQL database for Galaxy. The default SQLite3 database makes test fail randomly because of "database locked" error.
+ - createdb -U postgres galaxy-travis
+ - if [ -f universe_wsgi.ini.sample ]; then GALAXY_CONFIG_FILE=universe_wsgi.ini; else GALAXY_CONFIG_FILE=config/galaxy.ini; fi
+ - if [ -f universe_wsgi.ini.sample ]; then GALAXY_CONFIG_DIR=.; else GALAXY_CONFIG_DIR=config; fi
+ # Change Galaxy configuration to use the newly created PostgreSQL database
+ - sed -e 's|^#database_connection.*|database_connection = postgresql://postgres:@localhost/galaxy-travis|' $GALAXY_CONFIG_FILE.sample > $GALAXY_CONFIG_FILE
+ # Setup Galaxy master API key and admin user
+ - GALAXY_MASTER_API_KEY=`date --rfc-3339=ns | md5sum | cut -f 1 -d ' '`
+ - GALAXY_USER_EMAIL=${USER}@localhost.localdomain
+ - sed -i -e "s/^#master_api_key.*/master_api_key = $GALAXY_MASTER_API_KEY/" -e "s/^#admin_users.*/admin_users = $GALAXY_USER_EMAIL/" $GALAXY_CONFIG_FILE
+ # Change configuration needed by many tests
+ - sed -i -e 's/^#allow_user_dataset_purge.*/allow_user_dataset_purge = True/' $GALAXY_CONFIG_FILE
+ # Change Galaxy configuration needed by some library tests
+ - sed -i -e 's/^#allow_library_path_paste.*/allow_library_path_paste = True/' $GALAXY_CONFIG_FILE
+ - sed -i -e 's/^#enable_beta_workflow_modules.*/enable_beta_workflow_modules = True/' $GALAXY_CONFIG_FILE
+ - if [ -f test/functional/tools/samples_tool_conf.xml ]; then sed -i -e "s/^#tool_config_file.*/tool_config_file = $GALAXY_CONFIG_DIR\/tool_conf.xml.sample,$GALAXY_CONFIG_DIR\/shed_tool_conf.xml.sample,test\/functional\/tools\/samples_tool_conf.xml/" $GALAXY_CONFIG_FILE; fi
+ # Start Galaxy and wait for successful server start
+ - GALAXY_RUN_ALL=1 ../run_galaxy.sh --daemon --wait
+ # Use the master API key to create the admin user and get its API key
+ - export BIOBLEND_GALAXY_URL=http://localhost:8080
+ - GALAXY_USER=$USER
+ - GALAXY_USER_PASSWD=`date --rfc-3339=ns | md5sum | cut -f 1 -d ' '`
+ - export BIOBLEND_GALAXY_API_KEY=`python $TRAVIS_BUILD_DIR/docs/examples/create_user_get_api_key.py $BIOBLEND_GALAXY_URL $GALAXY_MASTER_API_KEY $GALAXY_USER $GALAXY_USER_EMAIL $GALAXY_USER_PASSWD`
+ - echo "Created new Galaxy user $GALAXY_USER with email $GALAXY_USER_EMAIL , password $GALAXY_USER_PASSWD and API key $BIOBLEND_GALAXY_API_KEY"
+
+script:
+ # Run nosetests through setuptools, so it will install test depedencies
+ - cd $TRAVIS_BUILD_DIR && tox -e $TOX_ENV
diff --git a/ABOUT.rst b/ABOUT.rst
new file mode 100644
index 0000000..1788697
--- /dev/null
+++ b/ABOUT.rst
@@ -0,0 +1,60 @@
+`BioBlend <http://bioblend.readthedocs.org/>`_ is a Python library for
+interacting with `CloudMan`_ and `Galaxy`_'s API.
+
+BioBlend is supported and tested on:
+
+- Python 2.6, 2.7, 3.3 and 3.4
+- Galaxy release_14.02 and later.
+
+Conceptually, it makes it possible to script and automate the process
+of cloud infrastructure provisioning and scaling via CloudMan, and
+running of analyses via Galaxy. In reality, it makes it possible to do
+things like this:
+
+- Create a CloudMan compute cluster, via an API and directly from your
+ local machine::
+
+ from bioblend.cloudman import CloudManConfig
+ from bioblend.cloudman import CloudManInstance
+ cfg = CloudManConfig('<your cloud access key>', '<your cloud secret key>', 'My CloudMan', 'ami-<ID>', 'm1.small', '<password>')
+ cmi = CloudManInstance.launch_instance(cfg)
+ cmi.get_status()
+
+- Reconnect to an existing CloudMan instance and manipulate it::
+
+ from bioblend.cloudman import CloudManInstance
+ cmi = CloudManInstance("<instance IP>", "<password>")
+ cmi.add_nodes(3)
+ cluster_status = cmi.get_status()
+ cmi.remove_nodes(2)
+
+- Interact with Galaxy via a straightforward API::
+
+ from bioblend.galaxy import GalaxyInstance
+ gi = GalaxyInstance('<Galaxy IP>', key='your API key')
+ libs = gi.libraries.get_libraries()
+ gi.workflows.show_workflow('workflow ID')
+ gi.workflows.run_workflow('workflow ID', input_dataset_map)
+
+- Interact with Galaxy via an object-oriented API::
+
+ from bioblend.galaxy.objects import GalaxyInstance
+ gi = GalaxyInstance("URL", "API_KEY")
+ wf = gi.workflows.list()[0]
+ hist = gi.histories.list()[0]
+ inputs = hist.get_datasets()[:2]
+ input_map = dict(zip(wf.input_labels, inputs))
+ params = {"Paste1": {"delimiter": "U"}}
+ wf.run(input_map, "wf_output", params=params)
+
+.. note::
+ Although this library allows you to blend these two services into
+ a cohesive unit, the library itself can be used with either
+ service irrespective of the other. For example, you can use it to
+ just manipulate CloudMan clusters or to script the interactions
+ with an instance of Galaxy running on your laptop.
+
+.. References/hyperlinks used above
+.. _CloudMan: http://usecloudman.org/
+.. _Galaxy: http://usegalaxy.org/
+.. _Git repository: https://github.com/afgane/bioblend
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 0000000..39f47e2
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,196 @@
+### BioBlend v0.7.0 - November 2, 2015
+
+* BioBlend.objects: enable import of workflows containing dataset collection
+ inputs.
+
+* Implement APIs for a modern Galaxy workflow APIs (i.e. delayed scheduling).
+
+* Implement APIs to search Tool Shed repositories and tools.
+
+* Add support for uploading (importing) from FTP (thanks to Eric Rasche).
+
+* Add ``to_posix_lines`` and ``space_to_tab`` params to ``upload_file()``,
+ ``upload_from_ftp()`` and ``paste_content()`` methods of ``ToolClient``.
+
+* BioBlend.objects: add ``upload_from_ftp()`` method to ``History``.
+
+* Updated the testing framework to work with Galaxy wheels; use TravisCI's
+ container infrastructure; test Galaxy release 15.07.
+
+* Updated CloudmanLauncher's ``launch`` method to accept ``subnet_id`` parameter,
+ for VPC support (thanks to Matthew Ralston).
+
+* Properly pass extra parameters to cloud instance userdata.
+
+* Update placement finding methods and `get_clusters_pd` method to return a
+ dict vs. lists so error messages can be included.
+
+* A numer of documentation improvements and minor updates/fixes (see individual
+ commits).
+
+### BioBlend v0.6.1 - July 27, 2015
+
+* BioBlend.objects: Rename ``ObjDatasetClient`` abstract class to
+ ``ObjDatasetContainerClient``.
+
+* BioBlend.objects: Add ``ABCMeta`` metaclass and ``list()`` method to
+ ``ObjClient``.
+
+* BioBlend.objects: Add ``io_details`` and ``link_details`` parameters to
+ ``ObjToolClient.get()`` method.
+
+* Open port 8800 when launching cloud instances for use by NodeJS proxy for
+ Galaxy IPython Interactive Environments.
+
+* When launching cloud instances, propagate error messages back to the called.
+ The return types for methods ``create_cm_security_group``, ``create_key_pair``
+ in ``CloudManLauncher`` class have changed as a result of this.
+
+### BioBlend v0.6.0 - June 30, 2015
+
+* Add support for Python >= 3.3.
+
+* Add ``get_library_permissions()`` method to ``LibraryClient``.
+
+* Add ``update_group()``, ``get_group_users()``, ``get_group_roles()``,
+ ``add_group_user()``, ``add_group_role()``, ``delete_group_user()`` and
+ ``delete_group_role()`` methods to ``GroupsClient``.
+
+* Add ``full_details`` parameter to ``JobsClient.show_job()`` (thanks to Rossano
+ Atzeni).
+
+* BioBlend.objects: add ``ObjJobClient`` and ``Job`` wrapper (thanks to Rossano
+ Atzeni).
+
+* BioBlend.objects: add check to verify that all tools in a workflow are
+ installed on the Galaxy instance (thanks to Gianmauro Cuccuru).
+
+* Remove several deprecated parameters: see commits [19e168f](https://github.com/galaxyproject/bioblend/commit/19e168f5342f4c791d37694d7039a85f2669df71)
+ and [442ae98](https://github.com/galaxyproject/bioblend/commit/442ae98037be7455d57be15542553dc848d99431).
+
+* Verify SSL certificates by default.
+
+* Add documentation about the Tool Shed and properly link all the docs on
+ ReadTheDocs.
+
+* Solidify automated testing by using [tox](https://tox.readthedocs.org/) and
+ [flake8](https://gitlab.com/pycqa/flake8).
+
+### BioBlend v0.5.3 - March 18, 2015
+
+* Project source moved to new URL - https://github.com/galaxyproject/bioblend
+
+* Huge improvements to automated testing, tests now run against Galaxy
+ release_14.02 and all later versions to ensure backward compatibility
+ (see `.travis.yml` for details).
+
+* Many documentation improvements (thanks to Eric Rasche).
+
+* Add Galaxy clients for the tool data tables, the roles, and library
+ folders (thanks to Anthony Bretaudeau).
+
+* Add method to get the standard error and standard output for the
+ job corresponding to a Galaxy dataset (thanks to Anthony Bretaudeau).
+
+* Add ``get_state()`` method to ``JobsClient``.
+
+* Add ``copy_from_dataset()`` method to ``LibraryClient``.
+
+* Add ``create_repository()`` method to ``ToolShedClient`` (thanks to Eric
+ Rasche).
+
+* Fix ``DatasetClient.download_dataset()`` for certain proxied Galaxy
+ deployments.
+
+* Make ``LibraryClient._get_root_folder_id()`` method safer and faster for
+ Galaxy release_13.06 and later.
+
+* Deprecate and ignore invalid ``deleted`` parameter to
+ ``WorkflowClient.get_workflows()``.
+
+* CloudMan: Add method to fetch instance types.
+
+* CloudMan: Update cluster options to reflect change to SLURM.
+
+* BioBlend.objects: Deprecate and ignore invalid ``deleted`` parameter
+ to ``ObjWorkflowClient.list()``.
+
+* BioBlend.objects: Add ``paste_content()`` method to ``History`` objects.
+
+* BioBlend.objects: Add ``copy_from_dataset()`` method and ``root_folder``
+ property to ``Library`` objects.
+
+* BioBlend.objects: Add ``container`` and ``deleted`` attributes to ``Folder``
+ objects.
+
+* BioBlend.objects: Set the ``parent`` attribute of a ``Folder`` object to its
+ parent folder object (thanks to John M. Eppley).
+
+* BioBlend.objects: Add ``deleted`` parameter to ``list()`` method
+ of libraries and histories.
+
+* BioBlend.objects: Add ``state`` and ``state_details`` attributes to
+ ``History`` objects (thanks to Gianmauro Cuccuru).
+
+* BioBlend.objects: Rename ``upload_dataset()`` method to ``upload_file()``
+ for ``History`` objects.
+
+* BioBlend.objects: Rename ``input_ids`` and ``output_ids`` attributes of
+ ``Workflow`` objects to ``source_ids`` and ``sink_ids`` respectively.
+
+* Add ``run_bioblend_tests.sh`` script (useful for Continuous Integration
+ testing).
+
+### BioBlend v0.5.2 - October 17, 2014
+
+* BioBlend.objects: enable email&password auth
+
+* Enable Tool Shed tar ball uploads
+
+* BioBlend.objects: allow deletion of history and library datasets
+
+* BioBlend.objects: fixed library dataset downloads
+
+* Fixed the Tool Shed tool installation method
+
+* Add 'deleted' attribute to DatasetContainer
+
+* Handle `data_type` changes in the Oct 2014 Galaxy release
+
+* Renamed `get_current_history()` to `get_most_recently_used_history()`
+
+* A number of documentation improvements and other small fixes (see
+ the commit messages for more details)
+
+### BioBlend v0.5.1 - August 19, 2014
+
+* Fixed url joining problem described in issue #82
+
+* Enabled Travis Continuous Inetgration testing
+
+* Added script to create a user and get its API key
+
+* Deprecated ``create_user()`` method in favor of clearer ``create_remote_user()``.
+ Added ``create_local_user()``.
+
+* Skip instead of fail tests when ``BIOBLEND_GALAXY_URL`` and
+ ``BIOBLEND_GALAXY_API_KEY`` environment variables are not defined.
+
+* Added export and download to objects API
+
+* Added export/download history
+
+* GalaxyClient: changed ``make_put_request`` to return whole ``requests``
+ response object
+
+* Added Tool wrapper to *BioBlend.objects* plus methods to list tools and get one
+
+* Added ``show_tool()`` method to ``ToolClient`` class
+
+* Added ``name``, ``in_panel`` and
+ ``trackster`` filters to ``get_tools()``
+
+* Added ``upload_dataset()`` method to ``History`` class.
+
+* Removed ``DataInput`` and ``Tool`` classes for workflow steps. ``Tool`` is to
+ be used for running single tools.
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..940b16a
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,9 @@
+global-exclude *.swp .gitignore
+
+include *.rst
+include setup.py
+
+graft bioblend
+graft docs
+graft tests
+
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..bdcf8e7
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,35 @@
+.. image:: https://img.shields.io/pypi/v/bioblend.svg
+ :target: https://pypi.python.org/pypi/bioblend/
+ :alt: latest version available on PyPI
+
+.. image:: https://img.shields.io/pypi/dm/bioblend.svg
+ :target: https://pypi.python.org/pypi/bioblend/
+ :alt: PyPI downloads in the last month
+
+.. image:: https://readthedocs.org/projects/bioblend/badge/
+ :alt: Documentation Status
+ :target: https://bioblend.readthedocs.org/
+
+.. image:: https://travis-ci.org/galaxyproject/bioblend.png
+ :target: https://travis-ci.org/galaxyproject/bioblend
+ :alt: Build Status
+
+.. image:: https://landscape.io/github/galaxyproject/bioblend/master/landscape.svg?style=flat
+ :target: https://landscape.io/github/galaxyproject/bioblend/master
+ :alt: Code Health
+
+
+BioBlend is a Python library for interacting with `CloudMan`_ and `Galaxy`_'s
+API.
+
+BioBlend is supported and tested on:
+
+- Python 2.6, 2.7, 3.3 and 3.4
+- Galaxy release_14.02 and later.
+
+Full docs are available at http://bioblend.readthedocs.org with a quick library
+overview also available in `ABOUT.rst <./ABOUT.rst>`_.
+
+.. References/hyperlinks used above
+.. _CloudMan: http://usecloudman.org/
+.. _Galaxy: http://usegalaxy.org/
diff --git a/bioblend/__init__.py b/bioblend/__init__.py
new file mode 100644
index 0000000..6949a04
--- /dev/null
+++ b/bioblend/__init__.py
@@ -0,0 +1,84 @@
+import logging
+import os
+
+from bioblend.config import Config, BioBlendConfigLocations
+
+# Current version of the library
+__version__ = '0.7.0'
+
+# default chunk size (in bytes) for reading remote data
+try:
+ import resource
+ CHUNK_SIZE = resource.getpagesize()
+except Exception:
+ CHUNK_SIZE = 4096
+
+
+config = Config()
+
+
+def get_version():
+ """
+ Returns a string with the current version of the library (e.g., "0.2.0")
+ """
+ return __version__
+
+
+def init_logging():
+ """
+ Initialize BioBlend's logging from a configuration file.
+ """
+ for config_file in BioBlendConfigLocations:
+ try:
+ logging.config.fileConfig(os.path.expanduser(config_file))
+ except:
+ pass
+
+
+class NullHandler(logging.Handler):
+ def emit(self, record):
+ pass
+
+# By default, do not force any logging by the library. If you want to see the
+# log messages in your scripts, add the following to the top of your script:
+# import logging
+# logging.basicConfig(filename="bioblend.log", level=logging.DEBUG)
+
+default_format_string = "%(asctime)s %(name)s [%(levelname)s]: %(message)s"
+log = logging.getLogger('bioblend')
+log.addHandler(NullHandler())
+init_logging()
+
+# Convenience functions to set logging to a particular file or stream
+# To enable either of these, simply add the following at the top of a
+# bioblend module:
+# import bioblend
+# bioblend.set_stream_logger(__name__)
+
+
+def set_file_logger(name, filepath, level=logging.INFO, format_string=None):
+ global log
+ if not format_string:
+ format_string = default_format_string
+ logger = logging.getLogger(name)
+ logger.setLevel(level)
+ fh = logging.FileHandler(filepath)
+ fh.setLevel(level)
+ formatter = logging.Formatter(format_string)
+ fh.setFormatter(formatter)
+ logger.addHandler(fh)
+ log = logger
+
+
+def set_stream_logger(name, level=logging.DEBUG, format_string=None):
+ global log
+ if not format_string:
+ format_string = default_format_string
+ logger = logging.getLogger(name)
+ logger.setLevel(level)
+ fh = logging.StreamHandler()
+ fh.setLevel(level)
+ formatter = logging.Formatter(format_string)
+ fh.setFormatter(formatter)
+ logger.addHandler(fh)
+ log = logger
diff --git a/bioblend/cloudman/__init__.py b/bioblend/cloudman/__init__.py
new file mode 100644
index 0000000..5aed3ba
--- /dev/null
+++ b/bioblend/cloudman/__init__.py
@@ -0,0 +1,729 @@
+"""
+API for interacting with a CloudMan instance.
+"""
+import functools
+import json
+import time
+
+import requests
+from six.moves import range
+from six.moves.urllib.parse import urlparse
+
+import bioblend
+from bioblend.cloudman.launch import CloudManLauncher
+from bioblend.util import Bunch
+
+
+def block_until_vm_ready(func):
+ """
+ This decorator exists to make sure that a launched VM is
+ ready and has received a public IP before allowing the wrapped
+ function call to continue. If the VM is not ready, the function will
+ block until the VM is ready. If the VM does not become ready
+ until the vm_ready_timeout elapses or the VM status returns an error,
+ a VMLaunchException will be thrown.
+
+ This decorator relies on the wait_until_instance_ready method defined in
+ class GenericVMInstance. All methods to which this decorator is applied
+ must be members of a class which inherit from GenericVMInstance.
+
+ The following two optional keyword arguments are recognized by this decorator:
+
+ :type vm_ready_timeout: int
+ :param vm_ready_timeout: Maximum length of time to block before timing out.
+ Once the timeout is reached, a VMLaunchException
+ will be thrown.
+
+ :type vm_ready_check_interval: int
+ :param vm_ready_check_interval: The number of seconds to pause between consecutive
+ calls when polling the VM's ready status.
+ """
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ obj = args[0]
+ timeout = kwargs.pop('vm_ready_timeout', 300)
+ interval = kwargs.pop('vm_ready_check_interval', 10)
+ try:
+ obj.wait_until_instance_ready(timeout, interval)
+ except AttributeError:
+ raise VMLaunchException("Decorated object does not define a wait_until_instance_ready method."
+ "Make sure that the object is of type GenericVMInstance.")
+ return func(*args, **kwargs)
+ return wrapper
+
+
+class VMLaunchException(Exception):
+ def __init__(self, value):
+ self.value = value
+
+ def __str__(self):
+ return repr(self.value)
+
+
+class CloudManConfig(object):
+
+ def __init__(self,
+ access_key=None,
+ secret_key=None,
+ cluster_name=None,
+ image_id=None,
+ instance_type='m1.medium',
+ password=None,
+ cloud_metadata=None,
+ cluster_type=None,
+ galaxy_data_option='',
+ initial_storage_size=10,
+ key_name='cloudman_key_pair',
+ security_groups=['CloudMan'],
+ placement='',
+ kernel_id=None,
+ ramdisk_id=None,
+ block_until_ready=False,
+ **kwargs):
+ """
+ Initializes a CloudMan launch configuration object.
+
+ :type access_key: str
+ :param access_key: Access credentials.
+
+ :type secret_key: str
+ :param secret_key: Access credentials.
+
+ :type cluster_name: str
+ :param cluster_name: Name used to identify this CloudMan cluster.
+
+ :type image_id: str
+ :param image_id: Machine image ID to use when launching this
+ CloudMan instance.
+
+ :type instance_type: str
+ :param instance_type: The type of the machine instance, as understood by
+ the chosen cloud provider. (e.g., ``m1.medium``)
+
+ :type password: str
+ :param password: The administrative password for this CloudMan instance.
+
+ :type cloud_metadata: Bunch
+ :param cloud_metadata: This object must define the properties required
+ to establish a `boto <https://github.com/boto/boto/>`_
+ connection to that cloud. See this method's implementation
+ for an example of the required fields. Note that as
+ long the as provided object defines the required fields,
+ it can really by implemented as anything (e.g.,
+ a Bunch, a database object, a custom class). If no
+ value for the ``cloud`` argument is provided, the
+ default is to use the Amazon cloud.
+
+ :type kernel_id: str
+ :param kernel_id: The ID of the kernel with which to launch the
+ instances
+
+ :type ramdisk_id: str
+ :param ramdisk_id: The ID of the RAM disk with which to launch the
+ instances
+
+ :type key_name: str
+ :param key_name: The name of the key pair with which to launch instances
+
+ :type security_groups: list of str
+ :param security_groups: The IDs of the security groups with which to
+ associate instances
+
+ :type placement: str
+ :param placement: The availability zone in which to launch the instances
+
+ :type cluster_type: str
+ :param cluster_type: The ``type``, either 'Galaxy', 'Data', or
+ 'Test', defines the type of cluster platform to initialize.
+
+ :type galaxy_data_option: str
+ :param galaxy_data_option: The storage type to use for this instance.
+ May be 'transient', 'custom_size' or ''. The default is '',
+ which will result in ignoring the bioblend specified
+ initial_storage_size. 'custom_size' must be used for
+ initial_storage_size to come into effect.
+
+ :type initial_storage_size: int
+ :param initial_storage_size: The initial storage to allocate for the instance.
+ This only applies if ``cluster_type`` is set
+ to either ``Galaxy`` or ``Data`` and ``galaxy_data_option``
+ is set to ``custom_size``
+
+ :type block_until_ready: bool
+ :param block_until_ready: Specifies whether the launch method will block
+ until the instance is ready and only return once
+ all initialization is complete. The default is False.
+ If False, the launch method will return immediately
+ without blocking. However, any subsequent calls
+ made will automatically block if the instance is
+ not ready and initialized. The blocking timeout
+ and polling interval can be configured by providing
+ extra parameters to the ``CloudManInstance.launch_instance``
+ method.
+ """
+ self.set_connection_parameters(access_key, secret_key, cloud_metadata)
+ self.set_pre_launch_parameters(
+ cluster_name, image_id, instance_type,
+ password, kernel_id, ramdisk_id, key_name, security_groups,
+ placement, block_until_ready)
+ self.set_post_launch_parameters(cluster_type, galaxy_data_option, initial_storage_size)
+ self.set_extra_parameters(**kwargs)
+
+ def set_connection_parameters(self, access_key, secret_key, cloud_metadata=None):
+ self.access_key = access_key
+ self.secret_key = secret_key
+ self.cloud_metadata = cloud_metadata
+
+ def set_pre_launch_parameters(
+ self, cluster_name, image_id, instance_type, password,
+ kernel_id=None, ramdisk_id=None, key_name='cloudman_key_pair',
+ security_groups=['CloudMan'], placement='',
+ block_until_ready=False):
+ self.cluster_name = cluster_name
+ self.image_id = image_id
+ self.instance_type = instance_type
+ self.password = password
+ self.kernel_id = kernel_id
+ self.ramdisk_id = ramdisk_id
+ self.key_name = key_name
+ self.security_groups = security_groups
+ self.placement = placement
+ self.block_until_ready = block_until_ready
+
+ def set_post_launch_parameters(self, cluster_type=None, galaxy_data_option='', initial_storage_size=10):
+ self.cluster_type = cluster_type
+ self.galaxy_data_option = galaxy_data_option
+ self.initial_storage_size = initial_storage_size
+
+ def set_extra_parameters(self, **kwargs):
+ self.kwargs = kwargs
+
+ class CustomTypeEncoder(json.JSONEncoder):
+ def default(self, obj):
+ if isinstance(obj, (CloudManConfig, Bunch)):
+ key = '__%s__' % obj.__class__.__name__
+ return {key: obj.__dict__}
+ return json.JSONEncoder.default(self, obj)
+
+ @staticmethod
+ def CustomTypeDecoder(dct):
+ if '__CloudManConfig__' in dct:
+ return CloudManConfig(**dct['__CloudManConfig__'])
+ elif '__Bunch__' in dct:
+ return Bunch(**dct['__Bunch__'])
+ else:
+ return dct
+
+ @staticmethod
+ def load_config(fp):
+ return json.load(fp, object_hook=CloudManConfig.CustomTypeDecoder)
+
+ def save_config(self, fp):
+ json.dump(self, fp, cls=self.CustomTypeEncoder)
+
+ def validate(self):
+ if self.access_key is None:
+ return "Access key must not be null"
+ elif self.secret_key is None:
+ return "Secret key must not be null"
+ elif self.cluster_name is None:
+ return "Cluster name must not be null"
+ elif self.image_id is None:
+ return "Image ID must not be null"
+ elif self.instance_type is None:
+ return "Instance type must not be null"
+ elif self.password is None:
+ return "Password must not be null"
+ elif self.cluster_type not in [None, 'Test', 'Data', 'Galaxy', 'Shared_cluster']:
+ return "Unrecognized cluster type ({0})".format(self.cluster_type)
+ elif self.galaxy_data_option not in [None, '', 'custom-size', 'transient']:
+ return "Unrecognized galaxy data option ({0})".format(self.galaxy_data_option)
+ elif self.key_name is None:
+ return "Key-pair name must not be null"
+ else:
+ return None
+
+
+class GenericVMInstance(object):
+
+ def __init__(self, launcher, launch_result):
+ """
+ Create an instance of the CloudMan API class, which is to be used when
+ manipulating that given CloudMan instance.
+
+ The ``url`` is a string defining the address of CloudMan, for
+ example "http://115.146.92.174". The ``password`` is CloudMan's password,
+ as defined in the user data sent to CloudMan on instance creation.
+ """
+ # Make sure the url scheme is defined (otherwise requests will not work)
+ self.vm_error = None
+ self.vm_status = None
+ self.host_name = None
+ self.launcher = launcher
+ self.launch_result = launch_result
+
+ def _update_host_name(self, host_name):
+ if self.host_name != host_name:
+ self.host_name = host_name
+
+ @property
+ def instance_id(self):
+ """
+ Returns the ID of this instance (e.g., ``i-87ey32dd``) if launch was
+ successful or ``None`` otherwise.
+ """
+ return None if self.launch_result is None else self.launch_result['instance_id']
+
+ @property
+ def key_pair_name(self):
+ """
+ Returns the name of the key pair used by this instance. If instance was
+ not launched properly, returns ``None``.
+ """
+ return None if self.launch_result is None else self.launch_result['kp_name']
+
+ @property
+ def key_pair_material(self):
+ """
+ Returns the private portion of the generated key pair. It does so only
+ if the instance was properly launched and key pair generated; ``None``
+ otherwise.
+ """
+ return None if self.launch_result is None else self.launch_result['kp_material']
+
+ def get_machine_status(self):
+ """
+ Check on the underlying VM status of an instance. This can be used to
+ determine whether the VM has finished booting up and if CloudMan
+ is up and running.
+
+ Return a ``state`` dict with the current ``instance_state``, ``public_ip``,
+ ``placement``, and ``error`` keys, which capture the current state (the
+ values for those keys default to empty string if no data is available from
+ the cloud).
+ """
+ if self.launcher:
+ return self.launcher.get_status(self.instance_id)
+ # elif self.host_name:
+
+ else:
+ state = {'instance_state': "",
+ 'public_ip': "",
+ 'placement': "",
+ 'error': "No reference to the instance object"}
+ return state
+
+ def _init_instance(self, host_name):
+ self._update_host_name(host_name)
+
+ def wait_until_instance_ready(self, vm_ready_timeout=300, vm_ready_check_interval=10):
+ """
+ Wait until the VM state changes to ready/error or timeout elapses.
+ Updates the host name once ready.
+ """
+ assert vm_ready_timeout > 0
+ assert vm_ready_timeout > vm_ready_check_interval
+ assert vm_ready_check_interval > 0
+
+ if self.host_name: # Host name available. Therefore, instance is ready
+ return
+
+ for time_left in range(vm_ready_timeout, 0, -vm_ready_check_interval):
+ status = self.get_machine_status()
+ if status['public_ip'] != '' and status['error'] == '':
+ self._init_instance(status['public_ip'])
+ return
+ elif status['error'] != '':
+ msg = "Error launching an instance: {0}".format(status['error'])
+ bioblend.log.error(msg)
+ raise VMLaunchException(msg)
+ else:
+ bioblend.log.warn("Instance not ready yet (it's in state '{0}'); waiting another {1} seconds..."
+ .format(status['instance_state'], time_left))
+ time.sleep(vm_ready_check_interval)
+
+ raise VMLaunchException("Waited too long for instance to become ready. Instance Id: %s"
+ % self.instance_id)
+
+
+class CloudManInstance(GenericVMInstance):
+
+ def __init__(self, url, password, **kwargs):
+ """
+ Create an instance of the CloudMan API class, which is to be used when
+ manipulating that given CloudMan instance.
+
... 13335 lines suppressed ...
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/python-bioblend.git
More information about the Python-modules-commits
mailing list