[med-svn] [snakemake] 01/05: New upstream version 3.13.3
Kevin Murray
daube-guest at moszumanska.debian.org
Tue Jul 18 01:06:11 UTC 2017
This is an automated email from the git hooks/post-receive script.
daube-guest pushed a commit to branch master
in repository snakemake.
commit 955a3266693ac25b6426e64bf56294ba5200c512
Author: Kevin Murray <kdmfoss at gmail.com>
Date: Tue Jul 18 10:08:47 2017 +1000
New upstream version 3.13.3
---
CHANGELOG.md | 65 +++
Dockerfile | 17 +-
README.md | 13 +-
docs/getting_started/examples.rst | 5 +-
docs/getting_started/installation.rst | 35 +-
docs/index.rst | 34 +-
docs/project_info/authors.rst | 1 +
docs/project_info/citations.rst | 4 +-
docs/project_info/contributing.rst | 4 +-
docs/project_info/faq.rst | 25 +
docs/project_info/more_resources.rst | 4 +
docs/snakefiles/configuration.rst | 32 ++
docs/snakefiles/deployment.rst | 38 +-
docs/snakefiles/modularization.rst | 79 ++-
docs/snakefiles/remote_files.rst | 119 ++++-
docs/snakefiles/rules.rst | 190 ++++++-
docs/tutorial/additional_features.rst | 120 ++++-
docs/tutorial/advanced.rst | 51 +-
docs/tutorial/basics.rst | 100 +++-
docs/tutorial/{welcome.rst => setup.rst} | 46 +-
docs/tutorial/tutorial.rst | 37 ++
environment.yml | 12 +-
examples/c/src/Snakefile | 2 +-
misc/vim/syntax/snakemake.vim | 2 +-
setup.py | 6 +-
snakemake/__init__.py | 116 ++++-
snakemake/benchmark.py | 259 ++++++++++
snakemake/common.py | 6 +
snakemake/conda.py | 262 ++++++----
snakemake/dag.py | 118 +++--
snakemake/exceptions.py | 20 +-
snakemake/executors.py | 203 +++++---
snakemake/io.py | 32 +-
snakemake/jobs.py | 60 ++-
snakemake/logging.py | 16 +-
snakemake/parser.py | 9 +-
snakemake/persistence.py | 13 +-
snakemake/remote/FTP.py | 89 +++-
snakemake/remote/GS.py | 23 +-
snakemake/remote/HTTP.py | 65 ++-
snakemake/remote/NCBI.py | 574 +++++++++++++++++++++
snakemake/remote/S3.py | 52 +-
.../test_remote => snakemake/remote}/S3Mocked.py | 14 +-
snakemake/remote/SFTP.py | 28 +-
snakemake/remote/XRootD.py | 197 +++++++
snakemake/remote/__init__.py | 114 ++--
snakemake/remote/dropbox.py | 47 +-
snakemake/report.py | 4 +-
snakemake/rules.py | 38 +-
snakemake/scheduler.py | 39 +-
snakemake/script.py | 55 +-
snakemake/shell.py | 12 +-
snakemake/utils.py | 9 +
snakemake/version.py | 2 +-
snakemake/workflow.py | 43 +-
snakemake/wrapper.py | 10 +-
environment.yml => test-environment.yml | 11 +-
tests/test_benchmark/Snakefile | 30 +-
.../{test.benchmark.txt => test.benchmark_run.txt} | 0
.../expected-results/test.benchmark_run_shell.txt} | 0
...est.benchmark.txt => test.benchmark_script.txt} | 0
...test.benchmark.txt => test.benchmark_shell.txt} | 0
tests/test_benchmark/script.py | 7 +
tests/test_conda/Snakefile | 4 +-
tests/test_conda_custom_prefix/Snakefile | 12 +
.../expected-results/test0.out | 1 +
.../expected-results/test1.out | 1 +
.../expected-results/test2.out | 1 +
tests/test_conda_custom_prefix/test-env.yaml | 4 +
tests/test_default_remote/Snakefile | 40 ++
.../expected-results/.gitignore} | 0
tests/test_default_remote/test.txt | 1 +
tests/test_dynamic_temp/Snakefile | 30 ++
.../expected-results/out.txt} | 0
tests/test_dynamic_temp/test1.txt | 8 +
tests/test_dynamic_temp/test2.txt | 8 +
tests/test_ftp_immediate_close/Snakefile | 11 +
.../expected-results/size.txt | 1 +
tests/test_get_log_none/Snakefile | 2 +-
tests/test_issue260/Snakefile | 32 ++
.../test_issue260/expected-results/output/done.txt | 1 +
.../expected-results/output/result.n1 | 1 +
.../expected-results/output/result.n2 | 1 +
.../expected-results/output/result.n3 | 1 +
tests/test_remote/Snakefile | 21 +-
tests/test_remote_ncbi/Snakefile | 26 +
tests/test_remote_ncbi/expected-results/sizes.txt | 4 +
tests/test_remote_ncbi_simple/Snakefile | 14 +
.../expected-results/sizes.txt | 1 +
tests/test_run_namedlist/Snakefile | 5 +
.../expected-results/file.txt} | 0
tests/test_script/Snakefile | 13 +
tests/test_script/expected-results/test.html | 234 +++++++++
tests/test_script/scripts/test.Rmd | 22 +
tests/test_subworkflows/Snakefile | 7 +-
tests/test_symlink_time_handling/Snakefile | 22 +-
tests/test_xrootd/Snakefile | 27 +
tests/tests.py | 56 +-
wercker.yml | 4 +
99 files changed, 3563 insertions(+), 671 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8710eb4..80419e9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,70 @@
# Change Log
+## [3.13.3] - 2017-06-23
+### Changed
+- Fix a followup bug in Namedlist where a single item was not returned as string.
+
+## [3.13.2] - 2017-06-20
+### Changed
+- The --wrapper-prefix flag now also affects where the corresponding environment definition is fetched from.
+- Fix bug where empty output file list was recognized as containing duplicates (issue #574).
+
+
+## [3.13.1] - 2017-06-20
+### Changed
+- Fix --conda-prefix to be passed to all jobs.
+- Fix cleanup issue with scripts that fail to download.
+
+## [3.13.0] - 2017-06-12
+### Added
+- An NCBI remote provider. By this, you can seamlessly integrate any NCBI resouce (reference genome, gene/protein sequences, ...) as input file.
+### Changed
+- Snakemake now detects if automatically generated conda environments have to be recreated because the workflow has been moved to a new path.
+- Remote functionality has been made more robust, in particular to avoid race conditions.
+- `--config` parameter evaluation has been fixed for non-string types.
+- The Snakemake docker container is now based on the official debian image.
+
+## [3.12.0] - 2017-05-09
+### Added
+- Support for RMarkdown (.Rmd) in script directives.
+- New option --debug-dag that prints all decisions while building the DAG of jobs. This helps to debug problems like cycles or unexpected MissingInputExceptions.
+- New option --conda-prefix to specify the place where conda environments are stored.
+
+### Changed
+- Benchmark files now also include the maximal RSS and VMS size of the Snakemake process and all sub processes.
+- Speedup conda environment creation.
+- Allow specification, of DRMAA log dir.
+- Pass cluster config to subworkflow.
+
+
+## [3.11.2] - 2017-03-15
+### Changed
+- Fixed fix handling of local URIs with the wrapper directive.
+
+
+## [3.11.1] - 2017-03-14
+### Changed
+- --touch ignores missing files
+- Fixed handling of local URIs with the wrapper directive.
+
+
+## [3.11.0] - 2017-03-08
+### Added
+- Param functions can now also refer to threads.
+### Changed
+- Improved tutorial and docs.
+- Made conda integration more robust.
+- None is converted to NULL in R scripts.
+
+
+## [3.10.2] - 2017-02-28
+### Changed
+- Improved config file handling and merging.
+- Output files can be referred in params functions (i.e. lambda wildcards, output: ...)
+- Improved conda-environment creation.
+- Jobs are cached, leading to reduced memory footprint.
+- Fixed subworkflow handling in input functions.
+
## [3.10.0] - 2017-01-18
### Added
- Workflows can now be archived to a tarball with `snakemake --archive my-workflow.tar.gz`. The archive contains all input files, source code versioned with git and all software packages that are defined via conda environments. Hence, the archive allows to fully reproduce a workflow on a different machine. Such an archive can be uploaded to Zenodo, such that your workflow is secured in a self-contained, executable way for the future.
diff --git a/Dockerfile b/Dockerfile
index ef6a970..5407fc8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,16 @@
-FROM continuumio/miniconda3
+FROM bitnami/minideb:jessie
MAINTAINER Johannes Köster <johannes.koester at tu-dortmund.de>
ADD . /tmp/repo
-RUN apt-get upgrade -y
-RUN conda update conda
-RUN conda env update --name root --file /tmp/repo/environment.yml
+# taken from condaforge/linux-anvil
+#RUN apt-get update && \
+# apt-get install -y wget bzip2 && \
+# rm -rf /var/lib/apt/lists/*
+RUN install_packages wget bzip2
+RUN wget --no-check-certificate https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+ bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \
+ rm Miniconda3-latest-Linux-x86_64.sh
+ENV PATH /opt/conda/bin:${PATH}
+ENV LANG C.UTF-8
+ENV SHELL /bin/bash
+RUN conda env update --name root --file /tmp/repo/environment.yml && conda clean --all -y
RUN pip install /tmp/repo
diff --git a/README.md b/README.md
index b488bdd..e0dd8eb 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,12 @@
-Snakemake -- a pythonic workflow system
+[](https://app.wercker.com/project/byKey/5b4faec0485e3b6ed5497f3e8e551b34)
-Snakemake is a workflow management system that aims to reduce the complexity of creating workflows by providing a fast and comfortable execution environment, together with a clean and modern specification language in python style. Snakemake workflows are essentially Python scripts extended by declarative code to define rules. Rules describe how to create output files from input files.
+# Snakemake - a pythonic workflow system
-Homepage: http://snakemake.bitbucket.org
+Snakemake is a workflow management system that aims to reduce the complexity of creating workflows
+by providing a fast and comfortable execution environment, together with a clean and readable
+specification language in Python style. Snakemake workflows are essentially Python scripts extended
+by declarative code to define rules. Rules describe how to create output files from input files.
-Copyright (c) 2016 Johannes Köster <johannes.koester at tu-dortmund.de> (see LICENSE)
+Homepage: https://snakemake.bitbucket.io
+
+Copyright (c) 2012-2017 Johannes Köster <johannes.koester at protonmail.com> (see LICENSE)
\ No newline at end of file
diff --git a/docs/getting_started/examples.rst b/docs/getting_started/examples.rst
index b6f7dfc..c370dc7 100644
--- a/docs/getting_started/examples.rst
+++ b/docs/getting_started/examples.rst
@@ -181,9 +181,10 @@ A Snakefile can be easily written as
output:
temp('{ODIR}/{name}.o')
input:
- '{name}.c', HEADERS
+ src='{name}.c',
+ headers=HEADERS
shell:
- "{CC} -c -o {output} {input} {CFLAGS}"
+ "{CC} -c -o {output} {input.src} {CFLAGS}"
rule clean:
"""clean up temporary files"""
diff --git a/docs/getting_started/installation.rst b/docs/getting_started/installation.rst
index 70e0def..4b3ea6c 100644
--- a/docs/getting_started/installation.rst
+++ b/docs/getting_started/installation.rst
@@ -7,6 +7,27 @@ Installation
Snakemake is available on PyPi as well as through Bioconda and also from source code.
You can use one of the following ways for installing Snakemake.
+Installation via Conda
+======================
+
+On **Linux** and **MacOSX**, this is the recommended way to install Snakemake,
+because it also enables Snakemake to :ref:`handle software dependencies of your
+workflow <integrated_package_management>`.
+
+First, you have to install the Miniconda Python3 distribution.
+See `here <https://conda.io/docs/install/quick.html>`_ for installation instructions.
+Make sure to ...
+
+* Install the **Python 3** version of Miniconda.
+* Answer yes to the question whether conda shall be put into your PATH.
+
+Then, you can install Snakemake with
+
+.. code-block:: console
+
+ $ conda install -c bioconda snakemake
+
+from the `Bioconda <https://bioconda.github.io>`_ channel.
Global Installation
===================
@@ -38,20 +59,6 @@ To create an installation in a virtual environment, use the following commands:
$ pip install snakemake
-Installing Conda
-================
-
-In case you have to install Python 3 yourself, we recommend to use the Miniconda Python 3 distribution (http://conda.pydata.org/miniconda.html).
-
-With Miniconda installed, you can issue
-
-.. code-block:: console
-
- $ conda install -c bioconda snakemake
-
-to install Snakemake from the bioconda channel.
-
-
Installing from Source
======================
diff --git a/docs/index.rst b/docs/index.rst
index aef65fc..e311f6d 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -16,8 +16,8 @@ Welcome to Snakemake's documentation!
.. image:: https://quay.io/repository/snakemake/snakemake/status
:target: https://quay.io/repository/snakemake/snakemake
-.. image:: https://img.shields.io/circleci/project/bitbucket/snakemake/snakemake.svg
- :target: https://circleci.com/bb/snakemake/snakemake/tree/master
+.. image:: https://app.wercker.com/status/5b4faec0485e3b6ed5497f3e8e551b34/s/master
+ :target: https://app.wercker.com/project/byKey/5b4faec0485e3b6ed5497f3e8e551b34
.. image:: https://img.shields.io/badge/stack-overflow-orange.svg
:target: http://stackoverflow.com/questions/tagged/snakemake
@@ -63,7 +63,7 @@ Quick Example
Getting started
---------------
-To get started, consider the :ref:`tutorial <tutorial-welcome>`, the `introductory slides <http://slides.com/johanneskoester/snakemake-tutorial-2016>`_, and the :ref:`FAQ <project_info-faq>`.
+To get started, consider the :ref:`tutorial`, the `introductory slides <http://slides.com/johanneskoester/snakemake-tutorial-2016>`_, and the :ref:`FAQ <project_info-faq>`.
.. _main-support:
@@ -109,6 +109,15 @@ Publications using Snakemake
In the following you find an incomplete list of publications making use of Snakemake for their analyses.
Please consider to add your own.
+* Uhlitz et al. 2017. `An immediate–late gene expression module decodes ERK signal duration <http://msb.embopress.org/content/13/5/928>`_. Molecular Systems Biology.
+* Akkouche et al. 2017. `Piwi Is Required during Drosophila Embryogenesis to License Dual-Strand piRNA Clusters for Transposon Repression in Adult Ovaries <http://www.sciencedirect.com/science/article/pii/S1097276517302071>`_. Molecular Cell.
+* Beatty et al. 2017. `Giardia duodenalis induces pathogenic dysbiosis of human intestinal microbiota biofilms <>`_. International Journal for Parasitology.
+* Meyer et al. 2017. `Differential Gene Expression in the Human Brain Is Associated with Conserved, but Not Accelerated, Noncoding Sequences <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5400397/>`_. Molecular Biology and Evolution.
+* Lonardo et al. 2017. `Priming of soil organic matter: Chemical structure of added compounds is more important than the energy content <http://www.sciencedirect.com/science/article/pii/S0038071716304539>`_. Soil Biology and Biochemistry.
+* Beisser et al. 2017. `Comprehensive transcriptome analysis provides new insights into nutritional strategies and phylogenetic relationships of chrysophytes <https://peerj.com/articles/2832/>`_. PeerJ.
+* Dimitrov et al 2017. `Successive DNA extractions improve characterization of soil microbial communities <https://peerj.com/articles/2915/>`_. PeerJ.
+* de Bourcy et al. 2016. `Phylogenetic analysis of the human antibody repertoire reveals quantitative signatures of immune senescence and aging <http://www.pnas.org/content/114/5/1105.short>`_. PNAS.
+* Bray et al. 2016. `Near-optimal probabilistic RNA-seq quantification<http://www.nature.com/nbt/journal/v34/n5/abs/nbt.3519.html>`_. Nature Biotechnology.
* Etournay et al. 2016. `TissueMiner: a multiscale analysis toolkit to quantify how cellular processes create tissue dynamics <https://elifesciences.org/content/5/e14334>`_. eLife Sciences.
* Townsend et al. 2016. `The Public Repository of Xenografts Enables Discovery and Randomized Phase II-like Trials in Mice <http://www.cell.com/cancer-cell/abstract/S1535-6108%2816%2930090-3>`_. Cancer Cell.
* Burrows et al. 2016. `Genetic Variation, Not Cell Type of Origin, Underlies the Majority of Identifiable Regulatory Differences in iPSCs <http://journals.plos.org/plosgenetics/article?id=10.1371/journal.pgen.1005793>`_. PLOS Genetics.
@@ -122,7 +131,6 @@ Please consider to add your own.
* Břinda et al. 2015. `Spaced seeds improve k-mer-based metagenomic classification <http://bioinformatics.oxfordjournals.org/content/early/2015/08/10/bioinformatics.btv419>`_. Bioinformatics.
* Spjuth et al. 2015. `Experiences with workflows for automating data-intensive bioinformatics <http://www.biologydirect.com/content/10/1/43>`_. Biology Direct.
* Schramm et al. 2015. `Mutational dynamics between primary and relapse neuroblastomas <http://www.nature.com/ng/journal/v47/n8/full/ng.3349.html>`_. Nature Genetics.
-* Bray et al. 2015. `Near-optimal RNA-Seq quantification <http://arxiv.org/abs/1505.02710>`_. Arxiv preprint.
* Berulava et al. 2015. `N6-Adenosine Methylation in MiRNAs <http://journals.plos.org/plosone/article?id=10.1371/journal.pone.0118438>`_. PLOS ONE.
* The Genome of the Netherlands Consortium 2014. `Whole-genome sequence variation, population structure and demographic history of the Dutch population <http://www.nature.com/ng/journal/v46/n8/full/ng.3021.html>`_. Nature Genetics.
* Patterson et al. 2014. `WhatsHap: Haplotype Assembly for Future-Generation Sequencing Reads <http://online.liebertpub.com/doi/10.1089/cmb.2014.0157>`_. Journal of Computational Biology.
@@ -138,33 +146,23 @@ Please consider to add your own.
.. toctree::
- :caption: Installation
- :name: installation
+ :caption: Getting started
+ :name: getting_started
:hidden:
:maxdepth: 1
getting_started/installation
getting_started/examples
+ tutorial/tutorial
.. toctree::
- :caption: Tutorial
- :name: tutorial
- :hidden:
- :maxdepth: 1
-
- tutorial/welcome
- tutorial/basics
- tutorial/advanced
- tutorial/additional_features
-
-.. toctree::
:caption: Executing workflows
:name: execution
:hidden:
:maxdepth: 1
- executable.rst
+ executable
.. toctree::
:caption: Defining workflows
diff --git a/docs/project_info/authors.rst b/docs/project_info/authors.rst
index d8091d4..1cba544 100644
--- a/docs/project_info/authors.rst
+++ b/docs/project_info/authors.rst
@@ -18,6 +18,7 @@ Development Team
- Tim Booth
- Manuel Holtgrewe
- Christian Arnold
+- Wibowo Arindrarto
Contributors
------------
diff --git a/docs/project_info/citations.rst b/docs/project_info/citations.rst
index 7477bb4..1b00ad7 100644
--- a/docs/project_info/citations.rst
+++ b/docs/project_info/citations.rst
@@ -39,13 +39,13 @@ Project Pages
If you publish a Snakemake workflow, consider to add this badge to your project page:
.. image:: https://img.shields.io/badge/snakemake-≥3.5.2-brightgreen.svg?style=flat-square
- :target: http://snakemake.bitbucket.org
+ :target: https://snakemake.bitbucket.io
The markdown syntax is
.. sourcecode:: text
- [](http://snakemake.bitbucket.org)
+ [](https://snakemake.bitbucket.io)
Replace the ``3.5.2`` with the minimum required Snakemake version.
You can also `change the style <http://shields.io/#styles>`_.
diff --git a/docs/project_info/contributing.rst b/docs/project_info/contributing.rst
index 7dec5b1..8972365 100644
--- a/docs/project_info/contributing.rst
+++ b/docs/project_info/contributing.rst
@@ -99,12 +99,12 @@ The easiest way to run your development version of Snakemake is perhaps to go to
.. code-block:: bash
conda env create -f environment.yml -n snakemake-testing
- pip install -e .
source activate snakemake-testing
+ pip install -e .
This will make your development version of Snakemake the one called when running snakemake. You do not need to run this command after each time you make code changes.
-From the base snakemake folder you call :code:`python setup.py nosetest` to run all the tests. (If it complains that you do not have nose installed, which is the testing framework we use, you can simply install it by running :code:`pip install nose`.)
+From the base snakemake folder you call :code:`python setup.py nosetests` to run all the tests. (If it complains that you do not have nose installed, which is the testing framework we use, you can simply install it by running :code:`pip install nose`.)
If you introduce a new feature you should add a new test to the tests directory. See the folder for examples.
diff --git a/docs/project_info/faq.rst b/docs/project_info/faq.rst
index e2597f4..2eff5c5 100644
--- a/docs/project_info/faq.rst
+++ b/docs/project_info/faq.rst
@@ -411,3 +411,28 @@ To remove all files created by snakemake as output files to start from scratch,
.. code-block:: console
rm $(snakemake --summary | tail -n+2 | cut -f1)
+
+
+Why can't I use the conda directive with a run block?
+-----------------------------------------------------
+
+The run block of a rule (see :ref:`snakefiles-rules`) has access to anything defined in the Snakefile, outside of the rule.
+Hence, it has to share the conda environment with the main Snakemake process.
+To avoid confusion we therefore disallow the conda directive together with the run block.
+It is recommended to use the script directive instead (see :ref:`snakefiles-external_scripts`).
+
+
+My workflow is very large, how to I stop Snakemake from printing all this rule/job information in a dry-run?
+------------------------------------------------------------------------------------------------------------
+
+Indeed, the information for each individual job can slow down a dryrun if there are tens of thousands of jobs.
+If you are just interested in the final summary, you can use the ``--quiet`` flag to suppress this.
+
+.. code-block:: console
+
+ $ snakemake -n --quiet
+
+Git is messing up the modification times of my input files, what can I do?
+--------------------------------------------------------------------------
+
+When you checkout a git repository, the modification times of updated files are set to the time of the checkout. If you rely on these files as input **and** output files in your workflow, this can cause trouble. For example, Snakemake could think that a certain (git-tracked) output has to be re-executed, just because its input has been checked out a bit later. In such cases, it is advisable to set the file modification dates to the last commit date after an update has been pulled. See `h [...]
diff --git a/docs/project_info/more_resources.rst b/docs/project_info/more_resources.rst
index 1e0472e..1e69171 100644
--- a/docs/project_info/more_resources.rst
+++ b/docs/project_info/more_resources.rst
@@ -25,6 +25,8 @@ Talks and Posters
External Resources
------------------
+These resources are not part of the official documentation.
+
* `Snakemake workflow used for the Kallisto paper <https://github.com/pachterlab/kallisto_paper_analysis>`_
* `An alternative tutorial for Snakemake <http://slowkow.com/notes/snakemake-tutorial/>`_
* `An Emacs mode for Snakemake <http://melpa.milkbox.net/#/snakemake-mode>`_
@@ -32,3 +34,5 @@ External Resources
* `Sandwiches with Snakemake <https://github.com/leipzig/SandwichesWithSnakemake>`_
* `A visualization of the past years of Snakemake development <http://youtu.be/bq3vXrWw1yk>`_
* `Japanese version of the Snakemake tutorial <https://github.com/joemphilips/Translate_Snakemake_Tutorial>`_
+* `Basic <http://bioinfo-fr.net/snakemake-pour-les-nuls>`_ and `advanced <http://bioinfo-fr.net/snakemake-aller-plus-loin-avec-la-parallelisation>`_ french Snakemake tutorial.
+* `Mini tutorial on Snakemake and Bioconda <https://github.com/dlaehnemann/TutMinicondaSnakemake>`_
diff --git a/docs/snakefiles/configuration.rst b/docs/snakefiles/configuration.rst
index 1421e35..f5aa183 100644
--- a/docs/snakefiles/configuration.rst
+++ b/docs/snakefiles/configuration.rst
@@ -37,6 +37,10 @@ In addition to the `configfile` statement, config values can be overwritten via
$ snakemake --config yourparam=1.5
Further, you can manually alter the config dictionary using any Python code **outside** of your rules. Changes made from within a rule won't be seen from other rules.
+Finally, you can use the `--configfile` command line argument to overwrite values from the `configfile` statement.
+Note that any values parsed into the `config` dictionary with any of above mechanisms are merged, i.e., all keys defined via a `configfile`
+statement, or the `--configfile` and `--config` command line arguments will end up in the final `config` dictionary, but if two methods define the same key, command line
+overwrites the `configfile` statement.
For adding config placeholders into a shell command, Python string formatting syntax requires you to leave out the quotes around the key name, like so:
@@ -98,6 +102,34 @@ Here ``__default__`` is a special object that specifies default parameters, thes
$ snakemake -j 999 --cluster-config cluster.json --cluster "sbatch -A {cluster.account} -p {cluster.partition} -n {cluster.n} -t {cluster.time}"
+For cluster systems using LSF/BSUB, a cluster config may look like this:
+
+.. code-block:: json
+
+ {
+ "__default__" :
+ {
+ "queue" : "medium_priority",
+ "nCPUs" : "16",
+ "memory" : 20000,
+ "resources" : "\"select[mem>20000] rusage[mem=20000] span[hosts=1]\"",
+ "name" : "JOBNAME.{rule}.{wildcards}",
+ "output" : "logs/cluster/{rule}.{wildcards}.out",
+ "error" : "logs/cluster/{rule}.{wildcards}.err"
+ },
+
+
+ "trimming_PE" :
+ {
+ "memory" : 30000,
+ "resources" : "\"select[mem>30000] rusage[mem=30000] span[hosts=1]\"",
+ }
+ }
+
+The advantage of this setup is that it is already pretty general by exploiting the wildcard possibilities that Snakemake provides via ``{rule}`` and ``{wildcards}``. So job names, output and error files all have reasonable and trackable default names, only the directies (``logs/cluster``) and job names (``JOBNAME``) have to adjusted accordingly.
+If a rule named ``bamCoverage`` is executed with the wildcard ``basename = sample1``, for example, the output and error files will be ``bamCoverage.basename=sample1.out`` and ``bamCoverage.basename=sample1.err``, respectively.
+
+
---------------------------
Configure Working Directory
---------------------------
diff --git a/docs/snakefiles/deployment.rst b/docs/snakefiles/deployment.rst
index cc2bcb0..505e7a1 100644
--- a/docs/snakefiles/deployment.rst
+++ b/docs/snakefiles/deployment.rst
@@ -51,7 +51,7 @@ Integrated Package Management
With Snakemake 3.9.0 it is possible to define isolated software environments per rule.
Upon execution of a workflow, the `Conda package manager <http://conda.pydata.org>`_ is used to obtain and deploy the defined software packages in the specified versions. Packages will be installed into your working directory, without requiring any admin/root priviledges.
Given that conda is available on your system (see `Miniconda <http://conda.pydata.org/miniconda.html>`_), to use the Conda integration, add the ``--use-conda`` flag to your workflow execution command, e.g. ``snakemake --cores 8 --use-conda``.
-When ``--use-conda`` is activated, Snakemake will automatically create software environments for any used wrapper (see above).
+When ``--use-conda`` is activated, Snakemake will automatically create software environments for any used wrapper (see :ref:`snakefiles-wrappers`).
Further, you can manually define environments via the ``conda`` directive, e.g.:
.. code-block:: python
@@ -78,4 +78,38 @@ with the following `environment definition <http://conda.pydata.org/docs/using/e
- r-ggplot2=2.1.0
Snakemake will store the environment persistently in ``.snakemake/conda/$hash`` with ``$hash`` being the MD5 hash of the environment definition file content. This way, updates to the environment definition are automatically detected.
-Note that you need to clean up environments manually for now. However, they are lightweight and consist only of symlinks to your central conda installation.
+Note that you need to clean up environments manually for now. However, in many cases they are lightweight and consist of symlinks to your central conda installation.
+
+
+--------------------------------------
+Sustainable and reproducible archiving
+--------------------------------------
+
+With Snakemake 3.10.0 it is possible to archive a workflow into a
+`tarball <https://en.wikipedia.org/wiki/Tar_(computing)>`_
+(`.tar`, `.tar.gz`, `.tar.bz2`, `.tar.xz`), via
+
+.. code-block:: bash
+
+ snakemake --archive my-workflow.tar.gz
+
+If above layout is followed, this will archive any code and config files that
+is under git version control. Further, all input files will be included into the
+archive. Finally, the software packages of each defined conda environment are included.
+This results in a self-contained workflow archive that can be re-executed on a
+vanilla machine that only has Conda and Snakemake installed via
+
+.. code-block:: bash
+
+ tar -xf my-workflow.tar.gz
+ snakemake -n
+
+Note that the archive is platform specific. For example, if created on Linux, it will
+run on any Linux newer than the minimum version that has been supported by the used
+Conda packages at the time of archiving (e.g. CentOS 6).
+
+A useful pattern when publishing data analyses is to create such an archive,
+upload it to `Zenodo <https://zenodo.org/>`_ and thereby obtain a
+`DOI <https://en.wikipedia.org/wiki/Digital_object_identifier>`_.
+Then, the DOI can be cited in manuscripts, and readers are able to download
+and reproduce the data analysis at any time in the future.
diff --git a/docs/snakefiles/modularization.rst b/docs/snakefiles/modularization.rst
index 1506ef7..ea15d7a 100644
--- a/docs/snakefiles/modularization.rst
+++ b/docs/snakefiles/modularization.rst
@@ -4,14 +4,44 @@
Modularization
==============
-Snakemake provides several means for modularization of your workflows.
-These features allow you to:
+Modularization in Snakemake comes at different levels.
-- distribute large workflows over multiple smaller files,
-- split workflows into different steps/sub workflows that
- - make things more clear by introducing structure and
- - allow for reuseable sub workflows, and
-- use reuseable wrapper scripts for certain tools instead of copy-and-paste code.
+1. The most fine-grained level are wrappers. They are available and can be published at the `Snakemake Wrapper Repository <https://snakemake-wrappers.readthedocs.io>`_. These wrappers can then be composed and customized according to your needs, by copying skeleton rules into your workflow. In combination with conda integration, wrappers also automatically deploy the needed software dependencies into isolated environments.
+2. For larger, reusable parts that shall be integrated into a common workflow, it is recommended to write small Snakefiles and include them into a master Snakefile via the include statement. In such a setup, all rules share a common config file.
+3. The third level of separation are subworkflows. Importantly, these are rather meant as links between otherwise separate data analyses.
+
+
+.. _snakefiles-wrappers:
+
+--------
+Wrappers
+--------
+
+With Snakemake 3.5.5, the wrapper directive is introduced (experimental).
+This directive allows to have re-usable wrapper scripts around e.g. command line tools. In contrast to modularization strategies like ``include`` or subworkflows, the wrapper directive allows to re-wire the DAG of jobs.
+For example
+
+.. code-block:: python
+
+ rule samtools_sort:
+ input:
+ "mapped/{sample}.bam"
+ output:
+ "mapped/{sample}.sorted.bam"
+ params:
+ "-m 4G"
+ threads: 8
+ wrapper:
+ "0.0.8/bio/samtools_sort"
+
+Refers to the wrapper ``"0.0.8/bio/samtools_sort"`` to create the output from the input.
+Snakemake will automatically download the wrapper from the `Snakemake Wrapper Repository <https://bitbucket.org/snakemake/snakemake-wrappers>`_.
+Thereby, 0.0.8 can be replaced with the git version tag you want to use, or a commit id (see `here <https://bitbucket.org/snakemake/snakemake-wrappers/commits>`_).
+This ensures reproducibility since changes in the wrapper implementation won't be propagated automatically to your workflow.
+Alternatively, e.g., for development, the wrapper directive can also point to full URLs, including URLs to local files with absolute paths ``file://`` or relative paths ``file:``.
+Examples for each wrapper can be found in the READMEs located in the wrapper subdirectories at the `Snakemake Wrapper Repository <https://bitbucket.org/snakemake/snakemake-wrappers>`_.
+
+The `Snakemake Wrapper Repository <https://bitbucket.org/snakemake/snakemake-wrappers>`_ is meant as a collaborative project and pull requests are very welcome.
.. _snakefiles-includes:
@@ -28,7 +58,7 @@ Another Snakefile with all its rules can be included into the current:
The default target rule (often called the ``all``-rule), won't be affected by the include.
I.e. it will always be the first rule in your Snakefile, no matter how many includes you have above your first rule.
-From version 3.2 on, includes are relative to the directory of the Snakefile in which they occur.
+Includes are relative to the directory of the Snakefile in which they occur.
For example, if above Snakefile resides in the directory ``my/dir``, then Snakemake will search for the include at ``my/dir/path/to/other/snakefile``, regardless of the working directory.
@@ -41,7 +71,7 @@ Sub-Workflows
In addition to including rules of another workflow, Snakemake allows to depend on the output of other workflows as sub-workflows.
A sub-workflow is executed independently before the current workflow is executed.
Thereby, Snakemake ensures that all files the current workflow depends on are created or updated if necessary.
-This allows to build nice hierarchies of separated workflows and even to separate the directories on disk.
+This allows to create links between otherwise separate data analyses.
.. code-block:: python
@@ -66,34 +96,3 @@ Then the current workflow is executed.
This can also happen recursively, since the subworkflow may have its own subworkflows as well.
-.. _snakefiles-wrappers:
-
---------
-Wrappers
---------
-
-With Snakemake 3.5.5, the wrapper directive is introduced (experimental).
-This directive allows to have re-usable wrapper scripts around e.g. command line tools. In contrast to modularization strategies like ``include`` or subworkflows, the wrapper directive allows to re-wire the DAG of jobs.
-For example
-
-.. code-block:: python
-
- rule samtools_sort:
- input:
- "mapped/{sample}.bam"
- output:
- "mapped/{sample}.sorted.bam"
- params:
- "-m 4G"
- threads: 8
- wrapper:
- "0.0.8/bio/samtools_sort"
-
-Refers to the wrapper ``"0.0.8/bio/samtools_sort"`` to create the output from the input.
-Snakemake will automatically download the wrapper from the `Snakemake Wrapper Repository <https://bitbucket.org/snakemake/snakemake-wrappers>`_.
-Thereby, 0.0.8 can be replaced with the git version tag you want to use, or a commit id (see `here <https://bitbucket.org/snakemake/snakemake-wrappers/commits>`_).
-This ensures reproducibility since changes in the wrapper implementation won't be propagated automatically to your workflow.
-Alternatively, e.g., for development, the wrapper directive can also point to full URLs, including URLs to local files with ``file://``.
-Examples for each wrapper can be found in the READMEs located in the wrapper subdirectories at the `Snakemake Wrapper Repository <https://bitbucket.org/snakemake/snakemake-wrappers>`_.
-
-The `Snakemake Wrapper Repository <https://bitbucket.org/snakemake/snakemake-wrappers>`_ is meant as a collaborative project and pull requests are very welcome.
diff --git a/docs/snakefiles/remote_files.rst b/docs/snakefiles/remote_files.rst
index a63baf6..1314acb 100644
--- a/docs/snakefiles/remote_files.rst
+++ b/docs/snakefiles/remote_files.rst
@@ -6,7 +6,7 @@ Remote files
In versions ``snakemake>=3.5``.
-The ``Snakefile`` supports a wrapper function, ``remote()``, indicating a file is on a remote storage provider (this is similar to ``temp()`` or ``protected()``). In order to use all types of remote files, the Python packages ``boto``, ``moto``, ``filechunkio``, ``pysftp``, ``dropbox``, ``requests``, and ``ftputil`` must be installed.
+The ``Snakefile`` supports a wrapper function, ``remote()``, indicating a file is on a remote storage provider (this is similar to ``temp()`` or ``protected()``). In order to use all types of remote files, the Python packages ``boto``, ``moto``, ``filechunkio``, ``pysftp``, ``dropbox``, ``requests``, ``ftputil``, ``XRootD``, and ``biopython`` must be installed.
During rule execution, a remote file (or object) specified is downloaded to the local ``cwd``, within a sub-directory bearing the same name as the remote provider. This sub-directory naming lets you have multiple remote origins with reduced likelihood of name collisions, and allows Snakemake to easily translate remote objects to local file paths. You can think of each local remote sub-directory as a local mirror of the remote system. The ``remote()`` wrapper is mutually-exclusive with th [...]
@@ -18,6 +18,8 @@ Snakemake includes the following remote providers, supported by the correspondin
* Read-only web (HTTP[S]): ``snakemake.remote.HTTP``
* File transfer protocol (FTP): ``snakemake.remote.FTP``
* Dropbox: ``snakemake.remote.dropbox``
+* XRootD: ``snakemake.remote.XRootD``
+* GenBank / NCBI Entrez: ``snakemake.remote.NCBI``
Amazon Simple Storage Service (S3)
@@ -88,6 +90,13 @@ If you wish to have a rule to simply download a file to a local copy, you can do
run:
shell("cp {output[0]} ./")
+In some cases the rule can use the data directly on the remote provider, in these cases ``stay_on_remote=True`` can be set to avoid downloading/uploading data unnecessarily. Additionally, if the backend supports it, any potentially corrupt output files will be removed from the remote. The default for ``stay_on_remote`` and ``keep_local`` can be configured by setting these properties on the remote provider object:
+
+.. code-block:: python
+
+ from snakemake.remote.S3 import RemoteProvider as S3RemoteProvider
+ S3 = S3RemoteProvider(access_key_id="MYACCESSKEY", secret_access_key="MYSECRET", keep_local=True, stay_on_remote=True)
+
The remote provider also supports a new ``glob_wildcards()`` (see :ref:`glob-wildcards`) which acts the same as the local version of ``glob_wildcards()``, but for remote files:
.. code-block:: python
@@ -355,6 +364,30 @@ Anonymous download of FTP resources is possible:
print(FTP.glob_wildcards("example.com/somedir/{file}.txt"))
+Setting `immediate_close=True` allows the use of a large number of remote FTP input files in a job where the endpoint server limits the number of concurrent connections. When `immediate_close=True`, Snakemake will terminate FTP connections after each remote file action (`exists()`, `size()`, `download()`, `mtime()`, etc.). This is in contrast to the default behavior which caches FTP details and leaves the connection open across actions to improve performance (closing the connection upon [...]
+
+.. code-block:: python
+
+ from snakemake.remote.FTP import RemoteProvider as FTPRemoteProvider
+ FTP = FTPRemoteProvider()
+
+ rule all:
+ input:
+ # only keep the file so we can move it out to the cwd
+ # This server limits the number of concurrent connections so we need to have Snakemake close each after each FTP action.
+ FTP.remote(expand("ftp.example.com/rel/path/to/{file}", file=large_list), keep_local=True, immediate_close=True)
+ run:
+ shell("mv {input} ./")
+
+``glob_wildcards()``:
+
+.. code-block:: python
+
+ from snakemake.remote.FTP import RemoteProvider as FTPRemoteProvider
+ FTP = FTPRemoteProvider(username="myusername", password="mypassword")
+
+ print(FTP.glob_wildcards("example.com/somedir/{file}.txt"))
+
Dropbox
=======
@@ -382,6 +415,90 @@ Using the Dropbox provider is straightforward:
Note that Dropbox paths are case-insensitive.
+XRootD
+=======
+
+Snakemake can be used with `XRootD <http://xrootd.org/>`_ backed storage provided the python bindings are installed.
+This is typically most useful when combined with the ``stay_on_remote`` flag to minimise local storage requirements.
+This flag can be overridden on a file by file basis as described in the S3 remote. Additionally ``glob_wildcards()`` is supported:
+
+.. code-block:: python
+
+ from snakemake.remote.XRootD import RemoteProvider as XRootDRemoteProvider
+
+ XRootD = XRootDRemoteProvider(stay_on_remote=True)
+ file_numbers = XRootD.glob_wildcards("root://eospublic.cern.ch//eos/opendata/lhcb/MasterclassDatasets/D0lifetime/2014/mclasseventv2_D0_{n}.root")
+
+ rule all:
+ input:
+ XRootD.remote(expand("local_data/mclasseventv2_D0_{n}.root", n=file_numbers))
+
+ rule make_data:
+ input:
+ XRootD.remote("root://eospublic.cern.ch//eos/opendata/lhcb/MasterclassDatasets/D0lifetime/2014/mclasseventv2_D0_{n}.root")
+ output:
+ 'local_data/mclasseventv2_D0_{n}.root'
+ shell:
+ 'xrdcp {input[0]} {output[0]}'
+
+GenBank / NCBI Entrez
+=====================
+
+Snakemake can directly source input files from `GenBank <https://www.ncbi.nlm.nih.gov/genbank/>`_ and other `NCBI Entrez databases <https://www.ncbi.nlm.nih.gov/books/NBK25497/table/chapter2.T._entrez_unique_identifiers_ui/?report=objectonly>`_ if the Biopython library is installed.
+
+.. code-block:: python
+
+ from snakemake.remote.NCBI import RemoteProvider as NCBIRemoteProvider
+ NCBI = NCBIRemoteProvider(email="someone at example.com") # email required by NCBI to prevent abuse
+
+ rule all:
+ input:
+ "size.txt"
+
+ rule download_and_count:
+ input:
+ NCBI.remote("KY785484.1.fasta", db="nuccore")
+ output:
+ "size.txt"
+ run:
+ shell("wc -c {input} > {output}")
+
+The output format and source database of a record retrieved from GenBank is inferred from the file extension specified. For example, ``NCBI.RemoteProvider().remote("KY785484.1.fasta", db="nuccore")`` will download a FASTA file while ``NCBI.RemoteProvider().remote("KY785484.1.gb", db="nuccore")`` will download a GenBank-format file. If the options are ambiguous, Snakemake will raise an exception and inform the user of possible format choices. To see available formats, consult the
+in a variety of `Entrez EFetch documentation <https://www.ncbi.nlm.nih.gov/books/NBK25499/table/chapter4.T._valid_values_of__retmode_and/?report=objectonly>`_. To view the valid file extensions for these formats, access ``NCBI.RemoteProvider()._gb.valid_extensions``, or instantiate an ``NCBI.NCBIHelper()`` and access ``NCBI.NCBIHelper().valid_extensions`` (this is a property).
+
+When used in conjunction with ``NCBI.RemoteProvider().search()``, Snakemake and ``NCBI.RemoteProvider().remote()`` can be used to find accessions by query and download them:
+
+.. code-block:: python
+
+ from snakemake.remote.NCBI import RemoteProvider as NCBIRemoteProvider
+ NCBI = NCBIRemoteProvider(email="someone at example.com") # email required by NCBI to prevent abuse
+
+ # get accessions for the first 3 results in a search for full-length Zika virus genomes
+ # the query parameter accepts standard GenBank search syntax
+ query = '"Zika virus"[Organism] AND (("9000"[SLEN] : "20000"[SLEN]) AND ("2017/03/20"[PDAT] : "2017/03/24"[PDAT])) '
+ accessions = NCBI.search(query, retmax=3)
+
+ # give the accessions a file extension to help the RemoteProvider determine the
+ # proper output type.
+ input_files = expand("{acc}.fasta", acc=accessions)
+
+ rule all:
+ input:
+ "sizes.txt"
+
+ rule download_and_count:
+ input:
+ # Since *.fasta files could come from several different databases, specify the database here.
+ # if the input files are ambiguous, the provider will alert the user with possible options
+ # standard options like "seq_start" are supported
+ NCBI.remote(input_files, db="nuccore", seq_start=5000)
+
+ output:
+ "sizes.txt"
+ run:
+ shell("wc -c {input} > sizes.txt")
+
+Normally, all accessions for a query are returned from ``NCBI.RemoteProvider.search()``. To truncate the results, specify ``retmax=<desired_number>``. Standard Entrez `fetch query options <https://www.ncbi.nlm.nih.gov/books/NBK25499/#chapter4.EFetch>`_ are supported as kwargs, and may be passed in to ``NCBI.RemoteProvider.remote()`` and ``NCBI.RemoteProvider.search()``.
Remote cross-provider transfers
===============================
diff --git a/docs/snakefiles/rules.rst b/docs/snakefiles/rules.rst
index 55ace0a..c184a3e 100644
--- a/docs/snakefiles/rules.rst
+++ b/docs/snakefiles/rules.rst
@@ -49,6 +49,8 @@ Further, this combination of python and shell commands, allows to iterate over t
Note that shell commands in Snakemake use the bash shell in `strict mode <http://redsymbol.net/articles/unofficial-bash-strict-mode/>`_ by default.
+.. _snakefiles-wildcards:
+
Wildcards
---------
@@ -72,6 +74,8 @@ For example, if another rule in the workflow requires the file the file ``101/fi
Thus, it requests file ``101/inputfile`` as input and executes the command ``somecommand --group A < 101/inputfile > 101/file.A.txt``.
Of course, the input file might have to be generated by another rule with different wildcards.
+Importantly, the wildcard names in input and output must be named identically. Most typically, the same wildcard is present in both input and output, but it is of course also possible to have wildcards only in the output but not the input section.
+
Multiple wildcards in one filename can cause ambiguity.
Consider the pattern ``{dataset}.{group}.txt`` and assume that a file ``101.B.normal.txt`` is available.
@@ -139,6 +143,11 @@ Above expression can be simplified to the following:
input: expand("{dataset}/file.A.txt", dataset=DATASETS)
+This may be used for "aggregation" rules for which files from multiple or all datasets are needed to produce a specific output (say, *allSamplesSummary.pdf*).
+Note that *dataset* is NOT a wildcard here because it is resolved by Snakemake due to the ``expand`` statement (see below also for more information).
+
+
+
The ``expand`` function thereby allows also to combine different variables, e.g.
.. code-block:: python
@@ -199,7 +208,7 @@ Further, a rule can be given a number of threads to use, i.e.
Snakemake can alter the number of cores available based on command line options. Therefore it is useful to propagate it via the built in variable ``threads`` rather than hardcoding it into the shell command.
In particular, it should be noted that the specified threads have to be seen as a maximum. When Snakemake is executed with fewer cores, the number of threads will be adjusted, i.e. ``threads = min(threads, cores)`` with ``cores`` being the number of cores specified at the command line (option ``--cores``). On a cluster node, Snakemake uses as many cores as available on that node. Hence, the number of threads used by a rule never exceeds the number of physically available cores on the nod [...]
-Starting from version 3.7, threads can also be a callable that returns an ``int`` value.
+Starting from version 3.7, threads can also be a callable that returns an ``int`` value. The signature of the callable should be ``callable(wildcards, [input])`` (input is an optional parameter). It is also possible to refer to a predefined variable (e.g, ``threads: threads_max``) so that the number of cores for a set of rules can be changed with one change only by altering the value of the variable ``threads_max``.
.. _snakefiles-resources:
@@ -246,6 +255,8 @@ When executing snakemake, a short summary for each running rule is given to the
message: "Executing somecommand with {threads} threads on the following files {input}."
shell: "somecommand --threads {threads} {input} {output}"
+Note that access to wildcards is also possible via the variable ``wildcards`` (e.g, ``{wildcards.sample}``), which is the same as with shell commands. It is important to have a namespace around wildcards in order to avoid clashes with other variable names.
+
Priorities
----------
@@ -286,6 +297,21 @@ The variable ``log`` can be used inside a shell command to tell the used tool to
log: "logs/abc.{dataset}.log"
+
+For programs that do not have an explicit ``log`` parameter, you may always use ``2> {log}`` to redirect standard output to a file (here, the ``log`` file) in Linux-based systems.
+Note that it is also supported to have multiple (named) log files being specified:
+
+.. code-block:: python
+
+ rule abc:
+ input: "input.txt"
+ output: "output.txt"
+ log: log1="logs/abc.log", log2="logs/xyz.log"
+ shell: "somecommand --log {log.log1} METRICS_FILE={log.log2} {input} {output}"
+
+
+
+
Non-file parameters for rules
-----------------------------
@@ -295,14 +321,41 @@ Sometimes you may want to define certain parameters separately from the rule bod
.. code-block:: python
rule:
- input: ...
- params: prefix="somedir/{sample}"
- output: "somedir/{sample}.csv"
- shell: "somecommand -o {params.prefix}"
+ input:
+ ...
+ params:
+ prefix="somedir/{sample}"
+ output:
+ "somedir/{sample}.csv"
+ shell:
+ "somecommand -o {params.prefix}"
+
+The ``params`` keyword allows you to specify additional parameters depending on the wildcards values. This allows you to circumvent the need to use ``run:`` and python code for non-standard commands like in the above case.
+Here, the command ``somecommand`` expects the prefix of the output file instead of the actual one. The ``params`` keyword helps here since you cannot simply add the prefix as an output file (as the file won't be created, Snakemake would throw an error after execution of the rule).
-The ``params`` keyword allows you to specify additional parameters depending on the wildcards values. This allows you to circumvent the need to use ``run:`` and python code for non-standard commands like in the above case. Here, the command ``somecommand`` expects the prefix of the output file instead of the actual one. The ``params`` keyword helps here since you cannot simply add the prefix as an output file (as the file won't be created, Snakemake would throw an error after execution o [...]
+Furthermore, for enhanced readability and clarity, the ``params`` section is also an excellent place to name and assign parameters and variables for your subsequent command.
+
+
+Similar to ``input``, ``params`` can take functions as well (see :ref:`snakefiles-input_functions`), e.g. you can write
+
+.. code-block:: python
+
+ rule:
+ input:
+ ...
+ params:
+ prefix=lambda wildcards, output: output[0][:-4]
+ output:
+ "somedir/{sample}.csv"
+ shell:
+ "somecommand -o {params.prefix}"
+
+to get the same effect as above. Note that in contrast to the ``input`` directive, the
+``params`` directive can optionally take more arguments than only ``wildcards``, namely ``input``, ``output``, ``threads``, and ``resources``.
+From the Python perspective, they can be seen as optional keyword arguments without a default value.
+Their order does not matter, apart from the fact that ``wildcards`` has to be the first argument.
+In the example above, this allows you to derive the prefix name from the output file.
-Similar to ``input``, ``params`` can take functions as well (see :ref:`snakefiles-input_functions`)
.. _snakefiles-external_scripts:
@@ -325,9 +378,11 @@ A rule can also point to an external script instead of a shell command or inline
The script path is always relative to the Snakefile (in contrast to the input and output file paths, which are relative to the working directory).
Inside the script, you have access to an object ``snakemake`` that provides access to the same objects that are available in the ``run`` and ``shell`` directives (input, output, params, wildcards, log, threads, resources, config), e.g. you can use ``snakemake.input[0]`` to access the first input file of above rule.
-Apart from Python scripts, this mechanism also allows you to integrate R_ scripts with Snakemake, e.g.
+
+Apart from Python scripts, this mechanism also allows you to integrate R_ and R Markdown_ scripts with Snakemake, e.g.
.. _R: https://www.r-project.org
+.. _Markdown: http://rmarkdown.rstudio.com
.. code-block:: python
@@ -367,6 +422,55 @@ An equivalent script written in R would look like this:
To debug R scripts, you can save the workspace with ``save.image()``, and invoke R after Snakemake has terminated. Then you can use the usual R debugging facilities while having access to the ``snakemake`` variable.
It is best practice to wrap the actual code into a separate function. This increases the portability if the code shall be invoked outside of Snakemake or from a different rule.
+An R Markdown file can be integrated in the same way as R and Python scripts, but only a single output (html) file can be used:
+
+.. code-block:: python
+
+ rule NAME:
+ input:
+ "path/to/inputfile",
+ "path/to/other/inputfile"
+ output:
+ "path/to/report.html",
+ script:
+ "path/to/report.Rmd"
+
+In the R Markdown file you can insert output from a R command, and access variables stored in the S4 object named ``snakemake``
+
+.. code-block:: R
+
+ ---
+ title: "Test Report"
+ author:
+ - "Your Name"
+ date: "`r format(Sys.time(), '%d %B, %Y')`"
+ params:
+ rmd: "report.Rmd"
+ output:
+ html_document:
+ highlight: tango
+ number_sections: no
+ theme: default
+ toc: yes
+ toc_depth: 3
+ toc_float:
+ collapsed: no
+ smooth_scroll: yes
+ ---
+
+ ## R Markdown
+
+ This is an R Markdown document.
+
+ Test include from snakemake `r snakemake at input`.
+
+ ## Source
+ <a download="report.Rmd" href="`r base64enc::dataURI(file = params$rmd, mime = 'text/rmd', encoding = 'base64')`">R Markdown source file (to produce this document)</a>
+
+A link to the R Markdown document with the snakemake object can be inserted. Therefore a variable called ``rmd`` needs to be added to the ``params`` section in the header of the ``report.Rmd`` file. The generated R Markdown file with snakemake object will be saved in the file specified in this ``rmd`` variable. This file can be embedded into the HTML document using base64 encoding and a link can be inserted as shown in the example above.
+Also other input and output files can be embedded in this way to make a portable report. Note that the above method with a data URI only works for small files. An experimental technology to embed larger files is using Javascript Blob object_.
+
+.. _object https://developer.mozilla.org/en-US/docs/Web/API/Blob
Protected and Temporary Files
-----------------------------
@@ -376,9 +480,12 @@ A particular output file may require a huge amount of computation time. Hence on
.. code-block:: python
rule NAME:
- input: "path/to/inputfile", "path/to/other/inputfile"
- output: protected("path/to/outputfile"), "path/to/another/outputfile"
- shell: "somecommand --threads {threads} {input} {output}"
+ input:
+ "path/to/inputfile"
+ output:
+ protected("path/to/outputfile")
+ shell:
+ "somecommand {input} {output}"
A protected file will be write-protected after the rule that produces it is completed.
@@ -387,9 +494,32 @@ Further, an output file marked as ``temp`` is deleted after all rules that use i
.. code-block:: python
rule NAME:
- input: "path/to/inputfile", "path/to/other/inputfile"
- output: temp("path/to/outputfile"), "path/to/another/outputfile"
- shell: "somecommand --threads {threads} {input} {output}"
+ input:
+ "path/to/inputfile"
+ output:
+ temp("path/to/outputfile")
+ shell:
+ "somecommand {input} {output}"
+
+Ignoring timestamps
+-------------------
+
+For determining whether output files have to be re-created, Snakemake checks whether the file modification date (i.e. the timestamp) of any input file of the same job is newer than the timestamp of the output file.
+This behavior can be overridden by marking an input file as ``ancient``.
+The timestamp of such files is ignored and always assumed to be older than any of the output files:
+
+.. code-block:: python
+
+ rule NAME:
+ input:
+ ancient("path/to/inputfile")
+ output:
+ "path/to/outputfile"
+ shell:
+ "somecommand {input} {output}"
+
+Here, this means that the file ``path/to/outputfile`` will not be triggered for re-creation after it has been generated once, even when the input file is modified in the future.
+Note that any flag that forces re-creation of files still also applies to files marked as ``ancient``.
Shadow rules
------------
@@ -516,6 +646,7 @@ Instead of specifying strings or lists of strings as input files, snakemake can
shell: "..."
The function has to accept a single argument that will be the wildcards object generated from the application of the rule to create some requested output files.
+Note that you can also use `lambda expressions <https://docs.python.org/3/tutorial/controlflow.html#lambda-expressions>`_ instead of full function definitions.
By this, rules can have entirely different input files (both in form and number) depending on the inferred wildcards. E.g. you can assign input files that appear in entirely different parts of your filesystem based on some wildcard value and a dictionary that maps the wildcard value to file paths.
Note that the function will be executed when the rule is evaluated and before the workflow actually starts to execute. Further note that using a function as input overrides the default mechanism of replacing wildcards with their values inferred from the output files. You have to take care of that yourself with the given wildcards object.
@@ -523,6 +654,8 @@ Note that the function will be executed when the rule is evaluated and before th
Finally, when implementing the input function, it is best practice to make sure that it can properly handle all possible wildcard values your rule can have.
In particular, input files should not be combined with very general rules that can be applied to create almost any file: Snakemake will try to apply the rule, and will report the exceptions of your input function as errors.
+For a practical example, see the :ref:`tutorial` (:ref:`tutorial-input_functions`).
+
.. _snakefiles-unpack:
Input Functions and ``unpack()``
@@ -605,6 +738,11 @@ A re-run can be automated by invoking Snakemake as follows:
$ snakemake -R `snakemake --list-version-changes`
+With the availability of the ``conda`` directive (see :ref:`integrated_package_management`)
+the ``version`` directive has become **obsolete** in favor of defining isolated
+software environments that can be automatically deployed via the conda package
+manager.
+
.. _snakefiles-code_tracking:
@@ -663,6 +801,21 @@ From verion 2.4.8 on, rules can also refer to the output of other rules in the S
Importantly, be aware that referring to rule a here requires that rule a was defined above rule b in the file, since the object has to be known already.
This feature also allows to resolve dependencies that are ambiguous when using filenames.
+Note that when the rule you refer to defines multiple output files but you want to require only a subset of those as input for another rule, you should name the output files and refer to them specifically:
+
+.. code-block:: python
+
+ rule a:
+ input: "path/to/input"
+ output: a = "path/to/output", b = "path/to/output2"
+ shell: ...
+
+ rule b:
+ input: rules.a.output.a
+ output: "path/to/output/of/b"
+ shell: ...
+
+
.. _snakefiles-ambiguous-rules:
Handling Ambiguous Rules
@@ -731,6 +884,13 @@ With the `benchmark` keyword, a rule can be declared to store a benchmark of its
"somecommand {input} {output}"
benchmarks the CPU and wall clock time of the command ``somecommand`` for the given output and input files.
-For this, the shell or run body of the rule is executed on that data, and all run times are stored into the given benchmark txt file (which will contain a tab-separated table of run times). Per default, Snakemake executes the job once, generating one run time.
+For this, the shell or run body of the rule is executed on that data, and all run times are stored into the given benchmark txt file (which will contain a tab-separated table of run times and memory usage in MiB).
+Per default, Snakemake executes the job once, generating one run time.
With ``snakemake --benchmark-repeats``, this number can be changed to e.g. generate timings for two or three runs.
The resulting txt file can be used as input for other rules, just like any other output file.
+
+.. note::
+
+ Note that benchmarking is only possible in a reliable fashion for subprocesses (thus for tasks run through the ``shell``, ``script``, and ``wrapper`` directive).
+ In the ``run`` block, the variable ``bench_record`` is available that you can pass to ``shell()`` as ``bench_record=bench_record``.
+ When using ``shell(..., bench_record=bench_record)``, the maximum of all measurements of all ``shell()`` calls will be used but the running time of the rule execution including any Python code.
diff --git a/docs/tutorial/additional_features.rst b/docs/tutorial/additional_features.rst
index 9a77a2a..bd3a92a 100644
--- a/docs/tutorial/additional_features.rst
+++ b/docs/tutorial/additional_features.rst
@@ -3,8 +3,8 @@
Additional features
-------------------
-.. _Snakemake: http://snakemake.bitbucket.org
-.. _Snakemake homepage: http://snakemake.bitbucket.org
+.. _Snakemake: https://snakemake.bitbucket.io
+.. _Snakemake homepage: https://snakemake.bitbucket.io
.. _GNU Make: https://www.gnu.org/software/make
.. _Python: http://www.python.org
.. _BWA: http://bio-bwa.sourceforge.net
@@ -34,7 +34,7 @@ Additional features
.. _slides: http://slides.com/johanneskoester/deck-1
In the following, we introduce some features that are beyond the scope of above example workflow.
-For details and even more features, see the :ref:`manual-main`, the :ref:`project_info-faq` and the command line help (``snakemake --help``).
+For details and even more features, see :ref:`user_manual-writing_snakefiles`, :ref:`project_info-faq` and the command line help (``snakemake --help``).
Benchmarking
@@ -54,7 +54,7 @@ We activate benchmarking for the rule ``bwa_map``:
params:
rg="@RG\tID:{sample}\tSM:{sample}"
log:
- "logs/bwa_map/{sample}.log"
+ "logs/bwa_mem/{sample}.log"
benchmark:
"benchmarks/{sample}.bwa.benchmark.txt"
threads: 8
@@ -64,7 +64,7 @@ We activate benchmarking for the rule ``bwa_map``:
The ``benchmark`` directive takes a string that points to the file where benchmarking results shall be stored.
Similar to output files, the path can contain wildcards (it must be the same wildcards as in the output files).
-When a job derived from the rule is executed, Snakemake will measure the wall clock time and store it in the file in tab-delimited format.
+When a job derived from the rule is executed, Snakemake will measure the wall clock time and memory usage (in MiB) and store it in the file in tab-delimited format.
With the command line flag ``--benchmark-repeats``, Snakemake can be instructed to perform repetitive measurements by executing benchmark jobs multiple times.
The repeated measurements occur as subsequent lines in the tab-delimited benchmark file.
@@ -121,7 +121,7 @@ A sub-workflow refers to a working directory with a complete Snakemake workflow.
Output files of that sub-workflow can be used in the current Snakefile.
When executing, Snakemake ensures that the output files of the sub-workflow are up-to-date before executing the current workflow.
This mechanism is particularly useful when you want to extend a previous analysis without modifying it.
-For details about sub-workflows, see the :ref:`manual-main`.
+For details about sub-workflows, see the :ref:`documentation <snakefiles-sub_workflows>`.
Exercise
@@ -137,7 +137,7 @@ Using the ``run`` directive as above is only reasonable for short Python scripts
As soon as your script becomes larger, it is reasonable to separate it from the
workflow definition.
For this purpose, Snakemake offers the ``script`` directive.
-Using this, ``report`` rule from above could instead look like this:
+Using this, the ``report`` rule from above could instead look like this:
.. code:: python
@@ -174,6 +174,11 @@ In the script, all properties of the rule like ``input``, ``output``, ``wildcard
Benchmark results for BWA can be found in the tables T2_.
""", snakemake.output[0], **snakemake.input)
+.. sidebar:: Note
+
+ It is best practice to use the script directive whenever a run block would have
+ more than a few lines of code.
+
Although there are other strategies to invoke separate scripts from your workflow
(e.g., invoking them via shell commands), the benefit of this is obvious:
the script logic is separated from the workflow logic (and can be even shared between workflows),
@@ -190,6 +195,106 @@ index, e.g. ``snakemake at input[["myfile"]]``.
For details and examples, see the :ref:`snakefiles-external_scripts` section in the Documentation.
+.. _tutorial-conda:
+
+Automatic deployment of software dependencies
+:::::::::::::::::::::::::::::::::::::::::::::
+
+In order to get a fully reproducible data analysis, it is not sufficient to
+be able to execute each step and document all used parameters.
+The used software tools and libraries have to be documented as well.
+In this tutorial, you have already seen how Conda_ can be used to specify an
+isolated software environment for a whole workflow. With Snakemake, you can
+go one step further and specify Conda environments per rule.
+This way, you can even make use of conflicting software versions (e.g. combine
+Python 2 with Python 3).
+
+In our example, instead of using an external environment we can specify
+environments per rule, e.g.:
+
+.. code:: python
+
+ rule samtools_index:
+ input:
+ "sorted_reads/{sample}.bam"
+ output:
+ "sorted_reads/{sample}.bam.bai"
+ conda:
+ "envs/samtools.yaml"
+ shell:
+ "samtools index {input}"
+
+with ``envs/samtools.yaml`` defined as
+
+.. code:: yaml
+
+ channels:
+ - bioconda
+ dependencies:
+ - samtools =1.3
+
+.. sidebar:: Note
+
+ The conda directive does not work in combination with ``run`` blocks, because
+ they have to share their Python environment with the surrounding snakefile.
+
+When Snakemake is executed with
+
+.. code:: console
+
+ snakemake --use-conda
+
+it will automatically create required environments and
+activate them before a job is executed.
+It is best practice to specify at least the `major and minor version <http://semver.org/>`_ of any packages
+in the environment definition. Specifying environments per rule in this way has two
+advantages.
+First, the workflow definition also documents all used software versions.
+Second, a workflow can be re-executed (without admin rights)
+on a vanilla system, without installing any
+prerequisites apart from Snakemake and Miniconda_.
+
+
+Tool wrappers
+:::::::::::::
+
+In order to simplify the utilization of popular tools, Snakemake provides a
+repository of so-called wrappers
+(the `Snakemake wrapper repository <https://snakemake-wrappers.readthedocs.io>`_).
+A wrapper is a short script that wraps (typically)
+a command line application and makes it directly addressable from within Snakemake.
+For this, Snakemake provides the ``wrapper`` directive that can be used instead of
+``shell``, ``script``, or ``run``.
+For example, the rule ``bwa_map`` could alternatively look like this:
+
+.. code:: python
+
+ rule bwa_mem:
+ input:
+ ref="data/genome.fa",
+ sample=lambda wildcards: config["samples"][wildcards.sample]
+ output:
+ temp("mapped_reads/{sample}.bam")
+ log:
+ "logs/bwa_mem/{sample}.log"
+ params:
+ "-R '@RG\tID:{sample}\tSM:{sample}'"
+ threads: 8
+ wrapper:
+ "0.15.3/bio/bwa/mem"
+
+.. sidebar:: Note
+
+ Updates to the Snakemake wrapper repository are automatically tested via
+ `continuous integration <https://en.wikipedia.org/wiki/Continuous_integration>`_.
+
+The wrapper directive expects a (partial) URL that points to a wrapper in the repository.
+These can be looked up in the corresponding `database <https://snakemake-wrappers.readthedocs.io>`_.
+The first part of the URL is a Git version tag. Upon invocation, Snakemake
+will automatically download the requested version of the wrapper.
+Furthermore, in combination with ``--use-conda`` (see :ref:`tutorial-conda`),
+the required software will be automatically deployed before execution.
+
Cluster execution
:::::::::::::::::
@@ -241,6 +346,7 @@ Snakemake uses regular expressions to match output files to input files and dete
Sometimes it is useful to constrain the values a wildcard can have.
This can be achieved by adding a regular expression that describes the set of allowed wildcard values.
For example, the wildcard ``sample`` in the output file ``"sorted_reads/{sample}.bam"`` can be constrained to only allow alphanumeric sample names as ``"sorted_reads/{sample,[A-Za-z0-9]+}.bam"``.
+Constrains may be defined per rule or globally using the ``wildcard_constraints`` keyword, as demonstrated in :ref:`snakefiles-wildcards`.
This mechanism helps to solve two kinds of ambiguity.
* It can help to avoid ambiguous rules, i.e. two or more rules that can be applied to generate the same output file. Other ways of handling ambiguous rules are described in the Section :ref:`snakefiles-ambiguous-rules`.
diff --git a/docs/tutorial/advanced.rst b/docs/tutorial/advanced.rst
index 0ea4102..6f3e3d6 100644
--- a/docs/tutorial/advanced.rst
+++ b/docs/tutorial/advanced.rst
@@ -3,8 +3,8 @@
Advanced: Decorating the example workflow
-----------------------------------------
-.. _Snakemake: http://snakemake.bitbucket.org
-.. _Snakemake homepage: http://snakemake.bitbucket.org
+.. _Snakemake: https://snakemake.bitbucket.io
+.. _Snakemake homepage: https://snakemake.bitbucket.io
.. _GNU Make: https://www.gnu.org/software/make
.. _Python: http://www.python.org
.. _BWA: http://bio-bwa.sourceforge.net
@@ -66,14 +66,16 @@ For example
$ snakemake --cores 10
+
+.. sidebar:: Note
+
+ Apart from the very common thread resource, Snakemake provides a ``resources`` directive that can be used to **specify arbitrary resources**, e.g., memory usage or auxiliary computing devices like GPUs.
+ Similar to threads, these can be considered by the scheduler when an available amount of that resource is given with the command line argument ``--resources`` (see :ref:`snakefiles-resources`).
+
would execute the workflow with 10 cores.
Since the rule ``bwa_map`` needs 8 threads, only one job of the rule can run at a time, and the Snakemake scheduler will try to saturate the remaining cores with other jobs like, e.g., ``samtools_sort``.
The threads directive in a rule is interpreted as a maximum: when **less cores than threads** are provided, the number of threads a rule uses will be **reduced to the number of given cores**.
-Apart from the very common thread resource, Snakemake provides a ``resources`` directive that can be used to **specify arbitrary resources**, e.g., memory usage or auxiliary computing devices like GPUs.
-Similar to threads, these can be considered by the scheduler when an available amount of that resource is given with the command line argument ``--resources``.
-Details can be found in the Snakemake :ref:`manual-main`.
-
Exercise
........
@@ -117,6 +119,7 @@ Now, we can remove the statement defining ``SAMPLES`` from the Snakefile and cha
"samtools mpileup -g -f {input.fa} {input.bam} | "
"bcftools call -mv - > {output}"
+.. _tutorial-input_functions:
Step 3: Input functions
:::::::::::::::::::::::
@@ -148,10 +151,21 @@ For the rule ``bwa_map`` this works as follows:
shell:
"bwa mem -t {threads} {input} | samtools view -Sb - > {output}"
-Here, we use an anonymous function, also called **lambda expression**.
+.. sidebar:: Note
+
+ Snakemake does not automatically rerun jobs when new input files are added as
+ in the excercise below. However, you can get a list of output files that
+ are affected by such changes with ``snakemake --list-input-changes``.
+ To trigger a rerun, some bash magic helps:
+
+ .. code:: console
+
+ snakemake -n --forcerun $(snakemake --list-input-changes)
+
+Here, we use an anonymous function, also called `lambda expression <https://docs.python.org/3/tutorial/controlflow.html#lambda-expressions>`_.
Any normal function would work as well.
Input functions take as **single argument** a ``wildcards`` object, that allows to access the wildcards values via attributes (here ``wildcards.sample``).
-They **return a string or a list of strings**, that are interpreted as paths to input files (here, we return the path that is stored for the sample in the config file).
+They have to **return a string or a list of strings**, that are interpreted as paths to input files (here, we return the path that is stored for the sample in the config file).
Input functions are evaluated once the wildcard values of a job are determined.
@@ -160,7 +174,6 @@ Exercise
* In the ``data/samples`` folder, there is an additional sample ``C.fastq``. Add that sample to the config file and see how Snakemake wants to recompute the part of the workflow belonging to the new sample, when invoking with ``snakemake -n --reason --forcerun bcftools_call``.
-
Step 4: Rule parameters
:::::::::::::::::::::::
@@ -184,8 +197,13 @@ We modify the rule ``bwa_map`` accordingly:
shell:
"bwa mem -R '{params.rg}' -t {threads} {input} | samtools view -Sb - > {output}"
-Similar to input and output files, ``params`` can be accessed from the shell command.
-Moreover, the ``params`` directive can also take functions like in Step 3 to defer initialization to the DAG phase.
+.. sidebar:: Note
+
+ The ``params`` directive can also take functions like in Step 3 to defer
+ initialization to the DAG phase. In contrast to input functions, these can
+ optionally take additional arguments ``input``, ``output``, ``threads``, and ``resources``.
+
+Similar to input and output files, ``params`` can be accessed from the shell command or the Python based ``run`` block (see :ref:`tutorial-report`).
Exercise
........
@@ -211,14 +229,17 @@ We modify our rule ``bwa_map`` as follows:
params:
rg="@RG\tID:{sample}\tSM:{sample}"
log:
- "logs/bwa_map/{sample}.log"
+ "logs/bwa_mem/{sample}.log"
threads: 8
shell:
"(bwa mem -R '{params.rg}' -t {threads} {input} | "
"samtools view -Sb - > {output}) 2> {log}"
+.. sidebar:: Note
+
+ It is best practice to store all log files in a subdirectory ``logs/``, prefixed by the rule or tool name.
+
The shell command is modified to collect STDERR output of both ``bwa`` and ``samtools`` and pipe it into the file referred by ``{log}``.
-It is best practice to store all log files in a ``logs`` subdirectory, prefixed by the rule or tool name.
Log files must contain exactly the same wildcards as the output files to avoid clashes.
Exercise
@@ -251,7 +272,7 @@ We use this mechanism for the output file of the rule ``bwa_map``:
params:
rg="@RG\tID:{sample}\tSM:{sample}"
log:
- "logs/bwa_map/{sample}.log"
+ "logs/bwa_mem/{sample}.log"
threads: 8
shell:
"(bwa mem -R '{params.rg}' -t {threads} {input} | "
@@ -305,7 +326,7 @@ The final version of our workflow looks like this:
params:
rg="@RG\tID:{sample}\tSM:{sample}"
log:
- "logs/bwa_map/{sample}.log"
+ "logs/bwa_mem/{sample}.log"
threads: 8
shell:
"(bwa mem -R '{params.rg}' -t {threads} {input} | "
diff --git a/docs/tutorial/basics.rst b/docs/tutorial/basics.rst
index 9bd21ae..dd3e8d2 100644
--- a/docs/tutorial/basics.rst
+++ b/docs/tutorial/basics.rst
@@ -3,8 +3,8 @@
Basics: An example workflow
---------------------------
-.. _Snakemake: http://snakemake.bitbucket.org
-.. _Snakemake homepage: http://snakemake.bitbucket.org
+.. _Snakemake: https://snakemake.bitbucket.io
+.. _Snakemake homepage: https://snakemake.bitbucket.io
.. _GNU Make: https://www.gnu.org/software/make
.. _Python: http://www.python.org
.. _BWA: http://bio-bwa.sourceforge.net
@@ -44,12 +44,44 @@ All added syntactic structures begin with a keyword followed by a code block tha
The resulting syntax resembles that of original Python constructs.
In the following, we will introduce the Snakemake syntax by creating an example workflow.
-The workflow will map sequencing reads to a reference genome and call variants on the mapped reads.
+The workflow comes from the domain of genome analysis.
+It maps sequencing reads to a reference genome and call variants on the mapped reads.
+The tutorial does not require you to know what this is about.
+Nevertheless, we provide some background in the following.
+
+.. _tutorial-background:
+
+Background
+::::::::::
+
+The genome of a living organism encodes its hereditary information.
+It serves as a blueprint for proteins, which form living cells, carry information
+and drive chemical reactions. Differences between populations, species, cancer
+cells and healthy tissue, as well as syndromes or diseases can be reflected and
+sometimes caused by changes in the genome.
+This makes the genome an major target of biological and medical research.
+Today, it is often analyzed with DNA sequencing, producing gigabytes of data from
+a single biological sample (e.g. a biopsy of some tissue).
+For technical reasons, DNA sequencing cuts the DNA of a sample into millions
+of small pieces, called **reads**.
+In order to recover the genome of the sample, one has to map these reads against
+a known **reference genome** (e.g., the human one obtained during the famous
+`human genome genome project <https://en.wikipedia.org/wiki/Human_Genome_Project>`_).
+This task is called **read mapping**.
+Often, it is of interest where an individual genome is different from the species-wide consensus
+represented with the reference genome.
+Such differences are called **variants**. They are responsible for harmless individual
+differences (like eye color), but can also cause diseases like cancer.
+By investigating the differences between the all mapped reads
+and the reference sequence at one position, variants can be detected.
+This is a statistical challenge, because they have
+to be distinguished from artifacts generated by the sequencing process.
Step 1: Mapping reads
:::::::::::::::::::::
-Our first Snakemake rule maps reads of a given sample to a given reference genome.
+Our first Snakemake rule maps reads of a given sample to a given reference genome (see :ref:`tutorial-background`).
+For this, we will use the tool bwa_, specifically the subcommand ``bwa mem``.
In the working directory, **create a new file** called ``Snakefile`` with an editor of your choice.
We propose to use the Atom_ editor, since it provides out-of-the-box syntax highlighting for Snakemake.
In the Snakefile, define the following rule:
@@ -65,6 +97,11 @@ In the Snakefile, define the following rule:
shell:
"bwa mem {input} | samtools view -Sb - > {output}"
+.. sidebar:: Note
+
+ A common error is to forget the comma between the input or output items.
+ Since Python concatenates subsequent strings, this can lead to unexpected behavior.
+
A Snakemake rule has a name (here ``bwa_map``) and a number of directives, here ``input``, ``output`` and ``shell``.
The ``input`` and ``output`` directives are followed by lists of files that are expected to be used or created by the rule.
In the simplest case, these are just explicit Python strings.
@@ -73,7 +110,7 @@ In the shell command string, we can refer to elements of the rule via braces not
Here, we refer to the output file by specifying ``{output}`` and to the input files by specifying ``{input}``.
Since the rule has multiple input files, Snakemake will concatenate them separated by a whitespace.
In other words, Snakemake will replace ``{input}`` with ``data/genome.fa data/samples/A.fastq`` before executing the command.
-The shell command invokes ``bwa mem`` with reference genome and reads, and pipes the output into ``samtools`` which creates a compressed BAM file containing the alignments.
+The shell command invokes ``bwa mem`` with reference genome and reads, and pipes the output into ``samtools`` which creates a compressed `BAM <https://en.wikipedia.org/wiki/Binary_Alignment_Map>`_ file containing the alignments.
The output of ``samtools`` is piped into the output file defined by the rule.
When a workflow is executed, Snakemake tries to generate given **target** files.
@@ -90,7 +127,7 @@ The ``-p`` flag instructs Snakemake to also print the resulting shell command fo
To generate the target files, **Snakemake applies the rules given in the Snakefile in a top-down way**.
The application of a rule to generate a set of output files is called **job**.
For each input file of a job, Snakemake again (i.e. recursively) determines rules that can be applied to generate it.
-This yields a directed acyclic graph (DAG) of jobs where the edges represent dependencies.
+This yields a `directed acyclic graph (DAG) <https://en.wikipedia.org/wiki/Directed_acyclic_graph>`_ of jobs where the edges represent dependencies.
So far, we only have a single rule, and the DAG of jobs consists of a single node.
Nevertheless, we can **execute our workflow** with
@@ -119,6 +156,12 @@ Simply replace the ``A`` in the second input file and in the output file with th
shell:
"bwa mem {input} | samtools view -Sb - > {output}"
+.. sidebar:: Note
+
+ Note that if a rule has multiple output files, Snakemake requires them to all
+ have exactly the same wildcards. Otherwise, it could happen
+ that two jobs from the same rule want to write the same file.
+
When Snakemake determines that this rule can be applied to generate a target file by replacing the wildcard ``{sample}`` in the output file with an appropriate value, it will propagate that value to all occurrences of ``{sample}`` in the input files and thereby determine the necessary input for the resulting job.
Note that you can have multiple wildcards in your file paths, however, to avoid conflicts with other jobs of the same rule, **all output files** of a rule have to **contain exactly the same wildcards**.
@@ -164,7 +207,7 @@ Step 3: Sorting read alignments
:::::::::::::::::::::::::::::::
For later steps, we need the read alignments in the BAM files to be sorted.
-This can be achieved with the ``samtools`` command.
+This can be achieved with the samtools_ command.
We add the following rule beneath the ``bwa_map`` rule:
.. code:: python
@@ -178,6 +221,10 @@ We add the following rule beneath the ``bwa_map`` rule:
"samtools sort -T sorted_reads/{wildcards.sample} "
"-O bam {input} > {output}"
+.. sidebar:: Note
+
+ It is best practice to have subsequent steps of a workflow in separate, unique, output folders. This keeps the working directory structured. Further, such unique prefixes allow Snakemake to prune the search space for dependencies.
+
This rule will take the input file from the ``mapped_reads`` directory and store a sorted version in the ``sorted_reads`` directory.
Note that Snakemake **automatically creates missing directories** before jobs are executed.
For sorting, ``samtools`` requires a prefix specified with the flag ``-T``.
@@ -196,7 +243,7 @@ as mentioned before, the dependencies are resolved automatically by matching fil
Step 4: Indexing read alignments and visualizing the DAG of jobs
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-Next, we need to index the sorted read alignments for random access.
+Next, we need to use samtools_ again to index the sorted read alignments for random access.
This can be done with the following rule:
.. code:: python
@@ -209,6 +256,13 @@ This can be done with the following rule:
shell:
"samtools index {input}"
+.. sidebar:: Note
+
+ Snakemake uses the Python format mini language to format shell commands.
+ Sometimes you have to use braces for something else in a shell command.
+ In that case, you have to escape them by doubling, e.g.,
+ ``ls {{A,B}}.txt``.
+
Having three steps already, it is a good time to take a closer look at the resulting DAG of jobs.
By executing
@@ -235,15 +289,16 @@ Exercise
Step 5: Calling genomic variants
::::::::::::::::::::::::::::::::
-The next step in our workflow will aggregate the aligned reads from all samples and jointly call genomic variants on them.
-Snakemake provides a **helper function for collecting input files**.
+The next step in our workflow will aggregate the mapped reads from all samples and jointly call genomic variants on them (see :ref:`tutorial-background`).
+For the variant calling, we will combine the two utilities samtools_ and bcftools_.
+Snakemake provides a **helper function for collecting input files** that helps us to describe the aggregation in this step.
With
.. code:: python
expand("sorted_reads/{sample}.bam", sample=SAMPLES)
-we obtain a list of files where the given pattern ``"sorted_reads/{sample}.bam"`` was formatted with the values in the given list of samples ``SAMPLES``, i.e.
+we obtain a list of files where the given pattern ``"sorted_reads/{sample}.bam"`` was formatted with the values in a given list of samples ``SAMPLES``, i.e.
.. code:: python
@@ -288,6 +343,11 @@ Now, we can add the following rule to our Snakefile:
"samtools mpileup -g -f {input.fa} {input.bam} | "
"bcftools call -mv - > {output}"
+.. sidebar:: Note
+
+ If you name input or output files like above, their order won't be preserved when referring them as ``{input}``.
+ Further, note that named and not named (i.e., positional) input and output files can be combined, but the positional ones must come first, equivalent to Python functions with keyword arguments.
+
With multiple input or output files, it is sometimes handy to refer them separately in the shell command.
This can be done by **specifying names for input or output files** (here, e.g., ``fa=...``).
The files can then be referred in the shell command via, e.g., ``{input.fa}``.
@@ -304,6 +364,9 @@ Exercise
.. image:: workflow/dag_call.png
:align: center
+
+.. _tutorial-report:
+
Step 6: Writing a report
::::::::::::::::::::::::
@@ -334,13 +397,19 @@ It is best practice to create reports in a separate rule that takes all desired
This resulted in {n_calls} variants (see Table T1_).
""", output[0], T1=input[0])
+.. sidebar:: Note
+
+ The run directive can be seen as a Python function with the arguments ``input``, ``output``, ``wildcards``, etc..
+ Hence, other than with the shell directive before, there is no need to enclose those objects in braces.
+
First, we notice that this rule does not entail a shell command.
Instead, we use the ``run`` directive, which is followed by plain Python code.
-Similar to the shell case, we have access to ``input`` and ``output`` files, which we can handle as plain Python objects (no braces notation here).
+Similar to the shell case, we have access to ``input`` and ``output`` files, which we can handle as plain Python objects.
We go through the ``run`` block line by line.
First, we import the ``report`` function from ``snakemake.utils``.
Second, we open the VCF file by accessing it via its index in the input files (i.e. ``input[0]``), and count the number of non-header lines (which is equivalent to the number of variant calls).
+Of course, this is only a silly example of what to do with variant calls.
Third, we create the report using the ``report`` function.
The function takes a string that contains RestructuredText_ markup.
In addition, we can use the familiar braces notation to access any Python variables (here the ``samples`` and ``n_calls`` variables we have defined before).
@@ -387,6 +456,13 @@ When executing Snakemake with
$ snakemake -n
+.. sidebar:: Note
+
+ In case you have mutliple reasonable sets of target files,
+ you can add multiple target rules at the top of the Snakefile. While
+ Snakemake will execute the first per default, you can target any of them via
+ the command line (e.g., ``snakemake -n mytarget``).
+
the execution plan for creating the file ``report.html`` which contains and summarizes all our results will be shown.
Note that, apart from Snakemake considering the first rule of the workflow as default target, **the appearance of rules in the Snakefile is arbitrary and does not influence the DAG of jobs**.
diff --git a/docs/tutorial/welcome.rst b/docs/tutorial/setup.rst
similarity index 71%
rename from docs/tutorial/welcome.rst
rename to docs/tutorial/setup.rst
index 18c95eb..888e546 100644
--- a/docs/tutorial/welcome.rst
+++ b/docs/tutorial/setup.rst
@@ -1,11 +1,11 @@
-.. _tutorial-welcome:
-==============
-Tutorial Setup
-==============
+.. _tutorial-setup:
+
+Setup
+-----
-.. _Snakemake: http://snakemake.bitbucket.org
-.. _Snakemake homepage: http://snakemake.bitbucket.org
+.. _Snakemake: http://snakemake.readthedocs.io
+.. _Snakemake homepage: http://snakemake.readthedocs.io
.. _GNU Make: https://www.gnu.org/software/make
.. _Python: http://www.python.org
.. _BWA: http://bio-bwa.sourceforge.net
@@ -16,41 +16,13 @@ Tutorial Setup
.. _Conda: http://conda.pydata.org
.. _Bash: http://www.tldp.org/LDP/Bash-Beginners-Guide/html
.. _Atom: https://atom.io
-.. _Anaconda: https://anaconda.org
.. _Graphviz: http://www.graphviz.org
-.. _RestructuredText: http://docutils.sourceforge.net/rst.html
-.. _data URI: https://developer.mozilla.org/en-US/docs/Web/HTTP/data_URIs
-.. _JSON: http://json.org
-.. _YAML: http://yaml.org
-.. _DRMAA: http://www.drmaa.org
-.. _rpy2: http://rpy.sourceforge.net
-.. _R: https://www.r-project.org
-.. _Rscript: https://stat.ethz.ch/R-manual/R-devel/library/utils/html/Rscript.html
.. _PyYAML: http://pyyaml.org
.. _Docutils: http://docutils.sourceforge.net
.. _Bioconda: https://bioconda.github.io
.. _Vagrant: https://www.vagrantup.com
.. _Vagrant Documentation: https://docs.vagrantup.com
.. _Blogpost: http://blog.osteel.me/posts/2015/01/25/how-to-use-vagrant-on-windows.html
-.. _slides: http://slides.com/johanneskoester/snakemake-tutorial-2016
-
-This tutorial introduces the text-based workflow system Snakemake_.
-Snakemake follows the `GNU Make`_ paradigm: workflows are defined in terms of rules that define how to create output files from input files.
-Dependencies between the rules are determined automatically, creating a DAG (directed acyclic graph) of jobs that can be automatically parallelized.
-
-Snakemake sets itself apart from existing text-based workflow systems in the following way.
-Hooking into the Python interpreter, Snakemake offers a definition language that is an extension of Python_ with syntax to define rules and workflow specific properties.
-This allows to combine the flexibility of a plain scripting language with a pythonic workflow definition.
-The Python language is known to be concise yet readable and can appear almost like pseudo-code.
-The syntactic extensions provided by Snakemake maintain this property for the definition of the workflow.
-Further, Snakemakes scheduling algorithm can be constrained by priorities, provided cores and customizable resources and it provides a generic support for distributed computing (e.g., cluster or batch systems).
-Hence, a Snakemake workflow scales without modification from single core workstations and multi-core servers to cluster or batch systems.
-
-While the examples presented here come from Bioinformatics, Snakemake is considered a general-purpose workflow management system for any discipline.
-
-Also have a look at the corresponding slides_.
-
-.. _tutorial-setup:
Requirements
::::::::::::
@@ -58,7 +30,7 @@ Requirements
To go through this tutorial, you need the following software installed:
* Python_ ≥3.3
-* Snakemake_ 3.9.0
+* Snakemake_ 3.11.0
* BWA_ 0.7.12
* SAMtools_ 1.3.1
* BCFtools_ 1.3.1
@@ -138,8 +110,8 @@ First, we download some example data on which the workflow shall be executed:
.. code:: console
- $ wget https://bitbucket.org/snakemake/snakemake-tutorial/get/v3.9.0-1.tar.bz2
- $ tar -xf v3.9.0-1.tar.bz2 --strip 1
+ $ wget https://bitbucket.org/snakemake/snakemake-tutorial/get/v3.11.0.tar.bz2
+ $ tar -xf v3.11.0.tar.bz2 --strip 1
This will create a folder ``data`` and a file ``environment.yaml`` in the working directory.
diff --git a/docs/tutorial/tutorial.rst b/docs/tutorial/tutorial.rst
new file mode 100644
index 0000000..f74f9d0
--- /dev/null
+++ b/docs/tutorial/tutorial.rst
@@ -0,0 +1,37 @@
+.. _tutorial:
+
+==================
+Snakemake Tutorial
+==================
+
+.. _Snakemake: http://snakemake.readthedocs.io
+.. _GNU Make: https://www.gnu.org/software/make
+.. _Python: http://www.python.org
+.. _slides: http://slides.com/johanneskoester/snakemake-tutorial-2016
+
+This tutorial introduces the text-based workflow system Snakemake_.
+Snakemake follows the `GNU Make`_ paradigm: workflows are defined in terms of rules that define how to create output files from input files.
+Dependencies between the rules are determined automatically, creating a DAG (directed acyclic graph) of jobs that can be automatically parallelized.
+
+Snakemake sets itself apart from existing text-based workflow systems in the following way.
+Hooking into the Python interpreter, Snakemake offers a definition language that is an extension of Python_ with syntax to define rules and workflow specific properties.
+This allows to combine the flexibility of a plain scripting language with a pythonic workflow definition.
+The Python language is known to be concise yet readable and can appear almost like pseudo-code.
+The syntactic extensions provided by Snakemake maintain this property for the definition of the workflow.
+Further, Snakemakes scheduling algorithm can be constrained by priorities, provided cores and customizable resources and it provides a generic support for distributed computing (e.g., cluster or batch systems).
+Hence, a Snakemake workflow scales without modification from single core workstations and multi-core servers to cluster or batch systems.
+
+The examples presented in this tutorial come from Bioinformatics.
+However, Snakemake is a general-purpose workflow management system for any discipline.
+We ensured that no bioinformatics knowledge is needed to understand the tutorial.
+
+Also have a look at the corresponding slides_.
+
+
+.. toctree::
+ :maxdepth: 2
+
+ setup
+ basics
+ advanced
+ additional_features
diff --git a/environment.yml b/environment.yml
index 97a2a32..b4f0d94 100644
--- a/environment.yml
+++ b/environment.yml
@@ -5,22 +5,18 @@ channels:
- conda-forge
dependencies:
- python >=3.3
- - rpy2 >=0.7.6
- boto
- moto
- - httpretty ==0.8.10
+ - httpretty
- filechunkio
- pyyaml
- - nose
- ftputil
- pysftp
- requests
- dropbox
- - numpy
- appdirs
- pytools
- docutils
- - sphinx
- - pip:
- - sphinxcontrib-napoleon
- - sphinx_rtd_theme
+ - psutil
+ - pandas
+ - nomkl
diff --git a/examples/c/src/Snakefile b/examples/c/src/Snakefile
index 29e5088..75984b5 100644
--- a/examples/c/src/Snakefile
+++ b/examples/c/src/Snakefile
@@ -31,7 +31,7 @@ rule c_to_o:
output:
temp('{ODIR}/{name}.o')
input:
- '{name}.c', HEADERS
+ '{name}.c'
shell:
"{CC} -c -o {output} {input} {CFLAGS}"
diff --git a/misc/vim/syntax/snakemake.vim b/misc/vim/syntax/snakemake.vim
index d8aef59..21d3c6b 100644
--- a/misc/vim/syntax/snakemake.vim
+++ b/misc/vim/syntax/snakemake.vim
@@ -51,7 +51,7 @@ syn keyword pythonStatement rule subworkflow nextgroup=pythonFunction skipwhite
" similar to special def and class treatment from python.vim, except
" parenthetical part of def and class
syn match pythonFunction
- \ "\%(\%(rule\s\|subworkflow\s\)\s*\)\@<=\h*" contained
+ \ "\%(\%(rule\s\|subworkflow\s\)\s*\)\@<=\h\w*" contained
syn sync match pythonSync grouphere NONE "^\s*\%(rule\|subworkflow\)\s\+\h\w*\s*"
diff --git a/setup.py b/setup.py
index 6c6fecf..5661c6a 100644
--- a/setup.py
+++ b/setup.py
@@ -53,7 +53,7 @@ setup(
'code to define rules. Rules describe how to create output files from input files.',
zip_safe=False,
license='MIT',
- url='http://snakemake.bitbucket.org',
+ url='http://snakemake.bitbucket.io',
packages=['snakemake', 'snakemake.remote'],
entry_points={
"console_scripts":
@@ -61,9 +61,9 @@ setup(
"snakemake-bash-completion = snakemake:bash_completion"]
},
package_data={'': ['*.css', '*.sh', '*.html']},
- install_requires=['wrapt',],
+ install_requires=['wrapt', 'requests'],
tests_require=['pytools', 'rpy2', 'httpretty==0.8.10', 'docutils', 'nose>=1.3', 'boto>=2.38.0', 'filechunkio>=1.6',
- 'moto>=0.4.14', 'ftputil>=3.2', 'pysftp>=0.2.8', 'requests>=2.8.1', 'dropbox>=5.2', 'pyyaml'],
+ 'moto>=0.4.14', 'ftputil>=3.2', 'pysftp>=0.2.8', 'requests>=2.8.1', 'dropbox>=5.2', 'pyyaml'],
test_suite='all',
cmdclass={'test': NoseTestCommand},
classifiers=
diff --git a/snakemake/__init__.py b/snakemake/__init__.py
index 6c802a2..5888e51 100644
--- a/snakemake/__init__.py
+++ b/snakemake/__init__.py
@@ -15,6 +15,7 @@ import inspect
import threading
import webbrowser
from functools import partial
+import importlib
from snakemake.workflow import Workflow
from snakemake.exceptions import print_exception
@@ -48,6 +49,7 @@ def snakemake(snakefile,
stats=None,
printreason=False,
printshellcmds=False,
+ debug_dag=False,
printdag=False,
printrulegraph=False,
printd3dag=False,
@@ -58,6 +60,7 @@ def snakemake(snakefile,
cluster_config=None,
cluster_sync=None,
drmaa=None,
+ drmaa_log_dir=None,
jobname="snakejob.{rulename}.{jobid}.sh",
immediate_submit=False,
standalone=False,
@@ -100,8 +103,11 @@ def snakemake(snakefile,
verbose=False,
force_use_threads=False,
use_conda=False,
+ conda_prefix=None,
mode=Mode.default,
- wrapper_prefix=None):
+ wrapper_prefix=None,
+ default_remote_provider=None,
+ default_remote_prefix=""):
"""Run snakemake on a given snakefile.
This function provides access to the whole snakemake functionality. It is not thread-safe.
@@ -136,6 +142,7 @@ def snakemake(snakefile,
cluster_config (str,list): configuration file for cluster options, or list thereof (default None)
cluster_sync (str): blocking cluster submission command (like SGE 'qsub -sync y') (default None)
drmaa (str): if not None use DRMAA for cluster support, str specifies native args passed to the cluster when submitting a job
+ drmaa_log_dir (str): the path to stdout and stderr output of DRMAA jobs (default None)
jobname (str): naming scheme for cluster job scripts (default "snakejob.{rulename}.{jobid}.sh")
immediate_submit (bool): immediately submit all cluster jobs, regardless of dependencies (default False)
standalone (bool): kill all processes very rudely in case of failure (do not use this if you use this API) (default False) (deprecated)
@@ -176,8 +183,11 @@ def snakemake(snakefile,
restart_times (int): number of times to restart failing jobs (default 1)
force_use_threads: whether to force use of threads over processes. helpful if shared memory is full or unavailable (default False)
use_conda (bool): create conda environments for each job (defined with conda directive of rules)
+ conda_prefix (str): the directories in which conda environments will be created (default None)
mode (snakemake.common.Mode): Execution mode
wrapper_prefix (str): Prefix for wrapper script URLs (default None)
+ default_remote_provider (str): Default remote provider to use instead of local files (S3, GS)
+ default_remote_prefix (str): Prefix for default remote provider (e.g. name of the bucket).
log_handler (function): redirect snakemake output to this custom log handler, a function that takes a log message dictionary (see below) as its only argument (default None). The log message dictionary for the log handler has to following entries:
:level:
@@ -223,6 +233,7 @@ def snakemake(snakefile,
bool: True if workflow execution was successful.
"""
+ assert not immediate_submit or (immediate_submit and notemp), "immediate_submit has to be combined with notemp (it does not support temp file handling)"
if updated_files is None:
updated_files = list()
@@ -250,12 +261,17 @@ def snakemake(snakefile,
# force thread use for any kind of cluster
use_threads = force_use_threads or (os.name != "posix") or cluster or cluster_sync or drmaa
if not keep_logger:
+ stdout = (
+ (dryrun and not (printdag or printd3dag or printrulegraph)) or
+ listrules or list_target_rules or list_resources
+ )
setup_logger(handler=log_handler,
quiet=quiet,
printreason=printreason,
printshellcmds=printshellcmds,
+ debug_dag=debug_dag,
nocolor=nocolor,
- stdout=dryrun and not (printdag or printd3dag or printrulegraph),
+ stdout=stdout,
debug=verbose,
timestamp=timestamp,
use_threads=use_threads,
@@ -286,6 +302,7 @@ def snakemake(snakefile,
overwrite_config = dict()
if configfile:
overwrite_config.update(load_configfile(configfile))
+ configfile = os.path.abspath(configfile)
if config:
overwrite_config.update(config)
@@ -297,6 +314,21 @@ def snakemake(snakefile,
os.makedirs(workdir)
workdir = os.path.abspath(workdir)
os.chdir(workdir)
+
+ # handle default remote provider
+ _default_remote_provider = None
+ if default_remote_provider is not None:
+ try:
+ rmt = importlib.import_module("snakemake.remote." +
+ default_remote_provider)
+ except ImportError as e:
+ raise WorkflowError("Unknown default remote provider.")
+ if rmt.RemoteProvider.supports_default:
+ _default_remote_provider = rmt.RemoteProvider()
+ else:
+ raise WorkflowError("Remote provider {} does not (yet) support to "
+ "be used as default provider.")
+
workflow = Workflow(snakefile=snakefile,
jobscript=jobscript,
overwrite_shellcmd=overwrite_shellcmd,
@@ -307,10 +339,13 @@ def snakemake(snakefile,
config_args=config_args,
debug=debug,
use_conda=use_conda,
+ conda_prefix=conda_prefix,
mode=mode,
wrapper_prefix=wrapper_prefix,
printshellcmds=printshellcmds,
- restart_times=restart_times)
+ restart_times=restart_times,
+ default_remote_provider=_default_remote_provider,
+ default_remote_prefix=default_remote_prefix)
success = True
try:
workflow.include(snakefile,
@@ -337,12 +372,14 @@ def snakemake(snakefile,
touch=touch,
printreason=printreason,
printshellcmds=printshellcmds,
+ debug_dag=debug_dag,
nocolor=nocolor,
quiet=quiet,
keepgoing=keepgoing,
cluster=cluster,
cluster_sync=cluster_sync,
drmaa=drmaa,
+ drmaa_log_dir=drmaa_log_dir,
jobname=jobname,
immediate_submit=immediate_submit,
standalone=standalone,
@@ -365,9 +402,15 @@ def snakemake(snakefile,
overwrite_shellcmd=overwrite_shellcmd,
config=config,
config_args=config_args,
+ cluster_config=cluster_config,
keep_logger=True,
keep_shadow=True,
- force_use_threads=use_threads)
+ force_use_threads=use_threads,
+ use_conda=use_conda,
+ conda_prefix=conda_prefix,
+ default_remote_provider=default_remote_provider,
+ default_remote_prefix=default_remote_prefix)
+
success = workflow.execute(
targets=targets,
dryrun=dryrun,
@@ -391,6 +434,7 @@ def snakemake(snakefile,
cluster_sync=cluster_sync,
jobname=jobname,
drmaa=drmaa,
+ drmaa_log_dir=drmaa_log_dir,
max_jobs_per_second=max_jobs_per_second,
printd3dag=printd3dag,
immediate_submit=immediate_submit,
@@ -486,7 +530,7 @@ def parse_config(args):
try:
v = parser(val)
# avoid accidental interpretation as function
- if not isinstance(v, callable):
+ if not callable(v):
break
except:
pass
@@ -582,6 +626,11 @@ def get_argument_parser():
action="store_true",
help="Print out the shell commands that will be executed.")
parser.add_argument(
+ "--debug-dag",
+ action="store_true",
+ help="Print candidate and selected jobs (including their wildcards) while "
+ "inferring DAG. This can help to debug unexpected DAG topology or errors.")
+ parser.add_argument(
"--dag",
action="store_true",
help="Do not execute anything and print the directed "
@@ -738,6 +787,16 @@ def get_argument_parser():
"with a leading whitespace.")
parser.add_argument(
+ "--drmaa-log-dir",
+ metavar="DIR",
+ help="Specify a directory in which stdout and stderr files of DRMAA"
+ " jobs will be written. The value may be given as a relative path,"
+ " in which case Snakemake will use the current invocation directory"
+ " as the origin. If given, this will override any given '-o' and/or"
+ " '-e' native specification. If not given, all DRMAA stdout and"
+ " stderr files are written to the current working directory.")
+
+ parser.add_argument(
"--cluster-config", "-u",
metavar="FILE",
default=[],
@@ -878,7 +937,9 @@ def get_argument_parser():
"--allowed-rules",
nargs="+",
help=
- "Only use given rules. If omitted, all rules in Snakefile are used.")
+ "Only consider given rules. If omitted, all rules in Snakefile are "
+ "used. Note that this is intended primarily for internal use and may "
+ "lead to unexpected results otherwise.")
parser.add_argument(
"--max-jobs-per-second", default=None, type=float,
help=
@@ -944,12 +1005,31 @@ def get_argument_parser():
help="If defined in the rule, create job specific conda environments. "
"If this flag is not set, the conda directive is ignored.")
parser.add_argument(
+ "--conda-prefix",
+ metavar="DIR",
+ help="Specify a directory in which the 'conda' and 'conda-archive' "
+ "directories are created. These are used to store conda environments "
+ "and their archives, respectively. If not supplied, the value is set "
+ "to the '.snakemake' directory relative to the invocation directory. "
+ "If supplied, the `--use-conda` flag must also be set. The value may "
+ "be given as a relative path, which will be extrapolated to the "
+ "invocation directory, or as an absolute path.")
+ parser.add_argument(
"--wrapper-prefix",
default="https://bitbucket.org/snakemake/snakemake-wrappers/raw/",
help="Prefix for URL created from wrapper directive (default: "
"https://bitbucket.org/snakemake/snakemake-wrappers/raw/). Set this to "
"a different URL to use your fork or a local clone of the repository."
)
+ parser.add_argument("--default-remote-provider",
+ choices=["S3", "GS", "SFTP", "S3Mocked"],
+ help="Specify default remote provider to be used for "
+ "all input and output files that don't yet specify "
+ "one.")
+ parser.add_argument("--default-remote-prefix",
+ default="",
+ help="Specify prefix for default remote provider. E.g. "
+ "a bucket name.")
parser.add_argument("--version", "-v",
action="version",
version=__version__)
@@ -988,10 +1068,27 @@ def main(argv=None):
elif args.cores is None:
args.cores = 1
+ if args.drmaa_log_dir is not None:
+ if not os.path.isabs(args.drmaa_log_dir):
+ args.drmaa_log_dir = os.path.abspath(os.path.expanduser(args.drmaa_log_dir))
+
if args.profile:
import yappi
yappi.start()
+ if args.immediate_submit and not args.notemp:
+ print(
+ "Error: --immediate-submit has to be combined with --notemp, "
+ "because temp file handling is not supported in this mode.",
+ file=sys.stderr)
+ sys.exit(1)
+
+ if args.conda_prefix and not args.use_conda:
+ print(
+ "Error: --use-conda must be set if --conda-prefix is set.",
+ file=sys.stderr)
+ sys.exit(1)
+
if args.gui is not None:
try:
import snakemake.gui as gui
@@ -1039,6 +1136,7 @@ def main(argv=None):
dryrun=args.dryrun,
printshellcmds=args.printshellcmds,
printreason=args.reason,
+ debug_dag=args.debug_dag,
printdag=args.dag,
printrulegraph=args.rulegraph,
printd3dag=args.d3dag,
@@ -1057,6 +1155,7 @@ def main(argv=None):
cluster_config=args.cluster_config,
cluster_sync=args.cluster_sync,
drmaa=args.drmaa,
+ drmaa_log_dir=args.drmaa_log_dir,
jobname=args.jobname,
immediate_submit=args.immediate_submit,
standalone=True,
@@ -1093,8 +1192,11 @@ def main(argv=None):
restart_times=args.restart_times,
force_use_threads=args.force_use_threads,
use_conda=args.use_conda,
+ conda_prefix=args.conda_prefix,
mode=args.mode,
- wrapper_prefix=args.wrapper_prefix)
+ wrapper_prefix=args.wrapper_prefix,
+ default_remote_provider=args.default_remote_provider,
+ default_remote_prefix=args.default_remote_prefix)
if args.profile:
with open(args.profile, "w") as out:
diff --git a/snakemake/benchmark.py b/snakemake/benchmark.py
new file mode 100644
index 0000000..36c1568
--- /dev/null
+++ b/snakemake/benchmark.py
@@ -0,0 +1,259 @@
+__author__ = "Manuel Holtgrewe"
+__copyright__ = "Copyright 2017, Manuel Holtgrewe"
+__email__ = "manuel.holtgrewe at bihealth.de"
+__license__ = "MIT"
+
+import contextlib
+import datetime
+from itertools import chain
+import os
+import sys
+import time
+import threading
+
+from snakemake.exceptions import WorkflowError
+
+try:
+ import psutil
+except ImportError:
+ raise WorkflowError(
+ "Python 3 package psutil needs to be installed to use the benchmarking.")
+
+
+#: Interval (in seconds) between measuring resource usage
+BENCHMARK_INTERVAL = 30
+#: Interval (in seconds) between measuring resource usage before
+#: BENCHMARK_INTERVAL
+BENCHMARK_INTERVAL_SHORT = 0.5
+
+
+class BenchmarkRecord:
+ """Record type for benchmark times"""
+
+ @classmethod
+ def get_header(klass):
+ return '\t'.join(
+ ('s', 'h:m:s', 'max_rss', 'max_vms', 'max_uss', 'max_pss', 'io_in', 'io_out',
+ 'mean_load'))
+
+ def __init__(self, running_time=None, max_rss=None, max_vms=None, max_uss=None, max_pss=None,
+ io_in=None, io_out=None, cpu_seconds=None):
+ #: Running time in seconds
+ self.running_time = running_time
+ #: Maximal RSS in MB
+ self.max_rss = max_rss
+ #: Maximal VMS in MB
+ self.max_vms = max_vms
+ #: Maximal USS in MB
+ self.max_uss = max_uss
+ #: Maximal PSS in MB
+ self.max_pss = max_pss
+ #: I/O read in bytes
+ self.io_in = io_in
+ #: I/O written in bytes
+ self.io_out = io_out
+ #: Count of CPU seconds, divide by running time to get mean load estimate
+ self.cpu_seconds = cpu_seconds or 0
+ #: First time when we measured CPU load, for estimating total running time
+ self.first_time = None
+ #: Previous point when measured CPU load, for estimating total running time
+ self.prev_time = None
+
+ def to_tsv(self):
+ """Return ``str`` with the TSV representation of this record"""
+ def to_tsv_str(x):
+ """Conversion of value to str for TSV (None becomes "-")"""
+ if x is None:
+ return '-'
+ elif isinstance(x, float):
+ return '{:.2f}'.format(x)
+ else:
+ return str(x)
+ def timedelta_to_str(x):
+ """Conversion of timedelta to str without fractions of seconds"""
+ mm, ss = divmod(x.seconds, 60)
+ hh, mm = divmod(mm, 60)
+ s = "%d:%02d:%02d" % (hh, mm, ss)
+ if x.days:
+ def plural(n):
+ return n, abs(n) != 1 and "s" or ""
+ s = ("%d day%s, " % plural(x.days)) + s
+ return s
+ return '\t'.join(map(
+ to_tsv_str, (
+ '{:.4f}'.format(self.running_time),
+ timedelta_to_str(datetime.timedelta(seconds=self.running_time)),
+ self.max_rss, self.max_vms, self.max_uss, self.max_pss, self.io_in, self.io_out,
+ 100.0 * self.cpu_seconds / self.running_time)))
+
+
+class DaemonTimer(threading.Thread):
+ """Variant of threading.Timer that is deaemonized"""
+
+ def __init__(self, interval, function, args=None, kwargs=None):
+ threading.Thread.__init__(self, daemon=True)
+ self.interval = interval
+ self.function = function
+ self.args = args if args is not None else []
+ self.kwargs = kwargs if kwargs is not None else {}
+ self.finished = threading.Event()
+
+ def cancel(self):
+ """Stop the timer if it hasn't finished yet."""
+ self.finished.set()
+
+ def run(self):
+ self.finished.wait(self.interval)
+ if not self.finished.is_set():
+ self.function(*self.args, **self.kwargs)
+ self.finished.set()
+
+
+class ScheduledPeriodicTimer:
+ """Scheduling of periodic events
+
+ Up to self._interval, schedule actions per second, above schedule events
+ in self._interval second gaps.
+ """
+
+ def __init__(self, interval):
+ self._times_called = 0
+ self._interval = interval
+ self._timer = None
+ self._stopped = True
+
+ def start(self):
+ """Start the intervalic timer"""
+ self.work()
+ self._times_called += 1
+ self._stopped = False
+ if self._times_called > self._interval:
+ self._timer = DaemonTimer(self._interval, self._action)
+ else:
+ self._timer = DaemonTimer(BENCHMARK_INTERVAL_SHORT, self._action)
+ self._timer.start()
+
+ def _action(self):
+ """Internally, called by timer"""
+ self.work()
+ self._times_called += 1
+ if self._times_called > self._interval:
+ self._timer = DaemonTimer(self._interval, self._action)
+ else:
+ self._timer = DaemonTimer(BENCHMARK_INTERVAL_SHORT, self._action)
+ self._timer.start()
+
+ def work(self):
+ """Override to perform the action"""
+ raise NotImplementedError('Override me!')
+
+ def cancel(self):
+ """Call to cancel any events"""
+ self._timer.cancel()
+ self._stopped = True
+
+
+class BenchmarkTimer(ScheduledPeriodicTimer):
+ """Allows easy observation of a given PID for resource usage"""
+
+ def __init__(self, pid, bench_record, interval=BENCHMARK_INTERVAL):
+ ScheduledPeriodicTimer.__init__(self, interval)
+ #: PID of observed process
+ self.pid = pid
+ #: ``BenchmarkRecord`` to write results to
+ self.bench_record = bench_record
+
+ def work(self):
+ """Write statistics"""
+ try:
+ self._update_record()
+ except psutil.NoSuchProcess:
+ pass # skip, process died in flight
+ except AttributeError:
+ pass # skip, process died in flight
+
+ def _update_record(self):
+ """Perform the actual measurement"""
+ # Memory measurements
+ rss, vms, uss, pss = 0, 0, 0, 0
+ # I/O measurements
+ io_in, io_out = 0, 0
+ # CPU seconds
+ cpu_seconds = 0
+ # Iterate over process and all children
+ try:
+ main = psutil.Process(self.pid)
+ this_time = time.time()
+ for proc in chain((main,), main.children(recursive=True)):
+ meminfo = proc.memory_full_info()
+ rss += meminfo.rss
+ vms += meminfo.vms
+ uss += meminfo.uss
+ pss += meminfo.pss
+ ioinfo = proc.io_counters()
+ io_in += ioinfo.read_bytes
+ io_out += ioinfo.write_bytes
+ if self.bench_record.prev_time:
+ cpu_seconds += proc.cpu_percent() / 100 * (
+ this_time - self.bench_record.prev_time)
+ self.bench_record.prev_time = this_time
+ if not self.bench_record.first_time:
+ self.bench_record.prev_time = this_time
+ rss /= 1024 * 1024
+ vms /= 1024 * 1024
+ uss /= 1024 * 1024
+ pss /= 1024 * 1024
+ io_in /= 1024 * 1024
+ io_out /= 1024 * 1024
+ except psutil.Error as e:
+ return
+ # Update benchmark record's RSS and VMS
+ self.bench_record.max_rss = max(self.bench_record.max_rss or 0, rss)
+ self.bench_record.max_vms = max(self.bench_record.max_vms or 0, vms)
+ self.bench_record.max_uss = max(self.bench_record.max_uss or 0, uss)
+ self.bench_record.max_pss = max(self.bench_record.max_pss or 0, pss)
+ self.bench_record.io_in = io_in
+ self.bench_record.io_out = io_out
+ self.bench_record.cpu_seconds += cpu_seconds
+
+
+ at contextlib.contextmanager
+def benchmarked(pid=None, benchmark_record=None, interval=BENCHMARK_INTERVAL):
+ """Measure benchmark parameters while within the context manager
+
+ Yields a ``BenchmarkRecord`` with the results (values are set after
+ leaving context).
+
+ If ``pid`` is ``None`` then the PID of the current process will be used.
+ If ``benchmark_record`` is ``None`` then a new ``BenchmarkRecord`` is
+ created and returned, otherwise, the object passed as this parameter is
+ returned.
+
+ Usage::
+
+ with benchmarked() as bench_result:
+ pass
+ """
+ result = benchmark_record or BenchmarkRecord()
+ if pid is False:
+ yield result
+ else:
+ start_time = time.time()
+ bench_thread = BenchmarkTimer(int(pid or os.getpid()), result, interval)
+ bench_thread.start()
+ yield result
+ bench_thread.cancel()
+ result.running_time = time.time() - start_time
+
+
+def print_benchmark_records(records, file_):
+ """Write benchmark records to file-like object"""
+ print(BenchmarkRecord.get_header(), file=file_)
+ for r in records:
+ print(r.to_tsv(), file=file_)
+
+
+def write_benchmark_records(records, path):
+ """Write benchmark records to file at path"""
+ with open(path, 'wt') as f:
+ print_benchmark_records(records, f)
diff --git a/snakemake/common.py b/snakemake/common.py
index e210db0..0feefd8 100644
--- a/snakemake/common.py
+++ b/snakemake/common.py
@@ -33,3 +33,9 @@ class lazy_property(property):
value = self.method(instance)
setattr(instance, self.cached, value)
return value
+
+
+def strip_prefix(text, prefix):
+ if text.startswith(prefix):
+ return text[len(prefix):]
+ return text
diff --git a/snakemake/conda.py b/snakemake/conda.py
index e80dc82..b9eadea 100644
--- a/snakemake/conda.py
+++ b/snakemake/conda.py
@@ -1,7 +1,7 @@
import os
import subprocess
import tempfile
-from urllib.request import urlopen, urlretrieve
+from urllib.request import urlopen
from urllib.parse import urlparse
import hashlib
import shutil
@@ -11,77 +11,167 @@ from glob import glob
from snakemake.exceptions import CreateCondaEnvironmentException, WorkflowError
from snakemake.logging import logger
+from snakemake.common import strip_prefix
+from snakemake import utils
-def get_env_archive(job, env_hash):
- """Get path to archived environment derived from given environment file."""
- return os.path.join(job.rule.workflow.persistence.conda_env_archive_path, env_hash)
+class Env:
+ """Conda environment from a given specification file."""
-def archive_env(job):
- """Create self-contained archive of environment."""
- try:
- import yaml
- except ImportError:
- raise WorkflowError("Error importing PyYAML. "
- "Please install PyYAML to archive workflows.")
+ def __init__(self, env_file, dag):
+ self.file = env_file
- env_archive = get_env_archive(job, get_env_hash(job.conda_env_file))
- if os.path.exists(env_archive):
- return env_archive
+ self._env_dir = dag.workflow.persistence.conda_env_path
+ self._env_archive_dir = dag.workflow.persistence.conda_env_archive_path
+
+ self._hash = None
+ self._content = None
+ self._path = None
+ self._archive_file = None
+
+ @property
+ def content(self):
+ if self._content is None:
+ env_file = self.file
+ if urlparse(env_file).scheme:
+ content = urlopen(env_file).read()
+ else:
+ with open(env_file, 'rb') as f:
+ content = f.read()
+ self._content = content
+ return self._content
+
+ @property
+ def hash(self):
+ if self._hash is None:
+ md5hash = hashlib.md5()
+ # Include the absolute path of the target env dir into the hash.
+ # By this, moving the working directory around automatically
+ # invalidates all environments. This is necessary, because binaries
+ # in conda environments can contain hardcoded absolute RPATHs.
+ assert os.path.isabs(self._env_dir)
+ md5hash.update(self._env_dir.encode())
+ md5hash.update(self.content)
+ self._hash = md5hash.hexdigest()
+ return self._hash
+
+ @property
+ def path(self):
+ """Path to directory of the conda environment.
+
+ First tries full hash, if it does not exist, (8-prefix) is used
+ as default.
+
+ """
+ hash = self.hash
+ env_dir = self._env_dir
+ for h in [hash, hash[:8]]:
+ path = os.path.join(env_dir, h)
+ if os.path.exists(path):
+ return path
+ return path
+
+ @property
+ def archive_file(self):
+ """Path to archive of the conda environment, which may or may not exist."""
+ if self._archive_file is None:
+ self._archive_file = os.path.join(self._env_archive_dir, self.hash)
+ return self._archive_file
+
+ def create_archive(self):
+ """Create self-contained archive of environment."""
+ try:
+ import yaml
+ except ImportError:
+ raise WorkflowError("Error importing PyYAML. "
+ "Please install PyYAML to archive workflows.")
+ # importing requests locally because it interferes with instantiating conda environments
+ import requests
+
+ env_archive = self.archive_file
+ if os.path.exists(env_archive):
+ return env_archive
- try:
- # Download
- logger.info("Downloading packages for conda environment {}...".format(job.conda_env_file))
- os.makedirs(env_archive, exist_ok=True)
try:
- out = subprocess.check_output(["conda", "list", "--explicit",
- "--prefix", job.conda_env],
- stderr=subprocess.STDOUT)
- logger.debug(out.decode())
- except subprocess.CalledProcessError as e:
- raise WorkflowError("Error exporting conda packages:\n" +
- e.output.decode())
- for l in out.decode().split("\n"):
- if l and not l.startswith("#") and not l.startswith("@"):
- pkg_url = l
- logger.info(pkg_url)
- parsed = urlparse(pkg_url)
- pkg_name = os.path.basename(parsed.path)
- urlretrieve(pkg_url, os.path.join(env_archive, pkg_name))
- except (Exception, BaseException) as e:
- shutil.rmtree(env_archive)
- raise e
- return env_archive
-
-
-def is_remote_env_file(env_file):
- return urlparse(env_file).scheme
-
-
-def get_env_hash(env_file):
- md5hash = hashlib.md5()
- if is_remote_env_file(env_file):
- md5hash.update(urlopen(env_file).read())
- else:
- with open(env_file, 'rb') as f:
- md5hash.update(f.read())
- return md5hash.hexdigest()
-
-
-def get_env_path(job, env_hash):
- """Return environment path from hash.
- First tries full hash, if it does not exist, (8-prefix) is used as
- default."""
- for h in [env_hash, env_hash[:8]]:
- path = os.path.join(job.rule.workflow.persistence.conda_env_path, h)
- if os.path.exists(path):
- return path
- return path
-
-
-def create_env(job):
- """ Create conda enviroment for the given job. """
+ # Download
+ logger.info("Downloading packages for conda environment {}...".format(self.file))
+ os.makedirs(env_archive, exist_ok=True)
+ try:
+ out = subprocess.check_output(["conda", "list", "--explicit",
+ "--prefix", self.path],
+ stderr=subprocess.STDOUT)
+ logger.debug(out.decode())
+ except subprocess.CalledProcessError as e:
+ raise WorkflowError("Error exporting conda packages:\n" +
+ e.output.decode())
+ for l in out.decode().split("\n"):
+ if l and not l.startswith("#") and not l.startswith("@"):
+ pkg_url = l
+ logger.info(pkg_url)
+ parsed = urlparse(pkg_url)
+ pkg_name = os.path.basename(parsed.path)
+ with open(os.path.join(env_archive, pkg_name), "wb") as copy:
+ copy.write(requests.get(pkg_url).content)
+ except (Exception, BaseException) as e:
+ shutil.rmtree(env_archive)
+ raise e
+ return env_archive
+
+ def create(self, dryrun=False):
+ """ Create the conda enviroment."""
+ # Read env file and create hash.
+ env_file = self.file
+ tmp_file = None
+
+ url_scheme, *_ = urlparse(env_file)
+ if url_scheme and not url_scheme == 'file':
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".yaml") as tmp:
+ tmp.write(self.content)
+ env_file = tmp.name
+ tmp_file = tmp.name
+
+ env_hash = self.hash
+ env_path = self.path
+ # Create environment if not already present.
+ if not os.path.exists(env_path):
+ if dryrun:
+ logger.info("Conda environment {} will be created.".format(utils.simplify_path(self.file)))
+ return env_path
+ logger.info("Creating conda environment {}...".format(
+ utils.simplify_path(self.file)))
+ # Check if env archive exists. Use that if present.
+ env_archive = self.archive_file
+ try:
+ if os.path.exists(env_archive):
+ # install packages manually from env archive
+ out = subprocess.check_output(["conda", "create", "--copy", "--prefix", env_path] +
+ glob(os.path.join(env_archive, "*.tar.bz2")),
+ stderr=subprocess.STDOUT
+ )
+ else:
+ out = subprocess.check_output(["conda", "env", "create",
+ "--file", env_file,
+ "--prefix", env_path],
+ stderr=subprocess.STDOUT)
+ logger.debug(out.decode())
+ logger.info("Environment for {} created (location: {})".format(
+ os.path.relpath(env_file), os.path.relpath(env_path)))
+ except subprocess.CalledProcessError as e:
+ # remove potential partially installed environment
+ shutil.rmtree(env_path, ignore_errors=True)
+ raise CreateCondaEnvironmentException(
+ "Could not create conda environment from {}:\n".format(env_file) +
+ e.output.decode())
+
+ if tmp_file:
+ # temporary file was created
+ os.remove(tmp_file)
+
+ return env_path
+
+
+def check_conda():
if shutil.which("conda") is None:
raise CreateCondaEnvironmentException("The 'conda' command is not available in $PATH.")
try:
@@ -94,45 +184,3 @@ def create_env(job):
raise CreateCondaEnvironmentException(
"Unable to check conda version:\n" + e.output.decode()
)
-
- # Read env file and create hash.
- env_file = job.conda_env_file
- tmp_file = None
- if is_remote_env_file(env_file):
- with tempfile.NamedTemporaryFile(delete=False) as tmp:
- tmp.write(urlopen(env_file).read())
- env_file = tmp.name
- tmp_file = tmp.name
- env_hash = get_env_hash(env_file)
- env_path = get_env_path(job, env_hash)
- # Create environment if not already present.
- if not os.path.exists(env_path):
- logger.info("Creating conda environment {}...".format(job.conda_env_file))
- # Check if env archive exists. Use that if present.
- env_archive = get_env_archive(job, env_hash)
- try:
- if os.path.exists(env_archive):
- # install packages manually from env archive
- out = subprocess.check_output(["conda", "create", "--prefix", env_path] +
- glob(os.path.join(env_archive, "*.tar.bz2")),
- stderr=subprocess.STDOUT
- )
- else:
- out = subprocess.check_output(["conda", "env", "create",
- "--file", env_file,
- "--prefix", env_path],
- stderr=subprocess.STDOUT)
- logger.debug(out.decode())
- logger.info("Environment for {} created.".format(job.conda_env_file))
- except subprocess.CalledProcessError as e:
- # remove potential partially installed environment
- shutil.rmtree(env_path, ignore_errors=True)
- raise CreateCondaEnvironmentException(
- "Could not create conda environment from {}:\n".format(job.conda_env_file) +
- e.output.decode())
-
- if tmp_file:
- # temporary file was created
- os.remove(tmp_file)
-
- return env_path
diff --git a/snakemake/dag.py b/snakemake/dag.py
index 9ebcc79..6acf6ec 100644
--- a/snakemake/dag.py
+++ b/snakemake/dag.py
@@ -22,7 +22,7 @@ from snakemake.exceptions import RuleException, MissingInputException
from snakemake.exceptions import MissingRuleException, AmbiguousRuleException
from snakemake.exceptions import CyclicGraphException, MissingOutputException
from snakemake.exceptions import IncompleteFilesException
-from snakemake.exceptions import PeriodicWildcardError
+from snakemake.exceptions import PeriodicWildcardError, WildcardError
from snakemake.exceptions import RemoteFileException, WorkflowError
from snakemake.exceptions import UnexpectedOutputException, InputFunctionException
from snakemake.logging import logger
@@ -83,6 +83,8 @@ class DAG:
self.notemp = notemp
self.keep_remote_local = keep_remote_local
self._jobid = dict()
+ self.job_cache = dict()
+ self.conda_envs = dict()
self.forcerules = set()
self.forcefiles = set()
@@ -141,6 +143,7 @@ class DAG:
self._jobid[job] = len(self._jobid)
def cleanup(self):
+ self.job_cache.clear()
final_jobs = set(self.jobs)
todelete = [job for job in self.dependencies if job not in final_jobs]
for job in todelete:
@@ -150,6 +153,24 @@ class DAG:
except KeyError:
pass
+ def create_conda_envs(self, dryrun=False):
+ conda.check_conda()
+ # First deduplicate based on job.conda_env_file
+ env_set = {job.conda_env_file for job in self.needrun_jobs
+ if job.conda_env_file}
+ # Then based on md5sum values
+ env_file_map = dict()
+ hash_set = set()
+ for env_file in env_set:
+ env = conda.Env(env_file, self)
+ hash = env.hash
+ env_file_map[env_file] = env
+ if hash not in hash_set:
+ env.create(dryrun)
+ hash_set.add(hash)
+
+ self.conda_envs = env_file_map
+
def update_output_index(self):
"""Update the OutputIndex."""
self.output_index = OutputIndex(self.rules)
@@ -286,24 +307,30 @@ class DAG:
return True
return False
- def check_and_touch_output(self, job, wait=3):
+ def check_and_touch_output(self, job, wait=3, ignore_missing_output=False):
""" Raise exception if output files of job are missing. """
expanded_output = [job.shadowed_path(path) for path in job.expanded_output]
- try:
- wait_for_files(expanded_output, latency_wait=wait)
- except IOError as e:
- raise MissingOutputException(str(e) + "\nThis might be due to "
- "filesystem latency. If that is the case, consider to increase the "
- "wait time with --latency-wait.", rule=job.rule)
+ if job.benchmark:
+ expanded_output.append(job.benchmark)
+
+ if ignore_missing_output is False:
+ try:
+ wait_for_files(expanded_output, latency_wait=wait)
+ except IOError as e:
+ raise MissingOutputException(str(e) + "\nThis might be due to "
+ "filesystem latency. If that is the case, consider to increase the "
+ "wait time with --latency-wait.", rule=job.rule)
#It is possible, due to archive expansion or cluster clock skew, that
#the files appear older than the input. But we know they must be new,
- #so touch them to update timestamps.
+ #so touch them to update timestamps. This also serves to touch outputs
+ #when using the --touch flag.
#Note that if the input files somehow have a future date then this will
#not currently be spotted and the job will always be re-run.
#Also, don't touch directories, as we can't guarantee they were removed.
for f in expanded_output:
- if not os.path.isdir(f):
+ #This will neither create missing files nor touch directories
+ if os.path.isfile(f):
f.touch()
def unshadow_output(self, job):
@@ -385,8 +412,13 @@ class DAG:
""" Remove local files if they are no longer needed, and upload to S3. """
if upload:
# handle output files
- for f in job.expanded_output:
- if f.is_remote:
+ files = list(job.expanded_output)
+ if job.benchmark:
+ files.append(job.benchmark)
+ if job.log:
+ files.extend(job.log)
+ for f in files:
+ if f.is_remote and not f.should_stay_on_remote:
f.upload_to_remote()
remote_mtime = f.mtime
# immediately force local mtime to match remote,
@@ -424,8 +456,9 @@ class DAG:
yield f
for f in unneeded_files():
- logger.info("Removing local output file: {}".format(f))
- f.remove()
+ if f.exists_local:
+ logger.info("Removing local output file: {}".format(f))
+ f.remove()
job.rmdir_empty_remote_dirs()
@@ -442,7 +475,9 @@ class DAG:
jobs = sorted(jobs, reverse=not self.ignore_ambiguity)
cycles = list()
+
for job in jobs:
+ logger.dag_debug(dict(status="candidate", job=job))
if file in job.input:
cycles.append(job)
continue
@@ -482,6 +517,9 @@ class DAG:
raise CyclicGraphException(job.rule, file, rule=job.rule)
if exceptions:
raise exceptions[0]
+
+ logger.dag_debug(dict(status="selected", job=job))
+
return producer
def update_(self, job, visited=None, skip_until_dynamic=False):
@@ -704,8 +742,6 @@ class DAG:
def finish(self, job, update_dynamic=True):
"""Finish a given job (e.g. remove from ready jobs, mark depending jobs
as ready)."""
- job.close_remote()
-
self._finished.add(job)
try:
self._ready_jobs.remove(job)
@@ -731,6 +767,19 @@ class DAG:
# add finished jobs to len as they are not counted after new postprocess
self._len += len(self._finished)
+ def new_job(self, rule, targetfile=None, format_wildcards=None):
+ """Create new job for given rule and (optional) targetfile.
+ This will reuse existing jobs with the same wildcards."""
+ key = (rule, targetfile)
+ if key in self.job_cache:
+ assert targetfile is not None
+ return self.job_cache[key]
+ wildcards_dict = rule.get_wildcards(targetfile)
+ job = Job(rule, self, wildcards_dict=wildcards_dict, format_wildcards=format_wildcards)
+ for f in job.output:
+ self.job_cache[(rule, f)] = job
+ return job
+
def update_dynamic(self, job):
"""Update the DAG by evaluating the output of the given job that
contains dynamic output files."""
@@ -747,18 +796,22 @@ class DAG:
self.specialize_rule(job.rule, newrule)
# no targetfile needed for job
- newjob = Job(newrule, self, format_wildcards=non_dynamic_wildcards)
+ newjob = self.new_job(newrule, format_wildcards=non_dynamic_wildcards)
self.replace_job(job, newjob)
for job_ in depending:
- if job_.dynamic_input:
+ needs_update = any(
+ f.get_wildcard_names() & dynamic_wildcards.keys()
+ for f in job_.rule.dynamic_input)
+
+ if needs_update:
newrule_ = job_.rule.dynamic_branch(dynamic_wildcards)
if newrule_ is not None:
self.specialize_rule(job_.rule, newrule_)
if not self.dynamic(job_):
logger.debug("Updating job {}.".format(job_))
- newjob_ = Job(newrule_,
- self,
- targetfile=job_.targetfile)
+ newjob_ = self.new_job(
+ newrule_,
+ targetfile=job_.output[0] if job_.output else None)
unexpected_output = self.reason(
job_).missing_output.intersection(
@@ -811,10 +864,12 @@ class DAG:
self.delete_job(job)
self.update([newjob])
+ logger.debug("Replace {} with dynamic branch {}".format(job, newjob))
for job_, files in depending:
- if not job_.dynamic_input:
- self.dependencies[job_][newjob].update(files)
- self.depending[newjob][job_].update(files)
+ #if not job_.dynamic_input:
+ logger.debug("updating depending job {}".format(job_))
+ self.dependencies[job_][newjob].update(files)
+ self.depending[newjob][job_].update(files)
def specialize_rule(self, rule, newrule):
"""Specialize the given rule by inserting newrule into the DAG."""
@@ -835,7 +890,7 @@ class DAG:
continue
try:
if file in job.dependencies:
- jobs = [Job(job.dependencies[file], self, targetfile=file)]
+ jobs = [self.new_job(job.dependencies[file], targetfile=file)]
else:
jobs = file2jobs(file)
dependencies[file].extend(jobs)
@@ -916,7 +971,7 @@ class DAG:
"""Generate a new job from a given rule."""
if targetrule.has_wildcards():
raise WorkflowError("Target rules may not contain wildcards. Please specify concrete files or a rule without wildcards.")
- return Job(targetrule, self)
+ return self.new_job(targetrule)
def file2jobs(self, targetfile):
rules = self.output_index.match(targetfile)
@@ -925,7 +980,7 @@ class DAG:
for rule in rules:
if rule.is_producer(targetfile):
try:
- jobs.append(Job(rule, self, targetfile=targetfile))
+ jobs.append(self.new_job(rule, targetfile=targetfile))
except InputFunctionException as e:
exceptions.append(e)
if not jobs:
@@ -1110,20 +1165,24 @@ class DAG:
try:
workdir = Path(os.path.abspath(os.getcwd()))
with tarfile.open(path, mode=mode, dereference=True) as archive:
+ archived = set()
def add(path):
if workdir not in Path(os.path.abspath(path)).parents:
logger.warning("Path {} cannot be archived: "
"not within working directory.".format(path))
else:
- archive.add(os.path.relpath(path))
+ f = os.path.relpath(path)
+ if f not in archived:
+ archive.add(f)
+ archived.add(f)
+ logger.info("archived " + f)
logger.info("Archiving files under version control...")
try:
out = subprocess.check_output(["git", "ls-files", "."])
for f in out.decode().split("\n"):
if f:
- logger.info(f)
add(f)
except subprocess.CalledProcessError as e:
raise WorkflowError("Error executing git.")
@@ -1134,7 +1193,6 @@ class DAG:
for f in job.input:
if not any(f in files for files in self.dependencies[job].values()):
# this is an input file that is not created by any job
- logger.info(f)
add(f)
logger.info("Archiving conda environments...")
diff --git a/snakemake/exceptions.py b/snakemake/exceptions.py
index 62e7a98..8bcda51 100644
--- a/snakemake/exceptions.py
+++ b/snakemake/exceptions.py
@@ -317,14 +317,22 @@ class DropboxFileException(RuleException):
def __init__(self, msg, lineno=None, snakefile=None):
super().__init__(msg, lineno=lineno, snakefile=snakefile)
+class XRootDFileException(RuleException):
+ def __init__(self, msg, lineno=None, snakefile=None):
+ super().__init__(msg, lineno=lineno, snakefile=snakefile)
+
+class NCBIFileException(RuleException):
+ def __init__(self, msg, lineno=None, snakefile=None):
+ super().__init__(msg, lineno=lineno, snakefile=snakefile)
+
class ClusterJobException(RuleException):
- def __init__(self, job, jobid, jobscript):
+ def __init__(self, job_info, jobid):
super().__init__(
- "Error executing rule {} on cluster (jobid: {}, jobscript: {}). "
- "For detailed error see the cluster log.".format(job.rule.name,
- jobid, jobscript),
- lineno=job.rule.lineno,
- snakefile=job.rule.snakefile)
+ "Error executing rule {} on cluster (jobid: {}, external: {}, jobscript: {}). "
+ "For detailed error see the cluster log.".format(job_info.job.rule.name,
+ jobid, job_info.jobid, job_info.jobscript),
+ lineno=job_info.job.rule.lineno,
+ snakefile=job_info.job.rule.snakefile)
class CreateRuleException(RuleException):
diff --git a/snakemake/executors.py b/snakemake/executors.py
index dd9a5d7..58e3f58 100644
--- a/snakemake/executors.py
+++ b/snakemake/executors.py
@@ -26,7 +26,7 @@ from snakemake.jobs import Job
from snakemake.shell import shell
from snakemake.logging import logger
from snakemake.stats import Stats
-from snakemake.utils import format, Unformattable
+from snakemake.utils import format, Unformattable, makedirs
from snakemake.io import get_wildcard_names, Wildcards
from snakemake.exceptions import print_exception, get_exception_origin
from snakemake.exceptions import format_error, RuleException, log_verbose_traceback
@@ -51,6 +51,15 @@ class AbstractExecutor:
self.latency_wait = latency_wait
self.benchmark_repeats = benchmark_repeats
+ def get_default_remote_provider_args(self):
+ if self.workflow.default_remote_provider:
+ return (
+ "--default-remote-provider {} "
+ "--default-remote-prefix {} ").format(
+ self.workflow.default_remote_provider.__module__.split(".")[-1],
+ self.workflow.default_remote_prefix)
+ return ""
+
def run(self, job,
callback=None,
submit_callback=None,
@@ -62,6 +71,9 @@ class AbstractExecutor:
def shutdown(self):
pass
+ def cancel(self):
+ pass
+
def _run(self, job):
self.printjob(job)
@@ -106,13 +118,11 @@ class AbstractExecutor:
logger.error("Error in job {} while creating output file{} {}.".format(
job, "s" if len(job.output) > 1 else "", ", ".join(job.output)))
- def finish_job(self, job, upload_remote=True):
- self.dag.handle_touch(job)
- self.dag.check_and_touch_output(job, wait=self.latency_wait)
- self.dag.unshadow_output(job)
- self.dag.handle_remote(job, upload=upload_remote)
- self.dag.handle_protected(job)
- self.dag.handle_temp(job)
+ def handle_job_success(self, job):
+ pass
+
+ def handle_job_error(self, job):
+ pass
class DryrunExecutor(AbstractExecutor):
@@ -149,8 +159,18 @@ class RealExecutor(AbstractExecutor):
"Please ensure write permissions for the "
"directory {}".format(e, self.workflow.persistence.path))
- def finish_job(self, job, upload_remote=True):
- super().finish_job(job, upload_remote=upload_remote)
+ def handle_job_success(self, job, upload_remote=True, ignore_missing_output=False):
+ self.dag.handle_touch(job)
+ self.dag.check_and_touch_output(
+ job,
+ wait=self.latency_wait,
+ ignore_missing_output=ignore_missing_output)
+ self.dag.unshadow_output(job)
+ self.dag.handle_remote(job, upload=upload_remote)
+ self.dag.handle_protected(job)
+ self.dag.handle_temp(job)
+ job.close_remote()
+
self.stats.report_job_end(job)
try:
self.workflow.persistence.finished(job)
@@ -160,6 +180,9 @@ class RealExecutor(AbstractExecutor):
"directory {}".format(e,
self.workflow.persistence.path))
+ def handle_job_error(self, job):
+ job.close_remote()
+
def format_job_pattern(self, pattern, job=None, **kwargs):
overwrite_workdir = []
if self.workflow.overwrite_workdir:
@@ -197,16 +220,16 @@ class TouchExecutor(RealExecutor):
error_callback=None):
super()._run(job)
try:
- #Touching of output files will be done by finish_job
- if job.benchmark:
- job.benchmark.touch()
+ #Touching of output files will be done by handle_job_success
time.sleep(0.1)
- self.finish_job(job)
callback(job)
except OSError as ex:
print_exception(ex, self.workflow.linemaps)
error_callback(job)
+ def handle_job_success(self, job):
+ super().handle_job_success(job, ignore_missing_output=True)
+
_ProcessPoolExceptions = (KeyboardInterrupt, )
try:
@@ -238,11 +261,14 @@ class CPUExecutor(RealExecutor):
'--force -j{cores} --keep-target-files --keep-shadow --keep-remote ',
'--benchmark-repeats {benchmark_repeats} ',
'--force-use-threads --wrapper-prefix {workflow.wrapper_prefix} ',
+ self.get_default_remote_provider_args(),
'{overwrite_workdir} {overwrite_config} {printshellcmds} ',
'--notemp --quiet --no-hooks --nolock --mode {} '.format(Mode.subprocess)))
if self.workflow.use_conda:
self.exec_job += " --use-conda "
+ if self.workflow.conda_prefix:
+ self.exec_job += " --conda-prefix " + self.workflow.conda_prefix + " "
self.use_threads = use_threads
self.cores = cores
@@ -258,17 +284,16 @@ class CPUExecutor(RealExecutor):
job.prepare()
conda_env = None
if self.workflow.use_conda:
- job.create_conda_env()
conda_env = job.conda_env
benchmark = None
if job.benchmark is not None:
benchmark = str(job.benchmark)
future = self.pool.submit(
- run_wrapper, job.rule.run_func, job.input.plainstrings(),
+ run_wrapper, job.rule, job.input.plainstrings(),
job.output.plainstrings(), job.params, job.wildcards, job.threads,
- job.resources, job.log.plainstrings(), job.rule.version, benchmark,
- self.benchmark_repeats, job.rule.name, conda_env,
+ job.resources, job.log.plainstrings(), benchmark,
+ self.benchmark_repeats, conda_env,
self.workflow.linemaps, self.workflow.debug,
shadow_dir=job.shadow_dir)
else:
@@ -299,24 +324,25 @@ class CPUExecutor(RealExecutor):
ex = future.exception()
if ex:
raise ex
- self.finish_job(job)
callback(job)
except _ProcessPoolExceptions:
- job.cleanup()
- self.workflow.persistence.cleanup(job)
+ self.handle_job_error(job)
# no error callback, just silently ignore the interrupt as the main scheduler is also killed
except SpawnedJobError:
# don't print error message, this is done by the spawned subprocess
- job.cleanup()
- self.workflow.persistence.cleanup(job)
error_callback(job)
except (Exception, BaseException) as ex:
self.print_job_error(job)
print_exception(ex, self.workflow.linemaps)
- job.cleanup()
- self.workflow.persistence.cleanup(job)
error_callback(job)
+ def handle_job_success(self, job):
+ super().handle_job_success(job)
+
+ def handle_job_error(self, job):
+ job.cleanup()
+ self.workflow.persistence.cleanup(job)
+
class ClusterExecutor(RealExecutor):
@@ -361,6 +387,7 @@ class ClusterExecutor(RealExecutor):
'--force -j{cores} --keep-target-files --keep-shadow --keep-remote ',
'--wait-for-files {wait_for_files} --latency-wait {latency_wait} ',
'--benchmark-repeats {benchmark_repeats} ',
+ self.get_default_remote_provider_args(),
'--force-use-threads --wrapper-prefix {workflow.wrapper_prefix} ',
'{overwrite_workdir} {overwrite_config} {printshellcmds} --nocolor ',
'--notemp --quiet --no-hooks --nolock'))
@@ -369,6 +396,8 @@ class ClusterExecutor(RealExecutor):
self.exec_job += " --printshellcmds "
if self.workflow.use_conda:
self.exec_job += " --use-conda "
+ if self.workflow.conda_prefix:
+ self.exec_job += " --conda-prefix " + self.workflow.conda_prefix + " "
# force threading.Lock() for cluster jobs
self.exec_job += " --force-use-threads "
@@ -417,6 +446,7 @@ class ClusterExecutor(RealExecutor):
if self.max_jobs_per_second:
self._limit_rate()
job.remove_existing_output()
+ job.download_remote_input()
super()._run(job, callback=callback, error_callback=error_callback)
logger.shellcmd(job.shellcmd)
@@ -427,15 +457,23 @@ class ClusterExecutor(RealExecutor):
return os.path.abspath(self._tmpdir)
def get_jobscript(self, job):
- return os.path.join(
- self.tmpdir,
- job.format_wildcards(self.jobname,
- rulename=job.rule.name,
- jobid=self.dag.jobid(job),
- cluster=self.cluster_wildcards(job)))
+ f = job.format_wildcards(self.jobname,
+ rulename=job.rule.name,
+ jobid=self.dag.jobid(job),
+ cluster=self.cluster_wildcards(job))
+ if os.path.sep in f:
+ raise WorkflowError("Path separator ({}) found in job name {}. "
+ "This is not supported.".format(
+ os.path.sep, f))
+
+ return os.path.join(self.tmpdir, f)
def spawn_jobscript(self, job, jobscript, **kwargs):
- wait_for_files = list(job.local_input) + [self.tmpdir]
+ wait_for_files = [self.tmpdir]
+ wait_for_files.extend(job.local_input)
+ wait_for_files.extend(f.local_file()
+ for f in job.remote_input if not f.stay_on_remote)
+
if job.shadow_dir:
wait_for_files.append(job.shadow_dir)
if self.workflow.use_conda and job.conda_env:
@@ -474,11 +512,16 @@ class ClusterExecutor(RealExecutor):
def cluster_wildcards(self, job):
return Wildcards(fromdict=self.cluster_params(job))
- def finish_job(self, job):
- super().finish_job(job, upload_remote=False)
+ def handle_job_success(self, job):
+ super().handle_job_success(job, upload_remote=False)
+ def handle_job_error(self, job):
+ # TODO what about removing empty remote dirs?? This cannot be decided
+ # on the cluster node.
+ super().handle_job_error(job)
-GenericClusterJob = namedtuple("GenericClusterJob", "job callback error_callback jobscript jobfinished jobfailed")
+
+GenericClusterJob = namedtuple("GenericClusterJob", "job jobid callback error_callback jobscript jobfinished jobfailed")
class GenericClusterExecutor(ClusterExecutor):
@@ -555,7 +598,7 @@ class GenericClusterExecutor(ClusterExecutor):
submit_callback(job)
with self.lock:
- self.active_jobs.append(GenericClusterJob(job, callback, error_callback, jobscript, jobfinished, jobfailed))
+ self.active_jobs.append(GenericClusterJob(job, ext_jobid, callback, error_callback, jobscript, jobfinished, jobfailed))
def _wait_for_jobs(self):
while True:
@@ -568,14 +611,12 @@ class GenericClusterExecutor(ClusterExecutor):
if os.path.exists(active_job.jobfinished):
os.remove(active_job.jobfinished)
os.remove(active_job.jobscript)
- self.finish_job(active_job.job)
active_job.callback(active_job.job)
elif os.path.exists(active_job.jobfailed):
os.remove(active_job.jobfailed)
os.remove(active_job.jobscript)
self.print_job_error(active_job.job)
- print_exception(ClusterJobException(active_job.job, self.dag.jobid(active_job.job),
- active_job.jobscript),
+ print_exception(ClusterJobException(active_job, self.dag.jobid(active_job.job)),
self.workflow.linemaps)
active_job.error_callback(active_job.job)
else:
@@ -583,7 +624,7 @@ class GenericClusterExecutor(ClusterExecutor):
time.sleep(1)
-SynchronousClusterJob = namedtuple("SynchronousClusterJob", "job callback error_callback jobscript process")
+SynchronousClusterJob = namedtuple("SynchronousClusterJob", "job jobid callback error_callback jobscript process")
class SynchronousClusterExecutor(ClusterExecutor):
@@ -647,7 +688,7 @@ class SynchronousClusterExecutor(ClusterExecutor):
submit_callback(job)
with self.lock:
- self.active_jobs.append(SynchronousClusterJob(job, callback, error_callback, jobscript, process))
+ self.active_jobs.append(SynchronousClusterJob(job, process.pid, callback, error_callback, jobscript, process))
def _wait_for_jobs(self):
while True:
@@ -664,14 +705,12 @@ class SynchronousClusterExecutor(ClusterExecutor):
elif exitcode == 0:
# job finished successfully
os.remove(active_job.jobscript)
- self.finish_job(active_job.job)
active_job.callback(active_job.job)
else:
# job failed
os.remove(active_job.jobscript)
self.print_job_error(active_job.job)
- print_exception(ClusterJobException(active_job.job, self.dag.jobid(active_job.job),
- active_job.jobscript),
+ print_exception(ClusterJobException(active_job, self.dag.jobid(active_job.job)),
self.workflow.linemaps)
active_job.error_callback(active_job.job)
time.sleep(1)
@@ -687,6 +726,7 @@ class DRMAAExecutor(ClusterExecutor):
quiet=False,
printshellcmds=False,
drmaa_args="",
+ drmaa_log_dir=None,
latency_wait=3,
benchmark_repeats=1,
cluster_config=None,
@@ -712,6 +752,7 @@ class DRMAAExecutor(ClusterExecutor):
raise WorkflowError("Error loading drmaa support:\n{}".format(e))
self.session = drmaa.Session()
self.drmaa_args = drmaa_args
+ self.drmaa_log_dir = drmaa_log_dir
self.session.initialize()
self.submitted = list()
@@ -742,10 +783,17 @@ class DRMAAExecutor(ClusterExecutor):
raise WorkflowError(str(e), rule=job.rule)
import drmaa
+
+ if self.drmaa_log_dir:
+ makedirs(self.drmaa_log_dir)
+
try:
jt = self.session.createJobTemplate()
jt.remoteCommand = jobscript
jt.nativeSpecification = drmaa_args
+ if self.drmaa_log_dir:
+ jt.outputPath = ":" + self.drmaa_log_dir
+ jt.errorPath = ":" + self.drmaa_log_dir
jt.jobName = os.path.basename(jobscript)
jobid = self.session.runJob(jt)
@@ -755,7 +803,7 @@ class DRMAAExecutor(ClusterExecutor):
self.workflow.linemaps)
error_callback(job)
return
- logger.info("Submitted DRMAA job (jobid {})".format(jobid))
+ logger.info("Submitted DRMAA job {} with external jobid {}.".format(self.dag.jobid(job), jobid))
self.submitted.append(jobid)
self.session.deleteJobTemplate(jt)
@@ -793,12 +841,11 @@ class DRMAAExecutor(ClusterExecutor):
# job exited
os.remove(active_job.jobscript)
if retval.hasExited and retval.exitStatus == 0:
- self.finish_job(active_job.job)
active_job.callback(active_job.job)
else:
self.print_job_error(active_job.job)
print_exception(
- ClusterJobException(active_job.job, self.dag.jobid(active_job.job), active_job.jobscript),
+ ClusterJobException(active_job, self.dag.jobid(active_job.job)),
self.workflow.linemaps)
active_job.error_callback(active_job.job)
time.sleep(1)
@@ -819,39 +866,64 @@ def change_working_directory(directory=None):
yield
-def run_wrapper(run, input, output, params, wildcards, threads, resources, log,
- version, benchmark, benchmark_repeats, rule, conda_env, linemaps, debug=False,
+def run_wrapper(job_rule, input, output, params, wildcards, threads, resources, log,
+ benchmark, benchmark_repeats, conda_env, linemaps, debug=False,
shadow_dir=None):
"""
Wrapper around the run method that handles exceptions and benchmarking.
Arguments
- run -- the run method
+ job_rule -- the ``job.rule`` member
input -- list of input files
output -- list of output files
wildcards -- so far processed wildcards
threads -- usable threads
log -- list of log files
- rule (str) -- rule name
shadow_dir -- optional shadow directory root
"""
+ # get shortcuts to job_rule members
+ run = job_rule.run_func
+ version = job_rule.version
+ rule = job_rule.name
+
if os.name == "posix" and debug:
sys.stdin = open('/dev/stdin')
- try:
- runs = 1 if benchmark is None else benchmark_repeats
- wallclock = []
- for i in range(runs):
- w = time.time()
- # execute the actual run method.
- with change_working_directory(shadow_dir):
- run(input, output, params, wildcards, threads, resources, log,
- version, rule, conda_env)
- w = time.time() - w
- wallclock.append(w)
+ if benchmark is not None:
+ from snakemake.benchmark import BenchmarkRecord, benchmarked, write_benchmark_records
+ try:
+ with change_working_directory(shadow_dir):
+ if benchmark:
+ bench_records = []
+ for i in range(benchmark_repeats):
+ # Determine whether to benchmark this process or do not
+ # benchmarking at all. We benchmark this process unless the
+ # execution is done through the ``shell:``, ``script:``, or
+ # ``wrapper:`` stanza.
+ is_sub = job_rule.shellcmd or job_rule.script or job_rule.wrapper
+ if is_sub:
+ # The benchmarking through ``benchmarked()`` is started
+ # in the execution of the shell fragment, script, wrapper
+ # etc, as the child PID is available there.
+ bench_record = BenchmarkRecord()
+ run(input, output, params, wildcards, threads, resources,
+ log, version, rule, conda_env, bench_record)
+ else:
+ # The benchmarking is started here as we have a run section
+ # and the generated Python function is executed in this
+ # process' thread.
+ with benchmarked() as bench_record:
+ run(input, output, params, wildcards, threads, resources,
+ log, version, rule, conda_env, bench_record)
+ # Store benchmark record for this iteration
+ bench_records.append(bench_record)
+ else:
+ run(input, output, params, wildcards, threads, resources,
+ log, version, rule, conda_env, None)
except (KeyboardInterrupt, SystemExit) as e:
- # re-raise the keyboard interrupt in order to record an error in the scheduler but ignore it
+ # Re-raise the keyboard interrupt in order to record an error in the
+ # scheduler but ignore it
raise e
except (Exception, BaseException) as ex:
log_verbose_traceback(ex)
@@ -864,9 +936,6 @@ def run_wrapper(run, input, output, params, wildcards, threads, resources, log,
if benchmark is not None:
try:
- with open(benchmark, "w") as f:
- print("s", "h:m:s", sep="\t", file=f)
- for t in wallclock:
- print(t, str(datetime.timedelta(seconds=t)), sep="\t", file=f)
+ write_benchmark_records(bench_records, benchmark)
except (Exception, BaseException) as ex:
raise WorkflowError(ex)
diff --git a/snakemake/io.py b/snakemake/io.py
index 8c1b5de..53e4aaa 100644
--- a/snakemake/io.py
+++ b/snakemake/io.py
@@ -123,6 +123,10 @@ class _IOFile(str):
return get_flag_value(self._file, "remote_object").keep_local
@property
+ def should_stay_on_remote(self):
+ return get_flag_value(self._file, "remote_object").stay_on_remote
+
+ @property
def remote_object(self):
self.update_remote_filepath()
return get_flag_value(self._file, "remote_object")
@@ -213,8 +217,9 @@ class _IOFile(str):
def download_from_remote(self):
if self.is_remote and self.remote_object.exists():
- logger.info("Downloading from remote: {}".format(self.file))
- self.remote_object.download()
+ if not self.should_stay_on_remote:
+ logger.info("Downloading from remote: {}".format(self.file))
+ self.remote_object.download()
else:
raise RemoteFileException(
"The file to be downloaded does not seem to exist remotely.")
@@ -248,7 +253,7 @@ class _IOFile(str):
lchmod(self.file, mode)
def remove(self, remove_non_empty_dir=False):
- remove(self.file, remove_non_empty_dir=remove_non_empty_dir)
+ remove(self, remove_non_empty_dir=remove_non_empty_dir)
def touch(self, times=None):
""" times must be 2-tuple: (atime, mtime) """
@@ -360,7 +365,10 @@ _wildcard_regex = re.compile(
def wait_for_files(files, latency_wait=3):
"""Wait for given files to be present in filesystem."""
files = list(files)
- get_missing = lambda: [f for f in files if not os.path.exists(f)]
+ get_missing = lambda: [
+ f for f in files if not
+ (f.exists_remote if (isinstance(f, _IOFile) and f.is_remote and f.should_stay_on_remote) else os.path.exists(f))
+ ]
missing = get_missing()
if missing:
logger.info("Waiting at most {} seconds for missing files.".format(
@@ -387,7 +395,10 @@ def contains_wildcard_constraints(pattern):
def remove(file, remove_non_empty_dir=False):
- if os.path.isdir(file) and not os.path.islink(file):
+ if file.is_remote and file.should_stay_on_remote:
+ if file.exists_remote:
+ file.remote_object.remove()
+ elif os.path.isdir(file) and not os.path.islink(file):
if remove_non_empty_dir:
shutil.rmtree(file)
else:
@@ -497,6 +508,7 @@ def get_flag_value(value, flag_type):
else:
return None
+
def ancient(value):
"""
A flag for an input file that shall be considered ancient; i.e. its timestamp shall have no effect on which jobs to run.
@@ -555,9 +567,11 @@ def dynamic(value):
def touch(value):
return flag(value, "touch")
+
def unpack(value):
return flag(value, "unpack")
+
def expand(*args, **wildcards):
"""
Expand wildcards in given filepatterns.
@@ -655,12 +669,12 @@ def update_wildcard_constraints(pattern,
return match.group(0)
examined_names.add(name)
# Don't override if constraint already set
- if not constraint is None:
+ if constraint is not None:
if name in wildcard_constraints:
raise ValueError("Wildcard {} is constrained by both the rule and the file pattern. Consider removing one of the constraints.")
return match.group(0)
# Only update if a new constraint has actually been set
- elif not newconstraint is None:
+ elif newconstraint is not None:
return "{{{},{}}}".format(name, newconstraint)
else:
return match.group(0)
@@ -675,7 +689,6 @@ def update_wildcard_constraints(pattern,
return updated
-
# TODO rewrite Namedlist!
class Namedlist(list):
"""
@@ -751,7 +764,8 @@ class Namedlist(list):
def allitems(self):
next = 0
for name, index in sorted(self._names.items(),
- key=lambda item: item[1][0]):
+ key=lambda item: (item[1][0], item[1][0] + 1 if item[1][1] is None else item[1][1])):
+
start, end = index
if end is None:
end = start + 1
diff --git a/snakemake/jobs.py b/snakemake/jobs.py
index 0cf824d..266abed 100644
--- a/snakemake/jobs.py
+++ b/snakemake/jobs.py
@@ -3,17 +3,19 @@ __copyright__ = "Copyright 2015, Johannes Köster"
__email__ = "koester at jimmy.harvard.edu"
__license__ = "MIT"
+import hashlib
import os
import sys
import base64
import tempfile
import subprocess
-import json
from collections import defaultdict
from itertools import chain
from functools import partial
from operator import attrgetter
+from urllib.request import urlopen
+from urllib.parse import urlparse
from snakemake.io import IOFile, Wildcards, Resources, _IOFile, is_flagged, contains_wildcard, lstat
from snakemake.utils import format, listfiles
@@ -31,7 +33,7 @@ def jobfiles(jobs, type):
class Job:
HIGHEST_PRIORITY = sys.maxsize
- __slots__ = ["rule", "dag", "targetfile", "wildcards_dict", "wildcards",
+ __slots__ = ["rule", "dag", "wildcards_dict", "wildcards",
"_format_wildcards", "input", "dependencies", "output",
"_params", "_log", "_benchmark", "_resources",
"_conda_env_file", "_conda_env", "shadow_dir", "_inputsize",
@@ -39,12 +41,11 @@ class Job:
"temp_output", "protected_output", "touch_output",
"subworkflow_input", "_hash"]
- def __init__(self, rule, dag, targetfile=None, format_wildcards=None):
+ def __init__(self, rule, dag, wildcards_dict=None, format_wildcards=None):
self.rule = rule
self.dag = dag
- self.targetfile = targetfile
- self.wildcards_dict = self.rule.get_wildcards(targetfile)
+ self.wildcards_dict = wildcards_dict
self.wildcards = Wildcards(fromdict=self.wildcards_dict)
self._format_wildcards = (self.wildcards if format_wildcards is None
else Wildcards(fromdict=format_wildcards))
@@ -84,6 +85,8 @@ class Job:
self.dynamic_input.add(f)
if f_ in self.rule.subworkflow_input:
self.subworkflow_input[f] = self.rule.subworkflow_input[f_]
+ elif "subworkflow" in f.flags:
+ self.subworkflow_input[f] = f.flags["subworkflow"]
self._hash = self.rule.__hash__()
for o in self.output:
self._hash ^= o.__hash__()
@@ -92,7 +95,7 @@ class Job:
"""Check if job is valid"""
# these properties have to work in dry-run as well. Hence we check them here:
resources = self.rule.expand_resources(self.wildcards_dict, self.input)
- self.rule.expand_params(self.wildcards_dict, self.input, resources)
+ self.rule.expand_params(self.wildcards_dict, self.input, self.output, resources)
self.rule.expand_benchmark(self.wildcards_dict)
self.rule.expand_log(self.wildcards_dict)
@@ -116,6 +119,7 @@ class Job:
if self._params is None:
self._params = self.rule.expand_params(self.wildcards_dict,
self.input,
+ self.output,
self.resources)
return self._params
@@ -141,30 +145,30 @@ class Job:
@property
def conda_env_file(self):
if self._conda_env_file is None:
- self._conda_env_file = self.rule.expand_conda_env(self.wildcards_dict)
+ expanded_env = self.rule.expand_conda_env(self.wildcards_dict)
+ scheme, _, path, *_ = urlparse(expanded_env)
+ # Normalize 'file:///my/path.yml' to '/my/path.yml'
+ if scheme == 'file' or not scheme:
+ self._conda_env_file = path
+ else:
+ self._conda_env_file = expanded_env
return self._conda_env_file
@property
def conda_env(self):
if self.conda_env_file:
if self._conda_env is None:
- raise ValueError("create_conda_env() must be called before calling conda_env")
- return self._conda_env
+ self._conda_env = self.dag.conda_envs.get(self.conda_env_file)
+ logger.debug("Accessing conda environment {}.".format(self._conda_env))
+ if self._conda_env is None:
+ raise ValueError("Conda environment {} not found in DAG.".format(self.conda_env_file))
+ return self._conda_env.path
return None
- def create_conda_env(self):
- """Create conda environment if specified."""
- if self.conda_env_file:
- try:
- self._conda_env = conda.create_env(self)
- except CreateCondaEnvironmentException as e:
- raise WorkflowError(e, rule=self.rule)
-
def archive_conda_env(self):
"""Archive a conda environment into a custom local channel."""
if self.conda_env_file:
- self.create_conda_env()
- return conda.archive_env(self)
+ return self.conda_env.create_archive()
return None
@property
@@ -448,6 +452,10 @@ class Job:
#No file == no problem
pass
+ def download_remote_input(self):
+ for f in self.files_to_download:
+ f.download_from_remote()
+
def prepare(self):
"""
Prepare execution of job.
@@ -471,8 +479,7 @@ class Job:
for f, f_ in zip(self.output, self.rule.output):
f.prepare()
- for f in self.files_to_download:
- f.download_from_remote()
+ self.download_remote_input()
for f in self.log:
f.prepare()
@@ -521,8 +528,11 @@ class Job:
""" Cleanup output files. """
to_remove = [f for f in self.expanded_output if f.exists]
- to_remove.extend([f for f in self.remote_input if f.exists])
- to_remove.extend([f for f in self.remote_output if f.exists_local])
+ to_remove.extend([f for f in self.remote_input if f.exists_local])
+ to_remove.extend([
+ f for f in self.remote_output
+ if (f.exists_remote if (f.is_remote and f.should_stay_on_remote) else f.exists_local)
+ ])
if to_remove:
logger.info("Removing output files of failed job {}"
" since they might be corrupted:\n{}".format(
@@ -535,7 +545,7 @@ class Job:
@property
def empty_remote_dirs(self):
for f in (set(self.output) | set(self.input)):
- if f.is_remote:
+ if f.is_remote and not f.should_stay_on_remote:
if os.path.exists(os.path.dirname(f)) and not len(os.listdir(
os.path.dirname(f))):
yield os.path.dirname(f)
@@ -582,7 +592,9 @@ class Job:
"local": self.dag.workflow.is_local(self.rule),
"input": self.input,
"output": self.output,
+ "wildcards": self.wildcards,
"params": params,
+ "log": self.log,
"threads": self.threads,
"resources": resources,
"jobid": self.dag.jobid(self)
diff --git a/snakemake/logging.py b/snakemake/logging.py
index 0dfcb0b..79fef1a 100644
--- a/snakemake/logging.py
+++ b/snakemake/logging.py
@@ -86,6 +86,7 @@ class Logger:
self.stream_handler = None
self.printshellcmds = False
self.printreason = False
+ self.debug_dag = False
self.quiet = False
self.logfile = None
self.last_msg_was_job_info = False
@@ -148,6 +149,9 @@ class Logger:
msg["level"] = "job_info"
self.handler(msg)
+ def dag_debug(self, msg):
+ self.handler(dict(level="dag_debug", **msg))
+
def shellcmd(self, msg):
if msg is not None:
self.handler(dict(level="shellcmd", msg=msg))
@@ -212,7 +216,7 @@ class Logger:
if not self.last_msg_was_job_info:
self.logger.info("")
if msg["msg"] is not None:
- self.logger.info(msg["msg"])
+ self.logger.info("Job {}: {}".format(msg["jobid"], msg["msg"]))
if self.printreason:
self.logger.info("Reason: {}".format(msg["reason"]))
else:
@@ -252,6 +256,14 @@ class Logger:
self.logger.info(" " + msg["docstring"])
elif level == "d3dag":
print(json.dumps({"nodes": msg["nodes"], "links": msg["edges"]}))
+ elif level == "dag_debug":
+ if self.debug_dag:
+ job = msg["job"]
+ self.logger.warning(
+ "{status} job {name}\n\twildcards: {wc}".format(
+ status=msg["status"],
+ name=job.rule.name,
+ wc=format_wildcards(job.wildcards)))
self.last_msg_was_job_info = False
@@ -277,6 +289,7 @@ def setup_logger(handler=None,
quiet=False,
printshellcmds=False,
printreason=False,
+ debug_dag=False,
nocolor=False,
stdout=False,
debug=False,
@@ -301,3 +314,4 @@ def setup_logger(handler=None,
logger.quiet = quiet
logger.printshellcmds = printshellcmds
logger.printreason = printreason
+ logger.debug_dag = debug_dag
diff --git a/snakemake/parser.py b/snakemake/parser.py
index be2e879..cf4784d 100644
--- a/snakemake/parser.py
+++ b/snakemake/parser.py
@@ -423,7 +423,7 @@ class Run(RuleKeywordState):
yield "@workflow.run"
yield "\n"
yield ("def __rule_{rulename}(input, output, params, wildcards, threads, "
- "resources, log, version, rule, conda_env):".format(
+ "resources, log, version, rule, conda_env, bench_record):".format(
rulename=self.rulename if self.rulename is not None else self.snakefile.rulecount))
def end(self):
@@ -509,6 +509,9 @@ class Shell(AbstractCmd):
start_func = "@workflow.shellcmd"
end_func = "shell"
+ def args(self):
+ yield ", bench_record=bench_record"
+
class Script(AbstractCmd):
start_func = "@workflow.script"
@@ -519,7 +522,7 @@ class Script(AbstractCmd):
yield ', "{}"'.format(
os.path.abspath(os.path.dirname(self.snakefile.path)))
# other args
- yield ", input, output, params, wildcards, threads, resources, log, config, rule, conda_env"
+ yield ", input, output, params, wildcards, threads, resources, log, config, rule, conda_env, bench_record"
class Wrapper(Script):
@@ -527,7 +530,7 @@ class Wrapper(Script):
end_func = "wrapper"
def args(self):
- yield ", input, output, params, wildcards, threads, resources, log, config, rule, conda_env, workflow.wrapper_prefix"
+ yield ", input, output, params, wildcards, threads, resources, log, config, rule, conda_env, bench_record, workflow.wrapper_prefix"
class Rule(GlobalKeywordState):
diff --git a/snakemake/persistence.py b/snakemake/persistence.py
index 5fd903f..bddb57b 100644
--- a/snakemake/persistence.py
+++ b/snakemake/persistence.py
@@ -18,7 +18,7 @@ from snakemake.utils import listfiles
class Persistence:
- def __init__(self, nolock=False, dag=None, warn_only=False):
+ def __init__(self, nolock=False, dag=None, conda_prefix=None, warn_only=False):
self.path = os.path.abspath(".snakemake")
if not os.path.exists(self.path):
os.mkdir(self.path)
@@ -38,16 +38,21 @@ class Persistence:
self._params_path = os.path.join(self.path, "params_tracking")
self._shellcmd_path = os.path.join(self.path, "shellcmd_tracking")
self.shadow_path = os.path.join(self.path, "shadow")
- self.conda_env_path = os.path.join(self.path, "conda")
self.conda_env_archive_path = os.path.join(self.path, "conda-archive")
for d in (self._incomplete_path, self._version_path, self._code_path,
self._rule_path, self._input_path, self._log_path, self._params_path,
- self._shellcmd_path, self.shadow_path, self.conda_env_path,
- self.conda_env_archive_path):
+ self._shellcmd_path, self.shadow_path, self.conda_env_archive_path):
if not os.path.exists(d):
os.mkdir(d)
+ if conda_prefix is None:
+ self.conda_env_path = os.path.join(self.path, "conda")
+ else:
+ self.conda_env_path = os.path.abspath(os.path.expanduser(conda_prefix))
+
+ os.makedirs(self.conda_env_path, exist_ok=True)
+
if nolock:
self.lock = self.noop
self.unlock = self.noop
diff --git a/snakemake/remote/FTP.py b/snakemake/remote/FTP.py
index 424595c..2826952 100644
--- a/snakemake/remote/FTP.py
+++ b/snakemake/remote/FTP.py
@@ -3,14 +3,16 @@ __copyright__ = "Copyright 2015, Christopher Tomkins-Tinch"
__email__ = "tomkinsc at broadinstitute.org"
__license__ = "MIT"
-import os, re, ftplib
-from itertools import product, chain
+import os
+import re
+import ftplib
+import collections
+from itertools import chain
from contextlib import contextmanager
# module-specific
from snakemake.remote import AbstractRemoteProvider, DomainObject
from snakemake.exceptions import FTPFileException, WorkflowError
-import snakemake.io
try:
# third-party modules
@@ -20,28 +22,72 @@ except ImportError as e:
raise WorkflowError("The Python 3 package 'ftputil' " +
"must be installed to use SFTP remote() file functionality. %s" % e.msg)
+
class RemoteProvider(AbstractRemoteProvider):
- def __init__(self, *args, **kwargs):
- super(RemoteProvider, self).__init__(*args, **kwargs)
+ def __init__(self, *args, stay_on_remote=False, immediate_close=False, **kwargs):
+ super(RemoteProvider, self).__init__(*args, stay_on_remote=stay_on_remote, **kwargs)
+
+ self.immediate_close = immediate_close
+
+ @property
+ def default_protocol(self):
+ """The protocol that is prepended to the path when no protocol is specified."""
+ return 'ftp://'
+
+ @property
+ def available_protocols(self):
+ """List of valid protocols for this remote provider."""
+ return ['ftp://', 'ftps://']
+
+ def remote(self, value, *args, encrypt_data_channel=None, immediate_close=None, **kwargs):
+ if isinstance(value, str):
+ values = [value]
+ elif isinstance(value, collections.Iterable):
+ values = value
+ else:
+ raise TypeError('Invalid type ({}) passed to remote: {}'.format(type(value), value))
+
+ for i, file in enumerate(values):
+ match = re.match('^(ftps?)://.+', file)
+ if match:
+ protocol, = match.groups()
+ if protocol == 'ftps' and encrypt_data_channel:
+ raise SyntaxError('encrypt_data_channel=False cannot be used with a ftps:// url')
+ if protocol == 'ftp' and encrypt_data_channel not in [None, False]:
+ raise SyntaxError('encrypt_data_channel=Trie cannot be used with a ftp:// url')
+ else:
+ if encrypt_data_channel:
+ values[i] = 'ftps://' + file
+ else:
+ values[i] = 'ftp://' + file
+
+ should_close = immediate_close if immediate_close else self.immediate_close
+ return super(RemoteProvider, self).remote(values, *args, encrypt_data_channel=encrypt_data_channel, immediate_close=should_close, **kwargs)
+
class RemoteObject(DomainObject):
""" This is a class to interact with an FTP server.
"""
- def __init__(self, *args, keep_local=False, provider=None, encrypt_data_channel=False, **kwargs):
+ def __init__(self, *args, keep_local=False, provider=None, encrypt_data_channel=False, immediate_close=False, **kwargs):
super(RemoteObject, self).__init__(*args, keep_local=keep_local, provider=provider, **kwargs)
self.encrypt_data_channel = encrypt_data_channel
+ self.immediate_close = immediate_close
def close(self):
- if hasattr(self, "conn") and isinstance(self.conn, ftputil.FTPHost):
- self.conn.close()
+ if hasattr(self, "conn") and isinstance(self.conn, ftputil.FTPHost) and not self.immediate_close:
+ try:
+ self.conn.keep_alive()
+ self.conn.close()
+ except:
+ pass
# === Implementations of abstract class members ===
@contextmanager #makes this a context manager. after 'yield' is __exit__()
def ftpc(self):
- if not hasattr(self, "conn") or (hasattr(self, "conn") and not isinstance(self.conn, ftputil.FTPHost)):
+ if (not hasattr(self, "conn") or (hasattr(self, "conn") and not isinstance(self.conn, ftputil.FTPHost))) or self.immediate_close:
# if args have been provided to remote(), use them over those given to RemoteProvider()
args_to_use = self.provider.args
if len(self.args):
@@ -70,8 +116,22 @@ class RemoteObject(DomainObject):
encrypt_data_channel= kwargs_to_use["encrypt_data_channel"],
debug_level=None)
- self.conn = ftputil.FTPHost(kwargs_to_use["host"], kwargs_to_use["username"], kwargs_to_use["password"], session_factory=ftp_session_factory)
- yield self.conn
+ conn = ftputil.FTPHost(kwargs_to_use["host"], kwargs_to_use["username"], kwargs_to_use["password"], session_factory=ftp_session_factory)
+ if self.immediate_close:
+ yield conn
+ else:
+ self.conn = conn
+ yield self.conn
+ elif not self.immediate_close:
+ yield self.conn
+
+ # after returning from the context manager, close the connection if the scope is local
+ if self.immediate_close:
+ try:
+ conn.keep_alive()
+ conn.close()
+ except:
+ pass
def exists(self):
if self._matched_address:
@@ -79,7 +139,7 @@ class RemoteObject(DomainObject):
return ftpc.path.exists(self.remote_path)
return False
else:
- raise FTPFileException("The file cannot be parsed as an FTP path in form 'host:port/abs/path/to/file': %s" % self.file())
+ raise FTPFileException("The file cannot be parsed as an FTP path in form 'host:port/abs/path/to/file': %s" % self.local_file())
def mtime(self):
if self.exists():
@@ -91,7 +151,7 @@ class RemoteObject(DomainObject):
pass
return ftpc.path.getmtime(self.remote_path)
else:
- raise FTPFileException("The file does not seem to exist remotely: %s" % self.file())
+ raise FTPFileException("The file does not seem to exist remotely: %s" % self.local_file())
def size(self):
if self.exists():
@@ -112,8 +172,9 @@ class RemoteObject(DomainObject):
except:
pass
ftpc.download(source=self.remote_path, target=self.local_path)
+ os.sync() # ensure flush to disk
else:
- raise FTPFileException("The file does not seem to exist remotely: %s" % self.file())
+ raise FTPFileException("The file does not seem to exist remotely: %s" % self.local_file())
def upload(self):
with self.ftpc() as ftpc:
diff --git a/snakemake/remote/GS.py b/snakemake/remote/GS.py
index fb9c3e5..99ac049 100644
--- a/snakemake/remote/GS.py
+++ b/snakemake/remote/GS.py
@@ -5,18 +5,19 @@ __license__ = "MIT"
# module-specific
from snakemake.remote.S3 import RemoteObject, RemoteProvider as S3RemoteProvider
-from snakemake.exceptions import WorkflowError
-try:
- # third-party modules
- import boto
- from boto.s3.key import Key
- from filechunkio import FileChunkIO
-except ImportError as e:
- raise WorkflowError("The Python 3 packages 'boto' and 'filechunkio' " +
- "need to be installed to use S3 remote() file functionality. %s" % e.msg)
class RemoteProvider(S3RemoteProvider):
- def __init__(self, *args, **kwargs):
+ def __init__(self, *args, stay_on_remote=False, **kwargs):
kwargs["host"] = "storage.googleapis.com"
- super(RemoteProvider, self).__init__(*args, **kwargs)
+ super(RemoteProvider, self).__init__(*args, stay_on_remote=stay_on_remote, **kwargs)
+
+ @property
+ def default_protocol(self):
+ """The protocol that is prepended to the path when no protocol is specified."""
+ return 'gs://'
+
+ @property
+ def available_protocols(self):
+ """List of valid protocols for this remote provider."""
+ return ['s3://', 'gs://']
diff --git a/snakemake/remote/HTTP.py b/snakemake/remote/HTTP.py
index d9941cd..c084c5d 100644
--- a/snakemake/remote/HTTP.py
+++ b/snakemake/remote/HTTP.py
@@ -3,15 +3,15 @@ __copyright__ = "Copyright 2015, Christopher Tomkins-Tinch"
__email__ = "tomkinsc at broadinstitute.org"
__license__ = "MIT"
-import os, re, http.client
+import os
+import re
+import collections
import email.utils
-#from itertools import product, chain
from contextlib import contextmanager
# module-specific
from snakemake.remote import AbstractRemoteProvider, DomainObject
from snakemake.exceptions import HTTPFileException, WorkflowError
-import snakemake.io
try:
# third-party modules
@@ -20,18 +20,52 @@ except ImportError as e:
raise WorkflowError("The Python 3 package 'requests' " +
"must be installed to use HTTP(S) remote() file functionality. %s" % e.msg)
+
class RemoteProvider(AbstractRemoteProvider):
- def __init__(self, *args, **kwargs):
- super(RemoteProvider, self).__init__(*args, **kwargs)
+ def __init__(self, *args, stay_on_remote=False, **kwargs):
+ super(RemoteProvider, self).__init__(*args, stay_on_remote=stay_on_remote, **kwargs)
+
+ @property
+ def default_protocol(self):
+ """The protocol that is prepended to the path when no protocol is specified."""
+ return 'https://'
+
+ @property
+ def available_protocols(self):
+ """List of valid protocols for this remote provider."""
+ return ['http://', 'https://']
+
+ def remote(self, value, *args, insecure=None, **kwargs):
+ if isinstance(value, str):
+ values = [value]
+ elif isinstance(value, collections.Iterable):
+ values = value
+ else:
+ raise TypeError('Invalid type ({}) passed to remote: {}'.format(type(value), value))
+
+ for i, file in enumerate(values):
+ match = re.match('^(https?)://.+', file)
+ if match:
+ protocol, = match.groups()
+ if protocol == 'https' and insecure:
+ raise SyntaxError('insecure=True cannot be used with a https:// url')
+ if protocol == 'http' and insecure not in [None, False]:
+ raise SyntaxError('insecure=False cannot be used with a http:// url')
+ else:
+ if insecure:
+ values[i] = 'http://' + file
+ else:
+ values[i] = 'https://' + file
+
+ return super(RemoteProvider, self).remote(values, *args, **kwargs)
+
class RemoteObject(DomainObject):
""" This is a class to interact with an HTTP server.
"""
- def __init__(self, *args, keep_local=False, provider=None, insecure=False, additional_request_string="", **kwargs):
+ def __init__(self, *args, keep_local=False, provider=None, additional_request_string="", **kwargs):
super(RemoteObject, self).__init__(*args, keep_local=keep_local, provider=provider, **kwargs)
-
- self.insecure = insecure
self.additional_request_string = additional_request_string
# === Implementations of abstract class members ===
@@ -68,13 +102,7 @@ class RemoteObject(DomainObject):
del kwargs_to_use["username"]
del kwargs_to_use["password"]
- url = self._iofile._file + self.additional_request_string
- # default to HTTPS
- if not self.insecure:
- protocol = "https://"
- else:
- protocol = "http://"
- url = protocol + url
+ url = self.remote_file() + self.additional_request_string
if verb.upper() == "GET":
r = requests.get(url, *args_to_use, stream=stream, **kwargs_to_use)
@@ -93,7 +121,7 @@ class RemoteObject(DomainObject):
return httpr.status_code == requests.codes.ok
return False
else:
- raise HTTPFileException("The file cannot be parsed as an HTTP path in form 'host:port/abs/path/to/file': %s" % self.file())
+ raise HTTPFileException("The file cannot be parsed as an HTTP path in form 'host:port/abs/path/to/file': %s" % self.local_file())
def mtime(self):
if self.exists():
@@ -106,7 +134,7 @@ class RemoteObject(DomainObject):
return epochTime
else:
- raise HTTPFileException("The file does not seem to exist remotely: %s" % self.file())
+ raise HTTPFileException("The file does not seem to exist remotely: %s" % self.remote_file())
def size(self):
if self.exists():
@@ -129,8 +157,9 @@ class RemoteObject(DomainObject):
for chunk in httpr.iter_content(chunk_size=1024):
if chunk: # filter out keep-alives
f.write(chunk)
+ os.sync() # ensure flush to disk
else:
- raise HTTPFileException("The file does not seem to exist remotely: %s" % self.file())
+ raise HTTPFileException("The file does not seem to exist remotely: %s" % self.remote_file())
def upload(self):
raise HTTPFileException("Upload is not permitted for the HTTP remote provider. Is an output set to HTTP.remote()?")
diff --git a/snakemake/remote/NCBI.py b/snakemake/remote/NCBI.py
new file mode 100644
index 0000000..989b73d
--- /dev/null
+++ b/snakemake/remote/NCBI.py
@@ -0,0 +1,574 @@
+__author__ = "Christopher Tomkins-Tinch"
+__copyright__ = "Copyright 2017, Christopher Tomkins-Tinch"
+__email__ = "tomkinsc at broadinstitute.org"
+__license__ = "MIT"
+
+# built-ins
+import time
+import os
+import re
+import json
+import logging
+import xml.etree.ElementTree as ET
+
+# module-specific
+from snakemake.remote import AbstractRemoteObject, AbstractRemoteProvider
+from snakemake.exceptions import WorkflowError, NCBIFileException
+from snakemake.logging import logger
+
+try:
+ # third-party modules
+ from Bio import Entrez
+except ImportError as e:
+ raise WorkflowError("The Python package 'biopython' needs to be installed to use NCBI Entrez remote() file functionality. %s" % e.msg)
+
+
+class RemoteProvider(AbstractRemoteProvider):
+ def __init__(self, *args, stay_on_remote=False, email=None, **kwargs):
+ super(RemoteProvider, self).__init__(*args, stay_on_remote=stay_on_remote, email=email, **kwargs)
+ self._ncbi = NCBIHelper(*args, email=email, **kwargs)
+
+ def remote_interface(self):
+ return self._ncbi
+
+ @property
+ def default_protocol(self):
+ """The protocol that is prepended to the path when no protocol is specified."""
+ return 'ncbi://'
+
+ @property
+ def available_protocols(self):
+ """List of valid protocols for this remote provider."""
+ return ['ncbi://']
+
+ def search(self, query, *args, db="nuccore", idtype="acc", retmode="json", **kwargs):
+ return list(self._ncbi.search(query, *args, db=db, idtype=idtype, retmode=retmode, **kwargs))
+
+
+class RemoteObject(AbstractRemoteObject):
+ """ This is a class to interact with NCBI / GenBank.
+ """
+
+ def __init__(self, *args, keep_local=False, stay_on_remote=False, provider=None, email=None, db=None, rettype=None, retmode=None, **kwargs):
+ super(RemoteObject, self).__init__(*args, keep_local=keep_local, stay_on_remote=stay_on_remote, provider=provider, email=email, db=db, rettype=rettype, retmode=retmode, **kwargs)
+ if provider:
+ self._ncbi = provider.remote_interface()
+ else:
+ self._ncbi = NCBIHelper(*args, email=email, **kwargs)
+
+ if db and not self._ncbi.is_valid_db(db):
+ raise NCBIFileException("DB specified is not valid. Options include: {dbs}".format(dbs=", ".join(self._ncbi.valid_dbs)))
+ else:
+ self.db = db
+
+ self.rettype = rettype
+ self.retmode = retmode
+ self.kwargs = kwargs
+
+ # === Implementations of abstract class members ===
+
+ def exists(self):
+ if not self.retmode or not self.rettype:
+ likely_request_options = self._ncbi.guess_db_options_for_extension(self.file_ext, db=self.db, rettype=self.rettype, retmode=self.retmode)
+ self.db = likely_request_options["db"]
+ self.retmode = likely_request_options["retmode"]
+ self.rettype = likely_request_options["rettype"]
+ return self._ncbi.exists(self.accession, db=self.db)
+
+ def mtime(self):
+ if self.exists():
+ return self._ncbi.mtime(self.accession, db=self.db)
+ else:
+ raise NCBIFileException("The record does not seem to exist remotely: %s" % self.accession)
+
+ def size(self):
+ if self.exists():
+ return self._ncbi.size(self.accession, db=self.db)
+ else:
+ return self._iofile.size_local
+
+ def download(self):
+ if self.exists():
+ self._ncbi.fetch_from_ncbi([self.accession], os.path.dirname(self.accession), rettype=self.rettype, retmode=self.retmode, file_ext=self.file_ext, db=self.db, **self.kwargs)
+ else:
+ raise NCBIFileException("The record does not seem to exist remotely: %s" % self.accession)
+
+ def upload(self):
+ raise NCBIFileException("Upload is not permitted for the NCBI remote provider. Is an output set to NCBI.RemoteProvider.remote()?")
+
+ @property
+ def list(self):
+ raise NCBIFileException("The NCBI Remote Provider does not currently support list-based operations like glob_wildcards().")
+
+ @property
+ def accession(self):
+ accession, version, file_ext = self._ncbi.parse_accession_str(self.local_file())
+ return accession + "." + version
+
+ @property
+ def file_ext(self):
+ accession, version, file_ext = self._ncbi.parse_accession_str(self.local_file())
+ return file_ext
+
+ @property
+ def version(self):
+ accession, version, file_ext = self._ncbi.parse_accession_str(self.local_file())
+ return version
+
+class NCBIHelper(object):
+ def __init__(self, *args, email=None, **kwargs):
+ if not email:
+ raise NCBIFileException("An e-mail address must be provided to either the remote file or the RemoteProvider() as email=<your_address>. The NCBI requires e-mail addresses for queries.")
+
+ self.email = email
+ self.entrez = Entrez
+ self.entrez.email = self.email
+ self.entrez.tool = "Snakemake"
+
+ # valid NCBI Entrez efetch options
+ # via https://www.ncbi.nlm.nih.gov/books/NBK25499/table/chapter4.T._valid_values_of__retmode_and/?report=objectonly
+ self.efetch_options = {
+ "bioproject": [
+ {"rettype":"xml", "retmode":"xml", "ext":"xml"}
+ ],
+ "biosample": [
+ {"rettype":"full", "retmode":"xml", "ext":"xml"},
+ {"rettype":"full", "retmode":"text", "ext":"txt"}
+ ],
+ "biosystems": [
+ {"rettype":"xml", "retmode":"xml", "ext":"xml"}
+ ],
+ "gds": [
+ {"rettype":"summary", "retmode":"text", "ext":"txt"}
+ ],
+ "gene": [
+ {"rettype":"null", "retmode":"asn.1", "ext":"asn1"},
+ {"rettype":"null", "retmode":"xml", "ext":"xml"},
+ {"rettype":"gene_table", "retmode":"text", "ext":"gene_table"}
+ ],
+ "homologene": [
+ {"rettype":"null", "retmode":"asn.1", "ext":"asn1"},
+ {"rettype":"null", "retmode":"xml", "ext":"xml"},
+ {"rettype":"alignmentscores", "retmode":"text", "ext":"alignmentscores"},
+ {"rettype":"fasta", "retmode":"text", "ext":"fasta"},
+ {"rettype":"homologene", "retmode":"text", "ext":"homologene"}
+ ],
+ "mesh": [
+ {"rettype":"full", "retmode":"text", "ext":"txt"}
+ ],
+ "nlmcatalog": [
+ {"rettype":"null", "retmode":"text", "ext":"txt"},
+ {"rettype":"null", "retmode":"xml", "ext":"xml"}
+ ],
+ "nuccore": [
+ {"rettype":"null", "retmode":"text", "ext":"txt"},
+ {"rettype":"null", "retmode":"asn.1", "ext":"asn1"},
+ {"rettype":"native", "retmode":"xml", "ext":"xml"},
+ {"rettype":"acc", "retmode":"text", "ext":"acc"},
+ {"rettype":"fasta", "retmode":"text", "ext":"fasta"},
+ {"rettype":"fasta", "retmode":"xml", "ext":"fasta.xml"},
+ {"rettype":"seqid", "retmode":"text", "ext":"seqid"},
+ {"rettype":"gb", "retmode":"text", "ext":"gb"},
+ {"rettype":"gb", "retmode":"xml", "ext":"gb.xml"},
+ {"rettype":"gbc", "retmode":"xml", "ext":"gbc"},
+ {"rettype":"ft", "retmode":"text", "ext":"ft"},
+ {"rettype":"gbwithparts", "retmode":"text", "ext":"gbwithparts"},
+ {"rettype":"fasta_cds_na", "retmode":"text", "ext":"fasta_cds_na"},
+ {"rettype":"fasta_cds_aa", "retmode":"text", "ext":"fasta_cds_aa"}
+ ],
+ "nucest": [
+ {"rettype":"null", "retmode":"text", "ext":"txt"},
+ {"rettype":"null", "retmode":"asn.1", "ext":"asn1"},
+ {"rettype":"native", "retmode":"xml", "ext":"xml"},
+ {"rettype":"acc", "retmode":"text", "ext":"acc"},
+ {"rettype":"fasta", "retmode":"text", "ext":"fasta"},
+ {"rettype":"fasta", "retmode":"xml", "ext":"fasta.xml"},
+ {"rettype":"seqid", "retmode":"text", "ext":"seqid"},
+ {"rettype":"gb", "retmode":"text", "ext":"gb"},
+ {"rettype":"gb", "retmode":"xml", "ext":"gb.xml"},
+ {"rettype":"gbc", "retmode":"xml", "ext":"gbc"},
+ {"rettype":"est", "retmode":"text", "ext":"est"}
+ ],
+ "nucgss": [
+ {"rettype":"null", "retmode":"text", "ext":"txt"},
+ {"rettype":"null", "retmode":"asn.1", "ext":"asn1"},
+ {"rettype":"native", "retmode":"xml", "ext":"xml"},
+ {"rettype":"acc", "retmode":"text", "ext":"acc"},
+ {"rettype":"fasta", "retmode":"text", "ext":"fasta"},
+ {"rettype":"fasta", "retmode":"xml", "ext":"fasta.xml"},
+ {"rettype":"seqid", "retmode":"text", "ext":"seqid"},
+ {"rettype":"gb", "retmode":"text", "ext":"gb"},
+ {"rettype":"gb", "retmode":"xml", "ext":"gb.xml"},
+ {"rettype":"gbc", "retmode":"xml", "ext":"gbc"},
+ {"rettype":"gss", "retmode":"text", "ext":"gss"}
+ ],
+ "protein": [
+ {"rettype":"null", "retmode":"text", "ext":"txt"},
+ {"rettype":"null", "retmode":"asn.1", "ext":"asn1"},
+ {"rettype":"native", "retmode":"xml", "ext":"xml"},
+ {"rettype":"acc", "retmode":"text", "ext":"acc"},
+ {"rettype":"fasta", "retmode":"text", "ext":"fasta"},
+ {"rettype":"fasta", "retmode":"xml", "ext":"fasta.xml"},
+ {"rettype":"seqid", "retmode":"text", "ext":"seqid"},
+ {"rettype":"ft", "retmode":"text", "ext":"ft"},
+ {"rettype":"gp", "retmode":"text", "ext":"gp"},
+ {"rettype":"gp", "retmode":"xml", "ext":"gp.xml"},
+ {"rettype":"gpc", "retmode":"xml", "ext":"gpc"},
+ {"rettype":"ipg", "retmode":"xml", "ext":"xml"}
+ ],
+ "popset": [
+ {"rettype":"null", "retmode":"text", "ext":"txt"},
+ {"rettype":"null", "retmode":"asn.1", "ext":"asn1"},
+ {"rettype":"native", "retmode":"xml", "ext":"xml"},
+ {"rettype":"acc", "retmode":"text", "ext":"acc"},
+ {"rettype":"fasta", "retmode":"text", "ext":"fasta"},
+ {"rettype":"fasta", "retmode":"xml", "ext":"fasta.xml"},
+ {"rettype":"seqid", "retmode":"text", "ext":"seqid"},
+ {"rettype":"gb", "retmode":"text", "ext":"gb"},
+ {"rettype":"gb", "retmode":"xml", "ext":"gb.xml"},
+ {"rettype":"gbc", "retmode":"xml", "ext":"gbc"}
+ ],
+ "pmc": [
+ {"rettype":"null", "retmode":"xml", "ext":"xml"},
+ {"rettype":"medline", "retmode":"text", "ext":"medline"}
+ ],
+ "pubmed": [
+ {"rettype":"null", "retmode":"asn.1", "ext":"asn1"},
+ {"rettype":"null", "retmode":"xml", "ext":"xml"},
+ {"rettype":"medline", "retmode":"text", "ext":"medline"},
+ {"rettype":"uilist", "retmode":"text", "ext":"uilist"},
+ {"rettype":"abstract", "retmode":"text", "ext":"abstract"}
+ ],
+ "sequences": [
+ {"rettype":"null", "retmode":"text", "ext":"txt"},
+ {"rettype":"acc", "retmode":"text", "ext":"acc"},
+ {"rettype":"fasta", "retmode":"text", "ext":"fasta"},
+ {"rettype":"seqid", "retmode":"text", "ext":"seqid"}
+ ],
+ "snp": [
+ {"rettype":"null", "retmode":"asn.1", "ext":"asn1"},
+ {"rettype":"null", "retmode":"xml", "ext":"xml"},
+ {"rettype":"flt", "retmode":"text", "ext":"flt"},
+ {"rettype":"fasta", "retmode":"text", "ext":"fasta"},
+ {"rettype":"rsr", "retmode":"text", "ext":"rsr"},
+ {"rettype":"ssexemplar", "retmode":"text", "ext":"ssexemplar"},
+ {"rettype":"chr", "retmode":"text", "ext":"chr"},
+ {"rettype":"docset", "retmode":"text", "ext":"docset"},
+ {"rettype":"uilist", "retmode":"text", "ext":"uilist"},
+ {"rettype":"uilist", "retmode":"xml", "ext":"uilist.xml"}
+ ],
+ "sra": [
+ {"rettype":"full", "retmode":"xml", "ext":"xml"}
+ ],
+ "taxonomy": [
+ {"rettype":"null", "retmode":"xml", "ext":"xml"},
+ {"rettype":"uilist", "retmode":"text", "ext":"uilist"},
+ {"rettype":"uilist", "retmode":"xml", "ext":"uilist.xml"}
+ ]
+ }
+
+ @property
+ def valid_extensions(self):
+ extensions = set()
+ for db, db_options in self.efetch_options.items():
+ for options in db_options:
+ extensions |= set([options["ext"]])
+ return list(extensions)
+
+ def dbs_for_options(self, file_ext, rettype=None, retmode=None):
+ possible_dbs = set()
+ for db, db_options in self.efetch_options.items():
+ for option_dict in db_options:
+ if option_dict["ext"] == file_ext:
+ if retmode and option_dict["retmode"]!=retmode:
+ continue
+ if rettype and option_dict["rettype"]!=rettype:
+ continue
+ possible_dbs |= set([db])
+ break
+ return possible_dbs
+
+ def options_for_db_and_extension(self, db, file_ext, rettype=None, retmode=None):
+ possible_options = []
+ assert file_ext, "file_ext must be defined"
+
+ if not self.is_valid_db(db):
+ raise NCBIFileException("DB specified is not valid. Options include: {dbs}".format(dbs=", ".join(self.valid_dbs)))
+
+ db_options = self.efetch_options[db]
+ for opt in db_options:
+ if file_ext == opt["ext"]:
+ if retmode and opt["retmode"]!=retmode:
+ continue
+ if rettype and opt["rettype"]!=rettype:
+ continue
+ possible_options.append(opt)
+
+ return possible_options
+
+ def guess_db_options_for_extension(self, file_ext, db=None, rettype=None, retmode=None):
+ if db and rettype and retmode:
+ if self.is_valid_db_request(db, rettype, retmode):
+ request_options = {}
+ request_options["db"] = db
+ request_options["rettype"] = rettype
+ request_options["retmode"] = retmode
+ request_options["ext"] = file_ext
+ return request_options
+
+ possible_dbs = [db] if db else self.dbs_for_options(file_ext, rettype, retmode)
+
+ if len(possible_dbs) > 1:
+ raise NCBIFileException('Ambigious db for file extension specified: "{}"; possible databases include: {}'.format(file_ext, ", ".join(list(possible_dbs))))
+ elif len(possible_dbs) == 1:
+ likely_db = possible_dbs.pop()
+
+ likely_options = self.options_for_db_and_extension(likely_db, file_ext, rettype, retmode)
+ if len(likely_options) == 1:
+ request_options = {}
+ request_options["db"] = likely_db
+ request_options["rettype"] = likely_options[0]["rettype"]
+ request_options["retmode"] = likely_options[0]["retmode"]
+ request_options["ext"] = likely_options[0]["ext"]
+ return request_options
+ elif len(likely_options) > 1:
+ raise NCBIFileException('Please clarify the rettype and retmode. Multiple request types are possible for the file extension ({}) specified: {}'.format(file_ext, likely_options))
+ else:
+ raise NCBIFileException("No request options found. Please check the file extension ({}), db ({}), rettype ({}), and retmode ({}) specified.".format(file_ext, db, rettype, retmode))
+
+ def is_valid_db_request(self, db, rettype, retmode):
+ if not self.is_valid_db(db):
+ raise NCBIFileException("DB specified is not valid. Options include: {dbs}".format(dbs=", ".join(self.valid_dbs)))
+ db_options = self.efetch_options[db]
+ for opt in db_options:
+ if opt["rettype"] == rettype and opt["retmode"] == retmode:
+ return True
+ return False
+
+ @property
+ def valid_dbs(self):
+ return self.efetch_options.keys()
+
+ def is_valid_db(self, db):
+ return db in self.valid_dbs
+
+ def parse_accession_str(self, id_str):
+ '''
+ This tries to match an NCBI accession as defined here:
+ http://www.ncbi.nlm.nih.gov/Sequin/acc.html
+ '''
+ m = re.search( r"(?P<accession>(?:[a-zA-Z]{1,6}|NC_|NM_|NR_)\d{1,10})(?:\.(?P<version>\d+))?(?:\.(?P<file_ext>\S+))?.*", id_str )
+ accession, version, file_ext = ("","","")
+ if m:
+ accession = m.group("accession")
+ version = m.group("version")
+ file_ext = m.group("file_ext")
+ assert file_ext, "file_ext must be defined: {}.{}.<file_ext>. Possible values include: {}".format(accession,version,", ".join(list(self.valid_extensions)))
+ assert version, "version must be defined: {}.<version>.{}".format(accession,file_ext)
+
+ return accession, version, file_ext
+
+ @staticmethod
+ def _seq_chunks(seq, n):
+ # http://stackoverflow.com/a/312464/190597 (Ned Batchelder)
+ """ Yield successive n-sized chunks from seq."""
+ for i in range(0, len(seq), n):
+ yield seq[i:i + n]
+
+ def _esummary_and_parse(self, accession, xpath_selector, db="nuccore", return_type=int, raise_on_failure=True, retmode="xml", **kwargs):
+ result = self.entrez.esummary(db=db, id=accession, **kwargs)
+
+ root = ET.fromstring(result.read())
+ nodes = root.findall(xpath_selector)
+
+ retval = 0
+ if len(nodes):
+ retval = return_type(nodes[0].text)
+ else:
+ if raise_on_failure:
+ raise NCBIFileException("The esummary query failed.")
+
+ return retval
+
+ def exists(self, accession, db="nuccore"):
+ result = self.entrez.esearch(db=db, term=accession, rettype="count")
+
+ root = ET.fromstring(result.read())
+ nodes = root.findall(".//Count")
+
+ count = 0
+ if len(nodes):
+ count = int(nodes[0].text)
+ else:
+ raise NCBIFileException("The esummary query failed.")
+
+ if count == 1:
+ return True
+ else:
+ logger.warning('The accession specified, "{acc}", could not be found in the database "{db}".\nConsider if you may need to specify a different database via "db=<db_id>".'.format(acc=accession, db=db))
+ return False
+
+ def size(self, accession, db="nuccore"):
+ return self._esummary_and_parse(accession, ".//*[@Name='Length']", db=db)
+
+ def mtime(self, accession, db="nuccore"):
+ update_date = self._esummary_and_parse(accession, ".//Item[@Name='UpdateDate']", db=db, return_type=str)
+
+ pattern = '%Y/%m/%d'
+ epoch_update_date = int(time.mktime(time.strptime(update_date, pattern)))
+
+ return epoch_update_date
+
+ def fetch_from_ncbi(self, accession_list, destination_dir,
+ force_overwrite=False, rettype="fasta", retmode="text",
+ file_ext=None, combined_file_prefix=None, remove_separate_files=False,
+ chunk_size=1, db="nuccore", **kwargs):
+ """
+ This function downloads and saves files from NCBI.
+ Adapted in part from the BSD-licensed code here:
+ https://github.com/broadinstitute/viral-ngs/blob/master/util/genbank.py
+ """
+
+ max_chunk_size = 500
+
+ # Conform to NCBI retreival guidelines by chunking into 500-accession chunks if
+ # >500 accessions are specified and chunk_size is set to 1
+ # Also clamp chunk size to 500 if the user specified a larger value.
+ if chunk_size > max_chunk_size or (len(accession_list) > max_chunk_size and chunk_size == 1):
+ chunk_size = max_chunk_size
+
+ outEx = {"fasta": "fasta", "ft": "tbl", "gb": "gbk"}
+
+ output_directory = os.path.abspath(os.path.expanduser(destination_dir))
+
+ if not os.path.exists(output_directory):
+ os.makedirs(output_directory)
+
+ output_extension = str(file_ext)
+
+ # ensure the extension starts with a ".", also allowing for passed-in
+ # extensions that already have it
+ if output_extension[:1] != ".":
+ output_extension = "." + output_extension
+
+ logger.info("Fetching {} entries from NCBI: {}\n".format(str(len(accession_list)), ", ".join(accession_list[:10])))
+ output_files = []
+
+ for chunk_num, chunk in enumerate(self._seq_chunks(accession_list, chunk_size)):
+ # sleep to throttle requests to 2 per second per NCBI guidelines:
+ # https://www.ncbi.nlm.nih.gov/books/NBK25497/#chapter2.Usage_Guidelines_and_Requiremen
+ time.sleep(0.5)
+ acc_string = ",".join(chunk)
+
+ # if the filename would be longer than Linux allows, simply say "chunk-chunk_num"
+ if len(acc_string) + len(output_extension) <= 254:
+ output_file_path = os.path.join(output_directory, acc_string + output_extension)
+ else:
+ output_file_path = os.path.join(output_directory, "chunk-{}".format(chunk_num) + output_extension)
+
+ if not force_overwrite:
+ logger.info("not overwriting, checking for existence")
+ assert not os.path.exists(output_file_path), """File %s already exists. Consider removing
+ this file or specifying a different output directory. The files for the accessions specified
+ can be overwritten if you add force_overwrite flag. Processing aborted.""" % output_file_path
+
+ try_count = 1
+ while True:
+ try:
+ logger.info("Fetching file {}: {}, try #{}".format(chunk_num + 1, acc_string, try_count))
+ handle = self.entrez.efetch(db=db, rettype=rettype, retmode=retmode, id=acc_string, **kwargs)
+
+ with open(output_file_path, "w") as outf:
+ for line in handle:
+ outf.write(line)
+ output_files.append(output_file_path)
+ except IOError:
+
+ logger.warning(
+ "Error fetching file {}: {}, try #{} probably because NCBI is too busy.".format(chunk_num + 1, acc_string,
+ try_count))
+
+ try_count += 1
+ if try_count > 4:
+ logger.warning("Tried too many times. Aborting.")
+ raise
+
+ # if the fetch failed, wait a few seconds and try again.
+ logger.info("Waiting and retrying...")
+ time.sleep(2)
+
+ continue
+ break
+
+ # assert that we are not trying to remove the intermediate files without writing a combined file
+ if remove_separate_files:
+ assert combined_file_prefix, """The intermediate files
+ can only be removed if a combined file is written via combined_file_prefix"""
+
+ # build a path to the combined genome file
+ if combined_file_prefix:
+ concatenated_genome_file_path = os.path.join(output_directory, combined_file_prefix + output_extension)
+
+ if not force_overwrite:
+ assert not os.path.exists(concatenated_genome_file_path), """File %s already exists. Consider removing
+ this file or specifying a different output directory. The files for the accessions specified
+ can be overwritten if you add force_overwrite flag. Processing aborted.""" % output_file_path
+
+ # concatenate the files together into one genome file
+ with open(concatenated_genome_file_path, 'w') as outfile:
+ for file_path in output_files:
+ with open(file_path) as infile:
+ for line in infile:
+ outfile.write(line)
+
+ # if the option is specified, remove the intermediate fasta files
+ if remove_separate_files:
+ while len(output_files) > 0:
+ os.unlink(output_files.pop())
+
+ # add the combined file to the list of files returned
+ output_files.append(concatenated_genome_file_path)
+
+ # return list of files
+ return output_files
+
+ def search(self, query, *args, db="nuccore", idtype="acc", **kwargs):
+ # enforce JSON return mode
+ kwargs["retmode"] = "json"
+
+ # if the user specifies retmax, use it and limit there
+ # otherwise page 200 at a time and return all
+ if 'retmax' not in kwargs or not kwargs['retmax']:
+ kwargs['retmax'] = 100000
+ return_all = True
+ else:
+ return_all = False
+
+ kwargs['retstart'] = kwargs.get('retstart', 0)
+
+ def esearch_json(term, *args, **kwargs):
+ handle = self.entrez.esearch(term=term, *args, **kwargs)
+ json_result = json.loads(handle.read())
+ return json_result
+
+ def result_ids(json):
+ if "esearchresult" in json_results and "idlist" in json_results["esearchresult"]:
+ return json_results["esearchresult"]["idlist"]
+ else:
+ raise NCBIFileException("ESearch error")
+
+ has_more = True
+
+ while has_more:
+ json_results = esearch_json(term=query, *args, db=db, idtype=idtype, **kwargs)
+
+ for acc in result_ids(json_results):
+ yield acc
+
+ if return_all and ("count" in json_results["esearchresult"] and int(json_results["esearchresult"]["count"]) > kwargs['retmax']+kwargs['retstart']):
+ kwargs['retstart'] += kwargs['retmax']
+ # sleep to throttle requests to <2 per second per NCBI guidelines:
+ # https://www.ncbi.nlm.nih.gov/books/NBK25497/#chapter2.Usage_Guidelines_and_Requiremen
+ time.sleep(0.5)
+ else:
+ has_more = False
diff --git a/snakemake/remote/S3.py b/snakemake/remote/S3.py
index 26324b5..c935c27 100644
--- a/snakemake/remote/S3.py
+++ b/snakemake/remote/S3.py
@@ -4,19 +4,16 @@ __email__ = "tomkinsc at broadinstitute.org"
__license__ = "MIT"
# built-ins
-import os, re, sys
+import os
+import re
import math
-import time
import email.utils
-from time import mktime
-import datetime
import functools
import concurrent.futures
# module-specific
from snakemake.remote import AbstractRemoteObject, AbstractRemoteProvider
-from snakemake.exceptions import MissingOutputException, WorkflowError, WildcardError, RemoteFileException, S3FileException
-import snakemake.io
+from snakemake.exceptions import WorkflowError, S3FileException
try:
# third-party modules
@@ -24,18 +21,33 @@ try:
from boto.s3.key import Key
from filechunkio import FileChunkIO
except ImportError as e:
- raise WorkflowError("The Python 3 packages 'boto' and 'filechunkio' " +
+ raise WorkflowError("The Python 3 packages 'boto' and 'filechunkio' " +
"need to be installed to use S3 remote() file functionality. %s" % e.msg)
+
class RemoteProvider(AbstractRemoteProvider):
- def __init__(self, *args, **kwargs):
- super(RemoteProvider, self).__init__(*args, **kwargs)
+
+ supports_default = True
+
+ def __init__(self, *args, stay_on_remote=False, **kwargs):
+ super(RemoteProvider, self).__init__(*args, stay_on_remote=stay_on_remote, **kwargs)
self._s3c = S3Helper(*args, **kwargs)
-
+
def remote_interface(self):
return self._s3c
+ @property
+ def default_protocol(self):
+ """The protocol that is prepended to the path when no protocol is specified."""
+ return 's3://'
+
+ @property
+ def available_protocols(self):
+ """List of valid protocols for this remote provider."""
+ return ['s3://']
+
+
class RemoteObject(AbstractRemoteObject):
""" This is a class to interact with the AWS S3 object store.
"""
@@ -54,13 +66,13 @@ class RemoteObject(AbstractRemoteObject):
if self._matched_s3_path:
return self._s3c.exists_in_bucket(self.s3_bucket, self.s3_key)
else:
- raise S3FileException("The file cannot be parsed as an s3 path in form 'bucket/key': %s" % self.file())
+ raise S3FileException("The file cannot be parsed as an s3 path in form 'bucket/key': %s" % self.local_file())
def mtime(self):
if self.exists():
return self._s3c.key_last_modified(self.s3_bucket, self.s3_key)
else:
- raise S3FileException("The file does not seem to exist remotely: %s" % self.file())
+ raise S3FileException("The file does not seem to exist remotely: %s" % self.local_file())
def size(self):
if self.exists():
@@ -69,13 +81,14 @@ class RemoteObject(AbstractRemoteObject):
return self._iofile.size_local
def download(self):
- self._s3c.download_from_s3(self.s3_bucket, self.s3_key, self.file())
+ self._s3c.download_from_s3(self.s3_bucket, self.s3_key, self.local_file())
+ os.sync() # ensure flush to disk
def upload(self):
if self.size() > 10 * 1024 * 1024: # S3 complains if multipart uploads are <10MB
- self._s3c.upload_to_s3_multipart(self.s3_bucket, self.file(), self.s3_key, encrypt_key=self.kwargs.get("encrypt_key", None))
+ self._s3c.upload_to_s3_multipart(self.s3_bucket, self.local_file(), self.s3_key, encrypt_key=self.kwargs.get("encrypt_key", None))
else:
- self._s3c.upload_to_s3(self.s3_bucket, self.file(), self.s3_key, encrypt_key=self.kwargs.get("encrypt_key", None))
+ self._s3c.upload_to_s3(self.s3_bucket, self.local_file(), self.s3_key, encrypt_key=self.kwargs.get("encrypt_key", None))
@property
def list(self):
@@ -85,7 +98,7 @@ class RemoteObject(AbstractRemoteObject):
@property
def _matched_s3_path(self):
- return re.search("(?P<bucket>[^/]*)/(?P<key>.*)", self.file())
+ return re.search("(?P<bucket>[^/]*)/(?P<key>.*)", self.local_file())
@property
def s3_bucket(self):
@@ -108,7 +121,8 @@ class RemoteObject(AbstractRemoteObject):
self._s3c.download_from_s3(self.s3_bucket, self.s3_key, self.file, create_stub_only=True)
else:
raise S3FileException("The file to be downloaded cannot be parsed as an s3 path in form 'bucket/key': %s" %
- self.file())
+ self.local_file())
+
class S3Helper(object):
@@ -118,7 +132,7 @@ class S3Helper(object):
# AWS_SECRET_ACCESS_KEY
# Otherwise these values need to be passed in as kwargs
- # allow key_id and secret to be specified with aws_, gs_, or no prefix.
+ # allow key_id and secret to be specified with aws_, gs_, or no prefix.
# Standardize to the aws_ prefix expected by boto.
if "gs_access_key_id" in kwargs:
kwargs["aws_access_key_id"] = kwargs.pop("gs_access_key_id")
@@ -128,7 +142,7 @@ class S3Helper(object):
kwargs["aws_access_key_id"] = kwargs.pop("access_key_id")
if "secret_access_key" in kwargs:
kwargs["aws_secret_access_key"] = kwargs.pop("secret_access_key")
-
+
self.conn = boto.connect_s3(*args, **kwargs)
def upload_to_s3(
diff --git a/tests/test_remote/S3Mocked.py b/snakemake/remote/S3Mocked.py
similarity index 95%
rename from tests/test_remote/S3Mocked.py
rename to snakemake/remote/S3Mocked.py
index 241869d..5540476 100644
--- a/tests/test_remote/S3Mocked.py
+++ b/snakemake/remote/S3Mocked.py
@@ -23,7 +23,7 @@ try:
from moto import mock_s3
import filechunkio
except ImportError as e:
- raise WorkflowError("The Python 3 packages 'moto', boto' and 'filechunkio' " +
+ raise WorkflowError("The Python 3 packages 'moto', boto' and 'filechunkio' " +
"need to be installed to use S3Mocked remote() file functionality. %s" % e.msg)
def noop():
@@ -33,7 +33,7 @@ def pickled_moto_wrapper(func):
"""
This is a class decorator that in turn decorates all methods within
a class to mock out boto calls with moto-simulated ones.
- Since the moto backends are not presistent across calls by default,
+ Since the moto backends are not presistent across calls by default,
the wrapper also pickles the bucket state after each function call,
and restores it before execution. This way uploaded files are available
for follow-on tasks. Since snakemake may execute with multiple threads
@@ -73,18 +73,18 @@ def pickled_moto_wrapper(func):
class RemoteProvider(S3RemoteProvider):
def __init__(self, *args, **kwargs):
super(RemoteProvider, self).__init__(*args, **kwargs)
-
+
@dec_all_methods(pickled_moto_wrapper, prefix=None)
class RemoteObject(S3RemoteObject):
- """
+ """
This is a derivative of the S3 remote provider that mocks
out boto-based S3 calls using the "moto" Python package.
- Only the initializer is different; it "uploads" the input
+ Only the initializer is different; it "uploads" the input
test file to the moto-simulated bucket at the start.
"""
- def __init__(self, *args, keep_local=False, provider=None, **kwargs):
- super(RemoteObject, self).__init__(*args, keep_local=keep_local, provider=provider, **kwargs)
+ def __init__(self, *args, keep_local=False, stay_on_remote=False, provider=None, **kwargs):
+ super(RemoteObject, self).__init__(*args, keep_local=keep_local, stay_on_remote=False, provider=provider, **kwargs)
bucket_name = 'test-remote-bucket'
test_file = "test.txt"
diff --git a/snakemake/remote/SFTP.py b/snakemake/remote/SFTP.py
index d466ca5..5c560bd 100644
--- a/snakemake/remote/SFTP.py
+++ b/snakemake/remote/SFTP.py
@@ -3,13 +3,12 @@ __copyright__ = "Copyright 2015, Christopher Tomkins-Tinch"
__email__ = "tomkinsc at broadinstitute.org"
__license__ = "MIT"
-import os, re
+import os
from contextlib import contextmanager
# module-specific
from snakemake.remote import AbstractRemoteProvider, DomainObject
from snakemake.exceptions import SFTPFileException, WorkflowError
-import snakemake.io
try:
# third-party modules
@@ -20,8 +19,22 @@ except ImportError as e:
class RemoteProvider(AbstractRemoteProvider):
- def __init__(self, *args, **kwargs):
- super(RemoteProvider, self).__init__(*args, **kwargs)
+
+ supports_default = True
+
+ def __init__(self, *args, stay_on_remote=False, **kwargs):
+ super(RemoteProvider, self).__init__(*args, stay_on_remote=stay_on_remote, **kwargs)
+
+ @property
+ def default_protocol(self):
+ """The protocol that is prepended to the path when no protocol is specified."""
+ return 'sftp://'
+
+ @property
+ def available_protocols(self):
+ """List of valid protocols for this remote provider."""
+ return ['ssh://', 'sftp://']
+
class RemoteObject(DomainObject):
""" This is a class to interact with an SFTP server.
@@ -62,7 +75,7 @@ class RemoteObject(DomainObject):
return sftpc.isfile(self.remote_path)
return False
else:
- raise SFTPFileException("The file cannot be parsed as an SFTP path in form 'host:port/path/to/file': %s" % self.file())
+ raise SFTPFileException("The file cannot be parsed as an SFTP path in form 'host:port/path/to/file': %s" % self.local_file())
def mtime(self):
if self.exists():
@@ -71,7 +84,7 @@ class RemoteObject(DomainObject):
attr = sftpc.lstat(self.remote_path)
return int(attr.st_mtime)
else:
- raise SFTPFileException("The file does not seem to exist remotely: %s" % self.file())
+ raise SFTPFileException("The file does not seem to exist remotely: %s" % self.local_file())
def is_newer(self, time):
""" Returns true of the file is newer than time, or if it is
@@ -96,8 +109,9 @@ class RemoteObject(DomainObject):
os.makedirs(os.path.dirname(self.local_path), exist_ok=True)
sftpc.get(remotepath=self.remote_path, localpath=self.local_path, preserve_mtime=True)
+ os.sync() # ensure flush to disk
else:
- raise SFTPFileException("The file does not seem to exist remotely: %s" % self.file())
+ raise SFTPFileException("The file does not seem to exist remotely: %s" % self.local_file())
def upload(self):
with self.sftpc() as sftpc:
diff --git a/snakemake/remote/XRootD.py b/snakemake/remote/XRootD.py
new file mode 100644
index 0000000..c14ec4b
--- /dev/null
+++ b/snakemake/remote/XRootD.py
@@ -0,0 +1,197 @@
+__author__ = "Chris Burr"
+__copyright__ = "Copyright 2017, Chris Burr"
+__email__ = "christopher.burr at cern.ch"
+__license__ = "MIT"
+
+import os
+from os.path import abspath, join, normpath
+import re
+
+from snakemake.remote import AbstractRemoteObject, AbstractRemoteProvider
+from snakemake.exceptions import WorkflowError, XRootDFileException
+
+try:
+ from XRootD import client
+ from XRootD.client.flags import DirListFlags, MkDirFlags, StatInfoFlags
+except ImportError as e:
+ raise WorkflowError(
+ "The Python 3 package 'XRootD' must be installed to use XRootD "
+ "remote() file functionality. %s" % e.msg
+ )
+
+
+class RemoteProvider(AbstractRemoteProvider):
+ def __init__(self, *args, stay_on_remote=False, **kwargs):
+ super(RemoteProvider, self).__init__(*args, stay_on_remote=stay_on_remote, **kwargs)
+
+ self._xrd = XRootDHelper()
+
+ def remote_interface(self):
+ return self._xrd
+
+ @property
+ def default_protocol(self):
+ """The protocol that is prepended to the path when no protocol is specified."""
+ return 'root://'
+
+ @property
+ def available_protocols(self):
+ """List of valid protocols for this remote provider."""
+ return ['root://', 'roots://', 'rootk://']
+
+
+class RemoteObject(AbstractRemoteObject):
+ """ This is a class to interact with XRootD servers."""
+
+ def __init__(self, *args, keep_local=False, stay_on_remote=False, provider=None, **kwargs):
+ super(RemoteObject, self).__init__(*args, keep_local=keep_local, stay_on_remote=stay_on_remote, provider=provider, **kwargs)
+
+ if provider:
+ self._xrd = provider.remote_interface()
+ else:
+ self._xrd = XRootDHelper()
+
+ # === Implementations of abstract class members ===
+
+ def exists(self):
+ return self._xrd.exists(self.remote_file())
+
+ def mtime(self):
+ if self.exists():
+ return self._xrd.file_last_modified(self.remote_file())
+ else:
+ raise XRootDFileException("The file does not seem to exist remotely: %s" % self.remote_file())
+
+ def size(self):
+ if self.exists():
+ return self._xrd.file_size(self.remote_file())
+ else:
+ return self._iofile.size_local
+
+ def download(self):
+ assert not self.stay_on_remote
+ self._xrd.copy(self.remote_file(), self.file())
+
+ def upload(self):
+ assert not self.stay_on_remote
+ self._xrd.copy(self.file(), self.remote_file())
+
+ @property
+ def name(self):
+ return self.local_file()
+
+ @property
+ def list(self):
+ dirname = os.path.dirname(self._iofile.constant_prefix())+'/'
+ files = list(self._xrd.list_directory_recursive(dirname))
+ return [normpath(f) for f in files]
+
+ def remove(self):
+ self._xrd.remove(self.remote_file())
+
+
+class XRootDHelper(object):
+
+ def __init__(self):
+ self._clients = {}
+
+ def get_client(self, domain):
+ try:
+ return self._clients[domain]
+ except KeyError:
+ self._clients[domain] = client.FileSystem(domain)
+ return self._clients[domain]
+
+ def _parse_url(self, url):
+ match = re.search('(?P<domain>(?:[A-Za-z]+://)[A-Za-z0-9:\_\-\.]+\:?/)(?P<path>/.+)', url)
+ if match is None:
+ return None
+
+ domain = match.group('domain')
+
+ dirname, filename = os.path.split(match.group('path'))
+ # We need a trailing / to keep XRootD happy
+ dirname += '/'
+ return domain, dirname, filename
+
+ def exists(self, url):
+ domain, dirname, filename = self._parse_url(url)
+ status, dirlist = self.get_client(domain).dirlist(dirname)
+ if not status.ok:
+ if status.errno == 3011:
+ return False
+ else:
+ raise XRootDFileException(
+ 'Error listing directory '+dirname+' on domain '+domain+
+ '\n'+repr(status)+'\n'+repr(dirlist))
+ return filename in [f.name for f in dirlist.dirlist]
+
+ def _get_statinfo(self, url):
+ domain, dirname, filename = self._parse_url(url)
+ matches = [f for f in self.list_directory(domain, dirname) if f.name == filename]
+ assert len(matches) == 1
+ return matches[0].statinfo
+
+ def file_last_modified(self, filename):
+ return self._get_statinfo(filename).modtime
+
+ def file_size(self, filename):
+ return self._get_statinfo(filename).size
+
+ def copy(self, source, destination):
+ # Prepare the source path for XRootD
+ if not self._parse_url(source):
+ source = abspath(source)
+ # Prepare the destination path for XRootD
+ assert os.path.basename(source) == os.path.basename(destination)
+ if self._parse_url(destination):
+ domain, dirname, filename = self._parse_url(destination)
+ self.makedirs(domain, dirname)
+ else:
+ destination = abspath(destination)
+ if not os.path.isdir(os.path.dirname(destination)):
+ os.makedirs(os.path.dirname(destination))
+ # Perform the copy operation
+ process = client.CopyProcess()
+ process.add_job(source, destination)
+ process.prepare()
+ status, returns = process.run()
+ if not status.ok or not returns[0]['status'].ok:
+ raise XRootDFileException('Error copying from '+source+' to '+destination, repr(status), repr(returns))
+
+ def makedirs(self, domain, dirname):
+ print('Making directories', domain, dirname)
+ assert dirname.endswith('/')
+ status, _ = self.get_client(domain).mkdir(dirname, MkDirFlags.MAKEPATH)
+ if not status.ok:
+ raise XRootDFileException('Failed to create directory '+dirname, repr(status))
+
+ def list_directory(self, domain, dirname):
+ status, dirlist = self.get_client(domain).dirlist(dirname, DirListFlags.STAT)
+ if not status.ok:
+ raise XRootDFileException(
+ 'Error listing directory '+dirname+' on domain '+domain+
+ '\n'+repr(status)+'\n'+repr(dirlist)
+ )
+ return dirlist.dirlist
+
+ def list_directory_recursive(self, start_url):
+ assert start_url.endswith('/')
+ domain, dirname, filename = self._parse_url(start_url)
+ assert not filename
+ filename = join(dirname, filename)
+ for f in self.list_directory(domain, dirname):
+ if f.statinfo.flags & StatInfoFlags.IS_DIR:
+ for _f_name in self.list_directory_recursive(domain+dirname+f.name+'/'):
+ yield _f_name
+ else:
+ # Only yield files as directories don't have timestamps on XRootD
+ yield domain+dirname+f.name
+
+ def remove(self, url):
+ domain, dirname, filename = self._parse_url(url)
+ filename = join(dirname, filename)
+ status, _ = self.get_client(domain).rm(filename)
+ if not status.ok:
+ raise XRootDFileException(
+ 'Failed to remove file '+filename+' from remote '+domain+'\n'+repr(status))
diff --git a/snakemake/remote/__init__.py b/snakemake/remote/__init__.py
index fa47b44..380ec5c 100644
--- a/snakemake/remote/__init__.py
+++ b/snakemake/remote/__init__.py
@@ -4,14 +4,16 @@ __email__ = "tomkinsc at broadinstitute.org"
__license__ = "MIT"
# built-ins
-import os, sys, re
+import os
+import sys
+import re
from abc import ABCMeta, abstractmethod
from wrapt import ObjectProxy
import copy
# module-specific
import snakemake.io
-from snakemake.exceptions import RemoteFileException
+
class StaticRemoteObjectProxy(ObjectProxy):
'''Proxy that implements static-ness for remote objects.
@@ -44,71 +46,123 @@ class AbstractRemoteProvider:
"""
__metaclass__ = ABCMeta
- def __init__(self, *args, **kwargs):
+ supports_default = False
+
+ def __init__(self, *args, keep_local=False, stay_on_remote=False, **kwargs):
self.args = args
+ self.stay_on_remote = stay_on_remote
+ self.keep_local = keep_local
self.kwargs = kwargs
- def remote(self, value, *args, keep_local=False, static=False, **kwargs):
+ def remote(self, value, *args, keep_local=None, stay_on_remote=None, static=False, **kwargs):
if snakemake.io.is_flagged(value, "temp"):
raise SyntaxError(
"Remote and temporary flags are mutually exclusive.")
if snakemake.io.is_flagged(value, "protected"):
raise SyntaxError(
"Remote and protected flags are mutually exclusive.")
-
- provider = sys.modules[self.__module__] # get module of derived class
- remote_object = provider.RemoteObject(*args, keep_local=keep_local, provider=provider.RemoteProvider(*self.args, **self.kwargs), **kwargs)
+ if keep_local is None:
+ keep_local = self.keep_local
+ if stay_on_remote is None:
+ stay_on_remote = self.stay_on_remote
+
+ def _set_protocol(value):
+ """Adds the default protocol to `value` if it doesn't already have one"""
+ for protocol in self.available_protocols:
+ if value.startswith(protocol):
+ break
+ if value.startswith(protocol):
+ value = value[len(protocol):]
+ protocol = protocol
+ else:
+ protocol = self.default_protocol
+ return protocol, value
+
+ if isinstance(value, str):
+ protocol, value = _set_protocol(value)
+ value = protocol+value if stay_on_remote else value
+ else:
+ protocol, value = list(zip(*[_set_protocol(v) for v in value]))
+ if len(set(protocol)) != 1:
+ raise SyntaxError('A single protocol must be used per RemoteObject')
+ protocol = set(protocol).pop()
+ value = [protocol+v if stay_on_remote else v for v in value]
+
+ provider = sys.modules[self.__module__] # get module of derived class
+ remote_object = provider.RemoteObject(
+ *args, protocol=protocol, keep_local=keep_local, stay_on_remote=stay_on_remote,
+ provider=provider.RemoteProvider(*self.args, **self.kwargs), **kwargs
+ )
if static:
remote_object = StaticRemoteObjectProxy(remote_object)
- return snakemake.io.flag(
- value,
- "remote_object",
- remote_object
- )
+ return snakemake.io.flag(value, "remote_object", remote_object)
def glob_wildcards(self, pattern, *args, **kwargs):
- args = self.args if not args else args
+ args = self.args if not args else args
kwargs = self.kwargs if not kwargs else kwargs
-
+
referenceObj = snakemake.io.IOFile(self.remote(pattern, *args, **kwargs))
- pattern = "./"+ referenceObj.remote_object.name
- pattern = os.path.normpath(pattern)
+ if not referenceObj.remote_object.stay_on_remote:
+ pattern = "./" + referenceObj.remote_object.name
+ pattern = os.path.normpath(pattern)
- key_list = [k for k in referenceObj.remote_object.list]
+ key_list = [k for k in referenceObj.remote_object.list]
return snakemake.io.glob_wildcards(pattern, files=key_list)
@abstractmethod
+ def default_protocol(self):
+ """The protocol that is prepended to the path when no protocol is specified."""
+ pass
+
+ @abstractmethod
+ def available_protocols(self):
+ """List of valid protocols for this remote provider."""
+ pass
+
+ @abstractmethod
def remote_interface(self):
pass
class AbstractRemoteObject:
- """ This is an abstract class to be used to derive remote object classes for
- different cloud storage providers. For example, there could be classes for interacting with
+ """ This is an abstract class to be used to derive remote object classes for
+ different cloud storage providers. For example, there could be classes for interacting with
Amazon AWS S3 and Google Cloud Storage, both derived from this common base class.
"""
__metaclass__ = ABCMeta
- def __init__(self, *args, keep_local=False, provider=None, **kwargs):
+ def __init__(self, *args, protocol=None, keep_local=False, stay_on_remote=False, provider=None, **kwargs):
+ assert protocol is not None
# self._iofile must be set before the remote object can be used, in io.py or elsewhere
self._iofile = None
self.args = args
self.kwargs = kwargs
self.keep_local = keep_local
+ self.stay_on_remote = stay_on_remote
self.provider = provider
+ self.protocol = protocol
@property
def _file(self):
if self._iofile is None:
return None
return self._iofile._file
-
+
def file(self):
return self._file
+ def local_file(self):
+ if self.stay_on_remote:
+ return self._file[len(self.protocol):]
+ else:
+ return self._file
+
+ def remote_file(self):
+ return self.protocol+self.local_file()
+
@abstractmethod
def close(self):
pass
@@ -142,9 +196,14 @@ class AbstractRemoteObject:
pass
@abstractmethod
- def remote(self, value, keep_local=False):
+ def remote(self, value, keep_local=False, stay_on_remote=False):
pass
+ @abstractmethod
+ def remove(self):
+ raise NotImplementedError("Removal of files is unavailable for this remote")
+
+
class DomainObject(AbstractRemoteObject):
"""This is a mixin related to parsing components
out of a location path specified as
@@ -155,16 +214,11 @@ class DomainObject(AbstractRemoteObject):
@property
def _matched_address(self):
- return re.search("^(?P<host>[A-Za-z0-9\-\.]+)(?:\:(?P<port>[0-9]+))?(?P<path_remainder>.*)$", self._iofile._file)
+ return re.search("^(?P<host>[A-Za-z0-9\-\.]+)(?:\:(?P<port>[0-9]+))?(?P<path_remainder>.*)$", self.local_file())
@property
def name(self):
return self.path_remainder
-
- @property
- def protocol(self):
- if self._matched_address:
- return self._matched_address.group("protocol")
@property
def host(self):
@@ -174,12 +228,12 @@ class DomainObject(AbstractRemoteObject):
@property
def port(self):
return self._matched_address.group("port")
-
+
@property
def path_prefix(self):
# this is the domain and port, however specified before the path remainder
return self._iofile._file[:self._iofile._file.index(self.path_remainder)]
-
+
@property
def path_remainder(self):
if self._matched_address:
diff --git a/snakemake/remote/dropbox.py b/snakemake/remote/dropbox.py
index 8067ba8..dbdc52b 100644
--- a/snakemake/remote/dropbox.py
+++ b/snakemake/remote/dropbox.py
@@ -3,17 +3,15 @@ __copyright__ = "Copyright 2015, Christopher Tomkins-Tinch"
__email__ = "tomkinsc at broadinstitute.org"
__license__ = "MIT"
-import os, re
-from contextlib import contextmanager
+import os
# module-specific
from snakemake.remote import AbstractRemoteProvider, AbstractRemoteObject
from snakemake.exceptions import DropboxFileException, WorkflowError
-import snakemake.io
try:
# third-party modules
- import dropbox # The official Dropbox API library
+ import dropbox # The official Dropbox API library
except ImportError as e:
raise WorkflowError("The Python 3 package 'dropbox' "
"must be installed to use Dropbox remote() file "
@@ -21,8 +19,8 @@ except ImportError as e:
class RemoteProvider(AbstractRemoteProvider):
- def __init__(self, *args, **kwargs):
- super(RemoteProvider, self).__init__(*args, **kwargs)
+ def __init__(self, *args, stay_on_remote=False, **kwargs):
+ super(RemoteProvider, self).__init__(*args, stay_on_remote=stay_on_remote, **kwargs)
self._dropboxc = dropbox.Dropbox(*args, **kwargs)
try:
@@ -33,6 +31,17 @@ class RemoteProvider(AbstractRemoteProvider):
def remote_interface(self):
return self._dropboxc
+ @property
+ def default_protocol(self):
+ """The protocol that is prepended to the path when no protocol is specified."""
+ return 'dropbox://'
+
+ @property
+ def available_protocols(self):
+ """List of valid protocols for this remote provider."""
+ return ['dropbox://']
+
+
class RemoteObject(AbstractRemoteObject):
""" This is a class to interact with the AWS S3 object store.
"""
@@ -53,22 +62,22 @@ class RemoteObject(AbstractRemoteObject):
def exists(self):
try:
- metadata = self._dropboxc.files_get_metadata(self.remote_file())
+ metadata = self._dropboxc.files_get_metadata(self.dropbox_file())
return True
except:
return False
def mtime(self):
if self.exists():
- metadata = self._dropboxc.files_get_metadata(self.remote_file())
+ metadata = self._dropboxc.files_get_metadata(self.dropbox_file())
epochTime = metadata.server_modified.timestamp()
return epochTime
else:
- raise DropboxFileException("The file does not seem to exist remotely: %s" % self.remote_file())
+ raise DropboxFileException("The file does not seem to exist remotely: %s" % self.dropbox_file())
def size(self):
if self.exists():
- metadata = self._dropboxc.files_get_metadata(self.remote_file())
+ metadata = self._dropboxc.files_get_metadata(self.dropbox_file())
return int(metadata.size)
else:
return self._iofile.size_local
@@ -77,17 +86,17 @@ class RemoteObject(AbstractRemoteObject):
if self.exists():
# if the destination path does not exist, make it
if make_dest_dirs:
- os.makedirs(os.path.dirname(self.file()), exist_ok=True)
+ os.makedirs(os.path.dirname(self.local_file()), exist_ok=True)
- self._dropboxc.files_download_to_file(self.file(), self.remote_file())
+ self._dropboxc.files_download_to_file(self.local_file(), self.dropbox_file())
+ os.sync() # ensure flush to disk
else:
- raise DropboxFileException("The file does not seem to exist remotely: %s" % self.remote_file())
+ raise DropboxFileException("The file does not seem to exist remotely: %s" % self.dropbox_file())
def upload(self, mode=dropbox.files.WriteMode('overwrite')):
- size = os.path.getsize(self.file())
# Chunk file into 10MB slices because Dropbox does not accept more than 150MB chunks
chunksize = 10000000
- with open(self.file(), mode='rb') as f:
+ with open(self.local_file(), mode='rb') as f:
data = f.read(chunksize)
# Start upload session
res = self._dropboxc.files_upload_session_start(data)
@@ -103,14 +112,14 @@ class RemoteObject(AbstractRemoteObject):
self._dropboxc.files_upload_session_finish(
f.read(chunksize),
dropbox.files.UploadSessionCursor(res.session_id, offset),
- dropbox.files.CommitInfo(path=self.remote_file(), mode=mode))
+ dropbox.files.CommitInfo(path=self.dropbox_file(), mode=mode))
- def remote_file(self):
- return "/"+self.file() if not self.file().startswith("/") else self.file()
+ def dropbox_file(self):
+ return "/"+self.local_file() if not self.local_file().startswith("/") else self.local_file()
@property
def name(self):
- return self.file()
+ return self.local_file()
@property
def list(self):
diff --git a/snakemake/report.py b/snakemake/report.py
index 1b8c15e..5f33279 100644
--- a/snakemake/report.py
+++ b/snakemake/report.py
@@ -108,11 +108,11 @@ def report(text, path,
if not isinstance(_files, list):
_files = [_files]
links = []
- for file in _files:
+ for file in sorted(_files):
data = data_uri(file)
links.append(':raw-html:`<a href="{data}" download="{filename}" draggable="true">{filename}</a>`'.format(
data=data, filename=os.path.basename(file)))
- links = "\n\n ".join(links)
+ links = "\n\n ".join(links)
attachments.append('''
.. container::
:name: {name}
diff --git a/snakemake/rules.py b/snakemake/rules.py
index 308b2b2..656f6bf 100644
--- a/snakemake/rules.py
+++ b/snakemake/rules.py
@@ -9,6 +9,7 @@ import sys
import inspect
import sre_constants
from collections import defaultdict, Iterable
+from urllib.parse import urljoin
from snakemake.io import IOFile, _IOFile, protected, temp, dynamic, Namedlist, AnnotatedString, contains_wildcard_constraints, update_wildcard_constraints
from snakemake.io import expand, InputFiles, OutputFiles, Wildcards, Params, Log, Resources
@@ -162,8 +163,11 @@ class Rule:
# TODO have a look into how to concretize dependencies here
branch._input, _, branch.dependencies = branch.expand_input(non_dynamic_wildcards)
branch._output, _ = branch.expand_output(non_dynamic_wildcards)
- branch.resources = dict(branch.expand_resources(non_dynamic_wildcards, branch._input).items())
- branch._params = branch.expand_params(non_dynamic_wildcards, branch._input, branch.resources)
+
+ resources = branch.expand_resources(non_dynamic_wildcards, branch._input)
+ branch._params = branch.expand_params(non_dynamic_wildcards, branch._input, branch._output, resources)
+ branch.resources = dict(resources.items())
+
branch._log = branch.expand_log(non_dynamic_wildcards)
branch._benchmark = branch.expand_benchmark(non_dynamic_wildcards)
branch._conda_env = branch.expand_conda_env(non_dynamic_wildcards)
@@ -192,6 +196,7 @@ class Rule:
@benchmark.setter
def benchmark(self, benchmark):
+ benchmark = self.apply_default_remote(benchmark)
self._benchmark = IOFile(benchmark, rule=self)
@property
@@ -279,6 +284,13 @@ class Rule:
self.name, seen[value], name or idx))
seen[value] = name or idx
+ def apply_default_remote(self, item):
+ if (not is_flagged(item, "remote_object") and
+ self.workflow.default_remote_provider is not None):
+ item = "{}/{}".format(self.workflow.default_remote_prefix, item)
+ return self.workflow.default_remote_provider.remote(item)
+ return item
+
def _set_inoutput_item(self, item, output=False, name=None):
"""
Set an item to be input or output.
@@ -290,6 +302,8 @@ class Rule:
"""
inoutput = self.output if output else self.input
if isinstance(item, str):
+ item = self.apply_default_remote(item)
+
# add the rule to the dependencies
if isinstance(item, _IOFile) and item.rule:
self.dependencies[item] = item.rule
@@ -337,6 +351,8 @@ class Rule:
if name:
inoutput.add_name(name)
elif callable(item):
+ item = self.apply_default_remote(item)
+
if output:
raise SyntaxError(
"Only input files can be specified as functions")
@@ -389,6 +405,7 @@ class Rule:
def _set_log_item(self, item, name=None):
if isinstance(item, str) or callable(item):
+ item = self.apply_default_remote(item)
self.log.append(IOFile(item,
rule=self) if isinstance(item, str) else
item)
@@ -494,7 +511,7 @@ class Rule:
concretize=concretize_iofile,
mapping=mapping)
except WildcardError as e:
- raise WorkflowError(
+ raise WildcardError(
"Wildcards in input files cannot be "
"determined from output files:",
str(e), rule=self)
@@ -514,8 +531,7 @@ class Rule:
return input, mapping, dependencies
- def expand_params(self, wildcards, input, resources):
- # TODO add output
+ def expand_params(self, wildcards, input, output, resources):
def concretize_param(p, wildcards):
if isinstance(p, str):
return apply_wildcards(p, wildcards)
@@ -530,9 +546,11 @@ class Rule:
check_return_type=False,
no_flattening=True,
aux_params={"input": input,
- "resources": resources})
+ "resources": resources,
+ "output": output,
+ "threads": resources._cores})
except WildcardError as e:
- raise WorkflowError(
+ raise WildcardError(
"Wildcards in params cannot be "
"determined from output files:",
str(e), rule=self)
@@ -571,7 +589,7 @@ class Rule:
wildcards,
concretize=concretize_logfile)
except WildcardError as e:
- raise WorkflowError(
+ raise WildcardError(
"Wildcards in log files cannot be "
"determined from output files:",
str(e), rule=self)
@@ -586,7 +604,7 @@ class Rule:
benchmark = self.benchmark.apply_wildcards(
wildcards) if self.benchmark else None
except WildcardError as e:
- raise WorkflowError(
+ raise WildcardError(
"Wildcards in benchmark file cannot be "
"determined from output files:",
str(e), rule=self)
@@ -615,7 +633,7 @@ class Rule:
conda_env = self.conda_env.apply_wildcards(
wildcards) if self.conda_env else None
except WildcardError as e:
- raise WorkflowError(
+ raise WildcardError(
"Wildcards in conda environment file cannot be "
"determined from output files:",
str(e), rule=self)
diff --git a/snakemake/scheduler.py b/snakemake/scheduler.py
index f50bfdb..39ed1b0 100644
--- a/snakemake/scheduler.py
+++ b/snakemake/scheduler.py
@@ -33,6 +33,7 @@ class JobScheduler:
cluster_config=None,
cluster_sync=None,
drmaa=None,
+ drmaa_log_dir=None,
jobname=None,
quiet=False,
printreason=False,
@@ -50,6 +51,7 @@ class JobScheduler:
self.dag = dag
self.workflow = workflow
self.dryrun = dryrun
+ self.touch = touch
self.quiet = quiet
self.keepgoing = keepgoing
self.running = set()
@@ -72,6 +74,7 @@ class JobScheduler:
update_dynamic=not self.dryrun,
print_progress=not self.quiet and not self.dryrun)
+ self._local_executor = None
if dryrun:
self._executor = DryrunExecutor(workflow, dag,
printreason=printreason,
@@ -96,7 +99,6 @@ class JobScheduler:
latency_wait=latency_wait,
benchmark_repeats=benchmark_repeats,
cores=local_cores)
- self.run = self.run_cluster_or_local
if cluster or cluster_sync:
constructor = SynchronousClusterExecutor if cluster_sync \
else GenericClusterExecutor
@@ -121,6 +123,7 @@ class JobScheduler:
self._executor = DRMAAExecutor(
workflow, dag, None,
drmaa_args=drmaa,
+ drmaa_log_dir=drmaa_log_dir,
jobname=jobname,
printreason=printreason,
quiet=quiet,
@@ -224,19 +227,18 @@ class JobScheduler:
job.cleanup()
return False
+ def get_executor(self, job):
+ if self._local_executor is None:
+ return self._executor
+ else:
+ return self._local_executor if self.workflow.is_local(
+ job.rule) else self._executor
+
def run(self, job):
- self._executor.run(job,
- callback=self._finish_callback,
- submit_callback=self._submit_callback,
- error_callback=self._error)
-
- def run_cluster_or_local(self, job):
- executor = self._local_executor if self.workflow.is_local(
- job.rule) else self._executor
- executor.run(job,
- callback=self._finish_callback,
- submit_callback=self._submit_callback,
- error_callback=self._error)
+ self.get_executor(job).run(job,
+ callback=self._finish_callback,
+ submit_callback=self._submit_callback,
+ error_callback=self._error)
def _noop(self, job):
pass
@@ -255,16 +257,18 @@ class JobScheduler:
update_resources=True):
""" Do stuff after job is finished. """
with self._lock:
+ # by calling this behind the lock, we avoid race conditions
+ self.get_executor(job).handle_job_success(job)
+ self.dag.finish(job, update_dynamic=update_dynamic)
+
if update_resources:
self.finished_jobs += 1
self.running.remove(job)
self._free_resources(job)
- self.dag.finish(job, update_dynamic=update_dynamic)
-
- logger.job_finished(jobid=self.dag.jobid(job))
if print_progress:
+ logger.job_finished(jobid=self.dag.jobid(job))
self.progress()
if any(self.open_jobs) or not self.running:
@@ -279,6 +283,7 @@ class JobScheduler:
try to run the job again.
"""
with self._lock:
+ self.get_executor(job).handle_job_error(job)
self.running.remove(job)
self._free_resources(job)
self._open_jobs.set()
@@ -368,7 +373,7 @@ Problem", Akcay, Li, Xu, Annals of Operations Research, 2012
def job_reward(self, job):
return (self.dag.priority(job), self.dag.temp_input_count(job), self.dag.downstream_size(job),
- job.inputsize)
+ 0 if self.touch else job.inputsize)
def dryrun_job_reward(self, job):
return (self.dag.priority(job), self.dag.temp_input_count(job), self.dag.downstream_size(job))
diff --git a/snakemake/script.py b/snakemake/script.py
index ff39823..c9b967d 100644
--- a/snakemake/script.py
+++ b/snakemake/script.py
@@ -13,7 +13,7 @@ import traceback
import subprocess
import collections
import re
-from urllib.request import urlopen
+from urllib.request import urlopen, pathname2url
from urllib.error import URLError
from snakemake.utils import format
@@ -23,7 +23,7 @@ from snakemake.shell import shell
from snakemake.version import MIN_PY_VERSION
-PY_VER_RE = re.compile("Python (?P<ver_min>\d+\.\d+).*:")
+PY_VER_RE = re.compile("Python (?P<ver_min>\d+\.\d+).*")
# TODO use this to find the right place for inserting the preamble
PY_PREAMBLE_RE = re.compile(r"from( )+__future__( )+import.*?(?P<end>[;\n])")
@@ -33,7 +33,9 @@ class REncoder:
@classmethod
def encode_value(cls, value):
- if isinstance(value, str):
+ if value is None:
+ return "NULL"
+ elif isinstance(value, str):
return repr(value)
elif isinstance(value, dict):
return cls.encode_dict(value)
@@ -149,7 +151,7 @@ class Snakemake:
def script(path, basedir, input, output, params, wildcards, threads, resources,
- log, config, rulename, conda_env):
+ log, config, rulename, conda_env, bench_record):
"""
Load a script from the given basedir + path and execute it.
Supports Python 3 and R.
@@ -157,13 +159,20 @@ def script(path, basedir, input, output, params, wildcards, threads, resources,
if not path.startswith("http"):
if path.startswith("file://"):
path = path[7:]
+ elif path.startswith("file:"):
+ path = path[5:]
if not os.path.isabs(path):
path = os.path.abspath(os.path.join(basedir, path))
path = "file://" + path
path = format(path, stepout=1)
+ if path.startswith("file://"):
+ sourceurl = "file:"+pathname2url(path[7:])
+ else:
+ sourceurl = path
+ f = None
try:
- with urlopen(path) as source:
+ with urlopen(sourceurl) as source:
if path.endswith(".py"):
snakemake = Snakemake(input, output, params, wildcards,
threads, resources, log, config, rulename)
@@ -176,7 +185,7 @@ def script(path, basedir, input, output, params, wildcards, threads, resources,
import sys; sys.path.insert(0, "{}"); import pickle; snakemake = pickle.loads({})
######## Original script #########
""").format(searchpath, snakemake)
- elif path.endswith(".R"):
+ elif path.endswith(".R") or path.endswith(".Rmd"):
preamble = textwrap.dedent("""
######## Snakemake header ########
library(methods)
@@ -218,7 +227,7 @@ def script(path, basedir, input, output, params, wildcards, threads, resources,
}), REncoder.encode_dict(config), REncoder.encode_value(rulename))
else:
raise ValueError(
- "Unsupported script: Expecting either Python (.py) or R (.R) script.")
+ "Unsupported script: Expecting either Python (.py), R (.R) or RMarkdown (.Rmd) script.")
if path.startswith("file://"):
# in case of local path, use the same directory
@@ -234,8 +243,22 @@ def script(path, basedir, input, output, params, wildcards, threads, resources,
prefix=prefix,
dir=dir,
delete=False) as f:
- f.write(preamble.encode())
- f.write(source.read())
+ if not path.endswith(".Rmd"):
+ f.write(preamble.encode())
+ f.write(source.read())
+ else:
+ # Insert Snakemake object after the RMarkdown header
+ code = source.read().decode()
+ pos = code.rfind("---")
+ f.write(str.encode(code[:pos+3]))
+ preamble = textwrap.dedent("""
+ ```{r, echo=FALSE, message=FALSE, warning=FALSE}
+ %s
+ ```
+ """ % preamble)
+ f.write(preamble.encode())
+ f.write(str.encode(code[pos+3:]))
+
if path.endswith(".py"):
py_exec = sys.executable
if conda_env is not None:
@@ -255,10 +278,18 @@ def script(path, basedir, input, output, params, wildcards, threads, resources,
"master process to execute "
"script.".format(*MIN_PY_VERSION))
# use the same Python as the running process or the one from the environment
- shell("{py_exec} {f.name}")
+ shell("{py_exec} {f.name}", bench_record=bench_record)
elif path.endswith(".R"):
- shell("Rscript {f.name}")
- os.remove(f.name)
+ shell("Rscript {f.name}", bench_record=bench_record)
+ elif path.endswith(".Rmd"):
+ if len(output) != 1:
+ raise WorkflowError("RMarkdown scripts (.Rmd) may only have a single output file.")
+ out = os.path.abspath(output[0])
+ shell("Rscript -e 'rmarkdown::render(\"{f.name}\", output_file=\"{out}\", quiet=TRUE, params = list(rmd=\"{f.name}\"))'",
+ bench_record=bench_record)
except URLError as e:
raise WorkflowError(e)
+ finally:
+ if f:
+ os.remove(f.name)
diff --git a/snakemake/shell.py b/snakemake/shell.py
index 7ca00b7..84eba46 100644
--- a/snakemake/shell.py
+++ b/snakemake/shell.py
@@ -12,6 +12,7 @@ import inspect
from snakemake.utils import format
from snakemake.logging import logger
+
__author__ = "Johannes Köster"
STDOUT = sys.stdout
@@ -43,7 +44,8 @@ class shell:
def __new__(cls, cmd, *args,
async=False,
iterable=False,
- read=False, **kwargs):
+ read=False, bench_record=None,
+ **kwargs):
if "stepout" in kwargs:
raise KeyError("Argument stepout is not allowed in shell command.")
cmd = format(cmd, *args, stepout=2, **kwargs)
@@ -75,7 +77,13 @@ class shell:
ret = proc.stdout.read()
elif async:
return proc
- retcode = proc.wait()
+ if bench_record is not None:
+ from snakemake.benchmark import benchmarked
+ # Note: benchmarking does not work in case of async=True
+ with benchmarked(proc.pid, bench_record):
+ retcode = proc.wait()
+ else:
+ retcode = proc.wait()
if retcode:
raise sp.CalledProcessError(retcode, cmd)
return ret
diff --git a/snakemake/utils.py b/snakemake/utils.py
index 2a7777e..cf639fe 100644
--- a/snakemake/utils.py
+++ b/snakemake/utils.py
@@ -22,6 +22,15 @@ from snakemake.exceptions import WorkflowError
import snakemake
+def simplify_path(path):
+ """Return a simplified version of the given path."""
+ relpath = os.path.relpath(path)
+ if relpath.startswith("../../"):
+ return path
+ else:
+ return relpath
+
+
def linecount(filename):
"""Return the number of lines of given file.
diff --git a/snakemake/version.py b/snakemake/version.py
index 11acea2..ee2cf61 100644
--- a/snakemake/version.py
+++ b/snakemake/version.py
@@ -1,3 +1,3 @@
-__version__ = "3.10.0"
+__version__ = "3.13.3"
MIN_PY_VERSION = (3, 3)
diff --git a/snakemake/workflow.py b/snakemake/workflow.py
index c11f1fe..12c1a74 100644
--- a/snakemake/workflow.py
+++ b/snakemake/workflow.py
@@ -13,6 +13,7 @@ from collections import OrderedDict
from itertools import filterfalse, chain
from functools import partial
from operator import attrgetter
+import copy
from snakemake.logging import logger, format_resources, format_resource_names
from snakemake.rules import Rule, Ruleorder
@@ -31,6 +32,7 @@ from snakemake.wrapper import wrapper
import snakemake.wrapper
from snakemake.common import Mode
+
class Workflow:
def __init__(self,
snakefile=None,
@@ -43,10 +45,13 @@ class Workflow:
config_args=None,
debug=False,
use_conda=False,
+ conda_prefix=None,
mode=Mode.default,
wrapper_prefix=None,
printshellcmds=False,
- restart_times=None):
+ restart_times=None,
+ default_remote_provider=None,
+ default_remote_prefix=""):
"""
Create the controller.
"""
@@ -81,18 +86,19 @@ class Workflow:
self.debug = debug
self._rulecount = 0
self.use_conda = use_conda
+ self.conda_prefix = conda_prefix
self.mode = mode
self.wrapper_prefix = wrapper_prefix
self.printshellcmds = printshellcmds
self.restart_times = restart_times
+ self.default_remote_provider = default_remote_provider
+ self.default_remote_prefix = default_remote_prefix
global config
- config = dict()
- config.update(self.overwrite_config)
+ config = copy.deepcopy(self.overwrite_config)
global cluster_config
- cluster_config = dict()
- cluster_config.update(self.overwrite_clusterconfig)
+ cluster_config = copy.deepcopy(self.overwrite_clusterconfig)
global rules
rules = Rules()
@@ -201,6 +207,7 @@ class Workflow:
printrulegraph=False,
printd3dag=False,
drmaa=None,
+ drmaa_log_dir=None,
stats=None,
force_incomplete=False,
ignore_incomplete=False,
@@ -313,6 +320,7 @@ class Workflow:
self.persistence = Persistence(
nolock=nolock,
dag=dag,
+ conda_prefix=self.conda_prefix,
warn_only=dryrun or printrulegraph or printdag or summary or archive or
list_version_changes or list_code_changes or list_input_changes or
list_params_changes)
@@ -438,6 +446,9 @@ class Workflow:
if not keep_shadow:
self.persistence.cleanup_shadow()
+ if self.use_conda:
+ dag.create_conda_envs(dryrun=dryrun)
+
scheduler = JobScheduler(self, dag, cores,
local_cores=local_cores,
dryrun=dryrun,
@@ -450,6 +461,7 @@ class Workflow:
quiet=quiet,
keepgoing=keepgoing,
drmaa=drmaa,
+ drmaa_log_dir=drmaa_log_dir,
printreason=printreason,
printshellcmds=printshellcmds,
latency_wait=latency_wait,
@@ -457,7 +469,7 @@ class Workflow:
greediness=greediness,
force_use_threads=force_use_threads)
- if not dryrun and not quiet:
+ if not dryrun:
if len(dag):
if cluster or cluster_sync or drmaa:
logger.resources_info(
@@ -469,13 +481,13 @@ class Workflow:
if provided_resources:
logger.resources_info(
"Provided resources: " + provided_resources)
- ignored_resources = format_resource_names(
+ unlimited_resources = format_resource_names(set(
resource for job in dag.needrun_jobs
for resource in job.resources.keys()
- if resource not in resources)
- if ignored_resources:
+ if resource not in resources))
+ if unlimited_resources:
logger.resources_info(
- "Ignored resources: " + ignored_resources)
+ "Unlimited resources: " + unlimited_resources)
logger.run_info("\n".join(dag.stats()))
else:
logger.info("Nothing to be done.")
@@ -489,7 +501,7 @@ class Workflow:
if success:
if dryrun:
- if not quiet and len(dag):
+ if len(dag):
logger.run_info("\n".join(dag.stats()))
elif stats:
scheduler.stats.to_json(stats)
@@ -589,7 +601,8 @@ class Workflow:
def subworkflow(self, name, snakefile=None, workdir=None, configfile=None):
# Take absolute path of config file, because it is relative to current
# workdir, which could be changed for the subworkflow.
- configfile = os.path.abspath(configfile)
+ if configfile:
+ configfile = os.path.abspath(configfile)
sw = Subworkflow(self, name, snakefile, workdir, configfile)
self._subworkflows[name] = sw
self.globals[name] = sw.target
@@ -649,8 +662,12 @@ class Workflow:
if ruleinfo.benchmark:
rule.benchmark = ruleinfo.benchmark
if ruleinfo.wrapper:
- rule.conda_env = snakemake.wrapper.get_conda_env(ruleinfo.wrapper)
+ rule.conda_env = snakemake.wrapper.get_conda_env(
+ ruleinfo.wrapper, prefix=self.wrapper_prefix)
if ruleinfo.conda_env:
+ if not (ruleinfo.script or ruleinfo.wrapper or ruleinfo.shellcmd):
+ raise RuleException("Conda environments are only allowed "
+ "with shell, script or wrapper directives (not with run).", rule=rule)
if not os.path.isabs(ruleinfo.conda_env):
ruleinfo.conda_env = os.path.join(self.current_basedir, ruleinfo.conda_env)
rule.conda_env = ruleinfo.conda_env
diff --git a/snakemake/wrapper.py b/snakemake/wrapper.py
index 52b1c95..fc44d6f 100644
--- a/snakemake/wrapper.py
+++ b/snakemake/wrapper.py
@@ -23,24 +23,24 @@ def get_path(path, prefix=None):
def get_script(path, prefix=None):
- path = get_path(path)
+ path = get_path(path, prefix=prefix)
if not is_script(path):
path += "/wrapper.py"
return path
-def get_conda_env(path):
- path = get_path(path)
+def get_conda_env(path, prefix=None):
+ path = get_path(path, prefix=prefix)
if is_script(path):
# URLs and posixpaths share the same separator. Hence use posixpath here.
path = posixpath.dirname(path)
return path + "/environment.yaml"
-def wrapper(path, input, output, params, wildcards, threads, resources, log, config, rulename, conda_env, prefix):
+def wrapper(path, input, output, params, wildcards, threads, resources, log, config, rulename, conda_env, bench_record, prefix):
"""
Load a wrapper from https://bitbucket.org/snakemake/snakemake-wrappers under
the given path + wrapper.py and execute it.
"""
path = get_script(path, prefix=prefix)
- script(path, "", input, output, params, wildcards, threads, resources, log, config, rulename, conda_env)
+ script(path, "", input, output, params, wildcards, threads, resources, log, config, rulename, conda_env, bench_record)
diff --git a/environment.yml b/test-environment.yml
similarity index 71%
copy from environment.yml
copy to test-environment.yml
index 97a2a32..63b70a9 100644
--- a/environment.yml
+++ b/test-environment.yml
@@ -20,7 +20,10 @@ dependencies:
- appdirs
- pytools
- docutils
- - sphinx
- - pip:
- - sphinxcontrib-napoleon
- - sphinx_rtd_theme
+ - pandoc
+ - r-rmarkdown
+ - xorg-libxrender
+ - xorg-libxext
+ - xorg-libxau
+ - xorg-libxdmcp
+ - psutil
diff --git a/tests/test_benchmark/Snakefile b/tests/test_benchmark/Snakefile
index 4393786..e7200a0 100644
--- a/tests/test_benchmark/Snakefile
+++ b/tests/test_benchmark/Snakefile
@@ -1,11 +1,35 @@
+import time
rule all:
input:
- "test.benchmark.txt"
+ "test.benchmark_shell.txt",
+ "test.benchmark_script.txt",
+ "test.benchmark_run.txt",
+ "test.benchmark_run_shell.txt",
-rule:
+
+rule bench_shell:
benchmark:
- "{v}.benchmark.txt"
+ "test.benchmark_shell.txt"
shell:
"sleep 1"
+
+
+rule bench_script:
+ benchmark:
+ "test.benchmark_script.txt"
+ script: 'script.py'
+
+
+rule bench_run:
+ benchmark:
+ "test.benchmark_run.txt"
+ run:
+ time.sleep(1)
+
+rule bench_run_shell:
+ benchmark:
+ "test.benchmark_run_shell.txt"
+ run:
+ shell("sleep 1")
diff --git a/tests/test_benchmark/expected-results/test.benchmark.txt b/tests/test_benchmark/expected-results/test.benchmark_run.txt
similarity index 100%
copy from tests/test_benchmark/expected-results/test.benchmark.txt
copy to tests/test_benchmark/expected-results/test.benchmark_run.txt
diff --git a/tests/test_remote/__init__.py b/tests/test_benchmark/expected-results/test.benchmark_run_shell.txt
similarity index 100%
rename from tests/test_remote/__init__.py
rename to tests/test_benchmark/expected-results/test.benchmark_run_shell.txt
diff --git a/tests/test_benchmark/expected-results/test.benchmark.txt b/tests/test_benchmark/expected-results/test.benchmark_script.txt
similarity index 100%
copy from tests/test_benchmark/expected-results/test.benchmark.txt
copy to tests/test_benchmark/expected-results/test.benchmark_script.txt
diff --git a/tests/test_benchmark/expected-results/test.benchmark.txt b/tests/test_benchmark/expected-results/test.benchmark_shell.txt
similarity index 100%
copy from tests/test_benchmark/expected-results/test.benchmark.txt
copy to tests/test_benchmark/expected-results/test.benchmark_shell.txt
diff --git a/tests/test_benchmark/script.py b/tests/test_benchmark/script.py
new file mode 100755
index 0000000..77fd158
--- /dev/null
+++ b/tests/test_benchmark/script.py
@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+import sys
+import time
+
+print('Hello World!', file=sys.stderr)
+time.sleep(1)
diff --git a/tests/test_conda/Snakefile b/tests/test_conda/Snakefile
index 6f11c5b..ea431b4 100644
--- a/tests/test_conda/Snakefile
+++ b/tests/test_conda/Snakefile
@@ -18,5 +18,5 @@ rule b:
"test{i}.out2"
conda:
"test-env.yaml"
- run:
- shell("snakemake --help > {output}")
+ shell:
+ "snakemake --help > {output}"
diff --git a/tests/test_conda_custom_prefix/Snakefile b/tests/test_conda_custom_prefix/Snakefile
new file mode 100644
index 0000000..d82c326
--- /dev/null
+++ b/tests/test_conda_custom_prefix/Snakefile
@@ -0,0 +1,12 @@
+rule all:
+ input:
+ expand("test{i}.out", i=range(3))
+
+rule a:
+ output:
+ "test{i}.out"
+ conda:
+ "test-env.yaml"
+ shell:
+ "test -e custom/*/bin/snakemake && "
+ "snakemake --version > {output}"
diff --git a/tests/test_conda_custom_prefix/expected-results/test0.out b/tests/test_conda_custom_prefix/expected-results/test0.out
new file mode 100644
index 0000000..a08ffae
--- /dev/null
+++ b/tests/test_conda_custom_prefix/expected-results/test0.out
@@ -0,0 +1 @@
+3.8.2
diff --git a/tests/test_conda_custom_prefix/expected-results/test1.out b/tests/test_conda_custom_prefix/expected-results/test1.out
new file mode 100644
index 0000000..a08ffae
--- /dev/null
+++ b/tests/test_conda_custom_prefix/expected-results/test1.out
@@ -0,0 +1 @@
+3.8.2
diff --git a/tests/test_conda_custom_prefix/expected-results/test2.out b/tests/test_conda_custom_prefix/expected-results/test2.out
new file mode 100644
index 0000000..a08ffae
--- /dev/null
+++ b/tests/test_conda_custom_prefix/expected-results/test2.out
@@ -0,0 +1 @@
+3.8.2
diff --git a/tests/test_conda_custom_prefix/test-env.yaml b/tests/test_conda_custom_prefix/test-env.yaml
new file mode 100644
index 0000000..d5af59b
--- /dev/null
+++ b/tests/test_conda_custom_prefix/test-env.yaml
@@ -0,0 +1,4 @@
+channels:
+ - bioconda
+dependencies:
+ - snakemake ==3.8.2
diff --git a/tests/test_default_remote/Snakefile b/tests/test_default_remote/Snakefile
new file mode 100644
index 0000000..9f64499
--- /dev/null
+++ b/tests/test_default_remote/Snakefile
@@ -0,0 +1,40 @@
+rule all:
+ input:
+ "test.3.txt"
+
+
+def check_remote(f):
+ if not f.startswith("test-remote-bucket/"):
+ raise ValueError("Input and output are not remote files.")
+
+
+rule a:
+ input:
+ "test.txt"
+ output:
+ "test.2.txt"
+ run:
+ check_remote(input[0])
+ check_remote(output[0])
+ shell("cp {input} {output}")
+
+rule b:
+ input:
+ "test.2.txt"
+ output:
+ "test.3.txt"
+ run:
+ check_remote(input[0])
+ check_remote(output[0])
+ shell("cp {input} {output}")
+
+
+# after we finish, we need to remove the pickle storing
+# the local moto "buckets" so we are starting fresh
+# next time this test is run. This file is created by
+# the moto wrapper defined in S3Mocked.py
+onsuccess:
+ shell("rm ./motoState.p")
+
+onerror:
+ shell("rm ./motoState.p")
diff --git a/tests/test_benchmark/expected-results/test.benchmark.txt b/tests/test_default_remote/expected-results/.gitignore
similarity index 100%
copy from tests/test_benchmark/expected-results/test.benchmark.txt
copy to tests/test_default_remote/expected-results/.gitignore
diff --git a/tests/test_default_remote/test.txt b/tests/test_default_remote/test.txt
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/test_default_remote/test.txt
@@ -0,0 +1 @@
+test
diff --git a/tests/test_dynamic_temp/Snakefile b/tests/test_dynamic_temp/Snakefile
new file mode 100644
index 0000000..ed0e10c
--- /dev/null
+++ b/tests/test_dynamic_temp/Snakefile
@@ -0,0 +1,30 @@
+# Snakefile
+# each input is 8 lines
+SCRATCH = 'test'
+
+
+rule all:
+ input:
+ 'out.txt'
+ run:
+ if os.listdir('test'):
+ raise ValueError('not all temp files have been deleted')
+
+
+rule gather:
+ input:
+ dynamic('{}/split_1.{{id}}'.format(SCRATCH)),
+ dynamic('{}/split_2.{{id}}'.format(SCRATCH))
+ output: 'out.txt'
+ shell:
+ 'touch {output}'
+
+rule scatter:
+ input: 'test1.txt', 'test2.txt'
+ output:
+ temp(dynamic('{}/split_1.{{id}}'.format(SCRATCH))),
+ temp(dynamic('{}/split_2.{{id}}'.format(SCRATCH)))
+ shell:
+ ('cat {{input[0]}} | head -8 | split -a 4 -l 4 - {}/split_1.; '
+ 'cat {{input[1]}} | head -8 | split -a 4 -l 4 - {}/split_2.'
+ .format(SCRATCH, SCRATCH))
diff --git a/tests/test_benchmark/expected-results/test.benchmark.txt b/tests/test_dynamic_temp/expected-results/out.txt
similarity index 100%
copy from tests/test_benchmark/expected-results/test.benchmark.txt
copy to tests/test_dynamic_temp/expected-results/out.txt
diff --git a/tests/test_dynamic_temp/test1.txt b/tests/test_dynamic_temp/test1.txt
new file mode 100644
index 0000000..535d2b0
--- /dev/null
+++ b/tests/test_dynamic_temp/test1.txt
@@ -0,0 +1,8 @@
+1
+2
+3
+4
+5
+6
+7
+8
diff --git a/tests/test_dynamic_temp/test2.txt b/tests/test_dynamic_temp/test2.txt
new file mode 100644
index 0000000..535d2b0
--- /dev/null
+++ b/tests/test_dynamic_temp/test2.txt
@@ -0,0 +1,8 @@
+1
+2
+3
+4
+5
+6
+7
+8
diff --git a/tests/test_ftp_immediate_close/Snakefile b/tests/test_ftp_immediate_close/Snakefile
new file mode 100644
index 0000000..84490c3
--- /dev/null
+++ b/tests/test_ftp_immediate_close/Snakefile
@@ -0,0 +1,11 @@
+from snakemake.remote.FTP import RemoteProvider
+FTP = RemoteProvider()
+
+rule a:
+ input:
+ FTP.remote("ftp://ftp.sra.ebi.ac.uk/vol1/ERA651/ERA651425/fastq/RAG16_R1.fastq.gz", immediate_close=True)
+ output:
+ "size.txt"
+ run:
+ shell("du -h {input} > {output}")
+
diff --git a/tests/test_ftp_immediate_close/expected-results/size.txt b/tests/test_ftp_immediate_close/expected-results/size.txt
new file mode 100644
index 0000000..c10c432
--- /dev/null
+++ b/tests/test_ftp_immediate_close/expected-results/size.txt
@@ -0,0 +1 @@
+9.9M ftp.sra.ebi.ac.uk/vol1/ERA651/ERA651425/fastq/RAG16_R1.fastq.gz
diff --git a/tests/test_get_log_none/Snakefile b/tests/test_get_log_none/Snakefile
index 441bd7b..abcb288 100644
--- a/tests/test_get_log_none/Snakefile
+++ b/tests/test_get_log_none/Snakefile
@@ -2,4 +2,4 @@ rule:
input: "test.in"
output: "test.out"
wrapper:
- 'file://wrapper.py'
+ 'file:wrapper.py'
diff --git a/tests/test_issue260/Snakefile b/tests/test_issue260/Snakefile
new file mode 100644
index 0000000..97265cf
--- /dev/null
+++ b/tests/test_issue260/Snakefile
@@ -0,0 +1,32 @@
+rule all:
+ input:
+ 'output/result.n3',
+ output:
+ 'output/done.txt',
+ shell:
+ 'echo all >> {output}'
+
+
+rule n3:
+ input:
+ dynamic("output/{id2}.n2"),
+ output:
+ "output/result.n3",
+ shell:
+ 'echo n3 > {output}'
+
+
+rule n2:
+ input:
+ dyn=dynamic("output/{id1}.n1"),
+ output:
+ dynamic("output/{id2}.n2"),
+ shell:
+ 'echo n2 > output/result.n2'
+
+
+rule n1:
+ output:
+ dyn=dynamic("output/{id1}.n1"),
+ shell:
+ 'echo n1 > output/result.n1'
diff --git a/tests/test_issue260/expected-results/output/done.txt b/tests/test_issue260/expected-results/output/done.txt
new file mode 100644
index 0000000..0702cb5
--- /dev/null
+++ b/tests/test_issue260/expected-results/output/done.txt
@@ -0,0 +1 @@
+all
diff --git a/tests/test_issue260/expected-results/output/result.n1 b/tests/test_issue260/expected-results/output/result.n1
new file mode 100644
index 0000000..3eac62e
--- /dev/null
+++ b/tests/test_issue260/expected-results/output/result.n1
@@ -0,0 +1 @@
+n1
diff --git a/tests/test_issue260/expected-results/output/result.n2 b/tests/test_issue260/expected-results/output/result.n2
new file mode 100644
index 0000000..819d993
--- /dev/null
+++ b/tests/test_issue260/expected-results/output/result.n2
@@ -0,0 +1 @@
+n2
diff --git a/tests/test_issue260/expected-results/output/result.n3 b/tests/test_issue260/expected-results/output/result.n3
new file mode 100644
index 0000000..65cd8a6
--- /dev/null
+++ b/tests/test_issue260/expected-results/output/result.n3
@@ -0,0 +1 @@
+n3
diff --git a/tests/test_remote/Snakefile b/tests/test_remote/Snakefile
index d24ef7d..1d2d179 100644
--- a/tests/test_remote/Snakefile
+++ b/tests/test_remote/Snakefile
@@ -1,6 +1,6 @@
#import re, os, sys
-from S3Mocked import RemoteProvider as S3RemoteProvider
+from snakemake.remote.S3Mocked import RemoteProvider as S3RemoteProvider
S3 = S3RemoteProvider()
@@ -25,16 +25,16 @@ rule split:
rule cut:
input: S3.remote('test-remote-bucket/prefix{split_id,[a-z][a-z]}.txt')
- output:
+ output:
S3.remote('test-remote-bucket/{split_id}_cut.txt')
shell: 'cut -f 1,2 {input} > {output}'
rule merge:
- input:
+ input:
S3.remote(dynamic('test-remote-bucket/{split_id}_cut.txt'))
- output:
+ output:
S3.remote('test-remote-bucket/out.txt'),
- run:
+ run:
shell('echo {input}; cat {input} > {output}')
# after we finish, we need to remove the pickle storing
@@ -70,17 +70,14 @@ onerror:
# rule cut:
# input: remote('test-remote-bucket/prefix{split_id,[a-z][a-z]}.txt', provider=S3Mocked, additional_kwargs={})
-# output:
+# output:
# remote('test-remote-bucket/{split_id}_cut.txt', provider=S3Mocked, additional_kwargs={})
# shell: 'cut -f 1,2 {input} > {output}'
# rule merge:
-# input:
+# input:
# remote(dynamic('test-remote-bucket/{split_id}_cut.txt'), provider=S3Mocked, additional_kwargs={})
-# output:
+# output:
# remote('test-remote-bucket/out.txt', provider=S3Mocked, additional_kwargs={}),
-# run:
+# run:
# shell('echo {input}; cat {input} > {output}')
-
-
-
diff --git a/tests/test_remote_ncbi/Snakefile b/tests/test_remote_ncbi/Snakefile
new file mode 100644
index 0000000..96edbb4
--- /dev/null
+++ b/tests/test_remote_ncbi/Snakefile
@@ -0,0 +1,26 @@
+from snakemake.remote.NCBI import RemoteProvider as NCBIRemoteProvider
+NCBI = NCBIRemoteProvider(email="someone at example.com") # email required by NCBI to prevent abuse
+
+# get accessions for the first 3 results in a search for full-length Zika virus genomes
+# the query parameter accepts standard NCBI search syntax
+query = '"Zika virus"[Organism] AND (("9000"[SLEN] : "20000"[SLEN]) AND ("2017/03/20"[PDAT] : "2017/03/24"[PDAT])) '
+accessions = NCBI.search(query, retmax=3, return_all=False)
+
+# give the accessions a file extension to help the RemoteProvider determine the
+# proper output type.
+input_files = expand("{acc}.fasta", acc=accessions)
+
+rule all:
+ input:
+ "sizes.txt"
+
+rule download_and_count:
+ input:
+ # Since *.fasta files could come from several different databases, specify the database here.
+ # if the input files are ambiguous, the provider will alert the user with possible options
+ NCBI.remote(input_files, db="nuccore", seq_start=5000)
+
+ output:
+ "sizes.txt"
+ run:
+ shell("wc -c {input} > sizes.txt")
diff --git a/tests/test_remote_ncbi/expected-results/sizes.txt b/tests/test_remote_ncbi/expected-results/sizes.txt
new file mode 100644
index 0000000..b74d8a1
--- /dev/null
+++ b/tests/test_remote_ncbi/expected-results/sizes.txt
@@ -0,0 +1,4 @@
+ 5801 KY785484.1.fasta
+ 5255 KY785481.1.fasta
+ 5318 KY785480.1.fasta
+ 16374 total
diff --git a/tests/test_remote_ncbi_simple/Snakefile b/tests/test_remote_ncbi_simple/Snakefile
new file mode 100644
index 0000000..c66b271
--- /dev/null
+++ b/tests/test_remote_ncbi_simple/Snakefile
@@ -0,0 +1,14 @@
+from snakemake.remote.NCBI import RemoteProvider as NCBIRemoteProvider
+NCBI = NCBIRemoteProvider(email="someone at example.com") # email required by NCBI to prevent abuse
+
+rule all:
+ input:
+ "sizes.txt"
+
+rule download_and_count:
+ input:
+ NCBI.remote("KY785484.1.fasta", db="nuccore")
+ output:
+ "sizes.txt"
+ run:
+ shell("wc -c {input} > sizes.txt")
diff --git a/tests/test_remote_ncbi_simple/expected-results/sizes.txt b/tests/test_remote_ncbi_simple/expected-results/sizes.txt
new file mode 100644
index 0000000..738724a
--- /dev/null
+++ b/tests/test_remote_ncbi_simple/expected-results/sizes.txt
@@ -0,0 +1 @@
+ 10861 KY785484.1.fasta
diff --git a/tests/test_run_namedlist/Snakefile b/tests/test_run_namedlist/Snakefile
new file mode 100644
index 0000000..ad0d5a8
--- /dev/null
+++ b/tests/test_run_namedlist/Snakefile
@@ -0,0 +1,5 @@
+rule:
+ output: txt='file.txt'
+ run:
+ shell('touch {output.txt}')
+ os.stat(output.txt)
diff --git a/tests/test_benchmark/expected-results/test.benchmark.txt b/tests/test_run_namedlist/expected-results/file.txt
similarity index 100%
rename from tests/test_benchmark/expected-results/test.benchmark.txt
rename to tests/test_run_namedlist/expected-results/file.txt
diff --git a/tests/test_script/Snakefile b/tests/test_script/Snakefile
index ff9dbae..b82c41c 100644
--- a/tests/test_script/Snakefile
+++ b/tests/test_script/Snakefile
@@ -1,6 +1,11 @@
configfile: "config.yaml"
+rule all:
+ input:
+ "test.out",
+ "test.html"
+
rule:
input:
@@ -17,3 +22,11 @@ rule:
"test.in"
script:
"scripts/test.py"
+
+rule:
+ output:
+ "test.html"
+ params:
+ test="testparam"
+ script:
+ "scripts/test.Rmd"
diff --git a/tests/test_script/expected-results/test.html b/tests/test_script/expected-results/test.html
new file mode 100644
index 0000000..72f2b27
--- /dev/null
+++ b/tests/test_script/expected-results/test.html
@@ -0,0 +1,234 @@
+<!DOCTYPE html>
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+
+<head>
+
+<meta charset="utf-8">
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+<meta name="generator" content="pandoc" />
+<meta name="viewport" content="width=device-width, initial-scale=1">
+
+<meta name="author" content="Mattias" />
+
+<meta name="date" content="2017-03-22" />
+
+<title>Test Report</title>
+
+<script src="data:application/x-javascript;base64,LyohIGpRdWVyeSB2MS4xMS4zIHwgKGMpIDIwMDUsIDIwMTUgalF1ZXJ5IEZvdW5kYXRpb24sIEluYy4gfCBqcXVlcnkub3JnL2xpY2Vuc2UgKi8KIWZ1bmN0aW9uKGEsYil7Im9iamVjdCI9PXR5cGVvZiBtb2R1bGUmJiJvYmplY3QiPT10eXBlb2YgbW9kdWxlLmV4cG9ydHM/bW9kdWxlLmV4cG9ydHM9YS5kb2N1bWVudD9iKGEsITApOmZ1bmN0aW9uKGEpe2lmKCFhLmRvY3VtZW50KXRocm93IG5ldyBFcnJvcigialF1ZXJ5IHJlcXVpcmVzIGEgd2luZG93IHdpdGggYSBkb2N1bWVudCIpO3JldHVybiBiKGEpfTpiKGEpfSgidW5kZWZpbmVkIiE9dHlwZW9mIHdpbmRvdz93aW5kb3c6dG [...]
+<script src="data:application/x-javascript;base64,LyohIGpRdWVyeSBVSSAtIHYxLjExLjQgLSAyMDE2LTAxLTA1CiogaHR0cDovL2pxdWVyeXVpLmNvbQoqIEluY2x1ZGVzOiBjb3JlLmpzLCB3aWRnZXQuanMsIG1vdXNlLmpzLCBwb3NpdGlvbi5qcywgZHJhZ2dhYmxlLmpzLCBkcm9wcGFibGUuanMsIHJlc2l6YWJsZS5qcywgc2VsZWN0YWJsZS5qcywgc29ydGFibGUuanMsIGFjY29yZGlvbi5qcywgYXV0b2NvbXBsZXRlLmpzLCBidXR0b24uanMsIGRpYWxvZy5qcywgbWVudS5qcywgcHJvZ3Jlc3NiYXIuanMsIHNlbGVjdG1lbnUuanMsIHNsaWRlci5qcywgc3Bpbm5lci5qcywgdGFicy5qcywgdG9vbHRpcC5qcywgZWZmZWN0LmpzLC [...]
+<link href="data:text/css;charset=utf-8,%0A%0A%2Etocify%20%7B%0Awidth%3A%2020%25%3B%0Amax%2Dheight%3A%2090%25%3B%0Aoverflow%3A%20auto%3B%0Amargin%2Dleft%3A%202%25%3B%0Aposition%3A%20fixed%3B%0Aborder%3A%201px%20solid%20%23ccc%3B%0Awebkit%2Dborder%2Dradius%3A%206px%3B%0Amoz%2Dborder%2Dradius%3A%206px%3B%0Aborder%2Dradius%3A%206px%3B%0A%7D%0A%0A%2Etocify%20ul%2C%20%2Etocify%20li%20%7B%0Alist%2Dstyle%3A%20none%3B%0Amargin%3A%200%3B%0Apadding%3A%200%3B%0Aborder%3A%20none%3B%0Aline%2Dheight%3 [...]
+<script src="data:application/x-javascript;base64,LyoganF1ZXJ5IFRvY2lmeSAtIHYxLjkuMSAtIDIwMTMtMTAtMjIKICogaHR0cDovL3d3dy5ncmVnZnJhbmtvLmNvbS9qcXVlcnkudG9jaWZ5LmpzLwogKiBDb3B5cmlnaHQgKGMpIDIwMTMgR3JlZyBGcmFua287IExpY2Vuc2VkIE1JVCAqLwoKLy8gSW1tZWRpYXRlbHktSW52b2tlZCBGdW5jdGlvbiBFeHByZXNzaW9uIChJSUZFKSBbQmVuIEFsbWFuIEJsb2cgUG9zdF0oaHR0cDovL2JlbmFsbWFuLmNvbS9uZXdzLzIwMTAvMTEvaW1tZWRpYXRlbHktaW52b2tlZC1mdW5jdGlvbi1leHByZXNzaW9uLykgdGhhdCBjYWxscyBhbm90aGVyIElJRkUgdGhhdCBjb250YWlucyBhbGwgb2YgdG [...]
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<link href="data:text/css;charset=utf-8,html%7Bfont%2Dfamily%3Asans%2Dserif%3B%2Dwebkit%2Dtext%2Dsize%2Dadjust%3A100%25%3B%2Dms%2Dtext%2Dsize%2Dadjust%3A100%25%7Dbody%7Bmargin%3A0%7Darticle%2Caside%2Cdetails%2Cfigcaption%2Cfigure%2Cfooter%2Cheader%2Chgroup%2Cmain%2Cmenu%2Cnav%2Csection%2Csummary%7Bdisplay%3Ablock%7Daudio%2Ccanvas%2Cprogress%2Cvideo%7Bdisplay%3Ainline%2Dblock%3Bvertical%2Dalign%3Abaseline%7Daudio%3Anot%28%5Bcontrols%5D%29%7Bdisplay%3Anone%3Bheight%3A0%7D%5Bhidden%5D%2Ctem [...]
+<script src="data:application/x-javascript;base64,LyohCiAqIEJvb3RzdHJhcCB2My4zLjUgKGh0dHA6Ly9nZXRib290c3RyYXAuY29tKQogKiBDb3B5cmlnaHQgMjAxMS0yMDE1IFR3aXR0ZXIsIEluYy4KICogTGljZW5zZWQgdW5kZXIgdGhlIE1JVCBsaWNlbnNlCiAqLwppZigidW5kZWZpbmVkIj09dHlwZW9mIGpRdWVyeSl0aHJvdyBuZXcgRXJyb3IoIkJvb3RzdHJhcCdzIEphdmFTY3JpcHQgcmVxdWlyZXMgalF1ZXJ5Iik7K2Z1bmN0aW9uKGEpeyJ1c2Ugc3RyaWN0Ijt2YXIgYj1hLmZuLmpxdWVyeS5zcGxpdCgiICIpWzBdLnNwbGl0KCIuIik7aWYoYlswXTwyJiZiWzFdPDl8fDE9PWJbMF0mJjk9PWJbMV0mJmJbMl08MSl0aHJvdy [...]
+<script src="data:application/x-javascript;base64,LyoqCiogQHByZXNlcnZlIEhUTUw1IFNoaXYgMy43LjIgfCBAYWZhcmthcyBAamRhbHRvbiBAam9uX25lYWwgQHJlbSB8IE1JVC9HUEwyIExpY2Vuc2VkCiovCi8vIE9ubHkgcnVuIHRoaXMgY29kZSBpbiBJRSA4CmlmICghIXdpbmRvdy5uYXZpZ2F0b3IudXNlckFnZW50Lm1hdGNoKCJNU0lFIDgiKSkgewohZnVuY3Rpb24oYSxiKXtmdW5jdGlvbiBjKGEsYil7dmFyIGM9YS5jcmVhdGVFbGVtZW50KCJwIiksZD1hLmdldEVsZW1lbnRzQnlUYWdOYW1lKCJoZWFkIilbMF18fGEuZG9jdW1lbnRFbGVtZW50O3JldHVybiBjLmlubmVySFRNTD0ieDxzdHlsZT4iK2IrIjwvc3R5bGU+IixkLm [...]
+<script src="data:application/x-javascript;base64,LyohIFJlc3BvbmQuanMgdjEuNC4yOiBtaW4vbWF4LXdpZHRoIG1lZGlhIHF1ZXJ5IHBvbHlmaWxsICogQ29weXJpZ2h0IDIwMTMgU2NvdHQgSmVobAogKiBMaWNlbnNlZCB1bmRlciBodHRwczovL2dpdGh1Yi5jb20vc2NvdHRqZWhsL1Jlc3BvbmQvYmxvYi9tYXN0ZXIvTElDRU5TRS1NSVQKICogICovCgovLyBPbmx5IHJ1biB0aGlzIGNvZGUgaW4gSUUgOAppZiAoISF3aW5kb3cubmF2aWdhdG9yLnVzZXJBZ2VudC5tYXRjaCgiTVNJRSA4IikpIHsKIWZ1bmN0aW9uKGEpeyJ1c2Ugc3RyaWN0IjthLm1hdGNoTWVkaWE9YS5tYXRjaE1lZGlhfHxmdW5jdGlvbihhKXt2YXIgYixjPWEuZG [...]
+
+
+
+
+
+</head>
+
+<body>
+
+<style type="text/css">
+.main-container {
+ max-width: 940px;
+ margin-left: auto;
+ margin-right: auto;
+}
+code {
+ color: inherit;
+ background-color: rgba(0, 0, 0, 0.04);
+}
+img {
+ max-width:100%;
+ height: auto;
+}
+h1 {
+ font-size: 34px;
+}
+h1.title {
+ font-size: 38px;
+}
+h2 {
+ font-size: 30px;
+}
+h3 {
+ font-size: 24px;
+}
+h4 {
+ font-size: 18px;
+}
+h5 {
+ font-size: 16px;
+}
+h6 {
+ font-size: 12px;
+}
+.tabbed-pane {
+ padding-top: 12px;
+}
+button.code-folding-btn:focus {
+ outline: none;
+}
+</style>
+
+
+<div class="container-fluid main-container">
+
+<!-- tabsets -->
+<script src="data:application/x-javascript;base64,Cgp3aW5kb3cuYnVpbGRUYWJzZXRzID0gZnVuY3Rpb24odG9jSUQpIHsKCiAgLy8gYnVpbGQgYSB0YWJzZXQgZnJvbSBhIHNlY3Rpb24gZGl2IHdpdGggdGhlIC50YWJzZXQgY2xhc3MKICBmdW5jdGlvbiBidWlsZFRhYnNldCh0YWJzZXQpIHsKCiAgICAvLyBjaGVjayBmb3IgZmFkZSBhbmQgcGlsbHMgb3B0aW9ucwogICAgdmFyIGZhZGUgPSB0YWJzZXQuaGFzQ2xhc3MoInRhYnNldC1mYWRlIik7CiAgICB2YXIgcGlsbHMgPSB0YWJzZXQuaGFzQ2xhc3MoInRhYnNldC1waWxscyIpOwogICAgdmFyIG5hdkNsYXNzID0gcGlsbHMgPyAibmF2LXBpbGxzIiA6ICJuYXYtdGFicyI7CgogIC [...]
+<script>
+$(document).ready(function () {
+ window.buildTabsets("TOC");
+});
+</script>
+
+<!-- code folding -->
+
+
+
+
+<script>
+$(document).ready(function () {
+ // establish options
+ var options = {
+ selectors: "h1,h2,h3",
+ theme: "bootstrap3",
+ context: '.toc-content',
+ hashGenerator: function (text) {
+ return text.replace(/[.\/?&!#<>]/g, '').replace(/\s/g, '_').toLowerCase();
+ },
+ ignoreSelector: "h1.title, .toc-ignore",
+ scrollTo: 0
+ };
+ options.showAndHide = false;
+ options.smoothScroll = true;
+
+ // tocify
+ var toc = $("#TOC").tocify(options).data("toc-tocify");
+});
+</script>
+
+<style type="text/css">
+
+#TOC {
+ margin: 25px 0px 20px 0px;
+}
+ at media (max-width: 768px) {
+#TOC {
+ position: relative;
+ width: 100%;
+}
+}
+
+.toc-content {
+ padding-left: 30px;
+ padding-right: 40px;
+}
+
+div.main-container {
+ max-width: 1200px;
+}
+
+div.tocify {
+ width: 20%;
+ max-width: 260px;
+ max-height: 85%;
+}
+
+ at media (min-width: 768px) and (max-width: 991px) {
+ div.tocify {
+ width: 25%;
+ }
+}
+
+ at media (max-width: 767px) {
+ div.tocify {
+ width: 100%;
+ max-width: none;
+ }
+}
+
+.tocify ul, .tocify li {
+ line-height: 20px;
+}
+
+.tocify-subheader .tocify-item {
+ font-size: 0.9em;
+ padding-left: 5px;
+}
+
+.tocify .list-group-item {
+ border-radius: 0px;
+}
+
+.tocify-subheader {
+ display: inline;
+}
+.tocify-subheader .tocify-item {
+ font-size: 0.95em;
+ padding-left: 10px;
+}
+
+</style>
+
+<!-- setup 3col/9col grid for toc_float and main content -->
+<div class="row-fluid">
+<div class="col-xs-12 col-sm-4 col-md-3">
+<div id="TOC" class="tocify">
+</div>
+</div>
+
+<div class="toc-content col-xs-12 col-sm-8 col-md-9">
+
+
+
+
+<div class="fluid-row" id="header">
+
+
+<h1 class="title">Test Report</h1>
+<h4 class="author"><em>Mattias</em></h4>
+<h4 class="date"><em>March 22, 2017</em></h4>
+
+</div>
+
+
+<div id="r-markdown" class="section level2">
+<h2>R Markdown</h2>
+<p>This is an R Markdown document.</p>
+<p>Test include from snakemake testparam.</p>
+</div>
+
+
+
+</div>
+</div>
+
+</div>
+
+<script>
+
+// add bootstrap table styles to pandoc tables
+$(document).ready(function () {
+ $('tr.header').parent('thead').parent('table').addClass('table table-condensed');
+});
+
+</script>
+
+<!-- dynamically load mathjax for compatibility with self-contained -->
+<script>
+ (function () {
+ var script = document.createElement("script");
+ script.type = "text/javascript";
+ script.src = "https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";
+ document.getElementsByTagName("head")[0].appendChild(script);
+ })();
+</script>
+
+</body>
+</html>
diff --git a/tests/test_script/scripts/test.Rmd b/tests/test_script/scripts/test.Rmd
new file mode 100644
index 0000000..d9ded9a
--- /dev/null
+++ b/tests/test_script/scripts/test.Rmd
@@ -0,0 +1,22 @@
+---
+title: "Test Report"
+author: "Mattias"
+date: "March 22, 2017"
+output:
+ html_document:
+ highlight: tango
+ number_sections: no
+ theme: default
+ toc: yes
+ toc_depth: 3
+ toc_float:
+ collapsed: no
+ smooth_scroll: yes
+---
+
+
+## R Markdown
+
+This is an R Markdown document.
+
+Test include from snakemake `r snakemake at params[["test"]]`.
diff --git a/tests/test_subworkflows/Snakefile b/tests/test_subworkflows/Snakefile
index 35e112a..58ff0d0 100644
--- a/tests/test_subworkflows/Snakefile
+++ b/tests/test_subworkflows/Snakefile
@@ -7,7 +7,12 @@ subworkflow test02:
rule:
input: "test.out"
+
+def test_in(wildcards):
+ return test02("test.out")
+
+
rule:
- input: test02("test.out")
+ input: test_in
output: "test.out"
shell: "cp {input} {output}"
diff --git a/tests/test_symlink_time_handling/Snakefile b/tests/test_symlink_time_handling/Snakefile
index c878ef1..af61482 100644
--- a/tests/test_symlink_time_handling/Snakefile
+++ b/tests/test_symlink_time_handling/Snakefile
@@ -27,20 +27,22 @@
"""
import os
-
import time
-def timestr(hr_delta=0):
- return time.strftime("%Y%m%d%H%M",
- time.localtime(time.time() - (hr_delta * 3600)))
-shell("touch -t {} input_file".format(timestr(1)))
-shell("ln -s input_file input_link")
-shell("touch -h -t {} input_link".format(timestr(4)))
+#Slightly crude way to stop this running twice
+if not os.path.exists("input_file"):
+ def timestr(hr_delta=0):
+ return time.strftime("%Y%m%d%H%M",
+ time.localtime(time.time() - (hr_delta * 3600) - 5))
+
+ shell("touch -t {} input_file".format(timestr(1)))
+ shell("ln -s input_file input_link")
+ shell("touch -h -t {} input_link".format(timestr(4)))
-shell("ln -s input_link output_link")
-shell("touch -h -t {} output_link".format(timestr(2)))
+ shell("ln -s input_link output_link")
+ shell("touch -h -t {} output_link".format(timestr(2)))
-shell("ls -lR > /dev/stderr")
+ shell("ls -lR > /dev/stderr")
rule main:
output: "time_diff.txt"
diff --git a/tests/test_xrootd/Snakefile b/tests/test_xrootd/Snakefile
new file mode 100644
index 0000000..e1e3c86
--- /dev/null
+++ b/tests/test_xrootd/Snakefile
@@ -0,0 +1,27 @@
+from snakemake.remote.XRootD import RemoteProvider as XRootDRemoteProvider
+
+XRootD = XRootDRemoteProvider(stay_on_remote=True)
+
+remote_path = 'root://eospublic.cern.ch//eos/opendata/lhcb/MasterclassDatasets/'
+my_files, = XRootD.glob_wildcards(remote_path+'D0lifetime/2014/mclasseventv2_D0_{n}.root')
+
+rule all:
+ input:
+ expand('access_remotely/mclasseventv2_D0_{n}.root', n=my_files),
+ expand('access_locally/mclasseventv2_D0_{n}.root', n=my_files)
+
+rule access_remotely:
+ input:
+ XRootD.remote(remote_path+'D0lifetime/2014/mclasseventv2_D0_{n}.root')
+ output:
+ 'access_remotely/mclasseventv2_D0_{n}.root'
+ shell:
+ 'xrdcp {input[0]} {output[0]}'
+
+rule access_locally:
+ input:
+ XRootD.remote(remote_path+'D0lifetime/2014/mclasseventv2_D0_{n}.root', stay_on_remote=False)
+ output:
+ 'access_locally/mclasseventv2_D0_{n}.root'
+ shell:
+ 'cp {input[0]} {output[0]}'
diff --git a/tests/tests.py b/tests/tests.py
index f27d531..2f03d3f 100644
--- a/tests/tests.py
+++ b/tests/tests.py
@@ -7,7 +7,7 @@ import sys
import os
from os.path import join
from subprocess import call
-from tempfile import mkdtemp
+import tempfile
import hashlib
import urllib
from shutil import rmtree, which
@@ -63,8 +63,7 @@ def run(path,
assert os.path.exists(snakefile)
assert os.path.exists(results_dir) and os.path.isdir(
results_dir), '{} does not exist'.format(results_dir)
- tmpdir = mkdtemp(prefix=".test", dir=os.path.abspath("."))
- try:
+ with tempfile.TemporaryDirectory(prefix=".test", dir=os.path.abspath(".")) as tmpdir:
config = {}
if subpath is not None:
# set up a working directory for the subworkflow and pass it in `config`
@@ -104,8 +103,6 @@ def run(path,
assert md5sum(targetfile) == md5sum(
expectedfile), 'wrong result produced for file "{}"'.format(
resultfile)
- finally:
- rmtree(tmpdir)
def test01():
@@ -370,6 +367,12 @@ def test_conda():
run(dpath("test_conda"), use_conda=True)
+def test_conda_custom_prefix():
+ if conda_available():
+ run(dpath("test_conda_custom_prefix"),
+ use_conda=True, conda_prefix="custom")
+
+
def test_wrapper():
if conda_available():
run(dpath("test_wrapper"), use_conda=True)
@@ -419,6 +422,25 @@ def test_static_remote():
except ImportError:
pass
+def test_remote_ncbi_simple():
+ try:
+ import Bio
+
+ # only run the remote file test if the dependencies
+ # are installed, otherwise do nothing
+ run(dpath("test_remote_ncbi_simple"))
+ except ImportError:
+ pass
+
+def test_remote_ncbi():
+ try:
+ import Bio
+
+ # only run the remote file test if the dependencies
+ # are installed, otherwise do nothing
+ run(dpath("test_remote_ncbi"))
+ except ImportError:
+ pass
def test_deferred_func_eval():
run(dpath("test_deferred_func_eval"))
@@ -488,6 +510,30 @@ def test_threads():
run(dpath("test_threads"), cores=20)
+def test_dynamic_temp():
+ run(dpath("test_dynamic_temp"))
+
+def test_ftp_immediate_close():
+ try:
+ import ftputil
+
+ # only run the remote file test if the dependencies
+ # are installed, otherwise do nothing
+ run(dpath("test_ftp_immediate_close"))
+ except ImportError:
+ pass
+
+def test_issue260():
+ run(dpath("test_issue260"))
+
+def test_default_remote():
+ run(dpath("test_default_remote"),
+ default_remote_provider="S3Mocked",
+ default_remote_prefix="test-remote-bucket")
+
+def test_run_namedlist():
+ run(dpath("test_run_namedlist"))
+
if __name__ == '__main__':
import nose
nose.run(defaultTest=__name__)
diff --git a/wercker.yml b/wercker.yml
index eb288c4..cbdd59d 100644
--- a/wercker.yml
+++ b/wercker.yml
@@ -6,6 +6,10 @@ box:
build:
steps:
- script:
+ name: env
+ code: conda env update --name root --file test-environment.yml
+
+ - script:
name: pip
code: pip install -e .
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/snakemake.git
More information about the debian-med-commit
mailing list