[med-svn] [Git][med-team/covpipe][upstream] New upstream version 3.0.6

[Andreas Tille (@tille)]

Andreas Tille pushed to branch upstream at Debian Med / covpipe


Commits:
f50c03dc by Andreas Tille at 2021-09-15T09:38:43+02:00
New upstream version 3.0.6
- - - - -


26 changed files:

- + .conda/3.0.3/build.sh
- + .conda/3.0.3/meta.yaml
- + .conda/3.0.4/build.sh
- + .conda/3.0.4/meta.yaml
- + .conda/3.0.5/build.sh
- + .conda/3.0.5/meta.yaml
- .conda/meta.yaml
- .gitlab-ci.yml
- Dockerfile
- README.md
- covpipe/__version__.py
- covpipe/ncov_minipipe.Rmd
- covpipe/ncov_minipipe.config
- covpipe/ncov_minipipe.py
- covpipe/ncov_minipipe.snake
- covpipe/rules/assign_lineage.smk
- covpipe/rules/classify_reads.smk
- covpipe/rules/get_version.smk
- covpipe/rules/trim_reads.smk
- covpipe_env.yaml
- + covpipe_tools/primerbed2bedpe.py
- covpipe_tools/update_pangolin.py
- ncov_minipipe.conda.setup/setup_env.sh
- setup.py
- tests/TestData/GEN_SAMPLES/mixture.csv
- tests/quicktest.sh


Changes:

=====================================
.conda/3.0.3/build.sh
=====================================
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+$PYTHON setup.py install 


=====================================
.conda/3.0.3/meta.yaml
=====================================
@@ -0,0 +1,34 @@
+{% set version = "3.0.3" %}
+package:
+  name: covpipe
+  version: {{ version }}
+
+source:
+  git_url: https://gitlab.com/RKIBioinformaticsPipelines/ncov_minipipe.git
+  git_rev: "v{{ version }}"
+
+
+build:
+    number: 0     
+
+
+requirements:
+  host:
+    - python
+    - pip
+  run:
+    - python
+    - snakemake >=5.26
+    - strictyaml
+    - pandas
+
+
+
+about:
+  home: https://gitlab.com/RKIBioinformaticsPipelines/ncov_minipipe.git
+  license: GPLv3
+  summary: 'Sars-Cov-2 NGS Pipeline'
+  description: |
+     NGS Anlysis of Sars-Cov-2 
+  dev_url: https://gitlab.com/RKIBioinformaticsPipelines/ncov_minipipe
+  doc_source_url: https://gitlab.com/RKIBioinformaticsPipelines/ncov_minipipe/-/blob/v{{ version }}/README.md 


=====================================
.conda/3.0.4/build.sh
=====================================
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+$PYTHON setup.py install 


=====================================
.conda/3.0.4/meta.yaml
=====================================
@@ -0,0 +1,34 @@
+{% set version = "3.0.4" %}
+package:
+  name: covpipe
+  version: {{ version }}
+
+source:
+  git_url: https://gitlab.com/RKIBioinformaticsPipelines/ncov_minipipe.git
+  git_rev: "v{{ version }}"
+
+
+build:
+    number: 0     
+
+
+requirements:
+  host:
+    - python
+    - pip
+  run:
+    - python
+    - snakemake >=5.26
+    - strictyaml
+    - pandas
+
+
+
+about:
+  home: https://gitlab.com/RKIBioinformaticsPipelines/ncov_minipipe.git
+  license: GPLv3
+  summary: 'Sars-Cov-2 NGS Pipeline'
+  description: |
+     NGS Anlysis of Sars-Cov-2 
+  dev_url: https://gitlab.com/RKIBioinformaticsPipelines/ncov_minipipe
+  doc_source_url: https://gitlab.com/RKIBioinformaticsPipelines/ncov_minipipe/-/blob/v{{ version }}/README.md 


=====================================
.conda/3.0.5/build.sh
=====================================
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+$PYTHON setup.py install 


=====================================
.conda/3.0.5/meta.yaml
=====================================
@@ -0,0 +1,34 @@
+{% set version = "3.0.5" %}
+package:
+  name: covpipe
+  version: {{ version }}
+
+source:
+  git_url: https://gitlab.com/RKIBioinformaticsPipelines/ncov_minipipe.git
+  git_rev: "v{{ version }}"
+
+
+build:
+    number: 0     
+
+
+requirements:
+  host:
+    - python
+    - pip
+  run:
+    - python
+    - snakemake >=5.26
+    - strictyaml
+    - pandas
+
+
+
+about:
+  home: https://gitlab.com/RKIBioinformaticsPipelines/ncov_minipipe.git
+  license: GPLv3
+  summary: 'Sars-Cov-2 NGS Pipeline'
+  description: |
+     NGS Anlysis of Sars-Cov-2 
+  dev_url: https://gitlab.com/RKIBioinformaticsPipelines/ncov_minipipe
+  doc_source_url: https://gitlab.com/RKIBioinformaticsPipelines/ncov_minipipe/-/blob/v{{ version }}/README.md 


=====================================
.conda/meta.yaml
=====================================
@@ -1,4 +1,4 @@
-{% set version = "3.0.3" %}
+{% set version = "3.0.6" %}
 package:
   name: covpipe
   version: {{ version }}


=====================================
.gitlab-ci.yml
=====================================
@@ -5,22 +5,26 @@ cache:
     - $CI_PROJECT_DIR/.conda_cache
     - $CI_PROJECT_DIR/.apt_cache
 before_script:
-  - conda update -y -n base conda
   - conda install -y -c conda-forge mamba
+  - mamba update -y -n base conda
   - mamba install -y conda-build
   - export APT_DIR=$CI_PROJECT_DIR/.apt_cache && export APT_STATE_LISTS=$APT_DIR/lists && export APT_CACHE_ARCHIVES=$APT_DIR/archives
   - printf "dir::state::lists    ${APT_STATE_LISTS};\ndir::cache::archives    ${APT_CACHE_ARCHIVES};\n" > /etc/apt/apt.conf
   - mkdir -p "${APT_STATE_LISTS}/partial" && mkdir -p "${APT_CACHE_ARCHIVES}/partial"
 
 stages:
-#  - build_env
   - quicktest
-  - pypi-gitlab
-  - git-release
-#  - simple_amplidata_test
+  - test
+  - build_pypackage
+  - migrate_to_prerelease
+  - deploy_git_release
+  - deploy_pypi_upload
+  - deploy_gitlab_index_upload
+  - deploy_bioconda_upload
+  - deploy_docker_build
 
 #simple_amplidata_test:
-#  stage: simple_amplidata_test
+#  stage: test
 #  timeout: 1h
 #  script:
 #    - ./tests/simple_amplidata_test/test.sh
@@ -36,6 +40,7 @@ stages:
 quicktest:
   only:
     - development_unstable
+    - test_pre_release
     - pre_release
     - next_milestone
     - master
@@ -52,47 +57,128 @@ quicktest:
         - "tests/QuickTest/test02/sample.config.yaml"
         - "tests/QuickTest/test03/output/sample.config.yaml"
 
-upload_gitlab_pypi:
+
+build_pip:
   only: 
     - master
-  stage: pypi-gitlab
+    - stage_pip_release
+  stage: build_pypackage
   script:
-    - pip install twine
+    - mamba env update -n base -f covpipe_env.yaml 
     - python setup.py sdist bdist_wheel
-    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/*
-    - TWINE_PASSWORD=${PYPI_RELEASE_TOKEN} TWINE_USERNAME=__token__ python -m twine upload --repository-url https://upload.pypi.org/legacy/ dist/*
+  artifacts:
+    paths:
+      - dist/*
 
-build_docker:
-  image: docker:19.03.12
-  services:
-    - docker:19.03.12-dind
-  only:
-    - stage_docker_build 
-  stage: git-release
-  before_script:
-    - echo "Do Nothing"
-  script:
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - docker build -t registry.gitlab.com/rkibioinformaticspipelines/ncov_minipipe .
-    - docker push registry.gitlab.com/rkibioinformaticspipelines/ncov_minipipe -p 
 
 
 make_new_release:
   only:
     - master
-  stage: git-release
+  stage: deploy_git_release
   before_script:
     - apt-get update
     - apt-get -y install software-properties-common
     - apt-get update
     - apt-get -y install php php-json git openssh-client curl
   script:
-    - VERSION="$( git tag | grep -E 'v[0-9]+\.[0-9]+\.[0-9]+$' | tail -n1 )" 
+    - git fetch --tags origin
+    - VERSION="$( git tag -l | grep -E 'v[0-9]+\.[0-9]+\.[0-9]+$' | tail -n1 )" 
     - DESCRIPTION="$( git cat-file -p $VERSION )"
-    - MESSAGE="$(php -r "echo json_encode(['name' => '$VERSION', 'tag_name' => '$VERSION', 'description' => '$DESCRIPTION']);")";
+    - SAFE_DESCR="$(sed 's/\([^\\]\)'"'"'/\1\\'"'/g" <<< "$DESCRIPTION")"
+    - MESSAGE="$(php -r "echo json_encode(['name' => '$VERSION', 'tag_name' => '$VERSION', 'description' => '$SAFE_DESCR']);")";
     - echo $MESSAGE
     - >
       curl --header 'Content-Type: application/json' --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \
            --data "$MESSAGE" \
            --request POST ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/releases
-      
+
+
+trigger_pre_release:
+  only:
+    - test_pre_release
+  stage: migrate_to_prerelease
+  script:
+    - mkdir -pvm 0700 $HOME/.ssh
+    - ssh-keyscan -t rsa gitlab.com >> $HOME/.ssh/known_hosts
+    - echo "$NeedToKnow" > $HOME/.ssh/id_rsa-gitlab-ci
+    - ls -la $HOME/.ssh/id_rsa-gitlab-ci
+    - chmod 0400 $HOME/.ssh/id_rsa-gitlab-ci
+    - git fetch
+    - git checkout pre_release
+    - git remote set-url --push origin $(perl -pe 's#.*@(.+?(\:\d+)?)/#git@\1:#' <<< $CI_REPOSITORY_URL)
+    - git remote -v
+    - git merge origin/test_pre_release --no-edit
+    - GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -i $HOME/.ssh/id_rsa-gitlab-ci" git push
+  after_script:
+    - rm -Rfv .ssh
+
+
+trigger_pypi_stage:
+  only:
+    - master
+  stage: deploy_git_release
+  script:
+    - mkdir -pvm 0700 $HOME/.ssh
+    - ssh-keyscan -t rsa gitlab.com >> $HOME/.ssh/known_hosts
+    - echo "$NeedToKnow" > $HOME/.ssh/id_rsa-gitlab-ci
+    - ls -la $HOME/.ssh/id_rsa-gitlab-ci
+    - chmod 0400 $HOME/.ssh/id_rsa-gitlab-ci
+    - git fetch
+    - git checkout stage_pip_release
+    - git remote set-url --push origin $(perl -pe 's#.*@(.+?(\:\d+)?)/#git@\1:#' <<< $CI_REPOSITORY_URL)
+    - git remote -v
+    - git merge origin/master --no-edit
+    - GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -i $HOME/.ssh/id_rsa-gitlab-ci" git push
+  after_script:
+    - rm -Rfv .ssh
+
+trigger_docker_stage:
+  only:
+    - master
+  stage: deploy_git_release
+  script:
+    - mkdir -pvm 0700 $HOME/.ssh
+    - ssh-keyscan -t rsa gitlab.com >> $HOME/.ssh/known_hosts
+    - echo "$NeedToKnow" > $HOME/.ssh/id_rsa-gitlab-ci
+    - ls -la $HOME/.ssh/id_rsa-gitlab-ci
+    - chmod 0400 $HOME/.ssh/id_rsa-gitlab-ci
+    - git fetch
+    - git checkout stage_docker_release
+    - git remote set-url --push origin $(perl -pe 's#.*@(.+?(\:\d+)?)/#git@\1:#' <<< $CI_REPOSITORY_URL)
+    - git remote -v
+    - git merge origin/master --no-edit
+    - GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -i $HOME/.ssh/id_rsa-gitlab-ci" git push
+  after_script:
+    - rm -Rfv .ssh
+
+upload_gitlab_pypi:
+  only: 
+    - stage_pip_release
+  stage: deploy_git_release
+  script:
+    - pip install twine
+    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/*
+
+upload_pypi:
+  only: 
+    - stage_pip_release
+  stage: deploy_pypi_upload
+  script:
+    - pip install twine
+    - TWINE_PASSWORD=${PYPI_RELEASE_TOKEN} TWINE_USERNAME=__token__ python -m twine upload --repository-url https://upload.pypi.org/legacy/ dist/*
+
+
+build_docker:
+  image: docker:19.03.12
+  services:
+    - docker:19.03.12-dind
+  only:
+    - stage_docker_release 
+  stage: deploy_docker_build
+  before_script:
+    - echo "Do Nothing"
+  script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker build -t registry.gitlab.com/rkibioinformaticspipelines/ncov_minipipe .
+    - docker push registry.gitlab.com/rkibioinformaticspipelines/ncov_minipipe 


=====================================
Dockerfile
=====================================
@@ -1,5 +1,5 @@
 FROM continuumio/miniconda3
-ENV VERSION 3.0.1.post2
+ENV VERSION 3.0.4
 ENV TOOL covpipe
 
 # meta data
@@ -11,32 +11,31 @@ LABEL about.home="https://gitlab.com/RKIBioinformaticsPipelines/ncov_minipipe"
 
 LABEL maintainer="RKI MF1 Bioinformatics <https://www.rki.de/EN/Content/Institute/DepartmentsUnits/MF/MF1/mf1_node.html>"
 
-# install basics
-RUN apt update && apt install -y procps wget gzip pigz bc build-essential libbz2-dev && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-RUN pip install Cython
-RUN conda install -c bioconda -c conda-forge python=3.7 mamba
-
-# install covpipe latest version
-RUN pip install covpipe==${VERSION} 
-
 
+# install basics & covpipe
+RUN apt update && apt install -y procps wget gzip pigz bc build-essential libbz2-dev && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN pip install Cython \
+    && conda install -c bioconda -c conda-forge python=3.7 mamba \
+    && pip install covpipe==${VERSION} \
+    && mamba clean -a \
+    && chmod -R a+w /opt/conda
 
 # install all environments according to snakemakes default path and hashing procedure
 COPY hash.py . 
-RUN chmod +x hash.py
-RUN for YAML in $(ls /opt/conda/lib/python3.7/site-packages/covpipe/envs/*.yaml);\
+RUN chmod +x hash.py \
+    && for YAML in $(ls /opt/conda/lib/python3.7/site-packages/covpipe/envs/*.yaml);\
         do export YAML; MD5=$(python hash.py | cut -c 1-8) \
            && mamba env create --prefix /.snakemake/conda/$MD5 -f $YAML \
            && rm -rf /tmp/* \
            && rm -rf /root/.conda/pkgs/* \
            && rm -rf /opt/conda/pkgs/* \
            && cp $YAML /.snakemake/conda/$MD5.yaml;\
-        done 
-
-# clean conda (keep image at least a bit smaller)
-RUN conda clean -a
+        done \
+   && mamba clean -a \
+   && chmod -R a+w /opt/conda && chmod -R a+w /.snakemake
 
 # integrate kraken database into the container (will increase container size by ~4 GB!)
-RUN wget -q -O kraken_db.tar.gz https://zenodo.org/record/4534746/files/GRCh38.p13_SC2_2021-02-08.tar.gz?download=1
-RUN tar zxvf kraken_db.tar.gz
-RUN rm kraken_db.tar.gz
+RUN wget -q -O kraken_db.tar.gz https://zenodo.org/record/4534746/files/GRCh38.p13_SC2_2021-02-08.tar.gz?download=1 \
+    && tar zxvf kraken_db.tar.gz \
+    && rm kraken_db.tar.gz \
+    && chmod -R a+rX /GRCh38.p13_SC2_2021-02-08


=====================================
README.md
=====================================
@@ -224,9 +224,40 @@ the same for the inserts, you can transform this into the desired bed file.
 create_bedpe AMPLICON_FILE.bed INSERT_FILE.bed -o myPrimerPositions.bedpe
 ```
 
-## 4.3 Activating taxonomic read filtering
+### 4.2.2 Create bedpe from primer bed file
 
-## 4.3 Activating taxonomic Read Filtering
+You might have a primer bed file, like the one provided in the artic repository.
+If the primer names have clear identifiers for forward and reverse primers youb can use the following script.
+
+```bash
+bed2bedpe nCoV-2019.primer.bed --output nCoV-2019.primer.bedpe
+```
+
+Take care: If you use artic you need to change the reference name to NC_045512.2, when using our default settings. 
+
+## 4.3 Adjusting Read Filtering
+
+### 4.3.1 Read Length
+
+Per default the minimum read length filter is set to 50. 
+We chose this, because we designed the pipeline with 159 BP pared end amplicons in mind. 
+We actually encountered a bunch of primer dimers that were shorter than 50 BP
+and found them enough of a nuisance to take this step.
+(Quick aside: We use the mapping rate as a quick quality measure and that gets polluted by the dimers.) 
+If you are using shorter amplicóns in your protocol, you will most likely need
+to set this option to something lower.
+
+(--filter_read_len 25) 
+
+### 4.3.2 Read Quality
+
+Qualitative read quality used by fastp to filter reads.
+By default the "--read_filter_qual" option uses a phredscore of 20 as a cutoff.
+Prior to version 3.0.4 it was hard coded to 15. 
+If you want to recreate your pipeline runs from prior versions, set this option
+to 15. 
+
+## 4.4 Activating taxonomic Read Filtering
 
 If necessary, reads not derived from SARS-COV-2 can be excluded. 
 Read classification is based on corresponding k-mer frequencies using a defined kraken2 database (`--kraken`). 
@@ -246,7 +277,7 @@ ncov_minipipe --kraken path/to/myKrakenDbFolder \
               -o path/to/myOutputFolder
 ```
 
-## 4.4 Adapting variant calling
+## 4.5 Adapting variant calling
 
 Sites considered for variant calling can be restricted based on the following parameters at the respective position.
 
@@ -264,7 +295,8 @@ ncov_minipipe --var_call_cov 20 \
               -o path/to/myOutputFolder
 ```
 
-## 4.5 Adapting variant filtering
+
+## 4.6 Adapting variant filtering
 
 Variants can be excluded, if the mean mapping quality of observed alternate alleles does not meet a given threshold (`--var_filter_mqm`; default: 40).
 The mapping quality measures how good reads align to the respective reference genome region. Good mapping qualities are around MQ 60. 
@@ -294,7 +326,7 @@ ncov_minipipe --var_filter_mqm 40 \
               -o path/to/myOutputFolder
 ```
 
-## 4.6 Adapting consensus generation
+## 4.7 Adapting consensus generation
 
 When generating the consensus sequence, all positions whose read coverage is below a defined threshold can be hard-masked by N (`--cns_min_cov`; default: 20). 
 In addtion, genotypes can be adjusted meaning that variants supported by a given fraction of all reads covering the respective site are called explicitely (`--cns_gt_adjust`; default: 0.9). 
@@ -310,7 +342,7 @@ ncov_minipipe --cns_min_cov 10 \
               -o path/to/myOutputFolder
 ```
 
-## 4.7 Lineage assignment 
+## 4.8 Lineage assignment 
 
 The generated consensus sequences can be submitted to a preliminary lineage assignment. 
 This will run [Pangolin](ttps://github.com/cov-lineages/pangolin) on the IUPAC consensus sequence.
@@ -329,7 +361,7 @@ ncov_minipipe --pangolin path/to/pangolin_conda_environment \
 
 ```
 
-### 4.7.1 Easy update or install of pangolin database
+### 4.8.1 Easy update or install of pangolin database
 
 You can easily install and or update pangolin, 
 if you have installed covpipe you should have access to the `update_pangolin` script. 


=====================================
covpipe/__version__.py
=====================================
@@ -3,10 +3,9 @@ import os
 from sys import version_info
 MAJOR=3
 MINOR=0
-PATCH=3
+PATCH=6
 VERSION="{MAJOR}.{MINOR}.{PATCH}".format(MAJOR=MAJOR, MINOR=MINOR, PATCH=PATCH)
 
-ADVANCED_VERSION = VERSION
 BRANCH="master"
 COMMIT=None
 AHEAD=0
@@ -37,17 +36,6 @@ def get_more_info():
     finally:
         os.chdir(old_cwd)
 
-
-def get_version():
-    global ADVANCED_VERSION
-    get_more_info()
-    ADVANCED_VERSION ="{vers}-{ahead}-{commit}-{branch}".format(
-            vers=VERSION, ahead=AHEAD, 
-            commit="release_commit" if COMMIT is None else COMMIT,
-            branch=BRANCH)
-    return ADVANCED_VERSION
-
-
 def mod_string(string):
     if (version_info > (3, 0)):
         return string.decode()


=====================================
covpipe/ncov_minipipe.Rmd
=====================================
@@ -628,8 +628,10 @@ Plotting the fragment sizes usually shows a distribution with multiple peaks. Id
 
 <div class=superhighimage>
 
-```{r plot_fragsizes, fig.width=10, fig.height=plot.height, fig.cap="Fragment size distribution determined by the distance of reads after mapping to reference genome."}
-ggplot(dt.fragsizes, aes(fragsize.abs, colour = sample, fill = sample)) +
+```{r plot_fragsizes, fig.width=10, fig.height=plot.height, fig.cap="Fragment size distribution determined by the distance of reads after mapping to reference genome.", error=T}
+
+tryCatch({
+    ggplot(dt.fragsizes, aes(fragsize.abs, colour = sample, fill = sample)) +
     geom_density() +
     facet_wrap(~sample, ncol = 4, scales = "free_y") +
     theme(axis.text.x = element_text(angle = 45, hjust = 1),
@@ -638,6 +640,9 @@ ggplot(dt.fragsizes, aes(fragsize.abs, colour = sample, fill = sample)) +
     scale_x_continuous(trans = "log10") +
     labs(title = "Fragment size histogram",
          x = "fragment size")
+    }, error = function(e) {
+        message("some error")
+    })
 
 ```
 


=====================================
covpipe/ncov_minipipe.config
=====================================
@@ -108,6 +108,22 @@ var_filter_sap: 60
 var_filter_qual: 10
 
 
+################################################################################
+# Read Filter                                                                  #
+################################################################################
+
+# Provide the qualified quality value as a Phread score.
+# This parameter was set to 15 prior to version 3.0.4
+
+read_filter_qual: 20
+
+# The minimum read length. We designed the pipeline with a roughly 150 BP
+# amplicon kit in mind. If you are using it for example with 
+# one of the illumina protocols generating 75 BP reads 
+# this should be adjusted down. In the latter case 25 might be a good target
+
+read_filter_len: 50
+
 ################################################################################
 # CONSENSUS GENERATION                                                         #
 ################################################################################


=====================================
covpipe/ncov_minipipe.py
=====================================
@@ -4,23 +4,26 @@ from __future__ import print_function
 from sys import version_info
 from datetime import datetime, timedelta
 from . import __version__ as version
+
 import argparse
-import logging
-import time
-import numpy as np
 import glob
 import hashlib
-import pandas
+import json
+import logging
+import numpy as np
 import os
+import pandas
 import pprint
 import re
+import shlex
 import strictyaml
 import subprocess
-import shlex
 import sys
+import time
+import traceback
+import urllib3
 import yaml
 
-
 version.get_more_info()
 
 #   > 1. Parser Setup
@@ -128,6 +131,29 @@ ILLUMINA_LEGAL = "Oligonucleotide sequences © 2020 Illumina, Inc."
 workdir_is_set = False
 config_is_set = False
 
+LEGACY_YAML_CONFIG_SCHEMA =  strictyaml.Map(
+    {"samples": strictyaml.Str(),
+     "output": strictyaml.Str(),
+     "reference": strictyaml.Str(),
+     "run_id": strictyaml.Str(),
+     "var_annotation": strictyaml.Bool(),
+     "cns_annotation": strictyaml.Str(), 
+     "pangolin": strictyaml.Str(),
+     "primer": strictyaml.Str(),
+     "adapter": strictyaml.Str(),
+     "var_special_interest": strictyaml.Str(),
+     "var_call_cov": strictyaml.Int(),
+     "var_call_count": strictyaml.Int(),
+     "var_call_frac": strictyaml.Float(),
+     "var_filter_mqm": strictyaml.Int(),
+     "var_filter_sap": strictyaml.Int(),
+     "var_filter_qual": strictyaml.Int(),
+     strictyaml.Optional("read_filter_len"): strictyaml.Int(),
+     strictyaml.Optional("read_filter_qual"): strictyaml.Int(),
+     "cns_min_cov": strictyaml.Int(),
+     "cns_gt_adjust": strictyaml.Float(),
+     "krakenDb": strictyaml.Str(),
+     "krakenTaxID": strictyaml.Int()})
 
 # Settings for Default Config Schema
 STRICT_YAML_CONFIG_SCHEMA = strictyaml.Map(
@@ -147,6 +173,8 @@ STRICT_YAML_CONFIG_SCHEMA = strictyaml.Map(
      "var_filter_mqm": strictyaml.Int(),
      "var_filter_sap": strictyaml.Int(),
      "var_filter_qual": strictyaml.Int(),
+     "read_filter_len": strictyaml.Int(),
+     "read_filter_qual": strictyaml.Int(),
      "cns_min_cov": strictyaml.Int(),
      "cns_gt_adjust": strictyaml.Float(),
      "krakenDb": strictyaml.Str(),
@@ -244,7 +272,7 @@ class ToolInfo():
             self.path = response
             self.tool_found = True
         except:
-            raise ToolNotFoundError("Could not find Tool", self)
+            raise ToolInfoNotFoundError("Could not find Tool", self)
 
 
     def get_version(self):
@@ -265,9 +293,18 @@ class ToolInfo():
 
 def get_snakemake_info():
     return ToolInfo("snakemake", ["snakemake", "--version"]) 
-    
+
+def has_mamba():
+    ret = False
+    try:
+        ToolInfo("mamba", ["mamba", "-V"])
+        ret = True
+    except ToolInfoNotFoundError:
+        pass
+    return ret
 
 def main(snakemake=None,cmd=None):
+    supress_error = False
     if snakemake is not None:
         cmd = get_snakecmd(snakemake) # Snakemake Support in >Utility Section<
     version_string = ("Version: {vers} - {ahead} ({commit}) {branch}".format(
@@ -299,6 +336,14 @@ def main(snakemake=None,cmd=None):
                              logging.DEBUG 
                              if args.debug 
                              else logging.INFO)
+            newer_version = get_newer_version()
+            if newer_version is not None:
+                logger.warning("!!There is a newer version available!!\n"
+                               "You are using         {vers}\n"
+                               "The newest release is {new_vers}\n\n"
+                               "Install it with \"pip install covpipe --upgrade\""
+                               "".format(vers=version.VERSION, 
+                                         new_vers=newer_version))
             logger.info("\n"
                         "  ################################\n"
                         "  #                              #\n"
@@ -363,6 +408,7 @@ def main(snakemake=None,cmd=None):
             #logger.exception(e)
             logger.error("\n\nSnakemake Stdout Log:\n{elog}".format(elog=e.stdout))
             logger.error("\n\nSnakemake Error Log:\n{elog}".format(elog=e.stderr))
+            supress_error = True
             raise e
     except QUIT_SAFE_ERROR:
         logger.error("\nQuitting because program encountered Error")
@@ -383,10 +429,32 @@ def main(snakemake=None,cmd=None):
                "incoming+rkibioinformaticspipelines-ncov-minipipe-17578161-issue- at incoming.gitlab.com\n"
                )
 
-        logger.exception(e)
+        if not supress_error:
+            logger.exception(e)
         exit(2)
 
 
+def get_newer_version(proxy_par=None):
+    retval = None
+    url = "http://gitlab.com/api/v4/projects/17578161/releases"
+    try:
+        if proxy_par is None:
+            proxy_par = os.getenv("HTTP_PROXY")
+        if proxy_par is not None:
+            proxy = urllib3.ProxyManager(proxy_par)
+            r = proxy.request("GET", url)
+        else:
+            http = urllib3.PoolManager()
+            r = http.request("GET", url)
+        data = json.loads(r.data)
+        new_vers_string = data[0]["tag_name"].strip("v")
+        new_vers = tuple(new_vers_string.split("."))
+        if tuple(version.VERSION.strip("v").split(".")) < new_vers:  
+            retval = new_vers_string
+    except json.decoder.JSONDecodeError:
+        pass
+    return retval
+
 
 #   # 1. Parser Setup
 
@@ -492,6 +560,14 @@ def get_parser(version_string="Version: {vers} "
                         help="Specify a common directory to store conda environments."
                              " Default is using the value of the environment variable SNAKE_CONDA_PREFIX."
                              " If said env var is unset, use the current working directory")
+    readgroup = parser.add_argument_group("Read Filter Options")
+    readgroup.add_argument("--read_filter_len", default=50, type=int,
+                           help="Minimum Read length. "
+                                "Defaults to 50 BP "
+                                "and should be lowered when using very short read protocols")
+    readgroup.add_argument("--read_filter_qual", default=20, type=int,
+                           help="Quality threshold for qualified quality "
+                                 "of read as a Phredscore. Default: 20")
     vargroup = parser.add_argument_group("Variation Calling in Filter Options")
     vargroup.add_argument("--var_call_cov", default=20, type=int, 
                           help="Minimum coverage value to use for variant call."
@@ -708,7 +784,8 @@ def check_pangolin(prefix, name):
                 """eval "$(conda shell.bash hook)"; {call}""".format(
                     call=" ".join(call)),
                 shell=True,
-                stderr=subprocess.STDOUT, 
+                stderr=subprocess.STDOUT,
+                executable="/bin/bash",
                 universal_newlines=True)
         is_conda = True
         env_list_lines = env_list.split("\n")
@@ -737,9 +814,10 @@ def prefix_augment(prefix):
 def setup_log_dir(args):
     outdir = args.output
     if args.log is None:
-        log_prefix = os.path.join(outdir, "ncov-minipipe-logs") 
+        log_prefix = os.path.join(outdir, "results", 
+                                  "intermediate_data", "00_logs", "_main_log") 
         mkdir_if_not_exists(log_prefix) 
-        logfile = "ncov-minipipe" 
+        logfile = "covpipe" 
         file_prefix = os.path.join(log_prefix,logfile)
         if os.path.isfile(file_prefix +".log"):
             logfile= "{_file}_{timestamp}.log".format(
@@ -991,6 +1069,7 @@ def build_config_from_args(args):
     if args.output:
         config_location=args.output
     schema = STRICT_YAML_CONFIG_SCHEMA
+    fallbackSchema = LEGACY_YAML_CONFIG_SCHEMA 
     found_config = False
     sample_conf_path = None
     config_file_path = None
@@ -999,13 +1078,19 @@ def build_config_from_args(args):
     with open(DEFAULT_CONFIG_TEMPLATE, "r") as conf_fh:
         try:
             config = strictyaml.load(conf_fh.read(), schema)
-        except strictyaml.exceptions.YAMLValidationError as e:
-            context = e.args[2]
-            logger.error("Error in Config:\n{before}\n"
-                         " #vvvv Error Line vvvv#{lines}".format(lines=context.lines(),
-                                                                 before=context.lines_before(10)))
-            logger.exception(e)
-            raise e 
+        except strictyaml.exceptions.YAMLValidationError as og_e:
+            try: 
+                conf_fh.seek(0)
+                config = strictyaml.load(conf_fh.read(), fallbackSchema) 
+            except strictyaml.exceptions.YAMLValidationError as e:
+                context = og_e.args[2]
+                logger.error(
+                    "Error in Config:\n{before}\n"
+                    " #vvvv Error Line vvvv#{lines}".format(
+                        lines=context.lines(),
+                        before=context.lines_before(10)))
+                logger.exception(og_e)
+                raise og_e 
         sample_conf_path=config["samples"].data
     # Check user provided
     if args.conf is not None:
@@ -1146,6 +1231,9 @@ def augment_config_with_args(args, config):
     config["var_filter_mqm"] = args.var_filter_mqm
     config["var_filter_sap"] = args.var_filter_sap
     config["var_filter_qual"] = args.var_filter_qual
+    
+    config["read_filter_len"] = args.read_filter_len
+    config["read_filter_qual"] = args.read_filter_qual
     config["cns_min_cov"] = args.cns_min_cov
     config["cns_gt_adjust"] = args.cns_gt_adjust
     if args.var_special_interest:
@@ -1532,10 +1620,13 @@ def prep_snake_call(args, rerun_dict, snake_cmd, name="Change"):
             outs, err = prep_proc.communicate()
             ret = prep_proc.wait()
             if ret!=0:
-                handle_snakemake_syntax_error(
-                        "{err}\n{out}".format(err=mod_string(err),
-                                              out=mod_string(outs)),
-                                              args.blame)
+                try:
+                    handle_snakemake_syntax_error(
+                            "{err}\n{out}".format(err=mod_string(err),
+                                                  out=mod_string(outs)),
+                                                  args.blame)
+                except AttributeError:
+                    pass
                 raise SNAKEMAKE_RUN_ERROR(ret, call=' '.join(snake_cmd),
                                           stdout=mod_string(outs),
                                           stderr=mod_string(err))
@@ -1564,13 +1655,16 @@ def real_snake_call(args, unknown_args, conf, samp_conf, rerun_list):
     if rerun_list:
         rerun_list = ["-R"] + rerun_list
     conda_prefix = []
+    conda_frontend = ["--conda-frontend", "mamba" if has_mamba() else "conda"] 
     if args.conda_prefix is not None:
         conda_prefix = ["-d", args.output,
                         '--conda-prefix', args.conda_prefix]
     snake_cmd=(BASIC_SNAKE_CALL + [ "--cores", str(args.cpus) ] +  ["--configfile", conf] +
                rerun_list + unknown_args + ["--use-conda"] + conda_prefix)
+    snake_cmd.extend(conda_frontend)
     short_snake_cmd =(BASIC_SNAKE_CALL + [ "--cores", str(args.cpus) ] +  ["--configfile", conf] +
                       unknown_args + ["--use-conda"] + conda_prefix)
+    short_snake_cmd.extend(conda_frontend)
     if unknown_args:
         logger.info("Arguments not known are being appended to the snakemake call.\n"
                     "Please refer to the snakemake help for issues encountered.\n"
@@ -1648,11 +1742,11 @@ def handle_snakemake_syntax_error(string, blame):
         if blame and match:
             match=match.groupdict()
             lnum = int(match["line"])
-            if lnum < 19:
-                bounds = "1,20"
-            else:
-                bounds = "{start},{end}".format(start=lnum-10,
-                                                end=lnum+10)
+            max_lnum = lnum+10
+            with open(match["file"], "r") as fh:
+                max_lnum = len(fh.readlines())
+            bounds = "{start},{end}".format(start=max(lnum-10,1),
+                                            end=min(lnum+10, max_lnum))
             old_cwd = os.getcwd()
             try:
                 os.chdir(SOURCE_DIR)


=====================================
covpipe/ncov_minipipe.snake
=====================================
@@ -13,7 +13,7 @@ import os
 import pprint
 import sys
 import yaml
-import __version__ as _vers_
+
 
 # DEBUGGING
 ## use as DEBUG(variable) for debugging at runtime
@@ -21,6 +21,12 @@ pp = pprint.PrettyPrinter(indent=4)
 DEBUG = pp.pprint
 
 
+def default_if_not(key, _dict, default):
+    try:
+        return _dict[key]
+    except KeyError:
+        return default
+
 # INPUT CHECK
 ## preparing checks
 def checkConfigKey(key, config):
@@ -140,6 +146,10 @@ VAR_FILTER_MQM = config['var_filter_mqm']
 VAR_FILTER_SAP = config['var_filter_sap']
 VAR_FILTER_QUAL = config['var_filter_qual']
 
+## read filtering
+READ_FILTER_QUAL = default_if_not("read_filter_qual", config, 20)
+READ_FILTER_LEN = default_if_not("read_filter_len", config, 50)
+
 ## consensus generation
 CNS_MIN_COV = config['cns_min_cov']
 CNS_GT_ADJUST = config["cns_gt_adjust"] if checkConfigKey('cns_gt_adjust', config) else None
@@ -210,6 +220,7 @@ def input_all(wildcards):
     if PANGOLIN:
         for sample in SAMPLES:
             files.append(os.path.join(DATAFOLDER["lineages"], sample, sample + ".lineage.txt"))
+        files.append(os.path.join(DATAFOLDER["lineages"], "all_samples.lineage.txt"))
 
     ## variant annotation
     if VAR_VOI != "no valid vcf input provided":


=====================================
covpipe/rules/assign_lineage.smk
=====================================
@@ -18,3 +18,15 @@ rule assign_lineage:
             conda activate {params.pangolin_env} &> {log}
             pangolin --outdir {params.dir} --outfile {params.fname} --tempdir {params.dir} --threads {threads} {input.fasta} &>> {log}
         """
+
+rule fuse_lineage:
+    input:
+        expand(os.path.join(DATAFOLDER["lineages"], "{sample}",
+               "{sample}.lineage.txt"), sample=SAMPLES)
+    output:
+        os.path.join(DATAFOLDER["lineages"], "all_samples.lineage.txt")
+    shell:
+        r"""
+        head -n1 {input[0]} > {output}
+        ls -1 {input} | xargs -I '{{}}' tail -n +2 {{}} >> {output}
+        """


=====================================
covpipe/rules/classify_reads.smk
=====================================
@@ -29,4 +29,5 @@ rule classifyReads:
                 --report {output.kraken_report} \
                 --gzip-compressed \
                 {input.PE1} {input.PE2} ) 2> {log}
+            #echo > {output.PE1} ; echo > {output.PE2} 
         """


=====================================
covpipe/rules/get_version.smk
=====================================
@@ -8,10 +8,8 @@ rule getVersion:
         sd = srcdir(".")
     log:
         os.path.join(DATAFOLDER["logs"], "version", "git.repository.version.log")
-    run:
-        with open(output[0],"w") as out_fh:
-            print(_vers_.get_version(), file=out_fh)
-#        r"""0
-#            cd {params.sd};
-#            git describe 1> {output} 2> {log} || echo 'unknown_version' 1> {output} 2> {log}
-#        """
+    shell:
+        r"""
+            cd {params.sd};
+            git describe 1> {output} 2> {log} || echo 'unknown_version' 1> {output} 2> {log}
+        """
\ No newline at end of file


=====================================
covpipe/rules/trim_reads.smk
=====================================
@@ -19,26 +19,40 @@ rule trimReads:
     log:
         os.path.join(DATAFOLDER["logs"], "trimming", "{sample}.log")
     params:
-        adapters = "--adapter_fasta " + ADAPTERS if ADAPTERS else ""
+        adapters = "--adapter_fasta " + ADAPTERS if ADAPTERS else "",
+        reqlen = ("--length_required {}".format(READ_FILTER_LEN) 
+                  if READ_FILTER_LEN != -1
+                  else ""),
+        qualfilter = "--qualified_quality_phred {}".format(READ_FILTER_QUAL) 
     conda:
         "../envs/fastp.yaml"
     threads:
         1
     shell:
         r"""
-            ( fastp \
-                --in1 {input[0]} \
-                --out1 {output.PE1} \
-                --in2 {input[1]} \
-                --out2 {output.PE2} \
-                --unpaired1 {output.SE1} \
-                --unpaired2 {output.SE2} \
-                --json {output.json} \
-                --html {output.html} \
+            
+            ( 
+                conda list -p $CONDA_PREFIX | grep fastp
+                set -x
+                fastp \
+                    --in1 {input[0]} \
+                    --out1 {output.PE1} \
+                    --in2 {input[1]} \
+                    --out2 {output.PE2} \
+                    --unpaired1 {output.SE1} \
+                    --unpaired2 {output.SE2} \
+                    --json {output.json} \
+                    --html {output.html} \
                 {params.adapters} \
-                --qualified_quality_phred 15 \
-                --length_required 50 \
+                {params.qualfilter} \
+                {params.reqlen} \
                 --low_complexity_filter \
                 --overrepresentation_analysis \
-                --thread {threads} ) &> {log}
+                --thread {threads} || {{
+                    ret=$?
+                    cp {log} $( sed -E 's/(.+).log/\1.err.log/' <<< {log} )
+                    exit $ret
+                }} 
+                set +x 
+            ) &> {log}
         """


=====================================
covpipe_env.yaml
=====================================
@@ -4,7 +4,7 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - snakemake>=5.26
+  - snakemake >=5.26, <6.1.0
   - pysam
   - biopython
   - strictyaml


=====================================
covpipe_tools/primerbed2bedpe.py
=====================================
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+import argparse
+import pandas as pd
+import re
+
+version="0.0.1"
+
+
+def main(cmd=None, snakemake=None):
+    parser = get_argparser()
+    cmd = from_snakemake(cmd, snakemake)
+    args = parser.parse_args(cmd)
+    reformat_bedpe(args)
+    
+
+
+
+def get_argparser():
+    parser = argparse.ArgumentParser("bed2bedpe")
+    parser.description = (
+            "Tool to quickly transform a primer bed file to a bedpe\n"
+            "It uses the naming sheme of primers to asign them.\n "
+            "amp1(_LEFT)\n"
+            "amp2(_RIGHT)\n"
+            "{_LEFT|_RIGHT} are the default identifiers the tool looks for. "
+            "When multiple pairs are possible for an amplicon,"
+            " the cartesian product is used.\n"
+            "Multiple paired end records with the same id will be outputted\n"
+            " and the score field will be incremented for each duplicate")
+    parser.add_argument("primer_bed", help="bed file with primer positions")
+    parser.add_argument("--forward_identifier", default="_LEFT",
+                        help="identifier used to identify left amplicon primer."
+                             " default: _LEFT")
+    parser.add_argument("--reverse_identifier", default="_RIGHT",
+                        help="identifier used to identify right amplicon primer."
+                             " default: _RIGHT")
+    parser.add_argument("-o","--output",default="primer.bedpe", 
+            help="Output file name.  default: primer.bedpe")
+    return parser
+
+
+def from_snakemake(cmd, snakemake):
+    return cmd
+
+def reformat_bedpe(args):
+    bed_file = args.primer_bed
+    outfile = args.output
+    for_id = args.forward_identifier
+    rev_id = args.reverse_identifier
+    bed =  pd.read_csv(bed_file, sep="\t",  
+                       names=["chrom","start","end","name","score", "strand"])
+    pattern = re.compile("(.+)(%s|%s)(.*)" % (for_id,rev_id))
+    func = lambda x: split_pattern(x, pattern, for_id, rev_id)
+    parse_bed_ids = pd.concat(list(bed.iloc[:,3].apply(func))) 
+    parse_bed_ids = pd.merge(bed, parse_bed_ids,how="left", left_on="name", right_on="key")
+    with open(outfile,"w") as o_fh:
+        for group_id, keys in parse_bed_ids.groupby("id"):
+            (left, l_data), (right, r_data) = keys.groupby("reverse") 
+            for (idx,(li,ri)) in enumerate([(x,y) for x in range(len(l_data)) for y in range(len(r_data))]):
+                out = []
+                l_temp = l_data.iloc[li,:]
+                r_temp = r_data.iloc[ri,:]
+                out.extend(list(l_temp[["chrom","start","end"]]))
+                out.extend(list(r_temp[["chrom","start","end"]]))
+                out.append(group_id)
+                out.append(idx)
+                out.extend(list(l_temp["strand"]))
+                out.extend(list(r_temp["strand"]))
+                print(*out, sep="\t", file=o_fh)
+
+
+def split_pattern(val, pattern, for_id, rev_id):
+    ret = {"key":val,
+           "id": None,
+           "reverse": False,
+           "orient_id": None,
+           "add": None}
+    match = pattern.match(val)
+    if match is not None:
+        ret["id"] = match.groups()[0]
+        ret["reverse"] = match.groups()[1] in rev_id
+        ret["orient_id"] = match.groups()[1]
+        ret["add"] = match.groups()[2]
+    return pd.DataFrame(ret, index=[ret["key"]])
+
+
+if __name__ == "__main__":
+    main(cmd=None)


=====================================
covpipe_tools/update_pangolin.py
=====================================
@@ -50,7 +50,7 @@ def main(CMD=None, snakemake=None):
         exit(2)
     try:
         if not args.use_conda_default:
-            prefix = os.path.join(covpipe_default_location, args.name)
+            prefix = args.prefix or os.path.join(covpipe_default_location, args.name)
             print("Checking Environment Availability:\n", prefix)
         tool_availability()
         has_conda, parsed_prefix, found_pangolin = (
@@ -202,6 +202,10 @@ def git_manual_update(prefix):
             except subprocess.CalledProcessError as e:
                 print(e.output)
                 raise 
+            run_unsafe_command_in_pangolin(
+                    prefix,
+                    "python -m pip install {git} --upgrade".format(
+                        git="git+https://github.com/cov-lineages/constellations.git"))
             print("Updating PangoLearn...")
             run_unsafe_command_in_pangolin(
                     prefix,
@@ -218,7 +222,7 @@ def run_unsafe_command_in_pangolin(prefix, cmd):
     with pushd(prefix):
         out = subprocess.check_output(
                 """ eval "$(conda shell.bash hook)"; conda activate ./ && {cmd} && conda deactivate""".format(cmd=cmd),
-                shell=True)
+                shell=True, executable='/bin/bash')
 
 
 


=====================================
ncov_minipipe.conda.setup/setup_env.sh
=====================================
@@ -49,7 +49,7 @@ update_conda_or_install(){
     $CONDA_FRONTEND install \
         -c conda-forge  -c bioconda -c defaults \
         "python>=3.6.0" \
-        "snakemake>=5.26" \
+        "snakemake>=5.26,<6.1" \
         "pysam" \
         "biopython" \
         strictyaml \


=====================================
setup.py
=====================================
@@ -15,7 +15,7 @@ setup(
     url="https://gitlab.com/RKIBioinformaticsPipelines/ncov_minipipe",
     tests_require=["pytest","PyYAML", "pandas"],
     install_requires=[
-        "snakemake>5.26",
+        "snakemake >5.26, <7",
         "strictyaml",
         "PyYAML",
         "biopython",
@@ -27,6 +27,7 @@ setup(
             'covpipe = covpipe.ncov_minipipe:main',
             'ncov_minipipe = covpipe.ncov_minipipe:main', 
             'create_bedpe = covpipe_tools.create_bedpe:main',
+            'bed2bedpe = covpipe_tools.primerbed2bedpe:main',
             'update_pangolin = covpipe_tools.update_pangolin:main'
         ]},
     include_package_data = True,


=====================================
tests/TestData/GEN_SAMPLES/mixture.csv
=====================================
@@ -5,3 +5,4 @@ Sample_4_r150_MSv3;WUHAN1_RAW_ALLVARS_50X_r159_MSv3;WUHAN1_NOVAR_MSv3_RAW_50X_L1
 Sample_5_r150_MSv3;WUHAN1_RAW_ALLVARS_50X_r159_MSv3;WUHAN1_NOVAR_MSv3_RAW_50X_L150;.30
 Sample_6_r150_MSv3;WUHAN1_RAW_ALLVARS_50X_r159_MSv3;WUHAN1_NOVAR_MSv3_RAW_50X_L150;.20
 Sample_7_r150_MSv3;Sample_2_r150_MSv3;ColdCorona_MSv3_RAW_50X_L150;.89
+Sample_8_r150_MSv3;ColdCorona_MSv3_RAW_50X_L150;ColdRhino_MSv3_RAW_50X_L150;.50


=====================================
tests/quicktest.sh
=====================================
@@ -56,6 +56,7 @@ stop_setup(){
 
 main(){
     local PROXY=""
+    local kraken=0
     snake_args=( "" ) 
 	local covpipe_package_exists=0
 	if conda list | grep covpipe ; then
@@ -68,7 +69,7 @@ main(){
     fi
 
     local skipconda=${covpipe_package_exists}
-    while getopts "p:sn" flag; do
+    while getopts "p:snk" flag; do
         case "$flag" in 
             p) PROXY=$OPTARG
                ;;
@@ -76,6 +77,8 @@ main(){
                ;;
             n) snake_args=( "${snake_args[@]}" "-n")
                ;;
+            k) kraken=1
+               ;;
         esac  
     done
     shift $(( $OPTIND - 1 ))  
@@ -117,12 +120,27 @@ run_tests(){
     activate_conda "$TESTENV"
     IFS=';' read -a READS  <<< "$("$GEN_SCR" Sample_7 gen)"
     IFS=';' read -a READS_TEST02  <<< "$("$GEN_SCR" Sample_6 gen)"
+    IFS=';' read -a READS_TEST04  <<< "$("$GEN_SCR" Sample_8 gen)"
     local REF="$( http_proxy=$http_proxy $REF_SCR SarsCov2Wuhan1 gen)"
     local VCF="$($VCF_SCR VCF_ALL_VARS gen)"
     local adapter="${GIT_ROOT}/adapter.fasta"
     deactivate_conda
     local testid=test01
     mkdir -p $testid
+
+    if [ $covpipe_package_exists -eq 0 ]; then  
+    	activate_conda "$PROGENV"
+	fi
+    #test01
+    test02   
+    test03
+    if [ $kraken -eq 1 ]; then
+        test04
+    fi
+    deactivate_conda
+}
+
+test01(){
     local testpath="$(realpath $testid)"
     local testsampleconf="${testpath}/sample.conf.yaml"
     local testconf="${testpath}/ncov_minipipe.config"
@@ -145,13 +163,13 @@ run_tests(){
             "    read2: \"${READS[1]}\""\
             > "$testsampleconf"
 
-    if [ $covpipe_package_exists -eq 0 ]; then  
-    	activate_conda "$PROGENV"
-	fi
-    #echo ncov_minipipe --conf "$testconf" -o $testid \
-    #    --conda_prefix "${TESTENVDIR}" --blame
-    #ncov_minipipe --conf "$testconf" -o $testid \
-    #    --conda_prefix "${TESTENVDIR}" --blame ${snake_args[@]}
+    echo ncov_minipipe --conf "$testconf" -o $testid \
+        --conda_prefix "${TESTENVDIR}" --blame
+    ncov_minipipe --conf "$testconf" -o $testid \
+        --conda_prefix "${TESTENVDIR}" --blame ${snake_args[@]}
+}
+
+test02(){
     echo ncov_minipipe --input ${READS_TEST02[@]} -o test02 --reference $REF \
         --conda_prefix "${TESTENVDIR}"  --blame
     ncov_minipipe --input ${READS_TEST02[@]} --reason -o test02 --reference $REF \
@@ -160,8 +178,10 @@ run_tests(){
     ncov_minipipe --reason \
         --conf "$( echo test02/*ncov_minipipe.config | cut -d' ' -f1 )" \
         --conda_prefix "${TESTENVDIR}"  --blame  ${snake_args[@]} --debug \
-    #  test3
 
+}
+
+test03(){
     mkdir -p test03/input
     mkdir -p test03/input2
     cp -rf ${READS_TEST02[0]} test03/input/200102_20-01222_nk0_S1_L001_R1_001.fastq.gz 
@@ -175,9 +195,22 @@ run_tests(){
 
     ncov_minipipe --input test03/input --input test03/input2 -o test03/output --reference $REF \
         --conda_prefix "${TESTENVDIR}" --reason --run_id "run01-202001" \
-        --blame ${snake_args[@]} --debug
+        --blame ${snake_args[@]} --debug --cpus 2
+}
+
+test04(){
+    mkdir -p krakendb
+    pushd krakendb
+    if [ ! -d GRCh*/ ]; then
+        wget -nc https://zenodo.org/record/4534746/files/GRCh38.p13_SC2_2021-02-08.tar.gz
+        tar -xzvf *.tar.gz -C ./
+    fi
+
+    popd 
+    ncov_minipipe --input ${READS_TEST04[@]} --reason -o test04 --reference $REF \
+        --conda_prefix "${TESTENVDIR}"  --blame  ${snake_args[@]} --debug \
+        --kraken "$( readlink -f krakendb/GRCh*/)"
 
-    deactivate_conda
 }
 
 



View it on GitLab: https://salsa.debian.org/med-team/covpipe/-/commit/f50c03dcc729b49f2ad4700cf2a335330f233250

-- 
View it on GitLab: https://salsa.debian.org/med-team/covpipe/-/commit/f50c03dcc729b49f2ad4700cf2a335330f233250
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20210915/3d4b2aea/attachment-0001.htm>