[med-svn] [Git][med-team/idseq-dag][master] 4 commits: Make examples available for test
Andreas Tille
gitlab at salsa.debian.org
Wed Apr 15 16:03:46 BST 2020
Andreas Tille pushed to branch master at Debian Med / idseq-dag
Commits:
aae9c062 by Andreas Tille at 2020-04-15T13:15:16+02:00
Make examples available for test
- - - - -
b81ec7be by Andreas Tille at 2020-04-15T13:22:31+02:00
Users do not have permissions to write to /mnt by default so change the default output dir to /var/tmp/idseq
- - - - -
8f481dbe by Andreas Tille at 2020-04-15T17:02:43+02:00
Seems bowtie2 is definitely needed
- - - - -
4507a009 by Andreas Tille at 2020-04-15T17:02:58+02:00
Try making tests with changed default_dir succeed but failed.
- - - - -
4 changed files:
- debian/control
- + debian/patches/series
- + debian/patches/set_default_output_dir_var_tmp.patch
- debian/rules
Changes:
=====================================
debian/control
=====================================
@@ -19,7 +19,8 @@ Package: python3-idseq-dag
Architecture: all
Depends: ${python3:Depends},
${misc:Depends},
- python3-boto3
+ python3-boto3,
+ bowtie2
Description: Pipeline engine for IDseq (Python 3)
Idseq_dag is the pipeline execution engine for idseq
(see idseq.net). It is a pipelining system that implements
=====================================
debian/patches/series
=====================================
@@ -0,0 +1 @@
+set_default_output_dir_var_tmp.patch
=====================================
debian/patches/set_default_output_dir_var_tmp.patch
=====================================
@@ -0,0 +1,101 @@
+Author: Andreas Tille <tille at debian.org>
+Last-Update: Wed, 15 Apr 2020 13:21:24 +0200
+Description: Users do not have permissions to write to /mnt by default so change the
+ default output dir to /var/tmp/idseq
+
+--- a/idseq_dag/engine/pipeline_flow.py
++++ b/idseq_dag/engine/pipeline_flow.py
+@@ -15,8 +15,8 @@ import idseq_dag.util.count as count
+ from idseq_dag.util.trace_lock import TraceLock
+ from idseq_dag.engine.pipeline_step import PipelineStep, InvalidInputFileError
+
+-DEFAULT_OUTPUT_DIR_LOCAL = '/mnt/idseq/results/%d' % os.getpid()
+-DEFAULT_REF_DIR_LOCAL = '/mnt/idseq/ref'
++DEFAULT_OUTPUT_DIR_LOCAL = '/var/tmp/idseq/results/%d' % os.getpid()
++DEFAULT_REF_DIR_LOCAL = '/var/tmp/idseq/ref'
+ PURGE_SENTINEL_DIR = DEFAULT_REF_DIR_LOCAL + "/purge_sentinel"
+ PURGE_SENTINEL = PURGE_SENTINEL_DIR + "/purge_nothing_newer_than_me"
+
+@@ -266,7 +266,7 @@ class PipelineFlow(object):
+ }
+ log.log_event("Reference downloads cache efficiency report", values=structured_report)
+ idseq_dag.util.s3.make_space() # make sure to touch this stage's files before deleting LRU ones
+- # N.B. the default results folder /mnt/idseq/results is removed by the aegea
++ # N.B. the default results folder /var/tmp/idseq/results is removed by the aegea
+ # command script driven through idseq-web --- so we don't have to worry about
+ # clearing that space here
+
+--- a/idseq_dag/steps/fetch_tax_info.py
++++ b/idseq_dag/steps/fetch_tax_info.py
+@@ -32,7 +32,7 @@ class PipelineStepFetchTaxInfo(PipelineS
+ id2namedict = {}
+ if namecsv:
+ # This is fetching a reference without fetch_reference; but ok because does not run from the actual pipeline
+- namecsvf = s3.fetch_from_s3(namecsv, "/mnt/idseq/ref")
++ namecsvf = s3.fetch_from_s3(namecsv, "/var/tmp/idseq/ref")
+ with open(namecsvf, 'r') as namef:
+ for line in namef:
+ fields = line.rstrip().split(",")
+--- a/idseq_dag/steps/run_bowtie2.py
++++ b/idseq_dag/steps/run_bowtie2.py
+@@ -77,10 +77,10 @@ class PipelineStepRunBowtie2(PipelineCou
+ bowtie2_params.extend(['-U', input_fas[0]])
+
+ # Example:
+- # bowtie2 -q -x /mnt/idseq/ref/bowtie2_genome/hg38_phiX_rRNA_mito_ERCC -f \
+- # --very-sensitive-local -S /mnt/idseq/results/589/bowtie2_human.sam \
++ # bowtie2 -q -x /var/tmp/idseq/ref/bowtie2_genome/hg38_phiX_rRNA_mito_ERCC -f \
++ # --very-sensitive-local -S /var/tmp/idseq/results/589/bowtie2_human.sam \
+ # -p 32 \
+- # -1 /mnt/idseq/results/589/unmapped_human_1.fa -2 /mnt/idseq/results/589/unmapped_human_2.fa
++ # -1 /var/tmp/idseq/results/589/unmapped_human_1.fa -2 /var/tmp/idseq/results/589/unmapped_human_2.fa
+ command.execute(
+ command_patterns.SingleCommand(
+ cmd='bowtie2',
+--- a/tests/unit/steps/test_download_accessions.py
++++ b/tests/unit/steps/test_download_accessions.py
+@@ -24,9 +24,9 @@ class TestDownloadAccessions(unittest.Te
+ ]
+ ],
+ output_files=["assembly/nr.refseq.fasta"],
+- output_dir_local="/mnt/idseq/results",
++ output_dir_local="/var/tmp/idseq/results",
+ output_dir_s3="s3://dummy_bucket",
+- ref_dir_local="/mnt/idseq/ref/",
++ ref_dir_local="/var/tmp/idseq/ref/",
+ additional_files={
+ "lineage_db": "s3://idseq-database/taxonomy/2018-12-01/taxid-lineages.sqlite3",
+ "loc_db": "s3://idseq-database/alignment_data/2018-12-01/nr_loc.sqlite3"
+@@ -75,4 +75,4 @@ class TestDownloadAccessions(unittest.Te
+ result = self.step._fix_ncbi_record(accession_data)
+
+ self.assertEqual(result, expected_result)
+-
+\ No newline at end of file
++
+--- a/tests/idseq_step_setup.py
++++ b/tests/idseq_step_setup.py
+@@ -30,10 +30,10 @@ class IdseqStepSetup(object):
+ "testrun_%s_%d_%d" % (step_name,
+ int(paired),
+ int(time.time())))
+- result_dir_local = "/mnt/idseq/results/%s_%d/%d" % (step_name,
++ result_dir_local = "/var/tmp/idseq/results/%s_%d/%d" % (step_name,
+ int(paired),
+ os.getpid())
+- ref_dir_local = '/mnt/idseq/ref'
++ ref_dir_local = '/var/tmp/idseq/ref'
+ command.make_dirs(result_dir_local)
+ command.make_dirs(ref_dir_local)
+
+@@ -77,8 +77,8 @@ class IdseqStepSetup(object):
+ od = output_dir_s3 or dag["output_dir_s3"]
+ output_dir_s3 = os.path.join(od,
+ f"testrun_{step_name}_{int(time.time())}")
+- result_dir_local = f"/mnt/idseq/results/{step_name}/{os.getpid()}"
+- ref_dir_local = '/mnt/idseq/ref'
++ result_dir_local = f"/var/tmp/idseq/results/{step_name}/{os.getpid()}"
++ ref_dir_local = '/var/tmp/idseq/ref'
+ command.make_dirs(result_dir_local)
+ command.make_dirs(ref_dir_local)
+
=====================================
debian/rules
=====================================
@@ -1,5 +1,14 @@
+#!/usr/bin/make -f
+
+# DH_VERBOSE := 1
export PYBUILD_NAME=idseq-dag
+export PYBUILD_BEFORE_TEST=cp -r {dir}/examples {build_dir}
%:
dh $@ --with python3 --buildsystem=pybuild
+override_dh_auto_test:
+ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
+ mkdir -p /var/tmp/idseq
+ dh_auto_test
+endif
View it on GitLab: https://salsa.debian.org/med-team/idseq-dag/-/compare/fe29b3b2adeaefe137e4887e251f12880d7cc8a1...4507a00935ca0916d011c421c78433883db16420
--
View it on GitLab: https://salsa.debian.org/med-team/idseq-dag/-/compare/fe29b3b2adeaefe137e4887e251f12880d7cc8a1...4507a00935ca0916d011c421c78433883db16420
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20200415/e4d853e2/attachment-0001.html>
More information about the debian-med-commit
mailing list