[med-svn] [Git][med-team/biomaj3][master] 11 commits: New upstream version 3.1.22
Andreas Tille (@tille)
gitlab at salsa.debian.org
Thu Jun 30 15:53:44 BST 2022
Andreas Tille pushed to branch master at Debian Med / biomaj3
Commits:
69a93dbb by Andreas Tille at 2022-06-29T15:28:10+02:00
New upstream version 3.1.22
- - - - -
d27b5aed by Andreas Tille at 2022-06-29T15:28:10+02:00
routine-update: New upstream version
- - - - -
da7a5789 by Andreas Tille at 2022-06-29T15:28:10+02:00
Update upstream source from tag 'upstream/3.1.22'
Update to upstream version '3.1.22'
with Debian dir 37a25f6039a2df9b48757ebf93e1860ac381f3da
- - - - -
322fea28 by Andreas Tille at 2022-06-29T15:28:11+02:00
routine-update: Standards-Version: 4.6.1
- - - - -
57e43edb by Andreas Tille at 2022-06-29T15:28:13+02:00
routine-update: Remove trailing whitespace in debian/changelog
- - - - -
59e327af by Andreas Tille at 2022-06-29T17:19:30+02:00
Re-enable influxdb which was fixed
- - - - -
6b2cf873 by Andreas Tille at 2022-06-29T17:26:01+02:00
Enable at least superflous autopkgtest
- - - - -
16c07078 by Andreas Tille at 2022-06-29T17:26:32+02:00
Fix lintian-overrides
- - - - -
8375e35f by Andreas Tille at 2022-06-29T17:27:44+02:00
DEP3
- - - - -
797f8588 by Andreas Tille at 2022-06-30T16:48:08+02:00
Fix import-name (cut-n-pasto)
- - - - -
429ff8ef by Andreas Tille at 2022-06-30T16:51:02+02:00
Upload to unstable
- - - - -
16 changed files:
- CHANGES.txt
- README.md
- biomaj/bank.py
- biomaj/process/processfactory.py
- biomaj/workflow.py
- debian/changelog
- debian/control
- debian/lintian-overrides
- debian/patches/fix_remove_drmaa.patch
- debian/patches/fix_remove_influxdb.patch
- debian/patches/series
- + debian/tests/pkg-python/import-name
- requirements.txt
- setup.py
- + tests/alu_list_error.properties
- tests/biomaj_tests.py
Changes:
=====================================
CHANGES.txt
=====================================
@@ -1,3 +1,13 @@
+3.1.21:
+ Freeze pymongo to 3.12.3 (4.x breaks)
+ Change isAlive() which is deprecated in python 3.9 to is_alive
+
+3.1.20:
+ Follow-up of #127 to get last release in file (refactor and bug fix)
+3.1.19:
+ Add tgz archive support
+ Add log file info to production info
+ #126 Issue with getting last release in file
3.1.18:
Python 3 support only
If multiple files match release.file, take most recent one
=====================================
README.md
=====================================
@@ -77,6 +77,7 @@ Application Features
* Remote access:
* Optional FTP server providing authenticated or anonymous data access
+ * HTTP access to bank files (/db endpoint, microservice setup only)
Dependencies
============
=====================================
biomaj/bank.py
=====================================
@@ -182,7 +182,7 @@ class Bank(object):
str(last_update),
str(release)])
# Bank production info header
- prod_info.append(["Session", "Remote release", "Release", "Directory", "Freeze", "Format(s)"])
+ prod_info.append(["Session", "Remote release", "Release", "Directory", "Freeze", "Format(s)", "Log file"])
for prod in _bank['production']:
data_dir = self.config.get('data.dir')
dir_version = self.config.get('dir.version')
@@ -205,7 +205,9 @@ class Bank(object):
prod['release'],
release_dir,
'yes' if 'freeze' in prod and prod['freeze'] else 'no',
- formats])
+ formats,
+ prod.get('log_file', '')
+ ])
# Bank pending info header
if 'pending' in _bank and len(_bank['pending']) > 0:
pend_info.append(["Pending release", "Last run"])
@@ -579,6 +581,7 @@ class Bank(object):
'data_dir': self.session._session['data_dir'],
'dir_version': self.session._session['dir_version'],
'prod_dir': prod_dir,
+ 'log_file': self.session._session['log_file'],
'freeze': False}
self.bank['production'].append(production)
self.banks.update({'name': self.name},
=====================================
biomaj/process/processfactory.py
=====================================
@@ -66,7 +66,7 @@ class ProcessFactory(object):
kill_received = True
for t in running_th:
t.kill_received = True
- running_th = [t.join(1000) for t in running_th if t is not None and t.isAlive()]
+ running_th = [t.join(1000) for t in running_th if t is not None and t.is_alive()]
except KeyboardInterrupt:
logging.warn("Ctrl-c received! Sending kill to threads...")
logging.warn("Running tasks will continue and process will stop.")
=====================================
biomaj/workflow.py
=====================================
@@ -26,6 +26,8 @@ from biomaj.process.processfactory import RemoveProcessFactory, PreProcessFactor
from biomaj_zipkin.zipkin import Zipkin
from yapsy.PluginManager import PluginManager
+from packaging.version import parse
+
class Workflow(object):
"""
@@ -516,25 +518,16 @@ class UpdateWorkflow(Workflow):
Try to find most release from releases input array
'''
release = releases[0]
- releaseElts = re.split(r'\.|-', release)
+ release_version = parse(release)
logging.debug('found a release %s' % (release))
for rel in releases:
if rel == release:
continue
logging.debug('compare next release %s' % (rel))
- relElts = re.split(r'\.|-', rel)
- index = 0
- for relElt in relElts:
- logging.debug("compare release major,minor,etc. : %s >? %s" % (relElt, releaseElts[index]))
- try:
- if int(relElt) > int(releaseElts[index]):
- release = rel
- logging.debug("found newer release %s" % (rel))
- break
- except ValueError:
- pass
- finally:
- index += 1
+ next_release = parse(rel)
+ if next_release > release_version:
+ release = rel
+ release_version = next_release
return release
def wf_release(self):
@@ -1663,9 +1656,7 @@ class UpdateWorkflow(Workflow):
nb_try = 1
origFile = self.session.get_offline_directory() + '/' + file['save_as']
is_archive = False
- if origFile.endswith('.tar.gz'):
- is_archive = True
- elif origFile.endswith('.tar'):
+ if origFile.endswith(('.tar.gz', '.tar', '.tgz')):
is_archive = True
elif origFile.endswith('.bz2'):
is_archive = True
=====================================
debian/changelog
=====================================
@@ -1,7 +1,20 @@
+biomaj3 (3.1.22-1) unstable; urgency=medium
+
+ * Team upload.
+ * New upstream version
+ * Standards-Version: 4.6.1 (routine-update)
+ * Remove trailing whitespace in debian/changelog (routine-update)
+ * Re-enable influxdb which was fixed
+ * Enable at least superfluous autopkgtest
+ * Fix lintian-overrides
+ * DEP3
+
+ -- Andreas Tille <tille at debian.org> Thu, 30 Jun 2022 16:48:27 +0200
+
biomaj3 (3.1.18-2) unstable; urgency=medium
* Remove drmaa support (optional feature) due to drmaa package
- migration issue on 0.7.9-2
+ migration issue on 0.7.9-2
-- Olivier Sallou <osallou at debian.org> Fri, 19 Feb 2021 10:25:44 +0000
=====================================
debian/control
=====================================
@@ -20,7 +20,7 @@ Build-Depends: debhelper-compat (= 13),
python3-biomaj3-download,
python3-biomaj3-user,
python3-biomaj3-process
-Standards-Version: 4.5.1
+Standards-Version: 4.6.1
Vcs-Browser: https://salsa.debian.org/med-team/biomaj3
Vcs-Git: https://salsa.debian.org/med-team/biomaj3.git
Homepage: https://github.com/genouest/biomaj
@@ -35,7 +35,6 @@ Recommends: ${python3:Recommends}
Suggests: ${python3:Suggests},
python3-biomaj3-cli,
python3-gunicorn,
- mongodb,
redis-server
Description: BioMAJ workflow management library
BioMAJ downloads remote data banks, checks their status and applies
=====================================
debian/lintian-overrides
=====================================
@@ -1,2 +1,2 @@
# see https://lists.debian.org/debian-med/2018/06/msg00043.html
-python3-biomaj3: script-with-language-extension usr/bin/*.*
+python3-biomaj3: script-with-language-extension
=====================================
debian/patches/fix_remove_drmaa.patch
=====================================
@@ -1,17 +1,21 @@
+Author: Olivier Sallou
+Last-Update: 2021-02-19 10:27:50 +0000
+Description: drmaa seems to be broken so disable this feature
+
--- a/docs/conf.py
+++ b/docs/conf.py
-@@ -22,7 +22,7 @@
+@@ -22,7 +22,7 @@ class Mock(MagicMock):
def __getattr__(cls, name):
return Mock()
--MOCK_MODULES = ['pycurl', 'pymongo', 'elasticsearch', 'drmaa',
-+MOCK_MODULES = ['pycurl', 'pymongo', 'elasticsearch',
+-MOCK_MODULES = ['pycurl', 'pymongo', 'elasticsearch', 'drmaa', 'influxdb',
++MOCK_MODULES = ['pycurl', 'pymongo', 'elasticsearch', 'influxdb',
'biomaj_download',
'biomaj_download.downloadclient',
'biomaj_download.download',
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
-@@ -4,7 +4,6 @@
+@@ -4,7 +4,6 @@ pymongo==3.2
tabulate
ldap3
py-bcrypt
@@ -21,7 +25,7 @@
biomaj_core
--- a/requirements.txt
+++ b/requirements.txt
-@@ -9,7 +9,6 @@
+@@ -9,7 +9,6 @@ pymongo==3.12.3
pycurl
tabulate
py-bcrypt
@@ -31,8 +35,8 @@
requests
--- a/setup.py
+++ b/setup.py
-@@ -65,7 +65,6 @@
- 'pymongo>=3.2',
+@@ -65,7 +65,6 @@ config = {
+ 'pymongo >=3.2, <4',
'pycurl',
'py-bcrypt',
- 'drmaa',
=====================================
debian/patches/fix_remove_influxdb.patch
=====================================
@@ -1,4 +1,5 @@
Subject: remove influxdb feature
+ --> This patch is deactivated since bug #950063 was done
Description: python-influxdb has a bug 950063
This patch remove influxdb optional feature waiting
for influx bug to be solved
=====================================
debian/patches/series
=====================================
@@ -1,2 +1,2 @@
-fix_remove_influxdb.patch
+# fix_remove_influxdb.patch
fix_remove_drmaa.patch
=====================================
debian/tests/pkg-python/import-name
=====================================
@@ -0,0 +1 @@
+biomaj
=====================================
requirements.txt
=====================================
@@ -5,7 +5,7 @@ biomaj_process>=3.0.12
biomaj_cli
mock
nose
-pymongo>=3.2
+pymongo==3.12.3
pycurl
tabulate
py-bcrypt
@@ -17,3 +17,4 @@ redis
influxdb
Yapsy==1.12.2
Jinja2
+packaging
=====================================
setup.py
=====================================
@@ -36,7 +36,7 @@ config = {
'url': 'http://biomaj.genouest.org',
'download_url': 'http://biomaj.genouest.org',
'author_email': 'olivier.sallou at irisa.fr',
- 'version': '3.1.18',
+ 'version': '3.1.22',
'classifiers': [
# How mature is this project? Common values are
# 3 - Alpha
@@ -62,7 +62,7 @@ config = {
'biomaj_user',
'biomaj_download',
'biomaj_process',
- 'pymongo>=3.2',
+ 'pymongo >=3.2, <4',
'pycurl',
'py-bcrypt',
'drmaa',
@@ -72,7 +72,8 @@ config = {
'redis',
'elasticsearch',
'influxdb',
- 'Yapsy==1.12.2'
+ 'Yapsy==1.12.2',
+ 'packaging'
],
'tests_require': ['nose', 'mock'],
'test_suite': 'nose.collector',
=====================================
tests/alu_list_error.properties
=====================================
@@ -0,0 +1,43 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="alu.n : alu repeat element. alu.a : translation of alu.n repeats"
+db.name=alu
+db.type=nucleic_protein
+
+offline.dir.name=offline/ncbi/blast/alu_tmp
+dir.version=ncbi/blast/alu
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=ftp.ncbi.nih.gov
+remote.dir=/blast/db/FASTA/foo/
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.files=^alu.*\.gz$
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^alu\.(a|n).*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+
+
+### Deployment ###
+
+keep.old.version=1
=====================================
tests/biomaj_tests.py
=====================================
@@ -80,7 +80,8 @@ class UtilsForTest():
def __copy_test_bank_properties(self):
if self.bank_properties is not None:
return
- self.bank_properties = ['alu', 'local', 'testhttp','directhttp']
+ self.bank_properties = ['alu', 'local', 'testhttp','directhttp',
+ 'alu_list_error']
curdir = os.path.dirname(os.path.realpath(__file__))
for b in self.bank_properties:
from_file = os.path.join(curdir, b+'.properties')
@@ -302,26 +303,33 @@ class TestBiomajSetup(unittest.TestCase):
class TestBiomajFunctional(unittest.TestCase):
+ # Banks used in tests
+ BANKS = ['local', 'alu_list_error']
+
def setUp(self):
self.utils = UtilsForTest()
- curdir = os.path.dirname(os.path.realpath(__file__))
BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)
- #Delete all banks
- b = Bank('local')
- b.banks.remove({})
-
- self.config = BiomajConfig('local')
- data_dir = self.config.get('data.dir')
- lock_file = os.path.join(data_dir,'local.lock')
- if os.path.exists(lock_file):
- os.remove(lock_file)
+ # Clean banks used in tests
+ for bank_name in self.BANKS:
+ # Delete all releases
+ b = Bank(bank_name)
+ b.banks.remove({})
+ # Delete lock files
+ config = BiomajConfig(bank_name)
+ data_dir = config.get('data.dir')
+ lock_file = os.path.join(data_dir, 'local.lock')
+ if os.path.exists(lock_file):
+ os.remove(lock_file)
def tearDown(self):
- data_dir = self.config.get('data.dir')
- lock_file = os.path.join(data_dir,'local.lock')
- if os.path.exists(lock_file):
- os.remove(lock_file)
+ # Delete lock files
+ for bank_name in self.BANKS:
+ config = BiomajConfig(bank_name)
+ data_dir = config.get('data.dir')
+ lock_file = os.path.join(data_dir,'local.lock')
+ if os.path.exists(lock_file):
+ os.remove(lock_file)
self.utils.clean()
def test_extract_release_from_file_name(self):
@@ -471,7 +479,7 @@ class TestBiomajFunctional(unittest.TestCase):
# Remove file
if os.path.exists(tmp_remote_file):
os.remove(tmp_remote_file)
-
+
def test_fromscratch_update(self):
"""
Try updating twice, at second time, bank should be updated (force with fromscratc)
@@ -858,3 +866,9 @@ class TestBiomajFunctional(unittest.TestCase):
self.fail('not owner, should not be allowed')
except Exception as e:
pass
+
+ @attr('network')
+ def test_bank_list_error(self):
+ b = Bank('alu_list_error')
+ res = b.update()
+ self.assertFalse(res)
View it on GitLab: https://salsa.debian.org/med-team/biomaj3/-/compare/7c6b110038480242fcb6a59c05728af260ecad65...429ff8ef5dc09baf08348f89d181c14a9a8d5bee
--
View it on GitLab: https://salsa.debian.org/med-team/biomaj3/-/compare/7c6b110038480242fcb6a59c05728af260ecad65...429ff8ef5dc09baf08348f89d181c14a9a8d5bee
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20220630/38ccf9da/attachment-0001.htm>
More information about the debian-med-commit
mailing list