[med-svn] [Git][med-team/biomaj3][upstream] New upstream version 3.1.22
Andreas Tille (@tille)
gitlab at salsa.debian.org
Thu Jun 30 15:53:50 BST 2022
Andreas Tille pushed to branch upstream at Debian Med / biomaj3
Commits:
69a93dbb by Andreas Tille at 2022-06-29T15:28:10+02:00
New upstream version 3.1.22
- - - - -
9 changed files:
- CHANGES.txt
- README.md
- biomaj/bank.py
- biomaj/process/processfactory.py
- biomaj/workflow.py
- requirements.txt
- setup.py
- + tests/alu_list_error.properties
- tests/biomaj_tests.py
Changes:
=====================================
CHANGES.txt
=====================================
@@ -1,3 +1,13 @@
+3.1.21:
+ Freeze pymongo to 3.12.3 (4.x breaks)
+ Change isAlive() which is deprecated in python 3.9 to is_alive
+
+3.1.20:
+ Follow-up of #127 to get last release in file (refactor and bug fix)
+3.1.19:
+ Add tgz archive support
+ Add log file info to production info
+ #126 Issue with getting last release in file
3.1.18:
Python 3 support only
If multiple files match release.file, take most recent one
=====================================
README.md
=====================================
@@ -77,6 +77,7 @@ Application Features
* Remote access:
* Optional FTP server providing authenticated or anonymous data access
+ * HTTP access to bank files (/db endpoint, microservice setup only)
Dependencies
============
=====================================
biomaj/bank.py
=====================================
@@ -182,7 +182,7 @@ class Bank(object):
str(last_update),
str(release)])
# Bank production info header
- prod_info.append(["Session", "Remote release", "Release", "Directory", "Freeze", "Format(s)"])
+ prod_info.append(["Session", "Remote release", "Release", "Directory", "Freeze", "Format(s)", "Log file"])
for prod in _bank['production']:
data_dir = self.config.get('data.dir')
dir_version = self.config.get('dir.version')
@@ -205,7 +205,9 @@ class Bank(object):
prod['release'],
release_dir,
'yes' if 'freeze' in prod and prod['freeze'] else 'no',
- formats])
+ formats,
+ prod.get('log_file', '')
+ ])
# Bank pending info header
if 'pending' in _bank and len(_bank['pending']) > 0:
pend_info.append(["Pending release", "Last run"])
@@ -579,6 +581,7 @@ class Bank(object):
'data_dir': self.session._session['data_dir'],
'dir_version': self.session._session['dir_version'],
'prod_dir': prod_dir,
+ 'log_file': self.session._session['log_file'],
'freeze': False}
self.bank['production'].append(production)
self.banks.update({'name': self.name},
=====================================
biomaj/process/processfactory.py
=====================================
@@ -66,7 +66,7 @@ class ProcessFactory(object):
kill_received = True
for t in running_th:
t.kill_received = True
- running_th = [t.join(1000) for t in running_th if t is not None and t.isAlive()]
+ running_th = [t.join(1000) for t in running_th if t is not None and t.is_alive()]
except KeyboardInterrupt:
logging.warn("Ctrl-c received! Sending kill to threads...")
logging.warn("Running tasks will continue and process will stop.")
=====================================
biomaj/workflow.py
=====================================
@@ -26,6 +26,8 @@ from biomaj.process.processfactory import RemoveProcessFactory, PreProcessFactor
from biomaj_zipkin.zipkin import Zipkin
from yapsy.PluginManager import PluginManager
+from packaging.version import parse
+
class Workflow(object):
"""
@@ -516,25 +518,16 @@ class UpdateWorkflow(Workflow):
Try to find most release from releases input array
'''
release = releases[0]
- releaseElts = re.split(r'\.|-', release)
+ release_version = parse(release)
logging.debug('found a release %s' % (release))
for rel in releases:
if rel == release:
continue
logging.debug('compare next release %s' % (rel))
- relElts = re.split(r'\.|-', rel)
- index = 0
- for relElt in relElts:
- logging.debug("compare release major,minor,etc. : %s >? %s" % (relElt, releaseElts[index]))
- try:
- if int(relElt) > int(releaseElts[index]):
- release = rel
- logging.debug("found newer release %s" % (rel))
- break
- except ValueError:
- pass
- finally:
- index += 1
+ next_release = parse(rel)
+ if next_release > release_version:
+ release = rel
+ release_version = next_release
return release
def wf_release(self):
@@ -1663,9 +1656,7 @@ class UpdateWorkflow(Workflow):
nb_try = 1
origFile = self.session.get_offline_directory() + '/' + file['save_as']
is_archive = False
- if origFile.endswith('.tar.gz'):
- is_archive = True
- elif origFile.endswith('.tar'):
+ if origFile.endswith(('.tar.gz', '.tar', '.tgz')):
is_archive = True
elif origFile.endswith('.bz2'):
is_archive = True
=====================================
requirements.txt
=====================================
@@ -5,7 +5,7 @@ biomaj_process>=3.0.12
biomaj_cli
mock
nose
-pymongo>=3.2
+pymongo==3.12.3
pycurl
tabulate
py-bcrypt
@@ -17,3 +17,4 @@ redis
influxdb
Yapsy==1.12.2
Jinja2
+packaging
=====================================
setup.py
=====================================
@@ -36,7 +36,7 @@ config = {
'url': 'http://biomaj.genouest.org',
'download_url': 'http://biomaj.genouest.org',
'author_email': 'olivier.sallou at irisa.fr',
- 'version': '3.1.18',
+ 'version': '3.1.22',
'classifiers': [
# How mature is this project? Common values are
# 3 - Alpha
@@ -62,7 +62,7 @@ config = {
'biomaj_user',
'biomaj_download',
'biomaj_process',
- 'pymongo>=3.2',
+ 'pymongo >=3.2, <4',
'pycurl',
'py-bcrypt',
'drmaa',
@@ -72,7 +72,8 @@ config = {
'redis',
'elasticsearch',
'influxdb',
- 'Yapsy==1.12.2'
+ 'Yapsy==1.12.2',
+ 'packaging'
],
'tests_require': ['nose', 'mock'],
'test_suite': 'nose.collector',
=====================================
tests/alu_list_error.properties
=====================================
@@ -0,0 +1,43 @@
+[GENERAL]
+######################
+### Initialization ###
+
+db.fullname="alu.n : alu repeat element. alu.a : translation of alu.n repeats"
+db.name=alu
+db.type=nucleic_protein
+
+offline.dir.name=offline/ncbi/blast/alu_tmp
+dir.version=ncbi/blast/alu
+
+frequency.update=0
+
+### Synchronization ###
+
+files.num.threads=1
+
+# NCBI (download fasta)
+protocol=ftp
+server=ftp.ncbi.nih.gov
+remote.dir=/blast/db/FASTA/foo/
+
+release.file=
+release.regexp=
+release.file.compressed=
+
+remote.files=^alu.*\.gz$
+
+#Uncomment if you don't want to extract the data files.
+#no.extract=true
+
+local.files=^alu\.(a|n).*
+
+## Post Process ## The files should be located in the projectfiles/process directory
+
+db.post.process=
+
+
+
+
+### Deployment ###
+
+keep.old.version=1
=====================================
tests/biomaj_tests.py
=====================================
@@ -80,7 +80,8 @@ class UtilsForTest():
def __copy_test_bank_properties(self):
if self.bank_properties is not None:
return
- self.bank_properties = ['alu', 'local', 'testhttp','directhttp']
+ self.bank_properties = ['alu', 'local', 'testhttp','directhttp',
+ 'alu_list_error']
curdir = os.path.dirname(os.path.realpath(__file__))
for b in self.bank_properties:
from_file = os.path.join(curdir, b+'.properties')
@@ -302,26 +303,33 @@ class TestBiomajSetup(unittest.TestCase):
class TestBiomajFunctional(unittest.TestCase):
+ # Banks used in tests
+ BANKS = ['local', 'alu_list_error']
+
def setUp(self):
self.utils = UtilsForTest()
- curdir = os.path.dirname(os.path.realpath(__file__))
BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)
- #Delete all banks
- b = Bank('local')
- b.banks.remove({})
-
- self.config = BiomajConfig('local')
- data_dir = self.config.get('data.dir')
- lock_file = os.path.join(data_dir,'local.lock')
- if os.path.exists(lock_file):
- os.remove(lock_file)
+ # Clean banks used in tests
+ for bank_name in self.BANKS:
+ # Delete all releases
+ b = Bank(bank_name)
+ b.banks.remove({})
+ # Delete lock files
+ config = BiomajConfig(bank_name)
+ data_dir = config.get('data.dir')
+ lock_file = os.path.join(data_dir, 'local.lock')
+ if os.path.exists(lock_file):
+ os.remove(lock_file)
def tearDown(self):
- data_dir = self.config.get('data.dir')
- lock_file = os.path.join(data_dir,'local.lock')
- if os.path.exists(lock_file):
- os.remove(lock_file)
+ # Delete lock files
+ for bank_name in self.BANKS:
+ config = BiomajConfig(bank_name)
+ data_dir = config.get('data.dir')
+ lock_file = os.path.join(data_dir,'local.lock')
+ if os.path.exists(lock_file):
+ os.remove(lock_file)
self.utils.clean()
def test_extract_release_from_file_name(self):
@@ -471,7 +479,7 @@ class TestBiomajFunctional(unittest.TestCase):
# Remove file
if os.path.exists(tmp_remote_file):
os.remove(tmp_remote_file)
-
+
def test_fromscratch_update(self):
"""
Try updating twice, at second time, bank should be updated (force with fromscratc)
@@ -858,3 +866,9 @@ class TestBiomajFunctional(unittest.TestCase):
self.fail('not owner, should not be allowed')
except Exception as e:
pass
+
+ @attr('network')
+ def test_bank_list_error(self):
+ b = Bank('alu_list_error')
+ res = b.update()
+ self.assertFalse(res)
View it on GitLab: https://salsa.debian.org/med-team/biomaj3/-/commit/69a93dbb10c58414557f6a50b61ee1b8f4c1e9d0
--
View it on GitLab: https://salsa.debian.org/med-team/biomaj3/-/commit/69a93dbb10c58414557f6a50b61ee1b8f4c1e9d0
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20220630/98978db4/attachment-0001.htm>
More information about the debian-med-commit
mailing list