[med-svn] [Git][med-team/biomaj3-download][master] 4 commits: New upstream version 3.0.21
Olivier Sallou
gitlab at salsa.debian.org
Sat Mar 9 10:11:33 GMT 2019
Olivier Sallou pushed to branch master at Debian Med / biomaj3-download
Commits:
5de3aa6a by Olivier Sallou at 2019-03-09T09:47:57Z
New upstream version 3.0.21
- - - - -
77200790 by Olivier Sallou at 2019-03-09T09:47:57Z
Update upstream source from tag 'upstream/3.0.21'
Update to upstream version '3.0.21'
with Debian dir 901aa17e9e50a8505565edf8ab321bb9be8fe37d
- - - - -
132af15f by Olivier Sallou at 2019-03-09T10:10:48Z
new upstream release 3.0.21
- - - - -
ed679a76 by Olivier Sallou at 2019-03-09T10:10:48Z
remove irods support as lib is not in debian
- - - - -
9 changed files:
- .travis.yml
- CHANGES.txt
- biomaj_download/biomaj_download_web.py
- debian/changelog
- + debian/patches/remove_irods.patch
- + debian/patches/series
- requirements.txt
- setup.py
- tests/biomaj_tests.py
Changes:
=====================================
.travis.yml
=====================================
@@ -1,29 +1,29 @@
language: python
sudo: false
python:
- - "2.7"
- - "3.4"
- - "3.5"
+- '2.7'
+- '3.4'
+- '3.5'
+- '3.6'
services:
- - redis
-# Apply only on main branches
+- redis
branches:
except:
- - /^feature.*$/
-# command to install dependencies
-#before_install:
-# - "sudo apt-get update -qq"
-# - "sudo apt-get install -qq libldap2-dev libsasl2-dev"
+ - "/^feature.*$/"
install:
- - "pip install -r requirements.txt"
- - "pip install coverage"
- - "pip install python-coveralls"
- - "python setup.py -q install"
-# - "echo data_file=$TRAVIS_BUILD_DIR/.coverage >> .coveragerc"
-# command to run tests
+- pip install -r requirements.txt
+- pip install coverage nose
+- pip install python-coveralls
+- python setup.py -q install
script:
- - python setup.py test
- - flake8 --ignore E501 biomaj_download/*.py biomaj_download/download
-#after_success:
-# - coveralls
-
+- nosetests -a '!network'
+- flake8 --ignore E501 biomaj_download/*.py biomaj_download/download
+deploy:
+ provider: pypi
+ skip_existing: true
+ user: osallou
+ password:
+ secure: fMEDYld7N2uRbICa++b7dTUXxt6VxSD1ZE+ZaTHQWxHHSK5fxbdwvsVHicdgKH3erMOXb7foumQ6uxBAtQTVmY+ZJbidydT/BEeoLjIH0NxVoTLCOllWKHOKm8LzgigVjbSVqXi7ZZ6rzigpQbg7NCdiD4wQr6xLO+gB3LrWovIgcPIlWDLWiMIp0P3b9nmoOdiSBBpLXI7i9W7ksL25Mopx/p4OoeWFEbg7aSkRVKufWLCoyLnYnWCnmC0MK7PF7V7mgiw+Gutv3XcOgfaZ/MU088+OrpvnLxfKplkwukcMBc4cuMV8mtWmDaQ9u2X9MLoeyRoQ6jHyIvftW+fn4Ck3wzyKZA3Pbz5+UVUG6GVpnwm3NIYvPl0m62brVVqs/wVaI8HB/7u5qn60ER9u9l9/YGIG5rsPrb2w12QFmcjwrge2T9h+aDW3+fQuzbjVGJMBCLqu9hUoiz9WHWqa74Ka8OfP+fTA3vpASGfuYhzdlJ5tDO6M1kjIe23ga5DhUMKHyl4j7uA/YVxc5numYU9naD/PyZw1qqms8zhJdVmIsBMtBP/MeH0YVrhmQS7p4ni7BB545JBYv8UyUfVXt2DcQHG9EAkMT69QfJhSl9KICe4SmlHVYy2Uldw/LE/FwYmngfDzZUYtndsn2xS+NVi3FwMNa1+6q0Sqqmzfmqc=
+ distributions: sdist bdist_wheel
+ on:
+ tags: true
=====================================
CHANGES.txt
=====================================
@@ -1,3 +1,8 @@
+3.0.21:
+ Fix traefik labels
+3.0.20:
+ Update pika dependency release
+ Add tags for traefik support
3.0.19:
Check archives after download
Fix python regexps syntax (deprecation)
=====================================
biomaj_download/biomaj_download_web.py
=====================================
@@ -30,7 +30,7 @@ download_time_metric = Counter("biomaj_download_file_time", "Bank download file
config_file = 'config.yml'
if 'BIOMAJ_CONFIG' in os.environ:
- config_file = os.environ['BIOMAJ_CONFIG']
+ config_file = os.environ['BIOMAJ_CONFIG']
config = None
with open(config_file, 'r') as ymlfile:
@@ -41,7 +41,19 @@ with open(config_file, 'r') as ymlfile:
def consul_declare(config):
if config['consul']['host']:
consul_agent = consul.Consul(host=config['consul']['host'])
- consul_agent.agent.service.register('biomaj-download', service_id=config['consul']['id'], address=config['web']['hostname'], port=config['web']['port'], tags=['biomaj'])
+ consul_agent.agent.service.register(
+ 'biomaj-download',
+ service_id=config['consul']['id'],
+ address=config['web']['hostname'],
+ port=config['web']['port'],
+ tags=[
+ 'biomaj',
+ 'api',
+ 'traefik-int.backend=biomaj-download',
+ 'traefik-int.frontend.rule=PathPrefix:/api/download',
+ 'traefik-int.enable=true'
+ ]
+ )
check = consul.Check.http(url='http://' + config['web']['hostname'] + ':' + str(config['web']['port']) + '/api/download', interval=20)
consul_agent.agent.check.register(config['consul']['id'] + '_check', check=check, service_id=config['consul']['id'])
=====================================
debian/changelog
=====================================
@@ -1,3 +1,12 @@
+biomaj3-download (3.0.21-1) unstable; urgency=medium
+
+ * New upstream release
+ * d/rules: use DEB_BUILD_MAINT_OPTIONS
+ * d/patches: remove irods protocol support as python lib is not
+ available in Debian.
+
+ -- Olivier Sallou <osallou at debian.org> Sat, 09 Mar 2019 09:48:16 +0000
+
biomaj3-download (3.0.19-1) unstable; urgency=medium
* Team upload.
=====================================
debian/patches/remove_irods.patch
=====================================
@@ -0,0 +1,135 @@
+Subject: python irods not available, remove it from supported protocols
+Description: biomaj supports irods as download protocol but irods is not
+ available in Debian. In the meanwhile remove support for this protocol
+Author: Olivier Sallou <osallou at debian.org>
+Last-Updated: 2019-03-09
+Forwarded: no
+--- a/requirements.txt
++++ b/requirements.txt
+@@ -14,4 +14,3 @@
+ biomaj_zipkin
+ flake8
+ humanfriendly
+-python-irodsclient
+--- a/setup.py
++++ b/setup.py
+@@ -54,8 +54,7 @@
+ 'prometheus_client>=0.0.18',
+ 'protobuf',
+ 'requests',
+- 'humanfriendly',
+- 'python-irodsclient'
++ 'humanfriendly'
+ ],
+ 'tests_require': ['nose', 'mock'],
+ 'test_suite': 'nose.collector',
+--- a/biomaj_download/download/protocolirods.py
++++ b/biomaj_download/download/protocolirods.py
+@@ -5,8 +5,6 @@
+
+ from biomaj_core.utils import Utils
+ from biomaj_download.download.interface import DownloadInterface
+-from irods.session import iRODSSession
+-from irods.models import Collection, DataObject, User
+
+
+ class IRODSDownload(DownloadInterface):
+@@ -31,27 +29,9 @@
+ self.zone = str(param['zone'])
+
+ def list(self, directory=''):
+- session = iRODSSession(host=self.server, port=self.port, user=self.user, password=self.password, zone=self.zone)
+ rfiles = []
+ rdirs = []
+- rfile = {}
+- date = None
+- for result in session.query(Collection.name, DataObject.name, DataObject.size, DataObject.owner_name, DataObject.modify_time).filter(User.name == self.user).get_results():
+- # if the user is biomaj : he will have access to all the irods data (biomaj ressource) : drwxr-xr-x
+- # Avoid duplication
+- if rfile != {} and rfile['name'] == str(result[DataObject.name]) and date == str(result[DataObject.modify_time]).split(" ")[0].split('-'):
+- continue
+- rfile = {}
+- date = str(result[DataObject.modify_time]).split(" ")[0].split('-')
+- rfile['permissions'] = "-rwxr-xr-x"
+- rfile['size'] = int(result[DataObject.size])
+- rfile['month'] = int(date[1])
+- rfile['day'] = int(date[2])
+- rfile['year'] = int(date[0])
+- rfile['name'] = str(result[DataObject.name])
+- rfile['download_path'] = str(result[Collection.name])
+- rfiles.append(rfile)
+- session.cleanup()
++ raise Exception("IRODS:NotSupported")
+ return (rfiles, rdirs)
+
+ def download(self, local_dir, keep_dirs=True):
+@@ -65,67 +45,10 @@
+ :return: list of downloaded files
+ '''
+ logging.debug('IRODS:Download')
+- try:
+- os.chdir(local_dir)
+- except TypeError:
+- logging.error("IRODS:list:Could not find offline_dir")
+- nb_files = len(self.files_to_download)
+- cur_files = 1
+- # give a working directory to copy the file from irods
+- remote_dir = self.remote_dir
+- for rfile in self.files_to_download:
+- if self.kill_received:
+- raise Exception('Kill request received, exiting')
+- file_dir = local_dir
+- if 'save_as' not in rfile or rfile['save_as'] is None:
+- rfile['save_as'] = rfile['name']
+- if keep_dirs:
+- file_dir = local_dir + os.path.dirname(rfile['save_as'])
+- file_path = file_dir + '/' + os.path.basename(rfile['save_as'])
+- # For unit tests only, workflow will take in charge directory creation before to avoid thread multi access
+- if not os.path.exists(file_dir):
+- os.makedirs(file_dir)
+-
+- logging.debug('IRODS:Download:Progress:' + str(cur_files) + '/' + str(nb_files) + ' downloading file ' + rfile['name'])
+- logging.debug('IRODS:Download:Progress:' + str(cur_files) + '/' + str(nb_files) + ' save as ' + rfile['save_as'])
+- cur_files += 1
+- start_time = datetime.now()
+- start_time = time.mktime(start_time.timetuple())
+- self.remote_dir = rfile['root']
+- error = self.irods_download(file_dir, str(self.remote_dir), str(rfile['name']))
+- if error:
+- rfile['download_time'] = 0
+- rfile['error'] = True
+- raise Exception("IRODS:Download:Error:" + rfile['root'] + '/' + rfile['name'])
+- else:
+- archive_status = Utils.archive_check(file_path)
+- if not archive_status:
+- self.logger.error('Archive is invalid or corrupted, deleting file')
+- rfile['error'] = True
+- if os.path.exists(file_path):
+- os.remove(file_path)
+- raise Exception("IRODS:Download:Error:" + rfile['root'] + '/' + rfile['name'])
+-
+- end_time = datetime.now()
+- end_time = time.mktime(end_time.timetuple())
+- rfile['download_time'] = end_time - start_time
+- self.set_permissions(file_path, rfile)
+- self.remote_dir = remote_dir
+- return(self.files_to_download)
++ raise Exception("IRODS:NotSupported")
+
+ def irods_download(self, file_dir, file_path, file_to_download):
+- error = False
+- logging.debug('IRODS:IRODS DOWNLOAD')
+- session = iRODSSession(host=self.server, port=self.port, user=self.user, password=self.password, zone=self.zone)
+- try:
+- file_to_get = str(file_path) + str(file_to_download)
+- # Write the file to download in the wanted file_dir : with the python-irods iget
+- obj = session.data_objects.get(file_to_get, file_dir)
+- except ExceptionIRODS as e:
+- logging.error("RsyncError:" + str(e))
+- logging.error("RsyncError: irods object" + str(obj))
+- session.cleanup()
+- return(error)
++ return("irods not supported")
+
+
+ class ExceptionIRODS(Exception):
=====================================
debian/patches/series
=====================================
@@ -0,0 +1 @@
+remove_irods.patch
=====================================
requirements.txt
=====================================
@@ -2,7 +2,7 @@ mock
nose
pycurl
py-bcrypt
-pika==0.11.2
+pika==0.13.0
redis
PyYAML
protobuf
@@ -10,7 +10,7 @@ flask
python-consul
prometheus_client>=0.0.18
requests
-biomaj_core>=3.0.12
+biomaj_core>=3.0.16
biomaj_zipkin
flake8
humanfriendly
=====================================
setup.py
=====================================
@@ -21,7 +21,7 @@ config = {
'url': 'http://biomaj.genouest.org',
'download_url': 'http://biomaj.genouest.org',
'author_email': 'olivier.sallou at irisa.fr',
- 'version': '3.0.19',
+ 'version': '3.0.21',
'classifiers': [
# How mature is this project? Common values are
# 3 - Alpha
@@ -46,7 +46,7 @@ config = {
'biomaj_zipkin',
'pycurl',
'py-bcrypt',
- 'pika==0.11.2',
+ 'pika==0.13.0',
'redis',
'PyYAML',
'flask',
=====================================
tests/biomaj_tests.py
=====================================
@@ -320,21 +320,21 @@ class TestBiomajDirectFTPDownload(unittest.TestCase):
self.utils.clean()
def test_ftp_list(self):
- file_list = ['/blast/db/FASTA/alu.n.gz.md5']
- ftpd = DirectFTPDownload('ftp', 'ftp.ncbi.nih.gov', '')
+ file_list = ['/debian/doc/mailing-lists.txt']
+ ftpd = DirectFTPDownload('ftp', 'ftp.fr.debian.org', '')
ftpd.set_files_to_download(file_list)
(file_list, dir_list) = ftpd.list()
ftpd.close()
self.assertTrue(len(file_list) == 1)
def test_download(self):
- file_list = ['/blast/db/FASTA/alu.n.gz.md5']
- ftpd = DirectFTPDownload('ftp', 'ftp.ncbi.nih.gov', '')
+ file_list = ['/debian/doc/mailing-lists.txt']
+ ftpd = DirectFTPDownload('ftp', 'ftp.fr.debian.org', '')
ftpd.set_files_to_download(file_list)
(file_list, dir_list) = ftpd.list()
ftpd.download(self.utils.data_dir, False)
ftpd.close()
- self.assertTrue(os.path.exists(os.path.join(self.utils.data_dir,'alu.n.gz.md5')))
+ self.assertTrue(os.path.exists(os.path.join(self.utils.data_dir,'mailing-lists.txt')))
@attr('directhttp')
@@ -431,17 +431,14 @@ class TestBiomajFTPDownload(unittest.TestCase):
def test_ftp_list(self):
ftpd = FTPDownload('ftp', 'speedtest.tele2.net', '/')
- # ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/db/FASTA/')
(file_list, dir_list) = ftpd.list()
ftpd.close()
self.assertTrue(len(file_list) > 1)
@attr('test')
def test_download(self):
- # ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/db/FASTA/')
ftpd = FTPDownload('ftp', 'speedtest.tele2.net', '/')
(file_list, dir_list) = ftpd.list()
- # ftpd.match([r'^alu.*\.gz$'], file_list, dir_list)
ftpd.match([r'^1.*KB\.zip$'], file_list, dir_list)
try:
ftpd.download(self.utils.data_dir)
@@ -453,11 +450,9 @@ class TestBiomajFTPDownload(unittest.TestCase):
# self.assertTrue(len(ftpd.files_to_download) == 2)
def test_download_skip_uncompress_checks(self):
- # ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/db/FASTA/')
os.environ['UNCOMPRESS_SKIP_CHECK'] = "1"
ftpd = FTPDownload('ftp', 'speedtest.tele2.net', '/')
(file_list, dir_list) = ftpd.list()
- # ftpd.match([r'^alu.*\.gz$'], file_list, dir_list)
ftpd.match([r'^1.*KB\.zip$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
@@ -465,19 +460,19 @@ class TestBiomajFTPDownload(unittest.TestCase):
del os.environ['UNCOMPRESS_SKIP_CHECK']
def test_download_in_subdir(self):
- ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/')
+ ftpd = FTPDownload('ftp', 'ftp.fr.debian.org', '/debian/')
(file_list, dir_list) = ftpd.list()
try:
- ftpd.match([r'^db/FASTA/alu.*\.gz$'], file_list, dir_list)
+ ftpd.match([r'^doc/mailing-lists.txt$'], file_list, dir_list)
except Exception as e:
print("Error: " + str(e))
self.skipTest("Skipping test due to remote server error")
ftpd.download(self.utils.data_dir)
ftpd.close()
- self.assertTrue(len(ftpd.files_to_download) == 2)
+ self.assertTrue(len(ftpd.files_to_download) == 1)
def test_download_or_copy(self):
- ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/')
+ ftpd = FTPDownload('ftp', 'ftp.fr.debian.org', '/debian/')
ftpd.files_to_download = [
{'name':'/test1', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test2', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
View it on GitLab: https://salsa.debian.org/med-team/biomaj3-download/compare/cafc530e3f261d5b92a7f34fd873edc2b7708eaf...ed679a76b1b461ea9a602d52c04a8dad1d429b14
--
View it on GitLab: https://salsa.debian.org/med-team/biomaj3-download/compare/cafc530e3f261d5b92a7f34fd873edc2b7708eaf...ed679a76b1b461ea9a602d52c04a8dad1d429b14
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20190309/f1fbb8a5/attachment-0001.html>
More information about the debian-med-commit
mailing list