[med-svn] [Git][med-team/biomaj3-download][upstream] New upstream version 3.0.18
Olivier Sallou
gitlab at salsa.debian.org
Thu Oct 25 10:04:26 BST 2018
Olivier Sallou pushed to branch upstream at Debian Med / biomaj3-download
Commits:
1bd9f064 by Olivier Sallou at 2018-10-25T08:51:29Z
New upstream version 3.0.18
- - - - -
11 changed files:
- CHANGES.txt
- README.md
- biomaj_download/biomaj_download_web.py
- + biomaj_download/download/protocolirods.py
- biomaj_download/downloadclient.py
- biomaj_download/downloadservice.py
- biomaj_download/message/message.proto → biomaj_download/message/downmessage.proto
- biomaj_download/message/message_pb2.py → biomaj_download/message/downmessage_pb2.py
- requirements.txt
- setup.py
- tests/biomaj_tests.py
Changes:
=====================================
CHANGES.txt
=====================================
@@ -1,3 +1,13 @@
+3.0.18:
+ Rename protobuf and use specific package to avoid conflicts
+3.0.17:
+ Regenerate protobuf message desc, failing on python3
+3.0.16:
+ Add missing req in setup.py
+3.0.15:
+ Fix progress download control where could have infinite loop
+ Add irods download
+
3.0.14:
Allow setup of local_endpoint per service, else use default local_endpoint
=====================================
README.md
=====================================
@@ -41,3 +41,5 @@ If you cloned the repository and installed it via python setup.py install, just
gunicorn -c gunicorn_conf.py biomaj_download.biomaj_download_web:app
Web processes should be behind a proxy/load balancer, API base url /api/download
+
+Prometheus endpoint metrics are exposed via /metrics on web server
=====================================
biomaj_download/biomaj_download_web.py
=====================================
@@ -16,7 +16,7 @@ from prometheus_client import multiprocess
from prometheus_client import CollectorRegistry
import consul
-from biomaj_download.message import message_pb2
+from biomaj_download.message import downmessage_pb2
from biomaj_download.downloadservice import DownloadService
from biomaj_core.utils import Utils
@@ -86,7 +86,7 @@ def list_status(bank, session):
Check if listing request is over
'''
dserv = DownloadService(config_file, rabbitmq=False)
- biomaj_file_info = message_pb2.DownloadFile()
+ biomaj_file_info = downmessage_pb2.DownloadFile()
biomaj_file_info.bank = bank
biomaj_file_info.session = session
biomaj_file_info.local_dir = '/tmp'
@@ -100,7 +100,7 @@ def download_status(bank, session):
Get number of downloads and errors for bank and session. Progress includes successful download and errored downloads.
'''
dserv = DownloadService(config_file, rabbitmq=False)
- biomaj_file_info = message_pb2.DownloadFile()
+ biomaj_file_info = downmessage_pb2.DownloadFile()
biomaj_file_info.bank = bank
biomaj_file_info.session = session
biomaj_file_info.local_dir = '/tmp'
@@ -114,7 +114,7 @@ def download_error(bank, session):
Get errors info for bank and session
'''
dserv = DownloadService(config_file, rabbitmq=False)
- biomaj_file_info = message_pb2.DownloadFile()
+ biomaj_file_info = downmessage_pb2.DownloadFile()
biomaj_file_info.bank = bank
biomaj_file_info.session = session
biomaj_file_info.local_dir = '/tmp'
@@ -128,7 +128,7 @@ def list_result(bank, session):
Get file listing for bank and session, using FileList protobuf serialized string
'''
dserv = DownloadService(config_file, rabbitmq=False)
- biomaj_file_info = message_pb2.DownloadFile()
+ biomaj_file_info = downmessage_pb2.DownloadFile()
biomaj_file_info.bank = bank
biomaj_file_info.session = session
biomaj_file_info.local_dir = '/tmp'
@@ -146,7 +146,7 @@ def create_session(bank):
@app.route('/api/download/session/<bank>/<session>', methods=['DELETE'])
def clean_session(bank, session):
dserv = DownloadService(config_file, rabbitmq=False)
- biomaj_file_info = message_pb2.DownloadFile()
+ biomaj_file_info = downmessage_pb2.DownloadFile()
biomaj_file_info.bank = bank
biomaj_file_info.session = session
dserv.clean(biomaj_file_info)
=====================================
biomaj_download/download/protocolirods.py
=====================================
@@ -0,0 +1,126 @@
+import logging
+import os
+from datetime import datetime
+import time
+
+from biomaj_download.download.interface import DownloadInterface
+from irods.session import iRODSSession
+from irods.models import Collection, DataObject, User
+
+
+class IRODSDownload(DownloadInterface):
+ # To connect to irods session : sess = iRODSSession(host='localhost', port=1247, user='rods', password='rods', zone='tempZone')
+ # password : self.credentials
+ def __init__(self, protocol, server, remote_dir):
+ DownloadInterface.__init__(self)
+ self.port = None
+ self.remote_dir = remote_dir # directory on the remote server : zone
+ self.rootdir = remote_dir
+ self.user = None
+ self.password = None
+ self.server = server
+ self.zone = None
+
+ def set_param(self, param):
+ # self.param is a dictionnary which has the following form :{'password': u'biomaj', 'protocol': u'iget', 'user': u'biomaj', 'port': u'port'}
+ self.param = param
+ self.port = int(param['port'])
+ self.user = str(param['user'])
+ self.password = str(param['password'])
+ self.zone = str(param['zone'])
+
+ def list(self, directory=''):
+ session = iRODSSession(host=self.server, port=self.port, user=self.user, password=self.password, zone=self.zone)
+ rfiles = []
+ rdirs = []
+ rfile = {}
+ date = None
+ for result in session.query(Collection.name, DataObject.name, DataObject.size, DataObject.owner_name, DataObject.modify_time).filter(User.name == self.user).get_results():
+ # if the user is biomaj : he will have access to all the irods data (biomaj ressource) : drwxr-xr-x
+ # Avoid duplication
+ if rfile != {} and rfile['name'] == str(result[DataObject.name]) and date == str(result[DataObject.modify_time]).split(" ")[0].split('-'):
+ continue
+ rfile = {}
+ date = str(result[DataObject.modify_time]).split(" ")[0].split('-')
+ rfile['permissions'] = "-rwxr-xr-x"
+ rfile['size'] = int(result[DataObject.size])
+ rfile['month'] = int(date[1])
+ rfile['day'] = int(date[2])
+ rfile['year'] = int(date[0])
+ rfile['name'] = str(result[DataObject.name])
+ rfile['download_path'] = str(result[Collection.name])
+ rfiles.append(rfile)
+ session.cleanup()
+ return (rfiles, rdirs)
+
+ def download(self, local_dir, keep_dirs=True):
+ '''
+ Download remote files to local_dir
+
+ :param local_dir: Directory where files should be downloaded
+ :type local_dir: str
+ :param keep_dirs: keep file name directory structure or copy file in local_dir directly
+ :param keep_dirs: bool
+ :return: list of downloaded files
+ '''
+ logging.debug('IRODS:Download')
+ try:
+ os.chdir(local_dir)
+ except TypeError:
+ logging.error("IRODS:list:Could not find offline_dir")
+ nb_files = len(self.files_to_download)
+ cur_files = 1
+ # give a working directory to copy the file from irods
+ remote_dir = self.remote_dir
+ for rfile in self.files_to_download:
+ if self.kill_received:
+ raise Exception('Kill request received, exiting')
+ file_dir = local_dir
+ if 'save_as' not in rfile or rfile['save_as'] is None:
+ rfile['save_as'] = rfile['name']
+ if keep_dirs:
+ file_dir = local_dir + os.path.dirname(rfile['save_as'])
+ file_path = file_dir + '/' + os.path.basename(rfile['save_as'])
+ # For unit tests only, workflow will take in charge directory creation before to avoid thread multi access
+ if not os.path.exists(file_dir):
+ os.makedirs(file_dir)
+
+ logging.debug('IRODS:Download:Progress:' + str(cur_files) + '/' + str(nb_files) + ' downloading file ' + rfile['name'])
+ logging.debug('IRODS:Download:Progress:' + str(cur_files) + '/' + str(nb_files) + ' save as ' + rfile['save_as'])
+ cur_files += 1
+ start_time = datetime.now()
+ start_time = time.mktime(start_time.timetuple())
+ self.remote_dir = rfile['root']
+ error = self.irods_download(file_dir, str(self.remote_dir), str(rfile['name']))
+ if error:
+ rfile['download_time'] = 0
+ rfile['error'] = True
+ raise Exception("IRODS:Download:Error:" + rfile['root'] + '/' + rfile['name'])
+ end_time = datetime.now()
+ end_time = time.mktime(end_time.timetuple())
+ rfile['download_time'] = end_time - start_time
+ self.set_permissions(file_path, rfile)
+ self.remote_dir = remote_dir
+ return(self.files_to_download)
+
+ def irods_download(self, file_dir, file_path, file_to_download):
+ error = False
+ logging.debug('IRODS:IRODS DOWNLOAD')
+ session = iRODSSession(host=self.server, port=self.port, user=self.user, password=self.password, zone=self.zone)
+ try:
+ file_to_get = str(file_path) + str(file_to_download)
+ # Write the file to download in the wanted file_dir : with the python-irods iget
+ obj = session.data_objects.get(file_to_get, file_dir)
+ except ExceptionIRODS as e:
+ logging.error("RsyncError:" + str(e))
+ logging.error("RsyncError: irods object" + str(obj))
+ session.cleanup()
+ return(error)
+
+
+class ExceptionIRODS(Exception):
+ def __init__(self, exception_reason):
+ self.exception_reason = exception_reason
+
+ def __str__(self):
+ return self.exception_reason
=====================================
biomaj_download/downloadclient.py
=====================================
@@ -7,7 +7,7 @@ import sys
import pika
from biomaj_download.download.downloadthreads import DownloadThread
-from biomaj_download.message import message_pb2
+from biomaj_download.message import downmessage_pb2
if sys.version_info[0] < 3:
from Queue import Queue
@@ -92,15 +92,15 @@ class DownloadClient(DownloadService):
'''
for downloader in downloaders:
for file_to_download in downloader.files_to_download:
- operation = message_pb2.Operation()
+ operation = downmessage_pb2.Operation()
operation.type = 1
- message = message_pb2.DownloadFile()
+ message = downmessage_pb2.DownloadFile()
message.bank = self.bank
message.session = self.session
message.local_dir = offline_dir
- remote_file = message_pb2.DownloadFile.RemoteFile()
+ remote_file = downmessage_pb2.DownloadFile.RemoteFile()
protocol = downloader.protocol
- remote_file.protocol = message_pb2.DownloadFile.Protocol.Value(protocol.upper())
+ remote_file.protocol = downmessage_pb2.DownloadFile.Protocol.Value(protocol.upper())
remote_file.server = downloader.server
if cf.get('remote.dir'):
remote_file.remote_dir = cf.get('remote.dir')
@@ -135,7 +135,7 @@ class DownloadClient(DownloadService):
if 'md5' in file_to_download and file_to_download['md5']:
biomaj_file.metadata.md5 = file_to_download['md5']
- message.http_method = message_pb2.DownloadFile.HTTP_METHOD.Value(downloader.method.upper())
+ message.http_method = downmessage_pb2.DownloadFile.HTTP_METHOD.Value(downloader.method.upper())
timeout_download = cf.get('timeout.download', None)
if timeout_download:
@@ -216,7 +216,7 @@ class DownloadClient(DownloadService):
self.ask_download(operation)
nb_submitted += 1
- if progress == nb_files_to_download:
+ if progress >= nb_files_to_download:
over = True
logging.info("Workflow:wf_download:RemoteDownload:Completed:" + str(progress))
logging.info("Workflow:wf_download:RemoteDownload:Errors:" + str(error))
=====================================
biomaj_download/downloadservice.py
=====================================
@@ -18,11 +18,11 @@ from biomaj_download.download.http import HTTPDownload
from biomaj_download.download.direct import DirectFTPDownload
from biomaj_download.download.direct import DirectHttpDownload
from biomaj_download.download.localcopy import LocalDownload
-from biomaj_download.message import message_pb2
+from biomaj_download.message import downmessage_pb2
from biomaj_download.download.rsync import RSYNCDownload
from biomaj_core.utils import Utils
from biomaj_zipkin.zipkin import Zipkin
-
+from biomaj_download.download.protocolirods import IRODSDownload
app = Flask(__name__)
app_log = logging.getLogger('werkzeug')
@@ -123,7 +123,7 @@ class DownloadService(object):
credentials=None, http_parse=None, http_method=None, param=None,
proxy=None, proxy_auth='',
save_as=None, timeout_download=None, offline_dir=None):
- protocol = message_pb2.DownloadFile.Protocol.Value(protocol_name.upper())
+ protocol = downmessage_pb2.DownloadFile.Protocol.Value(protocol_name.upper())
downloader = None
if protocol in [0, 1]:
downloader = FTPDownload(protocol_name, server, remote_dir)
@@ -139,13 +139,14 @@ class DownloadService(object):
downloader = DirectHttpDownload('https', server, '/')
if protocol == 8:
downloader = RSYNCDownload('rsync', server, remote_dir)
+ if protocol == 9:
+ downloader = IRODSDownload('irods', server, remote_dir)
if downloader is None:
return None
for remote_file in remote_files:
if remote_file['save_as']:
save_as = remote_file['save_as']
-
# For direct protocol, we only keep base name
if protocol in [4, 5, 6]:
tmp_remote = []
@@ -170,7 +171,6 @@ class DownloadService(object):
if save_as:
downloader.set_save_as(save_as)
-
if param:
downloader.set_param(param)
@@ -191,7 +191,7 @@ class DownloadService(object):
server = biomaj_file_info.remote_file.server
remote_dir = biomaj_file_info.remote_file.remote_dir
- protocol_name = message_pb2.DownloadFile.Protocol.Name(protocol).lower()
+ protocol_name = downmessage_pb2.DownloadFile.Protocol.Name(protocol).lower()
self.logger.debug('%s request to download from %s://%s' % (biomaj_file_info.bank, protocol_name, server))
remote_files = []
@@ -220,7 +220,7 @@ class DownloadService(object):
remote_files=remote_files,
credentials=biomaj_file_info.remote_file.credentials,
http_parse=biomaj_file_info.remote_file.http_parse,
- http_method=message_pb2.DownloadFile.HTTP_METHOD.Name(biomaj_file_info.http_method),
+ http_method=downmessage_pb2.DownloadFile.HTTP_METHOD.Name(biomaj_file_info.http_method),
param=params,
proxy=proxy,
proxy_auth=proxy_auth,
@@ -294,7 +294,7 @@ class DownloadService(object):
file_list = self.redis_client.get(self.config['redis']['prefix'] + ':' + biomaj_file_info.bank + ':session:' + biomaj_file_info.session + ':files')
if protobuf_decode:
- file_list_pb2 = message_pb2.FileList()
+ file_list_pb2 = downmessage_pb2.FileList()
file_list_pb2.ParseFromString(file_list_pb2)
return file_list_pb2
@@ -306,7 +306,7 @@ class DownloadService(object):
'''
file_list = []
dir_list = []
- file_list_pb2 = message_pb2.FileList()
+ file_list_pb2 = downmessage_pb2.FileList()
try:
(file_list, dir_list) = download_handler.list()
@@ -318,7 +318,6 @@ class DownloadService(object):
else:
self.logger.debug('End of download for %s session %s' % (biomaj_file_info.bank, biomaj_file_info.session))
for file_elt in download_handler.files_to_download:
- # file_pb2 = message_pb2.File()
file_pb2 = file_list_pb2.files.add()
file_pb2.name = file_elt['name']
file_pb2.root = file_elt['root']
@@ -331,7 +330,7 @@ class DownloadService(object):
param = file_list_pb2.param.add()
param.name = key
param.value = file_elt['param'][key]
- metadata = message_pb2.File.MetaData()
+ metadata = downmessage_pb2.File.MetaData()
metadata.permissions = file_elt['permissions']
metadata.group = file_elt['group']
metadata.size = int(file_elt['size'])
@@ -438,7 +437,7 @@ class DownloadService(object):
Manage download and send ACK message
'''
try:
- operation = message_pb2.Operation()
+ operation = downmessage_pb2.Operation()
operation.ParseFromString(body)
message = operation.download
span = None
=====================================
biomaj_download/message/message.proto → biomaj_download/message/downmessage.proto
=====================================
@@ -1,4 +1,4 @@
-package biomaj;
+package biomaj.download;
message File {
// Name of the file
@@ -72,6 +72,7 @@ message DownloadFile {
DIRECTHTTPS = 6;
LOCAL = 7;
RSYNC = 8;
+ IRODS = 9;
}
message Param {
=====================================
biomaj_download/message/message_pb2.py → biomaj_download/message/downmessage_pb2.py
=====================================
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
-# source: message.proto
+# source: downmessage.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -16,9 +16,9 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
- name='message.proto',
- package='biomaj',
- serialized_pb=_b('\n\rmessage.proto\x12\x06\x62iomaj\"\x94\x02\n\x04\x46ile\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\x0c\n\x04root\x18\x02 \x01(\t\x12\x0f\n\x07save_as\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\'\n\x08metadata\x18\x05 \x01(\x0b\x32\x15.biomaj.File.MetaData\x1a\xa8\x01\n\x08MetaData\x12\x13\n\x0bpermissions\x18\x01 \x01(\t\x12\r\n\x05group\x18\x02 \x01(\t\x12\x0c\n\x04size\x18\x03 \x01(\x03\x12\x0c\n\x04hash\x18\x04 \x01(\t\x12\x0c\n\x04year\x18\x05 \x01(\x05\x12\r\n\x05month\x18\x06 \x01(\x05\x12\x0b\n\x03\x64\x61y\x18\x07 \x01(\x05\x12\x0e\n\x06\x66ormat\x18\x08 \x01(\t\x12\x0b\n\x03md5\x18\t \x01(\t\x12\x15\n\rdownload_time\x18\n \x01(\x03\"\'\n\x08\x46ileList\x12\x1b\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x0c.biomaj.File\"\x86\x02\n\tOperation\x12)\n\x04type\x18\x01 \x02(\x0e\x32\x1b.biomaj.Operation.OPERATION\x12&\n\x08\x64ownload\x18\x02 \x01(\x0b\x32\x14.biomaj.DownloadFile\x12 \n\x07process\x18\x03 \x01(\x0b\x32\x0f.biomaj.Process\x12&\n\x05trace\x18\x04 \x01(\x0b\x32\x17.biomaj.Operation.Trace\x1a*\n\x05Trace\x12\x10\n\x08trace_id\x18\x01 \x02(\t\x12\x0f\n\x07span_id\x18\x02 \x02(\t\"0\n\tOPERATION\x12\x08\n\x04LIST\x10\x00\x12\x0c\n\x08\x44OWNLOAD\x10\x01\x12\x0b\n\x07PROCESS\x10\x02\"\x17\n\x07Process\x12\x0c\n\x04\x65xec\x18\x01 \x02(\t\"\xcc\t\n\x0c\x44ownloadFile\x12\x0c\n\x04\x62\x61nk\x18\x01 \x02(\t\x12\x0f\n\x07session\x18\x02 \x02(\t\x12\x11\n\tlocal_dir\x18\x03 \x02(\t\x12\x18\n\x10timeout_download\x18\x04 \x01(\x05\x12\x34\n\x0bremote_file\x18\x05 \x02(\x0b\x32\x1f.biomaj.DownloadFile.RemoteFile\x12)\n\x05proxy\x18\x06 \x01(\x0b\x32\x1a.biomaj.DownloadFile.Proxy\x12:\n\x0bhttp_method\x18\x08 \x01(\x0e\x32 .biomaj.DownloadFile.HTTP_METHOD:\x03GET\x1a$\n\x05Param\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\r\n\x05value\x18\x02 \x02(\t\x1a\xcd\x03\n\tHttpParse\x12\x91\x01\n\x08\x64ir_line\x18\x01 \x02(\t:\x7f<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[DIR\\]\"[\\s]*/?>[\\s]*<a[\\s]+href=\"([\\S]+)/\"[\\s]*>.*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})\x12\xa5\x01\n\tfile_line\x18\x02 \x02(\t:\x91\x01<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[[\\s]+\\]\"[\\s]*/?>[\\s]<a[\\s]+href=\"([\\S]+)\".*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})[\\s]+([\\d\\.]+[MKG]{0,1})\x12\x13\n\x08\x64ir_name\x18\x03 \x02(\x05:\x01\x31\x12\x13\n\x08\x64ir_date\x18\x04 \x02(\x05:\x01\x32\x12\x14\n\tfile_name\x18\x05 \x02(\x05:\x01\x31\x12\x14\n\tfile_date\x18\x06 \x02(\x05:\x01\x32\x12\x18\n\x10\x66ile_date_format\x18\x07 \x01(\t\x12\x14\n\tfile_size\x18\x08 \x02(\x05:\x01\x33\x1a\x94\x02\n\nRemoteFile\x12\x1b\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x0c.biomaj.File\x12/\n\x08protocol\x18\x02 \x02(\x0e\x32\x1d.biomaj.DownloadFile.Protocol\x12\x0e\n\x06server\x18\x03 \x02(\t\x12\x12\n\nremote_dir\x18\x04 \x02(\t\x12\x0f\n\x07save_as\x18\x05 \x01(\t\x12)\n\x05param\x18\x06 \x03(\x0b\x32\x1a.biomaj.DownloadFile.Param\x12\x32\n\nhttp_parse\x18\x07 \x01(\x0b\x32\x1e.biomaj.DownloadFile.HttpParse\x12\x13\n\x0b\x63redentials\x18\x08 \x01(\t\x12\x0f\n\x07matches\x18\t \x03(\t\x1a*\n\x05Proxy\x12\r\n\x05proxy\x18\x01 \x02(\t\x12\x12\n\nproxy_auth\x18\x02 \x01(\t\"x\n\x08Protocol\x12\x07\n\x03\x46TP\x10\x00\x12\x08\n\x04SFTP\x10\x01\x12\x08\n\x04HTTP\x10\x02\x12\t\n\x05HTTPS\x10\x03\x12\r\n\tDIRECTFTP\x10\x04\x12\x0e\n\nDIRECTHTTP\x10\x05\x12\x0f\n\x0b\x44IRECTHTTPS\x10\x06\x12\t\n\x05LOCAL\x10\x07\x12\t\n\x05RSYNC\x10\x08\" \n\x0bHTTP_METHOD\x12\x07\n\x03GET\x10\x00\x12\x08\n\x04POST\x10\x01')
+ name='downmessage.proto',
+ package='biomaj.download',
+ serialized_pb=_b('\n\x11\x64ownmessage.proto\x12\x0f\x62iomaj.download\"\x9d\x02\n\x04\x46ile\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\x0c\n\x04root\x18\x02 \x01(\t\x12\x0f\n\x07save_as\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\x30\n\x08metadata\x18\x05 \x01(\x0b\x32\x1e.biomaj.download.File.MetaData\x1a\xa8\x01\n\x08MetaData\x12\x13\n\x0bpermissions\x18\x01 \x01(\t\x12\r\n\x05group\x18\x02 \x01(\t\x12\x0c\n\x04size\x18\x03 \x01(\x03\x12\x0c\n\x04hash\x18\x04 \x01(\t\x12\x0c\n\x04year\x18\x05 \x01(\x05\x12\r\n\x05month\x18\x06 \x01(\x05\x12\x0b\n\x03\x64\x61y\x18\x07 \x01(\x05\x12\x0e\n\x06\x66ormat\x18\x08 \x01(\t\x12\x0b\n\x03md5\x18\t \x01(\t\x12\x15\n\rdownload_time\x18\n \x01(\x03\"0\n\x08\x46ileList\x12$\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x15.biomaj.download.File\"\xaa\x02\n\tOperation\x12\x32\n\x04type\x18\x01 \x02(\x0e\x32$.biomaj.download.Operation.OPERATION\x12/\n\x08\x64ownload\x18\x02 \x01(\x0b\x32\x1d.biomaj.download.DownloadFile\x12)\n\x07process\x18\x03 \x01(\x0b\x32\x18.biomaj.download.Process\x12/\n\x05trace\x18\x04 \x01(\x0b\x32 .biomaj.download.Operation.Trace\x1a*\n\x05Trace\x12\x10\n\x08trace_id\x18\x01 \x02(\t\x12\x0f\n\x07span_id\x18\x02 \x02(\t\"0\n\tOPERATION\x12\x08\n\x04LIST\x10\x00\x12\x0c\n\x08\x44OWNLOAD\x10\x01\x12\x0b\n\x07PROCESS\x10\x02\"\x17\n\x07Process\x12\x0c\n\x04\x65xec\x18\x01 \x02(\t\"\x97\n\n\x0c\x44ownloadFile\x12\x0c\n\x04\x62\x61nk\x18\x01 \x02(\t\x12\x0f\n\x07session\x18\x02 \x02(\t\x12\x11\n\tlocal_dir\x18\x03 \x02(\t\x12\x18\n\x10timeout_download\x18\x04 \x01(\x05\x12=\n\x0bremote_file\x18\x05 \x02(\x0b\x32(.biomaj.download.DownloadFile.RemoteFile\x12\x32\n\x05proxy\x18\x06 \x01(\x0b\x32#.biomaj.download.DownloadFile.Proxy\x12\x43\n\x0bhttp_method\x18\x08 \x01(\x0e\x32).biomaj.download.DownloadFile.HTTP_METHOD:\x03GET\x1a$\n\x05Param\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\r\n\x05value\x18\x02 \x02(\t\x1a\xcd\x03\n\tHttpParse\x12\x91\x01\n\x08\x64ir_line\x18\x01 \x02(\t:\x7f<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[DIR\\]\"[\\s]*/?>[\\s]*<a[\\s]+href=\"([\\S]+)/\"[\\s]*>.*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})\x12\xa5\x01\n\tfile_line\x18\x02 \x02(\t:\x91\x01<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[[\\s]+\\]\"[\\s]*/?>[\\s]<a[\\s]+href=\"([\\S]+)\".*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})[\\s]+([\\d\\.]+[MKG]{0,1})\x12\x13\n\x08\x64ir_name\x18\x03 \x02(\x05:\x01\x31\x12\x13\n\x08\x64ir_date\x18\x04 \x02(\x05:\x01\x32\x12\x14\n\tfile_name\x18\x05 \x02(\x05:\x01\x31\x12\x14\n\tfile_date\x18\x06 \x02(\x05:\x01\x32\x12\x18\n\x10\x66ile_date_format\x18\x07 \x01(\t\x12\x14\n\tfile_size\x18\x08 \x02(\x05:\x01\x33\x1a\xb8\x02\n\nRemoteFile\x12$\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x15.biomaj.download.File\x12\x38\n\x08protocol\x18\x02 \x02(\x0e\x32&.biomaj.download.DownloadFile.Protocol\x12\x0e\n\x06server\x18\x03 \x02(\t\x12\x12\n\nremote_dir\x18\x04 \x02(\t\x12\x0f\n\x07save_as\x18\x05 \x01(\t\x12\x32\n\x05param\x18\x06 \x03(\x0b\x32#.biomaj.download.DownloadFile.Param\x12;\n\nhttp_parse\x18\x07 \x01(\x0b\x32\'.biomaj.download.DownloadFile.HttpParse\x12\x13\n\x0b\x63redentials\x18\x08 \x01(\t\x12\x0f\n\x07matches\x18\t \x03(\t\x1a*\n\x05Proxy\x12\r\n\x05proxy\x18\x01 \x02(\t\x12\x12\n\nproxy_auth\x18\x02 \x01(\t\"\x83\x01\n\x08Protocol\x12\x07\n\x03\x46TP\x10\x00\x12\x08\n\x04SFTP\x10\x01\x12\x08\n\x04HTTP\x10\x02\x12\t\n\x05HTTPS\x10\x03\x12\r\n\tDIRECTFTP\x10\x04\x12\x0e\n\nDIRECTHTTP\x10\x05\x12\x0f\n\x0b\x44IRECTHTTPS\x10\x06\x12\t\n\x05LOCAL\x10\x07\x12\t\n\x05RSYNC\x10\x08\x12\t\n\x05IRODS\x10\t\" \n\x0bHTTP_METHOD\x12\x07\n\x03GET\x10\x00\x12\x08\n\x04POST\x10\x01')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
@@ -26,7 +26,7 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
_OPERATION_OPERATION = _descriptor.EnumDescriptor(
name='OPERATION',
- full_name='biomaj.Operation.OPERATION',
+ full_name='biomaj.download.Operation.OPERATION',
filename=None,
file=DESCRIPTOR,
values=[
@@ -45,14 +45,14 @@ _OPERATION_OPERATION = _descriptor.EnumDescriptor(
],
containing_type=None,
options=None,
- serialized_start=560,
- serialized_end=608,
+ serialized_start=627,
+ serialized_end=675,
)
_sym_db.RegisterEnumDescriptor(_OPERATION_OPERATION)
_DOWNLOADFILE_PROTOCOL = _descriptor.EnumDescriptor(
name='Protocol',
- full_name='biomaj.DownloadFile.Protocol',
+ full_name='biomaj.download.DownloadFile.Protocol',
filename=None,
file=DESCRIPTOR,
values=[
@@ -92,17 +92,21 @@ _DOWNLOADFILE_PROTOCOL = _descriptor.EnumDescriptor(
name='RSYNC', index=8, number=8,
options=None,
type=None),
+ _descriptor.EnumValueDescriptor(
+ name='IRODS', index=9, number=9,
+ options=None,
+ type=None),
],
containing_type=None,
options=None,
- serialized_start=1710,
- serialized_end=1830,
+ serialized_start=1841,
+ serialized_end=1972,
)
_sym_db.RegisterEnumDescriptor(_DOWNLOADFILE_PROTOCOL)
_DOWNLOADFILE_HTTP_METHOD = _descriptor.EnumDescriptor(
name='HTTP_METHOD',
- full_name='biomaj.DownloadFile.HTTP_METHOD',
+ full_name='biomaj.download.DownloadFile.HTTP_METHOD',
filename=None,
file=DESCRIPTOR,
values=[
@@ -117,84 +121,84 @@ _DOWNLOADFILE_HTTP_METHOD = _descriptor.EnumDescriptor(
],
containing_type=None,
options=None,
- serialized_start=1832,
- serialized_end=1864,
+ serialized_start=1974,
+ serialized_end=2006,
)
_sym_db.RegisterEnumDescriptor(_DOWNLOADFILE_HTTP_METHOD)
_FILE_METADATA = _descriptor.Descriptor(
name='MetaData',
- full_name='biomaj.File.MetaData',
+ full_name='biomaj.download.File.MetaData',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='permissions', full_name='biomaj.File.MetaData.permissions', index=0,
+ name='permissions', full_name='biomaj.download.File.MetaData.permissions', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='group', full_name='biomaj.File.MetaData.group', index=1,
+ name='group', full_name='biomaj.download.File.MetaData.group', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='size', full_name='biomaj.File.MetaData.size', index=2,
+ name='size', full_name='biomaj.download.File.MetaData.size', index=2,
number=3, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='hash', full_name='biomaj.File.MetaData.hash', index=3,
+ name='hash', full_name='biomaj.download.File.MetaData.hash', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='year', full_name='biomaj.File.MetaData.year', index=4,
+ name='year', full_name='biomaj.download.File.MetaData.year', index=4,
number=5, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='month', full_name='biomaj.File.MetaData.month', index=5,
+ name='month', full_name='biomaj.download.File.MetaData.month', index=5,
number=6, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='day', full_name='biomaj.File.MetaData.day', index=6,
+ name='day', full_name='biomaj.download.File.MetaData.day', index=6,
number=7, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='format', full_name='biomaj.File.MetaData.format', index=7,
+ name='format', full_name='biomaj.download.File.MetaData.format', index=7,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='md5', full_name='biomaj.File.MetaData.md5', index=8,
+ name='md5', full_name='biomaj.download.File.MetaData.md5', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='download_time', full_name='biomaj.File.MetaData.download_time', index=9,
+ name='download_time', full_name='biomaj.download.File.MetaData.download_time', index=9,
number=10, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
@@ -211,47 +215,47 @@ _FILE_METADATA = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=134,
- serialized_end=302,
+ serialized_start=156,
+ serialized_end=324,
)
_FILE = _descriptor.Descriptor(
name='File',
- full_name='biomaj.File',
+ full_name='biomaj.download.File',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='name', full_name='biomaj.File.name', index=0,
+ name='name', full_name='biomaj.download.File.name', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='root', full_name='biomaj.File.root', index=1,
+ name='root', full_name='biomaj.download.File.root', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='save_as', full_name='biomaj.File.save_as', index=2,
+ name='save_as', full_name='biomaj.download.File.save_as', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='url', full_name='biomaj.File.url', index=3,
+ name='url', full_name='biomaj.download.File.url', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='metadata', full_name='biomaj.File.metadata', index=4,
+ name='metadata', full_name='biomaj.download.File.metadata', index=4,
number=5, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
@@ -268,20 +272,20 @@ _FILE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=26,
- serialized_end=302,
+ serialized_start=39,
+ serialized_end=324,
)
_FILELIST = _descriptor.Descriptor(
name='FileList',
- full_name='biomaj.FileList',
+ full_name='biomaj.download.FileList',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='files', full_name='biomaj.FileList.files', index=0,
+ name='files', full_name='biomaj.download.FileList.files', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
@@ -298,27 +302,27 @@ _FILELIST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=304,
- serialized_end=343,
+ serialized_start=326,
+ serialized_end=374,
)
_OPERATION_TRACE = _descriptor.Descriptor(
name='Trace',
- full_name='biomaj.Operation.Trace',
+ full_name='biomaj.download.Operation.Trace',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='trace_id', full_name='biomaj.Operation.Trace.trace_id', index=0,
+ name='trace_id', full_name='biomaj.download.Operation.Trace.trace_id', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='span_id', full_name='biomaj.Operation.Trace.span_id', index=1,
+ name='span_id', full_name='biomaj.download.Operation.Trace.span_id', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
@@ -335,40 +339,40 @@ _OPERATION_TRACE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=516,
- serialized_end=558,
+ serialized_start=583,
+ serialized_end=625,
)
_OPERATION = _descriptor.Descriptor(
name='Operation',
- full_name='biomaj.Operation',
+ full_name='biomaj.download.Operation',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='type', full_name='biomaj.Operation.type', index=0,
+ name='type', full_name='biomaj.download.Operation.type', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='download', full_name='biomaj.Operation.download', index=1,
+ name='download', full_name='biomaj.download.Operation.download', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='process', full_name='biomaj.Operation.process', index=2,
+ name='process', full_name='biomaj.download.Operation.process', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='trace', full_name='biomaj.Operation.trace', index=3,
+ name='trace', full_name='biomaj.download.Operation.trace', index=3,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
@@ -386,20 +390,20 @@ _OPERATION = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=346,
- serialized_end=608,
+ serialized_start=377,
+ serialized_end=675,
)
_PROCESS = _descriptor.Descriptor(
name='Process',
- full_name='biomaj.Process',
+ full_name='biomaj.download.Process',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='exec', full_name='biomaj.Process.exec', index=0,
+ name='exec', full_name='biomaj.download.Process.exec', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
@@ -416,27 +420,27 @@ _PROCESS = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=610,
- serialized_end=633,
+ serialized_start=677,
+ serialized_end=700,
)
_DOWNLOADFILE_PARAM = _descriptor.Descriptor(
name='Param',
- full_name='biomaj.DownloadFile.Param',
+ full_name='biomaj.download.DownloadFile.Param',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='name', full_name='biomaj.DownloadFile.Param.name', index=0,
+ name='name', full_name='biomaj.download.DownloadFile.Param.name', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='value', full_name='biomaj.DownloadFile.Param.value', index=1,
+ name='value', full_name='biomaj.download.DownloadFile.Param.value', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
@@ -453,68 +457,68 @@ _DOWNLOADFILE_PARAM = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=885,
- serialized_end=921,
+ serialized_start=979,
+ serialized_end=1015,
)
_DOWNLOADFILE_HTTPPARSE = _descriptor.Descriptor(
name='HttpParse',
- full_name='biomaj.DownloadFile.HttpParse',
+ full_name='biomaj.download.DownloadFile.HttpParse',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='dir_line', full_name='biomaj.DownloadFile.HttpParse.dir_line', index=0,
+ name='dir_line', full_name='biomaj.download.DownloadFile.HttpParse.dir_line', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=True, default_value=_b("<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[DIR\\]\"[\\s]*/?>[\\s]*<a[\\s]+href=\"([\\S]+)/\"[\\s]*>.*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='file_line', full_name='biomaj.DownloadFile.HttpParse.file_line', index=1,
+ name='file_line', full_name='biomaj.download.DownloadFile.HttpParse.file_line', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=True, default_value=_b("<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[[\\s]+\\]\"[\\s]*/?>[\\s]<a[\\s]+href=\"([\\S]+)\".*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})[\\s]+([\\d\\.]+[MKG]{0,1})").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='dir_name', full_name='biomaj.DownloadFile.HttpParse.dir_name', index=2,
+ name='dir_name', full_name='biomaj.download.DownloadFile.HttpParse.dir_name', index=2,
number=3, type=5, cpp_type=1, label=2,
has_default_value=True, default_value=1,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='dir_date', full_name='biomaj.DownloadFile.HttpParse.dir_date', index=3,
+ name='dir_date', full_name='biomaj.download.DownloadFile.HttpParse.dir_date', index=3,
number=4, type=5, cpp_type=1, label=2,
has_default_value=True, default_value=2,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='file_name', full_name='biomaj.DownloadFile.HttpParse.file_name', index=4,
+ name='file_name', full_name='biomaj.download.DownloadFile.HttpParse.file_name', index=4,
number=5, type=5, cpp_type=1, label=2,
has_default_value=True, default_value=1,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='file_date', full_name='biomaj.DownloadFile.HttpParse.file_date', index=5,
+ name='file_date', full_name='biomaj.download.DownloadFile.HttpParse.file_date', index=5,
number=6, type=5, cpp_type=1, label=2,
has_default_value=True, default_value=2,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='file_date_format', full_name='biomaj.DownloadFile.HttpParse.file_date_format', index=6,
+ name='file_date_format', full_name='biomaj.download.DownloadFile.HttpParse.file_date_format', index=6,
number=7, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='file_size', full_name='biomaj.DownloadFile.HttpParse.file_size', index=7,
+ name='file_size', full_name='biomaj.download.DownloadFile.HttpParse.file_size', index=7,
number=8, type=5, cpp_type=1, label=2,
has_default_value=True, default_value=3,
message_type=None, enum_type=None, containing_type=None,
@@ -531,75 +535,75 @@ _DOWNLOADFILE_HTTPPARSE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=924,
- serialized_end=1385,
+ serialized_start=1018,
+ serialized_end=1479,
)
_DOWNLOADFILE_REMOTEFILE = _descriptor.Descriptor(
name='RemoteFile',
- full_name='biomaj.DownloadFile.RemoteFile',
+ full_name='biomaj.download.DownloadFile.RemoteFile',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='files', full_name='biomaj.DownloadFile.RemoteFile.files', index=0,
+ name='files', full_name='biomaj.download.DownloadFile.RemoteFile.files', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='protocol', full_name='biomaj.DownloadFile.RemoteFile.protocol', index=1,
+ name='protocol', full_name='biomaj.download.DownloadFile.RemoteFile.protocol', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='server', full_name='biomaj.DownloadFile.RemoteFile.server', index=2,
+ name='server', full_name='biomaj.download.DownloadFile.RemoteFile.server', index=2,
number=3, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='remote_dir', full_name='biomaj.DownloadFile.RemoteFile.remote_dir', index=3,
+ name='remote_dir', full_name='biomaj.download.DownloadFile.RemoteFile.remote_dir', index=3,
number=4, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='save_as', full_name='biomaj.DownloadFile.RemoteFile.save_as', index=4,
+ name='save_as', full_name='biomaj.download.DownloadFile.RemoteFile.save_as', index=4,
number=5, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='param', full_name='biomaj.DownloadFile.RemoteFile.param', index=5,
+ name='param', full_name='biomaj.download.DownloadFile.RemoteFile.param', index=5,
number=6, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='http_parse', full_name='biomaj.DownloadFile.RemoteFile.http_parse', index=6,
+ name='http_parse', full_name='biomaj.download.DownloadFile.RemoteFile.http_parse', index=6,
number=7, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='credentials', full_name='biomaj.DownloadFile.RemoteFile.credentials', index=7,
+ name='credentials', full_name='biomaj.download.DownloadFile.RemoteFile.credentials', index=7,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='matches', full_name='biomaj.DownloadFile.RemoteFile.matches', index=8,
+ name='matches', full_name='biomaj.download.DownloadFile.RemoteFile.matches', index=8,
number=9, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
@@ -616,26 +620,26 @@ _DOWNLOADFILE_REMOTEFILE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=1388,
- serialized_end=1664,
+ serialized_start=1482,
+ serialized_end=1794,
)
_DOWNLOADFILE_PROXY = _descriptor.Descriptor(
name='Proxy',
- full_name='biomaj.DownloadFile.Proxy',
+ full_name='biomaj.download.DownloadFile.Proxy',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='proxy', full_name='biomaj.DownloadFile.Proxy.proxy', index=0,
+ name='proxy', full_name='biomaj.download.DownloadFile.Proxy.proxy', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='proxy_auth', full_name='biomaj.DownloadFile.Proxy.proxy_auth', index=1,
+ name='proxy_auth', full_name='biomaj.download.DownloadFile.Proxy.proxy_auth', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
@@ -652,61 +656,61 @@ _DOWNLOADFILE_PROXY = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=1666,
- serialized_end=1708,
+ serialized_start=1796,
+ serialized_end=1838,
)
_DOWNLOADFILE = _descriptor.Descriptor(
name='DownloadFile',
- full_name='biomaj.DownloadFile',
+ full_name='biomaj.download.DownloadFile',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='bank', full_name='biomaj.DownloadFile.bank', index=0,
+ name='bank', full_name='biomaj.download.DownloadFile.bank', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='session', full_name='biomaj.DownloadFile.session', index=1,
+ name='session', full_name='biomaj.download.DownloadFile.session', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='local_dir', full_name='biomaj.DownloadFile.local_dir', index=2,
+ name='local_dir', full_name='biomaj.download.DownloadFile.local_dir', index=2,
number=3, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='timeout_download', full_name='biomaj.DownloadFile.timeout_download', index=3,
+ name='timeout_download', full_name='biomaj.download.DownloadFile.timeout_download', index=3,
number=4, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='remote_file', full_name='biomaj.DownloadFile.remote_file', index=4,
+ name='remote_file', full_name='biomaj.download.DownloadFile.remote_file', index=4,
number=5, type=11, cpp_type=10, label=2,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='proxy', full_name='biomaj.DownloadFile.proxy', index=5,
+ name='proxy', full_name='biomaj.download.DownloadFile.proxy', index=5,
number=6, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
- name='http_method', full_name='biomaj.DownloadFile.http_method', index=6,
+ name='http_method', full_name='biomaj.download.DownloadFile.http_method', index=6,
number=8, type=14, cpp_type=8, label=1,
has_default_value=True, default_value=0,
message_type=None, enum_type=None, containing_type=None,
@@ -725,8 +729,8 @@ _DOWNLOADFILE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=636,
- serialized_end=1864,
+ serialized_start=703,
+ serialized_end=2006,
)
_FILE_METADATA.containing_type = _FILE
@@ -761,21 +765,21 @@ File = _reflection.GeneratedProtocolMessageType('File', (_message.Message,), dic
MetaData = _reflection.GeneratedProtocolMessageType('MetaData', (_message.Message,), dict(
DESCRIPTOR = _FILE_METADATA,
- __module__ = 'message_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.File.MetaData)
+ __module__ = 'downmessage_pb2'
+ # @@protoc_insertion_point(class_scope:biomaj.download.File.MetaData)
))
,
DESCRIPTOR = _FILE,
- __module__ = 'message_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.File)
+ __module__ = 'downmessage_pb2'
+ # @@protoc_insertion_point(class_scope:biomaj.download.File)
))
_sym_db.RegisterMessage(File)
_sym_db.RegisterMessage(File.MetaData)
FileList = _reflection.GeneratedProtocolMessageType('FileList', (_message.Message,), dict(
DESCRIPTOR = _FILELIST,
- __module__ = 'message_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.FileList)
+ __module__ = 'downmessage_pb2'
+ # @@protoc_insertion_point(class_scope:biomaj.download.FileList)
))
_sym_db.RegisterMessage(FileList)
@@ -783,21 +787,21 @@ Operation = _reflection.GeneratedProtocolMessageType('Operation', (_message.Mess
Trace = _reflection.GeneratedProtocolMessageType('Trace', (_message.Message,), dict(
DESCRIPTOR = _OPERATION_TRACE,
- __module__ = 'message_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.Operation.Trace)
+ __module__ = 'downmessage_pb2'
+ # @@protoc_insertion_point(class_scope:biomaj.download.Operation.Trace)
))
,
DESCRIPTOR = _OPERATION,
- __module__ = 'message_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.Operation)
+ __module__ = 'downmessage_pb2'
+ # @@protoc_insertion_point(class_scope:biomaj.download.Operation)
))
_sym_db.RegisterMessage(Operation)
_sym_db.RegisterMessage(Operation.Trace)
Process = _reflection.GeneratedProtocolMessageType('Process', (_message.Message,), dict(
DESCRIPTOR = _PROCESS,
- __module__ = 'message_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.Process)
+ __module__ = 'downmessage_pb2'
+ # @@protoc_insertion_point(class_scope:biomaj.download.Process)
))
_sym_db.RegisterMessage(Process)
@@ -805,34 +809,34 @@ DownloadFile = _reflection.GeneratedProtocolMessageType('DownloadFile', (_messag
Param = _reflection.GeneratedProtocolMessageType('Param', (_message.Message,), dict(
DESCRIPTOR = _DOWNLOADFILE_PARAM,
- __module__ = 'message_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.DownloadFile.Param)
+ __module__ = 'downmessage_pb2'
+ # @@protoc_insertion_point(class_scope:biomaj.download.DownloadFile.Param)
))
,
HttpParse = _reflection.GeneratedProtocolMessageType('HttpParse', (_message.Message,), dict(
DESCRIPTOR = _DOWNLOADFILE_HTTPPARSE,
- __module__ = 'message_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.DownloadFile.HttpParse)
+ __module__ = 'downmessage_pb2'
+ # @@protoc_insertion_point(class_scope:biomaj.download.DownloadFile.HttpParse)
))
,
RemoteFile = _reflection.GeneratedProtocolMessageType('RemoteFile', (_message.Message,), dict(
DESCRIPTOR = _DOWNLOADFILE_REMOTEFILE,
- __module__ = 'message_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.DownloadFile.RemoteFile)
+ __module__ = 'downmessage_pb2'
+ # @@protoc_insertion_point(class_scope:biomaj.download.DownloadFile.RemoteFile)
))
,
Proxy = _reflection.GeneratedProtocolMessageType('Proxy', (_message.Message,), dict(
DESCRIPTOR = _DOWNLOADFILE_PROXY,
- __module__ = 'message_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.DownloadFile.Proxy)
+ __module__ = 'downmessage_pb2'
+ # @@protoc_insertion_point(class_scope:biomaj.download.DownloadFile.Proxy)
))
,
DESCRIPTOR = _DOWNLOADFILE,
- __module__ = 'message_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.DownloadFile)
+ __module__ = 'downmessage_pb2'
+ # @@protoc_insertion_point(class_scope:biomaj.download.DownloadFile)
))
_sym_db.RegisterMessage(DownloadFile)
_sym_db.RegisterMessage(DownloadFile.Param)
=====================================
requirements.txt
=====================================
@@ -10,7 +10,8 @@ flask
python-consul
prometheus_client>=0.0.18
requests
-biomaj_core>=3.0.10
+biomaj_core>=3.0.12
biomaj_zipkin
flake8
humanfriendly
+python-irodsclient
=====================================
setup.py
=====================================
@@ -21,7 +21,7 @@ config = {
'url': 'http://biomaj.genouest.org',
'download_url': 'http://biomaj.genouest.org',
'author_email': 'olivier.sallou at irisa.fr',
- 'version': '3.0.14',
+ 'version': '3.0.18',
'classifiers': [
# How mature is this project? Common values are
# 3 - Alpha
@@ -54,7 +54,8 @@ config = {
'prometheus_client>=0.0.18',
'protobuf',
'requests',
- 'humanfriendly'
+ 'humanfriendly',
+ 'python-irodsclient'
],
'tests_require': ['nose', 'mock'],
'test_suite': 'nose.collector',
=====================================
tests/biomaj_tests.py
=====================================
@@ -23,6 +23,9 @@ from biomaj_download.download.http import HTTPDownload, HTTPParse
from biomaj_download.download.localcopy import LocalDownload
from biomaj_download.download.downloadthreads import DownloadThread
from biomaj_download.download.rsync import RSYNCDownload
+from biomaj_download.download.protocolirods import IRODSDownload
+
+import pprint
import unittest
@@ -552,3 +555,106 @@ class TestBiomajRSYNCDownload(unittest.TestCase):
rsyncd.match([r'^/bank/test*'], file_list, dir_list, prefix='')
rsyncd.download(self.utils.data_dir)
self.assertTrue(len(rsyncd.files_to_download) == 3)
+
+
+class iRodsResult(object):
+
+ def __init__(self, collname, dataname, datasize, owner, modify):
+ self.Collname = 'tests/'
+ self.Dataname = 'test.fasta.gz'
+ self.Datasize = 45
+ self.Dataowner_name = 'biomaj'
+ self.Datamodify_time = '2017-04-10 00:00:00'
+
+ def __getitem__(self, index):
+ from irods.models import Collection, DataObject, User
+ if index.icat_id == DataObject.modify_time.icat_id:
+ return self.Datamodify_time
+ elif "DATA_SIZE" in str(index):
+ return self.Datasize
+ elif "DATA_NAME" in str(index):
+ return 'test.fasta.gz'
+ elif "COLL_NAME" in str(index):
+ return self.Collname
+ elif "D_OWNER_NAME" in str(index):
+ return self.Dataowner_name
+
+
+class MockiRODSSession(object):
+ '''
+ Simulation of python irods client
+ for result in session.query(Collection.name, DataObject.name, DataObject.size, DataObject.owner_name, DataObject.modify_time).filter(User.name == self.user).get_results():
+ '''
+ def __init__(self):
+ self.Collname="1"
+ self.Dataname="2"
+ self.Datasize="3"
+ self.Dataowner_name="4"
+ self.Datamodify_time="5"
+ self.Collid=""
+
+ def __getitem__(self, index):
+ from irods.data_object import iRODSDataObject
+ from irods.models import Collection, DataObject, User
+ print(index)
+ if "COLL_ID" in str(index):
+ return self.Collid
+ if "COLL_NAME" in str(index):
+ return self.Collname
+
+ def configure(self):
+ return MockiRODSSession()
+
+ def query(self,Collname, Dataname, Datasize, Dataowner_name, Datamodify_time):
+ return self
+
+ def all(self):
+ return self
+
+ def one(self):
+ return self
+
+ def filter(self,boo):
+ return self
+
+ def get_results(self):
+ get_result_dict= iRodsResult('tests/', 'test.fasta.gz', 45, 'biomaj', '2017-04-10 00:00:00')
+ return [get_result_dict]
+
+ def cleanup(self):
+ return self
+
+ def open(self,r):
+ my_test_file = open("tests/test.fasta.gz", "r+")
+ return(my_test_file)
+
+ at attr('irods')
+ at attr('roscoZone')
+ at attr('network')
+class TestBiomajIRODSDownload(unittest.TestCase):
+ '''
+ Test IRODS downloader
+ '''
+ def setUp(self):
+ self.utils = UtilsForTest()
+ self.curdir = os.path.dirname(os.path.realpath(__file__))
+ self.examples = os.path.join(self.curdir,'bank') + '/'
+ BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)
+
+ def tearDown(self):
+ self.utils.clean()
+
+ @patch('irods.session.iRODSSession.configure')
+ @patch('irods.session.iRODSSession.query')
+ @patch('irods.session.iRODSSession.cleanup')
+ def test_irods_list(self,initialize_mock, query_mock,cleanup_mock):
+ mock_session=MockiRODSSession()
+ initialize_mock.return_value=mock_session.configure()
+ query_mock.return_value = mock_session.query(None,None,None,None,None)
+ cleanup_mock.return_value = mock_session.cleanup()
+ irodsd = IRODSDownload('irods', self.examples, "")
+ irodsd.set_credentials(None)
+ irodsd.set_offline_dir(self.utils.data_dir)
+ (files_list, dir_list) = irodsd.list()
+ self.assertTrue(len(files_list) != 0)
+
View it on GitLab: https://salsa.debian.org/med-team/biomaj3-download/commit/1bd9f064d22f47cb4e71afae9562ddac03abe68d
--
View it on GitLab: https://salsa.debian.org/med-team/biomaj3-download/commit/1bd9f064d22f47cb4e71afae9562ddac03abe68d
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20181025/f74972c8/attachment-0001.html>
More information about the debian-med-commit
mailing list