[med-svn] [Git][med-team/biomaj3-download][upstream] New upstream version 3.0.27
Olivier Sallou
gitlab at salsa.debian.org
Wed Oct 16 14:31:11 BST 2019
Olivier Sallou pushed to branch upstream at Debian Med / biomaj3-download
Commits:
86037752 by Olivier Sallou at 2019-10-16T13:17:26Z
New upstream version 3.0.27
- - - - -
15 changed files:
- .travis.yml
- CHANGES.txt
- README.md
- biomaj_download/download/ftp.py
- biomaj_download/download/interface.py
- biomaj_download/download/localcopy.py
- biomaj_download/downloadclient.py
- biomaj_download/downloadservice.py
- biomaj_download/message/downmessage.proto
- biomaj_download/message/downmessage_pb2.py
- requirements.txt
- setup.py
- tests/biomaj_tests.py
- + tests/caert.demo.wftpserver.com.pem
- tests/global.properties
Changes:
=====================================
.travis.yml
=====================================
@@ -10,6 +10,10 @@ services:
branches:
except:
- "/^feature.*$/"
+addons:
+ apt:
+ packages:
+ - libgnutls-dev
install:
- pip install -r requirements.txt
- pip install coverage nose
=====================================
CHANGES.txt
=====================================
@@ -1,3 +1,17 @@
+3.0.27:
+ Fix previous release broken with a bug in direct protocols
+3.0.26:
+ Change default download timeout to 1h
+ #12 Allow FTPS protocol
+ #14 Add mechanism for protocol specific options
+3.0.25:
+ Allow to use hardlinks in LocalDownload
+3.0.24:
+ Remove debug logs
+3.0.23:
+ Support spaces in remote file names
+3.0.22:
+ Fix **/* remote.files parsing
3.0.21:
Fix traefik labels
3.0.20:
=====================================
README.md
=====================================
@@ -1,5 +1,7 @@
# About
+[](https://badge.fury.io/py/biomaj-download)
+
Microservice to manage the downloads of biomaj.
A protobuf interface is available in biomaj_download/message/message_pb2.py to exchange messages between BioMAJ and the download service.
@@ -9,7 +11,7 @@ Messages go through RabbitMQ (to be installed).
To compile protobuf, in biomaj_download/message:
- protoc --python_out=. message.proto
+ protoc --python_out=. downmessage.proto
# Development
=====================================
biomaj_download/download/ftp.py
=====================================
@@ -1,9 +1,11 @@
import pycurl
import re
import os
-from datetime import datetime
import time
+from datetime import datetime
+import stat
import hashlib
+import ftputil
from biomaj_core.utils import Utils
from biomaj_download.download.interface import DownloadInterface
@@ -13,6 +15,48 @@ try:
except ImportError:
from StringIO import StringIO as BytesIO
+# We use stat.filemode to convert from mode octal value to string.
+# In python < 3.3, stat.filmode is not defined.
+# This code is copied from the current implementation of stat.filemode.
+if 'filemode' not in stat.__dict__:
+ _filemode_table = (
+ ((stat.S_IFLNK, "l"), # noqa: E241
+ (stat.S_IFREG, "-"), # noqa: E241
+ (stat.S_IFBLK, "b"), # noqa: E241
+ (stat.S_IFDIR, "d"), # noqa: E241
+ (stat.S_IFCHR, "c"), # noqa: E241
+ (stat.S_IFIFO, "p")), # noqa: E241
+ ((stat.S_IRUSR, "r"),), # noqa: E241
+ ((stat.S_IWUSR, "w"),), # noqa: E241
+ ((stat.S_IXUSR | stat.S_ISUID, "s"), # noqa: E241
+ (stat.S_ISUID, "S"), # noqa: E241
+ (stat.S_IXUSR, "x")), # noqa: E241
+ ((stat.S_IRGRP, "r"),), # noqa: E241
+ ((stat.S_IWGRP, "w"),), # noqa: E241
+ ((stat.S_IXGRP | stat.S_ISGID, "s"), # noqa: E241
+ (stat.S_ISGID, "S"), # noqa: E241
+ (stat.S_IXGRP, "x")), # noqa: E241
+ ((stat.S_IROTH, "r"),), # noqa: E241
+ ((stat.S_IWOTH, "w"),), # noqa: E241
+ ((stat.S_IXOTH | stat.S_ISVTX, "t"), # noqa: E241
+ (stat.S_ISVTX, "T"), # noqa: E241
+ (stat.S_IXOTH, "x")) # noqa: E241
+ )
+
+ def _filemode(mode):
+ """Convert a file's mode to a string of the form '-rwxrwxrwx'."""
+ perm = []
+ for table in _filemode_table:
+ for bit, char in table:
+ if mode & bit == bit:
+ perm.append(char)
+ break
+ else:
+ perm.append("-")
+ return "".join(perm)
+
+ stat.filemode = _filemode
+
class FTPDownload(DownloadInterface):
'''
@@ -25,6 +69,12 @@ class FTPDownload(DownloadInterface):
remote.files=^alu.*\\.gz$
'''
+ # Utilities to parse ftp listings: UnixParser is the more common hence we
+ # put it first
+ ftp_listing_parsers = [
+ ftputil.stat.UnixParser(),
+ ftputil.stat.MSParser(),
+ ]
def __init__(self, protocol, host, rootdir):
DownloadInterface.__init__(self)
@@ -34,6 +84,25 @@ class FTPDownload(DownloadInterface):
self.rootdir = rootdir
self.url = url
self.headers = {}
+ # Initialize options
+ # Should we skip SSL verification (cURL -k/--insecure option)
+ self.ssl_verifyhost = True
+ self.ssl_verifypeer = True
+ # Path to the certificate of the server (cURL --cacert option; PEM format)
+ self.ssl_server_cert = None
+ # Keep alive
+ self.tcp_keepalive = 0
+
+ def set_options(self, protocol_options):
+ super(FTPDownload, self).set_options(protocol_options)
+ if "ssl_verifyhost" in protocol_options:
+ self.ssl_verifyhost = Utils.to_bool(protocol_options["ssl_verifyhost"])
+ if "ssl_verifypeer" in protocol_options:
+ self.ssl_verifypeer = Utils.to_bool(protocol_options["ssl_verifypeer"])
+ if "ssl_server_cert" in protocol_options:
+ self.ssl_server_cert = protocol_options["ssl_server_cert"]
+ if "tcp_keepalive" in protocol_options:
+ self.tcp_keepalive = Utils.to_int(protocol_options["tcp_keepalive"])
def match(self, patterns, file_list, dir_list=None, prefix='', submatch=False):
'''
@@ -63,19 +132,21 @@ class FTPDownload(DownloadInterface):
if subdir == '^':
subdirs_pattern = subdirs_pattern[1:]
subdir = subdirs_pattern[0]
+ # If getting all, get all files
+ if pattern == '**/*':
+ for rfile in file_list:
+ rfile['root'] = self.rootdir
+ if prefix != '':
+ rfile['name'] = prefix + '/' + rfile['name']
+ self.files_to_download.append(rfile)
+ self.logger.debug('Download:File:MatchRegExp:' + rfile['name'])
for direlt in dir_list:
subdir = direlt['name']
self.logger.debug('Download:File:Subdir:Check:' + subdir)
if pattern == '**/*':
(subfile_list, subdirs_list) = self.list(prefix + '/' + subdir + '/')
self.match([pattern], subfile_list, subdirs_list, prefix + '/' + subdir, True)
- for rfile in file_list:
- if pattern == '**/*' or re.match(pattern, rfile['name']):
- rfile['root'] = self.rootdir
- if prefix != '':
- rfile['name'] = prefix + '/' + rfile['name']
- self.files_to_download.append(rfile)
- self.logger.debug('Download:File:MatchRegExp:' + rfile['name'])
+
else:
if re.match(subdirs_pattern[0], subdir):
self.logger.debug('Download:File:Subdir:Match:' + subdir)
@@ -101,6 +172,27 @@ class FTPDownload(DownloadInterface):
while(error is True and nbtry < 3):
fp = open(file_path, "wb")
curl = pycurl.Curl()
+
+ # Configure TCP keepalive
+ if self.tcp_keepalive:
+ curl.setopt(pycurl.TCP_KEEPALIVE, True)
+ curl.setopt(pycurl.TCP_KEEPIDLE, self.tcp_keepalive * 2)
+ curl.setopt(pycurl.TCP_KEEPINTVL, self.tcp_keepalive)
+
+ # Configure SSL verification (on some platforms, disabling
+ # SSL_VERIFYPEER implies disabling SSL_VERIFYHOST so we set
+ # SSL_VERIFYPEER after)
+ curl.setopt(pycurl.SSL_VERIFYHOST, 2 if self.ssl_verifyhost else 0)
+ curl.setopt(pycurl.SSL_VERIFYPEER, 1 if self.ssl_verifypeer else 0)
+ if self.ssl_server_cert:
+ # cacert is the name of the option for the curl command. The
+ # corresponding cURL option is CURLOPT_CAINFO.
+ # See https://curl.haxx.se/libcurl/c/CURLOPT_CAINFO.html
+ # This is inspired by that https://curl.haxx.se/docs/sslcerts.html
+ # (section "Certificate Verification", option 2) but the option
+ # CURLOPT_CAPATH is for a directory of certificates.
+ curl.setopt(pycurl.CAINFO, self.ssl_server_cert)
+
try:
curl.setopt(pycurl.URL, file_to_download)
except Exception:
@@ -133,8 +225,7 @@ class FTPDownload(DownloadInterface):
nbtry += 1
curl.close()
fp.close()
- skip_check_uncompress = os.environ.get('UNCOMPRESS_SKIP_CHECK', None)
- if not error and skip_check_uncompress is None:
+ if not error and not self.skip_check_uncompress:
archive_status = Utils.archive_check(file_path)
if not archive_status:
self.logger.error('Archive is invalid or corrupted, deleting file and retrying download')
@@ -231,6 +322,18 @@ class FTPDownload(DownloadInterface):
'''
self.logger.debug('Download:List:' + self.url + self.rootdir + directory)
+ # Configure TCP keepalive
+ if self.tcp_keepalive:
+ self.crl.setopt(pycurl.TCP_KEEPALIVE, True)
+ self.crl.setopt(pycurl.TCP_KEEPIDLE, self.tcp_keepalive * 2)
+ self.crl.setopt(pycurl.TCP_KEEPINTVL, self.tcp_keepalive)
+
+ # See the corresponding lines in method:`curl_download`
+ self.crl.setopt(pycurl.SSL_VERIFYHOST, 2 if self.ssl_verifyhost else 0)
+ self.crl.setopt(pycurl.SSL_VERIFYPEER, 1 if self.ssl_verifypeer else 0)
+ if self.ssl_server_cert:
+ self.crl.setopt(pycurl.CAINFO, self.ssl_server_cert)
+
try:
self.crl.setopt(pycurl.URL, self.url + self.rootdir + directory)
except Exception:
@@ -252,6 +355,7 @@ class FTPDownload(DownloadInterface):
# Download should not take more than 5minutes
self.crl.setopt(pycurl.TIMEOUT, self.timeout)
self.crl.setopt(pycurl.NOSIGNAL, 1)
+
try:
self.crl.perform()
except Exception as e:
@@ -282,44 +386,46 @@ class FTPDownload(DownloadInterface):
rdirs = []
for line in lines:
- rfile = {}
- # lets print each part separately
- parts = line.split()
- # the individual fields in this list of parts
- if not parts:
+ # Skip empty lines (usually the last)
+ if not line:
continue
- rfile['permissions'] = parts[0]
- rfile['group'] = parts[2]
- rfile['user'] = parts[3]
- rfile['size'] = int(parts[4])
- rfile['month'] = Utils.month_to_num(parts[5])
- rfile['day'] = int(parts[6])
+ # Parse the line
+ for i, parser in enumerate(self.ftp_listing_parsers, 1):
+ try:
+ stats = parser.parse_line(line)
+ break
+ except ftputil.error.ParserError:
+ # If it's the last parser, re-raise the exception
+ if i == len(self.ftp_listing_parsers):
+ raise
+ else:
+ continue
+ # Put stats in a dict
+ rfile = {}
+ rfile['name'] = stats._st_name
+ # Reparse mode to a string
+ rfile['permissions'] = stat.filemode(stats.st_mode)
+ rfile['group'] = stats.st_gid
+ rfile['user'] = stats.st_uid
+ rfile['size'] = stats.st_size
+ mtime = time.localtime(stats.st_mtime)
+ rfile['year'] = mtime.tm_year
+ rfile['month'] = mtime.tm_mon
+ rfile['day'] = mtime.tm_mday
rfile['hash'] = hashlib.md5(line.encode('utf-8')).hexdigest()
- try:
- rfile['year'] = int(parts[7])
- except Exception:
- # specific ftp case issues at getting date info
- curdate = datetime.now()
- rfile['year'] = curdate.year
- # Year not precised, month feater than current means previous year
- if rfile['month'] > curdate.month:
- rfile['year'] = curdate.year - 1
- # Same month but later day => previous year
- if rfile['month'] == curdate.month and rfile['day'] > curdate.day:
- rfile['year'] = curdate.year - 1
- rfile['name'] = parts[8]
- if len(parts) >= 10 and parts[9] == '->':
- # Symlink, add to files AND dirs as we don't know the type of the link
- rdirs.append(rfile)
-
- is_dir = False
- if re.match('^d', rfile['permissions']):
- is_dir = True
- if not is_dir:
+ is_link = stat.S_ISLNK(stats.st_mode)
+ is_dir = stat.S_ISDIR(stats.st_mode)
+ # Append links to dirs and files since we don't know what the
+ # target is
+ if is_link:
rfiles.append(rfile)
- else:
rdirs.append(rfile)
+ else:
+ if not is_dir:
+ rfiles.append(rfile)
+ else:
+ rdirs.append(rfile)
return (rfiles, rdirs)
def chroot(self, cwd):
=====================================
biomaj_download/download/interface.py
=====================================
@@ -4,6 +4,8 @@ import datetime
import time
import re
+from biomaj_core.utils import Utils
+
class _FakeLock(object):
'''
@@ -39,7 +41,7 @@ class DownloadInterface(object):
self.kill_received = False
self.proxy = None
# 24h timeout
- self.timeout = 3600 * 24
+ self.timeout = 3600
# Optional save target for single file downloaders
self.save_as = None
self.logger = logging.getLogger('biomaj')
@@ -48,6 +50,9 @@ class DownloadInterface(object):
self.protocol = None
self.server = None
self.offline_dir = None
+ # Options
+ self.protocol_options = {}
+ self.skip_check_uncompress = False
def set_offline_dir(self, offline_dir):
self.offline_dir = offline_dir
@@ -266,6 +271,17 @@ class DownloadInterface(object):
'''
self.credentials = userpwd
+ def set_options(self, protocol_options):
+ """
+ Set protocol specific options.
+
+ Subclasses that override this method must call the
+ parent implementation.
+ """
+ self.protocol_options = protocol_options
+ if "skip_check_uncompress" in protocol_options:
+ self.skip_check_uncompress = Utils.to_bool(protocol_options["skip_check_uncompress"])
+
def close(self):
'''
Close connection
=====================================
biomaj_download/download/localcopy.py
=====================================
@@ -18,10 +18,11 @@ class LocalDownload(DownloadInterface):
'''
- def __init__(self, rootdir):
+ def __init__(self, rootdir, use_hardlinks=False):
DownloadInterface.__init__(self)
self.logger.debug('Download')
self.rootdir = rootdir
+ self.use_hardlinks = use_hardlinks
def download(self, local_dir):
'''
@@ -32,7 +33,9 @@ class LocalDownload(DownloadInterface):
:return: list of downloaded files
'''
self.logger.debug('Local:Download')
- Utils.copy_files(self.files_to_download, local_dir, lock=self.mkdir_lock)
+ Utils.copy_files(self.files_to_download, local_dir,
+ use_hardlinks=self.use_hardlinks,
+ lock=self.mkdir_lock)
for rfile in self.files_to_download:
rfile['download_time'] = 0
=====================================
biomaj_download/downloadclient.py
=====================================
@@ -80,7 +80,8 @@ class DownloadClient(DownloadService):
result = r.json()
return (result['progress'], result['errors'])
except Exception:
- logging.exception('Failed to connect to the download proxy: %s' % (url))
+ logging.exception('Failed to connect to the download proxy: %s, retrying in 2 seconds' % (url))
+ time.sleep(2)
raise Exception('Failed to connect to the download proxy')
def download_remote_files(self, cf, downloaders, offline_dir):
=====================================
biomaj_download/downloadservice.py
=====================================
@@ -28,6 +28,14 @@ app = Flask(__name__)
app_log = logging.getLogger('werkzeug')
app_log.setLevel(logging.ERROR)
+# Classify protocols from downmessage.proto
+# Note: those lists are based on the protocol numbers, not the protocol names
+ALL_PROTOCOLS = [item for key, item in downmessage_pb2.DownloadFile.Protocol.items()]
+DIRECT_PROTOCOLS = [
+ item for key, item in downmessage_pb2.DownloadFile.Protocol.items()
+ if key.startswith("DIRECT")
+]
+
@app.route('/api/download-message')
def ping():
@@ -122,7 +130,8 @@ class DownloadService(object):
def get_handler(self, protocol_name, server, remote_dir, remote_files=[],
credentials=None, http_parse=None, http_method=None, param=None,
proxy=None, proxy_auth='',
- save_as=None, timeout_download=None, offline_dir=None):
+ save_as=None, timeout_download=None, offline_dir=None,
+ protocol_options={}):
protocol = downmessage_pb2.DownloadFile.Protocol.Value(protocol_name.upper())
downloader = None
if protocol in [0, 1]:
@@ -133,6 +142,8 @@ class DownloadService(object):
downloader = LocalDownload(remote_dir)
if protocol == 4:
downloader = DirectFTPDownload('ftp', server, '/')
+ if protocol == 10:
+ downloader = DirectFTPDownload('ftps', server, '/')
if protocol == 5:
downloader = DirectHttpDownload('http', server, '/')
if protocol == 6:
@@ -147,8 +158,8 @@ class DownloadService(object):
for remote_file in remote_files:
if remote_file['save_as']:
save_as = remote_file['save_as']
- # For direct protocol, we only keep base name
- if protocol in [4, 5, 6]:
+ # For direct protocols, we only keep base name
+ if protocol in DIRECT_PROTOCOLS:
tmp_remote = []
for remote_file in remote_files:
tmp_remote.append(remote_file['name'])
@@ -178,6 +189,10 @@ class DownloadService(object):
downloader.set_protocol(protocol_name)
+ if protocol_options is not None:
+ self.logger.debug("Received protocol options: " + str(protocol_options))
+ downloader.set_options(protocol_options)
+
downloader.logger = self.logger
downloader.set_files_to_download(remote_files)
return downloader
@@ -226,7 +241,9 @@ class DownloadService(object):
proxy_auth=proxy_auth,
save_as=biomaj_file_info.remote_file.save_as,
timeout_download=biomaj_file_info.timeout_download,
- offline_dir=biomaj_file_info.local_dir)
+ offline_dir=biomaj_file_info.local_dir,
+ protocol_options=biomaj_file_info.protocol_options
+ )
def clean(self, biomaj_file_info=None):
'''
=====================================
biomaj_download/message/downmessage.proto
=====================================
@@ -1,3 +1,5 @@
+syntax = "proto2";
+
package biomaj.download;
message File {
@@ -64,7 +66,7 @@ message DownloadFile {
enum Protocol {
FTP = 0;
- SFTP = 1;
+ FTPS = 1;
HTTP = 2;
HTTPS = 3;
DIRECTFTP = 4;
@@ -73,6 +75,7 @@ message DownloadFile {
LOCAL = 7;
RSYNC = 8;
IRODS = 9;
+ DIRECTFTPS = 10;
}
message Param {
@@ -83,7 +86,7 @@ message DownloadFile {
message HttpParse {
required string dir_line = 1 [default = '<img[\\s]+src="[\\S]+"[\\s]+alt="\\[DIR\\]"[\\s]*/?>[\\s]*<a[\\s]+href="([\\S]+)/"[\\s]*>.*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})'];
required string file_line = 2 [default = '<img[\\s]+src="[\\S]+"[\\s]+alt="\\[[\\s]+\\]"[\\s]*/?>[\\s]<a[\\s]+href="([\\S]+)".*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})[\\s]+([\\d\\.]+[MKG]{0,1})'];
- required int32 dir_name = 3 [default =1];
+ required int32 dir_name = 3 [default = 1];
required int32 dir_date = 4 [default = 2];
required int32 file_name = 5 [default = 1];
required int32 file_date = 6 [default = 2];
@@ -119,5 +122,7 @@ message DownloadFile {
optional Proxy proxy = 6;
optional HTTP_METHOD http_method = 8 [ default = GET];
+
+ map<string, string> protocol_options = 9;
}
=====================================
biomaj_download/message/downmessage_pb2.py
=====================================
@@ -7,7 +7,6 @@ from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
-from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
@@ -18,9 +17,10 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='downmessage.proto',
package='biomaj.download',
- serialized_pb=_b('\n\x11\x64ownmessage.proto\x12\x0f\x62iomaj.download\"\x9d\x02\n\x04\x46ile\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\x0c\n\x04root\x18\x02 \x01(\t\x12\x0f\n\x07save_as\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\x30\n\x08metadata\x18\x05 \x01(\x0b\x32\x1e.biomaj.download.File.MetaData\x1a\xa8\x01\n\x08MetaData\x12\x13\n\x0bpermissions\x18\x01 \x01(\t\x12\r\n\x05group\x18\x02 \x01(\t\x12\x0c\n\x04size\x18\x03 \x01(\x03\x12\x0c\n\x04hash\x18\x04 \x01(\t\x12\x0c\n\x04year\x18\x05 \x01(\x05\x12\r\n\x05month\x18\x06 \x01(\x05\x12\x0b\n\x03\x64\x61y\x18\x07 \x01(\x05\x12\x0e\n\x06\x66ormat\x18\x08 \x01(\t\x12\x0b\n\x03md5\x18\t \x01(\t\x12\x15\n\rdownload_time\x18\n \x01(\x03\"0\n\x08\x46ileList\x12$\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x15.biomaj.download.File\"\xaa\x02\n\tOperation\x12\x32\n\x04type\x18\x01 \x02(\x0e\x32$.biomaj.download.Operation.OPERATION\x12/\n\x08\x64ownload\x18\x02 \x01(\x0b\x32\x1d.biomaj.download.DownloadFile\x12)\n\x07process\x18\x03 \x01(\x0b\x32\x18.biomaj.download.Process\x12/\n\x05trace\x18\x04 \x01(\x0b\x32 .biomaj.download.Operation.Trace\x1a*\n\x05Trace\x12\x10\n\x08trace_id\x18\x01 \x02(\t\x12\x0f\n\x07span_id\x18\x02 \x02(\t\"0\n\tOPERATION\x12\x08\n\x04LIST\x10\x00\x12\x0c\n\x08\x44OWNLOAD\x10\x01\x12\x0b\n\x07PROCESS\x10\x02\"\x17\n\x07Process\x12\x0c\n\x04\x65xec\x18\x01 \x02(\t\"\x97\n\n\x0c\x44ownloadFile\x12\x0c\n\x04\x62\x61nk\x18\x01 \x02(\t\x12\x0f\n\x07session\x18\x02 \x02(\t\x12\x11\n\tlocal_dir\x18\x03 \x02(\t\x12\x18\n\x10timeout_download\x18\x04 \x01(\x05\x12=\n\x0bremote_file\x18\x05 \x02(\x0b\x32(.biomaj.download.DownloadFile.RemoteFile\x12\x32\n\x05proxy\x18\x06 \x01(\x0b\x32#.biomaj.download.DownloadFile.Proxy\x12\x43\n\x0bhttp_method\x18\x08 \x01(\x0e\x32).biomaj.download.DownloadFile.HTTP_METHOD:\x03GET\x1a$\n\x05Param\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\r\n\x05value\x18\x02 \x02(\t\x1a\xcd\x03\n\tHttpParse\x12\x91\x01\n\x08\x64ir_line\x18\x01 \x02(\t:\x7f<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[DIR\\]\"[\\s]*/?>[\\s]*<a[\\s]+href=\"([\\S]+)/\"[\\s]*>.*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})\x12\xa5\x01\n\tfile_line\x18\x02 \x02(\t:\x91\x01<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[[\\s]+\\]\"[\\s]*/?>[\\s]<a[\\s]+href=\"([\\S]+)\".*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})[\\s]+([\\d\\.]+[MKG]{0,1})\x12\x13\n\x08\x64ir_name\x18\x03 \x02(\x05:\x01\x31\x12\x13\n\x08\x64ir_date\x18\x04 \x02(\x05:\x01\x32\x12\x14\n\tfile_name\x18\x05 \x02(\x05:\x01\x31\x12\x14\n\tfile_date\x18\x06 \x02(\x05:\x01\x32\x12\x18\n\x10\x66ile_date_format\x18\x07 \x01(\t\x12\x14\n\tfile_size\x18\x08 \x02(\x05:\x01\x33\x1a\xb8\x02\n\nRemoteFile\x12$\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x15.biomaj.download.File\x12\x38\n\x08protocol\x18\x02 \x02(\x0e\x32&.biomaj.download.DownloadFile.Protocol\x12\x0e\n\x06server\x18\x03 \x02(\t\x12\x12\n\nremote_dir\x18\x04 \x02(\t\x12\x0f\n\x07save_as\x18\x05 \x01(\t\x12\x32\n\x05param\x18\x06 \x03(\x0b\x32#.biomaj.download.DownloadFile.Param\x12;\n\nhttp_parse\x18\x07 \x01(\x0b\x32\'.biomaj.download.DownloadFile.HttpParse\x12\x13\n\x0b\x63redentials\x18\x08 \x01(\t\x12\x0f\n\x07matches\x18\t \x03(\t\x1a*\n\x05Proxy\x12\r\n\x05proxy\x18\x01 \x02(\t\x12\x12\n\nproxy_auth\x18\x02 \x01(\t\"\x83\x01\n\x08Protocol\x12\x07\n\x03\x46TP\x10\x00\x12\x08\n\x04SFTP\x10\x01\x12\x08\n\x04HTTP\x10\x02\x12\t\n\x05HTTPS\x10\x03\x12\r\n\tDIRECTFTP\x10\x04\x12\x0e\n\nDIRECTHTTP\x10\x05\x12\x0f\n\x0b\x44IRECTHTTPS\x10\x06\x12\t\n\x05LOCAL\x10\x07\x12\t\n\x05RSYNC\x10\x08\x12\t\n\x05IRODS\x10\t\" \n\x0bHTTP_METHOD\x12\x07\n\x03GET\x10\x00\x12\x08\n\x04POST\x10\x01')
+ syntax='proto2',
+ serialized_options=None,
+ serialized_pb=_b('\n\x11\x64ownmessage.proto\x12\x0f\x62iomaj.download\"\x9d\x02\n\x04\x46ile\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\x0c\n\x04root\x18\x02 \x01(\t\x12\x0f\n\x07save_as\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\x30\n\x08metadata\x18\x05 \x01(\x0b\x32\x1e.biomaj.download.File.MetaData\x1a\xa8\x01\n\x08MetaData\x12\x13\n\x0bpermissions\x18\x01 \x01(\t\x12\r\n\x05group\x18\x02 \x01(\t\x12\x0c\n\x04size\x18\x03 \x01(\x03\x12\x0c\n\x04hash\x18\x04 \x01(\t\x12\x0c\n\x04year\x18\x05 \x01(\x05\x12\r\n\x05month\x18\x06 \x01(\x05\x12\x0b\n\x03\x64\x61y\x18\x07 \x01(\x05\x12\x0e\n\x06\x66ormat\x18\x08 \x01(\t\x12\x0b\n\x03md5\x18\t \x01(\t\x12\x15\n\rdownload_time\x18\n \x01(\x03\"0\n\x08\x46ileList\x12$\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x15.biomaj.download.File\"\xaa\x02\n\tOperation\x12\x32\n\x04type\x18\x01 \x02(\x0e\x32$.biomaj.download.Operation.OPERATION\x12/\n\x08\x64ownload\x18\x02 \x01(\x0b\x32\x1d.biomaj.download.DownloadFile\x12)\n\x07process\x18\x03 \x01(\x0b\x32\x18.biomaj.download.Process\x12/\n\x05trace\x18\x04 \x01(\x0b\x32 .biomaj.download.Operation.Trace\x1a*\n\x05Trace\x12\x10\n\x08trace_id\x18\x01 \x02(\t\x12\x0f\n\x07span_id\x18\x02 \x02(\t\"0\n\tOPERATION\x12\x08\n\x04LIST\x10\x00\x12\x0c\n\x08\x44OWNLOAD\x10\x01\x12\x0b\n\x07PROCESS\x10\x02\"\x17\n\x07Process\x12\x0c\n\x04\x65xec\x18\x01 \x02(\t\"\xad\x0b\n\x0c\x44ownloadFile\x12\x0c\n\x04\x62\x61nk\x18\x01 \x02(\t\x12\x0f\n\x07session\x18\x02 \x02(\t\x12\x11\n\tlocal_dir\x18\x03 \x02(\t\x12\x18\n\x10timeout_download\x18\x04 \x01(\x05\x12=\n\x0bremote_file\x18\x05 \x02(\x0b\x32(.biomaj.download.DownloadFile.RemoteFile\x12\x32\n\x05proxy\x18\x06 \x01(\x0b\x32#.biomaj.download.DownloadFile.Proxy\x12\x43\n\x0bhttp_method\x18\x08 \x01(\x0e\x32).biomaj.download.DownloadFile.HTTP_METHOD:\x03GET\x12L\n\x10protocol_options\x18\t \x03(\x0b\x32\x32.biomaj.download.DownloadFile.ProtocolOptionsEntry\x1a$\n\x05Param\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\r\n\x05value\x18\x02 \x02(\t\x1a\xcd\x03\n\tHttpParse\x12\x91\x01\n\x08\x64ir_line\x18\x01 \x02(\t:\x7f<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[DIR\\]\"[\\s]*/?>[\\s]*<a[\\s]+href=\"([\\S]+)/\"[\\s]*>.*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})\x12\xa5\x01\n\tfile_line\x18\x02 \x02(\t:\x91\x01<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[[\\s]+\\]\"[\\s]*/?>[\\s]<a[\\s]+href=\"([\\S]+)\".*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})[\\s]+([\\d\\.]+[MKG]{0,1})\x12\x13\n\x08\x64ir_name\x18\x03 \x02(\x05:\x01\x31\x12\x13\n\x08\x64ir_date\x18\x04 \x02(\x05:\x01\x32\x12\x14\n\tfile_name\x18\x05 \x02(\x05:\x01\x31\x12\x14\n\tfile_date\x18\x06 \x02(\x05:\x01\x32\x12\x18\n\x10\x66ile_date_format\x18\x07 \x01(\t\x12\x14\n\tfile_size\x18\x08 \x02(\x05:\x01\x33\x1a\xb8\x02\n\nRemoteFile\x12$\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x15.biomaj.download.File\x12\x38\n\x08protocol\x18\x02 \x02(\x0e\x32&.biomaj.download.DownloadFile.Protocol\x12\x0e\n\x06server\x18\x03 \x02(\t\x12\x12\n\nremote_dir\x18\x04 \x02(\t\x12\x0f\n\x07save_as\x18\x05 \x01(\t\x12\x32\n\x05param\x18\x06 \x03(\x0b\x32#.biomaj.download.DownloadFile.Param\x12;\n\nhttp_parse\x18\x07 \x01(\x0b\x32\'.biomaj.download.DownloadFile.HttpParse\x12\x13\n\x0b\x63redentials\x18\x08 \x01(\t\x12\x0f\n\x07matches\x18\t \x03(\t\x1a*\n\x05Proxy\x12\r\n\x05proxy\x18\x01 \x02(\t\x12\x12\n\nproxy_auth\x18\x02 \x01(\t\x1a\x36\n\x14ProtocolOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x93\x01\n\x08Protocol\x12\x07\n\x03\x46TP\x10\x00\x12\x08\n\x04\x46TPS\x10\x01\x12\x08\n\x04HTTP\x10\x02\x12\t\n\x05HTTPS\x10\x03\x12\r\n\tDIRECTFTP\x10\x04\x12\x0e\n\nDIRECTHTTP\x10\x05\x12\x0f\n\x0b\x44IRECTHTTPS\x10\x06\x12\t\n\x05LOCAL\x10\x07\x12\t\n\x05RSYNC\x10\x08\x12\t\n\x05IRODS\x10\t\x12\x0e\n\nDIRECTFTPS\x10\n\" \n\x0bHTTP_METHOD\x12\x07\n\x03GET\x10\x00\x12\x08\n\x04POST\x10\x01')
)
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
@@ -32,19 +32,19 @@ _OPERATION_OPERATION = _descriptor.EnumDescriptor(
values=[
_descriptor.EnumValueDescriptor(
name='LIST', index=0, number=0,
- options=None,
+ serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='DOWNLOAD', index=1, number=1,
- options=None,
+ serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='PROCESS', index=2, number=2,
- options=None,
+ serialized_options=None,
type=None),
],
containing_type=None,
- options=None,
+ serialized_options=None,
serialized_start=627,
serialized_end=675,
)
@@ -58,49 +58,53 @@ _DOWNLOADFILE_PROTOCOL = _descriptor.EnumDescriptor(
values=[
_descriptor.EnumValueDescriptor(
name='FTP', index=0, number=0,
- options=None,
+ serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
- name='SFTP', index=1, number=1,
- options=None,
+ name='FTPS', index=1, number=1,
+ serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='HTTP', index=2, number=2,
- options=None,
+ serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='HTTPS', index=3, number=3,
- options=None,
+ serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='DIRECTFTP', index=4, number=4,
- options=None,
+ serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='DIRECTHTTP', index=5, number=5,
- options=None,
+ serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='DIRECTHTTPS', index=6, number=6,
- options=None,
+ serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='LOCAL', index=7, number=7,
- options=None,
+ serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='RSYNC', index=8, number=8,
- options=None,
+ serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='IRODS', index=9, number=9,
- options=None,
+ serialized_options=None,
+ type=None),
+ _descriptor.EnumValueDescriptor(
+ name='DIRECTFTPS', index=10, number=10,
+ serialized_options=None,
type=None),
],
containing_type=None,
- options=None,
- serialized_start=1841,
- serialized_end=1972,
+ serialized_options=None,
+ serialized_start=1975,
+ serialized_end=2122,
)
_sym_db.RegisterEnumDescriptor(_DOWNLOADFILE_PROTOCOL)
@@ -112,17 +116,17 @@ _DOWNLOADFILE_HTTP_METHOD = _descriptor.EnumDescriptor(
values=[
_descriptor.EnumValueDescriptor(
name='GET', index=0, number=0,
- options=None,
+ serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='POST', index=1, number=1,
- options=None,
+ serialized_options=None,
type=None),
],
containing_type=None,
- options=None,
- serialized_start=1974,
- serialized_end=2006,
+ serialized_options=None,
+ serialized_start=2124,
+ serialized_end=2156,
)
_sym_db.RegisterEnumDescriptor(_DOWNLOADFILE_HTTP_METHOD)
@@ -140,78 +144,79 @@ _FILE_METADATA = _descriptor.Descriptor(
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='group', full_name='biomaj.download.File.MetaData.group', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='size', full_name='biomaj.download.File.MetaData.size', index=2,
number=3, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='hash', full_name='biomaj.download.File.MetaData.hash', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='year', full_name='biomaj.download.File.MetaData.year', index=4,
number=5, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='month', full_name='biomaj.download.File.MetaData.month', index=5,
number=6, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='day', full_name='biomaj.download.File.MetaData.day', index=6,
number=7, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='format', full_name='biomaj.download.File.MetaData.format', index=7,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='md5', full_name='biomaj.download.File.MetaData.md5', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='download_time', full_name='biomaj.download.File.MetaData.download_time', index=9,
number=10, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
- options=None,
+ serialized_options=None,
is_extendable=False,
+ syntax='proto2',
extension_ranges=[],
oneofs=[
],
@@ -232,43 +237,44 @@ _FILE = _descriptor.Descriptor(
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='root', full_name='biomaj.download.File.root', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='save_as', full_name='biomaj.download.File.save_as', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='url', full_name='biomaj.download.File.url', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='metadata', full_name='biomaj.download.File.metadata', index=4,
number=5, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[_FILE_METADATA, ],
enum_types=[
],
- options=None,
+ serialized_options=None,
is_extendable=False,
+ syntax='proto2',
extension_ranges=[],
oneofs=[
],
@@ -290,15 +296,16 @@ _FILELIST = _descriptor.Descriptor(
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
- options=None,
+ serialized_options=None,
is_extendable=False,
+ syntax='proto2',
extension_ranges=[],
oneofs=[
],
@@ -320,22 +327,23 @@ _OPERATION_TRACE = _descriptor.Descriptor(
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='span_id', full_name='biomaj.download.Operation.Trace.span_id', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
- options=None,
+ serialized_options=None,
is_extendable=False,
+ syntax='proto2',
extension_ranges=[],
oneofs=[
],
@@ -356,28 +364,28 @@ _OPERATION = _descriptor.Descriptor(
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='download', full_name='biomaj.download.Operation.download', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='process', full_name='biomaj.download.Operation.process', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='trace', full_name='biomaj.download.Operation.trace', index=3,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
@@ -385,8 +393,9 @@ _OPERATION = _descriptor.Descriptor(
enum_types=[
_OPERATION_OPERATION,
],
- options=None,
+ serialized_options=None,
is_extendable=False,
+ syntax='proto2',
extension_ranges=[],
oneofs=[
],
@@ -408,15 +417,16 @@ _PROCESS = _descriptor.Descriptor(
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
- options=None,
+ serialized_options=None,
is_extendable=False,
+ syntax='proto2',
extension_ranges=[],
oneofs=[
],
@@ -438,27 +448,28 @@ _DOWNLOADFILE_PARAM = _descriptor.Descriptor(
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='value', full_name='biomaj.download.DownloadFile.Param.value', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
- options=None,
+ serialized_options=None,
is_extendable=False,
+ syntax='proto2',
extension_ranges=[],
oneofs=[
],
- serialized_start=979,
- serialized_end=1015,
+ serialized_start=1057,
+ serialized_end=1093,
)
_DOWNLOADFILE_HTTPPARSE = _descriptor.Descriptor(
@@ -474,69 +485,70 @@ _DOWNLOADFILE_HTTPPARSE = _descriptor.Descriptor(
has_default_value=True, default_value=_b("<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[DIR\\]\"[\\s]*/?>[\\s]*<a[\\s]+href=\"([\\S]+)/\"[\\s]*>.*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='file_line', full_name='biomaj.download.DownloadFile.HttpParse.file_line', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=True, default_value=_b("<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[[\\s]+\\]\"[\\s]*/?>[\\s]<a[\\s]+href=\"([\\S]+)\".*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})[\\s]+([\\d\\.]+[MKG]{0,1})").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='dir_name', full_name='biomaj.download.DownloadFile.HttpParse.dir_name', index=2,
number=3, type=5, cpp_type=1, label=2,
has_default_value=True, default_value=1,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='dir_date', full_name='biomaj.download.DownloadFile.HttpParse.dir_date', index=3,
number=4, type=5, cpp_type=1, label=2,
has_default_value=True, default_value=2,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='file_name', full_name='biomaj.download.DownloadFile.HttpParse.file_name', index=4,
number=5, type=5, cpp_type=1, label=2,
has_default_value=True, default_value=1,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='file_date', full_name='biomaj.download.DownloadFile.HttpParse.file_date', index=5,
number=6, type=5, cpp_type=1, label=2,
has_default_value=True, default_value=2,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='file_date_format', full_name='biomaj.download.DownloadFile.HttpParse.file_date_format', index=6,
number=7, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='file_size', full_name='biomaj.download.DownloadFile.HttpParse.file_size', index=7,
number=8, type=5, cpp_type=1, label=2,
has_default_value=True, default_value=3,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
- options=None,
+ serialized_options=None,
is_extendable=False,
+ syntax='proto2',
extension_ranges=[],
oneofs=[
],
- serialized_start=1018,
- serialized_end=1479,
+ serialized_start=1096,
+ serialized_end=1557,
)
_DOWNLOADFILE_REMOTEFILE = _descriptor.Descriptor(
@@ -552,76 +564,77 @@ _DOWNLOADFILE_REMOTEFILE = _descriptor.Descriptor(
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='protocol', full_name='biomaj.download.DownloadFile.RemoteFile.protocol', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='server', full_name='biomaj.download.DownloadFile.RemoteFile.server', index=2,
number=3, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='remote_dir', full_name='biomaj.download.DownloadFile.RemoteFile.remote_dir', index=3,
number=4, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='save_as', full_name='biomaj.download.DownloadFile.RemoteFile.save_as', index=4,
number=5, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='param', full_name='biomaj.download.DownloadFile.RemoteFile.param', index=5,
number=6, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='http_parse', full_name='biomaj.download.DownloadFile.RemoteFile.http_parse', index=6,
number=7, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='credentials', full_name='biomaj.download.DownloadFile.RemoteFile.credentials', index=7,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='matches', full_name='biomaj.download.DownloadFile.RemoteFile.matches', index=8,
number=9, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
- options=None,
+ serialized_options=None,
is_extendable=False,
+ syntax='proto2',
extension_ranges=[],
oneofs=[
],
- serialized_start=1482,
- serialized_end=1794,
+ serialized_start=1560,
+ serialized_end=1872,
)
_DOWNLOADFILE_PROXY = _descriptor.Descriptor(
@@ -637,27 +650,65 @@ _DOWNLOADFILE_PROXY = _descriptor.Descriptor(
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='proxy_auth', full_name='biomaj.download.DownloadFile.Proxy.proxy_auth', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto2',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=1874,
+ serialized_end=1916,
+)
+
+_DOWNLOADFILE_PROTOCOLOPTIONSENTRY = _descriptor.Descriptor(
+ name='ProtocolOptionsEntry',
+ full_name='biomaj.download.DownloadFile.ProtocolOptionsEntry',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name='key', full_name='biomaj.download.DownloadFile.ProtocolOptionsEntry.key', index=0,
+ number=1, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=_b("").decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
+ _descriptor.FieldDescriptor(
+ name='value', full_name='biomaj.download.DownloadFile.ProtocolOptionsEntry.value', index=1,
+ number=2, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=_b("").decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
- options=None,
+ serialized_options=_b('8\001'),
is_extendable=False,
+ syntax='proto2',
extension_ranges=[],
oneofs=[
],
- serialized_start=1796,
- serialized_end=1838,
+ serialized_start=1918,
+ serialized_end=1972,
)
_DOWNLOADFILE = _descriptor.Descriptor(
@@ -673,64 +724,72 @@ _DOWNLOADFILE = _descriptor.Descriptor(
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='session', full_name='biomaj.download.DownloadFile.session', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='local_dir', full_name='biomaj.download.DownloadFile.local_dir', index=2,
number=3, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timeout_download', full_name='biomaj.download.DownloadFile.timeout_download', index=3,
number=4, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='remote_file', full_name='biomaj.download.DownloadFile.remote_file', index=4,
number=5, type=11, cpp_type=10, label=2,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='proxy', full_name='biomaj.download.DownloadFile.proxy', index=5,
number=6, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='http_method', full_name='biomaj.download.DownloadFile.http_method', index=6,
number=8, type=14, cpp_type=8, label=1,
has_default_value=True, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
- options=None),
+ serialized_options=None, file=DESCRIPTOR),
+ _descriptor.FieldDescriptor(
+ name='protocol_options', full_name='biomaj.download.DownloadFile.protocol_options', index=7,
+ number=9, type=11, cpp_type=10, label=3,
+ has_default_value=False, default_value=[],
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
- nested_types=[_DOWNLOADFILE_PARAM, _DOWNLOADFILE_HTTPPARSE, _DOWNLOADFILE_REMOTEFILE, _DOWNLOADFILE_PROXY, ],
+ nested_types=[_DOWNLOADFILE_PARAM, _DOWNLOADFILE_HTTPPARSE, _DOWNLOADFILE_REMOTEFILE, _DOWNLOADFILE_PROXY, _DOWNLOADFILE_PROTOCOLOPTIONSENTRY, ],
enum_types=[
_DOWNLOADFILE_PROTOCOL,
_DOWNLOADFILE_HTTP_METHOD,
],
- options=None,
+ serialized_options=None,
is_extendable=False,
+ syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=703,
- serialized_end=2006,
+ serialized_end=2156,
)
_FILE_METADATA.containing_type = _FILE
@@ -750,9 +809,11 @@ _DOWNLOADFILE_REMOTEFILE.fields_by_name['param'].message_type = _DOWNLOADFILE_PA
_DOWNLOADFILE_REMOTEFILE.fields_by_name['http_parse'].message_type = _DOWNLOADFILE_HTTPPARSE
_DOWNLOADFILE_REMOTEFILE.containing_type = _DOWNLOADFILE
_DOWNLOADFILE_PROXY.containing_type = _DOWNLOADFILE
+_DOWNLOADFILE_PROTOCOLOPTIONSENTRY.containing_type = _DOWNLOADFILE
_DOWNLOADFILE.fields_by_name['remote_file'].message_type = _DOWNLOADFILE_REMOTEFILE
_DOWNLOADFILE.fields_by_name['proxy'].message_type = _DOWNLOADFILE_PROXY
_DOWNLOADFILE.fields_by_name['http_method'].enum_type = _DOWNLOADFILE_HTTP_METHOD
+_DOWNLOADFILE.fields_by_name['protocol_options'].message_type = _DOWNLOADFILE_PROTOCOLOPTIONSENTRY
_DOWNLOADFILE_PROTOCOL.containing_type = _DOWNLOADFILE
_DOWNLOADFILE_HTTP_METHOD.containing_type = _DOWNLOADFILE
DESCRIPTOR.message_types_by_name['File'] = _FILE
@@ -760,6 +821,7 @@ DESCRIPTOR.message_types_by_name['FileList'] = _FILELIST
DESCRIPTOR.message_types_by_name['Operation'] = _OPERATION
DESCRIPTOR.message_types_by_name['Process'] = _PROCESS
DESCRIPTOR.message_types_by_name['DownloadFile'] = _DOWNLOADFILE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
File = _reflection.GeneratedProtocolMessageType('File', (_message.Message,), dict(
@@ -834,6 +896,13 @@ DownloadFile = _reflection.GeneratedProtocolMessageType('DownloadFile', (_messag
# @@protoc_insertion_point(class_scope:biomaj.download.DownloadFile.Proxy)
))
,
+
+ ProtocolOptionsEntry = _reflection.GeneratedProtocolMessageType('ProtocolOptionsEntry', (_message.Message,), dict(
+ DESCRIPTOR = _DOWNLOADFILE_PROTOCOLOPTIONSENTRY,
+ __module__ = 'downmessage_pb2'
+ # @@protoc_insertion_point(class_scope:biomaj.download.DownloadFile.ProtocolOptionsEntry)
+ ))
+ ,
DESCRIPTOR = _DOWNLOADFILE,
__module__ = 'downmessage_pb2'
# @@protoc_insertion_point(class_scope:biomaj.download.DownloadFile)
@@ -843,6 +912,8 @@ _sym_db.RegisterMessage(DownloadFile.Param)
_sym_db.RegisterMessage(DownloadFile.HttpParse)
_sym_db.RegisterMessage(DownloadFile.RemoteFile)
_sym_db.RegisterMessage(DownloadFile.Proxy)
+_sym_db.RegisterMessage(DownloadFile.ProtocolOptionsEntry)
+_DOWNLOADFILE_PROTOCOLOPTIONSENTRY._options = None
# @@protoc_insertion_point(module_scope)
=====================================
requirements.txt
=====================================
@@ -10,7 +10,7 @@ flask
python-consul
prometheus_client>=0.0.18
requests
-biomaj_core>=3.0.16
+biomaj_core>=3.0.19
biomaj_zipkin
flake8
humanfriendly
=====================================
setup.py
=====================================
@@ -17,11 +17,12 @@ with open(os.path.join(here, 'CHANGES.txt')) as f:
config = {
'description': 'BioMAJ download service',
'long_description': README + '\n\n' + CHANGES,
+ 'long_description_content_type': 'text/markdown',
'author': 'Olivier Sallou',
'url': 'http://biomaj.genouest.org',
'download_url': 'http://biomaj.genouest.org',
'author_email': 'olivier.sallou at irisa.fr',
- 'version': '3.0.21',
+ 'version': '3.0.27',
'classifiers': [
# How mature is this project? Common values are
# 3 - Alpha
@@ -45,6 +46,7 @@ config = {
'biomaj_core',
'biomaj_zipkin',
'pycurl',
+ 'ftputil',
'py-bcrypt',
'pika==0.13.0',
'redis',
=====================================
tests/biomaj_tests.py
=====================================
@@ -4,6 +4,7 @@ from nose.plugins.attrib import attr
import json
import shutil
import os
+import sys
import tempfile
import logging
import copy
@@ -213,7 +214,31 @@ class TestBiomajLocalDownload(unittest.TestCase):
locald.close()
self.assertTrue(len(locald.files_to_download) == 1)
-
+ def test_local_download_hardlinks(self):
+ """
+ Test download with hardlinks: we download a file from conf/ to data_dir.
+ This should work unless /tmp don't accept hardlinks so the last assert is
+ optional.
+ """
+ test_file = "conf/global.properties"
+ locald = LocalDownload(self.utils.test_dir, use_hardlinks=True)
+ (file_list, dir_list) = locald.list()
+ locald.match([r'^/' + test_file + '$'], file_list, dir_list)
+ locald.download(self.utils.data_dir)
+ locald.close()
+ self.assertTrue(len(locald.files_to_download) == 1)
+ # Test if data/conf/global.properties is a hard link to
+ # conf/global.properties
+ local_global_properties = os.path.join(self.utils.test_dir, test_file)
+ copy_global_properties = os.path.join(self.utils.data_dir, test_file)
+ try:
+ self.assertTrue(
+ os.path.samefile(local_global_properties, copy_global_properties)
+ )
+ except Exception:
+ msg = "In %s: copy worked but hardlinks were not used." % self.id()
+ logging.info(msg)
+
@attr('network')
@attr('http')
class TestBiomajHTTPDownload(unittest.TestCase):
@@ -337,6 +362,40 @@ class TestBiomajDirectFTPDownload(unittest.TestCase):
self.assertTrue(os.path.exists(os.path.join(self.utils.data_dir,'mailing-lists.txt')))
+
+ at attr('directftps')
+ at attr('network')
+class TestBiomajDirectFTPSDownload(unittest.TestCase):
+ """
+ Test DirectFTP downloader with FTPS.
+ """
+
+ def setUp(self):
+ self.utils = UtilsForTest()
+
+ def tearDown(self):
+ self.utils.clean()
+
+ def test_ftps_list(self):
+ file_list = ['/readme.txt']
+ ftpd = DirectFTPDownload('ftps', 'test.rebex.net', '')
+ ftpd.set_credentials('demo:password')
+ ftpd.set_files_to_download(file_list)
+ (file_list, dir_list) = ftpd.list()
+ ftpd.close()
+ self.assertTrue(len(file_list) == 1)
+
+ def test_download(self):
+ file_list = ['/readme.txt']
+ ftpd = DirectFTPDownload('ftps', 'test.rebex.net', '')
+ ftpd.set_credentials('demo:password')
+ ftpd.set_files_to_download(file_list)
+ (file_list, dir_list) = ftpd.list()
+ ftpd.download(self.utils.data_dir, False)
+ ftpd.close()
+ self.assertTrue(os.path.exists(os.path.join(self.utils.data_dir,'readme.txt')))
+
+
@attr('directhttp')
@attr('network')
class TestBiomajDirectHTTPDownload(unittest.TestCase):
@@ -440,24 +499,27 @@ class TestBiomajFTPDownload(unittest.TestCase):
ftpd = FTPDownload('ftp', 'speedtest.tele2.net', '/')
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^1.*KB\.zip$'], file_list, dir_list)
+ # This tests fails because the zip file is fake. We intercept the failure
+ # and continue.
+ # See test_download_skip_uncompress_checks
try:
ftpd.download(self.utils.data_dir)
except Exception:
self.assertTrue(1==1)
else:
- self.assertTrue(1==0)
+ # In case it works, this is the real assertion
+ self.assertTrue(len(ftpd.files_to_download) == 2)
ftpd.close()
- # self.assertTrue(len(ftpd.files_to_download) == 2)
- def test_download_skip_uncompress_checks(self):
- os.environ['UNCOMPRESS_SKIP_CHECK'] = "1"
+ def test_download_skip_checks_uncompress(self):
+ # This test is similar to test_download but we skip test of zip file.
ftpd = FTPDownload('ftp', 'speedtest.tele2.net', '/')
+ ftpd.set_options(dict(skip_check_uncompress=True))
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^1.*KB\.zip$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 2)
- del os.environ['UNCOMPRESS_SKIP_CHECK']
def test_download_in_subdir(self):
ftpd = FTPDownload('ftp', 'ftp.fr.debian.org', '/debian/')
@@ -503,6 +565,104 @@ class TestBiomajFTPDownload(unittest.TestCase):
self.assertTrue(release['month']=='11')
self.assertTrue(release['day']=='12')
+ def test_ms_server(self):
+ ftpd = FTPDownload("ftp", "test.rebex.net", "/")
+ ftpd.set_credentials("demo:password")
+ (file_list, dir_list) = ftpd.list()
+ ftpd.match(["^readme.txt$"], file_list, dir_list)
+ ftpd.download(self.utils.data_dir)
+ ftpd.close()
+ self.assertTrue(len(ftpd.files_to_download) == 1)
+
+ def test_download_tcp_keepalive(self):
+ """
+ Test setting tcp_keepalive (it probably doesn't change anything here but
+ we test that there is no obvious mistake in the code).
+ """
+ ftpd = FTPDownload("ftp", "test.rebex.net", "/")
+ ftpd.set_options(dict(tcp_keepalive=10))
+ ftpd.set_credentials("demo:password")
+ (file_list, dir_list) = ftpd.list()
+ ftpd.match(["^readme.txt$"], file_list, dir_list)
+ ftpd.download(self.utils.data_dir)
+ ftpd.close()
+ self.assertTrue(len(ftpd.files_to_download) == 1)
+
+
+ at attr('ftps')
+ at attr('network')
+class TestBiomajFTPSDownload(unittest.TestCase):
+ """
+ Test FTP downloader with FTPS.
+ """
+ PROTOCOL = "ftps"
+
+ def setUp(self):
+ self.utils = UtilsForTest()
+
+ def tearDown(self):
+ self.utils.clean()
+
+ def test_ftps_list(self):
+ ftpd = FTPDownload(self.PROTOCOL, "test.rebex.net", "/")
+ ftpd.set_credentials("demo:password")
+ (file_list, dir_list) = ftpd.list()
+ ftpd.close()
+ self.assertTrue(len(file_list) == 1)
+
+ def test_download(self):
+ ftpd = FTPDownload(self.PROTOCOL, "test.rebex.net", "/")
+ ftpd.set_credentials("demo:password")
+ (file_list, dir_list) = ftpd.list()
+ ftpd.match([r'^readme.txt$'], file_list, dir_list)
+ ftpd.download(self.utils.data_dir)
+ ftpd.close()
+ self.assertTrue(len(ftpd.files_to_download) == 1)
+
+ def test_ftps_list_no_ssl(self):
+ # This server is misconfigured hence we disable all SSL verification
+ SERVER = "demo.wftpserver.com"
+ DIRECTORY = "/download/"
+ CREDENTIALS = "demo-user:demo-user"
+ ftpd = FTPDownload(self.PROTOCOL, SERVER, DIRECTORY)
+ ftpd.set_options(dict(ssl_verifyhost="False", ssl_verifypeer="False"))
+ ftpd.set_credentials(CREDENTIALS)
+ (file_list, dir_list) = ftpd.list()
+ ftpd.close()
+ self.assertTrue(len(file_list) > 1)
+
+ def test_download_no_ssl(self):
+ # This server is misconfigured hence we disable all SSL verification
+ SERVER = "demo.wftpserver.com"
+ DIRECTORY = "/download/"
+ CREDENTIALS = "demo-user:demo-user"
+ ftpd = FTPDownload(self.PROTOCOL, SERVER, DIRECTORY)
+ ftpd.set_options(dict(ssl_verifyhost="False", ssl_verifypeer="False"))
+ ftpd.set_credentials(CREDENTIALS)
+ (file_list, dir_list) = ftpd.list()
+ ftpd.match([r'^manual_en.pdf$'], file_list, dir_list)
+ ftpd.download(self.utils.data_dir)
+ ftpd.close()
+ self.assertTrue(len(ftpd.files_to_download) == 1)
+
+ def test_download_ssl_certficate(self):
+ # This server is misconfigured but we use its certificate
+ # The hostname is wrong so we disable host verification
+ SERVER = "demo.wftpserver.com"
+ DIRECTORY = "/download/"
+ CREDENTIALS = "demo-user:demo-user"
+ ftpd = FTPDownload(self.PROTOCOL, SERVER, DIRECTORY)
+ curdir = os.path.dirname(os.path.realpath(__file__))
+ cert_file = os.path.join(curdir, "caert.demo.wftpserver.com.pem")
+ ftpd.set_options(dict(ssl_verifyhost="False", ssl_server_cert=cert_file))
+ ftpd.set_credentials(CREDENTIALS)
+ (file_list, dir_list) = ftpd.list()
+ ftpd.match([r'^manual_en.pdf$'], file_list, dir_list)
+ ftpd.download(self.utils.data_dir)
+ ftpd.close()
+ self.assertTrue(len(ftpd.files_to_download) == 1)
+
+
@attr('rsync')
@attr('local')
class TestBiomajRSYNCDownload(unittest.TestCase):
=====================================
tests/caert.demo.wftpserver.com.pem
=====================================
@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDszCCApugAwIBAgIBADANBgkqhkiG9w0BAQsFADCBnDEYMBYGA1UEAwwPV2lu
+ZyBGVFAgU2VydmVyMQswCQYDVQQGEwJVUzELMAkGA1UECAwCTlkxCzAJBgNVBAcM
+Ak5NMRgwFgYDVQQKDA9XaW5nIEZUUCBTZXJ2ZXIxGDAWBgNVBAsMD1dpbmcgRlRQ
+IFNlcnZlcjElMCMGCSqGSIb3DQEJARYWc3VwcG9ydEB3ZnRwc2VydmVyLmNvbTAe
+Fw0xNjEwMDcxNjI3MDZaFw0yNjEwMDUxNjI3MDZaMIGcMRgwFgYDVQQDDA9XaW5n
+IEZUUCBTZXJ2ZXIxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJOWTELMAkGA1UEBwwC
+Tk0xGDAWBgNVBAoMD1dpbmcgRlRQIFNlcnZlcjEYMBYGA1UECwwPV2luZyBGVFAg
+U2VydmVyMSUwIwYJKoZIhvcNAQkBFhZzdXBwb3J0QHdmdHBzZXJ2ZXIuY29tMIIB
+IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4qCiFwqgJX9EFvf18bNL4aGl
+lFOpzGTbyS5AKiDwdf6dEcJd0p9s8PD2So1g+wYmAMD1aUAn9yyvU11oQ5gy+T3P
+ZaPes1bH4ugnq8inwzgy46wP4eN8CJzrxZvMAkdh/UbNiH8GLELR3Pex1BfrMlkN
+iO9STcMz7hVA2YhH59eolEJlsqTOSCgaXbCaDcQpof/Hbz/GtLu34x2LpA6GEvtr
+78gyuU8MPakISDyXAkaOr2KpJEabsq2xqvJTZUZJHAjFk3DREUYlLbY4HF0KjqH1
+VZtJcerBjNszHTrgR7DMy6FIMFnlF9jG0sMkG0kAYu55dqoMEiCTXLpgQWyoEwID
+AQABMA0GCSqGSIb3DQEBCwUAA4IBAQAvvnwJrqczqqow20eL77voXn9aTqbex/0C
+8kSTVetrThCh8sO+GH507fW4PkyxFfulosSRY18Bj17dVOILMbh959y7PkTWcNA1
+I5NxuU0lC2Ctc6sO6WtnKHh3nQaJKYix0CTwN4ZFDeBDWkbT+aqiCDzWDiAvUOaO
+wgOvkWaGy+6rB8fT/mcRaK2BH7H374tk5KqPrQwlVl0d/y+lBrp0ISebC/aKV9UE
+CqOXL36u0MdNINY/p/wH6aHfrcSe9EVTg7Euw5uq5wmMqrdUf9DyEtY2N18ShBZD
+f2c8ZXeb7abPkgef3cbwMHrqQ8ADiQqaLngazlNXU/a7/C1M0Llx
+-----END CERTIFICATE-----
=====================================
tests/global.properties
=====================================
@@ -121,3 +121,13 @@ formatter = generic
[formatter_generic]
format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
+
+#-----------------
+# Protocol options
+#-----------------
+
+# Set options
+options.name=skip_check_uncompress
+# Don't skip the test of compressed files
+options.skip_check_uncompress=1
+
View it on GitLab: https://salsa.debian.org/med-team/biomaj3-download/commit/860377520c0d8ead0119662629f59a95fba596a0
--
View it on GitLab: https://salsa.debian.org/med-team/biomaj3-download/commit/860377520c0d8ead0119662629f59a95fba596a0
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20191016/41dc4058/attachment-0001.html>
More information about the debian-med-commit
mailing list