[med-svn] [Git][med-team/biomaj3-download][upstream] New upstream version 3.2.12
Alexandre Detiste (@detiste-guest)
gitlab at salsa.debian.org
Mon Aug 26 19:56:19 BST 2024
Alexandre Detiste pushed to branch upstream at Debian Med / biomaj3-download
Commits:
7730fe65 by Alexandre Detiste at 2024-08-26T20:49:02+02:00
New upstream version 3.2.12
- - - - -
12 changed files:
- + .github/workflows/ci.yml
- − .travis.yml
- CHANGES.txt
- README.md
- bin/biomaj_download_consumer.py
- biomaj_download/download/direct.py
- biomaj_download/downloadclient.py
- biomaj_download/message/downmessage_pb2.py
- requirements.txt
- setup.cfg
- setup.py
- tests/biomaj_tests.py
Changes:
=====================================
.github/workflows/ci.yml
=====================================
@@ -0,0 +1,83 @@
+name: Lint, test and publish
+
+on: ["push"]
+
+jobs:
+
+ lint:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout at v4
+ - uses: actions/setup-python at v1
+ with:
+ python-version: 3.8
+ - name: Install Flake8
+ run: |
+ pip install flake8
+ - name: Flake8
+ run: flake8 --ignore=E501,W504 biomaj_download/*.py biomaj_download/download
+ - name: Install Protoc
+ uses: arduino/setup-protoc at v3
+ with:
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+ - name: Check that protobuf-compiler has been run
+ run: |
+ cd biomaj_download/message
+ protoc --python_out=. downmessage.proto
+ git status --porcelain
+ git diff
+ if [[ `git status --porcelain` ]]; then
+ echo "Protobuf-compiler has not been run, please do it and commit the modifed python files"
+ exit 1
+ fi
+
+ test:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout at v4
+ - uses: actions/setup-python at v1
+ with:
+ python-version: 3.8
+ - name: Install requirements
+ run: |
+ sudo apt update
+ sudo apt install libcurl4-openssl-dev libssl-dev
+ pip install pytest
+ python setup.py install
+ - name: Run tests
+ run: |
+ LOCAL_IRODS=0 NETWORK=0 pytest -v tests/biomaj_tests.py
+
+ pkg_build:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout at v4
+ - uses: actions/setup-python at v1
+ with:
+ python-version: 3.8
+ - name: Check that the package build works
+ run: |
+ pip install -U pip setuptools build
+ python -m build --sdist --wheel --outdir dist/ .
+
+ pypi:
+ runs-on: ubuntu-latest
+ needs: [lint, test, pkg_build]
+ name: Deploy release to Pypi
+ steps:
+ - name: Checkout
+ uses: actions/checkout at v4
+ - uses: actions/setup-python at v1
+ with:
+ python-version: 3.8
+ - name: Python install
+ run: pip install -U pip setuptools build
+ - name: Build a binary wheel and a source tarball
+ run: python -m build --sdist --wheel --outdir dist/ .
+ - name: Publish distribution 📦 to PyPI
+ if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+ uses: pypa/gh-action-pypi-publish at master
+ with:
+ password: ${{ secrets.pypi_password }}
=====================================
.travis.yml deleted
=====================================
@@ -1,35 +0,0 @@
-arch:
-- amd64
-- ppc64le
-language: python
-sudo: false
-python:
-- '3.6'
-- '3.7'
-- '3.8'
-services:
-- redis
-branches:
- except:
- - "/^feature.*$/"
-addons:
- apt:
- packages:
- - libgnutls-dev
-install:
-- pip install -r requirements.txt
-- pip install coverage pytest
-- pip install python-coveralls
-- python setup.py -q install
-script:
-- LOCAL_IRODS=0 NETWORK=0 pytest -v tests/biomaj_tests.py
-- flake8 --ignore E501 biomaj_download/*.py biomaj_download/download
-deploy:
- provider: pypi
- skip_existing: true
- user: osallou
- password:
- secure: fMEDYld7N2uRbICa++b7dTUXxt6VxSD1ZE+ZaTHQWxHHSK5fxbdwvsVHicdgKH3erMOXb7foumQ6uxBAtQTVmY+ZJbidydT/BEeoLjIH0NxVoTLCOllWKHOKm8LzgigVjbSVqXi7ZZ6rzigpQbg7NCdiD4wQr6xLO+gB3LrWovIgcPIlWDLWiMIp0P3b9nmoOdiSBBpLXI7i9W7ksL25Mopx/p4OoeWFEbg7aSkRVKufWLCoyLnYnWCnmC0MK7PF7V7mgiw+Gutv3XcOgfaZ/MU088+OrpvnLxfKplkwukcMBc4cuMV8mtWmDaQ9u2X9MLoeyRoQ6jHyIvftW+fn4Ck3wzyKZA3Pbz5+UVUG6GVpnwm3NIYvPl0m62brVVqs/wVaI8HB/7u5qn60ER9u9l9/YGIG5rsPrb2w12QFmcjwrge2T9h+aDW3+fQuzbjVGJMBCLqu9hUoiz9WHWqa74Ka8OfP+fTA3vpASGfuYhzdlJ5tDO6M1kjIe23ga5DhUMKHyl4j7uA/YVxc5numYU9naD/PyZw1qqms8zhJdVmIsBMtBP/MeH0YVrhmQS7p4ni7BB545JBYv8UyUfVXt2DcQHG9EAkMT69QfJhSl9KICe4SmlHVYy2Uldw/LE/FwYmngfDzZUYtndsn2xS+NVi3FwMNa1+6q0Sqqmzfmqc=
- distributions: sdist bdist_wheel
- on:
- tags: true
=====================================
CHANGES.txt
=====================================
@@ -1,3 +1,12 @@
+3.2.12:
+ Sort downloads in proper alphabetical order
+
+3.2.11:
+ fix direct handler in case of plugin usage
+
+3.2.10:
+ Regenerate proto messages
+
3.2.9:
Replace nose by pytest for unit testing
=====================================
README.md
=====================================
@@ -1,66 +1,81 @@
-# About
+# Biomaj-download
+
+## About
[](https://badge.fury.io/py/biomaj-download)
Microservice to manage the downloads of biomaj.
-A protobuf interface is available in biomaj_download/message/message_pb2.py to exchange messages between BioMAJ and the download service.
+A protobuf interface is available in `biomaj_download/message/downmessage_pb2.py` to exchange messages between BioMAJ and the download service.
Messages go through RabbitMQ (to be installed).
Python3 support only, python2 support is dropped
-# Protobuf
-
-To compile protobuf, in biomaj_download/message:
+## Protobuf
- protoc --python_out=. downmessage.proto
+If you make changes to protobuf code, you need to compile it to python code:
-# Development
+```bash
+cd /tmp/protoc/
+PB_REL="https://github.com/protocolbuffers/protobuf/releases"
+curl -LO $PB_REL/download/v23.2/protoc-23.2-linux-x86_64.zip # Version used by GitHub Actions currently
+unzip protoc-23.2-linux-x86_64.zip
+cd ..../biomaj_download/message/
+/tmp/protoc/bin/protoc --python_out=. downmessage.proto
+```
- flake8 --ignore E501 biomaj_download/\*.py biomaj_download/download
+## Development
-# Test
+## Test
To run the test suite, use:
- LOCAL_IRODS=0 pytest -v tests/biomaj_tests.py
+```bash
+LOCAL_IRODS=0 pytest -v tests/biomaj_tests.py
+```
This command skips the test that need a local iRODS server.
Some test might fail due to network connection. You can skip them with:
- NETWORK=0 pytest -v tests/biomaj_tests.py
+```bash
+NETWORK=0 pytest -v tests/biomaj_tests.py
+```
+## Run
-# Run
+### Message consumer
-## Message consumer:
+```bash
+export BIOMAJ_CONFIG=path_to_config.yml
+python bin/biomaj_download_consumer.py
+```
- export BIOMAJ_CONFIG=path_to_config.yml
- python bin/biomaj_download_consumer.py
-
-## Web server
+### Web server
If package is installed via pip, you need a file named *gunicorn_conf.py* containing somehwhere on local server:
- def worker_exit(server, worker):
- from prometheus_client import multiprocess
- multiprocess.mark_process_dead(worker.pid)
+```python
+def worker_exit(server, worker):
+ from prometheus_client import multiprocess
+ multiprocess.mark_process_dead(worker.pid)
+```
If you cloned the repository and installed it via python setup.py install, just refer to the *gunicorn_conf.py* in the cloned repository.
-
- export BIOMAJ_CONFIG=path_to_config.yml
- rm -rf ..path_to/prometheus-multiproc
- mkdir -p ..path_to/prometheus-multiproc
- export prometheus_multiproc_dir=..path_to/prometheus-multiproc
- gunicorn -c gunicorn_conf.py biomaj_download.biomaj_download_web:app
+```bash
+export BIOMAJ_CONFIG=path_to_config.yml
+rm -rf ..path_to/prometheus-multiproc
+mkdir -p ..path_to/prometheus-multiproc
+export prometheus_multiproc_dir=..path_to/prometheus-multiproc
+gunicorn -c gunicorn_conf.py biomaj_download.biomaj_download_web:app
+```
Web processes should be behind a proxy/load balancer, API base url /api/download
Prometheus endpoint metrics are exposed via /metrics on web server
-# Retrying
+## Retrying
A common problem when downloading a large number of files is the handling of temporary failures (network issues, server too busy to answer, etc.).
Since version 3.1.2, `biomaj-download` uses the [Tenacity library](https://github.com/jd/tenacity) which is designed to handle this.
@@ -73,14 +88,14 @@ For bank configuration those options also parse strings read from the configurat
This parsing is based on the [Simple Eval library](https://github.com/danthedeckie/simpleeval).
The rules are straightforward:
- * All concrete stop and wait classes defined in Tenacity (i.e. classes inheriting from `stop_base` and `wait_base` respectively) can be used
- by calling their constructor with the expected parameters.
- For example, the string `"stop_after_attempt(5)"` will create the desired object.
- Note that stop and wait classes that need no argument must be used as constants (i.e. use `"stop_never"` and not `"stop_never()"`).
- Currently, this is the case for `"stop_never"` (as in Tenacity) and `"wait_none"` (this slightly differs from Tenacity where it is `"wait_none()"`).
- * You can use classes that allow to combine other stop conditions (namely `stop_all` and `stop_any`) or wait policies (namely `wait_combine`).
- * Operator `+` can be used to add wait policies (similar to `wait_combine`).
- * Operators `&` and `|` can be used to compose stop conditions (similar to `wait_all` and `wait_none` respectively).
+* All concrete stop and wait classes defined in Tenacity (i.e. classes inheriting from `stop_base` and `wait_base` respectively) can be used
+ by calling their constructor with the expected parameters.
+ For example, the string `"stop_after_attempt(5)"` will create the desired object.
+Note that stop and wait classes that need no argument must be used as constants (i.e. use `"stop_never"` and not `"stop_never()"`).
+Currently, this is the case for `"stop_never"` (as in Tenacity) and `"wait_none"` (this slightly differs from Tenacity where it is `"wait_none()"`).
+* You can use classes that allow to combine other stop conditions (namely `stop_all` and `stop_any`) or wait policies (namely `wait_combine`).
+* Operator `+` can be used to add wait policies (similar to `wait_combine`).
+* Operators `&` and `|` can be used to compose stop conditions (similar to `wait_all` and `wait_none` respectively).
However, in this case, you can't use your own conditions.
The complete list of stop conditions is:
@@ -107,10 +122,10 @@ Please refer to [Tenacity doc](https://tenacity.readthedocs.io/en/latest/) for t
Examples (inspired by Tenacity doc):
- * `"wait_fixed(3) + wait_random(0, 2)"` and `"wait_combine(wait_fixed(3), wait_random(0, 2))"` are equivalent and will wait 3 seconds + up to 2 seconds of random delay
- * `"wait_chain(*([wait_fixed(3) for i in range(3)] + [wait_fixed(7) for i in range(2)] + [wait_fixed(9)]))"` will wait 3s for 3 attempts, 7s for the next 2 attempts and 9s for all attempts thereafter (here `+` is the list concatenation).
- * `"wait_none + wait_random(1,2)"` will wait between 1s and 2s (since `wait_none` doesn't wait).
- * `"stop_never | stop_after_attempt(5)"` will stop after 5 attempts (since `stop_never` never stops).
+* `"wait_fixed(3) + wait_random(0, 2)"` and `"wait_combine(wait_fixed(3), wait_random(0, 2))"` are equivalent and will wait 3 seconds + up to 2 seconds of random delay
+* `"wait_chain(*([wait_fixed(3) for i in range(3)] + [wait_fixed(7) for i in range(2)] + [wait_fixed(9)]))"` will wait 3s for 3 attempts, 7s for the next 2 attempts and 9s for all attempts thereafter (here `+` is the list concatenation).
+* `"wait_none + wait_random(1,2)"` will wait between 1s and 2s (since `wait_none` doesn't wait).
+* `"stop_never | stop_after_attempt(5)"` will stop after 5 attempts (since `stop_never` never stops).
Note that some protocols (e.g. FTP) classify errors as temporary or permanent (for example trying to download inexisting file).
More generally, we could distinguish permanent errors based on error codes, etc. and not retry in this case.
@@ -118,96 +133,96 @@ However in our experience, so called permanent errors may well be temporary.
Therefore downloaders always retry whatever the error.
In some cases, this is a waste of time but generally this is worth it.
-# Host keys
+## Host keys
When using the `sftp` protocol, `biomaj-download` must check the host key.
Those keys are stored in a file (for instance `~/.ssh/known_hosts`).
Two options are available to configure this:
- - **ssh_hosts_file** which sets the file to use
- - **ssh_new_host** which sets what to do for a new host
+* **ssh_hosts_file** which sets the file to use
+* **ssh_new_host** which sets what to do for a new host
When the host and the key are found in the file, the connection is accepted.
If the host is found but the key missmatches, the connection is rejected
(this usually indicates a problem or a change of configuration on the remote server).
When the host is not found, the decision depends on the value of **ssh_new_host**:
- - `reject` means that the connection is rejected
- - `accept` means that the connection is accepted
- - `add` means that the connection is accepted and the key is added to the file
+* `reject` means that the connection is rejected
+* `accept` means that the connection is accepted
+* `add` means that the connection is accepted and the key is added to the file
See the description of the options in [Download options](#download-options).
-# Download options
+## Download options
Since version 3.0.26, you can use the `set_options` method to pass a dictionary of downloader-specific options.
The following list shows some options and their effect (the option to set is the key and the parameter is the associated value):
- * **stop_condition**:
- * parameter: an instance of Tenacity `stop_base` or a string (see [Retrying](#retrying)).
- * downloader(s): all (except `LocalDownload`).
- * effect: sets the condition on which we should stop retrying to download a file.
- * default: `stop_after_attempt(3)` (i.e. stop after 3 attempts).
- * note: introduced in version 3.2.1.
- * **wait_policy**:
- * parameter: an instance of Tenacity `wait_base` or a string (see [Retrying](#retrying)).
- * downloader(s): all (except `LocalDownload`).
- * effect: sets the wait policy between download attempts.
- * default: `wait_fixed(3)` (i.e. wait 3 seconds between attempts).
- * note: introduced in version 3.2.1.
- * **skip_check_uncompress**:
- * parameter: bool.
- * downloader(s): all (except `LocalDownload`).
- * effect: if true, don't test the archives after download.
- * default: false (i.e. test the archives).
- * **ssl_verifyhost**:
- * parameter: bool.
- * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`).
- * effect: if false, don't check that the name of the remote server is the same than in the SSL certificate.
- * default: true (i.e. check host name).
- * note: it's generally a bad idea to disable this verification. However some servers are badly configured. See [here](https://curl.haxx.se/libcurl/c/CURLOPT_SSL_VERIFYHOST.html) for the corresponding cURL option.
- * **ssl_verifypeer**:
- * parameter: bool.
- * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`).
- * effect: if false, don't check the authenticity of the peer's certificate.
- * default: true (i.e. check authenticity).
- * note: it's generally a bad idea to disable this verification. However some servers are badly configured. See [here](https://curl.haxx.se/libcurl/c/CURLOPT_SSL_VERIFYPEER.html) for the corresponding cURL option.
- * **ssl_server_cert**:
- * parameter: path of the certificate file.
- * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`).
- * effect: use the certificate(s) in this file to verify the peer with.
- * default: use OS certificates.
- * note: see [here](https://curl.haxx.se/libcurl/c/CURLOPT_CAINFO.html) for the corresponding cURL option.
- * parameter: int.
- * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`).
- * effect: sets the interval, in seconds, that the operating system will wait between sending keepalive probes.
- * default: cURL default (60s at the time of this writing).
- * note: see [here](https://curl.haxx.se/libcurl/c/CURLOPT_TCP_KEEPINTVL.html) for the corresponding cURL option.
- * **ftp_method**:
- * parameter: one of `default`, `multicwd`, `nocwd`, `singlecwd` (case insensitive).
- * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`) - only used for `FTP(S)`.
- * effect: sets the method used to reach a file on a FTP(S) server (`nocwd` and `singlecwd` are usually faster but not always supported).
- * default: `default` (which is `multicwd` at the time of this writing as in cURL).
- * note: see [here](https://curl.haxx.se/libcurl/c/CURLOPT_FTP_FILEMETHOD.html) for the corresponding cURL option; introduced in version 3.1.2.
- * **ssh_hosts_file**:
- * parameter: path of the known hosts file.
- * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`) - only used for `SFTP`.
- * effect: sets the file used to read/store host keys for `SFTP`.
- * default: `~/.ssh/known_hosts` (where `~` is the home directory of the current user).
- * note: see [here](https://curl.haxx.se/libcurl/c/CURLOPT_SSH_KNOWNHOSTS.html) for the corresponding cURL option and the option below; introduced in version 3.2.1.
- * **ssh_new_host**:
- * parameter: one of `reject`, `accept`, `add`.
- * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`) - only used for `SFTP`.
- * effect: sets the policy to use for an unknown host.
- * default: `reject` (i.e. refuse new hosts - you must add them in the file for instance with `ssh` or `sftp`).
- * note: see [here](https://curl.haxx.se/libcurl/c/CURLOPT_SSH_KEYFUNCTION.html) for the corresponding cURL option and the option above; introduced in version 3.2.1.
- * *allow_redirections*:
- * parameter: bool.
- * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`) - only used for `HTTPS(S)`.
- * effect: sets the policy for `HTTP` redirections.
- * default: `true` (i.e. follow redirections).
- * note: see [here](https://curl.haxx.se/libcurl/c/CURLOPT_FOLLOWLOCATION.html) for the corresponding cURL option; introduced in version 3.2.3.
+* **stop_condition**:
+ * parameter: an instance of Tenacity `stop_base` or a string (see [Retrying](#retrying)).
+ * downloader(s): all (except `LocalDownload`).
+ * effect: sets the condition on which we should stop retrying to download a file.
+ * default: `stop_after_attempt(3)` (i.e. stop after 3 attempts).
+ * note: introduced in version 3.2.1.
+* **wait_policy**:
+ * parameter: an instance of Tenacity `wait_base` or a string (see [Retrying](#retrying)).
+ * downloader(s): all (except `LocalDownload`).
+ * effect: sets the wait policy between download attempts.
+ * default: `wait_fixed(3)` (i.e. wait 3 seconds between attempts).
+ * note: introduced in version 3.2.1.
+* **skip_check_uncompress**:
+ * parameter: bool.
+ * downloader(s): all (except `LocalDownload`).
+ * effect: if true, don't test the archives after download.
+ * default: false (i.e. test the archives).
+* **ssl_verifyhost**:
+ * parameter: bool.
+ * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`).
+ * effect: if false, don't check that the name of the remote server is the same than in the SSL certificate.
+ * default: true (i.e. check host name).
+ * note: it's generally a bad idea to disable this verification. However some servers are badly configured. See [here](https://curl.haxx.se/libcurl/c/CURLOPT_SSL_VERIFYHOST.html) for the corresponding cURL option.
+* **ssl_verifypeer**:
+ * parameter: bool.
+ * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`).
+ * effect: if false, don't check the authenticity of the peer's certificate.
+ * default: true (i.e. check authenticity).
+ * note: it's generally a bad idea to disable this verification. However some servers are badly configured. See [here](https://curl.haxx.se/libcurl/c/CURLOPT_SSL_VERIFYPEER.html) for the corresponding cURL option.
+* **ssl_server_cert**:
+ * parameter: path of the certificate file.
+ * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`).
+ * effect: use the certificate(s) in this file to verify the peer with.
+ * default: use OS certificates.
+ * note: see [here](https://curl.haxx.se/libcurl/c/CURLOPT_CAINFO.html) for the corresponding cURL option.
+ * parameter: int.
+ * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`).
+ * effect: sets the interval, in seconds, that the operating system will wait between sending keepalive probes.
+ * default: cURL default (60s at the time of this writing).
+ * note: see [here](https://curl.haxx.se/libcurl/c/CURLOPT_TCP_KEEPINTVL.html) for the corresponding cURL option.
+* **ftp_method**:
+ * parameter: one of `default`, `multicwd`, `nocwd`, `singlecwd` (case insensitive).
+ * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`) - only used for `FTP(S)`.
+ * effect: sets the method used to reach a file on a FTP(S) server (`nocwd` and `singlecwd` are usually faster but not always supported).
+ * default: `default` (which is `multicwd` at the time of this writing as in cURL).
+ * note: see [here](https://curl.haxx.se/libcurl/c/CURLOPT_FTP_FILEMETHOD.html) for the corresponding cURL option; introduced in version 3.1.2.
+* **ssh_hosts_file**:
+ * parameter: path of the known hosts file.
+ * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`) - only used for `SFTP`.
+ * effect: sets the file used to read/store host keys for `SFTP`.
+ * default: `~/.ssh/known_hosts` (where `~` is the home directory of the current user).
+ * note: see [here](https://curl.haxx.se/libcurl/c/CURLOPT_SSH_KNOWNHOSTS.html) for the corresponding cURL option and the option below; introduced in version 3.2.1.
+* **ssh_new_host**:
+ * parameter: one of `reject`, `accept`, `add`.
+ * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`) - only used for `SFTP`.
+ * effect: sets the policy to use for an unknown host.
+ * default: `reject` (i.e. refuse new hosts - you must add them in the file for instance with `ssh` or `sftp`).
+ * note: see [here](https://curl.haxx.se/libcurl/c/CURLOPT_SSH_KEYFUNCTION.html) for the corresponding cURL option and the option above; introduced in version 3.2.1.
+* *allow_redirections*:
+ * parameter: bool.
+ * downloader(s): `CurlDownload` (and derived classes: `DirectFTPDownload`, `DirectHTTPDownload`) - only used for `HTTPS(S)`.
+ * effect: sets the policy for `HTTP` redirections.
+ * default: `true` (i.e. follow redirections).
+ * note: see [here](https://curl.haxx.se/libcurl/c/CURLOPT_FOLLOWLOCATION.html) for the corresponding cURL option; introduced in version 3.2.3.
Those options can be set in bank properties.
See file `global.properties.example` in [biomaj module](https://github.com/genouest/biomaj).
=====================================
bin/biomaj_download_consumer.py
=====================================
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
'''
Message consumer for download requests
'''
=====================================
biomaj_download/download/direct.py
=====================================
@@ -47,6 +47,8 @@ class DirectFTPDownload(CurlDownload):
# biomaj workflow should fix this
if isinstance(rfile, dict):
filename = rfile['name']
+ if self.save_as is None and rfile.get('save_as', None):
+ self.save_as = rfile['save_as']
else:
# direct protocol send directly some filename
filename = rfile
=====================================
biomaj_download/downloadclient.py
=====================================
@@ -209,7 +209,7 @@ class DownloadClient(DownloadService):
for i in range(max_submit):
if self.download_pool:
logging.debug('Workflow:wf_download:RemoteDownload:RequestNewFile')
- operation = self.download_pool.pop()
+ operation = self.download_pool.pop(0)
self.ask_download(operation)
nb_submitted += 1
=====================================
biomaj_download/message/downmessage_pb2.py
=====================================
@@ -1,12 +1,11 @@
+# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: downmessage.proto
-
-import sys
-_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
-from google.protobuf import message as _message
-from google.protobuf import reflection as _reflection
+from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
@@ -14,906 +13,44 @@ _sym_db = _symbol_database.Default()
-DESCRIPTOR = _descriptor.FileDescriptor(
- name='downmessage.proto',
- package='biomaj.download',
- syntax='proto2',
- serialized_options=None,
- serialized_pb=_b('\n\x11\x64ownmessage.proto\x12\x0f\x62iomaj.download\"\x9d\x02\n\x04\x46ile\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\x0c\n\x04root\x18\x02 \x01(\t\x12\x0f\n\x07save_as\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\x30\n\x08metadata\x18\x05 \x01(\x0b\x32\x1e.biomaj.download.File.MetaData\x1a\xa8\x01\n\x08MetaData\x12\x13\n\x0bpermissions\x18\x01 \x01(\t\x12\r\n\x05group\x18\x02 \x01(\t\x12\x0c\n\x04size\x18\x03 \x01(\x03\x12\x0c\n\x04hash\x18\x04 \x01(\t\x12\x0c\n\x04year\x18\x05 \x01(\x05\x12\r\n\x05month\x18\x06 \x01(\x05\x12\x0b\n\x03\x64\x61y\x18\x07 \x01(\x05\x12\x0e\n\x06\x66ormat\x18\x08 \x01(\t\x12\x0b\n\x03md5\x18\t \x01(\t\x12\x15\n\rdownload_time\x18\n \x01(\x03\"0\n\x08\x46ileList\x12$\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x15.biomaj.download.File\"\xaa\x02\n\tOperation\x12\x32\n\x04type\x18\x01 \x02(\x0e\x32$.biomaj.download.Operation.OPERATION\x12/\n\x08\x64ownload\x18\x02 \x01(\x0b\x32\x1d.biomaj.download.DownloadFile\x12)\n\x07process\x18\x03 \x01(\x0b\x32\x18.biomaj.download.Process\x12/\n\x05trace\x18\x04 \x01(\x0b\x32 .biomaj.download.Operation.Trace\x1a*\n\x05Trace\x12\x10\n\x08trace_id\x18\x01 \x02(\t\x12\x0f\n\x07span_id\x18\x02 \x02(\t\"0\n\tOPERATION\x12\x08\n\x04LIST\x10\x00\x12\x0c\n\x08\x44OWNLOAD\x10\x01\x12\x0b\n\x07PROCESS\x10\x02\"\x17\n\x07Process\x12\x0c\n\x04\x65xec\x18\x01 \x02(\t\"\x94\x0b\n\x0c\x44ownloadFile\x12\x0c\n\x04\x62\x61nk\x18\x01 \x02(\t\x12\x0f\n\x07session\x18\x02 \x02(\t\x12\x11\n\tlocal_dir\x18\x03 \x02(\t\x12\x18\n\x10timeout_download\x18\x04 \x01(\x05\x12=\n\x0bremote_file\x18\x05 \x02(\x0b\x32(.biomaj.download.DownloadFile.RemoteFile\x12\x32\n\x05proxy\x18\x06 \x01(\x0b\x32#.biomaj.download.DownloadFile.Proxy\x12\x43\n\x0bhttp_method\x18\x08 \x01(\x0e\x32).biomaj.download.DownloadFile.HTTP_METHOD:\x03GET\x12;\n\x07options\x18\t \x03(\x0b\x32*.biomaj.download.DownloadFile.OptionsEntry\x1a$\n\x05Param\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\r\n\x05value\x18\x02 \x02(\t\x1a\xcd\x03\n\tHttpParse\x12\x91\x01\n\x08\x64ir_line\x18\x01 \x02(\t:\x7f<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[DIR\\]\"[\\s]*/?>[\\s]*<a[\\s]+href=\"([\\S]+)/\"[\\s]*>.*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})\x12\xa5\x01\n\tfile_line\x18\x02 \x02(\t:\x91\x01<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[[\\s]+\\]\"[\\s]*/?>[\\s]<a[\\s]+href=\"([\\S]+)\".*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})[\\s]+([\\d\\.]+[MKG]{0,1})\x12\x13\n\x08\x64ir_name\x18\x03 \x02(\x05:\x01\x31\x12\x13\n\x08\x64ir_date\x18\x04 \x02(\x05:\x01\x32\x12\x14\n\tfile_name\x18\x05 \x02(\x05:\x01\x31\x12\x14\n\tfile_date\x18\x06 \x02(\x05:\x01\x32\x12\x18\n\x10\x66ile_date_format\x18\x07 \x01(\t\x12\x14\n\tfile_size\x18\x08 \x02(\x05:\x01\x33\x1a\xb8\x02\n\nRemoteFile\x12$\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x15.biomaj.download.File\x12\x38\n\x08protocol\x18\x02 \x02(\x0e\x32&.biomaj.download.DownloadFile.Protocol\x12\x0e\n\x06server\x18\x03 \x02(\t\x12\x12\n\nremote_dir\x18\x04 \x02(\t\x12\x0f\n\x07save_as\x18\x05 \x01(\t\x12\x32\n\x05param\x18\x06 \x03(\x0b\x32#.biomaj.download.DownloadFile.Param\x12;\n\nhttp_parse\x18\x07 \x01(\x0b\x32\'.biomaj.download.DownloadFile.HttpParse\x12\x13\n\x0b\x63redentials\x18\x08 \x01(\t\x12\x0f\n\x07matches\x18\t \x03(\t\x1a*\n\x05Proxy\x12\r\n\x05proxy\x18\x01 \x02(\t\x12\x12\n\nproxy_auth\x18\x02 \x01(\t\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x93\x01\n\x08Protocol\x12\x07\n\x03\x46TP\x10\x00\x12\x08\n\x04\x46TPS\x10\x01\x12\x08\n\x04HTTP\x10\x02\x12\t\n\x05HTTPS\x10\x03\x12\r\n\tDIRECTFTP\x10\x04\x12\x0e\n\nDIRECTHTTP\x10\x05\x12\x0f\n\x0b\x44IRECTHTTPS\x10\x06\x12\t\n\x05LOCAL\x10\x07\x12\t\n\x05RSYNC\x10\x08\x12\t\n\x05IRODS\x10\t\x12\x0e\n\nDIRECTFTPS\x10\n\" \n\x0bHTTP_METHOD\x12\x07\n\x03GET\x10\x00\x12\x08\n\x04POST\x10\x01')
-)
-
-
-
-_OPERATION_OPERATION = _descriptor.EnumDescriptor(
- name='OPERATION',
- full_name='biomaj.download.Operation.OPERATION',
- filename=None,
- file=DESCRIPTOR,
- values=[
- _descriptor.EnumValueDescriptor(
- name='LIST', index=0, number=0,
- serialized_options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='DOWNLOAD', index=1, number=1,
- serialized_options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='PROCESS', index=2, number=2,
- serialized_options=None,
- type=None),
- ],
- containing_type=None,
- serialized_options=None,
- serialized_start=627,
- serialized_end=675,
-)
-_sym_db.RegisterEnumDescriptor(_OPERATION_OPERATION)
-
-_DOWNLOADFILE_PROTOCOL = _descriptor.EnumDescriptor(
- name='Protocol',
- full_name='biomaj.download.DownloadFile.Protocol',
- filename=None,
- file=DESCRIPTOR,
- values=[
- _descriptor.EnumValueDescriptor(
- name='FTP', index=0, number=0,
- serialized_options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='FTPS', index=1, number=1,
- serialized_options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='HTTP', index=2, number=2,
- serialized_options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='HTTPS', index=3, number=3,
- serialized_options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='DIRECTFTP', index=4, number=4,
- serialized_options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='DIRECTHTTP', index=5, number=5,
- serialized_options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='DIRECTHTTPS', index=6, number=6,
- serialized_options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='LOCAL', index=7, number=7,
- serialized_options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='RSYNC', index=8, number=8,
- serialized_options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='IRODS', index=9, number=9,
- serialized_options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='DIRECTFTPS', index=10, number=10,
- serialized_options=None,
- type=None),
- ],
- containing_type=None,
- serialized_options=None,
- serialized_start=1950,
- serialized_end=2097,
-)
-_sym_db.RegisterEnumDescriptor(_DOWNLOADFILE_PROTOCOL)
-
-_DOWNLOADFILE_HTTP_METHOD = _descriptor.EnumDescriptor(
- name='HTTP_METHOD',
- full_name='biomaj.download.DownloadFile.HTTP_METHOD',
- filename=None,
- file=DESCRIPTOR,
- values=[
- _descriptor.EnumValueDescriptor(
- name='GET', index=0, number=0,
- serialized_options=None,
- type=None),
- _descriptor.EnumValueDescriptor(
- name='POST', index=1, number=1,
- serialized_options=None,
- type=None),
- ],
- containing_type=None,
- serialized_options=None,
- serialized_start=2099,
- serialized_end=2131,
-)
-_sym_db.RegisterEnumDescriptor(_DOWNLOADFILE_HTTP_METHOD)
-
-
-_FILE_METADATA = _descriptor.Descriptor(
- name='MetaData',
- full_name='biomaj.download.File.MetaData',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='permissions', full_name='biomaj.download.File.MetaData.permissions', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='group', full_name='biomaj.download.File.MetaData.group', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='size', full_name='biomaj.download.File.MetaData.size', index=2,
- number=3, type=3, cpp_type=2, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='hash', full_name='biomaj.download.File.MetaData.hash', index=3,
- number=4, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='year', full_name='biomaj.download.File.MetaData.year', index=4,
- number=5, type=5, cpp_type=1, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='month', full_name='biomaj.download.File.MetaData.month', index=5,
- number=6, type=5, cpp_type=1, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='day', full_name='biomaj.download.File.MetaData.day', index=6,
- number=7, type=5, cpp_type=1, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='format', full_name='biomaj.download.File.MetaData.format', index=7,
- number=8, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='md5', full_name='biomaj.download.File.MetaData.md5', index=8,
- number=9, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='download_time', full_name='biomaj.download.File.MetaData.download_time', index=9,
- number=10, type=3, cpp_type=2, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto2',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=156,
- serialized_end=324,
-)
-
-_FILE = _descriptor.Descriptor(
- name='File',
- full_name='biomaj.download.File',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='name', full_name='biomaj.download.File.name', index=0,
- number=1, type=9, cpp_type=9, label=2,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='root', full_name='biomaj.download.File.root', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='save_as', full_name='biomaj.download.File.save_as', index=2,
- number=3, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='url', full_name='biomaj.download.File.url', index=3,
- number=4, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='metadata', full_name='biomaj.download.File.metadata', index=4,
- number=5, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[_FILE_METADATA, ],
- enum_types=[
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto2',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=39,
- serialized_end=324,
-)
-
-
-_FILELIST = _descriptor.Descriptor(
- name='FileList',
- full_name='biomaj.download.FileList',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='files', full_name='biomaj.download.FileList.files', index=0,
- number=1, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto2',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=326,
- serialized_end=374,
-)
-
-
-_OPERATION_TRACE = _descriptor.Descriptor(
- name='Trace',
- full_name='biomaj.download.Operation.Trace',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='trace_id', full_name='biomaj.download.Operation.Trace.trace_id', index=0,
- number=1, type=9, cpp_type=9, label=2,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='span_id', full_name='biomaj.download.Operation.Trace.span_id', index=1,
- number=2, type=9, cpp_type=9, label=2,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto2',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=583,
- serialized_end=625,
-)
-
-_OPERATION = _descriptor.Descriptor(
- name='Operation',
- full_name='biomaj.download.Operation',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='type', full_name='biomaj.download.Operation.type', index=0,
- number=1, type=14, cpp_type=8, label=2,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='download', full_name='biomaj.download.Operation.download', index=1,
- number=2, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='process', full_name='biomaj.download.Operation.process', index=2,
- number=3, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='trace', full_name='biomaj.download.Operation.trace', index=3,
- number=4, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[_OPERATION_TRACE, ],
- enum_types=[
- _OPERATION_OPERATION,
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto2',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=377,
- serialized_end=675,
-)
-
-
-_PROCESS = _descriptor.Descriptor(
- name='Process',
- full_name='biomaj.download.Process',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='exec', full_name='biomaj.download.Process.exec', index=0,
- number=1, type=9, cpp_type=9, label=2,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto2',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=677,
- serialized_end=700,
-)
-
-
-_DOWNLOADFILE_PARAM = _descriptor.Descriptor(
- name='Param',
- full_name='biomaj.download.DownloadFile.Param',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='name', full_name='biomaj.download.DownloadFile.Param.name', index=0,
- number=1, type=9, cpp_type=9, label=2,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='value', full_name='biomaj.download.DownloadFile.Param.value', index=1,
- number=2, type=9, cpp_type=9, label=2,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto2',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=1040,
- serialized_end=1076,
-)
-
-_DOWNLOADFILE_HTTPPARSE = _descriptor.Descriptor(
- name='HttpParse',
- full_name='biomaj.download.DownloadFile.HttpParse',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='dir_line', full_name='biomaj.download.DownloadFile.HttpParse.dir_line', index=0,
- number=1, type=9, cpp_type=9, label=2,
- has_default_value=True, default_value=_b("<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[DIR\\]\"[\\s]*/?>[\\s]*<a[\\s]+href=\"([\\S]+)/\"[\\s]*>.*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='file_line', full_name='biomaj.download.DownloadFile.HttpParse.file_line', index=1,
- number=2, type=9, cpp_type=9, label=2,
- has_default_value=True, default_value=_b("<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[[\\s]+\\]\"[\\s]*/?>[\\s]<a[\\s]+href=\"([\\S]+)\".*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})[\\s]+([\\d\\.]+[MKG]{0,1})").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='dir_name', full_name='biomaj.download.DownloadFile.HttpParse.dir_name', index=2,
- number=3, type=5, cpp_type=1, label=2,
- has_default_value=True, default_value=1,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='dir_date', full_name='biomaj.download.DownloadFile.HttpParse.dir_date', index=3,
- number=4, type=5, cpp_type=1, label=2,
- has_default_value=True, default_value=2,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='file_name', full_name='biomaj.download.DownloadFile.HttpParse.file_name', index=4,
- number=5, type=5, cpp_type=1, label=2,
- has_default_value=True, default_value=1,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='file_date', full_name='biomaj.download.DownloadFile.HttpParse.file_date', index=5,
- number=6, type=5, cpp_type=1, label=2,
- has_default_value=True, default_value=2,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='file_date_format', full_name='biomaj.download.DownloadFile.HttpParse.file_date_format', index=6,
- number=7, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='file_size', full_name='biomaj.download.DownloadFile.HttpParse.file_size', index=7,
- number=8, type=5, cpp_type=1, label=2,
- has_default_value=True, default_value=3,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto2',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=1079,
- serialized_end=1540,
-)
-
-_DOWNLOADFILE_REMOTEFILE = _descriptor.Descriptor(
- name='RemoteFile',
- full_name='biomaj.download.DownloadFile.RemoteFile',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='files', full_name='biomaj.download.DownloadFile.RemoteFile.files', index=0,
- number=1, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='protocol', full_name='biomaj.download.DownloadFile.RemoteFile.protocol', index=1,
- number=2, type=14, cpp_type=8, label=2,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='server', full_name='biomaj.download.DownloadFile.RemoteFile.server', index=2,
- number=3, type=9, cpp_type=9, label=2,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='remote_dir', full_name='biomaj.download.DownloadFile.RemoteFile.remote_dir', index=3,
- number=4, type=9, cpp_type=9, label=2,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='save_as', full_name='biomaj.download.DownloadFile.RemoteFile.save_as', index=4,
- number=5, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='param', full_name='biomaj.download.DownloadFile.RemoteFile.param', index=5,
- number=6, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='http_parse', full_name='biomaj.download.DownloadFile.RemoteFile.http_parse', index=6,
- number=7, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='credentials', full_name='biomaj.download.DownloadFile.RemoteFile.credentials', index=7,
- number=8, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='matches', full_name='biomaj.download.DownloadFile.RemoteFile.matches', index=8,
- number=9, type=9, cpp_type=9, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto2',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=1543,
- serialized_end=1855,
-)
-
-_DOWNLOADFILE_PROXY = _descriptor.Descriptor(
- name='Proxy',
- full_name='biomaj.download.DownloadFile.Proxy',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='proxy', full_name='biomaj.download.DownloadFile.Proxy.proxy', index=0,
- number=1, type=9, cpp_type=9, label=2,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='proxy_auth', full_name='biomaj.download.DownloadFile.Proxy.proxy_auth', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto2',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=1857,
- serialized_end=1899,
-)
-
-_DOWNLOADFILE_OPTIONSENTRY = _descriptor.Descriptor(
- name='OptionsEntry',
- full_name='biomaj.download.DownloadFile.OptionsEntry',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='biomaj.download.DownloadFile.OptionsEntry.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='value', full_name='biomaj.download.DownloadFile.OptionsEntry.value', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- serialized_options=_b('8\001'),
- is_extendable=False,
- syntax='proto2',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=1901,
- serialized_end=1947,
-)
-
-_DOWNLOADFILE = _descriptor.Descriptor(
- name='DownloadFile',
- full_name='biomaj.download.DownloadFile',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='bank', full_name='biomaj.download.DownloadFile.bank', index=0,
- number=1, type=9, cpp_type=9, label=2,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='session', full_name='biomaj.download.DownloadFile.session', index=1,
- number=2, type=9, cpp_type=9, label=2,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='local_dir', full_name='biomaj.download.DownloadFile.local_dir', index=2,
- number=3, type=9, cpp_type=9, label=2,
- has_default_value=False, default_value=_b("").decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='timeout_download', full_name='biomaj.download.DownloadFile.timeout_download', index=3,
- number=4, type=5, cpp_type=1, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='remote_file', full_name='biomaj.download.DownloadFile.remote_file', index=4,
- number=5, type=11, cpp_type=10, label=2,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='proxy', full_name='biomaj.download.DownloadFile.proxy', index=5,
- number=6, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='http_method', full_name='biomaj.download.DownloadFile.http_method', index=6,
- number=8, type=14, cpp_type=8, label=1,
- has_default_value=True, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='options', full_name='biomaj.download.DownloadFile.options', index=7,
- number=9, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[_DOWNLOADFILE_PARAM, _DOWNLOADFILE_HTTPPARSE, _DOWNLOADFILE_REMOTEFILE, _DOWNLOADFILE_PROXY, _DOWNLOADFILE_OPTIONSENTRY, ],
- enum_types=[
- _DOWNLOADFILE_PROTOCOL,
- _DOWNLOADFILE_HTTP_METHOD,
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto2',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=703,
- serialized_end=2131,
-)
-
-_FILE_METADATA.containing_type = _FILE
-_FILE.fields_by_name['metadata'].message_type = _FILE_METADATA
-_FILELIST.fields_by_name['files'].message_type = _FILE
-_OPERATION_TRACE.containing_type = _OPERATION
-_OPERATION.fields_by_name['type'].enum_type = _OPERATION_OPERATION
-_OPERATION.fields_by_name['download'].message_type = _DOWNLOADFILE
-_OPERATION.fields_by_name['process'].message_type = _PROCESS
-_OPERATION.fields_by_name['trace'].message_type = _OPERATION_TRACE
-_OPERATION_OPERATION.containing_type = _OPERATION
-_DOWNLOADFILE_PARAM.containing_type = _DOWNLOADFILE
-_DOWNLOADFILE_HTTPPARSE.containing_type = _DOWNLOADFILE
-_DOWNLOADFILE_REMOTEFILE.fields_by_name['files'].message_type = _FILE
-_DOWNLOADFILE_REMOTEFILE.fields_by_name['protocol'].enum_type = _DOWNLOADFILE_PROTOCOL
-_DOWNLOADFILE_REMOTEFILE.fields_by_name['param'].message_type = _DOWNLOADFILE_PARAM
-_DOWNLOADFILE_REMOTEFILE.fields_by_name['http_parse'].message_type = _DOWNLOADFILE_HTTPPARSE
-_DOWNLOADFILE_REMOTEFILE.containing_type = _DOWNLOADFILE
-_DOWNLOADFILE_PROXY.containing_type = _DOWNLOADFILE
-_DOWNLOADFILE_OPTIONSENTRY.containing_type = _DOWNLOADFILE
-_DOWNLOADFILE.fields_by_name['remote_file'].message_type = _DOWNLOADFILE_REMOTEFILE
-_DOWNLOADFILE.fields_by_name['proxy'].message_type = _DOWNLOADFILE_PROXY
-_DOWNLOADFILE.fields_by_name['http_method'].enum_type = _DOWNLOADFILE_HTTP_METHOD
-_DOWNLOADFILE.fields_by_name['options'].message_type = _DOWNLOADFILE_OPTIONSENTRY
-_DOWNLOADFILE_PROTOCOL.containing_type = _DOWNLOADFILE
-_DOWNLOADFILE_HTTP_METHOD.containing_type = _DOWNLOADFILE
-DESCRIPTOR.message_types_by_name['File'] = _FILE
-DESCRIPTOR.message_types_by_name['FileList'] = _FILELIST
-DESCRIPTOR.message_types_by_name['Operation'] = _OPERATION
-DESCRIPTOR.message_types_by_name['Process'] = _PROCESS
-DESCRIPTOR.message_types_by_name['DownloadFile'] = _DOWNLOADFILE
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
-
-File = _reflection.GeneratedProtocolMessageType('File', (_message.Message,), dict(
-
- MetaData = _reflection.GeneratedProtocolMessageType('MetaData', (_message.Message,), dict(
- DESCRIPTOR = _FILE_METADATA,
- __module__ = 'downmessage_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.download.File.MetaData)
- ))
- ,
- DESCRIPTOR = _FILE,
- __module__ = 'downmessage_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.download.File)
- ))
-_sym_db.RegisterMessage(File)
-_sym_db.RegisterMessage(File.MetaData)
-
-FileList = _reflection.GeneratedProtocolMessageType('FileList', (_message.Message,), dict(
- DESCRIPTOR = _FILELIST,
- __module__ = 'downmessage_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.download.FileList)
- ))
-_sym_db.RegisterMessage(FileList)
-
-Operation = _reflection.GeneratedProtocolMessageType('Operation', (_message.Message,), dict(
-
- Trace = _reflection.GeneratedProtocolMessageType('Trace', (_message.Message,), dict(
- DESCRIPTOR = _OPERATION_TRACE,
- __module__ = 'downmessage_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.download.Operation.Trace)
- ))
- ,
- DESCRIPTOR = _OPERATION,
- __module__ = 'downmessage_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.download.Operation)
- ))
-_sym_db.RegisterMessage(Operation)
-_sym_db.RegisterMessage(Operation.Trace)
-
-Process = _reflection.GeneratedProtocolMessageType('Process', (_message.Message,), dict(
- DESCRIPTOR = _PROCESS,
- __module__ = 'downmessage_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.download.Process)
- ))
-_sym_db.RegisterMessage(Process)
-
-DownloadFile = _reflection.GeneratedProtocolMessageType('DownloadFile', (_message.Message,), dict(
-
- Param = _reflection.GeneratedProtocolMessageType('Param', (_message.Message,), dict(
- DESCRIPTOR = _DOWNLOADFILE_PARAM,
- __module__ = 'downmessage_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.download.DownloadFile.Param)
- ))
- ,
-
- HttpParse = _reflection.GeneratedProtocolMessageType('HttpParse', (_message.Message,), dict(
- DESCRIPTOR = _DOWNLOADFILE_HTTPPARSE,
- __module__ = 'downmessage_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.download.DownloadFile.HttpParse)
- ))
- ,
-
- RemoteFile = _reflection.GeneratedProtocolMessageType('RemoteFile', (_message.Message,), dict(
- DESCRIPTOR = _DOWNLOADFILE_REMOTEFILE,
- __module__ = 'downmessage_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.download.DownloadFile.RemoteFile)
- ))
- ,
-
- Proxy = _reflection.GeneratedProtocolMessageType('Proxy', (_message.Message,), dict(
- DESCRIPTOR = _DOWNLOADFILE_PROXY,
- __module__ = 'downmessage_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.download.DownloadFile.Proxy)
- ))
- ,
-
- OptionsEntry = _reflection.GeneratedProtocolMessageType('OptionsEntry', (_message.Message,), dict(
- DESCRIPTOR = _DOWNLOADFILE_OPTIONSENTRY,
- __module__ = 'downmessage_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.download.DownloadFile.OptionsEntry)
- ))
- ,
- DESCRIPTOR = _DOWNLOADFILE,
- __module__ = 'downmessage_pb2'
- # @@protoc_insertion_point(class_scope:biomaj.download.DownloadFile)
- ))
-_sym_db.RegisterMessage(DownloadFile)
-_sym_db.RegisterMessage(DownloadFile.Param)
-_sym_db.RegisterMessage(DownloadFile.HttpParse)
-_sym_db.RegisterMessage(DownloadFile.RemoteFile)
-_sym_db.RegisterMessage(DownloadFile.Proxy)
-_sym_db.RegisterMessage(DownloadFile.OptionsEntry)
-
-
-_DOWNLOADFILE_OPTIONSENTRY._options = None
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x11\x64ownmessage.proto\x12\x0f\x62iomaj.download\"\x9d\x02\n\x04\x46ile\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\x0c\n\x04root\x18\x02 \x01(\t\x12\x0f\n\x07save_as\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\x30\n\x08metadata\x18\x05 \x01(\x0b\x32\x1e.biomaj.download.File.MetaData\x1a\xa8\x01\n\x08MetaData\x12\x13\n\x0bpermissions\x18\x01 \x01(\t\x12\r\n\x05group\x18\x02 \x01(\t\x12\x0c\n\x04size\x18\x03 \x01(\x03\x12\x0c\n\x04hash\x18\x04 \x01(\t\x12\x0c\n\x04year\x18\x05 \x01(\x05\x12\r\n\x05month\x18\x06 \x01(\x05\x12\x0b\n\x03\x64\x61y\x18\x07 \x01(\x05\x12\x0e\n\x06\x66ormat\x18\x08 \x01(\t\x12\x0b\n\x03md5\x18\t \x01(\t\x12\x15\n\rdownload_time\x18\n \x01(\x03\"0\n\x08\x46ileList\x12$\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x15.biomaj.download.File\"\xaa\x02\n\tOperation\x12\x32\n\x04type\x18\x01 \x02(\x0e\x32$.biomaj.download.Operation.OPERATION\x12/\n\x08\x64ownload\x18\x02 \x01(\x0b\x32\x1d.biomaj.download.DownloadFile\x12)\n\x07process\x18\x03 \x01(\x0b\x32\x18.biomaj.download.Process\x12/\n\x05trace\x18\x04 \x01(\x0b\x32 .biomaj.download.Operation.Trace\x1a*\n\x05Trace\x12\x10\n\x08trace_id\x18\x01 \x02(\t\x12\x0f\n\x07span_id\x18\x02 \x02(\t\"0\n\tOPERATION\x12\x08\n\x04LIST\x10\x00\x12\x0c\n\x08\x44OWNLOAD\x10\x01\x12\x0b\n\x07PROCESS\x10\x02\"\x17\n\x07Process\x12\x0c\n\x04\x65xec\x18\x01 \x02(\t\"\x94\x0b\n\x0c\x44ownloadFile\x12\x0c\n\x04\x62\x61nk\x18\x01 \x02(\t\x12\x0f\n\x07session\x18\x02 \x02(\t\x12\x11\n\tlocal_dir\x18\x03 \x02(\t\x12\x18\n\x10timeout_download\x18\x04 \x01(\x05\x12=\n\x0bremote_file\x18\x05 \x02(\x0b\x32(.biomaj.download.DownloadFile.RemoteFile\x12\x32\n\x05proxy\x18\x06 \x01(\x0b\x32#.biomaj.download.DownloadFile.Proxy\x12\x43\n\x0bhttp_method\x18\x08 \x01(\x0e\x32).biomaj.download.DownloadFile.HTTP_METHOD:\x03GET\x12;\n\x07options\x18\t \x03(\x0b\x32*.biomaj.download.DownloadFile.OptionsEntry\x1a$\n\x05Param\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\r\n\x05value\x18\x02 \x02(\t\x1a\xcd\x03\n\tHttpParse\x12\x91\x01\n\x08\x64ir_line\x18\x01 \x02(\t:\x7f<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[DIR\\]\"[\\s]*/?>[\\s]*<a[\\s]+href=\"([\\S]+)/\"[\\s]*>.*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})\x12\xa5\x01\n\tfile_line\x18\x02 \x02(\t:\x91\x01<img[\\s]+src=\"[\\S]+\"[\\s]+alt=\"\\[[\\s]+\\]\"[\\s]*/?>[\\s]<a[\\s]+href=\"([\\S]+)\".*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2})[\\s]+([\\d\\.]+[MKG]{0,1})\x12\x13\n\x08\x64ir_name\x18\x03 \x02(\x05:\x01\x31\x12\x13\n\x08\x64ir_date\x18\x04 \x02(\x05:\x01\x32\x12\x14\n\tfile_name\x18\x05 \x02(\x05:\x01\x31\x12\x14\n\tfile_date\x18\x06 \x02(\x05:\x01\x32\x12\x18\n\x10\x66ile_date_format\x18\x07 \x01(\t\x12\x14\n\tfile_size\x18\x08 \x02(\x05:\x01\x33\x1a\xb8\x02\n\nRemoteFile\x12$\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x15.biomaj.download.File\x12\x38\n\x08protocol\x18\x02 \x02(\x0e\x32&.biomaj.download.DownloadFile.Protocol\x12\x0e\n\x06server\x18\x03 \x02(\t\x12\x12\n\nremote_dir\x18\x04 \x02(\t\x12\x0f\n\x07save_as\x18\x05 \x01(\t\x12\x32\n\x05param\x18\x06 \x03(\x0b\x32#.biomaj.download.DownloadFile.Param\x12;\n\nhttp_parse\x18\x07 \x01(\x0b\x32\'.biomaj.download.DownloadFile.HttpParse\x12\x13\n\x0b\x63redentials\x18\x08 \x01(\t\x12\x0f\n\x07matches\x18\t \x03(\t\x1a*\n\x05Proxy\x12\r\n\x05proxy\x18\x01 \x02(\t\x12\x12\n\nproxy_auth\x18\x02 \x01(\t\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x93\x01\n\x08Protocol\x12\x07\n\x03\x46TP\x10\x00\x12\x08\n\x04\x46TPS\x10\x01\x12\x08\n\x04HTTP\x10\x02\x12\t\n\x05HTTPS\x10\x03\x12\r\n\tDIRECTFTP\x10\x04\x12\x0e\n\nDIRECTHTTP\x10\x05\x12\x0f\n\x0b\x44IRECTHTTPS\x10\x06\x12\t\n\x05LOCAL\x10\x07\x12\t\n\x05RSYNC\x10\x08\x12\t\n\x05IRODS\x10\t\x12\x0e\n\nDIRECTFTPS\x10\n\" \n\x0bHTTP_METHOD\x12\x07\n\x03GET\x10\x00\x12\x08\n\x04POST\x10\x01')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'downmessage_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+ DESCRIPTOR._options = None
+ _DOWNLOADFILE_OPTIONSENTRY._options = None
+ _DOWNLOADFILE_OPTIONSENTRY._serialized_options = b'8\001'
+ _globals['_FILE']._serialized_start=39
+ _globals['_FILE']._serialized_end=324
+ _globals['_FILE_METADATA']._serialized_start=156
+ _globals['_FILE_METADATA']._serialized_end=324
+ _globals['_FILELIST']._serialized_start=326
+ _globals['_FILELIST']._serialized_end=374
+ _globals['_OPERATION']._serialized_start=377
+ _globals['_OPERATION']._serialized_end=675
+ _globals['_OPERATION_TRACE']._serialized_start=583
+ _globals['_OPERATION_TRACE']._serialized_end=625
+ _globals['_OPERATION_OPERATION']._serialized_start=627
+ _globals['_OPERATION_OPERATION']._serialized_end=675
+ _globals['_PROCESS']._serialized_start=677
+ _globals['_PROCESS']._serialized_end=700
+ _globals['_DOWNLOADFILE']._serialized_start=703
+ _globals['_DOWNLOADFILE']._serialized_end=2131
+ _globals['_DOWNLOADFILE_PARAM']._serialized_start=1040
+ _globals['_DOWNLOADFILE_PARAM']._serialized_end=1076
+ _globals['_DOWNLOADFILE_HTTPPARSE']._serialized_start=1079
+ _globals['_DOWNLOADFILE_HTTPPARSE']._serialized_end=1540
+ _globals['_DOWNLOADFILE_REMOTEFILE']._serialized_start=1543
+ _globals['_DOWNLOADFILE_REMOTEFILE']._serialized_end=1855
+ _globals['_DOWNLOADFILE_PROXY']._serialized_start=1857
+ _globals['_DOWNLOADFILE_PROXY']._serialized_end=1899
+ _globals['_DOWNLOADFILE_OPTIONSENTRY']._serialized_start=1901
+ _globals['_DOWNLOADFILE_OPTIONSENTRY']._serialized_end=1947
+ _globals['_DOWNLOADFILE_PROTOCOL']._serialized_start=1950
+ _globals['_DOWNLOADFILE_PROTOCOL']._serialized_end=2097
+ _globals['_DOWNLOADFILE_HTTP_METHOD']._serialized_start=2099
+ _globals['_DOWNLOADFILE_HTTP_METHOD']._serialized_end=2131
# @@protoc_insertion_point(module_scope)
=====================================
requirements.txt
=====================================
@@ -1,19 +1,17 @@
-mock
-pytest
pycurl
py-bcrypt
pika==0.13.0
redis
PyYAML
-protobuf
+protobuf==3.20.3
flask
python-consul
prometheus_client>=0.0.18
requests
biomaj_core>=3.0.19
biomaj_zipkin
-flake8
humanfriendly
python-irodsclient
simpleeval
tenacity
+ftputil
=====================================
setup.cfg
=====================================
@@ -1,2 +1,8 @@
[bdist_wheel]
universal=1
+
+[metadata]
+description-file = README.md
+
+[flake8]
+ignore = E501,W504
=====================================
setup.py
=====================================
@@ -3,7 +3,6 @@ try:
except ImportError:
from distutils.core import setup
-from distutils.command.install import install
import os
@@ -14,15 +13,19 @@ with open(os.path.join(here, 'CHANGES.txt')) as f:
CHANGES = f.read()
+with open('requirements.txt') as f:
+ requirements = f.read().splitlines()
+
+
config = {
'description': 'BioMAJ download service',
'long_description': README + '\n\n' + CHANGES,
'long_description_content_type': 'text/markdown',
- 'author': 'Olivier Sallou',
- 'url': 'http://biomaj.genouest.org',
- 'download_url': 'http://biomaj.genouest.org',
- 'author_email': 'olivier.sallou at irisa.fr',
- 'version': '3.2.9',
+ 'author': 'GenOuest platform',
+ 'url': 'https://biomaj.genouest.org',
+ 'download_url': 'https://biomaj.genouest.org',
+ 'author_email': 'support at genouest.org',
+ 'version': '3.2.12',
'classifiers': [
# How mature is this project? Common values are
# 3 - Alpha
@@ -43,26 +46,8 @@ config = {
'Programming Language :: Python :: 3.6'
],
'python_requires': '>=3.6, <4',
- 'install_requires': [
- 'biomaj_core',
- 'biomaj_zipkin',
- 'pycurl',
- 'ftputil',
- 'tenacity',
- 'simpleeval',
- 'py-bcrypt',
- 'pika==0.13.0',
- 'redis',
- 'PyYAML',
- 'flask',
- 'python-consul',
- 'prometheus_client>=0.0.18',
- 'protobuf',
- 'requests',
- 'humanfriendly',
- 'python-irodsclient'
- ],
- 'tests_require': ['pytest', 'mock'],
+ 'install_requires': requirements,
+ 'tests_require': ['pytest'],
'packages': find_packages(),
'include_package_data': True,
'scripts': ['bin/biomaj_download_consumer.py'],
=====================================
tests/biomaj_tests.py
=====================================
@@ -14,7 +14,7 @@ import logging
import stat
import pytest
-from mock import patch
+from unittest.mock import patch
from irods.session import iRODSSession
View it on GitLab: https://salsa.debian.org/med-team/biomaj3-download/-/commit/7730fe656323f4c74c3201d2ad7b22b38a23d253
--
View it on GitLab: https://salsa.debian.org/med-team/biomaj3-download/-/commit/7730fe656323f4c74c3201d2ad7b22b38a23d253
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20240826/28290af3/attachment-0001.htm>
More information about the debian-med-commit
mailing list