[med-svn] [Git][med-team/cwltool][upstream] New upstream version 3.1.20230209161050
Michael R. Crusoe (@crusoe)
gitlab at salsa.debian.org
Thu Feb 9 18:20:08 GMT 2023
Michael R. Crusoe pushed to branch upstream at Debian Med / cwltool
Commits:
4b8c1168 by Michael R. Crusoe at 2023-02-09T17:52:59+01:00
New upstream version 3.1.20230209161050
- - - - -
17 changed files:
- PKG-INFO
- cwltool.egg-info/PKG-INFO
- cwltool/main.py
- cwltool/provenance_profile.py
- cwltool/software_requirements.py
- cwltool/utils.py
- mypy-requirements.txt
- pyproject.toml
- setup.cfg
- tests/test_dependencies.py
- tests/test_examples.py
- tests/test_iwdr.py
- tests/test_js_sandbox.py
- tests/test_provenance.py
- tests/test_singularity.py
- tests/test_subgraph.py
- tox.ini
Changes:
=====================================
PKG-INFO
=====================================
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: cwltool
-Version: 3.1.20230201224320
+Version: 3.1.20230209161050
Summary: Common workflow language reference implementation
Home-page: https://github.com/common-workflow-language/cwltool
Author: Common workflow language working group
=====================================
cwltool.egg-info/PKG-INFO
=====================================
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: cwltool
-Version: 3.1.20230201224320
+Version: 3.1.20230209161050
Summary: Common workflow language reference implementation
Home-page: https://github.com/common-workflow-language/cwltool
Author: Common workflow language working group
=====================================
cwltool/main.py
=====================================
@@ -1289,6 +1289,7 @@ def main(
find_default_container,
default_container=runtimeContext.default_container,
use_biocontainers=args.beta_use_biocontainers,
+ container_image_cache_path=args.beta_dependencies_directory,
)
(out, status) = real_executor(
@@ -1410,10 +1411,13 @@ def find_default_container(
builder: HasReqsHints,
default_container: Optional[str] = None,
use_biocontainers: Optional[bool] = None,
+ container_image_cache_path: Optional[str] = None,
) -> Optional[str]:
"""Find a container."""
if not default_container and use_biocontainers:
- default_container = get_container_from_software_requirements(use_biocontainers, builder)
+ default_container = get_container_from_software_requirements(
+ use_biocontainers, builder, container_image_cache_path
+ )
return default_container
=====================================
cwltool/provenance_profile.py
=====================================
@@ -386,6 +386,10 @@ class ProvenanceProfile:
(PROV_TYPE, RO["Folder"]),
],
)
+
+ if "basename" in value:
+ coll.add_attributes({CWLPROV["basename"]: cast(str, value["basename"])})
+
# ORE description of ro:Folder, saved separately
coll_b = dir_bundle.entity(
dir_id,
=====================================
cwltool/software_requirements.py
=====================================
@@ -136,7 +136,7 @@ def get_dependencies(builder: HasReqsHints) -> ToolRequirements:
def get_container_from_software_requirements(
- use_biocontainers: bool, builder: HasReqsHints
+ use_biocontainers: bool, builder: HasReqsHints, container_image_cache_path: Optional[str] = "."
) -> Optional[str]:
if use_biocontainers:
ensure_galaxy_lib_available()
@@ -147,7 +147,7 @@ def get_container_from_software_requirements(
app_info: AppInfo = AppInfo(
involucro_auto_init=True,
enable_mulled_containers=True,
- container_image_cache_path=".",
+ container_image_cache_path=container_image_cache_path,
)
container_registry: ContainerRegistry = ContainerRegistry(app_info)
requirements = get_dependencies(builder)
=====================================
cwltool/utils.py
=====================================
@@ -245,9 +245,9 @@ def random_outdir() -> str:
fcntl: Optional[ModuleType] = None
msvcrt: Optional[ModuleType] = None
try:
- import fcntl # type: ignore
+ import fcntl
except ImportError:
- import msvcrt # type: ignore
+ import msvcrt
def shared_file_lock(fd: IO[Any]) -> None:
=====================================
mypy-requirements.txt
=====================================
@@ -1,4 +1,4 @@
-mypy==0.991
+mypy==1.0.0 # also update pyproject.toml
ruamel.yaml>=0.16.0,<0.17.22
schema-salad>=8.2.20211104054942,<9
cwl-utils >=0.19
=====================================
pyproject.toml
=====================================
@@ -2,7 +2,7 @@
requires = [
"setuptools>=45",
'mypy==0.971; python_version == "3.6"', # last version for Python 3.6
- 'mypy==0.991; python_version >= "3.7"',
+ 'mypy==1.0; python_version >= "3.7"', # also update mypy-requirements.txt
"types-pkg_resources",
"types-requests",
"types-psutil",
=====================================
setup.cfg
=====================================
@@ -12,6 +12,6 @@ line_length = 88
ignore-words-list = ORE,ore,RO,ro
[egg_info]
-tag_build = .20230201224320
+tag_build = .20230209161050
tag_date = 0
=====================================
tests/test_dependencies.py
=====================================
@@ -1,9 +1,11 @@
"""Tests of satisfying SoftwareRequirement via dependencies."""
import os
+import tempfile
+from getpass import getuser
from pathlib import Path
from shutil import which
from types import ModuleType
-from typing import Optional
+from typing import Optional, Tuple
import pytest
@@ -26,7 +28,15 @@ def test_biocontainers(tmp_path: Path) -> None:
wflow = get_data("tests/seqtk_seq.cwl")
job = get_data("tests/seqtk_seq_job.json")
error_code, _, _ = get_main_output(
- ["--outdir", str(tmp_path), "--beta-use-biocontainers", wflow, job]
+ [
+ "--outdir",
+ str(tmp_path / "out"),
+ "--beta-use-biocontainers",
+ "--beta-dependencies-directory",
+ str(tmp_path / "deps"),
+ wflow,
+ job,
+ ]
)
assert error_code == 0
@@ -38,31 +48,85 @@ def test_biocontainers_resolution(tmp_path: Path) -> None:
"""Confirm expected container name for --beta-use-biocontainers."""
tool = load_tool(get_data("tests/seqtk_seq.cwl"), LoadingContext())
assert (
- get_container_from_software_requirements(True, tool) == "quay.io/biocontainers/seqtk:r93--0"
+ get_container_from_software_requirements(
+ True, tool, container_image_cache_path=str(tmp_path)
+ )
+ == "quay.io/biocontainers/seqtk:r93--0"
)
- at pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed")
-def test_bioconda(tmp_path: Path) -> None:
+ at pytest.fixture(scope="session")
+def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]:
+ """
+ Caches the conda environment created for seqtk_seq.cwl.
+
+ Respects ``--basetemp`` via code copied from
+ :py:method:`pytest.TempPathFactory.getbasetemp`.
+ """
+
+ assert request.config.cache
+ deps_dir = request.config.cache.get("bioconda_deps", None)
+ if deps_dir is not None and not Path(deps_dir).exists():
+ # cache value set, but cache is gone :( ... recreate
+ deps_dir = None
+
+ if deps_dir is None:
+ given_basetemp = request.config.option.basetemp
+ if given_basetemp is not None:
+ basetemp = Path(os.path.abspath(str(given_basetemp))).resolve()
+ deps_dir = basetemp / "bioconda"
+ else:
+ from_env = os.environ.get("PYTEST_DEBUG_TEMPROOT")
+ temproot = Path(from_env or tempfile.gettempdir()).resolve()
+ rootdir = temproot.joinpath(f"pytest-of-{getuser() or 'unknown'}")
+ try:
+ rootdir.mkdir(mode=0o700, exist_ok=True)
+ except OSError:
+ rootdir = temproot.joinpath("pytest-of-unknown")
+ rootdir.mkdir(mode=0o700, exist_ok=True)
+ deps_dir = rootdir / "bioconda"
+ request.config.cache.set("bioconda_deps", str(deps_dir))
+
+ deps_dirpath = Path(deps_dir)
+ deps_dirpath.mkdir(parents=True, exist_ok=True)
+
wflow = get_data("tests/seqtk_seq.cwl")
job = get_data("tests/seqtk_seq_job.json")
error_code, _, stderr = get_main_output(
- ["--outdir", str(tmp_path), "--beta-conda-dependencies", "--debug", wflow, job]
+ [
+ "--outdir",
+ str(deps_dirpath / "out"),
+ "--beta-conda-dependencies",
+ "--beta-dependencies-directory",
+ str(deps_dirpath / "deps"),
+ "--debug",
+ wflow,
+ job,
+ ]
)
+ return error_code, stderr
+
+ at pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed")
+def test_bioconda(bioconda_setup: Tuple[Optional[int], str]) -> None:
+ error_code, stderr = bioconda_setup
assert error_code == 0, stderr
@pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed")
@pytest.mark.skipif(not which("modulecmd"), reason="modulecmd not installed")
-def test_modules(monkeypatch: pytest.MonkeyPatch) -> None:
+def test_modules(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""Do a basic smoke test using environment modules to satisfy a SoftwareRequirement."""
wflow = get_data("tests/random_lines.cwl")
job = get_data("tests/random_lines_job.json")
monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles"))
error_code, _, stderr = get_main_output(
[
+ "--outdir",
+ str(tmp_path / "out"),
"--beta-dependency-resolvers-configuration",
+ "--beta-dependencies-directory",
+ str(tmp_path / "deps"),
"tests/test_deps_env_modules_resolvers_conf.yml",
"--debug",
wflow,
=====================================
tests/test_examples.py
=====================================
@@ -1291,6 +1291,8 @@ def test_cache_relative_paths(tmp_path: Path, factor: str) -> None:
commands = factor.split()
commands.extend(
[
+ "--out",
+ str(tmp_path / "out"),
"--cachedir",
cache_dir,
get_data(f"tests/{test_file}"),
@@ -1306,6 +1308,8 @@ def test_cache_relative_paths(tmp_path: Path, factor: str) -> None:
commands = factor.split()
commands.extend(
[
+ "--out",
+ str(tmp_path / "out2"),
"--cachedir",
cache_dir,
get_data(f"tests/{test_file}"),
@@ -1324,6 +1328,8 @@ def test_cache_relative_paths(tmp_path: Path, factor: str) -> None:
def test_write_summary(tmp_path: Path) -> None:
"""Test --write-summary."""
commands = [
+ "--out",
+ str(tmp_path / "out1"),
get_data("tests/wf/no-parameters-echo.cwl"),
]
error_code, stdout, stderr = get_main_output(commands)
@@ -1332,6 +1338,8 @@ def test_write_summary(tmp_path: Path) -> None:
final_output_path = str(tmp_path / "final-output.json")
commands_no = [
+ "--out",
+ str(tmp_path / "out2"),
"--write-summary",
final_output_path,
get_data("tests/wf/no-parameters-echo.cwl"),
@@ -1347,10 +1355,11 @@ def test_write_summary(tmp_path: Path) -> None:
@needs_docker
-def test_compute_checksum() -> None:
+def test_compute_checksum(tmp_path: Path) -> None:
runtime_context = RuntimeContext()
runtime_context.compute_checksum = True
runtime_context.use_container = False
+ runtime_context.outdir = str(tmp_path)
factory = cwltool.factory.Factory(runtime_context=runtime_context)
echo = factory.make(get_data("tests/wf/cat-tool.cwl"))
output = echo(
@@ -1413,10 +1422,12 @@ def test_bad_stdout_expr_error() -> None:
@needs_docker
-def test_stdin_with_id_preset() -> None:
+def test_stdin_with_id_preset(tmp_path: Path) -> None:
"""Confirm that a type: stdin with a preset id does not give an error."""
error_code, _, stderr = get_main_output(
[
+ "--out",
+ str(tmp_path),
get_data("tests/wf/1590.cwl"),
"--file1",
get_data("tests/wf/whale.txt"),
@@ -1590,7 +1601,7 @@ def test_v1_0_arg_empty_prefix_separate_false() -> None:
def test_scatter_output_filenames(tmp_path: Path) -> None:
"""Confirm that the final output is renamed correctly from identically named scatter outputs."""
- cwd = Path.cwd()
+ cwd = tmp_path
with working_directory(tmp_path):
rtc = RuntimeContext()
rtc.outdir = str(cwd)
@@ -1726,10 +1737,10 @@ def test_command_line_tool_class() -> None:
assert str(expression_tool) == f"CommandLineTool: file://{tool_path}"
-def test_record_default_with_long() -> None:
+def test_record_default_with_long(tmp_path: Path) -> None:
"""Confirm that record defaults are respected."""
tool_path = get_data("tests/wf/paramref_arguments_roundtrip.cwl")
- err_code, stdout, stderr = get_main_output([tool_path])
+ err_code, stdout, stderr = get_main_output(["--outdir", str(tmp_path), tool_path])
assert err_code == 0
result = json.loads(stdout)["same_record"]
assert result["first"] == "y"
=====================================
tests/test_iwdr.py
=====================================
@@ -5,6 +5,8 @@ from pathlib import Path
from stat import S_IWGRP, S_IWOTH, S_IWRITE
from typing import Any
+import pytest
+
from cwltool.factory import Factory
from cwltool.main import main
@@ -246,7 +248,9 @@ def test_iwdr_permutations_inplace(tmp_path_factory: Any) -> None:
@needs_singularity
-def test_iwdr_permutations_singularity(tmp_path_factory: Any) -> None:
+def test_iwdr_permutations_singularity(
+ tmp_path_factory: pytest.TempPathFactory, monkeypatch: pytest.MonkeyPatch
+) -> None:
misc = tmp_path_factory.mktemp("misc")
fifth = misc / "fifth"
fifth.mkdir()
@@ -269,6 +273,8 @@ def test_iwdr_permutations_singularity(tmp_path_factory: Any) -> None:
twelfth = misc / "twelfth"
twelfth.touch()
outdir = str(tmp_path_factory.mktemp("outdir"))
+ singularity_dir = str(tmp_path_factory.mktemp("singularity"))
+ monkeypatch.setenv("CWL_SINGULARITY_CACHE", singularity_dir)
err_code, stdout, _ = get_main_output(
[
"--outdir",
@@ -306,7 +312,9 @@ def test_iwdr_permutations_singularity(tmp_path_factory: Any) -> None:
@needs_singularity
-def test_iwdr_permutations_singularity_inplace(tmp_path_factory: Any) -> None:
+def test_iwdr_permutations_singularity_inplace(
+ tmp_path_factory: pytest.TempPathFactory, monkeypatch: pytest.MonkeyPatch
+) -> None:
"""IWDR tests using --singularity and a forced InplaceUpdateRequirement."""
misc = tmp_path_factory.mktemp("misc")
fifth = misc / "fifth"
@@ -330,6 +338,8 @@ def test_iwdr_permutations_singularity_inplace(tmp_path_factory: Any) -> None:
twelfth = misc / "twelfth"
twelfth.touch()
outdir = str(tmp_path_factory.mktemp("outdir"))
+ singularity_dir = str(tmp_path_factory.mktemp("singularity"))
+ monkeypatch.setenv("CWL_SINGULARITY_CACHE", singularity_dir)
assert (
main(
[
=====================================
tests/test_js_sandbox.py
=====================================
@@ -63,6 +63,10 @@ def hide_nodejs(temp_dir: Path) -> str:
if entry not in ("nodejs", "node"):
os.symlink(os.path.join(dirname, entry), new_dir / entry)
paths.append(str(new_dir))
+ dirname_path = Path(dirname)
+ for path in paths:
+ if Path(path).resolve() == dirname_path:
+ paths.remove(path)
return ":".join(paths)
@@ -94,12 +98,17 @@ def test_value_from_two_concatenated_expressions_singularity(
js_engine = sandboxjs.get_js_engine()
js_engine.have_node_slim = False # type: ignore[attr-defined]
js_engine.localdata = threading.local() # type: ignore[attr-defined]
- new_paths = hide_nodejs(tmp_path)
+ hide_base = tmp_path / "hide"
+ hide_base.mkdir()
+ new_paths = hide_nodejs(hide_base)
+ singularity_cache = tmp_path / "singularity"
+ singularity_cache.mkdir()
factory = Factory()
factory.loading_context.singularity = True
factory.loading_context.debug = True
factory.runtime_context.debug = True
with monkeypatch.context() as m:
+ m.setenv("CWL_SINGULARITY_CACHE", str(singularity_cache))
m.setenv("PATH", new_paths)
echo = factory.make(get_data("tests/wf/vf-concat.cwl"))
file = {"class": "File", "location": get_data("tests/wf/whale.txt")}
=====================================
tests/test_provenance.py
=====================================
@@ -539,6 +539,7 @@ def check_prov(
assert (d, RDF.type, PROV.Dictionary) in g
assert (d, RDF.type, PROV.Collection) in g
assert (d, RDF.type, PROV.Entity) in g
+ assert len(list(g.objects(d, CWLPROV.basename))) == 1
files = set()
for entry in g.objects(d, PROV.hadDictionaryMember):
@@ -590,8 +591,8 @@ def check_prov(
@pytest.fixture
-def research_object() -> Generator[ResearchObject, None, None]:
- re_ob = ResearchObject(StdFsAccess(""))
+def research_object(tmp_path: Path) -> Generator[ResearchObject, None, None]:
+ re_ob = ResearchObject(StdFsAccess(str(tmp_path / "ro")), temp_prefix_ro=str(tmp_path / "tmp"))
yield re_ob
re_ob.close()
=====================================
tests/test_singularity.py
=====================================
@@ -3,6 +3,8 @@ import shutil
from pathlib import Path
from typing import Any
+import pytest
+
from cwltool.main import main
from .util import (
@@ -58,7 +60,10 @@ def test_singularity_workflow(tmp_path: Path) -> None:
assert error_code == 0
-def test_singularity_iwdr() -> None:
+def test_singularity_iwdr(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ singularity_dir = tmp_path / "singularity"
+ singularity_dir.mkdir()
+ monkeypatch.setenv("CWL_SINGULARITY_CACHE", str(singularity_dir))
result_code = main(
[
"--singularity",
=====================================
tests/test_subgraph.py
=====================================
@@ -219,10 +219,12 @@ def test_single_process_packed_subwf_step(tmp_path: Path) -> None:
@needs_docker
-def test_single_process_subwf_subwf_inline_step() -> None:
+def test_single_process_subwf_subwf_inline_step(tmp_path: Path) -> None:
"""Test --single-process on an inline sub-sub-workflow step."""
err_code, stdout, stderr = get_main_output(
[
+ "--outdir",
+ str(tmp_path),
"--single-process",
"step1/stepX/stepY",
get_data("tests/subgraph/count-lines17-wf.cwl.json"),
=====================================
tox.ini
=====================================
@@ -11,7 +11,7 @@ envlist =
skip_missing_interpreters = True
[pytest]
-addopts=--ignore cwltool/schemas --basetemp ./tmp -n auto
+addopts=--ignore cwltool/schemas -n auto
testpaths = tests
[gh-actions]
View it on GitLab: https://salsa.debian.org/med-team/cwltool/-/commit/4b8c1168b3b7dc3fc4216affa3fd582ef4cb22d0
--
View it on GitLab: https://salsa.debian.org/med-team/cwltool/-/commit/4b8c1168b3b7dc3fc4216affa3fd582ef4cb22d0
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20230209/93ea2b19/attachment-0001.htm>
More information about the debian-med-commit
mailing list